diff --git a/apps/host-selfhost/executor.config.ts b/apps/host-selfhost/executor.config.ts index 1a29f4507..4e43f45e1 100644 --- a/apps/host-selfhost/executor.config.ts +++ b/apps/host-selfhost/executor.config.ts @@ -6,8 +6,10 @@ import { mcpHttpPlugin } from "@executor-js/plugin-mcp/api"; import { graphqlHttpPlugin } from "@executor-js/plugin-graphql/api"; import { encryptedSecretsPlugin } from "@executor-js/plugin-encrypted-secrets"; import { toolkitsPlugin } from "@executor-js/plugin-toolkits/server"; +import { appsPlugin } from "@executor-js/plugin-apps/api"; import { resolveSecretKey } from "./src/config"; +import { getSelfHostAppsBackings } from "./src/apps-backings"; // --------------------------------------------------------------------------- // Single source of truth for the self-hosted app's plugin list. @@ -36,6 +38,7 @@ export default defineExecutorConfig({ mcpHttpPlugin({ dangerouslyAllowStdioMCP: false }), graphqlHttpPlugin(), toolkitsPlugin({ activeToolkitSlug }), + appsPlugin({ backings: getSelfHostAppsBackings() }), // First writable secret provider -> the default for `secrets.set`. encryptedSecretsPlugin({ key: resolveSecretKey() }), ] as const, diff --git a/apps/host-selfhost/package.json b/apps/host-selfhost/package.json index 1c73a8cc8..bb3119599 100644 --- a/apps/host-selfhost/package.json +++ b/apps/host-selfhost/package.json @@ -26,6 +26,7 @@ "@executor-js/execution": "workspace:*", "@executor-js/fumadb": "workspace:*", "@executor-js/host-mcp": "workspace:*", + "@executor-js/plugin-apps": "workspace:*", "@executor-js/plugin-encrypted-secrets": "workspace:*", "@executor-js/plugin-google": "workspace:*", "@executor-js/plugin-graphql": "workspace:*", @@ -48,6 +49,7 @@ }, "devDependencies": { "@effect/vitest": "catalog:", + "@executor-js/emulate": "^0.13.2", "@executor-js/vite-plugin": "workspace:*", "@tailwindcss/vite": "catalog:", "@tanstack/router-plugin": "^1.167.12", diff --git a/apps/host-selfhost/src/app.ts b/apps/host-selfhost/src/app.ts index c4a080f3b..0bf33f64c 100644 --- a/apps/host-selfhost/src/app.ts +++ b/apps/host-selfhost/src/app.ts @@ -22,6 +22,7 @@ import { makeSelfHostMcpSeams } from "./mcp"; import { selfHostPlugins } from "./plugins"; import { ErrorCaptureLive } from "./observability"; import { oauthCallbackSignInRedirectLocation } from "./auth/oauth-callback-login"; +import { closeSelfHostAppsBackings } from "./apps-backings"; // =========================================================================== // The self-hosted Executor app, as ONE `ExecutorApp.make` call. @@ -139,6 +140,7 @@ export const makeSelfHostApp = async (options: MakeSelfHostAppOptions = {}) => { toWebHandler, betterAuth, closeDb: async () => { + await closeSelfHostAppsBackings(); await mcp.close(); await dbHandle.close(); }, diff --git a/apps/host-selfhost/src/apps-backings.ts b/apps/host-selfhost/src/apps-backings.ts new file mode 100644 index 000000000..6bebe52cb --- /dev/null +++ b/apps/host-selfhost/src/apps-backings.ts @@ -0,0 +1,43 @@ +import { join } from "node:path"; + +import { + makeGitArtifactStore, + makeQuickjsToolSandbox, + makeSqliteAppsStore, + type AppsBackings, +} from "@executor-js/plugin-apps/api"; + +import { resolveDataDir } from "./config"; + +interface SelfHostAppsBackings { + readonly backings: AppsBackings; + readonly close: () => Promise; +} + +const createBackings = (dataDir: string): SelfHostAppsBackings => { + const appsDir = join(dataDir, "apps"); + return { + backings: { + artifactStore: makeGitArtifactStore({ root: join(appsDir, "artifacts") }), + sandbox: makeQuickjsToolSandbox(), + store: makeSqliteAppsStore({ path: join(appsDir, "store.sqlite") }), + }, + close: async () => {}, + }; +}; + +let current: { readonly dataDir: string; readonly value: SelfHostAppsBackings } | undefined; + +export const getSelfHostAppsBackings = (): AppsBackings => { + const dataDir = resolveDataDir(); + if (current && current.dataDir === dataDir) return current.value.backings; + const value = createBackings(dataDir); + current = { dataDir, value }; + return value.backings; +}; + +export const closeSelfHostAppsBackings = async (): Promise => { + const value = current?.value; + current = undefined; + await value?.close(); +}; diff --git a/apps/host-selfhost/src/testing/test-app.ts b/apps/host-selfhost/src/testing/test-app.ts index 9dc707d90..e922f63f7 100644 --- a/apps/host-selfhost/src/testing/test-app.ts +++ b/apps/host-selfhost/src/testing/test-app.ts @@ -30,6 +30,7 @@ import { } from "../mcp/session-store"; import { selfHostPlugins } from "../plugins"; import { ErrorCaptureLive } from "../observability"; +import { closeSelfHostAppsBackings } from "../apps-backings"; // =========================================================================== // Self-host TEST harness — the throwaway composition tests use to exercise the @@ -217,6 +218,7 @@ export const makeSelfHostTestApp = async ( handler: web.handler, dispose: async () => { await web.dispose(); + await closeSelfHostAppsBackings(); await sessionStore.close(); await dbHandle.close(); }, diff --git a/apps/host-selfhost/tsconfig.json b/apps/host-selfhost/tsconfig.json index e214693ed..35302d3e8 100644 --- a/apps/host-selfhost/tsconfig.json +++ b/apps/host-selfhost/tsconfig.json @@ -6,6 +6,7 @@ "strict": true, "esModuleInterop": true, "skipLibCheck": true, + "jsx": "react-jsx", "types": ["bun-types"], "noUnusedLocals": true, "noImplicitOverride": true, diff --git a/bun.lock b/bun.lock index e1f7250b2..a097c7321 100644 --- a/bun.lock +++ b/bun.lock @@ -224,6 +224,7 @@ "@executor-js/execution": "workspace:*", "@executor-js/fumadb": "workspace:*", "@executor-js/host-mcp": "workspace:*", + "@executor-js/plugin-apps": "workspace:*", "@executor-js/plugin-encrypted-secrets": "workspace:*", "@executor-js/plugin-google": "workspace:*", "@executor-js/plugin-graphql": "workspace:*", @@ -246,6 +247,7 @@ }, "devDependencies": { "@effect/vitest": "catalog:", + "@executor-js/emulate": "^0.13.2", "@executor-js/vite-plugin": "workspace:*", "@tailwindcss/vite": "catalog:", "@tanstack/router-plugin": "^1.167.12", @@ -781,6 +783,44 @@ "effect": "catalog:", }, }, + "packages/plugins/apps": { + "name": "@executor-js/plugin-apps", + "version": "0.1.0", + "dependencies": { + "@executor-js/codemode-core": "workspace:*", + "@executor-js/runtime-quickjs": "workspace:*", + "@executor-js/sdk": "workspace:*", + "@libsql/client": "catalog:", + "esbuild": "^0.27.3", + "zod": "4.3.6", + }, + "devDependencies": { + "@effect/atom-react": "catalog:", + "@effect/vitest": "catalog:", + "@executor-js/api": "workspace:*", + "@executor-js/react": "workspace:*", + "@types/node": "catalog:", + "@types/react": "catalog:", + "bun-types": "catalog:", + "effect": "catalog:", + "react": "catalog:", + "tsup": "catalog:", + "vitest": "catalog:", + }, + "peerDependencies": { + "@effect/atom-react": "catalog:", + "@executor-js/api": "workspace:*", + "@executor-js/react": "workspace:*", + "effect": "catalog:", + "react": "catalog:", + }, + "optionalPeers": [ + "@effect/atom-react", + "@executor-js/api", + "@executor-js/react", + "react", + ], + }, "packages/plugins/desktop-settings": { "name": "@executor-js/plugin-desktop-settings", "version": "1.5.28", @@ -1215,9 +1255,9 @@ }, }, "patchedDependencies": { - "@1password/sdk-core@0.4.1-beta.1": "patches/@1password%2Fsdk-core@0.4.1-beta.1.patch", "@electric-sql/pglite-socket@0.1.4": "patches/@electric-sql%2Fpglite-socket@0.1.4.patch", "libsql@0.5.29": "patches/libsql@0.5.29.patch", + "@1password/sdk-core@0.4.1-beta.1": "patches/@1password%2Fsdk-core@0.4.1-beta.1.patch", "agents@0.17.3": "patches/agents@0.17.3.patch", "postgres@3.4.9": "patches/postgres@3.4.9.patch", }, @@ -1776,6 +1816,8 @@ "@executor-js/motel": ["@executor-js/motel@0.2.5-executor.1", "", { "dependencies": { "@effect/atom-react": "^4.0.0-beta.49", "@effect/opentelemetry": "^4.0.0-beta.49", "@effect/platform-bun": "^4.0.0-beta.50", "@opentelemetry/api": "^1.9.0", "@opentelemetry/exporter-logs-otlp-http": "^0.214.0", "@opentelemetry/exporter-trace-otlp-http": "^0.211.0", "@opentelemetry/sdk-logs": "^0.214.0", "@opentelemetry/sdk-node": "^0.214.0", "@opentelemetry/sdk-trace-base": "^2.5.0", "@opentelemetry/sdk-trace-node": "^2.6.1", "@opentui/core": "^0.1.99", "@opentui/react": "^0.1.99", "effect": "^4.0.0-beta.49", "react": "^19.2.5", "scheduler": "^0.27.0" }, "bin": { "motel": "src/motel.ts", "motel-mcp": "src/mcp.ts" } }, "sha512-fLGJ5FIIBYd+4L2OurpYFjWzvcCbACvmdNofh6THXQJE1Gku93EKURtgn27/XddyM9SKGST6/znTRcmwKiYdXw=="], + "@executor-js/plugin-apps": ["@executor-js/plugin-apps@workspace:packages/plugins/apps"], + "@executor-js/plugin-desktop-settings": ["@executor-js/plugin-desktop-settings@workspace:packages/plugins/desktop-settings"], "@executor-js/plugin-encrypted-secrets": ["@executor-js/plugin-encrypted-secrets@workspace:packages/plugins/encrypted-secrets"], diff --git a/e2e/scenarios/custom-tools.test.ts b/e2e/scenarios/custom-tools.test.ts new file mode 100644 index 000000000..0647deafa --- /dev/null +++ b/e2e/scenarios/custom-tools.test.ts @@ -0,0 +1,613 @@ +import { randomBytes } from "node:crypto"; +import { createServer } from "node:net"; + +import { expect } from "@effect/vitest"; +import { Effect } from "effect"; +import { createEmulator, type Emulator } from "@executor-js/emulate"; + +import { scenario } from "../src/scenario"; +import { Api, Browser, Target } from "../src/services"; +import type { Identity, Target as TargetShape } from "../src/target"; +import type { BrowserSurface } from "../src/surfaces/browser"; + +const OWNER = "syncer"; +const GITHUB_CONNECTION = "tools.github.user.main"; + +const unique = (prefix: string) => `${prefix}-${randomBytes(4).toString("hex")}`; + +const bearerTemplate = { + slug: "bearer", + type: "apiKey", + label: "Bearer token", + headers: { Authorization: ["Bearer ", { type: "variable", name: "token" }] }, +}; + +const availablePort = Effect.callback((resume) => { + const server = createServer(); + server.listen(0, "127.0.0.1", () => { + const address = server.address(); + const port = typeof address === "object" && address ? address.port : 0; + server.close(() => resume(Effect.succeed(port))); + }); +}); + +const githubEmulator = (repo: string) => + Effect.acquireRelease( + Effect.gen(function* () { + const port = yield* availablePort; + return yield* Effect.promise(() => + createEmulator({ + service: "github", + port, + seed: { + github: { + users: [{ login: OWNER, name: "Custom Tools Syncer" }], + repos: [{ owner: OWNER, name: repo, auto_init: true }], + }, + }, + }), + ); + }), + (emulator: Emulator) => Effect.promise(() => emulator.close()).pipe(Effect.ignore), + ); + +interface SyncResult { + readonly status: "published" | "up-to-date" | "failed"; + readonly snapshotId?: string; + readonly upstreamSha?: string; + readonly tools: readonly string[]; + readonly skipped: readonly { readonly path: string; readonly reason: string }[]; + readonly errors?: readonly { readonly message?: string }[]; +} + +interface ToolRow { + readonly address: string; + readonly name: string; + readonly integration: string; +} + +interface ExecuteResponse { + readonly status: "completed" | "paused"; + readonly text: string; + readonly structured: { + readonly result?: unknown; + readonly error?: unknown; + }; + readonly isError?: boolean; +} + +const parseJson = async (response: Response): Promise => { + const text = await response.text(); + return (text.length > 0 ? JSON.parse(text) : null) as T; +}; + +const request = async ( + target: TargetShape, + identity: Identity, + path: string, + init: RequestInit = {}, + expectedStatus = 200, +): Promise<{ readonly body: T; readonly text: string }> => { + const headers = new Headers(init.headers); + headers.set("origin", new URL(target.baseUrl).origin); + for (const [name, value] of Object.entries(identity.headers ?? {})) { + headers.set(name, value); + } + if (init.body !== undefined && !headers.has("content-type")) { + headers.set("content-type", "application/json"); + } + const response = await fetch(new URL(path, target.baseUrl), { ...init, headers }); + const text = await response.text(); + expect(response.status, `${init.method ?? "GET"} ${path}: ${text}`).toBe(expectedStatus); + return { body: (text.length > 0 ? JSON.parse(text) : null) as T, text }; +}; + +const postJson = ( + target: TargetShape, + identity: Identity, + path: string, + body: unknown, + expectedStatus = 200, +): Promise<{ readonly body: T; readonly text: string }> => + request( + target, + identity, + path, + { + method: "POST", + body: JSON.stringify(body), + }, + expectedStatus, + ); + +const deletePath = ( + target: TargetShape, + identity: Identity, + path: string, +): Promise<{ readonly status: number; readonly text: string }> => + fetch(new URL(path, target.baseUrl), { + method: "DELETE", + headers: { + ...(identity.headers ?? {}), + origin: new URL(target.baseUrl).origin, + }, + }).then(async (response) => ({ status: response.status, text: await response.text() })); + +const githubFetch = async ( + emulator: Emulator, + token: string, + path: string, + init: RequestInit = {}, +): Promise => { + const headers = new Headers(init.headers); + headers.set("authorization", `Bearer ${token}`); + headers.set("accept", "application/vnd.github+json"); + if (init.body !== undefined) headers.set("content-type", "application/json"); + const response = await fetch(`${emulator.url}${path}`, { ...init, headers }); + expect(response.ok, `${init.method ?? "GET"} ${path}: ${await response.clone().text()}`).toBe( + true, + ); + return parseJson(response); +}; + +const createIssue = ( + emulator: Emulator, + token: string, + repo: string, + title: string, +): Promise => + githubFetch(emulator, token, `/repos/${OWNER}/${repo}/issues`, { + method: "POST", + body: JSON.stringify({ title }), + }); + +const putRepoFiles = async ( + emulator: Emulator, + token: string, + repo: string, + files: Readonly>, +): Promise => { + const ref = await githubFetch<{ object: { sha: string } }>( + emulator, + token, + `/repos/${OWNER}/${repo}/git/ref/heads/main`, + ); + const tree = await githubFetch<{ sha: string }>( + emulator, + token, + `/repos/${OWNER}/${repo}/git/trees`, + { + method: "POST", + body: JSON.stringify({ + tree: Object.entries(files).map(([path, content]) => ({ + path, + mode: "100644", + type: "blob", + content, + })), + }), + }, + ); + const commit = await githubFetch<{ sha: string }>( + emulator, + token, + `/repos/${OWNER}/${repo}/git/commits`, + { + method: "POST", + body: JSON.stringify({ + message: `Update custom tools ${randomBytes(3).toString("hex")}`, + tree: tree.sha, + parents: [ref.object.sha], + }), + }, + ); + await githubFetch(emulator, token, `/repos/${OWNER}/${repo}/git/refs/heads/main`, { + method: "PATCH", + body: JSON.stringify({ sha: commit.sha }), + }); + return commit.sha; +}; + +const executorJson = JSON.stringify( + { + $schema: "https://executor.sh/schemas/executor.json", + description: "Custom tools e2e fixture.", + }, + null, + 2, +); + +const dealPipelineSyncSource = `import { z } from "zod"; +import { defineTool, integration } from "executor:app"; + +export default defineTool({ + description: "Summarize open GitHub issues for pipeline review.", + integrations: { + github: integration("github"), + }, + input: z.object({ + owner: z.string(), + repo: z.string(), + }), + output: z.object({ + synced: z.number(), + issues: z.array(z.object({ number: z.number(), title: z.string() })), + }), + annotations: { readOnly: false, destructive: false }, + async handler({ owner, repo }, { github }) { + const issues = await github.repos.listIssues({ owner, repo, state: "open" }); + return { + synced: issues.length, + issues: issues.map((issue) => ({ number: issue.number, title: issue.title })), + }; + }, +}); +`; + +const findDealDocsSource = `import { z } from "zod"; +import { defineTool, integration } from "executor:app"; + +export default defineTool({ + description: "Return issue titles as deal-document search results.", + integrations: { + github: integration("github"), + }, + input: z.object({ + owner: z.string(), + repo: z.string(), + limit: z.number().int().max(50).default(20), + }), + output: z.object({ + documents: z.array(z.object({ name: z.string(), number: z.number() })), + }), + annotations: { readOnly: true, destructive: false }, + async handler({ owner, repo, limit }, { github }) { + const issues = await github.repos.listIssues({ owner, repo, state: "open" }); + return { + documents: issues.slice(0, limit).map((issue) => ({ + name: issue.title, + number: issue.number, + })), + }; + }, +}); +`; + +const extraToolSource = `import { z } from "zod"; +import { defineTool } from "executor:app"; + +export default defineTool({ + description: "Return a static custom-tools health marker.", + input: z.object({}), + output: z.object({ ok: z.boolean() }), + annotations: { readOnly: true, destructive: false }, + async handler() { + return { ok: true }; + }, +}); +`; + +const sourceFiles = (): Record => ({ + "executor.json": executorJson, + "tools/deal-pipeline-sync.ts": dealPipelineSyncSource, + "tools/find-deal-docs.ts": findDealDocsSource, +}); + +const sourceUrl = (repo: string): string => `https://github.com/${OWNER}/${repo}`; + +const registerGithubIntegration = async ( + target: TargetShape, + identity: Identity, + emulator: Emulator, + token: string, +): Promise => { + await deletePath(target, identity, "/api/connections/user/github/main"); + await deletePath(target, identity, "/api/openapi/integrations/github"); + const added = await postJson<{ slug?: string }>(target, identity, "/api/openapi/specs", { + spec: { kind: "url", url: emulator.openapiUrl }, + slug: "github", + baseUrl: emulator.url, + authenticationTemplate: [bearerTemplate], + }); + expect(added.body.slug).toBe("github"); + const connection = await postJson<{ address: string }>(target, identity, "/api/connections", { + owner: "user", + name: "main", + integration: "github", + template: "bearer", + value: token, + }); + expect(connection.body.address).toBe(GITHUB_CONNECTION); +}; + +const execute = (target: TargetShape, identity: Identity, code: string): Promise => + postJson(target, identity, "/api/executions", { + code, + autoApprove: true, + }).then((response) => response.body); + +const executeResult = async ( + target: TargetShape, + identity: Identity, + code: string, +): Promise => { + const response = await execute(target, identity, code); + expect(response.status, response.text).toBe("completed"); + expect(response.isError, response.text).toBe(false); + return response.structured.result; +}; + +const callAppToolCode = (namespace: string, toolName: string, args: unknown): string => ` +const found = await tools.search({ namespace: ${JSON.stringify(namespace)}, query: ${JSON.stringify( + toolName, +)}, limit: 20 }); +const item = found.items.find((candidate) => candidate.path.endsWith(${JSON.stringify(toolName)})); +if (!item) return { ok: false, missing: ${JSON.stringify(toolName)}, found }; +let fn = tools; +for (const segment of item.path.split(".")) fn = fn[segment]; +const result = await fn(${JSON.stringify(args)}); +return { path: item.path, result }; +`; + +const addSourceThroughConsole = (input: { + readonly target: TargetShape; + readonly browser: BrowserSurface; + readonly identity: Identity; + readonly repo: string; + readonly appSlug: string; + readonly token: string; +}) => + input.browser.session(input.identity, async ({ page, step }) => { + await step("Open the integrations page", async () => { + await page.goto(new URL("/integrations", input.target.baseUrl).toString(), { + waitUntil: "networkidle", + }); + }); + + await step("Detect the GitHub repository as custom tools", async () => { + await page.getByRole("button", { name: "Connect", exact: true }).click(); + const dialog = page.getByRole("dialog"); + await dialog.getByRole("textbox").fill(sourceUrl(input.repo)); + await dialog.getByRole("button", { name: "Detect" }).click(); + await page.getByRole("heading", { name: "Add custom tools" }).waitFor(); + }); + + await step("Sync the custom tools source", async () => { + await page.locator('input[type="password"]').fill(input.token); + await page.getByRole("button", { name: "Sync repo" }).click(); + await page.waitForURL(new RegExp(`/integrations/${input.appSlug}(?:\\?|$)`), { + timeout: 60_000, + }); + await page.getByRole("link", { name: "2 tools" }).waitFor({ timeout: 60_000 }); + }); + }); + +const syncSourceInConsole = (input: { + readonly target: TargetShape; + readonly browser: BrowserSurface; + readonly identity: Identity; + readonly appSlug: string; + readonly expectedNotice: string; + readonly expectedToolCount: string; +}) => + input.browser.session(input.identity, async ({ page, step }) => { + await step(`Sync ${input.appSlug}`, async () => { + await page.goto( + new URL(`/integrations/${input.appSlug}?tab=source`, input.target.baseUrl).toString(), + { + waitUntil: "networkidle", + }, + ); + await page.getByRole("button", { name: "Sync" }).click(); + await page.getByText(input.expectedNotice).waitFor({ timeout: 60_000 }); + await page.getByRole("link", { name: input.expectedToolCount }).waitFor({ timeout: 60_000 }); + }); + }); + +const removeSourceThroughConsole = (input: { + readonly target: TargetShape; + readonly browser: BrowserSurface; + readonly identity: Identity; + readonly appSlug: string; +}) => + input.browser.session(input.identity, async ({ page, step }) => { + await step("Remove the custom tools source", async () => { + await page.goto( + new URL(`/integrations/${input.appSlug}?tab=source`, input.target.baseUrl).toString(), + { + waitUntil: "networkidle", + }, + ); + await page.getByRole("button", { name: "Remove" }).click(); + await page.getByRole("button", { name: "Remove source" }).click(); + await page.waitForURL(/\/integrations(?:\?|$)/, { timeout: 60_000 }); + }); + }); + +scenario( + "Custom tools · GitHub source syncs, invokes, refreshes, and removes", + { timeout: 300_000 }, + Effect.scoped( + Effect.gen(function* () { + const target = yield* Target; + if (target.name !== "selfhost") return; + const browser = yield* Browser; + yield* Api; + const identity = yield* target.newIdentity(); + const repo = unique("custom-tools"); + const appSlug = repo; + const emulator = yield* githubEmulator(repo); + const credential = yield* Effect.promise(() => + emulator.credentials.mint({ type: "api-key", login: OWNER, scopes: ["repo", "user"] }), + ); + const token = credential.token; + if (!token) return yield* Effect.die("GitHub emulator did not mint a token."); + + yield* Effect.ensuring( + Effect.gen(function* () { + const unauthorized = yield* Effect.promise(() => + fetch(new URL("/api/apps/sources/github/sync", target.baseUrl), { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ url: sourceUrl(repo), token }), + }), + ); + expect(unauthorized.status, "sync requires authentication").toBe(401); + yield* Effect.promise(() => unauthorized.text()); + + yield* Effect.promise(() => registerGithubIntegration(target, identity, emulator, token)); + yield* Effect.promise(() => createIssue(emulator, token, repo, "Acme renewal diligence")); + yield* Effect.promise(() => createIssue(emulator, token, repo, "Beta pipeline memo")); + yield* Effect.promise(() => putRepoFiles(emulator, token, repo, sourceFiles())); + + yield* addSourceThroughConsole({ target, browser, identity, repo, appSlug, token }); + + const sources = yield* Effect.promise(() => + request<{ sources: readonly { readonly slug: string; readonly hasToken: boolean }[] }>( + target, + identity, + "/api/apps/sources/github", + ), + ); + expect(sources.text).not.toContain(token); + expect(sources.body.sources.find((source) => source.slug === appSlug)?.hasToken).toBe( + true, + ); + + const detail = yield* Effect.promise(() => + request<{ source: { readonly slug: string; readonly hasToken: boolean } | null }>( + target, + identity, + `/api/apps/sources/github/${encodeURIComponent(appSlug)}`, + ), + ); + expect(detail.text).not.toContain(token); + expect(detail.body.source).toMatchObject({ slug: appSlug, hasToken: true }); + + const tools = yield* Effect.promise(() => + request( + target, + identity, + `/api/tools?integration=${encodeURIComponent(appSlug)}`, + ), + ); + expect(tools.body.map((tool) => tool.name).sort()).toEqual([ + "deal-pipeline-sync", + "find-deal-docs", + ]); + const syncTool = tools.body.find((tool) => tool.name === "deal-pipeline-sync"); + expect(syncTool).toBeDefined(); + + const schema = yield* Effect.promise(() => + request<{ + inputSchema: { + readonly properties?: Record< + string, + { readonly enum?: readonly string[]; readonly default?: string } + >; + readonly required?: readonly string[]; + }; + }>( + target, + identity, + `/api/tools/schema?address=${encodeURIComponent(syncTool!.address)}`, + ), + ); + expect(schema.body.inputSchema.properties?.github).toMatchObject({ + enum: [GITHUB_CONNECTION], + default: GITHUB_CONNECTION, + }); + expect(schema.body.inputSchema.required ?? []).not.toContain("github"); + + const invoked = (yield* Effect.promise(() => + executeResult( + target, + identity, + callAppToolCode(appSlug, "deal-pipeline-sync", { owner: OWNER, repo }), + ), + )) as { + readonly result?: { + readonly ok?: boolean; + readonly data?: { readonly synced?: number }; + }; + }; + expect(invoked.result?.ok).toBe(true); + expect(invoked.result?.data?.synced).toBe(2); + + const ledger = yield* Effect.promise(() => emulator.ledger.list()); + const issueList = ledger.find( + (entry) => + entry.operationId === "issues/listForRepo" && + entry.path === `/repos/${OWNER}/${repo}/issues`, + ); + expect( + issueList?.identity.user?.login, + "custom tool called GitHub with the connection", + ).toBe(OWNER); + + const upToDate = yield* Effect.promise(() => + postJson(target, identity, "/api/apps/sources/github/sync", { + slug: appSlug, + }), + ); + expect(upToDate.text).not.toContain(token); + expect(upToDate.body.status).toBe("up-to-date"); + + yield* syncSourceInConsole({ + target, + browser, + identity, + appSlug, + expectedNotice: "Already up to date.", + expectedToolCount: "2 tools", + }); + + yield* Effect.promise(() => + putRepoFiles(emulator, token, repo, { + ...sourceFiles(), + "tools/extra-tool.ts": extraToolSource, + }), + ); + yield* syncSourceInConsole({ + target, + browser, + identity, + appSlug, + expectedNotice: "Added: extra-tool", + expectedToolCount: "3 tools", + }); + + yield* Effect.promise(() => putRepoFiles(emulator, token, repo, sourceFiles())); + yield* syncSourceInConsole({ + target, + browser, + identity, + appSlug, + expectedNotice: "Removed: extra-tool", + expectedToolCount: "2 tools", + }); + + yield* removeSourceThroughConsole({ target, browser, identity, appSlug }); + + const afterRemove = yield* Effect.promise(() => + request( + target, + identity, + `/api/tools?integration=${encodeURIComponent(appSlug)}`, + ), + ); + expect(afterRemove.body).toEqual([]); + }), + Effect.promise(async () => { + await deletePath( + target, + identity, + `/api/apps/sources/github/${encodeURIComponent(appSlug)}`, + ); + await deletePath(target, identity, "/api/connections/user/github/main"); + await deletePath(target, identity, "/api/openapi/integrations/github"); + }).pipe(Effect.ignore), + ); + }), + ), +); diff --git a/packages/core/execution/src/tool-invoker.test.ts b/packages/core/execution/src/tool-invoker.test.ts index f117f1706..9216b8f4f 100644 --- a/packages/core/execution/src/tool-invoker.test.ts +++ b/packages/core/execution/src/tool-invoker.test.ts @@ -266,6 +266,27 @@ const errorPlugin = makeTestPlugin({ ], }); +const bindingErrorPlugin = makeTestPlugin({ + pluginId: "binding-error-test", + integration: "binding", + tools: [ + { + name: "sync", + description: "Sync with a caller-selected connection", + inputJsonSchema: EmptyInputJson, + validator: EmptyValidator, + handler: () => + Effect.fail({ + _tag: "BindingError", + message: 'unknown connection "personal" for role "github" (github)', + role: "github", + integration: "github", + requestedConnection: "personal", + }), + }, + ], +}); + const oauthErrorPlugin = definePlugin(() => ({ id: "oauth-error-test" as const, storage: () => ({}), @@ -1075,6 +1096,33 @@ describe("tool discovery", () => { }), ); + it.effect("returns binding errors as ToolResult.fail", () => + Effect.gen(function* () { + const executor = yield* makeExecutorWith([bindingErrorPlugin] as const); + yield* provision(executor as never, [ + { pluginId: "binding-error-test", integration: "binding" }, + ]); + const invoker = makeExecutorToolInvoker(executor, { + invokeOptions: { onElicitation: acceptAll }, + }); + + const result = yield* invoker.invoke({ path: "binding.org.main.sync", args: {} }); + + expect(result).toEqual({ + ok: false, + error: { + code: "binding_error", + message: 'unknown connection "personal" for role "github" (github)', + details: { + role: "github", + integration: "github", + requestedConnection: "personal", + }, + }, + }); + }), + ); + it.effect("preserves nested upstream error bodies through ToolResult.fail", () => Effect.gen(function* () { const executor = yield* makeExecutorWith([structuredFailurePlugin] as const); diff --git a/packages/core/execution/src/tool-invoker.ts b/packages/core/execution/src/tool-invoker.ts index be6105d6d..2b47970f6 100644 --- a/packages/core/execution/src/tool-invoker.ts +++ b/packages/core/execution/src/tool-invoker.ts @@ -188,6 +188,28 @@ const credentialResolutionToolFailure = (input: { }, }); +const bindingToolFailure = (value: unknown): ToolError | null => { + if (!Predicate.isTagged(value, "BindingError")) return null; + const maybeBinding = value as { + readonly message?: unknown; + readonly role?: unknown; + readonly integration?: unknown; + readonly requestedConnection?: unknown; + }; + const details: Record = {}; + if (typeof maybeBinding.role === "string") details.role = maybeBinding.role; + if (typeof maybeBinding.integration === "string") details.integration = maybeBinding.integration; + if (typeof maybeBinding.requestedConnection === "string") { + details.requestedConnection = maybeBinding.requestedConnection; + } + return { + code: "binding_error", + message: + typeof maybeBinding.message === "string" ? maybeBinding.message : "Tool binding failed.", + ...(Object.keys(details).length > 0 ? { details } : {}), + }; +}; + const expectedToolFailure = (value: unknown): ToolError | null => { if (Predicate.isTagged(value, "ToolNotFoundError") && "address" in value) { const suggestions = @@ -210,6 +232,8 @@ const expectedToolFailure = (value: unknown): ToolError | null => { } if (Predicate.isTagged(value, "ToolInvocationError")) { const cause = (value as { readonly cause?: unknown }).cause; + const binding = bindingToolFailure(cause); + if (binding) return binding; const issues = validationIssues(cause); if (issues) { return { diff --git a/packages/core/sdk/src/executor.ts b/packages/core/sdk/src/executor.ts index 071d4a21b..352866793 100644 --- a/packages/core/sdk/src/executor.ts +++ b/packages/core/sdk/src/executor.ts @@ -1927,6 +1927,19 @@ export const createExecutor = + pluginStorageFailure(existing.plugin_id, "removeIntegration", cause), + ), + ); + } // Drop owned connections / tools / definitions for this integration. const where = (b: AnyCb) => b("integration", "=", String(slug)); yield* core.deleteMany("tool", { where }); @@ -2071,6 +2084,7 @@ export const createExecutor = + pluginStorageFailure(row.plugin_id, "projectToolSchema", cause), + ), + ) + : null; + const inputSchema = + projected && Object.prototype.hasOwnProperty.call(projected, "inputSchema") + ? projected.inputSchema + : tool.inputSchema; + const outputSchema = + projected && Object.prototype.hasOwnProperty.call(projected, "outputSchema") + ? projected.outputSchema + : tool.outputSchema; + const definitionRows = yield* core.findMany("definition", { where: (b: AnyCb) => b.and( @@ -3099,15 +3137,12 @@ export const createExecutor = (); for (const def of definitionRows) defs.set(def.name, decodeJsonColumn(def.schema)); - const referenced = collectReferencedDefinitions( - [tool.inputSchema, tool.outputSchema], - defs, - ); + const referenced = collectReferencedDefinitions([inputSchema, outputSchema], defs); const preview = yield* Effect.tryPromise({ try: () => buildToolTypeScriptPreview({ - inputSchema: tool.inputSchema, - outputSchema: tool.outputSchema, + inputSchema, + outputSchema, defs, }), catch: (cause) => @@ -3119,8 +3154,8 @@ export const createExecutor = 0 ? (referenced as Record) @@ -3147,6 +3182,53 @@ export const createExecutor = => + Effect.gen(function* () { + const provider = credentialProviders.get(String(key)); + if (!provider) return null; + return yield* provider.get(id); + }); + + const providersHas = ( + key: ProviderKey, + id: ProviderItemId, + ): Effect.Effect => + Effect.gen(function* () { + const provider = credentialProviders.get(String(key)); + if (!provider) return false; + if (provider.has) return yield* provider.has(id); + const value = yield* provider.get(id); + return value !== null; + }); + + const providersSetDefault = ( + id: ProviderItemId, + value: string, + ): Effect.Effect => + Effect.gen(function* () { + const provider = defaultWritableProvider(); + if (!provider || !provider.set) { + return yield* new CredentialProviderNotRegisteredError({ + provider: ProviderKey.make("default"), + }); + } + yield* provider.set(id, value); + return provider.key; + }); + + const providersRemove = ( + key: ProviderKey, + id: ProviderItemId, + ): Effect.Effect => + Effect.gen(function* () { + const provider = credentialProviders.get(String(key)); + if (!provider || !provider.delete) return; + yield* provider.delete(id); + }); + // ------------------------------------------------------------------ // Policies — owner-ranked (user=0 inner, org=1 outer). // ------------------------------------------------------------------ @@ -3576,6 +3658,7 @@ export const createExecutor = providersList(), items: (key) => providersItems(key), + get: (key, id) => providersGet(key, id), + has: (key, id) => providersHas(key, id), + setDefault: (id, value) => providersSetDefault(id, value), + remove: (key, id) => providersRemove(key, id), }, oauth, + execute: (address, args, options) => execute(address, args, options), transaction: (effect: Effect.Effect) => transaction(effect), }; diff --git a/packages/core/sdk/src/plugin.ts b/packages/core/sdk/src/plugin.ts index 31dfa07eb..3885a11ba 100644 --- a/packages/core/sdk/src/plugin.ts +++ b/packages/core/sdk/src/plugin.ts @@ -26,9 +26,11 @@ import type { ConnectionName, IntegrationSlug, Owner, + ProviderItemId, ProviderKey, Subject, Tenant, + ToolAddress, } from "./ids"; import type { IntegrationDetectionResult } from "./types"; import type { @@ -36,8 +38,10 @@ import type { ElicitationHandler, ElicitationRequest, ElicitationResponse, + InvokeOptions, } from "./elicitation"; import type { + ExecuteError, ConnectionNotFoundError, CredentialProviderNotRegisteredError, IntegrationNotFoundError, @@ -244,11 +248,41 @@ export interface PluginCtx { readonly items: ( provider: ProviderKey, ) => Effect.Effect; + /** Read an opaque item from a provider. Plugins use this for secret values + * they own that are not modeled as connections. */ + readonly get: ( + provider: ProviderKey, + id: ProviderItemId, + ) => Effect.Effect; + readonly has: ( + provider: ProviderKey, + id: ProviderItemId, + ) => Effect.Effect; + /** Write through the executor's default writable provider and return the + * provider key that owns the item. */ + readonly setDefault: ( + id: ProviderItemId, + value: string, + ) => Effect.Effect; + readonly remove: ( + provider: ProviderKey, + id: ProviderItemId, + ) => Effect.Effect; }; /** Shared OAuth service. */ readonly oauth: OAuthService; + /** Invoke another catalog tool through the same executor request context: + * policy, approval, credential resolution, and plugin dispatch all stay in + * the core path. Intended for plugin sandboxes that expose higher-level + * virtual tools over existing integration tools. */ + readonly execute: ( + address: ToolAddress, + args: unknown, + options?: InvokeOptions, + ) => Effect.Effect; + /** Run `effect` inside a FumaDB transaction (atomic across plugin storage + * core integration/tool writes). */ readonly transaction: (effect: Effect.Effect) => Effect.Effect; @@ -261,6 +295,7 @@ export interface PluginCtx { // --------------------------------------------------------------------------- export interface ResolveToolsInput { + readonly ctx?: PluginCtx; /** The catalog record (public projection) whose connection is being resolved. */ readonly integration: Integration; /** The plugin's stored opaque config for that integration. */ @@ -299,6 +334,18 @@ export interface ResolveToolsResult { readonly incomplete?: boolean; } +export interface ProjectToolSchemaInput { + readonly ctx: PluginCtx; + readonly toolRow: ToolInvocationRow; + readonly inputSchema?: unknown; + readonly outputSchema?: unknown; +} + +export interface ProjectToolSchemaResult { + readonly inputSchema?: unknown; + readonly outputSchema?: unknown; +} + // --------------------------------------------------------------------------- // Resolved credential handed to `invokeTool` so the plugin renders auth onto // the request (D11: "auth state derived into the auth-template format"). @@ -454,6 +501,8 @@ export interface InvokeToolInput { readonly credential: ToolInvocationCredential; readonly args: unknown; readonly elicit: Elicit; + /** Original caller options for nested same-request tool calls. */ + readonly invokeOptions?: InvokeOptions; } /** Input for `validateToolArgs` — no credential/elicit: validation runs @@ -474,6 +523,11 @@ export interface ConnectionLifecycleInput { readonly connection: ConnectionRef; } +export interface IntegrationLifecycleInput { + readonly ctx: PluginCtx; + readonly integration: IntegrationRecord; +} + export interface ConfigureIntegrationHandlerInput { readonly ctx: PluginCtx; readonly integration: IntegrationSlug; @@ -585,6 +639,13 @@ export interface PluginSpec< * for catalogs derived purely from stored state (specs, static config). */ readonly remoteToolCatalog?: boolean; + /** Project a persisted tool row's stable schemas into the request-visible + * schema view. Use this only for volatile presentation data that should be + * current at read time, not persisted at catalog-refresh time. */ + readonly projectToolSchema?: ( + input: ProjectToolSchemaInput, + ) => Effect.Effect; + /** Invoke a dynamic tool. Called when the static-handler map doesn't have the * address. The plugin applies `input.credential` to the outbound request. */ readonly invokeTool?: (input: InvokeToolInput) => Effect.Effect; @@ -612,6 +673,12 @@ export interface PluginSpec< input: ConnectionLifecycleInput, ) => Effect.Effect; + /** Plugin-side cleanup when a removable integration is removed. Core still + * owns deleting the integration, connection, tool, and definition rows. */ + readonly removeIntegration?: ( + input: IntegrationLifecycleInput, + ) => Effect.Effect; + /** Core-dispatched integration configuration (beyond auth). */ readonly integrationConfigure?: IntegrationConfigureDecl; diff --git a/packages/plugins/apps/CHANGELOG.md b/packages/plugins/apps/CHANGELOG.md new file mode 100644 index 000000000..3605a25c4 --- /dev/null +++ b/packages/plugins/apps/CHANGELOG.md @@ -0,0 +1 @@ +# @executor-js/plugin-apps diff --git a/packages/plugins/apps/package.json b/packages/plugins/apps/package.json new file mode 100644 index 000000000..6d04fbdac --- /dev/null +++ b/packages/plugins/apps/package.json @@ -0,0 +1,74 @@ +{ + "name": "@executor-js/plugin-apps", + "version": "0.1.0", + "homepage": "https://github.com/UsefulSoftwareCo/executor/tree/main/packages/plugins/apps", + "bugs": { + "url": "https://github.com/UsefulSoftwareCo/executor/issues" + }, + "license": "MIT", + "repository": { + "type": "git", + "url": "git+https://github.com/UsefulSoftwareCo/executor.git", + "directory": "packages/plugins/apps" + }, + "files": [ + "dist" + ], + "type": "module", + "exports": { + ".": "./src/index.ts", + "./api": "./src/api.ts", + "./client": "./src/react/plugin-client.tsx", + "./seams": "./src/seams/index.ts", + "./testing": "./src/testing/index.ts" + }, + "scripts": { + "build": "tsup && (tsc --declaration --emitDeclarationOnly --outDir dist --rootDir src || true)", + "typecheck": "tsgo --noEmit", + "test": "vitest run", + "test:watch": "vitest", + "typecheck:slow": "bunx tsc --noEmit -p tsconfig.json" + }, + "dependencies": { + "@executor-js/codemode-core": "workspace:*", + "@executor-js/runtime-quickjs": "workspace:*", + "@executor-js/sdk": "workspace:*", + "@libsql/client": "catalog:", + "esbuild": "^0.27.3", + "zod": "4.3.6" + }, + "devDependencies": { + "@effect/atom-react": "catalog:", + "@effect/vitest": "catalog:", + "@executor-js/api": "workspace:*", + "@executor-js/react": "workspace:*", + "@types/node": "catalog:", + "@types/react": "catalog:", + "bun-types": "catalog:", + "effect": "catalog:", + "react": "catalog:", + "tsup": "catalog:", + "vitest": "catalog:" + }, + "peerDependencies": { + "@effect/atom-react": "catalog:", + "@executor-js/api": "workspace:*", + "@executor-js/react": "workspace:*", + "effect": "catalog:", + "react": "catalog:" + }, + "peerDependenciesMeta": { + "@effect/atom-react": { + "optional": true + }, + "@executor-js/api": { + "optional": true + }, + "@executor-js/react": { + "optional": true + }, + "react": { + "optional": true + } + } +} diff --git a/packages/plugins/apps/src/api.ts b/packages/plugins/apps/src/api.ts new file mode 100644 index 000000000..b028b16e4 --- /dev/null +++ b/packages/plugins/apps/src/api.ts @@ -0,0 +1,51 @@ +// Host-facing plugin surface for @executor-js/plugin-apps. +import { definePlugin } from "@executor-js/sdk/core"; + +import { + appsPlugin as appsCorePlugin, + APPS_INTEGRATION_SLUG, + APPS_PLUGIN_ID, + type AppsPluginOptions, +} from "./plugin/apps-plugin"; +import { AppsGroup } from "./plugin/routes"; +import { AppsHandlers, AppsExtensionService } from "./plugin/handlers"; + +export const appsPlugin = definePlugin((options?: AppsPluginOptions) => ({ + ...appsCorePlugin(options), + routes: () => AppsGroup, + handlers: () => AppsHandlers, + extensionService: AppsExtensionService, +})); + +export { APPS_INTEGRATION_SLUG, APPS_PLUGIN_ID, type AppsPluginOptions }; +export { AppsGroup } from "./plugin/routes"; +export { AppsHandlers, AppsExtensionService } from "./plugin/handlers"; +export { + makeSelfHostAppsRuntime, + type SelfHostAppsRuntime, + type SelfHostAppsRuntimeOptions, +} from "./plugin/self-host-runtime"; +export { type AppsRuntime, type GitHubCustomToolsSourceSummary } from "./plugin/runtime"; +export { makeAppsRuntimeFromBackings, type AppsBackings } from "./plugin/backings"; +export { makeGitArtifactStore } from "./backing/git-artifact-store"; +export { makeQuickjsToolSandbox } from "./backing/quickjs-tool-sandbox"; +export { makeSqliteAppsStore } from "./backing/sqlite-apps-store"; +export { + BindingError, + type ClientResolver, + type ConnectionCandidate, + type RoleBindings, +} from "./plugin/bindings"; +export { makePluginCtxAppsResolver } from "./plugin/resolver"; +export { SourceOriginError, assertSourceOrigin } from "./plugin/apps-plugin"; +export { + fetchGitHubSource, + parseGitHubSourceUrl, + syncGitHubSource, + GitHubSourceError, + type GitHubSourceInput, + type GitHubSourceSnapshot, + type GitHubSyncResult, + type SyncErrorData, + type SyncGitHubSourceInput, +} from "./source/github-source"; diff --git a/packages/plugins/apps/src/authoring.ts b/packages/plugins/apps/src/authoring.ts new file mode 100644 index 000000000..67ee1262c --- /dev/null +++ b/packages/plugins/apps/src/authoring.ts @@ -0,0 +1,61 @@ +import type { StandardSchemaV1 } from "./standard-schema"; + +export type JsonPrimitive = string | number | boolean | null; +export type JsonValue = JsonPrimitive | JsonObject | readonly JsonValue[]; +export interface JsonObject { + readonly [key: string]: JsonValue; +} + +export type ToolSchema = StandardSchemaV1 | JsonObject; + +export interface IntegrationDeclaration { + readonly integration: Slug; +} + +export type ToolHandlerContext< + TIntegrations extends Readonly> | undefined, +> = { + readonly [K in keyof NonNullable]: unknown; +}; + +type InferToolInput = TSchema extends StandardSchemaV1 + ? StandardSchemaV1.InferOutput + : unknown; + +type InferToolOutput = TSchema extends StandardSchemaV1 + ? StandardSchemaV1.InferOutput + : unknown; + +export interface DefineToolOptions< + TInputSchema extends ToolSchema, + TOutputSchema extends ToolSchema | undefined = undefined, + TIntegrations extends Readonly> | undefined = undefined, +> { + readonly description: string; + readonly integrations?: TIntegrations; + readonly input: TInputSchema; + readonly output?: TOutputSchema; + readonly annotations?: { + readonly readOnly?: boolean; + readonly destructive?: boolean; + readonly requiresApproval?: boolean; + }; + readonly handler: ( + input: InferToolInput, + context: ToolHandlerContext, + ) => + | Promise : unknown> + | (TOutputSchema extends ToolSchema ? InferToolOutput : unknown); +} + +export const integration = (slug: Slug): IntegrationDeclaration => ({ + integration: slug, +}); + +export const defineTool = < + TInputSchema extends ToolSchema, + TOutputSchema extends ToolSchema | undefined = undefined, + TIntegrations extends Readonly> | undefined = undefined, +>( + definition: DefineToolOptions, +): DefineToolOptions => definition; diff --git a/packages/plugins/apps/src/backing/git-artifact-store.test.ts b/packages/plugins/apps/src/backing/git-artifact-store.test.ts new file mode 100644 index 000000000..1b0320891 --- /dev/null +++ b/packages/plugins/apps/src/backing/git-artifact-store.test.ts @@ -0,0 +1,35 @@ +import { mkdtempSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +import { describe, expect, it } from "@effect/vitest"; +import { Effect, Exit } from "effect"; + +import { artifactStoreConformance } from "../seams/artifact-store.conformance"; +import { scopeAddress, scopeAddressStorageKey } from "../seams/scope-address"; +import { compareAndSwapSnapshotRef, makeGitArtifactStore } from "./git-artifact-store"; + +artifactStoreConformance("git", () => + makeGitArtifactStore({ root: mkdtempSync(join(tmpdir(), "apps-artifacts-")) }), +); + +const run = (effect: Effect.Effect): Promise => Effect.runPromise(effect); + +describe("git ArtifactStore CAS", () => { + it("returns a typed conflict when the expected head is stale", async () => { + const root = mkdtempSync(join(tmpdir(), "apps-artifacts-cas-")); + const address = scopeAddress("org", "s"); + const scope = await run(makeGitArtifactStore({ root }).forScope(address)); + const first = await run(scope.commit(new Map([["tools/a.ts", "a"]]), "first")); + await run(scope.commit(new Map([["tools/a.ts", "b"]]), "second")); + + const repoDir = join(root, `${scopeAddressStorageKey(address)}.git`); + const exit = await Effect.runPromiseExit( + compareAndSwapSnapshotRef(repoDir, String(first.id), String(first.id)), + ); + + expect(Exit.isFailure(exit)).toBe(true); + expect(JSON.stringify(exit)).toContain("publish conflict"); + expect(JSON.stringify(exit)).toContain('"conflict":true'); + }); +}); diff --git a/packages/plugins/apps/src/backing/git-artifact-store.ts b/packages/plugins/apps/src/backing/git-artifact-store.ts new file mode 100644 index 000000000..fbd5beb23 --- /dev/null +++ b/packages/plugins/apps/src/backing/git-artifact-store.ts @@ -0,0 +1,327 @@ +import { execFile } from "node:child_process"; +import { mkdir, rm } from "node:fs/promises"; +import { join } from "node:path"; + +import { Effect } from "effect"; + +import { + ArtifactStoreError, + asSnapshotId, + type ArtifactStore, + type FileSet, + type ScopeArtifactStore, + type SnapshotId, + type SnapshotMeta, +} from "../seams/artifact-store"; +import { scopeAddressStorageKey } from "../seams/scope-address"; + +// --------------------------------------------------------------------------- +// Git-backed ArtifactStore (self-hosted). One bare git repo per scope under +// `/.git`. Snapshots are commits, written via git plumbing +// (hash-object -> update-index -> write-tree -> commit-tree) so no working tree +// is ever checked out — the runtime only reads committed snapshots. The commit +// hash IS the SnapshotId; git guarantees immutability (content-addressed). +// --------------------------------------------------------------------------- + +const BRANCH = "refs/heads/main"; + +// The "must not exist" sentinel for `git update-ref `: an empty +// old-value asserts the ref currently has NO value (the first publish to a fresh +// scope repo). git treats the empty string as "the ref must not already exist". +const EMPTY_OID = ""; + +const stderrText = (stderr: string | Buffer): string => + Buffer.isBuffer(stderr) ? stderr.toString("utf8").trim() : String(stderr).trim(); + +const gitFailureMessage = (command: string, stderr: string | Buffer): string => { + const detail = stderrText(stderr); + return detail ? `git ${command} failed: ${detail}` : `git ${command} failed`; +}; + +const run = ( + cwd: string, + args: readonly string[], + input?: string | Buffer, +): Effect.Effect => + Effect.callback((resume) => { + const child = execFile( + "git", + args, + { cwd, maxBuffer: 256 * 1024 * 1024, encoding: "buffer" }, + (error, stdout, stderr) => { + if (error) { + resume( + Effect.fail( + new ArtifactStoreError({ + message: gitFailureMessage(args[0] ?? "command", stderr), + cause: error, + }), + ), + ); + return; + } + resume(Effect.succeed((stdout as Buffer).toString("utf8"))); + }, + ); + if (input !== undefined) { + child.stdin?.write(input); + child.stdin?.end(); + } + }); + +export const compareAndSwapSnapshotRef = ( + repoDir: string, + commitHash: string, + expectedOld: string | null, +): Effect.Effect => + run(repoDir, ["update-ref", BRANCH, commitHash, expectedOld ?? EMPTY_OID]).pipe( + Effect.asVoid, + Effect.mapError( + (cause) => + new ArtifactStoreError({ + message: `publish conflict: ${BRANCH} moved concurrently (expected ${ + expectedOld ?? EMPTY_OID + }); retry from the new head`, + conflict: true, + cause, + }), + ), + ); + +const makeScopeStore = (repoDir: string): ScopeArtifactStore => { + // Environment forcing a deterministic author/committer so a snapshot's hash + // depends only on its content + parent, not on wall-clock/identity drift. + const commitEnv = (message: string) => + Effect.gen(function* () { + const parent = yield* headCommit(); + // Build a tree from the provided file set held in a bare index file. + return { parent, message }; + }); + + const headCommit = (): Effect.Effect => + run(repoDir, ["rev-parse", "--verify", "--quiet", BRANCH]).pipe( + Effect.map((out) => out.trim() || null), + Effect.catch(() => Effect.succeed(null)), + ); + + const readMeta = (id: string): Effect.Effect => + run(repoDir, ["show", "-s", "--format=%H%n%ct%n%s", id]).pipe( + Effect.map((out) => { + const [hash, committed, ...subjectParts] = out.split("\n"); + return { + id: asSnapshotId(hash.trim()), + committedAt: Number(committed.trim()) * 1000, + message: subjectParts.join("\n").trim(), + } satisfies SnapshotMeta; + }), + ); + + return { + commit: (files: FileSet, message: string) => + Effect.gen(function* () { + const { parent } = yield* commitEnv(message); + // Use a throwaway index so we never touch a working tree. + const indexFile = join( + repoDir, + `commit-index-${Date.now()}-${Math.random().toString(36).slice(2)}`, + ); + const withIndex = (args: readonly string[], input?: string | Buffer) => + Effect.callback((resume) => { + const child = execFile( + "git", + args, + { + cwd: repoDir, + maxBuffer: 256 * 1024 * 1024, + encoding: "buffer", + env: { ...process.env, GIT_INDEX_FILE: indexFile }, + }, + (error, stdout, stderr) => { + if (error) { + resume( + Effect.fail( + new ArtifactStoreError({ + message: gitFailureMessage(args[0] ?? "command", stderr), + cause: error, + }), + ), + ); + return; + } + resume(Effect.succeed((stdout as Buffer).toString("utf8"))); + }, + ); + if (input !== undefined) { + child.stdin?.write(input); + child.stdin?.end(); + } + }); + + // Start from an empty index each publish (full file set, not a diff). + yield* withIndex(["read-tree", "--empty"]); + for (const [path, contents] of files) { + const blobHash = (yield* withIndex( + ["hash-object", "-w", "--stdin"], + Buffer.from(contents, "utf8"), + )).trim(); + yield* withIndex(["update-index", "--add", "--cacheinfo", `100644,${blobHash},${path}`]); + } + const treeHash = (yield* withIndex(["write-tree"])).trim(); + const commitArgs = ["commit-tree", treeHash, "-m", message]; + if (parent) commitArgs.push("-p", parent); + const commitEnvVars = { + ...process.env, + GIT_AUTHOR_NAME: "executor-apps", + GIT_AUTHOR_EMAIL: "apps@executor.local", + GIT_COMMITTER_NAME: "executor-apps", + GIT_COMMITTER_EMAIL: "apps@executor.local", + }; + const commitHash = (yield* Effect.callback((resume) => { + execFile( + "git", + commitArgs, + { cwd: repoDir, encoding: "buffer", env: commitEnvVars }, + (error, stdout, stderr) => { + if (error) { + resume( + Effect.fail( + new ArtifactStoreError({ + message: gitFailureMessage("commit-tree", stderr), + cause: error, + }), + ), + ); + return; + } + resume(Effect.succeed((stdout as Buffer).toString("utf8"))); + }, + ); + })).trim(); + // Compare-and-swap the branch ref: `update-ref ` fails + // if HEAD moved since we read `parent`, so a concurrent publish that + // committed first cannot be silently clobbered. On the first commit there + // is no parent, so we assert the ref is absent (the empty-oid form). A CAS + // failure surfaces as a typed conflict for the caller to retry from a + // fresh parent. + yield* compareAndSwapSnapshotRef(repoDir, commitHash, parent); + return yield* readMeta(commitHash); + }), + + read: (id: SnapshotId) => + run(repoDir, ["ls-tree", "-r", "--name-only", id]).pipe( + Effect.flatMap((out) => { + const paths = out + .split("\n") + .map((p) => p.trim()) + .filter(Boolean); + return Effect.forEach( + paths, + (path) => + run(repoDir, ["cat-file", "blob", `${id}:${path}`]).pipe( + Effect.map((contents) => [path, contents] as const), + ), + { concurrency: 8 }, + ); + }), + Effect.map((entries) => new Map(entries) as FileSet), + ), + + readFile: (id: SnapshotId, path: string) => + run(repoDir, ["cat-file", "blob", `${id}:${path}`]).pipe( + Effect.map((contents) => contents as string | null), + Effect.catch(() => Effect.succeed(null)), + ), + + list: (id: SnapshotId) => + run(repoDir, ["ls-tree", "-r", "--name-only", id]).pipe( + Effect.map((out) => + out + .split("\n") + .map((p) => p.trim()) + .filter(Boolean), + ), + ), + + latest: () => + headCommit().pipe(Effect.flatMap((head) => (head ? readMeta(head) : Effect.succeed(null)))), + + log: (limit = 50) => + headCommit().pipe( + Effect.flatMap((head) => { + if (!head) return Effect.succeed([] as readonly SnapshotMeta[]); + return run(repoDir, [ + "log", + `--max-count=${limit}`, + "--format=%H%x1f%ct%x1f%s", + BRANCH, + ]).pipe( + Effect.map((out) => + out + .split("\n") + .map((line) => line.trim()) + .filter(Boolean) + .map((line) => { + const [hash, committed, subject] = line.split("\x1f"); + return { + id: asSnapshotId(hash), + committedAt: Number(committed) * 1000, + message: subject ?? "", + } satisfies SnapshotMeta; + }), + ), + ); + }), + ), + }; +}; + +export interface GitArtifactStoreOptions { + /** Directory holding one bare repo per tenant/scope. Created on demand. */ + readonly root: string; +} + +/** Build the git-backed ArtifactStore. Each tenant/scope pair gets a + * lazily-initialized bare repo under ``. */ +export const makeGitArtifactStore = (options: GitArtifactStoreOptions): ArtifactStore => { + const initialized = new Map>(); + + const init = async (key: string): Promise => { + const repoDir = join(options.root, `${key}.git`); + await mkdir(repoDir, { recursive: true }); + await Effect.runPromise(run(repoDir, ["init", "--bare", "--quiet"])); + return makeScopeStore(repoDir); + }; + + return { + forScope: (address) => + Effect.tryPromise({ + try: () => { + const key = scopeAddressStorageKey(address); + let existing = initialized.get(key); + if (!existing) { + existing = init(key); + initialized.set(key, existing); + } + return existing; + }, + catch: (cause) => + new ArtifactStoreError({ + message: `failed to open scope repo: ${address.tenant}/${address.scope}`, + cause, + }), + }), + removeScope: (address) => + Effect.tryPromise({ + try: async () => { + const key = scopeAddressStorageKey(address); + initialized.delete(key); + await rm(join(options.root, `${key}.git`), { recursive: true, force: true }); + }, + catch: (cause) => + new ArtifactStoreError({ + message: `failed to remove scope repo: ${address.tenant}/${address.scope}`, + cause, + }), + }), + }; +}; diff --git a/packages/plugins/apps/src/backing/quickjs-tool-sandbox.conformance.test.ts b/packages/plugins/apps/src/backing/quickjs-tool-sandbox.conformance.test.ts new file mode 100644 index 000000000..53e0a9a66 --- /dev/null +++ b/packages/plugins/apps/src/backing/quickjs-tool-sandbox.conformance.test.ts @@ -0,0 +1,7 @@ +import { toolSandboxConformance } from "../seams/tool-sandbox.conformance"; +import { makeQuickjsToolSandbox } from "./quickjs-tool-sandbox"; + +// Short timeout so the "kills a runaway handler" case finishes quickly. +toolSandboxConformance("quickjs", () => + makeQuickjsToolSandbox({ collectTimeoutMs: 5_000, invokeTimeoutMs: 2_000 }), +); diff --git a/packages/plugins/apps/src/backing/quickjs-tool-sandbox.test.ts b/packages/plugins/apps/src/backing/quickjs-tool-sandbox.test.ts new file mode 100644 index 000000000..f1ed4bda1 --- /dev/null +++ b/packages/plugins/apps/src/backing/quickjs-tool-sandbox.test.ts @@ -0,0 +1,201 @@ +import { describe, expect, it } from "@effect/vitest"; +import { Effect, Exit } from "effect"; + +import { bundleEntry } from "../pipeline/bundle"; +import { makeQuickjsToolSandbox } from "./quickjs-tool-sandbox"; +import { ISSUES_SYNC_TS } from "../testing/daily-brief"; +import { InputValidationError, type HandleBridge } from "../seams/tool-sandbox"; + +const run = (effect: Effect.Effect): Promise => Effect.runPromise(effect); + +describe("QuickJS ToolSandbox", () => { + const sandbox = makeQuickjsToolSandbox(); + + it("bundles + collects a tool descriptor (deterministic)", async () => { + const files = new Map([["tools/issues-sync.ts", ISSUES_SYNC_TS]]); + const { code } = await run(bundleEntry({ files, entry: "tools/issues-sync.ts" })); + const result = await run(sandbox.collect(code)); + const descriptor = result.artifacts.default.descriptor as { + kind: string; + integrations: Record; + inputJsonSchema: { type: string; properties: Record }; + hasHandler: boolean; + }; + expect(descriptor.kind).toBe("tool"); + expect(descriptor.integrations.github).toEqual( + expect.objectContaining({ integration: "github" }), + ); + expect(descriptor.hasHandler).toBe(true); + expect(descriptor.inputJsonSchema.type).toBe("object"); + expect(descriptor.inputJsonSchema.properties).toHaveProperty("repos"); + expect(descriptor.inputJsonSchema.properties).toHaveProperty("since"); + }); + + it("invokes a tool, routing injected-client calls through the bridge", async () => { + const files = new Map([["tools/issues-sync.ts", ISSUES_SYNC_TS]]); + const { code } = await run(bundleEntry({ files, entry: "tools/issues-sync.ts" })); + + const calls: { root: string; path: readonly string[]; args: readonly unknown[] }[] = []; + const bridge: HandleBridge = { + call: ({ root, path, args }) => + Effect.sync(() => { + calls.push({ root, path, args }); + const key = `${root}.${path.join(".")}`; + if (key === "github.repos.listForAuthenticatedUser") { + return [{ full_name: "acme/app" }]; + } + if (key === "github.issues.listForRepo") { + return [ + { + number: 1, + title: "Bug", + labels: [{ name: "bug" }], + assignee: { login: "rhys" }, + updated_at: "2026-01-01T00:00:00Z", + html_url: "https://github.com/acme/app/issues/1", + }, + ]; + } + return []; + }), + }; + + const result = await run( + sandbox.invoke( + code, + { + artifact: "issues-sync", + kind: "tool", + input: {}, + roots: { github: { kind: "single" } }, + }, + bridge, + ), + ); + + expect(result.output).toEqual({ + synced: 1, + repos: 1, + issues: [{ repo: "acme/app", number: 1, title: "Bug" }], + }); + expect(calls.some((c) => c.root === "github")).toBe(true); + }); + + it("reports storage as unavailable when an old handler calls db.sql", async () => { + const source = `import { defineTool } from "executor:app"; +import { z } from "zod"; +export default defineTool({ + description: "legacy storage", + input: z.object({}), + async handler(_input, { db }) { + await db.sql\`SELECT 1\`; + return { ok: true }; + }, +});`; + const files = new Map([["tools/storage.ts", source]]); + const { code } = await run(bundleEntry({ files, entry: "tools/storage.ts" })); + const bridge: HandleBridge = { call: () => Effect.succeed(null) }; + + const exit = await Effect.runPromiseExit( + sandbox.invoke(code, { artifact: "storage", kind: "tool", input: {}, roots: {} }, bridge), + ); + + expect(Exit.isFailure(exit)).toBe(true); + expect(JSON.stringify(exit)).toContain("storage is not available yet"); + }); + + it("surfaces Standard Schema validation issues before the handler runs", async () => { + const source = `import { defineTool } from "executor:app"; +import { z } from "zod"; +export default defineTool({ + description: "validates", + input: z.object({ q: z.string() }), + async handler(){ return { ok: true }; }, +});`; + const files = new Map([["tools/validate.ts", source]]); + const { code } = await run(bundleEntry({ files, entry: "tools/validate.ts" })); + const bridge: HandleBridge = { call: () => Effect.succeed(null) }; + + const exit = await Effect.runPromiseExit( + sandbox.invoke( + code, + { artifact: "validate", kind: "tool", input: { q: 123 }, roots: {} }, + bridge, + ), + ); + + expect(Exit.isFailure(exit)).toBe(true); + expect(JSON.stringify(exit)).toContain("InputValidationError"); + expect(JSON.stringify(exit)).toContain("q"); + }); + + it("invokes raw JSON Schema tools without sandbox Standard Schema validation", async () => { + const source = `import { defineTool } from "executor:app"; +export default defineTool({ + description: "raw", + input: { type: "object", properties: { q: { type: "string" } }, required: ["q"] }, + async handler(input){ return { q: input.q }; }, +});`; + const files = new Map([["tools/raw.ts", source]]); + const { code } = await run(bundleEntry({ files, entry: "tools/raw.ts" })); + const bridge: HandleBridge = { call: () => Effect.succeed(null) }; + + const result = await run( + sandbox.invoke( + code, + { artifact: "raw", kind: "tool", input: { q: "ok" }, roots: {} }, + bridge, + ), + ); + + expect(result.output).toEqual({ q: "ok" }); + }); + + it("rejects a non-deterministic descriptor (Math.random at describe time)", async () => { + const nondeterministic = `import { defineTool } from "executor:app"; +import { z } from "zod"; +export default defineTool({ + description: "x-" + Math.random(), + input: z.object({}), + async handler() { return {}; }, +});`; + const files = new Map([["tools/rng.ts", nondeterministic]]); + const { code } = await run(bundleEntry({ files, entry: "tools/rng.ts" })); + const exit = await Effect.runPromiseExit(sandbox.collect(code)); + expect(Exit.isFailure(exit)).toBe(true); + }); + + it("denies network (fetch throws in the sandbox)", async () => { + const usesFetch = `import { defineTool } from "executor:app"; +import { z } from "zod"; +export default defineTool({ + description: "fetches", + input: z.object({}), + async handler() { await fetch("https://example.com"); return {}; }, +});`; + const files = new Map([["tools/net.ts", usesFetch]]); + const { code } = await run(bundleEntry({ files, entry: "tools/net.ts" })); + const bridge: HandleBridge = { call: () => Effect.succeed(null) }; + const exit = await Effect.runPromiseExit( + sandbox.invoke(code, { artifact: "net", kind: "tool", input: {}, roots: {} }, bridge), + ); + expect(Exit.isFailure(exit)).toBe(true); + }); + + it("returns typed InputValidationError from promise rejection", async () => { + const source = `import { defineTool } from "executor:app"; +import { z } from "zod"; +export default defineTool({ + description: "typed", + input: z.object({ name: z.string() }), + async handler(){ return {}; }, +});`; + const files = new Map([["tools/typed.ts", source]]); + const { code } = await run(bundleEntry({ files, entry: "tools/typed.ts" })); + const bridge: HandleBridge = { call: () => Effect.succeed(null) }; + + await expect( + run(sandbox.invoke(code, { artifact: "typed", kind: "tool", input: {}, roots: {} }, bridge)), + ).rejects.toBeInstanceOf(InputValidationError); + }); +}); diff --git a/packages/plugins/apps/src/backing/quickjs-tool-sandbox.ts b/packages/plugins/apps/src/backing/quickjs-tool-sandbox.ts new file mode 100644 index 000000000..df87118cc --- /dev/null +++ b/packages/plugins/apps/src/backing/quickjs-tool-sandbox.ts @@ -0,0 +1,398 @@ +import { Effect, Option, Predicate, Schema } from "effect"; +import type { SandboxToolInvoker } from "@executor-js/codemode-core"; +import { makeQuickJsExecutor } from "@executor-js/runtime-quickjs"; + +import { + InputValidationError, + OutputValidationError, + ToolSandboxError, + type CollectResult, + type CollectRequest, + type HandleBridge, + type InvokeRequest, + type InvokeResult, + type ValidationIssue, + type ToolSandbox, +} from "../seams/tool-sandbox"; +import { stableStringify } from "../pipeline/descriptor"; + +// --------------------------------------------------------------------------- +// QuickJS-backed ToolSandbox (self-hosted). +// +// The published bundle is a CJS string. The sandbox body prepends a `require` +// shim providing `executor:app` (defineTool/integration), +// then executes the bundle so `module.exports.default` is the artifact. A +// driver appended after the bundle either collects the descriptor (nothing +// bound) or runs one handler with injected clients. +// +// Injected clients are Proxies that turn `github.issues.listForRepo(args)` into +// `await __invokeTool("__handle__", { root, path, args })` — the ONE bridge the +// QuickJS runtime already provides (`tools`/`__invokeTool`). Our +// `SandboxToolInvoker` decodes that and forwards to the host `HandleBridge`. +// The old `db` handle is intentionally unavailable in v1; handlers that still +// try it get a clear error before any host storage call exists. +// Everything crossing is JSON (the cloud version is RPC), so the interface +// stays honest. +// +// Determinism: `collect` runs the collection body twice and byte-compares the +// descriptor JSON. Effectful top-levels (Math.random, Date.now) diverge and are +// rejected. QuickJS denies `fetch` and enforces a deadline + memory cap. +// --------------------------------------------------------------------------- + +const COLLECT_TIMEOUT_MS = 10_000; +const INVOKE_TIMEOUT_MS = 30_000; + +// The shim + module system injected before the bundle. Kept as a plain string +// (QuickJS evals a string). `defineTool` records its def so the driver can read +// it back. Clients are built by `__mkHandle`. +const runtimePrelude = ` +var __modules = {}; +var __defs = { tool: null }; +function __recordDefault(mod) { return mod; } +var __handleBridge = function(root, path, args) { + // Route every injected-client method call through the single tool bridge. + return tools.__handle__({ root: root, path: path, args: args }); +}; +function __mkHandle(root, prefix) { + var target = function(){}; + return new Proxy(target, { + get: function(_t, prop) { + if (prop === 'then' || typeof prop === 'symbol') return undefined; + return __mkHandle(root, prefix.concat([String(prop)])); + }, + apply: function(_t, _this, callArgs) { + return __handleBridge(root, prefix, callArgs); + } + }); +} +function __unavailableStorage() { + return new Proxy({}, { + get: function() { + return function() { throw new Error('storage is not available yet'); }; + }, + apply: function() { + throw new Error('storage is not available yet'); + } + }); +} +var __executorApp = { + integration: function(integration) { return { integration: integration }; }, + defineTool: function(def) { __defs.tool = def; return def; }, +}; +function __require(id) { + if (id === 'executor:app') return __executorApp; + throw new Error('module not available in sandbox: ' + id); +} +`; + +// Run the CJS bundle. The bundle's virtual entry sets `globalThis.__artifact` +// (the def object returned by defineTool). `require` is our shim; `defineTool` +// also records into `__defs` as a fallback. +const wrapBundle = (bundle: string): string => ` +var module = { exports: {} }; +var exports = module.exports; +var require = __require; +(function(module, exports, require){ +${bundle} +})(module, exports, require); +`; + +const decodeJsonMarker = Schema.decodeUnknownOption(Schema.UnknownFromJsonString); + +const isRecord = (value: unknown): value is Record => + value !== null && typeof value === "object" && !Array.isArray(value); + +const parseMarker = (message: string): unknown => { + const parse = (text: string): unknown => Option.getOrNull(decodeJsonMarker(text)); + const direct = parse(message); + if (direct) return direct; + const firstBrace = message.indexOf("{"); + const lastBrace = message.lastIndexOf("}"); + if (firstBrace !== -1 && lastBrace > firstBrace) { + return parse(message.slice(firstBrace, lastBrace + 1)); + } + return null; +}; + +const isMarker = (value: unknown, tag: string): value is Record => { + if (!isRecord(value)) return false; + return Predicate.isTagged(tag)(value); +}; + +const markerMessage = (marker: Record, fallback: string): string => + typeof marker.message === "string" ? marker.message : fallback; + +const markerIssues = (marker: Record): readonly ValidationIssue[] => + Array.isArray(marker.issues) ? marker.issues.filter(isValidationIssue) : []; + +const isValidationIssue = (value: unknown): value is ValidationIssue => + isRecord(value) && + typeof value.message === "string" && + (value.path === undefined || Array.isArray(value.path)); + +// Collect driver: describe the artifact's integrations + input/output schema. +// Deterministic JSON only — no handler execution. +const collectDriver = (artifact: string): string => ` +return await (async () => { + var artifactName = ${JSON.stringify(artifact)}; + var fail = function(tag, data) { + var payload = data || {}; + payload._tag = tag; + throw JSON.stringify(payload); + }; + var sanitizeJson = function(value) { + if (value === undefined) return undefined; + if (value === null || typeof value !== 'object') return value; + if (Array.isArray(value)) return value.map(sanitizeJson); + var out = {}; + for (var key in value) { + if (key === '~standard') continue; + if (typeof value[key] === 'function' || value[key] === undefined) continue; + out[key] = sanitizeJson(value[key]); + } + return out; + }; + var schemaToJson = function(field, schema, direction) { + if (schema === undefined) return undefined; + if (schema && typeof schema === 'object' && schema['~standard']) { + var standard = schema['~standard']; + var vendor = standard && standard.vendor ? String(standard.vendor) : 'unknown'; + var jsonSchema = standard && standard.jsonSchema; + if (!jsonSchema || typeof jsonSchema[direction] !== 'function') { + fail('SchemaExportError', { + tool: artifactName, + field: field, + vendor: vendor, + message: 'tool "' + artifactName + '" ' + field + ' schema from "' + vendor + '" is not supported for schema export', + }); + } + return sanitizeJson(jsonSchema[direction]({ target: 'draft-07' })); + } + if (schema && typeof schema === 'object' && !Array.isArray(schema)) { + return sanitizeJson(schema); + } + fail('SchemaExportError', { + tool: artifactName, + field: field, + vendor: 'raw', + message: 'tool "' + artifactName + '" ' + field + ' schema must be a Standard Schema or raw JSON Schema object', + }); + }; + var def = __defs.tool || (globalThis.__artifact && (globalThis.__artifact.default || globalThis.__artifact)); + if (def && typeof def.handler === 'function') { + var handlerSource = Function.prototype.toString.call(def.handler); + if (/\\bdb\\b/.test(handlerSource)) { + fail('StorageUnavailableError', { + tool: artifactName, + field: 'handler', + message: 'tool "' + artifactName + '" uses storage, but storage is not available yet', + }); + } + } + var integrations = {}; + if (def && def.integrations) { + for (var k in def.integrations) { + var c = def.integrations[k]; + integrations[k] = { integration: c && c.integration ? String(c.integration) : '' }; + } + } + return { + artifact: artifactName, + kind: 'tool', + description: def && def.description, + integrations: integrations, + annotations: def && def.annotations, + hasHandler: !!(def && def.handler), + inputJsonSchema: def ? schemaToJson('input', def.input, 'input') : undefined, + outputJsonSchema: def && def.output !== undefined ? schemaToJson('output', def.output, 'output') : undefined, + }; +})() +`; + +const buildCollectCode = (bundle: string, request?: CollectRequest): string => + runtimePrelude + wrapBundle(bundle) + collectDriver(request?.artifact ?? "default"); + +// Invoke driver: build injected clients from the request roots, validate +// Standard Schema input/output inside the sandbox, then call the artifact's +// handler with (input, injected). +const buildInvokeDriver = (request: InvokeRequest): string => { + const rootsLiteral = JSON.stringify(request.roots); + const inputLiteral = JSON.stringify(request.input ?? {}); + return ` +return await (async () => { + var def = __defs.tool || (globalThis.__artifact && (globalThis.__artifact.default || globalThis.__artifact)); + if (!def || typeof def.handler !== 'function') throw new Error('artifact has no handler: ${request.artifact}'); + var failValidation = function(tag, field, issues) { + throw JSON.stringify({ + _tag: tag, + tool: '${request.artifact}', + field: field, + message: 'tool "${request.artifact}" ' + field + ' validation failed', + issues: JSON.parse(JSON.stringify(issues || [])), + }); + }; + var validateWithStandardSchema = async function(field, schema, value) { + if (!schema || typeof schema !== 'object' || !schema['~standard']) return value; + var standard = schema['~standard']; + if (!standard || typeof standard.validate !== 'function') return value; + var result = await standard.validate(value); + if (result && result.issues) { + failValidation(field === 'input' ? 'InputValidationError' : 'OutputValidationError', field, result.issues); + } + return result && Object.prototype.hasOwnProperty.call(result, 'value') ? result.value : value; + }; + var roots = ${rootsLiteral}; + var injected = {}; + for (var name in roots) { + var spec = roots[name]; + if (spec.kind !== 'single') throw new Error('unsupported handle root kind: ' + spec.kind); + injected[name] = __mkHandle(name, []); + } + injected.db = __unavailableStorage(); + var input = await validateWithStandardSchema('input', def.input, ${inputLiteral}); + var out = await def.handler(input, injected); + out = await validateWithStandardSchema('output', def.output, out); + return out; +})() +`; +}; + +const buildInvokeCode = (bundle: string, request: InvokeRequest): string => + runtimePrelude + wrapBundle(bundle) + buildInvokeDriver(request); + +// A no-op invoker for collect: no handle calls should happen; if they do +// (misbehaving describe path), fail loudly. +const collectInvoker: SandboxToolInvoker = { + invoke: () => + Effect.fail( + new ToolSandboxError({ kind: "collect", message: "collect must not make handle calls" }), + ), +}; + +export interface QuickjsToolSandboxOptions { + readonly collectTimeoutMs?: number; + readonly invokeTimeoutMs?: number; +} + +export const makeQuickjsToolSandbox = (options: QuickjsToolSandboxOptions = {}): ToolSandbox => { + const collectExecutor = makeQuickJsExecutor({ + timeoutMs: options.collectTimeoutMs ?? COLLECT_TIMEOUT_MS, + }); + const invokeExecutor = makeQuickJsExecutor({ + timeoutMs: options.invokeTimeoutMs ?? INVOKE_TIMEOUT_MS, + }); + + const runCollect = (code: string): Effect.Effect => + collectExecutor.execute(code, collectInvoker).pipe( + Effect.mapError( + (cause) => new ToolSandboxError({ kind: "collect", message: "collect run failed", cause }), + ), + Effect.flatMap((result) => { + if (result.error) { + const marker = parseMarker(result.error); + if ( + isMarker(marker, "SchemaExportError") || + isMarker(marker, "StorageUnavailableError") + ) { + return Effect.fail( + new ToolSandboxError({ + kind: "collect", + message: markerMessage(marker, result.error), + cause: marker, + }), + ); + } + return Effect.fail(new ToolSandboxError({ kind: "collect", message: result.error })); + } + return Effect.succeed(result.result); + }), + ); + + return { + collect: (bundle: string, request?: CollectRequest) => + Effect.gen(function* () { + // Run twice, byte-compare (determinism gate). Key-sorted stringify so a + // false mismatch never comes from property-order luck — a real + // divergence (Math.random / Date.now) still fails. + const first = yield* runCollect(buildCollectCode(bundle, request)); + const second = yield* runCollect(buildCollectCode(bundle, request)); + const a = stableStringify(first); + const b = stableStringify(second); + if (a !== b) { + return yield* new ToolSandboxError({ + kind: "nondeterministic", + message: + "descriptor collection is non-deterministic (an artifact read Math.random/Date.now or otherwise diverged between runs)", + }); + } + const descriptor = first as { artifact?: string }; + const result: CollectResult = { + artifacts: { + [String(descriptor.artifact ?? "default")]: { + kind: "tool", + descriptor: first, + }, + }, + }; + return result; + }), + + invoke: (bundle: string, request: InvokeRequest, bridge: HandleBridge) => + Effect.gen(function* () { + // The invoker decodes the routed handle call and forwards to the host + // bridge. Path 0 is `__handle__`; the single arg is {root, path, args}. + const invoker: SandboxToolInvoker = { + invoke: (input: { path: string; args: unknown }) => { + // Strictness (grafted from A): the ONLY reserved bridge path the + // invoke phase accepts is `__handle__`. Anything else is a hard + // error, never silently ignored — a handler must not reach the host + // through an unexpected channel. + if (input.path !== "__handle__") { + return Effect.fail( + new ToolSandboxError({ + kind: "invoke", + message: `unexpected sandbox bridge path: ${input.path}`, + }), + ); + } + const call = input.args as { + root: string; + path: readonly string[]; + args: readonly unknown[]; + }; + if (!call || typeof call.root !== "string" || !Array.isArray(call.path)) { + return Effect.fail( + new ToolSandboxError({ kind: "invoke", message: "malformed sandbox bridge call" }), + ); + } + return bridge.call({ root: call.root, path: call.path, args: call.args }) as never; + }, + }; + const result = yield* invokeExecutor + .execute(buildInvokeCode(bundle, request), invoker) + .pipe( + Effect.mapError( + (cause) => + new ToolSandboxError({ kind: "invoke", message: "invoke run failed", cause }), + ), + ); + if (result.error) { + const marker = parseMarker(result.error); + if (isMarker(marker, "InputValidationError")) { + return yield* new InputValidationError({ + message: markerMessage(marker, "input validation failed"), + issues: markerIssues(marker), + }); + } + if (isMarker(marker, "OutputValidationError")) { + return yield* new OutputValidationError({ + message: markerMessage(marker, "output validation failed"), + issues: markerIssues(marker), + }); + } + return yield* new ToolSandboxError({ kind: "invoke", message: result.error }); + } + return { output: result.result, logs: result.logs ?? [] } satisfies InvokeResult; + }), + }; +}; diff --git a/packages/plugins/apps/src/backing/scope-collision.test.ts b/packages/plugins/apps/src/backing/scope-collision.test.ts new file mode 100644 index 000000000..72e08f3da --- /dev/null +++ b/packages/plugins/apps/src/backing/scope-collision.test.ts @@ -0,0 +1,38 @@ +import { describe, expect, it } from "@effect/vitest"; +import { Effect } from "effect"; + +import { makeSqliteAppsStore } from "./sqlite-apps-store"; +import { DESCRIPTOR_VERSION, type AppDescriptor } from "../pipeline/descriptor"; + +const run = (effect: Effect.Effect): Promise => Effect.runPromise(effect); + +// --------------------------------------------------------------------------- +// Finding 9 regression: scopes that differ only by a character an old naming +// scheme collapsed ("my-scope" vs "my_scope") must not share descriptor +// storage. +// --------------------------------------------------------------------------- + +describe("scope collision (Fix 9)", () => { + it("stores distinct descriptor scopes independently", async () => { + const store = makeSqliteAppsStore({ path: ":memory:" }); + const descriptor = (scope: string): AppDescriptor => ({ + version: DESCRIPTOR_VERSION, + tenant: "org", + scope, + description: "test", + snapshotId: `${scope}-snapshot`, + toolchain: { bundler: "esbuild", bundlerVersion: "test", target: "test" }, + tools: [], + workflows: [], + ui: [], + skills: [], + skipped: [], + }); + + await run(store.putDescriptor("org", "org", descriptor("my-scope"))); + await run(store.putDescriptor("org", "org", descriptor("my_scope"))); + + expect((await run(store.getDescriptor("org", "my-scope")))?.scope).toBe("my-scope"); + expect((await run(store.getDescriptor("org", "my_scope")))?.scope).toBe("my_scope"); + }); +}); diff --git a/packages/plugins/apps/src/backing/sqlite-apps-store.ts b/packages/plugins/apps/src/backing/sqlite-apps-store.ts new file mode 100644 index 000000000..efab5ceb4 --- /dev/null +++ b/packages/plugins/apps/src/backing/sqlite-apps-store.ts @@ -0,0 +1,131 @@ +import { mkdirSync } from "node:fs"; +import { dirname, resolve } from "node:path"; + +import { createClient, type Client } from "@libsql/client"; +import { Effect, Schema } from "effect"; + +import { StorageError, type StorageFailure } from "@executor-js/sdk"; + +import type { AppDescriptor } from "../pipeline/descriptor"; +import type { AppsStore } from "../plugin/store"; + +// --------------------------------------------------------------------------- +// SQLite-backed AppsStore (self-hosted). One small SQLite file stores the +// published descriptor per source scope. +// --------------------------------------------------------------------------- + +const toUrl = (path: string): string => (path === ":memory:" ? path : `file:${resolve(path)}`); + +const SCHEMA = ` +CREATE TABLE IF NOT EXISTS descriptors (tenant TEXT NOT NULL, scope TEXT NOT NULL, snapshot_id TEXT NOT NULL, descriptor TEXT NOT NULL, published_at INTEGER NOT NULL, PRIMARY KEY (tenant, scope)); +`; + +const storageFail = (message: string, cause: unknown): StorageFailure => + new StorageError({ message, cause }); + +const decodeDescriptorJson = Schema.decodeUnknownEffect(Schema.UnknownFromJsonString); + +const decodeDescriptor = ( + value: unknown, + message: string, +): Effect.Effect => + decodeDescriptorJson(String(value)).pipe( + Effect.map((descriptor) => descriptor as AppDescriptor), + Effect.mapError((cause) => storageFail(message, cause)), + ); + +export interface SqliteAppsStoreOptions { + readonly path: string; +} + +export const makeSqliteAppsStore = (options: SqliteAppsStoreOptions): AppsStore => { + if (options.path !== ":memory:") mkdirSync(dirname(resolve(options.path)), { recursive: true }); + const client: Client = createClient({ url: toUrl(options.path) }); + let ready: Promise | undefined; + const init = async () => { + if (!ready) { + ready = (async () => { + for (const stmt of SCHEMA.split(";")) { + const s = stmt.trim(); + if (s) await client.execute(s); + } + })(); + } + return ready; + }; + + return { + putDescriptor: (tenant, _owner, descriptor) => + Effect.tryPromise({ + try: async () => { + await init(); + await client.execute({ + sql: `INSERT INTO descriptors (tenant, scope, snapshot_id, descriptor, published_at) + VALUES (?, ?, ?, ?, ?) + ON CONFLICT(tenant, scope) DO UPDATE SET snapshot_id=excluded.snapshot_id, descriptor=excluded.descriptor, published_at=excluded.published_at`, + args: [ + tenant, + descriptor.scope, + descriptor.snapshotId, + JSON.stringify(descriptor), + Date.now(), + ], + }); + }, + catch: (cause) => storageFail("putDescriptor failed", cause), + }), + getDescriptor: (tenant, scope) => + Effect.tryPromise({ + try: async () => { + await init(); + const res = await client.execute({ + sql: "SELECT descriptor FROM descriptors WHERE tenant = ? AND scope = ?", + args: [tenant, scope], + }); + const row = res.rows[0]; + return row?.descriptor ?? null; + }, + catch: (cause) => storageFail("getDescriptor failed", cause), + }).pipe( + Effect.flatMap((descriptor) => + descriptor === null + ? Effect.succeed(null) + : decodeDescriptor(descriptor, "getDescriptor failed"), + ), + ), + listDescriptors: (tenant) => + Effect.tryPromise({ + try: async () => { + await init(); + const res = await client.execute({ + sql: "SELECT descriptor, published_at FROM descriptors WHERE tenant = ? ORDER BY published_at DESC", + args: [tenant], + }); + return res.rows; + }, + catch: (cause) => storageFail("listDescriptors failed", cause), + }).pipe( + Effect.flatMap((rows) => + Effect.forEach(rows, (row) => + decodeDescriptor(row.descriptor, "listDescriptors failed").pipe( + Effect.map((descriptor) => ({ + descriptor, + publishedAt: Number(row.published_at), + })), + ), + ), + ), + ), + removeDescriptor: (tenant, scope) => + Effect.tryPromise({ + try: async () => { + await init(); + await client.execute({ + sql: "DELETE FROM descriptors WHERE tenant = ? AND scope = ?", + args: [tenant, scope], + }); + }, + catch: (cause) => storageFail("removeDescriptor failed", cause), + }), + }; +}; diff --git a/packages/plugins/apps/src/index.ts b/packages/plugins/apps/src/index.ts new file mode 100644 index 000000000..8b82ff2b0 --- /dev/null +++ b/packages/plugins/apps/src/index.ts @@ -0,0 +1,65 @@ +// @executor-js/plugin-apps — custom tool publishing for executor. +// +// Custom tools are published into a per-scope store and invoked through the +// platform catalog path. Publish is the compiler (FDI); each substrate-specific +// capability sits behind a seam. + +export * from "./seams"; +export * from "./authoring"; +export * from "./standard-schema"; +export * from "./pipeline/descriptor"; +export { discover, PublishError } from "./pipeline/discover"; +export { bundleEntry, PLATFORM_MODULES, INLINABLE_MODULES } from "./pipeline/bundle"; +export { + publish, + type PublishInput, + type PublishOutput, + type PublishDeps, +} from "./pipeline/publish"; +export { + fetchGitHubSource, + syncGitHubSource, + GitHubSourceError, + type GitHubSourceInput, + type GitHubSourceSnapshot, + type GitHubSyncResult, + type SyncErrorData, + type SyncGitHubSourceInput, +} from "./source/github-source"; + +export { makeAppsRuntime, type AppsRuntime, type AppsRuntimeDeps } from "./plugin/runtime"; +export { makeAppsRuntimeFromBackings, type AppsBackings } from "./plugin/backings"; +export { + makeAppsStore, + type AppsStore, + type AppsStoreDeps, + type GitHubSourceTokenRef, + descriptorCollection, +} from "./plugin/store"; +export { + slugifyCustomToolsAppName, + validateCustomToolsAppSlug, + CUSTOM_TOOLS_APP_SLUG_PATTERN, +} from "./source/app-slug"; +export { + buildBridge, + rootsFor, + resolveIntegrationBindings, + BindingError, + type ConnectionCandidate, + type RoleBindings, + type ClientResolver, + type BindingContext, +} from "./plugin/bindings"; +export { makePluginCtxAppsResolver } from "./plugin/resolver"; +export { SourceOriginError, assertSourceOrigin } from "./plugin/apps-plugin"; +export { + makeSelfHostAppsRuntime, + type SelfHostAppsRuntime, + type SelfHostAppsRuntimeOptions, +} from "./plugin/self-host-runtime"; + +// Self-host seam backings. +export { makeGitArtifactStore } from "./backing/git-artifact-store"; +export { makeQuickjsToolSandbox } from "./backing/quickjs-tool-sandbox"; +export { makeSqliteAppsStore } from "./backing/sqlite-apps-store"; diff --git a/packages/plugins/apps/src/pipeline/bundle.ts b/packages/plugins/apps/src/pipeline/bundle.ts new file mode 100644 index 000000000..892f03bcf --- /dev/null +++ b/packages/plugins/apps/src/pipeline/bundle.ts @@ -0,0 +1,221 @@ +import { build, version as esbuildVersion } from "esbuild"; + +import { Effect, Option, Schema } from "effect"; + +import { ToolSandboxError } from "../seams/tool-sandbox"; +import type { ToolchainRef } from "./descriptor"; + +const BUNDLE_TARGET = "es2022"; + +/** The toolchain (esbuild version + target) recorded into the descriptor so a + * re-collect on a different esbuild is not falsely claimed byte-identical. */ +export const toolchainRef = (): ToolchainRef => ({ + bundler: "esbuild", + bundlerVersion: esbuildVersion, + target: BUNDLE_TARGET, +}); + +// --------------------------------------------------------------------------- +// bundle — the FDI pipeline's bundle stage. +// +// One esbuild pass per artifact entry. The platform module (`executor:app`) +// stays EXTERNAL, the sandbox shim provides the real behavior. A small allowlist +// of schema libraries (`zod`) is INLINED from node_modules so the author's +// schema runs in the sandbox and exposes the Standard Schema JSON-schema +// extension. Every other bare import is rejected with a diagnostic: npm deps in +// user bundles are out of scope. +// +// Output is a single CJS string with the externals left as `require(...)` +// calls the sandbox resolves. +// --------------------------------------------------------------------------- + +/** Modules the platform provides inside the sandbox (kept external, resolved by + * the sandbox `require` shim). */ +export const PLATFORM_MODULES = new Set(["executor:app"]); + +/** npm modules we deliberately inline (schema runtime that must run in the + * sandbox). Everything else bare is rejected. */ +export const INLINABLE_MODULES = new Set(["zod"]); + +const isInlinable = (path: string): boolean => { + if (INLINABLE_MODULES.has(path)) return true; + // zod subpaths (`zod/v4`, `zod/mini`) are fine too. + return [...INLINABLE_MODULES].some((m) => path === m || path.startsWith(`${m}/`)); +}; + +export interface BundleInput { + readonly files: ReadonlyMap; + readonly entry: string; +} + +export interface BundleOutput { + readonly code: string; +} + +const EsbuildDiagnostic = Schema.Struct({ + text: Schema.String, +}); + +const EsbuildFailure = Schema.Struct({ + errors: Schema.Array(EsbuildDiagnostic), +}); + +const decodeEsbuildFailure = Schema.decodeUnknownOption(EsbuildFailure); + +const bundleFailureMessage = (entry: string, cause: unknown): string => { + const decoded = Option.getOrNull(decodeEsbuildFailure(cause)); + const detail = decoded?.errors + .map((error) => error.text) + .filter((text) => text.length > 0) + .join("; "); + return detail ? `bundle failed for ${entry}: ${detail}` : `bundle failed for ${entry}`; +}; + +const resolveRelative = (base: string, rel: string): string => { + const parts = (base ? base.split("/") : []).concat(rel.split("/")); + const out: string[] = []; + for (const part of parts) { + if (part === "" || part === ".") continue; + if (part === "..") out.pop(); + else out.push(part); + } + return out.join("/"); +}; + +const candidates = (path: string): string[] => { + if (/\.(tsx?|jsx?)$/.test(path)) return [path]; + return [ + path, + `${path}.ts`, + `${path}.tsx`, + `${path}.js`, + `${path}.jsx`, + `${path}/index.ts`, + `${path}/index.tsx`, + ]; +}; + +const FILESET_NS = "fileset"; +const VIRTUAL_ENTRY = "executor-apps://entry"; + +/** The virtual entry esbuild starts from: re-export the author entry. */ +const virtualEntrySource = (authorEntry: string) => ` +import __artifactModule from ${JSON.stringify(`/${authorEntry}`)}; +globalThis.__artifact = __artifactModule; +`; + +const fileSetPlugin = (files: ReadonlyMap, authorEntry: string) => ({ + name: "executor-apps-fileset", + // eslint-disable-next-line @typescript-eslint/no-explicit-any + setup(build2: any) { + const norm = (p: string) => p.replace(/^\.\//, "").replace(/\/\.\//g, "/"); + + build2.onResolve( + { filter: /.*/ }, + (args: { path: string; importer: string; namespace?: string; resolveDir?: string }) => { + if (args.path === VIRTUAL_ENTRY) { + return { path: VIRTUAL_ENTRY, namespace: "virtual" }; + } + // Imports originating OUTSIDE the file set (from inlined npm modules + // like zod, or the virtual entry's `import "zod"`) are left to + // esbuild's default node resolution. We only govern file-set imports. + const fromFileSet = args.namespace === FILESET_NS || args.namespace === "virtual"; + if (!fromFileSet) return undefined; + + if (PLATFORM_MODULES.has(args.path)) { + return { path: args.path, external: true }; + } + if (isInlinable(args.path)) { + // Inline from node_modules via default resolution. + return undefined; + } + // Absolute (virtual entry uses `/tools/x.ts`) or relative -> file set. + if (args.path.startsWith("/")) { + const joined = norm(args.path.slice(1)); + for (const candidate of candidates(joined)) { + if (files.has(candidate)) return { path: candidate, namespace: FILESET_NS }; + } + return { errors: [{ text: `cannot resolve "${args.path}" in the app file set` }] }; + } + if (args.path.startsWith(".")) { + const base = args.importer.split("/").slice(0, -1).join("/"); + const joined = norm(resolveRelative(base, args.path)); + for (const candidate of candidates(joined)) { + if (files.has(candidate)) return { path: candidate, namespace: FILESET_NS }; + } + return { + errors: [ + { text: `cannot resolve "${args.path}" from "${args.importer}" in the app file set` }, + ], + }; + } + return { + errors: [ + { + text: `bare import "${args.path}" is not allowed: apps may only import platform modules (${[...PLATFORM_MODULES].join(", ")}), the schema runtime (zod), or files within the scope`, + }, + ], + }; + }, + ); + + build2.onLoad({ filter: /.*/, namespace: "virtual" }, () => ({ + contents: virtualEntrySource(authorEntry), + loader: "ts", + resolveDir: process.cwd(), + })); + + build2.onLoad({ filter: /.*/, namespace: FILESET_NS }, (args: { path: string }) => { + const contents = files.get(args.path); + if (contents === undefined) { + return { errors: [{ text: `missing file in set: ${args.path}` }] }; + } + const loader = args.path.endsWith(".tsx") + ? "tsx" + : args.path.endsWith(".ts") + ? "ts" + : args.path.endsWith(".jsx") + ? "jsx" + : "js"; + return { contents, loader, resolveDir: process.cwd() }; + }); + }, +}); + +/** Bundle one entry from the file set to a single CJS string. */ +export const bundleEntry = (input: BundleInput): Effect.Effect => + Effect.tryPromise({ + try: () => + build({ + entryPoints: [VIRTUAL_ENTRY], + bundle: true, + write: false, + format: "cjs", + platform: "neutral", + target: BUNDLE_TARGET, + minify: false, + treeShaking: true, + jsx: "automatic", + logLevel: "silent", + external: [...PLATFORM_MODULES], + plugins: [fileSetPlugin(input.files, input.entry) as never], + }), + catch: (cause) => + new ToolSandboxError({ + kind: "bundle", + message: bundleFailureMessage(input.entry, cause), + cause, + }), + }).pipe( + Effect.flatMap((result) => { + const out = result.outputFiles?.[0]?.text; + return out === undefined + ? Effect.fail( + new ToolSandboxError({ + kind: "bundle", + message: `bundle failed for ${input.entry}: esbuild produced no output`, + }), + ) + : Effect.succeed({ code: out }); + }), + ); diff --git a/packages/plugins/apps/src/pipeline/descriptor.ts b/packages/plugins/apps/src/pipeline/descriptor.ts new file mode 100644 index 000000000..565184e3f --- /dev/null +++ b/packages/plugins/apps/src/pipeline/descriptor.ts @@ -0,0 +1,123 @@ +// --------------------------------------------------------------------------- +// The versioned app descriptor, extracted from source at publish. +// +// Catalog rows are projections of this descriptor. Identity is the file path: +// there are no authored `name` or `id` fields. +// --------------------------------------------------------------------------- + +/** Descriptor schema version. Bumped on any breaking shape change. A reader + * refuses a descriptor from a version it does not understand. */ +export const DESCRIPTOR_VERSION = 5 as const; + +/** Where an entry came from: path + content hash. Lets a projection point back + * at the exact source bytes without re-reading the snapshot, and makes the + * determinism byte-compare include per-entry provenance. (Grafted from C.) */ +export interface ModuleSourceRef { + /** Path within the scope repo, e.g. `tools/issues-sync.ts`. */ + readonly path: string; + /** SHA-256 of the source bytes (hex). */ + readonly sourceHash: string; +} + +/** The toolchain that produced the bundles, recorded so a re-collect on a + * different esbuild is not falsely claimed byte-identical. (Grafted from C.) */ +export interface ToolchainRef { + readonly bundler: "esbuild"; + readonly bundlerVersion: string; + readonly target: string; +} + +export interface IntegrationDecl { + readonly integration: string; +} + +export interface SourceSkippedArtifact { + readonly path: string; + readonly reason: "not supported yet" | "unsupported file type" | "ignored"; +} + +export interface GitHubSourceRef { + readonly kind: "github"; + readonly url: string; + readonly repo: string; + readonly ref: string; + readonly upstreamSha: string; + readonly skipped?: readonly SourceSkippedArtifact[]; +} + +export type AppSourceRef = GitHubSourceRef; + +export interface ToolDescriptor { + /** Path identity, e.g. `issues-sync` (from `tools/issues-sync.ts`). */ + readonly name: string; + readonly sourcePath: string; + /** Path + source-hash provenance for this entry. */ + readonly source: ModuleSourceRef; + readonly description: string; + /** role -> integration declaration, collected from `integrations:`. */ + readonly integrations: Readonly>; + readonly inputSchema?: unknown; + readonly outputSchema?: unknown; + readonly annotations?: { + readonly readOnly?: boolean; + readonly destructive?: boolean; + readonly requiresApproval?: boolean; + }; +} + +export interface DeferredDescriptor { + readonly name: string; + readonly sourcePath: string; + readonly source: ModuleSourceRef; + readonly description?: string; +} + +export const FLOW_ENTRIES_KEY = "workflows"; +export const GUIDE_ENTRIES_KEY = "skills"; + +export interface AppDescriptor { + readonly version: typeof DESCRIPTOR_VERSION; + readonly tenant: string; + readonly scope: string; + readonly description?: string; + readonly source?: AppSourceRef; + /** The snapshot (commit hash) this descriptor was extracted from. */ + readonly snapshotId: string; + /** The toolchain that produced the compiled bundles. */ + readonly toolchain: ToolchainRef; + readonly tools: readonly ToolDescriptor[]; + readonly [FLOW_ENTRIES_KEY]: readonly DeferredDescriptor[]; + readonly ui: readonly DeferredDescriptor[]; + readonly [GUIDE_ENTRIES_KEY]: readonly DeferredDescriptor[]; + readonly skipped: readonly { + readonly path: string; + readonly reason: "not supported yet"; + }[]; + /** Shared JSON-schema `$defs` reachable from tool schemas. */ + readonly definitions?: Record; +} + +/** The path inside a committed snapshot where the extracted descriptor is + * written, so projections can be recovered from the commit alone. */ +export const DESCRIPTOR_SNAPSHOT_PATH = ".executor/descriptor.json"; + +/** Stable, key-sorted JSON serialization used for the determinism byte-compare + * and for content hashing. Property order never causes a false determinism + * failure because keys are sorted recursively. (Grafted from C.) */ +export const stableStringify = (value: unknown): string => { + const seen = new WeakSet(); + const walk = (v: unknown): unknown => { + if (v === null || typeof v !== "object") return v; + if (seen.has(v)) return "[Circular]"; + seen.add(v); + if (Array.isArray(v)) return v.map(walk); + const out: Record = {}; + for (const key of Object.keys(v as Record).sort()) { + const inner = (v as Record)[key]; + if (inner === undefined) continue; + out[key] = walk(inner); + } + return out; + }; + return JSON.stringify(walk(value)); +}; diff --git a/packages/plugins/apps/src/pipeline/discover.ts b/packages/plugins/apps/src/pipeline/discover.ts new file mode 100644 index 000000000..bc886f7a7 --- /dev/null +++ b/packages/plugins/apps/src/pipeline/discover.ts @@ -0,0 +1,92 @@ +import { Data } from "effect"; + +// --------------------------------------------------------------------------- +// discover: shape validation over the flat scope layout, with zero imports. +// Supported now: +// tools/.ts +// Deferred known folders are skipped and reported: +// workflows/ ui/ skills/ +// Unknown top-level files and folders are ignored. +// --------------------------------------------------------------------------- + +export interface FileDiagnostic { + readonly path: string; + readonly message: string; +} + +/** Typed publish failure carrying per-file diagnostics. Nothing is persisted on + * a failed publish. */ +export class PublishError extends Data.TaggedError("PublishError")<{ + readonly message: string; + readonly stage: "discover" | "bundle" | "collect" | "project"; + readonly diagnostics: readonly FileDiagnostic[]; +}> {} + +export type ArtifactKind = "tool"; + +export interface DiscoveredArtifact { + readonly kind: ArtifactKind; + /** Path identity, e.g. `issues-sync`. */ + readonly name: string; + /** The entry file path in the set, e.g. `tools/issues-sync.ts`. */ + readonly entry: string; +} + +export interface SkippedArtifact { + readonly path: string; + readonly reason: "not supported yet"; +} + +export interface DiscoverResult { + readonly artifacts: readonly DiscoveredArtifact[]; + readonly skipped: readonly SkippedArtifact[]; +} + +const TOOL_RE = /^tools\/([a-z0-9][a-z0-9-]*)\.(ts|tsx|js|jsx)$/; +const DEFERRED_RE = /^(workflows|ui|skills)\//; + +export const discover = (files: ReadonlyMap): DiscoverResult | PublishError => { + const diagnostics: FileDiagnostic[] = []; + const artifacts: DiscoveredArtifact[] = []; + const skipped: SkippedArtifact[] = []; + const seen = new Set(); + + for (const [path] of files) { + if (path === "executor.json") continue; + + const tool = path.match(TOOL_RE); + if (tool) { + const name = tool[1]; + const key = `tool:${name}`; + if (seen.has(key)) { + diagnostics.push({ path, message: `duplicate artifact identity: ${key}` }); + } else { + seen.add(key); + artifacts.push({ kind: "tool", name, entry: path }); + } + continue; + } + + if (path.startsWith("tools/")) { + diagnostics.push({ + path, + message: "file does not match the expected layout for tools/", + }); + continue; + } + + if (DEFERRED_RE.test(path)) { + skipped.push({ path, reason: "not supported yet" }); + } + } + + if (diagnostics.length > 0) { + return new PublishError({ + message: `discover found ${diagnostics.length} problem(s)`, + stage: "discover", + diagnostics, + }); + } + + return { artifacts, skipped }; +}; diff --git a/packages/plugins/apps/src/pipeline/publish.test.ts b/packages/plugins/apps/src/pipeline/publish.test.ts new file mode 100644 index 000000000..2f83f0d37 --- /dev/null +++ b/packages/plugins/apps/src/pipeline/publish.test.ts @@ -0,0 +1,224 @@ +import { mkdtempSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +import { describe, expect, it } from "@effect/vitest"; +import { Effect, Exit } from "effect"; + +import { makeGitArtifactStore } from "../backing/git-artifact-store"; +import { makeQuickjsToolSandbox } from "../backing/quickjs-tool-sandbox"; +import { scopeAddress } from "../seams/scope-address"; +import { dailyBriefFileSet } from "../testing/daily-brief"; +import { FLOW_ENTRIES_KEY, GUIDE_ENTRIES_KEY } from "./descriptor"; +import { publish, PUBLISH_LIMITS } from "./publish"; + +const run = (effect: Effect.Effect): Promise => Effect.runPromise(effect); + +const makeDeps = () => ({ + artifactStore: makeGitArtifactStore({ root: mkdtempSync(join(tmpdir(), "apps-pub-")) }), + sandbox: makeQuickjsToolSandbox(), +}); + +describe("publish pipeline (discover -> bundle -> collect -> project)", () => { + it("compiles the daily-brief set into a descriptor", async () => { + const deps = makeDeps(); + const out = await run(publish(deps, { scope: "rhys", files: dailyBriefFileSet() })); + + expect(out.snapshotId).toBeTruthy(); + const d = out.descriptor; + expect(d.scope).toBe("rhys"); + expect(d.snapshotId).toBe(out.snapshotId); + + const toolNames = d.tools.map((t) => t.name).sort(); + expect(toolNames).toEqual(["issues-sync", "search-all-mail"]); + expect(d[FLOW_ENTRIES_KEY]).toEqual([]); + expect(d.ui).toEqual([]); + expect(d[GUIDE_ENTRIES_KEY]).toEqual([]); + expect(out.skipped).toEqual([]); + + const sync = d.tools.find((t) => t.name === "issues-sync")!; + expect(sync.integrations.github).toEqual(expect.objectContaining({ integration: "github" })); + expect((sync.inputSchema as { type: string }).type).toBe("object"); + + const mail = d.tools.find((t) => t.name === "search-all-mail")!; + expect(mail.integrations.inbox).toEqual(expect.objectContaining({ integration: "gmail" })); + }); + + it("publishes tools while reporting deferred known folders under skipped", async () => { + const deps = makeDeps(); + const flowPath = "workflows/y.ts"; + const guidePath = "skills/z/SKILL.md"; + const files = new Map([ + [ + "tools/x.ts", + `import { defineTool } from "executor:app"; +import { z } from "zod"; +export default defineTool({ description: "x", input: z.object({}), async handler(){ return { ok: true }; } });`, + ], + [flowPath, "export default {};"], + [guidePath, "# z"], + ]); + + const out = await run(publish(deps, { scope: "s", files })); + + expect(out.descriptor.tools.map((t) => t.name)).toEqual(["x"]); + expect(out.descriptor[FLOW_ENTRIES_KEY]).toEqual([]); + expect(out.descriptor.ui).toEqual([]); + expect(out.descriptor[GUIDE_ENTRIES_KEY]).toEqual([]); + expect(out.skipped).toEqual([ + { path: flowPath, reason: "not supported yet" }, + { path: guidePath, reason: "not supported yet" }, + ]); + }); + + it("rejects an authored input field that collides with an integration role", async () => { + const deps = makeDeps(); + const files = new Map([ + [ + "tools/sync.ts", + `import { defineTool, integration } from "executor:app"; +import { z } from "zod"; +export default defineTool({ + description: "sync", + integrations: { crm: integration("dealcloud") }, + input: z.object({ crm: z.string() }), + async handler(){ return {}; }, +});`, + ], + ]); + + const exit = await Effect.runPromiseExit(publish(deps, { scope: "s", files })); + + expect(Exit.isFailure(exit)).toBe(true); + expect(JSON.stringify(exit)).toContain("collides"); + expect(JSON.stringify(exit)).toContain("crm"); + }); + + it("rejects tools that reference storage", async () => { + const deps = makeDeps(); + const files = new Map([ + [ + "tools/storage.ts", + `import { defineTool } from "executor:app"; +import { z } from "zod"; +export default defineTool({ + description: "storage", + input: z.object({}), + async handler(_input, { db }) { + await db.sql\`SELECT 1\`; + return { ok: true }; + }, +});`, + ], + ]); + + const exit = await Effect.runPromiseExit(publish(deps, { scope: "s", files })); + + expect(Exit.isFailure(exit)).toBe(true); + expect(JSON.stringify(exit)).toContain("storage is not available yet"); + const latest = await run( + Effect.flatMap(deps.artifactStore.forScope(scopeAddress("org", "s")), (s) => s.latest()), + ); + expect(latest).toBeNull(); + }); + + it("rejects a Standard Schema vendor without JSON Schema export", async () => { + const deps = makeDeps(); + const files = new Map([ + [ + "tools/vendor.ts", + `import { defineTool } from "executor:app"; +const schema = { + "~standard": { + version: 1, + vendor: "vendor-without-json-schema", + validate(value) { return { value }; }, + }, +}; +export default defineTool({ + description: "vendor", + input: schema, + async handler(input){ return input; }, +});`, + ], + ]); + + const exit = await Effect.runPromiseExit(publish(deps, { scope: "s", files })); + + expect(Exit.isFailure(exit)).toBe(true); + expect(JSON.stringify(exit)).toContain("vendor-without-json-schema"); + expect(JSON.stringify(exit)).toContain("vendor"); + expect(JSON.stringify(exit)).toContain("input"); + }); + + it("accepts raw JSON Schema input", async () => { + const deps = makeDeps(); + const files = new Map([ + [ + "tools/raw.ts", + `import { defineTool } from "executor:app"; +export default defineTool({ + description: "raw", + input: { type: "object", properties: { q: { type: "string" } }, required: ["q"] }, + async handler(input){ return input; }, +});`, + ], + ]); + + const out = await run(publish(deps, { scope: "s", files })); + + expect(out.descriptor.tools[0].inputSchema).toEqual({ + type: "object", + properties: { q: { type: "string" } }, + required: ["q"], + }); + }); + + it("rejects a bare npm import (npm deps out of scope)", async () => { + const deps = makeDeps(); + const files = new Map([ + [ + "tools/bad.ts", + `import { defineTool } from "executor:app";\nimport { chunk } from "lodash";\nimport { z } from "zod";\nexport default defineTool({ description: "b", input: z.object({}), async handler(){ return chunk([1,2,3], 2); } });`, + ], + ]); + const exit = await Effect.runPromiseExit(publish(deps, { scope: "s", files })); + expect(Exit.isFailure(exit)).toBe(true); + expect(JSON.stringify(exit)).toContain("bundle"); + }); + + it("rejects an oversized publish set (too many files) and persists nothing", async () => { + const deps = makeDeps(); + const files = new Map(); + for (let i = 0; i < PUBLISH_LIMITS.maxFiles + 5; i++) { + files.set(`tools/t${i}.ts`, "// noop"); + } + const exit = await Effect.runPromiseExit(publish(deps, { scope: "s", files })); + expect(Exit.isFailure(exit)).toBe(true); + expect(JSON.stringify(exit)).toContain("exceeding the limit"); + const latest = await run( + Effect.flatMap(deps.artifactStore.forScope(scopeAddress("org", "s")), (s) => s.latest()), + ); + expect(latest).toBeNull(); + }); + + it("rejects a single file over the per-file byte limit", async () => { + const deps = makeDeps(); + const big = "x".repeat(PUBLISH_LIMITS.maxFileBytes + 1); + const files = new Map([["tools/big.ts", big]]); + const exit = await Effect.runPromiseExit(publish(deps, { scope: "s", files })); + expect(Exit.isFailure(exit)).toBe(true); + expect(JSON.stringify(exit)).toContain("per-file limit"); + }); + + it("rejects a set over the total byte limit", async () => { + const deps = makeDeps(); + const half = "y".repeat(Math.floor(PUBLISH_LIMITS.maxFileBytes)); + const files = new Map(); + const count = Math.ceil(PUBLISH_LIMITS.maxTotalBytes / PUBLISH_LIMITS.maxFileBytes) + 1; + for (let i = 0; i < count; i++) files.set(`tools/t${i}.ts`, half); + const exit = await Effect.runPromiseExit(publish(deps, { scope: "s", files })); + expect(Exit.isFailure(exit)).toBe(true); + expect(JSON.stringify(exit)).toContain("total limit"); + }); +}); diff --git a/packages/plugins/apps/src/pipeline/publish.ts b/packages/plugins/apps/src/pipeline/publish.ts new file mode 100644 index 000000000..3fe8e74af --- /dev/null +++ b/packages/plugins/apps/src/pipeline/publish.ts @@ -0,0 +1,314 @@ +import { Effect, Predicate, Schema } from "effect"; + +import type { ArtifactStore, FileSet, SnapshotId } from "../seams/artifact-store"; +import { scopeAddress } from "../seams/scope-address"; +import type { ToolSandbox } from "../seams/tool-sandbox"; +import { bundleEntry, toolchainRef } from "./bundle"; +import { + DESCRIPTOR_SNAPSHOT_PATH, + DESCRIPTOR_VERSION, + FLOW_ENTRIES_KEY, + GUIDE_ENTRIES_KEY, + stableStringify, + type AppDescriptor, + type AppSourceRef, + type IntegrationDecl, + type ModuleSourceRef, + type ToolDescriptor, +} from "./descriptor"; +import { discover, PublishError, type FileDiagnostic, type SkippedArtifact } from "./discover"; + +// --------------------------------------------------------------------------- +// publish: discover -> bundle -> collect -> project. +// +// All fallible work happens before persistence. The extracted descriptor is +// written into the snapshot itself (`.executor/descriptor.json`) and the +// snapshot is committed last. The runtime writes the published-descriptor +// pointer only after this commit succeeds. +// --------------------------------------------------------------------------- + +export const PUBLISH_LIMITS = { + /** Max number of files in one publish. */ + maxFiles: 256, + /** Max bytes for any single file. */ + maxFileBytes: 1024 * 1024, + /** Max total bytes across all files. */ + maxTotalBytes: 4 * 1024 * 1024, +} as const; + +const byteLength = (value: string): number => Buffer.byteLength(value, "utf8"); + +export const enforcePublishLimits = (files: FileSet): PublishError | null => { + const diagnostics: FileDiagnostic[] = []; + if (files.size > PUBLISH_LIMITS.maxFiles) { + diagnostics.push({ + path: "", + message: `publish has ${files.size} files, exceeding the limit of ${PUBLISH_LIMITS.maxFiles}`, + }); + } + let total = 0; + for (const [path, contents] of files) { + const size = byteLength(contents); + total += size; + if (size > PUBLISH_LIMITS.maxFileBytes) { + diagnostics.push({ + path, + message: `file is ${size} bytes, exceeding the per-file limit of ${PUBLISH_LIMITS.maxFileBytes} bytes`, + }); + } + } + if (total > PUBLISH_LIMITS.maxTotalBytes) { + diagnostics.push({ + path: "", + message: `publish total is ${total} bytes, exceeding the total limit of ${PUBLISH_LIMITS.maxTotalBytes} bytes`, + }); + } + if (diagnostics.length === 0) return null; + return new PublishError({ + message: `publish payload exceeds limits (${diagnostics.length} problem(s))`, + stage: "discover", + diagnostics, + }); +}; + +export interface PublishInput { + readonly tenant?: string; + readonly scope: string; + readonly files: FileSet; + readonly commitMessage?: string; + readonly description?: string; + readonly source?: AppSourceRef; +} + +export interface PublishOutput { + readonly snapshotId: SnapshotId; + readonly descriptor: AppDescriptor; + readonly skipped: readonly SkippedArtifact[]; +} + +const sha256Hex = (text: string): Effect.Effect => + Effect.promise(async () => { + const digest = await crypto.subtle.digest("SHA-256", new TextEncoder().encode(text)); + return Array.from(new Uint8Array(digest), (b) => b.toString(16).padStart(2, "0")).join(""); + }); + +const sourceRef = (path: string, source: string): Effect.Effect => + sha256Hex(source).pipe(Effect.map((sourceHash) => ({ path, sourceHash }))); + +interface CollectedDescriptor { + readonly kind: "tool"; + readonly artifact?: string; + readonly description?: string; + readonly integrations?: Record; + readonly annotations?: ToolDescriptor["annotations"]; + readonly inputJsonSchema?: unknown; + readonly outputJsonSchema?: unknown; +} + +const toIntegrationDecls = ( + raw: CollectedDescriptor["integrations"], +): Record => { + const out: Record = {}; + for (const [role, c] of Object.entries(raw ?? {})) { + out[role] = { integration: c.integration ?? "" }; + } + return out; +}; + +const isRecord = (value: unknown): value is Record => + value !== null && typeof value === "object" && !Array.isArray(value); + +const rejectRoleInputCollisions = ( + artifactName: string, + sourcePath: string, + inputSchema: unknown, + integrations: Readonly>, +): PublishError | null => { + if (!isRecord(inputSchema)) return null; + const properties = inputSchema.properties; + if (!isRecord(properties)) return null; + + for (const role of Object.keys(integrations)) { + if (Object.prototype.hasOwnProperty.call(properties, role)) { + return new PublishError({ + message: `tool "${artifactName}" input field "${role}" collides with a platform integration argument`, + stage: "collect", + diagnostics: [ + { + path: sourcePath, + message: `input field "${role}" collides with declared integration role "${role}"`, + }, + ], + }); + } + } + return null; +}; + +interface AssembledApp { + readonly tools: readonly ToolDescriptor[]; + readonly skipped: readonly SkippedArtifact[]; +} + +export interface PublishDeps { + readonly artifactStore: ArtifactStore; + readonly sandbox: ToolSandbox; +} + +const isPublishError = Predicate.isTagged("PublishError") as ( + value: unknown, +) => value is PublishError; + +const decodeDescriptorJson = Schema.decodeUnknownEffect(Schema.UnknownFromJsonString); + +const assemble = (deps: PublishDeps, files: FileSet): Effect.Effect => + Effect.gen(function* () { + const discovered = discover(files); + if (isPublishError(discovered)) return yield* discovered; + + const tools: ToolDescriptor[] = []; + + for (const artifact of discovered.artifacts) { + const bundle = yield* bundleEntry({ files, entry: artifact.entry }).pipe( + Effect.mapError((bundleFailure) => { + const message = bundleFailure.message; + return new PublishError({ + message, + stage: "bundle", + diagnostics: [{ path: artifact.entry, message }], + }); + }), + ); + const collected = yield* deps.sandbox.collect(bundle.code, { artifact: artifact.name }).pipe( + Effect.mapError((collectFailure) => { + const message = collectFailure.message; + return new PublishError({ + message, + stage: "collect", + diagnostics: [{ path: artifact.entry, message }], + }); + }), + ); + const raw = collected.artifacts.default?.descriptor as CollectedDescriptor | undefined; + const keyed = collected.artifacts[artifact.name]?.descriptor as + | CollectedDescriptor + | undefined; + const descriptor = keyed ?? raw; + if (!descriptor) { + return yield* new PublishError({ + message: `no descriptor collected from ${artifact.entry}`, + stage: "collect", + diagnostics: [{ path: artifact.entry, message: "defineTool did not run" }], + }); + } + const integrations = toIntegrationDecls(descriptor.integrations); + const collision = rejectRoleInputCollisions( + artifact.name, + artifact.entry, + descriptor.inputJsonSchema, + integrations, + ); + if (collision) return yield* collision; + const source = yield* sourceRef(artifact.entry, files.get(artifact.entry) ?? ""); + tools.push({ + name: artifact.name, + sourcePath: artifact.entry, + source, + description: descriptor.description ?? "", + integrations, + inputSchema: descriptor.inputJsonSchema, + outputSchema: descriptor.outputJsonSchema, + annotations: descriptor.annotations, + }); + } + + return { tools, skipped: discovered.skipped }; + }); + +const descriptorBody = ( + tenant: string, + scope: string, + assembled: AssembledApp, + input: Pick, +): Omit => ({ + version: DESCRIPTOR_VERSION, + tenant, + scope, + ...(input.description !== undefined ? { description: input.description } : {}), + ...(input.source !== undefined ? { source: input.source } : {}), + toolchain: toolchainRef(), + tools: assembled.tools, + [FLOW_ENTRIES_KEY]: [], + ui: [], + [GUIDE_ENTRIES_KEY]: [], + skipped: assembled.skipped, +}); + +export const publish = ( + deps: PublishDeps, + input: PublishInput, +): Effect.Effect => + Effect.gen(function* () { + const tenant = input.tenant ?? "org"; + const overLimit = enforcePublishLimits(input.files); + if (overLimit) return yield* overLimit; + + const assembled = yield* assemble(deps, input.files); + const body = descriptorBody(tenant, input.scope, assembled, input); + + const scopeStore = yield* deps.artifactStore.forScope(scopeAddress(tenant, input.scope)).pipe( + Effect.mapError((storeFailure) => { + const message = storeFailure.message; + return new PublishError({ message, stage: "project", diagnostics: [] }); + }), + ); + + const filesWithDescriptor = new Map(input.files); + filesWithDescriptor.set(DESCRIPTOR_SNAPSHOT_PATH, stableStringify(body)); + + const meta = yield* scopeStore + .commit(filesWithDescriptor, input.commitMessage ?? `publish ${new Date().toISOString()}`) + .pipe( + Effect.mapError((storeFailure) => { + const message = storeFailure.message; + return new PublishError({ message, stage: "project", diagnostics: [] }); + }), + ); + + const descriptor: AppDescriptor = { ...body, snapshotId: meta.id }; + + return { snapshotId: meta.id, descriptor, skipped: assembled.skipped }; + }); + +export const loadDescriptorFromSnapshot = ( + store: ArtifactStore, + tenant: string, + scope: string, + snapshotId: SnapshotId, +): Effect.Effect => + Effect.gen(function* () { + const scopeStore = yield* store.forScope(scopeAddress(tenant, scope)).pipe( + Effect.mapError((storeFailure) => { + const message = storeFailure.message; + return new PublishError({ message, stage: "project", diagnostics: [] }); + }), + ); + const raw = yield* scopeStore.readFile(snapshotId, DESCRIPTOR_SNAPSHOT_PATH).pipe( + Effect.mapError((storeFailure) => { + const message = storeFailure.message; + return new PublishError({ message, stage: "project", diagnostics: [] }); + }), + ); + if (raw == null) return null; + const body = yield* decodeDescriptorJson(raw).pipe( + Effect.mapError( + (_cause) => + new PublishError({ + message: "descriptor snapshot is not valid JSON", + stage: "project", + diagnostics: [], + }), + ), + ); + return { ...(body as Omit), snapshotId }; + }); diff --git a/packages/plugins/apps/src/plugin/apps-plugin.test.ts b/packages/plugins/apps/src/plugin/apps-plugin.test.ts new file mode 100644 index 000000000..2e5d50a25 --- /dev/null +++ b/packages/plugins/apps/src/plugin/apps-plugin.test.ts @@ -0,0 +1,210 @@ +import { mkdtempSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +import { describe, expect, it } from "@effect/vitest"; +import { Effect } from "effect"; +import { IntegrationSlug } from "@executor-js/sdk"; +import { makeTestExecutor } from "@executor-js/sdk/testing"; + +import { SourceOriginError, appsPlugin, assertSourceOrigin } from "./apps-plugin"; +import { makeSelfHostAppsRuntime } from "./self-host-runtime"; +import { makeInMemoryAppsStore, makeTestResolver } from "../testing"; +import { dailyBriefFileSet } from "../testing/daily-brief"; + +const run = (effect: Effect.Effect): Promise => Effect.runPromise(effect); + +describe("appsPlugin custom-tools contract", () => { + it.effect("detects GitHub repo URLs for console auto-detect", () => + Effect.gen(function* () { + const executor = yield* makeTestExecutor({ plugins: [appsPlugin()] as const }); + + const repo = yield* executor.integrations.detect( + "https://github.com/RhysSullivan/executor-custom-tools-demo", + ); + const tree = yield* executor.integrations.detect( + "https://github.com/RhysSullivan/executor-custom-tools-demo/tree/feature/custom-tools", + ); + const commit = yield* executor.integrations.detect( + "https://github.com/RhysSullivan/executor-custom-tools-demo/commit/abc1234", + ); + + expect(repo).toEqual([ + { + kind: "apps", + confidence: "high", + endpoint: "https://github.com/RhysSullivan/executor-custom-tools-demo", + name: "Add custom tools from RhysSullivan/executor-custom-tools-demo", + slug: "executor-custom-tools-demo", + }, + ]); + expect(tree[0]?.endpoint).toBe( + "https://github.com/RhysSullivan/executor-custom-tools-demo/tree/feature/custom-tools", + ); + expect(commit[0]?.endpoint).toBe( + "https://github.com/RhysSullivan/executor-custom-tools-demo/commit/abc1234", + ); + }), + ); + + it.effect("leaves non-repo GitHub URLs unclaimed by custom tools detection", () => + Effect.gen(function* () { + const executor = yield* makeTestExecutor({ plugins: [appsPlugin()] as const }); + + const gist = yield* executor.integrations.detect( + "https://gist.github.com/RhysSullivan/abc1234", + ); + const file = yield* executor.integrations.detect( + "https://github.com/RhysSullivan/executor-custom-tools-demo/blob/main/openapi.json", + ); + + expect(gist).toEqual([]); + expect(file).toEqual([]); + }), + ); + + it.effect("rejects publishing through a different source door for a GitHub-managed app", () => + Effect.gen(function* () { + const failure = yield* Effect.flip(assertSourceOrigin("github", "mcp")); + + expect(failure).toBeInstanceOf(SourceOriginError); + expect(failure.message).toBe("this app is managed by its GitHub repo"); + expect(failure.existingOrigin).toBe("github"); + expect(failure.requestedOrigin).toBe("mcp"); + }), + ); + + it("round-trips custom tool files through publish, resolveTools, and invokeTool", async () => { + let issueListArgs: unknown; + const resolver = makeTestResolver( + { + github: { + "issues.listForRepo": (args) => { + issueListArgs = args[0]; + return [{ number: 7, title: "Renewal diligence" }]; + }, + }, + gmail: { + "messages.search": () => ({ messages: [] }), + }, + }, + [ + { + address: "tools.github.user.github-main", + integration: "github", + name: "github-main", + }, + { + address: "tools.gmail.user.inbox-main", + integration: "gmail", + name: "inbox-main", + }, + ], + ); + const host = makeSelfHostAppsRuntime({ + dataDir: mkdtempSync(join(tmpdir(), "apps-plugin-")), + store: makeInMemoryAppsStore(), + resolver, + inMemory: true, + }); + const runtime = host.runtime; + const plugin = appsPlugin({ backings: host.backings }); + const appIntegration = IntegrationSlug.make("rhys-tools"); + const appConfig = { + origin: "github", + kind: "github", + repoUrl: "https://github.com/rhys/tools", + repo: "rhys/tools", + scope: "rhys", + }; + const ctx = { + owner: { tenant: "org" }, + core: { + integrations: { + get: (slug: IntegrationSlug) => + Effect.succeed( + String(slug) === String(appIntegration) + ? { + slug: appIntegration, + name: "Rhys tools", + description: "Rhys tools", + kind: "apps", + canRemove: true, + canRefresh: false, + authMethods: [], + config: appConfig, + } + : null, + ), + }, + }, + }; + + await run(runtime.publish({ scope: "rhys", files: dailyBriefFileSet() })); + + const resolved = await run( + plugin.resolveTools!({ + ctx, + config: appConfig, + connection: { name: "main" }, + } as never), + ); + const syncTool = resolved.tools.find((tool) => String(tool.name) === "issues-sync"); + expect(syncTool).toBeTruthy(); + const persistedInputSchema = syncTool!.inputSchema as { + properties: Record; + }; + expect(persistedInputSchema.properties.github).toBeUndefined(); + + const projected = await run( + plugin.projectToolSchema!({ + ctx, + toolRow: { + name: "issues-sync", + integration: appIntegration, + connection: "main", + }, + inputSchema: syncTool!.inputSchema, + outputSchema: syncTool!.outputSchema, + } as never), + ); + const inputSchema = projected.inputSchema as { + properties: Record; + required?: string[]; + }; + expect(inputSchema.properties.github.enum).toEqual(["tools.github.user.github-main"]); + expect(inputSchema.properties.github.default).toBe("tools.github.user.github-main"); + expect(inputSchema.properties.github.description).toBe("Connection to use for github (github)"); + expect(inputSchema.required ?? []).not.toContain("github"); + + const output = await run( + plugin.invokeTool!({ + ctx, + toolRow: { + name: "issues-sync", + integration: appIntegration, + connection: "main", + }, + args: { + github: "tools.github.user.github-main", + repos: ["acme/tools"], + since: "2026-01-01T00:00:00Z", + }, + } as never), + ); + + expect(output).toEqual({ + synced: 1, + repos: 1, + issues: [{ repo: "acme/tools", number: 7, title: "Renewal diligence" }], + }); + expect(issueListArgs).toEqual({ + owner: "acme", + repo: "tools", + state: "open", + since: "2026-01-01T00:00:00Z", + per_page: 100, + }); + await host.close(); + }); +}); diff --git a/packages/plugins/apps/src/plugin/apps-plugin.ts b/packages/plugins/apps/src/plugin/apps-plugin.ts new file mode 100644 index 000000000..4108c478d --- /dev/null +++ b/packages/plugins/apps/src/plugin/apps-plugin.ts @@ -0,0 +1,801 @@ +import { Data, Effect, Result } from "effect"; + +import { + AuthTemplateSlug, + definePlugin, + IntegrationDetectionResult, + IntegrationSlug, + ProviderItemId, + ProviderKey, + ToolName, + connectionIdentifier, + tool, + type IntegrationRecord, + type InvokeToolInput, + type PluginCtx, + type ResolveToolsInput, + type ResolveToolsResult, + StorageError, + type StorageFailure, + type ToolDef, + type IntegrationRemovalNotAllowedError, +} from "@executor-js/sdk"; + +import type { IntegrationDecl, ToolDescriptor } from "../pipeline/descriptor"; +import { PublishError } from "../pipeline/discover"; +import { ArtifactStoreError } from "../seams/artifact-store"; +import { ToolSandboxError } from "../seams/tool-sandbox"; +import { + parseGitHubSourceUrl, + syncGitHubSource, + type GitHubSkippedArtifact, + type GitHubSyncResult, +} from "../source/github-source"; +import { slugifyCustomToolsAppName, validateCustomToolsAppSlug } from "../source/app-slug"; +import type { AppsRuntime, GitHubCustomToolsSourceSummary } from "./runtime"; +import { makeAppsStore, type AppDescriptorRecord, type GitHubSourceTokenRef } from "./store"; +import { BindingError, type ClientResolver, type ConnectionCandidate } from "./bindings"; +import { makePluginCtxAppsResolver } from "./resolver"; +import { makeAppsRuntimeFromBackings, type AppsBackings } from "./backings"; + +export const APPS_INTEGRATION_SLUG = "apps"; +export const APPS_PLUGIN_ID = "apps"; + +const APP_CONNECTION_NAME = connectionIdentifier("main"); + +interface AppsGitHubSourceConfig { + readonly origin: "github"; + readonly kind: "github"; + readonly repoUrl: string; + readonly repo: string; + readonly scope: string; + readonly ref?: string; + readonly token?: GitHubSourceTokenRef; +} + +type ResolverFactory = (input: { + readonly ctx: unknown; + readonly scope: string; + readonly tool: string; +}) => ClientResolver; + +export class SourceOriginError extends Data.TaggedError("SourceOriginError")<{ + readonly message: string; + readonly existingOrigin: string; + readonly requestedOrigin: string; +}> {} + +export const assertSourceOrigin = ( + existingOrigin: string, + requestedOrigin: string, +): Effect.Effect => + existingOrigin === requestedOrigin + ? Effect.void + : Effect.fail( + new SourceOriginError({ + existingOrigin, + requestedOrigin, + message: + existingOrigin === "github" + ? "this app is managed by its GitHub repo" + : `this app is managed by its ${existingOrigin} source`, + }), + ); + +export type AppsPluginOptions = + | { + readonly backings: AppsBackings; + readonly runtime?: never; + readonly makeResolver?: ResolverFactory; + } + | { + readonly runtime: AppsRuntime; + readonly backings?: never; + readonly makeResolver?: ResolverFactory; + }; + +interface AppsStoreShape { + readonly runtime: AppsRuntime; +} + +export interface AppsPluginExtension { + readonly runtime: AppsRuntime; + readonly syncGitHubSource: (input: unknown) => Effect.Effect; + readonly listGitHubSources: () => Effect.Effect<{ + readonly sources: readonly GitHubCustomToolsSourceSummary[]; + }>; + readonly getGitHubSource: (slug: string) => Effect.Effect<{ + readonly source: GitHubCustomToolsSourceSummary | null; + }>; + readonly removeGitHubSource: ( + slug: string, + ) => Effect.Effect< + { readonly removed: true }, + StorageFailure | IntegrationRemovalNotAllowedError + >; +} + +const isRecord = (value: unknown): value is Record => + value !== null && typeof value === "object" && !Array.isArray(value); + +const asString = (value: unknown): string | undefined => + typeof value === "string" && value.trim().length > 0 ? value.trim() : undefined; + +const unique = (values: readonly string[]): readonly string[] => [...new Set(values)]; + +const isPluginCtx = (value: unknown): value is PluginCtx => + isRecord(value) && + isRecord(value.connections) && + typeof value.connections.list === "function" && + typeof value.connections.get === "function" && + typeof value.execute === "function"; + +const syncFailure = (message: string, path?: string): GitHubSyncResult => ({ + status: "failed", + tools: [], + skipped: [], + errors: [ + { + stage: "source", + message, + ...(path ? { diagnostics: [{ path, message }] } : {}), + }, + ], +}); + +const missingResolver = (): ClientResolver => ({ + listConnections: () => Effect.succeed([]), + resolveConnection: () => Effect.succeed(null), + call: ({ integration, connection }) => + Effect.fail( + new BindingError({ + role: integration, + integration, + requestedConnection: connection, + message: "apps resolver is unavailable outside a request-scoped executor", + }), + ), +}); + +const missingBackingsMessage = "apps plugin requires backings or runtime"; + +const missingPublishError = (): PublishError => + new PublishError({ + message: missingBackingsMessage, + stage: "project", + diagnostics: [], + }); + +const missingStorageError = (): StorageError => + new StorageError({ + message: missingBackingsMessage, + cause: missingBackingsMessage, + }); + +const missingRuntime = (): AppsRuntime => ({ + publish: () => Effect.fail(missingPublishError()), + getDescriptor: () => Effect.succeed(null), + listGitHubSources: () => Effect.succeed([]), + removeSource: () => Effect.fail(missingPublishError()), + repair: () => Effect.fail(missingPublishError()), + invokeTool: () => Effect.fail(missingPublishError()), + deps: { + artifactStore: { + forScope: () => + Effect.fail( + new ArtifactStoreError({ + message: missingBackingsMessage, + }), + ), + removeScope: () => + Effect.fail( + new ArtifactStoreError({ + message: missingBackingsMessage, + }), + ), + }, + sandbox: { + collect: () => + Effect.fail( + new ToolSandboxError({ + message: missingBackingsMessage, + kind: "collect", + }), + ), + invoke: () => + Effect.fail( + new ToolSandboxError({ + message: missingBackingsMessage, + kind: "invoke", + }), + ), + }, + store: { + putDescriptor: () => Effect.fail(missingStorageError()), + getDescriptor: () => Effect.succeed(null), + removeDescriptor: () => Effect.fail(missingStorageError()), + listDescriptors: () => Effect.succeed([]), + }, + resolver: missingResolver(), + }, +}); + +const sourceTokenItemId = (tenant: string, scope: string): ProviderItemId => + ProviderItemId.make(`apps:github-source:${tenant}:${scope}:token`); + +const configBaseUrl = (config: unknown): string | undefined => + isRecord(config) && typeof config.baseUrl === "string" && config.baseUrl.length > 0 + ? config.baseUrl + : undefined; + +const decodeSourceConfig = (config: unknown): AppsGitHubSourceConfig | null => { + if (!isRecord(config) || config.origin !== "github" || config.kind !== "github") return null; + const repoUrl = asString(config.repoUrl); + const repo = asString(config.repo); + const scope = asString(config.scope); + if (!repoUrl || !repo || !scope) return null; + const ref = asString(config.ref); + const token = isRecord(config.token) + ? { + provider: asString(config.token.provider) ?? "", + itemId: asString(config.token.itemId) ?? "", + updatedAt: typeof config.token.updatedAt === "number" ? config.token.updatedAt : 0, + } + : undefined; + return { + origin: "github", + kind: "github", + repoUrl, + repo, + scope, + ...(ref ? { ref } : {}), + ...(token && token.provider && token.itemId ? { token } : {}), + }; +}; + +const projectInputSchema = ( + schema: unknown, + integrations: Readonly>, + byRole: Readonly>, +): unknown => { + const base: Record = isRecord(schema) ? { ...schema } : { type: "object" }; + const properties = isRecord(base.properties) ? { ...base.properties } : {}; + const required = new Set( + Array.isArray(base.required) + ? base.required.filter((value): value is string => typeof value === "string") + : [], + ); + + for (const [role, decl] of Object.entries(integrations)) { + const addresses = unique((byRole[role] ?? []).map((c) => c.address)); + const roleSchema: Record = { + type: "string", + enum: addresses, + description: `Connection to use for ${role} (${decl.integration})`, + }; + if (addresses.length === 1) { + roleSchema.default = addresses[0]; + required.delete(role); + } else { + required.add(role); + } + properties[role] = roleSchema; + } + + const projected: Record = { + ...base, + type: typeof base.type === "string" ? base.type : "object", + properties, + }; + if (required.size > 0) projected.required = [...required]; + else delete projected.required; + return projected; +}; + +const projectTool = (descriptor: ToolDescriptor): ToolDef => ({ + name: ToolName.make(descriptor.name), + description: descriptor.description, + inputSchema: descriptor.inputSchema, + outputSchema: descriptor.outputSchema, + annotations: { + requiresApproval: descriptor.annotations?.destructive === true, + }, +}); + +export const appsPlugin = definePlugin((options?: AppsPluginOptions) => { + const runtime = + options?.runtime ?? + (options?.backings + ? makeAppsRuntimeFromBackings(options.backings, missingResolver()) + : missingRuntime()); + const makeResolver = options?.makeResolver; + + const tenantFor = (ctx: Pick | undefined): string => { + const tenant = ctx?.owner?.tenant; + return tenant === undefined ? "org" : String(tenant); + }; + + const storeSourceToken = ( + ctx: PluginCtx, + tenant: string, + scope: string, + token: string, + ): Effect.Effect => + Effect.gen(function* () { + const itemId = sourceTokenItemId(tenant, scope); + const provider = yield* ctx.providers.setDefault(itemId, token).pipe(Effect.result); + if (Result.isFailure(provider)) return null; + return { + provider: String(provider.success), + itemId: String(itemId), + updatedAt: Date.now(), + }; + }); + + const storedSourceToken = ( + ctx: PluginCtx, + ref: GitHubSourceTokenRef | undefined, + ): Effect.Effect => { + if (!ref) return Effect.succeed(null); + return ctx.providers + .get(ProviderKey.make(ref.provider), ProviderItemId.make(ref.itemId)) + .pipe(Effect.orElseSucceed(() => null)); + }; + + const removeStoredSourceToken = ( + ctx: PluginCtx, + ref: GitHubSourceTokenRef | undefined, + ): Effect.Effect => + ref + ? ctx.providers.remove(ProviderKey.make(ref.provider), ProviderItemId.make(ref.itemId)) + : Effect.void; + + const ensureAppConnection = (slug: IntegrationSlug, ctx: PluginCtx) => + Effect.gen(function* () { + const conns = yield* ctx.connections + .list({ integration: slug }) + .pipe(Effect.orElseSucceed(() => [])); + if (!conns.some((connection) => String(connection.name) === String(APP_CONNECTION_NAME))) { + yield* ctx.connections.create({ + owner: "user", + name: APP_CONNECTION_NAME, + integration: slug, + template: AuthTemplateSlug.make("none"), + value: "", + }); + } + return { owner: "user" as const, integration: slug, name: APP_CONNECTION_NAME }; + }); + + const refreshAppConnection = (slug: IntegrationSlug, ctx: PluginCtx) => + Effect.gen(function* () { + const ref = yield* ensureAppConnection(slug, ctx); + yield* ctx.connections.refresh(ref).pipe(Effect.orElseSucceed(() => [])); + return ref; + }); + + const descriptorRecordByScope = (tenant: string) => + runtime.deps.store.listDescriptors(tenant).pipe( + Effect.orElseSucceed(() => []), + Effect.map((records) => new Map(records.map((record) => [record.descriptor.scope, record]))), + ); + + const sourceSummaryFor = ( + integration: IntegrationRecord, + recordsByScope: ReadonlyMap, + ): GitHubCustomToolsSourceSummary | null => { + const config = decodeSourceConfig(integration.config); + if (!config) return null; + const record = recordsByScope.get(config.scope); + if (!record) return null; + const descriptor = record.descriptor; + const source = descriptor?.source; + if (source?.kind !== "github") return null; + return { + slug: String(integration.slug), + name: integration.name, + scope: descriptor.scope, + url: source.url, + repo: source.repo, + ref: source.ref, + hasToken: config.token !== undefined, + upstreamSha: source.upstreamSha, + snapshotId: descriptor.snapshotId, + ...(descriptor.description ? { description: descriptor.description } : {}), + publishedAt: new Date(record.publishedAt).toISOString(), + tools: descriptor.tools.map((toolDesc) => toolDesc.name), + skipped: [ + ...(source.skipped ?? []), + ...(descriptor.skipped as readonly GitHubSkippedArtifact[]), + ], + }; + }; + + const listSources = (ctx: PluginCtx) => + Effect.gen(function* () { + const tenant = tenantFor(ctx); + const recordsByScope = yield* descriptorRecordByScope(tenant); + const integrations = yield* ctx.core.integrations.list().pipe(Effect.orElseSucceed(() => [])); + const sources: GitHubCustomToolsSourceSummary[] = []; + for (const integration of integrations) { + if (integration.kind !== APPS_PLUGIN_ID) continue; + const record = yield* ctx.core.integrations + .get(integration.slug) + .pipe(Effect.orElseSucceed(() => null)); + if (!record) continue; + const summary = sourceSummaryFor(record, recordsByScope); + if (summary) sources.push(summary); + } + return sources.sort((a, b) => b.publishedAt.localeCompare(a.publishedAt)); + }); + + const getSource = (ctx: PluginCtx, slug: string) => + Effect.gen(function* () { + const tenant = tenantFor(ctx); + const recordsByScope = yield* descriptorRecordByScope(tenant); + const record = yield* ctx.core.integrations + .get(IntegrationSlug.make(slug)) + .pipe(Effect.orElseSucceed(() => null)); + return record && record.kind === APPS_PLUGIN_ID + ? sourceSummaryFor(record, recordsByScope) + : null; + }); + + const syncExistingSource = ( + ctx: PluginCtx, + slugValue: string, + providedToken: string | null, + ) => + Effect.gen(function* () { + const slug = IntegrationSlug.make(slugValue); + const record = yield* ctx.core.integrations.get(slug).pipe(Effect.orElseSucceed(() => null)); + if (!record) { + return syncFailure(`Custom tools source "${slugValue}" does not exist.`); + } + if (record.kind === APPS_PLUGIN_ID && isRecord(record.config)) { + const origin = asString(record.config.origin); + if (origin) { + const originCheck = yield* assertSourceOrigin(origin, "github").pipe(Effect.result); + if (Result.isFailure(originCheck)) return syncFailure(originCheck.failure.message); + } + } + const config = record.kind === APPS_PLUGIN_ID ? decodeSourceConfig(record.config) : null; + if (!config) return syncFailure(`Custom tools source "${slugValue}" does not exist.`); + const github = yield* ctx.core.integrations + .get(IntegrationSlug.make("github")) + .pipe(Effect.orElseSucceed(() => null)); + const tenant = tenantFor(ctx); + const token = providedToken ?? (yield* storedSourceToken(ctx, config.token)); + const result = yield* syncGitHubSource({ + runtime, + tenant, + scope: config.scope, + url: config.repoUrl, + ...(config.ref ? { ref: config.ref } : {}), + token, + baseUrl: configBaseUrl(github?.config), + }); + if (result.status === "failed") return result; + + const tokenRef = providedToken + ? yield* storeSourceToken(ctx, tenant, config.scope, providedToken) + : config.token; + if (providedToken && !tokenRef) { + return syncFailure( + "No writable credential provider is available to store the GitHub source token.", + ); + } + const descriptor = yield* runtime.getDescriptor(tenant, config.scope); + const source = descriptor?.source?.kind === "github" ? descriptor.source : null; + const nextConfig: AppsGitHubSourceConfig = { + ...config, + ...(source + ? { + repoUrl: source.url, + repo: source.repo, + ref: source.ref, + } + : {}), + ...(tokenRef ? { token: tokenRef } : {}), + }; + yield* ctx.core.integrations.update(slug, { + ...(descriptor?.description ? { description: descriptor.description } : {}), + config: nextConfig, + }); + yield* refreshAppConnection(slug, ctx); + return result; + }); + + const addSource = ( + ctx: PluginCtx, + input: { + readonly url: string; + readonly ref?: string; + readonly name?: string; + readonly token: string | null; + }, + ) => + Effect.gen(function* () { + const parsed = parseGitHubSourceUrl(input.url, { ref: input.ref }); + if (!parsed.ok) return syncFailure(parsed.message, input.url); + const rawName = input.name && input.name.trim().length > 0 ? input.name : parsed.value.name; + const slugValue = slugifyCustomToolsAppName(rawName); + const slugError = validateCustomToolsAppSlug(slugValue); + if (slugError) return syncFailure(slugError); + const slug = IntegrationSlug.make(slugValue); + const existing = yield* ctx.core.integrations + .get(slug) + .pipe(Effect.orElseSucceed(() => null)); + if (existing) { + return syncFailure(`Integration "${slugValue}" already exists. Choose another name.`); + } + + const github = yield* ctx.core.integrations + .get(IntegrationSlug.make("github")) + .pipe(Effect.orElseSucceed(() => null)); + const tenant = tenantFor(ctx); + const scope = slugValue; + const result = yield* syncGitHubSource({ + runtime, + tenant, + scope, + url: input.url, + ...(input.ref ? { ref: input.ref } : {}), + token: input.token, + baseUrl: configBaseUrl(github?.config), + }); + if (result.status === "failed") return result; + + const tokenRef = input.token + ? yield* storeSourceToken(ctx, tenant, scope, input.token) + : undefined; + if (input.token && !tokenRef) { + yield* runtime.removeSource({ tenant, scope }).pipe(Effect.orElseSucceed(() => undefined)); + return syncFailure( + "No writable credential provider is available to store the GitHub source token.", + ); + } + + const descriptor = yield* runtime.getDescriptor(tenant, scope); + const source = descriptor?.source?.kind === "github" ? descriptor.source : null; + const config: AppsGitHubSourceConfig = { + origin: "github", + kind: "github", + repoUrl: source?.url ?? input.url.trim(), + repo: source?.repo ?? parsed.value.repo, + scope, + ...(source?.ref ? { ref: source.ref } : input.ref ? { ref: input.ref } : {}), + ...(tokenRef ? { token: tokenRef } : {}), + }; + yield* ctx.core.integrations.register({ + slug, + name: slugValue, + description: + descriptor?.description ?? + `Custom tools synced from ${source?.repo ?? parsed.value.repo}.`, + config, + canRemove: true, + canRefresh: false, + }); + yield* refreshAppConnection(slug, ctx); + return result; + }); + + const configForToolRow = ( + ctx: PluginCtx, + integration: IntegrationSlug, + ): Effect.Effect => + ctx.core.integrations.get(integration).pipe( + Effect.map((record) => + record && record.kind === APPS_PLUGIN_ID ? decodeSourceConfig(record.config) : null, + ), + Effect.orElseSucceed(() => null), + ); + + const requestResolver = (input: { + readonly ctx: unknown; + readonly scope: string; + readonly tool: string; + }): ClientResolver => + makeResolver + ? makeResolver(input) + : isPluginCtx(input.ctx) + ? makePluginCtxAppsResolver({ ctx: input.ctx }) + : runtime.deps.resolver; + + const syncSourceFromPayload = (ctx: PluginCtx, args: unknown) => + Effect.gen(function* () { + const payload = isRecord(args) ? args : {}; + const url = + typeof payload.url === "string" + ? payload.url.trim() + : typeof payload.repo === "string" + ? payload.repo.trim() + : ""; + const ref = asString(payload.ref); + const providedToken = asString(payload.token) ?? null; + const slug = asString(payload.slug); + if (slug && !url) return yield* syncExistingSource(ctx, slug, providedToken); + if (!url) return syncFailure('sync_github_source requires "url"'); + return yield* addSource(ctx, { + url, + ...(ref ? { ref } : {}), + ...(asString(payload.name) ? { name: asString(payload.name) } : {}), + token: providedToken, + }); + }).pipe( + Effect.catchTags({ + CredentialProviderNotRegisteredError: (err) => Effect.succeed(syncFailure(err.message)), + IntegrationNotFoundError: (err) => Effect.succeed(syncFailure(err.message)), + InvalidConnectionInputError: (err) => Effect.succeed(syncFailure(err.message)), + }), + ); + + const extensionFor = (ctx: PluginCtx): AppsPluginExtension => ({ + runtime, + syncGitHubSource: (input) => syncSourceFromPayload(ctx, input), + listGitHubSources: () => + Effect.gen(function* () { + const sources = yield* listSources(ctx); + return { sources }; + }), + getGitHubSource: (slug) => + Effect.gen(function* () { + if (!slug) return { source: null }; + const source = yield* getSource(ctx, slug); + return { source }; + }), + removeGitHubSource: (slug) => + Effect.gen(function* () { + yield* ctx.core.integrations.remove(IntegrationSlug.make(slug)); + return { removed: true as const }; + }), + }); + + return { + id: APPS_PLUGIN_ID as "apps", + packageName: "@executor-js/plugin-apps", + + storage: (deps): AppsStoreShape => { + void makeAppsStore({ + pluginStorage: deps.pluginStorage, + }); + return { runtime }; + }, + + pluginStorage: { + published_descriptor: { + name: "published_descriptor", + schema: { Type: {} as Record }, + indexes: [], + }, + }, + + extension: extensionFor, + + staticSources: (self) => [ + { + id: APPS_PLUGIN_ID, + kind: "executor", + name: "Apps", + tools: [ + tool({ + name: "sync_github_source", + description: "Sync a GitHub repository containing custom tool source files.", + execute: (args) => self.syncGitHubSource(args), + }), + tool({ + name: "list_github_sources", + description: "List synced GitHub repositories that publish custom tools.", + execute: () => self.listGitHubSources(), + }), + tool({ + name: "get_github_source", + description: "Read one synced GitHub custom-tools source.", + execute: (args) => + self.getGitHubSource(isRecord(args) ? (asString(args.slug) ?? "") : ""), + }), + ], + }, + ], + + removeIntegration: ({ ctx, integration }) => + Effect.gen(function* () { + const config = decodeSourceConfig(integration.config); + if (!config) return; + yield* removeStoredSourceToken(ctx, config.token); + yield* runtime.removeSource({ tenant: tenantFor(ctx), scope: config.scope }); + }), + + describeIntegrationDisplay: (integration) => { + const config = decodeSourceConfig(integration.config); + return config ? { url: config.repoUrl } : {}; + }, + + detect: ({ url }) => + Effect.sync(() => { + const parsed = parseGitHubSourceUrl(url); + if (!parsed.ok) return null; + return IntegrationDetectionResult.make({ + kind: APPS_PLUGIN_ID, + confidence: "high", + endpoint: parsed.value.url, + name: `Add custom tools from ${parsed.value.repo}`, + slug: slugifyCustomToolsAppName(parsed.value.name), + }); + }), + + resolveTools: ({ ctx, config }: ResolveToolsInput) => + Effect.gen(function* () { + const source = decodeSourceConfig(config); + if (!source) return { tools: [] } satisfies ResolveToolsResult; + const tenant = ctx ? tenantFor(ctx) : "org"; + const descriptor = yield* runtime.getDescriptor(tenant, source.scope); + if (!descriptor) return { tools: [] } satisfies ResolveToolsResult; + const tools: ToolDef[] = []; + for (const t of descriptor.tools) { + tools.push(projectTool(t)); + } + return { tools } satisfies ResolveToolsResult; + }), + + projectToolSchema: ({ ctx, toolRow, inputSchema, outputSchema }) => + Effect.gen(function* () { + const tenant = tenantFor(ctx); + const source = yield* configForToolRow(ctx, IntegrationSlug.make(toolRow.integration)); + if (!source) return { inputSchema, outputSchema }; + const descriptor = yield* runtime.getDescriptor(tenant, source.scope); + const toolDesc = descriptor?.tools.find((t) => t.name === toolRow.name); + if (!toolDesc) return { inputSchema, outputSchema }; + const resolver = requestResolver({ ctx, scope: source.scope, tool: toolRow.name }); + const byRole: Record = {}; + for (const [role, decl] of Object.entries(toolDesc.integrations)) { + byRole[role] = yield* resolver + .listConnections({ integration: decl.integration }) + .pipe(Effect.orElseSucceed(() => [])); + } + return { + inputSchema: projectInputSchema(toolDesc.inputSchema, toolDesc.integrations, byRole), + outputSchema: toolDesc.outputSchema, + }; + }), + + invokeTool: ({ ctx, toolRow, args, invokeOptions }: InvokeToolInput) => + Effect.gen(function* () { + const tenant = tenantFor(ctx); + const source = yield* configForToolRow(ctx, IntegrationSlug.make(toolRow.integration)); + if (!source) { + return yield* new PublishError({ + message: `apps integration "${toolRow.integration}" has no custom-tools source config`, + stage: "project", + diagnostics: [], + }); + } + const descriptor = yield* runtime.getDescriptor(tenant, source.scope); + if (!descriptor) { + return yield* new PublishError({ + message: `apps scope "${source.scope}" has no published app`, + stage: "project", + diagnostics: [], + }); + } + const toolDesc = descriptor.tools.find((t) => t.name === toolRow.name); + if (!toolDesc) { + return yield* new PublishError({ + message: `apps tool "${toolRow.name}" is not published in scope "${source.scope}"`, + stage: "project", + diagnostics: [], + }); + } + const resolver = requestResolver({ ctx, scope: source.scope, tool: toolRow.name }); + return yield* runtime.invokeTool({ + tenant, + scope: source.scope, + tool: toolRow.name, + args, + resolver, + invokeOptions, + }); + }), + }; +}); diff --git a/packages/plugins/apps/src/plugin/backings.ts b/packages/plugins/apps/src/plugin/backings.ts new file mode 100644 index 000000000..5f2c0268e --- /dev/null +++ b/packages/plugins/apps/src/plugin/backings.ts @@ -0,0 +1,25 @@ +import type { ArtifactStore } from "../seams/artifact-store"; +import type { ToolSandbox } from "../seams/tool-sandbox"; +import { makeAppsRuntime, type AppsRuntime } from "./runtime"; +import type { AppsStore } from "./store"; +import type { ClientResolver } from "./bindings"; + +export interface AppsBackings { + readonly artifactStore: ArtifactStore; + readonly sandbox: ToolSandbox; + readonly store: AppsStore; + readonly defaultTenant?: string; + readonly resolver?: ClientResolver; +} + +export const makeAppsRuntimeFromBackings = ( + backings: AppsBackings, + fallbackResolver: ClientResolver, +): AppsRuntime => + makeAppsRuntime({ + artifactStore: backings.artifactStore, + sandbox: backings.sandbox, + store: backings.store, + resolver: backings.resolver ?? fallbackResolver, + defaultTenant: backings.defaultTenant, + }); diff --git a/packages/plugins/apps/src/plugin/bindings.strict.test.ts b/packages/plugins/apps/src/plugin/bindings.strict.test.ts new file mode 100644 index 000000000..d27b1f5c8 --- /dev/null +++ b/packages/plugins/apps/src/plugin/bindings.strict.test.ts @@ -0,0 +1,125 @@ +import { describe, expect, it } from "@effect/vitest"; +import { Effect, Exit } from "effect"; + +import { buildBridge, resolveIntegrationBindings, type ClientResolver } from "./bindings"; +import type { IntegrationDecl } from "../pipeline/descriptor"; + +// --------------------------------------------------------------------------- +// Strict HandleBridge dispatch (Fix 4, grafted from build A). The bridge is the +// ONE channel out of the sandbox, so its dispatch must reject anything +// malformed, reserved, undeclared, or out-of-range rather than resolve it. +// --------------------------------------------------------------------------- + +const okResolver: ClientResolver = { + listConnections: ({ integration }) => + Effect.succeed([ + { + address: `tools.${integration}.user.main`, + integration, + name: "main", + owner: "user", + }, + ]), + resolveConnection: ({ connection }) => + Effect.succeed( + connection === "tools.github.user.main" + ? { + address: "tools.github.user.main", + integration: "github", + name: "main", + owner: "user", + } + : null, + ), + call: () => Effect.succeed({ ok: true }), +}; + +const fails = (effect: Effect.Effect): Promise => + Effect.runPromiseExit(effect).then((exit) => Exit.isFailure(exit)); + +const failureText = (effect: Effect.Effect): Promise => + Effect.runPromiseExit(effect).then((exit) => JSON.stringify(exit)); + +describe("HandleBridge strict dispatch", () => { + const declared: Record = { + gh: { integration: "github" }, + }; + const bindings = { + gh: "tools.github.user.main", + }; + const bridge = buildBridge({ declared, bindings, resolver: okResolver }); + + it("rejects an empty method path", async () => { + expect(await fails(bridge.call({ root: "gh", path: [], args: [] }))).toBe(true); + }); + + it("rejects a reserved root", async () => { + expect(await fails(bridge.call({ root: "__proto__", path: ["x"], args: [] }))).toBe(true); + }); + + it("rejects an undeclared root", async () => { + expect(await fails(bridge.call({ root: "ghost", path: ["x"], args: [] }))).toBe(true); + }); + + it("rejects an index on an integration role", async () => { + expect(await fails(bridge.call({ root: "gh#0", path: ["repos", "list"], args: [] }))).toBe( + true, + ); + }); + + it("routes a valid single-connection call through the resolver", async () => { + const out = await Effect.runPromise( + bridge.call({ root: "gh", path: ["repos", "list"], args: [{}] }), + ); + expect(out).toEqual({ ok: true }); + }); + + it("defaults a missing role property when exactly one connection exists", async () => { + const resolved = await Effect.runPromise(resolveIntegrationBindings(declared, {}, okResolver)); + expect(resolved.bindings).toEqual({ gh: "tools.github.user.main" }); + expect(resolved.input).toEqual({}); + }); + + it("fails when several connections exist and the role property is missing", async () => { + const resolver: ClientResolver = { + ...okResolver, + listConnections: () => + Effect.succeed([ + { address: "tools.github.user.one", integration: "github" }, + { address: "tools.github.user.two", integration: "github" }, + ]), + }; + expect(await fails(resolveIntegrationBindings(declared, {}, resolver))).toBe(true); + }); + + it("fails when a requested connection belongs to a different integration", async () => { + const resolver: ClientResolver = { + ...okResolver, + resolveConnection: () => + Effect.succeed({ address: "tools.gmail.user.main", integration: "gmail" }), + }; + expect( + await fails(resolveIntegrationBindings(declared, { gh: "tools.gmail.user.main" }, resolver)), + ).toBe(true); + }); + + it("fails when a requested connection is unknown", async () => { + expect( + await fails( + resolveIntegrationBindings(declared, { gh: "tools.github.user.missing" }, okResolver), + ), + ).toBe(true); + }); + + it("rejects a bare connection name and lists valid addresses", async () => { + const text = await failureText( + resolveIntegrationBindings(declared, { gh: "main" }, okResolver), + ); + expect(text).toContain('"message"'); + expect(text).toContain("unknown connection"); + expect(text).toContain("for role"); + expect(text).toContain('"requestedConnection":"main"'); + expect(text).toContain("tools.github.user.main"); + expect(text).not.toContain("unambiguous connection name"); + }); +}); diff --git a/packages/plugins/apps/src/plugin/bindings.ts b/packages/plugins/apps/src/plugin/bindings.ts new file mode 100644 index 000000000..c9fd7a17f --- /dev/null +++ b/packages/plugins/apps/src/plugin/bindings.ts @@ -0,0 +1,251 @@ +import { Effect } from "effect"; +import { Data } from "effect"; +import type { InvokeOptions } from "@executor-js/sdk"; + +import type { HandleBridge, HandleRootSpec } from "../seams/tool-sandbox"; +import { ToolSandboxError } from "../seams/tool-sandbox"; +import type { IntegrationDecl } from "../pipeline/descriptor"; + +// --------------------------------------------------------------------------- +// Integration DI: declare in source, choose at invocation. +// +// A tool descriptor declares `integrations` (role -> integration slug). The +// caller supplies one connection address per role in the tool-call payload, and +// this layer peels those role properties off before the handler runs. If an org +// has exactly one connection for a role's integration, that connection is used +// as the default. Missing, unknown, or wrong-integration choices are typed +// BindingError failures. The handler receives clients under the declared role +// names. +// --------------------------------------------------------------------------- + +export class BindingError extends Data.TaggedError("BindingError")<{ + readonly message: string; + readonly role: string; + readonly integration: string; + readonly requestedConnection?: string; +}> {} + +export interface ConnectionCandidate { + readonly address: string; + readonly integration: string; + readonly name?: string; + readonly owner?: string; +} + +/** The resolved connections for a tool invocation: role -> connection address. */ +export type RoleBindings = Readonly>; + +/** Resolves connection inventory and method calls for declared integration + * roles. Host wiring will back this with core connection lookup and normal + * policy/credential invocation. */ +export interface ClientResolver { + readonly listConnections: (input: { + readonly integration: string; + }) => Effect.Effect; + readonly resolveConnection: (input: { + readonly connection: string; + }) => Effect.Effect; + readonly call: (input: { + readonly integration: string; + readonly connection: string; + readonly path: readonly string[]; + readonly args: readonly unknown[]; + readonly invokeOptions?: InvokeOptions; + }) => Effect.Effect; +} + +export interface BindingContext { + /** Declared integration roles from the tool descriptor. */ + readonly declared: Readonly>; + /** Resolved role -> connection address choices for this invocation. */ + readonly bindings: RoleBindings; + /** Routes a bound method call to the real integration. */ + readonly resolver: ClientResolver; + /** Caller-supplied invoke options to preserve approval/elicitation context. */ + readonly invokeOptions?: InvokeOptions; +} + +const isRecord = (value: unknown): value is Record => + value !== null && typeof value === "object" && !Array.isArray(value); + +const findDefaultConnection = ( + role: string, + decl: IntegrationDecl, + resolver: ClientResolver, +): Effect.Effect => + resolver.listConnections({ integration: decl.integration }).pipe( + Effect.flatMap((connections) => { + if (connections.length === 1) return Effect.succeed(connections[0]!.address); + return Effect.fail( + new BindingError({ + role, + integration: decl.integration, + message: + connections.length === 0 + ? `missing required connection for role "${role}" (${decl.integration}); no connections are available` + : `missing required connection for role "${role}" (${decl.integration}); choose one of ${connections + .map((c) => c.address) + .join(", ")}`, + }), + ); + }), + ); + +const resolveRequestedConnection = ( + role: string, + decl: IntegrationDecl, + requested: string, + resolver: ClientResolver, +): Effect.Effect => + Effect.gen(function* () { + const connection = yield* resolver.resolveConnection({ connection: requested }); + if (connection) { + if (connection.integration !== decl.integration) { + return yield* new BindingError({ + role, + integration: decl.integration, + message: `connection "${requested}" belongs to integration "${connection.integration}", not "${decl.integration}" for role "${role}"`, + requestedConnection: requested, + }); + } + return connection.address; + } + + const connections = yield* resolver.listConnections({ integration: decl.integration }); + return yield* new BindingError({ + role, + integration: decl.integration, + message: `unknown connection "${requested}" for role "${role}" (${decl.integration})${connections.length > 0 ? `; choose one of ${connections.map((candidate) => candidate.address).join(", ")}` : ""}`, + requestedConnection: requested, + }); + }); + +export interface ResolvedIntegrationBindings { + readonly input: unknown; + readonly bindings: RoleBindings; +} + +/** Peel platform-added role properties from the caller payload and resolve them + * to concrete connection addresses. */ +export const resolveIntegrationBindings = ( + declared: Readonly>, + args: unknown, + resolver: ClientResolver, +): Effect.Effect => + Effect.gen(function* () { + const roles = Object.entries(declared); + if (roles.length === 0) return { input: args, bindings: {} }; + + const payload = isRecord(args) ? args : {}; + const input: Record = {}; + for (const [key, value] of Object.entries(payload)) { + if (!Object.prototype.hasOwnProperty.call(declared, key)) input[key] = value; + } + + const bindings: Record = {}; + for (const [role, decl] of roles) { + const raw = payload[role]; + if (raw === undefined) { + bindings[role] = yield* findDefaultConnection(role, decl, resolver); + continue; + } + if (typeof raw !== "string" || raw.length === 0) { + return yield* new BindingError({ + role, + integration: decl.integration, + message: `connection for role "${role}" (${decl.integration}) must be a non-empty string`, + ...(typeof raw === "string" ? { requestedConnection: raw } : {}), + }); + } + bindings[role] = yield* resolveRequestedConnection(role, decl, raw, resolver); + } + + return { input, bindings }; + }); + +/** Compute the sandbox handle roots: one single root per declared integration + * role. */ +export const rootsFor = ( + declared: Readonly>, +): Readonly> => { + const roots: Record = {}; + for (const role of Object.keys(declared)) { + roots[role] = { kind: "single" }; + } + return roots; +}; + +// Parse an indexed root name. Indexed roots are not supported in v1, but the +// parser lets the bridge reject `role#0` explicitly instead of misrouting it. +const parseRoot = (root: string): { role: string; index?: number } => { + const hash = root.indexOf("#"); + if (hash === -1) return { role: root }; + return { role: root.slice(0, hash), index: Number(root.slice(hash + 1)) }; +}; + +const RESERVED_ROOTS = new Set(["__proto__", "constructor", "prototype"]); + +const invokeErr = (message: string): ToolSandboxError => + new ToolSandboxError({ kind: "invoke", message }); + +/** + * Build the HandleBridge the sandbox calls out through. A declared role routes + * to its resolved connection through the `ClientResolver`. The dispatch is + * STRICT: an empty/malformed path, a reserved root, an undeclared root, or an + * indexed root is an error, never a silent success. A `.account` read on a + * client returns bound-connection metadata without a round-trip. + */ +export const buildBridge = (context: BindingContext): HandleBridge => ({ + call: ({ root, path, args }) => { + if (typeof root !== "string" || root.length === 0) { + return Effect.fail(invokeErr("bridge call is missing a handle root")); + } + if (!Array.isArray(path) || path.length === 0) { + return Effect.fail(invokeErr(`bridge call to "${root}" has an empty method path`)); + } + if (RESERVED_ROOTS.has(root)) { + return Effect.fail(invokeErr(`reserved handle root is not callable: ${root}`)); + } + + const { role, index } = parseRoot(root); + if (index !== undefined) { + return Effect.fail( + invokeErr(`single integration role "${role}" was addressed with an index`), + ); + } + + const decl = context.declared[role]; + if (!decl) { + return Effect.fail(invokeErr(`undeclared handle root: ${root}`)); + } + + const connectionName = context.bindings[role]; + if (!connectionName) { + return Effect.fail(invokeErr(`no binding for role ${role}`)); + } + if (path.length === 1 && path[0] === "account") { + // Connection-derived placeholders only; this is not verified upstream + // account identity. + return Effect.succeed({ email: connectionName, login: connectionName, name: connectionName }); + } + + return context.resolver + .call({ + integration: decl.integration, + connection: connectionName, + path, + args, + invokeOptions: context.invokeOptions, + }) + .pipe( + Effect.mapError( + (cause) => + new ToolSandboxError({ + kind: "invoke", + message: "integration call failed", + cause, + }), + ), + ); + }, +}); diff --git a/packages/plugins/apps/src/plugin/handlers.ts b/packages/plugins/apps/src/plugin/handlers.ts new file mode 100644 index 000000000..9d978180e --- /dev/null +++ b/packages/plugins/apps/src/plugin/handlers.ts @@ -0,0 +1,49 @@ +import { Context, Effect } from "effect"; +import { HttpApiBuilder } from "effect/unstable/httpapi"; + +import { addGroup, capture } from "@executor-js/api"; +import type { AppsPluginExtension } from "./apps-plugin"; +import { AppsGroup } from "./routes"; + +export class AppsExtensionService extends Context.Service< + AppsExtensionService, + AppsPluginExtension +>()("AppsExtensionService") {} + +const ExecutorApiWithApps = addGroup(AppsGroup); + +export const AppsHandlers = HttpApiBuilder.group(ExecutorApiWithApps, "apps", (handlers) => + handlers + .handle("listGithubSources", () => + capture( + Effect.gen(function* () { + const ext = yield* AppsExtensionService; + return yield* ext.listGitHubSources(); + }), + ), + ) + .handle("syncGithubSource", ({ payload }) => + capture( + Effect.gen(function* () { + const ext = yield* AppsExtensionService; + return yield* ext.syncGitHubSource(payload); + }), + ), + ) + .handle("getGithubSource", ({ params }) => + capture( + Effect.gen(function* () { + const ext = yield* AppsExtensionService; + return yield* ext.getGitHubSource(params.slug); + }), + ), + ) + .handle("removeGithubSource", ({ params }) => + capture( + Effect.gen(function* () { + const ext = yield* AppsExtensionService; + return yield* ext.removeGitHubSource(params.slug); + }), + ), + ), +); diff --git a/packages/plugins/apps/src/plugin/publish-concurrency.test.ts b/packages/plugins/apps/src/plugin/publish-concurrency.test.ts new file mode 100644 index 000000000..abb7d4c64 --- /dev/null +++ b/packages/plugins/apps/src/plugin/publish-concurrency.test.ts @@ -0,0 +1,84 @@ +import { mkdtempSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +import { describe, expect, it } from "@effect/vitest"; +import { Effect } from "effect"; + +import { makeSelfHostAppsRuntime } from "./self-host-runtime"; +import { makeInMemoryAppsStore, makeTestResolver } from "../testing"; +import { scopeAddress } from "../seams/scope-address"; + +const run = (effect: Effect.Effect): Promise => Effect.runPromise(effect); + +// --------------------------------------------------------------------------- +// Finding 6 regression: concurrent publishes to one scope must not race the git +// ref update or the descriptor-pointer write. They serialize (both succeed in +// sequence) or one gets a typed conflict; afterwards the committed HEAD and the +// stored descriptor pointer always AGREE. Before the fix the ref update had no +// expected-old-value (last-writer-wins clobber) and the pointer write raced. +// +// Uses the REAL git-backed artifact store (the git-CAS is the mechanism under +// test) via `inMemory: false`. +// --------------------------------------------------------------------------- + +const toolFiles = (tag: string): ReadonlyMap => + new Map([ + [ + "tools/echo.ts", + `import { defineTool } from "executor:app"; +import { z } from "zod"; +export default defineTool({ + description: "echo ${tag}", + input: z.object({}), + async handler(){ return { tag: "${tag}" }; }, +});`, + ], + ]); + +describe("concurrent publishes to one scope (Fix 6)", () => { + it("serialize with head and descriptor pointer always in agreement", async () => { + const store = makeInMemoryAppsStore(); + const resolver = makeTestResolver({}); + const host = makeSelfHostAppsRuntime({ + dataDir: mkdtempSync(join(tmpdir(), "apps-pubrace-")), + store, + resolver, + // Real git-backed artifact store: exercise the ref compare-and-swap. + inMemory: false, + }); + const { runtime } = host; + + // Fire many concurrent publishes to the SAME scope. + const N = 8; + const results = await Promise.allSettled( + Array.from({ length: N }, (_v, i) => + run(runtime.publish({ scope: "s", files: toolFiles(`v${i}`) })), + ), + ); + + // Every publish either succeeded or failed with a typed conflict; none + // clobbered another silently. + const succeeded = results.filter((r) => r.status === "fulfilled"); + const rejectedReasons = results.flatMap((r) => + r.status === "rejected" ? [JSON.stringify(r.reason)] : [], + ); + expect(succeeded.length).toBeGreaterThan(0); + // A conflict is the only acceptable failure mode. + expect(rejectedReasons.every((reason) => /conflict/i.test(reason))).toBe(true); + + // HEAD and the descriptor pointer AGREE: the store's current descriptor's + // snapshotId equals the artifact store's latest committed snapshot. + const latest = await run( + runtime.deps.artifactStore + .forScope(scopeAddress("org", "s")) + .pipe(Effect.flatMap((s) => s.latest())), + ); + expect(latest).not.toBeNull(); + const pointer = await run(runtime.getDescriptor("s")); + expect(pointer).not.toBeNull(); + expect(pointer!.snapshotId).toBe(latest!.id); + + await host.close(); + }, 60_000); +}); diff --git a/packages/plugins/apps/src/plugin/resolver.test.ts b/packages/plugins/apps/src/plugin/resolver.test.ts new file mode 100644 index 000000000..b675bc4ee --- /dev/null +++ b/packages/plugins/apps/src/plugin/resolver.test.ts @@ -0,0 +1,46 @@ +import { describe, expect, it } from "@effect/vitest"; +import { Effect } from "effect"; +import type { InvokeOptions } from "@executor-js/sdk"; + +import { makePluginCtxAppsResolver, type AppsResolverPluginCtx } from "./resolver"; + +const run = (effect: Effect.Effect): Promise => Effect.runPromise(effect); + +describe("makePluginCtxAppsResolver", () => { + it("passes caller invoke options through bridged integration calls", async () => { + const invokeOptions = { onElicitation: "accept-all" } satisfies InvokeOptions; + let seenAddress = ""; + let seenPayload: unknown; + let seenOptions: InvokeOptions | undefined; + + const ctx: AppsResolverPluginCtx = { + connections: { + list: () => Effect.succeed([]), + get: () => Effect.succeed(null), + }, + execute: (address: unknown, payload: unknown, options?: InvokeOptions) => + Effect.sync(() => { + seenAddress = String(address); + seenPayload = payload; + seenOptions = options; + return "called"; + }), + }; + + const resolver = makePluginCtxAppsResolver({ ctx }); + const result = await run( + resolver.call({ + integration: "github", + connection: "tools.github.user.main", + path: ["issues", "listForRepo"], + args: [{ owner: "acme", repo: "tools" }], + invokeOptions, + }), + ); + + expect(result).toBe("called"); + expect(seenAddress).toBe("tools.github.user.main.issues.listForRepo"); + expect(seenPayload).toEqual({ owner: "acme", repo: "tools" }); + expect(seenOptions).toBe(invokeOptions); + }); +}); diff --git a/packages/plugins/apps/src/plugin/resolver.ts b/packages/plugins/apps/src/plugin/resolver.ts new file mode 100644 index 000000000..e4fd70c65 --- /dev/null +++ b/packages/plugins/apps/src/plugin/resolver.ts @@ -0,0 +1,123 @@ +import { Effect } from "effect"; + +import { + ConnectionName, + IntegrationSlug, + ToolAddress, + isToolResult, + type ConnectionRef, + type Owner, + type PluginCtx, +} from "@executor-js/sdk"; +import { BindingError, type ClientResolver, type ConnectionCandidate } from "./bindings"; + +export interface AppsResolverPluginCtx { + readonly connections: Pick; + readonly execute: PluginCtx["execute"]; +} + +const parseConnectionAddress = (address: string): ConnectionRef | null => { + const parts = address.split("."); + if (parts.length !== 4 || parts[0] !== "tools") return null; + const [, integration, owner, name] = parts; + if (!integration || !name) return null; + if (owner !== "org" && owner !== "user") return null; + return { + owner: owner as Owner, + integration: IntegrationSlug.make(integration), + name: ConnectionName.make(name), + }; +}; + +const toCandidate = (connection: { + readonly address: unknown; + readonly integration: unknown; + readonly name?: unknown; + readonly owner?: unknown; +}): ConnectionCandidate => ({ + address: String(connection.address), + integration: String(connection.integration), + ...(connection.name !== undefined ? { name: String(connection.name) } : {}), + ...(connection.owner !== undefined ? { owner: String(connection.owner) } : {}), +}); + +export const makePluginCtxAppsResolver = (input: { + readonly ctx: AppsResolverPluginCtx; +}): ClientResolver => ({ + listConnections: ({ integration }) => + input.ctx.connections.list({ integration: IntegrationSlug.make(integration) }).pipe( + Effect.map((connections) => connections.map(toCandidate)), + Effect.mapError( + () => + new BindingError({ + role: integration, + integration, + message: `failed to list ${integration} connections`, + }), + ), + ), + + resolveConnection: ({ connection }) => + Effect.gen(function* () { + const ref = parseConnectionAddress(connection); + if (!ref) return null; + const row = yield* input.ctx.connections.get(ref).pipe( + Effect.mapError( + () => + new BindingError({ + role: String(ref.integration), + integration: String(ref.integration), + requestedConnection: connection, + message: `failed to resolve connection ${connection}`, + }), + ), + ); + return row ? toCandidate(row) : null; + }), + + call: ({ integration, connection, path, args, invokeOptions }) => + Effect.gen(function* () { + const ref = parseConnectionAddress(connection); + if (!ref) { + return yield* new BindingError({ + role: integration, + integration, + requestedConnection: connection, + message: `invalid connection address ${connection}`, + }); + } + if (String(ref.integration) !== integration) { + return yield* new BindingError({ + role: integration, + integration, + requestedConnection: connection, + message: `connection "${connection}" belongs to integration "${ref.integration}", not "${integration}"`, + }); + } + const tool = path.join("."); + const address = ToolAddress.make(`${connection}.${tool}`); + const payload = args[0] ?? {}; + const result = yield* input.ctx.execute(address, payload, invokeOptions).pipe( + Effect.mapError( + () => + new BindingError({ + role: integration, + integration, + requestedConnection: connection, + message: `failed to execute ${address}`, + }), + ), + ); + if (isToolResult(result)) { + if (result.ok) return result.data; + const { message } = result.error; + return yield* new BindingError({ + role: integration, + integration, + requestedConnection: connection, + message, + }); + } + return result; + }), +}); diff --git a/packages/plugins/apps/src/plugin/routes.ts b/packages/plugins/apps/src/plugin/routes.ts new file mode 100644 index 000000000..9640c2552 --- /dev/null +++ b/packages/plugins/apps/src/plugin/routes.ts @@ -0,0 +1,104 @@ +import { Schema } from "effect"; +import { HttpApiEndpoint, HttpApiGroup } from "effect/unstable/httpapi"; +import { IntegrationRemovalNotAllowedError, InternalError } from "@executor-js/sdk/shared"; + +const GitHubSourcePayload = Schema.Struct({ + url: Schema.optional(Schema.String), + repo: Schema.optional(Schema.String), + name: Schema.optional(Schema.String), + slug: Schema.optional(Schema.String), + ref: Schema.optional(Schema.String), + token: Schema.optional(Schema.String), +}); + +const SourceSlugParams = { + slug: Schema.String, +}; + +const SkippedArtifact = Schema.Struct({ + path: Schema.String, + reason: Schema.Literals(["not supported yet", "unsupported file type", "ignored"]), +}); + +const FileDiagnostic = Schema.Struct({ + path: Schema.String, + message: Schema.String, +}); + +const SyncErrorData = Schema.Struct({ + stage: Schema.Literals(["source", "discover", "bundle", "collect", "project"]), + message: Schema.String, + diagnostics: Schema.optional(Schema.Array(FileDiagnostic)), +}); + +const GitHubSyncResponse = Schema.Struct({ + status: Schema.Literals(["published", "up-to-date", "failed"]), + snapshotId: Schema.optional(Schema.String), + upstreamSha: Schema.optional(Schema.String), + tools: Schema.Array(Schema.String), + skipped: Schema.Array(SkippedArtifact), + errors: Schema.optional(Schema.Array(SyncErrorData)), +}); + +const GitHubSourceSummary = Schema.Struct({ + slug: Schema.String, + name: Schema.String, + scope: Schema.String, + url: Schema.String, + repo: Schema.String, + ref: Schema.String, + hasToken: Schema.Boolean, + upstreamSha: Schema.String, + snapshotId: Schema.String, + description: Schema.optional(Schema.String), + publishedAt: Schema.String, + tools: Schema.Array(Schema.String), + skipped: Schema.Array(SkippedArtifact), +}); + +const GitHubSourcesResponse = Schema.Struct({ + sources: Schema.Array(GitHubSourceSummary), +}); + +const GitHubSourceResponse = Schema.Struct({ + source: Schema.NullOr(GitHubSourceSummary), +}); + +const RemoveSourceResponse = Schema.Struct({ + removed: Schema.Boolean, +}); + +const DomainErrors = [InternalError] as const; +const IntegrationRemovalNotAllowed = IntegrationRemovalNotAllowedError.annotate({ + httpApiStatus: 409, +}); +const RemoveSourceErrors = [InternalError, IntegrationRemovalNotAllowed] as const; + +export const AppsGroup = HttpApiGroup.make("apps") + .add( + HttpApiEndpoint.get("listGithubSources", "/apps/sources/github", { + success: GitHubSourcesResponse, + error: DomainErrors, + }), + ) + .add( + HttpApiEndpoint.post("syncGithubSource", "/apps/sources/github/sync", { + payload: GitHubSourcePayload, + success: GitHubSyncResponse, + error: DomainErrors, + }), + ) + .add( + HttpApiEndpoint.get("getGithubSource", "/apps/sources/github/:slug", { + params: SourceSlugParams, + success: GitHubSourceResponse, + error: DomainErrors, + }), + ) + .add( + HttpApiEndpoint.delete("removeGithubSource", "/apps/sources/github/:slug", { + params: SourceSlugParams, + success: RemoveSourceResponse, + error: RemoveSourceErrors, + }), + ); diff --git a/packages/plugins/apps/src/plugin/runtime.test.ts b/packages/plugins/apps/src/plugin/runtime.test.ts new file mode 100644 index 000000000..d004dd0fd --- /dev/null +++ b/packages/plugins/apps/src/plugin/runtime.test.ts @@ -0,0 +1,252 @@ +import { mkdtempSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +import { describe, expect, it } from "@effect/vitest"; +import { Effect, Exit } from "effect"; + +import { makeSelfHostAppsRuntime } from "./self-host-runtime"; +import { makeInMemoryAppsStore, makeTestResolver, dailyBriefFileSet } from "../testing"; + +const run = (effect: Effect.Effect): Promise => Effect.runPromise(effect); + +const githubHandlers = { + github: { + "repos.listForAuthenticatedUser": () => [{ full_name: "acme/app" }], + "issues.listForRepo": () => [ + { + number: 1, + title: "Fresh bug", + labels: [{ name: "bug" }], + assignee: { login: "rhys" }, + updated_at: new Date().toISOString(), + html_url: "https://github.com/acme/app/issues/1", + }, + { + number: 2, + title: "Old bug", + labels: [], + assignee: null, + updated_at: "2020-01-01T00:00:00Z", + html_url: "https://github.com/acme/app/issues/2", + }, + ], + }, +}; + +describe("AppsRuntime end-to-end (publish -> invoke)", () => { + it("publishes daily-brief and invokes the tool through declared integrations", async () => { + const store = makeInMemoryAppsStore(); + const resolver = makeTestResolver(githubHandlers, [ + { address: "tools.github.user.rhys-github", integration: "github", name: "rhys-github" }, + ]); + const host = makeSelfHostAppsRuntime({ + dataDir: mkdtempSync(join(tmpdir(), "apps-rt-")), + store, + resolver, + inMemory: true, + }); + const { runtime } = host; + + const published = await run(runtime.publish({ scope: "rhys", files: dailyBriefFileSet() })); + expect(published.descriptor.tools.map((t) => t.name).sort()).toEqual([ + "issues-sync", + "search-all-mail", + ]); + + const syncResult = (await run( + runtime.invokeTool({ + scope: "rhys", + tool: "issues-sync", + args: { github: "tools.github.user.rhys-github" }, + }), + )) as { + synced: number; + repos: number; + issues: readonly { repo: string; number: number; title: string }[]; + }; + expect(syncResult).toEqual({ + synced: 2, + repos: 1, + issues: [ + { repo: "acme/app", number: 1, title: "Fresh bug" }, + { repo: "acme/app", number: 2, title: "Old bug" }, + ], + }); + + await host.close(); + }); + + it("applies the single available connection as the default", async () => { + const store = makeInMemoryAppsStore(); + const resolver = makeTestResolver( + { + dealcloud: { + "deals.list": () => [], + }, + }, + [{ address: "tools.dealcloud.user.crm-main", integration: "dealcloud", name: "crm-main" }], + ); + const host = makeSelfHostAppsRuntime({ + dataDir: mkdtempSync(join(tmpdir(), "apps-default-")), + store, + resolver, + inMemory: true, + }); + + await run( + host.runtime.publish({ + scope: "s", + files: new Map([ + [ + "tools/sync.ts", + `import { defineTool, integration } from "executor:app"; +import { z } from "zod"; +export default defineTool({ + description: "sync", + integrations: { crm: integration("dealcloud") }, + input: z.object({}), + async handler(_input, { crm }) { + await crm.deals.list({}); + return { ok: true }; + }, +});`, + ], + ]), + }), + ); + + const out = await run(host.runtime.invokeTool({ scope: "s", tool: "sync", args: {} })); + + expect(out).toEqual({ ok: true }); + expect(resolver.calls[0]).toEqual({ + integration: "dealcloud", + connection: "tools.dealcloud.user.crm-main", + method: "deals.list", + }); + await host.close(); + }); + + it("fails with BindingError when multiple connections exist and the role property is missing", async () => { + const store = makeInMemoryAppsStore(); + const resolver = makeTestResolver({}, [ + { address: "tools.dealcloud.user.one", integration: "dealcloud" }, + { address: "tools.dealcloud.user.two", integration: "dealcloud" }, + ]); + const host = makeSelfHostAppsRuntime({ + dataDir: mkdtempSync(join(tmpdir(), "apps-missing-")), + store, + resolver, + inMemory: true, + }); + + await run( + host.runtime.publish({ + scope: "s", + files: new Map([ + [ + "tools/sync.ts", + `import { defineTool, integration } from "executor:app"; +import { z } from "zod"; +export default defineTool({ + description: "sync", + integrations: { crm: integration("dealcloud") }, + input: z.object({}), + async handler(){ return {}; }, +});`, + ], + ]), + }), + ); + + const exit = await Effect.runPromiseExit( + host.runtime.invokeTool({ scope: "s", tool: "sync", args: {} }), + ); + + expect(Exit.isFailure(exit)).toBe(true); + expect(JSON.stringify(exit)).toContain("BindingError"); + expect(JSON.stringify(exit)).toContain("crm"); + expect(JSON.stringify(exit)).toContain("dealcloud"); + await host.close(); + }); + + it("fails with BindingError when the named connection belongs to another integration", async () => { + const store = makeInMemoryAppsStore(); + const resolver = makeTestResolver({}, [ + { address: "tools.github.user.main", integration: "github" }, + ]); + const host = makeSelfHostAppsRuntime({ + dataDir: mkdtempSync(join(tmpdir(), "apps-wrong-")), + store, + resolver, + inMemory: true, + }); + + await run( + host.runtime.publish({ + scope: "s", + files: new Map([ + [ + "tools/sync.ts", + `import { defineTool, integration } from "executor:app"; +import { z } from "zod"; +export default defineTool({ + description: "sync", + integrations: { crm: integration("dealcloud") }, + input: z.object({}), + async handler(){ return {}; }, +});`, + ], + ]), + }), + ); + + const exit = await Effect.runPromiseExit( + host.runtime.invokeTool({ + scope: "s", + tool: "sync", + args: { crm: "tools.github.user.main" }, + }), + ); + + expect(Exit.isFailure(exit)).toBe(true); + expect(JSON.stringify(exit)).toContain("BindingError"); + expect(JSON.stringify(exit)).toContain("github"); + expect(JSON.stringify(exit)).toContain("dealcloud"); + expect(JSON.stringify(exit)).toContain("tools.github.user.main"); + await host.close(); + }); + + it("accepts raw JSON Schema input end to end", async () => { + const store = makeInMemoryAppsStore(); + const resolver = makeTestResolver({}); + const host = makeSelfHostAppsRuntime({ + dataDir: mkdtempSync(join(tmpdir(), "apps-raw-")), + store, + resolver, + inMemory: true, + }); + + await run( + host.runtime.publish({ + scope: "s", + files: new Map([ + [ + "tools/raw.ts", + `import { defineTool } from "executor:app"; +export default defineTool({ + description: "raw", + input: { type: "object", properties: { q: { type: "string" } }, required: ["q"] }, + async handler(input){ return { q: input.q }; }, +});`, + ], + ]), + }), + ); + + const out = await run(host.runtime.invokeTool({ scope: "s", tool: "raw", args: { q: "ok" } })); + + expect(out).toEqual({ q: "ok" }); + await host.close(); + }); +}); diff --git a/packages/plugins/apps/src/plugin/runtime.ts b/packages/plugins/apps/src/plugin/runtime.ts new file mode 100644 index 000000000..8f5ec6d9e --- /dev/null +++ b/packages/plugins/apps/src/plugin/runtime.ts @@ -0,0 +1,449 @@ +import { Effect, Predicate } from "effect"; +import type { InvokeOptions } from "@executor-js/sdk"; + +import type { ArtifactStore } from "../seams/artifact-store"; +import { + InputValidationError, + OutputValidationError, + type ToolSandbox, +} from "../seams/tool-sandbox"; +import { + publish as runPublish, + loadDescriptorFromSnapshot, + type PublishOutput, +} from "../pipeline/publish"; +import { PublishError } from "../pipeline/discover"; +import type { AppDescriptor, AppSourceRef, ToolDescriptor } from "../pipeline/descriptor"; +import { bundleEntry } from "../pipeline/bundle"; +import { + buildBridge, + rootsFor, + resolveIntegrationBindings, + type ClientResolver, + BindingError, +} from "./bindings"; +import type { AppsStore } from "./store"; +import { scopeAddress } from "../seams/scope-address"; +import type { GitHubSkippedArtifact } from "../source/github-source"; + +// --------------------------------------------------------------------------- +// AppsRuntime: the substrate-neutral core for published custom tools. +// --------------------------------------------------------------------------- + +export interface AppsRuntimeDeps { + readonly artifactStore: ArtifactStore; + readonly sandbox: ToolSandbox; + readonly store: AppsStore; + /** Routes a bound integration method call to the real API (policy/audit). */ + readonly resolver: ClientResolver; + readonly defaultTenant?: string; +} + +export interface GitHubCustomToolsSourceSummary { + readonly slug: string; + readonly name: string; + readonly scope: string; + readonly url: string; + readonly repo: string; + readonly ref: string; + readonly hasToken: boolean; + readonly upstreamSha: string; + readonly snapshotId: string; + readonly description?: string; + readonly publishedAt: string; + readonly tools: readonly string[]; + readonly skipped: readonly GitHubSkippedArtifact[]; +} + +export interface AppsRuntime { + readonly publish: (input: { + readonly tenant?: string; + readonly scope: string; + readonly files: ReadonlyMap; + readonly message?: string; + readonly description?: string; + readonly source?: AppSourceRef; + }) => Effect.Effect; + readonly getDescriptor: ( + tenantOrScope: string, + scope?: string, + ) => Effect.Effect; + readonly listGitHubSources: ( + tenant?: string, + ) => Effect.Effect; + readonly removeSource: (input: { + readonly tenant?: string; + readonly scope: string; + }) => Effect.Effect; + /** Re-derive the published-descriptor pointer from the latest committed + * snapshot. */ + readonly repair: ( + tenantOrScope: string, + scope?: string, + ) => Effect.Effect; + readonly invokeTool: (input: { + readonly tenant?: string; + readonly scope: string; + readonly tool: string; + readonly args: unknown; + /** Optional per-request resolver override. The catalog invoke path supplies + * one built from the request's executor context. */ + readonly resolver?: ClientResolver; + /** Optional caller options forwarded to bridged sub-calls. */ + readonly invokeOptions?: InvokeOptions; + }) => Effect.Effect< + unknown, + PublishError | BindingError | InputValidationError | OutputValidationError + >; + readonly deps: AppsRuntimeDeps; +} + +const failNoDescriptor = (scope: string): PublishError => + new PublishError({ + message: `scope "${scope}" has no published app`, + stage: "project", + diagnostics: [], + }); + +const isInvokePassthroughError = ( + cause: unknown, +): cause is BindingError | InputValidationError | OutputValidationError => + Predicate.isTagged("BindingError")(cause) || + Predicate.isTagged("InputValidationError")(cause) || + Predicate.isTagged("OutputValidationError")(cause); + +export const makeAppsRuntime = (deps: AppsRuntimeDeps): AppsRuntime => { + const defaultTenant = deps.defaultTenant ?? "org"; + const bundleCache = new Map(); + const publishChains = new Map>(); + const resolveTenant = (tenant?: string): string => tenant ?? defaultTenant; + const resolveTenantScope = ( + tenantOrScope: string, + maybeScope?: string, + ): { tenant: string; scope: string } => + maybeScope === undefined + ? { tenant: defaultTenant, scope: tenantOrScope } + : { tenant: tenantOrScope, scope: maybeScope }; + + const withScopePublishLock = ( + tenant: string, + scope: string, + run: () => Promise, + ): Promise => { + const key = `${tenant}:${scope}`; + const prior = publishChains.get(key) ?? Promise.resolve(); + const afterPrior = prior.then( + () => undefined, + () => undefined, + ); + const next = afterPrior.then(run); + publishChains.set( + key, + next.then( + () => undefined, + () => undefined, + ), + ); + return next; + }; + + const bundleFor = ( + descriptor: AppDescriptor, + sourcePath: string, + ): Effect.Effect => + Effect.gen(function* () { + const cacheKey = `${descriptor.snapshotId}:${sourcePath}`; + const cached = bundleCache.get(cacheKey); + if (cached) return cached; + const scopeStore = yield* deps.artifactStore + .forScope(scopeAddress(descriptor.tenant, descriptor.scope)) + .pipe( + Effect.mapError( + (c) => + new PublishError({ + message: c.message, + stage: "project", + diagnostics: [], + }), + ), + ); + const files = yield* scopeStore.read(descriptor.snapshotId as never).pipe( + Effect.mapError( + (c) => + new PublishError({ + message: c.message, + stage: "project", + diagnostics: [], + }), + ), + ); + const bundle = yield* bundleEntry({ files, entry: sourcePath }).pipe( + Effect.mapError( + (c) => + new PublishError({ + message: c.message, + stage: "bundle", + diagnostics: [{ path: sourcePath, message: c.message }], + }), + ), + ); + bundleCache.set(cacheKey, bundle.code); + return bundle.code; + }); + + const recoverFromSnapshot = ( + tenant: string, + scope: string, + ): Effect.Effect => + Effect.gen(function* () { + const scopeStore = yield* deps.artifactStore.forScope(scopeAddress(tenant, scope)).pipe( + Effect.mapError( + (c) => + new PublishError({ + message: c.message, + stage: "project", + diagnostics: [], + }), + ), + ); + const latest = yield* scopeStore.latest().pipe( + Effect.mapError( + (c) => + new PublishError({ + message: c.message, + stage: "project", + diagnostics: [], + }), + ), + ); + if (!latest) return null; + return yield* loadDescriptorFromSnapshot(deps.artifactStore, tenant, scope, latest.id); + }); + + const requireDescriptor = ( + tenant: string, + scope: string, + ): Effect.Effect => + deps.store.getDescriptor(tenant, scope).pipe( + Effect.mapError( + (c) => + new PublishError({ + message: String(c), + stage: "project", + diagnostics: [], + }), + ), + Effect.flatMap((d) => + d + ? Effect.succeed(d) + : recoverFromSnapshot(tenant, scope).pipe(Effect.orElseSucceed(() => null)), + ), + Effect.flatMap((d) => (d ? Effect.succeed(d) : Effect.fail(failNoDescriptor(scope)))), + ); + + const putDescriptorPointer = ( + tenant: string, + descriptor: AppDescriptor, + ): Effect.Effect => + deps.store.putDescriptor(tenant, "org", descriptor).pipe( + Effect.mapError( + (c) => + new PublishError({ + message: String(c), + stage: "project", + diagnostics: [], + }), + ), + ); + + const repairScope = ( + tenant: string, + scope: string, + ): Effect.Effect => + Effect.gen(function* () { + const descriptor = yield* recoverFromSnapshot(tenant, scope); + if (!descriptor) return null; + yield* putDescriptorPointer(tenant, descriptor); + return descriptor; + }); + + const invokeToolInternal = ( + scope: string, + descriptor: AppDescriptor, + toolDesc: ToolDescriptor, + args: unknown, + resolver?: ClientResolver, + invokeOptions?: InvokeOptions, + ): Effect.Effect< + unknown, + PublishError | BindingError | InputValidationError | OutputValidationError + > => + Effect.gen(function* () { + const activeResolver = resolver ?? deps.resolver; + const resolved = yield* resolveIntegrationBindings( + toolDesc.integrations, + args, + activeResolver, + ); + const roots = rootsFor(toolDesc.integrations); + const code = yield* bundleFor(descriptor, toolDesc.sourcePath); + const bridge = buildBridge({ + declared: toolDesc.integrations, + bindings: resolved.bindings, + resolver: activeResolver, + invokeOptions, + }); + const result = yield* deps.sandbox + .invoke( + code, + { artifact: toolDesc.name, kind: "tool", input: resolved.input, roots }, + bridge, + ) + .pipe( + Effect.mapError((c) => { + if (isInvokePassthroughError(c)) { + return c; + } + return new PublishError({ + message: c.message, + stage: "project", + diagnostics: [{ path: toolDesc.sourcePath, message: c.message }], + }); + }), + ); + return result.output; + }); + + return { + deps, + publish: (input) => + Effect.flatMap( + Effect.tryPromise({ + try: () => { + const tenant = resolveTenant(input.tenant); + return withScopePublishLock(tenant, input.scope, () => + Effect.runPromiseExit( + Effect.gen(function* () { + const out = yield* runPublish( + { + artifactStore: deps.artifactStore, + sandbox: deps.sandbox, + }, + { + tenant, + scope: input.scope, + files: input.files, + commitMessage: input.message, + description: input.description, + source: input.source, + }, + ); + yield* putDescriptorPointer(tenant, out.descriptor); + return out; + }), + ), + ); + }, + catch: (_cause) => + new PublishError({ + message: "publish failed before pipeline completed", + stage: "project", + diagnostics: [], + }), + }), + (exit) => exit, + ), + + getDescriptor: (tenantOrScope, maybeScope) => { + const { tenant, scope } = resolveTenantScope(tenantOrScope, maybeScope); + return deps.store.getDescriptor(tenant, scope).pipe( + Effect.orElseSucceed(() => null), + Effect.flatMap((pointer) => + pointer + ? Effect.succeed(pointer) + : recoverFromSnapshot(tenant, scope).pipe(Effect.orElseSucceed(() => null)), + ), + ); + }, + + listGitHubSources: (tenantInput) => { + const tenant = resolveTenant(tenantInput); + return deps.store.listDescriptors(tenant).pipe( + Effect.orElseSucceed(() => []), + Effect.flatMap((records) => { + const githubRecords = records.flatMap((record) => { + const source = record.descriptor.source; + return source?.kind === "github" ? [{ record, source }] : []; + }); + return Effect.forEach(githubRecords, ({ record, source }) => { + const { descriptor, publishedAt } = record; + return Effect.succeed({ + slug: descriptor.scope, + name: source.repo.split("/").at(-1) ?? descriptor.scope, + scope: descriptor.scope, + url: source.url, + repo: source.repo, + ref: source.ref, + hasToken: false, + upstreamSha: source.upstreamSha, + snapshotId: descriptor.snapshotId, + ...(descriptor.description ? { description: descriptor.description } : {}), + publishedAt: new Date(publishedAt).toISOString(), + tools: descriptor.tools.map((tool) => tool.name), + skipped: [ + ...(source.skipped ?? []), + ...(descriptor.skipped as readonly GitHubSkippedArtifact[]), + ], + } satisfies GitHubCustomToolsSourceSummary); + }); + }), + ); + }, + + repair: (tenantOrScope, maybeScope) => { + const { tenant, scope } = resolveTenantScope(tenantOrScope, maybeScope); + return repairScope(tenant, scope); + }, + + removeSource: (input) => { + const tenant = resolveTenant(input.tenant); + const address = scopeAddress(tenant, input.scope); + const toPublishError = (message: string) => (_cause: unknown) => + new PublishError({ + message, + stage: "project", + diagnostics: [], + }); + return Effect.gen(function* () { + yield* deps.store + .removeDescriptor(tenant, input.scope) + .pipe(Effect.mapError(toPublishError("removeDescriptor failed"))); + yield* deps.artifactStore + .removeScope(address) + .pipe(Effect.mapError(toPublishError("remove artifact scope failed"))); + }); + }, + + invokeTool: (input) => + Effect.gen(function* () { + const tenant = resolveTenant(input.tenant); + const descriptor = yield* requireDescriptor(tenant, input.scope); + const toolDesc = descriptor.tools.find((t) => t.name === input.tool); + if (!toolDesc) { + return yield* new PublishError({ + message: `tool "${input.tool}" is not published in scope "${input.scope}"`, + stage: "project", + diagnostics: [], + }); + } + return yield* invokeToolInternal( + input.scope, + descriptor, + toolDesc, + input.args, + input.resolver, + input.invokeOptions, + ); + }), + }; +}; diff --git a/packages/plugins/apps/src/plugin/self-host-runtime.ts b/packages/plugins/apps/src/plugin/self-host-runtime.ts new file mode 100644 index 000000000..d2276ff4b --- /dev/null +++ b/packages/plugins/apps/src/plugin/self-host-runtime.ts @@ -0,0 +1,50 @@ +import { join } from "node:path"; + +import { makeGitArtifactStore } from "../backing/git-artifact-store"; +import { makeQuickjsToolSandbox } from "../backing/quickjs-tool-sandbox"; +import { type AppsRuntime } from "./runtime"; +import type { AppsStore } from "./store"; +import type { ClientResolver } from "./bindings"; +import { makeAppsRuntimeFromBackings, type AppsBackings } from "./backings"; + +export interface SelfHostAppsRuntimeOptions { + /** Data dir root; `/artifacts`. */ + readonly dataDir: string; + /** Default tenant for direct runtime calls; request paths pass tenant explicitly. */ + readonly tenant?: string; + readonly store: AppsStore; + /** Routes bound integration calls to real APIs (policy/audit). */ + readonly resolver: ClientResolver; + /** In-memory backings for tests. */ + readonly inMemory?: boolean; +} + +export interface SelfHostAppsRuntime { + readonly runtime: AppsRuntime; + readonly backings: AppsBackings; + readonly close: () => Promise; +} + +export const makeSelfHostAppsRuntime = ( + options: SelfHostAppsRuntimeOptions, +): SelfHostAppsRuntime => { + const inMem = options.inMemory === true; + const artifactStore = makeGitArtifactStore({ + root: inMem ? options.dataDir : join(options.dataDir, "artifacts"), + }); + const sandbox = makeQuickjsToolSandbox(); + const backings: AppsBackings = { + artifactStore, + sandbox, + store: options.store, + resolver: options.resolver, + defaultTenant: options.tenant, + }; + const runtime = makeAppsRuntimeFromBackings(backings, options.resolver); + + return { + runtime, + backings, + close: async () => {}, + }; +}; diff --git a/packages/plugins/apps/src/plugin/store.ts b/packages/plugins/apps/src/plugin/store.ts new file mode 100644 index 000000000..8c0db58f4 --- /dev/null +++ b/packages/plugins/apps/src/plugin/store.ts @@ -0,0 +1,92 @@ +import { Effect } from "effect"; + +import { + definePluginStorageCollection, + type PluginStorageFacade, + type StorageFailure, +} from "@executor-js/sdk"; + +import type { AppDescriptor } from "../pipeline/descriptor"; + +// --------------------------------------------------------------------------- +// AppsStore: descriptor pointer persistence. +// --------------------------------------------------------------------------- + +export const descriptorCollection = definePluginStorageCollection("published_descriptor", { + Type: {} as { + readonly tenant: string; + readonly scope: string; + readonly snapshotId: string; + readonly descriptor: AppDescriptor; + readonly publishedAt: number; + }, +}); + +export interface AppDescriptorRecord { + readonly descriptor: AppDescriptor; + readonly publishedAt: number; +} + +export interface GitHubSourceTokenRef { + readonly provider: string; + readonly itemId: string; + readonly updatedAt: number; +} + +export interface AppsStore { + readonly putDescriptor: ( + tenant: string, + owner: "org" | "user", + descriptor: AppDescriptor, + ) => Effect.Effect; + readonly getDescriptor: ( + tenant: string, + scope: string, + ) => Effect.Effect; + readonly removeDescriptor: (tenant: string, scope: string) => Effect.Effect; + readonly listDescriptors: ( + tenant: string, + ) => Effect.Effect; +} + +export interface AppsStoreDeps { + readonly pluginStorage: PluginStorageFacade; +} + +export const makeAppsStore = (deps: AppsStoreDeps): AppsStore => { + const descriptors = deps.pluginStorage.collection(descriptorCollection); + // Tenant is now part of apps storage keys. The apps subsystem had not shipped + // before this key shape, so no migration from the old scope-only keys is needed. + const keyFor = (tenant: string, key: string): string => `${tenant}:${key}`; + return { + putDescriptor: (tenant, owner, descriptor) => + descriptors + .put({ + owner, + key: keyFor(tenant, descriptor.scope), + data: { + tenant, + scope: descriptor.scope, + snapshotId: descriptor.snapshotId, + descriptor, + publishedAt: Date.now(), + }, + }) + .pipe(Effect.asVoid), + getDescriptor: (tenant, scope) => + descriptors + .get({ key: keyFor(tenant, scope) }) + .pipe(Effect.map((entry) => entry?.data.descriptor ?? null)), + removeDescriptor: (tenant, scope) => + descriptors.remove({ owner: "org", key: keyFor(tenant, scope) }), + listDescriptors: (tenant) => + descriptors.list({ keyPrefix: `${tenant}:` }).pipe( + Effect.map((entries) => + entries.map((entry) => ({ + descriptor: entry.data.descriptor, + publishedAt: entry.data.publishedAt, + })), + ), + ), + }; +}; diff --git a/packages/plugins/apps/src/plugin/tenant-isolation.test.ts b/packages/plugins/apps/src/plugin/tenant-isolation.test.ts new file mode 100644 index 000000000..93befbe23 --- /dev/null +++ b/packages/plugins/apps/src/plugin/tenant-isolation.test.ts @@ -0,0 +1,122 @@ +import { mkdtempSync, readdirSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +import { describe, expect, it } from "@effect/vitest"; +import { Effect } from "effect"; + +import { makeSqliteAppsStore } from "../backing/sqlite-apps-store"; +import { syncGitHubSource } from "../source/github-source"; +import { makeTestResolver } from "../testing"; +import { makeSelfHostAppsRuntime } from "./self-host-runtime"; + +const run = (effect: Effect.Effect): Promise => Effect.runPromise(effect); + +const toolSource = (name: string): string => `import { defineTool } from "executor:app"; +import { z } from "zod"; +export default defineTool({ + description: "${name}", + input: z.object({}), + output: z.object({ ok: z.boolean(), name: z.string() }), + async handler() { + return { ok: true, name: ${JSON.stringify(name)} }; + }, +});`; + +const json = (value: unknown, status = 200): Response => + new Response(JSON.stringify(value), { + status, + headers: { "content-type": "application/json" }, + }); + +const makeGitHubFetch = (input: { readonly toolName: string; readonly upstreamSha: string }) => { + const repoPath = "/repos/acme/tools"; + const treeSha = `${input.upstreamSha}-tree`; + const content = toolSource(input.toolName); + const fetch = (async (rawUrl: string) => { + const url = new URL(rawUrl); + if (url.pathname === repoPath) return json({ default_branch: "main" }); + if (url.pathname === `${repoPath}/git/ref/heads%2Fmain`) { + return json({ object: { sha: input.upstreamSha } }); + } + if (url.pathname === `${repoPath}/git/commits/${input.upstreamSha}`) { + return json({ sha: input.upstreamSha, tree: { sha: treeSha } }); + } + if (url.pathname === `${repoPath}/git/trees/${treeSha}`) { + return json({ + tree: [ + { + path: `tools/${input.toolName}.ts`, + type: "blob", + mode: "100644", + sha: `${input.upstreamSha}-blob`, + size: Buffer.byteLength(content, "utf8"), + }, + ], + }); + } + if (url.pathname === `${repoPath}/git/blobs/${input.upstreamSha}-blob`) { + return json({ + encoding: "base64", + content: Buffer.from(content, "utf8").toString("base64"), + }); + } + return json({ message: "not found" }, 404); + }) as typeof globalThis.fetch; + return fetch; +}; + +const makeTenantRuntime = (dataDir: string, tenant: string, storePath: string) => + makeSelfHostAppsRuntime({ + dataDir, + tenant, + store: makeSqliteAppsStore({ path: storePath }), + resolver: makeTestResolver({}), + }); + +describe("tenant scope isolation", () => { + it("keeps the same source-shaped scope distinct across tenants", async () => { + const dataDir = mkdtempSync(join(tmpdir(), "apps-tenant-")); + const storePath = join(dataDir, "store.sqlite"); + const scope = "githubSourceSameRepoConnection"; + const tenantA = "org-a"; + const tenantB = "org-b"; + const hostA = makeTenantRuntime(dataDir, tenantA, storePath); + const hostB = makeTenantRuntime(dataDir, tenantB, storePath); + + const publishedA = await run( + syncGitHubSource({ + runtime: hostA.runtime, + tenant: tenantA, + scope, + url: "https://github.com/acme/tools", + fetch: makeGitHubFetch({ toolName: "alpha", upstreamSha: "sha-a" }), + }), + ); + const publishedB = await run( + syncGitHubSource({ + runtime: hostB.runtime, + tenant: tenantB, + scope, + url: "https://github.com/acme/tools", + fetch: makeGitHubFetch({ toolName: "beta", upstreamSha: "sha-b" }), + }), + ); + + expect(publishedA.status).toBe("published"); + expect(publishedB.status).toBe("published"); + + const artifactDirs = readdirSync(join(dataDir, "artifacts")).filter((name) => + name.endsWith(".git"), + ); + expect(artifactDirs.length).toBe(2); + + const descriptorA = await run(hostA.runtime.getDescriptor(tenantA, scope)); + const descriptorB = await run(hostB.runtime.getDescriptor(tenantB, scope)); + expect(descriptorA?.tools.map((tool) => tool.name)).toEqual(["alpha"]); + expect(descriptorB?.tools.map((tool) => tool.name)).toEqual(["beta"]); + + await hostA.close(); + await hostB.close(); + }); +}); diff --git a/packages/plugins/apps/src/react/AddCustomToolsSource.tsx b/packages/plugins/apps/src/react/AddCustomToolsSource.tsx new file mode 100644 index 000000000..65325ebc2 --- /dev/null +++ b/packages/plugins/apps/src/react/AddCustomToolsSource.tsx @@ -0,0 +1,183 @@ +import { useState } from "react"; +import { Effect, Exit } from "effect"; + +import { Button } from "@executor-js/react/components/button"; +import { + CardStack, + CardStackContent, + CardStackEntryField, +} from "@executor-js/react/components/card-stack"; +import { FloatActions } from "@executor-js/react/components/float-actions"; +import { Input } from "@executor-js/react/components/input"; +import { toast } from "@executor-js/react/components/sonner"; +import { FormErrorAlert, useSlugAlreadyExists } from "@executor-js/react/lib/integration-add"; + +import { + formatSyncErrors, + slugifyCustomToolsAppName, + suggestCustomToolsAppName, + syncCustomToolSourceEffect, + syncStatusLabel, + validateCustomToolsAppSlug, + validateGitHubSourceUrl, +} from "./custom-tools-client"; + +export default function AddCustomToolsSource(props: { + readonly onComplete: (slug?: string) => void; + readonly onCancel: () => void; + readonly initialUrl?: string; + readonly initialNamespace?: string; +}) { + const [url, setUrl] = useState(props.initialUrl ?? ""); + const [name, setName] = useState( + props.initialNamespace ? slugifyCustomToolsAppName(props.initialNamespace) : "", + ); + const [nameTouched, setNameTouched] = useState(props.initialNamespace !== undefined); + const [token, setToken] = useState(""); + const [tokenRevealed, setTokenRevealed] = useState(false); + const [urlError, setUrlError] = useState(null); + const [nameError, setNameError] = useState(null); + const [syncing, setSyncing] = useState(false); + const [syncError, setSyncError] = useState(null); + const effectiveName = nameTouched + ? name + : slugifyCustomToolsAppName(suggestCustomToolsAppName(url)); + const slug = effectiveName; + const slugAlreadyExists = useSlugAlreadyExists(slug); + + const submit = async () => { + const validation = validateGitHubSourceUrl(url); + const nextNameError = validateCustomToolsAppSlug(slug); + setUrlError(validation); + setNameError(nextNameError); + setSyncError(null); + if (validation || nextNameError) return; + if (slugAlreadyExists) { + setSyncError(`An integration named "${slug}" already exists. Choose another name.`); + return; + } + + setSyncing(true); + const exit = await Effect.runPromiseExit( + syncCustomToolSourceEffect({ + name: slug, + url, + token, + }), + ); + if (Exit.isFailure(exit)) { + setSyncError("Failed to sync custom tools."); + setSyncing(false); + return; + } + const result = exit.value; + if (result.status === "failed") { + setSyncError(formatSyncErrors(result).join("\n") || "Sync failed."); + setSyncing(false); + return; + } + toast.success(syncStatusLabel(result)); + props.onComplete(slug); + }; + + return ( +
+
+

Add custom tools

+

+ Sync a GitHub repository that defines tools with executor:app. +

+
+ + + + +
+ { + setUrl((event.target as HTMLInputElement).value); + setUrlError(null); + setNameError(null); + setSyncError(null); + }} + onBlur={() => setUrlError(validateGitHubSourceUrl(url))} + placeholder="https://github.com/UsefulSoftwareCo/executor" + className="font-mono text-sm" + aria-invalid={urlError ? true : undefined} + /> + {urlError &&

{urlError}

} +
+ + + +
+ { + setNameTouched(true); + setName(slugifyCustomToolsAppName((event.target as HTMLInputElement).value)); + setNameError(null); + setSyncError(null); + }} + placeholder="executor-custom-tools-demo" + className="text-sm" + aria-invalid={nameError ? true : undefined} + /> + {nameError &&

{nameError}

} + {slugAlreadyExists && !syncing && !nameError && ( +

+ An integration named "{slug}" already exists. +

+ )} +
+ + + +
+ { + setToken((event.target as HTMLInputElement).value); + setSyncError(null); + }} + autoComplete="off" + className="font-mono text-sm" + /> + +
+ + + + + {syncError && } + + + + + +
+ ); +} diff --git a/packages/plugins/apps/src/react/CustomToolsAccountsPanel.tsx b/packages/plugins/apps/src/react/CustomToolsAccountsPanel.tsx new file mode 100644 index 000000000..ec875c2c4 --- /dev/null +++ b/packages/plugins/apps/src/react/CustomToolsAccountsPanel.tsx @@ -0,0 +1,292 @@ +import { useEffect, useState } from "react"; +import { Effect, Exit } from "effect"; + +import { + AlertDialog, + AlertDialogAction, + AlertDialogCancel, + AlertDialogContent, + AlertDialogDescription, + AlertDialogFooter, + AlertDialogHeader, + AlertDialogTitle, + AlertDialogTrigger, +} from "@executor-js/react/components/alert-dialog"; +import { Alert, AlertDescription, AlertTitle } from "@executor-js/react/components/alert"; +import { Button } from "@executor-js/react/components/button"; + +import type { GitHubCustomToolsSourceSummary } from "../api"; +import { + consoleIntegrationHref, + getCustomToolSourceEffect, + removeCustomToolSourceEffect, + syncCustomToolSourceEffect, +} from "./custom-tools-client"; +import { sourcePanelModel, syncNoticeFromResult, type SyncNoticeModel } from "./source-panel-model"; + +type LoadState = + | { readonly status: "loading" } + | { readonly status: "error"; readonly message: string } + | { readonly status: "missing" } + | { readonly status: "ready"; readonly source: GitHubCustomToolsSourceSummary }; + +export default function CustomToolsAccountsPanel(props: { + readonly sourceId: string; + readonly integrationName: string; +}) { + const [loadState, setLoadState] = useState({ status: "loading" }); + const [syncing, setSyncing] = useState(false); + const [removing, setRemoving] = useState(false); + const [notice, setNotice] = useState(null); + const [removeError, setRemoveError] = useState(null); + + const loadSource = async () => { + setLoadState({ status: "loading" }); + const exit = await Effect.runPromiseExit(getCustomToolSourceEffect(props.sourceId)); + if (Exit.isFailure(exit)) { + setLoadState({ status: "error", message: "Failed to load custom tools source." }); + return; + } + const source = exit.value.source; + setLoadState(source ? { status: "ready", source } : { status: "missing" }); + }; + + useEffect(() => { + let active = true; + void (async () => { + const exit = await Effect.runPromiseExit(getCustomToolSourceEffect(props.sourceId)); + if (!active) return; + if (Exit.isFailure(exit)) { + setLoadState({ status: "error", message: "Failed to load custom tools source." }); + return; + } + const source = exit.value.source; + setLoadState(source ? { status: "ready", source } : { status: "missing" }); + })(); + return () => { + active = false; + }; + }, [props.sourceId]); + + const syncSource = async (source: GitHubCustomToolsSourceSummary) => { + setSyncing(true); + setNotice(null); + const beforeTools = source.tools; + const exit = await Effect.runPromiseExit(syncCustomToolSourceEffect({ slug: source.slug })); + if (Exit.isFailure(exit)) { + setNotice({ + status: "failed", + message: "Sync failed.", + added: [], + removed: [], + errors: ["Failed to sync custom tools."], + skipped: [], + }); + setSyncing(false); + return; + } + const result = exit.value; + setNotice(syncNoticeFromResult(result, beforeTools)); + if (result.status !== "failed") await loadSource(); + setSyncing(false); + }; + + const removeSource = async (source: GitHubCustomToolsSourceSummary) => { + setRemoving(true); + setRemoveError(null); + const exit = await Effect.runPromiseExit(removeCustomToolSourceEffect(source.slug)); + if (Exit.isFailure(exit)) { + setRemoveError("Failed to remove custom tools source."); + setRemoving(false); + return; + } + window.location.assign(consoleIntegrationHref("/integrations")); + }; + + return ( +
+ {loadState.status === "loading" && ( +
+ Loading source... +
+ )} + + {loadState.status === "error" && ( + void loadSource()} /> + )} + + {loadState.status === "missing" && ( + + Source not found + + The custom tools source for {props.integrationName} is no longer available. + + + )} + + {loadState.status === "ready" && ( + void syncSource(loadState.source)} + onRemove={() => void removeSource(loadState.source)} + /> + )} +
+ ); +} + +function ErrorWithRetry(props: { readonly message: string; readonly onRetry: () => void }) { + return ( + + Failed to load source + +
+

{props.message}

+ +
+ + + ); +} + +function SourceDetail(props: { + readonly source: GitHubCustomToolsSourceSummary; + readonly notice: SyncNoticeModel | null; + readonly removeError: string | null; + readonly syncing: boolean; + readonly removing: boolean; + readonly onSync: () => void; + readonly onRemove: () => void; +}) { + const { source, notice } = props; + const model = sourcePanelModel(source); + const noticeHasDetails = + notice !== null && + (notice.added.length > 0 || + notice.removed.length > 0 || + notice.errors.length > 0 || + notice.skipped.length > 0 || + notice.upstreamSha !== undefined); + return ( +
+
+ +
+ + + + + + + + Remove {source.name}? + + This removes {source.tools.length} {source.tools.length === 1 ? "tool" : "tools"}{" "} + from the catalog. The GitHub repository is untouched; re-add it to sync again. + + + + Cancel + + {props.removing ? "Removing..." : "Remove source"} + + + + +
+
+ +
+
+

{model.lastSynced}

+

+ + {model.publishedTools.label} + {" "} + published +

+
+ + {notice && ( + + {notice.message} + {noticeHasDetails && ( + +
+ {notice.added.length > 0 &&

Added: {notice.added.join(", ")}

} + {notice.removed.length > 0 &&

Removed: {notice.removed.join(", ")}

} + {notice.errors.map((error) => ( +

{error}

+ ))} + {notice.skipped.length > 0 && ( +
+

Skipped:

+
    + {notice.skipped.map((entry) => ( +
  • + + {entry.path} + + + {entry.reason} + +
    • + ))} +
+
+ )} + {notice.upstreamSha && ( +

+ Commit {notice.upstreamSha} +

+ )} +
+ + )} + + )} + + {props.removeError && } +
+
+ ); +} + +function FormErrorMessage(props: { readonly message: string }) { + return ( +
+

{props.message}

+
+ ); +} diff --git a/packages/plugins/apps/src/react/custom-tools-client.ts b/packages/plugins/apps/src/react/custom-tools-client.ts new file mode 100644 index 000000000..a717c9640 --- /dev/null +++ b/packages/plugins/apps/src/react/custom-tools-client.ts @@ -0,0 +1,212 @@ +import { Data, Effect, Schema } from "effect"; + +import type { GitHubCustomToolsSourceSummary, GitHubSyncResult } from "../api"; +import { parseGitHubSourceUrl } from "../source/github-url"; +import { slugifyCustomToolsAppName, validateCustomToolsAppSlug } from "../source/app-slug"; + +export const CUSTOM_TOOLS_PLUGIN_KEY = "apps"; +export const CUSTOM_TOOLS_LABEL = "Custom tools"; + +export interface GitHubSourcesListResponse { + readonly sources: readonly GitHubCustomToolsSourceSummary[]; +} + +export interface GitHubSourceDetailResponse { + readonly source: GitHubCustomToolsSourceSummary | null; +} + +export interface SyncGitHubSourceRequest { + readonly url?: string; + readonly name?: string; + readonly slug?: string; + readonly ref?: string; + readonly token?: string; +} + +export type CustomToolsFetch = (input: RequestInfo | URL, init?: RequestInit) => Promise; + +export class CustomToolsClientError extends Data.TaggedError("CustomToolsClientError")<{ + readonly message: string; +}> {} + +export { parseGitHubSourceUrl }; +export { slugifyCustomToolsAppName, validateCustomToolsAppSlug }; + +export const suggestCustomToolsAppName = (url: string): string => { + const parsed = parseGitHubSourceUrl(url); + return parsed.ok ? parsed.value.name : ""; +}; + +export const validateGitHubSourceUrl = (url: string): string | null => { + const parsed = parseGitHubSourceUrl(url); + return parsed.ok ? null : parsed.message; +}; + +export const validateGitHubRepo = validateGitHubSourceUrl; + +const decodeJsonText = Schema.decodeUnknownEffect(Schema.UnknownFromJsonString); + +const isRecord = (value: unknown): value is Record => + value !== null && typeof value === "object" && !Array.isArray(value); + +const responseErrorMessage = (body: unknown, fallback: string): string => + isRecord(body) && typeof body.error === "string" ? body.error : fallback; + +const parseJsonResponseEffect = ( + response: Response, + fallback: string, +): Effect.Effect => + Effect.gen(function* () { + const text = yield* Effect.tryPromise({ + try: () => response.text(), + catch: () => new CustomToolsClientError({ message: fallback }), + }); + const body = + text.length > 0 + ? yield* decodeJsonText(text).pipe( + Effect.mapError(() => new CustomToolsClientError({ message: fallback })), + ) + : null; + if (!response.ok) { + return yield* new CustomToolsClientError({ + message: responseErrorMessage(body, fallback), + }); + } + return body as A; + }); + +export const listCustomToolSourcesEffect = ( + fetchImpl: CustomToolsFetch = fetch, +): Effect.Effect => + Effect.tryPromise({ + try: () => + fetchImpl("/api/apps/sources/github", { + credentials: "same-origin", + }), + catch: () => new CustomToolsClientError({ message: "Failed to load custom tools." }), + }).pipe( + Effect.flatMap((response) => + parseJsonResponseEffect(response, "Failed to load custom tools."), + ), + ); + +export const listCustomToolSources = ( + fetchImpl: CustomToolsFetch = fetch, +): Promise => Effect.runPromise(listCustomToolSourcesEffect(fetchImpl)); + +export const getCustomToolSourceEffect = ( + slug: string, + fetchImpl: CustomToolsFetch = fetch, +): Effect.Effect => + Effect.tryPromise({ + try: () => + fetchImpl(`/api/apps/sources/github/${encodeURIComponent(slug)}`, { + credentials: "same-origin", + }), + catch: () => new CustomToolsClientError({ message: "Failed to load custom tools source." }), + }).pipe( + Effect.flatMap((response) => + parseJsonResponseEffect( + response, + "Failed to load custom tools source.", + ), + ), + ); + +export const getCustomToolSource = ( + slug: string, + fetchImpl: CustomToolsFetch = fetch, +): Promise => + Effect.runPromise(getCustomToolSourceEffect(slug, fetchImpl)); + +export const syncCustomToolSourceEffect = ( + input: SyncGitHubSourceRequest, + fetchImpl: CustomToolsFetch = fetch, +): Effect.Effect => + Effect.tryPromise({ + try: () => + fetchImpl("/api/apps/sources/github/sync", { + method: "POST", + headers: { "content-type": "application/json" }, + credentials: "same-origin", + body: JSON.stringify({ + ...(input.url?.trim() ? { url: input.url.trim() } : {}), + ...(input.name?.trim() ? { name: input.name.trim() } : {}), + ...(input.slug?.trim() ? { slug: input.slug.trim() } : {}), + ...(input.ref?.trim() ? { ref: input.ref.trim() } : {}), + ...(input.token?.trim() ? { token: input.token.trim() } : {}), + }), + }), + catch: () => new CustomToolsClientError({ message: "Failed to sync custom tools." }), + }).pipe( + Effect.flatMap((response) => + parseJsonResponseEffect(response, "Failed to sync custom tools."), + ), + ); + +export const syncCustomToolSource = ( + input: SyncGitHubSourceRequest, + fetchImpl: CustomToolsFetch = fetch, +): Promise => Effect.runPromise(syncCustomToolSourceEffect(input, fetchImpl)); + +export const removeCustomToolSourceEffect = ( + slug: string, + fetchImpl: CustomToolsFetch = fetch, +): Effect.Effect<{ readonly removed: boolean }, CustomToolsClientError> => + Effect.tryPromise({ + try: () => + fetchImpl(`/api/apps/sources/github/${encodeURIComponent(slug)}`, { + method: "DELETE", + credentials: "same-origin", + }), + catch: () => new CustomToolsClientError({ message: "Failed to remove custom tools source." }), + }).pipe( + Effect.flatMap((response) => + parseJsonResponseEffect<{ readonly removed: boolean }>( + response, + "Failed to remove custom tools source.", + ), + ), + ); + +export const removeCustomToolSource = ( + slug: string, + fetchImpl: CustomToolsFetch = fetch, +): Promise<{ readonly removed: boolean }> => + Effect.runPromise(removeCustomToolSourceEffect(slug, fetchImpl)); + +export const syncStatusLabel = (result: GitHubSyncResult): string => { + if (result.status === "published") return `Published ${result.tools.length} tools.`; + if (result.status === "up-to-date") return "Already up to date."; + return "Sync failed."; +}; + +export const formatSyncErrors = (result: GitHubSyncResult): readonly string[] => { + if (result.status !== "failed") return []; + return result.errors.map((entry) => { + const message = entry.message; + const stage = entry.stage; + const details = entry.diagnostics?.map((d) => `${d.path}: ${d.message}`).join("; "); + return details ? `${stage}: ${message} (${details})` : `${stage}: ${message}`; + }); +}; + +export const toolDiff = ( + before: readonly string[], + after: readonly string[], +): { readonly added: readonly string[]; readonly removed: readonly string[] } => { + const beforeSet = new Set(before); + const afterSet = new Set(after); + return { + added: after.filter((tool) => !beforeSet.has(tool)), + removed: before.filter((tool) => !afterSet.has(tool)), + }; +}; + +export const consoleIntegrationHref = (path: string): string => { + if (typeof window === "undefined") return path; + const marker = "/integrations"; + const index = window.location.pathname.indexOf(marker); + const prefix = index === -1 ? "" : window.location.pathname.slice(0, index); + return `${prefix}${path}`; +}; diff --git a/packages/plugins/apps/src/react/custom-tools-client.web.test.ts b/packages/plugins/apps/src/react/custom-tools-client.web.test.ts new file mode 100644 index 000000000..c5a4e091b --- /dev/null +++ b/packages/plugins/apps/src/react/custom-tools-client.web.test.ts @@ -0,0 +1,275 @@ +import { describe, expect, it } from "@effect/vitest"; + +import appsClientPlugin, { + CUSTOM_TOOLS_LABEL, + CUSTOM_TOOLS_PLUGIN_KEY, + type CustomToolsFetch, + appsIntegrationPlugin, + formatSyncErrors, + getCustomToolSource, + listCustomToolSources, + removeCustomToolSource, + parseGitHubSourceUrl, + slugifyCustomToolsAppName, + suggestCustomToolsAppName, + syncCustomToolSource, + syncStatusLabel, + validateGitHubSourceUrl, +} from "./plugin-client"; +import { asSnapshotId } from "../seams/artifact-store"; +import { sourcePanelModel, syncNoticeFromResult } from "./source-panel-model"; + +const jsonResponse = (body: unknown, init?: ResponseInit): Response => + new Response(JSON.stringify(body), { + status: 200, + headers: { "content-type": "application/json" }, + ...init, + }); + +describe("custom tools console client", () => { + const demoSource = { + slug: "demo-tools", + name: "Demo tools", + scope: "demo-tools", + url: "https://github.com/RhysSullivan/executor-custom-tools-demo", + repo: "RhysSullivan/executor-custom-tools-demo", + ref: "main", + hasToken: true, + upstreamSha: "abc123", + snapshotId: "snap1", + publishedAt: "2026-07-06T12:00:00.000Z", + tools: ["repo-summary", "stale-issues"], + skipped: [{ path: "README.md", reason: "ignored" as const }], + }; + + it("registers the manual Custom tools tile", () => { + const manualTiles = [appsIntegrationPlugin].map((plugin) => ({ + href: `/integrations/add/${plugin.key}`, + label: plugin.label, + })); + + expect(appsClientPlugin.id).toBe(CUSTOM_TOOLS_PLUGIN_KEY); + expect(appsClientPlugin.integrationPlugin?.label).toBe(CUSTOM_TOOLS_LABEL); + expect(manualTiles).toContainEqual({ + href: "/integrations/add/apps", + label: "Custom tools", + }); + }); + + it("validates the GitHub URL shape", () => { + expect(validateGitHubSourceUrl("https://github.com/UsefulSoftwareCo/executor")).toBeNull(); + expect(validateGitHubSourceUrl("UsefulSoftwareCo/executor")).toBeNull(); + expect(validateGitHubSourceUrl("https://gitlab.com/UsefulSoftwareCo/executor")).toBe( + "GitHub source URLs must use github.com.", + ); + expect(validateGitHubSourceUrl("UsefulSoftwareCo")).toBe( + "Use a GitHub repo URL like https://github.com/owner/repo, optionally with /tree/ or /commit/.", + ); + expect(validateGitHubSourceUrl("")).toBe("Enter a GitHub URL."); + }); + + it("carries detected repo URL variants into the add form defaults", () => { + const treeUrl = + "https://github.com/RhysSullivan/executor-custom-tools-demo/tree/feature/custom-tools"; + const commitUrl = "https://github.com/RhysSullivan/executor-custom-tools-demo/commit/abc1234"; + + const tree = parseGitHubSourceUrl(treeUrl); + const commit = parseGitHubSourceUrl(commitUrl); + + expect(tree).toMatchObject({ + ok: true, + value: { + repo: "RhysSullivan/executor-custom-tools-demo", + ref: "feature/custom-tools", + url: treeUrl, + }, + }); + expect(commit).toMatchObject({ + ok: true, + value: { + repo: "RhysSullivan/executor-custom-tools-demo", + ref: "abc1234", + url: commitUrl, + }, + }); + expect(suggestCustomToolsAppName(treeUrl)).toBe("executor-custom-tools-demo"); + expect(slugifyCustomToolsAppName(suggestCustomToolsAppName(treeUrl))).toBe( + "executor-custom-tools-demo", + ); + }); + + it("does not classify non-repo GitHub URLs as custom-tools add input", () => { + expect(parseGitHubSourceUrl("https://gist.github.com/RhysSullivan/abc1234").ok).toBe(false); + expect( + parseGitHubSourceUrl( + "https://github.com/RhysSullivan/executor-custom-tools-demo/blob/main/openapi.json", + ).ok, + ).toBe(false); + }); + + it("models the custom-tools detail panel without default debug fields", () => { + const panel = sourcePanelModel(demoSource, { + now: Date.parse("2026-07-06T12:05:00.000Z"), + }); + + expect(panel.title).toBe("Demo tools"); + expect(panel.repository).toEqual({ + href: "https://github.com/RhysSullivan/executor-custom-tools-demo", + label: "github.com/RhysSullivan/executor-custom-tools-demo", + }); + expect(panel.lastSynced).toBe("Last synced 5m ago"); + expect(panel.publishedTools).toEqual({ + href: "/integrations/demo-tools?tab=tools", + label: "2 tools", + }); + expect(panel).not.toHaveProperty("toolNames"); + expect(panel).not.toHaveProperty("skipped"); + expect(panel).not.toHaveProperty("upstreamSha"); + expect(panel).not.toHaveProperty("hasToken"); + }); + + it("keeps skipped files and upstream SHA in the post-sync result details", () => { + const notice = syncNoticeFromResult( + { + status: "published", + snapshotId: asSnapshotId("snap2"), + upstreamSha: "def456", + tools: ["repo-summary", "stale-issues", "third-tool"], + skipped: [{ path: "workflows/x.ts", reason: "not supported yet" }], + }, + demoSource.tools, + ); + + expect(notice.message).toBe("Published 3 tools."); + expect(notice.added).toEqual(["third-tool"]); + expect(notice.skipped).toEqual([{ path: "workflows/x.ts", reason: "not supported yet" }]); + expect(notice.upstreamSha).toBe("def456"); + }); + + it("surfaces successful sync and source detail data", async () => { + let syncBody = ""; + let removed = false; + const fetchImpl: CustomToolsFetch = async (input, init) => { + const url = String(input); + if (url.endsWith("/sync")) { + syncBody = String(init?.body ?? ""); + return jsonResponse({ + status: "published", + snapshotId: "snap1", + upstreamSha: "abc123", + tools: ["repo-summary", "stale-issues"], + skipped: [], + }); + } + if (url.endsWith("/demo-tools")) { + if (init?.method === "DELETE") { + removed = true; + return jsonResponse({ removed: true }); + } + return jsonResponse({ + source: { + slug: "demo-tools", + name: "demo-tools", + scope: "demo-tools", + url: "https://github.com/RhysSullivan/executor-custom-tools-demo", + repo: "RhysSullivan/executor-custom-tools-demo", + ref: "main", + hasToken: true, + upstreamSha: "abc123", + snapshotId: "snap1", + publishedAt: "2026-07-06T12:00:00.000Z", + tools: ["repo-summary", "stale-issues"], + skipped: [], + }, + }); + } + return jsonResponse({ + sources: [ + { + slug: "demo-tools", + name: "demo-tools", + scope: "demo-tools", + url: "https://github.com/RhysSullivan/executor-custom-tools-demo", + repo: "RhysSullivan/executor-custom-tools-demo", + ref: "main", + hasToken: true, + upstreamSha: "abc123", + snapshotId: "snap1", + publishedAt: "2026-07-06T12:00:00.000Z", + tools: ["repo-summary", "stale-issues"], + skipped: [], + }, + ], + }); + }; + + const syncResult = await syncCustomToolSource( + { + name: "demo-tools", + url: "https://github.com/RhysSullivan/executor-custom-tools-demo", + token: "ghp_demo", + }, + fetchImpl, + ); + const listed = await listCustomToolSources(fetchImpl); + const detail = await getCustomToolSource("demo-tools", fetchImpl); + const remove = await removeCustomToolSource("demo-tools", fetchImpl); + + expect(syncStatusLabel(syncResult)).toBe("Published 2 tools."); + expect(syncBody).toContain("ghp_demo"); + expect(syncBody).toContain("demo-tools"); + expect(syncResult.tools).toEqual(["repo-summary", "stale-issues"]); + expect(detail.source?.slug).toBe("demo-tools"); + expect(listed.sources[0]?.hasToken).toBe(true); + expect(listed.sources[0]?.tools).toEqual(["repo-summary", "stale-issues"]); + expect(remove).toEqual({ removed: true }); + expect(removed).toBe(true); + }); + + it("renders failed sync errors readably", async () => { + const fetchImpl: CustomToolsFetch = async () => + jsonResponse({ + status: "failed", + tools: [], + skipped: [], + errors: [ + { + stage: "collect", + message: "schema library not supported for schema export", + diagnostics: [{ path: "tools/x.ts", message: "vendor: nope" }], + }, + ], + }); + + const result = await syncCustomToolSource( + { + url: "https://github.com/RhysSullivan/executor-custom-tools-demo", + }, + fetchImpl, + ); + + expect(syncStatusLabel(result)).toBe("Sync failed."); + expect(formatSyncErrors(result)).toEqual([ + "collect: schema library not supported for schema export (tools/x.ts: vendor: nope)", + ]); + }); + + it("shows the up-to-date sync state", async () => { + const fetchImpl: CustomToolsFetch = async () => + jsonResponse({ + status: "up-to-date", + upstreamSha: "abc123", + tools: ["repo-summary"], + skipped: [], + }); + + const result = await syncCustomToolSource( + { + url: "https://github.com/RhysSullivan/executor-custom-tools-demo", + }, + fetchImpl, + ); + + expect(syncStatusLabel(result)).toBe("Already up to date."); + }); +}); diff --git a/packages/plugins/apps/src/react/plugin-client.tsx b/packages/plugins/apps/src/react/plugin-client.tsx new file mode 100644 index 000000000..d3d2ba8a0 --- /dev/null +++ b/packages/plugins/apps/src/react/plugin-client.tsx @@ -0,0 +1,11 @@ +import { defineClientPlugin } from "@executor-js/sdk/client"; + +import { appsIntegrationPlugin } from "./source-plugin"; + +export { appsIntegrationPlugin } from "./source-plugin"; +export * from "./custom-tools-client"; + +export default defineClientPlugin({ + id: "apps", + integrationPlugin: appsIntegrationPlugin, +}); diff --git a/packages/plugins/apps/src/react/source-panel-model.ts b/packages/plugins/apps/src/react/source-panel-model.ts new file mode 100644 index 000000000..707703dae --- /dev/null +++ b/packages/plugins/apps/src/react/source-panel-model.ts @@ -0,0 +1,92 @@ +import type { GitHubCustomToolsSourceSummary, GitHubSyncResult } from "../api"; +import { + consoleIntegrationHref, + formatSyncErrors, + syncStatusLabel, + toolDiff, +} from "./custom-tools-client"; + +export interface SourcePanelModel { + readonly title: string; + readonly repository: { + readonly href: string; + readonly label: string; + }; + readonly lastSynced: string; + readonly publishedTools: { + readonly href: string; + readonly label: string; + }; +} + +export interface SyncNoticeModel { + readonly status: GitHubSyncResult["status"]; + readonly message: string; + readonly added: readonly string[]; + readonly removed: readonly string[]; + readonly errors: readonly string[]; + readonly skipped: GitHubSyncResult["skipped"]; + readonly upstreamSha?: string; +} + +export const formatRelativeSyncTime = (iso: string, now = Date.now()): string => { + const then = new Date(iso).getTime(); + if (Number.isNaN(then)) return iso; + const diffMs = Math.max(0, now - then); + const min = Math.floor(diffMs / 60_000); + if (min < 1) return "just now"; + if (min < 60) return `${min}m ago`; + const hrs = Math.floor(min / 60); + if (hrs < 24) return `${hrs}h ago`; + const days = Math.floor(hrs / 24); + if (days < 7) return `${days}d ago`; + const weeks = Math.floor(days / 7); + if (weeks < 5) return `${weeks}w ago`; + const months = Math.floor(days / 30); + if (months < 12) return `${months}mo ago`; + return `${Math.floor(days / 365)}y ago`; +}; + +export const sourceRepositoryDisplay = ( + source: GitHubCustomToolsSourceSummary, +): SourcePanelModel["repository"] => { + const explicitRef = /\/(tree|commit)\//.test(source.url); + const base = `github.com/${source.repo}`; + return { + href: source.url, + label: explicitRef ? `${base} @ ${source.ref}` : base, + }; +}; + +export const toolsCountLabel = (count: number): string => + `${count} ${count === 1 ? "tool" : "tools"}`; + +export const sourcePanelModel = ( + source: GitHubCustomToolsSourceSummary, + options?: { readonly now?: number }, +): SourcePanelModel => ({ + title: source.name, + repository: sourceRepositoryDisplay(source), + lastSynced: `Last synced ${formatRelativeSyncTime(source.publishedAt, options?.now)}`, + publishedTools: { + href: consoleIntegrationHref(`/integrations/${encodeURIComponent(source.slug)}?tab=tools`), + label: toolsCountLabel(source.tools.length), + }, +}); + +export const syncNoticeFromResult = ( + result: GitHubSyncResult, + beforeTools: readonly string[], +): SyncNoticeModel => { + const diff = + result.status === "failed" ? { added: [], removed: [] } : toolDiff(beforeTools, result.tools); + return { + status: result.status, + message: syncStatusLabel(result), + added: diff.added, + removed: diff.removed, + errors: formatSyncErrors(result), + skipped: result.skipped, + ...(result.upstreamSha ? { upstreamSha: result.upstreamSha } : {}), + }; +}; diff --git a/packages/plugins/apps/src/react/source-plugin.ts b/packages/plugins/apps/src/react/source-plugin.ts new file mode 100644 index 000000000..950409ab2 --- /dev/null +++ b/packages/plugins/apps/src/react/source-plugin.ts @@ -0,0 +1,18 @@ +import { lazy } from "react"; +import type { IntegrationPlugin } from "@executor-js/sdk/client"; + +import { CUSTOM_TOOLS_LABEL, CUSTOM_TOOLS_PLUGIN_KEY } from "./custom-tools-client"; + +const importAdd = () => import("./AddCustomToolsSource"); +const importAccounts = () => import("./CustomToolsAccountsPanel"); + +export const appsIntegrationPlugin: IntegrationPlugin = { + key: CUSTOM_TOOLS_PLUGIN_KEY, + label: CUSTOM_TOOLS_LABEL, + add: lazy(importAdd), + accounts: lazy(importAccounts), + preload: () => { + void importAdd(); + void importAccounts(); + }, +}; diff --git a/packages/plugins/apps/src/seams/artifact-store.conformance.ts b/packages/plugins/apps/src/seams/artifact-store.conformance.ts new file mode 100644 index 000000000..b1da0e36e --- /dev/null +++ b/packages/plugins/apps/src/seams/artifact-store.conformance.ts @@ -0,0 +1,92 @@ +import { describe, expect, it } from "@effect/vitest"; +import { Effect } from "effect"; + +import type { ArtifactStore, FileSet } from "./artifact-store"; +import { scopeAddress } from "./scope-address"; + +// --------------------------------------------------------------------------- +// ArtifactStore conformance suite. Runs against the INTERFACE, not a specific +// backing — pass a factory that yields a fresh store. Any future backing +// (Cloudflare Artifacts) must pass this same suite. Covers: round-trip +// (write a file set, read it back identical), snapshot immutability (a second +// publish does not change the first snapshot's bytes), latest/log ordering. +// --------------------------------------------------------------------------- + +const run = (effect: Effect.Effect): Promise => Effect.runPromise(effect); + +const fileSet = (entries: Record): FileSet => new Map(Object.entries(entries)); + +export const artifactStoreConformance = ( + name: string, + makeStore: () => Promise | ArtifactStore, +): void => { + describe(`ArtifactStore conformance: ${name}`, () => { + it("round-trips a file set through a snapshot", async () => { + const store = await makeStore(); + const scope = await run(store.forScope(scopeAddress("org", "s1"))); + const files = fileSet({ + "tools/a.ts": "export const a = 1;\n", + "notes/x.md": "# x\n", + }); + const meta = await run(scope.commit(files, "publish 1")); + expect(meta.id).toBeTruthy(); + expect(meta.message).toBe("publish 1"); + + const readBack = await run(scope.read(meta.id)); + expect(readBack.get("tools/a.ts")).toBe("export const a = 1;\n"); + expect(readBack.get("notes/x.md")).toBe("# x\n"); + + const paths = await run(scope.list(meta.id)); + expect([...paths].sort()).toEqual(["notes/x.md", "tools/a.ts"]); + + const one = await run(scope.readFile(meta.id, "tools/a.ts")); + expect(one).toBe("export const a = 1;\n"); + const missing = await run(scope.readFile(meta.id, "tools/missing.ts")); + expect(missing).toBeNull(); + }); + + it("keeps a snapshot immutable across a later publish", async () => { + const store = await makeStore(); + const scope = await run(store.forScope(scopeAddress("org", "s2"))); + const first = await run(scope.commit(fileSet({ "tools/a.ts": "v1" }), "first")); + const second = await run( + scope.commit(fileSet({ "tools/a.ts": "v2", "tools/b.ts": "new" }), "second"), + ); + + expect(second.id).not.toBe(first.id); + // The first snapshot still reads its original bytes. + const firstFiles = await run(scope.read(first.id)); + expect(firstFiles.get("tools/a.ts")).toBe("v1"); + expect(firstFiles.has("tools/b.ts")).toBe(false); + // The second snapshot has the new bytes. + const secondFiles = await run(scope.read(second.id)); + expect(secondFiles.get("tools/a.ts")).toBe("v2"); + expect(secondFiles.get("tools/b.ts")).toBe("new"); + }); + + it("tracks latest and logs newest-first", async () => { + const store = await makeStore(); + const scope = await run(store.forScope(scopeAddress("org", "s3"))); + expect(await run(scope.latest())).toBeNull(); + + const a = await run(scope.commit(fileSet({ "tools/a.ts": "1" }), "a")); + const b = await run(scope.commit(fileSet({ "tools/a.ts": "2" }), "b")); + + const latest = await run(scope.latest()); + expect(latest?.id).toBe(b.id); + + const log = await run(scope.log()); + expect(log[0].id).toBe(b.id); + expect(log[1].id).toBe(a.id); + expect(log.map((m) => m.message)).toEqual(["b", "a"]); + }); + + it("isolates scopes", async () => { + const store = await makeStore(); + const s1 = await run(store.forScope(scopeAddress("org", "iso-1"))); + const s2 = await run(store.forScope(scopeAddress("org", "iso-2"))); + await run(s1.commit(fileSet({ "tools/a.ts": "one" }), "one")); + expect(await run(s2.latest())).toBeNull(); + }); + }); +}; diff --git a/packages/plugins/apps/src/seams/artifact-store.ts b/packages/plugins/apps/src/seams/artifact-store.ts new file mode 100644 index 000000000..033a913df --- /dev/null +++ b/packages/plugins/apps/src/seams/artifact-store.ts @@ -0,0 +1,68 @@ +import type { Effect } from "effect"; +import { Data } from "effect"; +import type { ScopeAddress } from "./scope-address"; + +// --------------------------------------------------------------------------- +// ArtifactStore — the per-scope git-backed source store. +// +// A scope's source is stored as a bare git repository; a publish writes a commit +// and the commit hash IS the snapshot id (immutable, content-addressed by git). +// The self-hosted backing is a bare git repo per scope on disk (git CLI via +// subprocess). The cloud backing (future) is Cloudflare Artifacts. The seam is +// substrate-neutral: read/write/list/latest/log over a flat file set, plus a +// snapshot immutability guarantee (a committed snapshot's bytes never change). +// --------------------------------------------------------------------------- + +/** A published snapshot id. In the git backing this is the commit hash. */ +export type SnapshotId = string & { readonly __snapshotId: unique symbol }; + +export const asSnapshotId = (value: string): SnapshotId => value as SnapshotId; + +/** A flat set of source files, path -> UTF-8 contents. Paths are POSIX, + * relative to the scope root (e.g. `tools/issues-sync.ts`). */ +export type FileSet = ReadonlyMap; + +export interface SnapshotMeta { + readonly id: SnapshotId; + readonly message: string; + /** Epoch ms the snapshot was written. */ + readonly committedAt: number; +} + +export class ArtifactStoreError extends Data.TaggedError("ArtifactStoreError")<{ + readonly message: string; + /** True when the failure is a concurrent-write conflict (the snapshot ref + * moved under a compare-and-swap). Callers retry from the fresh head or + * surface a typed conflict rather than clobbering the other writer. */ + readonly conflict?: boolean; + readonly cause?: unknown; +}> {} + +/** + * One scope's source store. `commit` writes a new snapshot from a full file set + * and returns its id. `read` materializes a snapshot back to a file set. + * `latest` is the most recent snapshot on the default branch (null when the + * scope has never published). `log` lists snapshots newest-first. + */ +export interface ScopeArtifactStore { + readonly commit: ( + files: FileSet, + message: string, + ) => Effect.Effect; + readonly read: (id: SnapshotId) => Effect.Effect; + readonly readFile: ( + id: SnapshotId, + path: string, + ) => Effect.Effect; + readonly list: (id: SnapshotId) => Effect.Effect; + readonly latest: () => Effect.Effect; + readonly log: (limit?: number) => Effect.Effect; +} + +/** The substrate-neutral store: hands out a per-scope store by tenant + scope. */ +export interface ArtifactStore { + readonly forScope: ( + address: ScopeAddress, + ) => Effect.Effect; + readonly removeScope: (address: ScopeAddress) => Effect.Effect; +} diff --git a/packages/plugins/apps/src/seams/index.ts b/packages/plugins/apps/src/seams/index.ts new file mode 100644 index 000000000..02df3b451 --- /dev/null +++ b/packages/plugins/apps/src/seams/index.ts @@ -0,0 +1,5 @@ +// The substrate-neutral seams. Self-hosted backings are under +// `src/backing/`; each seam has a conformance suite (`*.conformance.ts`) that a +// backing must pass, keeping a future Cloudflare backing honest. +export * from "./artifact-store"; +export * from "./tool-sandbox"; diff --git a/packages/plugins/apps/src/seams/scope-address.ts b/packages/plugins/apps/src/seams/scope-address.ts new file mode 100644 index 000000000..9e5c2c27d --- /dev/null +++ b/packages/plugins/apps/src/seams/scope-address.ts @@ -0,0 +1,11 @@ +export interface ScopeAddress { + readonly tenant: string; + readonly scope: string; +} + +export const scopeAddress = (tenant: string, scope: string): ScopeAddress => ({ tenant, scope }); + +// Tenant-aware keys were introduced before the apps subsystem shipped, so no +// on-disk migration is needed for the older scope-only layout. +export const scopeAddressStorageKey = (address: ScopeAddress): string => + `v2-${Buffer.from(JSON.stringify([address.tenant, address.scope]), "utf8").toString("hex")}`; diff --git a/packages/plugins/apps/src/seams/tool-sandbox.conformance.ts b/packages/plugins/apps/src/seams/tool-sandbox.conformance.ts new file mode 100644 index 000000000..2e647ead6 --- /dev/null +++ b/packages/plugins/apps/src/seams/tool-sandbox.conformance.ts @@ -0,0 +1,110 @@ +import { describe, expect, it } from "@effect/vitest"; +import { Effect, Exit } from "effect"; + +import type { HandleBridge, ToolSandbox } from "./tool-sandbox"; +import { bundleEntry } from "../pipeline/bundle"; + +// --------------------------------------------------------------------------- +// ToolSandbox conformance suite. Runs against the interface. Covers: +// - collect determinism catches Math.random (double-run byte-compare) +// - network denial (fetch throws) +// - timeout kill (an infinite loop is interrupted) +// - handle bridge round-trip for declared integration roles +// A future Worker Loaders backing must pass this same suite. +// --------------------------------------------------------------------------- + +const run = (effect: Effect.Effect): Promise => Effect.runPromise(effect); + +const bundle = (entry: string, source: string): Promise => + run(bundleEntry({ files: new Map([[entry, source]]), entry })).then((b) => b.code); + +export const toolSandboxConformance = (name: string, makeSandbox: () => ToolSandbox): void => { + describe(`ToolSandbox conformance: ${name}`, () => { + it("collects deterministically and rejects Math.random at describe time", async () => { + const sandbox = makeSandbox(); + const stable = `import { defineTool } from "executor:app"; +import { z } from "zod"; +export default defineTool({ description: "stable", input: z.object({ a: z.string() }), async handler(){ return {}; } });`; + const stableBundle = await bundle("tools/s.ts", stable); + const res = await run(sandbox.collect(stableBundle)); + expect(res.artifacts.default.descriptor).toBeTruthy(); + + const rng = `import { defineTool } from "executor:app"; +import { z } from "zod"; +export default defineTool({ description: "x" + Math.random(), input: z.object({}), async handler(){ return {}; } });`; + const rngBundle = await bundle("tools/rng.ts", rng); + const exit = await Effect.runPromiseExit(sandbox.collect(rngBundle)); + expect(Exit.isFailure(exit)).toBe(true); + }); + + it("denies network access", async () => { + const sandbox = makeSandbox(); + const src = `import { defineTool } from "executor:app"; +import { z } from "zod"; +export default defineTool({ description: "net", input: z.object({}), async handler(){ await fetch("https://x.test"); return {}; } });`; + const b = await bundle("tools/net.ts", src); + const bridge: HandleBridge = { call: () => Effect.succeed(null) }; + const exit = await Effect.runPromiseExit( + sandbox.invoke(b, { artifact: "net", kind: "tool", input: {}, roots: {} }, bridge), + ); + expect(Exit.isFailure(exit)).toBe(true); + }); + + it("kills a runaway handler on timeout", async () => { + const sandbox = makeSandbox(); + const src = `import { defineTool } from "executor:app"; +import { z } from "zod"; +export default defineTool({ description: "loop", input: z.object({}), async handler(){ while (true) {} } });`; + const b = await bundle("tools/loop.ts", src); + const bridge: HandleBridge = { call: () => Effect.succeed(null) }; + const exit = await Effect.runPromiseExit( + sandbox.invoke(b, { artifact: "loop", kind: "tool", input: {}, roots: {} }, bridge), + ); + expect(Exit.isFailure(exit)).toBe(true); + }); + + it("round-trips the handle bridge for a declared integration role", async () => { + const sandbox = makeSandbox(); + const src = `import { defineTool, integration } from "executor:app"; +import { z } from "zod"; +export default defineTool({ + description: "search", + integrations: { inbox: integration("gmail") }, + input: z.object({ q: z.string() }), + async handler({ q }, { inbox }) { + const r = await inbox.messages.search({ q }); + return { total: r.count }; + }, +});`; + const b = await bundle("tools/search.ts", src); + + const seen: { root: string; path: readonly string[]; args: readonly unknown[] }[] = []; + const bridge: HandleBridge = { + call: ({ root, path, args }) => + Effect.sync(() => { + seen.push({ root, path, args }); + return { count: 3 }; + }), + }; + + const result = await run( + sandbox.invoke( + b, + { + artifact: "search", + kind: "tool", + input: { q: "invoice" }, + roots: { inbox: { kind: "single" } }, + }, + bridge, + ), + ); + expect(result.output).toEqual({ total: 3 }); + const roots = new Set(seen.map((s) => s.root)); + expect(roots.has("inbox")).toBe(true); + // The method path and JSON args crossed the boundary intact. + expect(seen[0].path).toEqual(["messages", "search"]); + expect(seen[0].args[0]).toMatchObject({ q: "invoice" }); + }); + }); +}; diff --git a/packages/plugins/apps/src/seams/tool-sandbox.ts b/packages/plugins/apps/src/seams/tool-sandbox.ts new file mode 100644 index 000000000..bc2f6c7da --- /dev/null +++ b/packages/plugins/apps/src/seams/tool-sandbox.ts @@ -0,0 +1,126 @@ +import type { Effect } from "effect"; +import { Data } from "effect"; + +// --------------------------------------------------------------------------- +// ToolSandbox — the isolated substrate that runs published bundles. +// +// Two operations, both over a bundled JS string (esbuild output, platform +// modules external, zod inlined): +// +// collect(bundle): import the bundle with NOTHING bound. `defineTool` returns +// a descriptor; a collector shim gathers it and returns JSON. This is how +// the publish pipeline extracts the versioned descriptor from source without +// ever running effectful code. Run twice + byte-compare = the determinism +// gate (a bundle that reads Math.random / Date.now at the top level or in a +// describe path diverges and is rejected). +// +// invoke(bundle, request): run one artifact's handler. The handler receives +// pre-bound clients whose method calls cross OUT through a serializable +// bridge (`HandleBridge.call`). EVERYTHING crossing the boundary is +// serializable — the cloud version of this seam is an RPC, so the interface +// forbids passing functions or runtime objects across. +// +// The self-hosted backing is QuickJS (packages/kernel/runtime-quickjs), whose +// `SandboxToolInvoker.invoke({path, args})` already matches the bridge shape. +// The cloud backing (future) is Worker Loaders. The Deno subprocess kernel is +// the harder-isolation escalation behind this same seam. +// --------------------------------------------------------------------------- + +export class ToolSandboxError extends Data.TaggedError("ToolSandboxError")<{ + readonly message: string; + readonly kind: "collect" | "invoke" | "timeout" | "network" | "nondeterministic" | "bundle"; + readonly cause?: unknown; +}> {} + +export interface ValidationIssue { + readonly message: string; + readonly path?: readonly unknown[]; +} + +export class InputValidationError extends Data.TaggedError("InputValidationError")<{ + readonly message: string; + readonly issues: readonly ValidationIssue[]; +}> {} + +export class OutputValidationError extends Data.TaggedError("OutputValidationError")<{ + readonly message: string; + readonly issues: readonly ValidationIssue[]; +}> {} + +/** + * The serializable bridge the sandbox calls out through. `root` names an + * injected handle (a connection role or one element of a fan-out set); + * `path` is the method chain (`["events", "list"]`); `args` is the JSON call + * arguments. The return value is JSON. This is the ONE way sandboxed code + * reaches the host — nothing else is wired. + */ +export interface HandleBridge { + readonly call: (input: { + readonly root: string; + readonly path: readonly string[]; + readonly args: readonly unknown[]; + }) => Effect.Effect; +} + +/** Which handle roots to inject. Each declared integration role is a single + * root. Everything the handler can see is enumerated here; undeclared roots + * are simply absent. */ +export interface HandleRootSpec { + readonly kind: "single"; +} + +export interface InvokeRequest { + /** The artifact whose handler to run (path identity, e.g. `issues-sync`). */ + readonly artifact: string; + /** The kind selects the wrapper the sandbox uses to reach the handler. */ + readonly kind: "tool"; + /** JSON input passed to the handler. */ + readonly input: unknown; + /** The handle roots to inject, keyed by the name the handler destructures + * (`github`, `inboxes`). */ + readonly roots: Readonly>; +} + +export interface InvokeResult { + readonly output: unknown; + readonly logs: readonly string[]; +} + +/** A collected artifact descriptor — the JSON `defineTool` returns. The pipeline + * refines this into the versioned descriptor; the sandbox only guarantees it + * is deterministic JSON. */ +export interface CollectedArtifact { + readonly kind: "tool"; + readonly descriptor: unknown; +} + +export interface CollectResult { + /** Descriptors keyed by artifact path identity. */ + readonly artifacts: Readonly>; +} + +export interface CollectRequest { + /** The path-derived artifact identity, used only for diagnostics. */ + readonly artifact?: string; +} + +export interface ToolSandbox { + /** + * Import the bundle with nothing bound and gather the `defineTool` descriptor. + * Runs the collection twice internally and byte-compares; a mismatch fails + * with `kind: "nondeterministic"`. This is the determinism gate. + */ + readonly collect: ( + bundle: string, + request?: CollectRequest, + ) => Effect.Effect; + /** + * Run one artifact's handler with injected handles bridged through `bridge`. + * Network is denied; a per-call timeout kills a runaway handler. + */ + readonly invoke: ( + bundle: string, + request: InvokeRequest, + bridge: HandleBridge, + ) => Effect.Effect; +} diff --git a/packages/plugins/apps/src/source/app-slug.ts b/packages/plugins/apps/src/source/app-slug.ts new file mode 100644 index 000000000..7da700111 --- /dev/null +++ b/packages/plugins/apps/src/source/app-slug.ts @@ -0,0 +1,17 @@ +export const CUSTOM_TOOLS_APP_SLUG_PATTERN = /^[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?$/; + +export const slugifyCustomToolsAppName = (input: string): string => { + const base = input + .trim() + .toLowerCase() + .replace(/[^a-z0-9]+/g, "-") + .replace(/^-+|-+$/g, ""); + return base.slice(0, 63).replace(/-+$/g, ""); +}; + +export const validateCustomToolsAppSlug = (slug: string): string | null => { + if (slug.length === 0) return "Enter a name with at least one letter or number."; + return CUSTOM_TOOLS_APP_SLUG_PATTERN.test(slug) + ? null + : "Use lowercase letters, numbers, and hyphens. Start and end with a letter or number."; +}; diff --git a/packages/plugins/apps/src/source/github-source.test.ts b/packages/plugins/apps/src/source/github-source.test.ts new file mode 100644 index 000000000..b7d410fe2 --- /dev/null +++ b/packages/plugins/apps/src/source/github-source.test.ts @@ -0,0 +1,399 @@ +import { mkdtempSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +import { describe, expect, it } from "@effect/vitest"; +import { Effect, Exit } from "effect"; + +import { makeSelfHostAppsRuntime } from "../plugin/self-host-runtime"; +import { makeInMemoryAppsStore, makeTestResolver } from "../testing"; +import { PUBLISH_LIMITS } from "../pipeline/publish"; +import { scopeAddress } from "../seams/scope-address"; +import { fetchGitHubSource, parseGitHubSourceUrl, syncGitHubSource } from "./github-source"; + +const run = (effect: Effect.Effect): Promise => Effect.runPromise(effect); + +const toolSource = (name = "ok"): string => `import { defineTool } from "executor:app"; +import { z } from "zod"; +export default defineTool({ + description: "${name}", + input: z.object({ value: z.string().default("${name}") }), + async handler(input) { return { value: input.value }; }, +});`; + +const makeRuntime = () => + makeSelfHostAppsRuntime({ + dataDir: mkdtempSync(join(tmpdir(), "apps-gh-src-")), + store: makeInMemoryAppsStore(), + resolver: makeTestResolver({}), + inMemory: true, + }); + +const json = (value: unknown, status = 200): Response => + new Response(JSON.stringify(value), { + status, + headers: { "content-type": "application/json" }, + }); + +interface GitHubTreeFixtureEntry { + readonly path: string; + readonly type: string; + readonly mode?: string; + readonly content?: string; + readonly size?: number; +} + +const makeGitHubFetch = (input: { + readonly files: ReadonlyMap; + readonly entries?: readonly GitHubTreeFixtureEntry[]; + readonly upstreamSha?: string; + readonly treeSha?: string; +}) => { + const repoPath = "/repos/acme/tools"; + const upstreamSha = input.upstreamSha ?? "commit-1"; + const treeSha = input.treeSha ?? "tree-1"; + const blobBySha = new Map(); + const treeEntries: readonly GitHubTreeFixtureEntry[] = + input.entries ?? + [...input.files].map(([path, content]) => ({ + path, + type: "blob", + mode: "100644", + content, + })); + let index = 0; + const shaByPath = new Map(); + for (const entry of treeEntries) { + if (entry.content !== undefined) { + const sha = `blob-${++index}`; + shaByPath.set(entry.path, sha); + blobBySha.set(sha, { path: entry.path, content: entry.content }); + } + } + let blobCalls = 0; + const authHeaders: (string | null)[] = []; + const fetch = (async (rawUrl: string, init?: RequestInit) => { + authHeaders.push(new Headers(init?.headers).get("authorization")); + const url = new URL(rawUrl); + if (url.pathname === repoPath) return json({ default_branch: "main" }); + if (url.pathname === `${repoPath}/git/ref/heads%2Fmain`) { + return json({ object: { sha: upstreamSha } }); + } + if (url.pathname === `${repoPath}/git/commits/${upstreamSha}`) { + return json({ sha: upstreamSha, tree: { sha: treeSha } }); + } + if (url.pathname === `${repoPath}/git/trees/${treeSha}`) { + return json({ + tree: treeEntries.map((entry, entryIndex) => { + const sha = shaByPath.get(entry.path) ?? `tree-${entryIndex}`; + return { + path: entry.path, + type: entry.type, + mode: entry.mode, + sha, + size: entry.size ?? Buffer.byteLength(entry.content ?? "", "utf8"), + }; + }), + }); + } + const blobPrefix = `${repoPath}/git/blobs/`; + if (url.pathname.startsWith(blobPrefix)) { + blobCalls++; + const sha = decodeURIComponent(url.pathname.slice(blobPrefix.length)); + const blob = blobBySha.get(sha); + if (!blob) return json({ message: "not found" }, 404); + return json({ + encoding: "base64", + content: Buffer.from(blob.content, "utf8").toString("base64"), + }); + } + return json({ message: "not found" }, 404); + }) as typeof globalThis.fetch; + return { + fetch, + blobCalls: () => blobCalls, + authHeaders: () => authHeaders, + }; +}; + +describe("GitHub custom-tools source", () => { + it("parses strict GitHub source URLs", () => { + expect(parseGitHubSourceUrl("https://github.com/UsefulSoftwareCo/executor")).toEqual({ + ok: true, + value: { + owner: "UsefulSoftwareCo", + name: "executor", + repo: "UsefulSoftwareCo/executor", + url: "https://github.com/UsefulSoftwareCo/executor", + }, + }); + expect(parseGitHubSourceUrl("https://github.com/UsefulSoftwareCo/executor.git/")).toEqual({ + ok: true, + value: { + owner: "UsefulSoftwareCo", + name: "executor", + repo: "UsefulSoftwareCo/executor", + url: "https://github.com/UsefulSoftwareCo/executor", + }, + }); + expect(parseGitHubSourceUrl("https://github.com/UsefulSoftwareCo/executor/tree/main")).toEqual({ + ok: true, + value: { + owner: "UsefulSoftwareCo", + name: "executor", + repo: "UsefulSoftwareCo/executor", + ref: "main", + url: "https://github.com/UsefulSoftwareCo/executor/tree/main", + }, + }); + expect( + parseGitHubSourceUrl("https://github.com/UsefulSoftwareCo/executor/tree/feature/tools"), + ).toEqual({ + ok: true, + value: { + owner: "UsefulSoftwareCo", + name: "executor", + repo: "UsefulSoftwareCo/executor", + ref: "feature/tools", + url: "https://github.com/UsefulSoftwareCo/executor/tree/feature/tools", + }, + }); + expect( + parseGitHubSourceUrl( + "https://github.com/UsefulSoftwareCo/executor/commit/0123456789abcdef0123456789abcdef01234567", + ), + ).toEqual({ + ok: true, + value: { + owner: "UsefulSoftwareCo", + name: "executor", + repo: "UsefulSoftwareCo/executor", + ref: "0123456789abcdef0123456789abcdef01234567", + url: "https://github.com/UsefulSoftwareCo/executor/commit/0123456789abcdef0123456789abcdef01234567", + }, + }); + expect(parseGitHubSourceUrl("UsefulSoftwareCo/executor")).toEqual({ + ok: true, + value: { + owner: "UsefulSoftwareCo", + name: "executor", + repo: "UsefulSoftwareCo/executor", + url: "https://github.com/UsefulSoftwareCo/executor", + }, + }); + expect(parseGitHubSourceUrl("https://gitlab.com/UsefulSoftwareCo/executor")).toEqual({ + ok: false, + message: "GitHub source URLs must use github.com.", + }); + expect(parseGitHubSourceUrl("https://github.com/UsefulSoftwareCo")).toEqual({ + ok: false, + message: + "Use a GitHub repo URL like https://github.com/owner/repo, optionally with /tree/ or /commit/.", + }); + expect(parseGitHubSourceUrl("https://github.com/UsefulSoftwareCo/executor/issues")).toEqual({ + ok: false, + message: + "Use a GitHub repo URL like https://github.com/owner/repo, optionally with /tree/ or /commit/.", + }); + expect( + parseGitHubSourceUrl("https://github.com/UsefulSoftwareCo/executor/commit/main"), + ).toEqual({ + ok: false, + message: "GitHub commit URLs must include a commit SHA.", + }); + }); + + it("fetches a repo fileset and publishes provenance, description, and skipped entries", async () => { + const files = new Map([ + [ + "executor.json", + JSON.stringify({ + $schema: "https://example.test/schema", + description: "Acme tools", + ignored: true, + }), + ], + ["tools/hello.ts", toolSource("hello")], + ["workflows/deferred.ts", "export default {};"], + ["docs/readme.md", "ignored"], + ]); + const github = makeGitHubFetch({ files, upstreamSha: "commit-a" }); + const snapshot = await run( + fetchGitHubSource({ url: "https://github.com/acme/tools", fetch: github.fetch }), + ); + expect([...snapshot.files.keys()].sort()).toEqual(["executor.json", "tools/hello.ts"]); + expect(github.authHeaders().every((header) => header === null)).toBe(true); + expect(snapshot.skipped).toEqual([ + { path: "workflows/deferred.ts", reason: "not supported yet" }, + { path: "docs/readme.md", reason: "ignored" }, + ]); + expect(snapshot.description).toBe("Acme tools"); + + const host = makeRuntime(); + const result = await run( + syncGitHubSource({ + runtime: host.runtime, + scope: "githubTools", + url: "https://github.com/acme/tools", + fetch: github.fetch, + }), + ); + expect(result.status).toBe("published"); + expect(result.tools).toEqual(["hello"]); + expect(result.skipped).toEqual([ + { path: "workflows/deferred.ts", reason: "not supported yet" }, + { path: "docs/readme.md", reason: "ignored" }, + ]); + const descriptor = await run(host.runtime.getDescriptor("githubTools")); + expect(descriptor?.description).toBe("Acme tools"); + expect(descriptor?.source).toEqual({ + kind: "github", + url: "https://github.com/acme/tools", + repo: "acme/tools", + ref: "main", + upstreamSha: "commit-a", + skipped: [ + { path: "workflows/deferred.ts", reason: "not supported yet" }, + { path: "docs/readme.md", reason: "ignored" }, + ], + }); + await host.close(); + }); + + it("reports up-to-date when the upstream commit SHA is unchanged", async () => { + const github = makeGitHubFetch({ + files: new Map([["tools/hello.ts", toolSource("hello")]]), + upstreamSha: "same-sha", + }); + const host = makeRuntime(); + const first = await run( + syncGitHubSource({ + runtime: host.runtime, + scope: "githubTools", + url: "https://github.com/acme/tools", + fetch: github.fetch, + }), + ); + const second = await run( + syncGitHubSource({ + runtime: host.runtime, + scope: "githubTools", + url: "https://github.com/acme/tools", + fetch: github.fetch, + }), + ); + expect(first.status).toBe("published"); + expect(second).toEqual({ + status: "up-to-date", + upstreamSha: "same-sha", + tools: ["hello"], + skipped: [], + }); + await host.close(); + }); + + it("rejects oversized trees before fetching blobs", async () => { + const files = new Map(); + for (let i = 0; i < PUBLISH_LIMITS.maxFiles + 1; i++) { + files.set(`tools/t${i}.ts`, toolSource(`t${i}`)); + } + const github = makeGitHubFetch({ files }); + const exit = await Effect.runPromiseExit( + fetchGitHubSource({ url: "https://github.com/acme/tools", fetch: github.fetch }), + ); + expect(Exit.isFailure(exit)).toBe(true); + expect(JSON.stringify(exit)).toContain("exceeding the limit"); + expect(github.blobCalls()).toBe(0); + }); + + it("returns typed failure data for publish errors", async () => { + const github = makeGitHubFetch({ + files: new Map([ + [ + "tools/bad.ts", + `import { defineTool } from "executor:app"; +import { chunk } from "lodash"; +export default defineTool({ description: "bad", input: { type: "object" }, async handler(){ return chunk([1], 1); } });`, + ], + ]), + upstreamSha: "bad-sha", + }); + const host = makeRuntime(); + const result = await run( + syncGitHubSource({ + runtime: host.runtime, + scope: "githubTools", + url: "https://github.com/acme/tools", + fetch: github.fetch, + }), + ); + expect(result.status).toBe("failed"); + expect(result.upstreamSha).toBe("bad-sha"); + expect(result.errors?.[0]?.stage).toBe("bundle"); + expect(result.errors?.[0]?.message).toContain('bare import "lodash" is not allowed'); + await host.close(); + }); + + it("skips unsupported tree entries and invalid paths without committing them", async () => { + const github = makeGitHubFetch({ + files: new Map(), + entries: [ + { + path: "tools/ok.ts", + type: "blob", + mode: "100644", + content: toolSource("ok"), + }, + { + path: "tools/link.ts", + type: "blob", + mode: "120000", + content: "../outside.ts", + }, + { + path: "tools/submodule.ts", + type: "commit", + mode: "160000", + }, + { + path: "tools/Bad.Name.ts", + type: "blob", + mode: "100644", + content: toolSource("bad"), + }, + ], + upstreamSha: "mixed-sha", + }); + const host = makeRuntime(); + const result = await run( + syncGitHubSource({ + runtime: host.runtime, + scope: "githubTools", + url: "https://github.com/acme/tools", + token: "source-token", + fetch: github.fetch, + }), + ); + + expect(result.status).toBe("published"); + expect(github.authHeaders().every((header) => header === "Bearer source-token")).toBe(true); + expect(result.tools).toEqual(["ok"]); + expect(result.skipped).toEqual([ + { path: "tools/link.ts", reason: "unsupported file type" }, + { path: "tools/submodule.ts", reason: "unsupported file type" }, + { path: "tools/Bad.Name.ts", reason: "ignored" }, + ]); + + const descriptor = await run(host.runtime.getDescriptor("githubTools")); + expect(descriptor?.tools.map((tool) => tool.name)).toEqual(["ok"]); + const scopeStore = await run( + host.runtime.deps.artifactStore.forScope(scopeAddress("org", "githubTools")), + ); + const snapshotPaths = await run(scopeStore.list(descriptor!.snapshotId as never)); + expect(snapshotPaths).toContain("tools/ok.ts"); + expect(snapshotPaths).not.toContain("tools/link.ts"); + expect(snapshotPaths).not.toContain("tools/submodule.ts"); + expect(snapshotPaths).not.toContain("tools/Bad.Name.ts"); + await host.close(); + }); +}); diff --git a/packages/plugins/apps/src/source/github-source.ts b/packages/plugins/apps/src/source/github-source.ts new file mode 100644 index 000000000..0bc38fe2e --- /dev/null +++ b/packages/plugins/apps/src/source/github-source.ts @@ -0,0 +1,455 @@ +import { Data, Effect, Predicate, Result, Schema } from "effect"; + +import { PublishError, type FileDiagnostic } from "../pipeline/discover"; +import { PUBLISH_LIMITS, enforcePublishLimits } from "../pipeline/publish"; +import type { AppSourceRef, SourceSkippedArtifact } from "../pipeline/descriptor"; +import type { AppsRuntime } from "../plugin/runtime"; +import type { FileSet, SnapshotId } from "../seams/artifact-store"; +import { parseGitHubSourceUrl, type ParsedGitHubSourceUrl } from "./github-url"; + +export interface GitHubSourceInput { + readonly url: string; + readonly ref?: string; + readonly token?: string | null; + readonly baseUrl?: string; + readonly fetch?: typeof globalThis.fetch; +} + +export interface GitHubSourceSnapshot { + readonly files: FileSet; + readonly url: string; + readonly repo: string; + readonly ref: string; + readonly upstreamSha: string; + readonly description?: string; + readonly skipped: readonly GitHubSkippedArtifact[]; +} + +export class GitHubSourceError extends Data.TaggedError("GitHubSourceError")<{ + readonly message: string; + readonly status?: number; + readonly path?: string; + readonly cause?: unknown; +}> {} + +export interface SyncErrorData { + readonly stage: "source" | "discover" | "bundle" | "collect" | "project"; + readonly message: string; + readonly diagnostics?: readonly FileDiagnostic[]; +} + +export type GitHubSkippedArtifact = SourceSkippedArtifact; + +export type GitHubSyncResult = + | { + readonly status: "published"; + readonly snapshotId: SnapshotId; + readonly upstreamSha: string; + readonly tools: readonly string[]; + readonly skipped: readonly GitHubSkippedArtifact[]; + readonly errors?: undefined; + } + | { + readonly status: "up-to-date"; + readonly upstreamSha: string; + readonly tools: readonly string[]; + readonly skipped: readonly GitHubSkippedArtifact[]; + readonly errors?: undefined; + } + | { + readonly status: "failed"; + readonly upstreamSha?: string; + readonly tools: readonly string[]; + readonly skipped: readonly GitHubSkippedArtifact[]; + readonly errors: readonly SyncErrorData[]; + }; + +export interface SyncGitHubSourceInput extends GitHubSourceInput { + readonly runtime: AppsRuntime; + readonly tenant?: string; + readonly scope: string; +} + +const TOOL_RE = /^tools\/([a-z0-9][a-z0-9-]*)\.(ts|tsx|js|jsx)$/; +const DEFERRED_RE = /^(workflows|ui|skills)\//; +const REGULAR_FILE_MODES = new Set(["100644", "100755"]); + +const trimBaseUrl = (baseUrl: string): string => baseUrl.replace(/\/+$/, ""); + +const encodedRepoPath = (source: ParsedGitHubSourceUrl): string => + `${encodeURIComponent(source.owner)}/${encodeURIComponent(source.name)}`; + +const parseSourceInput = ( + input: GitHubSourceInput, +): Effect.Effect => { + const parsed = parseGitHubSourceUrl(input.url, { ref: input.ref }); + return parsed.ok + ? Effect.succeed(parsed.value) + : Effect.fail( + new GitHubSourceError({ + message: parsed.message, + path: input.url, + }), + ); +}; + +const acceptedPath = (path: string): boolean => path === "executor.json" || TOOL_RE.test(path); + +const isRecord = (value: unknown): value is Record => + value !== null && typeof value === "object" && !Array.isArray(value); + +const asString = (value: unknown): string | undefined => + typeof value === "string" && value.length > 0 ? value : undefined; + +const requestJson = ( + input: GitHubSourceInput, + path: string, +): Effect.Effect => + Effect.gen(function* () { + const response = yield* Effect.tryPromise({ + try: async () => { + const fetchImpl = input.fetch ?? globalThis.fetch; + const headers: Record = { + accept: "application/vnd.github+json", + "user-agent": "executor-apps-github-source", + }; + if (input.token) headers.authorization = `Bearer ${input.token}`; + return fetchImpl(`${trimBaseUrl(input.baseUrl ?? "https://api.github.com")}${path}`, { + headers, + }); + }, + catch: (cause) => + new GitHubSourceError({ + message: `GitHub request failed: GET ${path}`, + path, + cause, + }), + }); + if (!response.ok) { + return yield* new GitHubSourceError({ + message: `GitHub request failed: GET ${path} -> ${response.status}`, + status: response.status, + path, + }); + } + return yield* Effect.tryPromise({ + try: () => response.json() as Promise, + catch: (cause) => + new GitHubSourceError({ + message: `GitHub response was not valid JSON: GET ${path}`, + path, + cause, + }), + }); + }); + +interface RepoResponse { + readonly default_branch?: unknown; +} + +interface CommitResponse { + readonly sha?: unknown; + readonly commit?: { + readonly tree?: { + readonly sha?: unknown; + }; + }; +} + +interface GitCommitResponse { + readonly sha?: unknown; + readonly tree?: { + readonly sha?: unknown; + }; +} + +interface RefResponse { + readonly object?: { + readonly sha?: unknown; + }; +} + +interface TreeEntry { + readonly path?: unknown; + readonly type?: unknown; + readonly mode?: unknown; + readonly sha?: unknown; + readonly size?: unknown; +} + +interface TreeResponse { + readonly tree?: readonly TreeEntry[]; + readonly truncated?: unknown; +} + +interface BlobResponse { + readonly content?: unknown; + readonly encoding?: unknown; +} + +const commitTreeSha = (commit: CommitResponse | GitCommitResponse): string | undefined => { + if ("commit" in commit) return asString(commit.commit?.tree?.sha); + if ("tree" in commit) return asString(commit.tree?.sha); + return undefined; +}; + +const limitError = (diagnostics: readonly FileDiagnostic[]): PublishError => + new PublishError({ + message: `publish payload exceeds limits (${diagnostics.length} problem(s))`, + stage: "discover", + diagnostics, + }); + +const checkTreeLimits = (entries: readonly TreeEntry[]): PublishError | null => { + const diagnostics: FileDiagnostic[] = []; + if (entries.length > PUBLISH_LIMITS.maxFiles) { + diagnostics.push({ + path: "", + message: `publish has ${entries.length} files, exceeding the limit of ${PUBLISH_LIMITS.maxFiles}`, + }); + } + let total = 0; + for (const entry of entries) { + const path = String(entry.path ?? ""); + const size = typeof entry.size === "number" ? entry.size : 0; + total += size; + if (size > PUBLISH_LIMITS.maxFileBytes) { + diagnostics.push({ + path, + message: `file is ${size} bytes, exceeding the per-file limit of ${PUBLISH_LIMITS.maxFileBytes} bytes`, + }); + } + } + if (total > PUBLISH_LIMITS.maxTotalBytes) { + diagnostics.push({ + path: "", + message: `publish total is ${total} bytes, exceeding the total limit of ${PUBLISH_LIMITS.maxTotalBytes} bytes`, + }); + } + return diagnostics.length === 0 ? null : limitError(diagnostics); +}; + +const classifyTreeEntry = ( + entry: TreeEntry, +): + | { + readonly kind: "fetch"; + readonly entry: TreeEntry & { readonly path: string; readonly sha: string }; + } + | { readonly kind: "skip"; readonly skipped: GitHubSkippedArtifact } + | null => { + const path = asString(entry.path); + if (!path) return null; + + const type = asString(entry.type); + const mode = asString(entry.mode); + if (type === "tree") return null; + if (type !== "blob" || !mode || !REGULAR_FILE_MODES.has(mode)) { + return { kind: "skip", skipped: { path, reason: "unsupported file type" } }; + } + + const sha = asString(entry.sha); + if (!sha) return { kind: "skip", skipped: { path, reason: "unsupported file type" } }; + if (acceptedPath(path)) return { kind: "fetch", entry: { ...entry, path, sha } }; + if (DEFERRED_RE.test(path)) { + return { kind: "skip", skipped: { path, reason: "not supported yet" } }; + } + return { kind: "skip", skipped: { path, reason: "ignored" } }; +}; + +const decodeBlob = (path: string, blob: BlobResponse): Effect.Effect => + Effect.gen(function* () { + const encoding = asString(blob.encoding); + const content = asString(blob.content); + if (!content || encoding !== "base64") { + return yield* new GitHubSourceError({ + message: `GitHub blob ${path} did not return base64 content`, + path, + }); + } + return Buffer.from(content.replace(/\s/g, ""), "base64").toString("utf8"); + }); + +const decodeExecutorJson = Schema.decodeUnknownEffect(Schema.UnknownFromJsonString); + +const executorDescription = ( + files: FileSet, +): Effect.Effect => + Effect.gen(function* () { + const raw = files.get("executor.json"); + if (!raw) return undefined; + const parsed = yield* decodeExecutorJson(raw).pipe( + Effect.mapError( + (cause) => + new GitHubSourceError({ + message: "executor.json is not valid JSON", + path: "executor.json", + cause, + }), + ), + ); + if (!isRecord(parsed)) return undefined; + const description = parsed.description; + return typeof description === "string" ? description : undefined; + }); + +export const fetchGitHubSource = ( + input: GitHubSourceInput, +): Effect.Effect => + Effect.gen(function* () { + const source = yield* parseSourceInput(input); + const repoPath = encodedRepoPath(source); + const repo = yield* requestJson(input, `/repos/${repoPath}`); + const ref = source.ref ?? asString(repo.default_branch) ?? "main"; + const branchRef = yield* requestJson( + input, + `/repos/${repoPath}/git/ref/${encodeURIComponent(`heads/${ref}`)}`, + ).pipe(Effect.result); + const commitPath = + Result.isSuccess(branchRef) && asString(branchRef.success.object?.sha) + ? `/repos/${repoPath}/git/commits/${encodeURIComponent(String(branchRef.success.object?.sha))}` + : `/repos/${repoPath}/commits/${encodeURIComponent(ref)}`; + const commit = yield* requestJson(input, commitPath); + const upstreamSha = asString(commit.sha); + const treeSha = commitTreeSha(commit); + if (!upstreamSha || !treeSha) { + return yield* new GitHubSourceError({ + message: `GitHub commit ${ref} did not include a commit SHA and tree SHA`, + path: commitPath, + }); + } + + const tree = yield* requestJson( + input, + `/repos/${repoPath}/git/trees/${encodeURIComponent(treeSha)}?recursive=1`, + ); + if (tree.truncated === true) { + return yield* new GitHubSourceError({ + message: "GitHub returned a truncated repository tree; custom tools source is too large", + path: `/repos/${repoPath}/git/trees/${treeSha}`, + }); + } + const entries: (TreeEntry & { readonly path: string; readonly sha: string })[] = []; + const skipped: GitHubSkippedArtifact[] = []; + for (const entry of tree.tree ?? []) { + const classified = classifyTreeEntry(entry); + if (!classified) continue; + if (classified.kind === "fetch") entries.push(classified.entry); + else skipped.push(classified.skipped); + } + const treeLimitError = checkTreeLimits(entries); + if (treeLimitError) return yield* treeLimitError; + + const files = new Map(); + for (const entry of entries) { + const path = String(entry.path); + const blob = yield* requestJson( + input, + `/repos/${repoPath}/git/blobs/${encodeURIComponent(String(entry.sha))}`, + ); + files.set(path, yield* decodeBlob(path, blob)); + } + const payloadLimitError = enforcePublishLimits(files); + if (payloadLimitError) return yield* payloadLimitError; + + return { + files, + url: source.ref ? source.url : `https://github.com/${source.repo}`, + repo: source.repo, + ref, + upstreamSha, + description: yield* executorDescription(files), + skipped, + }; + }); + +const publishErrorToSyncError = (publishFailure: PublishError): SyncErrorData => { + const stage = publishFailure.stage; + const message = publishFailure.message; + const diagnostics = publishFailure.diagnostics; + return { stage, message, diagnostics }; +}; + +const sourceErrorToSyncError = (sourceFailure: GitHubSourceError): SyncErrorData => { + const message = sourceFailure.message; + const path = sourceFailure.path; + return { + stage: "source", + message, + diagnostics: path ? [{ path, message }] : [], + }; +}; + +const sourceRef = (snapshot: GitHubSourceSnapshot): AppSourceRef => ({ + kind: "github", + url: snapshot.url, + repo: snapshot.repo, + ref: snapshot.ref, + upstreamSha: snapshot.upstreamSha, + skipped: snapshot.skipped, +}); + +export const syncGitHubSource = (input: SyncGitHubSourceInput): Effect.Effect => + Effect.gen(function* () { + const fetched = yield* fetchGitHubSource(input).pipe(Effect.result); + if (Result.isFailure(fetched)) { + const error = fetched.failure; + return { + status: "failed", + tools: [], + skipped: [], + errors: [ + Predicate.isTagged("PublishError")(error) + ? publishErrorToSyncError(error) + : sourceErrorToSyncError(error), + ], + } satisfies GitHubSyncResult; + } + const snapshot = fetched.success; + const current = input.tenant + ? yield* input.runtime.getDescriptor(input.tenant, input.scope) + : yield* input.runtime.getDescriptor(input.scope); + if ( + current?.source?.kind === "github" && + current.source.repo === snapshot.repo && + current.source.upstreamSha === snapshot.upstreamSha + ) { + return { + status: "up-to-date", + upstreamSha: snapshot.upstreamSha, + tools: current.tools.map((tool) => tool.name), + skipped: [...snapshot.skipped, ...(current.skipped ?? [])], + } satisfies GitHubSyncResult; + } + + const published = yield* input.runtime + .publish({ + tenant: input.tenant, + scope: input.scope, + files: snapshot.files, + description: snapshot.description, + source: sourceRef(snapshot), + message: `sync ${snapshot.repo}@${snapshot.upstreamSha}`, + }) + .pipe(Effect.result); + if (Result.isFailure(published)) { + return { + status: "failed", + upstreamSha: snapshot.upstreamSha, + tools: [], + skipped: snapshot.skipped, + errors: [publishErrorToSyncError(published.failure)], + } satisfies GitHubSyncResult; + } + + return { + status: "published", + snapshotId: published.success.snapshotId, + upstreamSha: snapshot.upstreamSha, + tools: published.success.descriptor.tools.map((tool) => tool.name), + skipped: [...snapshot.skipped, ...published.success.skipped], + } satisfies GitHubSyncResult; + }); + +export { parseGitHubSourceUrl } from "./github-url"; +export type { ParsedGitHubSourceUrl } from "./github-url"; diff --git a/packages/plugins/apps/src/source/github-url.ts b/packages/plugins/apps/src/source/github-url.ts new file mode 100644 index 000000000..6f291180c --- /dev/null +++ b/packages/plugins/apps/src/source/github-url.ts @@ -0,0 +1,108 @@ +export interface ParsedGitHubSourceUrl { + readonly owner: string; + readonly name: string; + readonly repo: string; + readonly ref?: string; + readonly url: string; +} + +export type GitHubSourceUrlParseResult = + | { readonly ok: true; readonly value: ParsedGitHubSourceUrl } + | { readonly ok: false; readonly message: string }; + +const OWNER_RE = /^[A-Za-z0-9](?:[A-Za-z0-9-]{0,38})$/; +const REPO_RE = /^[A-Za-z0-9_.-]+$/; +const COMMIT_RE = /^[0-9a-f]{7,40}$/i; +const URL_RE = /^https?:\/\/([^/?#]+)(?:\/([^?#]*))?$/i; + +const trimSlashes = (value: string): string => value.replace(/^\/+|\/+$/g, ""); + +const normalizeRepoName = (value: string): string => + value.endsWith(".git") ? value.slice(0, -4) : value; + +const invalidShape = + "Use a GitHub repo URL like https://github.com/owner/repo, optionally with /tree/ or /commit/."; + +const validateOwnerRepo = ( + owner: string | undefined, + nameInput: string | undefined, +): { readonly owner: string; readonly name: string } | null => { + const name = nameInput ? normalizeRepoName(nameInput) : ""; + if (!owner || !name) return null; + if (!OWNER_RE.test(owner) || !REPO_RE.test(name)) return null; + return { owner, name }; +}; + +const canonicalUrl = ( + owner: string, + name: string, + kind: "base" | "tree" | "commit", + ref?: string, +): string => { + const base = `https://github.com/${owner}/${name}`; + if (!ref || kind === "base") return base; + return `${base}/${kind}/${ref}`; +}; + +export const parseGitHubSourceUrl = ( + input: string, + options?: { readonly ref?: string | undefined }, +): GitHubSourceUrlParseResult => { + const trimmed = input.trim(); + const overrideRef = options?.ref?.trim(); + if (trimmed.length === 0) return { ok: false, message: "Enter a GitHub URL." }; + + const urlMatch = URL_RE.exec(trimmed); + const path = urlMatch ? trimSlashes(urlMatch[2] ?? "") : trimSlashes(trimmed); + if (urlMatch && urlMatch[1]?.toLowerCase() !== "github.com") { + return { ok: false, message: "GitHub source URLs must use github.com." }; + } + if (!urlMatch && /^[A-Za-z][A-Za-z0-9+.-]*:\/\//.test(trimmed)) { + return { ok: false, message: "GitHub source URLs must use github.com." }; + } + + const segments = path.split("/").filter((segment) => segment.length > 0); + const validated = validateOwnerRepo(segments[0], segments[1]); + if (!validated) return { ok: false, message: invalidShape }; + const { owner, name } = validated; + const repo = `${owner}/${name}`; + + if (segments.length === 2) { + const ref = overrideRef || undefined; + return { + ok: true, + value: { + owner, + name, + repo, + ...(ref ? { ref } : {}), + url: canonicalUrl(owner, name, ref ? "tree" : "base", ref), + }, + }; + } + + const route = segments[2]; + if (route === "tree") { + const ref = overrideRef || segments.slice(3).join("/"); + if (!ref) + return { ok: false, message: "GitHub tree URLs must include a branch, tag, or commit SHA." }; + return { + ok: true, + value: { owner, name, repo, ref, url: canonicalUrl(owner, name, "tree", ref) }, + }; + } + + if (route === "commit") { + const sourceRef = segments.length === 4 ? segments[3] : ""; + const ref = overrideRef || sourceRef; + if (!ref || !COMMIT_RE.test(ref)) { + return { ok: false, message: "GitHub commit URLs must include a commit SHA." }; + } + return { + ok: true, + value: { owner, name, repo, ref, url: canonicalUrl(owner, name, "commit", ref) }, + }; + } + + return { ok: false, message: invalidShape }; +}; diff --git a/packages/plugins/apps/src/standard-schema.ts b/packages/plugins/apps/src/standard-schema.ts new file mode 100644 index 000000000..8aa04a05e --- /dev/null +++ b/packages/plugins/apps/src/standard-schema.ts @@ -0,0 +1,153 @@ +// Type-only vendored subset of @standard-schema/spec 1.1.0. +// Source: https://github.com/standard-schema/standard-schema + +/** The Standard Typed interface. This is a base type extended by other specs. */ +export interface StandardTypedV1 { + /** The Standard properties. */ + readonly "~standard": StandardTypedV1.Props; +} + +export namespace StandardTypedV1 { + /** The Standard Typed properties interface. */ + export interface Props { + /** The version number of the standard. */ + readonly version: 1; + /** The vendor name of the schema library. */ + readonly vendor: string; + /** Inferred types associated with the schema. */ + readonly types?: Types | undefined; + } + + /** The Standard Typed types interface. */ + export interface Types { + /** The input type of the schema. */ + readonly input: Input; + /** The output type of the schema. */ + readonly output: Output; + } + + /** Infers the input type of a Standard Typed. */ + export type InferInput = NonNullable< + Schema["~standard"]["types"] + >["input"]; + + /** Infers the output type of a Standard Typed. */ + export type InferOutput = NonNullable< + Schema["~standard"]["types"] + >["output"]; +} + +/** The Standard Schema interface. */ +export interface StandardSchemaV1 { + /** The Standard Schema properties. */ + readonly "~standard": StandardSchemaV1.Props; +} + +export namespace StandardSchemaV1 { + /** The Standard Schema properties interface. */ + export interface Props extends StandardTypedV1.Props< + Input, + Output + > { + /** Validates unknown input values. */ + readonly validate: ( + value: unknown, + options?: StandardSchemaV1.Options | undefined, + ) => Result | Promise>; + } + + /** The result interface of the validate function. */ + export type Result = SuccessResult | FailureResult; + + /** The result interface if validation succeeds. */ + export interface SuccessResult { + /** The typed output value. */ + readonly value: Output; + /** A falsy value for `issues` indicates success. */ + readonly issues?: undefined; + } + + export interface Options { + /** Explicit support for additional vendor-specific parameters, if needed. */ + readonly libraryOptions?: Record | undefined; + } + + /** The result interface if validation fails. */ + export interface FailureResult { + /** The issues of failed validation. */ + readonly issues: ReadonlyArray; + } + + /** The issue interface of the failure output. */ + export interface Issue { + /** The error message of the issue. */ + readonly message: string; + /** The path of the issue, if any. */ + readonly path?: ReadonlyArray | undefined; + } + + /** The path segment interface of the issue. */ + export interface PathSegment { + /** The key representing a path segment. */ + readonly key: PropertyKey; + } + + /** The Standard types interface. */ + export interface Types extends StandardTypedV1.Types< + Input, + Output + > {} + + /** Infers the input type of a Standard. */ + export type InferInput = StandardTypedV1.InferInput; + + /** Infers the output type of a Standard. */ + export type InferOutput = StandardTypedV1.InferOutput; +} + +/** The Standard JSON Schema interface. */ +export interface StandardJSONSchemaV1 { + /** The Standard JSON Schema properties. */ + readonly "~standard": StandardJSONSchemaV1.Props; +} + +export namespace StandardJSONSchemaV1 { + /** The Standard JSON Schema properties interface. */ + export interface Props extends StandardTypedV1.Props< + Input, + Output + > { + /** Methods for generating the input/output JSON Schema. */ + readonly jsonSchema: StandardJSONSchemaV1.Converter; + } + + /** The Standard JSON Schema converter interface. */ + export interface Converter { + /** Converts the input type to JSON Schema. */ + readonly input: (options: StandardJSONSchemaV1.Options) => Record; + /** Converts the output type to JSON Schema. */ + readonly output: (options: StandardJSONSchemaV1.Options) => Record; + } + + export type Target = "draft-2020-12" | "draft-07" | "openapi-3.0" | ({} & string); + + /** The options for the input/output methods. */ + export interface Options { + /** Specifies the target version of the generated JSON Schema. */ + readonly target: Target; + /** Explicit support for additional vendor-specific parameters, if needed. */ + readonly libraryOptions?: Record | undefined; + } + + /** The Standard types interface. */ + export interface Types extends StandardTypedV1.Types< + Input, + Output + > {} + + /** Infers the input type of a Standard. */ + export type InferInput = StandardTypedV1.InferInput; + + /** Infers the output type of a Standard. */ + export type InferOutput = StandardTypedV1.InferOutput; +} diff --git a/packages/plugins/apps/src/testing/daily-brief.ts b/packages/plugins/apps/src/testing/daily-brief.ts new file mode 100644 index 000000000..3f7e5018f --- /dev/null +++ b/packages/plugins/apps/src/testing/daily-brief.ts @@ -0,0 +1,81 @@ +export const DAILY_BRIEF_MANIFEST = `{ + "$schema": "https://executor.sh/schemas/scope-manifest.json", + "scope": "rhys", + "description": "Personal scope artifacts, GitHub issues brief.", + "artifacts": { "tools": "tools/" } +} +`; + +export const ISSUES_SYNC_TS = `import { z } from "zod"; +import { defineTool, integration } from "executor:app"; + +export default defineTool({ + description: + "Summarize open GitHub issues across the given repos (default: every repo the connection can see).", + integrations: { + github: integration("github"), + }, + input: z.object({ + repos: z.array(z.string()).optional().describe("owner/repo entries; omit to sync all accessible repos"), + since: z.string().optional().describe("ISO timestamp, only issues updated after this"), + }), + output: z.object({ synced: z.number(), repos: z.number(), issues: z.array(z.object({ repo: z.string(), number: z.number(), title: z.string() })) }), + annotations: { readOnly: false, destructive: false }, + async handler({ repos, since }, { github }) { + const targets = + repos ?? + (await github.repos.listForAuthenticatedUser({ per_page: 100 })).map((r) => r.full_name); + + let synced = 0; + const collected = []; + for (const target of targets) { + const [owner, repo] = target.split("/"); + const issues = await github.issues.listForRepo({ owner, repo, state: "open", since, per_page: 100 }); + for (const issue of issues) { + if (issue.pull_request) continue; + collected.push({ repo: target, number: issue.number, title: issue.title }); + synced++; + } + } + return { synced, repos: targets.length, issues: collected }; + }, +}); +`; + +export const SEARCH_ALL_MAIL_TS = `import { z } from "zod"; +import { defineTool, integration } from "executor:app"; + +export default defineTool({ + description: + "Search a connected Gmail account. Returns matches newest-first.", + integrations: { + inbox: integration("gmail"), + }, + input: z.object({ + query: z.string().describe("Gmail search syntax, e.g. from:acme subject:invoice"), + limit: z.number().int().min(1).max(100).default(25), + }), + output: z.object({ + results: z.array(z.object({ + inbox: z.string(), id: z.string(), from: z.string(), + subject: z.string(), snippet: z.string(), date: z.string(), + })), + }), + annotations: { readOnly: true }, + async handler({ query, limit }, { inbox }) { + const { messages } = await inbox.messages.search({ q: query, maxResults: limit }); + const results = messages.map((m) => ({ + inbox: inbox.account.email, id: m.id, from: m.from, + subject: m.subject, snippet: m.snippet, date: m.date, + })).sort((a, b) => b.date.localeCompare(a.date)).slice(0, limit); + return { results }; + }, +}); +`; + +export const dailyBriefFileSet = (): Map => + new Map([ + ["executor.json", DAILY_BRIEF_MANIFEST], + ["tools/issues-sync.ts", ISSUES_SYNC_TS], + ["tools/search-all-mail.ts", SEARCH_ALL_MAIL_TS], + ]); diff --git a/packages/plugins/apps/src/testing/index.ts b/packages/plugins/apps/src/testing/index.ts new file mode 100644 index 000000000..9ac064e31 --- /dev/null +++ b/packages/plugins/apps/src/testing/index.ts @@ -0,0 +1,146 @@ +import { Effect } from "effect"; + +import type { AppDescriptor } from "../pipeline/descriptor"; +import type { AppsStore } from "../plugin/store"; +import { BindingError, type ClientResolver, type ConnectionCandidate } from "../plugin/bindings"; +import { + ArtifactStoreError, + asSnapshotId, + type ArtifactStore, + type FileSet, + type ScopeArtifactStore, + type SnapshotId, +} from "../seams/artifact-store"; +import { scopeAddressStorageKey } from "../seams/scope-address"; + +export * from "./daily-brief"; + +export const makeInMemoryAppsStore = (): AppsStore & { + readonly descriptors: Map; +} => { + const descriptors = new Map(); + const publishedAt = new Map(); + const keyFor = (tenant: string, key: string): string => `${tenant}:${key}`; + return { + descriptors, + putDescriptor: (tenant, _owner, descriptor) => + Effect.sync(() => { + const key = keyFor(tenant, descriptor.scope); + descriptors.set(key, descriptor); + publishedAt.set(key, Date.now()); + }), + getDescriptor: (tenant, scope) => + Effect.sync(() => descriptors.get(keyFor(tenant, scope)) ?? null), + removeDescriptor: (tenant, scope) => + Effect.sync(() => { + const key = keyFor(tenant, scope); + descriptors.delete(key); + publishedAt.delete(key); + }), + listDescriptors: (tenant) => + Effect.sync(() => + [...descriptors.entries()] + .filter(([key]) => key.startsWith(`${tenant}:`)) + .map(([key, descriptor]) => ({ + descriptor, + publishedAt: publishedAt.get(key) ?? 0, + })), + ), + }; +}; + +export const makeInMemoryArtifactStore = (): ArtifactStore => { + const scopes = new Map>(); + const order = new Map(); + let counter = 0; + const forScope = (key: string): ScopeArtifactStore => { + const snaps = scopes.get(key) ?? new Map(); + scopes.set(key, snaps); + const seq = order.get(key) ?? []; + order.set(key, seq); + return { + commit: (files, message) => + Effect.sync(() => { + const id = `mem${(++counter).toString(16).padStart(40, "0")}`; + snaps.set(id, new Map(files)); + seq.push(id); + return { id: asSnapshotId(id), message, committedAt: Date.now() }; + }), + read: (id) => + snaps.has(id) + ? Effect.succeed(snaps.get(id)!) + : Effect.fail(new ArtifactStoreError({ message: `no snapshot ${id}` })), + readFile: (id, path) => Effect.succeed(snaps.get(id)?.get(path) ?? null), + list: (id) => Effect.succeed([...(snaps.get(id)?.keys() ?? [])]), + latest: () => + Effect.sync(() => { + const last = seq.at(-1); + if (!last) return null; + return { id: asSnapshotId(last), message: "latest", committedAt: Date.now() }; + }), + log: (limit) => + Effect.sync(() => + [...seq] + .reverse() + .slice(0, limit ?? seq.length) + .map((id) => ({ id: asSnapshotId(id), message: "", committedAt: Date.now() })), + ), + }; + }; + return { + forScope: (address) => Effect.succeed(forScope(scopeAddressStorageKey(address))), + removeScope: (address) => + Effect.sync(() => { + const key = scopeAddressStorageKey(address); + scopes.delete(key); + order.delete(key); + }), + }; +}; + +export type { SnapshotId }; + +export const makeTestResolver = ( + handlers: Record unknown>>, + connections?: readonly ConnectionCandidate[], +): ClientResolver & { + readonly calls: { integration: string; connection: string; method: string }[]; +} => { + const calls: { integration: string; connection: string; method: string }[] = []; + const knownConnections = + connections ?? + Object.keys(handlers).map((integration) => ({ + address: `tools.${integration}.user.${integration}`, + integration, + name: integration, + owner: "user", + })); + return { + calls, + listConnections: ({ integration }) => + Effect.succeed( + knownConnections.filter((connection) => connection.integration === integration), + ), + resolveConnection: ({ connection }) => + Effect.succeed( + knownConnections.find( + (candidate) => candidate.address === connection || candidate.name === connection, + ) ?? null, + ), + call: ({ integration, connection, path, args }) => { + const method = path.join("."); + calls.push({ integration, connection, method }); + const handler = handlers[integration]?.[method]; + if (!handler) { + return Effect.fail( + new BindingError({ + message: `no test handler for ${integration}.${method}`, + role: integration, + integration, + }), + ); + } + return Effect.sync(() => handler(args)); + }, + }; +}; diff --git a/packages/plugins/apps/tsconfig.json b/packages/plugins/apps/tsconfig.json new file mode 100644 index 000000000..1504bed72 --- /dev/null +++ b/packages/plugins/apps/tsconfig.json @@ -0,0 +1,23 @@ +{ + "compilerOptions": { + "target": "ES2022", + "module": "ESNext", + "moduleResolution": "Bundler", + "strict": true, + "skipLibCheck": true, + "lib": ["ES2022", "DOM"], + "types": ["bun-types", "node"], + "noUnusedLocals": true, + "noImplicitOverride": true, + "jsx": "react-jsx", + "plugins": [ + { + "name": "@effect/language-service", + "ignoreEffectSuggestionsInTscExitCode": true, + "ignoreEffectWarningsInTscExitCode": true, + "diagnosticSeverity": {} + } + ] + }, + "include": ["src/**/*.ts", "src/**/*.tsx"] +} diff --git a/packages/plugins/apps/tsup.config.ts b/packages/plugins/apps/tsup.config.ts new file mode 100644 index 000000000..b0f69dc9b --- /dev/null +++ b/packages/plugins/apps/tsup.config.ts @@ -0,0 +1,15 @@ +import { defineConfig } from "tsup"; + +export default defineConfig({ + entry: { + index: "src/index.ts", + api: "src/api.ts", + "seams/index": "src/seams/index.ts", + "testing/index": "src/testing/index.ts", + }, + format: ["esm"], + dts: false, + sourcemap: true, + clean: true, + external: [/^@executor-js\//, /^effect/, /^@effect\//, "esbuild"], +}); diff --git a/packages/plugins/apps/vitest.config.ts b/packages/plugins/apps/vitest.config.ts new file mode 100644 index 000000000..698a11896 --- /dev/null +++ b/packages/plugins/apps/vitest.config.ts @@ -0,0 +1,9 @@ +import { defineConfig } from "vitest/config"; + +export default defineConfig({ + test: { + include: ["src/**/*.test.ts"], + testTimeout: 30_000, + hookTimeout: 30_000, + }, +}); diff --git a/packages/react/src/components/json-schema-form.test.ts b/packages/react/src/components/json-schema-form.test.ts index 5074fa909..e97d03ad8 100644 --- a/packages/react/src/components/json-schema-form.test.ts +++ b/packages/react/src/components/json-schema-form.test.ts @@ -5,6 +5,7 @@ import * as Schema from "effect/Schema"; import { coerceNumber, defaultForSchema, + emptyEnumGuidance, isRenderableObjectSchema, missingRequiredFields, removeProperty, @@ -101,10 +102,20 @@ describe("json-schema-form value transforms", () => { expect(defaultForSchema({ type: "boolean" }, root)).toBe(false); expect(defaultForSchema({ type: "object" }, root)).toEqual({}); expect(defaultForSchema({ type: "array" }, root)).toEqual([]); + expect(defaultForSchema({ type: "string", default: "tools.github.user.main" }, root)).toBe( + "tools.github.user.main", + ); expect(defaultForSchema({ enum: ["a", "b"] }, root)).toBe("a"); expect(defaultForSchema({ const: 7 }, root)).toBe(7); }); + it("empty enum guidance names connection integrations", () => { + expect(emptyEnumGuidance("Connection to use for github (github)")).toBe( + "Connect github first.", + ); + expect(emptyEnumGuidance("Choose a value")).toBe("No options available."); + }); + it("isRenderableObjectSchema gates form-mode availability", () => { expect(isRenderableObjectSchema(SCHEMA)).toBe(true); diff --git a/packages/react/src/components/json-schema-form.tsx b/packages/react/src/components/json-schema-form.tsx index de26787a6..52fc90d1a 100644 --- a/packages/react/src/components/json-schema-form.tsx +++ b/packages/react/src/components/json-schema-form.tsx @@ -95,6 +95,11 @@ const schemaDescription = (schema: JsonSchema): string | undefined => { return undefined; }; +export const emptyEnumGuidance = (description: string | undefined): string => { + const integration = description?.match(/^Connection to use for .+ \(([^)]+)\)$/)?.[1]; + return integration ? `Connect ${integration} first.` : "No options available."; +}; + const DEFAULT_BY_TYPE: Record unknown> = { string: () => "", number: () => 0, @@ -105,9 +110,10 @@ const DEFAULT_BY_TYPE: Record unknown> = { }; /** Seed used when the user clicks "Add" for an optional property or array item. - * Never auto-injected on load — only on explicit add. */ + * Schema defaults can also be displayed without being injected into JSON. */ export const defaultForSchema = (schema: JsonSchema, root: JsonSchema): unknown => { const s = deepResolve(schema, root); + if (s.default !== undefined) return s.default; if (s.const !== undefined) return s.const; if (Array.isArray(s.enum) && s.enum.length > 0) return s.enum[0]; const t = primaryType(s); @@ -223,8 +229,8 @@ export function JsonSchemaForm(props: { } // --------------------------------------------------------------------------- -// ObjectFields — renders an object's properties (required first, then present -// optionals, then "Add" rows for absent optionals) +// ObjectFields — renders an object's properties (required/default-backed first, +// then present optionals, then "Add" rows for absent optionals) // --------------------------------------------------------------------------- function ObjectFields(props: { @@ -242,11 +248,15 @@ function ObjectFields(props: { const requiredSet = new Set(schema.required ?? []); const entries = Object.entries(properties); - const present = entries.filter( - ([key]: [string, JsonSchema]) => key in value || requiredSet.has(key), - ); + const present = entries.filter(([key, propSchema]: [string, JsonSchema]) => { + const resolved = deepResolve(propSchema, root); + return key in value || requiredSet.has(key) || resolved.default !== undefined; + }); const addable = entries.filter( - ([key]: [string, JsonSchema]) => !requiredSet.has(key) && !(key in value), + ([key, propSchema]: [string, JsonSchema]) => + !requiredSet.has(key) && + !(key in value) && + deepResolve(propSchema, root).default === undefined, ); // No properties → render nothing (no "takes no arguments" noise); a zero-arg @@ -316,6 +326,7 @@ function JsonSchemaField(props: { const { root, name, fieldId, required, value, present, onChange, depth, disabled } = props; const schema = deepResolve(props.schema, root); const description = schemaDescription(schema); + const fieldValue = value === undefined && schema.default !== undefined ? schema.default : value; const header = (
@@ -353,7 +364,7 @@ function JsonSchemaField(props: { schema={schema} root={root} fieldId={fieldId} - value={value} + value={fieldValue} onChange={onChange} depth={depth} disabled={disabled} @@ -407,6 +418,18 @@ function FieldControl(props: { ); } + if (Array.isArray(schema.enum) && schema.enum.length === 0) { + const guidance = emptyEnumGuidance(schemaDescription(schema)); + return ( +
+ + {guidance} + +

{guidance}

+
+ ); + } + // enum — native select. Optional/unset gets a leading empty option. if (Array.isArray(schema.enum) && schema.enum.length > 0) { const options = schema.enum; diff --git a/packages/react/src/lib/integration-detail-tabs.test.ts b/packages/react/src/lib/integration-detail-tabs.test.ts new file mode 100644 index 000000000..09096181b --- /dev/null +++ b/packages/react/src/lib/integration-detail-tabs.test.ts @@ -0,0 +1,23 @@ +import { describe, expect, it } from "@effect/vitest"; + +import { + integrationDetailInternalTabFromSearch, + integrationDetailSearchTabForInternal, + integrationDetailTabForAddCompletion, +} from "./integration-detail-tabs"; + +describe("integration detail tab routing", () => { + it("lands custom-tools add completion on the same source tab a reload reads", () => { + const addCompletionTab = integrationDetailTabForAddCompletion("apps"); + + expect(addCompletionTab).toBe("source"); + expect(integrationDetailInternalTabFromSearch(addCompletionTab)).toBe("accounts"); + expect(integrationDetailSearchTabForInternal("apps", "accounts")).toBe(addCompletionTab); + }); + + it("keeps non-app integrations on the accounts/tools tab vocabulary", () => { + expect(integrationDetailTabForAddCompletion("openapi")).toBeUndefined(); + expect(integrationDetailSearchTabForInternal("openapi", "accounts")).toBe("accounts"); + expect(integrationDetailSearchTabForInternal("openapi", "tools")).toBe("tools"); + }); +}); diff --git a/packages/react/src/lib/integration-detail-tabs.ts b/packages/react/src/lib/integration-detail-tabs.ts new file mode 100644 index 000000000..90495205d --- /dev/null +++ b/packages/react/src/lib/integration-detail-tabs.ts @@ -0,0 +1,16 @@ +export type IntegrationDetailSearchTab = "accounts" | "source" | "tools"; +export type IntegrationDetailInternalTab = "accounts" | "tools"; + +export const integrationDetailTabForAddCompletion = ( + pluginKey: string, +): IntegrationDetailSearchTab | undefined => (pluginKey === "apps" ? "source" : undefined); + +export const integrationDetailInternalTabFromSearch = ( + tab: IntegrationDetailSearchTab | undefined, +): IntegrationDetailInternalTab => (tab === "tools" ? "tools" : "accounts"); + +export const integrationDetailSearchTabForInternal = ( + integrationKind: string | undefined, + tab: IntegrationDetailInternalTab, +): IntegrationDetailSearchTab => + tab === "tools" ? "tools" : integrationKind === "apps" ? "source" : "accounts"; diff --git a/packages/react/src/pages/integration-add.tsx b/packages/react/src/pages/integration-add.tsx index df1faf752..1c6f74270 100644 --- a/packages/react/src/pages/integration-add.tsx +++ b/packages/react/src/pages/integration-add.tsx @@ -1,8 +1,11 @@ import { Suspense } from "react"; +import { useAtomRefresh } from "@effect/atom-react"; import { Link, useNavigate } from "@tanstack/react-router"; import { useIntegrationPlugins } from "@executor-js/sdk/client"; +import { integrationsOptimisticAtom } from "../api/atoms"; import { trackEvent } from "../api/analytics"; import { useExecutorDocumentTitle } from "../lib/document-title"; +import { integrationDetailTabForAddCompletion } from "../lib/integration-detail-tabs"; // --------------------------------------------------------------------------- // Page @@ -18,6 +21,7 @@ export function AddIntegrationPage(props: { const { pluginKey, url, preset, namespace } = props; const navigate = useNavigate(); const integrationPlugins = useIntegrationPlugins(); + const refreshIntegrations = useAtomRefresh(integrationsOptimisticAtom); const plugin = integrationPlugins.find((p) => p.key === pluginKey); @@ -59,9 +63,15 @@ export function AddIntegrationPage(props: { plugin_key: pluginKey, ...(slug ? { integration_slug: slug } : {}), }); + refreshIntegrations(); + const tab = integrationDetailTabForAddCompletion(pluginKey); void navigate( slug - ? { to: "/{-$orgSlug}/integrations/$namespace", params: { namespace: slug } } + ? { + to: "/{-$orgSlug}/integrations/$namespace", + params: { namespace: slug }, + search: tab ? { tab } : {}, + } : { to: "/{-$orgSlug}" }, ); }} diff --git a/packages/react/src/pages/integration-detail.tsx b/packages/react/src/pages/integration-detail.tsx index 7462543c0..dd35ba380 100644 --- a/packages/react/src/pages/integration-detail.tsx +++ b/packages/react/src/pages/integration-detail.tsx @@ -36,6 +36,12 @@ import { Skeleton } from "../components/skeleton"; import { useExecutorDocumentTitle } from "../lib/document-title"; import { ErrorState } from "../components/error-state"; import { isAsyncResultLoading } from "../lib/async-result"; +import { + integrationDetailInternalTabFromSearch, + integrationDetailSearchTabForInternal, + type IntegrationDetailInternalTab, + type IntegrationDetailSearchTab, +} from "../lib/integration-detail-tabs"; // v2: the route's `namespace` param is the integration slug. Tools belong to // the integration's per-owner connections; a tool's policy id is @@ -52,7 +58,10 @@ type ToolRow = { readonly static?: boolean; }; -export function IntegrationDetailPage(props: { namespace: string }) { +export function IntegrationDetailPage(props: { + namespace: string; + tab?: IntegrationDetailSearchTab; +}) { const { namespace } = props; const slug = IntegrationSlug.make(namespace); const integrationPlugins = useIntegrationPlugins(); @@ -89,7 +98,9 @@ export function IntegrationDetailPage(props: { namespace: string }) { const [deleting, setDeleting] = useState(false); const [refreshing, setRefreshing] = useState(false); const [editSheetOpen, setEditSheetOpen] = useState(false); - const [activeTab, setActiveTab] = useState<"accounts" | "tools">("accounts"); + const [activeTab, setActiveTab] = useState(() => + integrationDetailInternalTabFromSearch(props.tab), + ); const [manualAccountHandoff, setManualAccountHandoff] = useState(null); const [locationSearch] = useState(() => @@ -101,9 +112,14 @@ export function IntegrationDetailPage(props: { namespace: string }) { setEditSheetOpen(false); }, [namespace]); + useEffect(() => { + setActiveTab(integrationDetailInternalTabFromSearch(props.tab)); + }, [namespace, props.tab]); + const integrationData = AsyncResult.isSuccess(integration) ? integration.value : null; useExecutorDocumentTitle(integrationData?.name || namespace); const isBuiltInIntegration = namespace === "executor" || integrationData?.kind === "built-in"; + const isAppsIntegration = integrationData?.kind === "apps"; const currentTab = isBuiltInIntegration ? "tools" : activeTab; const canRefresh = integrationData?.canRefresh ?? false; const canRemove = integrationData?.canRemove ?? false; @@ -153,6 +169,9 @@ export function IntegrationDetailPage(props: { namespace: string }) { // Find the plugin edit component based on integration kind const editPlugin = useMemo(() => { if (!integrationData) return null; + if (integrationData.kind === "apps") { + return integrationPlugins.find((p) => p.key === "apps") ?? null; + } return integrationPlugins.find((p) => p.key === integrationData.kind) ?? null; }, [integrationData, integrationPlugins]); @@ -347,6 +366,18 @@ export function IntegrationDetailPage(props: { namespace: string }) { setManualAccountHandoff({ key: `manual:${String(slug)}:${Date.now()}` }); }; + const handleTabChange = (value: string) => { + const nextTab = value === "tools" ? "tools" : "accounts"; + setActiveTab(nextTab); + void navigate({ + to: "/{-$orgSlug}/integrations/$namespace", + params: { namespace }, + search: { + tab: integrationDetailSearchTabForInternal(integrationData?.kind, nextTab), + }, + }); + }; + return (
{/* Header bar */} @@ -363,7 +394,7 @@ export function IntegrationDetailPage(props: { namespace: string }) {
- {!confirmDelete && !isBuiltInIntegration && integrationData && ( + {!confirmDelete && !isBuiltInIntegration && !isAppsIntegration && integrationData && ( @@ -381,6 +412,7 @@ export function IntegrationDetailPage(props: { namespace: string }) { )} {canRemove && + !isAppsIntegration && (confirmDelete ? (