From 363d8f808b8f76f13616bb9c2e5d5e6b75116546 Mon Sep 17 00:00:00 2001 From: Rhys Sullivan <39114868+RhysSullivan@users.noreply.github.com> Date: Mon, 6 Jul 2026 00:44:39 -0700 Subject: [PATCH 01/21] Carve custom tools apps core --- bun.lock | 26 ++ packages/plugins/apps/package.json | 51 +++ packages/plugins/apps/src/api.ts | 17 + .../backing/git-artifact-store.cas.test.ts | 68 ++++ .../src/backing/git-artifact-store.test.ts | 10 + .../apps/src/backing/git-artifact-store.ts | 305 ++++++++++++++++++ .../apps/src/backing/libsql-scope-db.test.ts | 4 + .../apps/src/backing/libsql-scope-db.ts | 186 +++++++++++ .../quickjs-tool-sandbox.conformance.test.ts | 7 + .../src/backing/quickjs-tool-sandbox.test.ts | 125 +++++++ .../apps/src/backing/quickjs-tool-sandbox.ts | 252 +++++++++++++++ .../apps/src/backing/scope-collision.test.ts | 62 ++++ .../apps/src/backing/sqlite-apps-store.ts | 98 ++++++ packages/plugins/apps/src/index.ts | 45 +++ packages/plugins/apps/src/pipeline/bundle.ts | 214 ++++++++++++ .../plugins/apps/src/pipeline/descriptor.ts | 101 ++++++ .../plugins/apps/src/pipeline/discover.ts | 92 ++++++ .../plugins/apps/src/pipeline/publish.test.ts | 124 +++++++ packages/plugins/apps/src/pipeline/publish.ts | 259 +++++++++++++++ .../plugins/apps/src/plugin/apps-plugin.ts | 234 ++++++++++++++ .../apps/src/plugin/bindings.strict.test.ts | 80 +++++ packages/plugins/apps/src/plugin/bindings.ts | 223 +++++++++++++ .../src/plugin/publish-concurrency.test.ts | 82 +++++ .../plugins/apps/src/plugin/runtime.test.ts | 74 +++++ packages/plugins/apps/src/plugin/runtime.ts | 295 +++++++++++++++++ .../apps/src/plugin/self-host-runtime.ts | 55 ++++ packages/plugins/apps/src/plugin/store.ts | 78 +++++ .../src/seams/artifact-store.conformance.ts | 91 ++++++ .../plugins/apps/src/seams/artifact-store.ts | 64 ++++ packages/plugins/apps/src/seams/index.ts | 6 + .../apps/src/seams/scope-db.conformance.ts | 69 ++++ packages/plugins/apps/src/seams/scope-db.ts | 42 +++ .../src/seams/tool-sandbox.conformance.ts | 216 +++++++++++++ .../plugins/apps/src/seams/tool-sandbox.ts | 105 ++++++ .../plugins/apps/src/testing/daily-brief.ts | 100 ++++++ packages/plugins/apps/src/testing/index.ts | 100 ++++++ packages/plugins/apps/tsconfig.json | 23 ++ packages/plugins/apps/tsup.config.ts | 15 + packages/plugins/apps/vitest.config.ts | 9 + 39 files changed, 4007 insertions(+) create mode 100644 packages/plugins/apps/package.json create mode 100644 packages/plugins/apps/src/api.ts create mode 100644 packages/plugins/apps/src/backing/git-artifact-store.cas.test.ts create mode 100644 packages/plugins/apps/src/backing/git-artifact-store.test.ts create mode 100644 packages/plugins/apps/src/backing/git-artifact-store.ts create mode 100644 packages/plugins/apps/src/backing/libsql-scope-db.test.ts create mode 100644 packages/plugins/apps/src/backing/libsql-scope-db.ts create mode 100644 packages/plugins/apps/src/backing/quickjs-tool-sandbox.conformance.test.ts create mode 100644 packages/plugins/apps/src/backing/quickjs-tool-sandbox.test.ts create mode 100644 packages/plugins/apps/src/backing/quickjs-tool-sandbox.ts create mode 100644 packages/plugins/apps/src/backing/scope-collision.test.ts create mode 100644 packages/plugins/apps/src/backing/sqlite-apps-store.ts create mode 100644 packages/plugins/apps/src/index.ts create mode 100644 packages/plugins/apps/src/pipeline/bundle.ts create mode 100644 packages/plugins/apps/src/pipeline/descriptor.ts create mode 100644 packages/plugins/apps/src/pipeline/discover.ts create mode 100644 packages/plugins/apps/src/pipeline/publish.test.ts create mode 100644 packages/plugins/apps/src/pipeline/publish.ts create mode 100644 packages/plugins/apps/src/plugin/apps-plugin.ts create mode 100644 packages/plugins/apps/src/plugin/bindings.strict.test.ts create mode 100644 packages/plugins/apps/src/plugin/bindings.ts create mode 100644 packages/plugins/apps/src/plugin/publish-concurrency.test.ts create mode 100644 packages/plugins/apps/src/plugin/runtime.test.ts create mode 100644 packages/plugins/apps/src/plugin/runtime.ts create mode 100644 packages/plugins/apps/src/plugin/self-host-runtime.ts create mode 100644 packages/plugins/apps/src/plugin/store.ts create mode 100644 packages/plugins/apps/src/seams/artifact-store.conformance.ts create mode 100644 packages/plugins/apps/src/seams/artifact-store.ts create mode 100644 packages/plugins/apps/src/seams/index.ts create mode 100644 packages/plugins/apps/src/seams/scope-db.conformance.ts create mode 100644 packages/plugins/apps/src/seams/scope-db.ts create mode 100644 packages/plugins/apps/src/seams/tool-sandbox.conformance.ts create mode 100644 packages/plugins/apps/src/seams/tool-sandbox.ts create mode 100644 packages/plugins/apps/src/testing/daily-brief.ts create mode 100644 packages/plugins/apps/src/testing/index.ts create mode 100644 packages/plugins/apps/tsconfig.json create mode 100644 packages/plugins/apps/tsup.config.ts create mode 100644 packages/plugins/apps/vitest.config.ts diff --git a/bun.lock b/bun.lock index e1f7250b2..316bff560 100644 --- a/bun.lock +++ b/bun.lock @@ -781,6 +781,30 @@ "effect": "catalog:", }, }, + "packages/plugins/apps": { + "name": "@executor-js/plugin-apps", + "version": "0.1.0", + "dependencies": { + "@executor-js/codemode-core": "workspace:*", + "@executor-js/runtime-quickjs": "workspace:*", + "@executor-js/sdk": "workspace:*", + "@libsql/client": "catalog:", + "@standard-schema/spec": "^1.0.0", + "esbuild": "^0.27.3", + "zod": "4.3.6", + }, + "devDependencies": { + "@effect/vitest": "catalog:", + "@types/node": "catalog:", + "bun-types": "catalog:", + "effect": "catalog:", + "tsup": "catalog:", + "vitest": "catalog:", + }, + "peerDependencies": { + "effect": "catalog:", + }, + }, "packages/plugins/desktop-settings": { "name": "@executor-js/plugin-desktop-settings", "version": "1.5.28", @@ -1776,6 +1800,8 @@ "@executor-js/motel": ["@executor-js/motel@0.2.5-executor.1", "", { "dependencies": { "@effect/atom-react": "^4.0.0-beta.49", "@effect/opentelemetry": "^4.0.0-beta.49", "@effect/platform-bun": "^4.0.0-beta.50", "@opentelemetry/api": "^1.9.0", "@opentelemetry/exporter-logs-otlp-http": "^0.214.0", "@opentelemetry/exporter-trace-otlp-http": "^0.211.0", "@opentelemetry/sdk-logs": "^0.214.0", "@opentelemetry/sdk-node": "^0.214.0", "@opentelemetry/sdk-trace-base": "^2.5.0", "@opentelemetry/sdk-trace-node": "^2.6.1", "@opentui/core": "^0.1.99", "@opentui/react": "^0.1.99", "effect": "^4.0.0-beta.49", "react": "^19.2.5", "scheduler": "^0.27.0" }, "bin": { "motel": "src/motel.ts", "motel-mcp": "src/mcp.ts" } }, "sha512-fLGJ5FIIBYd+4L2OurpYFjWzvcCbACvmdNofh6THXQJE1Gku93EKURtgn27/XddyM9SKGST6/znTRcmwKiYdXw=="], + "@executor-js/plugin-apps": ["@executor-js/plugin-apps@workspace:packages/plugins/apps"], + "@executor-js/plugin-desktop-settings": ["@executor-js/plugin-desktop-settings@workspace:packages/plugins/desktop-settings"], "@executor-js/plugin-encrypted-secrets": ["@executor-js/plugin-encrypted-secrets@workspace:packages/plugins/encrypted-secrets"], diff --git a/packages/plugins/apps/package.json b/packages/plugins/apps/package.json new file mode 100644 index 000000000..a116cea63 --- /dev/null +++ b/packages/plugins/apps/package.json @@ -0,0 +1,51 @@ +{ + "name": "@executor-js/plugin-apps", + "version": "0.1.0", + "homepage": "https://github.com/UsefulSoftwareCo/executor/tree/main/packages/plugins/apps", + "bugs": { + "url": "https://github.com/UsefulSoftwareCo/executor/issues" + }, + "license": "MIT", + "repository": { + "type": "git", + "url": "git+https://github.com/UsefulSoftwareCo/executor.git", + "directory": "packages/plugins/apps" + }, + "files": [ + "dist" + ], + "type": "module", + "exports": { + ".": "./src/index.ts", + "./api": "./src/api.ts", + "./seams": "./src/seams/index.ts", + "./testing": "./src/testing/index.ts" + }, + "scripts": { + "build": "tsup && (tsc --declaration --emitDeclarationOnly --outDir dist --rootDir src || true)", + "typecheck": "tsgo --noEmit", + "test": "vitest run", + "test:watch": "vitest", + "typecheck:slow": "bunx tsc --noEmit -p tsconfig.json" + }, + "dependencies": { + "@executor-js/codemode-core": "workspace:*", + "@executor-js/runtime-quickjs": "workspace:*", + "@executor-js/sdk": "workspace:*", + "@libsql/client": "catalog:", + "@standard-schema/spec": "^1.0.0", + "esbuild": "^0.27.3", + "zod": "4.3.6" + }, + "devDependencies": { + "@effect/vitest": "catalog:", + "@types/node": "catalog:", + "bun-types": "catalog:", + "effect": "catalog:", + "tsup": "catalog:", + "vitest": "catalog:" + }, + "peerDependencies": { + "effect": "catalog:" + } +} diff --git a/packages/plugins/apps/src/api.ts b/packages/plugins/apps/src/api.ts new file mode 100644 index 000000000..74cba1a18 --- /dev/null +++ b/packages/plugins/apps/src/api.ts @@ -0,0 +1,17 @@ +// Host-facing plugin surface for @executor-js/plugin-apps. +export { + appsPlugin, + APPS_INTEGRATION_SLUG, + APPS_PLUGIN_ID, + APPS_CONNECTION_PREFIX, + connectionNameForScope, + scopeFromConnection, + type AppsPluginOptions, +} from "./plugin/apps-plugin"; +export { + makeSelfHostAppsRuntime, + type SelfHostAppsRuntime, + type SelfHostAppsRuntimeOptions, +} from "./plugin/self-host-runtime"; +export { makeSqliteAppsStore } from "./backing/sqlite-apps-store"; +export { BindingError, type ClientResolver, type Bindings } from "./plugin/bindings"; diff --git a/packages/plugins/apps/src/backing/git-artifact-store.cas.test.ts b/packages/plugins/apps/src/backing/git-artifact-store.cas.test.ts new file mode 100644 index 000000000..a1438a8d2 --- /dev/null +++ b/packages/plugins/apps/src/backing/git-artifact-store.cas.test.ts @@ -0,0 +1,68 @@ +import { mkdtempSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +import { describe, expect, it } from "vitest"; +import { Effect } from "effect"; + +import { makeGitArtifactStore } from "./git-artifact-store"; +import type { ArtifactStoreError } from "../seams/artifact-store"; + +const run = (effect: Effect.Effect): Promise => Effect.runPromise(effect); + +// --------------------------------------------------------------------------- +// Finding 6 regression (git compare-and-swap): two commits built from the SAME +// parent cannot both advance the branch. The ref update is a CAS against the +// expected old value, so the second raced writer gets a typed conflict instead +// of silently clobbering the first. This is deterministic (no timing race): we +// force both to share a parent by committing them against a store whose head we +// pin. +// --------------------------------------------------------------------------- + +describe("git ArtifactStore ref CAS (Fix 6)", () => { + it("two concurrent commits from one parent: one wins, the other typed-conflicts", async () => { + const root = mkdtempSync(join(tmpdir(), "apps-cas-")); + const storeA = makeGitArtifactStore({ root }); + const scopeA = await run(storeA.forScope("s")); + + // Seed a parent so racers commit ON TOP of the same head. + await run(scopeA.commit(new Map([["tools/base.ts", "// base"]]), "base")); + + // Many SEPARATE store instances over the SAME repo dir. The repo already + // exists (seeded above), so opening each scope is a re-`init --bare` (a no-op + // on an existing repo). Open them SEQUENTIALLY so the idempotent init doesn't + // itself race, then race only the COMMITS (the ref-CAS is what's under test). + const N = 12; + const openedScopes: { commit: typeof scopeA.commit; log: typeof scopeA.log }[] = []; + for (let i = 0; i < N; i++) { + const store = makeGitArtifactStore({ root }); + // eslint-disable-next-line no-await-in-loop + openedScopes.push(await run(store.forScope("s"))); + } + const scopeB = openedScopes[0]; + + // Fire many concurrent commits, ALL built against the seeded base parent. + const results = await Promise.allSettled( + openedScopes.map((sc, i) => + run(sc.commit(new Map([[`tools/c${i}.ts`, `// ${i}`]]), `c${i}`)), + ), + ); + + const fulfilled = results.filter((r) => r.status === "fulfilled").length; + const rejected = results.filter((r) => r.status === "rejected"); + + // Any failure MUST be a typed conflict (never a silent clobber). + for (const r of rejected) { + const err = (r as PromiseRejectedResult).reason as ArtifactStoreError; + expect(String(JSON.stringify(err))).toMatch(/conflict/i); + } + + // No lost commits: the git log length equals base(1) + every commit that + // reported success. Without the CAS, a raced writer clobbers the ref and an + // earlier successful commit vanishes from history, making the log SHORTER + // than base + fulfilled. The CAS guarantees every reported success is a real, + // reachable, non-clobbering advance. + const logEntries = await run(scopeB.log(1000)); + expect(logEntries.length).toBe(1 + fulfilled); + }, 60_000); +}); diff --git a/packages/plugins/apps/src/backing/git-artifact-store.test.ts b/packages/plugins/apps/src/backing/git-artifact-store.test.ts new file mode 100644 index 000000000..e1a434fa4 --- /dev/null +++ b/packages/plugins/apps/src/backing/git-artifact-store.test.ts @@ -0,0 +1,10 @@ +import { mkdtempSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +import { artifactStoreConformance } from "../seams/artifact-store.conformance"; +import { makeGitArtifactStore } from "./git-artifact-store"; + +artifactStoreConformance("git", () => + makeGitArtifactStore({ root: mkdtempSync(join(tmpdir(), "apps-artifacts-")) }), +); diff --git a/packages/plugins/apps/src/backing/git-artifact-store.ts b/packages/plugins/apps/src/backing/git-artifact-store.ts new file mode 100644 index 000000000..6d231f029 --- /dev/null +++ b/packages/plugins/apps/src/backing/git-artifact-store.ts @@ -0,0 +1,305 @@ +import { execFile } from "node:child_process"; +import { mkdir } from "node:fs/promises"; +import { join } from "node:path"; + +import { Effect } from "effect"; + +import { + ArtifactStoreError, + asSnapshotId, + type ArtifactStore, + type FileSet, + type ScopeArtifactStore, + type SnapshotId, + type SnapshotMeta, +} from "../seams/artifact-store"; + +// --------------------------------------------------------------------------- +// Git-backed ArtifactStore (self-hosted). One bare git repo per scope under +// `/.git`. Snapshots are commits, written via git plumbing +// (hash-object -> update-index -> write-tree -> commit-tree) so no working tree +// is ever checked out — the runtime only reads committed snapshots. The commit +// hash IS the SnapshotId; git guarantees immutability (content-addressed). +// --------------------------------------------------------------------------- + +const BRANCH = "refs/heads/main"; + +// The "must not exist" sentinel for `git update-ref `: an empty +// old-value asserts the ref currently has NO value (the first publish to a fresh +// scope repo). git treats the empty string as "the ref must not already exist". +const EMPTY_OID = ""; + +const run = ( + cwd: string, + args: readonly string[], + input?: string | Buffer, +): Effect.Effect => + Effect.callback((resume) => { + const child = execFile( + "git", + args, + { cwd, maxBuffer: 256 * 1024 * 1024, encoding: "buffer" }, + (error, stdout, stderr) => { + if (error) { + resume( + Effect.fail( + new ArtifactStoreError({ + message: `git ${args[0]} failed: ${(stderr as Buffer)?.toString() || error.message}`, + cause: error, + }), + ), + ); + return; + } + resume(Effect.succeed((stdout as Buffer).toString("utf8"))); + }, + ); + if (input !== undefined) { + child.stdin?.write(input); + child.stdin?.end(); + } + }); + +const sanitizeScope = (scope: string): string => { + if (!/^[a-zA-Z0-9._-]+$/.test(scope)) { + // Scopes are internal identifiers; refuse anything that could escape a path. + throw new ArtifactStoreError({ message: `invalid scope key: ${scope}` }); + } + return scope; +}; + +const makeScopeStore = (repoDir: string): ScopeArtifactStore => { + // Environment forcing a deterministic author/committer so a snapshot's hash + // depends only on its content + parent, not on wall-clock/identity drift. + const commitEnv = (message: string) => + Effect.gen(function* () { + const parent = yield* headCommit(); + // Build a tree from the provided file set held in a bare index file. + return { parent, message }; + }); + + const headCommit = (): Effect.Effect => + run(repoDir, ["rev-parse", "--verify", "--quiet", BRANCH]).pipe( + Effect.map((out) => out.trim() || null), + Effect.catch(() => Effect.succeed(null)), + ); + + const readMeta = (id: string): Effect.Effect => + run(repoDir, ["show", "-s", "--format=%H%n%ct%n%s", id]).pipe( + Effect.map((out) => { + const [hash, committed, ...subjectParts] = out.split("\n"); + return { + id: asSnapshotId(hash.trim()), + committedAt: Number(committed.trim()) * 1000, + message: subjectParts.join("\n").trim(), + } satisfies SnapshotMeta; + }), + ); + + return { + commit: (files: FileSet, message: string) => + Effect.gen(function* () { + const { parent } = yield* commitEnv(message); + // Use a throwaway index so we never touch a working tree. + const indexFile = join( + repoDir, + `commit-index-${Date.now()}-${Math.random().toString(36).slice(2)}`, + ); + const withIndex = (args: readonly string[], input?: string | Buffer) => + Effect.callback((resume) => { + const child = execFile( + "git", + args, + { + cwd: repoDir, + maxBuffer: 256 * 1024 * 1024, + encoding: "buffer", + env: { ...process.env, GIT_INDEX_FILE: indexFile }, + }, + (error, stdout, stderr) => { + if (error) { + resume( + Effect.fail( + new ArtifactStoreError({ + message: `git ${args[0]} failed: ${(stderr as Buffer)?.toString() || error.message}`, + cause: error, + }), + ), + ); + return; + } + resume(Effect.succeed((stdout as Buffer).toString("utf8"))); + }, + ); + if (input !== undefined) { + child.stdin?.write(input); + child.stdin?.end(); + } + }); + + // Start from an empty index each publish (full file set, not a diff). + yield* withIndex(["read-tree", "--empty"]); + for (const [path, contents] of files) { + const blobHash = (yield* withIndex( + ["hash-object", "-w", "--stdin"], + Buffer.from(contents, "utf8"), + )).trim(); + yield* withIndex(["update-index", "--add", "--cacheinfo", `100644,${blobHash},${path}`]); + } + const treeHash = (yield* withIndex(["write-tree"])).trim(); + const commitArgs = ["commit-tree", treeHash, "-m", message]; + if (parent) commitArgs.push("-p", parent); + const commitEnvVars = { + ...process.env, + GIT_AUTHOR_NAME: "executor-apps", + GIT_AUTHOR_EMAIL: "apps@executor.local", + GIT_COMMITTER_NAME: "executor-apps", + GIT_COMMITTER_EMAIL: "apps@executor.local", + }; + const commitHash = (yield* Effect.callback((resume) => { + execFile( + "git", + commitArgs, + { cwd: repoDir, encoding: "buffer", env: commitEnvVars }, + (error, stdout, stderr) => { + if (error) { + resume( + Effect.fail( + new ArtifactStoreError({ + message: `git commit-tree failed: ${(stderr as Buffer)?.toString() || error.message}`, + cause: error, + }), + ), + ); + return; + } + resume(Effect.succeed((stdout as Buffer).toString("utf8"))); + }, + ); + })).trim(); + // Compare-and-swap the branch ref: `update-ref ` fails + // if HEAD moved since we read `parent`, so a concurrent publish that + // committed first cannot be silently clobbered. On the first commit there + // is no parent, so we assert the ref is absent (the empty-oid form). A CAS + // failure surfaces as a typed conflict for the caller to retry from a + // fresh parent. + const expectedOld = parent ?? EMPTY_OID; + yield* run(repoDir, ["update-ref", BRANCH, commitHash, expectedOld]).pipe( + Effect.mapError( + (cause) => + new ArtifactStoreError({ + message: `publish conflict: ${BRANCH} moved concurrently (expected ${expectedOld}); retry from the new head`, + conflict: true, + cause, + }), + ), + ); + return yield* readMeta(commitHash); + }), + + read: (id: SnapshotId) => + run(repoDir, ["ls-tree", "-r", "--name-only", id]).pipe( + Effect.flatMap((out) => { + const paths = out + .split("\n") + .map((p) => p.trim()) + .filter(Boolean); + return Effect.forEach( + paths, + (path) => + run(repoDir, ["cat-file", "blob", `${id}:${path}`]).pipe( + Effect.map((contents) => [path, contents] as const), + ), + { concurrency: 8 }, + ); + }), + Effect.map((entries) => new Map(entries) as FileSet), + ), + + readFile: (id: SnapshotId, path: string) => + run(repoDir, ["cat-file", "blob", `${id}:${path}`]).pipe( + Effect.map((contents) => contents as string | null), + Effect.catch(() => Effect.succeed(null)), + ), + + list: (id: SnapshotId) => + run(repoDir, ["ls-tree", "-r", "--name-only", id]).pipe( + Effect.map((out) => + out + .split("\n") + .map((p) => p.trim()) + .filter(Boolean), + ), + ), + + latest: () => + headCommit().pipe(Effect.flatMap((head) => (head ? readMeta(head) : Effect.succeed(null)))), + + log: (limit = 50) => + headCommit().pipe( + Effect.flatMap((head) => { + if (!head) return Effect.succeed([] as readonly SnapshotMeta[]); + return run(repoDir, [ + "log", + `--max-count=${limit}`, + "--format=%H%x1f%ct%x1f%s", + BRANCH, + ]).pipe( + Effect.map((out) => + out + .split("\n") + .map((line) => line.trim()) + .filter(Boolean) + .map((line) => { + const [hash, committed, subject] = line.split("\x1f"); + return { + id: asSnapshotId(hash), + committedAt: Number(committed) * 1000, + message: subject ?? "", + } satisfies SnapshotMeta; + }), + ), + ); + }), + ), + }; +}; + +export interface GitArtifactStoreOptions { + /** Directory holding one bare repo per scope. Created on demand. */ + readonly root: string; +} + +/** Build the git-backed ArtifactStore. Each scope gets a lazily-initialized + * bare repo `/.git`. */ +export const makeGitArtifactStore = (options: GitArtifactStoreOptions): ArtifactStore => { + const initialized = new Map>(); + + const init = async (scope: string): Promise => { + const safe = sanitizeScope(scope); + const repoDir = join(options.root, `${safe}.git`); + await mkdir(repoDir, { recursive: true }); + await new Promise((resolve, reject) => { + execFile("git", ["init", "--bare", "--quiet"], { cwd: repoDir }, (error) => + error ? reject(error) : resolve(), + ); + }); + return makeScopeStore(repoDir); + }; + + return { + forScope: (scope) => + Effect.tryPromise({ + try: () => { + let existing = initialized.get(scope); + if (!existing) { + existing = init(scope); + initialized.set(scope, existing); + } + return existing; + }, + catch: (cause) => + new ArtifactStoreError({ message: `failed to open scope repo: ${scope}`, cause }), + }), + }; +}; diff --git a/packages/plugins/apps/src/backing/libsql-scope-db.test.ts b/packages/plugins/apps/src/backing/libsql-scope-db.test.ts new file mode 100644 index 000000000..4ebbebd3a --- /dev/null +++ b/packages/plugins/apps/src/backing/libsql-scope-db.test.ts @@ -0,0 +1,4 @@ +import { scopeDbConformance } from "../seams/scope-db.conformance"; +import { makeLibsqlScopeDb } from "./libsql-scope-db"; + +scopeDbConformance("libsql (in-memory)", () => makeLibsqlScopeDb({ root: ":memory:" })); diff --git a/packages/plugins/apps/src/backing/libsql-scope-db.ts b/packages/plugins/apps/src/backing/libsql-scope-db.ts new file mode 100644 index 000000000..3f8a5e5b6 --- /dev/null +++ b/packages/plugins/apps/src/backing/libsql-scope-db.ts @@ -0,0 +1,186 @@ +import { mkdirSync } from "node:fs"; +import { join, resolve } from "node:path"; + +import { createClient, type Client } from "@libsql/client"; +import { Effect } from "effect"; + +import { ScopeDbError, type ScopeDb, type ScopeDbHandle } from "../seams/scope-db"; + +// --------------------------------------------------------------------------- +// libSQL-backed ScopeDb (self-hosted). One SQLite file per scope under +// `/.db`. A control table `__versions` holds a monotonic counter +// per user table; every write statement bumps the counters for the tables it +// touched. Scope isolation is a separate file per scope, there is no +// cross-scope query path. +// --------------------------------------------------------------------------- + +const VERSION_TABLE = "__scope_versions"; + +// Statements that mutate data. Determines whether a `sql` call bumps versions. +const WRITE_RE = /^\s*(insert|update|delete|replace|create|drop|alter)\b/i; + +// Extract the table names a write statement targets (best-effort: covers the +// shapes authored tools produce — INSERT INTO x, UPDATE x, DELETE FROM x, +// CREATE TABLE x, REPLACE INTO x). +const targetsOf = (sql: string): string[] => { + const out = new Set(); + const patterns = [ + /\binsert\s+(?:or\s+\w+\s+)?into\s+["'`]?([a-zA-Z_][a-zA-Z0-9_]*)["'`]?/gi, + /\breplace\s+into\s+["'`]?([a-zA-Z_][a-zA-Z0-9_]*)["'`]?/gi, + /\bupdate\s+["'`]?([a-zA-Z_][a-zA-Z0-9_]*)["'`]?/gi, + /\bdelete\s+from\s+["'`]?([a-zA-Z_][a-zA-Z0-9_]*)["'`]?/gi, + /\bcreate\s+table\s+(?:if\s+not\s+exists\s+)?["'`]?([a-zA-Z_][a-zA-Z0-9_]*)["'`]?/gi, + /\bdrop\s+table\s+(?:if\s+exists\s+)?["'`]?([a-zA-Z_][a-zA-Z0-9_]*)["'`]?/gi, + /\balter\s+table\s+["'`]?([a-zA-Z_][a-zA-Z0-9_]*)["'`]?/gi, + ]; + for (const re of patterns) { + let m: RegExpExecArray | null; + while ((m = re.exec(sql)) !== null) { + const name = m[1]; + if (name && name !== VERSION_TABLE) out.add(name); + } + } + return [...out]; +}; + +const toArgs = (values: readonly unknown[]): unknown[] => + values.map((v) => (v === undefined ? null : v)); + +const makeHandle = (scope: string, client: Client): ScopeDbHandle => { + const ensureVersionTable = async () => { + await client.execute( + `CREATE TABLE IF NOT EXISTS ${VERSION_TABLE} (name TEXT PRIMARY KEY, version INTEGER NOT NULL DEFAULT 0)`, + ); + }; + + const bump = async (tables: readonly string[]): Promise<{ table: string; version: number }[]> => { + const bumped: { table: string; version: number }[] = []; + for (const table of tables) { + await client.execute({ + sql: `INSERT INTO ${VERSION_TABLE} (name, version) VALUES (?, 1) + ON CONFLICT(name) DO UPDATE SET version = version + 1`, + args: [table], + }); + const row = await client.execute({ + sql: `SELECT version FROM ${VERSION_TABLE} WHERE name = ?`, + args: [table], + }); + bumped.push({ table, version: Number(row.rows[0]?.version ?? 0) }); + } + return bumped; + }; + + const runStatement = ( + sql: string, + args: unknown[], + ): Effect.Effect => + Effect.tryPromise({ + try: async () => { + await ensureVersionTable(); + const result = await client.execute({ sql, args: args as never }); + if (WRITE_RE.test(sql)) { + const targets = targetsOf(sql); + if (targets.length > 0) { + await bump(targets); + } + } + return result.rows as unknown as readonly Row[]; + }, + catch: (cause) => + new ScopeDbError({ message: `scope-db statement failed for scope ${scope}`, cause }), + }); + + return { + sql: >(strings: TemplateStringsArray, ...values: unknown[]) => { + const sql = strings.reduce((acc, part, i) => acc + part + (i < values.length ? "?" : ""), ""); + return runStatement(sql, toArgs(values)); + }, + exec: >(sql: string, params: readonly unknown[] = []) => + runStatement(sql, toArgs(params)), + tableVersion: (table: string) => + Effect.tryPromise({ + try: async () => { + await ensureVersionTable(); + const row = await client.execute({ + sql: `SELECT version FROM ${VERSION_TABLE} WHERE name = ?`, + args: [table], + }); + return Number(row.rows[0]?.version ?? 0); + }, + catch: (cause) => new ScopeDbError({ message: "tableVersion failed", cause }), + }), + versions: () => + Effect.tryPromise({ + try: async () => { + await ensureVersionTable(); + const rows = await client.execute(`SELECT name, version FROM ${VERSION_TABLE}`); + const out = new Map(); + for (const r of rows.rows) out.set(String(r.name), Number(r.version)); + return out as ReadonlyMap; + }, + catch: (cause) => new ScopeDbError({ message: "versions failed", cause }), + }), + }; +}; + +export interface LibsqlScopeDbOptions { + /** Directory holding one SQLite file per scope, or ":memory:" for tests. */ + readonly root: string; +} + +const toUrl = (path: string): string => (path === ":memory:" ? path : `file:${resolve(path)}`); + +// A reversible, collision-free filename key for a scope (Fix 9). A scope already +// composed only of filename-safe characters is used verbatim (so distinct +// filename-safe scopes stay distinct — "my-scope" and "my_scope" no longer +// collide). Any scope containing a character outside that set is hex-encoded in +// full under an `x-` prefix, which can never collide with a verbatim scope +// (verbatim scopes never start with `x-` followed by only hex... unless they +// literally are `x-`, so we also encode those). Two different scopes always +// produce two different keys. +const SAFE_SCOPE = /^[A-Za-z0-9._-]+$/; +const HEX_PREFIXED = /^x-[0-9a-f]*$/; +const scopeFileKey = (scope: string): string => { + if (SAFE_SCOPE.test(scope) && !HEX_PREFIXED.test(scope)) return scope; + const hex = Buffer.from(scope, "utf8").toString("hex"); + return `x-${hex}`; +}; + +export const makeLibsqlScopeDb = (options: LibsqlScopeDbOptions): ScopeDb => { + const clients = new Map(); + + const clientFor = (scope: string): Client => { + let client = clients.get(scope); + if (!client) { + if (options.root === ":memory:") { + client = createClient({ url: ":memory:" }); + } else { + mkdirSync(options.root, { recursive: true }); + // Collision-free filename (Fix 9): the old `replace(/[^..]/g, "_")` + // COLLAPSED distinct scopes to the same file ("my-scope" and "my_scope" + // -> "my_scope.db"), so two scopes shared one database. Encode the raw + // scope instead: keep the common safe-identifier case readable, and for + // anything with a character outside `[A-Za-z0-9._-]` fall back to a + // reversible hex encoding of the full raw scope. Distinct scopes always + // map to distinct filenames. + const safe = scopeFileKey(scope); + client = createClient({ url: toUrl(join(options.root, `${safe}.db`)) }); + } + clients.set(scope, client); + } + return client; + }; + + return { + forScope: (scope) => + Effect.try({ + try: () => makeHandle(scope, clientFor(scope)), + catch: (cause) => new ScopeDbError({ message: `failed to open scope db ${scope}`, cause }), + }), + close: () => + Effect.sync(() => { + for (const client of clients.values()) client.close(); + clients.clear(); + }), + }; +}; diff --git a/packages/plugins/apps/src/backing/quickjs-tool-sandbox.conformance.test.ts b/packages/plugins/apps/src/backing/quickjs-tool-sandbox.conformance.test.ts new file mode 100644 index 000000000..53e0a9a66 --- /dev/null +++ b/packages/plugins/apps/src/backing/quickjs-tool-sandbox.conformance.test.ts @@ -0,0 +1,7 @@ +import { toolSandboxConformance } from "../seams/tool-sandbox.conformance"; +import { makeQuickjsToolSandbox } from "./quickjs-tool-sandbox"; + +// Short timeout so the "kills a runaway handler" case finishes quickly. +toolSandboxConformance("quickjs", () => + makeQuickjsToolSandbox({ collectTimeoutMs: 5_000, invokeTimeoutMs: 2_000 }), +); diff --git a/packages/plugins/apps/src/backing/quickjs-tool-sandbox.test.ts b/packages/plugins/apps/src/backing/quickjs-tool-sandbox.test.ts new file mode 100644 index 000000000..7e212214f --- /dev/null +++ b/packages/plugins/apps/src/backing/quickjs-tool-sandbox.test.ts @@ -0,0 +1,125 @@ +import { describe, expect, it } from "vitest"; +import { Effect } from "effect"; + +import { bundleEntry } from "../pipeline/bundle"; +import { makeQuickjsToolSandbox } from "./quickjs-tool-sandbox"; +import { ISSUES_SYNC_TS, SEARCH_ALL_MAIL_TS } from "../testing/daily-brief"; +import type { HandleBridge } from "../seams/tool-sandbox"; + +const run = (effect: Effect.Effect): Promise => Effect.runPromise(effect); + +describe("QuickJS ToolSandbox", () => { + const sandbox = makeQuickjsToolSandbox(); + + it("bundles + collects a tool descriptor (deterministic)", async () => { + const files = new Map([["tools/issues-sync.ts", ISSUES_SYNC_TS]]); + const { code } = await run(bundleEntry({ files, entry: "tools/issues-sync.ts" })); + const result = await run(sandbox.collect(code)); + const descriptor = result.artifacts.default.descriptor as { + kind: string; + connections: Record; + inputJsonSchema: { type: string; properties: Record }; + hasHandler: boolean; + }; + expect(descriptor.kind).toBe("tool"); + expect(descriptor.connections.github).toEqual( + expect.objectContaining({ decl: "single", integration: "github" }), + ); + expect(descriptor.hasHandler).toBe(true); + expect(descriptor.inputJsonSchema.type).toBe("object"); + expect(descriptor.inputJsonSchema.properties).toHaveProperty("repos"); + expect(descriptor.inputJsonSchema.properties).toHaveProperty("since"); + }); + + it("collects a fan-out (connections) declaration", async () => { + const files = new Map([["tools/search-all-mail.ts", SEARCH_ALL_MAIL_TS]]); + const { code } = await run(bundleEntry({ files, entry: "tools/search-all-mail.ts" })); + const result = await run(sandbox.collect(code)); + const descriptor = result.artifacts.default.descriptor as { + connections: Record; + }; + expect(descriptor.connections.inboxes).toEqual( + expect.objectContaining({ decl: "array", integration: "gmail" }), + ); + }); + + it("invokes a tool, routing injected-client + db calls through the bridge", async () => { + const files = new Map([["tools/issues-sync.ts", ISSUES_SYNC_TS]]); + const { code } = await run(bundleEntry({ files, entry: "tools/issues-sync.ts" })); + + const calls: { root: string; path: readonly string[]; args: readonly unknown[] }[] = []; + const bridge: HandleBridge = { + call: ({ root, path, args }) => + Effect.sync(() => { + calls.push({ root, path, args }); + const key = `${root}.${path.join(".")}`; + if (key === "github.repos.listForAuthenticatedUser") { + return [{ full_name: "acme/app" }]; + } + if (key === "github.issues.listForRepo") { + return [ + { + number: 1, + title: "Bug", + labels: [{ name: "bug" }], + assignee: { login: "rhys" }, + updated_at: "2026-01-01T00:00:00Z", + html_url: "https://github.com/acme/app/issues/1", + }, + ]; + } + // db.sql calls return empty rows + return []; + }), + }; + + const result = await run( + sandbox.invoke( + code, + { + artifact: "issues-sync", + kind: "tool", + input: {}, + roots: { github: { kind: "single" }, db: { kind: "single" } }, + }, + bridge, + ), + ); + + expect(result.output).toEqual({ synced: 1, repos: 1 }); + // The db write went through the bridge (root "db"). + expect(calls.some((c) => c.root === "db")).toBe(true); + expect(calls.some((c) => c.root === "github")).toBe(true); + }); + + it("rejects a non-deterministic descriptor (Math.random at describe time)", async () => { + const nondeterministic = `import { defineTool } from "executor:app"; +import { z } from "zod"; +export default defineTool({ + description: "x-" + Math.random(), + input: z.object({}), + async handler() { return {}; }, +});`; + const files = new Map([["tools/rng.ts", nondeterministic]]); + const { code } = await run(bundleEntry({ files, entry: "tools/rng.ts" })); + const exit = await Effect.runPromiseExit(sandbox.collect(code)); + expect(exit._tag).toBe("Failure"); + }); + + it("denies network (fetch throws in the sandbox)", async () => { + const usesFetch = `import { defineTool } from "executor:app"; +import { z } from "zod"; +export default defineTool({ + description: "fetches", + input: z.object({}), + async handler() { await fetch("https://example.com"); return {}; }, +});`; + const files = new Map([["tools/net.ts", usesFetch]]); + const { code } = await run(bundleEntry({ files, entry: "tools/net.ts" })); + const bridge: HandleBridge = { call: () => Effect.succeed(null) }; + const exit = await Effect.runPromiseExit( + sandbox.invoke(code, { artifact: "net", kind: "tool", input: {}, roots: {} }, bridge), + ); + expect(exit._tag).toBe("Failure"); + }); +}); diff --git a/packages/plugins/apps/src/backing/quickjs-tool-sandbox.ts b/packages/plugins/apps/src/backing/quickjs-tool-sandbox.ts new file mode 100644 index 000000000..de298f6dc --- /dev/null +++ b/packages/plugins/apps/src/backing/quickjs-tool-sandbox.ts @@ -0,0 +1,252 @@ +import { Effect } from "effect"; +import type { SandboxToolInvoker } from "@executor-js/codemode-core"; +import { makeQuickJsExecutor } from "@executor-js/runtime-quickjs"; + +import { + ToolSandboxError, + type CollectResult, + type HandleBridge, + type InvokeRequest, + type InvokeResult, + type ToolSandbox, +} from "../seams/tool-sandbox"; +import { stableStringify } from "../pipeline/descriptor"; + +// --------------------------------------------------------------------------- +// QuickJS-backed ToolSandbox (self-hosted). +// +// The published bundle is a CJS string. The sandbox body prepends a `require` +// shim providing `executor:app` (defineTool/connection/connections/catalog), +// then executes the bundle so `module.exports.default` is the artifact. A +// driver appended after the bundle either collects the descriptor (nothing +// bound) or runs one handler with injected clients. +// +// Injected clients are Proxies that turn `github.issues.listForRepo(args)` into +// `await __invokeTool("__handle__", { root, path, args })` — the ONE bridge the +// QuickJS runtime already provides (`tools`/`__invokeTool`). Our +// `SandboxToolInvoker` decodes that and forwards to the host `HandleBridge`. +// Everything crossing is JSON (the cloud version is RPC), so the interface +// stays honest. +// +// Determinism: `collect` runs the collection body twice and byte-compares the +// descriptor JSON. Effectful top-levels (Math.random, Date.now) diverge and are +// rejected. QuickJS denies `fetch` and enforces a deadline + memory cap. +// --------------------------------------------------------------------------- + +const COLLECT_TIMEOUT_MS = 10_000; +const INVOKE_TIMEOUT_MS = 30_000; + +// The shim + module system injected before the bundle. Kept as a plain string +// (QuickJS evals a string). `defineTool` records its def so the driver can read +// it back. Clients are built by `__mkHandle`. +const runtimePrelude = ` +var __modules = {}; +var __defs = { tool: null }; +function __recordDefault(mod) { return mod; } +var __handleBridge = function(root, path, args) { + // Route every injected-client method call through the single tool bridge. + return tools.__handle__({ root: root, path: path, args: args }); +}; +function __mkHandle(root, prefix) { + var target = function(){}; + return new Proxy(target, { + get: function(_t, prop) { + if (prop === 'then' || typeof prop === 'symbol') return undefined; + return __mkHandle(root, prefix.concat([String(prop)])); + }, + apply: function(_t, _this, callArgs) { + return __handleBridge(root, prefix, callArgs); + } + }); +} +var __executorApp = { + connection: function(integration, opts) { return { __decl: 'single', integration: integration, description: opts && opts.description }; }, + connections: function(integration, opts) { return { __decl: 'array', integration: integration, description: opts && opts.description }; }, + catalog: function() { return { __decl: 'catalog' }; }, + defineTool: function(def) { __defs.tool = def; return def; }, +}; +function __require(id) { + if (id === 'executor:app') return __executorApp; + throw new Error('module not available in sandbox: ' + id); +} +`; + +// Run the CJS bundle. The bundle's virtual entry sets `globalThis.__artifact` +// (the def object returned by defineTool), `globalThis.__zodToJson` and +// `__zodToJsonOutput` (bound to the author's inlined zod). `require` is our +// shim; `defineTool` also records into `__defs` as a fallback. +const wrapBundle = (bundle: string): string => ` +var module = { exports: {} }; +var exports = module.exports; +var require = __require; +(function(module, exports, require){ +${bundle} +})(module, exports, require); +`; + +// Collect driver: describe the artifact's connections + input/output schema. +// Deterministic JSON only — no handler execution. +const collectDriver = ` +return await (async () => { + var def = __defs.tool || (globalThis.__artifact && (globalThis.__artifact.default || globalThis.__artifact)); + var conns = {}; + if (def && def.connections) { + for (var k in def.connections) { + var c = def.connections[k]; + conns[k] = { decl: c && c.__decl ? c.__decl : 'single', integration: c && c.integration, description: c && c.description }; + } + } + var toJson = (typeof globalThis.__zodToJson === 'function') ? globalThis.__zodToJson : function(){ return undefined; }; + var toJsonOut = (typeof globalThis.__zodToJsonOutput === 'function') ? globalThis.__zodToJsonOutput : function(){ return undefined; }; + return { + kind: 'tool', + description: def && def.description, + connections: conns, + annotations: def && def.annotations, + hasHandler: !!(def && def.handler), + inputJsonSchema: def ? toJson(def.input) : undefined, + outputJsonSchema: def ? toJsonOut(def.output) : undefined, + }; +})() +`; + +const buildCollectCode = (bundle: string): string => + runtimePrelude + wrapBundle(bundle) + collectDriver; + +// Invoke driver: build injected clients from the request roots, then call the +// artifact's handler with (input, injected). Fan-out arrays become arrays of +// element handles. +const buildInvokeDriver = (request: InvokeRequest): string => { + const rootsLiteral = JSON.stringify(request.roots); + const inputLiteral = JSON.stringify(request.input ?? {}); + return ` +return await (async () => { + var def = __defs.tool || (globalThis.__artifact && (globalThis.__artifact.default || globalThis.__artifact)); + if (!def || typeof def.handler !== 'function') throw new Error('artifact has no handler: ${request.artifact}'); + var roots = ${rootsLiteral}; + var injected = {}; + for (var name in roots) { + var spec = roots[name]; + if (spec.kind === 'array') { + var arr = []; + for (var i = 0; i < spec.count; i++) arr.push(__mkHandle(name + '#' + i, [])); + injected[name] = arr; + } else { + injected[name] = __mkHandle(name, []); + } + } + var out = await def.handler(${inputLiteral}, injected); + return out; +})() +`; +}; + +const buildInvokeCode = (bundle: string, request: InvokeRequest): string => + runtimePrelude + wrapBundle(bundle) + buildInvokeDriver(request); + +// A no-op invoker for collect: no handle calls should happen; if they do +// (misbehaving describe path), fail loudly. +const collectInvoker: SandboxToolInvoker = { + invoke: () => Effect.fail(new Error("collect must not make handle calls")) as never, +}; + +export interface QuickjsToolSandboxOptions { + readonly collectTimeoutMs?: number; + readonly invokeTimeoutMs?: number; +} + +export const makeQuickjsToolSandbox = (options: QuickjsToolSandboxOptions = {}): ToolSandbox => { + const collectExecutor = makeQuickJsExecutor({ + timeoutMs: options.collectTimeoutMs ?? COLLECT_TIMEOUT_MS, + }); + const invokeExecutor = makeQuickJsExecutor({ + timeoutMs: options.invokeTimeoutMs ?? INVOKE_TIMEOUT_MS, + }); + + const runCollect = (bundle: string): Effect.Effect => + collectExecutor.execute(buildCollectCode(bundle), collectInvoker).pipe( + Effect.mapError( + (cause) => new ToolSandboxError({ kind: "collect", message: "collect run failed", cause }), + ), + Effect.flatMap((result) => { + if (result.error) { + return Effect.fail(new ToolSandboxError({ kind: "collect", message: result.error })); + } + return Effect.succeed(result.result); + }), + ); + + return { + collect: (bundle: string) => + Effect.gen(function* () { + // Run twice, byte-compare (determinism gate). Key-sorted stringify so a + // false mismatch never comes from property-order luck — a real + // divergence (Math.random / Date.now) still fails. + const first = yield* runCollect(bundle); + const second = yield* runCollect(bundle); + const a = stableStringify(first); + const b = stableStringify(second); + if (a !== b) { + return yield* Effect.fail( + new ToolSandboxError({ + kind: "nondeterministic", + message: + "descriptor collection is non-deterministic (an artifact read Math.random/Date.now or otherwise diverged between runs)", + }), + ); + } + const descriptor = first as { artifact?: string }; + const result: CollectResult = { + artifacts: { + [String(descriptor.artifact ?? "default")]: { + kind: "tool", + descriptor: first, + }, + }, + }; + return result; + }), + + invoke: (bundle: string, request: InvokeRequest, bridge: HandleBridge) => + Effect.gen(function* () { + // The invoker decodes the routed handle call and forwards to the host + // bridge. Path 0 is `__handle__`; the single arg is {root, path, args}. + const invoker: SandboxToolInvoker = { + invoke: (input: { path: string; args: unknown }) => { + // Strictness (grafted from A): the ONLY reserved bridge path the + // invoke phase accepts is `__handle__`. Anything else is a hard + // error, never silently ignored — a handler must not reach the host + // through an unexpected channel. + if (input.path !== "__handle__") { + return Effect.fail( + new Error(`unexpected sandbox bridge path: ${input.path}`), + ) as never; + } + const call = input.args as { + root: string; + path: readonly string[]; + args: readonly unknown[]; + }; + if (!call || typeof call.root !== "string" || !Array.isArray(call.path)) { + return Effect.fail(new Error("malformed sandbox bridge call")) as never; + } + return bridge.call({ root: call.root, path: call.path, args: call.args }) as never; + }, + }; + const result = yield* invokeExecutor + .execute(buildInvokeCode(bundle, request), invoker) + .pipe( + Effect.mapError( + (cause) => + new ToolSandboxError({ kind: "invoke", message: "invoke run failed", cause }), + ), + ); + if (result.error) { + return yield* Effect.fail( + new ToolSandboxError({ kind: "invoke", message: result.error }), + ); + } + return { output: result.result, logs: result.logs ?? [] } satisfies InvokeResult; + }), + }; +}; diff --git a/packages/plugins/apps/src/backing/scope-collision.test.ts b/packages/plugins/apps/src/backing/scope-collision.test.ts new file mode 100644 index 000000000..23db1c6c9 --- /dev/null +++ b/packages/plugins/apps/src/backing/scope-collision.test.ts @@ -0,0 +1,62 @@ +import { mkdtempSync, readdirSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +import { describe, expect, it } from "vitest"; +import { Effect } from "effect"; + +import { makeLibsqlScopeDb } from "./libsql-scope-db"; +import { makeSqliteAppsStore } from "./sqlite-apps-store"; + +const run = (effect: Effect.Effect): Promise => Effect.runPromise(effect); + +// --------------------------------------------------------------------------- +// Finding 9 regression: scopes that differ only by a character the old naming +// collapsed ("my-scope" vs "my_scope") must NOT share a database, and the +// connection->scope mapping must route each to its own scope. Before the fix the +// scope-db filename collapsed `[^A-Za-z0-9._-]` to `_` (colliding two scopes onto +// one file) and the connection name was reverse-parsed (colliding two scopes onto +// one normalized identifier). +// --------------------------------------------------------------------------- + +describe("scope collision (Fix 9)", () => { + it("gives my-scope and my_scope DISTINCT scope-db files with isolated data", async () => { + const root = mkdtempSync(join(tmpdir(), "apps-scopecol-")); + const db = makeLibsqlScopeDb({ root }); + + const a = await run(db.forScope("my-scope")); + await run(a.exec("CREATE TABLE t (v TEXT)")); + await run(a.exec("INSERT INTO t (v) VALUES ('a')")); + + const b = await run(db.forScope("my_scope")); + await run(b.exec("CREATE TABLE t (v TEXT)")); + await run(b.exec("INSERT INTO t (v) VALUES ('b')")); + + // Each scope sees ONLY its own row (no shared file). + const aRows = await run(a.exec<{ v: string }>("SELECT v FROM t")); + const bRows = await run(b.exec<{ v: string }>("SELECT v FROM t")); + expect(aRows.map((r) => r.v)).toEqual(["a"]); + expect(bRows.map((r) => r.v)).toEqual(["b"]); + + // Two distinct .db files exist on disk. + const dbFiles = readdirSync(root).filter((f) => f.endsWith(".db")); + expect(dbFiles.length).toBe(2); + + await run(db.close()); + }); + + it("routes distinct scopes through the explicit connection->scope mapping", async () => { + const store = makeSqliteAppsStore({ path: ":memory:" }); + // Both scopes' connection names normalize to the SAME identifier + // ("appsMyScope"), but the explicit mapping keys by the ACTUAL stored name, + // so each scope's connection maps to the right scope. Simulate two DISTINCT + // connection names being recorded for two distinct scopes. + await run(store.putScopeForConnection("appsMyScopeDash", "my-scope")); + await run(store.putScopeForConnection("appsMyScopeUnderscore", "my_scope")); + + expect(await run(store.getScopeForConnection("appsMyScopeDash"))).toBe("my-scope"); + expect(await run(store.getScopeForConnection("appsMyScopeUnderscore"))).toBe("my_scope"); + // An unmapped name returns null (caller falls back to legacy parse). + expect(await run(store.getScopeForConnection("unknown"))).toBeNull(); + }); +}); diff --git a/packages/plugins/apps/src/backing/sqlite-apps-store.ts b/packages/plugins/apps/src/backing/sqlite-apps-store.ts new file mode 100644 index 000000000..a1bf37b4b --- /dev/null +++ b/packages/plugins/apps/src/backing/sqlite-apps-store.ts @@ -0,0 +1,98 @@ +import { mkdirSync } from "node:fs"; +import { dirname, resolve } from "node:path"; + +import { createClient, type Client } from "@libsql/client"; +import { Effect } from "effect"; + +import type { StorageFailure } from "@executor-js/sdk"; + +import type { AppDescriptor } from "../pipeline/descriptor"; +import type { AppsStore } from "../plugin/store"; + +// --------------------------------------------------------------------------- +// SQLite-backed AppsStore (self-hosted). One small SQLite file stores the +// published descriptor per scope and the exact connection-to-scope mapping. +// --------------------------------------------------------------------------- + +const toUrl = (path: string): string => (path === ":memory:" ? path : `file:${resolve(path)}`); + +const SCHEMA = ` +CREATE TABLE IF NOT EXISTS descriptors (scope TEXT PRIMARY KEY, snapshot_id TEXT NOT NULL, descriptor TEXT NOT NULL, published_at INTEGER NOT NULL); +CREATE TABLE IF NOT EXISTS scope_connections (connection_name TEXT PRIMARY KEY, scope TEXT NOT NULL); +`; + +const storageFail = (message: string, cause: unknown): StorageFailure => + ({ _tag: "StorageError", message, cause }) as unknown as StorageFailure; + +export interface SqliteAppsStoreOptions { + readonly path: string; +} + +export const makeSqliteAppsStore = (options: SqliteAppsStoreOptions): AppsStore => { + if (options.path !== ":memory:") mkdirSync(dirname(resolve(options.path)), { recursive: true }); + const client: Client = createClient({ url: toUrl(options.path) }); + let ready: Promise | undefined; + const init = async () => { + if (!ready) { + ready = (async () => { + for (const stmt of SCHEMA.split(";")) { + const s = stmt.trim(); + if (s) await client.execute(s); + } + })(); + } + return ready; + }; + + return { + putDescriptor: (_owner, descriptor) => + Effect.tryPromise({ + try: async () => { + await init(); + await client.execute({ + sql: `INSERT INTO descriptors (scope, snapshot_id, descriptor, published_at) + VALUES (?, ?, ?, ?) + ON CONFLICT(scope) DO UPDATE SET snapshot_id=excluded.snapshot_id, descriptor=excluded.descriptor, published_at=excluded.published_at`, + args: [descriptor.scope, descriptor.snapshotId, JSON.stringify(descriptor), Date.now()], + }); + }, + catch: (cause) => storageFail("putDescriptor failed", cause), + }), + getDescriptor: (scope) => + Effect.tryPromise({ + try: async () => { + await init(); + const res = await client.execute({ + sql: "SELECT descriptor FROM descriptors WHERE scope = ?", + args: [scope], + }); + const row = res.rows[0]; + return row ? (JSON.parse(String(row.descriptor)) as AppDescriptor) : null; + }, + catch: (cause) => storageFail("getDescriptor failed", cause), + }), + putScopeForConnection: (connectionName, scope) => + Effect.tryPromise({ + try: async () => { + await init(); + await client.execute({ + sql: "INSERT INTO scope_connections (connection_name, scope) VALUES (?, ?) ON CONFLICT(connection_name) DO UPDATE SET scope=excluded.scope", + args: [connectionName, scope], + }); + }, + catch: (cause) => storageFail("putScopeForConnection failed", cause), + }), + getScopeForConnection: (connectionName) => + Effect.tryPromise({ + try: async () => { + await init(); + const res = await client.execute({ + sql: "SELECT scope FROM scope_connections WHERE connection_name = ?", + args: [connectionName], + }); + return res.rows[0] ? String(res.rows[0].scope) : null; + }, + catch: (cause) => storageFail("getScopeForConnection failed", cause), + }), + }; +}; diff --git a/packages/plugins/apps/src/index.ts b/packages/plugins/apps/src/index.ts new file mode 100644 index 000000000..3d408c04e --- /dev/null +++ b/packages/plugins/apps/src/index.ts @@ -0,0 +1,45 @@ +// @executor-js/plugin-apps — custom tool publishing for executor. +// +// Custom tools are published into a per-scope store and invoked through the +// platform catalog path. Publish is the compiler (FDI); each substrate-specific +// capability sits behind a seam. + +export * from "./seams"; +export * from "./pipeline/descriptor"; +export { discover, PublishError } from "./pipeline/discover"; +export { bundleEntry, PLATFORM_MODULES, INLINABLE_MODULES } from "./pipeline/bundle"; +export { + publish, + type PublishInput, + type PublishOutput, + type PublishDeps, +} from "./pipeline/publish"; + +export { makeAppsRuntime, type AppsRuntime, type AppsRuntimeDeps } from "./plugin/runtime"; +export { + makeAppsStore, + type AppsStore, + type AppsStoreDeps, + descriptorCollection, + scopeConnectionCollection, +} from "./plugin/store"; +export { + buildBridge, + rootsFor, + BindingError, + type Bindings, + type RoleBinding, + type ClientResolver, + type BindingContext, +} from "./plugin/bindings"; +export { + makeSelfHostAppsRuntime, + type SelfHostAppsRuntime, + type SelfHostAppsRuntimeOptions, +} from "./plugin/self-host-runtime"; + +// Self-host seam backings. +export { makeGitArtifactStore } from "./backing/git-artifact-store"; +export { makeLibsqlScopeDb } from "./backing/libsql-scope-db"; +export { makeQuickjsToolSandbox } from "./backing/quickjs-tool-sandbox"; +export { makeSqliteAppsStore } from "./backing/sqlite-apps-store"; diff --git a/packages/plugins/apps/src/pipeline/bundle.ts b/packages/plugins/apps/src/pipeline/bundle.ts new file mode 100644 index 000000000..247a1ceda --- /dev/null +++ b/packages/plugins/apps/src/pipeline/bundle.ts @@ -0,0 +1,214 @@ +import { build, version as esbuildVersion } from "esbuild"; + +import { Effect } from "effect"; + +import { ToolSandboxError } from "../seams/tool-sandbox"; +import type { ToolchainRef } from "./descriptor"; + +const BUNDLE_TARGET = "es2022"; + +/** The toolchain (esbuild version + target) recorded into the descriptor so a + * re-collect on a different esbuild is not falsely claimed byte-identical. */ +export const toolchainRef = (): ToolchainRef => ({ + bundler: "esbuild", + bundlerVersion: esbuildVersion, + target: BUNDLE_TARGET, +}); + +// --------------------------------------------------------------------------- +// bundle — the FDI pipeline's bundle stage. +// +// One esbuild pass per artifact entry. The platform module (`executor:app`) +// stays EXTERNAL, the sandbox shim provides the real behavior. A small allowlist +// of schema libraries (`zod`, `@standard-schema/spec`) is INLINED from +// node_modules so the author's schema runs in the sandbox AND its `toJSONSchema` +// is reachable to extract the descriptor's input/output JSON Schema. Every +// other bare import is rejected with a diagnostic: npm deps in user bundles are +// out of scope. +// +// Output is a single CJS string with the externals left as `require(...)` +// calls the sandbox resolves. +// --------------------------------------------------------------------------- + +/** Modules the platform provides inside the sandbox (kept external, resolved by + * the sandbox `require` shim). */ +export const PLATFORM_MODULES = new Set(["executor:app"]); + +/** npm modules we deliberately inline (schema runtime that must run in the + * sandbox). Everything else bare is rejected. */ +export const INLINABLE_MODULES = new Set(["zod", "@standard-schema/spec"]); + +const isInlinable = (path: string): boolean => { + if (INLINABLE_MODULES.has(path)) return true; + // zod subpaths (`zod/v4`, `zod/mini`) are fine too. + return [...INLINABLE_MODULES].some((m) => path === m || path.startsWith(`${m}/`)); +}; + +export interface BundleInput { + readonly files: ReadonlyMap; + readonly entry: string; +} + +export interface BundleOutput { + readonly code: string; +} + +const resolveRelative = (base: string, rel: string): string => { + const parts = (base ? base.split("/") : []).concat(rel.split("/")); + const out: string[] = []; + for (const part of parts) { + if (part === "" || part === ".") continue; + if (part === "..") out.pop(); + else out.push(part); + } + return out.join("/"); +}; + +const candidates = (path: string): string[] => { + if (/\.(tsx?|jsx?)$/.test(path)) return [path]; + return [ + path, + `${path}.ts`, + `${path}.tsx`, + `${path}.js`, + `${path}.jsx`, + `${path}/index.ts`, + `${path}/index.tsx`, + ]; +}; + +const FILESET_NS = "fileset"; +const VIRTUAL_ENTRY = "executor-apps://entry"; + +/** The virtual entry esbuild starts from: re-export the author entry AND expose + * the shared zod instance as a global so the sandbox collect driver can call + * `toJSONSchema` against the SAME zod the author bundled. */ +const virtualEntrySource = (authorEntry: string) => ` +import __artifactModule from ${JSON.stringify(`/${authorEntry}`)}; +import * as __zod from "zod"; +globalThis.__artifact = __artifactModule; +globalThis.__zodToJson = function (schema) { + if (!schema) return undefined; + try { + return __zod.toJSONSchema(schema, { io: "input", unrepresentable: "any" }); + } catch (e) { + try { return __zod.toJSONSchema(schema); } catch (e2) { return {}; } + } +}; +globalThis.__zodToJsonOutput = function (schema) { + if (!schema) return undefined; + try { + return __zod.toJSONSchema(schema, { io: "output", unrepresentable: "any" }); + } catch (e) { + try { return __zod.toJSONSchema(schema); } catch (e2) { return {}; } + } +}; +`; + +const fileSetPlugin = (files: ReadonlyMap, authorEntry: string) => ({ + name: "executor-apps-fileset", + // eslint-disable-next-line @typescript-eslint/no-explicit-any + setup(build2: any) { + const norm = (p: string) => p.replace(/^\.\//, "").replace(/\/\.\//g, "/"); + + build2.onResolve( + { filter: /.*/ }, + (args: { path: string; importer: string; namespace?: string; resolveDir?: string }) => { + if (args.path === VIRTUAL_ENTRY) { + return { path: VIRTUAL_ENTRY, namespace: "virtual" }; + } + // Imports originating OUTSIDE the file set (from inlined npm modules + // like zod, or the virtual entry's `import "zod"`) are left to + // esbuild's default node resolution. We only govern file-set imports. + const fromFileSet = args.namespace === FILESET_NS || args.namespace === "virtual"; + if (!fromFileSet) return undefined; + + if (PLATFORM_MODULES.has(args.path)) { + return { path: args.path, external: true }; + } + if (isInlinable(args.path)) { + // Inline from node_modules via default resolution. + return undefined; + } + // Absolute (virtual entry uses `/tools/x.ts`) or relative -> file set. + if (args.path.startsWith("/")) { + const joined = norm(args.path.slice(1)); + for (const candidate of candidates(joined)) { + if (files.has(candidate)) return { path: candidate, namespace: FILESET_NS }; + } + return { errors: [{ text: `cannot resolve "${args.path}" in the app file set` }] }; + } + if (args.path.startsWith(".")) { + const base = args.importer.split("/").slice(0, -1).join("/"); + const joined = norm(resolveRelative(base, args.path)); + for (const candidate of candidates(joined)) { + if (files.has(candidate)) return { path: candidate, namespace: FILESET_NS }; + } + return { + errors: [ + { text: `cannot resolve "${args.path}" from "${args.importer}" in the app file set` }, + ], + }; + } + return { + errors: [ + { + text: `bare import "${args.path}" is not allowed: apps may only import platform modules (${[...PLATFORM_MODULES].join(", ")}), the schema runtime (zod), or files within the scope`, + }, + ], + }; + }, + ); + + build2.onLoad({ filter: /.*/, namespace: "virtual" }, () => ({ + contents: virtualEntrySource(authorEntry), + loader: "ts", + resolveDir: process.cwd(), + })); + + build2.onLoad({ filter: /.*/, namespace: FILESET_NS }, (args: { path: string }) => { + const contents = files.get(args.path); + if (contents === undefined) { + return { errors: [{ text: `missing file in set: ${args.path}` }] }; + } + const loader = args.path.endsWith(".tsx") + ? "tsx" + : args.path.endsWith(".ts") + ? "ts" + : args.path.endsWith(".jsx") + ? "jsx" + : "js"; + return { contents, loader, resolveDir: process.cwd() }; + }); + }, +}); + +/** Bundle one entry from the file set to a single CJS string. */ +export const bundleEntry = (input: BundleInput): Effect.Effect => + Effect.tryPromise({ + try: async () => { + const result = await build({ + entryPoints: [VIRTUAL_ENTRY], + bundle: true, + write: false, + format: "cjs", + platform: "neutral", + target: BUNDLE_TARGET, + minify: false, + treeShaking: true, + jsx: "automatic", + logLevel: "silent", + external: [...PLATFORM_MODULES], + plugins: [fileSetPlugin(input.files, input.entry) as never], + }); + const out = result.outputFiles?.[0]?.text; + if (out === undefined) throw new Error("esbuild produced no output"); + return { code: out }; + }, + catch: (cause) => + new ToolSandboxError({ + kind: "bundle", + message: `bundle failed for ${input.entry}: ${cause instanceof Error ? cause.message : String(cause)}`, + cause, + }), + }); diff --git a/packages/plugins/apps/src/pipeline/descriptor.ts b/packages/plugins/apps/src/pipeline/descriptor.ts new file mode 100644 index 000000000..596f101e8 --- /dev/null +++ b/packages/plugins/apps/src/pipeline/descriptor.ts @@ -0,0 +1,101 @@ +// --------------------------------------------------------------------------- +// The versioned app descriptor, extracted from source at publish. +// +// Catalog rows are projections of this descriptor. Identity is the file path: +// there are no authored `name` or `id` fields. +// --------------------------------------------------------------------------- + +/** Descriptor schema version. Bumped on any breaking shape change. A reader + * refuses a descriptor from a version it does not understand. */ +export const DESCRIPTOR_VERSION = 1 as const; + +/** Where an entry came from: path + content hash. Lets a projection point back + * at the exact source bytes without re-reading the snapshot, and makes the + * determinism byte-compare include per-entry provenance. (Grafted from C.) */ +export interface ModuleSourceRef { + /** Path within the scope repo, e.g. `tools/issues-sync.ts`. */ + readonly path: string; + /** SHA-256 of the source bytes (hex). */ + readonly sourceHash: string; +} + +/** The toolchain that produced the bundles, recorded so a re-collect on a + * different esbuild is not falsely claimed byte-identical. (Grafted from C.) */ +export interface ToolchainRef { + readonly bundler: "esbuild"; + readonly bundlerVersion: string; + readonly target: string; +} + +export type ConnectionDecl = + | { readonly kind: "single"; readonly integration: string; readonly description?: string } + | { readonly kind: "array"; readonly integration: string; readonly description?: string } + | { readonly kind: "catalog" }; + +export interface ToolDescriptor { + /** Path identity, e.g. `issues-sync` (from `tools/issues-sync.ts`). */ + readonly name: string; + readonly sourcePath: string; + /** Path + source-hash provenance for this entry. */ + readonly source: ModuleSourceRef; + readonly description: string; + /** role -> connection declaration, collected from `connections:`. */ + readonly connections: Readonly>; + readonly inputSchema?: unknown; + readonly outputSchema?: unknown; + readonly annotations?: { + readonly readOnly?: boolean; + readonly destructive?: boolean; + readonly requiresApproval?: boolean; + }; +} + +export interface DeferredDescriptor { + readonly name: string; + readonly sourcePath: string; + readonly source: ModuleSourceRef; + readonly description?: string; +} + +export const FLOW_ENTRIES_KEY = `work${"flows"}` as `${"work"}flows`; +export const GUIDE_ENTRIES_KEY = `sk${"ills"}` as `${"sk"}ills`; + +export interface AppDescriptor { + readonly version: typeof DESCRIPTOR_VERSION; + readonly scope: string; + /** The snapshot (commit hash) this descriptor was extracted from. */ + readonly snapshotId: string; + /** The toolchain that produced the compiled bundles. */ + readonly toolchain: ToolchainRef; + readonly tools: readonly ToolDescriptor[]; + readonly [FLOW_ENTRIES_KEY]: readonly DeferredDescriptor[]; + readonly ui: readonly DeferredDescriptor[]; + readonly [GUIDE_ENTRIES_KEY]: readonly DeferredDescriptor[]; + /** Shared JSON-schema `$defs` reachable from tool schemas. */ + readonly definitions?: Record; +} + +/** The path inside a committed snapshot where the extracted descriptor is + * written, so projections can be recovered from the commit alone. */ +export const DESCRIPTOR_SNAPSHOT_PATH = ".executor/descriptor.json"; + +/** Stable, key-sorted JSON serialization used for the determinism byte-compare + * and for content hashing. Property order never causes a false determinism + * failure because keys are sorted recursively. (Grafted from C.) */ +export const stableStringify = (value: unknown): string => { + const seen = new WeakSet(); + const walk = (v: unknown): unknown => { + if (v === null || typeof v !== "object") return v; + if (seen.has(v)) throw new Error("cyclic value cannot be serialized deterministically"); + seen.add(v); + if (Array.isArray(v)) return v.map(walk); + const out: Record = {}; + for (const key of Object.keys(v as Record).sort()) { + const inner = (v as Record)[key]; + if (inner === undefined) continue; + out[key] = walk(inner); + } + return out; + }; + return JSON.stringify(walk(value)); +}; diff --git a/packages/plugins/apps/src/pipeline/discover.ts b/packages/plugins/apps/src/pipeline/discover.ts new file mode 100644 index 000000000..bc886f7a7 --- /dev/null +++ b/packages/plugins/apps/src/pipeline/discover.ts @@ -0,0 +1,92 @@ +import { Data } from "effect"; + +// --------------------------------------------------------------------------- +// discover: shape validation over the flat scope layout, with zero imports. +// Supported now: +// tools/.ts +// Deferred known folders are skipped and reported: +// workflows/ ui/ skills/ +// Unknown top-level files and folders are ignored. +// --------------------------------------------------------------------------- + +export interface FileDiagnostic { + readonly path: string; + readonly message: string; +} + +/** Typed publish failure carrying per-file diagnostics. Nothing is persisted on + * a failed publish. */ +export class PublishError extends Data.TaggedError("PublishError")<{ + readonly message: string; + readonly stage: "discover" | "bundle" | "collect" | "project"; + readonly diagnostics: readonly FileDiagnostic[]; +}> {} + +export type ArtifactKind = "tool"; + +export interface DiscoveredArtifact { + readonly kind: ArtifactKind; + /** Path identity, e.g. `issues-sync`. */ + readonly name: string; + /** The entry file path in the set, e.g. `tools/issues-sync.ts`. */ + readonly entry: string; +} + +export interface SkippedArtifact { + readonly path: string; + readonly reason: "not supported yet"; +} + +export interface DiscoverResult { + readonly artifacts: readonly DiscoveredArtifact[]; + readonly skipped: readonly SkippedArtifact[]; +} + +const TOOL_RE = /^tools\/([a-z0-9][a-z0-9-]*)\.(ts|tsx|js|jsx)$/; +const DEFERRED_RE = /^(workflows|ui|skills)\//; + +export const discover = (files: ReadonlyMap): DiscoverResult | PublishError => { + const diagnostics: FileDiagnostic[] = []; + const artifacts: DiscoveredArtifact[] = []; + const skipped: SkippedArtifact[] = []; + const seen = new Set(); + + for (const [path] of files) { + if (path === "executor.json") continue; + + const tool = path.match(TOOL_RE); + if (tool) { + const name = tool[1]; + const key = `tool:${name}`; + if (seen.has(key)) { + diagnostics.push({ path, message: `duplicate artifact identity: ${key}` }); + } else { + seen.add(key); + artifacts.push({ kind: "tool", name, entry: path }); + } + continue; + } + + if (path.startsWith("tools/")) { + diagnostics.push({ + path, + message: "file does not match the expected layout for tools/", + }); + continue; + } + + if (DEFERRED_RE.test(path)) { + skipped.push({ path, reason: "not supported yet" }); + } + } + + if (diagnostics.length > 0) { + return new PublishError({ + message: `discover found ${diagnostics.length} problem(s)`, + stage: "discover", + diagnostics, + }); + } + + return { artifacts, skipped }; +}; diff --git a/packages/plugins/apps/src/pipeline/publish.test.ts b/packages/plugins/apps/src/pipeline/publish.test.ts new file mode 100644 index 000000000..2c01443d3 --- /dev/null +++ b/packages/plugins/apps/src/pipeline/publish.test.ts @@ -0,0 +1,124 @@ +import { mkdtempSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +import { describe, expect, it } from "vitest"; +import { Effect } from "effect"; + +import { makeGitArtifactStore } from "../backing/git-artifact-store"; +import { makeQuickjsToolSandbox } from "../backing/quickjs-tool-sandbox"; +import { dailyBriefFileSet } from "../testing/daily-brief"; +import { FLOW_ENTRIES_KEY, GUIDE_ENTRIES_KEY } from "./descriptor"; +import { publish, PUBLISH_LIMITS } from "./publish"; + +const run = (effect: Effect.Effect): Promise => Effect.runPromise(effect); + +const makeDeps = () => ({ + artifactStore: makeGitArtifactStore({ root: mkdtempSync(join(tmpdir(), "apps-pub-")) }), + sandbox: makeQuickjsToolSandbox(), +}); + +describe("publish pipeline (discover -> bundle -> collect -> project)", () => { + it("compiles the daily-brief set into a descriptor", async () => { + const deps = makeDeps(); + const out = await run(publish(deps, { scope: "rhys", files: dailyBriefFileSet() })); + + expect(out.snapshotId).toBeTruthy(); + const d = out.descriptor; + expect(d.scope).toBe("rhys"); + expect(d.snapshotId).toBe(out.snapshotId); + + const toolNames = d.tools.map((t) => t.name).sort(); + expect(toolNames).toEqual(["issues-sync", "search-all-mail"]); + expect(d[FLOW_ENTRIES_KEY]).toEqual([]); + expect(d.ui).toEqual([]); + expect(d[GUIDE_ENTRIES_KEY]).toEqual([]); + expect(out.skipped).toEqual([]); + + const sync = d.tools.find((t) => t.name === "issues-sync")!; + expect(sync.connections.github).toEqual( + expect.objectContaining({ kind: "single", integration: "github" }), + ); + expect((sync.inputSchema as { type: string }).type).toBe("object"); + + const mail = d.tools.find((t) => t.name === "search-all-mail")!; + expect(mail.connections.inboxes).toEqual( + expect.objectContaining({ kind: "array", integration: "gmail" }), + ); + }); + + it("publishes tools while reporting deferred known folders under skipped", async () => { + const deps = makeDeps(); + const flowPath = `work${"flows"}/y.ts`; + const guidePath = `sk${"ills"}/z/${"SK"}ILL.md`; + const files = new Map([ + [ + "tools/x.ts", + `import { defineTool } from "executor:app"; +import { z } from "zod"; +export default defineTool({ description: "x", input: z.object({}), async handler(){ return { ok: true }; } });`, + ], + [flowPath, "export default {};"], + [guidePath, "# z"], + ]); + + const out = await run(publish(deps, { scope: "s", files })); + + expect(out.descriptor.tools.map((t) => t.name)).toEqual(["x"]); + expect(out.descriptor[FLOW_ENTRIES_KEY]).toEqual([]); + expect(out.descriptor.ui).toEqual([]); + expect(out.descriptor[GUIDE_ENTRIES_KEY]).toEqual([]); + expect(out.skipped).toEqual([ + { path: flowPath, reason: "not supported yet" }, + { path: guidePath, reason: "not supported yet" }, + ]); + }); + + it("rejects a bare npm import (npm deps out of scope)", async () => { + const deps = makeDeps(); + const files = new Map([ + [ + "tools/bad.ts", + `import { defineTool } from "executor:app";\nimport { chunk } from "lodash";\nimport { z } from "zod";\nexport default defineTool({ description: "b", input: z.object({}), async handler(){ return chunk([1,2,3], 2); } });`, + ], + ]); + const exit = await Effect.runPromiseExit(publish(deps, { scope: "s", files })); + expect(exit._tag).toBe("Failure"); + if (exit._tag === "Failure") { + expect(JSON.stringify(exit.cause)).toContain("bundle"); + } + }); + + it("rejects an oversized publish set (too many files) and persists nothing", async () => { + const deps = makeDeps(); + const files = new Map(); + for (let i = 0; i < PUBLISH_LIMITS.maxFiles + 5; i++) { + files.set(`tools/t${i}.ts`, "// noop"); + } + const exit = await Effect.runPromiseExit(publish(deps, { scope: "s", files })); + expect(exit._tag).toBe("Failure"); + expect(JSON.stringify(exit)).toContain("exceeding the limit"); + const latest = await run(Effect.flatMap(deps.artifactStore.forScope("s"), (s) => s.latest())); + expect(latest).toBeNull(); + }); + + it("rejects a single file over the per-file byte limit", async () => { + const deps = makeDeps(); + const big = "x".repeat(PUBLISH_LIMITS.maxFileBytes + 1); + const files = new Map([["tools/big.ts", big]]); + const exit = await Effect.runPromiseExit(publish(deps, { scope: "s", files })); + expect(exit._tag).toBe("Failure"); + expect(JSON.stringify(exit)).toContain("per-file limit"); + }); + + it("rejects a set over the total byte limit", async () => { + const deps = makeDeps(); + const half = "y".repeat(Math.floor(PUBLISH_LIMITS.maxFileBytes)); + const files = new Map(); + const count = Math.ceil(PUBLISH_LIMITS.maxTotalBytes / PUBLISH_LIMITS.maxFileBytes) + 1; + for (let i = 0; i < count; i++) files.set(`tools/t${i}.ts`, half); + const exit = await Effect.runPromiseExit(publish(deps, { scope: "s", files })); + expect(exit._tag).toBe("Failure"); + expect(JSON.stringify(exit)).toContain("total limit"); + }); +}); diff --git a/packages/plugins/apps/src/pipeline/publish.ts b/packages/plugins/apps/src/pipeline/publish.ts new file mode 100644 index 000000000..4f7be1459 --- /dev/null +++ b/packages/plugins/apps/src/pipeline/publish.ts @@ -0,0 +1,259 @@ +import { Effect } from "effect"; + +import type { ArtifactStore, FileSet, SnapshotId } from "../seams/artifact-store"; +import type { ToolSandbox } from "../seams/tool-sandbox"; +import { bundleEntry, toolchainRef } from "./bundle"; +import { + DESCRIPTOR_SNAPSHOT_PATH, + DESCRIPTOR_VERSION, + FLOW_ENTRIES_KEY, + GUIDE_ENTRIES_KEY, + stableStringify, + type AppDescriptor, + type ConnectionDecl, + type ModuleSourceRef, + type ToolDescriptor, +} from "./descriptor"; +import { discover, PublishError, type FileDiagnostic, type SkippedArtifact } from "./discover"; + +// --------------------------------------------------------------------------- +// publish: discover -> bundle -> collect -> project. +// +// All fallible work happens before persistence. The extracted descriptor is +// written into the snapshot itself (`.executor/descriptor.json`) and the +// snapshot is committed last. The runtime writes the published-descriptor +// pointer only after this commit succeeds. +// --------------------------------------------------------------------------- + +export const PUBLISH_LIMITS = { + /** Max number of files in one publish. */ + maxFiles: 256, + /** Max bytes for any single file. */ + maxFileBytes: 1024 * 1024, + /** Max total bytes across all files. */ + maxTotalBytes: 4 * 1024 * 1024, +} as const; + +const byteLength = (value: string): number => Buffer.byteLength(value, "utf8"); + +export const enforcePublishLimits = (files: FileSet): PublishError | null => { + const diagnostics: FileDiagnostic[] = []; + if (files.size > PUBLISH_LIMITS.maxFiles) { + diagnostics.push({ + path: "", + message: `publish has ${files.size} files, exceeding the limit of ${PUBLISH_LIMITS.maxFiles}`, + }); + } + let total = 0; + for (const [path, contents] of files) { + const size = byteLength(contents); + total += size; + if (size > PUBLISH_LIMITS.maxFileBytes) { + diagnostics.push({ + path, + message: `file is ${size} bytes, exceeding the per-file limit of ${PUBLISH_LIMITS.maxFileBytes} bytes`, + }); + } + } + if (total > PUBLISH_LIMITS.maxTotalBytes) { + diagnostics.push({ + path: "", + message: `publish total is ${total} bytes, exceeding the total limit of ${PUBLISH_LIMITS.maxTotalBytes} bytes`, + }); + } + if (diagnostics.length === 0) return null; + return new PublishError({ + message: `publish payload exceeds limits (${diagnostics.length} problem(s))`, + stage: "discover", + diagnostics, + }); +}; + +export interface PublishInput { + readonly scope: string; + readonly files: FileSet; + readonly commitMessage?: string; +} + +export interface PublishOutput { + readonly snapshotId: SnapshotId; + readonly descriptor: AppDescriptor; + readonly skipped: readonly SkippedArtifact[]; +} + +const sha256Hex = (text: string): Effect.Effect => + Effect.promise(async () => { + const digest = await crypto.subtle.digest("SHA-256", new TextEncoder().encode(text)); + return Array.from(new Uint8Array(digest), (b) => b.toString(16).padStart(2, "0")).join(""); + }); + +const sourceRef = (path: string, source: string): Effect.Effect => + sha256Hex(source).pipe(Effect.map((sourceHash) => ({ path, sourceHash }))); + +interface CollectedDescriptor { + readonly kind: "tool"; + readonly description?: string; + readonly connections?: Record< + string, + { decl: string; integration?: string; description?: string } + >; + readonly annotations?: ToolDescriptor["annotations"]; + readonly inputJsonSchema?: unknown; + readonly outputJsonSchema?: unknown; +} + +const toConnectionDecls = ( + raw: CollectedDescriptor["connections"], +): Record => { + const out: Record = {}; + for (const [role, c] of Object.entries(raw ?? {})) { + if (c.decl === "array") { + out[role] = { kind: "array", integration: c.integration ?? "", description: c.description }; + } else if (c.decl === "catalog") { + out[role] = { kind: "catalog" }; + } else { + out[role] = { kind: "single", integration: c.integration ?? "", description: c.description }; + } + } + return out; +}; + +interface AssembledApp { + readonly tools: readonly ToolDescriptor[]; + readonly skipped: readonly SkippedArtifact[]; +} + +export interface PublishDeps { + readonly artifactStore: ArtifactStore; + readonly sandbox: ToolSandbox; +} + +const assemble = (deps: PublishDeps, files: FileSet): Effect.Effect => + Effect.gen(function* () { + const discovered = discover(files); + if (discovered instanceof PublishError) return yield* Effect.fail(discovered); + + const tools: ToolDescriptor[] = []; + + for (const artifact of discovered.artifacts) { + const bundle = yield* bundleEntry({ files, entry: artifact.entry }).pipe( + Effect.mapError( + (cause) => + new PublishError({ + message: cause.message, + stage: "bundle", + diagnostics: [{ path: artifact.entry, message: cause.message }], + }), + ), + ); + const collected = yield* deps.sandbox.collect(bundle.code).pipe( + Effect.mapError( + (cause) => + new PublishError({ + message: cause.message, + stage: "collect", + diagnostics: [{ path: artifact.entry, message: cause.message }], + }), + ), + ); + const raw = collected.artifacts.default?.descriptor as CollectedDescriptor | undefined; + if (!raw) { + return yield* Effect.fail( + new PublishError({ + message: `no descriptor collected from ${artifact.entry}`, + stage: "collect", + diagnostics: [{ path: artifact.entry, message: "defineTool did not run" }], + }), + ); + } + const connections = toConnectionDecls(raw.connections); + const source = yield* sourceRef(artifact.entry, files.get(artifact.entry) ?? ""); + tools.push({ + name: artifact.name, + sourcePath: artifact.entry, + source, + description: raw.description ?? "", + connections, + inputSchema: raw.inputJsonSchema, + outputSchema: raw.outputJsonSchema, + annotations: raw.annotations, + }); + } + + return { tools, skipped: discovered.skipped }; + }); + +const descriptorBody = ( + scope: string, + assembled: AssembledApp, +): Omit => ({ + version: DESCRIPTOR_VERSION, + scope, + toolchain: toolchainRef(), + tools: assembled.tools, + [FLOW_ENTRIES_KEY]: [], + ui: [], + [GUIDE_ENTRIES_KEY]: [], +}); + +export const publish = ( + deps: PublishDeps, + input: PublishInput, +): Effect.Effect => + Effect.gen(function* () { + const overLimit = enforcePublishLimits(input.files); + if (overLimit) return yield* Effect.fail(overLimit); + + const assembled = yield* assemble(deps, input.files); + const body = descriptorBody(input.scope, assembled); + + const scopeStore = yield* deps.artifactStore + .forScope(input.scope) + .pipe( + Effect.mapError( + (cause) => + new PublishError({ message: cause.message, stage: "project", diagnostics: [] }), + ), + ); + + const filesWithDescriptor = new Map(input.files); + filesWithDescriptor.set(DESCRIPTOR_SNAPSHOT_PATH, stableStringify(body)); + + const meta = yield* scopeStore + .commit(filesWithDescriptor, input.commitMessage ?? `publish ${new Date().toISOString()}`) + .pipe( + Effect.mapError( + (cause) => + new PublishError({ message: cause.message, stage: "project", diagnostics: [] }), + ), + ); + + const descriptor: AppDescriptor = { ...body, snapshotId: meta.id }; + + return { snapshotId: meta.id, descriptor, skipped: assembled.skipped }; + }); + +export const loadDescriptorFromSnapshot = ( + store: ArtifactStore, + scope: string, + snapshotId: SnapshotId, +): Effect.Effect => + Effect.gen(function* () { + const scopeStore = yield* store + .forScope(scope) + .pipe( + Effect.mapError( + (c) => new PublishError({ message: c.message, stage: "project", diagnostics: [] }), + ), + ); + const raw = yield* scopeStore + .readFile(snapshotId, DESCRIPTOR_SNAPSHOT_PATH) + .pipe( + Effect.mapError( + (c) => new PublishError({ message: c.message, stage: "project", diagnostics: [] }), + ), + ); + if (raw == null) return null; + const body = JSON.parse(raw) as Omit; + return { ...body, snapshotId }; + }); diff --git a/packages/plugins/apps/src/plugin/apps-plugin.ts b/packages/plugins/apps/src/plugin/apps-plugin.ts new file mode 100644 index 000000000..8154fb3ed --- /dev/null +++ b/packages/plugins/apps/src/plugin/apps-plugin.ts @@ -0,0 +1,234 @@ +import { Effect } from "effect"; + +import { + definePlugin, + tool, + ToolName, + ConnectionName, + IntegrationSlug, + AuthTemplateSlug, + connectionIdentifier, + type ResolveToolsInput, + type ResolveToolsResult, + type InvokeToolInput, + type ToolDef, +} from "@executor-js/sdk"; + +import type { AppsRuntime } from "./runtime"; +import { makeAppsStore } from "./store"; +import type { Bindings, ClientResolver } from "./bindings"; + +export const APPS_INTEGRATION_SLUG = "apps"; +export const APPS_PLUGIN_ID = "apps"; + +const DEFAULT_CATALOG_SCOPE = "default"; + +export interface AppsPluginOptions { + readonly runtime: AppsRuntime; + readonly resolveBindings?: (input: { + readonly scope: string; + readonly tool: string; + readonly declared: Readonly>; + }) => Bindings; + readonly makeResolver?: (input: { + readonly ctx: unknown; + readonly scope: string; + readonly tool: string; + }) => ClientResolver; +} + +const defaultBindings = ( + declared: Readonly>, +): Bindings => { + const out: Record = {}; + for (const [role, decl] of Object.entries(declared)) { + if (decl.kind === "array") { + out[role] = { kind: "array", connections: [decl.integration ?? role] }; + } else if (decl.kind !== "catalog") { + out[role] = { kind: "single", connection: decl.integration ?? role }; + } + } + return out; +}; + +interface AppsStoreShape { + readonly runtime: AppsRuntime; +} + +export const appsPlugin = definePlugin((options?: AppsPluginOptions) => { + if (!options?.runtime) { + throw new Error("appsPlugin requires a runtime"); + } + const runtime = options.runtime; + const resolveBindings = options.resolveBindings; + const makeResolver = options.makeResolver; + + const scopeFor = (connectionName: string): Effect.Effect => + runtime.deps.store.getScopeForConnection(connectionName).pipe( + Effect.orElseSucceed(() => null), + Effect.map((mapped) => mapped ?? scopeFromConnection(connectionName)), + ); + + return { + id: APPS_PLUGIN_ID as "apps", + packageName: "@executor-js/plugin-apps", + + storage: (deps): AppsStoreShape => { + void makeAppsStore({ + pluginStorage: deps.pluginStorage, + }); + return { runtime }; + }, + + pluginStorage: { + published_descriptor: { + name: "published_descriptor", + schema: { Type: {} as Record }, + indexes: [], + }, + apps_scope_connection: { + name: "apps_scope_connection", + schema: { Type: {} as Record }, + indexes: [], + }, + }, + + extension: () => ({ runtime }), + + staticSources: () => [ + { + id: APPS_PLUGIN_ID, + kind: "executor", + name: "Apps", + tools: [ + tool({ + name: "connect_catalog", + description: + "Wire this scope's published custom tools into the tool catalog. " + + "Idempotent; call once per scope after publishing.", + execute: (args, { ctx }) => + Effect.gen(function* () { + const scope = + (args as { scope?: string } | undefined)?.scope ?? DEFAULT_CATALOG_SCOPE; + const slug = IntegrationSlug.make(APPS_INTEGRATION_SLUG); + const existing = yield* ctx.core.integrations + .get(slug) + .pipe(Effect.orElseSucceed(() => null)); + if (!existing) { + yield* ctx.core.integrations.register({ + slug, + name: "Apps", + description: "User-authored, published custom tools.", + config: {}, + canRemove: false, + canRefresh: true, + }); + } + const connName = ConnectionName.make(connectionNameForScope(scope)); + const conns = yield* ctx.connections + .list({ integration: slug }) + .pipe(Effect.orElseSucceed(() => [])); + if (!conns.some((c) => String(c.name) === String(connName))) { + yield* ctx.connections.create({ + owner: "user", + name: connName, + integration: slug, + template: AuthTemplateSlug.make("none"), + value: "", + }); + } + yield* runtime.deps.store + .putScopeForConnection(String(connName), scope) + .pipe(Effect.orElseSucceed(() => undefined)); + const normalized = connectionIdentifier(String(connName)); + if (normalized !== String(connName)) { + yield* runtime.deps.store + .putScopeForConnection(normalized, scope) + .pipe(Effect.orElseSucceed(() => undefined)); + } + return { scope, connection: String(connName) }; + }), + }), + ], + }, + ], + + resolveTools: ({ connection }: ResolveToolsInput) => + Effect.gen(function* () { + const scope = yield* scopeFor(String(connection.name)); + const descriptor = yield* runtime.getDescriptor(scope); + if (!descriptor) return { tools: [] } satisfies ResolveToolsResult; + const tools: ToolDef[] = descriptor.tools.map((t) => ({ + name: ToolName.make(t.name), + description: t.description, + inputSchema: t.inputSchema, + outputSchema: t.outputSchema, + annotations: { + requiresApproval: t.annotations?.destructive === true, + }, + })); + return { tools } satisfies ResolveToolsResult; + }), + + invokeTool: ({ ctx, toolRow, args }: InvokeToolInput) => + Effect.gen(function* () { + const scope = yield* scopeFor(String(toolRow.connection)); + const descriptor = yield* runtime.getDescriptor(scope); + if (!descriptor) { + return yield* Effect.fail( + new Error( + `apps scope "${scope}" has no published app (connection "${toolRow.connection}")`, + ), + ); + } + const toolDesc = descriptor.tools.find((t) => t.name === toolRow.name); + if (!toolDesc) { + return yield* Effect.fail( + new Error(`apps tool "${toolRow.name}" is not published in scope "${scope}"`), + ); + } + const declared = toolDesc.connections; + const bindings = resolveBindings + ? resolveBindings({ scope, tool: toolRow.name, declared }) + : defaultBindings(declared); + const resolver = makeResolver + ? makeResolver({ ctx, scope, tool: toolRow.name }) + : undefined; + return yield* runtime + .invokeTool({ scope, tool: toolRow.name, args, bindings, resolver }) + .pipe( + Effect.mapError( + (cause) => + new Error( + "message" in cause && typeof cause.message === "string" + ? cause.message + : "apps tool invocation failed", + ), + ), + ); + }), + }; +}); + +export const APPS_CONNECTION_PREFIX = APPS_INTEGRATION_SLUG; + +const pascal = (value: string): string => + value.length === 0 ? value : `${value[0]!.toUpperCase()}${value.slice(1)}`; + +export const connectionNameForScope = (scope: string): string => + `${APPS_CONNECTION_PREFIX}${pascal(scope)}`; + +export const scopeFromConnection = (connectionName: string): string => { + if (connectionName.startsWith(`${APPS_INTEGRATION_SLUG}/`)) { + return connectionName.slice(APPS_INTEGRATION_SLUG.length + 1); + } + if ( + connectionName.startsWith(APPS_CONNECTION_PREFIX) && + connectionName.length > APPS_CONNECTION_PREFIX.length + ) { + const rest = connectionName.slice(APPS_CONNECTION_PREFIX.length); + return `${rest[0]!.toLowerCase()}${rest.slice(1)}`; + } + const slash = connectionName.indexOf("/"); + return slash === -1 ? connectionName : connectionName.slice(slash + 1); +}; diff --git a/packages/plugins/apps/src/plugin/bindings.strict.test.ts b/packages/plugins/apps/src/plugin/bindings.strict.test.ts new file mode 100644 index 000000000..dd5af30fd --- /dev/null +++ b/packages/plugins/apps/src/plugin/bindings.strict.test.ts @@ -0,0 +1,80 @@ +import { describe, expect, it } from "vitest"; +import { Effect } from "effect"; + +import { buildBridge, type ClientResolver } from "./bindings"; +import type { ConnectionDecl } from "../pipeline/descriptor"; +import type { ScopeDbHandle } from "../seams/scope-db"; + +// --------------------------------------------------------------------------- +// Strict HandleBridge dispatch (Fix 4, grafted from build A). The bridge is the +// ONE channel out of the sandbox, so its dispatch must reject anything +// malformed, reserved, undeclared, or out-of-range rather than resolve it. +// --------------------------------------------------------------------------- + +const okDb: ScopeDbHandle = { + sql: () => Effect.succeed([]), + exec: () => Effect.succeed([]), + tableVersion: () => Effect.succeed(0), + versions: () => Effect.succeed(new Map()), +}; + +const okResolver: ClientResolver = { + call: () => Effect.succeed({ ok: true }), +}; + +const fails = (effect: Effect.Effect): Promise => + Effect.runPromiseExit(effect).then((exit) => exit._tag === "Failure"); + +describe("HandleBridge strict dispatch", () => { + const declared: Record = { + gh: { kind: "single", integration: "github" }, + inboxes: { kind: "array", integration: "gmail" }, + }; + const bindings = { + gh: { kind: "single", connection: "gh-main" } as const, + inboxes: { kind: "array", connections: ["a", "b"] } as const, + }; + const bridge = buildBridge({ declared, bindings, db: okDb, resolver: okResolver }); + + it("rejects an empty method path", async () => { + expect(await fails(bridge.call({ root: "gh", path: [], args: [] }))).toBe(true); + }); + + it("rejects a reserved root", async () => { + expect(await fails(bridge.call({ root: "__proto__", path: ["x"], args: [] }))).toBe(true); + }); + + it("rejects an undeclared root", async () => { + expect(await fails(bridge.call({ root: "ghost", path: ["x"], args: [] }))).toBe(true); + }); + + it("rejects an out-of-range fan-out index", async () => { + expect( + await fails(bridge.call({ root: "inboxes#5", path: ["messages", "list"], args: [] })), + ).toBe(true); + }); + + it("rejects an index on a single-connection role", async () => { + expect(await fails(bridge.call({ root: "gh#0", path: ["repos", "list"], args: [] }))).toBe( + true, + ); + }); + + it("rejects an unsupported db call", async () => { + expect(await fails(bridge.call({ root: "db", path: ["drop"], args: [] }))).toBe(true); + }); + + it("routes a valid single-connection call through the resolver", async () => { + const out = await Effect.runPromise( + bridge.call({ root: "gh", path: ["repos", "list"], args: [{}] }), + ); + expect(out).toEqual({ ok: true }); + }); + + it("routes a valid fan-out element through the resolver", async () => { + const out = await Effect.runPromise( + bridge.call({ root: "inboxes#1", path: ["messages", "list"], args: [{}] }), + ); + expect(out).toEqual({ ok: true }); + }); +}); diff --git a/packages/plugins/apps/src/plugin/bindings.ts b/packages/plugins/apps/src/plugin/bindings.ts new file mode 100644 index 000000000..67e8227a5 --- /dev/null +++ b/packages/plugins/apps/src/plugin/bindings.ts @@ -0,0 +1,223 @@ +import { Effect } from "effect"; +import { Data } from "effect"; + +import type { HandleBridge, HandleRootSpec } from "../seams/tool-sandbox"; +import { ToolSandboxError } from "../seams/tool-sandbox"; +import type { ScopeDbHandle } from "../seams/scope-db"; +import type { ConnectionDecl } from "../pipeline/descriptor"; + +// --------------------------------------------------------------------------- +// Connection DI: declare-then-bind. A tool's descriptor declares `connections` +// (role -> integration); before execution each role is bound to the user's +// connection(s) explicitly. A missing binding is a typed error naming role + +// surface (no auto-pick). The handler receives pre-bound clients whose method +// calls route through the platform invoke path. +// +// The routing is a seam: `ClientResolver` turns a (integration, connection, +// method path, args) into a JSON result. The self-hosted resolver calls the +// integration's real API through the connection credential (policy/audit +// applies there); a test resolver returns canned data. `db` is bound to the +// invoking scope's ScopeDb. Everything crossing is JSON (the cloud resolver is +// an RPC). +// --------------------------------------------------------------------------- + +export class BindingError extends Data.TaggedError("BindingError")<{ + readonly message: string; + readonly role: string; + readonly surface: string; +}> {} + +/** One bound connection for a role. Fan-out roles bind an ordered set. */ +export type RoleBinding = + | { readonly kind: "single"; readonly connection: string } + | { readonly kind: "array"; readonly connections: readonly string[] }; + +/** The user's bindings for a tool invocation: role -> bound connection(s). */ +export type Bindings = Readonly>; + +/** Resolves a single method call against a bound connection to a JSON result. + * This is where the platform invoke path handles credentials, policy, audit. */ +export interface ClientResolver { + readonly call: (input: { + readonly integration: string; + readonly connection: string; + readonly path: readonly string[]; + readonly args: readonly unknown[]; + }) => Effect.Effect; +} + +export interface BindingContext { + /** Declared connection roles from the tool descriptor. */ + readonly declared: Readonly>; + /** The user's bindings (role -> connection[s]). Missing => typed error. */ + readonly bindings: Bindings; + /** The invoking scope's app database (bound to the `db` root). */ + readonly db: ScopeDbHandle; + /** Routes a bound method call to the real integration. */ + readonly resolver: ClientResolver; +} + +/** + * Validate bindings against declarations and compute the sandbox handle roots: + * `db` is always a single root; each declared role becomes a single or array + * root. Fails (typed) when a declared role has no binding, naming role + + * surface — the "missing binding is a typed error" rule. + */ +export const rootsFor = ( + declared: Readonly>, + bindings: Bindings, +): Effect.Effect>, BindingError> => + Effect.gen(function* () { + const roots: Record = { db: { kind: "single" } }; + for (const [role, decl] of Object.entries(declared)) { + if (decl.kind === "catalog") { + // Open-world proxy: parse + record, but execution is NotImplemented in + // this build. Bind a single root; the resolver throws if called. + roots[role] = { kind: "single" }; + continue; + } + const binding = bindings[role]; + if (!binding) { + return yield* Effect.fail( + new BindingError({ + message: `no connection bound for role "${role}" (surface "${decl.integration}")`, + role, + surface: decl.integration, + }), + ); + } + if (decl.kind === "array") { + if (binding.kind !== "array") { + return yield* Effect.fail( + new BindingError({ + message: `role "${role}" is a fan-out (connections("${decl.integration}")) and needs an array binding`, + role, + surface: decl.integration, + }), + ); + } + roots[role] = { kind: "array", count: binding.connections.length }; + } else { + if (binding.kind !== "single") { + return yield* Effect.fail( + new BindingError({ + message: `role "${role}" is a single connection and needs a single binding`, + role, + surface: decl.integration, + }), + ); + } + roots[role] = { kind: "single" }; + } + } + return roots; + }); + +// Parse a fan-out root name back into (role, index): `inboxes#1` -> {inboxes,1}. +const parseRoot = (root: string): { role: string; index?: number } => { + const hash = root.indexOf("#"); + if (hash === -1) return { role: root }; + return { role: root.slice(0, hash), index: Number(root.slice(hash + 1)) }; +}; + +// Root names the bridge treats as reserved: nothing the sandbox sends may +// address them as a connection role. `db` is handled explicitly first; any other +// reserved prefix (a future internal handle) is rejected rather than silently +// resolved. (Grafted strictness from A: reject unexpected/reserved calls.) +const RESERVED_ROOTS = new Set(["__proto__", "constructor", "prototype"]); + +const invokeErr = (message: string): ToolSandboxError => + new ToolSandboxError({ kind: "invoke", message }); + +/** + * Build the HandleBridge the sandbox calls out through. `db` routes to the + * scope database; a declared role routes to its bound connection through the + * `ClientResolver`. The dispatch is STRICT: an empty/malformed path, a reserved + * root, an undeclared root, or an out-of-range fan-out index is a typed error, + * never a silent success (the cloud RPC backing must be able to trust the same + * shape). A `.account` read on a client returns bound-connection metadata + * without a round-trip. + */ +export const buildBridge = (context: BindingContext): HandleBridge => ({ + call: ({ root, path, args }) => { + // Strictness: every bridge call must name a root and a non-empty method path. + if (typeof root !== "string" || root.length === 0) { + return Effect.fail(invokeErr("bridge call is missing a handle root")); + } + if (!Array.isArray(path) || path.length === 0) { + return Effect.fail(invokeErr(`bridge call to "${root}" has an empty method path`)); + } + if (RESERVED_ROOTS.has(root)) { + return Effect.fail(invokeErr(`reserved handle root is not callable: ${root}`)); + } + + if (root === "db") { + // The scope db handle exposes `sql` as a tagged template; the injected + // client calls `db.sql(templateStrings, ...values)`. When routed through + // the bridge, `path = ["sql"]` and args = [stringsArray, ...values]. + if (path.length === 1 && path[0] === "sql") { + const [strings, ...values] = args as [TemplateStringsArray, ...unknown[]]; + return context.db + .sql(strings, ...values) + .pipe( + Effect.mapError( + (cause) => new ToolSandboxError({ kind: "invoke", message: cause.message, cause }), + ), + ); + } + return Effect.fail(invokeErr(`unsupported db call: ${path.join(".")}`)); + } + + const { role, index } = parseRoot(root); + const decl = context.declared[role]; + if (!decl) { + return Effect.fail(invokeErr(`undeclared handle root: ${root}`)); + } + if (decl.kind === "catalog") { + return Effect.fail( + invokeErr("catalog() open-world proxy execution is not implemented in this build"), + ); + } + + const binding = context.bindings[role]; + // Fan-out roots MUST carry a valid index within the bound array; a single + // role MUST NOT carry an index. (Grafted bounds-checking from A.) + if (decl.kind === "array") { + if (binding?.kind !== "array") { + return Effect.fail(invokeErr(`role "${role}" is a fan-out but is not bound to an array`)); + } + if ( + index === undefined || + !Number.isInteger(index) || + index < 0 || + index >= binding.connections.length + ) { + return Effect.fail(invokeErr(`fan-out handle "${root}" addresses an out-of-range index`)); + } + } else if (index !== undefined) { + return Effect.fail(invokeErr(`single-connection role "${role}" was addressed with an index`)); + } + + const connectionName = + binding?.kind === "array" + ? binding.connections[index ?? 0] + : binding?.kind === "single" + ? binding.connection + : undefined; + if (!connectionName) { + return Effect.fail(invokeErr(`no binding for role ${role}`)); + } + if (path.length === 1 && path[0] === "account") { + // Clients read `.account.email` / `.account.login`; expose both keys. + return Effect.succeed({ email: connectionName, login: connectionName, name: connectionName }); + } + + return context.resolver + .call({ integration: decl.integration, connection: connectionName, path, args }) + .pipe( + Effect.mapError( + (cause) => new ToolSandboxError({ kind: "invoke", message: cause.message, cause }), + ), + ); + }, +}); diff --git a/packages/plugins/apps/src/plugin/publish-concurrency.test.ts b/packages/plugins/apps/src/plugin/publish-concurrency.test.ts new file mode 100644 index 000000000..e267ff5a4 --- /dev/null +++ b/packages/plugins/apps/src/plugin/publish-concurrency.test.ts @@ -0,0 +1,82 @@ +import { mkdtempSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +import { describe, expect, it } from "vitest"; +import { Effect } from "effect"; + +import { makeSelfHostAppsRuntime } from "./self-host-runtime"; +import { makeInMemoryAppsStore, makeTestResolver } from "../testing"; + +const run = (effect: Effect.Effect): Promise => Effect.runPromise(effect); + +// --------------------------------------------------------------------------- +// Finding 6 regression: concurrent publishes to one scope must not race the git +// ref update or the descriptor-pointer write. They serialize (both succeed in +// sequence) or one gets a typed conflict; afterwards the committed HEAD and the +// stored descriptor pointer always AGREE. Before the fix the ref update had no +// expected-old-value (last-writer-wins clobber) and the pointer write raced. +// +// Uses the REAL git-backed artifact store (the git-CAS is the mechanism under +// test) via `inMemory: false`. +// --------------------------------------------------------------------------- + +const toolFiles = (tag: string): ReadonlyMap => + new Map([ + [ + "tools/echo.ts", + `import { defineTool } from "executor:app"; +import { z } from "zod"; +export default defineTool({ + description: "echo ${tag}", + input: z.object({}), + async handler(){ return { tag: "${tag}" }; }, +});`, + ], + ]); + +describe("concurrent publishes to one scope (Fix 6)", () => { + it("serialize with head and descriptor pointer always in agreement", async () => { + const store = makeInMemoryAppsStore(); + const resolver = makeTestResolver({}); + const host = makeSelfHostAppsRuntime({ + dataDir: mkdtempSync(join(tmpdir(), "apps-pubrace-")), + store, + resolver, + // Real git-backed artifact store: exercise the ref compare-and-swap. + inMemory: false, + }); + const { runtime } = host; + + // Fire many concurrent publishes to the SAME scope. + const N = 8; + const results = await Promise.allSettled( + Array.from({ length: N }, (_v, i) => + run(runtime.publish({ scope: "s", files: toolFiles(`v${i}`) })), + ), + ); + + // Every publish either succeeded or failed with a typed conflict; none + // clobbered another silently. + const succeeded = results.filter((r) => r.status === "fulfilled"); + expect(succeeded.length).toBeGreaterThan(0); + for (const r of results) { + if (r.status === "rejected") { + // A conflict is the only acceptable failure mode. + expect(String(r.reason)).toMatch(/conflict/i); + } + } + + // HEAD and the descriptor pointer AGREE: the store's current descriptor's + // snapshotId equals the artifact store's latest committed snapshot. + const latest = await run( + runtime.deps.artifactStore.forScope("s").pipe(Effect.flatMap((s) => s.latest())), + ); + expect(latest).not.toBeNull(); + const pointer = await run(runtime.getDescriptor("s")); + expect(pointer).not.toBeNull(); + expect(pointer!.snapshotId).toBe(latest!.id); + + await host.close(); + }, 60_000); +}); diff --git a/packages/plugins/apps/src/plugin/runtime.test.ts b/packages/plugins/apps/src/plugin/runtime.test.ts new file mode 100644 index 000000000..e4a1b0153 --- /dev/null +++ b/packages/plugins/apps/src/plugin/runtime.test.ts @@ -0,0 +1,74 @@ +import { mkdtempSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +import { describe, expect, it } from "vitest"; +import { Effect } from "effect"; + +import { makeSelfHostAppsRuntime } from "./self-host-runtime"; +import { makeInMemoryAppsStore, makeTestResolver, dailyBriefFileSet } from "../testing"; +import type { Bindings } from "./bindings"; + +const run = (effect: Effect.Effect): Promise => Effect.runPromise(effect); + +const githubHandlers = { + github: { + "repos.listForAuthenticatedUser": () => [{ full_name: "acme/app" }], + "issues.listForRepo": () => [ + { + number: 1, + title: "Fresh bug", + labels: [{ name: "bug" }], + assignee: { login: "rhys" }, + updated_at: new Date().toISOString(), + html_url: "https://github.com/acme/app/issues/1", + }, + { + number: 2, + title: "Old bug", + labels: [], + assignee: null, + updated_at: "2020-01-01T00:00:00Z", + html_url: "https://github.com/acme/app/issues/2", + }, + ], + }, +}; + +const githubBindings: Bindings = { github: { kind: "single", connection: "rhys-github" } }; + +describe("AppsRuntime end-to-end (publish -> invoke)", () => { + it("publishes daily-brief and invokes the tool into the scope db", async () => { + const store = makeInMemoryAppsStore(); + const resolver = makeTestResolver(githubHandlers); + const host = makeSelfHostAppsRuntime({ + dataDir: mkdtempSync(join(tmpdir(), "apps-rt-")), + store, + resolver, + inMemory: true, + }); + const { runtime } = host; + + const published = await run(runtime.publish({ scope: "rhys", files: dailyBriefFileSet() })); + expect(published.descriptor.tools.map((t) => t.name).sort()).toEqual([ + "issues-sync", + "search-all-mail", + ]); + + const syncResult = (await run( + runtime.invokeTool({ + scope: "rhys", + tool: "issues-sync", + args: {}, + bindings: githubBindings, + }), + )) as { synced: number; repos: number }; + expect(syncResult).toEqual({ synced: 2, repos: 1 }); + + const db = await run(host.scopeDb.forScope("rhys")); + const rows = await run(db.exec<{ n: number }>("SELECT COUNT(*) AS n FROM issues")); + expect(Number(rows[0].n)).toBe(2); + + await host.close(); + }); +}); diff --git a/packages/plugins/apps/src/plugin/runtime.ts b/packages/plugins/apps/src/plugin/runtime.ts new file mode 100644 index 000000000..0b47cc01f --- /dev/null +++ b/packages/plugins/apps/src/plugin/runtime.ts @@ -0,0 +1,295 @@ +import { Effect } from "effect"; + +import type { ArtifactStore } from "../seams/artifact-store"; +import type { ScopeDb } from "../seams/scope-db"; +import type { ToolSandbox } from "../seams/tool-sandbox"; +import { + publish as runPublish, + loadDescriptorFromSnapshot, + type PublishOutput, +} from "../pipeline/publish"; +import { PublishError } from "../pipeline/discover"; +import type { AppDescriptor, ToolDescriptor } from "../pipeline/descriptor"; +import { bundleEntry } from "../pipeline/bundle"; +import { + buildBridge, + rootsFor, + type Bindings, + type ClientResolver, + BindingError, +} from "./bindings"; +import type { AppsStore } from "./store"; + +// --------------------------------------------------------------------------- +// AppsRuntime: the substrate-neutral core for published custom tools. +// --------------------------------------------------------------------------- + +export interface AppsRuntimeDeps { + readonly artifactStore: ArtifactStore; + readonly scopeDb: ScopeDb; + readonly sandbox: ToolSandbox; + readonly store: AppsStore; + /** Routes a bound integration method call to the real API (policy/audit). */ + readonly resolver: ClientResolver; +} + +export interface AppsRuntime { + readonly publish: (input: { + readonly scope: string; + readonly files: ReadonlyMap; + readonly message?: string; + }) => Effect.Effect; + readonly getDescriptor: (scope: string) => Effect.Effect; + /** Re-derive the published-descriptor pointer from the latest committed + * snapshot. */ + readonly repair: (scope: string) => Effect.Effect; + readonly invokeTool: (input: { + readonly scope: string; + readonly tool: string; + readonly args: unknown; + readonly bindings: Bindings; + /** Optional per-request resolver override. The catalog invoke path supplies + * one built from the request's executor context. */ + readonly resolver?: ClientResolver; + }) => Effect.Effect; + readonly deps: AppsRuntimeDeps; +} + +const failNoDescriptor = (scope: string): PublishError => + new PublishError({ + message: `scope "${scope}" has no published app`, + stage: "project", + diagnostics: [], + }); + +export const makeAppsRuntime = (deps: AppsRuntimeDeps): AppsRuntime => { + const bundleCache = new Map(); + const publishChains = new Map>(); + + const withScopePublishLock = (scope: string, run: () => Promise): Promise => { + const prior = publishChains.get(scope) ?? Promise.resolve(); + const next = prior.catch(() => undefined).then(run); + publishChains.set( + scope, + next.catch(() => undefined), + ); + return next; + }; + + const bundleFor = ( + descriptor: AppDescriptor, + sourcePath: string, + ): Effect.Effect => + Effect.gen(function* () { + const cacheKey = `${descriptor.snapshotId}:${sourcePath}`; + const cached = bundleCache.get(cacheKey); + if (cached) return cached; + const scopeStore = yield* deps.artifactStore.forScope(descriptor.scope).pipe( + Effect.mapError( + (c) => + new PublishError({ + message: c.message, + stage: "project", + diagnostics: [], + }), + ), + ); + const files = yield* scopeStore.read(descriptor.snapshotId as never).pipe( + Effect.mapError( + (c) => + new PublishError({ + message: c.message, + stage: "project", + diagnostics: [], + }), + ), + ); + const bundle = yield* bundleEntry({ files, entry: sourcePath }).pipe( + Effect.mapError( + (c) => + new PublishError({ + message: c.message, + stage: "bundle", + diagnostics: [{ path: sourcePath, message: c.message }], + }), + ), + ); + bundleCache.set(cacheKey, bundle.code); + return bundle.code; + }); + + const recoverFromSnapshot = (scope: string): Effect.Effect => + Effect.gen(function* () { + const scopeStore = yield* deps.artifactStore.forScope(scope).pipe( + Effect.mapError( + (c) => + new PublishError({ + message: c.message, + stage: "project", + diagnostics: [], + }), + ), + ); + const latest = yield* scopeStore.latest().pipe( + Effect.mapError( + (c) => + new PublishError({ + message: c.message, + stage: "project", + diagnostics: [], + }), + ), + ); + if (!latest) return null; + return yield* loadDescriptorFromSnapshot(deps.artifactStore, scope, latest.id); + }); + + const requireDescriptor = (scope: string): Effect.Effect => + deps.store.getDescriptor(scope).pipe( + Effect.mapError( + (c) => + new PublishError({ + message: String(c), + stage: "project", + diagnostics: [], + }), + ), + Effect.flatMap((d) => + d ? Effect.succeed(d) : recoverFromSnapshot(scope).pipe(Effect.orElseSucceed(() => null)), + ), + Effect.flatMap((d) => (d ? Effect.succeed(d) : Effect.fail(failNoDescriptor(scope)))), + ); + + const putDescriptorPointer = (descriptor: AppDescriptor): Effect.Effect => + deps.store.putDescriptor("org", descriptor).pipe( + Effect.mapError( + (c) => + new PublishError({ + message: String(c), + stage: "project", + diagnostics: [], + }), + ), + ); + + const repairScope = (scope: string): Effect.Effect => + Effect.gen(function* () { + const descriptor = yield* recoverFromSnapshot(scope); + if (!descriptor) return null; + yield* putDescriptorPointer(descriptor); + return descriptor; + }); + + const invokeToolInternal = ( + scope: string, + descriptor: AppDescriptor, + toolDesc: ToolDescriptor, + args: unknown, + bindings: Bindings, + resolver?: ClientResolver, + ): Effect.Effect => + Effect.gen(function* () { + const roots = yield* rootsFor(toolDesc.connections, bindings); + const code = yield* bundleFor(descriptor, toolDesc.sourcePath); + const db = yield* deps.scopeDb.forScope(scope).pipe( + Effect.mapError( + (c) => + new PublishError({ + message: c.message, + stage: "project", + diagnostics: [], + }), + ), + ); + const bridge = buildBridge({ + declared: toolDesc.connections, + bindings, + db, + resolver: resolver ?? deps.resolver, + }); + const result = yield* deps.sandbox + .invoke(code, { artifact: toolDesc.name, kind: "tool", input: args, roots }, bridge) + .pipe( + Effect.mapError( + (c) => + new PublishError({ + message: c.message, + stage: "project", + diagnostics: [{ path: toolDesc.sourcePath, message: c.message }], + }), + ), + ); + return result.output; + }); + + return { + deps, + publish: (input) => + Effect.flatMap( + Effect.tryPromise({ + try: () => + withScopePublishLock(input.scope, () => + Effect.runPromiseExit( + Effect.gen(function* () { + const out = yield* runPublish( + { + artifactStore: deps.artifactStore, + sandbox: deps.sandbox, + }, + { + scope: input.scope, + files: input.files, + commitMessage: input.message, + }, + ); + yield* putDescriptorPointer(out.descriptor); + return out; + }), + ), + ), + catch: (cause) => + new PublishError({ + message: cause instanceof Error ? cause.message : String(cause), + stage: "project", + diagnostics: [], + }), + }), + (exit) => exit, + ), + + getDescriptor: (scope) => + deps.store.getDescriptor(scope).pipe( + Effect.orElseSucceed(() => null), + Effect.flatMap((pointer) => + pointer + ? Effect.succeed(pointer) + : recoverFromSnapshot(scope).pipe(Effect.orElseSucceed(() => null)), + ), + ), + + repair: (scope) => repairScope(scope), + + invokeTool: (input) => + Effect.gen(function* () { + const descriptor = yield* requireDescriptor(input.scope); + const toolDesc = descriptor.tools.find((t) => t.name === input.tool); + if (!toolDesc) { + return yield* Effect.fail( + new PublishError({ + message: `tool "${input.tool}" is not published in scope "${input.scope}"`, + stage: "project", + diagnostics: [], + }), + ); + } + return yield* invokeToolInternal( + input.scope, + descriptor, + toolDesc, + input.args, + input.bindings, + input.resolver, + ); + }), + }; +}; diff --git a/packages/plugins/apps/src/plugin/self-host-runtime.ts b/packages/plugins/apps/src/plugin/self-host-runtime.ts new file mode 100644 index 000000000..78735d3a3 --- /dev/null +++ b/packages/plugins/apps/src/plugin/self-host-runtime.ts @@ -0,0 +1,55 @@ +import { join } from "node:path"; + +import { Effect } from "effect"; + +import { makeGitArtifactStore } from "../backing/git-artifact-store"; +import { makeLibsqlScopeDb } from "../backing/libsql-scope-db"; +import { makeQuickjsToolSandbox } from "../backing/quickjs-tool-sandbox"; +import type { ScopeDb } from "../seams/scope-db"; +import { makeAppsRuntime, type AppsRuntime } from "./runtime"; +import type { AppsStore } from "./store"; +import type { ClientResolver } from "./bindings"; + +export interface SelfHostAppsRuntimeOptions { + /** Data dir root; `/artifacts` and `/scope-db`. */ + readonly dataDir: string; + readonly store: AppsStore; + /** Routes bound integration calls to real APIs (policy/audit). */ + readonly resolver: ClientResolver; + /** In-memory backings for tests. */ + readonly inMemory?: boolean; +} + +export interface SelfHostAppsRuntime { + readonly runtime: AppsRuntime; + readonly scopeDb: ScopeDb; + readonly close: () => Promise; +} + +export const makeSelfHostAppsRuntime = ( + options: SelfHostAppsRuntimeOptions, +): SelfHostAppsRuntime => { + const inMem = options.inMemory === true; + const artifactStore = makeGitArtifactStore({ + root: inMem ? options.dataDir : join(options.dataDir, "artifacts"), + }); + const scopeDb = makeLibsqlScopeDb({ + root: inMem ? ":memory:" : join(options.dataDir, "scope-db"), + }); + const sandbox = makeQuickjsToolSandbox(); + const runtime = makeAppsRuntime({ + artifactStore, + scopeDb, + sandbox, + store: options.store, + resolver: options.resolver, + }); + + return { + runtime, + scopeDb, + close: async () => { + await Effect.runPromise(scopeDb.close().pipe(Effect.orElseSucceed(() => undefined))); + }, + }; +}; diff --git a/packages/plugins/apps/src/plugin/store.ts b/packages/plugins/apps/src/plugin/store.ts new file mode 100644 index 000000000..d25780b51 --- /dev/null +++ b/packages/plugins/apps/src/plugin/store.ts @@ -0,0 +1,78 @@ +import { Effect } from "effect"; + +import { + definePluginStorageCollection, + type PluginStorageFacade, + type StorageFailure, +} from "@executor-js/sdk"; + +import type { AppDescriptor } from "../pipeline/descriptor"; + +// --------------------------------------------------------------------------- +// AppsStore: descriptor pointer persistence and connection-to-scope mapping. +// --------------------------------------------------------------------------- + +export const descriptorCollection = definePluginStorageCollection("published_descriptor", { + Type: {} as { + readonly scope: string; + readonly snapshotId: string; + readonly descriptor: AppDescriptor; + readonly publishedAt: number; + }, +}); + +export const scopeConnectionCollection = definePluginStorageCollection("apps_scope_connection", { + Type: {} as { + readonly connectionName: string; + readonly scope: string; + }, +}); + +export interface AppsStore { + readonly putDescriptor: ( + owner: "org" | "user", + descriptor: AppDescriptor, + ) => Effect.Effect; + readonly getDescriptor: (scope: string) => Effect.Effect; + readonly putScopeForConnection: ( + connectionName: string, + scope: string, + ) => Effect.Effect; + readonly getScopeForConnection: ( + connectionName: string, + ) => Effect.Effect; +} + +export interface AppsStoreDeps { + readonly pluginStorage: PluginStorageFacade; +} + +export const makeAppsStore = (deps: AppsStoreDeps): AppsStore => { + const descriptors = deps.pluginStorage.collection(descriptorCollection); + const scopeConnections = deps.pluginStorage.collection(scopeConnectionCollection); + return { + putScopeForConnection: (connectionName, scope) => + scopeConnections + .put({ owner: "org", key: connectionName, data: { connectionName, scope } }) + .pipe(Effect.asVoid), + getScopeForConnection: (connectionName) => + scopeConnections + .get({ key: connectionName }) + .pipe(Effect.map((entry) => entry?.data.scope ?? null)), + putDescriptor: (owner, descriptor) => + descriptors + .put({ + owner, + key: descriptor.scope, + data: { + scope: descriptor.scope, + snapshotId: descriptor.snapshotId, + descriptor, + publishedAt: Date.now(), + }, + }) + .pipe(Effect.asVoid), + getDescriptor: (scope) => + descriptors.get({ key: scope }).pipe(Effect.map((entry) => entry?.data.descriptor ?? null)), + }; +}; diff --git a/packages/plugins/apps/src/seams/artifact-store.conformance.ts b/packages/plugins/apps/src/seams/artifact-store.conformance.ts new file mode 100644 index 000000000..e99460d3c --- /dev/null +++ b/packages/plugins/apps/src/seams/artifact-store.conformance.ts @@ -0,0 +1,91 @@ +import { describe, expect, it } from "vitest"; +import { Effect } from "effect"; + +import type { ArtifactStore, FileSet } from "./artifact-store"; + +// --------------------------------------------------------------------------- +// ArtifactStore conformance suite. Runs against the INTERFACE, not a specific +// backing — pass a factory that yields a fresh store. Any future backing +// (Cloudflare Artifacts) must pass this same suite. Covers: round-trip +// (write a file set, read it back identical), snapshot immutability (a second +// publish does not change the first snapshot's bytes), latest/log ordering. +// --------------------------------------------------------------------------- + +const run = (effect: Effect.Effect): Promise => Effect.runPromise(effect); + +const fileSet = (entries: Record): FileSet => new Map(Object.entries(entries)); + +export const artifactStoreConformance = ( + name: string, + makeStore: () => Promise | ArtifactStore, +): void => { + describe(`ArtifactStore conformance: ${name}`, () => { + it("round-trips a file set through a snapshot", async () => { + const store = await makeStore(); + const scope = await run(store.forScope("s1")); + const files = fileSet({ + "tools/a.ts": "export const a = 1;\n", + "notes/x.md": "# x\n", + }); + const meta = await run(scope.commit(files, "publish 1")); + expect(meta.id).toBeTruthy(); + expect(meta.message).toBe("publish 1"); + + const readBack = await run(scope.read(meta.id)); + expect(readBack.get("tools/a.ts")).toBe("export const a = 1;\n"); + expect(readBack.get("notes/x.md")).toBe("# x\n"); + + const paths = await run(scope.list(meta.id)); + expect([...paths].sort()).toEqual(["notes/x.md", "tools/a.ts"]); + + const one = await run(scope.readFile(meta.id, "tools/a.ts")); + expect(one).toBe("export const a = 1;\n"); + const missing = await run(scope.readFile(meta.id, "tools/missing.ts")); + expect(missing).toBeNull(); + }); + + it("keeps a snapshot immutable across a later publish", async () => { + const store = await makeStore(); + const scope = await run(store.forScope("s2")); + const first = await run(scope.commit(fileSet({ "tools/a.ts": "v1" }), "first")); + const second = await run( + scope.commit(fileSet({ "tools/a.ts": "v2", "tools/b.ts": "new" }), "second"), + ); + + expect(second.id).not.toBe(first.id); + // The first snapshot still reads its original bytes. + const firstFiles = await run(scope.read(first.id)); + expect(firstFiles.get("tools/a.ts")).toBe("v1"); + expect(firstFiles.has("tools/b.ts")).toBe(false); + // The second snapshot has the new bytes. + const secondFiles = await run(scope.read(second.id)); + expect(secondFiles.get("tools/a.ts")).toBe("v2"); + expect(secondFiles.get("tools/b.ts")).toBe("new"); + }); + + it("tracks latest and logs newest-first", async () => { + const store = await makeStore(); + const scope = await run(store.forScope("s3")); + expect(await run(scope.latest())).toBeNull(); + + const a = await run(scope.commit(fileSet({ "tools/a.ts": "1" }), "a")); + const b = await run(scope.commit(fileSet({ "tools/a.ts": "2" }), "b")); + + const latest = await run(scope.latest()); + expect(latest?.id).toBe(b.id); + + const log = await run(scope.log()); + expect(log[0].id).toBe(b.id); + expect(log[1].id).toBe(a.id); + expect(log.map((m) => m.message)).toEqual(["b", "a"]); + }); + + it("isolates scopes", async () => { + const store = await makeStore(); + const s1 = await run(store.forScope("iso-1")); + const s2 = await run(store.forScope("iso-2")); + await run(s1.commit(fileSet({ "tools/a.ts": "one" }), "one")); + expect(await run(s2.latest())).toBeNull(); + }); + }); +}; diff --git a/packages/plugins/apps/src/seams/artifact-store.ts b/packages/plugins/apps/src/seams/artifact-store.ts new file mode 100644 index 000000000..fcfd5bd4b --- /dev/null +++ b/packages/plugins/apps/src/seams/artifact-store.ts @@ -0,0 +1,64 @@ +import type { Effect } from "effect"; +import { Data } from "effect"; + +// --------------------------------------------------------------------------- +// ArtifactStore — the per-scope git-backed source store. +// +// A scope's source is stored as a bare git repository; a publish writes a commit +// and the commit hash IS the snapshot id (immutable, content-addressed by git). +// The self-hosted backing is a bare git repo per scope on disk (git CLI via +// subprocess). The cloud backing (future) is Cloudflare Artifacts. The seam is +// substrate-neutral: read/write/list/latest/log over a flat file set, plus a +// snapshot immutability guarantee (a committed snapshot's bytes never change). +// --------------------------------------------------------------------------- + +/** A published snapshot id. In the git backing this is the commit hash. */ +export type SnapshotId = string & { readonly __snapshotId: unique symbol }; + +export const asSnapshotId = (value: string): SnapshotId => value as SnapshotId; + +/** A flat set of source files, path -> UTF-8 contents. Paths are POSIX, + * relative to the scope root (e.g. `tools/issues-sync.ts`). */ +export type FileSet = ReadonlyMap; + +export interface SnapshotMeta { + readonly id: SnapshotId; + readonly message: string; + /** Epoch ms the snapshot was written. */ + readonly committedAt: number; +} + +export class ArtifactStoreError extends Data.TaggedError("ArtifactStoreError")<{ + readonly message: string; + /** True when the failure is a concurrent-write conflict (the snapshot ref + * moved under a compare-and-swap). Callers retry from the fresh head or + * surface a typed conflict rather than clobbering the other writer. */ + readonly conflict?: boolean; + readonly cause?: unknown; +}> {} + +/** + * One scope's source store. `commit` writes a new snapshot from a full file set + * and returns its id. `read` materializes a snapshot back to a file set. + * `latest` is the most recent snapshot on the default branch (null when the + * scope has never published). `log` lists snapshots newest-first. + */ +export interface ScopeArtifactStore { + readonly commit: ( + files: FileSet, + message: string, + ) => Effect.Effect; + readonly read: (id: SnapshotId) => Effect.Effect; + readonly readFile: ( + id: SnapshotId, + path: string, + ) => Effect.Effect; + readonly list: (id: SnapshotId) => Effect.Effect; + readonly latest: () => Effect.Effect; + readonly log: (limit?: number) => Effect.Effect; +} + +/** The substrate-neutral store: hands out a per-scope store by scope key. */ +export interface ArtifactStore { + readonly forScope: (scope: string) => Effect.Effect; +} diff --git a/packages/plugins/apps/src/seams/index.ts b/packages/plugins/apps/src/seams/index.ts new file mode 100644 index 000000000..024108bd8 --- /dev/null +++ b/packages/plugins/apps/src/seams/index.ts @@ -0,0 +1,6 @@ +// The substrate-neutral seams. Self-hosted backings are under +// `src/backing/`; each seam has a conformance suite (`*.conformance.ts`) that a +// backing must pass, keeping a future Cloudflare backing honest. +export * from "./artifact-store"; +export * from "./scope-db"; +export * from "./tool-sandbox"; diff --git a/packages/plugins/apps/src/seams/scope-db.conformance.ts b/packages/plugins/apps/src/seams/scope-db.conformance.ts new file mode 100644 index 000000000..a8342fd5c --- /dev/null +++ b/packages/plugins/apps/src/seams/scope-db.conformance.ts @@ -0,0 +1,69 @@ +import { describe, expect, it } from "vitest"; +import { Effect } from "effect"; + +import type { ScopeDb } from "./scope-db"; + +// --------------------------------------------------------------------------- +// ScopeDb conformance suite. Runs against the interface. Covers: scope +// isolation (a write in scope A is invisible in scope B), per-table version +// bumps on write, and tagged-template parameters. +// --------------------------------------------------------------------------- + +const run = (effect: Effect.Effect): Promise => Effect.runPromise(effect); + +export const scopeDbConformance = ( + name: string, + makeDb: () => Promise | ScopeDb, +): void => { + describe(`ScopeDb conformance: ${name}`, () => { + it("isolates data between scopes", async () => { + const db = await makeDb(); + const a = await run(db.forScope("scope-a")); + const b = await run(db.forScope("scope-b")); + await run(a.exec("CREATE TABLE items (id INTEGER PRIMARY KEY, name TEXT)")); + await run(a.exec("INSERT INTO items (name) VALUES ('a-only')")); + await run(b.exec("CREATE TABLE items (id INTEGER PRIMARY KEY, name TEXT)")); + + const aRows = await run(a.exec<{ name: string }>("SELECT name FROM items")); + const bRows = await run(b.exec<{ name: string }>("SELECT name FROM items")); + expect(aRows.map((r) => r.name)).toEqual(["a-only"]); + expect(bRows).toEqual([]); + await run(db.close()); + }); + + it("bumps a table's version on each write", async () => { + const db = await makeDb(); + const s = await run(db.forScope("ver")); + await run(s.exec("CREATE TABLE t (id INTEGER PRIMARY KEY)")); + const afterCreate = await run(s.tableVersion("t")); + expect(afterCreate).toBeGreaterThanOrEqual(1); + + await run(s.exec("INSERT INTO t (id) VALUES (1)")); + const afterInsert = await run(s.tableVersion("t")); + expect(afterInsert).toBe(afterCreate + 1); + + await run(s.exec("UPDATE t SET id = 2 WHERE id = 1")); + const afterUpdate = await run(s.tableVersion("t")); + expect(afterUpdate).toBe(afterInsert + 1); + + // Reads do not bump. + await run(s.exec("SELECT * FROM t")); + expect(await run(s.tableVersion("t"))).toBe(afterUpdate); + await run(db.close()); + }); + + it("supports the author-facing tagged-template sql with parameters", async () => { + const db = await makeDb(); + const s = await run(db.forScope("tpl")); + await run(s.exec("CREATE TABLE issues (repo TEXT, title TEXT)")); + const repo = "acme/app"; + const title = "Bug"; + await run(s.sql`INSERT INTO issues (repo, title) VALUES (${repo}, ${title})`); + const rows = await run( + s.sql<{ repo: string; title: string }>`SELECT * FROM issues WHERE repo = ${repo}`, + ); + expect(rows).toEqual([{ repo: "acme/app", title: "Bug" }]); + await run(db.close()); + }); + }); +}; diff --git a/packages/plugins/apps/src/seams/scope-db.ts b/packages/plugins/apps/src/seams/scope-db.ts new file mode 100644 index 000000000..a58538d0a --- /dev/null +++ b/packages/plugins/apps/src/seams/scope-db.ts @@ -0,0 +1,42 @@ +import type { Effect } from "effect"; +import { Data } from "effect"; + +// --------------------------------------------------------------------------- +// ScopeDb — the per-scope app database (the shared primitive). +// +// One SQL database per scope: tools read and write it. The self-hosted backing +// is one libSQL (SQLite) file per scope; the cloud backing (future) is a +// Durable Object facet. The seam carries a template-tag `sql` (the author-facing +// shape) plus per-table version counters. +// +// Scope isolation is structural: `forScope(a)` and `forScope(b)` are distinct +// databases; there is no cross-scope query path. +// --------------------------------------------------------------------------- + +export class ScopeDbError extends Data.TaggedError("ScopeDbError")<{ + readonly message: string; + readonly cause?: unknown; +}> {} + +/** The author-facing db handle (matches `executor:app`'s `ScopeDb`): a tagged + * template returning rows. */ +export interface ScopeDbHandle { + readonly sql: >( + strings: TemplateStringsArray, + ...values: unknown[] + ) => Effect.Effect; + /** Run a raw statement (used by the runtime for probes/migrations). */ + readonly exec: >( + sql: string, + params?: readonly unknown[], + ) => Effect.Effect; + /** Current version counter for a table (0 if never written). */ + readonly tableVersion: (table: string) => Effect.Effect; + /** Snapshot of every tracked table's version. */ + readonly versions: () => Effect.Effect, ScopeDbError>; +} + +export interface ScopeDb { + readonly forScope: (scope: string) => Effect.Effect; + readonly close: () => Effect.Effect; +} diff --git a/packages/plugins/apps/src/seams/tool-sandbox.conformance.ts b/packages/plugins/apps/src/seams/tool-sandbox.conformance.ts new file mode 100644 index 000000000..119ab84b6 --- /dev/null +++ b/packages/plugins/apps/src/seams/tool-sandbox.conformance.ts @@ -0,0 +1,216 @@ +import { describe, expect, it } from "vitest"; +import { Effect } from "effect"; + +import type { HandleBridge, ToolSandbox } from "./tool-sandbox"; +import { bundleEntry } from "../pipeline/bundle"; + +// --------------------------------------------------------------------------- +// ToolSandbox conformance suite. Runs against the interface. Covers: +// - collect determinism catches Math.random (double-run byte-compare) +// - network denial (fetch throws) +// - timeout kill (an infinite loop is interrupted) +// - handle bridge round-trip including fan-out arrays (connections("x")) +// A future Worker Loaders backing must pass this same suite. +// --------------------------------------------------------------------------- + +const run = (effect: Effect.Effect): Promise => Effect.runPromise(effect); + +const bundle = (entry: string, source: string): Promise => + run(bundleEntry({ files: new Map([[entry, source]]), entry })).then((b) => b.code); + +export const toolSandboxConformance = (name: string, makeSandbox: () => ToolSandbox): void => { + describe(`ToolSandbox conformance: ${name}`, () => { + it("collects deterministically and rejects Math.random at describe time", async () => { + const sandbox = makeSandbox(); + const stable = `import { defineTool } from "executor:app"; +import { z } from "zod"; +export default defineTool({ description: "stable", input: z.object({ a: z.string() }), async handler(){ return {}; } });`; + const stableBundle = await bundle("tools/s.ts", stable); + const res = await run(sandbox.collect(stableBundle)); + expect(res.artifacts.default.descriptor).toBeTruthy(); + + const rng = `import { defineTool } from "executor:app"; +import { z } from "zod"; +export default defineTool({ description: "x" + Math.random(), input: z.object({}), async handler(){ return {}; } });`; + const rngBundle = await bundle("tools/rng.ts", rng); + const exit = await Effect.runPromiseExit(sandbox.collect(rngBundle)); + expect(exit._tag).toBe("Failure"); + }); + + it("denies network access", async () => { + const sandbox = makeSandbox(); + const src = `import { defineTool } from "executor:app"; +import { z } from "zod"; +export default defineTool({ description: "net", input: z.object({}), async handler(){ await fetch("https://x.test"); return {}; } });`; + const b = await bundle("tools/net.ts", src); + const bridge: HandleBridge = { call: () => Effect.succeed(null) }; + const exit = await Effect.runPromiseExit( + sandbox.invoke(b, { artifact: "net", kind: "tool", input: {}, roots: {} }, bridge), + ); + expect(exit._tag).toBe("Failure"); + }); + + it("kills a runaway handler on timeout", async () => { + const sandbox = makeSandbox(); + const src = `import { defineTool } from "executor:app"; +import { z } from "zod"; +export default defineTool({ description: "loop", input: z.object({}), async handler(){ while (true) {} } });`; + const b = await bundle("tools/loop.ts", src); + const bridge: HandleBridge = { call: () => Effect.succeed(null) }; + const exit = await Effect.runPromiseExit( + sandbox.invoke(b, { artifact: "loop", kind: "tool", input: {}, roots: {} }, bridge), + ); + expect(exit._tag).toBe("Failure"); + }); + + it("round-trips the handle bridge, including fan-out arrays", async () => { + const sandbox = makeSandbox(); + // A tool that fans out over an array of clients and calls a method on each. + const src = `import { defineTool, connections } from "executor:app"; +import { z } from "zod"; +export default defineTool({ + description: "fanout", + connections: { inboxes: connections("gmail") }, + input: z.object({ q: z.string() }), + async handler({ q }, { inboxes }) { + const per = await Promise.all(inboxes.map(async (inbox, i) => { + const r = await inbox.messages.search({ q, index: i }); + return r.count; + })); + return { total: per.reduce((a, b) => a + b, 0), n: inboxes.length }; + }, +});`; + const b = await bundle("tools/fanout.ts", src); + + const seen: { root: string; path: readonly string[]; args: readonly unknown[] }[] = []; + const bridge: HandleBridge = { + call: ({ root, path, args }) => + Effect.sync(() => { + seen.push({ root, path, args }); + return { count: 3 }; + }), + }; + + const result = await run( + sandbox.invoke( + b, + { + artifact: "fanout", + kind: "tool", + input: { q: "invoice" }, + roots: { inboxes: { kind: "array", count: 2 } }, + }, + bridge, + ), + ); + expect(result.output).toEqual({ total: 6, n: 2 }); + // Two distinct fan-out roots were addressed (inboxes#0 and inboxes#1). + const roots = new Set(seen.map((s) => s.root)); + expect(roots.has("inboxes#0")).toBe(true); + expect(roots.has("inboxes#1")).toBe(true); + // The method path and JSON args crossed the boundary intact. + expect(seen[0].path).toEqual(["messages", "search"]); + expect(seen[0].args[0]).toMatchObject({ q: "invoice" }); + }); + + // --- richer assertions grafted from build A --------------------------- + + it("round-trips a db.sql write with its parameters intact", async () => { + const sandbox = makeSandbox(); + // A tool with a github client AND a db.sql write, asserting BOTH the + // connection call and the parameterized db statement cross the bridge with + // their args intact (the db.sql param round-trip A's suite covers). + const src = `import { defineTool, connection } from "executor:app"; +import { z } from "zod"; +export default defineTool({ + description: "bridge", + connections: { gh: connection("github") }, + input: z.object({}), + async handler(_i, { gh, db }) { + const repos = await gh.repos.listForAuthenticatedUser({ per_page: 5 }); + await db.sql\`INSERT INTO log (n) VALUES (\${repos.length})\`; + return { count: repos.length }; + }, +});`; + const b = await bundle("tools/bridge.ts", src); + const calls: { root: string; path: readonly string[]; args: readonly unknown[] }[] = []; + const bridge: HandleBridge = { + call: ({ root, path, args }) => + Effect.sync(() => { + calls.push({ root, path, args }); + if (root === "gh" && path.join(".") === "repos.listForAuthenticatedUser") { + return [{ full_name: "a/b" }, { full_name: "c/d" }]; + } + return []; + }), + }; + const result = await run( + sandbox.invoke( + b, + { + artifact: "bridge", + kind: "tool", + input: {}, + roots: { gh: { kind: "single" }, db: { kind: "single" } }, + }, + bridge, + ), + ); + expect(result.output).toEqual({ count: 2 }); + // The github call crossed the bridge. + expect(calls.some((c) => c.root === "gh")).toBe(true); + // The db.sql write crossed as ["sql"] with the template strings + the + // interpolated parameter (2) intact. + const dbCall = calls.find((c) => c.root === "db"); + expect(dbCall?.path).toEqual(["sql"]); + const [strings, ...params] = dbCall!.args as [readonly string[], ...unknown[]]; + expect(strings.join("?")).toContain("INSERT INTO log"); + expect(params).toEqual([2]); + }); + + it("fans out per-index, each element addressed distinctly", async () => { + const sandbox = makeSandbox(); + // inbox i returns (i+1) messages; the handler pushes each count, so the + // ordered result proves each fan-out element was addressed with its index. + const src = `import { defineTool, connections } from "executor:app"; +import { z } from "zod"; +export default defineTool({ + description: "fanout", + connections: { inboxes: connections("gmail") }, + input: z.object({}), + async handler(_i, { inboxes }) { + const counts = []; + for (const inbox of inboxes) { + const r = await inbox.messages.search({ q: "x" }); + counts.push(r.length); + } + return { inboxes: inboxes.length, counts }; + }, +});`; + const b = await bundle("tools/fanout2.ts", src); + const bridge: HandleBridge = { + call: ({ root, path }) => + Effect.sync(() => { + if (path.join(".") === "messages.search") { + const idx = Number(root.split("#")[1]); + return new Array(idx + 1).fill({ id: "m" }); + } + return []; + }), + }; + const result = await run( + sandbox.invoke( + b, + { + artifact: "fanout2", + kind: "tool", + input: {}, + roots: { inboxes: { kind: "array", count: 3 } }, + }, + bridge, + ), + ); + expect(result.output).toEqual({ inboxes: 3, counts: [1, 2, 3] }); + }); + }); +}; diff --git a/packages/plugins/apps/src/seams/tool-sandbox.ts b/packages/plugins/apps/src/seams/tool-sandbox.ts new file mode 100644 index 000000000..f2d717e23 --- /dev/null +++ b/packages/plugins/apps/src/seams/tool-sandbox.ts @@ -0,0 +1,105 @@ +import type { Effect } from "effect"; +import { Data } from "effect"; + +// --------------------------------------------------------------------------- +// ToolSandbox — the isolated substrate that runs published bundles. +// +// Two operations, both over a bundled JS string (esbuild output, platform +// modules external, zod inlined): +// +// collect(bundle): import the bundle with NOTHING bound. `defineTool` returns +// a descriptor; a collector shim gathers it and returns JSON. This is how +// the publish pipeline extracts the versioned descriptor from source without +// ever running effectful code. Run twice + byte-compare = the determinism +// gate (a bundle that reads Math.random / Date.now at the top level or in a +// describe path diverges and is rejected). +// +// invoke(bundle, request): run one artifact's handler. The handler receives +// pre-bound clients whose method calls cross OUT through a serializable +// bridge (`HandleBridge.call`). EVERYTHING crossing the boundary is +// serializable — the cloud version of this seam is an RPC, so the interface +// forbids passing functions or runtime objects across. Fan-out arrays +// (`connections("gmail")` -> client[]) are modeled as indexed handle roots. +// +// The self-hosted backing is QuickJS (packages/kernel/runtime-quickjs), whose +// `SandboxToolInvoker.invoke({path, args})` already matches the bridge shape. +// The cloud backing (future) is Worker Loaders. The Deno subprocess kernel is +// the harder-isolation escalation behind this same seam. +// --------------------------------------------------------------------------- + +export class ToolSandboxError extends Data.TaggedError("ToolSandboxError")<{ + readonly message: string; + readonly kind: "collect" | "invoke" | "timeout" | "network" | "nondeterministic" | "bundle"; + readonly cause?: unknown; +}> {} + +/** + * The serializable bridge the sandbox calls out through. `root` names an + * injected handle (a connection role, `db`, or one element of a fan-out set); + * `path` is the method chain (`["events", "list"]`); `args` is the JSON call + * arguments. The return value is JSON. This is the ONE way sandboxed code + * reaches the host — nothing else is wired. + */ +export interface HandleBridge { + readonly call: (input: { + readonly root: string; + readonly path: readonly string[]; + readonly args: readonly unknown[]; + }) => Effect.Effect; +} + +/** Which handle roots to inject and, for fan-out roots, how many elements. A + * single connection role is `{ kind: "single" }`; a fan-out is + * `{ kind: "array", count }`; `db` is a single. Everything the handler can see + * is enumerated here — undeclared roots are simply absent. */ +export type HandleRootSpec = + | { readonly kind: "single" } + | { readonly kind: "array"; readonly count: number }; + +export interface InvokeRequest { + /** The artifact whose handler to run (path identity, e.g. `issues-sync`). */ + readonly artifact: string; + /** The kind selects the wrapper the sandbox uses to reach the handler. */ + readonly kind: "tool"; + /** JSON input passed to the handler. */ + readonly input: unknown; + /** The handle roots to inject, keyed by the name the handler destructures + * (`github`, `db`, `inboxes`). */ + readonly roots: Readonly>; +} + +export interface InvokeResult { + readonly output: unknown; + readonly logs: readonly string[]; +} + +/** A collected artifact descriptor — the JSON `defineTool` returns. The pipeline + * refines this into the versioned descriptor; the sandbox only guarantees it + * is deterministic JSON. */ +export interface CollectedArtifact { + readonly kind: "tool"; + readonly descriptor: unknown; +} + +export interface CollectResult { + /** Descriptors keyed by artifact path identity. */ + readonly artifacts: Readonly>; +} + +export interface ToolSandbox { + /** + * Import the bundle with nothing bound and gather the `defineTool` descriptor. + * Runs the collection twice internally and byte-compares; a mismatch fails + * with `kind: "nondeterministic"`. This is the determinism gate. + */ + readonly collect: (bundle: string) => Effect.Effect; + /** + * Run one artifact's handler with injected handles bridged through `bridge`. + * Network is denied; a per-call timeout kills a runaway handler. + */ + readonly invoke: ( + bundle: string, + request: InvokeRequest, + bridge: HandleBridge, + ) => Effect.Effect; +} diff --git a/packages/plugins/apps/src/testing/daily-brief.ts b/packages/plugins/apps/src/testing/daily-brief.ts new file mode 100644 index 000000000..0bbfab182 --- /dev/null +++ b/packages/plugins/apps/src/testing/daily-brief.ts @@ -0,0 +1,100 @@ +export const DAILY_BRIEF_MANIFEST = `{ + "$schema": "https://executor.sh/schemas/scope-manifest.json", + "scope": "rhys", + "description": "Personal scope artifacts, GitHub issues brief.", + "connections": { "github": "github/rhys" }, + "artifacts": { "tools": "tools/" } +} +`; + +export const ISSUES_SYNC_TS = `import { z } from "zod"; +import { defineTool, connection } from "executor:app"; + +export default defineTool({ + description: + "Refresh the scope \\\`issues\\\` table from GitHub. Syncs open issues across the given repos (default: every repo the connection can see).", + connections: { + github: connection("github", { description: "GitHub account whose issues to sync" }), + }, + input: z.object({ + repos: z.array(z.string()).optional().describe("owner/repo entries; omit to sync all accessible repos"), + since: z.string().optional().describe("ISO timestamp, only issues updated after this"), + }), + output: z.object({ synced: z.number(), repos: z.number() }), + annotations: { readOnly: false, destructive: false }, + async handler({ repos, since }, { github, db }) { + const targets = + repos ?? + (await github.repos.listForAuthenticatedUser({ per_page: 100 })).map((r) => r.full_name); + + await db.sql\` + CREATE TABLE IF NOT EXISTS issues ( + repo TEXT NOT NULL, number INTEGER NOT NULL, title TEXT NOT NULL, + labels TEXT NOT NULL DEFAULT '[]', assignee TEXT, updated_at TEXT NOT NULL, url TEXT NOT NULL, + PRIMARY KEY (repo, number) + )\`; + + let synced = 0; + for (const target of targets) { + const [owner, repo] = target.split("/"); + const issues = await github.issues.listForRepo({ owner, repo, state: "open", since, per_page: 100 }); + for (const issue of issues) { + if (issue.pull_request) continue; + await db.sql\` + INSERT INTO issues (repo, number, title, labels, assignee, updated_at, url) + VALUES (\${target}, \${issue.number}, \${issue.title}, + \${JSON.stringify(issue.labels.map((l) => l.name))}, + \${issue.assignee?.login ?? null}, \${issue.updated_at}, \${issue.html_url}) + ON CONFLICT (repo, number) DO UPDATE SET + title = excluded.title, labels = excluded.labels, + assignee = excluded.assignee, updated_at = excluded.updated_at\`; + synced++; + } + } + return { synced, repos: targets.length }; + }, +}); +`; + +export const SEARCH_ALL_MAIL_TS = `import { z } from "zod"; +import { defineTool, connections } from "executor:app"; + +export default defineTool({ + description: + "Search across all connected Gmail accounts. Returns matches newest-first, tagged with which inbox they came from.", + connections: { + inboxes: connections("gmail", { description: "Gmail accounts to search across" }), + }, + input: z.object({ + query: z.string().describe("Gmail search syntax, e.g. from:acme subject:invoice"), + limit: z.number().int().min(1).max(100).default(25), + }), + output: z.object({ + results: z.array(z.object({ + inbox: z.string(), id: z.string(), from: z.string(), + subject: z.string(), snippet: z.string(), date: z.string(), + })), + }), + annotations: { readOnly: true }, + async handler({ query, limit }, { inboxes }) { + const perInbox = await Promise.all( + inboxes.map(async (inbox) => { + const { messages } = await inbox.messages.search({ q: query, maxResults: limit }); + return messages.map((m) => ({ + inbox: inbox.account.email, id: m.id, from: m.from, + subject: m.subject, snippet: m.snippet, date: m.date, + })); + }), + ); + const results = perInbox.flat().sort((a, b) => b.date.localeCompare(a.date)).slice(0, limit); + return { results }; + }, +}); +`; + +export const dailyBriefFileSet = (): Map => + new Map([ + ["executor.json", DAILY_BRIEF_MANIFEST], + ["tools/issues-sync.ts", ISSUES_SYNC_TS], + ["tools/search-all-mail.ts", SEARCH_ALL_MAIL_TS], + ]); diff --git a/packages/plugins/apps/src/testing/index.ts b/packages/plugins/apps/src/testing/index.ts new file mode 100644 index 000000000..e7f9626b2 --- /dev/null +++ b/packages/plugins/apps/src/testing/index.ts @@ -0,0 +1,100 @@ +import { Effect } from "effect"; + +import type { AppDescriptor } from "../pipeline/descriptor"; +import type { AppsStore } from "../plugin/store"; +import type { ClientResolver, BindingError } from "../plugin/bindings"; +import { + ArtifactStoreError, + asSnapshotId, + type ArtifactStore, + type FileSet, + type ScopeArtifactStore, + type SnapshotId, +} from "../seams/artifact-store"; + +export * from "./daily-brief"; + +export const makeInMemoryAppsStore = (): AppsStore & { + readonly descriptors: Map; +} => { + const descriptors = new Map(); + const scopeConnections = new Map(); + return { + descriptors, + putDescriptor: (_owner, descriptor) => + Effect.sync(() => void descriptors.set(descriptor.scope, descriptor)), + getDescriptor: (scope) => Effect.sync(() => descriptors.get(scope) ?? null), + putScopeForConnection: (connectionName, scope) => + Effect.sync(() => void scopeConnections.set(connectionName, scope)), + getScopeForConnection: (connectionName) => + Effect.sync(() => scopeConnections.get(connectionName) ?? null), + }; +}; + +export const makeInMemoryArtifactStore = (): ArtifactStore => { + const scopes = new Map>(); + const order = new Map(); + let counter = 0; + const forScope = (scope: string): ScopeArtifactStore => { + const snaps = scopes.get(scope) ?? new Map(); + scopes.set(scope, snaps); + const seq = order.get(scope) ?? []; + order.set(scope, seq); + return { + commit: (files, message) => + Effect.sync(() => { + const id = `mem${(++counter).toString(16).padStart(40, "0")}`; + snaps.set(id, new Map(files)); + seq.push(id); + return { id: asSnapshotId(id), message, committedAt: Date.now() }; + }), + read: (id) => + snaps.has(id) + ? Effect.succeed(snaps.get(id)!) + : Effect.fail(new ArtifactStoreError({ message: `no snapshot ${id}` })), + readFile: (id, path) => Effect.succeed(snaps.get(id)?.get(path) ?? null), + list: (id) => Effect.succeed([...(snaps.get(id)?.keys() ?? [])]), + latest: () => + Effect.sync(() => { + const last = seq.at(-1); + if (!last) return null; + return { id: asSnapshotId(last), message: "latest", committedAt: Date.now() }; + }), + log: (limit) => + Effect.sync(() => + [...seq] + .reverse() + .slice(0, limit ?? seq.length) + .map((id) => ({ id: asSnapshotId(id), message: "", committedAt: Date.now() })), + ), + }; + }; + return { forScope: (scope) => Effect.succeed(forScope(scope)) }; +}; + +export type { SnapshotId }; + +export const makeTestResolver = ( + handlers: Record unknown>>, +): ClientResolver & { + readonly calls: { integration: string; connection: string; method: string }[]; +} => { + const calls: { integration: string; connection: string; method: string }[] = []; + return { + calls, + call: ({ integration, connection, path, args }) => { + const method = path.join("."); + calls.push({ integration, connection, method }); + const handler = handlers[integration]?.[method]; + if (!handler) { + return Effect.fail({ + _tag: "BindingError", + message: `no test handler for ${integration}.${method}`, + role: integration, + surface: integration, + } as unknown as BindingError); + } + return Effect.sync(() => handler(args)); + }, + }; +}; diff --git a/packages/plugins/apps/tsconfig.json b/packages/plugins/apps/tsconfig.json new file mode 100644 index 000000000..1504bed72 --- /dev/null +++ b/packages/plugins/apps/tsconfig.json @@ -0,0 +1,23 @@ +{ + "compilerOptions": { + "target": "ES2022", + "module": "ESNext", + "moduleResolution": "Bundler", + "strict": true, + "skipLibCheck": true, + "lib": ["ES2022", "DOM"], + "types": ["bun-types", "node"], + "noUnusedLocals": true, + "noImplicitOverride": true, + "jsx": "react-jsx", + "plugins": [ + { + "name": "@effect/language-service", + "ignoreEffectSuggestionsInTscExitCode": true, + "ignoreEffectWarningsInTscExitCode": true, + "diagnosticSeverity": {} + } + ] + }, + "include": ["src/**/*.ts", "src/**/*.tsx"] +} diff --git a/packages/plugins/apps/tsup.config.ts b/packages/plugins/apps/tsup.config.ts new file mode 100644 index 000000000..b0f69dc9b --- /dev/null +++ b/packages/plugins/apps/tsup.config.ts @@ -0,0 +1,15 @@ +import { defineConfig } from "tsup"; + +export default defineConfig({ + entry: { + index: "src/index.ts", + api: "src/api.ts", + "seams/index": "src/seams/index.ts", + "testing/index": "src/testing/index.ts", + }, + format: ["esm"], + dts: false, + sourcemap: true, + clean: true, + external: [/^@executor-js\//, /^effect/, /^@effect\//, "esbuild"], +}); diff --git a/packages/plugins/apps/vitest.config.ts b/packages/plugins/apps/vitest.config.ts new file mode 100644 index 000000000..698a11896 --- /dev/null +++ b/packages/plugins/apps/vitest.config.ts @@ -0,0 +1,9 @@ +import { defineConfig } from "vitest/config"; + +export default defineConfig({ + test: { + include: ["src/**/*.test.ts"], + testTimeout: 30_000, + hookTimeout: 30_000, + }, +}); From 41eb3aea44f171f0b74b869e849690c0da53373f Mon Sep 17 00:00:00 2001 From: Rhys Sullivan <39114868+RhysSullivan@users.noreply.github.com> Date: Mon, 6 Jul 2026 01:34:29 -0700 Subject: [PATCH 02/21] Align custom tools authoring contract --- bun.lock | 1 - packages/plugins/apps/package.json | 1 - packages/plugins/apps/src/api.ts | 7 +- packages/plugins/apps/src/authoring.ts | 64 ++++ .../src/backing/quickjs-tool-sandbox.test.ts | 87 ++++-- .../apps/src/backing/quickjs-tool-sandbox.ts | 190 +++++++++--- packages/plugins/apps/src/index.ts | 7 +- packages/plugins/apps/src/pipeline/bundle.ts | 32 +- .../plugins/apps/src/pipeline/descriptor.ts | 17 +- .../plugins/apps/src/pipeline/publish.test.ts | 87 +++++- packages/plugins/apps/src/pipeline/publish.ts | 81 ++++-- .../apps/src/plugin/apps-plugin.test.ts | 107 +++++++ .../plugins/apps/src/plugin/apps-plugin.ts | 105 ++++--- .../apps/src/plugin/bindings.strict.test.ts | 80 +++-- packages/plugins/apps/src/plugin/bindings.ts | 274 ++++++++++-------- .../plugins/apps/src/plugin/runtime.test.ts | 185 +++++++++++- packages/plugins/apps/src/plugin/runtime.ts | 63 ++-- .../src/seams/tool-sandbox.conformance.ts | 81 +----- .../plugins/apps/src/seams/tool-sandbox.ts | 43 ++- packages/plugins/apps/src/standard-schema.ts | 153 ++++++++++ .../plugins/apps/src/testing/daily-brief.ts | 32 +- packages/plugins/apps/src/testing/index.ts | 23 +- 22 files changed, 1302 insertions(+), 418 deletions(-) create mode 100644 packages/plugins/apps/src/authoring.ts create mode 100644 packages/plugins/apps/src/plugin/apps-plugin.test.ts create mode 100644 packages/plugins/apps/src/standard-schema.ts diff --git a/bun.lock b/bun.lock index 316bff560..31ae50848 100644 --- a/bun.lock +++ b/bun.lock @@ -789,7 +789,6 @@ "@executor-js/runtime-quickjs": "workspace:*", "@executor-js/sdk": "workspace:*", "@libsql/client": "catalog:", - "@standard-schema/spec": "^1.0.0", "esbuild": "^0.27.3", "zod": "4.3.6", }, diff --git a/packages/plugins/apps/package.json b/packages/plugins/apps/package.json index a116cea63..c0a0751bb 100644 --- a/packages/plugins/apps/package.json +++ b/packages/plugins/apps/package.json @@ -33,7 +33,6 @@ "@executor-js/runtime-quickjs": "workspace:*", "@executor-js/sdk": "workspace:*", "@libsql/client": "catalog:", - "@standard-schema/spec": "^1.0.0", "esbuild": "^0.27.3", "zod": "4.3.6" }, diff --git a/packages/plugins/apps/src/api.ts b/packages/plugins/apps/src/api.ts index 74cba1a18..05901c63d 100644 --- a/packages/plugins/apps/src/api.ts +++ b/packages/plugins/apps/src/api.ts @@ -14,4 +14,9 @@ export { type SelfHostAppsRuntimeOptions, } from "./plugin/self-host-runtime"; export { makeSqliteAppsStore } from "./backing/sqlite-apps-store"; -export { BindingError, type ClientResolver, type Bindings } from "./plugin/bindings"; +export { + BindingError, + type ClientResolver, + type ConnectionCandidate, + type RoleBindings, +} from "./plugin/bindings"; diff --git a/packages/plugins/apps/src/authoring.ts b/packages/plugins/apps/src/authoring.ts new file mode 100644 index 000000000..e60335d9f --- /dev/null +++ b/packages/plugins/apps/src/authoring.ts @@ -0,0 +1,64 @@ +import type { ScopeDbHandle } from "./seams/scope-db"; +import type { StandardSchemaV1 } from "./standard-schema"; + +export type JsonPrimitive = string | number | boolean | null; +export type JsonValue = JsonPrimitive | JsonObject | readonly JsonValue[]; +export interface JsonObject { + readonly [key: string]: JsonValue; +} + +export type ToolSchema = StandardSchemaV1 | JsonObject; + +export interface IntegrationDeclaration { + readonly integration: Slug; +} + +export type ToolHandlerContext< + TIntegrations extends Readonly> | undefined, +> = { + readonly db: ScopeDbHandle; +} & { + readonly [K in keyof NonNullable]: unknown; +}; + +type InferToolInput = TSchema extends StandardSchemaV1 + ? StandardSchemaV1.InferOutput + : unknown; + +type InferToolOutput = TSchema extends StandardSchemaV1 + ? StandardSchemaV1.InferOutput + : unknown; + +export interface DefineToolOptions< + TInputSchema extends ToolSchema, + TOutputSchema extends ToolSchema | undefined = undefined, + TIntegrations extends Readonly> | undefined = undefined, +> { + readonly description: string; + readonly integrations?: TIntegrations; + readonly input: TInputSchema; + readonly output?: TOutputSchema; + readonly annotations?: { + readonly readOnly?: boolean; + readonly destructive?: boolean; + readonly requiresApproval?: boolean; + }; + readonly handler: ( + input: InferToolInput, + context: ToolHandlerContext, + ) => + | Promise : unknown> + | (TOutputSchema extends ToolSchema ? InferToolOutput : unknown); +} + +export const integration = (slug: Slug): IntegrationDeclaration => ({ + integration: slug, +}); + +export const defineTool = < + TInputSchema extends ToolSchema, + TOutputSchema extends ToolSchema | undefined = undefined, + TIntegrations extends Readonly> | undefined = undefined, +>( + definition: DefineToolOptions, +): DefineToolOptions => definition; diff --git a/packages/plugins/apps/src/backing/quickjs-tool-sandbox.test.ts b/packages/plugins/apps/src/backing/quickjs-tool-sandbox.test.ts index 7e212214f..237f99718 100644 --- a/packages/plugins/apps/src/backing/quickjs-tool-sandbox.test.ts +++ b/packages/plugins/apps/src/backing/quickjs-tool-sandbox.test.ts @@ -3,8 +3,8 @@ import { Effect } from "effect"; import { bundleEntry } from "../pipeline/bundle"; import { makeQuickjsToolSandbox } from "./quickjs-tool-sandbox"; -import { ISSUES_SYNC_TS, SEARCH_ALL_MAIL_TS } from "../testing/daily-brief"; -import type { HandleBridge } from "../seams/tool-sandbox"; +import { ISSUES_SYNC_TS } from "../testing/daily-brief"; +import { InputValidationError, type HandleBridge } from "../seams/tool-sandbox"; const run = (effect: Effect.Effect): Promise => Effect.runPromise(effect); @@ -17,13 +17,13 @@ describe("QuickJS ToolSandbox", () => { const result = await run(sandbox.collect(code)); const descriptor = result.artifacts.default.descriptor as { kind: string; - connections: Record; + integrations: Record; inputJsonSchema: { type: string; properties: Record }; hasHandler: boolean; }; expect(descriptor.kind).toBe("tool"); - expect(descriptor.connections.github).toEqual( - expect.objectContaining({ decl: "single", integration: "github" }), + expect(descriptor.integrations.github).toEqual( + expect.objectContaining({ integration: "github" }), ); expect(descriptor.hasHandler).toBe(true); expect(descriptor.inputJsonSchema.type).toBe("object"); @@ -31,18 +31,6 @@ describe("QuickJS ToolSandbox", () => { expect(descriptor.inputJsonSchema.properties).toHaveProperty("since"); }); - it("collects a fan-out (connections) declaration", async () => { - const files = new Map([["tools/search-all-mail.ts", SEARCH_ALL_MAIL_TS]]); - const { code } = await run(bundleEntry({ files, entry: "tools/search-all-mail.ts" })); - const result = await run(sandbox.collect(code)); - const descriptor = result.artifacts.default.descriptor as { - connections: Record; - }; - expect(descriptor.connections.inboxes).toEqual( - expect.objectContaining({ decl: "array", integration: "gmail" }), - ); - }); - it("invokes a tool, routing injected-client + db calls through the bridge", async () => { const files = new Map([["tools/issues-sync.ts", ISSUES_SYNC_TS]]); const { code } = await run(bundleEntry({ files, entry: "tools/issues-sync.ts" })); @@ -92,6 +80,54 @@ describe("QuickJS ToolSandbox", () => { expect(calls.some((c) => c.root === "github")).toBe(true); }); + it("surfaces Standard Schema validation issues before the handler runs", async () => { + const source = `import { defineTool } from "executor:app"; +import { z } from "zod"; +export default defineTool({ + description: "validates", + input: z.object({ q: z.string() }), + async handler(){ return { ok: true }; }, +});`; + const files = new Map([["tools/validate.ts", source]]); + const { code } = await run(bundleEntry({ files, entry: "tools/validate.ts" })); + const bridge: HandleBridge = { call: () => Effect.succeed(null) }; + + const exit = await Effect.runPromiseExit( + sandbox.invoke( + code, + { artifact: "validate", kind: "tool", input: { q: 123 }, roots: {} }, + bridge, + ), + ); + + expect(exit._tag).toBe("Failure"); + if (exit._tag !== "Failure") throw new Error("expected validation failure"); + expect(JSON.stringify(exit.cause)).toContain("InputValidationError"); + expect(JSON.stringify(exit.cause)).toContain("q"); + }); + + it("invokes raw JSON Schema tools without sandbox Standard Schema validation", async () => { + const source = `import { defineTool } from "executor:app"; +export default defineTool({ + description: "raw", + input: { type: "object", properties: { q: { type: "string" } }, required: ["q"] }, + async handler(input){ return { q: input.q }; }, +});`; + const files = new Map([["tools/raw.ts", source]]); + const { code } = await run(bundleEntry({ files, entry: "tools/raw.ts" })); + const bridge: HandleBridge = { call: () => Effect.succeed(null) }; + + const result = await run( + sandbox.invoke( + code, + { artifact: "raw", kind: "tool", input: { q: "ok" }, roots: {} }, + bridge, + ), + ); + + expect(result.output).toEqual({ q: "ok" }); + }); + it("rejects a non-deterministic descriptor (Math.random at describe time)", async () => { const nondeterministic = `import { defineTool } from "executor:app"; import { z } from "zod"; @@ -122,4 +158,21 @@ export default defineTool({ ); expect(exit._tag).toBe("Failure"); }); + + it("returns typed InputValidationError from promise rejection", async () => { + const source = `import { defineTool } from "executor:app"; +import { z } from "zod"; +export default defineTool({ + description: "typed", + input: z.object({ name: z.string() }), + async handler(){ return {}; }, +});`; + const files = new Map([["tools/typed.ts", source]]); + const { code } = await run(bundleEntry({ files, entry: "tools/typed.ts" })); + const bridge: HandleBridge = { call: () => Effect.succeed(null) }; + + await expect( + run(sandbox.invoke(code, { artifact: "typed", kind: "tool", input: {}, roots: {} }, bridge)), + ).rejects.toBeInstanceOf(InputValidationError); + }); }); diff --git a/packages/plugins/apps/src/backing/quickjs-tool-sandbox.ts b/packages/plugins/apps/src/backing/quickjs-tool-sandbox.ts index de298f6dc..ca0f4043c 100644 --- a/packages/plugins/apps/src/backing/quickjs-tool-sandbox.ts +++ b/packages/plugins/apps/src/backing/quickjs-tool-sandbox.ts @@ -3,11 +3,15 @@ import type { SandboxToolInvoker } from "@executor-js/codemode-core"; import { makeQuickJsExecutor } from "@executor-js/runtime-quickjs"; import { + InputValidationError, + OutputValidationError, ToolSandboxError, type CollectResult, + type CollectRequest, type HandleBridge, type InvokeRequest, type InvokeResult, + type ValidationIssue, type ToolSandbox, } from "../seams/tool-sandbox"; import { stableStringify } from "../pipeline/descriptor"; @@ -16,7 +20,7 @@ import { stableStringify } from "../pipeline/descriptor"; // QuickJS-backed ToolSandbox (self-hosted). // // The published bundle is a CJS string. The sandbox body prepends a `require` -// shim providing `executor:app` (defineTool/connection/connections/catalog), +// shim providing `executor:app` (defineTool/integration), // then executes the bundle so `module.exports.default` is the artifact. A // driver appended after the bundle either collects the descriptor (nothing // bound) or runs one handler with injected clients. @@ -60,9 +64,7 @@ function __mkHandle(root, prefix) { }); } var __executorApp = { - connection: function(integration, opts) { return { __decl: 'single', integration: integration, description: opts && opts.description }; }, - connections: function(integration, opts) { return { __decl: 'array', integration: integration, description: opts && opts.description }; }, - catalog: function() { return { __decl: 'catalog' }; }, + integration: function(integration) { return { integration: integration }; }, defineTool: function(def) { __defs.tool = def; return def; }, }; function __require(id) { @@ -72,9 +74,8 @@ function __require(id) { `; // Run the CJS bundle. The bundle's virtual entry sets `globalThis.__artifact` -// (the def object returned by defineTool), `globalThis.__zodToJson` and -// `__zodToJsonOutput` (bound to the author's inlined zod). `require` is our -// shim; `defineTool` also records into `__defs` as a fallback. +// (the def object returned by defineTool). `require` is our shim; `defineTool` +// also records into `__defs` as a fallback. const wrapBundle = (bundle: string): string => ` var module = { exports: {} }; var exports = module.exports; @@ -84,38 +85,110 @@ ${bundle} })(module, exports, require); `; -// Collect driver: describe the artifact's connections + input/output schema. +const parseMarker = (message: string): unknown => { + const parse = (text: string): unknown => { + try { + return JSON.parse(text); + } catch { + return null; + } + }; + const direct = parse(message); + if (direct) return direct; + const firstBrace = message.indexOf("{"); + const lastBrace = message.lastIndexOf("}"); + if (firstBrace !== -1 && lastBrace > firstBrace) { + return parse(message.slice(firstBrace, lastBrace + 1)); + } + return null; +}; + +const isMarker = (value: unknown, tag: string): value is Record => { + if (value === null || typeof value !== "object") return false; + return (value as { readonly _tag?: unknown })._tag === tag; +}; + +const markerMessage = (marker: Record, fallback: string): string => + typeof marker.message === "string" ? marker.message : fallback; + +const markerIssues = (marker: Record): readonly ValidationIssue[] => + Array.isArray(marker.issues) ? (marker.issues as readonly ValidationIssue[]) : []; + +// Collect driver: describe the artifact's integrations + input/output schema. // Deterministic JSON only — no handler execution. -const collectDriver = ` +const collectDriver = (artifact: string): string => ` return await (async () => { + var artifactName = ${JSON.stringify(artifact)}; + var fail = function(tag, data) { + var payload = data || {}; + payload._tag = tag; + throw JSON.stringify(payload); + }; + var sanitizeJson = function(value) { + if (value === undefined) return undefined; + if (value === null || typeof value !== 'object') return value; + if (Array.isArray(value)) return value.map(sanitizeJson); + var out = {}; + for (var key in value) { + if (key === '~standard') continue; + if (typeof value[key] === 'function' || value[key] === undefined) continue; + out[key] = sanitizeJson(value[key]); + } + return out; + }; + var schemaToJson = function(field, schema, direction) { + if (schema === undefined) return undefined; + if (schema && typeof schema === 'object' && schema['~standard']) { + var standard = schema['~standard']; + var vendor = standard && standard.vendor ? String(standard.vendor) : 'unknown'; + var jsonSchema = standard && standard.jsonSchema; + if (!jsonSchema || typeof jsonSchema[direction] !== 'function') { + fail('SchemaExportError', { + tool: artifactName, + field: field, + vendor: vendor, + message: 'tool "' + artifactName + '" ' + field + ' schema from "' + vendor + '" is not supported for schema export', + }); + } + return sanitizeJson(jsonSchema[direction]({ target: 'draft-07' })); + } + if (schema && typeof schema === 'object' && !Array.isArray(schema)) { + return sanitizeJson(schema); + } + fail('SchemaExportError', { + tool: artifactName, + field: field, + vendor: 'raw', + message: 'tool "' + artifactName + '" ' + field + ' schema must be a Standard Schema or raw JSON Schema object', + }); + }; var def = __defs.tool || (globalThis.__artifact && (globalThis.__artifact.default || globalThis.__artifact)); - var conns = {}; - if (def && def.connections) { - for (var k in def.connections) { - var c = def.connections[k]; - conns[k] = { decl: c && c.__decl ? c.__decl : 'single', integration: c && c.integration, description: c && c.description }; + var integrations = {}; + if (def && def.integrations) { + for (var k in def.integrations) { + var c = def.integrations[k]; + integrations[k] = { integration: c && c.integration ? String(c.integration) : '' }; } } - var toJson = (typeof globalThis.__zodToJson === 'function') ? globalThis.__zodToJson : function(){ return undefined; }; - var toJsonOut = (typeof globalThis.__zodToJsonOutput === 'function') ? globalThis.__zodToJsonOutput : function(){ return undefined; }; return { + artifact: artifactName, kind: 'tool', description: def && def.description, - connections: conns, + integrations: integrations, annotations: def && def.annotations, hasHandler: !!(def && def.handler), - inputJsonSchema: def ? toJson(def.input) : undefined, - outputJsonSchema: def ? toJsonOut(def.output) : undefined, + inputJsonSchema: def ? schemaToJson('input', def.input, 'input') : undefined, + outputJsonSchema: def && def.output !== undefined ? schemaToJson('output', def.output, 'output') : undefined, }; })() `; -const buildCollectCode = (bundle: string): string => - runtimePrelude + wrapBundle(bundle) + collectDriver; +const buildCollectCode = (bundle: string, request?: CollectRequest): string => + runtimePrelude + wrapBundle(bundle) + collectDriver(request?.artifact ?? "default"); -// Invoke driver: build injected clients from the request roots, then call the -// artifact's handler with (input, injected). Fan-out arrays become arrays of -// element handles. +// Invoke driver: build injected clients from the request roots, validate +// Standard Schema input/output inside the sandbox, then call the artifact's +// handler with (input, injected). const buildInvokeDriver = (request: InvokeRequest): string => { const rootsLiteral = JSON.stringify(request.roots); const inputLiteral = JSON.stringify(request.input ?? {}); @@ -123,19 +196,35 @@ const buildInvokeDriver = (request: InvokeRequest): string => { return await (async () => { var def = __defs.tool || (globalThis.__artifact && (globalThis.__artifact.default || globalThis.__artifact)); if (!def || typeof def.handler !== 'function') throw new Error('artifact has no handler: ${request.artifact}'); + var failValidation = function(tag, field, issues) { + throw JSON.stringify({ + _tag: tag, + tool: '${request.artifact}', + field: field, + message: 'tool "${request.artifact}" ' + field + ' validation failed', + issues: JSON.parse(JSON.stringify(issues || [])), + }); + }; + var validateWithStandardSchema = async function(field, schema, value) { + if (!schema || typeof schema !== 'object' || !schema['~standard']) return value; + var standard = schema['~standard']; + if (!standard || typeof standard.validate !== 'function') return value; + var result = await standard.validate(value); + if (result && result.issues) { + failValidation(field === 'input' ? 'InputValidationError' : 'OutputValidationError', field, result.issues); + } + return result && Object.prototype.hasOwnProperty.call(result, 'value') ? result.value : value; + }; var roots = ${rootsLiteral}; var injected = {}; for (var name in roots) { var spec = roots[name]; - if (spec.kind === 'array') { - var arr = []; - for (var i = 0; i < spec.count; i++) arr.push(__mkHandle(name + '#' + i, [])); - injected[name] = arr; - } else { - injected[name] = __mkHandle(name, []); - } + if (spec.kind !== 'single') throw new Error('unsupported handle root kind: ' + spec.kind); + injected[name] = __mkHandle(name, []); } - var out = await def.handler(${inputLiteral}, injected); + var input = await validateWithStandardSchema('input', def.input, ${inputLiteral}); + var out = await def.handler(input, injected); + out = await validateWithStandardSchema('output', def.output, out); return out; })() `; @@ -163,13 +252,23 @@ export const makeQuickjsToolSandbox = (options: QuickjsToolSandboxOptions = {}): timeoutMs: options.invokeTimeoutMs ?? INVOKE_TIMEOUT_MS, }); - const runCollect = (bundle: string): Effect.Effect => - collectExecutor.execute(buildCollectCode(bundle), collectInvoker).pipe( + const runCollect = (code: string): Effect.Effect => + collectExecutor.execute(code, collectInvoker).pipe( Effect.mapError( (cause) => new ToolSandboxError({ kind: "collect", message: "collect run failed", cause }), ), Effect.flatMap((result) => { if (result.error) { + const marker = parseMarker(result.error); + if (isMarker(marker, "SchemaExportError")) { + return Effect.fail( + new ToolSandboxError({ + kind: "collect", + message: markerMessage(marker, result.error), + cause: marker, + }), + ); + } return Effect.fail(new ToolSandboxError({ kind: "collect", message: result.error })); } return Effect.succeed(result.result); @@ -177,13 +276,13 @@ export const makeQuickjsToolSandbox = (options: QuickjsToolSandboxOptions = {}): ); return { - collect: (bundle: string) => + collect: (bundle: string, request?: CollectRequest) => Effect.gen(function* () { // Run twice, byte-compare (determinism gate). Key-sorted stringify so a // false mismatch never comes from property-order luck — a real // divergence (Math.random / Date.now) still fails. - const first = yield* runCollect(bundle); - const second = yield* runCollect(bundle); + const first = yield* runCollect(buildCollectCode(bundle, request)); + const second = yield* runCollect(buildCollectCode(bundle, request)); const a = stableStringify(first); const b = stableStringify(second); if (a !== b) { @@ -242,6 +341,23 @@ export const makeQuickjsToolSandbox = (options: QuickjsToolSandboxOptions = {}): ), ); if (result.error) { + const marker = parseMarker(result.error); + if (isMarker(marker, "InputValidationError")) { + return yield* Effect.fail( + new InputValidationError({ + message: markerMessage(marker, "input validation failed"), + issues: markerIssues(marker), + }), + ); + } + if (isMarker(marker, "OutputValidationError")) { + return yield* Effect.fail( + new OutputValidationError({ + message: markerMessage(marker, "output validation failed"), + issues: markerIssues(marker), + }), + ); + } return yield* Effect.fail( new ToolSandboxError({ kind: "invoke", message: result.error }), ); diff --git a/packages/plugins/apps/src/index.ts b/packages/plugins/apps/src/index.ts index 3d408c04e..dc829d6e1 100644 --- a/packages/plugins/apps/src/index.ts +++ b/packages/plugins/apps/src/index.ts @@ -5,6 +5,8 @@ // capability sits behind a seam. export * from "./seams"; +export * from "./authoring"; +export * from "./standard-schema"; export * from "./pipeline/descriptor"; export { discover, PublishError } from "./pipeline/discover"; export { bundleEntry, PLATFORM_MODULES, INLINABLE_MODULES } from "./pipeline/bundle"; @@ -26,9 +28,10 @@ export { export { buildBridge, rootsFor, + resolveIntegrationBindings, BindingError, - type Bindings, - type RoleBinding, + type ConnectionCandidate, + type RoleBindings, type ClientResolver, type BindingContext, } from "./plugin/bindings"; diff --git a/packages/plugins/apps/src/pipeline/bundle.ts b/packages/plugins/apps/src/pipeline/bundle.ts index 247a1ceda..b7d6c95f1 100644 --- a/packages/plugins/apps/src/pipeline/bundle.ts +++ b/packages/plugins/apps/src/pipeline/bundle.ts @@ -20,11 +20,10 @@ export const toolchainRef = (): ToolchainRef => ({ // // One esbuild pass per artifact entry. The platform module (`executor:app`) // stays EXTERNAL, the sandbox shim provides the real behavior. A small allowlist -// of schema libraries (`zod`, `@standard-schema/spec`) is INLINED from -// node_modules so the author's schema runs in the sandbox AND its `toJSONSchema` -// is reachable to extract the descriptor's input/output JSON Schema. Every -// other bare import is rejected with a diagnostic: npm deps in user bundles are -// out of scope. +// of schema libraries (`zod`) is INLINED from node_modules so the author's +// schema runs in the sandbox and exposes the Standard Schema JSON-schema +// extension. Every other bare import is rejected with a diagnostic: npm deps in +// user bundles are out of scope. // // Output is a single CJS string with the externals left as `require(...)` // calls the sandbox resolves. @@ -36,7 +35,7 @@ export const PLATFORM_MODULES = new Set(["executor:app"]); /** npm modules we deliberately inline (schema runtime that must run in the * sandbox). Everything else bare is rejected. */ -export const INLINABLE_MODULES = new Set(["zod", "@standard-schema/spec"]); +export const INLINABLE_MODULES = new Set(["zod"]); const isInlinable = (path: string): boolean => { if (INLINABLE_MODULES.has(path)) return true; @@ -80,29 +79,10 @@ const candidates = (path: string): string[] => { const FILESET_NS = "fileset"; const VIRTUAL_ENTRY = "executor-apps://entry"; -/** The virtual entry esbuild starts from: re-export the author entry AND expose - * the shared zod instance as a global so the sandbox collect driver can call - * `toJSONSchema` against the SAME zod the author bundled. */ +/** The virtual entry esbuild starts from: re-export the author entry. */ const virtualEntrySource = (authorEntry: string) => ` import __artifactModule from ${JSON.stringify(`/${authorEntry}`)}; -import * as __zod from "zod"; globalThis.__artifact = __artifactModule; -globalThis.__zodToJson = function (schema) { - if (!schema) return undefined; - try { - return __zod.toJSONSchema(schema, { io: "input", unrepresentable: "any" }); - } catch (e) { - try { return __zod.toJSONSchema(schema); } catch (e2) { return {}; } - } -}; -globalThis.__zodToJsonOutput = function (schema) { - if (!schema) return undefined; - try { - return __zod.toJSONSchema(schema, { io: "output", unrepresentable: "any" }); - } catch (e) { - try { return __zod.toJSONSchema(schema); } catch (e2) { return {}; } - } -}; `; const fileSetPlugin = (files: ReadonlyMap, authorEntry: string) => ({ diff --git a/packages/plugins/apps/src/pipeline/descriptor.ts b/packages/plugins/apps/src/pipeline/descriptor.ts index 596f101e8..66f44edb1 100644 --- a/packages/plugins/apps/src/pipeline/descriptor.ts +++ b/packages/plugins/apps/src/pipeline/descriptor.ts @@ -7,7 +7,7 @@ /** Descriptor schema version. Bumped on any breaking shape change. A reader * refuses a descriptor from a version it does not understand. */ -export const DESCRIPTOR_VERSION = 1 as const; +export const DESCRIPTOR_VERSION = 2 as const; /** Where an entry came from: path + content hash. Lets a projection point back * at the exact source bytes without re-reading the snapshot, and makes the @@ -27,10 +27,9 @@ export interface ToolchainRef { readonly target: string; } -export type ConnectionDecl = - | { readonly kind: "single"; readonly integration: string; readonly description?: string } - | { readonly kind: "array"; readonly integration: string; readonly description?: string } - | { readonly kind: "catalog" }; +export interface IntegrationDecl { + readonly integration: string; +} export interface ToolDescriptor { /** Path identity, e.g. `issues-sync` (from `tools/issues-sync.ts`). */ @@ -39,8 +38,8 @@ export interface ToolDescriptor { /** Path + source-hash provenance for this entry. */ readonly source: ModuleSourceRef; readonly description: string; - /** role -> connection declaration, collected from `connections:`. */ - readonly connections: Readonly>; + /** role -> integration declaration, collected from `integrations:`. */ + readonly integrations: Readonly>; readonly inputSchema?: unknown; readonly outputSchema?: unknown; readonly annotations?: { @@ -57,8 +56,8 @@ export interface DeferredDescriptor { readonly description?: string; } -export const FLOW_ENTRIES_KEY = `work${"flows"}` as `${"work"}flows`; -export const GUIDE_ENTRIES_KEY = `sk${"ills"}` as `${"sk"}ills`; +export const FLOW_ENTRIES_KEY = "workflows"; +export const GUIDE_ENTRIES_KEY = "skills"; export interface AppDescriptor { readonly version: typeof DESCRIPTOR_VERSION; diff --git a/packages/plugins/apps/src/pipeline/publish.test.ts b/packages/plugins/apps/src/pipeline/publish.test.ts index 2c01443d3..8f8042eb6 100644 --- a/packages/plugins/apps/src/pipeline/publish.test.ts +++ b/packages/plugins/apps/src/pipeline/publish.test.ts @@ -36,21 +36,17 @@ describe("publish pipeline (discover -> bundle -> collect -> project)", () => { expect(out.skipped).toEqual([]); const sync = d.tools.find((t) => t.name === "issues-sync")!; - expect(sync.connections.github).toEqual( - expect.objectContaining({ kind: "single", integration: "github" }), - ); + expect(sync.integrations.github).toEqual(expect.objectContaining({ integration: "github" })); expect((sync.inputSchema as { type: string }).type).toBe("object"); const mail = d.tools.find((t) => t.name === "search-all-mail")!; - expect(mail.connections.inboxes).toEqual( - expect.objectContaining({ kind: "array", integration: "gmail" }), - ); + expect(mail.integrations.inbox).toEqual(expect.objectContaining({ integration: "gmail" })); }); it("publishes tools while reporting deferred known folders under skipped", async () => { const deps = makeDeps(); - const flowPath = `work${"flows"}/y.ts`; - const guidePath = `sk${"ills"}/z/${"SK"}ILL.md`; + const flowPath = "workflows/y.ts"; + const guidePath = "skills/z/SKILL.md"; const files = new Map([ [ "tools/x.ts", @@ -74,6 +70,81 @@ export default defineTool({ description: "x", input: z.object({}), async handler ]); }); + it("rejects an authored input field that collides with an integration role", async () => { + const deps = makeDeps(); + const files = new Map([ + [ + "tools/sync.ts", + `import { defineTool, integration } from "executor:app"; +import { z } from "zod"; +export default defineTool({ + description: "sync", + integrations: { crm: integration("dealcloud") }, + input: z.object({ crm: z.string() }), + async handler(){ return {}; }, +});`, + ], + ]); + + const exit = await Effect.runPromiseExit(publish(deps, { scope: "s", files })); + + expect(exit._tag).toBe("Failure"); + expect(JSON.stringify(exit)).toContain("collides"); + expect(JSON.stringify(exit)).toContain("crm"); + }); + + it("rejects a Standard Schema vendor without JSON Schema export", async () => { + const deps = makeDeps(); + const files = new Map([ + [ + "tools/vendor.ts", + `import { defineTool } from "executor:app"; +const schema = { + "~standard": { + version: 1, + vendor: "vendor-without-json-schema", + validate(value) { return { value }; }, + }, +}; +export default defineTool({ + description: "vendor", + input: schema, + async handler(input){ return input; }, +});`, + ], + ]); + + const exit = await Effect.runPromiseExit(publish(deps, { scope: "s", files })); + + expect(exit._tag).toBe("Failure"); + expect(JSON.stringify(exit)).toContain("vendor-without-json-schema"); + expect(JSON.stringify(exit)).toContain("vendor"); + expect(JSON.stringify(exit)).toContain("input"); + }); + + it("accepts raw JSON Schema input", async () => { + const deps = makeDeps(); + const files = new Map([ + [ + "tools/raw.ts", + `import { defineTool } from "executor:app"; +export default defineTool({ + description: "raw", + input: { type: "object", properties: { q: { type: "string" } }, required: ["q"] }, + async handler(input){ return input; }, +});`, + ], + ]); + + const out = await run(publish(deps, { scope: "s", files })); + + expect(out.descriptor.tools[0].inputSchema).toEqual({ + type: "object", + properties: { q: { type: "string" } }, + required: ["q"], + }); + }); + it("rejects a bare npm import (npm deps out of scope)", async () => { const deps = makeDeps(); const files = new Map([ diff --git a/packages/plugins/apps/src/pipeline/publish.ts b/packages/plugins/apps/src/pipeline/publish.ts index 4f7be1459..91b779e2b 100644 --- a/packages/plugins/apps/src/pipeline/publish.ts +++ b/packages/plugins/apps/src/pipeline/publish.ts @@ -10,7 +10,7 @@ import { GUIDE_ENTRIES_KEY, stableStringify, type AppDescriptor, - type ConnectionDecl, + type IntegrationDecl, type ModuleSourceRef, type ToolDescriptor, } from "./descriptor"; @@ -92,32 +92,54 @@ const sourceRef = (path: string, source: string): Effect.Effect interface CollectedDescriptor { readonly kind: "tool"; + readonly artifact?: string; readonly description?: string; - readonly connections?: Record< - string, - { decl: string; integration?: string; description?: string } - >; + readonly integrations?: Record; readonly annotations?: ToolDescriptor["annotations"]; readonly inputJsonSchema?: unknown; readonly outputJsonSchema?: unknown; } -const toConnectionDecls = ( - raw: CollectedDescriptor["connections"], -): Record => { - const out: Record = {}; +const toIntegrationDecls = ( + raw: CollectedDescriptor["integrations"], +): Record => { + const out: Record = {}; for (const [role, c] of Object.entries(raw ?? {})) { - if (c.decl === "array") { - out[role] = { kind: "array", integration: c.integration ?? "", description: c.description }; - } else if (c.decl === "catalog") { - out[role] = { kind: "catalog" }; - } else { - out[role] = { kind: "single", integration: c.integration ?? "", description: c.description }; - } + out[role] = { integration: c.integration ?? "" }; } return out; }; +const isRecord = (value: unknown): value is Record => + value !== null && typeof value === "object" && !Array.isArray(value); + +const rejectRoleInputCollisions = ( + artifactName: string, + sourcePath: string, + inputSchema: unknown, + integrations: Readonly>, +): PublishError | null => { + if (!isRecord(inputSchema)) return null; + const properties = inputSchema.properties; + if (!isRecord(properties)) return null; + + for (const role of Object.keys(integrations)) { + if (Object.prototype.hasOwnProperty.call(properties, role)) { + return new PublishError({ + message: `tool "${artifactName}" input field "${role}" collides with a platform integration argument`, + stage: "collect", + diagnostics: [ + { + path: sourcePath, + message: `input field "${role}" collides with declared integration role "${role}"`, + }, + ], + }); + } + } + return null; +}; + interface AssembledApp { readonly tools: readonly ToolDescriptor[]; readonly skipped: readonly SkippedArtifact[]; @@ -146,7 +168,7 @@ const assemble = (deps: PublishDeps, files: FileSet): Effect.Effect new PublishError({ @@ -157,7 +179,11 @@ const assemble = (deps: PublishDeps, files: FileSet): Effect.Effect(effect: Effect.Effect): Promise => Effect.runPromise(effect); + +const PROTOTYPE_ROOT = "/Users/rhyssullivan/agent-workspace/prototypes/custom-tools"; + +const prototypeFileSet = (): Map => + new Map([ + ["executor.json", readFileSync(join(PROTOTYPE_ROOT, "executor.json"), "utf8")], + [ + "tools/deal-pipeline-sync.ts", + readFileSync(join(PROTOTYPE_ROOT, "tools/deal-pipeline-sync.ts"), "utf8"), + ], + [ + "tools/find-deal-docs.ts", + readFileSync(join(PROTOTYPE_ROOT, "tools/find-deal-docs.ts"), "utf8"), + ], + ]); + +describe("appsPlugin custom-tools contract", () => { + it("round-trips prototype files through publish, resolveTools, and invokeTool", async () => { + let dealListArgs: unknown; + const resolver = makeTestResolver( + { + dealcloud: { + "deals.list": (args) => { + dealListArgs = args[0]; + return []; + }, + }, + "microsoft-sharepoint": { + "search.query": () => [], + }, + }, + [ + { + address: "tools.dealcloud.user.crm-main", + integration: "dealcloud", + name: "crm-main", + }, + { + address: "tools.microsoft-sharepoint.user.sharepoint-main", + integration: "microsoft-sharepoint", + name: "sharepoint-main", + }, + ], + ); + const host = makeSelfHostAppsRuntime({ + dataDir: mkdtempSync(join(tmpdir(), "apps-plugin-")), + store: makeInMemoryAppsStore(), + resolver, + inMemory: true, + }); + const runtime = host.runtime; + const plugin = appsPlugin({ runtime }); + const appConnection = connectionNameForScope("rhys"); + + await run(runtime.publish({ scope: "rhys", files: prototypeFileSet() })); + + const resolved = await run( + plugin.resolveTools!({ + connection: { name: appConnection }, + } as never), + ); + const syncTool = resolved.tools.find((tool) => String(tool.name) === "deal-pipeline-sync"); + expect(syncTool).toBeTruthy(); + const inputSchema = syncTool!.inputSchema as { + properties: Record; + required?: string[]; + }; + expect(inputSchema.properties.crm.enum).toEqual(["tools.dealcloud.user.crm-main"]); + expect(inputSchema.properties.crm.default).toBe("tools.dealcloud.user.crm-main"); + expect(inputSchema.properties.crm.description).toBe("Connection to use for crm (dealcloud)"); + expect(inputSchema.required ?? []).not.toContain("crm"); + + const output = await run( + plugin.invokeTool!({ + ctx: {}, + toolRow: { + name: "deal-pipeline-sync", + connection: appConnection, + }, + args: { + crm: "tools.dealcloud.user.crm-main", + updatedSince: "2026-01-01T00:00:00Z", + }, + } as never), + ); + + expect(output).toEqual({ synced: 0 }); + expect(dealListArgs).toEqual({ + status: "active", + updatedSince: "2026-01-01T00:00:00Z", + pageSize: 200, + }); + await host.close(); + }); +}); diff --git a/packages/plugins/apps/src/plugin/apps-plugin.ts b/packages/plugins/apps/src/plugin/apps-plugin.ts index 8154fb3ed..076b38c1a 100644 --- a/packages/plugins/apps/src/plugin/apps-plugin.ts +++ b/packages/plugins/apps/src/plugin/apps-plugin.ts @@ -16,7 +16,8 @@ import { import type { AppsRuntime } from "./runtime"; import { makeAppsStore } from "./store"; -import type { Bindings, ClientResolver } from "./bindings"; +import type { ClientResolver, ConnectionCandidate } from "./bindings"; +import type { IntegrationDecl, ToolDescriptor } from "../pipeline/descriptor"; export const APPS_INTEGRATION_SLUG = "apps"; export const APPS_PLUGIN_ID = "apps"; @@ -25,11 +26,6 @@ const DEFAULT_CATALOG_SCOPE = "default"; export interface AppsPluginOptions { readonly runtime: AppsRuntime; - readonly resolveBindings?: (input: { - readonly scope: string; - readonly tool: string; - readonly declared: Readonly>; - }) => Bindings; readonly makeResolver?: (input: { readonly ctx: unknown; readonly scope: string; @@ -37,30 +33,59 @@ export interface AppsPluginOptions { }) => ClientResolver; } -const defaultBindings = ( - declared: Readonly>, -): Bindings => { - const out: Record = {}; - for (const [role, decl] of Object.entries(declared)) { - if (decl.kind === "array") { - out[role] = { kind: "array", connections: [decl.integration ?? role] }; - } else if (decl.kind !== "catalog") { - out[role] = { kind: "single", connection: decl.integration ?? role }; - } - } - return out; -}; - interface AppsStoreShape { readonly runtime: AppsRuntime; } +const isRecord = (value: unknown): value is Record => + value !== null && typeof value === "object" && !Array.isArray(value); + +const unique = (values: readonly string[]): readonly string[] => [...new Set(values)]; + +const projectInputSchema = ( + schema: unknown, + integrations: Readonly>, + byRole: Readonly>, +): unknown => { + const base: Record = isRecord(schema) ? { ...schema } : { type: "object" }; + const properties = isRecord(base.properties) ? { ...base.properties } : {}; + const required = new Set( + Array.isArray(base.required) + ? base.required.filter((value): value is string => typeof value === "string") + : [], + ); + + for (const [role, decl] of Object.entries(integrations)) { + const addresses = unique((byRole[role] ?? []).map((c) => c.address)); + const roleSchema: Record = { + type: "string", + enum: addresses, + description: `Connection to use for ${role} (${decl.integration})`, + }; + if (addresses.length === 1) { + roleSchema.default = addresses[0]; + required.delete(role); + } else { + required.add(role); + } + properties[role] = roleSchema; + } + + const projected: Record = { + ...base, + type: typeof base.type === "string" ? base.type : "object", + properties, + }; + if (required.size > 0) projected.required = [...required]; + else delete projected.required; + return projected; +}; + export const appsPlugin = definePlugin((options?: AppsPluginOptions) => { if (!options?.runtime) { throw new Error("appsPlugin requires a runtime"); } const runtime = options.runtime; - const resolveBindings = options.resolveBindings; const makeResolver = options.makeResolver; const scopeFor = (connectionName: string): Effect.Effect => @@ -158,15 +183,16 @@ export const appsPlugin = definePlugin((options?: AppsPluginOptions) => { const scope = yield* scopeFor(String(connection.name)); const descriptor = yield* runtime.getDescriptor(scope); if (!descriptor) return { tools: [] } satisfies ResolveToolsResult; - const tools: ToolDef[] = descriptor.tools.map((t) => ({ - name: ToolName.make(t.name), - description: t.description, - inputSchema: t.inputSchema, - outputSchema: t.outputSchema, - annotations: { - requiresApproval: t.annotations?.destructive === true, - }, - })); + const tools: ToolDef[] = []; + for (const t of descriptor.tools) { + const byRole: Record = {}; + for (const [role, decl] of Object.entries(t.integrations)) { + byRole[role] = yield* runtime.deps.resolver + .listConnections({ integration: decl.integration }) + .pipe(Effect.orElseSucceed(() => [])); + } + tools.push(projectTool(t, byRole)); + } return { tools } satisfies ResolveToolsResult; }), @@ -187,15 +213,11 @@ export const appsPlugin = definePlugin((options?: AppsPluginOptions) => { new Error(`apps tool "${toolRow.name}" is not published in scope "${scope}"`), ); } - const declared = toolDesc.connections; - const bindings = resolveBindings - ? resolveBindings({ scope, tool: toolRow.name, declared }) - : defaultBindings(declared); const resolver = makeResolver ? makeResolver({ ctx, scope, tool: toolRow.name }) : undefined; return yield* runtime - .invokeTool({ scope, tool: toolRow.name, args, bindings, resolver }) + .invokeTool({ scope, tool: toolRow.name, args, resolver }) .pipe( Effect.mapError( (cause) => @@ -210,6 +232,19 @@ export const appsPlugin = definePlugin((options?: AppsPluginOptions) => { }; }); +const projectTool = ( + descriptor: ToolDescriptor, + byRole: Readonly>, +): ToolDef => ({ + name: ToolName.make(descriptor.name), + description: descriptor.description, + inputSchema: projectInputSchema(descriptor.inputSchema, descriptor.integrations, byRole), + outputSchema: descriptor.outputSchema, + annotations: { + requiresApproval: descriptor.annotations?.destructive === true, + }, +}); + export const APPS_CONNECTION_PREFIX = APPS_INTEGRATION_SLUG; const pascal = (value: string): string => diff --git a/packages/plugins/apps/src/plugin/bindings.strict.test.ts b/packages/plugins/apps/src/plugin/bindings.strict.test.ts index dd5af30fd..6256e7708 100644 --- a/packages/plugins/apps/src/plugin/bindings.strict.test.ts +++ b/packages/plugins/apps/src/plugin/bindings.strict.test.ts @@ -1,8 +1,8 @@ import { describe, expect, it } from "vitest"; import { Effect } from "effect"; -import { buildBridge, type ClientResolver } from "./bindings"; -import type { ConnectionDecl } from "../pipeline/descriptor"; +import { buildBridge, resolveIntegrationBindings, type ClientResolver } from "./bindings"; +import type { IntegrationDecl } from "../pipeline/descriptor"; import type { ScopeDbHandle } from "../seams/scope-db"; // --------------------------------------------------------------------------- @@ -19,6 +19,26 @@ const okDb: ScopeDbHandle = { }; const okResolver: ClientResolver = { + listConnections: ({ integration }) => + Effect.succeed([ + { + address: `tools.${integration}.user.main`, + integration, + name: "main", + owner: "user", + }, + ]), + resolveConnection: ({ connection }) => + Effect.succeed( + connection === "tools.github.user.main" || connection === "main" + ? { + address: "tools.github.user.main", + integration: "github", + name: "main", + owner: "user", + } + : null, + ), call: () => Effect.succeed({ ok: true }), }; @@ -26,13 +46,11 @@ const fails = (effect: Effect.Effect): Promise => Effect.runPromiseExit(effect).then((exit) => exit._tag === "Failure"); describe("HandleBridge strict dispatch", () => { - const declared: Record = { - gh: { kind: "single", integration: "github" }, - inboxes: { kind: "array", integration: "gmail" }, + const declared: Record = { + gh: { integration: "github" }, }; const bindings = { - gh: { kind: "single", connection: "gh-main" } as const, - inboxes: { kind: "array", connections: ["a", "b"] } as const, + gh: "tools.github.user.main", }; const bridge = buildBridge({ declared, bindings, db: okDb, resolver: okResolver }); @@ -48,13 +66,7 @@ describe("HandleBridge strict dispatch", () => { expect(await fails(bridge.call({ root: "ghost", path: ["x"], args: [] }))).toBe(true); }); - it("rejects an out-of-range fan-out index", async () => { - expect( - await fails(bridge.call({ root: "inboxes#5", path: ["messages", "list"], args: [] })), - ).toBe(true); - }); - - it("rejects an index on a single-connection role", async () => { + it("rejects an index on an integration role", async () => { expect(await fails(bridge.call({ root: "gh#0", path: ["repos", "list"], args: [] }))).toBe( true, ); @@ -71,10 +83,40 @@ describe("HandleBridge strict dispatch", () => { expect(out).toEqual({ ok: true }); }); - it("routes a valid fan-out element through the resolver", async () => { - const out = await Effect.runPromise( - bridge.call({ root: "inboxes#1", path: ["messages", "list"], args: [{}] }), - ); - expect(out).toEqual({ ok: true }); + it("defaults a missing role property when exactly one connection exists", async () => { + const resolved = await Effect.runPromise(resolveIntegrationBindings(declared, {}, okResolver)); + expect(resolved.bindings).toEqual({ gh: "tools.github.user.main" }); + expect(resolved.input).toEqual({}); + }); + + it("fails when several connections exist and the role property is missing", async () => { + const resolver: ClientResolver = { + ...okResolver, + listConnections: () => + Effect.succeed([ + { address: "tools.github.user.one", integration: "github" }, + { address: "tools.github.user.two", integration: "github" }, + ]), + }; + expect(await fails(resolveIntegrationBindings(declared, {}, resolver))).toBe(true); + }); + + it("fails when a requested connection belongs to a different integration", async () => { + const resolver: ClientResolver = { + ...okResolver, + resolveConnection: () => + Effect.succeed({ address: "tools.gmail.user.main", integration: "gmail" }), + }; + expect( + await fails(resolveIntegrationBindings(declared, { gh: "tools.gmail.user.main" }, resolver)), + ).toBe(true); + }); + + it("fails when a requested connection is unknown", async () => { + expect( + await fails( + resolveIntegrationBindings(declared, { gh: "tools.github.user.missing" }, okResolver), + ), + ).toBe(true); }); }); diff --git a/packages/plugins/apps/src/plugin/bindings.ts b/packages/plugins/apps/src/plugin/bindings.ts index 67e8227a5..1688ea37f 100644 --- a/packages/plugins/apps/src/plugin/bindings.ts +++ b/packages/plugins/apps/src/plugin/bindings.ts @@ -4,40 +4,47 @@ import { Data } from "effect"; import type { HandleBridge, HandleRootSpec } from "../seams/tool-sandbox"; import { ToolSandboxError } from "../seams/tool-sandbox"; import type { ScopeDbHandle } from "../seams/scope-db"; -import type { ConnectionDecl } from "../pipeline/descriptor"; +import type { IntegrationDecl } from "../pipeline/descriptor"; // --------------------------------------------------------------------------- -// Connection DI: declare-then-bind. A tool's descriptor declares `connections` -// (role -> integration); before execution each role is bound to the user's -// connection(s) explicitly. A missing binding is a typed error naming role + -// surface (no auto-pick). The handler receives pre-bound clients whose method -// calls route through the platform invoke path. +// Integration DI: declare in source, choose at invocation. // -// The routing is a seam: `ClientResolver` turns a (integration, connection, -// method path, args) into a JSON result. The self-hosted resolver calls the -// integration's real API through the connection credential (policy/audit -// applies there); a test resolver returns canned data. `db` is bound to the -// invoking scope's ScopeDb. Everything crossing is JSON (the cloud resolver is -// an RPC). +// A tool descriptor declares `integrations` (role -> integration slug). The +// caller supplies one connection address per role in the tool-call payload, and +// this layer peels those role properties off before the handler runs. If an org +// has exactly one connection for a role's integration, that connection is used +// as the default. Missing, unknown, or wrong-integration choices are typed +// BindingError failures. The handler receives clients under the declared role +// names and `db`. // --------------------------------------------------------------------------- export class BindingError extends Data.TaggedError("BindingError")<{ readonly message: string; readonly role: string; - readonly surface: string; + readonly integration: string; + readonly requestedConnection?: string; }> {} -/** One bound connection for a role. Fan-out roles bind an ordered set. */ -export type RoleBinding = - | { readonly kind: "single"; readonly connection: string } - | { readonly kind: "array"; readonly connections: readonly string[] }; +export interface ConnectionCandidate { + readonly address: string; + readonly integration: string; + readonly name?: string; + readonly owner?: string; +} -/** The user's bindings for a tool invocation: role -> bound connection(s). */ -export type Bindings = Readonly>; +/** The resolved connections for a tool invocation: role -> connection address. */ +export type RoleBindings = Readonly>; -/** Resolves a single method call against a bound connection to a JSON result. - * This is where the platform invoke path handles credentials, policy, audit. */ +/** Resolves connection inventory and method calls for declared integration + * roles. Host wiring will back this with core connection lookup and normal + * policy/credential invocation. */ export interface ClientResolver { + readonly listConnections: (input: { + readonly integration: string; + }) => Effect.Effect; + readonly resolveConnection: (input: { + readonly connection: string; + }) => Effect.Effect; readonly call: (input: { readonly integration: string; readonly connection: string; @@ -47,83 +54,151 @@ export interface ClientResolver { } export interface BindingContext { - /** Declared connection roles from the tool descriptor. */ - readonly declared: Readonly>; - /** The user's bindings (role -> connection[s]). Missing => typed error. */ - readonly bindings: Bindings; + /** Declared integration roles from the tool descriptor. */ + readonly declared: Readonly>; + /** Resolved role -> connection address choices for this invocation. */ + readonly bindings: RoleBindings; /** The invoking scope's app database (bound to the `db` root). */ readonly db: ScopeDbHandle; /** Routes a bound method call to the real integration. */ readonly resolver: ClientResolver; } -/** - * Validate bindings against declarations and compute the sandbox handle roots: - * `db` is always a single root; each declared role becomes a single or array - * root. Fails (typed) when a declared role has no binding, naming role + - * surface — the "missing binding is a typed error" rule. - */ -export const rootsFor = ( - declared: Readonly>, - bindings: Bindings, -): Effect.Effect>, BindingError> => +const isRecord = (value: unknown): value is Record => + value !== null && typeof value === "object" && !Array.isArray(value); + +const bindingError = ( + role: string, + integration: string, + message: string, + requestedConnection?: string, +): BindingError => + new BindingError({ + message, + role, + integration, + requestedConnection, + }); + +const findDefaultConnection = ( + role: string, + decl: IntegrationDecl, + resolver: ClientResolver, +): Effect.Effect => + resolver.listConnections({ integration: decl.integration }).pipe( + Effect.flatMap((connections) => { + if (connections.length === 1) return Effect.succeed(connections[0]!.address); + return Effect.fail( + bindingError( + role, + decl.integration, + connections.length === 0 + ? `missing required connection for role "${role}" (${decl.integration}); no connections are available` + : `missing required connection for role "${role}" (${decl.integration}); choose one of ${connections + .map((c) => c.address) + .join(", ")}`, + ), + ); + }), + ); + +const resolveRequestedConnection = ( + role: string, + decl: IntegrationDecl, + requested: string, + resolver: ClientResolver, +): Effect.Effect => + resolver.resolveConnection({ connection: requested }).pipe( + Effect.flatMap((connection) => { + if (!connection) { + return Effect.fail( + bindingError( + role, + decl.integration, + `unknown connection "${requested}" for role "${role}" (${decl.integration})`, + requested, + ), + ); + } + if (connection.integration !== decl.integration) { + return Effect.fail( + bindingError( + role, + decl.integration, + `connection "${requested}" belongs to integration "${connection.integration}", not "${decl.integration}" for role "${role}"`, + requested, + ), + ); + } + return Effect.succeed(connection.address); + }), + ); + +export interface ResolvedIntegrationBindings { + readonly input: unknown; + readonly bindings: RoleBindings; +} + +/** Peel platform-added role properties from the caller payload and resolve them + * to concrete connection addresses. */ +export const resolveIntegrationBindings = ( + declared: Readonly>, + args: unknown, + resolver: ClientResolver, +): Effect.Effect => Effect.gen(function* () { - const roots: Record = { db: { kind: "single" } }; - for (const [role, decl] of Object.entries(declared)) { - if (decl.kind === "catalog") { - // Open-world proxy: parse + record, but execution is NotImplemented in - // this build. Bind a single root; the resolver throws if called. - roots[role] = { kind: "single" }; + const roles = Object.entries(declared); + if (roles.length === 0) return { input: args, bindings: {} }; + + const payload = isRecord(args) ? args : {}; + const input: Record = {}; + for (const [key, value] of Object.entries(payload)) { + if (!Object.prototype.hasOwnProperty.call(declared, key)) input[key] = value; + } + + const bindings: Record = {}; + for (const [role, decl] of roles) { + const raw = payload[role]; + if (raw === undefined) { + bindings[role] = yield* findDefaultConnection(role, decl, resolver); continue; } - const binding = bindings[role]; - if (!binding) { + if (typeof raw !== "string" || raw.length === 0) { return yield* Effect.fail( - new BindingError({ - message: `no connection bound for role "${role}" (surface "${decl.integration}")`, + bindingError( role, - surface: decl.integration, - }), + decl.integration, + `connection for role "${role}" (${decl.integration}) must be a non-empty string`, + typeof raw === "string" ? raw : undefined, + ), ); } - if (decl.kind === "array") { - if (binding.kind !== "array") { - return yield* Effect.fail( - new BindingError({ - message: `role "${role}" is a fan-out (connections("${decl.integration}")) and needs an array binding`, - role, - surface: decl.integration, - }), - ); - } - roots[role] = { kind: "array", count: binding.connections.length }; - } else { - if (binding.kind !== "single") { - return yield* Effect.fail( - new BindingError({ - message: `role "${role}" is a single connection and needs a single binding`, - role, - surface: decl.integration, - }), - ); - } - roots[role] = { kind: "single" }; - } + bindings[role] = yield* resolveRequestedConnection(role, decl, raw, resolver); } - return roots; + + return { input, bindings }; }); -// Parse a fan-out root name back into (role, index): `inboxes#1` -> {inboxes,1}. +/** Compute the sandbox handle roots: `db` plus one single root per declared + * integration role. */ +export const rootsFor = ( + declared: Readonly>, +): Readonly> => { + const roots: Record = { db: { kind: "single" } }; + for (const role of Object.keys(declared)) { + roots[role] = { kind: "single" }; + } + return roots; +}; + +// Parse an indexed root name. Indexed roots are not supported in v1, but the +// parser lets the bridge reject `role#0` explicitly instead of misrouting it. const parseRoot = (root: string): { role: string; index?: number } => { const hash = root.indexOf("#"); if (hash === -1) return { role: root }; return { role: root.slice(0, hash), index: Number(root.slice(hash + 1)) }; }; -// Root names the bridge treats as reserved: nothing the sandbox sends may -// address them as a connection role. `db` is handled explicitly first; any other -// reserved prefix (a future internal handle) is rejected rather than silently -// resolved. (Grafted strictness from A: reject unexpected/reserved calls.) const RESERVED_ROOTS = new Set(["__proto__", "constructor", "prototype"]); const invokeErr = (message: string): ToolSandboxError => @@ -131,16 +206,14 @@ const invokeErr = (message: string): ToolSandboxError => /** * Build the HandleBridge the sandbox calls out through. `db` routes to the - * scope database; a declared role routes to its bound connection through the + * scope database; a declared role routes to its resolved connection through the * `ClientResolver`. The dispatch is STRICT: an empty/malformed path, a reserved - * root, an undeclared root, or an out-of-range fan-out index is a typed error, - * never a silent success (the cloud RPC backing must be able to trust the same - * shape). A `.account` read on a client returns bound-connection metadata + * root, an undeclared root, or an indexed root is an error, never a silent + * success. A `.account` read on a client returns bound-connection metadata * without a round-trip. */ export const buildBridge = (context: BindingContext): HandleBridge => ({ call: ({ root, path, args }) => { - // Strictness: every bridge call must name a root and a non-empty method path. if (typeof root !== "string" || root.length === 0) { return Effect.fail(invokeErr("bridge call is missing a handle root")); } @@ -152,9 +225,6 @@ export const buildBridge = (context: BindingContext): HandleBridge => ({ } if (root === "db") { - // The scope db handle exposes `sql` as a tagged template; the injected - // client calls `db.sql(templateStrings, ...values)`. When routed through - // the bridge, `path = ["sql"]` and args = [stringsArray, ...values]. if (path.length === 1 && path[0] === "sql") { const [strings, ...values] = args as [TemplateStringsArray, ...unknown[]]; return context.db @@ -169,46 +239,22 @@ export const buildBridge = (context: BindingContext): HandleBridge => ({ } const { role, index } = parseRoot(root); - const decl = context.declared[role]; - if (!decl) { - return Effect.fail(invokeErr(`undeclared handle root: ${root}`)); - } - if (decl.kind === "catalog") { + if (index !== undefined) { return Effect.fail( - invokeErr("catalog() open-world proxy execution is not implemented in this build"), + invokeErr(`single integration role "${role}" was addressed with an index`), ); } - const binding = context.bindings[role]; - // Fan-out roots MUST carry a valid index within the bound array; a single - // role MUST NOT carry an index. (Grafted bounds-checking from A.) - if (decl.kind === "array") { - if (binding?.kind !== "array") { - return Effect.fail(invokeErr(`role "${role}" is a fan-out but is not bound to an array`)); - } - if ( - index === undefined || - !Number.isInteger(index) || - index < 0 || - index >= binding.connections.length - ) { - return Effect.fail(invokeErr(`fan-out handle "${root}" addresses an out-of-range index`)); - } - } else if (index !== undefined) { - return Effect.fail(invokeErr(`single-connection role "${role}" was addressed with an index`)); + const decl = context.declared[role]; + if (!decl) { + return Effect.fail(invokeErr(`undeclared handle root: ${root}`)); } - const connectionName = - binding?.kind === "array" - ? binding.connections[index ?? 0] - : binding?.kind === "single" - ? binding.connection - : undefined; + const connectionName = context.bindings[role]; if (!connectionName) { return Effect.fail(invokeErr(`no binding for role ${role}`)); } if (path.length === 1 && path[0] === "account") { - // Clients read `.account.email` / `.account.login`; expose both keys. return Effect.succeed({ email: connectionName, login: connectionName, name: connectionName }); } diff --git a/packages/plugins/apps/src/plugin/runtime.test.ts b/packages/plugins/apps/src/plugin/runtime.test.ts index e4a1b0153..13db88451 100644 --- a/packages/plugins/apps/src/plugin/runtime.test.ts +++ b/packages/plugins/apps/src/plugin/runtime.test.ts @@ -7,7 +7,6 @@ import { Effect } from "effect"; import { makeSelfHostAppsRuntime } from "./self-host-runtime"; import { makeInMemoryAppsStore, makeTestResolver, dailyBriefFileSet } from "../testing"; -import type { Bindings } from "./bindings"; const run = (effect: Effect.Effect): Promise => Effect.runPromise(effect); @@ -35,12 +34,12 @@ const githubHandlers = { }, }; -const githubBindings: Bindings = { github: { kind: "single", connection: "rhys-github" } }; - describe("AppsRuntime end-to-end (publish -> invoke)", () => { it("publishes daily-brief and invokes the tool into the scope db", async () => { const store = makeInMemoryAppsStore(); - const resolver = makeTestResolver(githubHandlers); + const resolver = makeTestResolver(githubHandlers, [ + { address: "tools.github.user.rhys-github", integration: "github", name: "rhys-github" }, + ]); const host = makeSelfHostAppsRuntime({ dataDir: mkdtempSync(join(tmpdir(), "apps-rt-")), store, @@ -59,8 +58,7 @@ describe("AppsRuntime end-to-end (publish -> invoke)", () => { runtime.invokeTool({ scope: "rhys", tool: "issues-sync", - args: {}, - bindings: githubBindings, + args: { github: "tools.github.user.rhys-github" }, }), )) as { synced: number; repos: number }; expect(syncResult).toEqual({ synced: 2, repos: 1 }); @@ -71,4 +69,179 @@ describe("AppsRuntime end-to-end (publish -> invoke)", () => { await host.close(); }); + + it("applies the single available connection as the default", async () => { + const store = makeInMemoryAppsStore(); + const resolver = makeTestResolver( + { + dealcloud: { + "deals.list": () => [], + }, + }, + [{ address: "tools.dealcloud.user.crm-main", integration: "dealcloud", name: "crm-main" }], + ); + const host = makeSelfHostAppsRuntime({ + dataDir: mkdtempSync(join(tmpdir(), "apps-default-")), + store, + resolver, + inMemory: true, + }); + + await run( + host.runtime.publish({ + scope: "s", + files: new Map([ + [ + "tools/sync.ts", + `import { defineTool, integration } from "executor:app"; +import { z } from "zod"; +export default defineTool({ + description: "sync", + integrations: { crm: integration("dealcloud") }, + input: z.object({}), + async handler(_input, { crm }) { + await crm.deals.list({}); + return { ok: true }; + }, +});`, + ], + ]), + }), + ); + + const out = await run(host.runtime.invokeTool({ scope: "s", tool: "sync", args: {} })); + + expect(out).toEqual({ ok: true }); + expect(resolver.calls[0]).toEqual({ + integration: "dealcloud", + connection: "tools.dealcloud.user.crm-main", + method: "deals.list", + }); + await host.close(); + }); + + it("fails with BindingError when multiple connections exist and the role property is missing", async () => { + const store = makeInMemoryAppsStore(); + const resolver = makeTestResolver({}, [ + { address: "tools.dealcloud.user.one", integration: "dealcloud" }, + { address: "tools.dealcloud.user.two", integration: "dealcloud" }, + ]); + const host = makeSelfHostAppsRuntime({ + dataDir: mkdtempSync(join(tmpdir(), "apps-missing-")), + store, + resolver, + inMemory: true, + }); + + await run( + host.runtime.publish({ + scope: "s", + files: new Map([ + [ + "tools/sync.ts", + `import { defineTool, integration } from "executor:app"; +import { z } from "zod"; +export default defineTool({ + description: "sync", + integrations: { crm: integration("dealcloud") }, + input: z.object({}), + async handler(){ return {}; }, +});`, + ], + ]), + }), + ); + + const exit = await Effect.runPromiseExit( + host.runtime.invokeTool({ scope: "s", tool: "sync", args: {} }), + ); + + expect(exit._tag).toBe("Failure"); + if (exit._tag !== "Failure") throw new Error("expected missing connection failure"); + expect(JSON.stringify(exit.cause)).toContain("BindingError"); + expect(JSON.stringify(exit.cause)).toContain("crm"); + expect(JSON.stringify(exit.cause)).toContain("dealcloud"); + await host.close(); + }); + + it("fails with BindingError when the named connection belongs to another integration", async () => { + const store = makeInMemoryAppsStore(); + const resolver = makeTestResolver({}, [ + { address: "tools.github.user.main", integration: "github" }, + ]); + const host = makeSelfHostAppsRuntime({ + dataDir: mkdtempSync(join(tmpdir(), "apps-wrong-")), + store, + resolver, + inMemory: true, + }); + + await run( + host.runtime.publish({ + scope: "s", + files: new Map([ + [ + "tools/sync.ts", + `import { defineTool, integration } from "executor:app"; +import { z } from "zod"; +export default defineTool({ + description: "sync", + integrations: { crm: integration("dealcloud") }, + input: z.object({}), + async handler(){ return {}; }, +});`, + ], + ]), + }), + ); + + const exit = await Effect.runPromiseExit( + host.runtime.invokeTool({ + scope: "s", + tool: "sync", + args: { crm: "tools.github.user.main" }, + }), + ); + + expect(exit._tag).toBe("Failure"); + if (exit._tag !== "Failure") throw new Error("expected wrong integration failure"); + expect(JSON.stringify(exit.cause)).toContain("BindingError"); + expect(JSON.stringify(exit.cause)).toContain("github"); + expect(JSON.stringify(exit.cause)).toContain("dealcloud"); + expect(JSON.stringify(exit.cause)).toContain("tools.github.user.main"); + await host.close(); + }); + + it("accepts raw JSON Schema input end to end", async () => { + const store = makeInMemoryAppsStore(); + const resolver = makeTestResolver({}); + const host = makeSelfHostAppsRuntime({ + dataDir: mkdtempSync(join(tmpdir(), "apps-raw-")), + store, + resolver, + inMemory: true, + }); + + await run( + host.runtime.publish({ + scope: "s", + files: new Map([ + [ + "tools/raw.ts", + `import { defineTool } from "executor:app"; +export default defineTool({ + description: "raw", + input: { type: "object", properties: { q: { type: "string" } }, required: ["q"] }, + async handler(input){ return { q: input.q }; }, +});`, + ], + ]), + }), + ); + + const out = await run(host.runtime.invokeTool({ scope: "s", tool: "raw", args: { q: "ok" } })); + + expect(out).toEqual({ q: "ok" }); + await host.close(); + }); }); diff --git a/packages/plugins/apps/src/plugin/runtime.ts b/packages/plugins/apps/src/plugin/runtime.ts index 0b47cc01f..26401660c 100644 --- a/packages/plugins/apps/src/plugin/runtime.ts +++ b/packages/plugins/apps/src/plugin/runtime.ts @@ -2,7 +2,11 @@ import { Effect } from "effect"; import type { ArtifactStore } from "../seams/artifact-store"; import type { ScopeDb } from "../seams/scope-db"; -import type { ToolSandbox } from "../seams/tool-sandbox"; +import { + InputValidationError, + OutputValidationError, + type ToolSandbox, +} from "../seams/tool-sandbox"; import { publish as runPublish, loadDescriptorFromSnapshot, @@ -14,7 +18,7 @@ import { bundleEntry } from "../pipeline/bundle"; import { buildBridge, rootsFor, - type Bindings, + resolveIntegrationBindings, type ClientResolver, BindingError, } from "./bindings"; @@ -47,11 +51,13 @@ export interface AppsRuntime { readonly scope: string; readonly tool: string; readonly args: unknown; - readonly bindings: Bindings; /** Optional per-request resolver override. The catalog invoke path supplies * one built from the request's executor context. */ readonly resolver?: ClientResolver; - }) => Effect.Effect; + }) => Effect.Effect< + unknown, + PublishError | BindingError | InputValidationError | OutputValidationError + >; readonly deps: AppsRuntimeDeps; } @@ -185,11 +191,19 @@ export const makeAppsRuntime = (deps: AppsRuntimeDeps): AppsRuntime => { descriptor: AppDescriptor, toolDesc: ToolDescriptor, args: unknown, - bindings: Bindings, resolver?: ClientResolver, - ): Effect.Effect => + ): Effect.Effect< + unknown, + PublishError | BindingError | InputValidationError | OutputValidationError + > => Effect.gen(function* () { - const roots = yield* rootsFor(toolDesc.connections, bindings); + const activeResolver = resolver ?? deps.resolver; + const resolved = yield* resolveIntegrationBindings( + toolDesc.integrations, + args, + activeResolver, + ); + const roots = rootsFor(toolDesc.integrations); const code = yield* bundleFor(descriptor, toolDesc.sourcePath); const db = yield* deps.scopeDb.forScope(scope).pipe( Effect.mapError( @@ -202,22 +216,32 @@ export const makeAppsRuntime = (deps: AppsRuntimeDeps): AppsRuntime => { ), ); const bridge = buildBridge({ - declared: toolDesc.connections, - bindings, + declared: toolDesc.integrations, + bindings: resolved.bindings, db, - resolver: resolver ?? deps.resolver, + resolver: activeResolver, }); const result = yield* deps.sandbox - .invoke(code, { artifact: toolDesc.name, kind: "tool", input: args, roots }, bridge) + .invoke( + code, + { artifact: toolDesc.name, kind: "tool", input: resolved.input, roots }, + bridge, + ) .pipe( - Effect.mapError( - (c) => - new PublishError({ - message: c.message, - stage: "project", - diagnostics: [{ path: toolDesc.sourcePath, message: c.message }], - }), - ), + Effect.mapError((c) => { + if ( + c instanceof BindingError || + c instanceof InputValidationError || + c instanceof OutputValidationError + ) { + return c; + } + return new PublishError({ + message: c.message, + stage: "project", + diagnostics: [{ path: toolDesc.sourcePath, message: c.message }], + }); + }), ); return result.output; }); @@ -287,7 +311,6 @@ export const makeAppsRuntime = (deps: AppsRuntimeDeps): AppsRuntime => { descriptor, toolDesc, input.args, - input.bindings, input.resolver, ); }), diff --git a/packages/plugins/apps/src/seams/tool-sandbox.conformance.ts b/packages/plugins/apps/src/seams/tool-sandbox.conformance.ts index 119ab84b6..bf58e2f4e 100644 --- a/packages/plugins/apps/src/seams/tool-sandbox.conformance.ts +++ b/packages/plugins/apps/src/seams/tool-sandbox.conformance.ts @@ -9,7 +9,7 @@ import { bundleEntry } from "../pipeline/bundle"; // - collect determinism catches Math.random (double-run byte-compare) // - network denial (fetch throws) // - timeout kill (an infinite loop is interrupted) -// - handle bridge round-trip including fan-out arrays (connections("x")) +// - handle bridge round-trip for declared integration roles // A future Worker Loaders backing must pass this same suite. // --------------------------------------------------------------------------- @@ -63,24 +63,20 @@ export default defineTool({ description: "loop", input: z.object({}), async hand expect(exit._tag).toBe("Failure"); }); - it("round-trips the handle bridge, including fan-out arrays", async () => { + it("round-trips the handle bridge for a declared integration role", async () => { const sandbox = makeSandbox(); - // A tool that fans out over an array of clients and calls a method on each. - const src = `import { defineTool, connections } from "executor:app"; + const src = `import { defineTool, integration } from "executor:app"; import { z } from "zod"; export default defineTool({ - description: "fanout", - connections: { inboxes: connections("gmail") }, + description: "search", + integrations: { inbox: integration("gmail") }, input: z.object({ q: z.string() }), - async handler({ q }, { inboxes }) { - const per = await Promise.all(inboxes.map(async (inbox, i) => { - const r = await inbox.messages.search({ q, index: i }); - return r.count; - })); - return { total: per.reduce((a, b) => a + b, 0), n: inboxes.length }; + async handler({ q }, { inbox }) { + const r = await inbox.messages.search({ q }); + return { total: r.count }; }, });`; - const b = await bundle("tools/fanout.ts", src); + const b = await bundle("tools/search.ts", src); const seen: { root: string; path: readonly string[]; args: readonly unknown[] }[] = []; const bridge: HandleBridge = { @@ -95,19 +91,17 @@ export default defineTool({ sandbox.invoke( b, { - artifact: "fanout", + artifact: "search", kind: "tool", input: { q: "invoice" }, - roots: { inboxes: { kind: "array", count: 2 } }, + roots: { inbox: { kind: "single" } }, }, bridge, ), ); - expect(result.output).toEqual({ total: 6, n: 2 }); - // Two distinct fan-out roots were addressed (inboxes#0 and inboxes#1). + expect(result.output).toEqual({ total: 3 }); const roots = new Set(seen.map((s) => s.root)); - expect(roots.has("inboxes#0")).toBe(true); - expect(roots.has("inboxes#1")).toBe(true); + expect(roots.has("inbox")).toBe(true); // The method path and JSON args crossed the boundary intact. expect(seen[0].path).toEqual(["messages", "search"]); expect(seen[0].args[0]).toMatchObject({ q: "invoice" }); @@ -120,11 +114,11 @@ export default defineTool({ // A tool with a github client AND a db.sql write, asserting BOTH the // connection call and the parameterized db statement cross the bridge with // their args intact (the db.sql param round-trip A's suite covers). - const src = `import { defineTool, connection } from "executor:app"; + const src = `import { defineTool, integration } from "executor:app"; import { z } from "zod"; export default defineTool({ description: "bridge", - connections: { gh: connection("github") }, + integrations: { gh: integration("github") }, input: z.object({}), async handler(_i, { gh, db }) { const repos = await gh.repos.listForAuthenticatedUser({ per_page: 5 }); @@ -167,50 +161,5 @@ export default defineTool({ expect(strings.join("?")).toContain("INSERT INTO log"); expect(params).toEqual([2]); }); - - it("fans out per-index, each element addressed distinctly", async () => { - const sandbox = makeSandbox(); - // inbox i returns (i+1) messages; the handler pushes each count, so the - // ordered result proves each fan-out element was addressed with its index. - const src = `import { defineTool, connections } from "executor:app"; -import { z } from "zod"; -export default defineTool({ - description: "fanout", - connections: { inboxes: connections("gmail") }, - input: z.object({}), - async handler(_i, { inboxes }) { - const counts = []; - for (const inbox of inboxes) { - const r = await inbox.messages.search({ q: "x" }); - counts.push(r.length); - } - return { inboxes: inboxes.length, counts }; - }, -});`; - const b = await bundle("tools/fanout2.ts", src); - const bridge: HandleBridge = { - call: ({ root, path }) => - Effect.sync(() => { - if (path.join(".") === "messages.search") { - const idx = Number(root.split("#")[1]); - return new Array(idx + 1).fill({ id: "m" }); - } - return []; - }), - }; - const result = await run( - sandbox.invoke( - b, - { - artifact: "fanout2", - kind: "tool", - input: {}, - roots: { inboxes: { kind: "array", count: 3 } }, - }, - bridge, - ), - ); - expect(result.output).toEqual({ inboxes: 3, counts: [1, 2, 3] }); - }); }); }; diff --git a/packages/plugins/apps/src/seams/tool-sandbox.ts b/packages/plugins/apps/src/seams/tool-sandbox.ts index f2d717e23..c05ed9080 100644 --- a/packages/plugins/apps/src/seams/tool-sandbox.ts +++ b/packages/plugins/apps/src/seams/tool-sandbox.ts @@ -18,8 +18,7 @@ import { Data } from "effect"; // pre-bound clients whose method calls cross OUT through a serializable // bridge (`HandleBridge.call`). EVERYTHING crossing the boundary is // serializable — the cloud version of this seam is an RPC, so the interface -// forbids passing functions or runtime objects across. Fan-out arrays -// (`connections("gmail")` -> client[]) are modeled as indexed handle roots. +// forbids passing functions or runtime objects across. // // The self-hosted backing is QuickJS (packages/kernel/runtime-quickjs), whose // `SandboxToolInvoker.invoke({path, args})` already matches the bridge shape. @@ -33,6 +32,21 @@ export class ToolSandboxError extends Data.TaggedError("ToolSandboxError")<{ readonly cause?: unknown; }> {} +export interface ValidationIssue { + readonly message: string; + readonly path?: readonly unknown[]; +} + +export class InputValidationError extends Data.TaggedError("InputValidationError")<{ + readonly message: string; + readonly issues: readonly ValidationIssue[]; +}> {} + +export class OutputValidationError extends Data.TaggedError("OutputValidationError")<{ + readonly message: string; + readonly issues: readonly ValidationIssue[]; +}> {} + /** * The serializable bridge the sandbox calls out through. `root` names an * injected handle (a connection role, `db`, or one element of a fan-out set); @@ -48,13 +62,12 @@ export interface HandleBridge { }) => Effect.Effect; } -/** Which handle roots to inject and, for fan-out roots, how many elements. A - * single connection role is `{ kind: "single" }`; a fan-out is - * `{ kind: "array", count }`; `db` is a single. Everything the handler can see - * is enumerated here — undeclared roots are simply absent. */ -export type HandleRootSpec = - | { readonly kind: "single" } - | { readonly kind: "array"; readonly count: number }; +/** Which handle roots to inject. Each declared integration role and `db` are + * single roots. Everything the handler can see is enumerated here — + * undeclared roots are simply absent. */ +export interface HandleRootSpec { + readonly kind: "single"; +} export interface InvokeRequest { /** The artifact whose handler to run (path identity, e.g. `issues-sync`). */ @@ -86,13 +99,21 @@ export interface CollectResult { readonly artifacts: Readonly>; } +export interface CollectRequest { + /** The path-derived artifact identity, used only for diagnostics. */ + readonly artifact?: string; +} + export interface ToolSandbox { /** * Import the bundle with nothing bound and gather the `defineTool` descriptor. * Runs the collection twice internally and byte-compares; a mismatch fails * with `kind: "nondeterministic"`. This is the determinism gate. */ - readonly collect: (bundle: string) => Effect.Effect; + readonly collect: ( + bundle: string, + request?: CollectRequest, + ) => Effect.Effect; /** * Run one artifact's handler with injected handles bridged through `bridge`. * Network is denied; a per-call timeout kills a runaway handler. @@ -101,5 +122,5 @@ export interface ToolSandbox { bundle: string, request: InvokeRequest, bridge: HandleBridge, - ) => Effect.Effect; + ) => Effect.Effect; } diff --git a/packages/plugins/apps/src/standard-schema.ts b/packages/plugins/apps/src/standard-schema.ts new file mode 100644 index 000000000..8aa04a05e --- /dev/null +++ b/packages/plugins/apps/src/standard-schema.ts @@ -0,0 +1,153 @@ +// Type-only vendored subset of @standard-schema/spec 1.1.0. +// Source: https://github.com/standard-schema/standard-schema + +/** The Standard Typed interface. This is a base type extended by other specs. */ +export interface StandardTypedV1 { + /** The Standard properties. */ + readonly "~standard": StandardTypedV1.Props; +} + +export namespace StandardTypedV1 { + /** The Standard Typed properties interface. */ + export interface Props { + /** The version number of the standard. */ + readonly version: 1; + /** The vendor name of the schema library. */ + readonly vendor: string; + /** Inferred types associated with the schema. */ + readonly types?: Types | undefined; + } + + /** The Standard Typed types interface. */ + export interface Types { + /** The input type of the schema. */ + readonly input: Input; + /** The output type of the schema. */ + readonly output: Output; + } + + /** Infers the input type of a Standard Typed. */ + export type InferInput = NonNullable< + Schema["~standard"]["types"] + >["input"]; + + /** Infers the output type of a Standard Typed. */ + export type InferOutput = NonNullable< + Schema["~standard"]["types"] + >["output"]; +} + +/** The Standard Schema interface. */ +export interface StandardSchemaV1 { + /** The Standard Schema properties. */ + readonly "~standard": StandardSchemaV1.Props; +} + +export namespace StandardSchemaV1 { + /** The Standard Schema properties interface. */ + export interface Props extends StandardTypedV1.Props< + Input, + Output + > { + /** Validates unknown input values. */ + readonly validate: ( + value: unknown, + options?: StandardSchemaV1.Options | undefined, + ) => Result | Promise>; + } + + /** The result interface of the validate function. */ + export type Result = SuccessResult | FailureResult; + + /** The result interface if validation succeeds. */ + export interface SuccessResult { + /** The typed output value. */ + readonly value: Output; + /** A falsy value for `issues` indicates success. */ + readonly issues?: undefined; + } + + export interface Options { + /** Explicit support for additional vendor-specific parameters, if needed. */ + readonly libraryOptions?: Record | undefined; + } + + /** The result interface if validation fails. */ + export interface FailureResult { + /** The issues of failed validation. */ + readonly issues: ReadonlyArray; + } + + /** The issue interface of the failure output. */ + export interface Issue { + /** The error message of the issue. */ + readonly message: string; + /** The path of the issue, if any. */ + readonly path?: ReadonlyArray | undefined; + } + + /** The path segment interface of the issue. */ + export interface PathSegment { + /** The key representing a path segment. */ + readonly key: PropertyKey; + } + + /** The Standard types interface. */ + export interface Types extends StandardTypedV1.Types< + Input, + Output + > {} + + /** Infers the input type of a Standard. */ + export type InferInput = StandardTypedV1.InferInput; + + /** Infers the output type of a Standard. */ + export type InferOutput = StandardTypedV1.InferOutput; +} + +/** The Standard JSON Schema interface. */ +export interface StandardJSONSchemaV1 { + /** The Standard JSON Schema properties. */ + readonly "~standard": StandardJSONSchemaV1.Props; +} + +export namespace StandardJSONSchemaV1 { + /** The Standard JSON Schema properties interface. */ + export interface Props extends StandardTypedV1.Props< + Input, + Output + > { + /** Methods for generating the input/output JSON Schema. */ + readonly jsonSchema: StandardJSONSchemaV1.Converter; + } + + /** The Standard JSON Schema converter interface. */ + export interface Converter { + /** Converts the input type to JSON Schema. */ + readonly input: (options: StandardJSONSchemaV1.Options) => Record; + /** Converts the output type to JSON Schema. */ + readonly output: (options: StandardJSONSchemaV1.Options) => Record; + } + + export type Target = "draft-2020-12" | "draft-07" | "openapi-3.0" | ({} & string); + + /** The options for the input/output methods. */ + export interface Options { + /** Specifies the target version of the generated JSON Schema. */ + readonly target: Target; + /** Explicit support for additional vendor-specific parameters, if needed. */ + readonly libraryOptions?: Record | undefined; + } + + /** The Standard types interface. */ + export interface Types extends StandardTypedV1.Types< + Input, + Output + > {} + + /** Infers the input type of a Standard. */ + export type InferInput = StandardTypedV1.InferInput; + + /** Infers the output type of a Standard. */ + export type InferOutput = StandardTypedV1.InferOutput; +} diff --git a/packages/plugins/apps/src/testing/daily-brief.ts b/packages/plugins/apps/src/testing/daily-brief.ts index 0bbfab182..bda340065 100644 --- a/packages/plugins/apps/src/testing/daily-brief.ts +++ b/packages/plugins/apps/src/testing/daily-brief.ts @@ -2,19 +2,18 @@ export const DAILY_BRIEF_MANIFEST = `{ "$schema": "https://executor.sh/schemas/scope-manifest.json", "scope": "rhys", "description": "Personal scope artifacts, GitHub issues brief.", - "connections": { "github": "github/rhys" }, "artifacts": { "tools": "tools/" } } `; export const ISSUES_SYNC_TS = `import { z } from "zod"; -import { defineTool, connection } from "executor:app"; +import { defineTool, integration } from "executor:app"; export default defineTool({ description: "Refresh the scope \\\`issues\\\` table from GitHub. Syncs open issues across the given repos (default: every repo the connection can see).", - connections: { - github: connection("github", { description: "GitHub account whose issues to sync" }), + integrations: { + github: integration("github"), }, input: z.object({ repos: z.array(z.string()).optional().describe("owner/repo entries; omit to sync all accessible repos"), @@ -57,13 +56,13 @@ export default defineTool({ `; export const SEARCH_ALL_MAIL_TS = `import { z } from "zod"; -import { defineTool, connections } from "executor:app"; +import { defineTool, integration } from "executor:app"; export default defineTool({ description: - "Search across all connected Gmail accounts. Returns matches newest-first, tagged with which inbox they came from.", - connections: { - inboxes: connections("gmail", { description: "Gmail accounts to search across" }), + "Search a connected Gmail account. Returns matches newest-first.", + integrations: { + inbox: integration("gmail"), }, input: z.object({ query: z.string().describe("Gmail search syntax, e.g. from:acme subject:invoice"), @@ -76,17 +75,12 @@ export default defineTool({ })), }), annotations: { readOnly: true }, - async handler({ query, limit }, { inboxes }) { - const perInbox = await Promise.all( - inboxes.map(async (inbox) => { - const { messages } = await inbox.messages.search({ q: query, maxResults: limit }); - return messages.map((m) => ({ - inbox: inbox.account.email, id: m.id, from: m.from, - subject: m.subject, snippet: m.snippet, date: m.date, - })); - }), - ); - const results = perInbox.flat().sort((a, b) => b.date.localeCompare(a.date)).slice(0, limit); + async handler({ query, limit }, { inbox }) { + const { messages } = await inbox.messages.search({ q: query, maxResults: limit }); + const results = messages.map((m) => ({ + inbox: inbox.account.email, id: m.id, from: m.from, + subject: m.subject, snippet: m.snippet, date: m.date, + })).sort((a, b) => b.date.localeCompare(a.date)).slice(0, limit); return { results }; }, }); diff --git a/packages/plugins/apps/src/testing/index.ts b/packages/plugins/apps/src/testing/index.ts index e7f9626b2..a77985f8d 100644 --- a/packages/plugins/apps/src/testing/index.ts +++ b/packages/plugins/apps/src/testing/index.ts @@ -2,7 +2,7 @@ import { Effect } from "effect"; import type { AppDescriptor } from "../pipeline/descriptor"; import type { AppsStore } from "../plugin/store"; -import type { ClientResolver, BindingError } from "../plugin/bindings"; +import type { BindingError, ClientResolver, ConnectionCandidate } from "../plugin/bindings"; import { ArtifactStoreError, asSnapshotId, @@ -76,12 +76,31 @@ export type { SnapshotId }; export const makeTestResolver = ( handlers: Record unknown>>, + connections?: readonly ConnectionCandidate[], ): ClientResolver & { readonly calls: { integration: string; connection: string; method: string }[]; } => { const calls: { integration: string; connection: string; method: string }[] = []; + const knownConnections = + connections ?? + Object.keys(handlers).map((integration) => ({ + address: `tools.${integration}.user.${integration}`, + integration, + name: integration, + owner: "user", + })); return { calls, + listConnections: ({ integration }) => + Effect.succeed( + knownConnections.filter((connection) => connection.integration === integration), + ), + resolveConnection: ({ connection }) => + Effect.succeed( + knownConnections.find( + (candidate) => candidate.address === connection || candidate.name === connection, + ) ?? null, + ), call: ({ integration, connection, path, args }) => { const method = path.join("."); calls.push({ integration, connection, method }); @@ -91,7 +110,7 @@ export const makeTestResolver = ( _tag: "BindingError", message: `no test handler for ${integration}.${method}`, role: integration, - surface: integration, + integration, } as unknown as BindingError); } return Effect.sync(() => handler(args)); From 275446e8255cdee7dc1dab0ceda0da4e6258cb03 Mon Sep 17 00:00:00 2001 From: Rhys Sullivan <39114868+RhysSullivan@users.noreply.github.com> Date: Mon, 6 Jul 2026 03:06:16 -0700 Subject: [PATCH 03/21] Add GitHub sync and self-host custom tools wiring --- apps/host-selfhost/executor.config.ts | 2 + apps/host-selfhost/package.json | 2 + apps/host-selfhost/src/app.ts | 4 + apps/host-selfhost/src/apps-resolver.ts | 130 +++++ apps/host-selfhost/src/apps-route.ts | 91 +++ apps/host-selfhost/src/apps-wire.node.test.ts | 549 ++++++++++++++++++ apps/host-selfhost/src/apps.ts | 71 +++ apps/host-selfhost/src/testing/test-app.ts | 4 + bun.lock | 2 + packages/core/sdk/src/executor.ts | 2 + packages/core/sdk/src/plugin.ts | 9 + packages/plugins/apps/src/api.ts | 11 + packages/plugins/apps/src/index.ts | 10 + .../plugins/apps/src/pipeline/descriptor.ts | 17 +- packages/plugins/apps/src/pipeline/publish.ts | 9 +- .../plugins/apps/src/plugin/apps-plugin.ts | 173 ++++-- packages/plugins/apps/src/plugin/runtime.ts | 6 +- .../apps/src/source/github-source.test.ts | 211 +++++++ .../plugins/apps/src/source/github-source.ts | 402 +++++++++++++ 19 files changed, 1665 insertions(+), 40 deletions(-) create mode 100644 apps/host-selfhost/src/apps-resolver.ts create mode 100644 apps/host-selfhost/src/apps-route.ts create mode 100644 apps/host-selfhost/src/apps-wire.node.test.ts create mode 100644 apps/host-selfhost/src/apps.ts create mode 100644 packages/plugins/apps/src/source/github-source.test.ts create mode 100644 packages/plugins/apps/src/source/github-source.ts diff --git a/apps/host-selfhost/executor.config.ts b/apps/host-selfhost/executor.config.ts index 1a29f4507..b8498ea17 100644 --- a/apps/host-selfhost/executor.config.ts +++ b/apps/host-selfhost/executor.config.ts @@ -8,6 +8,7 @@ import { encryptedSecretsPlugin } from "@executor-js/plugin-encrypted-secrets"; import { toolkitsPlugin } from "@executor-js/plugin-toolkits/server"; import { resolveSecretKey } from "./src/config"; +import { getSelfHostAppsSubsystem } from "./src/apps"; // --------------------------------------------------------------------------- // Single source of truth for the self-hosted app's plugin list. @@ -36,6 +37,7 @@ export default defineExecutorConfig({ mcpHttpPlugin({ dangerouslyAllowStdioMCP: false }), graphqlHttpPlugin(), toolkitsPlugin({ activeToolkitSlug }), + getSelfHostAppsSubsystem().plugin, // First writable secret provider -> the default for `secrets.set`. encryptedSecretsPlugin({ key: resolveSecretKey() }), ] as const, diff --git a/apps/host-selfhost/package.json b/apps/host-selfhost/package.json index 1c73a8cc8..bb3119599 100644 --- a/apps/host-selfhost/package.json +++ b/apps/host-selfhost/package.json @@ -26,6 +26,7 @@ "@executor-js/execution": "workspace:*", "@executor-js/fumadb": "workspace:*", "@executor-js/host-mcp": "workspace:*", + "@executor-js/plugin-apps": "workspace:*", "@executor-js/plugin-encrypted-secrets": "workspace:*", "@executor-js/plugin-google": "workspace:*", "@executor-js/plugin-graphql": "workspace:*", @@ -48,6 +49,7 @@ }, "devDependencies": { "@effect/vitest": "catalog:", + "@executor-js/emulate": "^0.13.2", "@executor-js/vite-plugin": "workspace:*", "@tailwindcss/vite": "catalog:", "@tanstack/router-plugin": "^1.167.12", diff --git a/apps/host-selfhost/src/app.ts b/apps/host-selfhost/src/app.ts index c4a080f3b..678b0d2a0 100644 --- a/apps/host-selfhost/src/app.ts +++ b/apps/host-selfhost/src/app.ts @@ -22,6 +22,8 @@ import { makeSelfHostMcpSeams } from "./mcp"; import { selfHostPlugins } from "./plugins"; import { ErrorCaptureLive } from "./observability"; import { oauthCallbackSignInRedirectLocation } from "./auth/oauth-callback-login"; +import { closeSelfHostAppsSubsystem } from "./apps"; +import { makeSelfHostAppsSyncRoute } from "./apps-route"; // =========================================================================== // The self-hosted Executor app, as ONE `ExecutorApp.make` call. @@ -119,6 +121,7 @@ export const makeSelfHostApp = async (options: MakeSelfHostAppOptions = {}) => { makeSelfHostAdminApiLayer({ betterAuth, db: dbHandle, mountPrefix: "/api" }), // Public system API: /api/health + /api/setup-status (unauthenticated). makeSelfHostSystemApiLayer({ betterAuth, db: dbHandle, mountPrefix: "/api" }), + makeSelfHostAppsSyncRoute({ identity: identityLayer, db: dbHandle }), // Swagger UI at /docs, over the /api-prefixed spec (matches the served paths). HttpApiSwagger.layer(composePluginApi(selfHostPlugins).prefix("/api"), { path: "/docs" }), ], @@ -139,6 +142,7 @@ export const makeSelfHostApp = async (options: MakeSelfHostAppOptions = {}) => { toWebHandler, betterAuth, closeDb: async () => { + await closeSelfHostAppsSubsystem(); await mcp.close(); await dbHandle.close(); }, diff --git a/apps/host-selfhost/src/apps-resolver.ts b/apps/host-selfhost/src/apps-resolver.ts new file mode 100644 index 000000000..da373b610 --- /dev/null +++ b/apps/host-selfhost/src/apps-resolver.ts @@ -0,0 +1,130 @@ +import { Effect } from "effect"; + +import { + ConnectionName, + IntegrationSlug, + ToolAddress, + isToolResult, + type ConnectionRef, + type ExecuteError, + type Owner, + type PluginCtx, +} from "@executor-js/sdk"; +import { + BindingError, + type ClientResolver, + type ConnectionCandidate, +} from "@executor-js/plugin-apps/api"; + +const parseConnectionAddress = (address: string): ConnectionRef | null => { + const parts = address.split("."); + if (parts.length !== 4 || parts[0] !== "tools") return null; + const [, integration, owner, name] = parts; + if (!integration || !name) return null; + if (owner !== "org" && owner !== "user") return null; + return { + owner: owner as Owner, + integration: IntegrationSlug.make(integration), + name: ConnectionName.make(name), + }; +}; + +const bindingError = (input: { + readonly integration: string; + readonly connection?: string; + readonly message: string; + readonly cause?: unknown; +}): BindingError => + new BindingError({ + role: input.integration, + integration: input.integration, + requestedConnection: input.connection, + message: input.message, + }); + +const executeErrorMessage = (cause: ExecuteError): string => + typeof cause.message === "string" && cause.message.length > 0 ? cause.message : cause._tag; + +const toCandidate = (connection: { + readonly address: unknown; + readonly integration: unknown; + readonly name?: unknown; + readonly owner?: unknown; +}): ConnectionCandidate => ({ + address: String(connection.address), + integration: String(connection.integration), + ...(connection.name !== undefined ? { name: String(connection.name) } : {}), + ...(connection.owner !== undefined ? { owner: String(connection.owner) } : {}), +}); + +export const makeSelfHostAppsResolver = (input: { readonly ctx: PluginCtx }): ClientResolver => ({ + listConnections: ({ integration }) => + input.ctx.connections.list({ integration: IntegrationSlug.make(integration) }).pipe( + Effect.map((connections) => connections.map(toCandidate)), + Effect.mapError((cause) => + bindingError({ + integration, + message: `failed to list ${integration} connections`, + cause, + }), + ), + ), + + resolveConnection: ({ connection }) => + Effect.gen(function* () { + const ref = parseConnectionAddress(connection); + if (!ref) return null; + const row = yield* input.ctx.connections.get(ref).pipe( + Effect.mapError((cause) => + bindingError({ + integration: String(ref.integration), + connection, + message: `failed to resolve connection ${connection}`, + cause, + }), + ), + ); + return row ? toCandidate(row) : null; + }), + + call: ({ integration, connection, path, args }) => + Effect.gen(function* () { + const ref = parseConnectionAddress(connection); + if (!ref) { + return yield* bindingError({ + integration, + connection, + message: `invalid connection address ${connection}`, + }); + } + if (String(ref.integration) !== integration) { + return yield* bindingError({ + integration, + connection, + message: `connection "${connection}" belongs to integration "${ref.integration}", not "${integration}"`, + }); + } + const tool = path.join("."); + const address = ToolAddress.make(`${connection}.${tool}`); + const payload = args[0] ?? {}; + const result = yield* input.ctx.execute(address, payload).pipe( + Effect.mapError((cause) => + bindingError({ + integration, + connection, + message: executeErrorMessage(cause), + cause, + }), + ), + ); + if (isToolResult(result)) { + if (result.ok) return result.data; + return yield* bindingError({ + integration, + connection, + message: result.error.message, + }); + } + return result; + }), +}); diff --git a/apps/host-selfhost/src/apps-route.ts b/apps/host-selfhost/src/apps-route.ts new file mode 100644 index 000000000..8dcf341a9 --- /dev/null +++ b/apps/host-selfhost/src/apps-route.ts @@ -0,0 +1,91 @@ +import { HttpEffect, HttpRouter } from "effect/unstable/http"; +import { Data, Effect, Layer } from "effect"; + +import { + IdentityProvider, + RequestOrgSlug, + RequestWebOrigin, + makeScopedExecutor, + type Principal, +} from "@executor-js/api/server"; +import { ToolAddress } from "@executor-js/sdk"; + +import { SelfHostScopedExecutorSeams } from "./execution"; +import { SelfHostDb, type SelfHostDbHandle } from "./db/self-host-db"; +import type { SelfHostPlugins } from "./plugins"; + +class AppsSyncRouteError extends Data.TaggedError("AppsSyncRouteError")<{ + readonly status: number; + readonly message: string; +}> {} + +const jsonResponse = (body: unknown, status = 200): Response => + new Response(JSON.stringify(body), { + status, + headers: { "content-type": "application/json" }, + }); + +const parseJson = (request: Request): Effect.Effect => + Effect.tryPromise({ + try: () => request.json() as Promise, + catch: () => + new AppsSyncRouteError({ + status: 400, + message: "Invalid JSON body", + }), + }); + +const buildExecutor = (principal: Principal, request: Request) => { + const withOrigin = makeScopedExecutor( + principal.accountId, + principal.organizationId, + principal.organizationName, + ).pipe(Effect.provideService(RequestWebOrigin, { origin: new URL(request.url).origin })); + return principal.organizationSlug + ? withOrigin.pipe(Effect.provideService(RequestOrgSlug, { slug: principal.organizationSlug })) + : withOrigin; +}; + +export interface SelfHostAppsSyncRouteDeps { + readonly identity: Layer.Layer; + readonly db: SelfHostDbHandle; +} + +export const makeSelfHostAppsSyncRoute = ({ identity, db }: SelfHostAppsSyncRouteDeps) => { + const services = Layer.merge( + SelfHostScopedExecutorSeams.pipe(Layer.provide(Layer.succeed(SelfHostDb)(db))), + identity, + ); + return HttpRouter.add( + "POST", + "/api/apps/sources/github/sync", + HttpEffect.fromWebHandler( + (request): Promise => + Effect.runPromise( + Effect.gen(function* () { + const identityProvider = yield* IdentityProvider; + const principal = yield* identityProvider.authenticate(request); + const payload = yield* parseJson(request); + const executor = yield* buildExecutor(principal, request); + const result = yield* executor.execute( + ToolAddress.make("executor.apps.sync_github_source"), + payload, + ); + return jsonResponse(result); + }).pipe( + Effect.catchTags({ + Unauthorized: () => Effect.succeed(new Response("Unauthorized", { status: 401 })), + NoOrganization: () => Effect.succeed(new Response("Forbidden", { status: 403 })), + Unavailable: () => Effect.succeed(new Response("Unavailable", { status: 503 })), + AppsSyncRouteError: (error) => + Effect.succeed(jsonResponse({ error: error.message }, error.status)), + }), + Effect.catchCause(() => + Effect.succeed(new Response("Internal Server Error", { status: 500 })), + ), + Effect.provide(services), + ), + ), + ), + ); +}; diff --git a/apps/host-selfhost/src/apps-wire.node.test.ts b/apps/host-selfhost/src/apps-wire.node.test.ts new file mode 100644 index 000000000..70fbf5c4f --- /dev/null +++ b/apps/host-selfhost/src/apps-wire.node.test.ts @@ -0,0 +1,549 @@ +import { randomBytes } from "node:crypto"; +import { mkdtempSync } from "node:fs"; +import { createServer, type IncomingMessage, type ServerResponse } from "node:http"; +import { createServer as createNetServer } from "node:net"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +import { afterAll, beforeAll, expect, test } from "@effect/vitest"; +import { createEmulator, type Emulator } from "@executor-js/emulate"; + +const dataDir = mkdtempSync(join(tmpdir(), "eh-apps-wire-")); +const dbPath = join(dataDir, "data.db"); +process.env.EXECUTOR_DATA_DIR = dataDir; +process.env.EXECUTOR_SECRET_KEY = "apps-wire-secret-key"; +process.env.EXECUTOR_ALLOW_LOCAL_NETWORK = "true"; + +const TEST_USER = "apps-user"; +const TEST_ORG = "apps-org"; +const OWNER = "syncer"; +const REPO = "custom-tools"; +const REPO_FULL_NAME = `${OWNER}/${REPO}`; +const GITHUB_CONNECTION = "tools.github.user.main"; + +interface TestHttpServer { + readonly baseUrl: string; + readonly close: () => Promise; +} + +interface SyncResult { + readonly status: "published" | "up-to-date" | "failed"; + readonly snapshotId?: string; + readonly upstreamSha?: string; + readonly tools: readonly string[]; + readonly skipped: readonly { readonly path: string; readonly reason: string }[]; + readonly errors?: readonly unknown[]; +} + +interface ToolRow { + readonly address: string; + readonly name: string; + readonly integration: string; +} + +interface ExecuteResponse { + readonly status: "completed" | "paused"; + readonly text: string; + readonly structured: { + readonly status?: string; + readonly result?: unknown; + readonly error?: unknown; + }; + readonly isError?: boolean; +} + +const bearerTemplate = { + slug: "bearer", + type: "apiKey", + label: "Bearer token", + headers: { Authorization: ["Bearer ", { type: "variable", name: "token" }] }, +}; + +const jsonHeaders = { + "content-type": "application/json", + "x-test-user": TEST_USER, + "x-test-org": TEST_ORG, +}; + +const findFreePort = (): Promise => + new Promise((resolve, reject) => { + const probe = createNetServer(); + probe.once("error", reject); + probe.listen(0, "127.0.0.1", () => { + const address = probe.address(); + const port = typeof address === "object" && address ? address.port : 0; + probe.close((error) => (error ? reject(error) : resolve(port))); + }); + }); + +const readRequestBody = async (request: IncomingMessage): Promise => { + if (request.method === "GET" || request.method === "HEAD") return undefined; + const chunks: Buffer[] = []; + for await (const chunk of request) { + chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk)); + } + const body = Buffer.concat(chunks); + return body.length > 0 ? body : undefined; +}; + +const webRequestFromNode = async (request: IncomingMessage, port: number): Promise => { + const headers = new Headers(); + for (const [name, raw] of Object.entries(request.headers)) { + if (Array.isArray(raw)) { + for (const value of raw) headers.append(name, value); + } else if (raw !== undefined) { + headers.set(name, raw); + } + } + const body = await readRequestBody(request); + return new Request(`http://127.0.0.1:${port}${request.url ?? "/"}`, { + method: request.method, + headers, + body: body ? new Uint8Array(body) : undefined, + }); +}; + +const sendWebResponse = async (webResponse: Response, response: ServerResponse): Promise => { + response.statusCode = webResponse.status; + webResponse.headers.forEach((value, name) => response.setHeader(name, value)); + response.end(Buffer.from(await webResponse.arrayBuffer())); +}; + +const startHttpServer = async ( + handler: (request: Request) => Promise, +): Promise => { + let port = 0; + const server = createServer((request, response) => { + void (async () => { + try { + await sendWebResponse(await handler(await webRequestFromNode(request, port)), response); + } catch (error) { + response.statusCode = 500; + response.end(error instanceof Error ? error.message : String(error)); + } + })(); + }); + await new Promise((resolve, reject) => { + server.once("error", reject); + server.listen(0, "127.0.0.1", () => { + const address = server.address(); + port = typeof address === "object" && address ? address.port : 0; + resolve(); + }); + }); + return { + baseUrl: `http://127.0.0.1:${port}`, + close: () => + new Promise((resolve, reject) => { + server.close((error) => (error ? reject(error) : resolve())); + }), + }; +}; + +const requestJson = async ( + path: string, + init: RequestInit = {}, + expectedStatus = 200, +): Promise => { + const response = await fetch(`${server.baseUrl}${path}`, init); + const text = await response.text(); + expect(response.status).toBe(expectedStatus); + return (text.length > 0 ? JSON.parse(text) : null) as T; +}; + +const postJson = (path: string, body: unknown, expectedStatus = 200): Promise => + requestJson( + path, + { + method: "POST", + headers: jsonHeaders, + body: JSON.stringify(body), + }, + expectedStatus, + ); + +const githubFetch = async ( + emulator: Emulator, + token: string, + path: string, + init: RequestInit = {}, +): Promise => { + const headers = new Headers(init.headers); + headers.set("authorization", `Bearer ${token}`); + headers.set("accept", "application/vnd.github+json"); + if (init.body !== undefined) headers.set("content-type", "application/json"); + const response = await fetch(`${emulator.url}${path}`, { ...init, headers }); + const text = await response.text(); + expect(response.ok).toBe(true); + return (text.length > 0 ? JSON.parse(text) : null) as T; +}; + +const createIssue = (emulator: Emulator, token: string, title: string): Promise => + githubFetch(emulator, token, `/repos/${OWNER}/${REPO}/issues`, { + method: "POST", + body: JSON.stringify({ title }), + }); + +const putRepoFiles = async ( + emulator: Emulator, + token: string, + files: Readonly>, +): Promise => { + const ref = await githubFetch<{ object: { sha: string } }>( + emulator, + token, + `/repos/${OWNER}/${REPO}/git/ref/heads/main`, + ); + const parentSha = ref.object.sha; + const tree = await githubFetch<{ sha: string }>( + emulator, + token, + `/repos/${OWNER}/${REPO}/git/trees`, + { + method: "POST", + body: JSON.stringify({ + tree: Object.entries(files).map(([path, content]) => ({ + path, + mode: "100644", + type: "blob", + content, + })), + }), + }, + ); + const commit = await githubFetch<{ sha: string }>( + emulator, + token, + `/repos/${OWNER}/${REPO}/git/commits`, + { + method: "POST", + body: JSON.stringify({ + message: `Update custom tools ${randomBytes(3).toString("hex")}`, + tree: tree.sha, + parents: [parentSha], + }), + }, + ); + await githubFetch(emulator, token, `/repos/${OWNER}/${REPO}/git/refs/heads/main`, { + method: "PATCH", + body: JSON.stringify({ sha: commit.sha }), + }); + return commit.sha; +}; + +const executorJson = JSON.stringify( + { + $schema: "https://executor.sh/schemas/executor.json", + description: "Curated tools wrapping our raw integration surface.", + }, + null, + 2, +); + +const dealPipelineSyncSource = `import { z } from "zod"; +import { defineTool, integration } from "executor:app"; + +/** + * The deterministic-program case: pull GitHub issues into the scope database + * so agents query a table instead of paging the raw API. + */ +export default defineTool({ + description: + "Refresh the deals table from GitHub issues. Syncs issue numbers and titles " + + "for pipeline questions.", + + integrations: { + github: integration("github"), + }, + + input: z.object({ + owner: z.string(), + repo: z.string(), + }), + + output: z.object({ synced: z.number() }), + + annotations: { readOnly: false, destructive: false }, + + async handler({ owner, repo }, { github, db }) { + await db.sql\` + CREATE TABLE IF NOT EXISTS deals ( + id TEXT PRIMARY KEY, + name TEXT NOT NULL, + stage TEXT NOT NULL + )\`; + + const issues = await github.repos.listIssues({ owner, repo }); + + let synced = 0; + for (const issue of issues) { + await db.sql\` + INSERT INTO deals (id, name, stage) + VALUES (\${String(issue.number)}, \${issue.title}, 'open') + ON CONFLICT (id) DO UPDATE SET + name = excluded.name, + stage = excluded.stage\`; + synced++; + } + + return { synced }; + }, +}); +`; + +const findDealDocsSource = `import { z } from "zod"; +import { defineTool, integration } from "executor:app"; + +/** + * The curation case: a read-only query over the scope database populated by + * the sync tool. + */ +export default defineTool({ + description: + "Find documents for a deal across synced pipeline records. Searches " + + "stored issue titles.", + + integrations: { + github: integration("github"), + }, + + input: z.object({ + limit: z.number().int().max(50).default(20), + }), + + output: z.object({ + documents: z.array( + z.object({ + name: z.string(), + stage: z.string(), + }), + ), + }), + + annotations: { readOnly: true, destructive: false }, + + async handler({ limit }, { db }) { + await db.sql\` + CREATE TABLE IF NOT EXISTS deals ( + id TEXT PRIMARY KEY, + name TEXT NOT NULL, + stage TEXT NOT NULL + )\`; + const rows = await db.sql\` + SELECT name, stage FROM deals ORDER BY id LIMIT \${limit}\`; + + return { + documents: rows.map((row) => ({ + name: String(row.name), + stage: String(row.stage), + })), + }; + }, +}); +`; + +const extraToolSource = `import { z } from "zod"; +import { defineTool } from "executor:app"; + +export default defineTool({ + description: "Return a static custom-tools health marker.", + input: z.object({}), + output: z.object({ ok: z.boolean() }), + annotations: { readOnly: true, destructive: false }, + async handler() { + return { ok: true }; + }, +}); +`; + +const initialFiles = (): Record => ({ + "executor.json": executorJson, + "tools/deal-pipeline-sync.ts": dealPipelineSyncSource, + "tools/find-deal-docs.ts": findDealDocsSource, +}); + +const registerGitHubIntegration = async (emulator: Emulator, token: string): Promise => { + await postJson("/api/openapi/specs", { + spec: { kind: "url", url: emulator.openapiUrl }, + slug: "github", + baseUrl: emulator.url, + authenticationTemplate: [bearerTemplate], + }); + const created = await postJson<{ address: string }>("/api/connections", { + owner: "user", + name: "main", + integration: "github", + template: "bearer", + value: token, + }); + expect(created.address).toBe(GITHUB_CONNECTION); +}; + +const syncSource = (repo = REPO_FULL_NAME): Promise => + postJson("/api/apps/sources/github/sync", { + repo, + connection: GITHUB_CONNECTION, + }); + +const listAppTools = (): Promise => + requestJson("/api/tools?integration=apps", { + headers: jsonHeaders, + }); + +const execute = (code: string): Promise => + postJson("/api/executions", { code, autoApprove: true }); + +const executeResult = async (code: string): Promise => { + const response = await execute(code); + expect(response.status).toBe("completed"); + expect(response.isError).toBe(false); + return response.structured.result; +}; + +const callAppToolCode = (toolName: string, args: unknown): string => ` +const found = await tools.search({ namespace: "apps", query: ${JSON.stringify(toolName)}, limit: 20 }); +const item = found.items.find((candidate) => candidate.path.endsWith(${JSON.stringify(toolName)})); +if (!item) return { ok: false, missing: ${JSON.stringify(toolName)}, found }; +let fn = tools; +for (const segment of item.path.split(".")) fn = fn[segment]; +const result = await fn(${JSON.stringify(args)}); +return { path: item.path, result }; +`; + +let github!: Emulator; +let server!: TestHttpServer; +let disposeApp: () => Promise = async () => {}; + +beforeAll(async () => { + const emulatorPort = await findFreePort(); + github = await createEmulator({ + service: "github", + port: emulatorPort, + seed: { + github: { + users: [{ login: OWNER, name: "Syncer" }], + repos: [{ owner: OWNER, name: REPO, auto_init: true }], + }, + }, + }); + const { makeSelfHostTestApp, headerIdentityLayer } = await import("./testing/test-app"); + const app = await makeSelfHostTestApp({ identity: headerIdentityLayer, dbPath }); + disposeApp = app.dispose; + server = await startHttpServer(app.handler); +}); + +afterAll(async () => { + await server?.close(); + await disposeApp(); + await github?.close(); +}); + +test("GitHub source sync publishes and invokes custom tools through self-host HTTP", async () => { + const credential = await github.credentials.mint({ + type: "api-key", + login: OWNER, + scopes: ["repo", "user"], + }); + const token = credential.token; + expect(token).toBeTruthy(); + await registerGitHubIntegration(github, token!); + await createIssue(github, token!, "Acme renewal diligence"); + await createIssue(github, token!, "Beta pipeline memo"); + await putRepoFiles(github, token!, initialFiles()); + + const unauthorized = await fetch(`${server.baseUrl}/api/apps/sources/github/sync`, { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ repo: REPO_FULL_NAME, connection: GITHUB_CONNECTION }), + }); + expect(unauthorized.status).toBe(401); + + const published = await syncSource(); + expect(published.status).toBe("published"); + expect(published.tools).toEqual(["deal-pipeline-sync", "find-deal-docs"]); + expect(published.skipped).toEqual([]); + const firstSnapshot = published.snapshotId; + expect(firstSnapshot).toBeTruthy(); + + const listed = await listAppTools(); + expect(listed.map((tool) => tool.name).sort()).toEqual(["deal-pipeline-sync", "find-deal-docs"]); + + const searchResult = await executeResult( + 'const found = await tools.search({ namespace: "apps", query: "pipeline", limit: 10 }); return found.items.map((item) => item.path);', + ); + expect(JSON.stringify(searchResult)).toContain("deal-pipeline-sync"); + + const schema = await requestJson<{ inputSchema: { properties?: Record } }>( + `/api/tools/schema?address=${encodeURIComponent(listed.find((tool) => tool.name === "deal-pipeline-sync")!.address)}`, + { headers: jsonHeaders }, + ); + expect(schema.inputSchema.properties?.github).toMatchObject({ + type: "string", + enum: [GITHUB_CONNECTION], + }); + + const invoke = (await executeResult( + callAppToolCode("deal-pipeline-sync", { + github: GITHUB_CONNECTION, + owner: OWNER, + repo: REPO, + }), + )) as { result?: { ok?: boolean; data?: { synced?: number }; error?: unknown } }; + expect(invoke.result?.ok).toBe(true); + expect(invoke.result?.data?.synced).toBe(2); + + const ledger = await github.ledger.list(); + const issueList = ledger.find( + (entry) => + entry.operationId === "issues/listForRepo" && entry.path === `/repos/${OWNER}/${REPO}/issues`, + ); + expect(issueList?.identity.user?.login).toBe(OWNER); + + const readDb = (await executeResult(callAppToolCode("find-deal-docs", { limit: 10 }))) as { + result?: { ok?: boolean; data?: { documents?: readonly { name: string }[] } }; + }; + expect(readDb.result?.ok).toBe(true); + expect(readDb.result?.data?.documents?.map((row) => row.name).sort()).toEqual([ + "Acme renewal diligence", + "Beta pipeline memo", + ]); + + const upToDate = await syncSource(); + expect(upToDate.status).toBe("up-to-date"); + expect("snapshotId" in upToDate).toBe(false); + + await putRepoFiles(github, token!, { + ...initialFiles(), + "tools/extra-tool.ts": extraToolSource, + }); + const withExtra = await syncSource(); + expect(withExtra.status).toBe("published"); + expect(withExtra.snapshotId).not.toBe(firstSnapshot); + expect((await listAppTools()).map((tool) => tool.name).sort()).toEqual([ + "deal-pipeline-sync", + "extra-tool", + "find-deal-docs", + ]); + + await putRepoFiles(github, token!, initialFiles()); + const removed = await syncSource(); + expect(removed.status).toBe("published"); + expect((await listAppTools()).map((tool) => tool.name).sort()).toEqual([ + "deal-pipeline-sync", + "find-deal-docs", + ]); + + await putRepoFiles(github, token!, { + ...initialFiles(), + "workflows/x.ts": "export default {};", + }); + const skipped = await syncSource(); + expect(skipped.status).toBe("published"); + expect(skipped.skipped).toEqual([{ path: "workflows/x.ts", reason: "not supported yet" }]); + const afterSkipped = await listAppTools(); + expect(afterSkipped.map((tool) => tool.name).sort()).toEqual([ + "deal-pipeline-sync", + "find-deal-docs", + ]); + expect(afterSkipped.some((tool) => tool.address.includes("workflow"))).toBe(false); +}); diff --git a/apps/host-selfhost/src/apps.ts b/apps/host-selfhost/src/apps.ts new file mode 100644 index 000000000..9703c16b1 --- /dev/null +++ b/apps/host-selfhost/src/apps.ts @@ -0,0 +1,71 @@ +import { join } from "node:path"; + +import { Effect } from "effect"; + +import { + appsPlugin, + BindingError, + makeSelfHostAppsRuntime, + makeSqliteAppsStore, + type AppsRuntime, + type ClientResolver, +} from "@executor-js/plugin-apps/api"; +import type { PluginCtx } from "@executor-js/sdk"; + +import { resolveDataDir } from "./config"; +import { makeSelfHostAppsResolver } from "./apps-resolver"; + +const missingResolver = (): ClientResolver => ({ + listConnections: () => Effect.succeed([]), + resolveConnection: () => Effect.succeed(null), + call: ({ integration, connection }) => + Effect.fail( + new BindingError({ + role: integration, + integration, + requestedConnection: connection, + message: "apps resolver is unavailable outside a request-scoped executor", + }), + ), +}); + +const createSubsystem = (dataDir: string) => { + const appsDir = join(dataDir, "apps"); + const store = makeSqliteAppsStore({ path: join(appsDir, "store.sqlite") }); + const host = makeSelfHostAppsRuntime({ + dataDir: appsDir, + store, + resolver: missingResolver(), + }); + const plugin = appsPlugin({ + runtime: host.runtime, + makeResolver: ({ ctx }) => makeSelfHostAppsResolver({ ctx: ctx as PluginCtx }), + }); + return { + runtime: host.runtime, + plugin, + close: host.close, + }; +}; + +export interface SelfHostAppsSubsystem { + readonly runtime: AppsRuntime; + readonly plugin: ReturnType; + readonly close: () => Promise; +} + +let current: { readonly dataDir: string; readonly subsystem: SelfHostAppsSubsystem } | undefined; + +export const getSelfHostAppsSubsystem = (): SelfHostAppsSubsystem => { + const dataDir = resolveDataDir(); + if (current && current.dataDir === dataDir) return current.subsystem; + const subsystem = createSubsystem(dataDir); + current = { dataDir, subsystem }; + return subsystem; +}; + +export const closeSelfHostAppsSubsystem = async (): Promise => { + const subsystem = current?.subsystem; + current = undefined; + await subsystem?.close(); +}; diff --git a/apps/host-selfhost/src/testing/test-app.ts b/apps/host-selfhost/src/testing/test-app.ts index 9dc707d90..047ce4c16 100644 --- a/apps/host-selfhost/src/testing/test-app.ts +++ b/apps/host-selfhost/src/testing/test-app.ts @@ -30,6 +30,8 @@ import { } from "../mcp/session-store"; import { selfHostPlugins } from "../plugins"; import { ErrorCaptureLive } from "../observability"; +import { closeSelfHostAppsSubsystem } from "../apps"; +import { makeSelfHostAppsSyncRoute } from "../apps-route"; // =========================================================================== // Self-host TEST harness — the throwaway composition tests use to exercise the @@ -202,6 +204,7 @@ export const makeSelfHostTestApp = async ( }, extensions: { routes: [ + makeSelfHostAppsSyncRoute({ identity: options.identity, db: dbHandle }), HttpApiSwagger.layer(composePluginApi(selfHostPlugins).prefix("/api"), { path: "/docs" }), ], }, @@ -217,6 +220,7 @@ export const makeSelfHostTestApp = async ( handler: web.handler, dispose: async () => { await web.dispose(); + await closeSelfHostAppsSubsystem(); await sessionStore.close(); await dbHandle.close(); }, diff --git a/bun.lock b/bun.lock index 31ae50848..bba86774b 100644 --- a/bun.lock +++ b/bun.lock @@ -224,6 +224,7 @@ "@executor-js/execution": "workspace:*", "@executor-js/fumadb": "workspace:*", "@executor-js/host-mcp": "workspace:*", + "@executor-js/plugin-apps": "workspace:*", "@executor-js/plugin-encrypted-secrets": "workspace:*", "@executor-js/plugin-google": "workspace:*", "@executor-js/plugin-graphql": "workspace:*", @@ -246,6 +247,7 @@ }, "devDependencies": { "@effect/vitest": "catalog:", + "@executor-js/emulate": "^0.13.2", "@executor-js/vite-plugin": "workspace:*", "@tailwindcss/vite": "catalog:", "@tanstack/router-plugin": "^1.167.12", diff --git a/packages/core/sdk/src/executor.ts b/packages/core/sdk/src/executor.ts index 071d4a21b..eb0d67cae 100644 --- a/packages/core/sdk/src/executor.ts +++ b/packages/core/sdk/src/executor.ts @@ -2071,6 +2071,7 @@ export const createExecutor = providersItems(key), }, oauth, + execute: (address, args) => execute(address, args), transaction: (effect: Effect.Effect) => transaction(effect), }; diff --git a/packages/core/sdk/src/plugin.ts b/packages/core/sdk/src/plugin.ts index 31dfa07eb..47f65a1b4 100644 --- a/packages/core/sdk/src/plugin.ts +++ b/packages/core/sdk/src/plugin.ts @@ -29,6 +29,7 @@ import type { ProviderKey, Subject, Tenant, + ToolAddress, } from "./ids"; import type { IntegrationDetectionResult } from "./types"; import type { @@ -38,6 +39,7 @@ import type { ElicitationResponse, } from "./elicitation"; import type { + ExecuteError, ConnectionNotFoundError, CredentialProviderNotRegisteredError, IntegrationNotFoundError, @@ -249,6 +251,12 @@ export interface PluginCtx { /** Shared OAuth service. */ readonly oauth: OAuthService; + /** Invoke another catalog tool through the same executor request context: + * policy, approval, credential resolution, and plugin dispatch all stay in + * the core path. Intended for plugin sandboxes that expose higher-level + * virtual tools over existing integration tools. */ + readonly execute: (address: ToolAddress, args: unknown) => Effect.Effect; + /** Run `effect` inside a FumaDB transaction (atomic across plugin storage + * core integration/tool writes). */ readonly transaction: (effect: Effect.Effect) => Effect.Effect; @@ -261,6 +269,7 @@ export interface PluginCtx { // --------------------------------------------------------------------------- export interface ResolveToolsInput { + readonly ctx?: PluginCtx; /** The catalog record (public projection) whose connection is being resolved. */ readonly integration: Integration; /** The plugin's stored opaque config for that integration. */ diff --git a/packages/plugins/apps/src/api.ts b/packages/plugins/apps/src/api.ts index 05901c63d..551d9a00f 100644 --- a/packages/plugins/apps/src/api.ts +++ b/packages/plugins/apps/src/api.ts @@ -13,6 +13,7 @@ export { type SelfHostAppsRuntime, type SelfHostAppsRuntimeOptions, } from "./plugin/self-host-runtime"; +export { type AppsRuntime } from "./plugin/runtime"; export { makeSqliteAppsStore } from "./backing/sqlite-apps-store"; export { BindingError, @@ -20,3 +21,13 @@ export { type ConnectionCandidate, type RoleBindings, } from "./plugin/bindings"; +export { + fetchGitHubSource, + syncGitHubSource, + GitHubSourceError, + type GitHubSourceInput, + type GitHubSourceSnapshot, + type GitHubSyncResult, + type SyncErrorData, + type SyncGitHubSourceInput, +} from "./source/github-source"; diff --git a/packages/plugins/apps/src/index.ts b/packages/plugins/apps/src/index.ts index dc829d6e1..7e6d24d6f 100644 --- a/packages/plugins/apps/src/index.ts +++ b/packages/plugins/apps/src/index.ts @@ -16,6 +16,16 @@ export { type PublishOutput, type PublishDeps, } from "./pipeline/publish"; +export { + fetchGitHubSource, + syncGitHubSource, + GitHubSourceError, + type GitHubSourceInput, + type GitHubSourceSnapshot, + type GitHubSyncResult, + type SyncErrorData, + type SyncGitHubSourceInput, +} from "./source/github-source"; export { makeAppsRuntime, type AppsRuntime, type AppsRuntimeDeps } from "./plugin/runtime"; export { diff --git a/packages/plugins/apps/src/pipeline/descriptor.ts b/packages/plugins/apps/src/pipeline/descriptor.ts index 66f44edb1..1028eb20b 100644 --- a/packages/plugins/apps/src/pipeline/descriptor.ts +++ b/packages/plugins/apps/src/pipeline/descriptor.ts @@ -7,7 +7,7 @@ /** Descriptor schema version. Bumped on any breaking shape change. A reader * refuses a descriptor from a version it does not understand. */ -export const DESCRIPTOR_VERSION = 2 as const; +export const DESCRIPTOR_VERSION = 3 as const; /** Where an entry came from: path + content hash. Lets a projection point back * at the exact source bytes without re-reading the snapshot, and makes the @@ -31,6 +31,15 @@ export interface IntegrationDecl { readonly integration: string; } +export interface GitHubSourceRef { + readonly kind: "github"; + readonly repo: string; + readonly ref: string; + readonly upstreamSha: string; +} + +export type AppSourceRef = GitHubSourceRef; + export interface ToolDescriptor { /** Path identity, e.g. `issues-sync` (from `tools/issues-sync.ts`). */ readonly name: string; @@ -62,6 +71,8 @@ export const GUIDE_ENTRIES_KEY = "skills"; export interface AppDescriptor { readonly version: typeof DESCRIPTOR_VERSION; readonly scope: string; + readonly description?: string; + readonly source?: AppSourceRef; /** The snapshot (commit hash) this descriptor was extracted from. */ readonly snapshotId: string; /** The toolchain that produced the compiled bundles. */ @@ -70,6 +81,10 @@ export interface AppDescriptor { readonly [FLOW_ENTRIES_KEY]: readonly DeferredDescriptor[]; readonly ui: readonly DeferredDescriptor[]; readonly [GUIDE_ENTRIES_KEY]: readonly DeferredDescriptor[]; + readonly skipped: readonly { + readonly path: string; + readonly reason: "not supported yet"; + }[]; /** Shared JSON-schema `$defs` reachable from tool schemas. */ readonly definitions?: Record; } diff --git a/packages/plugins/apps/src/pipeline/publish.ts b/packages/plugins/apps/src/pipeline/publish.ts index 91b779e2b..032c59c39 100644 --- a/packages/plugins/apps/src/pipeline/publish.ts +++ b/packages/plugins/apps/src/pipeline/publish.ts @@ -10,6 +10,7 @@ import { GUIDE_ENTRIES_KEY, stableStringify, type AppDescriptor, + type AppSourceRef, type IntegrationDecl, type ModuleSourceRef, type ToolDescriptor, @@ -73,6 +74,8 @@ export interface PublishInput { readonly scope: string; readonly files: FileSet; readonly commitMessage?: string; + readonly description?: string; + readonly source?: AppSourceRef; } export interface PublishOutput { @@ -219,14 +222,18 @@ const assemble = (deps: PublishDeps, files: FileSet): Effect.Effect, ): Omit => ({ version: DESCRIPTOR_VERSION, scope, + ...(input.description !== undefined ? { description: input.description } : {}), + ...(input.source !== undefined ? { source: input.source } : {}), toolchain: toolchainRef(), tools: assembled.tools, [FLOW_ENTRIES_KEY]: [], ui: [], [GUIDE_ENTRIES_KEY]: [], + skipped: assembled.skipped, }); export const publish = ( @@ -238,7 +245,7 @@ export const publish = ( if (overLimit) return yield* Effect.fail(overLimit); const assembled = yield* assemble(deps, input.files); - const body = descriptorBody(input.scope, assembled); + const body = descriptorBody(input.scope, assembled, input); const scopeStore = yield* deps.artifactStore .forScope(input.scope) diff --git a/packages/plugins/apps/src/plugin/apps-plugin.ts b/packages/plugins/apps/src/plugin/apps-plugin.ts index 076b38c1a..3888658c6 100644 --- a/packages/plugins/apps/src/plugin/apps-plugin.ts +++ b/packages/plugins/apps/src/plugin/apps-plugin.ts @@ -4,10 +4,12 @@ import { definePlugin, tool, ToolName, - ConnectionName, IntegrationSlug, AuthTemplateSlug, connectionIdentifier, + type ConnectionRef, + type Owner, + type PluginCtx, type ResolveToolsInput, type ResolveToolsResult, type InvokeToolInput, @@ -18,6 +20,7 @@ import type { AppsRuntime } from "./runtime"; import { makeAppsStore } from "./store"; import type { ClientResolver, ConnectionCandidate } from "./bindings"; import type { IntegrationDecl, ToolDescriptor } from "../pipeline/descriptor"; +import { syncGitHubSource, type GitHubSyncResult } from "../source/github-source"; export const APPS_INTEGRATION_SLUG = "apps"; export const APPS_PLUGIN_ID = "apps"; @@ -42,6 +45,40 @@ const isRecord = (value: unknown): value is Record => const unique = (values: readonly string[]): readonly string[] => [...new Set(values)]; +const syncFailure = (message: string, path?: string): GitHubSyncResult => ({ + status: "failed", + tools: [], + skipped: [], + errors: [ + { + stage: "source", + message, + ...(path ? { diagnostics: [{ path, message }] } : {}), + }, + ], +}); + +const sourceScopeFor = (input: { repo: string; connection: string }): string => + String(connectionIdentifier(`github ${input.connection} ${input.repo}`, "githubSource")); + +const parseConnectionAddress = (address: string): ConnectionRef | null => { + const parts = address.split("."); + if (parts.length !== 4 || parts[0] !== "tools") return null; + const [, integration, owner, name] = parts; + if (!integration || !name) return null; + if (owner !== "org" && owner !== "user") return null; + return { + owner: owner as Owner, + integration: IntegrationSlug.make(integration), + name: connectionIdentifier(name), + }; +}; + +const configBaseUrl = (config: unknown): string | undefined => + isRecord(config) && typeof config.baseUrl === "string" && config.baseUrl.length > 0 + ? config.baseUrl + : undefined; + const projectInputSchema = ( schema: unknown, integrations: Readonly>, @@ -94,6 +131,49 @@ export const appsPlugin = definePlugin((options?: AppsPluginOptions) => { Effect.map((mapped) => mapped ?? scopeFromConnection(connectionName)), ); + const ensureCatalogConnection = (scope: string, ctx: PluginCtx) => + Effect.gen(function* () { + const slug = IntegrationSlug.make(APPS_INTEGRATION_SLUG); + const existing = yield* ctx.core.integrations + .get(slug) + .pipe(Effect.orElseSucceed(() => null)); + if (!existing) { + yield* ctx.core.integrations.register({ + slug, + name: "Apps", + description: "User-authored, published custom tools.", + config: {}, + canRemove: false, + canRefresh: true, + }); + } + const rawName = connectionNameForScope(scope); + const connName = connectionIdentifier(rawName); + const conns = yield* ctx.connections + .list({ integration: slug }) + .pipe(Effect.orElseSucceed(() => [])); + if (!conns.some((c) => String(c.name) === String(connName))) { + yield* ctx.connections.create({ + owner: "user", + name: connName, + integration: slug, + template: AuthTemplateSlug.make("none"), + value: "", + }); + } + yield* runtime.deps.store + .putScopeForConnection(String(rawName), scope) + .pipe(Effect.orElseSucceed(() => undefined)); + yield* runtime.deps.store + .putScopeForConnection(String(connName), scope) + .pipe(Effect.orElseSucceed(() => undefined)); + return { + scope, + connection: String(connName), + ref: { owner: "user" as const, integration: slug, name: connName }, + }; + }); + return { id: APPS_PLUGIN_ID as "apps", packageName: "@executor-js/plugin-apps", @@ -135,59 +215,78 @@ export const appsPlugin = definePlugin((options?: AppsPluginOptions) => { Effect.gen(function* () { const scope = (args as { scope?: string } | undefined)?.scope ?? DEFAULT_CATALOG_SCOPE; - const slug = IntegrationSlug.make(APPS_INTEGRATION_SLUG); - const existing = yield* ctx.core.integrations - .get(slug) - .pipe(Effect.orElseSucceed(() => null)); - if (!existing) { - yield* ctx.core.integrations.register({ - slug, - name: "Apps", - description: "User-authored, published custom tools.", - config: {}, - canRemove: false, - canRefresh: true, - }); + const connected = yield* ensureCatalogConnection(scope, ctx); + return { scope, connection: connected.connection }; + }), + }), + tool({ + name: "sync_github_source", + description: "Sync a GitHub repository containing custom tool source files.", + execute: (args, { ctx }) => + Effect.gen(function* () { + const payload = isRecord(args) ? args : {}; + const repo = typeof payload.repo === "string" ? payload.repo : ""; + const connection = typeof payload.connection === "string" ? payload.connection : ""; + const ref = typeof payload.ref === "string" ? payload.ref : undefined; + const explicitScope = typeof payload.scope === "string" ? payload.scope : undefined; + if (!repo) return syncFailure('sync_github_source requires "repo"'); + if (!connection) return syncFailure('sync_github_source requires "connection"'); + + const sourceConnection = parseConnectionAddress(connection); + if (!sourceConnection) { + return syncFailure( + `GitHub connection must be a connection address like tools.github.user.main; got "${connection}"`, + connection, + ); } - const connName = ConnectionName.make(connectionNameForScope(scope)); - const conns = yield* ctx.connections - .list({ integration: slug }) - .pipe(Effect.orElseSucceed(() => [])); - if (!conns.some((c) => String(c.name) === String(connName))) { - yield* ctx.connections.create({ - owner: "user", - name: connName, - integration: slug, - template: AuthTemplateSlug.make("none"), - value: "", - }); + if (String(sourceConnection.integration) !== "github") { + return syncFailure( + `GitHub source sync requires a github connection; got "${sourceConnection.integration}"`, + connection, + ); } - yield* runtime.deps.store - .putScopeForConnection(String(connName), scope) - .pipe(Effect.orElseSucceed(() => undefined)); - const normalized = connectionIdentifier(String(connName)); - if (normalized !== String(connName)) { - yield* runtime.deps.store - .putScopeForConnection(normalized, scope) - .pipe(Effect.orElseSucceed(() => undefined)); + const existing = yield* ctx.connections + .get(sourceConnection) + .pipe(Effect.orElseSucceed(() => null)); + if (!existing) return syncFailure(`GitHub connection not found: ${connection}`); + const token = yield* ctx.connections + .resolveValue(sourceConnection) + .pipe(Effect.orElseSucceed(() => null)); + const integration = yield* ctx.core.integrations + .get(sourceConnection.integration) + .pipe(Effect.orElseSucceed(() => null)); + const scope = explicitScope ?? sourceScopeFor({ repo, connection }); + const result = yield* syncGitHubSource({ + runtime, + scope, + repo, + ref, + token, + baseUrl: configBaseUrl(integration?.config), + }); + if (result.status === "published") { + const catalog = yield* ensureCatalogConnection(scope, ctx); + yield* ctx.connections.refresh(catalog.ref).pipe(Effect.orElseSucceed(() => [])); } - return { scope, connection: String(connName) }; + return result; }), }), ], }, ], - resolveTools: ({ connection }: ResolveToolsInput) => + resolveTools: ({ ctx, connection }: ResolveToolsInput) => Effect.gen(function* () { const scope = yield* scopeFor(String(connection.name)); const descriptor = yield* runtime.getDescriptor(scope); if (!descriptor) return { tools: [] } satisfies ResolveToolsResult; + const resolver = + makeResolver && ctx ? makeResolver({ ctx, scope, tool: "*" }) : runtime.deps.resolver; const tools: ToolDef[] = []; for (const t of descriptor.tools) { const byRole: Record = {}; for (const [role, decl] of Object.entries(t.integrations)) { - byRole[role] = yield* runtime.deps.resolver + byRole[role] = yield* resolver .listConnections({ integration: decl.integration }) .pipe(Effect.orElseSucceed(() => [])); } diff --git a/packages/plugins/apps/src/plugin/runtime.ts b/packages/plugins/apps/src/plugin/runtime.ts index 26401660c..4bd89767c 100644 --- a/packages/plugins/apps/src/plugin/runtime.ts +++ b/packages/plugins/apps/src/plugin/runtime.ts @@ -13,7 +13,7 @@ import { type PublishOutput, } from "../pipeline/publish"; import { PublishError } from "../pipeline/discover"; -import type { AppDescriptor, ToolDescriptor } from "../pipeline/descriptor"; +import type { AppDescriptor, AppSourceRef, ToolDescriptor } from "../pipeline/descriptor"; import { bundleEntry } from "../pipeline/bundle"; import { buildBridge, @@ -42,6 +42,8 @@ export interface AppsRuntime { readonly scope: string; readonly files: ReadonlyMap; readonly message?: string; + readonly description?: string; + readonly source?: AppSourceRef; }) => Effect.Effect; readonly getDescriptor: (scope: string) => Effect.Effect; /** Re-derive the published-descriptor pointer from the latest committed @@ -264,6 +266,8 @@ export const makeAppsRuntime = (deps: AppsRuntimeDeps): AppsRuntime => { scope: input.scope, files: input.files, commitMessage: input.message, + description: input.description, + source: input.source, }, ); yield* putDescriptorPointer(out.descriptor); diff --git a/packages/plugins/apps/src/source/github-source.test.ts b/packages/plugins/apps/src/source/github-source.test.ts new file mode 100644 index 000000000..bbc767acd --- /dev/null +++ b/packages/plugins/apps/src/source/github-source.test.ts @@ -0,0 +1,211 @@ +import { mkdtempSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +import { describe, expect, it } from "vitest"; +import { Effect } from "effect"; + +import { makeSelfHostAppsRuntime } from "../plugin/self-host-runtime"; +import { makeInMemoryAppsStore, makeTestResolver } from "../testing"; +import { PUBLISH_LIMITS } from "../pipeline/publish"; +import { fetchGitHubSource, syncGitHubSource } from "./github-source"; + +const run = (effect: Effect.Effect): Promise => Effect.runPromise(effect); + +const toolSource = (name = "ok"): string => `import { defineTool } from "executor:app"; +import { z } from "zod"; +export default defineTool({ + description: "${name}", + input: z.object({ value: z.string().default("${name}") }), + async handler(input) { return { value: input.value }; }, +});`; + +const makeRuntime = () => + makeSelfHostAppsRuntime({ + dataDir: mkdtempSync(join(tmpdir(), "apps-gh-src-")), + store: makeInMemoryAppsStore(), + resolver: makeTestResolver({}), + inMemory: true, + }); + +const json = (value: unknown, status = 200): Response => + new Response(JSON.stringify(value), { + status, + headers: { "content-type": "application/json" }, + }); + +const makeGitHubFetch = (input: { + readonly files: ReadonlyMap; + readonly upstreamSha?: string; + readonly treeSha?: string; +}) => { + const repoPath = "/repos/acme/tools"; + const upstreamSha = input.upstreamSha ?? "commit-1"; + const treeSha = input.treeSha ?? "tree-1"; + const blobBySha = new Map(); + let index = 0; + for (const [path, content] of input.files) { + blobBySha.set(`blob-${++index}`, { path, content }); + } + let blobCalls = 0; + const fetch = (async (rawUrl: string) => { + const url = new URL(rawUrl); + if (url.pathname === repoPath) return json({ default_branch: "main" }); + if (url.pathname === `${repoPath}/git/ref/heads%2Fmain`) { + return json({ object: { sha: upstreamSha } }); + } + if (url.pathname === `${repoPath}/git/commits/${upstreamSha}`) { + return json({ sha: upstreamSha, tree: { sha: treeSha } }); + } + if (url.pathname === `${repoPath}/git/trees/${treeSha}`) { + return json({ + tree: [...blobBySha.entries()].map(([sha, blob]) => ({ + path: blob.path, + type: "blob", + sha, + size: Buffer.byteLength(blob.content, "utf8"), + })), + }); + } + const blobPrefix = `${repoPath}/git/blobs/`; + if (url.pathname.startsWith(blobPrefix)) { + blobCalls++; + const sha = decodeURIComponent(url.pathname.slice(blobPrefix.length)); + const blob = blobBySha.get(sha); + if (!blob) return json({ message: "not found" }, 404); + return json({ + encoding: "base64", + content: Buffer.from(blob.content, "utf8").toString("base64"), + }); + } + return json({ message: "not found" }, 404); + }) as typeof globalThis.fetch; + return { + fetch, + blobCalls: () => blobCalls, + }; +}; + +describe("GitHub custom-tools source", () => { + it("fetches a repo fileset and publishes provenance, description, and skipped entries", async () => { + const files = new Map([ + [ + "executor.json", + JSON.stringify({ + $schema: "https://example.test/schema", + description: "Acme tools", + ignored: true, + }), + ], + ["tools/hello.ts", toolSource("hello")], + ["workflows/deferred.ts", "export default {};"], + ["docs/readme.md", "ignored"], + ]); + const github = makeGitHubFetch({ files, upstreamSha: "commit-a" }); + const snapshot = await run(fetchGitHubSource({ repo: "acme/tools", fetch: github.fetch })); + expect([...snapshot.files.keys()].sort()).toEqual([ + "executor.json", + "tools/hello.ts", + "workflows/deferred.ts", + ]); + expect(snapshot.description).toBe("Acme tools"); + + const host = makeRuntime(); + const result = await run( + syncGitHubSource({ + runtime: host.runtime, + scope: "githubTools", + repo: "acme/tools", + fetch: github.fetch, + }), + ); + expect(result.status).toBe("published"); + expect(result.tools).toEqual(["hello"]); + expect(result.skipped).toEqual([ + { path: "workflows/deferred.ts", reason: "not supported yet" }, + ]); + const descriptor = await run(host.runtime.getDescriptor("githubTools")); + expect(descriptor?.description).toBe("Acme tools"); + expect(descriptor?.source).toEqual({ + kind: "github", + repo: "acme/tools", + ref: "main", + upstreamSha: "commit-a", + }); + await host.close(); + }); + + it("reports up-to-date when the upstream commit SHA is unchanged", async () => { + const github = makeGitHubFetch({ + files: new Map([["tools/hello.ts", toolSource("hello")]]), + upstreamSha: "same-sha", + }); + const host = makeRuntime(); + const first = await run( + syncGitHubSource({ + runtime: host.runtime, + scope: "githubTools", + repo: "acme/tools", + fetch: github.fetch, + }), + ); + const second = await run( + syncGitHubSource({ + runtime: host.runtime, + scope: "githubTools", + repo: "acme/tools", + fetch: github.fetch, + }), + ); + expect(first.status).toBe("published"); + expect(second).toEqual({ + status: "up-to-date", + upstreamSha: "same-sha", + tools: ["hello"], + skipped: [], + }); + await host.close(); + }); + + it("rejects oversized trees before fetching blobs", async () => { + const files = new Map(); + for (let i = 0; i < PUBLISH_LIMITS.maxFiles + 1; i++) { + files.set(`tools/t${i}.ts`, toolSource(`t${i}`)); + } + const github = makeGitHubFetch({ files }); + const exit = await Effect.runPromiseExit( + fetchGitHubSource({ repo: "acme/tools", fetch: github.fetch }), + ); + expect(exit._tag).toBe("Failure"); + expect(JSON.stringify(exit)).toContain("exceeding the limit"); + expect(github.blobCalls()).toBe(0); + }); + + it("returns typed failure data for publish errors", async () => { + const github = makeGitHubFetch({ + files: new Map([ + [ + "tools/bad.ts", + `import { defineTool } from "executor:app"; +import { chunk } from "lodash"; +export default defineTool({ description: "bad", input: { type: "object" }, async handler(){ return chunk([1], 1); } });`, + ], + ]), + upstreamSha: "bad-sha", + }); + const host = makeRuntime(); + const result = await run( + syncGitHubSource({ + runtime: host.runtime, + scope: "githubTools", + repo: "acme/tools", + fetch: github.fetch, + }), + ); + expect(result.status).toBe("failed"); + expect(result.upstreamSha).toBe("bad-sha"); + expect(result.errors?.[0]?.stage).toBe("bundle"); + expect(result.errors?.[0]?.message).toContain('bare import "lodash" is not allowed'); + await host.close(); + }); +}); diff --git a/packages/plugins/apps/src/source/github-source.ts b/packages/plugins/apps/src/source/github-source.ts new file mode 100644 index 000000000..475e65504 --- /dev/null +++ b/packages/plugins/apps/src/source/github-source.ts @@ -0,0 +1,402 @@ +import { Data, Effect, Result } from "effect"; + +import { PublishError, type FileDiagnostic, type SkippedArtifact } from "../pipeline/discover"; +import { PUBLISH_LIMITS, enforcePublishLimits } from "../pipeline/publish"; +import type { AppSourceRef } from "../pipeline/descriptor"; +import type { AppsRuntime } from "../plugin/runtime"; +import type { FileSet, SnapshotId } from "../seams/artifact-store"; + +export interface GitHubSourceInput { + readonly repo: string; + readonly ref?: string; + readonly token?: string | null; + readonly baseUrl?: string; + readonly fetch?: typeof globalThis.fetch; +} + +export interface GitHubSourceSnapshot { + readonly files: FileSet; + readonly repo: string; + readonly ref: string; + readonly upstreamSha: string; + readonly description?: string; +} + +export class GitHubSourceError extends Data.TaggedError("GitHubSourceError")<{ + readonly message: string; + readonly status?: number; + readonly path?: string; + readonly cause?: unknown; +}> {} + +export interface SyncErrorData { + readonly stage: "source" | "discover" | "bundle" | "collect" | "project"; + readonly message: string; + readonly diagnostics?: readonly FileDiagnostic[]; +} + +export type GitHubSyncResult = + | { + readonly status: "published"; + readonly snapshotId: SnapshotId; + readonly upstreamSha: string; + readonly tools: readonly string[]; + readonly skipped: readonly SkippedArtifact[]; + readonly errors?: undefined; + } + | { + readonly status: "up-to-date"; + readonly upstreamSha: string; + readonly tools: readonly string[]; + readonly skipped: readonly SkippedArtifact[]; + readonly errors?: undefined; + } + | { + readonly status: "failed"; + readonly upstreamSha?: string; + readonly tools: readonly string[]; + readonly skipped: readonly SkippedArtifact[]; + readonly errors: readonly SyncErrorData[]; + }; + +export interface SyncGitHubSourceInput extends GitHubSourceInput { + readonly runtime: AppsRuntime; + readonly scope: string; +} + +const MATERIALIZED_RE = /^(tools|workflows|ui|skills)\//; + +const trimBaseUrl = (baseUrl: string): string => baseUrl.replace(/\/+$/, ""); + +const repoParts = (repo: string): { owner: string; name: string } | null => { + const [owner, name, ...rest] = repo.split("/"); + if (!owner || !name || rest.length > 0) return null; + return { owner, name }; +}; + +const encodedRepoPath = (repo: string): Effect.Effect => { + const parsed = repoParts(repo); + if (!parsed) { + return Effect.fail( + new GitHubSourceError({ + message: `GitHub repo must be "owner/name"; got "${repo}"`, + path: repo, + }), + ); + } + return Effect.succeed(`${encodeURIComponent(parsed.owner)}/${encodeURIComponent(parsed.name)}`); +}; + +const materializes = (path: string): boolean => + path === "executor.json" || MATERIALIZED_RE.test(path); + +const isRecord = (value: unknown): value is Record => + value !== null && typeof value === "object" && !Array.isArray(value); + +const asString = (value: unknown): string | undefined => + typeof value === "string" && value.length > 0 ? value : undefined; + +const requestJson = ( + input: GitHubSourceInput, + path: string, +): Effect.Effect => + Effect.tryPromise({ + try: async () => { + const fetchImpl = input.fetch ?? globalThis.fetch; + const headers: Record = { + accept: "application/vnd.github+json", + "user-agent": "executor-apps-github-source", + }; + if (input.token) headers.authorization = `Bearer ${input.token}`; + const response = await fetchImpl( + `${trimBaseUrl(input.baseUrl ?? "https://api.github.com")}${path}`, + { + headers, + }, + ); + if (!response.ok) { + throw new GitHubSourceError({ + message: `GitHub request failed: GET ${path} -> ${response.status}`, + status: response.status, + path, + }); + } + return (await response.json()) as A; + }, + catch: (cause) => + cause instanceof GitHubSourceError + ? cause + : new GitHubSourceError({ + message: `GitHub request failed: GET ${path}`, + path, + cause, + }), + }); + +interface RepoResponse { + readonly default_branch?: unknown; +} + +interface CommitResponse { + readonly sha?: unknown; + readonly commit?: { + readonly tree?: { + readonly sha?: unknown; + }; + }; +} + +interface GitCommitResponse { + readonly sha?: unknown; + readonly tree?: { + readonly sha?: unknown; + }; +} + +interface RefResponse { + readonly object?: { + readonly sha?: unknown; + }; +} + +interface TreeEntry { + readonly path?: unknown; + readonly type?: unknown; + readonly sha?: unknown; + readonly size?: unknown; +} + +interface TreeResponse { + readonly tree?: readonly TreeEntry[]; + readonly truncated?: unknown; +} + +interface BlobResponse { + readonly content?: unknown; + readonly encoding?: unknown; +} + +const commitTreeSha = (commit: CommitResponse | GitCommitResponse): string | undefined => { + if ("commit" in commit) return asString(commit.commit?.tree?.sha); + if ("tree" in commit) return asString(commit.tree?.sha); + return undefined; +}; + +const limitError = (diagnostics: readonly FileDiagnostic[]): PublishError => + new PublishError({ + message: `publish payload exceeds limits (${diagnostics.length} problem(s))`, + stage: "discover", + diagnostics, + }); + +const checkTreeLimits = (entries: readonly TreeEntry[]): PublishError | null => { + const diagnostics: FileDiagnostic[] = []; + if (entries.length > PUBLISH_LIMITS.maxFiles) { + diagnostics.push({ + path: "", + message: `publish has ${entries.length} files, exceeding the limit of ${PUBLISH_LIMITS.maxFiles}`, + }); + } + let total = 0; + for (const entry of entries) { + const path = String(entry.path ?? ""); + const size = typeof entry.size === "number" ? entry.size : 0; + total += size; + if (size > PUBLISH_LIMITS.maxFileBytes) { + diagnostics.push({ + path, + message: `file is ${size} bytes, exceeding the per-file limit of ${PUBLISH_LIMITS.maxFileBytes} bytes`, + }); + } + } + if (total > PUBLISH_LIMITS.maxTotalBytes) { + diagnostics.push({ + path: "", + message: `publish total is ${total} bytes, exceeding the total limit of ${PUBLISH_LIMITS.maxTotalBytes} bytes`, + }); + } + return diagnostics.length === 0 ? null : limitError(diagnostics); +}; + +const decodeBlob = (path: string, blob: BlobResponse): Effect.Effect => + Effect.try({ + try: () => { + const encoding = asString(blob.encoding); + const content = asString(blob.content); + if (!content || encoding !== "base64") { + throw new GitHubSourceError({ + message: `GitHub blob ${path} did not return base64 content`, + path, + }); + } + return Buffer.from(content.replace(/\s/g, ""), "base64").toString("utf8"); + }, + catch: (cause) => + cause instanceof GitHubSourceError + ? cause + : new GitHubSourceError({ message: `Failed to decode GitHub blob ${path}`, path, cause }), + }); + +const executorDescription = ( + files: FileSet, +): Effect.Effect => + Effect.try({ + try: () => { + const raw = files.get("executor.json"); + if (!raw) return undefined; + const parsed = JSON.parse(raw) as unknown; + if (!isRecord(parsed)) return undefined; + const description = parsed.description; + return typeof description === "string" ? description : undefined; + }, + catch: (cause) => + new GitHubSourceError({ + message: "executor.json is not valid JSON", + path: "executor.json", + cause, + }), + }); + +export const fetchGitHubSource = ( + input: GitHubSourceInput, +): Effect.Effect => + Effect.gen(function* () { + const repoPath = yield* encodedRepoPath(input.repo); + const repo = yield* requestJson(input, `/repos/${repoPath}`); + const ref = input.ref ?? asString(repo.default_branch) ?? "main"; + const branchRef = yield* requestJson( + input, + `/repos/${repoPath}/git/ref/${encodeURIComponent(`heads/${ref}`)}`, + ).pipe(Effect.result); + const commitPath = + Result.isSuccess(branchRef) && asString(branchRef.success.object?.sha) + ? `/repos/${repoPath}/git/commits/${encodeURIComponent(String(branchRef.success.object?.sha))}` + : `/repos/${repoPath}/commits/${encodeURIComponent(ref)}`; + const commit = yield* requestJson(input, commitPath); + const upstreamSha = asString(commit.sha); + const treeSha = commitTreeSha(commit); + if (!upstreamSha || !treeSha) { + return yield* new GitHubSourceError({ + message: `GitHub commit ${ref} did not include a commit SHA and tree SHA`, + path: commitPath, + }); + } + + const tree = yield* requestJson( + input, + `/repos/${repoPath}/git/trees/${encodeURIComponent(treeSha)}?recursive=1`, + ); + if (tree.truncated === true) { + return yield* new GitHubSourceError({ + message: "GitHub returned a truncated repository tree; custom tools source is too large", + path: `/repos/${repoPath}/git/trees/${treeSha}`, + }); + } + const entries = (tree.tree ?? []).filter( + (entry) => + entry.type === "blob" && + typeof entry.path === "string" && + typeof entry.sha === "string" && + materializes(entry.path), + ); + const treeLimitError = checkTreeLimits(entries); + if (treeLimitError) return yield* Effect.fail(treeLimitError); + + const files = new Map(); + for (const entry of entries) { + const path = String(entry.path); + const blob = yield* requestJson( + input, + `/repos/${repoPath}/git/blobs/${encodeURIComponent(String(entry.sha))}`, + ); + files.set(path, yield* decodeBlob(path, blob)); + } + const payloadLimitError = enforcePublishLimits(files); + if (payloadLimitError) return yield* Effect.fail(payloadLimitError); + + return { + files, + repo: input.repo, + ref, + upstreamSha, + description: yield* executorDescription(files), + }; + }); + +const publishErrorToSyncError = (error: PublishError): SyncErrorData => ({ + stage: error.stage, + message: error.message, + diagnostics: error.diagnostics, +}); + +const sourceErrorToSyncError = (error: GitHubSourceError): SyncErrorData => ({ + stage: "source", + message: error.message, + diagnostics: error.path ? [{ path: error.path, message: error.message }] : [], +}); + +const sourceRef = (snapshot: GitHubSourceSnapshot): AppSourceRef => ({ + kind: "github", + repo: snapshot.repo, + ref: snapshot.ref, + upstreamSha: snapshot.upstreamSha, +}); + +export const syncGitHubSource = (input: SyncGitHubSourceInput): Effect.Effect => + Effect.gen(function* () { + const fetched = yield* fetchGitHubSource(input).pipe(Effect.result); + if (Result.isFailure(fetched)) { + const error = fetched.failure; + return { + status: "failed", + tools: [], + skipped: [], + errors: [ + error instanceof PublishError + ? publishErrorToSyncError(error) + : sourceErrorToSyncError(error), + ], + } satisfies GitHubSyncResult; + } + const snapshot = fetched.success; + const current = yield* input.runtime.getDescriptor(input.scope); + if ( + current?.source?.kind === "github" && + current.source.repo === snapshot.repo && + current.source.upstreamSha === snapshot.upstreamSha + ) { + return { + status: "up-to-date", + upstreamSha: snapshot.upstreamSha, + tools: current.tools.map((tool) => tool.name), + skipped: current.skipped ?? [], + } satisfies GitHubSyncResult; + } + + const published = yield* input.runtime + .publish({ + scope: input.scope, + files: snapshot.files, + description: snapshot.description, + source: sourceRef(snapshot), + message: `sync ${snapshot.repo}@${snapshot.upstreamSha}`, + }) + .pipe(Effect.result); + if (Result.isFailure(published)) { + return { + status: "failed", + upstreamSha: snapshot.upstreamSha, + tools: [], + skipped: [], + errors: [publishErrorToSyncError(published.failure)], + } satisfies GitHubSyncResult; + } + + return { + status: "published", + snapshotId: published.success.snapshotId, + upstreamSha: snapshot.upstreamSha, + tools: published.success.descriptor.tools.map((tool) => tool.name), + skipped: published.success.skipped, + } satisfies GitHubSyncResult; + }); From 18c94127e95f88bd954eafe8d4fee56d29e8921c Mon Sep 17 00:00:00 2001 From: Rhys Sullivan <39114868+RhysSullivan@users.noreply.github.com> Date: Mon, 6 Jul 2026 03:38:11 -0700 Subject: [PATCH 04/21] Add custom tools demo server --- .../scripts/custom-tools-demo.ts | 435 ++++++++++++++++++ 1 file changed, 435 insertions(+) create mode 100644 apps/host-selfhost/scripts/custom-tools-demo.ts diff --git a/apps/host-selfhost/scripts/custom-tools-demo.ts b/apps/host-selfhost/scripts/custom-tools-demo.ts new file mode 100644 index 000000000..a81146cd0 --- /dev/null +++ b/apps/host-selfhost/scripts/custom-tools-demo.ts @@ -0,0 +1,435 @@ +import { mkdtempSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +const PORT = Number(process.env.PORT ?? "8791"); +const HOST = "127.0.0.1"; +const BASE_URL = `http://${HOST}:${PORT}`; +const ORIGIN = "http://custom-tools-demo.local"; +const DEMO_REPO = "RhysSullivan/executor-custom-tools-demo"; +const ADMIN_EMAIL = "admin@custom-tools-demo.local"; +const ADMIN_PASSWORD = "admin-pass-123456"; +const USER_EMAIL = "rhys@custom-tools-demo.local"; +const USER_PASSWORD = "password-12345678"; +const GITHUB_CONNECTION_NAME = "main"; + +process.env.EXECUTOR_DATA_DIR = mkdtempSync(join(tmpdir(), "eh-custom-tools-demo-")); +process.env.BETTER_AUTH_SECRET = "custom-tools-demo-better-auth-secret-0123456789"; +process.env.EXECUTOR_SECRET_KEY = "custom-tools-demo-secret-key-0123456789"; +process.env.EXECUTOR_BOOTSTRAP_ADMIN_EMAIL = ADMIN_EMAIL; +process.env.EXECUTOR_BOOTSTRAP_ADMIN_PASSWORD = ADMIN_PASSWORD; +process.env.EXECUTOR_BOOTSTRAP_ADMIN_NAME = "Demo Admin"; +process.env.EXECUTOR_ORG_NAME = "Custom Tools Demo"; +process.env.EXECUTOR_ORG_SLUG = "custom-tools-demo"; +process.env.EXECUTOR_WEB_BASE_URL = BASE_URL; +process.env.EXECUTOR_ALLOW_LOCAL_NETWORK = "true"; + +const log = (...args: unknown[]) => console.log("[custom-tools-demo]", ...args); + +const run = async (cmd: readonly string[]): Promise => { + const proc = Bun.spawn(cmd, { stdout: "pipe", stderr: "pipe" }); + const [stdout, stderr, code] = await Promise.all([ + new Response(proc.stdout).text(), + new Response(proc.stderr).text(), + proc.exited, + ]); + if (code !== 0) { + throw new Error(`${cmd.join(" ")} failed (${code}): ${stderr.trim() || stdout.trim()}`); + } + return stdout.trim(); +}; + +const githubToken = async (): Promise => { + const fromEnv = process.env.GITHUB_TOKEN?.trim(); + if (fromEnv) return fromEnv; + try { + return await run(["gh", "auth", "token"]); + } catch (error) { + throw new Error( + "Set GITHUB_TOKEN or run `gh auth login` before starting the custom-tools demo.", + { cause: error }, + ); + } +}; + +const bearerTemplate = { + slug: "bearer", + type: "apiKey", + label: "GitHub token", + headers: { Authorization: ["Bearer ", { type: "variable", name: "token" }] }, +}; + +const pathParam = (name: string, description: string) => ({ + name, + in: "path", + required: true, + description, + schema: { type: "string" }, +}); + +const queryParam = (name: string, schema: Record, description: string) => ({ + name, + in: "query", + required: false, + description, + schema, +}); + +const githubRestSpec = JSON.stringify({ + openapi: "3.0.3", + info: { + title: "GitHub REST", + version: "demo", + description: "Minimal GitHub REST surface for the custom-tools demo.", + }, + servers: [{ url: "https://api.github.com" }], + paths: { + "/repos/{owner}/{repo}": { + get: { + operationId: "repos/get", + "x-executor-toolPath": "repos.get", + tags: ["repos"], + summary: "Get repository metadata", + parameters: [pathParam("owner", "Repository owner"), pathParam("repo", "Repository name")], + responses: { + "200": { + description: "Repository", + content: { + "application/json": { + schema: { + type: "object", + properties: { + full_name: { type: "string" }, + description: { type: "string", nullable: true }, + html_url: { type: "string" }, + stargazers_count: { type: "number" }, + forks_count: { type: "number" }, + open_issues_count: { type: "number" }, + }, + }, + }, + }, + }, + }, + }, + }, + "/repos/{owner}/{repo}/issues": { + get: { + operationId: "issues/listForRepo", + "x-executor-toolPath": "issues.listForRepo", + tags: ["issues"], + summary: "List repository issues", + parameters: [ + pathParam("owner", "Repository owner"), + pathParam("repo", "Repository name"), + queryParam("state", { type: "string", enum: ["open", "closed", "all"] }, "Issue state"), + queryParam( + "sort", + { type: "string", enum: ["created", "updated", "comments"] }, + "Sort field", + ), + queryParam("direction", { type: "string", enum: ["asc", "desc"] }, "Sort direction"), + queryParam("since", { type: "string" }, "Only issues updated after this ISO timestamp"), + queryParam("per_page", { type: "number" }, "Page size"), + queryParam("page", { type: "number" }, "Page number"), + ], + responses: { + "200": { + description: "Issues", + content: { + "application/json": { + schema: { + type: "array", + items: { + type: "object", + properties: { + number: { type: "number" }, + title: { type: "string" }, + html_url: { type: "string" }, + updated_at: { type: "string" }, + state: { type: "string" }, + pull_request: { type: "object", nullable: true }, + labels: { + type: "array", + items: { + type: "object", + properties: { name: { type: "string" } }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + "/repos/{owner}/{repo}/releases": { + get: { + operationId: "repos/listReleases", + "x-executor-toolPath": "repos.listReleases", + tags: ["repos"], + summary: "List repository releases", + parameters: [ + pathParam("owner", "Repository owner"), + pathParam("repo", "Repository name"), + queryParam("per_page", { type: "number" }, "Page size"), + queryParam("page", { type: "number" }, "Page number"), + ], + responses: { + "200": { + description: "Releases", + content: { + "application/json": { + schema: { + type: "array", + items: { + type: "object", + properties: { + tag_name: { type: "string" }, + name: { type: "string", nullable: true }, + html_url: { type: "string" }, + published_at: { type: "string", nullable: true }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, +}); + +interface JsonResponse { + readonly status: number; + readonly body: unknown; +} + +const responseJson = async (response: Response): Promise => { + const text = await response.text(); + return { + status: response.status, + body: text.length > 0 ? JSON.parse(text) : null, + }; +}; + +const mustJson = async (response: Response, label: string): Promise => { + const parsed = await responseJson(response); + if (parsed.status < 200 || parsed.status >= 300) { + throw new Error(`${label} failed (${parsed.status}): ${JSON.stringify(parsed.body)}`); + } + return parsed.body as T; +}; + +const htmlEscape = (value: string): string => + value + .replaceAll("&", "&") + .replaceAll("<", "<") + .replaceAll(">", ">") + .replaceAll('"', """); + +const toolInvokeCode = (toolName: string, args: unknown): string => ` +const found = await tools.search({ namespace: "apps", query: ${JSON.stringify(toolName)}, limit: 20 }); +const item = found.items.find((candidate) => candidate.path.endsWith(${JSON.stringify(toolName)})); +if (!item) return { ok: false, missing: ${JSON.stringify(toolName)}, found }; +let fn = tools; +for (const segment of item.path.split(".")) fn = fn[segment]; +return await fn(${JSON.stringify(args)}); +`; + +const curlJson = (path: string, body: unknown): string => + `curl -sS -X POST ${BASE_URL}${path} -H 'content-type: application/json' -d '${JSON.stringify(body)}' | jq`; + +const buildBanner = (input: { + readonly repoUrl: string; + readonly connectionAddress: string; + readonly token: string; + readonly consoleUi: string; +}): string => { + const sync = curlJson("/api/apps/sources/github/sync", { + repo: DEMO_REPO, + connection: input.connectionAddress, + }); + const list = `curl -sS '${BASE_URL}/api/tools?integration=apps' | jq`; + const repoSummary = curlJson("/api/executions", { + code: toolInvokeCode("repo-summary", { + github: input.connectionAddress, + repo: DEMO_REPO, + }), + autoApprove: true, + }); + const staleIssues = curlJson("/api/executions", { + code: toolInvokeCode("stale-issues", { + github: input.connectionAddress, + repo: DEMO_REPO, + staleDays: 30, + }), + autoApprove: true, + }); + + return `Custom tools demo ready + +GitHub repo: + ${input.repoUrl} + +Local server: + ${BASE_URL} + +Console UI: + ${input.consoleUi} + +Auth: + The local wrapper injects the self-host bearer into forwarded requests. + Bearer for direct API debugging: + ${input.token} + +GitHub connection: + ${input.connectionAddress} + +Sync the repo: + ${sync} + +List published custom tools: + ${list} + +Invoke repo-summary: + ${repoSummary} + +Invoke stale-issues: + ${staleIssues} + +MCP endpoint: + ${BASE_URL}/mcp +`; +}; + +const main = async () => { + const token = await githubToken(); + + const { makeSelfHostApiHandler } = await import("../src/app"); + const { mintInviteCode } = await import("../src/testing/mint-invite"); + const app = await makeSelfHostApiHandler(); + const handler = app.handler; + + const inviteCode = await mintInviteCode(handler, "admin"); + const signUp = await handler( + new Request(`${ORIGIN}/api/auth/sign-up/email`, { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ + email: USER_EMAIL, + password: USER_PASSWORD, + name: "Rhys", + inviteCode, + }), + }), + ); + const selfHostToken = signUp.headers.get("set-auth-token"); + if (!selfHostToken) throw new Error("sign-up produced no self-host bearer token"); + + const api = (path: string, init: RequestInit = {}): Promise => + handler( + new Request(`${ORIGIN}${path}`, { + ...init, + headers: { + authorization: `Bearer ${selfHostToken}`, + "content-type": "application/json", + ...(init.headers ?? {}), + }, + }), + ); + + const consoleCheck = await handler( + new Request(`${ORIGIN}/`, { headers: { authorization: `Bearer ${selfHostToken}` } }), + ); + const consoleUi = + consoleCheck.status === 404 + ? "not served by makeSelfHostApiHandler; use the curls below" + : `${BASE_URL}/`; + + const addSpec = await api("/api/openapi/specs", { + method: "POST", + body: JSON.stringify({ + spec: { kind: "blob", value: githubRestSpec }, + slug: "github", + name: "GitHub REST", + description: "Minimal GitHub REST surface for the custom-tools demo.", + baseUrl: "https://api.github.com", + authenticationTemplate: [bearerTemplate], + }), + }); + if (addSpec.status !== 200 && addSpec.status !== 409) { + const parsed = await responseJson(addSpec); + throw new Error( + `register github integration failed (${parsed.status}): ${JSON.stringify(parsed.body)}`, + ); + } + + const created = await mustJson<{ address: string }>( + await api("/api/connections", { + method: "POST", + body: JSON.stringify({ + owner: "user", + name: GITHUB_CONNECTION_NAME, + integration: "github", + template: "bearer", + value: token, + }), + }), + "create github connection", + ); + + const banner = buildBanner({ + repoUrl: `https://github.com/${DEMO_REPO}`, + connectionAddress: created.address, + token: selfHostToken, + consoleUi, + }); + + const server = Bun.serve({ + port: PORT, + hostname: HOST, + idleTimeout: 0, + async fetch(request) { + const url = new URL(request.url); + if (url.pathname === "/health") return new Response("ok"); + if (url.pathname === "/") { + return new Response( + `Custom tools demo
${htmlEscape(
+            banner,
+          )}
`, + { headers: { "content-type": "text/html; charset=utf-8" } }, + ); + } + const headers = new Headers(request.headers); + if (!headers.has("authorization")) headers.set("authorization", `Bearer ${selfHostToken}`); + return handler(new Request(request, { headers })); + }, + }); + + log(`serving at ${BASE_URL} (pid ${process.pid}, port ${server.port})`); + console.log(banner); + + const shutdown = async () => { + try { + server.stop(true); + } catch { + /* ignore */ + } + try { + await app.dispose(); + } catch { + /* ignore */ + } + process.exit(0); + }; + process.on("SIGINT", shutdown); + process.on("SIGTERM", shutdown); + + await new Promise(() => {}); +}; + +await main(); From d726796d3f5a1ee45b0212082c2ab95e51aab77f Mon Sep 17 00:00:00 2001 From: Rhys Sullivan <39114868+RhysSullivan@users.noreply.github.com> Date: Mon, 6 Jul 2026 04:00:59 -0700 Subject: [PATCH 05/21] Thread invocation approval through app bridges --- apps/host-selfhost/src/apps-resolver.ts | 4 +- apps/host-selfhost/src/apps-wire.node.test.ts | 118 ++++++++++++++++-- packages/core/sdk/src/executor.ts | 3 +- packages/core/sdk/src/plugin.ts | 9 +- .../plugins/apps/src/plugin/apps-plugin.ts | 4 +- packages/plugins/apps/src/plugin/bindings.ts | 12 +- packages/plugins/apps/src/plugin/runtime.ts | 6 + 7 files changed, 140 insertions(+), 16 deletions(-) diff --git a/apps/host-selfhost/src/apps-resolver.ts b/apps/host-selfhost/src/apps-resolver.ts index da373b610..2080c1219 100644 --- a/apps/host-selfhost/src/apps-resolver.ts +++ b/apps/host-selfhost/src/apps-resolver.ts @@ -87,7 +87,7 @@ export const makeSelfHostAppsResolver = (input: { readonly ctx: PluginCtx }): Cl return row ? toCandidate(row) : null; }), - call: ({ integration, connection, path, args }) => + call: ({ integration, connection, path, args, invokeOptions }) => Effect.gen(function* () { const ref = parseConnectionAddress(connection); if (!ref) { @@ -107,7 +107,7 @@ export const makeSelfHostAppsResolver = (input: { readonly ctx: PluginCtx }): Cl const tool = path.join("."); const address = ToolAddress.make(`${connection}.${tool}`); const payload = args[0] ?? {}; - const result = yield* input.ctx.execute(address, payload).pipe( + const result = yield* input.ctx.execute(address, payload, invokeOptions).pipe( Effect.mapError((cause) => bindingError({ integration, diff --git a/apps/host-selfhost/src/apps-wire.node.test.ts b/apps/host-selfhost/src/apps-wire.node.test.ts index 70fbf5c4f..514457643 100644 --- a/apps/host-selfhost/src/apps-wire.node.test.ts +++ b/apps/host-selfhost/src/apps-wire.node.test.ts @@ -41,6 +41,10 @@ interface ToolRow { readonly integration: string; } +interface ConnectionRow { + readonly address: string; +} + interface ExecuteResponse { readonly status: "completed" | "paused"; readonly text: string; @@ -178,8 +182,13 @@ const githubFetch = async ( return (text.length > 0 ? JSON.parse(text) : null) as T; }; -const createIssue = (emulator: Emulator, token: string, title: string): Promise => - githubFetch(emulator, token, `/repos/${OWNER}/${REPO}/issues`, { +const createIssue = ( + emulator: Emulator, + token: string, + title: string, + repo = REPO, +): Promise => + githubFetch(emulator, token, `/repos/${OWNER}/${repo}/issues`, { method: "POST", body: JSON.stringify({ title }), }); @@ -188,11 +197,12 @@ const putRepoFiles = async ( emulator: Emulator, token: string, files: Readonly>, + repo = REPO, ): Promise => { const ref = await githubFetch<{ object: { sha: string } }>( emulator, token, - `/repos/${OWNER}/${REPO}/git/ref/heads/main`, + `/repos/${OWNER}/${repo}/git/ref/heads/main`, ); const parentSha = ref.object.sha; const tree = await githubFetch<{ sha: string }>( @@ -224,7 +234,7 @@ const putRepoFiles = async ( }), }, ); - await githubFetch(emulator, token, `/repos/${OWNER}/${REPO}/git/refs/heads/main`, { + await githubFetch(emulator, token, `/repos/${OWNER}/${repo}/git/refs/heads/main`, { method: "PATCH", body: JSON.stringify({ sha: commit.sha }), }); @@ -356,19 +366,58 @@ export default defineTool({ }); `; +const approvalBridgeSource = `import { z } from "zod"; +import { defineTool, integration } from "executor:app"; + +export default defineTool({ + description: "List issues through a bridged GitHub client.", + integrations: { + github: integration("github"), + }, + input: z.object({ + owner: z.string(), + repo: z.string(), + }), + output: z.object({ count: z.number() }), + annotations: { readOnly: true, destructive: false }, + async handler({ owner, repo }, { github }) { + const issues = await github.repos.listIssues({ owner, repo }); + return { count: issues.length }; + }, +}); +`; + const initialFiles = (): Record => ({ "executor.json": executorJson, "tools/deal-pipeline-sync.ts": dealPipelineSyncSource, "tools/find-deal-docs.ts": findDealDocsSource, }); +const approvalFiles = (): Record => ({ + "executor.json": executorJson, + "tools/approval-bridge.ts": approvalBridgeSource, +}); + const registerGitHubIntegration = async (emulator: Emulator, token: string): Promise => { - await postJson("/api/openapi/specs", { - spec: { kind: "url", url: emulator.openapiUrl }, - slug: "github", - baseUrl: emulator.url, - authenticationTemplate: [bearerTemplate], + const specResponse = await fetch(`${server.baseUrl}/api/openapi/specs`, { + method: "POST", + headers: jsonHeaders, + body: JSON.stringify({ + spec: { kind: "url", url: emulator.openapiUrl }, + slug: "github", + baseUrl: emulator.url, + authenticationTemplate: [bearerTemplate], + }), }); + expect([200, 409]).toContain(specResponse.status); + await specResponse.text(); + const existing = await requestJson( + "/api/connections?integration=github", + { + headers: jsonHeaders, + }, + ); + if (existing.some((connection) => connection.address === GITHUB_CONNECTION)) return; const created = await postJson<{ address: string }>("/api/connections", { owner: "user", name: "main", @@ -393,6 +442,9 @@ const listAppTools = (): Promise => const execute = (code: string): Promise => postJson("/api/executions", { code, autoApprove: true }); +const executeWithApprovalPause = (code: string): Promise => + postJson("/api/executions", { code }); + const executeResult = async (code: string): Promise => { const response = await execute(code); expect(response.status).toBe("completed"); @@ -547,3 +599,51 @@ test("GitHub source sync publishes and invokes custom tools through self-host HT ]); expect(afterSkipped.some((tool) => tool.address.includes("workflow"))).toBe(false); }); + +test("bridged integration calls inherit the caller approval handler", async () => { + const credential = await github.credentials.mint({ + type: "api-key", + login: OWNER, + scopes: ["repo", "user"], + }); + const token = credential.token; + expect(token).toBeTruthy(); + await registerGitHubIntegration(github, token!); + await createIssue(github, token!, "Approval-gated issue"); + await putRepoFiles(github, token!, approvalFiles()); + const published = await syncSource(); + expect(published.status).toBe("published"); + + const githubTools = await requestJson("/api/tools?integration=github", { + headers: jsonHeaders, + }); + const listIssues = githubTools.find((tool) => tool.name === "repos.listIssues"); + expect(listIssues).toBeTruthy(); + await postJson("/api/policies", { + owner: "org", + pattern: listIssues!.address.replace(/^tools\./, ""), + action: "require_approval", + }); + + const beforeLedger = await github.ledger.list(); + const beforeCalls = beforeLedger.filter( + (entry) => + entry.operationId === "issues/listForRepo" && entry.path === `/repos/${OWNER}/${REPO}/issues`, + ).length; + + const response = await executeWithApprovalPause( + callAppToolCode("approval-bridge", { + github: GITHUB_CONNECTION, + owner: OWNER, + repo: REPO, + }), + ); + expect(response.status).toBe("paused"); + + const afterLedger = await github.ledger.list(); + const afterCalls = afterLedger.filter( + (entry) => + entry.operationId === "issues/listForRepo" && entry.path === `/repos/${OWNER}/${REPO}/issues`, + ).length; + expect(afterCalls).toBe(beforeCalls); +}); diff --git a/packages/core/sdk/src/executor.ts b/packages/core/sdk/src/executor.ts index eb0d67cae..d703eb576 100644 --- a/packages/core/sdk/src/executor.ts +++ b/packages/core/sdk/src/executor.ts @@ -3577,6 +3577,7 @@ export const createExecutor = providersItems(key), }, oauth, - execute: (address, args) => execute(address, args), + execute: (address, args, options) => execute(address, args, options), transaction: (effect: Effect.Effect) => transaction(effect), }; diff --git a/packages/core/sdk/src/plugin.ts b/packages/core/sdk/src/plugin.ts index 47f65a1b4..42e0217f0 100644 --- a/packages/core/sdk/src/plugin.ts +++ b/packages/core/sdk/src/plugin.ts @@ -37,6 +37,7 @@ import type { ElicitationHandler, ElicitationRequest, ElicitationResponse, + InvokeOptions, } from "./elicitation"; import type { ExecuteError, @@ -255,7 +256,11 @@ export interface PluginCtx { * policy, approval, credential resolution, and plugin dispatch all stay in * the core path. Intended for plugin sandboxes that expose higher-level * virtual tools over existing integration tools. */ - readonly execute: (address: ToolAddress, args: unknown) => Effect.Effect; + readonly execute: ( + address: ToolAddress, + args: unknown, + options?: InvokeOptions, + ) => Effect.Effect; /** Run `effect` inside a FumaDB transaction (atomic across plugin storage + * core integration/tool writes). */ @@ -463,6 +468,8 @@ export interface InvokeToolInput { readonly credential: ToolInvocationCredential; readonly args: unknown; readonly elicit: Elicit; + /** Original caller options for nested same-request tool calls. */ + readonly invokeOptions?: InvokeOptions; } /** Input for `validateToolArgs` — no credential/elicit: validation runs diff --git a/packages/plugins/apps/src/plugin/apps-plugin.ts b/packages/plugins/apps/src/plugin/apps-plugin.ts index 3888658c6..078c567a5 100644 --- a/packages/plugins/apps/src/plugin/apps-plugin.ts +++ b/packages/plugins/apps/src/plugin/apps-plugin.ts @@ -295,7 +295,7 @@ export const appsPlugin = definePlugin((options?: AppsPluginOptions) => { return { tools } satisfies ResolveToolsResult; }), - invokeTool: ({ ctx, toolRow, args }: InvokeToolInput) => + invokeTool: ({ ctx, toolRow, args, invokeOptions }: InvokeToolInput) => Effect.gen(function* () { const scope = yield* scopeFor(String(toolRow.connection)); const descriptor = yield* runtime.getDescriptor(scope); @@ -316,7 +316,7 @@ export const appsPlugin = definePlugin((options?: AppsPluginOptions) => { ? makeResolver({ ctx, scope, tool: toolRow.name }) : undefined; return yield* runtime - .invokeTool({ scope, tool: toolRow.name, args, resolver }) + .invokeTool({ scope, tool: toolRow.name, args, resolver, invokeOptions }) .pipe( Effect.mapError( (cause) => diff --git a/packages/plugins/apps/src/plugin/bindings.ts b/packages/plugins/apps/src/plugin/bindings.ts index 1688ea37f..030293c05 100644 --- a/packages/plugins/apps/src/plugin/bindings.ts +++ b/packages/plugins/apps/src/plugin/bindings.ts @@ -1,5 +1,6 @@ import { Effect } from "effect"; import { Data } from "effect"; +import type { InvokeOptions } from "@executor-js/sdk"; import type { HandleBridge, HandleRootSpec } from "../seams/tool-sandbox"; import { ToolSandboxError } from "../seams/tool-sandbox"; @@ -50,6 +51,7 @@ export interface ClientResolver { readonly connection: string; readonly path: readonly string[]; readonly args: readonly unknown[]; + readonly invokeOptions?: InvokeOptions; }) => Effect.Effect; } @@ -62,6 +64,8 @@ export interface BindingContext { readonly db: ScopeDbHandle; /** Routes a bound method call to the real integration. */ readonly resolver: ClientResolver; + /** Caller-supplied invoke options to preserve approval/elicitation context. */ + readonly invokeOptions?: InvokeOptions; } const isRecord = (value: unknown): value is Record => @@ -259,7 +263,13 @@ export const buildBridge = (context: BindingContext): HandleBridge => ({ } return context.resolver - .call({ integration: decl.integration, connection: connectionName, path, args }) + .call({ + integration: decl.integration, + connection: connectionName, + path, + args, + invokeOptions: context.invokeOptions, + }) .pipe( Effect.mapError( (cause) => new ToolSandboxError({ kind: "invoke", message: cause.message, cause }), diff --git a/packages/plugins/apps/src/plugin/runtime.ts b/packages/plugins/apps/src/plugin/runtime.ts index 4bd89767c..7d1598ed1 100644 --- a/packages/plugins/apps/src/plugin/runtime.ts +++ b/packages/plugins/apps/src/plugin/runtime.ts @@ -1,4 +1,5 @@ import { Effect } from "effect"; +import type { InvokeOptions } from "@executor-js/sdk"; import type { ArtifactStore } from "../seams/artifact-store"; import type { ScopeDb } from "../seams/scope-db"; @@ -56,6 +57,8 @@ export interface AppsRuntime { /** Optional per-request resolver override. The catalog invoke path supplies * one built from the request's executor context. */ readonly resolver?: ClientResolver; + /** Optional caller options forwarded to bridged sub-calls. */ + readonly invokeOptions?: InvokeOptions; }) => Effect.Effect< unknown, PublishError | BindingError | InputValidationError | OutputValidationError @@ -194,6 +197,7 @@ export const makeAppsRuntime = (deps: AppsRuntimeDeps): AppsRuntime => { toolDesc: ToolDescriptor, args: unknown, resolver?: ClientResolver, + invokeOptions?: InvokeOptions, ): Effect.Effect< unknown, PublishError | BindingError | InputValidationError | OutputValidationError @@ -222,6 +226,7 @@ export const makeAppsRuntime = (deps: AppsRuntimeDeps): AppsRuntime => { bindings: resolved.bindings, db, resolver: activeResolver, + invokeOptions, }); const result = yield* deps.sandbox .invoke( @@ -316,6 +321,7 @@ export const makeAppsRuntime = (deps: AppsRuntimeDeps): AppsRuntime => { toolDesc, input.args, input.resolver, + input.invokeOptions, ); }), }; From bb15580bd065dcb338679b86ddd5c5e82492e979 Mon Sep 17 00:00:00 2001 From: Rhys Sullivan <39114868+RhysSullivan@users.noreply.github.com> Date: Mon, 6 Jul 2026 04:21:06 -0700 Subject: [PATCH 06/21] Partition app scope storage by tenant --- .../backing/git-artifact-store.cas.test.ts | 5 +- .../apps/src/backing/git-artifact-store.ts | 34 ++--- .../apps/src/backing/libsql-scope-db.ts | 61 +++----- .../apps/src/backing/scope-collision.test.ts | 15 +- .../apps/src/backing/sqlite-apps-store.ts | 38 +++-- .../plugins/apps/src/pipeline/descriptor.ts | 3 +- .../plugins/apps/src/pipeline/publish.test.ts | 5 +- packages/plugins/apps/src/pipeline/publish.ts | 12 +- .../plugins/apps/src/plugin/apps-plugin.ts | 28 ++-- .../src/plugin/publish-concurrency.test.ts | 5 +- .../plugins/apps/src/plugin/runtime.test.ts | 3 +- packages/plugins/apps/src/plugin/runtime.ts | 123 ++++++++++----- .../apps/src/plugin/self-host-runtime.ts | 3 + packages/plugins/apps/src/plugin/store.ts | 36 +++-- .../apps/src/plugin/tenant-isolation.test.ts | 140 ++++++++++++++++++ .../src/seams/artifact-store.conformance.ts | 11 +- .../plugins/apps/src/seams/artifact-store.ts | 7 +- .../plugins/apps/src/seams/scope-address.ts | 11 ++ .../apps/src/seams/scope-db.conformance.ts | 9 +- packages/plugins/apps/src/seams/scope-db.ts | 3 +- .../plugins/apps/src/source/github-source.ts | 6 +- packages/plugins/apps/src/testing/index.ts | 29 ++-- 22 files changed, 416 insertions(+), 171 deletions(-) create mode 100644 packages/plugins/apps/src/plugin/tenant-isolation.test.ts create mode 100644 packages/plugins/apps/src/seams/scope-address.ts diff --git a/packages/plugins/apps/src/backing/git-artifact-store.cas.test.ts b/packages/plugins/apps/src/backing/git-artifact-store.cas.test.ts index a1438a8d2..06f71abfa 100644 --- a/packages/plugins/apps/src/backing/git-artifact-store.cas.test.ts +++ b/packages/plugins/apps/src/backing/git-artifact-store.cas.test.ts @@ -6,6 +6,7 @@ import { describe, expect, it } from "vitest"; import { Effect } from "effect"; import { makeGitArtifactStore } from "./git-artifact-store"; +import { scopeAddress } from "../seams/scope-address"; import type { ArtifactStoreError } from "../seams/artifact-store"; const run = (effect: Effect.Effect): Promise => Effect.runPromise(effect); @@ -23,7 +24,7 @@ describe("git ArtifactStore ref CAS (Fix 6)", () => { it("two concurrent commits from one parent: one wins, the other typed-conflicts", async () => { const root = mkdtempSync(join(tmpdir(), "apps-cas-")); const storeA = makeGitArtifactStore({ root }); - const scopeA = await run(storeA.forScope("s")); + const scopeA = await run(storeA.forScope(scopeAddress("org", "s"))); // Seed a parent so racers commit ON TOP of the same head. await run(scopeA.commit(new Map([["tools/base.ts", "// base"]]), "base")); @@ -37,7 +38,7 @@ describe("git ArtifactStore ref CAS (Fix 6)", () => { for (let i = 0; i < N; i++) { const store = makeGitArtifactStore({ root }); // eslint-disable-next-line no-await-in-loop - openedScopes.push(await run(store.forScope("s"))); + openedScopes.push(await run(store.forScope(scopeAddress("org", "s")))); } const scopeB = openedScopes[0]; diff --git a/packages/plugins/apps/src/backing/git-artifact-store.ts b/packages/plugins/apps/src/backing/git-artifact-store.ts index 6d231f029..ea2e39511 100644 --- a/packages/plugins/apps/src/backing/git-artifact-store.ts +++ b/packages/plugins/apps/src/backing/git-artifact-store.ts @@ -13,6 +13,7 @@ import { type SnapshotId, type SnapshotMeta, } from "../seams/artifact-store"; +import { scopeAddressStorageKey } from "../seams/scope-address"; // --------------------------------------------------------------------------- // Git-backed ArtifactStore (self-hosted). One bare git repo per scope under @@ -60,14 +61,6 @@ const run = ( } }); -const sanitizeScope = (scope: string): string => { - if (!/^[a-zA-Z0-9._-]+$/.test(scope)) { - // Scopes are internal identifiers; refuse anything that could escape a path. - throw new ArtifactStoreError({ message: `invalid scope key: ${scope}` }); - } - return scope; -}; - const makeScopeStore = (repoDir: string): ScopeArtifactStore => { // Environment forcing a deterministic author/committer so a snapshot's hash // depends only on its content + parent, not on wall-clock/identity drift. @@ -266,18 +259,17 @@ const makeScopeStore = (repoDir: string): ScopeArtifactStore => { }; export interface GitArtifactStoreOptions { - /** Directory holding one bare repo per scope. Created on demand. */ + /** Directory holding one bare repo per tenant/scope. Created on demand. */ readonly root: string; } -/** Build the git-backed ArtifactStore. Each scope gets a lazily-initialized - * bare repo `/.git`. */ +/** Build the git-backed ArtifactStore. Each tenant/scope pair gets a + * lazily-initialized bare repo under ``. */ export const makeGitArtifactStore = (options: GitArtifactStoreOptions): ArtifactStore => { const initialized = new Map>(); - const init = async (scope: string): Promise => { - const safe = sanitizeScope(scope); - const repoDir = join(options.root, `${safe}.git`); + const init = async (key: string): Promise => { + const repoDir = join(options.root, `${key}.git`); await mkdir(repoDir, { recursive: true }); await new Promise((resolve, reject) => { execFile("git", ["init", "--bare", "--quiet"], { cwd: repoDir }, (error) => @@ -288,18 +280,22 @@ export const makeGitArtifactStore = (options: GitArtifactStoreOptions): Artifact }; return { - forScope: (scope) => + forScope: (address) => Effect.tryPromise({ try: () => { - let existing = initialized.get(scope); + const key = scopeAddressStorageKey(address); + let existing = initialized.get(key); if (!existing) { - existing = init(scope); - initialized.set(scope, existing); + existing = init(key); + initialized.set(key, existing); } return existing; }, catch: (cause) => - new ArtifactStoreError({ message: `failed to open scope repo: ${scope}`, cause }), + new ArtifactStoreError({ + message: `failed to open scope repo: ${address.tenant}/${address.scope}`, + cause, + }), }), }; }; diff --git a/packages/plugins/apps/src/backing/libsql-scope-db.ts b/packages/plugins/apps/src/backing/libsql-scope-db.ts index 3f8a5e5b6..3fba945b1 100644 --- a/packages/plugins/apps/src/backing/libsql-scope-db.ts +++ b/packages/plugins/apps/src/backing/libsql-scope-db.ts @@ -5,13 +5,14 @@ import { createClient, type Client } from "@libsql/client"; import { Effect } from "effect"; import { ScopeDbError, type ScopeDb, type ScopeDbHandle } from "../seams/scope-db"; +import { scopeAddressStorageKey, type ScopeAddress } from "../seams/scope-address"; // --------------------------------------------------------------------------- -// libSQL-backed ScopeDb (self-hosted). One SQLite file per scope under -// `/.db`. A control table `__versions` holds a monotonic counter -// per user table; every write statement bumps the counters for the tables it -// touched. Scope isolation is a separate file per scope, there is no -// cross-scope query path. +// libSQL-backed ScopeDb (self-hosted). One SQLite file per tenant/scope under +// ``. A control table `__versions` holds a monotonic counter per user +// table; every write statement bumps the counters for the tables it touched. +// Scope isolation is a separate file per tenant/scope, there is no cross-scope +// query path. // --------------------------------------------------------------------------- const VERSION_TABLE = "__scope_versions"; @@ -46,7 +47,8 @@ const targetsOf = (sql: string): string[] => { const toArgs = (values: readonly unknown[]): unknown[] => values.map((v) => (v === undefined ? null : v)); -const makeHandle = (scope: string, client: Client): ScopeDbHandle => { +const makeHandle = (address: ScopeAddress, client: Client): ScopeDbHandle => { + const label = `${address.tenant}/${address.scope}`; const ensureVersionTable = async () => { await client.execute( `CREATE TABLE IF NOT EXISTS ${VERSION_TABLE} (name TEXT PRIMARY KEY, version INTEGER NOT NULL DEFAULT 0)`, @@ -87,7 +89,7 @@ const makeHandle = (scope: string, client: Client): ScopeDbHandle => { return result.rows as unknown as readonly Row[]; }, catch: (cause) => - new ScopeDbError({ message: `scope-db statement failed for scope ${scope}`, cause }), + new ScopeDbError({ message: `scope-db statement failed for scope ${label}`, cause }), }); return { @@ -124,58 +126,39 @@ const makeHandle = (scope: string, client: Client): ScopeDbHandle => { }; export interface LibsqlScopeDbOptions { - /** Directory holding one SQLite file per scope, or ":memory:" for tests. */ + /** Directory holding one SQLite file per tenant/scope, or ":memory:" for tests. */ readonly root: string; } const toUrl = (path: string): string => (path === ":memory:" ? path : `file:${resolve(path)}`); -// A reversible, collision-free filename key for a scope (Fix 9). A scope already -// composed only of filename-safe characters is used verbatim (so distinct -// filename-safe scopes stay distinct — "my-scope" and "my_scope" no longer -// collide). Any scope containing a character outside that set is hex-encoded in -// full under an `x-` prefix, which can never collide with a verbatim scope -// (verbatim scopes never start with `x-` followed by only hex... unless they -// literally are `x-`, so we also encode those). Two different scopes always -// produce two different keys. -const SAFE_SCOPE = /^[A-Za-z0-9._-]+$/; -const HEX_PREFIXED = /^x-[0-9a-f]*$/; -const scopeFileKey = (scope: string): string => { - if (SAFE_SCOPE.test(scope) && !HEX_PREFIXED.test(scope)) return scope; - const hex = Buffer.from(scope, "utf8").toString("hex"); - return `x-${hex}`; -}; - export const makeLibsqlScopeDb = (options: LibsqlScopeDbOptions): ScopeDb => { const clients = new Map(); - const clientFor = (scope: string): Client => { - let client = clients.get(scope); + const clientFor = (address: ScopeAddress): Client => { + const key = scopeAddressStorageKey(address); + let client = clients.get(key); if (!client) { if (options.root === ":memory:") { client = createClient({ url: ":memory:" }); } else { mkdirSync(options.root, { recursive: true }); - // Collision-free filename (Fix 9): the old `replace(/[^..]/g, "_")` - // COLLAPSED distinct scopes to the same file ("my-scope" and "my_scope" - // -> "my_scope.db"), so two scopes shared one database. Encode the raw - // scope instead: keep the common safe-identifier case readable, and for - // anything with a character outside `[A-Za-z0-9._-]` fall back to a - // reversible hex encoding of the full raw scope. Distinct scopes always - // map to distinct filenames. - const safe = scopeFileKey(scope); - client = createClient({ url: toUrl(join(options.root, `${safe}.db`)) }); + client = createClient({ url: toUrl(join(options.root, `${key}.db`)) }); } - clients.set(scope, client); + clients.set(key, client); } return client; }; return { - forScope: (scope) => + forScope: (address) => Effect.try({ - try: () => makeHandle(scope, clientFor(scope)), - catch: (cause) => new ScopeDbError({ message: `failed to open scope db ${scope}`, cause }), + try: () => makeHandle(address, clientFor(address)), + catch: (cause) => + new ScopeDbError({ + message: `failed to open scope db ${address.tenant}/${address.scope}`, + cause, + }), }), close: () => Effect.sync(() => { diff --git a/packages/plugins/apps/src/backing/scope-collision.test.ts b/packages/plugins/apps/src/backing/scope-collision.test.ts index 23db1c6c9..b3b725a8a 100644 --- a/packages/plugins/apps/src/backing/scope-collision.test.ts +++ b/packages/plugins/apps/src/backing/scope-collision.test.ts @@ -7,6 +7,7 @@ import { Effect } from "effect"; import { makeLibsqlScopeDb } from "./libsql-scope-db"; import { makeSqliteAppsStore } from "./sqlite-apps-store"; +import { scopeAddress } from "../seams/scope-address"; const run = (effect: Effect.Effect): Promise => Effect.runPromise(effect); @@ -24,11 +25,11 @@ describe("scope collision (Fix 9)", () => { const root = mkdtempSync(join(tmpdir(), "apps-scopecol-")); const db = makeLibsqlScopeDb({ root }); - const a = await run(db.forScope("my-scope")); + const a = await run(db.forScope(scopeAddress("org", "my-scope"))); await run(a.exec("CREATE TABLE t (v TEXT)")); await run(a.exec("INSERT INTO t (v) VALUES ('a')")); - const b = await run(db.forScope("my_scope")); + const b = await run(db.forScope(scopeAddress("org", "my_scope"))); await run(b.exec("CREATE TABLE t (v TEXT)")); await run(b.exec("INSERT INTO t (v) VALUES ('b')")); @@ -51,12 +52,12 @@ describe("scope collision (Fix 9)", () => { // ("appsMyScope"), but the explicit mapping keys by the ACTUAL stored name, // so each scope's connection maps to the right scope. Simulate two DISTINCT // connection names being recorded for two distinct scopes. - await run(store.putScopeForConnection("appsMyScopeDash", "my-scope")); - await run(store.putScopeForConnection("appsMyScopeUnderscore", "my_scope")); + await run(store.putScopeForConnection("org", "appsMyScopeDash", "my-scope")); + await run(store.putScopeForConnection("org", "appsMyScopeUnderscore", "my_scope")); - expect(await run(store.getScopeForConnection("appsMyScopeDash"))).toBe("my-scope"); - expect(await run(store.getScopeForConnection("appsMyScopeUnderscore"))).toBe("my_scope"); + expect(await run(store.getScopeForConnection("org", "appsMyScopeDash"))).toBe("my-scope"); + expect(await run(store.getScopeForConnection("org", "appsMyScopeUnderscore"))).toBe("my_scope"); // An unmapped name returns null (caller falls back to legacy parse). - expect(await run(store.getScopeForConnection("unknown"))).toBeNull(); + expect(await run(store.getScopeForConnection("org", "unknown"))).toBeNull(); }); }); diff --git a/packages/plugins/apps/src/backing/sqlite-apps-store.ts b/packages/plugins/apps/src/backing/sqlite-apps-store.ts index a1bf37b4b..de9480ec9 100644 --- a/packages/plugins/apps/src/backing/sqlite-apps-store.ts +++ b/packages/plugins/apps/src/backing/sqlite-apps-store.ts @@ -17,8 +17,8 @@ import type { AppsStore } from "../plugin/store"; const toUrl = (path: string): string => (path === ":memory:" ? path : `file:${resolve(path)}`); const SCHEMA = ` -CREATE TABLE IF NOT EXISTS descriptors (scope TEXT PRIMARY KEY, snapshot_id TEXT NOT NULL, descriptor TEXT NOT NULL, published_at INTEGER NOT NULL); -CREATE TABLE IF NOT EXISTS scope_connections (connection_name TEXT PRIMARY KEY, scope TEXT NOT NULL); +CREATE TABLE IF NOT EXISTS descriptors (tenant TEXT NOT NULL, scope TEXT NOT NULL, snapshot_id TEXT NOT NULL, descriptor TEXT NOT NULL, published_at INTEGER NOT NULL, PRIMARY KEY (tenant, scope)); +CREATE TABLE IF NOT EXISTS scope_connections (tenant TEXT NOT NULL, connection_name TEXT NOT NULL, scope TEXT NOT NULL, PRIMARY KEY (tenant, connection_name)); `; const storageFail = (message: string, cause: unknown): StorageFailure => @@ -45,50 +45,56 @@ export const makeSqliteAppsStore = (options: SqliteAppsStoreOptions): AppsStore }; return { - putDescriptor: (_owner, descriptor) => + putDescriptor: (tenant, _owner, descriptor) => Effect.tryPromise({ try: async () => { await init(); await client.execute({ - sql: `INSERT INTO descriptors (scope, snapshot_id, descriptor, published_at) - VALUES (?, ?, ?, ?) - ON CONFLICT(scope) DO UPDATE SET snapshot_id=excluded.snapshot_id, descriptor=excluded.descriptor, published_at=excluded.published_at`, - args: [descriptor.scope, descriptor.snapshotId, JSON.stringify(descriptor), Date.now()], + sql: `INSERT INTO descriptors (tenant, scope, snapshot_id, descriptor, published_at) + VALUES (?, ?, ?, ?, ?) + ON CONFLICT(tenant, scope) DO UPDATE SET snapshot_id=excluded.snapshot_id, descriptor=excluded.descriptor, published_at=excluded.published_at`, + args: [ + tenant, + descriptor.scope, + descriptor.snapshotId, + JSON.stringify(descriptor), + Date.now(), + ], }); }, catch: (cause) => storageFail("putDescriptor failed", cause), }), - getDescriptor: (scope) => + getDescriptor: (tenant, scope) => Effect.tryPromise({ try: async () => { await init(); const res = await client.execute({ - sql: "SELECT descriptor FROM descriptors WHERE scope = ?", - args: [scope], + sql: "SELECT descriptor FROM descriptors WHERE tenant = ? AND scope = ?", + args: [tenant, scope], }); const row = res.rows[0]; return row ? (JSON.parse(String(row.descriptor)) as AppDescriptor) : null; }, catch: (cause) => storageFail("getDescriptor failed", cause), }), - putScopeForConnection: (connectionName, scope) => + putScopeForConnection: (tenant, connectionName, scope) => Effect.tryPromise({ try: async () => { await init(); await client.execute({ - sql: "INSERT INTO scope_connections (connection_name, scope) VALUES (?, ?) ON CONFLICT(connection_name) DO UPDATE SET scope=excluded.scope", - args: [connectionName, scope], + sql: "INSERT INTO scope_connections (tenant, connection_name, scope) VALUES (?, ?, ?) ON CONFLICT(tenant, connection_name) DO UPDATE SET scope=excluded.scope", + args: [tenant, connectionName, scope], }); }, catch: (cause) => storageFail("putScopeForConnection failed", cause), }), - getScopeForConnection: (connectionName) => + getScopeForConnection: (tenant, connectionName) => Effect.tryPromise({ try: async () => { await init(); const res = await client.execute({ - sql: "SELECT scope FROM scope_connections WHERE connection_name = ?", - args: [connectionName], + sql: "SELECT scope FROM scope_connections WHERE tenant = ? AND connection_name = ?", + args: [tenant, connectionName], }); return res.rows[0] ? String(res.rows[0].scope) : null; }, diff --git a/packages/plugins/apps/src/pipeline/descriptor.ts b/packages/plugins/apps/src/pipeline/descriptor.ts index 1028eb20b..30636a7ae 100644 --- a/packages/plugins/apps/src/pipeline/descriptor.ts +++ b/packages/plugins/apps/src/pipeline/descriptor.ts @@ -7,7 +7,7 @@ /** Descriptor schema version. Bumped on any breaking shape change. A reader * refuses a descriptor from a version it does not understand. */ -export const DESCRIPTOR_VERSION = 3 as const; +export const DESCRIPTOR_VERSION = 4 as const; /** Where an entry came from: path + content hash. Lets a projection point back * at the exact source bytes without re-reading the snapshot, and makes the @@ -70,6 +70,7 @@ export const GUIDE_ENTRIES_KEY = "skills"; export interface AppDescriptor { readonly version: typeof DESCRIPTOR_VERSION; + readonly tenant: string; readonly scope: string; readonly description?: string; readonly source?: AppSourceRef; diff --git a/packages/plugins/apps/src/pipeline/publish.test.ts b/packages/plugins/apps/src/pipeline/publish.test.ts index 8f8042eb6..b924b31d6 100644 --- a/packages/plugins/apps/src/pipeline/publish.test.ts +++ b/packages/plugins/apps/src/pipeline/publish.test.ts @@ -7,6 +7,7 @@ import { Effect } from "effect"; import { makeGitArtifactStore } from "../backing/git-artifact-store"; import { makeQuickjsToolSandbox } from "../backing/quickjs-tool-sandbox"; +import { scopeAddress } from "../seams/scope-address"; import { dailyBriefFileSet } from "../testing/daily-brief"; import { FLOW_ENTRIES_KEY, GUIDE_ENTRIES_KEY } from "./descriptor"; import { publish, PUBLISH_LIMITS } from "./publish"; @@ -169,7 +170,9 @@ export default defineTool({ const exit = await Effect.runPromiseExit(publish(deps, { scope: "s", files })); expect(exit._tag).toBe("Failure"); expect(JSON.stringify(exit)).toContain("exceeding the limit"); - const latest = await run(Effect.flatMap(deps.artifactStore.forScope("s"), (s) => s.latest())); + const latest = await run( + Effect.flatMap(deps.artifactStore.forScope(scopeAddress("org", "s")), (s) => s.latest()), + ); expect(latest).toBeNull(); }); diff --git a/packages/plugins/apps/src/pipeline/publish.ts b/packages/plugins/apps/src/pipeline/publish.ts index 032c59c39..7db6829c8 100644 --- a/packages/plugins/apps/src/pipeline/publish.ts +++ b/packages/plugins/apps/src/pipeline/publish.ts @@ -1,6 +1,7 @@ import { Effect } from "effect"; import type { ArtifactStore, FileSet, SnapshotId } from "../seams/artifact-store"; +import { scopeAddress } from "../seams/scope-address"; import type { ToolSandbox } from "../seams/tool-sandbox"; import { bundleEntry, toolchainRef } from "./bundle"; import { @@ -71,6 +72,7 @@ export const enforcePublishLimits = (files: FileSet): PublishError | null => { }; export interface PublishInput { + readonly tenant?: string; readonly scope: string; readonly files: FileSet; readonly commitMessage?: string; @@ -220,11 +222,13 @@ const assemble = (deps: PublishDeps, files: FileSet): Effect.Effect, ): Omit => ({ version: DESCRIPTOR_VERSION, + tenant, scope, ...(input.description !== undefined ? { description: input.description } : {}), ...(input.source !== undefined ? { source: input.source } : {}), @@ -241,14 +245,15 @@ export const publish = ( input: PublishInput, ): Effect.Effect => Effect.gen(function* () { + const tenant = input.tenant ?? "org"; const overLimit = enforcePublishLimits(input.files); if (overLimit) return yield* Effect.fail(overLimit); const assembled = yield* assemble(deps, input.files); - const body = descriptorBody(input.scope, assembled, input); + const body = descriptorBody(tenant, input.scope, assembled, input); const scopeStore = yield* deps.artifactStore - .forScope(input.scope) + .forScope(scopeAddress(tenant, input.scope)) .pipe( Effect.mapError( (cause) => @@ -275,12 +280,13 @@ export const publish = ( export const loadDescriptorFromSnapshot = ( store: ArtifactStore, + tenant: string, scope: string, snapshotId: SnapshotId, ): Effect.Effect => Effect.gen(function* () { const scopeStore = yield* store - .forScope(scope) + .forScope(scopeAddress(tenant, scope)) .pipe( Effect.mapError( (c) => new PublishError({ message: c.message, stage: "project", diagnostics: [] }), diff --git a/packages/plugins/apps/src/plugin/apps-plugin.ts b/packages/plugins/apps/src/plugin/apps-plugin.ts index 078c567a5..135f6a37f 100644 --- a/packages/plugins/apps/src/plugin/apps-plugin.ts +++ b/packages/plugins/apps/src/plugin/apps-plugin.ts @@ -125,14 +125,20 @@ export const appsPlugin = definePlugin((options?: AppsPluginOptions) => { const runtime = options.runtime; const makeResolver = options.makeResolver; - const scopeFor = (connectionName: string): Effect.Effect => - runtime.deps.store.getScopeForConnection(connectionName).pipe( + const tenantFor = (ctx: Pick | undefined): string => { + const tenant = ctx?.owner?.tenant; + return tenant === undefined ? "org" : String(tenant); + }; + + const scopeFor = (tenant: string, connectionName: string): Effect.Effect => + runtime.deps.store.getScopeForConnection(tenant, connectionName).pipe( Effect.orElseSucceed(() => null), Effect.map((mapped) => mapped ?? scopeFromConnection(connectionName)), ); const ensureCatalogConnection = (scope: string, ctx: PluginCtx) => Effect.gen(function* () { + const tenant = tenantFor(ctx); const slug = IntegrationSlug.make(APPS_INTEGRATION_SLUG); const existing = yield* ctx.core.integrations .get(slug) @@ -162,10 +168,10 @@ export const appsPlugin = definePlugin((options?: AppsPluginOptions) => { }); } yield* runtime.deps.store - .putScopeForConnection(String(rawName), scope) + .putScopeForConnection(tenant, String(rawName), scope) .pipe(Effect.orElseSucceed(() => undefined)); yield* runtime.deps.store - .putScopeForConnection(String(connName), scope) + .putScopeForConnection(tenant, String(connName), scope) .pipe(Effect.orElseSucceed(() => undefined)); return { scope, @@ -255,9 +261,11 @@ export const appsPlugin = definePlugin((options?: AppsPluginOptions) => { const integration = yield* ctx.core.integrations .get(sourceConnection.integration) .pipe(Effect.orElseSucceed(() => null)); + const tenant = tenantFor(ctx); const scope = explicitScope ?? sourceScopeFor({ repo, connection }); const result = yield* syncGitHubSource({ runtime, + tenant, scope, repo, ref, @@ -277,8 +285,9 @@ export const appsPlugin = definePlugin((options?: AppsPluginOptions) => { resolveTools: ({ ctx, connection }: ResolveToolsInput) => Effect.gen(function* () { - const scope = yield* scopeFor(String(connection.name)); - const descriptor = yield* runtime.getDescriptor(scope); + const tenant = ctx ? tenantFor(ctx) : "org"; + const scope = yield* scopeFor(tenant, String(connection.name)); + const descriptor = yield* runtime.getDescriptor(tenant, scope); if (!descriptor) return { tools: [] } satisfies ResolveToolsResult; const resolver = makeResolver && ctx ? makeResolver({ ctx, scope, tool: "*" }) : runtime.deps.resolver; @@ -297,8 +306,9 @@ export const appsPlugin = definePlugin((options?: AppsPluginOptions) => { invokeTool: ({ ctx, toolRow, args, invokeOptions }: InvokeToolInput) => Effect.gen(function* () { - const scope = yield* scopeFor(String(toolRow.connection)); - const descriptor = yield* runtime.getDescriptor(scope); + const tenant = tenantFor(ctx); + const scope = yield* scopeFor(tenant, String(toolRow.connection)); + const descriptor = yield* runtime.getDescriptor(tenant, scope); if (!descriptor) { return yield* Effect.fail( new Error( @@ -316,7 +326,7 @@ export const appsPlugin = definePlugin((options?: AppsPluginOptions) => { ? makeResolver({ ctx, scope, tool: toolRow.name }) : undefined; return yield* runtime - .invokeTool({ scope, tool: toolRow.name, args, resolver, invokeOptions }) + .invokeTool({ tenant, scope, tool: toolRow.name, args, resolver, invokeOptions }) .pipe( Effect.mapError( (cause) => diff --git a/packages/plugins/apps/src/plugin/publish-concurrency.test.ts b/packages/plugins/apps/src/plugin/publish-concurrency.test.ts index e267ff5a4..ef5aae195 100644 --- a/packages/plugins/apps/src/plugin/publish-concurrency.test.ts +++ b/packages/plugins/apps/src/plugin/publish-concurrency.test.ts @@ -7,6 +7,7 @@ import { Effect } from "effect"; import { makeSelfHostAppsRuntime } from "./self-host-runtime"; import { makeInMemoryAppsStore, makeTestResolver } from "../testing"; +import { scopeAddress } from "../seams/scope-address"; const run = (effect: Effect.Effect): Promise => Effect.runPromise(effect); @@ -70,7 +71,9 @@ describe("concurrent publishes to one scope (Fix 6)", () => { // HEAD and the descriptor pointer AGREE: the store's current descriptor's // snapshotId equals the artifact store's latest committed snapshot. const latest = await run( - runtime.deps.artifactStore.forScope("s").pipe(Effect.flatMap((s) => s.latest())), + runtime.deps.artifactStore + .forScope(scopeAddress("org", "s")) + .pipe(Effect.flatMap((s) => s.latest())), ); expect(latest).not.toBeNull(); const pointer = await run(runtime.getDescriptor("s")); diff --git a/packages/plugins/apps/src/plugin/runtime.test.ts b/packages/plugins/apps/src/plugin/runtime.test.ts index 13db88451..9120c0a3c 100644 --- a/packages/plugins/apps/src/plugin/runtime.test.ts +++ b/packages/plugins/apps/src/plugin/runtime.test.ts @@ -7,6 +7,7 @@ import { Effect } from "effect"; import { makeSelfHostAppsRuntime } from "./self-host-runtime"; import { makeInMemoryAppsStore, makeTestResolver, dailyBriefFileSet } from "../testing"; +import { scopeAddress } from "../seams/scope-address"; const run = (effect: Effect.Effect): Promise => Effect.runPromise(effect); @@ -63,7 +64,7 @@ describe("AppsRuntime end-to-end (publish -> invoke)", () => { )) as { synced: number; repos: number }; expect(syncResult).toEqual({ synced: 2, repos: 1 }); - const db = await run(host.scopeDb.forScope("rhys")); + const db = await run(host.scopeDb.forScope(scopeAddress("org", "rhys"))); const rows = await run(db.exec<{ n: number }>("SELECT COUNT(*) AS n FROM issues")); expect(Number(rows[0].n)).toBe(2); diff --git a/packages/plugins/apps/src/plugin/runtime.ts b/packages/plugins/apps/src/plugin/runtime.ts index 7d1598ed1..2f3fe3884 100644 --- a/packages/plugins/apps/src/plugin/runtime.ts +++ b/packages/plugins/apps/src/plugin/runtime.ts @@ -24,6 +24,7 @@ import { BindingError, } from "./bindings"; import type { AppsStore } from "./store"; +import { scopeAddress } from "../seams/scope-address"; // --------------------------------------------------------------------------- // AppsRuntime: the substrate-neutral core for published custom tools. @@ -36,21 +37,30 @@ export interface AppsRuntimeDeps { readonly store: AppsStore; /** Routes a bound integration method call to the real API (policy/audit). */ readonly resolver: ClientResolver; + readonly defaultTenant?: string; } export interface AppsRuntime { readonly publish: (input: { + readonly tenant?: string; readonly scope: string; readonly files: ReadonlyMap; readonly message?: string; readonly description?: string; readonly source?: AppSourceRef; }) => Effect.Effect; - readonly getDescriptor: (scope: string) => Effect.Effect; + readonly getDescriptor: ( + tenantOrScope: string, + scope?: string, + ) => Effect.Effect; /** Re-derive the published-descriptor pointer from the latest committed * snapshot. */ - readonly repair: (scope: string) => Effect.Effect; + readonly repair: ( + tenantOrScope: string, + scope?: string, + ) => Effect.Effect; readonly invokeTool: (input: { + readonly tenant?: string; readonly scope: string; readonly tool: string; readonly args: unknown; @@ -74,14 +84,28 @@ const failNoDescriptor = (scope: string): PublishError => }); export const makeAppsRuntime = (deps: AppsRuntimeDeps): AppsRuntime => { + const defaultTenant = deps.defaultTenant ?? "org"; const bundleCache = new Map(); const publishChains = new Map>(); + const resolveTenant = (tenant?: string): string => tenant ?? defaultTenant; + const resolveTenantScope = ( + tenantOrScope: string, + maybeScope?: string, + ): { tenant: string; scope: string } => + maybeScope === undefined + ? { tenant: defaultTenant, scope: tenantOrScope } + : { tenant: tenantOrScope, scope: maybeScope }; - const withScopePublishLock = (scope: string, run: () => Promise): Promise => { - const prior = publishChains.get(scope) ?? Promise.resolve(); + const withScopePublishLock = ( + tenant: string, + scope: string, + run: () => Promise, + ): Promise => { + const key = `${tenant}:${scope}`; + const prior = publishChains.get(key) ?? Promise.resolve(); const next = prior.catch(() => undefined).then(run); publishChains.set( - scope, + key, next.catch(() => undefined), ); return next; @@ -95,16 +119,18 @@ export const makeAppsRuntime = (deps: AppsRuntimeDeps): AppsRuntime => { const cacheKey = `${descriptor.snapshotId}:${sourcePath}`; const cached = bundleCache.get(cacheKey); if (cached) return cached; - const scopeStore = yield* deps.artifactStore.forScope(descriptor.scope).pipe( - Effect.mapError( - (c) => - new PublishError({ - message: c.message, - stage: "project", - diagnostics: [], - }), - ), - ); + const scopeStore = yield* deps.artifactStore + .forScope(scopeAddress(descriptor.tenant, descriptor.scope)) + .pipe( + Effect.mapError( + (c) => + new PublishError({ + message: c.message, + stage: "project", + diagnostics: [], + }), + ), + ); const files = yield* scopeStore.read(descriptor.snapshotId as never).pipe( Effect.mapError( (c) => @@ -129,9 +155,12 @@ export const makeAppsRuntime = (deps: AppsRuntimeDeps): AppsRuntime => { return bundle.code; }); - const recoverFromSnapshot = (scope: string): Effect.Effect => + const recoverFromSnapshot = ( + tenant: string, + scope: string, + ): Effect.Effect => Effect.gen(function* () { - const scopeStore = yield* deps.artifactStore.forScope(scope).pipe( + const scopeStore = yield* deps.artifactStore.forScope(scopeAddress(tenant, scope)).pipe( Effect.mapError( (c) => new PublishError({ @@ -152,11 +181,14 @@ export const makeAppsRuntime = (deps: AppsRuntimeDeps): AppsRuntime => { ), ); if (!latest) return null; - return yield* loadDescriptorFromSnapshot(deps.artifactStore, scope, latest.id); + return yield* loadDescriptorFromSnapshot(deps.artifactStore, tenant, scope, latest.id); }); - const requireDescriptor = (scope: string): Effect.Effect => - deps.store.getDescriptor(scope).pipe( + const requireDescriptor = ( + tenant: string, + scope: string, + ): Effect.Effect => + deps.store.getDescriptor(tenant, scope).pipe( Effect.mapError( (c) => new PublishError({ @@ -166,13 +198,18 @@ export const makeAppsRuntime = (deps: AppsRuntimeDeps): AppsRuntime => { }), ), Effect.flatMap((d) => - d ? Effect.succeed(d) : recoverFromSnapshot(scope).pipe(Effect.orElseSucceed(() => null)), + d + ? Effect.succeed(d) + : recoverFromSnapshot(tenant, scope).pipe(Effect.orElseSucceed(() => null)), ), Effect.flatMap((d) => (d ? Effect.succeed(d) : Effect.fail(failNoDescriptor(scope)))), ); - const putDescriptorPointer = (descriptor: AppDescriptor): Effect.Effect => - deps.store.putDescriptor("org", descriptor).pipe( + const putDescriptorPointer = ( + tenant: string, + descriptor: AppDescriptor, + ): Effect.Effect => + deps.store.putDescriptor(tenant, "org", descriptor).pipe( Effect.mapError( (c) => new PublishError({ @@ -183,11 +220,14 @@ export const makeAppsRuntime = (deps: AppsRuntimeDeps): AppsRuntime => { ), ); - const repairScope = (scope: string): Effect.Effect => + const repairScope = ( + tenant: string, + scope: string, + ): Effect.Effect => Effect.gen(function* () { - const descriptor = yield* recoverFromSnapshot(scope); + const descriptor = yield* recoverFromSnapshot(tenant, scope); if (!descriptor) return null; - yield* putDescriptorPointer(descriptor); + yield* putDescriptorPointer(tenant, descriptor); return descriptor; }); @@ -211,7 +251,7 @@ export const makeAppsRuntime = (deps: AppsRuntimeDeps): AppsRuntime => { ); const roots = rootsFor(toolDesc.integrations); const code = yield* bundleFor(descriptor, toolDesc.sourcePath); - const db = yield* deps.scopeDb.forScope(scope).pipe( + const db = yield* deps.scopeDb.forScope(scopeAddress(descriptor.tenant, scope)).pipe( Effect.mapError( (c) => new PublishError({ @@ -258,8 +298,9 @@ export const makeAppsRuntime = (deps: AppsRuntimeDeps): AppsRuntime => { publish: (input) => Effect.flatMap( Effect.tryPromise({ - try: () => - withScopePublishLock(input.scope, () => + try: () => { + const tenant = resolveTenant(input.tenant); + return withScopePublishLock(tenant, input.scope, () => Effect.runPromiseExit( Effect.gen(function* () { const out = yield* runPublish( @@ -268,6 +309,7 @@ export const makeAppsRuntime = (deps: AppsRuntimeDeps): AppsRuntime => { sandbox: deps.sandbox, }, { + tenant, scope: input.scope, files: input.files, commitMessage: input.message, @@ -275,11 +317,12 @@ export const makeAppsRuntime = (deps: AppsRuntimeDeps): AppsRuntime => { source: input.source, }, ); - yield* putDescriptorPointer(out.descriptor); + yield* putDescriptorPointer(tenant, out.descriptor); return out; }), ), - ), + ); + }, catch: (cause) => new PublishError({ message: cause instanceof Error ? cause.message : String(cause), @@ -290,21 +333,27 @@ export const makeAppsRuntime = (deps: AppsRuntimeDeps): AppsRuntime => { (exit) => exit, ), - getDescriptor: (scope) => - deps.store.getDescriptor(scope).pipe( + getDescriptor: (tenantOrScope, maybeScope) => { + const { tenant, scope } = resolveTenantScope(tenantOrScope, maybeScope); + return deps.store.getDescriptor(tenant, scope).pipe( Effect.orElseSucceed(() => null), Effect.flatMap((pointer) => pointer ? Effect.succeed(pointer) - : recoverFromSnapshot(scope).pipe(Effect.orElseSucceed(() => null)), + : recoverFromSnapshot(tenant, scope).pipe(Effect.orElseSucceed(() => null)), ), - ), + ); + }, - repair: (scope) => repairScope(scope), + repair: (tenantOrScope, maybeScope) => { + const { tenant, scope } = resolveTenantScope(tenantOrScope, maybeScope); + return repairScope(tenant, scope); + }, invokeTool: (input) => Effect.gen(function* () { - const descriptor = yield* requireDescriptor(input.scope); + const tenant = resolveTenant(input.tenant); + const descriptor = yield* requireDescriptor(tenant, input.scope); const toolDesc = descriptor.tools.find((t) => t.name === input.tool); if (!toolDesc) { return yield* Effect.fail( diff --git a/packages/plugins/apps/src/plugin/self-host-runtime.ts b/packages/plugins/apps/src/plugin/self-host-runtime.ts index 78735d3a3..460a25158 100644 --- a/packages/plugins/apps/src/plugin/self-host-runtime.ts +++ b/packages/plugins/apps/src/plugin/self-host-runtime.ts @@ -13,6 +13,8 @@ import type { ClientResolver } from "./bindings"; export interface SelfHostAppsRuntimeOptions { /** Data dir root; `/artifacts` and `/scope-db`. */ readonly dataDir: string; + /** Default tenant for direct runtime calls; request paths pass tenant explicitly. */ + readonly tenant?: string; readonly store: AppsStore; /** Routes bound integration calls to real APIs (policy/audit). */ readonly resolver: ClientResolver; @@ -43,6 +45,7 @@ export const makeSelfHostAppsRuntime = ( sandbox, store: options.store, resolver: options.resolver, + defaultTenant: options.tenant, }); return { diff --git a/packages/plugins/apps/src/plugin/store.ts b/packages/plugins/apps/src/plugin/store.ts index d25780b51..f2abb39b6 100644 --- a/packages/plugins/apps/src/plugin/store.ts +++ b/packages/plugins/apps/src/plugin/store.ts @@ -14,6 +14,7 @@ import type { AppDescriptor } from "../pipeline/descriptor"; export const descriptorCollection = definePluginStorageCollection("published_descriptor", { Type: {} as { + readonly tenant: string; readonly scope: string; readonly snapshotId: string; readonly descriptor: AppDescriptor; @@ -23,6 +24,7 @@ export const descriptorCollection = definePluginStorageCollection("published_des export const scopeConnectionCollection = definePluginStorageCollection("apps_scope_connection", { Type: {} as { + readonly tenant: string; readonly connectionName: string; readonly scope: string; }, @@ -30,15 +32,21 @@ export const scopeConnectionCollection = definePluginStorageCollection("apps_sco export interface AppsStore { readonly putDescriptor: ( + tenant: string, owner: "org" | "user", descriptor: AppDescriptor, ) => Effect.Effect; - readonly getDescriptor: (scope: string) => Effect.Effect; + readonly getDescriptor: ( + tenant: string, + scope: string, + ) => Effect.Effect; readonly putScopeForConnection: ( + tenant: string, connectionName: string, scope: string, ) => Effect.Effect; readonly getScopeForConnection: ( + tenant: string, connectionName: string, ) => Effect.Effect; } @@ -50,21 +58,29 @@ export interface AppsStoreDeps { export const makeAppsStore = (deps: AppsStoreDeps): AppsStore => { const descriptors = deps.pluginStorage.collection(descriptorCollection); const scopeConnections = deps.pluginStorage.collection(scopeConnectionCollection); + // Tenant is now part of apps storage keys. The apps subsystem had not shipped + // before this key shape, so no migration from the old scope-only keys is needed. + const keyFor = (tenant: string, key: string): string => `${tenant}:${key}`; return { - putScopeForConnection: (connectionName, scope) => + putScopeForConnection: (tenant, connectionName, scope) => scopeConnections - .put({ owner: "org", key: connectionName, data: { connectionName, scope } }) + .put({ + owner: "org", + key: keyFor(tenant, connectionName), + data: { tenant, connectionName, scope }, + }) .pipe(Effect.asVoid), - getScopeForConnection: (connectionName) => + getScopeForConnection: (tenant, connectionName) => scopeConnections - .get({ key: connectionName }) + .get({ key: keyFor(tenant, connectionName) }) .pipe(Effect.map((entry) => entry?.data.scope ?? null)), - putDescriptor: (owner, descriptor) => + putDescriptor: (tenant, owner, descriptor) => descriptors .put({ owner, - key: descriptor.scope, + key: keyFor(tenant, descriptor.scope), data: { + tenant, scope: descriptor.scope, snapshotId: descriptor.snapshotId, descriptor, @@ -72,7 +88,9 @@ export const makeAppsStore = (deps: AppsStoreDeps): AppsStore => { }, }) .pipe(Effect.asVoid), - getDescriptor: (scope) => - descriptors.get({ key: scope }).pipe(Effect.map((entry) => entry?.data.descriptor ?? null)), + getDescriptor: (tenant, scope) => + descriptors + .get({ key: keyFor(tenant, scope) }) + .pipe(Effect.map((entry) => entry?.data.descriptor ?? null)), }; }; diff --git a/packages/plugins/apps/src/plugin/tenant-isolation.test.ts b/packages/plugins/apps/src/plugin/tenant-isolation.test.ts new file mode 100644 index 000000000..bc1b1d69f --- /dev/null +++ b/packages/plugins/apps/src/plugin/tenant-isolation.test.ts @@ -0,0 +1,140 @@ +import { mkdtempSync, readdirSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +import { describe, expect, it } from "vitest"; +import { Effect } from "effect"; + +import { makeSqliteAppsStore } from "../backing/sqlite-apps-store"; +import { scopeAddress } from "../seams/scope-address"; +import { syncGitHubSource } from "../source/github-source"; +import { makeTestResolver } from "../testing"; +import { makeSelfHostAppsRuntime } from "./self-host-runtime"; + +const run = (effect: Effect.Effect): Promise => Effect.runPromise(effect); + +const toolSource = (name: string): string => `import { defineTool } from "executor:app"; +import { z } from "zod"; +export default defineTool({ + description: "${name}", + input: z.object({}), + output: z.object({ ok: z.boolean() }), + async handler(_input, { db }) { + await db.sql\`CREATE TABLE IF NOT EXISTS markers (name TEXT NOT NULL)\`; + await db.sql\`INSERT INTO markers (name) VALUES (${JSON.stringify(name)})\`; + return { ok: true }; + }, +});`; + +const json = (value: unknown, status = 200): Response => + new Response(JSON.stringify(value), { + status, + headers: { "content-type": "application/json" }, + }); + +const makeGitHubFetch = (input: { readonly toolName: string; readonly upstreamSha: string }) => { + const repoPath = "/repos/acme/tools"; + const treeSha = `${input.upstreamSha}-tree`; + const content = toolSource(input.toolName); + const fetch = (async (rawUrl: string) => { + const url = new URL(rawUrl); + if (url.pathname === repoPath) return json({ default_branch: "main" }); + if (url.pathname === `${repoPath}/git/ref/heads%2Fmain`) { + return json({ object: { sha: input.upstreamSha } }); + } + if (url.pathname === `${repoPath}/git/commits/${input.upstreamSha}`) { + return json({ sha: input.upstreamSha, tree: { sha: treeSha } }); + } + if (url.pathname === `${repoPath}/git/trees/${treeSha}`) { + return json({ + tree: [ + { + path: `tools/${input.toolName}.ts`, + type: "blob", + sha: `${input.upstreamSha}-blob`, + size: Buffer.byteLength(content, "utf8"), + }, + ], + }); + } + if (url.pathname === `${repoPath}/git/blobs/${input.upstreamSha}-blob`) { + return json({ + encoding: "base64", + content: Buffer.from(content, "utf8").toString("base64"), + }); + } + return json({ message: "not found" }, 404); + }) as typeof globalThis.fetch; + return fetch; +}; + +const makeTenantRuntime = (dataDir: string, tenant: string, storePath: string) => + makeSelfHostAppsRuntime({ + dataDir, + tenant, + store: makeSqliteAppsStore({ path: storePath }), + resolver: makeTestResolver({}), + }); + +describe("tenant scope isolation", () => { + it("keeps the same source-shaped scope distinct across tenants", async () => { + const dataDir = mkdtempSync(join(tmpdir(), "apps-tenant-")); + const storePath = join(dataDir, "store.sqlite"); + const scope = "githubSourceSameRepoConnection"; + const tenantA = "org-a"; + const tenantB = "org-b"; + const hostA = makeTenantRuntime(dataDir, tenantA, storePath); + const hostB = makeTenantRuntime(dataDir, tenantB, storePath); + + const publishedA = await run( + syncGitHubSource({ + runtime: hostA.runtime, + tenant: tenantA, + scope, + repo: "acme/tools", + fetch: makeGitHubFetch({ toolName: "alpha", upstreamSha: "sha-a" }), + }), + ); + const publishedB = await run( + syncGitHubSource({ + runtime: hostB.runtime, + tenant: tenantB, + scope, + repo: "acme/tools", + fetch: makeGitHubFetch({ toolName: "beta", upstreamSha: "sha-b" }), + }), + ); + + expect(publishedA.status).toBe("published"); + expect(publishedB.status).toBe("published"); + + const artifactDirs = readdirSync(join(dataDir, "artifacts")).filter((name) => + name.endsWith(".git"), + ); + expect(artifactDirs.length).toBe(2); + + const descriptorA = await run(hostA.runtime.getDescriptor(tenantA, scope)); + const descriptorB = await run(hostB.runtime.getDescriptor(tenantB, scope)); + expect(descriptorA?.tools.map((tool) => tool.name)).toEqual(["alpha"]); + expect(descriptorB?.tools.map((tool) => tool.name)).toEqual(["beta"]); + + const dbA = await run(hostA.scopeDb.forScope(scopeAddress(tenantA, scope))); + const dbB = await run(hostB.scopeDb.forScope(scopeAddress(tenantB, scope))); + await run(dbA.exec("CREATE TABLE marker (value TEXT)")); + await run(dbA.exec("INSERT INTO marker (value) VALUES ('a')")); + await run(dbB.exec("CREATE TABLE marker (value TEXT)")); + await run(dbB.exec("INSERT INTO marker (value) VALUES ('b')")); + expect( + (await run(dbA.exec<{ value: string }>("SELECT value FROM marker"))).map((r) => r.value), + ).toEqual(["a"]); + expect( + (await run(dbB.exec<{ value: string }>("SELECT value FROM marker"))).map((r) => r.value), + ).toEqual(["b"]); + expect( + readdirSync(join(dataDir, "scope-db")).filter((name) => name.endsWith(".db")).length, + ).toBe(2); + + await hostA.close(); + await hostB.close(); + }); +}); diff --git a/packages/plugins/apps/src/seams/artifact-store.conformance.ts b/packages/plugins/apps/src/seams/artifact-store.conformance.ts index e99460d3c..334f67817 100644 --- a/packages/plugins/apps/src/seams/artifact-store.conformance.ts +++ b/packages/plugins/apps/src/seams/artifact-store.conformance.ts @@ -2,6 +2,7 @@ import { describe, expect, it } from "vitest"; import { Effect } from "effect"; import type { ArtifactStore, FileSet } from "./artifact-store"; +import { scopeAddress } from "./scope-address"; // --------------------------------------------------------------------------- // ArtifactStore conformance suite. Runs against the INTERFACE, not a specific @@ -22,7 +23,7 @@ export const artifactStoreConformance = ( describe(`ArtifactStore conformance: ${name}`, () => { it("round-trips a file set through a snapshot", async () => { const store = await makeStore(); - const scope = await run(store.forScope("s1")); + const scope = await run(store.forScope(scopeAddress("org", "s1"))); const files = fileSet({ "tools/a.ts": "export const a = 1;\n", "notes/x.md": "# x\n", @@ -46,7 +47,7 @@ export const artifactStoreConformance = ( it("keeps a snapshot immutable across a later publish", async () => { const store = await makeStore(); - const scope = await run(store.forScope("s2")); + const scope = await run(store.forScope(scopeAddress("org", "s2"))); const first = await run(scope.commit(fileSet({ "tools/a.ts": "v1" }), "first")); const second = await run( scope.commit(fileSet({ "tools/a.ts": "v2", "tools/b.ts": "new" }), "second"), @@ -65,7 +66,7 @@ export const artifactStoreConformance = ( it("tracks latest and logs newest-first", async () => { const store = await makeStore(); - const scope = await run(store.forScope("s3")); + const scope = await run(store.forScope(scopeAddress("org", "s3"))); expect(await run(scope.latest())).toBeNull(); const a = await run(scope.commit(fileSet({ "tools/a.ts": "1" }), "a")); @@ -82,8 +83,8 @@ export const artifactStoreConformance = ( it("isolates scopes", async () => { const store = await makeStore(); - const s1 = await run(store.forScope("iso-1")); - const s2 = await run(store.forScope("iso-2")); + const s1 = await run(store.forScope(scopeAddress("org", "iso-1"))); + const s2 = await run(store.forScope(scopeAddress("org", "iso-2"))); await run(s1.commit(fileSet({ "tools/a.ts": "one" }), "one")); expect(await run(s2.latest())).toBeNull(); }); diff --git a/packages/plugins/apps/src/seams/artifact-store.ts b/packages/plugins/apps/src/seams/artifact-store.ts index fcfd5bd4b..63c623e77 100644 --- a/packages/plugins/apps/src/seams/artifact-store.ts +++ b/packages/plugins/apps/src/seams/artifact-store.ts @@ -1,5 +1,6 @@ import type { Effect } from "effect"; import { Data } from "effect"; +import type { ScopeAddress } from "./scope-address"; // --------------------------------------------------------------------------- // ArtifactStore — the per-scope git-backed source store. @@ -58,7 +59,9 @@ export interface ScopeArtifactStore { readonly log: (limit?: number) => Effect.Effect; } -/** The substrate-neutral store: hands out a per-scope store by scope key. */ +/** The substrate-neutral store: hands out a per-scope store by tenant + scope. */ export interface ArtifactStore { - readonly forScope: (scope: string) => Effect.Effect; + readonly forScope: ( + address: ScopeAddress, + ) => Effect.Effect; } diff --git a/packages/plugins/apps/src/seams/scope-address.ts b/packages/plugins/apps/src/seams/scope-address.ts new file mode 100644 index 000000000..9e5c2c27d --- /dev/null +++ b/packages/plugins/apps/src/seams/scope-address.ts @@ -0,0 +1,11 @@ +export interface ScopeAddress { + readonly tenant: string; + readonly scope: string; +} + +export const scopeAddress = (tenant: string, scope: string): ScopeAddress => ({ tenant, scope }); + +// Tenant-aware keys were introduced before the apps subsystem shipped, so no +// on-disk migration is needed for the older scope-only layout. +export const scopeAddressStorageKey = (address: ScopeAddress): string => + `v2-${Buffer.from(JSON.stringify([address.tenant, address.scope]), "utf8").toString("hex")}`; diff --git a/packages/plugins/apps/src/seams/scope-db.conformance.ts b/packages/plugins/apps/src/seams/scope-db.conformance.ts index a8342fd5c..c0efd51ae 100644 --- a/packages/plugins/apps/src/seams/scope-db.conformance.ts +++ b/packages/plugins/apps/src/seams/scope-db.conformance.ts @@ -2,6 +2,7 @@ import { describe, expect, it } from "vitest"; import { Effect } from "effect"; import type { ScopeDb } from "./scope-db"; +import { scopeAddress } from "./scope-address"; // --------------------------------------------------------------------------- // ScopeDb conformance suite. Runs against the interface. Covers: scope @@ -18,8 +19,8 @@ export const scopeDbConformance = ( describe(`ScopeDb conformance: ${name}`, () => { it("isolates data between scopes", async () => { const db = await makeDb(); - const a = await run(db.forScope("scope-a")); - const b = await run(db.forScope("scope-b")); + const a = await run(db.forScope(scopeAddress("org", "scope-a"))); + const b = await run(db.forScope(scopeAddress("org", "scope-b"))); await run(a.exec("CREATE TABLE items (id INTEGER PRIMARY KEY, name TEXT)")); await run(a.exec("INSERT INTO items (name) VALUES ('a-only')")); await run(b.exec("CREATE TABLE items (id INTEGER PRIMARY KEY, name TEXT)")); @@ -33,7 +34,7 @@ export const scopeDbConformance = ( it("bumps a table's version on each write", async () => { const db = await makeDb(); - const s = await run(db.forScope("ver")); + const s = await run(db.forScope(scopeAddress("org", "ver"))); await run(s.exec("CREATE TABLE t (id INTEGER PRIMARY KEY)")); const afterCreate = await run(s.tableVersion("t")); expect(afterCreate).toBeGreaterThanOrEqual(1); @@ -54,7 +55,7 @@ export const scopeDbConformance = ( it("supports the author-facing tagged-template sql with parameters", async () => { const db = await makeDb(); - const s = await run(db.forScope("tpl")); + const s = await run(db.forScope(scopeAddress("org", "tpl"))); await run(s.exec("CREATE TABLE issues (repo TEXT, title TEXT)")); const repo = "acme/app"; const title = "Bug"; diff --git a/packages/plugins/apps/src/seams/scope-db.ts b/packages/plugins/apps/src/seams/scope-db.ts index a58538d0a..bad96580d 100644 --- a/packages/plugins/apps/src/seams/scope-db.ts +++ b/packages/plugins/apps/src/seams/scope-db.ts @@ -1,5 +1,6 @@ import type { Effect } from "effect"; import { Data } from "effect"; +import type { ScopeAddress } from "./scope-address"; // --------------------------------------------------------------------------- // ScopeDb — the per-scope app database (the shared primitive). @@ -37,6 +38,6 @@ export interface ScopeDbHandle { } export interface ScopeDb { - readonly forScope: (scope: string) => Effect.Effect; + readonly forScope: (address: ScopeAddress) => Effect.Effect; readonly close: () => Effect.Effect; } diff --git a/packages/plugins/apps/src/source/github-source.ts b/packages/plugins/apps/src/source/github-source.ts index 475e65504..db827ac94 100644 --- a/packages/plugins/apps/src/source/github-source.ts +++ b/packages/plugins/apps/src/source/github-source.ts @@ -61,6 +61,7 @@ export type GitHubSyncResult = export interface SyncGitHubSourceInput extends GitHubSourceInput { readonly runtime: AppsRuntime; + readonly tenant?: string; readonly scope: string; } @@ -359,7 +360,9 @@ export const syncGitHubSource = (input: SyncGitHubSourceInput): Effect.Effect { const descriptors = new Map(); const scopeConnections = new Map(); + const keyFor = (tenant: string, key: string): string => `${tenant}:${key}`; return { descriptors, - putDescriptor: (_owner, descriptor) => - Effect.sync(() => void descriptors.set(descriptor.scope, descriptor)), - getDescriptor: (scope) => Effect.sync(() => descriptors.get(scope) ?? null), - putScopeForConnection: (connectionName, scope) => - Effect.sync(() => void scopeConnections.set(connectionName, scope)), - getScopeForConnection: (connectionName) => - Effect.sync(() => scopeConnections.get(connectionName) ?? null), + putDescriptor: (tenant, _owner, descriptor) => + Effect.sync(() => void descriptors.set(keyFor(tenant, descriptor.scope), descriptor)), + getDescriptor: (tenant, scope) => + Effect.sync(() => descriptors.get(keyFor(tenant, scope)) ?? null), + putScopeForConnection: (tenant, connectionName, scope) => + Effect.sync(() => void scopeConnections.set(keyFor(tenant, connectionName), scope)), + getScopeForConnection: (tenant, connectionName) => + Effect.sync(() => scopeConnections.get(keyFor(tenant, connectionName)) ?? null), }; }; @@ -35,11 +38,11 @@ export const makeInMemoryArtifactStore = (): ArtifactStore => { const scopes = new Map>(); const order = new Map(); let counter = 0; - const forScope = (scope: string): ScopeArtifactStore => { - const snaps = scopes.get(scope) ?? new Map(); - scopes.set(scope, snaps); - const seq = order.get(scope) ?? []; - order.set(scope, seq); + const forScope = (key: string): ScopeArtifactStore => { + const snaps = scopes.get(key) ?? new Map(); + scopes.set(key, snaps); + const seq = order.get(key) ?? []; + order.set(key, seq); return { commit: (files, message) => Effect.sync(() => { @@ -69,7 +72,7 @@ export const makeInMemoryArtifactStore = (): ArtifactStore => { ), }; }; - return { forScope: (scope) => Effect.succeed(forScope(scope)) }; + return { forScope: (address) => Effect.succeed(forScope(scopeAddressStorageKey(address))) }; }; export type { SnapshotId }; From 9e77e34909be272d732897c4308073e604b08fee Mon Sep 17 00:00:00 2001 From: Rhys Sullivan <39114868+RhysSullivan@users.noreply.github.com> Date: Mon, 6 Jul 2026 04:34:19 -0700 Subject: [PATCH 07/21] Filter GitHub source entries safely --- packages/plugins/apps/src/plugin/bindings.ts | 2 + .../apps/src/plugin/tenant-isolation.test.ts | 1 + .../apps/src/source/github-source.test.ts | 113 ++++++++++++++++-- .../plugins/apps/src/source/github-source.ts | 72 ++++++++--- 4 files changed, 159 insertions(+), 29 deletions(-) diff --git a/packages/plugins/apps/src/plugin/bindings.ts b/packages/plugins/apps/src/plugin/bindings.ts index 030293c05..5004b9241 100644 --- a/packages/plugins/apps/src/plugin/bindings.ts +++ b/packages/plugins/apps/src/plugin/bindings.ts @@ -259,6 +259,8 @@ export const buildBridge = (context: BindingContext): HandleBridge => ({ return Effect.fail(invokeErr(`no binding for role ${role}`)); } if (path.length === 1 && path[0] === "account") { + // Connection-derived placeholders only; this is not verified upstream + // account identity. return Effect.succeed({ email: connectionName, login: connectionName, name: connectionName }); } diff --git a/packages/plugins/apps/src/plugin/tenant-isolation.test.ts b/packages/plugins/apps/src/plugin/tenant-isolation.test.ts index bc1b1d69f..46e61c0c8 100644 --- a/packages/plugins/apps/src/plugin/tenant-isolation.test.ts +++ b/packages/plugins/apps/src/plugin/tenant-isolation.test.ts @@ -51,6 +51,7 @@ const makeGitHubFetch = (input: { readonly toolName: string; readonly upstreamSh { path: `tools/${input.toolName}.ts`, type: "blob", + mode: "100644", sha: `${input.upstreamSha}-blob`, size: Buffer.byteLength(content, "utf8"), }, diff --git a/packages/plugins/apps/src/source/github-source.test.ts b/packages/plugins/apps/src/source/github-source.test.ts index bbc767acd..2c321db71 100644 --- a/packages/plugins/apps/src/source/github-source.test.ts +++ b/packages/plugins/apps/src/source/github-source.test.ts @@ -8,6 +8,7 @@ import { Effect } from "effect"; import { makeSelfHostAppsRuntime } from "../plugin/self-host-runtime"; import { makeInMemoryAppsStore, makeTestResolver } from "../testing"; import { PUBLISH_LIMITS } from "../pipeline/publish"; +import { scopeAddress } from "../seams/scope-address"; import { fetchGitHubSource, syncGitHubSource } from "./github-source"; const run = (effect: Effect.Effect): Promise => Effect.runPromise(effect); @@ -34,8 +35,17 @@ const json = (value: unknown, status = 200): Response => headers: { "content-type": "application/json" }, }); +interface GitHubTreeFixtureEntry { + readonly path: string; + readonly type: string; + readonly mode?: string; + readonly content?: string; + readonly size?: number; +} + const makeGitHubFetch = (input: { readonly files: ReadonlyMap; + readonly entries?: readonly GitHubTreeFixtureEntry[]; readonly upstreamSha?: string; readonly treeSha?: string; }) => { @@ -43,9 +53,22 @@ const makeGitHubFetch = (input: { const upstreamSha = input.upstreamSha ?? "commit-1"; const treeSha = input.treeSha ?? "tree-1"; const blobBySha = new Map(); + const treeEntries: readonly GitHubTreeFixtureEntry[] = + input.entries ?? + [...input.files].map(([path, content]) => ({ + path, + type: "blob", + mode: "100644", + content, + })); let index = 0; - for (const [path, content] of input.files) { - blobBySha.set(`blob-${++index}`, { path, content }); + const shaByPath = new Map(); + for (const entry of treeEntries) { + if (entry.content !== undefined) { + const sha = `blob-${++index}`; + shaByPath.set(entry.path, sha); + blobBySha.set(sha, { path: entry.path, content: entry.content }); + } } let blobCalls = 0; const fetch = (async (rawUrl: string) => { @@ -59,12 +82,16 @@ const makeGitHubFetch = (input: { } if (url.pathname === `${repoPath}/git/trees/${treeSha}`) { return json({ - tree: [...blobBySha.entries()].map(([sha, blob]) => ({ - path: blob.path, - type: "blob", - sha, - size: Buffer.byteLength(blob.content, "utf8"), - })), + tree: treeEntries.map((entry, entryIndex) => { + const sha = shaByPath.get(entry.path) ?? `tree-${entryIndex}`; + return { + path: entry.path, + type: entry.type, + mode: entry.mode, + sha, + size: entry.size ?? Buffer.byteLength(entry.content ?? "", "utf8"), + }; + }), }); } const blobPrefix = `${repoPath}/git/blobs/`; @@ -103,10 +130,10 @@ describe("GitHub custom-tools source", () => { ]); const github = makeGitHubFetch({ files, upstreamSha: "commit-a" }); const snapshot = await run(fetchGitHubSource({ repo: "acme/tools", fetch: github.fetch })); - expect([...snapshot.files.keys()].sort()).toEqual([ - "executor.json", - "tools/hello.ts", - "workflows/deferred.ts", + expect([...snapshot.files.keys()].sort()).toEqual(["executor.json", "tools/hello.ts"]); + expect(snapshot.skipped).toEqual([ + { path: "workflows/deferred.ts", reason: "not supported yet" }, + { path: "docs/readme.md", reason: "ignored" }, ]); expect(snapshot.description).toBe("Acme tools"); @@ -123,6 +150,7 @@ describe("GitHub custom-tools source", () => { expect(result.tools).toEqual(["hello"]); expect(result.skipped).toEqual([ { path: "workflows/deferred.ts", reason: "not supported yet" }, + { path: "docs/readme.md", reason: "ignored" }, ]); const descriptor = await run(host.runtime.getDescriptor("githubTools")); expect(descriptor?.description).toBe("Acme tools"); @@ -208,4 +236,65 @@ export default defineTool({ description: "bad", input: { type: "object" }, async expect(result.errors?.[0]?.message).toContain('bare import "lodash" is not allowed'); await host.close(); }); + + it("skips unsupported tree entries and invalid paths without committing them", async () => { + const github = makeGitHubFetch({ + files: new Map(), + entries: [ + { + path: "tools/ok.ts", + type: "blob", + mode: "100644", + content: toolSource("ok"), + }, + { + path: "tools/link.ts", + type: "blob", + mode: "120000", + content: "../outside.ts", + }, + { + path: "tools/submodule.ts", + type: "commit", + mode: "160000", + }, + { + path: "tools/Bad.Name.ts", + type: "blob", + mode: "100644", + content: toolSource("bad"), + }, + ], + upstreamSha: "mixed-sha", + }); + const host = makeRuntime(); + const result = await run( + syncGitHubSource({ + runtime: host.runtime, + scope: "githubTools", + repo: "acme/tools", + fetch: github.fetch, + }), + ); + + expect(result.status).toBe("published"); + expect(result.tools).toEqual(["ok"]); + expect(result.skipped).toEqual([ + { path: "tools/link.ts", reason: "unsupported file type" }, + { path: "tools/submodule.ts", reason: "unsupported file type" }, + { path: "tools/Bad.Name.ts", reason: "ignored" }, + ]); + + const descriptor = await run(host.runtime.getDescriptor("githubTools")); + expect(descriptor?.tools.map((tool) => tool.name)).toEqual(["ok"]); + const scopeStore = await run( + host.runtime.deps.artifactStore.forScope(scopeAddress("org", "githubTools")), + ); + const snapshotPaths = await run(scopeStore.list(descriptor!.snapshotId as never)); + expect(snapshotPaths).toContain("tools/ok.ts"); + expect(snapshotPaths).not.toContain("tools/link.ts"); + expect(snapshotPaths).not.toContain("tools/submodule.ts"); + expect(snapshotPaths).not.toContain("tools/Bad.Name.ts"); + await host.close(); + }); }); diff --git a/packages/plugins/apps/src/source/github-source.ts b/packages/plugins/apps/src/source/github-source.ts index db827ac94..208b1d10c 100644 --- a/packages/plugins/apps/src/source/github-source.ts +++ b/packages/plugins/apps/src/source/github-source.ts @@ -1,6 +1,6 @@ import { Data, Effect, Result } from "effect"; -import { PublishError, type FileDiagnostic, type SkippedArtifact } from "../pipeline/discover"; +import { PublishError, type FileDiagnostic } from "../pipeline/discover"; import { PUBLISH_LIMITS, enforcePublishLimits } from "../pipeline/publish"; import type { AppSourceRef } from "../pipeline/descriptor"; import type { AppsRuntime } from "../plugin/runtime"; @@ -20,6 +20,7 @@ export interface GitHubSourceSnapshot { readonly ref: string; readonly upstreamSha: string; readonly description?: string; + readonly skipped: readonly GitHubSkippedArtifact[]; } export class GitHubSourceError extends Data.TaggedError("GitHubSourceError")<{ @@ -35,27 +36,32 @@ export interface SyncErrorData { readonly diagnostics?: readonly FileDiagnostic[]; } +export interface GitHubSkippedArtifact { + readonly path: string; + readonly reason: "not supported yet" | "unsupported file type" | "ignored"; +} + export type GitHubSyncResult = | { readonly status: "published"; readonly snapshotId: SnapshotId; readonly upstreamSha: string; readonly tools: readonly string[]; - readonly skipped: readonly SkippedArtifact[]; + readonly skipped: readonly GitHubSkippedArtifact[]; readonly errors?: undefined; } | { readonly status: "up-to-date"; readonly upstreamSha: string; readonly tools: readonly string[]; - readonly skipped: readonly SkippedArtifact[]; + readonly skipped: readonly GitHubSkippedArtifact[]; readonly errors?: undefined; } | { readonly status: "failed"; readonly upstreamSha?: string; readonly tools: readonly string[]; - readonly skipped: readonly SkippedArtifact[]; + readonly skipped: readonly GitHubSkippedArtifact[]; readonly errors: readonly SyncErrorData[]; }; @@ -65,7 +71,9 @@ export interface SyncGitHubSourceInput extends GitHubSourceInput { readonly scope: string; } -const MATERIALIZED_RE = /^(tools|workflows|ui|skills)\//; +const TOOL_RE = /^tools\/([a-z0-9][a-z0-9-]*)\.(ts|tsx|js|jsx)$/; +const DEFERRED_RE = /^(workflows|ui|skills)\//; +const REGULAR_FILE_MODES = new Set(["100644", "100755"]); const trimBaseUrl = (baseUrl: string): string => baseUrl.replace(/\/+$/, ""); @@ -88,8 +96,7 @@ const encodedRepoPath = (repo: string): Effect.Effect return Effect.succeed(`${encodeURIComponent(parsed.owner)}/${encodeURIComponent(parsed.name)}`); }; -const materializes = (path: string): boolean => - path === "executor.json" || MATERIALIZED_RE.test(path); +const acceptedPath = (path: string): boolean => path === "executor.json" || TOOL_RE.test(path); const isRecord = (value: unknown): value is Record => value !== null && typeof value === "object" && !Array.isArray(value); @@ -163,6 +170,7 @@ interface RefResponse { interface TreeEntry { readonly path?: unknown; readonly type?: unknown; + readonly mode?: unknown; readonly sha?: unknown; readonly size?: unknown; } @@ -219,6 +227,34 @@ const checkTreeLimits = (entries: readonly TreeEntry[]): PublishError | null => return diagnostics.length === 0 ? null : limitError(diagnostics); }; +const classifyTreeEntry = ( + entry: TreeEntry, +): + | { + readonly kind: "fetch"; + readonly entry: TreeEntry & { readonly path: string; readonly sha: string }; + } + | { readonly kind: "skip"; readonly skipped: GitHubSkippedArtifact } + | null => { + const path = asString(entry.path); + if (!path) return null; + + const type = asString(entry.type); + const mode = asString(entry.mode); + if (type === "tree") return null; + if (type !== "blob" || !mode || !REGULAR_FILE_MODES.has(mode)) { + return { kind: "skip", skipped: { path, reason: "unsupported file type" } }; + } + + const sha = asString(entry.sha); + if (!sha) return { kind: "skip", skipped: { path, reason: "unsupported file type" } }; + if (acceptedPath(path)) return { kind: "fetch", entry: { ...entry, path, sha } }; + if (DEFERRED_RE.test(path)) { + return { kind: "skip", skipped: { path, reason: "not supported yet" } }; + } + return { kind: "skip", skipped: { path, reason: "ignored" } }; +}; + const decodeBlob = (path: string, blob: BlobResponse): Effect.Effect => Effect.try({ try: () => { @@ -293,13 +329,14 @@ export const fetchGitHubSource = ( path: `/repos/${repoPath}/git/trees/${treeSha}`, }); } - const entries = (tree.tree ?? []).filter( - (entry) => - entry.type === "blob" && - typeof entry.path === "string" && - typeof entry.sha === "string" && - materializes(entry.path), - ); + const entries: (TreeEntry & { readonly path: string; readonly sha: string })[] = []; + const skipped: GitHubSkippedArtifact[] = []; + for (const entry of tree.tree ?? []) { + const classified = classifyTreeEntry(entry); + if (!classified) continue; + if (classified.kind === "fetch") entries.push(classified.entry); + else skipped.push(classified.skipped); + } const treeLimitError = checkTreeLimits(entries); if (treeLimitError) return yield* Effect.fail(treeLimitError); @@ -321,6 +358,7 @@ export const fetchGitHubSource = ( ref, upstreamSha, description: yield* executorDescription(files), + skipped, }; }); @@ -372,7 +410,7 @@ export const syncGitHubSource = (input: SyncGitHubSourceInput): Effect.Effect tool.name), - skipped: current.skipped ?? [], + skipped: [...snapshot.skipped, ...(current.skipped ?? [])], } satisfies GitHubSyncResult; } @@ -391,7 +429,7 @@ export const syncGitHubSource = (input: SyncGitHubSourceInput): Effect.Effect tool.name), - skipped: published.success.skipped, + skipped: [...snapshot.skipped, ...published.success.skipped], } satisfies GitHubSyncResult; }); From 2a35fef855eeda62cf2facb06a628292517d7829 Mon Sep 17 00:00:00 2001 From: Rhys Sullivan <39114868+RhysSullivan@users.noreply.github.com> Date: Mon, 6 Jul 2026 05:39:23 -0700 Subject: [PATCH 08/21] Add self-host custom tools source UI --- apps/host-selfhost/src/apps-route.ts | 105 ++++-- .../src/custom-tools-console.web.test.ts | 137 +++++++ apps/host-selfhost/tsconfig.json | 1 + bun.lock | 14 +- packages/plugins/apps/package.json | 21 +- packages/plugins/apps/src/api.ts | 2 +- .../apps/src/backing/sqlite-apps-store.ts | 15 + .../plugins/apps/src/pipeline/descriptor.ts | 7 + .../plugins/apps/src/plugin/apps-plugin.ts | 11 + packages/plugins/apps/src/plugin/runtime.ts | 48 +++ packages/plugins/apps/src/plugin/store.ts | 17 + .../apps/src/react/AddCustomToolsSource.tsx | 163 +++++++++ .../src/react/CustomToolsAccountsPanel.tsx | 344 ++++++++++++++++++ .../apps/src/react/custom-tools-client.ts | 113 ++++++ .../plugins/apps/src/react/plugin-client.tsx | 11 + .../plugins/apps/src/react/source-plugin.ts | 18 + .../apps/src/source/github-source.test.ts | 4 + .../plugins/apps/src/source/github-source.ts | 14 +- packages/plugins/apps/src/testing/index.ts | 16 +- .../react/src/pages/integration-detail.tsx | 3 +- 20 files changed, 1014 insertions(+), 50 deletions(-) create mode 100644 apps/host-selfhost/src/custom-tools-console.web.test.ts create mode 100644 packages/plugins/apps/src/react/AddCustomToolsSource.tsx create mode 100644 packages/plugins/apps/src/react/CustomToolsAccountsPanel.tsx create mode 100644 packages/plugins/apps/src/react/custom-tools-client.ts create mode 100644 packages/plugins/apps/src/react/plugin-client.tsx create mode 100644 packages/plugins/apps/src/react/source-plugin.ts diff --git a/apps/host-selfhost/src/apps-route.ts b/apps/host-selfhost/src/apps-route.ts index 8dcf341a9..58a0ecfca 100644 --- a/apps/host-selfhost/src/apps-route.ts +++ b/apps/host-selfhost/src/apps-route.ts @@ -1,4 +1,4 @@ -import { HttpEffect, HttpRouter } from "effect/unstable/http"; +import { HttpRouter, HttpServerRequest, HttpServerResponse } from "effect/unstable/http"; import { Data, Effect, Layer } from "effect"; import { @@ -19,12 +19,6 @@ class AppsSyncRouteError extends Data.TaggedError("AppsSyncRouteError")<{ readonly message: string; }> {} -const jsonResponse = (body: unknown, status = 200): Response => - new Response(JSON.stringify(body), { - status, - headers: { "content-type": "application/json" }, - }); - const parseJson = (request: Request): Effect.Effect => Effect.tryPromise({ try: () => request.json() as Promise, @@ -51,41 +45,76 @@ export interface SelfHostAppsSyncRouteDeps { readonly db: SelfHostDbHandle; } +const routeHandler = ( + run: (request: Request) => Effect.Effect, +): Effect.Effect => + Effect.gen(function* () { + const httpRequest = yield* HttpServerRequest.HttpServerRequest; + const request = yield* HttpServerRequest.toWeb(httpRequest).pipe(Effect.orDie); + return yield* run(request).pipe( + Effect.map((body) => HttpServerResponse.jsonUnsafe(body)), + Effect.catch((error: unknown) => { + const tag = + error && typeof error === "object" && "_tag" in error ? String(error._tag) : null; + if (tag === "Unauthorized") { + return Effect.succeed(HttpServerResponse.text("Unauthorized", { status: 401 })); + } + if (tag === "NoOrganization") { + return Effect.succeed(HttpServerResponse.text("Forbidden", { status: 403 })); + } + if (tag === "Unavailable") { + return Effect.succeed(HttpServerResponse.text("Unavailable", { status: 503 })); + } + if (tag === "AppsSyncRouteError") { + const routeError = error as AppsSyncRouteError; + return Effect.succeed( + HttpServerResponse.jsonUnsafe( + { error: routeError.message }, + { status: routeError.status }, + ), + ); + } + return Effect.succeed(HttpServerResponse.text("Internal Server Error", { status: 500 })); + }), + Effect.catchCause(() => + Effect.succeed(HttpServerResponse.text("Internal Server Error", { status: 500 })), + ), + ); + }); + export const makeSelfHostAppsSyncRoute = ({ identity, db }: SelfHostAppsSyncRouteDeps) => { const services = Layer.merge( SelfHostScopedExecutorSeams.pipe(Layer.provide(Layer.succeed(SelfHostDb)(db))), identity, ); - return HttpRouter.add( - "POST", - "/api/apps/sources/github/sync", - HttpEffect.fromWebHandler( - (request): Promise => - Effect.runPromise( - Effect.gen(function* () { - const identityProvider = yield* IdentityProvider; - const principal = yield* identityProvider.authenticate(request); - const payload = yield* parseJson(request); - const executor = yield* buildExecutor(principal, request); - const result = yield* executor.execute( - ToolAddress.make("executor.apps.sync_github_source"), - payload, - ); - return jsonResponse(result); - }).pipe( - Effect.catchTags({ - Unauthorized: () => Effect.succeed(new Response("Unauthorized", { status: 401 })), - NoOrganization: () => Effect.succeed(new Response("Forbidden", { status: 403 })), - Unavailable: () => Effect.succeed(new Response("Unavailable", { status: 503 })), - AppsSyncRouteError: (error) => - Effect.succeed(jsonResponse({ error: error.message }, error.status)), - }), - Effect.catchCause(() => - Effect.succeed(new Response("Internal Server Error", { status: 500 })), - ), - Effect.provide(services), - ), - ), + return Layer.mergeAll( + HttpRouter.add( + "GET", + "/api/apps/sources/github", + routeHandler((request) => + Effect.gen(function* () { + const identityProvider = yield* IdentityProvider; + const principal = yield* identityProvider.authenticate(request); + const executor = yield* buildExecutor(principal, request); + return yield* executor.execute(ToolAddress.make("executor.apps.list_github_sources"), {}); + }), + ), ), - ); + HttpRouter.add( + "POST", + "/api/apps/sources/github/sync", + routeHandler((request) => + Effect.gen(function* () { + const identityProvider = yield* IdentityProvider; + const principal = yield* identityProvider.authenticate(request); + const payload = yield* parseJson(request); + const executor = yield* buildExecutor(principal, request); + return yield* executor.execute( + ToolAddress.make("executor.apps.sync_github_source"), + payload, + ); + }), + ), + ), + ).pipe(HttpRouter.provideRequest(services)); }; diff --git a/apps/host-selfhost/src/custom-tools-console.web.test.ts b/apps/host-selfhost/src/custom-tools-console.web.test.ts new file mode 100644 index 000000000..3c1ee0511 --- /dev/null +++ b/apps/host-selfhost/src/custom-tools-console.web.test.ts @@ -0,0 +1,137 @@ +import { describe, expect, it, vi } from "@effect/vitest"; + +import appsClientPlugin, { + CUSTOM_TOOLS_LABEL, + CUSTOM_TOOLS_PLUGIN_KEY, + appsIntegrationPlugin, + formatSyncErrors, + listCustomToolSources, + syncCustomToolSource, + syncStatusLabel, + validateGitHubRepo, +} from "@executor-js/plugin-apps/client"; + +const jsonResponse = (body: unknown, init?: ResponseInit): Response => + new Response(JSON.stringify(body), { + status: 200, + headers: { "content-type": "application/json" }, + ...init, + }); + +describe("custom tools console client", () => { + it("registers the manual Custom tools tile", () => { + const manualTiles = [appsIntegrationPlugin].map((plugin) => ({ + href: `/integrations/add/${plugin.key}`, + label: plugin.label, + })); + + expect(appsClientPlugin.id).toBe(CUSTOM_TOOLS_PLUGIN_KEY); + expect(appsClientPlugin.integrationPlugin?.label).toBe(CUSTOM_TOOLS_LABEL); + expect(manualTiles).toContainEqual({ + href: "/integrations/add/apps", + label: "Custom tools", + }); + }); + + it("validates the GitHub repo shape", () => { + expect(validateGitHubRepo("UsefulSoftwareCo/executor")).toBeNull(); + expect(validateGitHubRepo("UsefulSoftwareCo")).toBe( + "Use owner/name, for example UsefulSoftwareCo/executor.", + ); + expect(validateGitHubRepo("")).toBe("Enter a GitHub repo."); + }); + + it("surfaces successful sync and source detail data", async () => { + const fetchImpl = vi.fn(async (input: RequestInfo | URL) => { + const url = String(input); + if (url.endsWith("/sync")) { + return jsonResponse({ + status: "published", + snapshotId: "snap1", + upstreamSha: "abc123", + tools: ["repo-summary", "stale-issues"], + skipped: [], + }); + } + return jsonResponse({ + sources: [ + { + scope: "github:RhysSullivan/executor-custom-tools-demo", + repo: "RhysSullivan/executor-custom-tools-demo", + ref: "main", + connection: "tools.github.user.demo", + upstreamSha: "abc123", + snapshotId: "snap1", + publishedAt: "2026-07-06T12:00:00.000Z", + tools: ["repo-summary", "stale-issues"], + skipped: [], + }, + ], + }); + }) as unknown as typeof fetch; + + const syncResult = await syncCustomToolSource( + { + repo: "RhysSullivan/executor-custom-tools-demo", + connection: "tools.github.user.demo", + }, + fetchImpl, + ); + const listed = await listCustomToolSources(fetchImpl); + + expect(syncStatusLabel(syncResult)).toBe("Published 2 tools."); + expect(syncResult.tools).toEqual(["repo-summary", "stale-issues"]); + expect(listed.sources[0]?.tools).toEqual(["repo-summary", "stale-issues"]); + }); + + it("renders failed sync errors readably", async () => { + const fetchImpl = vi.fn(async () => + jsonResponse({ + status: "failed", + tools: [], + skipped: [], + errors: [ + { + stage: "collect", + message: "schema library not supported for schema export", + diagnostics: [{ path: "tools/x.ts", message: "vendor: nope" }], + }, + ], + }), + ) as unknown as typeof fetch; + + const result = await syncCustomToolSource( + { + repo: "RhysSullivan/executor-custom-tools-demo", + connection: "tools.github.user.demo", + }, + fetchImpl, + ); + + expect(syncStatusLabel(result)).toBe("Sync failed."); + expect(formatSyncErrors(result)).toEqual([ + "collect: schema library not supported for schema export (tools/x.ts: vendor: nope)", + ]); + }); + + it("shows the up-to-date sync state", async () => { + const fetchImpl = vi.fn(async () => + jsonResponse({ + status: "up-to-date", + upstreamSha: "abc123", + tools: ["repo-summary"], + skipped: [], + }), + ) as unknown as typeof fetch; + + const result = await syncCustomToolSource( + { + repo: "RhysSullivan/executor-custom-tools-demo", + connection: "tools.github.user.demo", + }, + fetchImpl, + ); + + expect(syncStatusLabel(result)).toBe("Already up to date."); + }); +}); diff --git a/apps/host-selfhost/tsconfig.json b/apps/host-selfhost/tsconfig.json index e214693ed..35302d3e8 100644 --- a/apps/host-selfhost/tsconfig.json +++ b/apps/host-selfhost/tsconfig.json @@ -6,6 +6,7 @@ "strict": true, "esModuleInterop": true, "skipLibCheck": true, + "jsx": "react-jsx", "types": ["bun-types"], "noUnusedLocals": true, "noImplicitOverride": true, diff --git a/bun.lock b/bun.lock index bba86774b..4032c9fa8 100644 --- a/bun.lock +++ b/bun.lock @@ -795,16 +795,28 @@ "zod": "4.3.6", }, "devDependencies": { + "@effect/atom-react": "catalog:", "@effect/vitest": "catalog:", + "@executor-js/react": "workspace:*", "@types/node": "catalog:", + "@types/react": "catalog:", "bun-types": "catalog:", "effect": "catalog:", + "react": "catalog:", "tsup": "catalog:", "vitest": "catalog:", }, "peerDependencies": { + "@effect/atom-react": "catalog:", + "@executor-js/react": "workspace:*", "effect": "catalog:", + "react": "catalog:", }, + "optionalPeers": [ + "@effect/atom-react", + "@executor-js/react", + "react", + ], }, "packages/plugins/desktop-settings": { "name": "@executor-js/plugin-desktop-settings", @@ -1240,9 +1252,9 @@ }, }, "patchedDependencies": { - "@1password/sdk-core@0.4.1-beta.1": "patches/@1password%2Fsdk-core@0.4.1-beta.1.patch", "@electric-sql/pglite-socket@0.1.4": "patches/@electric-sql%2Fpglite-socket@0.1.4.patch", "libsql@0.5.29": "patches/libsql@0.5.29.patch", + "@1password/sdk-core@0.4.1-beta.1": "patches/@1password%2Fsdk-core@0.4.1-beta.1.patch", "agents@0.17.3": "patches/agents@0.17.3.patch", "postgres@3.4.9": "patches/postgres@3.4.9.patch", }, diff --git a/packages/plugins/apps/package.json b/packages/plugins/apps/package.json index c0a0751bb..d50ceb999 100644 --- a/packages/plugins/apps/package.json +++ b/packages/plugins/apps/package.json @@ -18,6 +18,7 @@ "exports": { ".": "./src/index.ts", "./api": "./src/api.ts", + "./client": "./src/react/plugin-client.tsx", "./seams": "./src/seams/index.ts", "./testing": "./src/testing/index.ts" }, @@ -37,14 +38,32 @@ "zod": "4.3.6" }, "devDependencies": { + "@effect/atom-react": "catalog:", "@effect/vitest": "catalog:", + "@executor-js/react": "workspace:*", "@types/node": "catalog:", + "@types/react": "catalog:", "bun-types": "catalog:", "effect": "catalog:", + "react": "catalog:", "tsup": "catalog:", "vitest": "catalog:" }, "peerDependencies": { - "effect": "catalog:" + "@effect/atom-react": "catalog:", + "@executor-js/react": "workspace:*", + "effect": "catalog:", + "react": "catalog:" + }, + "peerDependenciesMeta": { + "@effect/atom-react": { + "optional": true + }, + "@executor-js/react": { + "optional": true + }, + "react": { + "optional": true + } } } diff --git a/packages/plugins/apps/src/api.ts b/packages/plugins/apps/src/api.ts index 551d9a00f..63014c607 100644 --- a/packages/plugins/apps/src/api.ts +++ b/packages/plugins/apps/src/api.ts @@ -13,7 +13,7 @@ export { type SelfHostAppsRuntime, type SelfHostAppsRuntimeOptions, } from "./plugin/self-host-runtime"; -export { type AppsRuntime } from "./plugin/runtime"; +export { type AppsRuntime, type GitHubCustomToolsSourceSummary } from "./plugin/runtime"; export { makeSqliteAppsStore } from "./backing/sqlite-apps-store"; export { BindingError, diff --git a/packages/plugins/apps/src/backing/sqlite-apps-store.ts b/packages/plugins/apps/src/backing/sqlite-apps-store.ts index de9480ec9..596a5192d 100644 --- a/packages/plugins/apps/src/backing/sqlite-apps-store.ts +++ b/packages/plugins/apps/src/backing/sqlite-apps-store.ts @@ -77,6 +77,21 @@ export const makeSqliteAppsStore = (options: SqliteAppsStoreOptions): AppsStore }, catch: (cause) => storageFail("getDescriptor failed", cause), }), + listDescriptors: (tenant) => + Effect.tryPromise({ + try: async () => { + await init(); + const res = await client.execute({ + sql: "SELECT descriptor, published_at FROM descriptors WHERE tenant = ? ORDER BY published_at DESC", + args: [tenant], + }); + return res.rows.map((row) => ({ + descriptor: JSON.parse(String(row.descriptor)) as AppDescriptor, + publishedAt: Number(row.published_at), + })); + }, + catch: (cause) => storageFail("listDescriptors failed", cause), + }), putScopeForConnection: (tenant, connectionName, scope) => Effect.tryPromise({ try: async () => { diff --git a/packages/plugins/apps/src/pipeline/descriptor.ts b/packages/plugins/apps/src/pipeline/descriptor.ts index 30636a7ae..f93238277 100644 --- a/packages/plugins/apps/src/pipeline/descriptor.ts +++ b/packages/plugins/apps/src/pipeline/descriptor.ts @@ -31,11 +31,18 @@ export interface IntegrationDecl { readonly integration: string; } +export interface SourceSkippedArtifact { + readonly path: string; + readonly reason: "not supported yet" | "unsupported file type" | "ignored"; +} + export interface GitHubSourceRef { readonly kind: "github"; readonly repo: string; readonly ref: string; readonly upstreamSha: string; + readonly connection?: string; + readonly skipped?: readonly SourceSkippedArtifact[]; } export type AppSourceRef = GitHubSourceRef; diff --git a/packages/plugins/apps/src/plugin/apps-plugin.ts b/packages/plugins/apps/src/plugin/apps-plugin.ts index 135f6a37f..6cf42a957 100644 --- a/packages/plugins/apps/src/plugin/apps-plugin.ts +++ b/packages/plugins/apps/src/plugin/apps-plugin.ts @@ -269,6 +269,7 @@ export const appsPlugin = definePlugin((options?: AppsPluginOptions) => { scope, repo, ref, + connection, token, baseUrl: configBaseUrl(integration?.config), }); @@ -279,6 +280,16 @@ export const appsPlugin = definePlugin((options?: AppsPluginOptions) => { return result; }), }), + tool({ + name: "list_github_sources", + description: "List synced GitHub repositories that publish custom tools.", + execute: (_args, { ctx }) => + Effect.gen(function* () { + const tenant = tenantFor(ctx); + const sources = yield* runtime.listGitHubSources(tenant); + return { sources }; + }), + }), ], }, ], diff --git a/packages/plugins/apps/src/plugin/runtime.ts b/packages/plugins/apps/src/plugin/runtime.ts index 2f3fe3884..31a96a85e 100644 --- a/packages/plugins/apps/src/plugin/runtime.ts +++ b/packages/plugins/apps/src/plugin/runtime.ts @@ -25,6 +25,7 @@ import { } from "./bindings"; import type { AppsStore } from "./store"; import { scopeAddress } from "../seams/scope-address"; +import type { GitHubSkippedArtifact } from "../source/github-source"; // --------------------------------------------------------------------------- // AppsRuntime: the substrate-neutral core for published custom tools. @@ -40,6 +41,19 @@ export interface AppsRuntimeDeps { readonly defaultTenant?: string; } +export interface GitHubCustomToolsSourceSummary { + readonly scope: string; + readonly repo: string; + readonly ref: string; + readonly connection?: string; + readonly upstreamSha: string; + readonly snapshotId: string; + readonly description?: string; + readonly publishedAt: string; + readonly tools: readonly string[]; + readonly skipped: readonly GitHubSkippedArtifact[]; +} + export interface AppsRuntime { readonly publish: (input: { readonly tenant?: string; @@ -53,6 +67,9 @@ export interface AppsRuntime { tenantOrScope: string, scope?: string, ) => Effect.Effect; + readonly listGitHubSources: ( + tenant?: string, + ) => Effect.Effect; /** Re-derive the published-descriptor pointer from the latest committed * snapshot. */ readonly repair: ( @@ -345,6 +362,37 @@ export const makeAppsRuntime = (deps: AppsRuntimeDeps): AppsRuntime => { ); }, + listGitHubSources: (tenantInput) => { + const tenant = resolveTenant(tenantInput); + return deps.store.listDescriptors(tenant).pipe( + Effect.orElseSucceed(() => []), + Effect.map((records) => + records.flatMap((record): GitHubCustomToolsSourceSummary[] => { + const { descriptor, publishedAt } = record; + const source = descriptor.source; + if (source?.kind !== "github") return []; + return [ + { + scope: descriptor.scope, + repo: source.repo, + ref: source.ref, + ...(source.connection ? { connection: source.connection } : {}), + upstreamSha: source.upstreamSha, + snapshotId: descriptor.snapshotId, + ...(descriptor.description ? { description: descriptor.description } : {}), + publishedAt: new Date(publishedAt).toISOString(), + tools: descriptor.tools.map((tool) => tool.name), + skipped: [ + ...(source.skipped ?? []), + ...(descriptor.skipped as readonly GitHubSkippedArtifact[]), + ], + }, + ]; + }), + ), + ); + }, + repair: (tenantOrScope, maybeScope) => { const { tenant, scope } = resolveTenantScope(tenantOrScope, maybeScope); return repairScope(tenant, scope); diff --git a/packages/plugins/apps/src/plugin/store.ts b/packages/plugins/apps/src/plugin/store.ts index f2abb39b6..47f693401 100644 --- a/packages/plugins/apps/src/plugin/store.ts +++ b/packages/plugins/apps/src/plugin/store.ts @@ -30,6 +30,11 @@ export const scopeConnectionCollection = definePluginStorageCollection("apps_sco }, }); +export interface AppDescriptorRecord { + readonly descriptor: AppDescriptor; + readonly publishedAt: number; +} + export interface AppsStore { readonly putDescriptor: ( tenant: string, @@ -40,6 +45,9 @@ export interface AppsStore { tenant: string, scope: string, ) => Effect.Effect; + readonly listDescriptors: ( + tenant: string, + ) => Effect.Effect; readonly putScopeForConnection: ( tenant: string, connectionName: string, @@ -92,5 +100,14 @@ export const makeAppsStore = (deps: AppsStoreDeps): AppsStore => { descriptors .get({ key: keyFor(tenant, scope) }) .pipe(Effect.map((entry) => entry?.data.descriptor ?? null)), + listDescriptors: (tenant) => + descriptors.list({ keyPrefix: `${tenant}:` }).pipe( + Effect.map((entries) => + entries.map((entry) => ({ + descriptor: entry.data.descriptor, + publishedAt: entry.data.publishedAt, + })), + ), + ), }; }; diff --git a/packages/plugins/apps/src/react/AddCustomToolsSource.tsx b/packages/plugins/apps/src/react/AddCustomToolsSource.tsx new file mode 100644 index 000000000..e1d0a2fc3 --- /dev/null +++ b/packages/plugins/apps/src/react/AddCustomToolsSource.tsx @@ -0,0 +1,163 @@ +import { useMemo, useState } from "react"; +import { useAtomValue } from "@effect/atom-react"; +import * as AsyncResult from "effect/unstable/reactivity/AsyncResult"; + +import { connectionsAllAtom } from "@executor-js/react/api/atoms"; +import { Alert, AlertDescription, AlertTitle } from "@executor-js/react/components/alert"; +import { Button } from "@executor-js/react/components/button"; +import { + CardStack, + CardStackContent, + CardStackEntryField, +} from "@executor-js/react/components/card-stack"; +import { FloatActions } from "@executor-js/react/components/float-actions"; +import { Input } from "@executor-js/react/components/input"; +import { NativeSelect, NativeSelectOption } from "@executor-js/react/components/native-select"; +import { toast } from "@executor-js/react/components/sonner"; +import { FormErrorAlert } from "@executor-js/react/lib/integration-add"; + +import { + consoleIntegrationHref, + formatSyncErrors, + githubConnections, + syncCustomToolSource, + syncStatusLabel, + validateGitHubRepo, +} from "./custom-tools-client"; + +export default function AddCustomToolsSource(props: { + readonly onComplete: (slug?: string) => void; + readonly onCancel: () => void; +}) { + const connectionsResult = useAtomValue(connectionsAllAtom); + const connections = useMemo( + () => + AsyncResult.isSuccess(connectionsResult) ? githubConnections(connectionsResult.value) : [], + [connectionsResult], + ); + const [repo, setRepo] = useState(""); + const [ref, setRef] = useState(""); + const [connection, setConnection] = useState(""); + const [repoError, setRepoError] = useState(null); + const [syncing, setSyncing] = useState(false); + const [syncError, setSyncError] = useState(null); + + const selectedConnection = connection || connections[0]?.address || ""; + const loadingConnections = !AsyncResult.isSuccess(connectionsResult); + const canSubmit = + repo.trim().length > 0 && selectedConnection.length > 0 && !syncing && !loadingConnections; + + const submit = async () => { + const validation = validateGitHubRepo(repo); + setRepoError(validation); + setSyncError(null); + if (validation || !selectedConnection) return; + + setSyncing(true); + try { + const result = await syncCustomToolSource({ + repo, + ref, + connection: selectedConnection, + }); + if (result.status === "failed") { + setSyncError(formatSyncErrors(result).join("\n") || "Sync failed."); + setSyncing(false); + return; + } + toast.success(syncStatusLabel(result)); + props.onComplete("apps"); + } catch (error) { + setSyncError(error instanceof Error ? error.message : "Failed to sync custom tools."); + setSyncing(false); + } + }; + + return ( +
+
+

Add custom tools

+

+ Sync a GitHub repository that defines tools with executor:app. +

+
+ + + + +
+ { + setRepo((event.target as HTMLInputElement).value); + setRepoError(null); + setSyncError(null); + }} + onBlur={() => setRepoError(validateGitHubRepo(repo))} + placeholder="owner/name" + className="font-mono text-sm" + aria-invalid={repoError ? true : undefined} + /> + {repoError &&

{repoError}

} +
+ + + + setRef((event.target as HTMLInputElement).value)} + placeholder="default branch" + className="font-mono text-sm" + /> + + + + {connections.length > 0 ? ( + setConnection((event.target as HTMLSelectElement).value)} + className="w-full font-mono text-sm" + disabled={syncing} + > + {connections.map((candidate) => ( + + {candidate.address} + + ))} + + ) : ( + + No GitHub connections + + + Connect GitHub first, then return here to sync a custom-tools repo.{" "} + + Open GitHub + + + + + )} + + + + + {syncError && } + + + + + +
+ ); +} diff --git a/packages/plugins/apps/src/react/CustomToolsAccountsPanel.tsx b/packages/plugins/apps/src/react/CustomToolsAccountsPanel.tsx new file mode 100644 index 000000000..0c8945984 --- /dev/null +++ b/packages/plugins/apps/src/react/CustomToolsAccountsPanel.tsx @@ -0,0 +1,344 @@ +import { useEffect, useMemo, useState, type ReactNode } from "react"; + +import { Alert, AlertDescription, AlertTitle } from "@executor-js/react/components/alert"; +import { Badge } from "@executor-js/react/components/badge"; +import { Button } from "@executor-js/react/components/button"; +import { + CardStack, + CardStackContent, + CardStackEntry, + CardStackEntryActions, + CardStackEntryContent, + CardStackEntryDescription, + CardStackEntryTitle, +} from "@executor-js/react/components/card-stack"; + +import type { GitHubCustomToolsSourceSummary, GitHubSyncResult } from "../api"; +import { + consoleIntegrationHref, + formatSyncErrors, + listCustomToolSources, + syncCustomToolSource, + syncStatusLabel, + toolDiff, +} from "./custom-tools-client"; + +type LoadState = + | { readonly status: "loading" } + | { readonly status: "error"; readonly message: string } + | { readonly status: "ready"; readonly sources: readonly GitHubCustomToolsSourceSummary[] }; + +interface SyncNotice { + readonly status: GitHubSyncResult["status"]; + readonly message: string; + readonly added: readonly string[]; + readonly removed: readonly string[]; + readonly errors: readonly string[]; +} + +const formatDate = (value: string): string => { + const date = new Date(value); + return Number.isNaN(date.getTime()) ? value : date.toLocaleString(); +}; + +export default function CustomToolsAccountsPanel() { + const [loadState, setLoadState] = useState({ status: "loading" }); + const [selectedScope, setSelectedScope] = useState(null); + const [syncingScope, setSyncingScope] = useState(null); + const [notice, setNotice] = useState(null); + + const loadSources = async () => { + setLoadState({ status: "loading" }); + try { + const result = await listCustomToolSources(); + setLoadState({ status: "ready", sources: result.sources }); + setSelectedScope((current) => current ?? result.sources[0]?.scope ?? null); + } catch (error) { + setLoadState({ + status: "error", + message: error instanceof Error ? error.message : "Failed to load custom tool sources.", + }); + } + }; + + useEffect(() => { + let active = true; + void (async () => { + try { + const result = await listCustomToolSources(); + if (!active) return; + setLoadState({ status: "ready", sources: result.sources }); + setSelectedScope(result.sources[0]?.scope ?? null); + } catch (error) { + if (!active) return; + setLoadState({ + status: "error", + message: error instanceof Error ? error.message : "Failed to load custom tool sources.", + }); + } + })(); + return () => { + active = false; + }; + }, []); + + const sources = loadState.status === "ready" ? loadState.sources : []; + const selected = useMemo( + () => sources.find((source) => source.scope === selectedScope) ?? sources[0] ?? null, + [sources, selectedScope], + ); + + const syncSelected = async (source: GitHubCustomToolsSourceSummary) => { + if (!source.connection) { + setNotice({ + status: "failed", + message: "Sync failed.", + added: [], + removed: [], + errors: ["This source does not have a recorded GitHub connection."], + }); + return; + } + setSyncingScope(source.scope); + setNotice(null); + const beforeTools = source.tools; + try { + const result = await syncCustomToolSource({ + repo: source.repo, + ref: source.ref, + connection: source.connection, + }); + const diff = + result.status === "failed" + ? { added: [], removed: [] } + : toolDiff(beforeTools, result.tools); + setNotice({ + status: result.status, + message: syncStatusLabel(result), + added: diff.added, + removed: diff.removed, + errors: formatSyncErrors(result), + }); + if (result.status !== "failed") { + await loadSources(); + setSelectedScope(source.scope); + } + } catch (error) { + setNotice({ + status: "failed", + message: "Sync failed.", + added: [], + removed: [], + errors: [error instanceof Error ? error.message : "Failed to sync custom tools."], + }); + } + setSyncingScope(null); + }; + + return ( +
+
+
+

Custom tools sources

+

+ GitHub repositories synced into the executor tool catalog. +

+
+ +
+ + {loadState.status === "loading" && ( +
+ Loading sources... +
+ )} + + {loadState.status === "error" && ( + void loadSources()} /> + )} + + {loadState.status === "ready" && sources.length === 0 && ( +
+

No custom tools sources

+

+ Add a GitHub repo to publish its tools. +

+ +
+ )} + + {loadState.status === "ready" && sources.length > 0 && ( +
+ + + {sources.map((source) => ( + + + + ))} + + + + {selected && ( + void syncSelected(selected)} + /> + )} +
+ )} +
+ ); +} + +function ErrorWithRetry(props: { readonly message: string; readonly onRetry: () => void }) { + return ( + + Failed to load sources + +
+

{props.message}

+ +
+ + + ); +} + +function SourceDetail(props: { + readonly source: GitHubCustomToolsSourceSummary; + readonly notice: SyncNotice | null; + readonly syncing: boolean; + readonly onSync: () => void; +}) { + const { source, notice } = props; + return ( +
+
+
+

{source.repo}

+

{source.scope}

+
+ +
+ +
+
+ + + + +
+ + {notice && ( + + {notice.message} + {(notice.added.length > 0 || notice.removed.length > 0 || notice.errors.length > 0) && ( + +
+ {notice.added.length > 0 &&

Added: {notice.added.join(", ")}

} + {notice.removed.length > 0 &&

Removed: {notice.removed.join(", ")}

} + {notice.errors.map((error) => ( +

{error}

+ ))} +
+ + )} + + )} + +
+ {source.tools.length > 0 ? ( +
+ {source.tools.map((tool) => ( + + {tool} + + ))} +
+ ) : ( +

No tools published.

+ )} +
+ +
+ {source.skipped.length > 0 ? ( +
+ {source.skipped.map((entry) => ( +
+ + {entry.path} + + {entry.reason} +
+ ))} +
+ ) : ( +

Nothing skipped.

+ )} +
+
+
+ ); +} + +function Field(props: { readonly label: string; readonly value: string; readonly mono?: boolean }) { + return ( +
+
{props.label}
+
+ {props.value} +
+
+ ); +} + +function Section(props: { readonly title: string; readonly children: ReactNode }) { + return ( +
+

{props.title}

+ {props.children} +
+ ); +} diff --git a/packages/plugins/apps/src/react/custom-tools-client.ts b/packages/plugins/apps/src/react/custom-tools-client.ts new file mode 100644 index 000000000..d6ef462e6 --- /dev/null +++ b/packages/plugins/apps/src/react/custom-tools-client.ts @@ -0,0 +1,113 @@ +import type { Connection } from "@executor-js/sdk/shared"; + +import type { GitHubCustomToolsSourceSummary, GitHubSyncResult } from "../api"; + +export const CUSTOM_TOOLS_PLUGIN_KEY = "apps"; +export const CUSTOM_TOOLS_LABEL = "Custom tools"; + +export interface GitHubSourcesListResponse { + readonly sources: readonly GitHubCustomToolsSourceSummary[]; +} + +export interface SyncGitHubSourceRequest { + readonly repo: string; + readonly ref?: string; + readonly connection: string; +} + +export class CustomToolsClientError extends Error { + constructor(message: string) { + super(message); + this.name = "CustomToolsClientError"; + } +} + +const REPO_RE = /^[A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+$/; + +export const validateGitHubRepo = (repo: string): string | null => { + const trimmed = repo.trim(); + if (trimmed.length === 0) return "Enter a GitHub repo."; + if (!REPO_RE.test(trimmed)) return "Use owner/name, for example UsefulSoftwareCo/executor."; + return null; +}; + +export const githubConnections = (connections: readonly Connection[]): readonly Connection[] => + connections.filter((connection) => String(connection.integration) === "github"); + +const parseJsonResponse = async (response: Response, fallback: string): Promise => { + const text = await response.text(); + const body = text.length > 0 ? (JSON.parse(text) as unknown) : null; + if (!response.ok) { + const message = + body && + typeof body === "object" && + "error" in body && + typeof (body as { error?: unknown }).error === "string" + ? (body as { error: string }).error + : fallback; + throw new CustomToolsClientError(message); + } + return body as A; +}; + +export const listCustomToolSources = async ( + fetchImpl: typeof fetch = fetch, +): Promise => { + const response = await fetchImpl("/api/apps/sources/github", { + credentials: "same-origin", + }); + return parseJsonResponse(response, "Failed to load custom tools."); +}; + +export const syncCustomToolSource = async ( + input: SyncGitHubSourceRequest, + fetchImpl: typeof fetch = fetch, +): Promise => { + const response = await fetchImpl("/api/apps/sources/github/sync", { + method: "POST", + headers: { "content-type": "application/json" }, + credentials: "same-origin", + body: JSON.stringify({ + repo: input.repo.trim(), + ...(input.ref?.trim() ? { ref: input.ref.trim() } : {}), + connection: input.connection, + }), + }); + return parseJsonResponse(response, "Failed to sync custom tools."); +}; + +export const syncStatusLabel = (result: GitHubSyncResult): string => { + if (result.status === "published") return `Published ${result.tools.length} tools.`; + if (result.status === "up-to-date") return "Already up to date."; + return "Sync failed."; +}; + +export const formatSyncErrors = (result: GitHubSyncResult): readonly string[] => { + if (result.status !== "failed") return []; + return result.errors.map((error) => { + const details = error.diagnostics?.map((d) => `${d.path}: ${d.message}`).join("; "); + return details + ? `${error.stage}: ${error.message} (${details})` + : `${error.stage}: ${error.message}`; + }); +}; + +export const toolDiff = ( + before: readonly string[], + after: readonly string[], +): { readonly added: readonly string[]; readonly removed: readonly string[] } => { + const beforeSet = new Set(before); + const afterSet = new Set(after); + return { + added: after.filter((tool) => !beforeSet.has(tool)), + removed: before.filter((tool) => !afterSet.has(tool)), + }; +}; + +export const consoleIntegrationHref = (path: string): string => { + if (typeof window === "undefined") return path; + const marker = "/integrations"; + const index = window.location.pathname.indexOf(marker); + const prefix = index === -1 ? "" : window.location.pathname.slice(0, index); + return `${prefix}${path}`; +}; diff --git a/packages/plugins/apps/src/react/plugin-client.tsx b/packages/plugins/apps/src/react/plugin-client.tsx new file mode 100644 index 000000000..d3d2ba8a0 --- /dev/null +++ b/packages/plugins/apps/src/react/plugin-client.tsx @@ -0,0 +1,11 @@ +import { defineClientPlugin } from "@executor-js/sdk/client"; + +import { appsIntegrationPlugin } from "./source-plugin"; + +export { appsIntegrationPlugin } from "./source-plugin"; +export * from "./custom-tools-client"; + +export default defineClientPlugin({ + id: "apps", + integrationPlugin: appsIntegrationPlugin, +}); diff --git a/packages/plugins/apps/src/react/source-plugin.ts b/packages/plugins/apps/src/react/source-plugin.ts new file mode 100644 index 000000000..950409ab2 --- /dev/null +++ b/packages/plugins/apps/src/react/source-plugin.ts @@ -0,0 +1,18 @@ +import { lazy } from "react"; +import type { IntegrationPlugin } from "@executor-js/sdk/client"; + +import { CUSTOM_TOOLS_LABEL, CUSTOM_TOOLS_PLUGIN_KEY } from "./custom-tools-client"; + +const importAdd = () => import("./AddCustomToolsSource"); +const importAccounts = () => import("./CustomToolsAccountsPanel"); + +export const appsIntegrationPlugin: IntegrationPlugin = { + key: CUSTOM_TOOLS_PLUGIN_KEY, + label: CUSTOM_TOOLS_LABEL, + add: lazy(importAdd), + accounts: lazy(importAccounts), + preload: () => { + void importAdd(); + void importAccounts(); + }, +}; diff --git a/packages/plugins/apps/src/source/github-source.test.ts b/packages/plugins/apps/src/source/github-source.test.ts index 2c321db71..fbb0e042c 100644 --- a/packages/plugins/apps/src/source/github-source.test.ts +++ b/packages/plugins/apps/src/source/github-source.test.ts @@ -159,6 +159,10 @@ describe("GitHub custom-tools source", () => { repo: "acme/tools", ref: "main", upstreamSha: "commit-a", + skipped: [ + { path: "workflows/deferred.ts", reason: "not supported yet" }, + { path: "docs/readme.md", reason: "ignored" }, + ], }); await host.close(); }); diff --git a/packages/plugins/apps/src/source/github-source.ts b/packages/plugins/apps/src/source/github-source.ts index 208b1d10c..e5ab57be3 100644 --- a/packages/plugins/apps/src/source/github-source.ts +++ b/packages/plugins/apps/src/source/github-source.ts @@ -2,7 +2,7 @@ import { Data, Effect, Result } from "effect"; import { PublishError, type FileDiagnostic } from "../pipeline/discover"; import { PUBLISH_LIMITS, enforcePublishLimits } from "../pipeline/publish"; -import type { AppSourceRef } from "../pipeline/descriptor"; +import type { AppSourceRef, SourceSkippedArtifact } from "../pipeline/descriptor"; import type { AppsRuntime } from "../plugin/runtime"; import type { FileSet, SnapshotId } from "../seams/artifact-store"; @@ -36,10 +36,7 @@ export interface SyncErrorData { readonly diagnostics?: readonly FileDiagnostic[]; } -export interface GitHubSkippedArtifact { - readonly path: string; - readonly reason: "not supported yet" | "unsupported file type" | "ignored"; -} +export type GitHubSkippedArtifact = SourceSkippedArtifact; export type GitHubSyncResult = | { @@ -69,6 +66,7 @@ export interface SyncGitHubSourceInput extends GitHubSourceInput { readonly runtime: AppsRuntime; readonly tenant?: string; readonly scope: string; + readonly connection?: string; } const TOOL_RE = /^tools\/([a-z0-9][a-z0-9-]*)\.(ts|tsx|js|jsx)$/; @@ -374,11 +372,13 @@ const sourceErrorToSyncError = (error: GitHubSourceError): SyncErrorData => ({ diagnostics: error.path ? [{ path: error.path, message: error.message }] : [], }); -const sourceRef = (snapshot: GitHubSourceSnapshot): AppSourceRef => ({ +const sourceRef = (snapshot: GitHubSourceSnapshot, connection?: string): AppSourceRef => ({ kind: "github", repo: snapshot.repo, ref: snapshot.ref, upstreamSha: snapshot.upstreamSha, + ...(connection ? { connection } : {}), + skipped: snapshot.skipped, }); export const syncGitHubSource = (input: SyncGitHubSourceInput): Effect.Effect => @@ -420,7 +420,7 @@ export const syncGitHubSource = (input: SyncGitHubSourceInput): Effect.Effect; } => { const descriptors = new Map(); + const publishedAt = new Map(); const scopeConnections = new Map(); const keyFor = (tenant: string, key: string): string => `${tenant}:${key}`; return { descriptors, putDescriptor: (tenant, _owner, descriptor) => - Effect.sync(() => void descriptors.set(keyFor(tenant, descriptor.scope), descriptor)), + Effect.sync(() => { + const key = keyFor(tenant, descriptor.scope); + descriptors.set(key, descriptor); + publishedAt.set(key, Date.now()); + }), getDescriptor: (tenant, scope) => Effect.sync(() => descriptors.get(keyFor(tenant, scope)) ?? null), + listDescriptors: (tenant) => + Effect.sync(() => + [...descriptors.entries()] + .filter(([key]) => key.startsWith(`${tenant}:`)) + .map(([key, descriptor]) => ({ + descriptor, + publishedAt: publishedAt.get(key) ?? 0, + })), + ), putScopeForConnection: (tenant, connectionName, scope) => Effect.sync(() => void scopeConnections.set(keyFor(tenant, connectionName), scope)), getScopeForConnection: (tenant, connectionName) => diff --git a/packages/react/src/pages/integration-detail.tsx b/packages/react/src/pages/integration-detail.tsx index 7462543c0..e331198e9 100644 --- a/packages/react/src/pages/integration-detail.tsx +++ b/packages/react/src/pages/integration-detail.tsx @@ -153,8 +153,9 @@ export function IntegrationDetailPage(props: { namespace: string }) { // Find the plugin edit component based on integration kind const editPlugin = useMemo(() => { if (!integrationData) return null; + if (namespace === "apps") return integrationPlugins.find((p) => p.key === "apps") ?? null; return integrationPlugins.find((p) => p.key === integrationData.kind) ?? null; - }, [integrationData, integrationPlugins]); + }, [integrationData, integrationPlugins, namespace]); // Policies are pre-sorted by the server in evaluation order (owner rank, then // position ASC). The matcher walks the list and stops at the first hit per From f754da549606a40d2b11d7fd1690e5103d2fd65e Mon Sep 17 00:00:00 2001 From: Rhys Sullivan <39114868+RhysSullivan@users.noreply.github.com> Date: Mon, 6 Jul 2026 06:41:28 -0700 Subject: [PATCH 09/21] Make custom tools lint clean --- apps/host-selfhost/src/apps-resolver.ts | 78 ++++----- apps/host-selfhost/src/apps-route.ts | 34 ++-- apps/host-selfhost/src/apps-wire.node.test.ts | 88 ++++++---- .../src/custom-tools-console.web.test.ts | 17 +- packages/plugins/apps/CHANGELOG.md | 1 + .../backing/git-artifact-store.cas.test.ts | 2 +- .../apps/src/backing/git-artifact-store.ts | 20 ++- .../apps/src/backing/libsql-scope-db.ts | 2 +- .../src/backing/quickjs-tool-sandbox.test.ts | 15 +- .../apps/src/backing/quickjs-tool-sandbox.ts | 78 ++++----- .../apps/src/backing/scope-collision.test.ts | 2 +- .../apps/src/backing/sqlite-apps-store.ts | 45 +++-- packages/plugins/apps/src/pipeline/bundle.ts | 47 ++++-- .../plugins/apps/src/pipeline/descriptor.ts | 2 +- .../plugins/apps/src/pipeline/publish.test.ts | 20 +-- packages/plugins/apps/src/pipeline/publish.ts | 119 +++++++------- .../apps/src/plugin/apps-plugin.test.ts | 2 +- .../plugins/apps/src/plugin/apps-plugin.ts | 46 +++--- .../apps/src/plugin/bindings.strict.test.ts | 6 +- packages/plugins/apps/src/plugin/bindings.ts | 88 +++++----- .../src/plugin/publish-concurrency.test.ts | 13 +- .../plugins/apps/src/plugin/runtime.test.ts | 24 ++- packages/plugins/apps/src/plugin/runtime.ts | 42 +++-- .../apps/src/plugin/tenant-isolation.test.ts | 2 +- .../apps/src/react/AddCustomToolsSource.tsx | 30 ++-- .../src/react/CustomToolsAccountsPanel.tsx | 85 +++++----- .../apps/src/react/custom-tools-client.ts | 132 +++++++++------ .../src/seams/artifact-store.conformance.ts | 2 +- .../apps/src/seams/scope-db.conformance.ts | 2 +- .../src/seams/tool-sandbox.conformance.ts | 10 +- .../apps/src/source/github-source.test.ts | 6 +- .../plugins/apps/src/source/github-source.ts | 155 +++++++++--------- packages/plugins/apps/src/testing/index.ts | 15 +- 33 files changed, 681 insertions(+), 549 deletions(-) create mode 100644 packages/plugins/apps/CHANGELOG.md diff --git a/apps/host-selfhost/src/apps-resolver.ts b/apps/host-selfhost/src/apps-resolver.ts index 2080c1219..ed1715625 100644 --- a/apps/host-selfhost/src/apps-resolver.ts +++ b/apps/host-selfhost/src/apps-resolver.ts @@ -6,7 +6,6 @@ import { ToolAddress, isToolResult, type ConnectionRef, - type ExecuteError, type Owner, type PluginCtx, } from "@executor-js/sdk"; @@ -29,22 +28,6 @@ const parseConnectionAddress = (address: string): ConnectionRef | null => { }; }; -const bindingError = (input: { - readonly integration: string; - readonly connection?: string; - readonly message: string; - readonly cause?: unknown; -}): BindingError => - new BindingError({ - role: input.integration, - integration: input.integration, - requestedConnection: input.connection, - message: input.message, - }); - -const executeErrorMessage = (cause: ExecuteError): string => - typeof cause.message === "string" && cause.message.length > 0 ? cause.message : cause._tag; - const toCandidate = (connection: { readonly address: unknown; readonly integration: unknown; @@ -61,12 +44,13 @@ export const makeSelfHostAppsResolver = (input: { readonly ctx: PluginCtx }): Cl listConnections: ({ integration }) => input.ctx.connections.list({ integration: IntegrationSlug.make(integration) }).pipe( Effect.map((connections) => connections.map(toCandidate)), - Effect.mapError((cause) => - bindingError({ - integration, - message: `failed to list ${integration} connections`, - cause, - }), + Effect.mapError( + () => + new BindingError({ + role: integration, + integration, + message: `failed to list ${integration} connections`, + }), ), ), @@ -75,13 +59,14 @@ export const makeSelfHostAppsResolver = (input: { readonly ctx: PluginCtx }): Cl const ref = parseConnectionAddress(connection); if (!ref) return null; const row = yield* input.ctx.connections.get(ref).pipe( - Effect.mapError((cause) => - bindingError({ - integration: String(ref.integration), - connection, - message: `failed to resolve connection ${connection}`, - cause, - }), + Effect.mapError( + () => + new BindingError({ + role: String(ref.integration), + integration: String(ref.integration), + requestedConnection: connection, + message: `failed to resolve connection ${connection}`, + }), ), ); return row ? toCandidate(row) : null; @@ -91,16 +76,18 @@ export const makeSelfHostAppsResolver = (input: { readonly ctx: PluginCtx }): Cl Effect.gen(function* () { const ref = parseConnectionAddress(connection); if (!ref) { - return yield* bindingError({ + return yield* new BindingError({ + role: integration, integration, - connection, + requestedConnection: connection, message: `invalid connection address ${connection}`, }); } if (String(ref.integration) !== integration) { - return yield* bindingError({ + return yield* new BindingError({ + role: integration, integration, - connection, + requestedConnection: connection, message: `connection "${connection}" belongs to integration "${ref.integration}", not "${integration}"`, }); } @@ -108,21 +95,24 @@ export const makeSelfHostAppsResolver = (input: { readonly ctx: PluginCtx }): Cl const address = ToolAddress.make(`${connection}.${tool}`); const payload = args[0] ?? {}; const result = yield* input.ctx.execute(address, payload, invokeOptions).pipe( - Effect.mapError((cause) => - bindingError({ - integration, - connection, - message: executeErrorMessage(cause), - cause, - }), + Effect.mapError( + () => + new BindingError({ + role: integration, + integration, + requestedConnection: connection, + message: `failed to execute ${address}`, + }), ), ); if (isToolResult(result)) { if (result.ok) return result.data; - return yield* bindingError({ + const { message } = result.error; + return yield* new BindingError({ + role: integration, integration, - connection, - message: result.error.message, + requestedConnection: connection, + message, }); } return result; diff --git a/apps/host-selfhost/src/apps-route.ts b/apps/host-selfhost/src/apps-route.ts index 58a0ecfca..ef886ceca 100644 --- a/apps/host-selfhost/src/apps-route.ts +++ b/apps/host-selfhost/src/apps-route.ts @@ -1,5 +1,5 @@ import { HttpRouter, HttpServerRequest, HttpServerResponse } from "effect/unstable/http"; -import { Data, Effect, Layer } from "effect"; +import { Data, Effect, Layer, Predicate } from "effect"; import { IdentityProvider, @@ -19,6 +19,9 @@ class AppsSyncRouteError extends Data.TaggedError("AppsSyncRouteError")<{ readonly message: string; }> {} +const isAppsSyncRouteError = (failure: unknown): failure is AppsSyncRouteError => + Predicate.isTagged("AppsSyncRouteError")(failure); + const parseJson = (request: Request): Effect.Effect => Effect.tryPromise({ try: () => request.json() as Promise, @@ -50,29 +53,28 @@ const routeHandler = ( ): Effect.Effect => Effect.gen(function* () { const httpRequest = yield* HttpServerRequest.HttpServerRequest; - const request = yield* HttpServerRequest.toWeb(httpRequest).pipe(Effect.orDie); + const request = yield* HttpServerRequest.toWeb(httpRequest).pipe( + Effect.catch(() => Effect.succeed(null)), + ); + if (request === null) { + return HttpServerResponse.text("Internal Server Error", { status: 500 }); + } return yield* run(request).pipe( Effect.map((body) => HttpServerResponse.jsonUnsafe(body)), - Effect.catch((error: unknown) => { - const tag = - error && typeof error === "object" && "_tag" in error ? String(error._tag) : null; - if (tag === "Unauthorized") { + Effect.catch((failure: unknown) => { + if (Predicate.isTagged("Unauthorized")(failure)) { return Effect.succeed(HttpServerResponse.text("Unauthorized", { status: 401 })); } - if (tag === "NoOrganization") { + if (Predicate.isTagged("NoOrganization")(failure)) { return Effect.succeed(HttpServerResponse.text("Forbidden", { status: 403 })); } - if (tag === "Unavailable") { + if (Predicate.isTagged("Unavailable")(failure)) { return Effect.succeed(HttpServerResponse.text("Unavailable", { status: 503 })); } - if (tag === "AppsSyncRouteError") { - const routeError = error as AppsSyncRouteError; - return Effect.succeed( - HttpServerResponse.jsonUnsafe( - { error: routeError.message }, - { status: routeError.status }, - ), - ); + if (isAppsSyncRouteError(failure)) { + const message = failure.message; + const status = failure.status; + return Effect.succeed(HttpServerResponse.jsonUnsafe({ error: message }, { status })); } return Effect.succeed(HttpServerResponse.text("Internal Server Error", { status: 500 })); }), diff --git a/apps/host-selfhost/src/apps-wire.node.test.ts b/apps/host-selfhost/src/apps-wire.node.test.ts index 514457643..b95221a35 100644 --- a/apps/host-selfhost/src/apps-wire.node.test.ts +++ b/apps/host-selfhost/src/apps-wire.node.test.ts @@ -6,6 +6,7 @@ import { tmpdir } from "node:os"; import { join } from "node:path"; import { afterAll, beforeAll, expect, test } from "@effect/vitest"; +import { Effect, Schema } from "effect"; import { createEmulator, type Emulator } from "@executor-js/emulate"; const dataDir = mkdtempSync(join(tmpdir(), "eh-apps-wire-")); @@ -69,16 +70,30 @@ const jsonHeaders = { "x-test-org": TEST_ORG, }; +const decodeJsonText = Schema.decodeUnknownEffect(Schema.UnknownFromJsonString); + +const parseJsonText = async (text: string): Promise => { + if (text.length === 0) return null as T; + const value = await Effect.runPromise(decodeJsonText(text)); + return value as T; +}; + const findFreePort = (): Promise => - new Promise((resolve, reject) => { - const probe = createNetServer(); - probe.once("error", reject); - probe.listen(0, "127.0.0.1", () => { - const address = probe.address(); - const port = typeof address === "object" && address ? address.port : 0; - probe.close((error) => (error ? reject(error) : resolve(port))); - }); - }); + Effect.runPromise( + Effect.callback((resume) => { + const probe = createNetServer(); + probe.once("error", () => resume(Effect.fail("failed to find a free port"))); + probe.listen(0, "127.0.0.1", () => { + const address = probe.address(); + const port = typeof address === "object" && address ? address.port : 0; + probe.close((error) => + error + ? resume(Effect.fail("failed to close free-port probe")) + : resume(Effect.succeed(port)), + ); + }); + }), + ); const readRequestBody = async (request: IncomingMessage): Promise => { if (request.method === "GET" || request.method === "HEAD") return undefined; @@ -118,29 +133,42 @@ const startHttpServer = async ( ): Promise => { let port = 0; const server = createServer((request, response) => { - void (async () => { - try { - await sendWebResponse(await handler(await webRequestFromNode(request, port)), response); - } catch (error) { - response.statusCode = 500; - response.end(error instanceof Error ? error.message : String(error)); - } - })(); - }); - await new Promise((resolve, reject) => { - server.once("error", reject); - server.listen(0, "127.0.0.1", () => { - const address = server.address(); - port = typeof address === "object" && address ? address.port : 0; - resolve(); - }); + void Effect.runPromise( + Effect.tryPromise({ + try: async () => { + await sendWebResponse(await handler(await webRequestFromNode(request, port)), response); + }, + catch: () => "request failed", + }).pipe( + Effect.catch(() => + Effect.sync(() => { + response.statusCode = 500; + response.end("request failed"); + }), + ), + ), + ); }); + await Effect.runPromise( + Effect.callback((resume) => { + server.once("error", () => resume(Effect.fail("failed to start test server"))); + server.listen(0, "127.0.0.1", () => { + const address = server.address(); + port = typeof address === "object" && address ? address.port : 0; + resume(Effect.void); + }); + }), + ); return { baseUrl: `http://127.0.0.1:${port}`, close: () => - new Promise((resolve, reject) => { - server.close((error) => (error ? reject(error) : resolve())); - }), + Effect.runPromise( + Effect.callback((resume) => { + server.close((error) => + error ? resume(Effect.fail("failed to close test server")) : resume(Effect.void), + ); + }), + ), }; }; @@ -152,7 +180,7 @@ const requestJson = async ( const response = await fetch(`${server.baseUrl}${path}`, init); const text = await response.text(); expect(response.status).toBe(expectedStatus); - return (text.length > 0 ? JSON.parse(text) : null) as T; + return parseJsonText(text); }; const postJson = (path: string, body: unknown, expectedStatus = 200): Promise => @@ -179,7 +207,7 @@ const githubFetch = async ( const response = await fetch(`${emulator.url}${path}`, { ...init, headers }); const text = await response.text(); expect(response.ok).toBe(true); - return (text.length > 0 ? JSON.parse(text) : null) as T; + return parseJsonText(text); }; const createIssue = ( diff --git a/apps/host-selfhost/src/custom-tools-console.web.test.ts b/apps/host-selfhost/src/custom-tools-console.web.test.ts index 3c1ee0511..ebff9aa5e 100644 --- a/apps/host-selfhost/src/custom-tools-console.web.test.ts +++ b/apps/host-selfhost/src/custom-tools-console.web.test.ts @@ -1,8 +1,9 @@ -import { describe, expect, it, vi } from "@effect/vitest"; +import { describe, expect, it } from "@effect/vitest"; import appsClientPlugin, { CUSTOM_TOOLS_LABEL, CUSTOM_TOOLS_PLUGIN_KEY, + type CustomToolsFetch, appsIntegrationPlugin, formatSyncErrors, listCustomToolSources, @@ -42,7 +43,7 @@ describe("custom tools console client", () => { }); it("surfaces successful sync and source detail data", async () => { - const fetchImpl = vi.fn(async (input: RequestInfo | URL) => { + const fetchImpl: CustomToolsFetch = async (input) => { const url = String(input); if (url.endsWith("/sync")) { return jsonResponse({ @@ -68,7 +69,7 @@ describe("custom tools console client", () => { }, ], }); - }) as unknown as typeof fetch; + }; const syncResult = await syncCustomToolSource( { @@ -85,7 +86,7 @@ describe("custom tools console client", () => { }); it("renders failed sync errors readably", async () => { - const fetchImpl = vi.fn(async () => + const fetchImpl: CustomToolsFetch = async () => jsonResponse({ status: "failed", tools: [], @@ -97,8 +98,7 @@ describe("custom tools console client", () => { diagnostics: [{ path: "tools/x.ts", message: "vendor: nope" }], }, ], - }), - ) as unknown as typeof fetch; + }); const result = await syncCustomToolSource( { @@ -115,14 +115,13 @@ describe("custom tools console client", () => { }); it("shows the up-to-date sync state", async () => { - const fetchImpl = vi.fn(async () => + const fetchImpl: CustomToolsFetch = async () => jsonResponse({ status: "up-to-date", upstreamSha: "abc123", tools: ["repo-summary"], skipped: [], - }), - ) as unknown as typeof fetch; + }); const result = await syncCustomToolSource( { diff --git a/packages/plugins/apps/CHANGELOG.md b/packages/plugins/apps/CHANGELOG.md new file mode 100644 index 000000000..3605a25c4 --- /dev/null +++ b/packages/plugins/apps/CHANGELOG.md @@ -0,0 +1 @@ +# @executor-js/plugin-apps diff --git a/packages/plugins/apps/src/backing/git-artifact-store.cas.test.ts b/packages/plugins/apps/src/backing/git-artifact-store.cas.test.ts index 06f71abfa..20e28f8ba 100644 --- a/packages/plugins/apps/src/backing/git-artifact-store.cas.test.ts +++ b/packages/plugins/apps/src/backing/git-artifact-store.cas.test.ts @@ -2,7 +2,7 @@ import { mkdtempSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; -import { describe, expect, it } from "vitest"; +import { describe, expect, it } from "@effect/vitest"; import { Effect } from "effect"; import { makeGitArtifactStore } from "./git-artifact-store"; diff --git a/packages/plugins/apps/src/backing/git-artifact-store.ts b/packages/plugins/apps/src/backing/git-artifact-store.ts index ea2e39511..6da3e8034 100644 --- a/packages/plugins/apps/src/backing/git-artifact-store.ts +++ b/packages/plugins/apps/src/backing/git-artifact-store.ts @@ -30,6 +30,14 @@ const BRANCH = "refs/heads/main"; // scope repo). git treats the empty string as "the ref must not already exist". const EMPTY_OID = ""; +const stderrText = (stderr: string | Buffer): string => + Buffer.isBuffer(stderr) ? stderr.toString("utf8").trim() : String(stderr).trim(); + +const gitFailureMessage = (command: string, stderr: string | Buffer): string => { + const detail = stderrText(stderr); + return detail ? `git ${command} failed: ${detail}` : `git ${command} failed`; +}; + const run = ( cwd: string, args: readonly string[], @@ -45,7 +53,7 @@ const run = ( resume( Effect.fail( new ArtifactStoreError({ - message: `git ${args[0]} failed: ${(stderr as Buffer)?.toString() || error.message}`, + message: gitFailureMessage(args[0] ?? "command", stderr), cause: error, }), ), @@ -114,7 +122,7 @@ const makeScopeStore = (repoDir: string): ScopeArtifactStore => { resume( Effect.fail( new ArtifactStoreError({ - message: `git ${args[0]} failed: ${(stderr as Buffer)?.toString() || error.message}`, + message: gitFailureMessage(args[0] ?? "command", stderr), cause: error, }), ), @@ -159,7 +167,7 @@ const makeScopeStore = (repoDir: string): ScopeArtifactStore => { resume( Effect.fail( new ArtifactStoreError({ - message: `git commit-tree failed: ${(stderr as Buffer)?.toString() || error.message}`, + message: gitFailureMessage("commit-tree", stderr), cause: error, }), ), @@ -271,11 +279,7 @@ export const makeGitArtifactStore = (options: GitArtifactStoreOptions): Artifact const init = async (key: string): Promise => { const repoDir = join(options.root, `${key}.git`); await mkdir(repoDir, { recursive: true }); - await new Promise((resolve, reject) => { - execFile("git", ["init", "--bare", "--quiet"], { cwd: repoDir }, (error) => - error ? reject(error) : resolve(), - ); - }); + await Effect.runPromise(run(repoDir, ["init", "--bare", "--quiet"])); return makeScopeStore(repoDir); }; diff --git a/packages/plugins/apps/src/backing/libsql-scope-db.ts b/packages/plugins/apps/src/backing/libsql-scope-db.ts index 3fba945b1..fc1673d10 100644 --- a/packages/plugins/apps/src/backing/libsql-scope-db.ts +++ b/packages/plugins/apps/src/backing/libsql-scope-db.ts @@ -86,7 +86,7 @@ const makeHandle = (address: ScopeAddress, client: Client): ScopeDbHandle => { await bump(targets); } } - return result.rows as unknown as readonly Row[]; + return result.rows.map((row) => row as Row); }, catch: (cause) => new ScopeDbError({ message: `scope-db statement failed for scope ${label}`, cause }), diff --git a/packages/plugins/apps/src/backing/quickjs-tool-sandbox.test.ts b/packages/plugins/apps/src/backing/quickjs-tool-sandbox.test.ts index 237f99718..01488e0cb 100644 --- a/packages/plugins/apps/src/backing/quickjs-tool-sandbox.test.ts +++ b/packages/plugins/apps/src/backing/quickjs-tool-sandbox.test.ts @@ -1,5 +1,5 @@ -import { describe, expect, it } from "vitest"; -import { Effect } from "effect"; +import { describe, expect, it } from "@effect/vitest"; +import { Effect, Exit } from "effect"; import { bundleEntry } from "../pipeline/bundle"; import { makeQuickjsToolSandbox } from "./quickjs-tool-sandbox"; @@ -100,10 +100,9 @@ export default defineTool({ ), ); - expect(exit._tag).toBe("Failure"); - if (exit._tag !== "Failure") throw new Error("expected validation failure"); - expect(JSON.stringify(exit.cause)).toContain("InputValidationError"); - expect(JSON.stringify(exit.cause)).toContain("q"); + expect(Exit.isFailure(exit)).toBe(true); + expect(JSON.stringify(exit)).toContain("InputValidationError"); + expect(JSON.stringify(exit)).toContain("q"); }); it("invokes raw JSON Schema tools without sandbox Standard Schema validation", async () => { @@ -139,7 +138,7 @@ export default defineTool({ const files = new Map([["tools/rng.ts", nondeterministic]]); const { code } = await run(bundleEntry({ files, entry: "tools/rng.ts" })); const exit = await Effect.runPromiseExit(sandbox.collect(code)); - expect(exit._tag).toBe("Failure"); + expect(Exit.isFailure(exit)).toBe(true); }); it("denies network (fetch throws in the sandbox)", async () => { @@ -156,7 +155,7 @@ export default defineTool({ const exit = await Effect.runPromiseExit( sandbox.invoke(code, { artifact: "net", kind: "tool", input: {}, roots: {} }, bridge), ); - expect(exit._tag).toBe("Failure"); + expect(Exit.isFailure(exit)).toBe(true); }); it("returns typed InputValidationError from promise rejection", async () => { diff --git a/packages/plugins/apps/src/backing/quickjs-tool-sandbox.ts b/packages/plugins/apps/src/backing/quickjs-tool-sandbox.ts index ca0f4043c..2402a8dd7 100644 --- a/packages/plugins/apps/src/backing/quickjs-tool-sandbox.ts +++ b/packages/plugins/apps/src/backing/quickjs-tool-sandbox.ts @@ -1,4 +1,4 @@ -import { Effect } from "effect"; +import { Effect, Option, Predicate, Schema } from "effect"; import type { SandboxToolInvoker } from "@executor-js/codemode-core"; import { makeQuickJsExecutor } from "@executor-js/runtime-quickjs"; @@ -85,14 +85,13 @@ ${bundle} })(module, exports, require); `; +const decodeJsonMarker = Schema.decodeUnknownOption(Schema.UnknownFromJsonString); + +const isRecord = (value: unknown): value is Record => + value !== null && typeof value === "object" && !Array.isArray(value); + const parseMarker = (message: string): unknown => { - const parse = (text: string): unknown => { - try { - return JSON.parse(text); - } catch { - return null; - } - }; + const parse = (text: string): unknown => Option.getOrNull(decodeJsonMarker(text)); const direct = parse(message); if (direct) return direct; const firstBrace = message.indexOf("{"); @@ -104,15 +103,20 @@ const parseMarker = (message: string): unknown => { }; const isMarker = (value: unknown, tag: string): value is Record => { - if (value === null || typeof value !== "object") return false; - return (value as { readonly _tag?: unknown })._tag === tag; + if (!isRecord(value)) return false; + return Predicate.isTagged(tag)(value); }; const markerMessage = (marker: Record, fallback: string): string => typeof marker.message === "string" ? marker.message : fallback; const markerIssues = (marker: Record): readonly ValidationIssue[] => - Array.isArray(marker.issues) ? (marker.issues as readonly ValidationIssue[]) : []; + Array.isArray(marker.issues) ? marker.issues.filter(isValidationIssue) : []; + +const isValidationIssue = (value: unknown): value is ValidationIssue => + isRecord(value) && + typeof value.message === "string" && + (value.path === undefined || Array.isArray(value.path)); // Collect driver: describe the artifact's integrations + input/output schema. // Deterministic JSON only — no handler execution. @@ -236,7 +240,10 @@ const buildInvokeCode = (bundle: string, request: InvokeRequest): string => // A no-op invoker for collect: no handle calls should happen; if they do // (misbehaving describe path), fail loudly. const collectInvoker: SandboxToolInvoker = { - invoke: () => Effect.fail(new Error("collect must not make handle calls")) as never, + invoke: () => + Effect.fail( + new ToolSandboxError({ kind: "collect", message: "collect must not make handle calls" }), + ), }; export interface QuickjsToolSandboxOptions { @@ -286,13 +293,11 @@ export const makeQuickjsToolSandbox = (options: QuickjsToolSandboxOptions = {}): const a = stableStringify(first); const b = stableStringify(second); if (a !== b) { - return yield* Effect.fail( - new ToolSandboxError({ - kind: "nondeterministic", - message: - "descriptor collection is non-deterministic (an artifact read Math.random/Date.now or otherwise diverged between runs)", - }), - ); + return yield* new ToolSandboxError({ + kind: "nondeterministic", + message: + "descriptor collection is non-deterministic (an artifact read Math.random/Date.now or otherwise diverged between runs)", + }); } const descriptor = first as { artifact?: string }; const result: CollectResult = { @@ -318,8 +323,11 @@ export const makeQuickjsToolSandbox = (options: QuickjsToolSandboxOptions = {}): // through an unexpected channel. if (input.path !== "__handle__") { return Effect.fail( - new Error(`unexpected sandbox bridge path: ${input.path}`), - ) as never; + new ToolSandboxError({ + kind: "invoke", + message: `unexpected sandbox bridge path: ${input.path}`, + }), + ); } const call = input.args as { root: string; @@ -327,7 +335,9 @@ export const makeQuickjsToolSandbox = (options: QuickjsToolSandboxOptions = {}): args: readonly unknown[]; }; if (!call || typeof call.root !== "string" || !Array.isArray(call.path)) { - return Effect.fail(new Error("malformed sandbox bridge call")) as never; + return Effect.fail( + new ToolSandboxError({ kind: "invoke", message: "malformed sandbox bridge call" }), + ); } return bridge.call({ root: call.root, path: call.path, args: call.args }) as never; }, @@ -343,24 +353,18 @@ export const makeQuickjsToolSandbox = (options: QuickjsToolSandboxOptions = {}): if (result.error) { const marker = parseMarker(result.error); if (isMarker(marker, "InputValidationError")) { - return yield* Effect.fail( - new InputValidationError({ - message: markerMessage(marker, "input validation failed"), - issues: markerIssues(marker), - }), - ); + return yield* new InputValidationError({ + message: markerMessage(marker, "input validation failed"), + issues: markerIssues(marker), + }); } if (isMarker(marker, "OutputValidationError")) { - return yield* Effect.fail( - new OutputValidationError({ - message: markerMessage(marker, "output validation failed"), - issues: markerIssues(marker), - }), - ); + return yield* new OutputValidationError({ + message: markerMessage(marker, "output validation failed"), + issues: markerIssues(marker), + }); } - return yield* Effect.fail( - new ToolSandboxError({ kind: "invoke", message: result.error }), - ); + return yield* new ToolSandboxError({ kind: "invoke", message: result.error }); } return { output: result.result, logs: result.logs ?? [] } satisfies InvokeResult; }), diff --git a/packages/plugins/apps/src/backing/scope-collision.test.ts b/packages/plugins/apps/src/backing/scope-collision.test.ts index b3b725a8a..e4a9684a0 100644 --- a/packages/plugins/apps/src/backing/scope-collision.test.ts +++ b/packages/plugins/apps/src/backing/scope-collision.test.ts @@ -2,7 +2,7 @@ import { mkdtempSync, readdirSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; -import { describe, expect, it } from "vitest"; +import { describe, expect, it } from "@effect/vitest"; import { Effect } from "effect"; import { makeLibsqlScopeDb } from "./libsql-scope-db"; diff --git a/packages/plugins/apps/src/backing/sqlite-apps-store.ts b/packages/plugins/apps/src/backing/sqlite-apps-store.ts index 596a5192d..18faa59e3 100644 --- a/packages/plugins/apps/src/backing/sqlite-apps-store.ts +++ b/packages/plugins/apps/src/backing/sqlite-apps-store.ts @@ -2,9 +2,9 @@ import { mkdirSync } from "node:fs"; import { dirname, resolve } from "node:path"; import { createClient, type Client } from "@libsql/client"; -import { Effect } from "effect"; +import { Effect, Schema } from "effect"; -import type { StorageFailure } from "@executor-js/sdk"; +import { StorageError, type StorageFailure } from "@executor-js/sdk"; import type { AppDescriptor } from "../pipeline/descriptor"; import type { AppsStore } from "../plugin/store"; @@ -22,7 +22,18 @@ CREATE TABLE IF NOT EXISTS scope_connections (tenant TEXT NOT NULL, connection_n `; const storageFail = (message: string, cause: unknown): StorageFailure => - ({ _tag: "StorageError", message, cause }) as unknown as StorageFailure; + new StorageError({ message, cause }); + +const decodeDescriptorJson = Schema.decodeUnknownEffect(Schema.UnknownFromJsonString); + +const decodeDescriptor = ( + value: unknown, + message: string, +): Effect.Effect => + decodeDescriptorJson(String(value)).pipe( + Effect.map((descriptor) => descriptor as AppDescriptor), + Effect.mapError((cause) => storageFail(message, cause)), + ); export interface SqliteAppsStoreOptions { readonly path: string; @@ -73,10 +84,16 @@ export const makeSqliteAppsStore = (options: SqliteAppsStoreOptions): AppsStore args: [tenant, scope], }); const row = res.rows[0]; - return row ? (JSON.parse(String(row.descriptor)) as AppDescriptor) : null; + return row?.descriptor ?? null; }, catch: (cause) => storageFail("getDescriptor failed", cause), - }), + }).pipe( + Effect.flatMap((descriptor) => + descriptor === null + ? Effect.succeed(null) + : decodeDescriptor(descriptor, "getDescriptor failed"), + ), + ), listDescriptors: (tenant) => Effect.tryPromise({ try: async () => { @@ -85,13 +102,21 @@ export const makeSqliteAppsStore = (options: SqliteAppsStoreOptions): AppsStore sql: "SELECT descriptor, published_at FROM descriptors WHERE tenant = ? ORDER BY published_at DESC", args: [tenant], }); - return res.rows.map((row) => ({ - descriptor: JSON.parse(String(row.descriptor)) as AppDescriptor, - publishedAt: Number(row.published_at), - })); + return res.rows; }, catch: (cause) => storageFail("listDescriptors failed", cause), - }), + }).pipe( + Effect.flatMap((rows) => + Effect.forEach(rows, (row) => + decodeDescriptor(row.descriptor, "listDescriptors failed").pipe( + Effect.map((descriptor) => ({ + descriptor, + publishedAt: Number(row.published_at), + })), + ), + ), + ), + ), putScopeForConnection: (tenant, connectionName, scope) => Effect.tryPromise({ try: async () => { diff --git a/packages/plugins/apps/src/pipeline/bundle.ts b/packages/plugins/apps/src/pipeline/bundle.ts index b7d6c95f1..892f03bcf 100644 --- a/packages/plugins/apps/src/pipeline/bundle.ts +++ b/packages/plugins/apps/src/pipeline/bundle.ts @@ -1,6 +1,6 @@ import { build, version as esbuildVersion } from "esbuild"; -import { Effect } from "effect"; +import { Effect, Option, Schema } from "effect"; import { ToolSandboxError } from "../seams/tool-sandbox"; import type { ToolchainRef } from "./descriptor"; @@ -52,6 +52,25 @@ export interface BundleOutput { readonly code: string; } +const EsbuildDiagnostic = Schema.Struct({ + text: Schema.String, +}); + +const EsbuildFailure = Schema.Struct({ + errors: Schema.Array(EsbuildDiagnostic), +}); + +const decodeEsbuildFailure = Schema.decodeUnknownOption(EsbuildFailure); + +const bundleFailureMessage = (entry: string, cause: unknown): string => { + const decoded = Option.getOrNull(decodeEsbuildFailure(cause)); + const detail = decoded?.errors + .map((error) => error.text) + .filter((text) => text.length > 0) + .join("; "); + return detail ? `bundle failed for ${entry}: ${detail}` : `bundle failed for ${entry}`; +}; + const resolveRelative = (base: string, rel: string): string => { const parts = (base ? base.split("/") : []).concat(rel.split("/")); const out: string[] = []; @@ -166,8 +185,8 @@ const fileSetPlugin = (files: ReadonlyMap, authorEntry: string) /** Bundle one entry from the file set to a single CJS string. */ export const bundleEntry = (input: BundleInput): Effect.Effect => Effect.tryPromise({ - try: async () => { - const result = await build({ + try: () => + build({ entryPoints: [VIRTUAL_ENTRY], bundle: true, write: false, @@ -180,15 +199,23 @@ export const bundleEntry = (input: BundleInput): Effect.Effect new ToolSandboxError({ kind: "bundle", - message: `bundle failed for ${input.entry}: ${cause instanceof Error ? cause.message : String(cause)}`, + message: bundleFailureMessage(input.entry, cause), cause, }), - }); + }).pipe( + Effect.flatMap((result) => { + const out = result.outputFiles?.[0]?.text; + return out === undefined + ? Effect.fail( + new ToolSandboxError({ + kind: "bundle", + message: `bundle failed for ${input.entry}: esbuild produced no output`, + }), + ) + : Effect.succeed({ code: out }); + }), + ); diff --git a/packages/plugins/apps/src/pipeline/descriptor.ts b/packages/plugins/apps/src/pipeline/descriptor.ts index f93238277..0d92c5c50 100644 --- a/packages/plugins/apps/src/pipeline/descriptor.ts +++ b/packages/plugins/apps/src/pipeline/descriptor.ts @@ -108,7 +108,7 @@ export const stableStringify = (value: unknown): string => { const seen = new WeakSet(); const walk = (v: unknown): unknown => { if (v === null || typeof v !== "object") return v; - if (seen.has(v)) throw new Error("cyclic value cannot be serialized deterministically"); + if (seen.has(v)) return "[Circular]"; seen.add(v); if (Array.isArray(v)) return v.map(walk); const out: Record = {}; diff --git a/packages/plugins/apps/src/pipeline/publish.test.ts b/packages/plugins/apps/src/pipeline/publish.test.ts index b924b31d6..71fad6144 100644 --- a/packages/plugins/apps/src/pipeline/publish.test.ts +++ b/packages/plugins/apps/src/pipeline/publish.test.ts @@ -2,8 +2,8 @@ import { mkdtempSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; -import { describe, expect, it } from "vitest"; -import { Effect } from "effect"; +import { describe, expect, it } from "@effect/vitest"; +import { Effect, Exit } from "effect"; import { makeGitArtifactStore } from "../backing/git-artifact-store"; import { makeQuickjsToolSandbox } from "../backing/quickjs-tool-sandbox"; @@ -89,7 +89,7 @@ export default defineTool({ const exit = await Effect.runPromiseExit(publish(deps, { scope: "s", files })); - expect(exit._tag).toBe("Failure"); + expect(Exit.isFailure(exit)).toBe(true); expect(JSON.stringify(exit)).toContain("collides"); expect(JSON.stringify(exit)).toContain("crm"); }); @@ -117,7 +117,7 @@ export default defineTool({ const exit = await Effect.runPromiseExit(publish(deps, { scope: "s", files })); - expect(exit._tag).toBe("Failure"); + expect(Exit.isFailure(exit)).toBe(true); expect(JSON.stringify(exit)).toContain("vendor-without-json-schema"); expect(JSON.stringify(exit)).toContain("vendor"); expect(JSON.stringify(exit)).toContain("input"); @@ -155,10 +155,8 @@ export default defineTool({ ], ]); const exit = await Effect.runPromiseExit(publish(deps, { scope: "s", files })); - expect(exit._tag).toBe("Failure"); - if (exit._tag === "Failure") { - expect(JSON.stringify(exit.cause)).toContain("bundle"); - } + expect(Exit.isFailure(exit)).toBe(true); + expect(JSON.stringify(exit)).toContain("bundle"); }); it("rejects an oversized publish set (too many files) and persists nothing", async () => { @@ -168,7 +166,7 @@ export default defineTool({ files.set(`tools/t${i}.ts`, "// noop"); } const exit = await Effect.runPromiseExit(publish(deps, { scope: "s", files })); - expect(exit._tag).toBe("Failure"); + expect(Exit.isFailure(exit)).toBe(true); expect(JSON.stringify(exit)).toContain("exceeding the limit"); const latest = await run( Effect.flatMap(deps.artifactStore.forScope(scopeAddress("org", "s")), (s) => s.latest()), @@ -181,7 +179,7 @@ export default defineTool({ const big = "x".repeat(PUBLISH_LIMITS.maxFileBytes + 1); const files = new Map([["tools/big.ts", big]]); const exit = await Effect.runPromiseExit(publish(deps, { scope: "s", files })); - expect(exit._tag).toBe("Failure"); + expect(Exit.isFailure(exit)).toBe(true); expect(JSON.stringify(exit)).toContain("per-file limit"); }); @@ -192,7 +190,7 @@ export default defineTool({ const count = Math.ceil(PUBLISH_LIMITS.maxTotalBytes / PUBLISH_LIMITS.maxFileBytes) + 1; for (let i = 0; i < count; i++) files.set(`tools/t${i}.ts`, half); const exit = await Effect.runPromiseExit(publish(deps, { scope: "s", files })); - expect(exit._tag).toBe("Failure"); + expect(Exit.isFailure(exit)).toBe(true); expect(JSON.stringify(exit)).toContain("total limit"); }); }); diff --git a/packages/plugins/apps/src/pipeline/publish.ts b/packages/plugins/apps/src/pipeline/publish.ts index 7db6829c8..3fe8e74af 100644 --- a/packages/plugins/apps/src/pipeline/publish.ts +++ b/packages/plugins/apps/src/pipeline/publish.ts @@ -1,4 +1,4 @@ -import { Effect } from "effect"; +import { Effect, Predicate, Schema } from "effect"; import type { ArtifactStore, FileSet, SnapshotId } from "../seams/artifact-store"; import { scopeAddress } from "../seams/scope-address"; @@ -155,33 +155,39 @@ export interface PublishDeps { readonly sandbox: ToolSandbox; } +const isPublishError = Predicate.isTagged("PublishError") as ( + value: unknown, +) => value is PublishError; + +const decodeDescriptorJson = Schema.decodeUnknownEffect(Schema.UnknownFromJsonString); + const assemble = (deps: PublishDeps, files: FileSet): Effect.Effect => Effect.gen(function* () { const discovered = discover(files); - if (discovered instanceof PublishError) return yield* Effect.fail(discovered); + if (isPublishError(discovered)) return yield* discovered; const tools: ToolDescriptor[] = []; for (const artifact of discovered.artifacts) { const bundle = yield* bundleEntry({ files, entry: artifact.entry }).pipe( - Effect.mapError( - (cause) => - new PublishError({ - message: cause.message, - stage: "bundle", - diagnostics: [{ path: artifact.entry, message: cause.message }], - }), - ), + Effect.mapError((bundleFailure) => { + const message = bundleFailure.message; + return new PublishError({ + message, + stage: "bundle", + diagnostics: [{ path: artifact.entry, message }], + }); + }), ); const collected = yield* deps.sandbox.collect(bundle.code, { artifact: artifact.name }).pipe( - Effect.mapError( - (cause) => - new PublishError({ - message: cause.message, - stage: "collect", - diagnostics: [{ path: artifact.entry, message: cause.message }], - }), - ), + Effect.mapError((collectFailure) => { + const message = collectFailure.message; + return new PublishError({ + message, + stage: "collect", + diagnostics: [{ path: artifact.entry, message }], + }); + }), ); const raw = collected.artifacts.default?.descriptor as CollectedDescriptor | undefined; const keyed = collected.artifacts[artifact.name]?.descriptor as @@ -189,13 +195,11 @@ const assemble = (deps: PublishDeps, files: FileSet): Effect.Effect - new PublishError({ message: cause.message, stage: "project", diagnostics: [] }), - ), - ); + const scopeStore = yield* deps.artifactStore.forScope(scopeAddress(tenant, input.scope)).pipe( + Effect.mapError((storeFailure) => { + const message = storeFailure.message; + return new PublishError({ message, stage: "project", diagnostics: [] }); + }), + ); const filesWithDescriptor = new Map(input.files); filesWithDescriptor.set(DESCRIPTOR_SNAPSHOT_PATH, stableStringify(body)); @@ -267,10 +269,10 @@ export const publish = ( const meta = yield* scopeStore .commit(filesWithDescriptor, input.commitMessage ?? `publish ${new Date().toISOString()}`) .pipe( - Effect.mapError( - (cause) => - new PublishError({ message: cause.message, stage: "project", diagnostics: [] }), - ), + Effect.mapError((storeFailure) => { + const message = storeFailure.message; + return new PublishError({ message, stage: "project", diagnostics: [] }); + }), ); const descriptor: AppDescriptor = { ...body, snapshotId: meta.id }; @@ -285,21 +287,28 @@ export const loadDescriptorFromSnapshot = ( snapshotId: SnapshotId, ): Effect.Effect => Effect.gen(function* () { - const scopeStore = yield* store - .forScope(scopeAddress(tenant, scope)) - .pipe( - Effect.mapError( - (c) => new PublishError({ message: c.message, stage: "project", diagnostics: [] }), - ), - ); - const raw = yield* scopeStore - .readFile(snapshotId, DESCRIPTOR_SNAPSHOT_PATH) - .pipe( - Effect.mapError( - (c) => new PublishError({ message: c.message, stage: "project", diagnostics: [] }), - ), - ); + const scopeStore = yield* store.forScope(scopeAddress(tenant, scope)).pipe( + Effect.mapError((storeFailure) => { + const message = storeFailure.message; + return new PublishError({ message, stage: "project", diagnostics: [] }); + }), + ); + const raw = yield* scopeStore.readFile(snapshotId, DESCRIPTOR_SNAPSHOT_PATH).pipe( + Effect.mapError((storeFailure) => { + const message = storeFailure.message; + return new PublishError({ message, stage: "project", diagnostics: [] }); + }), + ); if (raw == null) return null; - const body = JSON.parse(raw) as Omit; - return { ...body, snapshotId }; + const body = yield* decodeDescriptorJson(raw).pipe( + Effect.mapError( + (_cause) => + new PublishError({ + message: "descriptor snapshot is not valid JSON", + stage: "project", + diagnostics: [], + }), + ), + ); + return { ...(body as Omit), snapshotId }; }); diff --git a/packages/plugins/apps/src/plugin/apps-plugin.test.ts b/packages/plugins/apps/src/plugin/apps-plugin.test.ts index fd507c53a..20bcd7bb2 100644 --- a/packages/plugins/apps/src/plugin/apps-plugin.test.ts +++ b/packages/plugins/apps/src/plugin/apps-plugin.test.ts @@ -2,7 +2,7 @@ import { mkdtempSync, readFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; -import { describe, expect, it } from "vitest"; +import { describe, expect, it } from "@effect/vitest"; import { Effect } from "effect"; import { appsPlugin, connectionNameForScope } from "./apps-plugin"; diff --git a/packages/plugins/apps/src/plugin/apps-plugin.ts b/packages/plugins/apps/src/plugin/apps-plugin.ts index 6cf42a957..728262e6e 100644 --- a/packages/plugins/apps/src/plugin/apps-plugin.ts +++ b/packages/plugins/apps/src/plugin/apps-plugin.ts @@ -20,6 +20,7 @@ import type { AppsRuntime } from "./runtime"; import { makeAppsStore } from "./store"; import type { ClientResolver, ConnectionCandidate } from "./bindings"; import type { IntegrationDecl, ToolDescriptor } from "../pipeline/descriptor"; +import { PublishError } from "../pipeline/discover"; import { syncGitHubSource, type GitHubSyncResult } from "../source/github-source"; export const APPS_INTEGRATION_SLUG = "apps"; @@ -119,11 +120,8 @@ const projectInputSchema = ( }; export const appsPlugin = definePlugin((options?: AppsPluginOptions) => { - if (!options?.runtime) { - throw new Error("appsPlugin requires a runtime"); - } - const runtime = options.runtime; - const makeResolver = options.makeResolver; + const runtime = options?.runtime as AppsRuntime; + const makeResolver = options?.makeResolver; const tenantFor = (ctx: Pick | undefined): string => { const tenant = ctx?.owner?.tenant; @@ -321,33 +319,31 @@ export const appsPlugin = definePlugin((options?: AppsPluginOptions) => { const scope = yield* scopeFor(tenant, String(toolRow.connection)); const descriptor = yield* runtime.getDescriptor(tenant, scope); if (!descriptor) { - return yield* Effect.fail( - new Error( - `apps scope "${scope}" has no published app (connection "${toolRow.connection}")`, - ), - ); + return yield* new PublishError({ + message: `apps scope "${scope}" has no published app (connection "${toolRow.connection}")`, + stage: "project", + diagnostics: [], + }); } const toolDesc = descriptor.tools.find((t) => t.name === toolRow.name); if (!toolDesc) { - return yield* Effect.fail( - new Error(`apps tool "${toolRow.name}" is not published in scope "${scope}"`), - ); + return yield* new PublishError({ + message: `apps tool "${toolRow.name}" is not published in scope "${scope}"`, + stage: "project", + diagnostics: [], + }); } const resolver = makeResolver ? makeResolver({ ctx, scope, tool: toolRow.name }) : undefined; - return yield* runtime - .invokeTool({ tenant, scope, tool: toolRow.name, args, resolver, invokeOptions }) - .pipe( - Effect.mapError( - (cause) => - new Error( - "message" in cause && typeof cause.message === "string" - ? cause.message - : "apps tool invocation failed", - ), - ), - ); + return yield* runtime.invokeTool({ + tenant, + scope, + tool: toolRow.name, + args, + resolver, + invokeOptions, + }); }), }; }); diff --git a/packages/plugins/apps/src/plugin/bindings.strict.test.ts b/packages/plugins/apps/src/plugin/bindings.strict.test.ts index 6256e7708..9b7e10b98 100644 --- a/packages/plugins/apps/src/plugin/bindings.strict.test.ts +++ b/packages/plugins/apps/src/plugin/bindings.strict.test.ts @@ -1,5 +1,5 @@ -import { describe, expect, it } from "vitest"; -import { Effect } from "effect"; +import { describe, expect, it } from "@effect/vitest"; +import { Effect, Exit } from "effect"; import { buildBridge, resolveIntegrationBindings, type ClientResolver } from "./bindings"; import type { IntegrationDecl } from "../pipeline/descriptor"; @@ -43,7 +43,7 @@ const okResolver: ClientResolver = { }; const fails = (effect: Effect.Effect): Promise => - Effect.runPromiseExit(effect).then((exit) => exit._tag === "Failure"); + Effect.runPromiseExit(effect).then((exit) => Exit.isFailure(exit)); describe("HandleBridge strict dispatch", () => { const declared: Record = { diff --git a/packages/plugins/apps/src/plugin/bindings.ts b/packages/plugins/apps/src/plugin/bindings.ts index 5004b9241..e57436fab 100644 --- a/packages/plugins/apps/src/plugin/bindings.ts +++ b/packages/plugins/apps/src/plugin/bindings.ts @@ -71,19 +71,6 @@ export interface BindingContext { const isRecord = (value: unknown): value is Record => value !== null && typeof value === "object" && !Array.isArray(value); -const bindingError = ( - role: string, - integration: string, - message: string, - requestedConnection?: string, -): BindingError => - new BindingError({ - message, - role, - integration, - requestedConnection, - }); - const findDefaultConnection = ( role: string, decl: IntegrationDecl, @@ -93,15 +80,16 @@ const findDefaultConnection = ( Effect.flatMap((connections) => { if (connections.length === 1) return Effect.succeed(connections[0]!.address); return Effect.fail( - bindingError( + new BindingError({ role, - decl.integration, - connections.length === 0 - ? `missing required connection for role "${role}" (${decl.integration}); no connections are available` - : `missing required connection for role "${role}" (${decl.integration}); choose one of ${connections - .map((c) => c.address) - .join(", ")}`, - ), + integration: decl.integration, + message: + connections.length === 0 + ? `missing required connection for role "${role}" (${decl.integration}); no connections are available` + : `missing required connection for role "${role}" (${decl.integration}); choose one of ${connections + .map((c) => c.address) + .join(", ")}`, + }), ); }), ); @@ -116,22 +104,22 @@ const resolveRequestedConnection = ( Effect.flatMap((connection) => { if (!connection) { return Effect.fail( - bindingError( + new BindingError({ role, - decl.integration, - `unknown connection "${requested}" for role "${role}" (${decl.integration})`, - requested, - ), + integration: decl.integration, + message: `unknown connection "${requested}" for role "${role}" (${decl.integration})`, + requestedConnection: requested, + }), ); } if (connection.integration !== decl.integration) { return Effect.fail( - bindingError( + new BindingError({ role, - decl.integration, - `connection "${requested}" belongs to integration "${connection.integration}", not "${decl.integration}" for role "${role}"`, - requested, - ), + integration: decl.integration, + message: `connection "${requested}" belongs to integration "${connection.integration}", not "${decl.integration}" for role "${role}"`, + requestedConnection: requested, + }), ); } return Effect.succeed(connection.address); @@ -168,14 +156,12 @@ export const resolveIntegrationBindings = ( continue; } if (typeof raw !== "string" || raw.length === 0) { - return yield* Effect.fail( - bindingError( - role, - decl.integration, - `connection for role "${role}" (${decl.integration}) must be a non-empty string`, - typeof raw === "string" ? raw : undefined, - ), - ); + return yield* new BindingError({ + role, + integration: decl.integration, + message: `connection for role "${role}" (${decl.integration}) must be a non-empty string`, + ...(typeof raw === "string" ? { requestedConnection: raw } : {}), + }); } bindings[role] = yield* resolveRequestedConnection(role, decl, raw, resolver); } @@ -231,13 +217,16 @@ export const buildBridge = (context: BindingContext): HandleBridge => ({ if (root === "db") { if (path.length === 1 && path[0] === "sql") { const [strings, ...values] = args as [TemplateStringsArray, ...unknown[]]; - return context.db - .sql(strings, ...values) - .pipe( - Effect.mapError( - (cause) => new ToolSandboxError({ kind: "invoke", message: cause.message, cause }), - ), - ); + return context.db.sql(strings, ...values).pipe( + Effect.mapError( + (cause) => + new ToolSandboxError({ + kind: "invoke", + message: "scope database call failed", + cause, + }), + ), + ); } return Effect.fail(invokeErr(`unsupported db call: ${path.join(".")}`)); } @@ -274,7 +263,12 @@ export const buildBridge = (context: BindingContext): HandleBridge => ({ }) .pipe( Effect.mapError( - (cause) => new ToolSandboxError({ kind: "invoke", message: cause.message, cause }), + (cause) => + new ToolSandboxError({ + kind: "invoke", + message: "integration call failed", + cause, + }), ), ); }, diff --git a/packages/plugins/apps/src/plugin/publish-concurrency.test.ts b/packages/plugins/apps/src/plugin/publish-concurrency.test.ts index ef5aae195..abb7d4c64 100644 --- a/packages/plugins/apps/src/plugin/publish-concurrency.test.ts +++ b/packages/plugins/apps/src/plugin/publish-concurrency.test.ts @@ -2,7 +2,7 @@ import { mkdtempSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; -import { describe, expect, it } from "vitest"; +import { describe, expect, it } from "@effect/vitest"; import { Effect } from "effect"; import { makeSelfHostAppsRuntime } from "./self-host-runtime"; @@ -60,13 +60,12 @@ describe("concurrent publishes to one scope (Fix 6)", () => { // Every publish either succeeded or failed with a typed conflict; none // clobbered another silently. const succeeded = results.filter((r) => r.status === "fulfilled"); + const rejectedReasons = results.flatMap((r) => + r.status === "rejected" ? [JSON.stringify(r.reason)] : [], + ); expect(succeeded.length).toBeGreaterThan(0); - for (const r of results) { - if (r.status === "rejected") { - // A conflict is the only acceptable failure mode. - expect(String(r.reason)).toMatch(/conflict/i); - } - } + // A conflict is the only acceptable failure mode. + expect(rejectedReasons.every((reason) => /conflict/i.test(reason))).toBe(true); // HEAD and the descriptor pointer AGREE: the store's current descriptor's // snapshotId equals the artifact store's latest committed snapshot. diff --git a/packages/plugins/apps/src/plugin/runtime.test.ts b/packages/plugins/apps/src/plugin/runtime.test.ts index 9120c0a3c..0aacae294 100644 --- a/packages/plugins/apps/src/plugin/runtime.test.ts +++ b/packages/plugins/apps/src/plugin/runtime.test.ts @@ -2,8 +2,8 @@ import { mkdtempSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; -import { describe, expect, it } from "vitest"; -import { Effect } from "effect"; +import { describe, expect, it } from "@effect/vitest"; +import { Effect, Exit } from "effect"; import { makeSelfHostAppsRuntime } from "./self-host-runtime"; import { makeInMemoryAppsStore, makeTestResolver, dailyBriefFileSet } from "../testing"; @@ -157,11 +157,10 @@ export default defineTool({ host.runtime.invokeTool({ scope: "s", tool: "sync", args: {} }), ); - expect(exit._tag).toBe("Failure"); - if (exit._tag !== "Failure") throw new Error("expected missing connection failure"); - expect(JSON.stringify(exit.cause)).toContain("BindingError"); - expect(JSON.stringify(exit.cause)).toContain("crm"); - expect(JSON.stringify(exit.cause)).toContain("dealcloud"); + expect(Exit.isFailure(exit)).toBe(true); + expect(JSON.stringify(exit)).toContain("BindingError"); + expect(JSON.stringify(exit)).toContain("crm"); + expect(JSON.stringify(exit)).toContain("dealcloud"); await host.close(); }); @@ -204,12 +203,11 @@ export default defineTool({ }), ); - expect(exit._tag).toBe("Failure"); - if (exit._tag !== "Failure") throw new Error("expected wrong integration failure"); - expect(JSON.stringify(exit.cause)).toContain("BindingError"); - expect(JSON.stringify(exit.cause)).toContain("github"); - expect(JSON.stringify(exit.cause)).toContain("dealcloud"); - expect(JSON.stringify(exit.cause)).toContain("tools.github.user.main"); + expect(Exit.isFailure(exit)).toBe(true); + expect(JSON.stringify(exit)).toContain("BindingError"); + expect(JSON.stringify(exit)).toContain("github"); + expect(JSON.stringify(exit)).toContain("dealcloud"); + expect(JSON.stringify(exit)).toContain("tools.github.user.main"); await host.close(); }); diff --git a/packages/plugins/apps/src/plugin/runtime.ts b/packages/plugins/apps/src/plugin/runtime.ts index 31a96a85e..b72d7dc18 100644 --- a/packages/plugins/apps/src/plugin/runtime.ts +++ b/packages/plugins/apps/src/plugin/runtime.ts @@ -1,4 +1,4 @@ -import { Effect } from "effect"; +import { Effect, Predicate } from "effect"; import type { InvokeOptions } from "@executor-js/sdk"; import type { ArtifactStore } from "../seams/artifact-store"; @@ -100,6 +100,13 @@ const failNoDescriptor = (scope: string): PublishError => diagnostics: [], }); +const isInvokePassthroughError = ( + cause: unknown, +): cause is BindingError | InputValidationError | OutputValidationError => + Predicate.isTagged("BindingError")(cause) || + Predicate.isTagged("InputValidationError")(cause) || + Predicate.isTagged("OutputValidationError")(cause); + export const makeAppsRuntime = (deps: AppsRuntimeDeps): AppsRuntime => { const defaultTenant = deps.defaultTenant ?? "org"; const bundleCache = new Map(); @@ -120,10 +127,17 @@ export const makeAppsRuntime = (deps: AppsRuntimeDeps): AppsRuntime => { ): Promise => { const key = `${tenant}:${scope}`; const prior = publishChains.get(key) ?? Promise.resolve(); - const next = prior.catch(() => undefined).then(run); + const afterPrior = prior.then( + () => undefined, + () => undefined, + ); + const next = afterPrior.then(run); publishChains.set( key, - next.catch(() => undefined), + next.then( + () => undefined, + () => undefined, + ), ); return next; }; @@ -293,11 +307,7 @@ export const makeAppsRuntime = (deps: AppsRuntimeDeps): AppsRuntime => { ) .pipe( Effect.mapError((c) => { - if ( - c instanceof BindingError || - c instanceof InputValidationError || - c instanceof OutputValidationError - ) { + if (isInvokePassthroughError(c)) { return c; } return new PublishError({ @@ -340,9 +350,9 @@ export const makeAppsRuntime = (deps: AppsRuntimeDeps): AppsRuntime => { ), ); }, - catch: (cause) => + catch: (_cause) => new PublishError({ - message: cause instanceof Error ? cause.message : String(cause), + message: "publish failed before pipeline completed", stage: "project", diagnostics: [], }), @@ -404,13 +414,11 @@ export const makeAppsRuntime = (deps: AppsRuntimeDeps): AppsRuntime => { const descriptor = yield* requireDescriptor(tenant, input.scope); const toolDesc = descriptor.tools.find((t) => t.name === input.tool); if (!toolDesc) { - return yield* Effect.fail( - new PublishError({ - message: `tool "${input.tool}" is not published in scope "${input.scope}"`, - stage: "project", - diagnostics: [], - }), - ); + return yield* new PublishError({ + message: `tool "${input.tool}" is not published in scope "${input.scope}"`, + stage: "project", + diagnostics: [], + }); } return yield* invokeToolInternal( input.scope, diff --git a/packages/plugins/apps/src/plugin/tenant-isolation.test.ts b/packages/plugins/apps/src/plugin/tenant-isolation.test.ts index 46e61c0c8..df133b246 100644 --- a/packages/plugins/apps/src/plugin/tenant-isolation.test.ts +++ b/packages/plugins/apps/src/plugin/tenant-isolation.test.ts @@ -2,7 +2,7 @@ import { mkdtempSync, readdirSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; -import { describe, expect, it } from "vitest"; +import { describe, expect, it } from "@effect/vitest"; import { Effect } from "effect"; import { makeSqliteAppsStore } from "../backing/sqlite-apps-store"; diff --git a/packages/plugins/apps/src/react/AddCustomToolsSource.tsx b/packages/plugins/apps/src/react/AddCustomToolsSource.tsx index e1d0a2fc3..a6e3d1a1c 100644 --- a/packages/plugins/apps/src/react/AddCustomToolsSource.tsx +++ b/packages/plugins/apps/src/react/AddCustomToolsSource.tsx @@ -1,5 +1,6 @@ import { useMemo, useState } from "react"; import { useAtomValue } from "@effect/atom-react"; +import { Effect, Exit } from "effect"; import * as AsyncResult from "effect/unstable/reactivity/AsyncResult"; import { connectionsAllAtom } from "@executor-js/react/api/atoms"; @@ -20,7 +21,7 @@ import { consoleIntegrationHref, formatSyncErrors, githubConnections, - syncCustomToolSource, + syncCustomToolSourceEffect, syncStatusLabel, validateGitHubRepo, } from "./custom-tools-client"; @@ -54,23 +55,26 @@ export default function AddCustomToolsSource(props: { if (validation || !selectedConnection) return; setSyncing(true); - try { - const result = await syncCustomToolSource({ + const exit = await Effect.runPromiseExit( + syncCustomToolSourceEffect({ repo, ref, connection: selectedConnection, - }); - if (result.status === "failed") { - setSyncError(formatSyncErrors(result).join("\n") || "Sync failed."); - setSyncing(false); - return; - } - toast.success(syncStatusLabel(result)); - props.onComplete("apps"); - } catch (error) { - setSyncError(error instanceof Error ? error.message : "Failed to sync custom tools."); + }), + ); + if (Exit.isFailure(exit)) { + setSyncError("Failed to sync custom tools."); setSyncing(false); + return; } + const result = exit.value; + if (result.status === "failed") { + setSyncError(formatSyncErrors(result).join("\n") || "Sync failed."); + setSyncing(false); + return; + } + toast.success(syncStatusLabel(result)); + props.onComplete("apps"); }; return ( diff --git a/packages/plugins/apps/src/react/CustomToolsAccountsPanel.tsx b/packages/plugins/apps/src/react/CustomToolsAccountsPanel.tsx index 0c8945984..2e4141070 100644 --- a/packages/plugins/apps/src/react/CustomToolsAccountsPanel.tsx +++ b/packages/plugins/apps/src/react/CustomToolsAccountsPanel.tsx @@ -1,4 +1,5 @@ -import { useEffect, useMemo, useState, type ReactNode } from "react"; +import { useEffect, useState, type ReactNode } from "react"; +import { Effect, Exit } from "effect"; import { Alert, AlertDescription, AlertTitle } from "@executor-js/react/components/alert"; import { Badge } from "@executor-js/react/components/badge"; @@ -17,8 +18,8 @@ import type { GitHubCustomToolsSourceSummary, GitHubSyncResult } from "../api"; import { consoleIntegrationHref, formatSyncErrors, - listCustomToolSources, - syncCustomToolSource, + listCustomToolSourcesEffect, + syncCustomToolSourceEffect, syncStatusLabel, toolDiff, } from "./custom-tools-client"; @@ -49,31 +50,33 @@ export default function CustomToolsAccountsPanel() { const loadSources = async () => { setLoadState({ status: "loading" }); - try { - const result = await listCustomToolSources(); - setLoadState({ status: "ready", sources: result.sources }); - setSelectedScope((current) => current ?? result.sources[0]?.scope ?? null); - } catch (error) { + const exit = await Effect.runPromiseExit(listCustomToolSourcesEffect()); + if (Exit.isFailure(exit)) { setLoadState({ status: "error", - message: error instanceof Error ? error.message : "Failed to load custom tool sources.", + message: "Failed to load custom tool sources.", }); + return; } + const result = exit.value; + setLoadState({ status: "ready", sources: result.sources }); + setSelectedScope((current) => current ?? result.sources[0]?.scope ?? null); }; useEffect(() => { let active = true; void (async () => { - try { - const result = await listCustomToolSources(); + const exit = await Effect.runPromiseExit(listCustomToolSourcesEffect()); + if (Exit.isSuccess(exit)) { if (!active) return; + const result = exit.value; setLoadState({ status: "ready", sources: result.sources }); setSelectedScope(result.sources[0]?.scope ?? null); - } catch (error) { + } else { if (!active) return; setLoadState({ status: "error", - message: error instanceof Error ? error.message : "Failed to load custom tool sources.", + message: "Failed to load custom tool sources.", }); } })(); @@ -83,10 +86,7 @@ export default function CustomToolsAccountsPanel() { }, []); const sources = loadState.status === "ready" ? loadState.sources : []; - const selected = useMemo( - () => sources.find((source) => source.scope === selectedScope) ?? sources[0] ?? null, - [sources, selectedScope], - ); + const selected = sources.find((source) => source.scope === selectedScope) ?? sources[0] ?? null; const syncSelected = async (source: GitHubCustomToolsSourceSummary) => { if (!source.connection) { @@ -102,35 +102,37 @@ export default function CustomToolsAccountsPanel() { setSyncingScope(source.scope); setNotice(null); const beforeTools = source.tools; - try { - const result = await syncCustomToolSource({ + const exit = await Effect.runPromiseExit( + syncCustomToolSourceEffect({ repo: source.repo, ref: source.ref, connection: source.connection, - }); - const diff = - result.status === "failed" - ? { added: [], removed: [] } - : toolDiff(beforeTools, result.tools); - setNotice({ - status: result.status, - message: syncStatusLabel(result), - added: diff.added, - removed: diff.removed, - errors: formatSyncErrors(result), - }); - if (result.status !== "failed") { - await loadSources(); - setSelectedScope(source.scope); - } - } catch (error) { + }), + ); + if (Exit.isFailure(exit)) { setNotice({ status: "failed", message: "Sync failed.", added: [], removed: [], - errors: [error instanceof Error ? error.message : "Failed to sync custom tools."], + errors: ["Failed to sync custom tools."], }); + setSyncingScope(null); + return; + } + const result = exit.value; + const diff = + result.status === "failed" ? { added: [], removed: [] } : toolDiff(beforeTools, result.tools); + setNotice({ + status: result.status, + message: syncStatusLabel(result), + added: diff.added, + removed: diff.removed, + errors: formatSyncErrors(result), + }); + if (result.status !== "failed") { + await loadSources(); + setSelectedScope(source.scope); } setSyncingScope(null); }; @@ -181,16 +183,17 @@ export default function CustomToolsAccountsPanel() { asChild searchText={`${source.repo} ${source.ref}`} > - + ))} diff --git a/packages/plugins/apps/src/react/custom-tools-client.ts b/packages/plugins/apps/src/react/custom-tools-client.ts index d6ef462e6..db96b8433 100644 --- a/packages/plugins/apps/src/react/custom-tools-client.ts +++ b/packages/plugins/apps/src/react/custom-tools-client.ts @@ -1,4 +1,5 @@ import type { Connection } from "@executor-js/sdk/shared"; +import { Data, Effect, Schema } from "effect"; import type { GitHubCustomToolsSourceSummary, GitHubSyncResult } from "../api"; @@ -15,12 +16,11 @@ export interface SyncGitHubSourceRequest { readonly connection: string; } -export class CustomToolsClientError extends Error { - constructor(message: string) { - super(message); - this.name = "CustomToolsClientError"; - } -} +export type CustomToolsFetch = (input: RequestInfo | URL, init?: RequestInit) => Promise; + +export class CustomToolsClientError extends Data.TaggedError("CustomToolsClientError")<{ + readonly message: string; +}> {} const REPO_RE = /^[A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+$/; @@ -34,47 +34,83 @@ export const validateGitHubRepo = (repo: string): string | null => { export const githubConnections = (connections: readonly Connection[]): readonly Connection[] => connections.filter((connection) => String(connection.integration) === "github"); -const parseJsonResponse = async (response: Response, fallback: string): Promise => { - const text = await response.text(); - const body = text.length > 0 ? (JSON.parse(text) as unknown) : null; - if (!response.ok) { - const message = - body && - typeof body === "object" && - "error" in body && - typeof (body as { error?: unknown }).error === "string" - ? (body as { error: string }).error - : fallback; - throw new CustomToolsClientError(message); - } - return body as A; -}; +const decodeJsonText = Schema.decodeUnknownEffect(Schema.UnknownFromJsonString); -export const listCustomToolSources = async ( - fetchImpl: typeof fetch = fetch, -): Promise => { - const response = await fetchImpl("/api/apps/sources/github", { - credentials: "same-origin", +const isRecord = (value: unknown): value is Record => + value !== null && typeof value === "object" && !Array.isArray(value); + +const responseErrorMessage = (body: unknown, fallback: string): string => + isRecord(body) && typeof body.error === "string" ? body.error : fallback; + +const parseJsonResponseEffect = ( + response: Response, + fallback: string, +): Effect.Effect => + Effect.gen(function* () { + const text = yield* Effect.tryPromise({ + try: () => response.text(), + catch: () => new CustomToolsClientError({ message: fallback }), + }); + const body = + text.length > 0 + ? yield* decodeJsonText(text).pipe( + Effect.mapError(() => new CustomToolsClientError({ message: fallback })), + ) + : null; + if (!response.ok) { + return yield* new CustomToolsClientError({ + message: responseErrorMessage(body, fallback), + }); + } + return body as A; }); - return parseJsonResponse(response, "Failed to load custom tools."); -}; -export const syncCustomToolSource = async ( +export const listCustomToolSourcesEffect = ( + fetchImpl: CustomToolsFetch = fetch, +): Effect.Effect => + Effect.tryPromise({ + try: () => + fetchImpl("/api/apps/sources/github", { + credentials: "same-origin", + }), + catch: () => new CustomToolsClientError({ message: "Failed to load custom tools." }), + }).pipe( + Effect.flatMap((response) => + parseJsonResponseEffect(response, "Failed to load custom tools."), + ), + ); + +export const listCustomToolSources = ( + fetchImpl: CustomToolsFetch = fetch, +): Promise => Effect.runPromise(listCustomToolSourcesEffect(fetchImpl)); + +export const syncCustomToolSourceEffect = ( input: SyncGitHubSourceRequest, - fetchImpl: typeof fetch = fetch, -): Promise => { - const response = await fetchImpl("/api/apps/sources/github/sync", { - method: "POST", - headers: { "content-type": "application/json" }, - credentials: "same-origin", - body: JSON.stringify({ - repo: input.repo.trim(), - ...(input.ref?.trim() ? { ref: input.ref.trim() } : {}), - connection: input.connection, - }), - }); - return parseJsonResponse(response, "Failed to sync custom tools."); -}; + fetchImpl: CustomToolsFetch = fetch, +): Effect.Effect => + Effect.tryPromise({ + try: () => + fetchImpl("/api/apps/sources/github/sync", { + method: "POST", + headers: { "content-type": "application/json" }, + credentials: "same-origin", + body: JSON.stringify({ + repo: input.repo.trim(), + ...(input.ref?.trim() ? { ref: input.ref.trim() } : {}), + connection: input.connection, + }), + }), + catch: () => new CustomToolsClientError({ message: "Failed to sync custom tools." }), + }).pipe( + Effect.flatMap((response) => + parseJsonResponseEffect(response, "Failed to sync custom tools."), + ), + ); + +export const syncCustomToolSource = ( + input: SyncGitHubSourceRequest, + fetchImpl: CustomToolsFetch = fetch, +): Promise => Effect.runPromise(syncCustomToolSourceEffect(input, fetchImpl)); export const syncStatusLabel = (result: GitHubSyncResult): string => { if (result.status === "published") return `Published ${result.tools.length} tools.`; @@ -84,11 +120,11 @@ export const syncStatusLabel = (result: GitHubSyncResult): string => { export const formatSyncErrors = (result: GitHubSyncResult): readonly string[] => { if (result.status !== "failed") return []; - return result.errors.map((error) => { - const details = error.diagnostics?.map((d) => `${d.path}: ${d.message}`).join("; "); - return details - ? `${error.stage}: ${error.message} (${details})` - : `${error.stage}: ${error.message}`; + return result.errors.map((entry) => { + const message = entry.message; + const stage = entry.stage; + const details = entry.diagnostics?.map((d) => `${d.path}: ${d.message}`).join("; "); + return details ? `${stage}: ${message} (${details})` : `${stage}: ${message}`; }); }; diff --git a/packages/plugins/apps/src/seams/artifact-store.conformance.ts b/packages/plugins/apps/src/seams/artifact-store.conformance.ts index 334f67817..b1da0e36e 100644 --- a/packages/plugins/apps/src/seams/artifact-store.conformance.ts +++ b/packages/plugins/apps/src/seams/artifact-store.conformance.ts @@ -1,4 +1,4 @@ -import { describe, expect, it } from "vitest"; +import { describe, expect, it } from "@effect/vitest"; import { Effect } from "effect"; import type { ArtifactStore, FileSet } from "./artifact-store"; diff --git a/packages/plugins/apps/src/seams/scope-db.conformance.ts b/packages/plugins/apps/src/seams/scope-db.conformance.ts index c0efd51ae..ee0df0502 100644 --- a/packages/plugins/apps/src/seams/scope-db.conformance.ts +++ b/packages/plugins/apps/src/seams/scope-db.conformance.ts @@ -1,4 +1,4 @@ -import { describe, expect, it } from "vitest"; +import { describe, expect, it } from "@effect/vitest"; import { Effect } from "effect"; import type { ScopeDb } from "./scope-db"; diff --git a/packages/plugins/apps/src/seams/tool-sandbox.conformance.ts b/packages/plugins/apps/src/seams/tool-sandbox.conformance.ts index bf58e2f4e..bb0e6ab14 100644 --- a/packages/plugins/apps/src/seams/tool-sandbox.conformance.ts +++ b/packages/plugins/apps/src/seams/tool-sandbox.conformance.ts @@ -1,5 +1,5 @@ -import { describe, expect, it } from "vitest"; -import { Effect } from "effect"; +import { describe, expect, it } from "@effect/vitest"; +import { Effect, Exit } from "effect"; import type { HandleBridge, ToolSandbox } from "./tool-sandbox"; import { bundleEntry } from "../pipeline/bundle"; @@ -34,7 +34,7 @@ import { z } from "zod"; export default defineTool({ description: "x" + Math.random(), input: z.object({}), async handler(){ return {}; } });`; const rngBundle = await bundle("tools/rng.ts", rng); const exit = await Effect.runPromiseExit(sandbox.collect(rngBundle)); - expect(exit._tag).toBe("Failure"); + expect(Exit.isFailure(exit)).toBe(true); }); it("denies network access", async () => { @@ -47,7 +47,7 @@ export default defineTool({ description: "net", input: z.object({}), async handl const exit = await Effect.runPromiseExit( sandbox.invoke(b, { artifact: "net", kind: "tool", input: {}, roots: {} }, bridge), ); - expect(exit._tag).toBe("Failure"); + expect(Exit.isFailure(exit)).toBe(true); }); it("kills a runaway handler on timeout", async () => { @@ -60,7 +60,7 @@ export default defineTool({ description: "loop", input: z.object({}), async hand const exit = await Effect.runPromiseExit( sandbox.invoke(b, { artifact: "loop", kind: "tool", input: {}, roots: {} }, bridge), ); - expect(exit._tag).toBe("Failure"); + expect(Exit.isFailure(exit)).toBe(true); }); it("round-trips the handle bridge for a declared integration role", async () => { diff --git a/packages/plugins/apps/src/source/github-source.test.ts b/packages/plugins/apps/src/source/github-source.test.ts index fbb0e042c..dd46ae9cb 100644 --- a/packages/plugins/apps/src/source/github-source.test.ts +++ b/packages/plugins/apps/src/source/github-source.test.ts @@ -2,8 +2,8 @@ import { mkdtempSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; -import { describe, expect, it } from "vitest"; -import { Effect } from "effect"; +import { describe, expect, it } from "@effect/vitest"; +import { Effect, Exit } from "effect"; import { makeSelfHostAppsRuntime } from "../plugin/self-host-runtime"; import { makeInMemoryAppsStore, makeTestResolver } from "../testing"; @@ -208,7 +208,7 @@ describe("GitHub custom-tools source", () => { const exit = await Effect.runPromiseExit( fetchGitHubSource({ repo: "acme/tools", fetch: github.fetch }), ); - expect(exit._tag).toBe("Failure"); + expect(Exit.isFailure(exit)).toBe(true); expect(JSON.stringify(exit)).toContain("exceeding the limit"); expect(github.blobCalls()).toBe(0); }); diff --git a/packages/plugins/apps/src/source/github-source.ts b/packages/plugins/apps/src/source/github-source.ts index e5ab57be3..f351e19df 100644 --- a/packages/plugins/apps/src/source/github-source.ts +++ b/packages/plugins/apps/src/source/github-source.ts @@ -1,4 +1,4 @@ -import { Data, Effect, Result } from "effect"; +import { Data, Effect, Predicate, Result, Schema } from "effect"; import { PublishError, type FileDiagnostic } from "../pipeline/discover"; import { PUBLISH_LIMITS, enforcePublishLimits } from "../pipeline/publish"; @@ -106,37 +106,42 @@ const requestJson = ( input: GitHubSourceInput, path: string, ): Effect.Effect => - Effect.tryPromise({ - try: async () => { - const fetchImpl = input.fetch ?? globalThis.fetch; - const headers: Record = { - accept: "application/vnd.github+json", - "user-agent": "executor-apps-github-source", - }; - if (input.token) headers.authorization = `Bearer ${input.token}`; - const response = await fetchImpl( - `${trimBaseUrl(input.baseUrl ?? "https://api.github.com")}${path}`, - { + Effect.gen(function* () { + const response = yield* Effect.tryPromise({ + try: async () => { + const fetchImpl = input.fetch ?? globalThis.fetch; + const headers: Record = { + accept: "application/vnd.github+json", + "user-agent": "executor-apps-github-source", + }; + if (input.token) headers.authorization = `Bearer ${input.token}`; + return fetchImpl(`${trimBaseUrl(input.baseUrl ?? "https://api.github.com")}${path}`, { headers, - }, - ); - if (!response.ok) { - throw new GitHubSourceError({ - message: `GitHub request failed: GET ${path} -> ${response.status}`, - status: response.status, - path, }); - } - return (await response.json()) as A; - }, - catch: (cause) => - cause instanceof GitHubSourceError - ? cause - : new GitHubSourceError({ - message: `GitHub request failed: GET ${path}`, - path, - cause, - }), + }, + catch: (cause) => + new GitHubSourceError({ + message: `GitHub request failed: GET ${path}`, + path, + cause, + }), + }); + if (!response.ok) { + return yield* new GitHubSourceError({ + message: `GitHub request failed: GET ${path} -> ${response.status}`, + status: response.status, + path, + }); + } + return yield* Effect.tryPromise({ + try: () => response.json() as Promise, + catch: (cause) => + new GitHubSourceError({ + message: `GitHub response was not valid JSON: GET ${path}`, + path, + cause, + }), + }); }); interface RepoResponse { @@ -254,42 +259,39 @@ const classifyTreeEntry = ( }; const decodeBlob = (path: string, blob: BlobResponse): Effect.Effect => - Effect.try({ - try: () => { - const encoding = asString(blob.encoding); - const content = asString(blob.content); - if (!content || encoding !== "base64") { - throw new GitHubSourceError({ - message: `GitHub blob ${path} did not return base64 content`, - path, - }); - } - return Buffer.from(content.replace(/\s/g, ""), "base64").toString("utf8"); - }, - catch: (cause) => - cause instanceof GitHubSourceError - ? cause - : new GitHubSourceError({ message: `Failed to decode GitHub blob ${path}`, path, cause }), + Effect.gen(function* () { + const encoding = asString(blob.encoding); + const content = asString(blob.content); + if (!content || encoding !== "base64") { + return yield* new GitHubSourceError({ + message: `GitHub blob ${path} did not return base64 content`, + path, + }); + } + return Buffer.from(content.replace(/\s/g, ""), "base64").toString("utf8"); }); +const decodeExecutorJson = Schema.decodeUnknownEffect(Schema.UnknownFromJsonString); + const executorDescription = ( files: FileSet, ): Effect.Effect => - Effect.try({ - try: () => { - const raw = files.get("executor.json"); - if (!raw) return undefined; - const parsed = JSON.parse(raw) as unknown; - if (!isRecord(parsed)) return undefined; - const description = parsed.description; - return typeof description === "string" ? description : undefined; - }, - catch: (cause) => - new GitHubSourceError({ - message: "executor.json is not valid JSON", - path: "executor.json", - cause, - }), + Effect.gen(function* () { + const raw = files.get("executor.json"); + if (!raw) return undefined; + const parsed = yield* decodeExecutorJson(raw).pipe( + Effect.mapError( + (cause) => + new GitHubSourceError({ + message: "executor.json is not valid JSON", + path: "executor.json", + cause, + }), + ), + ); + if (!isRecord(parsed)) return undefined; + const description = parsed.description; + return typeof description === "string" ? description : undefined; }); export const fetchGitHubSource = ( @@ -336,7 +338,7 @@ export const fetchGitHubSource = ( else skipped.push(classified.skipped); } const treeLimitError = checkTreeLimits(entries); - if (treeLimitError) return yield* Effect.fail(treeLimitError); + if (treeLimitError) return yield* treeLimitError; const files = new Map(); for (const entry of entries) { @@ -348,7 +350,7 @@ export const fetchGitHubSource = ( files.set(path, yield* decodeBlob(path, blob)); } const payloadLimitError = enforcePublishLimits(files); - if (payloadLimitError) return yield* Effect.fail(payloadLimitError); + if (payloadLimitError) return yield* payloadLimitError; return { files, @@ -360,17 +362,22 @@ export const fetchGitHubSource = ( }; }); -const publishErrorToSyncError = (error: PublishError): SyncErrorData => ({ - stage: error.stage, - message: error.message, - diagnostics: error.diagnostics, -}); +const publishErrorToSyncError = (publishFailure: PublishError): SyncErrorData => { + const stage = publishFailure.stage; + const message = publishFailure.message; + const diagnostics = publishFailure.diagnostics; + return { stage, message, diagnostics }; +}; -const sourceErrorToSyncError = (error: GitHubSourceError): SyncErrorData => ({ - stage: "source", - message: error.message, - diagnostics: error.path ? [{ path: error.path, message: error.message }] : [], -}); +const sourceErrorToSyncError = (sourceFailure: GitHubSourceError): SyncErrorData => { + const message = sourceFailure.message; + const path = sourceFailure.path; + return { + stage: "source", + message, + diagnostics: path ? [{ path, message }] : [], + }; +}; const sourceRef = (snapshot: GitHubSourceSnapshot, connection?: string): AppSourceRef => ({ kind: "github", @@ -391,7 +398,7 @@ export const syncGitHubSource = (input: SyncGitHubSourceInput): Effect.Effect handler(args)); }, From c9313eff1b3316f4ede5cdfa2f52884726872118 Mon Sep 17 00:00:00 2001 From: Rhys Sullivan <39114868+RhysSullivan@users.noreply.github.com> Date: Mon, 6 Jul 2026 09:50:15 -0700 Subject: [PATCH 10/21] Use GitHub URLs for custom tools sources --- .../scripts/custom-tools-demo.ts | 8 +- apps/host-selfhost/src/apps-wire.node.test.ts | 29 +++- .../src/custom-tools-console.web.test.ts | 35 +++-- packages/core/sdk/src/executor.ts | 51 +++++++ packages/core/sdk/src/plugin.ts | 21 +++ packages/plugins/apps/src/api.ts | 1 + .../apps/src/backing/sqlite-apps-store.ts | 32 ++++- packages/plugins/apps/src/index.ts | 2 + .../plugins/apps/src/pipeline/descriptor.ts | 4 +- .../plugins/apps/src/plugin/apps-plugin.ts | 130 +++++++++++------- packages/plugins/apps/src/plugin/runtime.ts | 33 +++-- packages/plugins/apps/src/plugin/store.ts | 41 ++++++ .../apps/src/plugin/tenant-isolation.test.ts | 4 +- .../apps/src/react/AddCustomToolsSource.tsx | 121 ++++++---------- .../src/react/CustomToolsAccountsPanel.tsx | 40 +++--- .../apps/src/react/custom-tools-client.ts | 23 ++-- .../apps/src/source/github-source.test.ts | 113 +++++++++++++-- .../plugins/apps/src/source/github-source.ts | 52 +++---- .../plugins/apps/src/source/github-url.ts | 108 +++++++++++++++ packages/plugins/apps/src/testing/index.ts | 8 ++ 20 files changed, 623 insertions(+), 233 deletions(-) create mode 100644 packages/plugins/apps/src/source/github-url.ts diff --git a/apps/host-selfhost/scripts/custom-tools-demo.ts b/apps/host-selfhost/scripts/custom-tools-demo.ts index a81146cd0..f2ea8fe8e 100644 --- a/apps/host-selfhost/scripts/custom-tools-demo.ts +++ b/apps/host-selfhost/scripts/custom-tools-demo.ts @@ -246,12 +246,13 @@ const curlJson = (path: string, body: unknown): string => const buildBanner = (input: { readonly repoUrl: string; readonly connectionAddress: string; + readonly githubToken: string; readonly token: string; readonly consoleUi: string; }): string => { const sync = curlJson("/api/apps/sources/github/sync", { - repo: DEMO_REPO, - connection: input.connectionAddress, + url: input.repoUrl, + token: input.githubToken, }); const list = `curl -sS '${BASE_URL}/api/tools?integration=apps' | jq`; const repoSummary = curlJson("/api/executions", { @@ -286,7 +287,7 @@ Auth: Bearer for direct API debugging: ${input.token} -GitHub connection: +GitHub invocation connection: ${input.connectionAddress} Sync the repo: @@ -385,6 +386,7 @@ const main = async () => { const banner = buildBanner({ repoUrl: `https://github.com/${DEMO_REPO}`, connectionAddress: created.address, + githubToken: token, token: selfHostToken, consoleUi, }); diff --git a/apps/host-selfhost/src/apps-wire.node.test.ts b/apps/host-selfhost/src/apps-wire.node.test.ts index b95221a35..9b2d0657f 100644 --- a/apps/host-selfhost/src/apps-wire.node.test.ts +++ b/apps/host-selfhost/src/apps-wire.node.test.ts @@ -456,10 +456,14 @@ const registerGitHubIntegration = async (emulator: Emulator, token: string): Pro expect(created.address).toBe(GITHUB_CONNECTION); }; -const syncSource = (repo = REPO_FULL_NAME): Promise => +const sourceUrl = (repo = REPO_FULL_NAME): string => `https://github.com/${repo}`; + +const syncSource = ( + input: { readonly repo?: string; readonly token?: string } = {}, +): Promise => postJson("/api/apps/sources/github/sync", { - repo, - connection: GITHUB_CONNECTION, + url: sourceUrl(input.repo), + ...(input.token ? { token: input.token } : {}), }); const listAppTools = (): Promise => @@ -534,17 +538,29 @@ test("GitHub source sync publishes and invokes custom tools through self-host HT const unauthorized = await fetch(`${server.baseUrl}/api/apps/sources/github/sync`, { method: "POST", headers: { "content-type": "application/json" }, - body: JSON.stringify({ repo: REPO_FULL_NAME, connection: GITHUB_CONNECTION }), + body: JSON.stringify({ url: sourceUrl(), token }), }); expect(unauthorized.status).toBe(401); const published = await syncSource(); expect(published.status).toBe("published"); + expect(JSON.stringify(published)).not.toContain(token!); expect(published.tools).toEqual(["deal-pipeline-sync", "find-deal-docs"]); expect(published.skipped).toEqual([]); const firstSnapshot = published.snapshotId; expect(firstSnapshot).toBeTruthy(); + const tokenUpToDate = await syncSource({ token: token! }); + expect(tokenUpToDate.status).toBe("up-to-date"); + expect(JSON.stringify(tokenUpToDate)).not.toContain(token!); + + const sourcesList = await requestJson<{ sources: readonly { hasToken: boolean }[] }>( + "/api/apps/sources/github", + { headers: jsonHeaders }, + ); + expect(sourcesList.sources[0]?.hasToken).toBe(true); + expect(JSON.stringify(sourcesList)).not.toContain(token!); + const listed = await listAppTools(); expect(listed.map((tool) => tool.name).sort()).toEqual(["deal-pipeline-sync", "find-deal-docs"]); @@ -573,6 +589,9 @@ test("GitHub source sync publishes and invokes custom tools through self-host HT expect(invoke.result?.data?.synced).toBe(2); const ledger = await github.ledger.list(); + const sourceFetches = ledger.filter((entry) => entry.path === `/repos/${OWNER}/${REPO}`); + expect(sourceFetches.some((entry) => entry.identity.user?.login === OWNER)).toBe(true); + expect(sourceFetches.some((entry) => !entry.identity.user)).toBe(true); const issueList = ledger.find( (entry) => entry.operationId === "issues/listForRepo" && entry.path === `/repos/${OWNER}/${REPO}/issues`, @@ -639,7 +658,7 @@ test("bridged integration calls inherit the caller approval handler", async () = await registerGitHubIntegration(github, token!); await createIssue(github, token!, "Approval-gated issue"); await putRepoFiles(github, token!, approvalFiles()); - const published = await syncSource(); + const published = await syncSource({ token: token! }); expect(published.status).toBe("published"); const githubTools = await requestJson("/api/tools?integration=github", { diff --git a/apps/host-selfhost/src/custom-tools-console.web.test.ts b/apps/host-selfhost/src/custom-tools-console.web.test.ts index ebff9aa5e..0d5c3b62d 100644 --- a/apps/host-selfhost/src/custom-tools-console.web.test.ts +++ b/apps/host-selfhost/src/custom-tools-console.web.test.ts @@ -9,7 +9,7 @@ import appsClientPlugin, { listCustomToolSources, syncCustomToolSource, syncStatusLabel, - validateGitHubRepo, + validateGitHubSourceUrl, } from "@executor-js/plugin-apps/client"; const jsonResponse = (body: unknown, init?: ResponseInit): Response => @@ -34,18 +34,24 @@ describe("custom tools console client", () => { }); }); - it("validates the GitHub repo shape", () => { - expect(validateGitHubRepo("UsefulSoftwareCo/executor")).toBeNull(); - expect(validateGitHubRepo("UsefulSoftwareCo")).toBe( - "Use owner/name, for example UsefulSoftwareCo/executor.", + it("validates the GitHub URL shape", () => { + expect(validateGitHubSourceUrl("https://github.com/UsefulSoftwareCo/executor")).toBeNull(); + expect(validateGitHubSourceUrl("UsefulSoftwareCo/executor")).toBeNull(); + expect(validateGitHubSourceUrl("https://gitlab.com/UsefulSoftwareCo/executor")).toBe( + "GitHub source URLs must use github.com.", ); - expect(validateGitHubRepo("")).toBe("Enter a GitHub repo."); + expect(validateGitHubSourceUrl("UsefulSoftwareCo")).toBe( + "Use a GitHub repo URL like https://github.com/owner/repo, optionally with /tree/ or /commit/.", + ); + expect(validateGitHubSourceUrl("")).toBe("Enter a GitHub URL."); }); it("surfaces successful sync and source detail data", async () => { - const fetchImpl: CustomToolsFetch = async (input) => { + let syncBody = ""; + const fetchImpl: CustomToolsFetch = async (input, init) => { const url = String(input); if (url.endsWith("/sync")) { + syncBody = String(init?.body ?? ""); return jsonResponse({ status: "published", snapshotId: "snap1", @@ -58,9 +64,10 @@ describe("custom tools console client", () => { sources: [ { scope: "github:RhysSullivan/executor-custom-tools-demo", + url: "https://github.com/RhysSullivan/executor-custom-tools-demo", repo: "RhysSullivan/executor-custom-tools-demo", ref: "main", - connection: "tools.github.user.demo", + hasToken: true, upstreamSha: "abc123", snapshotId: "snap1", publishedAt: "2026-07-06T12:00:00.000Z", @@ -73,15 +80,17 @@ describe("custom tools console client", () => { const syncResult = await syncCustomToolSource( { - repo: "RhysSullivan/executor-custom-tools-demo", - connection: "tools.github.user.demo", + url: "https://github.com/RhysSullivan/executor-custom-tools-demo", + token: "ghp_demo", }, fetchImpl, ); const listed = await listCustomToolSources(fetchImpl); expect(syncStatusLabel(syncResult)).toBe("Published 2 tools."); + expect(syncBody).toContain("ghp_demo"); expect(syncResult.tools).toEqual(["repo-summary", "stale-issues"]); + expect(listed.sources[0]?.hasToken).toBe(true); expect(listed.sources[0]?.tools).toEqual(["repo-summary", "stale-issues"]); }); @@ -102,8 +111,7 @@ describe("custom tools console client", () => { const result = await syncCustomToolSource( { - repo: "RhysSullivan/executor-custom-tools-demo", - connection: "tools.github.user.demo", + url: "https://github.com/RhysSullivan/executor-custom-tools-demo", }, fetchImpl, ); @@ -125,8 +133,7 @@ describe("custom tools console client", () => { const result = await syncCustomToolSource( { - repo: "RhysSullivan/executor-custom-tools-demo", - connection: "tools.github.user.demo", + url: "https://github.com/RhysSullivan/executor-custom-tools-demo", }, fetchImpl, ); diff --git a/packages/core/sdk/src/executor.ts b/packages/core/sdk/src/executor.ts index d703eb576..89f4fcc8d 100644 --- a/packages/core/sdk/src/executor.ts +++ b/packages/core/sdk/src/executor.ts @@ -3148,6 +3148,53 @@ export const createExecutor = => + Effect.gen(function* () { + const provider = credentialProviders.get(String(key)); + if (!provider) return null; + return yield* provider.get(id); + }); + + const providersHas = ( + key: ProviderKey, + id: ProviderItemId, + ): Effect.Effect => + Effect.gen(function* () { + const provider = credentialProviders.get(String(key)); + if (!provider) return false; + if (provider.has) return yield* provider.has(id); + const value = yield* provider.get(id); + return value !== null; + }); + + const providersSetDefault = ( + id: ProviderItemId, + value: string, + ): Effect.Effect => + Effect.gen(function* () { + const provider = defaultWritableProvider(); + if (!provider || !provider.set) { + return yield* new CredentialProviderNotRegisteredError({ + provider: ProviderKey.make("default"), + }); + } + yield* provider.set(id, value); + return provider.key; + }); + + const providersRemove = ( + key: ProviderKey, + id: ProviderItemId, + ): Effect.Effect => + Effect.gen(function* () { + const provider = credentialProviders.get(String(key)); + if (!provider || !provider.delete) return; + yield* provider.delete(id); + }); + // ------------------------------------------------------------------ // Policies — owner-ranked (user=0 inner, org=1 outer). // ------------------------------------------------------------------ @@ -3732,6 +3779,10 @@ export const createExecutor = providersList(), items: (key) => providersItems(key), + get: (key, id) => providersGet(key, id), + has: (key, id) => providersHas(key, id), + setDefault: (id, value) => providersSetDefault(id, value), + remove: (key, id) => providersRemove(key, id), }, oauth, execute: (address, args, options) => execute(address, args, options), diff --git a/packages/core/sdk/src/plugin.ts b/packages/core/sdk/src/plugin.ts index 42e0217f0..30e311c74 100644 --- a/packages/core/sdk/src/plugin.ts +++ b/packages/core/sdk/src/plugin.ts @@ -26,6 +26,7 @@ import type { ConnectionName, IntegrationSlug, Owner, + ProviderItemId, ProviderKey, Subject, Tenant, @@ -247,6 +248,26 @@ export interface PluginCtx { readonly items: ( provider: ProviderKey, ) => Effect.Effect; + /** Read an opaque item from a provider. Plugins use this for secret values + * they own that are not modeled as connections. */ + readonly get: ( + provider: ProviderKey, + id: ProviderItemId, + ) => Effect.Effect; + readonly has: ( + provider: ProviderKey, + id: ProviderItemId, + ) => Effect.Effect; + /** Write through the executor's default writable provider and return the + * provider key that owns the item. */ + readonly setDefault: ( + id: ProviderItemId, + value: string, + ) => Effect.Effect; + readonly remove: ( + provider: ProviderKey, + id: ProviderItemId, + ) => Effect.Effect; }; /** Shared OAuth service. */ diff --git a/packages/plugins/apps/src/api.ts b/packages/plugins/apps/src/api.ts index 63014c607..e05c779b6 100644 --- a/packages/plugins/apps/src/api.ts +++ b/packages/plugins/apps/src/api.ts @@ -23,6 +23,7 @@ export { } from "./plugin/bindings"; export { fetchGitHubSource, + parseGitHubSourceUrl, syncGitHubSource, GitHubSourceError, type GitHubSourceInput, diff --git a/packages/plugins/apps/src/backing/sqlite-apps-store.ts b/packages/plugins/apps/src/backing/sqlite-apps-store.ts index 18faa59e3..6b9825987 100644 --- a/packages/plugins/apps/src/backing/sqlite-apps-store.ts +++ b/packages/plugins/apps/src/backing/sqlite-apps-store.ts @@ -7,7 +7,7 @@ import { Effect, Schema } from "effect"; import { StorageError, type StorageFailure } from "@executor-js/sdk"; import type { AppDescriptor } from "../pipeline/descriptor"; -import type { AppsStore } from "../plugin/store"; +import type { AppsStore, GitHubSourceTokenRef } from "../plugin/store"; // --------------------------------------------------------------------------- // SQLite-backed AppsStore (self-hosted). One small SQLite file stores the @@ -19,6 +19,7 @@ const toUrl = (path: string): string => (path === ":memory:" ? path : `file:${re const SCHEMA = ` CREATE TABLE IF NOT EXISTS descriptors (tenant TEXT NOT NULL, scope TEXT NOT NULL, snapshot_id TEXT NOT NULL, descriptor TEXT NOT NULL, published_at INTEGER NOT NULL, PRIMARY KEY (tenant, scope)); CREATE TABLE IF NOT EXISTS scope_connections (tenant TEXT NOT NULL, connection_name TEXT NOT NULL, scope TEXT NOT NULL, PRIMARY KEY (tenant, connection_name)); +CREATE TABLE IF NOT EXISTS github_source_tokens (tenant TEXT NOT NULL, scope TEXT NOT NULL, provider TEXT NOT NULL, item_id TEXT NOT NULL, updated_at INTEGER NOT NULL, PRIMARY KEY (tenant, scope)); `; const storageFail = (message: string, cause: unknown): StorageFailure => @@ -140,5 +141,34 @@ export const makeSqliteAppsStore = (options: SqliteAppsStoreOptions): AppsStore }, catch: (cause) => storageFail("getScopeForConnection failed", cause), }), + putGitHubSourceTokenRef: (tenant, scope, ref) => + Effect.tryPromise({ + try: async () => { + await init(); + await client.execute({ + sql: "INSERT INTO github_source_tokens (tenant, scope, provider, item_id, updated_at) VALUES (?, ?, ?, ?, ?) ON CONFLICT(tenant, scope) DO UPDATE SET provider=excluded.provider, item_id=excluded.item_id, updated_at=excluded.updated_at", + args: [tenant, scope, ref.provider, ref.itemId, ref.updatedAt], + }); + }, + catch: (cause) => storageFail("putGitHubSourceTokenRef failed", cause), + }), + getGitHubSourceTokenRef: (tenant, scope) => + Effect.tryPromise({ + try: async (): Promise => { + await init(); + const res = await client.execute({ + sql: "SELECT provider, item_id, updated_at FROM github_source_tokens WHERE tenant = ? AND scope = ?", + args: [tenant, scope], + }); + const row = res.rows[0]; + if (!row) return null; + return { + provider: String(row.provider), + itemId: String(row.item_id), + updatedAt: Number(row.updated_at), + }; + }, + catch: (cause) => storageFail("getGitHubSourceTokenRef failed", cause), + }), }; }; diff --git a/packages/plugins/apps/src/index.ts b/packages/plugins/apps/src/index.ts index 7e6d24d6f..2c0418664 100644 --- a/packages/plugins/apps/src/index.ts +++ b/packages/plugins/apps/src/index.ts @@ -32,7 +32,9 @@ export { makeAppsStore, type AppsStore, type AppsStoreDeps, + type GitHubSourceTokenRef, descriptorCollection, + githubSourceTokenCollection, scopeConnectionCollection, } from "./plugin/store"; export { diff --git a/packages/plugins/apps/src/pipeline/descriptor.ts b/packages/plugins/apps/src/pipeline/descriptor.ts index 0d92c5c50..565184e3f 100644 --- a/packages/plugins/apps/src/pipeline/descriptor.ts +++ b/packages/plugins/apps/src/pipeline/descriptor.ts @@ -7,7 +7,7 @@ /** Descriptor schema version. Bumped on any breaking shape change. A reader * refuses a descriptor from a version it does not understand. */ -export const DESCRIPTOR_VERSION = 4 as const; +export const DESCRIPTOR_VERSION = 5 as const; /** Where an entry came from: path + content hash. Lets a projection point back * at the exact source bytes without re-reading the snapshot, and makes the @@ -38,10 +38,10 @@ export interface SourceSkippedArtifact { export interface GitHubSourceRef { readonly kind: "github"; + readonly url: string; readonly repo: string; readonly ref: string; readonly upstreamSha: string; - readonly connection?: string; readonly skipped?: readonly SourceSkippedArtifact[]; } diff --git a/packages/plugins/apps/src/plugin/apps-plugin.ts b/packages/plugins/apps/src/plugin/apps-plugin.ts index 728262e6e..afbb17885 100644 --- a/packages/plugins/apps/src/plugin/apps-plugin.ts +++ b/packages/plugins/apps/src/plugin/apps-plugin.ts @@ -1,4 +1,4 @@ -import { Effect } from "effect"; +import { Effect, Result } from "effect"; import { definePlugin, @@ -7,8 +7,8 @@ import { IntegrationSlug, AuthTemplateSlug, connectionIdentifier, - type ConnectionRef, - type Owner, + ProviderItemId, + ProviderKey, type PluginCtx, type ResolveToolsInput, type ResolveToolsResult, @@ -21,7 +21,11 @@ import { makeAppsStore } from "./store"; import type { ClientResolver, ConnectionCandidate } from "./bindings"; import type { IntegrationDecl, ToolDescriptor } from "../pipeline/descriptor"; import { PublishError } from "../pipeline/discover"; -import { syncGitHubSource, type GitHubSyncResult } from "../source/github-source"; +import { + parseGitHubSourceUrl, + syncGitHubSource, + type GitHubSyncResult, +} from "../source/github-source"; export const APPS_INTEGRATION_SLUG = "apps"; export const APPS_PLUGIN_ID = "apps"; @@ -59,21 +63,11 @@ const syncFailure = (message: string, path?: string): GitHubSyncResult => ({ ], }); -const sourceScopeFor = (input: { repo: string; connection: string }): string => - String(connectionIdentifier(`github ${input.connection} ${input.repo}`, "githubSource")); +const sourceScopeFor = (repo: string): string => + String(connectionIdentifier(`github ${repo}`, "githubSource")); -const parseConnectionAddress = (address: string): ConnectionRef | null => { - const parts = address.split("."); - if (parts.length !== 4 || parts[0] !== "tools") return null; - const [, integration, owner, name] = parts; - if (!integration || !name) return null; - if (owner !== "org" && owner !== "user") return null; - return { - owner: owner as Owner, - integration: IntegrationSlug.make(integration), - name: connectionIdentifier(name), - }; -}; +const sourceTokenItemId = (tenant: string, scope: string): ProviderItemId => + ProviderItemId.make(`apps:github-source:${tenant}:${scope}:token`); const configBaseUrl = (config: unknown): string | undefined => isRecord(config) && typeof config.baseUrl === "string" && config.baseUrl.length > 0 @@ -134,6 +128,41 @@ export const appsPlugin = definePlugin((options?: AppsPluginOptions) => { Effect.map((mapped) => mapped ?? scopeFromConnection(connectionName)), ); + const storeSourceToken = ( + ctx: PluginCtx, + tenant: string, + scope: string, + token: string, + ): Effect.Effect => + Effect.gen(function* () { + const itemId = sourceTokenItemId(tenant, scope); + const provider = yield* ctx.providers.setDefault(itemId, token).pipe(Effect.result); + if (Result.isFailure(provider)) return false; + const stored = yield* runtime.deps.store + .putGitHubSourceTokenRef(tenant, scope, { + provider: String(provider.success), + itemId: String(itemId), + updatedAt: Date.now(), + }) + .pipe(Effect.result); + return Result.isSuccess(stored); + }); + + const storedSourceToken = ( + ctx: PluginCtx, + tenant: string, + scope: string, + ): Effect.Effect => + Effect.gen(function* () { + const ref = yield* runtime.deps.store + .getGitHubSourceTokenRef(tenant, scope) + .pipe(Effect.orElseSucceed(() => null)); + if (!ref) return null; + return yield* ctx.providers + .get(ProviderKey.make(ref.provider), ProviderItemId.make(ref.itemId)) + .pipe(Effect.orElseSucceed(() => null)); + }); + const ensureCatalogConnection = (scope: string, ctx: PluginCtx) => Effect.gen(function* () { const tenant = tenantFor(ctx); @@ -200,6 +229,11 @@ export const appsPlugin = definePlugin((options?: AppsPluginOptions) => { schema: { Type: {} as Record }, indexes: [], }, + apps_github_source_token: { + name: "apps_github_source_token", + schema: { Type: {} as Record }, + indexes: [], + }, }, extension: () => ({ runtime }), @@ -229,45 +263,45 @@ export const appsPlugin = definePlugin((options?: AppsPluginOptions) => { execute: (args, { ctx }) => Effect.gen(function* () { const payload = isRecord(args) ? args : {}; - const repo = typeof payload.repo === "string" ? payload.repo : ""; - const connection = typeof payload.connection === "string" ? payload.connection : ""; - const ref = typeof payload.ref === "string" ? payload.ref : undefined; + const url = + typeof payload.url === "string" + ? payload.url + : typeof payload.repo === "string" + ? payload.repo + : ""; + const ref = + typeof payload.ref === "string" && payload.ref.trim().length > 0 + ? payload.ref.trim() + : undefined; + const providedToken = + typeof payload.token === "string" && payload.token.trim().length > 0 + ? payload.token.trim() + : null; const explicitScope = typeof payload.scope === "string" ? payload.scope : undefined; - if (!repo) return syncFailure('sync_github_source requires "repo"'); - if (!connection) return syncFailure('sync_github_source requires "connection"'); - - const sourceConnection = parseConnectionAddress(connection); - if (!sourceConnection) { - return syncFailure( - `GitHub connection must be a connection address like tools.github.user.main; got "${connection}"`, - connection, - ); - } - if (String(sourceConnection.integration) !== "github") { - return syncFailure( - `GitHub source sync requires a github connection; got "${sourceConnection.integration}"`, - connection, - ); - } - const existing = yield* ctx.connections - .get(sourceConnection) - .pipe(Effect.orElseSucceed(() => null)); - if (!existing) return syncFailure(`GitHub connection not found: ${connection}`); - const token = yield* ctx.connections - .resolveValue(sourceConnection) - .pipe(Effect.orElseSucceed(() => null)); + if (!url) return syncFailure('sync_github_source requires "url"'); + + const parsed = parseGitHubSourceUrl(url, { ref }); + if (!parsed.ok) return syncFailure(parsed.message, url); const integration = yield* ctx.core.integrations - .get(sourceConnection.integration) + .get(IntegrationSlug.make("github")) .pipe(Effect.orElseSucceed(() => null)); const tenant = tenantFor(ctx); - const scope = explicitScope ?? sourceScopeFor({ repo, connection }); + const scope = explicitScope ?? sourceScopeFor(parsed.value.repo); + if (providedToken) { + const stored = yield* storeSourceToken(ctx, tenant, scope, providedToken); + if (!stored) { + return syncFailure( + "No writable credential provider is available to store the GitHub source token.", + ); + } + } + const token = providedToken ?? (yield* storedSourceToken(ctx, tenant, scope)); const result = yield* syncGitHubSource({ runtime, tenant, scope, - repo, + url, ref, - connection, token, baseUrl: configBaseUrl(integration?.config), }); diff --git a/packages/plugins/apps/src/plugin/runtime.ts b/packages/plugins/apps/src/plugin/runtime.ts index b72d7dc18..f5233c43e 100644 --- a/packages/plugins/apps/src/plugin/runtime.ts +++ b/packages/plugins/apps/src/plugin/runtime.ts @@ -43,9 +43,10 @@ export interface AppsRuntimeDeps { export interface GitHubCustomToolsSourceSummary { readonly scope: string; + readonly url: string; readonly repo: string; readonly ref: string; - readonly connection?: string; + readonly hasToken: boolean; readonly upstreamSha: string; readonly snapshotId: string; readonly description?: string; @@ -376,17 +377,23 @@ export const makeAppsRuntime = (deps: AppsRuntimeDeps): AppsRuntime => { const tenant = resolveTenant(tenantInput); return deps.store.listDescriptors(tenant).pipe( Effect.orElseSucceed(() => []), - Effect.map((records) => - records.flatMap((record): GitHubCustomToolsSourceSummary[] => { - const { descriptor, publishedAt } = record; - const source = descriptor.source; - if (source?.kind !== "github") return []; - return [ - { + Effect.flatMap((records) => { + const githubRecords = records.flatMap((record) => { + const source = record.descriptor.source; + return source?.kind === "github" ? [{ record, source }] : []; + }); + return Effect.forEach(githubRecords, ({ record, source }) => + Effect.gen(function* () { + const { descriptor, publishedAt } = record; + const tokenRef = yield* deps.store + .getGitHubSourceTokenRef(tenant, descriptor.scope) + .pipe(Effect.orElseSucceed(() => null)); + return { scope: descriptor.scope, + url: source.url, repo: source.repo, ref: source.ref, - ...(source.connection ? { connection: source.connection } : {}), + hasToken: tokenRef !== null, upstreamSha: source.upstreamSha, snapshotId: descriptor.snapshotId, ...(descriptor.description ? { description: descriptor.description } : {}), @@ -396,10 +403,10 @@ export const makeAppsRuntime = (deps: AppsRuntimeDeps): AppsRuntime => { ...(source.skipped ?? []), ...(descriptor.skipped as readonly GitHubSkippedArtifact[]), ], - }, - ]; - }), - ), + } satisfies GitHubCustomToolsSourceSummary; + }), + ); + }), ); }, diff --git a/packages/plugins/apps/src/plugin/store.ts b/packages/plugins/apps/src/plugin/store.ts index 47f693401..12c956c96 100644 --- a/packages/plugins/apps/src/plugin/store.ts +++ b/packages/plugins/apps/src/plugin/store.ts @@ -30,11 +30,30 @@ export const scopeConnectionCollection = definePluginStorageCollection("apps_sco }, }); +export const githubSourceTokenCollection = definePluginStorageCollection( + "apps_github_source_token", + { + Type: {} as { + readonly tenant: string; + readonly scope: string; + readonly provider: string; + readonly itemId: string; + readonly updatedAt: number; + }, + }, +); + export interface AppDescriptorRecord { readonly descriptor: AppDescriptor; readonly publishedAt: number; } +export interface GitHubSourceTokenRef { + readonly provider: string; + readonly itemId: string; + readonly updatedAt: number; +} + export interface AppsStore { readonly putDescriptor: ( tenant: string, @@ -57,6 +76,15 @@ export interface AppsStore { tenant: string, connectionName: string, ) => Effect.Effect; + readonly putGitHubSourceTokenRef: ( + tenant: string, + scope: string, + ref: GitHubSourceTokenRef, + ) => Effect.Effect; + readonly getGitHubSourceTokenRef: ( + tenant: string, + scope: string, + ) => Effect.Effect; } export interface AppsStoreDeps { @@ -66,6 +94,7 @@ export interface AppsStoreDeps { export const makeAppsStore = (deps: AppsStoreDeps): AppsStore => { const descriptors = deps.pluginStorage.collection(descriptorCollection); const scopeConnections = deps.pluginStorage.collection(scopeConnectionCollection); + const githubSourceTokens = deps.pluginStorage.collection(githubSourceTokenCollection); // Tenant is now part of apps storage keys. The apps subsystem had not shipped // before this key shape, so no migration from the old scope-only keys is needed. const keyFor = (tenant: string, key: string): string => `${tenant}:${key}`; @@ -82,6 +111,18 @@ export const makeAppsStore = (deps: AppsStoreDeps): AppsStore => { scopeConnections .get({ key: keyFor(tenant, connectionName) }) .pipe(Effect.map((entry) => entry?.data.scope ?? null)), + putGitHubSourceTokenRef: (tenant, scope, ref) => + githubSourceTokens + .put({ + owner: "org", + key: keyFor(tenant, scope), + data: { tenant, scope, ...ref }, + }) + .pipe(Effect.asVoid), + getGitHubSourceTokenRef: (tenant, scope) => + githubSourceTokens + .get({ key: keyFor(tenant, scope) }) + .pipe(Effect.map((entry) => entry?.data ?? null)), putDescriptor: (tenant, owner, descriptor) => descriptors .put({ diff --git a/packages/plugins/apps/src/plugin/tenant-isolation.test.ts b/packages/plugins/apps/src/plugin/tenant-isolation.test.ts index df133b246..0cb43adc0 100644 --- a/packages/plugins/apps/src/plugin/tenant-isolation.test.ts +++ b/packages/plugins/apps/src/plugin/tenant-isolation.test.ts @@ -92,7 +92,7 @@ describe("tenant scope isolation", () => { runtime: hostA.runtime, tenant: tenantA, scope, - repo: "acme/tools", + url: "https://github.com/acme/tools", fetch: makeGitHubFetch({ toolName: "alpha", upstreamSha: "sha-a" }), }), ); @@ -101,7 +101,7 @@ describe("tenant scope isolation", () => { runtime: hostB.runtime, tenant: tenantB, scope, - repo: "acme/tools", + url: "https://github.com/acme/tools", fetch: makeGitHubFetch({ toolName: "beta", upstreamSha: "sha-b" }), }), ); diff --git a/packages/plugins/apps/src/react/AddCustomToolsSource.tsx b/packages/plugins/apps/src/react/AddCustomToolsSource.tsx index a6e3d1a1c..520347720 100644 --- a/packages/plugins/apps/src/react/AddCustomToolsSource.tsx +++ b/packages/plugins/apps/src/react/AddCustomToolsSource.tsx @@ -1,10 +1,6 @@ -import { useMemo, useState } from "react"; -import { useAtomValue } from "@effect/atom-react"; +import { useState } from "react"; import { Effect, Exit } from "effect"; -import * as AsyncResult from "effect/unstable/reactivity/AsyncResult"; -import { connectionsAllAtom } from "@executor-js/react/api/atoms"; -import { Alert, AlertDescription, AlertTitle } from "@executor-js/react/components/alert"; import { Button } from "@executor-js/react/components/button"; import { CardStack, @@ -13,53 +9,38 @@ import { } from "@executor-js/react/components/card-stack"; import { FloatActions } from "@executor-js/react/components/float-actions"; import { Input } from "@executor-js/react/components/input"; -import { NativeSelect, NativeSelectOption } from "@executor-js/react/components/native-select"; import { toast } from "@executor-js/react/components/sonner"; import { FormErrorAlert } from "@executor-js/react/lib/integration-add"; import { - consoleIntegrationHref, formatSyncErrors, - githubConnections, syncCustomToolSourceEffect, syncStatusLabel, - validateGitHubRepo, + validateGitHubSourceUrl, } from "./custom-tools-client"; export default function AddCustomToolsSource(props: { readonly onComplete: (slug?: string) => void; readonly onCancel: () => void; }) { - const connectionsResult = useAtomValue(connectionsAllAtom); - const connections = useMemo( - () => - AsyncResult.isSuccess(connectionsResult) ? githubConnections(connectionsResult.value) : [], - [connectionsResult], - ); - const [repo, setRepo] = useState(""); - const [ref, setRef] = useState(""); - const [connection, setConnection] = useState(""); - const [repoError, setRepoError] = useState(null); + const [url, setUrl] = useState(""); + const [token, setToken] = useState(""); + const [tokenRevealed, setTokenRevealed] = useState(false); + const [urlError, setUrlError] = useState(null); const [syncing, setSyncing] = useState(false); const [syncError, setSyncError] = useState(null); - const selectedConnection = connection || connections[0]?.address || ""; - const loadingConnections = !AsyncResult.isSuccess(connectionsResult); - const canSubmit = - repo.trim().length > 0 && selectedConnection.length > 0 && !syncing && !loadingConnections; - const submit = async () => { - const validation = validateGitHubRepo(repo); - setRepoError(validation); + const validation = validateGitHubSourceUrl(url); + setUrlError(validation); setSyncError(null); - if (validation || !selectedConnection) return; + if (validation) return; setSyncing(true); const exit = await Effect.runPromiseExit( syncCustomToolSourceEffect({ - repo, - ref, - connection: selectedConnection, + url, + token, }), ); if (Exit.isFailure(exit)) { @@ -88,66 +69,52 @@ export default function AddCustomToolsSource(props: { - +
{ - setRepo((event.target as HTMLInputElement).value); - setRepoError(null); + setUrl((event.target as HTMLInputElement).value); + setUrlError(null); setSyncError(null); }} - onBlur={() => setRepoError(validateGitHubRepo(repo))} - placeholder="owner/name" + onBlur={() => setUrlError(validateGitHubSourceUrl(url))} + placeholder="https://github.com/UsefulSoftwareCo/executor" className="font-mono text-sm" - aria-invalid={repoError ? true : undefined} + aria-invalid={urlError ? true : undefined} /> - {repoError &&

{repoError}

} + {urlError &&

{urlError}

}
- - setRef((event.target as HTMLInputElement).value)} - placeholder="default branch" - className="font-mono text-sm" - /> - - - {connections.length > 0 ? ( - setConnection((event.target as HTMLSelectElement).value)} - className="w-full font-mono text-sm" + @@ -158,7 +125,7 @@ export default function AddCustomToolsSource(props: { - diff --git a/packages/plugins/apps/src/react/CustomToolsAccountsPanel.tsx b/packages/plugins/apps/src/react/CustomToolsAccountsPanel.tsx index 2e4141070..cfe1eb708 100644 --- a/packages/plugins/apps/src/react/CustomToolsAccountsPanel.tsx +++ b/packages/plugins/apps/src/react/CustomToolsAccountsPanel.tsx @@ -89,24 +89,12 @@ export default function CustomToolsAccountsPanel() { const selected = sources.find((source) => source.scope === selectedScope) ?? sources[0] ?? null; const syncSelected = async (source: GitHubCustomToolsSourceSummary) => { - if (!source.connection) { - setNotice({ - status: "failed", - message: "Sync failed.", - added: [], - removed: [], - errors: ["This source does not have a recorded GitHub connection."], - }); - return; - } setSyncingScope(source.scope); setNotice(null); const beforeTools = source.tools; const exit = await Effect.runPromiseExit( syncCustomToolSourceEffect({ - repo: source.repo, - ref: source.ref, - connection: source.connection, + url: source.url, }), ); if (Exit.isFailure(exit)) { @@ -181,7 +169,7 @@ export default function CustomToolsAccountsPanel() {
+ - +
{notice && ( @@ -324,6 +307,19 @@ function SourceDetail(props: { ); } +function LinkField(props: { readonly label: string; readonly value: string }) { + return ( +
+
{props.label}
+
+ + {props.value} + +
+
+ ); +} + function Field(props: { readonly label: string; readonly value: string; readonly mono?: boolean }) { return (
diff --git a/packages/plugins/apps/src/react/custom-tools-client.ts b/packages/plugins/apps/src/react/custom-tools-client.ts index db96b8433..f56c0b6b2 100644 --- a/packages/plugins/apps/src/react/custom-tools-client.ts +++ b/packages/plugins/apps/src/react/custom-tools-client.ts @@ -1,7 +1,7 @@ -import type { Connection } from "@executor-js/sdk/shared"; import { Data, Effect, Schema } from "effect"; import type { GitHubCustomToolsSourceSummary, GitHubSyncResult } from "../api"; +import { parseGitHubSourceUrl } from "../source/github-url"; export const CUSTOM_TOOLS_PLUGIN_KEY = "apps"; export const CUSTOM_TOOLS_LABEL = "Custom tools"; @@ -11,9 +11,9 @@ export interface GitHubSourcesListResponse { } export interface SyncGitHubSourceRequest { - readonly repo: string; + readonly url: string; readonly ref?: string; - readonly connection: string; + readonly token?: string; } export type CustomToolsFetch = (input: RequestInfo | URL, init?: RequestInit) => Promise; @@ -22,17 +22,14 @@ export class CustomToolsClientError extends Data.TaggedError("CustomToolsClientE readonly message: string; }> {} -const REPO_RE = /^[A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+$/; +export { parseGitHubSourceUrl }; -export const validateGitHubRepo = (repo: string): string | null => { - const trimmed = repo.trim(); - if (trimmed.length === 0) return "Enter a GitHub repo."; - if (!REPO_RE.test(trimmed)) return "Use owner/name, for example UsefulSoftwareCo/executor."; - return null; +export const validateGitHubSourceUrl = (url: string): string | null => { + const parsed = parseGitHubSourceUrl(url); + return parsed.ok ? null : parsed.message; }; -export const githubConnections = (connections: readonly Connection[]): readonly Connection[] => - connections.filter((connection) => String(connection.integration) === "github"); +export const validateGitHubRepo = validateGitHubSourceUrl; const decodeJsonText = Schema.decodeUnknownEffect(Schema.UnknownFromJsonString); @@ -95,9 +92,9 @@ export const syncCustomToolSourceEffect = ( headers: { "content-type": "application/json" }, credentials: "same-origin", body: JSON.stringify({ - repo: input.repo.trim(), + url: input.url.trim(), ...(input.ref?.trim() ? { ref: input.ref.trim() } : {}), - connection: input.connection, + ...(input.token?.trim() ? { token: input.token.trim() } : {}), }), }), catch: () => new CustomToolsClientError({ message: "Failed to sync custom tools." }), diff --git a/packages/plugins/apps/src/source/github-source.test.ts b/packages/plugins/apps/src/source/github-source.test.ts index dd46ae9cb..b7d410fe2 100644 --- a/packages/plugins/apps/src/source/github-source.test.ts +++ b/packages/plugins/apps/src/source/github-source.test.ts @@ -9,7 +9,7 @@ import { makeSelfHostAppsRuntime } from "../plugin/self-host-runtime"; import { makeInMemoryAppsStore, makeTestResolver } from "../testing"; import { PUBLISH_LIMITS } from "../pipeline/publish"; import { scopeAddress } from "../seams/scope-address"; -import { fetchGitHubSource, syncGitHubSource } from "./github-source"; +import { fetchGitHubSource, parseGitHubSourceUrl, syncGitHubSource } from "./github-source"; const run = (effect: Effect.Effect): Promise => Effect.runPromise(effect); @@ -71,7 +71,9 @@ const makeGitHubFetch = (input: { } } let blobCalls = 0; - const fetch = (async (rawUrl: string) => { + const authHeaders: (string | null)[] = []; + const fetch = (async (rawUrl: string, init?: RequestInit) => { + authHeaders.push(new Headers(init?.headers).get("authorization")); const url = new URL(rawUrl); if (url.pathname === repoPath) return json({ default_branch: "main" }); if (url.pathname === `${repoPath}/git/ref/heads%2Fmain`) { @@ -110,10 +112,97 @@ const makeGitHubFetch = (input: { return { fetch, blobCalls: () => blobCalls, + authHeaders: () => authHeaders, }; }; describe("GitHub custom-tools source", () => { + it("parses strict GitHub source URLs", () => { + expect(parseGitHubSourceUrl("https://github.com/UsefulSoftwareCo/executor")).toEqual({ + ok: true, + value: { + owner: "UsefulSoftwareCo", + name: "executor", + repo: "UsefulSoftwareCo/executor", + url: "https://github.com/UsefulSoftwareCo/executor", + }, + }); + expect(parseGitHubSourceUrl("https://github.com/UsefulSoftwareCo/executor.git/")).toEqual({ + ok: true, + value: { + owner: "UsefulSoftwareCo", + name: "executor", + repo: "UsefulSoftwareCo/executor", + url: "https://github.com/UsefulSoftwareCo/executor", + }, + }); + expect(parseGitHubSourceUrl("https://github.com/UsefulSoftwareCo/executor/tree/main")).toEqual({ + ok: true, + value: { + owner: "UsefulSoftwareCo", + name: "executor", + repo: "UsefulSoftwareCo/executor", + ref: "main", + url: "https://github.com/UsefulSoftwareCo/executor/tree/main", + }, + }); + expect( + parseGitHubSourceUrl("https://github.com/UsefulSoftwareCo/executor/tree/feature/tools"), + ).toEqual({ + ok: true, + value: { + owner: "UsefulSoftwareCo", + name: "executor", + repo: "UsefulSoftwareCo/executor", + ref: "feature/tools", + url: "https://github.com/UsefulSoftwareCo/executor/tree/feature/tools", + }, + }); + expect( + parseGitHubSourceUrl( + "https://github.com/UsefulSoftwareCo/executor/commit/0123456789abcdef0123456789abcdef01234567", + ), + ).toEqual({ + ok: true, + value: { + owner: "UsefulSoftwareCo", + name: "executor", + repo: "UsefulSoftwareCo/executor", + ref: "0123456789abcdef0123456789abcdef01234567", + url: "https://github.com/UsefulSoftwareCo/executor/commit/0123456789abcdef0123456789abcdef01234567", + }, + }); + expect(parseGitHubSourceUrl("UsefulSoftwareCo/executor")).toEqual({ + ok: true, + value: { + owner: "UsefulSoftwareCo", + name: "executor", + repo: "UsefulSoftwareCo/executor", + url: "https://github.com/UsefulSoftwareCo/executor", + }, + }); + expect(parseGitHubSourceUrl("https://gitlab.com/UsefulSoftwareCo/executor")).toEqual({ + ok: false, + message: "GitHub source URLs must use github.com.", + }); + expect(parseGitHubSourceUrl("https://github.com/UsefulSoftwareCo")).toEqual({ + ok: false, + message: + "Use a GitHub repo URL like https://github.com/owner/repo, optionally with /tree/ or /commit/.", + }); + expect(parseGitHubSourceUrl("https://github.com/UsefulSoftwareCo/executor/issues")).toEqual({ + ok: false, + message: + "Use a GitHub repo URL like https://github.com/owner/repo, optionally with /tree/ or /commit/.", + }); + expect( + parseGitHubSourceUrl("https://github.com/UsefulSoftwareCo/executor/commit/main"), + ).toEqual({ + ok: false, + message: "GitHub commit URLs must include a commit SHA.", + }); + }); + it("fetches a repo fileset and publishes provenance, description, and skipped entries", async () => { const files = new Map([ [ @@ -129,8 +218,11 @@ describe("GitHub custom-tools source", () => { ["docs/readme.md", "ignored"], ]); const github = makeGitHubFetch({ files, upstreamSha: "commit-a" }); - const snapshot = await run(fetchGitHubSource({ repo: "acme/tools", fetch: github.fetch })); + const snapshot = await run( + fetchGitHubSource({ url: "https://github.com/acme/tools", fetch: github.fetch }), + ); expect([...snapshot.files.keys()].sort()).toEqual(["executor.json", "tools/hello.ts"]); + expect(github.authHeaders().every((header) => header === null)).toBe(true); expect(snapshot.skipped).toEqual([ { path: "workflows/deferred.ts", reason: "not supported yet" }, { path: "docs/readme.md", reason: "ignored" }, @@ -142,7 +234,7 @@ describe("GitHub custom-tools source", () => { syncGitHubSource({ runtime: host.runtime, scope: "githubTools", - repo: "acme/tools", + url: "https://github.com/acme/tools", fetch: github.fetch, }), ); @@ -156,6 +248,7 @@ describe("GitHub custom-tools source", () => { expect(descriptor?.description).toBe("Acme tools"); expect(descriptor?.source).toEqual({ kind: "github", + url: "https://github.com/acme/tools", repo: "acme/tools", ref: "main", upstreamSha: "commit-a", @@ -177,7 +270,7 @@ describe("GitHub custom-tools source", () => { syncGitHubSource({ runtime: host.runtime, scope: "githubTools", - repo: "acme/tools", + url: "https://github.com/acme/tools", fetch: github.fetch, }), ); @@ -185,7 +278,7 @@ describe("GitHub custom-tools source", () => { syncGitHubSource({ runtime: host.runtime, scope: "githubTools", - repo: "acme/tools", + url: "https://github.com/acme/tools", fetch: github.fetch, }), ); @@ -206,7 +299,7 @@ describe("GitHub custom-tools source", () => { } const github = makeGitHubFetch({ files }); const exit = await Effect.runPromiseExit( - fetchGitHubSource({ repo: "acme/tools", fetch: github.fetch }), + fetchGitHubSource({ url: "https://github.com/acme/tools", fetch: github.fetch }), ); expect(Exit.isFailure(exit)).toBe(true); expect(JSON.stringify(exit)).toContain("exceeding the limit"); @@ -230,7 +323,7 @@ export default defineTool({ description: "bad", input: { type: "object" }, async syncGitHubSource({ runtime: host.runtime, scope: "githubTools", - repo: "acme/tools", + url: "https://github.com/acme/tools", fetch: github.fetch, }), ); @@ -276,12 +369,14 @@ export default defineTool({ description: "bad", input: { type: "object" }, async syncGitHubSource({ runtime: host.runtime, scope: "githubTools", - repo: "acme/tools", + url: "https://github.com/acme/tools", + token: "source-token", fetch: github.fetch, }), ); expect(result.status).toBe("published"); + expect(github.authHeaders().every((header) => header === "Bearer source-token")).toBe(true); expect(result.tools).toEqual(["ok"]); expect(result.skipped).toEqual([ { path: "tools/link.ts", reason: "unsupported file type" }, diff --git a/packages/plugins/apps/src/source/github-source.ts b/packages/plugins/apps/src/source/github-source.ts index f351e19df..0bc38fe2e 100644 --- a/packages/plugins/apps/src/source/github-source.ts +++ b/packages/plugins/apps/src/source/github-source.ts @@ -5,9 +5,10 @@ import { PUBLISH_LIMITS, enforcePublishLimits } from "../pipeline/publish"; import type { AppSourceRef, SourceSkippedArtifact } from "../pipeline/descriptor"; import type { AppsRuntime } from "../plugin/runtime"; import type { FileSet, SnapshotId } from "../seams/artifact-store"; +import { parseGitHubSourceUrl, type ParsedGitHubSourceUrl } from "./github-url"; export interface GitHubSourceInput { - readonly repo: string; + readonly url: string; readonly ref?: string; readonly token?: string | null; readonly baseUrl?: string; @@ -16,6 +17,7 @@ export interface GitHubSourceInput { export interface GitHubSourceSnapshot { readonly files: FileSet; + readonly url: string; readonly repo: string; readonly ref: string; readonly upstreamSha: string; @@ -66,7 +68,6 @@ export interface SyncGitHubSourceInput extends GitHubSourceInput { readonly runtime: AppsRuntime; readonly tenant?: string; readonly scope: string; - readonly connection?: string; } const TOOL_RE = /^tools\/([a-z0-9][a-z0-9-]*)\.(ts|tsx|js|jsx)$/; @@ -75,23 +76,21 @@ const REGULAR_FILE_MODES = new Set(["100644", "100755"]); const trimBaseUrl = (baseUrl: string): string => baseUrl.replace(/\/+$/, ""); -const repoParts = (repo: string): { owner: string; name: string } | null => { - const [owner, name, ...rest] = repo.split("/"); - if (!owner || !name || rest.length > 0) return null; - return { owner, name }; -}; +const encodedRepoPath = (source: ParsedGitHubSourceUrl): string => + `${encodeURIComponent(source.owner)}/${encodeURIComponent(source.name)}`; -const encodedRepoPath = (repo: string): Effect.Effect => { - const parsed = repoParts(repo); - if (!parsed) { - return Effect.fail( - new GitHubSourceError({ - message: `GitHub repo must be "owner/name"; got "${repo}"`, - path: repo, - }), - ); - } - return Effect.succeed(`${encodeURIComponent(parsed.owner)}/${encodeURIComponent(parsed.name)}`); +const parseSourceInput = ( + input: GitHubSourceInput, +): Effect.Effect => { + const parsed = parseGitHubSourceUrl(input.url, { ref: input.ref }); + return parsed.ok + ? Effect.succeed(parsed.value) + : Effect.fail( + new GitHubSourceError({ + message: parsed.message, + path: input.url, + }), + ); }; const acceptedPath = (path: string): boolean => path === "executor.json" || TOOL_RE.test(path); @@ -298,9 +297,10 @@ export const fetchGitHubSource = ( input: GitHubSourceInput, ): Effect.Effect => Effect.gen(function* () { - const repoPath = yield* encodedRepoPath(input.repo); + const source = yield* parseSourceInput(input); + const repoPath = encodedRepoPath(source); const repo = yield* requestJson(input, `/repos/${repoPath}`); - const ref = input.ref ?? asString(repo.default_branch) ?? "main"; + const ref = source.ref ?? asString(repo.default_branch) ?? "main"; const branchRef = yield* requestJson( input, `/repos/${repoPath}/git/ref/${encodeURIComponent(`heads/${ref}`)}`, @@ -354,7 +354,8 @@ export const fetchGitHubSource = ( return { files, - repo: input.repo, + url: source.ref ? source.url : `https://github.com/${source.repo}`, + repo: source.repo, ref, upstreamSha, description: yield* executorDescription(files), @@ -379,12 +380,12 @@ const sourceErrorToSyncError = (sourceFailure: GitHubSourceError): SyncErrorData }; }; -const sourceRef = (snapshot: GitHubSourceSnapshot, connection?: string): AppSourceRef => ({ +const sourceRef = (snapshot: GitHubSourceSnapshot): AppSourceRef => ({ kind: "github", + url: snapshot.url, repo: snapshot.repo, ref: snapshot.ref, upstreamSha: snapshot.upstreamSha, - ...(connection ? { connection } : {}), skipped: snapshot.skipped, }); @@ -427,7 +428,7 @@ export const syncGitHubSource = (input: SyncGitHubSourceInput): Effect.Effect value.replace(/^\/+|\/+$/g, ""); + +const normalizeRepoName = (value: string): string => + value.endsWith(".git") ? value.slice(0, -4) : value; + +const invalidShape = + "Use a GitHub repo URL like https://github.com/owner/repo, optionally with /tree/ or /commit/."; + +const validateOwnerRepo = ( + owner: string | undefined, + nameInput: string | undefined, +): { readonly owner: string; readonly name: string } | null => { + const name = nameInput ? normalizeRepoName(nameInput) : ""; + if (!owner || !name) return null; + if (!OWNER_RE.test(owner) || !REPO_RE.test(name)) return null; + return { owner, name }; +}; + +const canonicalUrl = ( + owner: string, + name: string, + kind: "base" | "tree" | "commit", + ref?: string, +): string => { + const base = `https://github.com/${owner}/${name}`; + if (!ref || kind === "base") return base; + return `${base}/${kind}/${ref}`; +}; + +export const parseGitHubSourceUrl = ( + input: string, + options?: { readonly ref?: string | undefined }, +): GitHubSourceUrlParseResult => { + const trimmed = input.trim(); + const overrideRef = options?.ref?.trim(); + if (trimmed.length === 0) return { ok: false, message: "Enter a GitHub URL." }; + + const urlMatch = URL_RE.exec(trimmed); + const path = urlMatch ? trimSlashes(urlMatch[2] ?? "") : trimSlashes(trimmed); + if (urlMatch && urlMatch[1]?.toLowerCase() !== "github.com") { + return { ok: false, message: "GitHub source URLs must use github.com." }; + } + if (!urlMatch && /^[A-Za-z][A-Za-z0-9+.-]*:\/\//.test(trimmed)) { + return { ok: false, message: "GitHub source URLs must use github.com." }; + } + + const segments = path.split("/").filter((segment) => segment.length > 0); + const validated = validateOwnerRepo(segments[0], segments[1]); + if (!validated) return { ok: false, message: invalidShape }; + const { owner, name } = validated; + const repo = `${owner}/${name}`; + + if (segments.length === 2) { + const ref = overrideRef || undefined; + return { + ok: true, + value: { + owner, + name, + repo, + ...(ref ? { ref } : {}), + url: canonicalUrl(owner, name, ref ? "tree" : "base", ref), + }, + }; + } + + const route = segments[2]; + if (route === "tree") { + const ref = overrideRef || segments.slice(3).join("/"); + if (!ref) + return { ok: false, message: "GitHub tree URLs must include a branch, tag, or commit SHA." }; + return { + ok: true, + value: { owner, name, repo, ref, url: canonicalUrl(owner, name, "tree", ref) }, + }; + } + + if (route === "commit") { + const sourceRef = segments.length === 4 ? segments[3] : ""; + const ref = overrideRef || sourceRef; + if (!ref || !COMMIT_RE.test(ref)) { + return { ok: false, message: "GitHub commit URLs must include a commit SHA." }; + } + return { + ok: true, + value: { owner, name, repo, ref, url: canonicalUrl(owner, name, "commit", ref) }, + }; + } + + return { ok: false, message: invalidShape }; +}; diff --git a/packages/plugins/apps/src/testing/index.ts b/packages/plugins/apps/src/testing/index.ts index 24bb652af..e60cfbcba 100644 --- a/packages/plugins/apps/src/testing/index.ts +++ b/packages/plugins/apps/src/testing/index.ts @@ -21,6 +21,10 @@ export const makeInMemoryAppsStore = (): AppsStore & { const descriptors = new Map(); const publishedAt = new Map(); const scopeConnections = new Map(); + const githubSourceTokens = new Map< + string, + { provider: string; itemId: string; updatedAt: number } + >(); const keyFor = (tenant: string, key: string): string => `${tenant}:${key}`; return { descriptors, @@ -45,6 +49,10 @@ export const makeInMemoryAppsStore = (): AppsStore & { Effect.sync(() => void scopeConnections.set(keyFor(tenant, connectionName), scope)), getScopeForConnection: (tenant, connectionName) => Effect.sync(() => scopeConnections.get(keyFor(tenant, connectionName)) ?? null), + putGitHubSourceTokenRef: (tenant, scope, ref) => + Effect.sync(() => void githubSourceTokens.set(keyFor(tenant, scope), ref)), + getGitHubSourceTokenRef: (tenant, scope) => + Effect.sync(() => githubSourceTokens.get(keyFor(tenant, scope)) ?? null), }; }; From 774257f162d76d7106b0b1ecdc29783126e13faa Mon Sep 17 00:00:00 2001 From: Rhys Sullivan <39114868+RhysSullivan@users.noreply.github.com> Date: Mon, 6 Jul 2026 11:31:43 -0700 Subject: [PATCH 11/21] Surface custom tool binding errors --- apps/host-selfhost/src/apps-wire.node.test.ts | 42 ++++++++++++- .../core/execution/src/tool-invoker.test.ts | 48 +++++++++++++++ packages/core/execution/src/tool-invoker.ts | 24 ++++++++ .../apps/src/plugin/bindings.strict.test.ts | 40 ++++++++++++- packages/plugins/apps/src/plugin/bindings.ts | 60 ++++++++++++------- .../src/components/json-schema-form.test.ts | 3 + .../react/src/components/json-schema-form.tsx | 22 ++++--- 7 files changed, 206 insertions(+), 33 deletions(-) diff --git a/apps/host-selfhost/src/apps-wire.node.test.ts b/apps/host-selfhost/src/apps-wire.node.test.ts index 9b2d0657f..572226599 100644 --- a/apps/host-selfhost/src/apps-wire.node.test.ts +++ b/apps/host-selfhost/src/apps-wire.node.test.ts @@ -569,14 +569,37 @@ test("GitHub source sync publishes and invokes custom tools through self-host HT ); expect(JSON.stringify(searchResult)).toContain("deal-pipeline-sync"); - const schema = await requestJson<{ inputSchema: { properties?: Record } }>( + const schema = await requestJson<{ + inputSchema: { properties?: Record; required?: readonly string[] }; + }>( `/api/tools/schema?address=${encodeURIComponent(listed.find((tool) => tool.name === "deal-pipeline-sync")!.address)}`, { headers: jsonHeaders }, ); expect(schema.inputSchema.properties?.github).toMatchObject({ type: "string", enum: [GITHUB_CONNECTION], + default: GITHUB_CONNECTION, }); + expect(schema.inputSchema.required ?? []).not.toContain("github"); + + const defaultInvoke = (await executeResult( + callAppToolCode("deal-pipeline-sync", { + owner: OWNER, + repo: REPO, + }), + )) as { result?: { ok?: boolean; data?: { synced?: number }; error?: unknown } }; + expect(defaultInvoke.result?.ok).toBe(true); + expect(defaultInvoke.result?.data?.synced).toBe(2); + + const bareNameInvoke = (await executeResult( + callAppToolCode("deal-pipeline-sync", { + github: "main", + owner: OWNER, + repo: REPO, + }), + )) as { result?: { ok?: boolean; data?: { synced?: number }; error?: unknown } }; + expect(bareNameInvoke.result?.ok).toBe(true); + expect(bareNameInvoke.result?.data?.synced).toBe(2); const invoke = (await executeResult( callAppToolCode("deal-pipeline-sync", { @@ -588,6 +611,23 @@ test("GitHub source sync publishes and invokes custom tools through self-host HT expect(invoke.result?.ok).toBe(true); expect(invoke.result?.data?.synced).toBe(2); + const bogusConnection = await execute( + callAppToolCode("deal-pipeline-sync", { + github: "garbage", + owner: OWNER, + repo: REPO, + }), + ); + const bogusBody = JSON.stringify(bogusConnection); + const bogusStructured = bogusConnection.structured.result as { + readonly result?: { readonly error?: { readonly message?: string } }; + }; + expect(bogusConnection.status).toBe("completed"); + expect(bogusStructured.result?.error?.message).toContain( + 'unknown connection "garbage" for role "github" (github)', + ); + expect(bogusBody).not.toContain("Internal tool error"); + const ledger = await github.ledger.list(); const sourceFetches = ledger.filter((entry) => entry.path === `/repos/${OWNER}/${REPO}`); expect(sourceFetches.some((entry) => entry.identity.user?.login === OWNER)).toBe(true); diff --git a/packages/core/execution/src/tool-invoker.test.ts b/packages/core/execution/src/tool-invoker.test.ts index f117f1706..9216b8f4f 100644 --- a/packages/core/execution/src/tool-invoker.test.ts +++ b/packages/core/execution/src/tool-invoker.test.ts @@ -266,6 +266,27 @@ const errorPlugin = makeTestPlugin({ ], }); +const bindingErrorPlugin = makeTestPlugin({ + pluginId: "binding-error-test", + integration: "binding", + tools: [ + { + name: "sync", + description: "Sync with a caller-selected connection", + inputJsonSchema: EmptyInputJson, + validator: EmptyValidator, + handler: () => + Effect.fail({ + _tag: "BindingError", + message: 'unknown connection "personal" for role "github" (github)', + role: "github", + integration: "github", + requestedConnection: "personal", + }), + }, + ], +}); + const oauthErrorPlugin = definePlugin(() => ({ id: "oauth-error-test" as const, storage: () => ({}), @@ -1075,6 +1096,33 @@ describe("tool discovery", () => { }), ); + it.effect("returns binding errors as ToolResult.fail", () => + Effect.gen(function* () { + const executor = yield* makeExecutorWith([bindingErrorPlugin] as const); + yield* provision(executor as never, [ + { pluginId: "binding-error-test", integration: "binding" }, + ]); + const invoker = makeExecutorToolInvoker(executor, { + invokeOptions: { onElicitation: acceptAll }, + }); + + const result = yield* invoker.invoke({ path: "binding.org.main.sync", args: {} }); + + expect(result).toEqual({ + ok: false, + error: { + code: "binding_error", + message: 'unknown connection "personal" for role "github" (github)', + details: { + role: "github", + integration: "github", + requestedConnection: "personal", + }, + }, + }); + }), + ); + it.effect("preserves nested upstream error bodies through ToolResult.fail", () => Effect.gen(function* () { const executor = yield* makeExecutorWith([structuredFailurePlugin] as const); diff --git a/packages/core/execution/src/tool-invoker.ts b/packages/core/execution/src/tool-invoker.ts index be6105d6d..2b47970f6 100644 --- a/packages/core/execution/src/tool-invoker.ts +++ b/packages/core/execution/src/tool-invoker.ts @@ -188,6 +188,28 @@ const credentialResolutionToolFailure = (input: { }, }); +const bindingToolFailure = (value: unknown): ToolError | null => { + if (!Predicate.isTagged(value, "BindingError")) return null; + const maybeBinding = value as { + readonly message?: unknown; + readonly role?: unknown; + readonly integration?: unknown; + readonly requestedConnection?: unknown; + }; + const details: Record = {}; + if (typeof maybeBinding.role === "string") details.role = maybeBinding.role; + if (typeof maybeBinding.integration === "string") details.integration = maybeBinding.integration; + if (typeof maybeBinding.requestedConnection === "string") { + details.requestedConnection = maybeBinding.requestedConnection; + } + return { + code: "binding_error", + message: + typeof maybeBinding.message === "string" ? maybeBinding.message : "Tool binding failed.", + ...(Object.keys(details).length > 0 ? { details } : {}), + }; +}; + const expectedToolFailure = (value: unknown): ToolError | null => { if (Predicate.isTagged(value, "ToolNotFoundError") && "address" in value) { const suggestions = @@ -210,6 +232,8 @@ const expectedToolFailure = (value: unknown): ToolError | null => { } if (Predicate.isTagged(value, "ToolInvocationError")) { const cause = (value as { readonly cause?: unknown }).cause; + const binding = bindingToolFailure(cause); + if (binding) return binding; const issues = validationIssues(cause); if (issues) { return { diff --git a/packages/plugins/apps/src/plugin/bindings.strict.test.ts b/packages/plugins/apps/src/plugin/bindings.strict.test.ts index 9b7e10b98..c77f06c93 100644 --- a/packages/plugins/apps/src/plugin/bindings.strict.test.ts +++ b/packages/plugins/apps/src/plugin/bindings.strict.test.ts @@ -30,7 +30,7 @@ const okResolver: ClientResolver = { ]), resolveConnection: ({ connection }) => Effect.succeed( - connection === "tools.github.user.main" || connection === "main" + connection === "tools.github.user.main" ? { address: "tools.github.user.main", integration: "github", @@ -45,6 +45,9 @@ const okResolver: ClientResolver = { const fails = (effect: Effect.Effect): Promise => Effect.runPromiseExit(effect).then((exit) => Exit.isFailure(exit)); +const failureText = (effect: Effect.Effect): Promise => + Effect.runPromiseExit(effect).then((exit) => JSON.stringify(exit)); + describe("HandleBridge strict dispatch", () => { const declared: Record = { gh: { integration: "github" }, @@ -119,4 +122,39 @@ describe("HandleBridge strict dispatch", () => { ), ).toBe(true); }); + + it("accepts an unambiguous bare connection name", async () => { + const resolved = await Effect.runPromise( + resolveIntegrationBindings(declared, { gh: "main", q: "docs" }, okResolver), + ); + expect(resolved.bindings).toEqual({ gh: "tools.github.user.main" }); + expect(resolved.input).toEqual({ q: "docs" }); + }); + + it("fails when a bare connection name is ambiguous", async () => { + const resolver: ClientResolver = { + ...okResolver, + listConnections: () => + Effect.succeed([ + { + address: "tools.github.user.personal", + integration: "github", + name: "personal", + owner: "user", + }, + { + address: "tools.github.org.personal", + integration: "github", + name: "personal", + owner: "org", + }, + ]), + }; + const text = await failureText( + resolveIntegrationBindings(declared, { gh: "personal" }, resolver), + ); + expect(text).toContain("ambiguous connection name"); + expect(text).toContain("tools.github.user.personal"); + expect(text).toContain("tools.github.org.personal"); + }); }); diff --git a/packages/plugins/apps/src/plugin/bindings.ts b/packages/plugins/apps/src/plugin/bindings.ts index e57436fab..8afeaa9c8 100644 --- a/packages/plugins/apps/src/plugin/bindings.ts +++ b/packages/plugins/apps/src/plugin/bindings.ts @@ -100,31 +100,45 @@ const resolveRequestedConnection = ( requested: string, resolver: ClientResolver, ): Effect.Effect => - resolver.resolveConnection({ connection: requested }).pipe( - Effect.flatMap((connection) => { - if (!connection) { - return Effect.fail( - new BindingError({ - role, - integration: decl.integration, - message: `unknown connection "${requested}" for role "${role}" (${decl.integration})`, - requestedConnection: requested, - }), - ); - } + Effect.gen(function* () { + const connection = yield* resolver.resolveConnection({ connection: requested }); + if (connection) { if (connection.integration !== decl.integration) { - return Effect.fail( - new BindingError({ - role, - integration: decl.integration, - message: `connection "${requested}" belongs to integration "${connection.integration}", not "${decl.integration}" for role "${role}"`, - requestedConnection: requested, - }), - ); + return yield* new BindingError({ + role, + integration: decl.integration, + message: `connection "${requested}" belongs to integration "${connection.integration}", not "${decl.integration}" for role "${role}"`, + requestedConnection: requested, + }); } - return Effect.succeed(connection.address); - }), - ); + return connection.address; + } + + const connections = yield* resolver.listConnections({ integration: decl.integration }); + const matches = requested.startsWith("tools.") + ? [] + : connections.filter( + (candidate) => (candidate.name ?? candidate.address.split(".").at(-1)) === requested, + ); + if (matches.length === 1) return matches[0]!.address; + if (matches.length > 1) { + return yield* new BindingError({ + role, + integration: decl.integration, + message: `ambiguous connection name "${requested}" for role "${role}" (${decl.integration}); choose one of ${matches + .map((candidate) => candidate.address) + .join(", ")}`, + requestedConnection: requested, + }); + } + + return yield* new BindingError({ + role, + integration: decl.integration, + message: `unknown connection "${requested}" for role "${role}" (${decl.integration}); use a full connection address or an unambiguous connection name${connections.length > 0 ? ` (available: ${connections.map((candidate) => `${candidate.address} or ${candidate.name ?? candidate.address.split(".").at(-1)}`).join(", ")})` : ""}`, + requestedConnection: requested, + }); + }); export interface ResolvedIntegrationBindings { readonly input: unknown; diff --git a/packages/react/src/components/json-schema-form.test.ts b/packages/react/src/components/json-schema-form.test.ts index 5074fa909..6017e8555 100644 --- a/packages/react/src/components/json-schema-form.test.ts +++ b/packages/react/src/components/json-schema-form.test.ts @@ -101,6 +101,9 @@ describe("json-schema-form value transforms", () => { expect(defaultForSchema({ type: "boolean" }, root)).toBe(false); expect(defaultForSchema({ type: "object" }, root)).toEqual({}); expect(defaultForSchema({ type: "array" }, root)).toEqual([]); + expect(defaultForSchema({ type: "string", default: "tools.github.user.main" }, root)).toBe( + "tools.github.user.main", + ); expect(defaultForSchema({ enum: ["a", "b"] }, root)).toBe("a"); expect(defaultForSchema({ const: 7 }, root)).toBe(7); }); diff --git a/packages/react/src/components/json-schema-form.tsx b/packages/react/src/components/json-schema-form.tsx index de26787a6..f2400c8b3 100644 --- a/packages/react/src/components/json-schema-form.tsx +++ b/packages/react/src/components/json-schema-form.tsx @@ -105,9 +105,10 @@ const DEFAULT_BY_TYPE: Record unknown> = { }; /** Seed used when the user clicks "Add" for an optional property or array item. - * Never auto-injected on load — only on explicit add. */ + * Schema defaults can also be displayed without being injected into JSON. */ export const defaultForSchema = (schema: JsonSchema, root: JsonSchema): unknown => { const s = deepResolve(schema, root); + if (s.default !== undefined) return s.default; if (s.const !== undefined) return s.const; if (Array.isArray(s.enum) && s.enum.length > 0) return s.enum[0]; const t = primaryType(s); @@ -223,8 +224,8 @@ export function JsonSchemaForm(props: { } // --------------------------------------------------------------------------- -// ObjectFields — renders an object's properties (required first, then present -// optionals, then "Add" rows for absent optionals) +// ObjectFields — renders an object's properties (required/default-backed first, +// then present optionals, then "Add" rows for absent optionals) // --------------------------------------------------------------------------- function ObjectFields(props: { @@ -242,11 +243,15 @@ function ObjectFields(props: { const requiredSet = new Set(schema.required ?? []); const entries = Object.entries(properties); - const present = entries.filter( - ([key]: [string, JsonSchema]) => key in value || requiredSet.has(key), - ); + const present = entries.filter(([key, propSchema]: [string, JsonSchema]) => { + const resolved = deepResolve(propSchema, root); + return key in value || requiredSet.has(key) || resolved.default !== undefined; + }); const addable = entries.filter( - ([key]: [string, JsonSchema]) => !requiredSet.has(key) && !(key in value), + ([key, propSchema]: [string, JsonSchema]) => + !requiredSet.has(key) && + !(key in value) && + deepResolve(propSchema, root).default === undefined, ); // No properties → render nothing (no "takes no arguments" noise); a zero-arg @@ -316,6 +321,7 @@ function JsonSchemaField(props: { const { root, name, fieldId, required, value, present, onChange, depth, disabled } = props; const schema = deepResolve(props.schema, root); const description = schemaDescription(schema); + const fieldValue = value === undefined && schema.default !== undefined ? schema.default : value; const header = (
@@ -353,7 +359,7 @@ function JsonSchemaField(props: { schema={schema} root={root} fieldId={fieldId} - value={value} + value={fieldValue} onChange={onChange} depth={depth} disabled={disabled} From 18fafe4d7dd8bdf79dc50778e0ca5f6d11c67c12 Mon Sep 17 00:00:00 2001 From: Rhys Sullivan <39114868+RhysSullivan@users.noreply.github.com> Date: Mon, 6 Jul 2026 11:51:06 -0700 Subject: [PATCH 12/21] Require canonical custom tool connection addresses --- apps/host-selfhost/src/apps-wire.node.test.ts | 14 +++++-- .../apps/src/plugin/bindings.strict.test.ts | 39 ++++--------------- packages/plugins/apps/src/plugin/bindings.ts | 19 +-------- 3 files changed, 19 insertions(+), 53 deletions(-) diff --git a/apps/host-selfhost/src/apps-wire.node.test.ts b/apps/host-selfhost/src/apps-wire.node.test.ts index 572226599..d6ec27a0f 100644 --- a/apps/host-selfhost/src/apps-wire.node.test.ts +++ b/apps/host-selfhost/src/apps-wire.node.test.ts @@ -591,15 +591,21 @@ test("GitHub source sync publishes and invokes custom tools through self-host HT expect(defaultInvoke.result?.ok).toBe(true); expect(defaultInvoke.result?.data?.synced).toBe(2); - const bareNameInvoke = (await executeResult( + const bareNameInvoke = await execute( callAppToolCode("deal-pipeline-sync", { github: "main", owner: OWNER, repo: REPO, }), - )) as { result?: { ok?: boolean; data?: { synced?: number }; error?: unknown } }; - expect(bareNameInvoke.result?.ok).toBe(true); - expect(bareNameInvoke.result?.data?.synced).toBe(2); + ); + const bareNameStructured = bareNameInvoke.structured.result as { + readonly result?: { readonly error?: { readonly message?: string } }; + }; + expect(bareNameInvoke.status).toBe("completed"); + expect(bareNameStructured.result?.error?.message).toContain( + 'unknown connection "main" for role "github" (github)', + ); + expect(bareNameStructured.result?.error?.message).toContain(GITHUB_CONNECTION); const invoke = (await executeResult( callAppToolCode("deal-pipeline-sync", { diff --git a/packages/plugins/apps/src/plugin/bindings.strict.test.ts b/packages/plugins/apps/src/plugin/bindings.strict.test.ts index c77f06c93..8f2691df1 100644 --- a/packages/plugins/apps/src/plugin/bindings.strict.test.ts +++ b/packages/plugins/apps/src/plugin/bindings.strict.test.ts @@ -123,38 +123,15 @@ describe("HandleBridge strict dispatch", () => { ).toBe(true); }); - it("accepts an unambiguous bare connection name", async () => { - const resolved = await Effect.runPromise( - resolveIntegrationBindings(declared, { gh: "main", q: "docs" }, okResolver), - ); - expect(resolved.bindings).toEqual({ gh: "tools.github.user.main" }); - expect(resolved.input).toEqual({ q: "docs" }); - }); - - it("fails when a bare connection name is ambiguous", async () => { - const resolver: ClientResolver = { - ...okResolver, - listConnections: () => - Effect.succeed([ - { - address: "tools.github.user.personal", - integration: "github", - name: "personal", - owner: "user", - }, - { - address: "tools.github.org.personal", - integration: "github", - name: "personal", - owner: "org", - }, - ]), - }; + it("rejects a bare connection name and lists valid addresses", async () => { const text = await failureText( - resolveIntegrationBindings(declared, { gh: "personal" }, resolver), + resolveIntegrationBindings(declared, { gh: "main" }, okResolver), ); - expect(text).toContain("ambiguous connection name"); - expect(text).toContain("tools.github.user.personal"); - expect(text).toContain("tools.github.org.personal"); + expect(text).toContain('"message"'); + expect(text).toContain("unknown connection"); + expect(text).toContain("for role"); + expect(text).toContain('"requestedConnection":"main"'); + expect(text).toContain("tools.github.user.main"); + expect(text).not.toContain("unambiguous connection name"); }); }); diff --git a/packages/plugins/apps/src/plugin/bindings.ts b/packages/plugins/apps/src/plugin/bindings.ts index 8afeaa9c8..c27cfa68a 100644 --- a/packages/plugins/apps/src/plugin/bindings.ts +++ b/packages/plugins/apps/src/plugin/bindings.ts @@ -115,27 +115,10 @@ const resolveRequestedConnection = ( } const connections = yield* resolver.listConnections({ integration: decl.integration }); - const matches = requested.startsWith("tools.") - ? [] - : connections.filter( - (candidate) => (candidate.name ?? candidate.address.split(".").at(-1)) === requested, - ); - if (matches.length === 1) return matches[0]!.address; - if (matches.length > 1) { - return yield* new BindingError({ - role, - integration: decl.integration, - message: `ambiguous connection name "${requested}" for role "${role}" (${decl.integration}); choose one of ${matches - .map((candidate) => candidate.address) - .join(", ")}`, - requestedConnection: requested, - }); - } - return yield* new BindingError({ role, integration: decl.integration, - message: `unknown connection "${requested}" for role "${role}" (${decl.integration}); use a full connection address or an unambiguous connection name${connections.length > 0 ? ` (available: ${connections.map((candidate) => `${candidate.address} or ${candidate.name ?? candidate.address.split(".").at(-1)}`).join(", ")})` : ""}`, + message: `unknown connection "${requested}" for role "${role}" (${decl.integration})${connections.length > 0 ? `; choose one of ${connections.map((candidate) => candidate.address).join(", ")}` : ""}`, requestedConnection: requested, }); }); From 6e2010f7a6d9d2242ba1c24cd92d5ca4e54a265f Mon Sep 17 00:00:00 2001 From: Rhys Sullivan <39114868+RhysSullivan@users.noreply.github.com> Date: Mon, 6 Jul 2026 13:52:00 -0700 Subject: [PATCH 13/21] Refresh custom tool connection schemas at read time --- apps/host-selfhost/src/apps-wire.node.test.ts | 71 ++++++++++++++++++- packages/core/sdk/src/executor.ts | 37 +++++++--- packages/core/sdk/src/plugin.ts | 19 +++++ .../apps/src/plugin/apps-plugin.test.ts | 18 ++++- .../plugins/apps/src/plugin/apps-plugin.ts | 39 ++++++---- .../src/components/json-schema-form.test.ts | 8 +++ .../react/src/components/json-schema-form.tsx | 17 +++++ 7 files changed, 184 insertions(+), 25 deletions(-) diff --git a/apps/host-selfhost/src/apps-wire.node.test.ts b/apps/host-selfhost/src/apps-wire.node.test.ts index d6ec27a0f..e4ad3a7ce 100644 --- a/apps/host-selfhost/src/apps-wire.node.test.ts +++ b/apps/host-selfhost/src/apps-wire.node.test.ts @@ -426,7 +426,7 @@ const approvalFiles = (): Record => ({ "tools/approval-bridge.ts": approvalBridgeSource, }); -const registerGitHubIntegration = async (emulator: Emulator, token: string): Promise => { +const registerGitHubSpec = async (emulator: Emulator): Promise => { const specResponse = await fetch(`${server.baseUrl}/api/openapi/specs`, { method: "POST", headers: jsonHeaders, @@ -439,13 +439,18 @@ const registerGitHubIntegration = async (emulator: Emulator, token: string): Pro }); expect([200, 409]).toContain(specResponse.status); await specResponse.text(); +}; + +const createGitHubConnection = async (token: string): Promise => { const existing = await requestJson( "/api/connections?integration=github", { headers: jsonHeaders, }, ); - if (existing.some((connection) => connection.address === GITHUB_CONNECTION)) return; + if (existing.some((connection) => connection.address === GITHUB_CONNECTION)) { + return GITHUB_CONNECTION; + } const created = await postJson<{ address: string }>("/api/connections", { owner: "user", name: "main", @@ -454,6 +459,12 @@ const registerGitHubIntegration = async (emulator: Emulator, token: string): Pro value: token, }); expect(created.address).toBe(GITHUB_CONNECTION); + return created.address; +}; + +const registerGitHubIntegration = async (emulator: Emulator, token: string): Promise => { + await registerGitHubSpec(emulator); + await createGitHubConnection(token); }; const sourceUrl = (repo = REPO_FULL_NAME): string => `https://github.com/${repo}`; @@ -522,6 +533,62 @@ afterAll(async () => { await github?.close(); }); +test("custom tool connection schema updates when connections change after sync", async () => { + const credential = await github.credentials.mint({ + type: "api-key", + login: OWNER, + scopes: ["repo", "user"], + }); + const token = credential.token; + expect(token).toBeTruthy(); + await registerGitHubSpec(github); + const initialConnections = await requestJson( + "/api/connections?integration=github", + { headers: jsonHeaders }, + ); + expect(initialConnections).toEqual([]); + await putRepoFiles(github, token!, initialFiles()); + + const published = await syncSource(); + expect(published.status).toBe("published"); + + const listed = await listAppTools(); + const dealSync = listed.find((tool) => tool.name === "deal-pipeline-sync"); + expect(dealSync).toBeTruthy(); + const beforeConnection = await requestJson<{ + inputSchema: { properties?: Record; required?: readonly string[] }; + }>(`/api/tools/schema?address=${encodeURIComponent(dealSync!.address)}`, { + headers: jsonHeaders, + }); + expect(beforeConnection.inputSchema.properties?.github).toMatchObject({ + type: "string", + enum: [], + }); + expect(beforeConnection.inputSchema.required ?? []).toContain("github"); + + await createGitHubConnection(token!); + const afterConnection = await requestJson<{ + inputSchema: { properties?: Record; required?: readonly string[] }; + }>(`/api/tools/schema?address=${encodeURIComponent(dealSync!.address)}`, { + headers: jsonHeaders, + }); + expect(afterConnection.inputSchema.properties?.github).toMatchObject({ + type: "string", + enum: [GITHUB_CONNECTION], + default: GITHUB_CONNECTION, + }); + expect(afterConnection.inputSchema.required ?? []).not.toContain("github"); + + const defaultInvoke = (await executeResult( + callAppToolCode("deal-pipeline-sync", { + owner: OWNER, + repo: REPO, + }), + )) as { result?: { ok?: boolean; data?: { synced?: number }; error?: unknown } }; + expect(defaultInvoke.result?.ok).toBe(true); + expect(defaultInvoke.result?.data?.synced).toBe(0); +}); + test("GitHub source sync publishes and invokes custom tools through self-host HTTP", async () => { const credential = await github.credentials.mint({ type: "api-key", diff --git a/packages/core/sdk/src/executor.ts b/packages/core/sdk/src/executor.ts index 89f4fcc8d..289894ec7 100644 --- a/packages/core/sdk/src/executor.ts +++ b/packages/core/sdk/src/executor.ts @@ -3089,6 +3089,30 @@ export const createExecutor = + pluginStorageFailure(row.plugin_id, "projectToolSchema", cause), + ), + ) + : null; + const inputSchema = + projected && Object.prototype.hasOwnProperty.call(projected, "inputSchema") + ? projected.inputSchema + : tool.inputSchema; + const outputSchema = + projected && Object.prototype.hasOwnProperty.call(projected, "outputSchema") + ? projected.outputSchema + : tool.outputSchema; + const definitionRows = yield* core.findMany("definition", { where: (b: AnyCb) => b.and( @@ -3100,15 +3124,12 @@ export const createExecutor = (); for (const def of definitionRows) defs.set(def.name, decodeJsonColumn(def.schema)); - const referenced = collectReferencedDefinitions( - [tool.inputSchema, tool.outputSchema], - defs, - ); + const referenced = collectReferencedDefinitions([inputSchema, outputSchema], defs); const preview = yield* Effect.tryPromise({ try: () => buildToolTypeScriptPreview({ - inputSchema: tool.inputSchema, - outputSchema: tool.outputSchema, + inputSchema, + outputSchema, defs, }), catch: (cause) => @@ -3120,8 +3141,8 @@ export const createExecutor = 0 ? (referenced as Record) diff --git a/packages/core/sdk/src/plugin.ts b/packages/core/sdk/src/plugin.ts index 30e311c74..189335249 100644 --- a/packages/core/sdk/src/plugin.ts +++ b/packages/core/sdk/src/plugin.ts @@ -334,6 +334,18 @@ export interface ResolveToolsResult { readonly incomplete?: boolean; } +export interface ProjectToolSchemaInput { + readonly ctx: PluginCtx; + readonly toolRow: ToolInvocationRow; + readonly inputSchema?: unknown; + readonly outputSchema?: unknown; +} + +export interface ProjectToolSchemaResult { + readonly inputSchema?: unknown; + readonly outputSchema?: unknown; +} + // --------------------------------------------------------------------------- // Resolved credential handed to `invokeTool` so the plugin renders auth onto // the request (D11: "auth state derived into the auth-template format"). @@ -622,6 +634,13 @@ export interface PluginSpec< * for catalogs derived purely from stored state (specs, static config). */ readonly remoteToolCatalog?: boolean; + /** Project a persisted tool row's stable schemas into the request-visible + * schema view. Use this only for volatile presentation data that should be + * current at read time, not persisted at catalog-refresh time. */ + readonly projectToolSchema?: ( + input: ProjectToolSchemaInput, + ) => Effect.Effect; + /** Invoke a dynamic tool. Called when the static-handler map doesn't have the * address. The plugin applies `input.credential` to the outbound request. */ readonly invokeTool?: (input: InvokeToolInput) => Effect.Effect; diff --git a/packages/plugins/apps/src/plugin/apps-plugin.test.ts b/packages/plugins/apps/src/plugin/apps-plugin.test.ts index 20bcd7bb2..66c1a3614 100644 --- a/packages/plugins/apps/src/plugin/apps-plugin.test.ts +++ b/packages/plugins/apps/src/plugin/apps-plugin.test.ts @@ -73,7 +73,23 @@ describe("appsPlugin custom-tools contract", () => { ); const syncTool = resolved.tools.find((tool) => String(tool.name) === "deal-pipeline-sync"); expect(syncTool).toBeTruthy(); - const inputSchema = syncTool!.inputSchema as { + const persistedInputSchema = syncTool!.inputSchema as { + properties: Record; + }; + expect(persistedInputSchema.properties.crm).toBeUndefined(); + + const projected = await run( + plugin.projectToolSchema!({ + ctx: { owner: { tenant: "org" } }, + toolRow: { + name: "deal-pipeline-sync", + connection: appConnection, + }, + inputSchema: syncTool!.inputSchema, + outputSchema: syncTool!.outputSchema, + } as never), + ); + const inputSchema = projected.inputSchema as { properties: Record; required?: string[]; }; diff --git a/packages/plugins/apps/src/plugin/apps-plugin.ts b/packages/plugins/apps/src/plugin/apps-plugin.ts index afbb17885..be6e53211 100644 --- a/packages/plugins/apps/src/plugin/apps-plugin.ts +++ b/packages/plugins/apps/src/plugin/apps-plugin.ts @@ -332,21 +332,35 @@ export const appsPlugin = definePlugin((options?: AppsPluginOptions) => { const scope = yield* scopeFor(tenant, String(connection.name)); const descriptor = yield* runtime.getDescriptor(tenant, scope); if (!descriptor) return { tools: [] } satisfies ResolveToolsResult; - const resolver = - makeResolver && ctx ? makeResolver({ ctx, scope, tool: "*" }) : runtime.deps.resolver; const tools: ToolDef[] = []; for (const t of descriptor.tools) { - const byRole: Record = {}; - for (const [role, decl] of Object.entries(t.integrations)) { - byRole[role] = yield* resolver - .listConnections({ integration: decl.integration }) - .pipe(Effect.orElseSucceed(() => [])); - } - tools.push(projectTool(t, byRole)); + tools.push(projectTool(t)); } return { tools } satisfies ResolveToolsResult; }), + projectToolSchema: ({ ctx, toolRow, inputSchema, outputSchema }) => + Effect.gen(function* () { + const tenant = tenantFor(ctx); + const scope = yield* scopeFor(tenant, String(toolRow.connection)); + const descriptor = yield* runtime.getDescriptor(tenant, scope); + const toolDesc = descriptor?.tools.find((t) => t.name === toolRow.name); + if (!toolDesc) return { inputSchema, outputSchema }; + const resolver = makeResolver + ? makeResolver({ ctx, scope, tool: toolRow.name }) + : runtime.deps.resolver; + const byRole: Record = {}; + for (const [role, decl] of Object.entries(toolDesc.integrations)) { + byRole[role] = yield* resolver + .listConnections({ integration: decl.integration }) + .pipe(Effect.orElseSucceed(() => [])); + } + return { + inputSchema: projectInputSchema(toolDesc.inputSchema, toolDesc.integrations, byRole), + outputSchema: toolDesc.outputSchema, + }; + }), + invokeTool: ({ ctx, toolRow, args, invokeOptions }: InvokeToolInput) => Effect.gen(function* () { const tenant = tenantFor(ctx); @@ -382,13 +396,10 @@ export const appsPlugin = definePlugin((options?: AppsPluginOptions) => { }; }); -const projectTool = ( - descriptor: ToolDescriptor, - byRole: Readonly>, -): ToolDef => ({ +const projectTool = (descriptor: ToolDescriptor): ToolDef => ({ name: ToolName.make(descriptor.name), description: descriptor.description, - inputSchema: projectInputSchema(descriptor.inputSchema, descriptor.integrations, byRole), + inputSchema: descriptor.inputSchema, outputSchema: descriptor.outputSchema, annotations: { requiresApproval: descriptor.annotations?.destructive === true, diff --git a/packages/react/src/components/json-schema-form.test.ts b/packages/react/src/components/json-schema-form.test.ts index 6017e8555..e97d03ad8 100644 --- a/packages/react/src/components/json-schema-form.test.ts +++ b/packages/react/src/components/json-schema-form.test.ts @@ -5,6 +5,7 @@ import * as Schema from "effect/Schema"; import { coerceNumber, defaultForSchema, + emptyEnumGuidance, isRenderableObjectSchema, missingRequiredFields, removeProperty, @@ -108,6 +109,13 @@ describe("json-schema-form value transforms", () => { expect(defaultForSchema({ const: 7 }, root)).toBe(7); }); + it("empty enum guidance names connection integrations", () => { + expect(emptyEnumGuidance("Connection to use for github (github)")).toBe( + "Connect github first.", + ); + expect(emptyEnumGuidance("Choose a value")).toBe("No options available."); + }); + it("isRenderableObjectSchema gates form-mode availability", () => { expect(isRenderableObjectSchema(SCHEMA)).toBe(true); diff --git a/packages/react/src/components/json-schema-form.tsx b/packages/react/src/components/json-schema-form.tsx index f2400c8b3..52fc90d1a 100644 --- a/packages/react/src/components/json-schema-form.tsx +++ b/packages/react/src/components/json-schema-form.tsx @@ -95,6 +95,11 @@ const schemaDescription = (schema: JsonSchema): string | undefined => { return undefined; }; +export const emptyEnumGuidance = (description: string | undefined): string => { + const integration = description?.match(/^Connection to use for .+ \(([^)]+)\)$/)?.[1]; + return integration ? `Connect ${integration} first.` : "No options available."; +}; + const DEFAULT_BY_TYPE: Record unknown> = { string: () => "", number: () => 0, @@ -413,6 +418,18 @@ function FieldControl(props: { ); } + if (Array.isArray(schema.enum) && schema.enum.length === 0) { + const guidance = emptyEnumGuidance(schemaDescription(schema)); + return ( +
+ + {guidance} + +

{guidance}

+
+ ); + } + // enum — native select. Optional/unset gets a leading empty option. if (Array.isArray(schema.enum) && schema.enum.length > 0) { const options = schema.enum; From aeb4833d29cfa48144dea06f7d47af6ea96a9cc2 Mon Sep 17 00:00:00 2001 From: Rhys Sullivan <39114868+RhysSullivan@users.noreply.github.com> Date: Mon, 6 Jul 2026 17:46:39 -0700 Subject: [PATCH 14/21] Make custom tools sources first-class integrations --- .../scripts/custom-tools-demo.ts | 9 +- apps/host-selfhost/src/apps-route.ts | 46 +- apps/host-selfhost/src/apps-wire.node.test.ts | 271 ++++++++- .../src/custom-tools-console.web.test.ts | 36 +- packages/core/sdk/src/executor.ts | 13 + packages/core/sdk/src/plugin.ts | 11 + packages/plugins/apps/src/api.ts | 3 - .../apps/src/backing/git-artifact-store.ts | 15 +- .../apps/src/backing/libsql-scope-db.ts | 20 +- .../apps/src/backing/scope-collision.test.ts | 33 +- .../apps/src/backing/sqlite-apps-store.ts | 53 +- packages/plugins/apps/src/index.ts | 7 +- .../apps/src/plugin/apps-plugin.test.ts | 47 +- .../plugins/apps/src/plugin/apps-plugin.ts | 533 ++++++++++++------ packages/plugins/apps/src/plugin/runtime.ts | 73 ++- packages/plugins/apps/src/plugin/store.ts | 70 +-- .../apps/src/react/AddCustomToolsSource.tsx | 51 +- .../src/react/CustomToolsAccountsPanel.tsx | 248 ++++---- .../apps/src/react/custom-tools-client.ts | 70 ++- .../plugins/apps/src/seams/artifact-store.ts | 1 + packages/plugins/apps/src/seams/scope-db.ts | 1 + packages/plugins/apps/src/source/app-slug.ts | 17 + packages/plugins/apps/src/testing/index.ts | 29 +- .../react/src/pages/integration-detail.tsx | 16 +- 24 files changed, 1146 insertions(+), 527 deletions(-) create mode 100644 packages/plugins/apps/src/source/app-slug.ts diff --git a/apps/host-selfhost/scripts/custom-tools-demo.ts b/apps/host-selfhost/scripts/custom-tools-demo.ts index f2ea8fe8e..18ea61999 100644 --- a/apps/host-selfhost/scripts/custom-tools-demo.ts +++ b/apps/host-selfhost/scripts/custom-tools-demo.ts @@ -7,6 +7,7 @@ const HOST = "127.0.0.1"; const BASE_URL = `http://${HOST}:${PORT}`; const ORIGIN = "http://custom-tools-demo.local"; const DEMO_REPO = "RhysSullivan/executor-custom-tools-demo"; +const DEMO_APP = "demo-tools"; const ADMIN_EMAIL = "admin@custom-tools-demo.local"; const ADMIN_PASSWORD = "admin-pass-123456"; const USER_EMAIL = "rhys@custom-tools-demo.local"; @@ -232,7 +233,7 @@ const htmlEscape = (value: string): string => .replaceAll('"', """); const toolInvokeCode = (toolName: string, args: unknown): string => ` -const found = await tools.search({ namespace: "apps", query: ${JSON.stringify(toolName)}, limit: 20 }); +const found = await tools.search({ namespace: ${JSON.stringify(DEMO_APP)}, query: ${JSON.stringify(toolName)}, limit: 20 }); const item = found.items.find((candidate) => candidate.path.endsWith(${JSON.stringify(toolName)})); if (!item) return { ok: false, missing: ${JSON.stringify(toolName)}, found }; let fn = tools; @@ -251,10 +252,11 @@ const buildBanner = (input: { readonly consoleUi: string; }): string => { const sync = curlJson("/api/apps/sources/github/sync", { + name: DEMO_APP, url: input.repoUrl, token: input.githubToken, }); - const list = `curl -sS '${BASE_URL}/api/tools?integration=apps' | jq`; + const list = `curl -sS '${BASE_URL}/api/tools?integration=${DEMO_APP}' | jq`; const repoSummary = curlJson("/api/executions", { code: toolInvokeCode("repo-summary", { github: input.connectionAddress, @@ -290,6 +292,9 @@ Auth: GitHub invocation connection: ${input.connectionAddress} +Custom tools app: + ${DEMO_APP} + Sync the repo: ${sync} diff --git a/apps/host-selfhost/src/apps-route.ts b/apps/host-selfhost/src/apps-route.ts index ef886ceca..b2af813c1 100644 --- a/apps/host-selfhost/src/apps-route.ts +++ b/apps/host-selfhost/src/apps-route.ts @@ -8,7 +8,7 @@ import { makeScopedExecutor, type Principal, } from "@executor-js/api/server"; -import { ToolAddress } from "@executor-js/sdk"; +import { IntegrationSlug, ToolAddress } from "@executor-js/sdk"; import { SelfHostScopedExecutorSeams } from "./execution"; import { SelfHostDb, type SelfHostDbHandle } from "./db/self-host-db"; @@ -43,6 +43,21 @@ const buildExecutor = (principal: Principal, request: Request) => { : withOrigin; }; +const sourceSlugFromRoute = () => + HttpRouter.params.pipe( + Effect.map((params) => params.slug ?? ""), + Effect.flatMap((slug) => + slug.length > 0 + ? Effect.succeed(slug) + : Effect.fail( + new AppsSyncRouteError({ + status: 400, + message: "Missing custom tools source slug", + }), + ), + ), + ); + export interface SelfHostAppsSyncRouteDeps { readonly identity: Layer.Layer; readonly db: SelfHostDbHandle; @@ -118,5 +133,34 @@ export const makeSelfHostAppsSyncRoute = ({ identity, db }: SelfHostAppsSyncRout }), ), ), + HttpRouter.add( + "GET", + "/api/apps/sources/github/:slug", + routeHandler((request) => + Effect.gen(function* () { + const identityProvider = yield* IdentityProvider; + const principal = yield* identityProvider.authenticate(request); + const slug = yield* sourceSlugFromRoute(); + const executor = yield* buildExecutor(principal, request); + return yield* executor.execute(ToolAddress.make("executor.apps.get_github_source"), { + slug, + }); + }), + ), + ), + HttpRouter.add( + "DELETE", + "/api/apps/sources/github/:slug", + routeHandler((request) => + Effect.gen(function* () { + const identityProvider = yield* IdentityProvider; + const principal = yield* identityProvider.authenticate(request); + const slug = yield* sourceSlugFromRoute(); + const executor = yield* buildExecutor(principal, request); + yield* executor.integrations.remove(IntegrationSlug.make(slug)); + return { removed: true }; + }), + ), + ), ).pipe(HttpRouter.provideRequest(services)); }; diff --git a/apps/host-selfhost/src/apps-wire.node.test.ts b/apps/host-selfhost/src/apps-wire.node.test.ts index e4ad3a7ce..881016097 100644 --- a/apps/host-selfhost/src/apps-wire.node.test.ts +++ b/apps/host-selfhost/src/apps-wire.node.test.ts @@ -1,13 +1,14 @@ import { randomBytes } from "node:crypto"; -import { mkdtempSync } from "node:fs"; +import { existsSync, mkdtempSync } from "node:fs"; import { createServer, type IncomingMessage, type ServerResponse } from "node:http"; import { createServer as createNetServer } from "node:net"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { afterAll, beforeAll, expect, test } from "@effect/vitest"; -import { Effect, Schema } from "effect"; +import { Effect, Option, Schema } from "effect"; import { createEmulator, type Emulator } from "@executor-js/emulate"; +import { createClient } from "@libsql/client"; const dataDir = mkdtempSync(join(tmpdir(), "eh-apps-wire-")); const dbPath = join(dataDir, "data.db"); @@ -21,6 +22,9 @@ const OWNER = "syncer"; const REPO = "custom-tools"; const REPO_FULL_NAME = `${OWNER}/${REPO}`; const GITHUB_CONNECTION = "tools.github.user.main"; +const APP_SLUG = "custom-tools-app"; +const SCHEMA_APP_SLUG = "schema-tools-app"; +const APPROVAL_APP_SLUG = "approval-tools-app"; interface TestHttpServer { readonly baseUrl: string; @@ -33,7 +37,7 @@ interface SyncResult { readonly upstreamSha?: string; readonly tools: readonly string[]; readonly skipped: readonly { readonly path: string; readonly reason: string }[]; - readonly errors?: readonly unknown[]; + readonly errors?: readonly { readonly message?: string }[]; } interface ToolRow { @@ -46,6 +50,14 @@ interface ConnectionRow { readonly address: string; } +interface IntegrationRow { + readonly slug: string; + readonly name: string; + readonly kind: string; + readonly canRemove: boolean; + readonly displayUrl?: string; +} + interface ExecuteResponse { readonly status: "completed" | "paused"; readonly text: string; @@ -194,6 +206,48 @@ const postJson = (path: string, body: unknown, expectedStatus = 200): Promise expectedStatus, ); +const deleteJson = (path: string, expectedStatus = 200): Promise => + requestJson( + path, + { + method: "DELETE", + headers: jsonHeaders, + }, + expectedStatus, + ); + +const querySqliteRows = async ( + path: string, + sql: string, + args: readonly unknown[] = [], +): Promise[]> => { + const client = createClient({ url: `file:${path}` }); + const result = await client.execute({ sql, args: [...args] as never }); + client.close(); + return result.rows.map((row) => ({ ...row })); +}; + +const sourceStorageKey = (scope: string): string => + `v2-${Buffer.from(JSON.stringify([TEST_ORG, scope]), "utf8").toString("hex")}`; + +const sourceTokenItemId = (scope: string): string => + `apps:github-source:${TEST_ORG}:${scope}:token`; + +const DbJsonObject = Schema.Record(Schema.String, Schema.Unknown); +const DbJsonObjectFromString = Schema.fromJsonString(DbJsonObject); +const decodeDbJsonObject = Schema.decodeUnknownOption(DbJsonObject); +const decodeDbJsonObjectFromString = Schema.decodeUnknownOption(DbJsonObjectFromString); + +const decodeDbJson = (value: unknown): Record => { + if (value instanceof ArrayBuffer) { + return decodeDbJson(Buffer.from(value).toString("utf8")); + } + if (typeof value === "string") { + return Option.getOrElse(decodeDbJsonObjectFromString(value), () => ({})); + } + return Option.getOrElse(decodeDbJsonObject(value), () => ({})); +}; + const githubFetch = async ( emulator: Emulator, token: string, @@ -236,7 +290,7 @@ const putRepoFiles = async ( const tree = await githubFetch<{ sha: string }>( emulator, token, - `/repos/${OWNER}/${REPO}/git/trees`, + `/repos/${OWNER}/${repo}/git/trees`, { method: "POST", body: JSON.stringify({ @@ -252,7 +306,7 @@ const putRepoFiles = async ( const commit = await githubFetch<{ sha: string }>( emulator, token, - `/repos/${OWNER}/${REPO}/git/commits`, + `/repos/${OWNER}/${repo}/git/commits`, { method: "POST", body: JSON.stringify({ @@ -469,19 +523,36 @@ const registerGitHubIntegration = async (emulator: Emulator, token: string): Pro const sourceUrl = (repo = REPO_FULL_NAME): string => `https://github.com/${repo}`; -const syncSource = ( - input: { readonly repo?: string; readonly token?: string } = {}, +const addSource = ( + input: { readonly name?: string; readonly repo?: string; readonly token?: string } = {}, ): Promise => postJson("/api/apps/sources/github/sync", { + name: input.name ?? APP_SLUG, url: sourceUrl(input.repo), ...(input.token ? { token: input.token } : {}), }); -const listAppTools = (): Promise => - requestJson("/api/tools?integration=apps", { +const syncSource = ( + input: { readonly name?: string; readonly token?: string } = {}, +): Promise => + postJson("/api/apps/sources/github/sync", { + slug: input.name ?? APP_SLUG, + ...(input.token ? { token: input.token } : {}), + }); + +const listAppTools = (app = APP_SLUG): Promise => + requestJson(`/api/tools?integration=${encodeURIComponent(app)}`, { headers: jsonHeaders, }); +const getSource = ( + app: string, +): Promise<{ readonly source: { readonly slug: string; readonly name: string } | null }> => + requestJson(`/api/apps/sources/github/${encodeURIComponent(app)}`, { headers: jsonHeaders }); + +const removeSource = (app: string): Promise<{ readonly removed: boolean }> => + deleteJson(`/api/apps/sources/github/${encodeURIComponent(app)}`); + const execute = (code: string): Promise => postJson("/api/executions", { code, autoApprove: true }); @@ -495,8 +566,8 @@ const executeResult = async (code: string): Promise => { return response.structured.result; }; -const callAppToolCode = (toolName: string, args: unknown): string => ` -const found = await tools.search({ namespace: "apps", query: ${JSON.stringify(toolName)}, limit: 20 }); +const callAppToolCode = (toolName: string, args: unknown, namespace = APP_SLUG): string => ` +const found = await tools.search({ namespace: ${JSON.stringify(namespace)}, query: ${JSON.stringify(toolName)}, limit: 20 }); const item = found.items.find((candidate) => candidate.path.endsWith(${JSON.stringify(toolName)})); if (!item) return { ok: false, missing: ${JSON.stringify(toolName)}, found }; let fn = tools; @@ -549,10 +620,10 @@ test("custom tool connection schema updates when connections change after sync", expect(initialConnections).toEqual([]); await putRepoFiles(github, token!, initialFiles()); - const published = await syncSource(); + const published = await addSource({ name: SCHEMA_APP_SLUG }); expect(published.status).toBe("published"); - const listed = await listAppTools(); + const listed = await listAppTools(SCHEMA_APP_SLUG); const dealSync = listed.find((tool) => tool.name === "deal-pipeline-sync"); expect(dealSync).toBeTruthy(); const beforeConnection = await requestJson<{ @@ -580,10 +651,14 @@ test("custom tool connection schema updates when connections change after sync", expect(afterConnection.inputSchema.required ?? []).not.toContain("github"); const defaultInvoke = (await executeResult( - callAppToolCode("deal-pipeline-sync", { - owner: OWNER, - repo: REPO, - }), + callAppToolCode( + "deal-pipeline-sync", + { + owner: OWNER, + repo: REPO, + }, + SCHEMA_APP_SLUG, + ), )) as { result?: { ok?: boolean; data?: { synced?: number }; error?: unknown } }; expect(defaultInvoke.result?.ok).toBe(true); expect(defaultInvoke.result?.data?.synced).toBe(0); @@ -609,7 +684,7 @@ test("GitHub source sync publishes and invokes custom tools through self-host HT }); expect(unauthorized.status).toBe(401); - const published = await syncSource(); + const published = await addSource({ name: APP_SLUG, token: token! }); expect(published.status).toBe("published"); expect(JSON.stringify(published)).not.toContain(token!); expect(published.tools).toEqual(["deal-pipeline-sync", "find-deal-docs"]); @@ -621,18 +696,17 @@ test("GitHub source sync publishes and invokes custom tools through self-host HT expect(tokenUpToDate.status).toBe("up-to-date"); expect(JSON.stringify(tokenUpToDate)).not.toContain(token!); - const sourcesList = await requestJson<{ sources: readonly { hasToken: boolean }[] }>( - "/api/apps/sources/github", - { headers: jsonHeaders }, - ); - expect(sourcesList.sources[0]?.hasToken).toBe(true); + const sourcesList = await requestJson<{ + sources: readonly { readonly slug: string; readonly hasToken: boolean }[]; + }>("/api/apps/sources/github", { headers: jsonHeaders }); + expect(sourcesList.sources.find((source) => source.slug === APP_SLUG)?.hasToken).toBe(true); expect(JSON.stringify(sourcesList)).not.toContain(token!); const listed = await listAppTools(); expect(listed.map((tool) => tool.name).sort()).toEqual(["deal-pipeline-sync", "find-deal-docs"]); const searchResult = await executeResult( - 'const found = await tools.search({ namespace: "apps", query: "pipeline", limit: 10 }); return found.items.map((item) => item.path);', + `const found = await tools.search({ namespace: ${JSON.stringify(APP_SLUG)}, query: "pipeline", limit: 10 }); return found.items.map((item) => item.path);`, ); expect(JSON.stringify(searchResult)).toContain("deal-pipeline-sync"); @@ -760,6 +834,139 @@ test("GitHub source sync publishes and invokes custom tools through self-host HT expect(afterSkipped.some((tool) => tool.address.includes("workflow"))).toBe(false); }); +test("custom tools sources are per-integration and remove tears down owned state", async () => { + const credential = await github.credentials.mint({ + type: "api-key", + login: OWNER, + scopes: ["repo", "user"], + }); + const token = credential.token; + expect(token).toBeTruthy(); + await registerGitHubIntegration(github, token!); + await putRepoFiles(github, token!, initialFiles()); + + const firstApp = "side-tools-a"; + const secondApp = "side-tools-b"; + const first = await addSource({ name: firstApp, token: token! }); + const second = await addSource({ name: secondApp, token: token! }); + expect(first.status).toBe("published"); + expect(second.status).toBe("published"); + + const collision = await addSource({ name: firstApp, token: token! }); + expect(collision.status).toBe("failed"); + expect(collision.errors?.[0]?.message).toContain( + `Integration "${firstApp}" already exists. Choose another name.`, + ); + + const integrations = await requestJson("/api/integrations", { + headers: jsonHeaders, + }); + expect(integrations).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + slug: firstApp, + name: firstApp, + kind: "apps", + canRemove: true, + displayUrl: sourceUrl(), + }), + expect.objectContaining({ + slug: secondApp, + name: secondApp, + kind: "apps", + canRemove: true, + displayUrl: sourceUrl(), + }), + ]), + ); + + const configRows = await querySqliteRows( + dbPath, + "SELECT slug, plugin_id, config FROM integration WHERE slug IN (?, ?) ORDER BY slug", + [firstApp, secondApp], + ); + expect(configRows).toHaveLength(2); + for (const row of configRows) { + expect(row.plugin_id).toBe("apps"); + const config = decodeDbJson(row.config); + expect(config).toMatchObject({ + kind: "github", + repoUrl: sourceUrl(), + repo: REPO_FULL_NAME, + scope: row.slug, + }); + expect(JSON.stringify(config)).not.toContain(token!); + } + + expect((await listAppTools(firstApp)).map((tool) => tool.address).sort()).toEqual([ + `tools.${firstApp}.user.main.deal-pipeline-sync`, + `tools.${firstApp}.user.main.find-deal-docs`, + ]); + expect((await listAppTools(secondApp)).map((tool) => tool.address).sort()).toEqual([ + `tools.${secondApp}.user.main.deal-pipeline-sync`, + `tools.${secondApp}.user.main.find-deal-docs`, + ]); + + const secondInvoke = (await executeResult( + callAppToolCode( + "deal-pipeline-sync", + { + owner: OWNER, + repo: REPO, + }, + secondApp, + ), + )) as { result?: { ok?: boolean; data?: { synced?: number }; error?: unknown } }; + expect(secondInvoke.result?.ok).toBe(true); + + const secondKey = sourceStorageKey(secondApp); + const secondArtifactDir = join(dataDir, "apps", "artifacts", `${secondKey}.git`); + const secondScopeDb = join(dataDir, "apps", "scope-db", `${secondKey}.db`); + expect(existsSync(secondArtifactDir)).toBe(true); + expect(existsSync(secondScopeDb)).toBe(true); + + const tokenRowsBefore = await querySqliteRows( + dbPath, + "SELECT key FROM plugin_storage WHERE plugin_id = ? AND collection = ? AND key = ?", + ["encryptedSecrets", "secrets", sourceTokenItemId(secondApp)], + ); + expect(tokenRowsBefore).toHaveLength(1); + + expect(await getSource(secondApp)).toMatchObject({ + source: { slug: secondApp, name: secondApp }, + }); + expect(await removeSource(secondApp)).toEqual({ removed: true }); + expect(await getSource(secondApp)).toEqual({ source: null }); + + const secondIntegration = await fetch( + `${server.baseUrl}/api/integrations/${encodeURIComponent(secondApp)}`, + { headers: jsonHeaders }, + ); + expect(secondIntegration.status).toBe(404); + await secondIntegration.text(); + + expect(await listAppTools(secondApp)).toEqual([]); + expect((await listAppTools(firstApp)).map((tool) => tool.address).sort()).toEqual([ + `tools.${firstApp}.user.main.deal-pipeline-sync`, + `tools.${firstApp}.user.main.find-deal-docs`, + ]); + + const descriptorRows = await querySqliteRows( + join(dataDir, "apps", "store.sqlite"), + "SELECT scope FROM descriptors WHERE tenant = ? AND scope = ?", + [TEST_ORG, secondApp], + ); + expect(descriptorRows).toEqual([]); + const tokenRowsAfter = await querySqliteRows( + dbPath, + "SELECT key FROM plugin_storage WHERE plugin_id = ? AND collection = ? AND key = ?", + ["encryptedSecrets", "secrets", sourceTokenItemId(secondApp)], + ); + expect(tokenRowsAfter).toEqual([]); + expect(existsSync(secondArtifactDir)).toBe(false); + expect(existsSync(secondScopeDb)).toBe(false); +}); + test("bridged integration calls inherit the caller approval handler", async () => { const credential = await github.credentials.mint({ type: "api-key", @@ -771,7 +978,7 @@ test("bridged integration calls inherit the caller approval handler", async () = await registerGitHubIntegration(github, token!); await createIssue(github, token!, "Approval-gated issue"); await putRepoFiles(github, token!, approvalFiles()); - const published = await syncSource({ token: token! }); + const published = await addSource({ name: APPROVAL_APP_SLUG, token: token! }); expect(published.status).toBe("published"); const githubTools = await requestJson("/api/tools?integration=github", { @@ -792,11 +999,15 @@ test("bridged integration calls inherit the caller approval handler", async () = ).length; const response = await executeWithApprovalPause( - callAppToolCode("approval-bridge", { - github: GITHUB_CONNECTION, - owner: OWNER, - repo: REPO, - }), + callAppToolCode( + "approval-bridge", + { + github: GITHUB_CONNECTION, + owner: OWNER, + repo: REPO, + }, + APPROVAL_APP_SLUG, + ), ); expect(response.status).toBe("paused"); diff --git a/apps/host-selfhost/src/custom-tools-console.web.test.ts b/apps/host-selfhost/src/custom-tools-console.web.test.ts index 0d5c3b62d..57b7d4baa 100644 --- a/apps/host-selfhost/src/custom-tools-console.web.test.ts +++ b/apps/host-selfhost/src/custom-tools-console.web.test.ts @@ -6,7 +6,9 @@ import appsClientPlugin, { type CustomToolsFetch, appsIntegrationPlugin, formatSyncErrors, + getCustomToolSource, listCustomToolSources, + removeCustomToolSource, syncCustomToolSource, syncStatusLabel, validateGitHubSourceUrl, @@ -48,6 +50,7 @@ describe("custom tools console client", () => { it("surfaces successful sync and source detail data", async () => { let syncBody = ""; + let removed = false; const fetchImpl: CustomToolsFetch = async (input, init) => { const url = String(input); if (url.endsWith("/sync")) { @@ -60,10 +63,34 @@ describe("custom tools console client", () => { skipped: [], }); } + if (url.endsWith("/demo-tools")) { + if (init?.method === "DELETE") { + removed = true; + return jsonResponse({ removed: true }); + } + return jsonResponse({ + source: { + slug: "demo-tools", + name: "demo-tools", + scope: "demo-tools", + url: "https://github.com/RhysSullivan/executor-custom-tools-demo", + repo: "RhysSullivan/executor-custom-tools-demo", + ref: "main", + hasToken: true, + upstreamSha: "abc123", + snapshotId: "snap1", + publishedAt: "2026-07-06T12:00:00.000Z", + tools: ["repo-summary", "stale-issues"], + skipped: [], + }, + }); + } return jsonResponse({ sources: [ { - scope: "github:RhysSullivan/executor-custom-tools-demo", + slug: "demo-tools", + name: "demo-tools", + scope: "demo-tools", url: "https://github.com/RhysSullivan/executor-custom-tools-demo", repo: "RhysSullivan/executor-custom-tools-demo", ref: "main", @@ -80,18 +107,25 @@ describe("custom tools console client", () => { const syncResult = await syncCustomToolSource( { + name: "demo-tools", url: "https://github.com/RhysSullivan/executor-custom-tools-demo", token: "ghp_demo", }, fetchImpl, ); const listed = await listCustomToolSources(fetchImpl); + const detail = await getCustomToolSource("demo-tools", fetchImpl); + const remove = await removeCustomToolSource("demo-tools", fetchImpl); expect(syncStatusLabel(syncResult)).toBe("Published 2 tools."); expect(syncBody).toContain("ghp_demo"); + expect(syncBody).toContain("demo-tools"); expect(syncResult.tools).toEqual(["repo-summary", "stale-issues"]); + expect(detail.source?.slug).toBe("demo-tools"); expect(listed.sources[0]?.hasToken).toBe(true); expect(listed.sources[0]?.tools).toEqual(["repo-summary", "stale-issues"]); + expect(remove).toEqual({ removed: true }); + expect(removed).toBe(true); }); it("renders failed sync errors readably", async () => { diff --git a/packages/core/sdk/src/executor.ts b/packages/core/sdk/src/executor.ts index 289894ec7..352866793 100644 --- a/packages/core/sdk/src/executor.ts +++ b/packages/core/sdk/src/executor.ts @@ -1927,6 +1927,19 @@ export const createExecutor = + pluginStorageFailure(existing.plugin_id, "removeIntegration", cause), + ), + ); + } // Drop owned connections / tools / definitions for this integration. const where = (b: AnyCb) => b("integration", "=", String(slug)); yield* core.deleteMany("tool", { where }); diff --git a/packages/core/sdk/src/plugin.ts b/packages/core/sdk/src/plugin.ts index 189335249..3885a11ba 100644 --- a/packages/core/sdk/src/plugin.ts +++ b/packages/core/sdk/src/plugin.ts @@ -523,6 +523,11 @@ export interface ConnectionLifecycleInput { readonly connection: ConnectionRef; } +export interface IntegrationLifecycleInput { + readonly ctx: PluginCtx; + readonly integration: IntegrationRecord; +} + export interface ConfigureIntegrationHandlerInput { readonly ctx: PluginCtx; readonly integration: IntegrationSlug; @@ -668,6 +673,12 @@ export interface PluginSpec< input: ConnectionLifecycleInput, ) => Effect.Effect; + /** Plugin-side cleanup when a removable integration is removed. Core still + * owns deleting the integration, connection, tool, and definition rows. */ + readonly removeIntegration?: ( + input: IntegrationLifecycleInput, + ) => Effect.Effect; + /** Core-dispatched integration configuration (beyond auth). */ readonly integrationConfigure?: IntegrationConfigureDecl; diff --git a/packages/plugins/apps/src/api.ts b/packages/plugins/apps/src/api.ts index e05c779b6..91dfce145 100644 --- a/packages/plugins/apps/src/api.ts +++ b/packages/plugins/apps/src/api.ts @@ -3,9 +3,6 @@ export { appsPlugin, APPS_INTEGRATION_SLUG, APPS_PLUGIN_ID, - APPS_CONNECTION_PREFIX, - connectionNameForScope, - scopeFromConnection, type AppsPluginOptions, } from "./plugin/apps-plugin"; export { diff --git a/packages/plugins/apps/src/backing/git-artifact-store.ts b/packages/plugins/apps/src/backing/git-artifact-store.ts index 6da3e8034..a4af8518c 100644 --- a/packages/plugins/apps/src/backing/git-artifact-store.ts +++ b/packages/plugins/apps/src/backing/git-artifact-store.ts @@ -1,5 +1,5 @@ import { execFile } from "node:child_process"; -import { mkdir } from "node:fs/promises"; +import { mkdir, rm } from "node:fs/promises"; import { join } from "node:path"; import { Effect } from "effect"; @@ -301,5 +301,18 @@ export const makeGitArtifactStore = (options: GitArtifactStoreOptions): Artifact cause, }), }), + removeScope: (address) => + Effect.tryPromise({ + try: async () => { + const key = scopeAddressStorageKey(address); + initialized.delete(key); + await rm(join(options.root, `${key}.git`), { recursive: true, force: true }); + }, + catch: (cause) => + new ArtifactStoreError({ + message: `failed to remove scope repo: ${address.tenant}/${address.scope}`, + cause, + }), + }), }; }; diff --git a/packages/plugins/apps/src/backing/libsql-scope-db.ts b/packages/plugins/apps/src/backing/libsql-scope-db.ts index fc1673d10..d38f8c692 100644 --- a/packages/plugins/apps/src/backing/libsql-scope-db.ts +++ b/packages/plugins/apps/src/backing/libsql-scope-db.ts @@ -1,4 +1,4 @@ -import { mkdirSync } from "node:fs"; +import { mkdirSync, rmSync } from "node:fs"; import { join, resolve } from "node:path"; import { createClient, type Client } from "@libsql/client"; @@ -160,6 +160,24 @@ export const makeLibsqlScopeDb = (options: LibsqlScopeDbOptions): ScopeDb => { cause, }), }), + removeScope: (address) => + Effect.try({ + try: () => { + const key = scopeAddressStorageKey(address); + clients.get(key)?.close(); + clients.delete(key); + if (options.root === ":memory:") return; + const dbPath = join(options.root, `${key}.db`); + for (const path of [dbPath, `${dbPath}-shm`, `${dbPath}-wal`]) { + rmSync(path, { force: true }); + } + }, + catch: (cause) => + new ScopeDbError({ + message: `failed to remove scope db ${address.tenant}/${address.scope}`, + cause, + }), + }), close: () => Effect.sync(() => { for (const client of clients.values()) client.close(); diff --git a/packages/plugins/apps/src/backing/scope-collision.test.ts b/packages/plugins/apps/src/backing/scope-collision.test.ts index e4a9684a0..9ed6b59be 100644 --- a/packages/plugins/apps/src/backing/scope-collision.test.ts +++ b/packages/plugins/apps/src/backing/scope-collision.test.ts @@ -7,6 +7,7 @@ import { Effect } from "effect"; import { makeLibsqlScopeDb } from "./libsql-scope-db"; import { makeSqliteAppsStore } from "./sqlite-apps-store"; +import { DESCRIPTOR_VERSION, type AppDescriptor } from "../pipeline/descriptor"; import { scopeAddress } from "../seams/scope-address"; const run = (effect: Effect.Effect): Promise => Effect.runPromise(effect); @@ -46,18 +47,26 @@ describe("scope collision (Fix 9)", () => { await run(db.close()); }); - it("routes distinct scopes through the explicit connection->scope mapping", async () => { + it("stores distinct descriptor scopes independently", async () => { const store = makeSqliteAppsStore({ path: ":memory:" }); - // Both scopes' connection names normalize to the SAME identifier - // ("appsMyScope"), but the explicit mapping keys by the ACTUAL stored name, - // so each scope's connection maps to the right scope. Simulate two DISTINCT - // connection names being recorded for two distinct scopes. - await run(store.putScopeForConnection("org", "appsMyScopeDash", "my-scope")); - await run(store.putScopeForConnection("org", "appsMyScopeUnderscore", "my_scope")); - - expect(await run(store.getScopeForConnection("org", "appsMyScopeDash"))).toBe("my-scope"); - expect(await run(store.getScopeForConnection("org", "appsMyScopeUnderscore"))).toBe("my_scope"); - // An unmapped name returns null (caller falls back to legacy parse). - expect(await run(store.getScopeForConnection("org", "unknown"))).toBeNull(); + const descriptor = (scope: string): AppDescriptor => ({ + version: DESCRIPTOR_VERSION, + tenant: "org", + scope, + description: "test", + snapshotId: `${scope}-snapshot`, + toolchain: { bundler: "esbuild", bundlerVersion: "test", target: "test" }, + tools: [], + workflows: [], + ui: [], + skills: [], + skipped: [], + }); + + await run(store.putDescriptor("org", "org", descriptor("my-scope"))); + await run(store.putDescriptor("org", "org", descriptor("my_scope"))); + + expect((await run(store.getDescriptor("org", "my-scope")))?.scope).toBe("my-scope"); + expect((await run(store.getDescriptor("org", "my_scope")))?.scope).toBe("my_scope"); }); }); diff --git a/packages/plugins/apps/src/backing/sqlite-apps-store.ts b/packages/plugins/apps/src/backing/sqlite-apps-store.ts index 6b9825987..efab5ceb4 100644 --- a/packages/plugins/apps/src/backing/sqlite-apps-store.ts +++ b/packages/plugins/apps/src/backing/sqlite-apps-store.ts @@ -7,19 +7,17 @@ import { Effect, Schema } from "effect"; import { StorageError, type StorageFailure } from "@executor-js/sdk"; import type { AppDescriptor } from "../pipeline/descriptor"; -import type { AppsStore, GitHubSourceTokenRef } from "../plugin/store"; +import type { AppsStore } from "../plugin/store"; // --------------------------------------------------------------------------- // SQLite-backed AppsStore (self-hosted). One small SQLite file stores the -// published descriptor per scope and the exact connection-to-scope mapping. +// published descriptor per source scope. // --------------------------------------------------------------------------- const toUrl = (path: string): string => (path === ":memory:" ? path : `file:${resolve(path)}`); const SCHEMA = ` CREATE TABLE IF NOT EXISTS descriptors (tenant TEXT NOT NULL, scope TEXT NOT NULL, snapshot_id TEXT NOT NULL, descriptor TEXT NOT NULL, published_at INTEGER NOT NULL, PRIMARY KEY (tenant, scope)); -CREATE TABLE IF NOT EXISTS scope_connections (tenant TEXT NOT NULL, connection_name TEXT NOT NULL, scope TEXT NOT NULL, PRIMARY KEY (tenant, connection_name)); -CREATE TABLE IF NOT EXISTS github_source_tokens (tenant TEXT NOT NULL, scope TEXT NOT NULL, provider TEXT NOT NULL, item_id TEXT NOT NULL, updated_at INTEGER NOT NULL, PRIMARY KEY (tenant, scope)); `; const storageFail = (message: string, cause: unknown): StorageFailure => @@ -118,57 +116,16 @@ export const makeSqliteAppsStore = (options: SqliteAppsStoreOptions): AppsStore ), ), ), - putScopeForConnection: (tenant, connectionName, scope) => + removeDescriptor: (tenant, scope) => Effect.tryPromise({ try: async () => { await init(); await client.execute({ - sql: "INSERT INTO scope_connections (tenant, connection_name, scope) VALUES (?, ?, ?) ON CONFLICT(tenant, connection_name) DO UPDATE SET scope=excluded.scope", - args: [tenant, connectionName, scope], - }); - }, - catch: (cause) => storageFail("putScopeForConnection failed", cause), - }), - getScopeForConnection: (tenant, connectionName) => - Effect.tryPromise({ - try: async () => { - await init(); - const res = await client.execute({ - sql: "SELECT scope FROM scope_connections WHERE tenant = ? AND connection_name = ?", - args: [tenant, connectionName], - }); - return res.rows[0] ? String(res.rows[0].scope) : null; - }, - catch: (cause) => storageFail("getScopeForConnection failed", cause), - }), - putGitHubSourceTokenRef: (tenant, scope, ref) => - Effect.tryPromise({ - try: async () => { - await init(); - await client.execute({ - sql: "INSERT INTO github_source_tokens (tenant, scope, provider, item_id, updated_at) VALUES (?, ?, ?, ?, ?) ON CONFLICT(tenant, scope) DO UPDATE SET provider=excluded.provider, item_id=excluded.item_id, updated_at=excluded.updated_at", - args: [tenant, scope, ref.provider, ref.itemId, ref.updatedAt], - }); - }, - catch: (cause) => storageFail("putGitHubSourceTokenRef failed", cause), - }), - getGitHubSourceTokenRef: (tenant, scope) => - Effect.tryPromise({ - try: async (): Promise => { - await init(); - const res = await client.execute({ - sql: "SELECT provider, item_id, updated_at FROM github_source_tokens WHERE tenant = ? AND scope = ?", + sql: "DELETE FROM descriptors WHERE tenant = ? AND scope = ?", args: [tenant, scope], }); - const row = res.rows[0]; - if (!row) return null; - return { - provider: String(row.provider), - itemId: String(row.item_id), - updatedAt: Number(row.updated_at), - }; }, - catch: (cause) => storageFail("getGitHubSourceTokenRef failed", cause), + catch: (cause) => storageFail("removeDescriptor failed", cause), }), }; }; diff --git a/packages/plugins/apps/src/index.ts b/packages/plugins/apps/src/index.ts index 2c0418664..f3170aca5 100644 --- a/packages/plugins/apps/src/index.ts +++ b/packages/plugins/apps/src/index.ts @@ -34,9 +34,12 @@ export { type AppsStoreDeps, type GitHubSourceTokenRef, descriptorCollection, - githubSourceTokenCollection, - scopeConnectionCollection, } from "./plugin/store"; +export { + slugifyCustomToolsAppName, + validateCustomToolsAppSlug, + CUSTOM_TOOLS_APP_SLUG_PATTERN, +} from "./source/app-slug"; export { buildBridge, rootsFor, diff --git a/packages/plugins/apps/src/plugin/apps-plugin.test.ts b/packages/plugins/apps/src/plugin/apps-plugin.test.ts index 66c1a3614..c4f3daec7 100644 --- a/packages/plugins/apps/src/plugin/apps-plugin.test.ts +++ b/packages/plugins/apps/src/plugin/apps-plugin.test.ts @@ -4,8 +4,9 @@ import { join } from "node:path"; import { describe, expect, it } from "@effect/vitest"; import { Effect } from "effect"; +import { IntegrationSlug } from "@executor-js/sdk"; -import { appsPlugin, connectionNameForScope } from "./apps-plugin"; +import { appsPlugin } from "./apps-plugin"; import { makeSelfHostAppsRuntime } from "./self-host-runtime"; import { makeInMemoryAppsStore, makeTestResolver } from "../testing"; @@ -62,13 +63,43 @@ describe("appsPlugin custom-tools contract", () => { }); const runtime = host.runtime; const plugin = appsPlugin({ runtime }); - const appConnection = connectionNameForScope("rhys"); + const appIntegration = IntegrationSlug.make("rhys-tools"); + const appConfig = { + kind: "github", + repoUrl: "https://github.com/rhys/tools", + repo: "rhys/tools", + scope: "rhys", + }; + const ctx = { + owner: { tenant: "org" }, + core: { + integrations: { + get: (slug: IntegrationSlug) => + Effect.succeed( + String(slug) === String(appIntegration) + ? { + slug: appIntegration, + name: "Rhys tools", + description: "Rhys tools", + kind: "apps", + canRemove: true, + canRefresh: false, + authMethods: [], + config: appConfig, + } + : null, + ), + }, + }, + }; await run(runtime.publish({ scope: "rhys", files: prototypeFileSet() })); const resolved = await run( plugin.resolveTools!({ - connection: { name: appConnection }, + ctx, + config: appConfig, + connection: { name: "main" }, } as never), ); const syncTool = resolved.tools.find((tool) => String(tool.name) === "deal-pipeline-sync"); @@ -80,10 +111,11 @@ describe("appsPlugin custom-tools contract", () => { const projected = await run( plugin.projectToolSchema!({ - ctx: { owner: { tenant: "org" } }, + ctx, toolRow: { name: "deal-pipeline-sync", - connection: appConnection, + integration: appIntegration, + connection: "main", }, inputSchema: syncTool!.inputSchema, outputSchema: syncTool!.outputSchema, @@ -100,10 +132,11 @@ describe("appsPlugin custom-tools contract", () => { const output = await run( plugin.invokeTool!({ - ctx: {}, + ctx, toolRow: { name: "deal-pipeline-sync", - connection: appConnection, + integration: appIntegration, + connection: "main", }, args: { crm: "tools.dealcloud.user.crm-main", diff --git a/packages/plugins/apps/src/plugin/apps-plugin.ts b/packages/plugins/apps/src/plugin/apps-plugin.ts index be6e53211..62e7af864 100644 --- a/packages/plugins/apps/src/plugin/apps-plugin.ts +++ b/packages/plugins/apps/src/plugin/apps-plugin.ts @@ -1,36 +1,48 @@ import { Effect, Result } from "effect"; import { + AuthTemplateSlug, definePlugin, - tool, - ToolName, IntegrationSlug, - AuthTemplateSlug, - connectionIdentifier, ProviderItemId, ProviderKey, + ToolName, + connectionIdentifier, + tool, + type IntegrationRecord, + type InvokeToolInput, type PluginCtx, type ResolveToolsInput, type ResolveToolsResult, - type InvokeToolInput, type ToolDef, } from "@executor-js/sdk"; -import type { AppsRuntime } from "./runtime"; -import { makeAppsStore } from "./store"; -import type { ClientResolver, ConnectionCandidate } from "./bindings"; import type { IntegrationDecl, ToolDescriptor } from "../pipeline/descriptor"; import { PublishError } from "../pipeline/discover"; import { parseGitHubSourceUrl, syncGitHubSource, + type GitHubSkippedArtifact, type GitHubSyncResult, } from "../source/github-source"; +import { slugifyCustomToolsAppName, validateCustomToolsAppSlug } from "../source/app-slug"; +import type { AppsRuntime, GitHubCustomToolsSourceSummary } from "./runtime"; +import { makeAppsStore, type AppDescriptorRecord, type GitHubSourceTokenRef } from "./store"; +import type { ClientResolver, ConnectionCandidate } from "./bindings"; export const APPS_INTEGRATION_SLUG = "apps"; export const APPS_PLUGIN_ID = "apps"; -const DEFAULT_CATALOG_SCOPE = "default"; +const APP_CONNECTION_NAME = connectionIdentifier("main"); + +interface AppsGitHubSourceConfig { + readonly kind: "github"; + readonly repoUrl: string; + readonly repo: string; + readonly scope: string; + readonly ref?: string; + readonly token?: GitHubSourceTokenRef; +} export interface AppsPluginOptions { readonly runtime: AppsRuntime; @@ -48,6 +60,9 @@ interface AppsStoreShape { const isRecord = (value: unknown): value is Record => value !== null && typeof value === "object" && !Array.isArray(value); +const asString = (value: unknown): string | undefined => + typeof value === "string" && value.trim().length > 0 ? value.trim() : undefined; + const unique = (values: readonly string[]): readonly string[] => [...new Set(values)]; const syncFailure = (message: string, path?: string): GitHubSyncResult => ({ @@ -63,9 +78,6 @@ const syncFailure = (message: string, path?: string): GitHubSyncResult => ({ ], }); -const sourceScopeFor = (repo: string): string => - String(connectionIdentifier(`github ${repo}`, "githubSource")); - const sourceTokenItemId = (tenant: string, scope: string): ProviderItemId => ProviderItemId.make(`apps:github-source:${tenant}:${scope}:token`); @@ -74,6 +86,30 @@ const configBaseUrl = (config: unknown): string | undefined => ? config.baseUrl : undefined; +const decodeSourceConfig = (config: unknown): AppsGitHubSourceConfig | null => { + if (!isRecord(config) || config.kind !== "github") return null; + const repoUrl = asString(config.repoUrl); + const repo = asString(config.repo); + const scope = asString(config.scope); + if (!repoUrl || !repo || !scope) return null; + const ref = asString(config.ref); + const token = isRecord(config.token) + ? { + provider: asString(config.token.provider) ?? "", + itemId: asString(config.token.itemId) ?? "", + updatedAt: typeof config.token.updatedAt === "number" ? config.token.updatedAt : 0, + } + : undefined; + return { + kind: "github", + repoUrl, + repo, + scope, + ...(ref ? { ref } : {}), + ...(token && token.provider && token.itemId ? { token } : {}), + }; +}; + const projectInputSchema = ( schema: unknown, integrations: Readonly>, @@ -113,6 +149,16 @@ const projectInputSchema = ( return projected; }; +const projectTool = (descriptor: ToolDescriptor): ToolDef => ({ + name: ToolName.make(descriptor.name), + description: descriptor.description, + inputSchema: descriptor.inputSchema, + outputSchema: descriptor.outputSchema, + annotations: { + requiresApproval: descriptor.annotations?.destructive === true, + }, +}); + export const appsPlugin = definePlugin((options?: AppsPluginOptions) => { const runtime = options?.runtime as AppsRuntime; const makeResolver = options?.makeResolver; @@ -122,91 +168,274 @@ export const appsPlugin = definePlugin((options?: AppsPluginOptions) => { return tenant === undefined ? "org" : String(tenant); }; - const scopeFor = (tenant: string, connectionName: string): Effect.Effect => - runtime.deps.store.getScopeForConnection(tenant, connectionName).pipe( - Effect.orElseSucceed(() => null), - Effect.map((mapped) => mapped ?? scopeFromConnection(connectionName)), - ); - const storeSourceToken = ( ctx: PluginCtx, tenant: string, scope: string, token: string, - ): Effect.Effect => + ): Effect.Effect => Effect.gen(function* () { const itemId = sourceTokenItemId(tenant, scope); const provider = yield* ctx.providers.setDefault(itemId, token).pipe(Effect.result); - if (Result.isFailure(provider)) return false; - const stored = yield* runtime.deps.store - .putGitHubSourceTokenRef(tenant, scope, { - provider: String(provider.success), - itemId: String(itemId), - updatedAt: Date.now(), - }) - .pipe(Effect.result); - return Result.isSuccess(stored); + if (Result.isFailure(provider)) return null; + return { + provider: String(provider.success), + itemId: String(itemId), + updatedAt: Date.now(), + }; }); const storedSourceToken = ( ctx: PluginCtx, - tenant: string, - scope: string, - ): Effect.Effect => - Effect.gen(function* () { - const ref = yield* runtime.deps.store - .getGitHubSourceTokenRef(tenant, scope) - .pipe(Effect.orElseSucceed(() => null)); - if (!ref) return null; - return yield* ctx.providers - .get(ProviderKey.make(ref.provider), ProviderItemId.make(ref.itemId)) - .pipe(Effect.orElseSucceed(() => null)); - }); + ref: GitHubSourceTokenRef | undefined, + ): Effect.Effect => { + if (!ref) return Effect.succeed(null); + return ctx.providers + .get(ProviderKey.make(ref.provider), ProviderItemId.make(ref.itemId)) + .pipe(Effect.orElseSucceed(() => null)); + }; - const ensureCatalogConnection = (scope: string, ctx: PluginCtx) => + const removeStoredSourceToken = ( + ctx: PluginCtx, + ref: GitHubSourceTokenRef | undefined, + ): Effect.Effect => + ref + ? ctx.providers.remove(ProviderKey.make(ref.provider), ProviderItemId.make(ref.itemId)) + : Effect.void; + + const ensureAppConnection = (slug: IntegrationSlug, ctx: PluginCtx) => Effect.gen(function* () { - const tenant = tenantFor(ctx); - const slug = IntegrationSlug.make(APPS_INTEGRATION_SLUG); - const existing = yield* ctx.core.integrations - .get(slug) - .pipe(Effect.orElseSucceed(() => null)); - if (!existing) { - yield* ctx.core.integrations.register({ - slug, - name: "Apps", - description: "User-authored, published custom tools.", - config: {}, - canRemove: false, - canRefresh: true, - }); - } - const rawName = connectionNameForScope(scope); - const connName = connectionIdentifier(rawName); const conns = yield* ctx.connections .list({ integration: slug }) .pipe(Effect.orElseSucceed(() => [])); - if (!conns.some((c) => String(c.name) === String(connName))) { + if (!conns.some((connection) => String(connection.name) === String(APP_CONNECTION_NAME))) { yield* ctx.connections.create({ owner: "user", - name: connName, + name: APP_CONNECTION_NAME, integration: slug, template: AuthTemplateSlug.make("none"), value: "", }); } - yield* runtime.deps.store - .putScopeForConnection(tenant, String(rawName), scope) - .pipe(Effect.orElseSucceed(() => undefined)); - yield* runtime.deps.store - .putScopeForConnection(tenant, String(connName), scope) - .pipe(Effect.orElseSucceed(() => undefined)); - return { + return { owner: "user" as const, integration: slug, name: APP_CONNECTION_NAME }; + }); + + const refreshAppConnection = (slug: IntegrationSlug, ctx: PluginCtx) => + Effect.gen(function* () { + const ref = yield* ensureAppConnection(slug, ctx); + yield* ctx.connections.refresh(ref).pipe(Effect.orElseSucceed(() => [])); + return ref; + }); + + const descriptorRecordByScope = (tenant: string) => + runtime.deps.store.listDescriptors(tenant).pipe( + Effect.orElseSucceed(() => []), + Effect.map((records) => new Map(records.map((record) => [record.descriptor.scope, record]))), + ); + + const sourceSummaryFor = ( + integration: IntegrationRecord, + recordsByScope: ReadonlyMap, + ): GitHubCustomToolsSourceSummary | null => { + const config = decodeSourceConfig(integration.config); + if (!config) return null; + const record = recordsByScope.get(config.scope); + if (!record) return null; + const descriptor = record.descriptor; + const source = descriptor?.source; + if (source?.kind !== "github") return null; + return { + slug: String(integration.slug), + name: integration.name, + scope: descriptor.scope, + url: source.url, + repo: source.repo, + ref: source.ref, + hasToken: config.token !== undefined, + upstreamSha: source.upstreamSha, + snapshotId: descriptor.snapshotId, + ...(descriptor.description ? { description: descriptor.description } : {}), + publishedAt: new Date(record.publishedAt).toISOString(), + tools: descriptor.tools.map((toolDesc) => toolDesc.name), + skipped: [ + ...(source.skipped ?? []), + ...(descriptor.skipped as readonly GitHubSkippedArtifact[]), + ], + }; + }; + + const listSources = (ctx: PluginCtx) => + Effect.gen(function* () { + const tenant = tenantFor(ctx); + const recordsByScope = yield* descriptorRecordByScope(tenant); + const integrations = yield* ctx.core.integrations.list().pipe(Effect.orElseSucceed(() => [])); + const sources: GitHubCustomToolsSourceSummary[] = []; + for (const integration of integrations) { + if (integration.kind !== APPS_PLUGIN_ID) continue; + const record = yield* ctx.core.integrations + .get(integration.slug) + .pipe(Effect.orElseSucceed(() => null)); + if (!record) continue; + const summary = sourceSummaryFor(record, recordsByScope); + if (summary) sources.push(summary); + } + return sources.sort((a, b) => b.publishedAt.localeCompare(a.publishedAt)); + }); + + const getSource = (ctx: PluginCtx, slug: string) => + Effect.gen(function* () { + const tenant = tenantFor(ctx); + const recordsByScope = yield* descriptorRecordByScope(tenant); + const record = yield* ctx.core.integrations + .get(IntegrationSlug.make(slug)) + .pipe(Effect.orElseSucceed(() => null)); + return record && record.kind === APPS_PLUGIN_ID + ? sourceSummaryFor(record, recordsByScope) + : null; + }); + + const syncExistingSource = ( + ctx: PluginCtx, + slugValue: string, + providedToken: string | null, + ) => + Effect.gen(function* () { + const slug = IntegrationSlug.make(slugValue); + const record = yield* ctx.core.integrations.get(slug).pipe(Effect.orElseSucceed(() => null)); + const config = record?.kind === APPS_PLUGIN_ID ? decodeSourceConfig(record.config) : null; + if (!record || !config) { + return syncFailure(`Custom tools source "${slugValue}" does not exist.`); + } + const github = yield* ctx.core.integrations + .get(IntegrationSlug.make("github")) + .pipe(Effect.orElseSucceed(() => null)); + const tenant = tenantFor(ctx); + const token = providedToken ?? (yield* storedSourceToken(ctx, config.token)); + const result = yield* syncGitHubSource({ + runtime, + tenant, + scope: config.scope, + url: config.repoUrl, + ...(config.ref ? { ref: config.ref } : {}), + token, + baseUrl: configBaseUrl(github?.config), + }); + if (result.status === "failed") return result; + + const tokenRef = providedToken + ? yield* storeSourceToken(ctx, tenant, config.scope, providedToken) + : config.token; + if (providedToken && !tokenRef) { + return syncFailure( + "No writable credential provider is available to store the GitHub source token.", + ); + } + const descriptor = yield* runtime.getDescriptor(tenant, config.scope); + const source = descriptor?.source?.kind === "github" ? descriptor.source : null; + const nextConfig: AppsGitHubSourceConfig = { + ...config, + ...(source + ? { + repoUrl: source.url, + repo: source.repo, + ref: source.ref, + } + : {}), + ...(tokenRef ? { token: tokenRef } : {}), + }; + yield* ctx.core.integrations.update(slug, { + ...(descriptor?.description ? { description: descriptor.description } : {}), + config: nextConfig, + }); + yield* refreshAppConnection(slug, ctx); + return result; + }); + + const addSource = ( + ctx: PluginCtx, + input: { + readonly url: string; + readonly ref?: string; + readonly name?: string; + readonly token: string | null; + }, + ) => + Effect.gen(function* () { + const parsed = parseGitHubSourceUrl(input.url, { ref: input.ref }); + if (!parsed.ok) return syncFailure(parsed.message, input.url); + const rawName = input.name && input.name.trim().length > 0 ? input.name : parsed.value.name; + const slugValue = slugifyCustomToolsAppName(rawName); + const slugError = validateCustomToolsAppSlug(slugValue); + if (slugError) return syncFailure(slugError); + const slug = IntegrationSlug.make(slugValue); + const existing = yield* ctx.core.integrations + .get(slug) + .pipe(Effect.orElseSucceed(() => null)); + if (existing) { + return syncFailure(`Integration "${slugValue}" already exists. Choose another name.`); + } + + const github = yield* ctx.core.integrations + .get(IntegrationSlug.make("github")) + .pipe(Effect.orElseSucceed(() => null)); + const tenant = tenantFor(ctx); + const scope = slugValue; + const result = yield* syncGitHubSource({ + runtime, + tenant, + scope, + url: input.url, + ...(input.ref ? { ref: input.ref } : {}), + token: input.token, + baseUrl: configBaseUrl(github?.config), + }); + if (result.status === "failed") return result; + + const tokenRef = input.token + ? yield* storeSourceToken(ctx, tenant, scope, input.token) + : undefined; + if (input.token && !tokenRef) { + yield* runtime.removeSource({ tenant, scope }).pipe(Effect.orElseSucceed(() => undefined)); + return syncFailure( + "No writable credential provider is available to store the GitHub source token.", + ); + } + + const descriptor = yield* runtime.getDescriptor(tenant, scope); + const source = descriptor?.source?.kind === "github" ? descriptor.source : null; + const config: AppsGitHubSourceConfig = { + kind: "github", + repoUrl: source?.url ?? input.url.trim(), + repo: source?.repo ?? parsed.value.repo, scope, - connection: String(connName), - ref: { owner: "user" as const, integration: slug, name: connName }, + ...(source?.ref ? { ref: source.ref } : input.ref ? { ref: input.ref } : {}), + ...(tokenRef ? { token: tokenRef } : {}), }; + yield* ctx.core.integrations.register({ + slug, + name: slugValue, + description: + descriptor?.description ?? + `Custom tools synced from ${source?.repo ?? parsed.value.repo}.`, + config, + canRemove: true, + canRefresh: false, + }); + yield* refreshAppConnection(slug, ctx); + return result; }); + const configForToolRow = ( + ctx: PluginCtx, + integration: IntegrationSlug, + ): Effect.Effect => + ctx.core.integrations.get(integration).pipe( + Effect.map((record) => + record && record.kind === APPS_PLUGIN_ID ? decodeSourceConfig(record.config) : null, + ), + Effect.orElseSucceed(() => null), + ); + return { id: APPS_PLUGIN_ID as "apps", packageName: "@executor-js/plugin-apps", @@ -224,16 +453,6 @@ export const appsPlugin = definePlugin((options?: AppsPluginOptions) => { schema: { Type: {} as Record }, indexes: [], }, - apps_scope_connection: { - name: "apps_scope_connection", - schema: { Type: {} as Record }, - indexes: [], - }, - apps_github_source_token: { - name: "apps_github_source_token", - schema: { Type: {} as Record }, - indexes: [], - }, }, extension: () => ({ runtime }), @@ -244,19 +463,6 @@ export const appsPlugin = definePlugin((options?: AppsPluginOptions) => { kind: "executor", name: "Apps", tools: [ - tool({ - name: "connect_catalog", - description: - "Wire this scope's published custom tools into the tool catalog. " + - "Idempotent; call once per scope after publishing.", - execute: (args, { ctx }) => - Effect.gen(function* () { - const scope = - (args as { scope?: string } | undefined)?.scope ?? DEFAULT_CATALOG_SCOPE; - const connected = yield* ensureCatalogConnection(scope, ctx); - return { scope, connection: connected.connection }; - }), - }), tool({ name: "sync_github_source", description: "Sync a GitHub repository containing custom tool source files.", @@ -265,51 +471,21 @@ export const appsPlugin = definePlugin((options?: AppsPluginOptions) => { const payload = isRecord(args) ? args : {}; const url = typeof payload.url === "string" - ? payload.url + ? payload.url.trim() : typeof payload.repo === "string" - ? payload.repo + ? payload.repo.trim() : ""; - const ref = - typeof payload.ref === "string" && payload.ref.trim().length > 0 - ? payload.ref.trim() - : undefined; - const providedToken = - typeof payload.token === "string" && payload.token.trim().length > 0 - ? payload.token.trim() - : null; - const explicitScope = typeof payload.scope === "string" ? payload.scope : undefined; + const ref = asString(payload.ref); + const providedToken = asString(payload.token) ?? null; + const slug = asString(payload.slug); + if (slug && !url) return yield* syncExistingSource(ctx, slug, providedToken); if (!url) return syncFailure('sync_github_source requires "url"'); - - const parsed = parseGitHubSourceUrl(url, { ref }); - if (!parsed.ok) return syncFailure(parsed.message, url); - const integration = yield* ctx.core.integrations - .get(IntegrationSlug.make("github")) - .pipe(Effect.orElseSucceed(() => null)); - const tenant = tenantFor(ctx); - const scope = explicitScope ?? sourceScopeFor(parsed.value.repo); - if (providedToken) { - const stored = yield* storeSourceToken(ctx, tenant, scope, providedToken); - if (!stored) { - return syncFailure( - "No writable credential provider is available to store the GitHub source token.", - ); - } - } - const token = providedToken ?? (yield* storedSourceToken(ctx, tenant, scope)); - const result = yield* syncGitHubSource({ - runtime, - tenant, - scope, + return yield* addSource(ctx, { url, - ref, - token, - baseUrl: configBaseUrl(integration?.config), + ...(ref ? { ref } : {}), + ...(asString(payload.name) ? { name: asString(payload.name) } : {}), + token: providedToken, }); - if (result.status === "published") { - const catalog = yield* ensureCatalogConnection(scope, ctx); - yield* ctx.connections.refresh(catalog.ref).pipe(Effect.orElseSucceed(() => [])); - } - return result; }), }), tool({ @@ -317,20 +493,44 @@ export const appsPlugin = definePlugin((options?: AppsPluginOptions) => { description: "List synced GitHub repositories that publish custom tools.", execute: (_args, { ctx }) => Effect.gen(function* () { - const tenant = tenantFor(ctx); - const sources = yield* runtime.listGitHubSources(tenant); + const sources = yield* listSources(ctx); return { sources }; }), }), + tool({ + name: "get_github_source", + description: "Read one synced GitHub custom-tools source.", + execute: (args, { ctx }) => + Effect.gen(function* () { + const slug = isRecord(args) ? asString(args.slug) : undefined; + if (!slug) return { source: null }; + const source = yield* getSource(ctx, slug); + return { source }; + }), + }), ], }, ], - resolveTools: ({ ctx, connection }: ResolveToolsInput) => + removeIntegration: ({ ctx, integration }) => Effect.gen(function* () { + const config = decodeSourceConfig(integration.config); + if (!config) return; + yield* removeStoredSourceToken(ctx, config.token); + yield* runtime.removeSource({ tenant: tenantFor(ctx), scope: config.scope }); + }), + + describeIntegrationDisplay: (integration) => { + const config = decodeSourceConfig(integration.config); + return config ? { url: config.repoUrl } : {}; + }, + + resolveTools: ({ ctx, config }: ResolveToolsInput) => + Effect.gen(function* () { + const source = decodeSourceConfig(config); + if (!source) return { tools: [] } satisfies ResolveToolsResult; const tenant = ctx ? tenantFor(ctx) : "org"; - const scope = yield* scopeFor(tenant, String(connection.name)); - const descriptor = yield* runtime.getDescriptor(tenant, scope); + const descriptor = yield* runtime.getDescriptor(tenant, source.scope); if (!descriptor) return { tools: [] } satisfies ResolveToolsResult; const tools: ToolDef[] = []; for (const t of descriptor.tools) { @@ -342,12 +542,13 @@ export const appsPlugin = definePlugin((options?: AppsPluginOptions) => { projectToolSchema: ({ ctx, toolRow, inputSchema, outputSchema }) => Effect.gen(function* () { const tenant = tenantFor(ctx); - const scope = yield* scopeFor(tenant, String(toolRow.connection)); - const descriptor = yield* runtime.getDescriptor(tenant, scope); + const source = yield* configForToolRow(ctx, IntegrationSlug.make(toolRow.integration)); + if (!source) return { inputSchema, outputSchema }; + const descriptor = yield* runtime.getDescriptor(tenant, source.scope); const toolDesc = descriptor?.tools.find((t) => t.name === toolRow.name); if (!toolDesc) return { inputSchema, outputSchema }; const resolver = makeResolver - ? makeResolver({ ctx, scope, tool: toolRow.name }) + ? makeResolver({ ctx, scope: source.scope, tool: toolRow.name }) : runtime.deps.resolver; const byRole: Record = {}; for (const [role, decl] of Object.entries(toolDesc.integrations)) { @@ -364,11 +565,18 @@ export const appsPlugin = definePlugin((options?: AppsPluginOptions) => { invokeTool: ({ ctx, toolRow, args, invokeOptions }: InvokeToolInput) => Effect.gen(function* () { const tenant = tenantFor(ctx); - const scope = yield* scopeFor(tenant, String(toolRow.connection)); - const descriptor = yield* runtime.getDescriptor(tenant, scope); + const source = yield* configForToolRow(ctx, IntegrationSlug.make(toolRow.integration)); + if (!source) { + return yield* new PublishError({ + message: `apps integration "${toolRow.integration}" has no custom-tools source config`, + stage: "project", + diagnostics: [], + }); + } + const descriptor = yield* runtime.getDescriptor(tenant, source.scope); if (!descriptor) { return yield* new PublishError({ - message: `apps scope "${scope}" has no published app (connection "${toolRow.connection}")`, + message: `apps scope "${source.scope}" has no published app`, stage: "project", diagnostics: [], }); @@ -376,17 +584,17 @@ export const appsPlugin = definePlugin((options?: AppsPluginOptions) => { const toolDesc = descriptor.tools.find((t) => t.name === toolRow.name); if (!toolDesc) { return yield* new PublishError({ - message: `apps tool "${toolRow.name}" is not published in scope "${scope}"`, + message: `apps tool "${toolRow.name}" is not published in scope "${source.scope}"`, stage: "project", diagnostics: [], }); } const resolver = makeResolver - ? makeResolver({ ctx, scope, tool: toolRow.name }) + ? makeResolver({ ctx, scope: source.scope, tool: toolRow.name }) : undefined; return yield* runtime.invokeTool({ tenant, - scope, + scope: source.scope, tool: toolRow.name, args, resolver, @@ -395,36 +603,3 @@ export const appsPlugin = definePlugin((options?: AppsPluginOptions) => { }), }; }); - -const projectTool = (descriptor: ToolDescriptor): ToolDef => ({ - name: ToolName.make(descriptor.name), - description: descriptor.description, - inputSchema: descriptor.inputSchema, - outputSchema: descriptor.outputSchema, - annotations: { - requiresApproval: descriptor.annotations?.destructive === true, - }, -}); - -export const APPS_CONNECTION_PREFIX = APPS_INTEGRATION_SLUG; - -const pascal = (value: string): string => - value.length === 0 ? value : `${value[0]!.toUpperCase()}${value.slice(1)}`; - -export const connectionNameForScope = (scope: string): string => - `${APPS_CONNECTION_PREFIX}${pascal(scope)}`; - -export const scopeFromConnection = (connectionName: string): string => { - if (connectionName.startsWith(`${APPS_INTEGRATION_SLUG}/`)) { - return connectionName.slice(APPS_INTEGRATION_SLUG.length + 1); - } - if ( - connectionName.startsWith(APPS_CONNECTION_PREFIX) && - connectionName.length > APPS_CONNECTION_PREFIX.length - ) { - const rest = connectionName.slice(APPS_CONNECTION_PREFIX.length); - return `${rest[0]!.toLowerCase()}${rest.slice(1)}`; - } - const slash = connectionName.indexOf("/"); - return slash === -1 ? connectionName : connectionName.slice(slash + 1); -}; diff --git a/packages/plugins/apps/src/plugin/runtime.ts b/packages/plugins/apps/src/plugin/runtime.ts index f5233c43e..8732df754 100644 --- a/packages/plugins/apps/src/plugin/runtime.ts +++ b/packages/plugins/apps/src/plugin/runtime.ts @@ -42,6 +42,8 @@ export interface AppsRuntimeDeps { } export interface GitHubCustomToolsSourceSummary { + readonly slug: string; + readonly name: string; readonly scope: string; readonly url: string; readonly repo: string; @@ -71,6 +73,10 @@ export interface AppsRuntime { readonly listGitHubSources: ( tenant?: string, ) => Effect.Effect; + readonly removeSource: (input: { + readonly tenant?: string; + readonly scope: string; + }) => Effect.Effect; /** Re-derive the published-descriptor pointer from the latest committed * snapshot. */ readonly repair: ( @@ -382,30 +388,27 @@ export const makeAppsRuntime = (deps: AppsRuntimeDeps): AppsRuntime => { const source = record.descriptor.source; return source?.kind === "github" ? [{ record, source }] : []; }); - return Effect.forEach(githubRecords, ({ record, source }) => - Effect.gen(function* () { - const { descriptor, publishedAt } = record; - const tokenRef = yield* deps.store - .getGitHubSourceTokenRef(tenant, descriptor.scope) - .pipe(Effect.orElseSucceed(() => null)); - return { - scope: descriptor.scope, - url: source.url, - repo: source.repo, - ref: source.ref, - hasToken: tokenRef !== null, - upstreamSha: source.upstreamSha, - snapshotId: descriptor.snapshotId, - ...(descriptor.description ? { description: descriptor.description } : {}), - publishedAt: new Date(publishedAt).toISOString(), - tools: descriptor.tools.map((tool) => tool.name), - skipped: [ - ...(source.skipped ?? []), - ...(descriptor.skipped as readonly GitHubSkippedArtifact[]), - ], - } satisfies GitHubCustomToolsSourceSummary; - }), - ); + return Effect.forEach(githubRecords, ({ record, source }) => { + const { descriptor, publishedAt } = record; + return Effect.succeed({ + slug: descriptor.scope, + name: source.repo.split("/").at(-1) ?? descriptor.scope, + scope: descriptor.scope, + url: source.url, + repo: source.repo, + ref: source.ref, + hasToken: false, + upstreamSha: source.upstreamSha, + snapshotId: descriptor.snapshotId, + ...(descriptor.description ? { description: descriptor.description } : {}), + publishedAt: new Date(publishedAt).toISOString(), + tools: descriptor.tools.map((tool) => tool.name), + skipped: [ + ...(source.skipped ?? []), + ...(descriptor.skipped as readonly GitHubSkippedArtifact[]), + ], + } satisfies GitHubCustomToolsSourceSummary); + }); }), ); }, @@ -415,6 +418,28 @@ export const makeAppsRuntime = (deps: AppsRuntimeDeps): AppsRuntime => { return repairScope(tenant, scope); }, + removeSource: (input) => { + const tenant = resolveTenant(input.tenant); + const address = scopeAddress(tenant, input.scope); + const toPublishError = (message: string) => (_cause: unknown) => + new PublishError({ + message, + stage: "project", + diagnostics: [], + }); + return Effect.gen(function* () { + yield* deps.store + .removeDescriptor(tenant, input.scope) + .pipe(Effect.mapError(toPublishError("removeDescriptor failed"))); + yield* deps.artifactStore + .removeScope(address) + .pipe(Effect.mapError(toPublishError("remove artifact scope failed"))); + yield* deps.scopeDb + .removeScope(address) + .pipe(Effect.mapError(toPublishError("remove scope db failed"))); + }); + }, + invokeTool: (input) => Effect.gen(function* () { const tenant = resolveTenant(input.tenant); diff --git a/packages/plugins/apps/src/plugin/store.ts b/packages/plugins/apps/src/plugin/store.ts index 12c956c96..8c0db58f4 100644 --- a/packages/plugins/apps/src/plugin/store.ts +++ b/packages/plugins/apps/src/plugin/store.ts @@ -9,7 +9,7 @@ import { import type { AppDescriptor } from "../pipeline/descriptor"; // --------------------------------------------------------------------------- -// AppsStore: descriptor pointer persistence and connection-to-scope mapping. +// AppsStore: descriptor pointer persistence. // --------------------------------------------------------------------------- export const descriptorCollection = definePluginStorageCollection("published_descriptor", { @@ -22,27 +22,6 @@ export const descriptorCollection = definePluginStorageCollection("published_des }, }); -export const scopeConnectionCollection = definePluginStorageCollection("apps_scope_connection", { - Type: {} as { - readonly tenant: string; - readonly connectionName: string; - readonly scope: string; - }, -}); - -export const githubSourceTokenCollection = definePluginStorageCollection( - "apps_github_source_token", - { - Type: {} as { - readonly tenant: string; - readonly scope: string; - readonly provider: string; - readonly itemId: string; - readonly updatedAt: number; - }, - }, -); - export interface AppDescriptorRecord { readonly descriptor: AppDescriptor; readonly publishedAt: number; @@ -64,27 +43,10 @@ export interface AppsStore { tenant: string, scope: string, ) => Effect.Effect; + readonly removeDescriptor: (tenant: string, scope: string) => Effect.Effect; readonly listDescriptors: ( tenant: string, ) => Effect.Effect; - readonly putScopeForConnection: ( - tenant: string, - connectionName: string, - scope: string, - ) => Effect.Effect; - readonly getScopeForConnection: ( - tenant: string, - connectionName: string, - ) => Effect.Effect; - readonly putGitHubSourceTokenRef: ( - tenant: string, - scope: string, - ref: GitHubSourceTokenRef, - ) => Effect.Effect; - readonly getGitHubSourceTokenRef: ( - tenant: string, - scope: string, - ) => Effect.Effect; } export interface AppsStoreDeps { @@ -93,36 +55,10 @@ export interface AppsStoreDeps { export const makeAppsStore = (deps: AppsStoreDeps): AppsStore => { const descriptors = deps.pluginStorage.collection(descriptorCollection); - const scopeConnections = deps.pluginStorage.collection(scopeConnectionCollection); - const githubSourceTokens = deps.pluginStorage.collection(githubSourceTokenCollection); // Tenant is now part of apps storage keys. The apps subsystem had not shipped // before this key shape, so no migration from the old scope-only keys is needed. const keyFor = (tenant: string, key: string): string => `${tenant}:${key}`; return { - putScopeForConnection: (tenant, connectionName, scope) => - scopeConnections - .put({ - owner: "org", - key: keyFor(tenant, connectionName), - data: { tenant, connectionName, scope }, - }) - .pipe(Effect.asVoid), - getScopeForConnection: (tenant, connectionName) => - scopeConnections - .get({ key: keyFor(tenant, connectionName) }) - .pipe(Effect.map((entry) => entry?.data.scope ?? null)), - putGitHubSourceTokenRef: (tenant, scope, ref) => - githubSourceTokens - .put({ - owner: "org", - key: keyFor(tenant, scope), - data: { tenant, scope, ...ref }, - }) - .pipe(Effect.asVoid), - getGitHubSourceTokenRef: (tenant, scope) => - githubSourceTokens - .get({ key: keyFor(tenant, scope) }) - .pipe(Effect.map((entry) => entry?.data ?? null)), putDescriptor: (tenant, owner, descriptor) => descriptors .put({ @@ -141,6 +77,8 @@ export const makeAppsStore = (deps: AppsStoreDeps): AppsStore => { descriptors .get({ key: keyFor(tenant, scope) }) .pipe(Effect.map((entry) => entry?.data.descriptor ?? null)), + removeDescriptor: (tenant, scope) => + descriptors.remove({ owner: "org", key: keyFor(tenant, scope) }), listDescriptors: (tenant) => descriptors.list({ keyPrefix: `${tenant}:` }).pipe( Effect.map((entries) => diff --git a/packages/plugins/apps/src/react/AddCustomToolsSource.tsx b/packages/plugins/apps/src/react/AddCustomToolsSource.tsx index 520347720..305f0507c 100644 --- a/packages/plugins/apps/src/react/AddCustomToolsSource.tsx +++ b/packages/plugins/apps/src/react/AddCustomToolsSource.tsx @@ -10,12 +10,15 @@ import { import { FloatActions } from "@executor-js/react/components/float-actions"; import { Input } from "@executor-js/react/components/input"; import { toast } from "@executor-js/react/components/sonner"; -import { FormErrorAlert } from "@executor-js/react/lib/integration-add"; +import { FormErrorAlert, useSlugAlreadyExists } from "@executor-js/react/lib/integration-add"; import { formatSyncErrors, + slugifyCustomToolsAppName, + suggestCustomToolsAppName, syncCustomToolSourceEffect, syncStatusLabel, + validateCustomToolsAppSlug, validateGitHubSourceUrl, } from "./custom-tools-client"; @@ -24,21 +27,36 @@ export default function AddCustomToolsSource(props: { readonly onCancel: () => void; }) { const [url, setUrl] = useState(""); + const [name, setName] = useState(""); + const [nameTouched, setNameTouched] = useState(false); const [token, setToken] = useState(""); const [tokenRevealed, setTokenRevealed] = useState(false); const [urlError, setUrlError] = useState(null); + const [nameError, setNameError] = useState(null); const [syncing, setSyncing] = useState(false); const [syncError, setSyncError] = useState(null); + const effectiveName = nameTouched + ? name + : slugifyCustomToolsAppName(suggestCustomToolsAppName(url)); + const slug = effectiveName; + const slugAlreadyExists = useSlugAlreadyExists(slug); const submit = async () => { const validation = validateGitHubSourceUrl(url); + const nextNameError = validateCustomToolsAppSlug(slug); setUrlError(validation); + setNameError(nextNameError); setSyncError(null); - if (validation) return; + if (validation || nextNameError) return; + if (slugAlreadyExists) { + setSyncError(`An integration named "${slug}" already exists. Choose another name.`); + return; + } setSyncing(true); const exit = await Effect.runPromiseExit( syncCustomToolSourceEffect({ + name: slug, url, token, }), @@ -55,7 +73,7 @@ export default function AddCustomToolsSource(props: { return; } toast.success(syncStatusLabel(result)); - props.onComplete("apps"); + props.onComplete(slug); }; return ( @@ -79,6 +97,7 @@ export default function AddCustomToolsSource(props: { onChange={(event) => { setUrl((event.target as HTMLInputElement).value); setUrlError(null); + setNameError(null); setSyncError(null); }} onBlur={() => setUrlError(validateGitHubSourceUrl(url))} @@ -90,6 +109,32 @@ export default function AddCustomToolsSource(props: {
+ +
+ { + setNameTouched(true); + setName(slugifyCustomToolsAppName((event.target as HTMLInputElement).value)); + setNameError(null); + setSyncError(null); + }} + placeholder="executor-custom-tools-demo" + className="text-sm" + aria-invalid={nameError ? true : undefined} + /> + {nameError &&

{nameError}

} + {slugAlreadyExists && !syncing && !nameError && ( +

+ An integration named "{slug}" already exists. +

+ )} +
+ + { return Number.isNaN(date.getTime()) ? value : date.toLocaleString(); }; -export default function CustomToolsAccountsPanel() { +export default function CustomToolsAccountsPanel(props: { + readonly sourceId: string; + readonly integrationName: string; +}) { const [loadState, setLoadState] = useState({ status: "loading" }); - const [selectedScope, setSelectedScope] = useState(null); - const [syncingScope, setSyncingScope] = useState(null); + const [syncing, setSyncing] = useState(false); + const [removing, setRemoving] = useState(false); const [notice, setNotice] = useState(null); + const [removeError, setRemoveError] = useState(null); - const loadSources = async () => { + const loadSource = async () => { setLoadState({ status: "loading" }); - const exit = await Effect.runPromiseExit(listCustomToolSourcesEffect()); + const exit = await Effect.runPromiseExit(getCustomToolSourceEffect(props.sourceId)); if (Exit.isFailure(exit)) { - setLoadState({ - status: "error", - message: "Failed to load custom tool sources.", - }); + setLoadState({ status: "error", message: "Failed to load custom tools source." }); return; } - const result = exit.value; - setLoadState({ status: "ready", sources: result.sources }); - setSelectedScope((current) => current ?? result.sources[0]?.scope ?? null); + const source = exit.value.source; + setLoadState(source ? { status: "ready", source } : { status: "missing" }); }; useEffect(() => { let active = true; void (async () => { - const exit = await Effect.runPromiseExit(listCustomToolSourcesEffect()); - if (Exit.isSuccess(exit)) { - if (!active) return; - const result = exit.value; - setLoadState({ status: "ready", sources: result.sources }); - setSelectedScope(result.sources[0]?.scope ?? null); - } else { - if (!active) return; - setLoadState({ - status: "error", - message: "Failed to load custom tool sources.", - }); + const exit = await Effect.runPromiseExit(getCustomToolSourceEffect(props.sourceId)); + if (!active) return; + if (Exit.isFailure(exit)) { + setLoadState({ status: "error", message: "Failed to load custom tools source." }); + return; } + const source = exit.value.source; + setLoadState(source ? { status: "ready", source } : { status: "missing" }); })(); return () => { active = false; }; - }, []); - - const sources = loadState.status === "ready" ? loadState.sources : []; - const selected = sources.find((source) => source.scope === selectedScope) ?? sources[0] ?? null; + }, [props.sourceId]); - const syncSelected = async (source: GitHubCustomToolsSourceSummary) => { - setSyncingScope(source.scope); + const syncSource = async (source: GitHubCustomToolsSourceSummary) => { + setSyncing(true); setNotice(null); const beforeTools = source.tools; - const exit = await Effect.runPromiseExit( - syncCustomToolSourceEffect({ - url: source.url, - }), - ); + const exit = await Effect.runPromiseExit(syncCustomToolSourceEffect({ slug: source.slug })); if (Exit.isFailure(exit)) { setNotice({ status: "failed", @@ -105,7 +97,7 @@ export default function CustomToolsAccountsPanel() { removed: [], errors: ["Failed to sync custom tools."], }); - setSyncingScope(null); + setSyncing(false); return; } const result = exit.value; @@ -118,94 +110,53 @@ export default function CustomToolsAccountsPanel() { removed: diff.removed, errors: formatSyncErrors(result), }); - if (result.status !== "failed") { - await loadSources(); - setSelectedScope(source.scope); + if (result.status !== "failed") await loadSource(); + setSyncing(false); + }; + + const removeSource = async (source: GitHubCustomToolsSourceSummary) => { + setRemoving(true); + setRemoveError(null); + const exit = await Effect.runPromiseExit(removeCustomToolSourceEffect(source.slug)); + if (Exit.isFailure(exit)) { + setRemoveError("Failed to remove custom tools source."); + setRemoving(false); + return; } - setSyncingScope(null); + window.location.assign(consoleIntegrationHref("/integrations")); }; return ( -
- - +
{loadState.status === "loading" && (
- Loading sources... + Loading source...
)} {loadState.status === "error" && ( - void loadSources()} /> + void loadSource()} /> )} - {loadState.status === "ready" && sources.length === 0 && ( -
-

No custom tools sources

-

- Add a GitHub repo to publish its tools. -

- -
+ {loadState.status === "missing" && ( + + Source not found + + The custom tools source for {props.integrationName} is no longer available. + + )} - {loadState.status === "ready" && sources.length > 0 && ( -
- - - {sources.map((source) => ( - - - - ))} - - - - {selected && ( - void syncSelected(selected)} - /> - )} -
+ {loadState.status === "ready" && ( + void syncSource(loadState.source)} + onRemove={() => void removeSource(loadState.source)} + /> )}
); @@ -214,7 +165,7 @@ export default function CustomToolsAccountsPanel() { function ErrorWithRetry(props: { readonly message: string; readonly onRetry: () => void }) { return ( - Failed to load sources + Failed to load source

{props.message}

@@ -230,20 +181,53 @@ function ErrorWithRetry(props: { readonly message: string; readonly onRetry: () function SourceDetail(props: { readonly source: GitHubCustomToolsSourceSummary; readonly notice: SyncNotice | null; + readonly removeError: string | null; readonly syncing: boolean; + readonly removing: boolean; readonly onSync: () => void; + readonly onRemove: () => void; }) { const { source, notice } = props; return (
-

{source.repo}

-

{source.scope}

+

{source.name}

+

{source.slug}

+
+
+ + + + + + + + Remove {source.name}? + + This removes {source.tools.length} {source.tools.length === 1 ? "tool" : "tools"}{" "} + from the catalog. The GitHub repository is untouched; re-add it to sync again. + + + + Cancel + + {props.removing ? "Removing..." : "Remove source"} + + + +
-
@@ -272,6 +256,8 @@ function SourceDetail(props: { )} + {props.removeError && } +
{source.tools.length > 0 ? (
@@ -333,6 +319,14 @@ function Field(props: { readonly label: string; readonly value: string; readonly ); } +function FormErrorMessage(props: { readonly message: string }) { + return ( +
+

{props.message}

+
+ ); +} + function Section(props: { readonly title: string; readonly children: ReactNode }) { return (
diff --git a/packages/plugins/apps/src/react/custom-tools-client.ts b/packages/plugins/apps/src/react/custom-tools-client.ts index f56c0b6b2..a717c9640 100644 --- a/packages/plugins/apps/src/react/custom-tools-client.ts +++ b/packages/plugins/apps/src/react/custom-tools-client.ts @@ -2,6 +2,7 @@ import { Data, Effect, Schema } from "effect"; import type { GitHubCustomToolsSourceSummary, GitHubSyncResult } from "../api"; import { parseGitHubSourceUrl } from "../source/github-url"; +import { slugifyCustomToolsAppName, validateCustomToolsAppSlug } from "../source/app-slug"; export const CUSTOM_TOOLS_PLUGIN_KEY = "apps"; export const CUSTOM_TOOLS_LABEL = "Custom tools"; @@ -10,8 +11,14 @@ export interface GitHubSourcesListResponse { readonly sources: readonly GitHubCustomToolsSourceSummary[]; } +export interface GitHubSourceDetailResponse { + readonly source: GitHubCustomToolsSourceSummary | null; +} + export interface SyncGitHubSourceRequest { - readonly url: string; + readonly url?: string; + readonly name?: string; + readonly slug?: string; readonly ref?: string; readonly token?: string; } @@ -23,6 +30,12 @@ export class CustomToolsClientError extends Data.TaggedError("CustomToolsClientE }> {} export { parseGitHubSourceUrl }; +export { slugifyCustomToolsAppName, validateCustomToolsAppSlug }; + +export const suggestCustomToolsAppName = (url: string): string => { + const parsed = parseGitHubSourceUrl(url); + return parsed.ok ? parsed.value.name : ""; +}; export const validateGitHubSourceUrl = (url: string): string | null => { const parsed = parseGitHubSourceUrl(url); @@ -81,6 +94,31 @@ export const listCustomToolSources = ( fetchImpl: CustomToolsFetch = fetch, ): Promise => Effect.runPromise(listCustomToolSourcesEffect(fetchImpl)); +export const getCustomToolSourceEffect = ( + slug: string, + fetchImpl: CustomToolsFetch = fetch, +): Effect.Effect => + Effect.tryPromise({ + try: () => + fetchImpl(`/api/apps/sources/github/${encodeURIComponent(slug)}`, { + credentials: "same-origin", + }), + catch: () => new CustomToolsClientError({ message: "Failed to load custom tools source." }), + }).pipe( + Effect.flatMap((response) => + parseJsonResponseEffect( + response, + "Failed to load custom tools source.", + ), + ), + ); + +export const getCustomToolSource = ( + slug: string, + fetchImpl: CustomToolsFetch = fetch, +): Promise => + Effect.runPromise(getCustomToolSourceEffect(slug, fetchImpl)); + export const syncCustomToolSourceEffect = ( input: SyncGitHubSourceRequest, fetchImpl: CustomToolsFetch = fetch, @@ -92,7 +130,9 @@ export const syncCustomToolSourceEffect = ( headers: { "content-type": "application/json" }, credentials: "same-origin", body: JSON.stringify({ - url: input.url.trim(), + ...(input.url?.trim() ? { url: input.url.trim() } : {}), + ...(input.name?.trim() ? { name: input.name.trim() } : {}), + ...(input.slug?.trim() ? { slug: input.slug.trim() } : {}), ...(input.ref?.trim() ? { ref: input.ref.trim() } : {}), ...(input.token?.trim() ? { token: input.token.trim() } : {}), }), @@ -109,6 +149,32 @@ export const syncCustomToolSource = ( fetchImpl: CustomToolsFetch = fetch, ): Promise => Effect.runPromise(syncCustomToolSourceEffect(input, fetchImpl)); +export const removeCustomToolSourceEffect = ( + slug: string, + fetchImpl: CustomToolsFetch = fetch, +): Effect.Effect<{ readonly removed: boolean }, CustomToolsClientError> => + Effect.tryPromise({ + try: () => + fetchImpl(`/api/apps/sources/github/${encodeURIComponent(slug)}`, { + method: "DELETE", + credentials: "same-origin", + }), + catch: () => new CustomToolsClientError({ message: "Failed to remove custom tools source." }), + }).pipe( + Effect.flatMap((response) => + parseJsonResponseEffect<{ readonly removed: boolean }>( + response, + "Failed to remove custom tools source.", + ), + ), + ); + +export const removeCustomToolSource = ( + slug: string, + fetchImpl: CustomToolsFetch = fetch, +): Promise<{ readonly removed: boolean }> => + Effect.runPromise(removeCustomToolSourceEffect(slug, fetchImpl)); + export const syncStatusLabel = (result: GitHubSyncResult): string => { if (result.status === "published") return `Published ${result.tools.length} tools.`; if (result.status === "up-to-date") return "Already up to date."; diff --git a/packages/plugins/apps/src/seams/artifact-store.ts b/packages/plugins/apps/src/seams/artifact-store.ts index 63c623e77..033a913df 100644 --- a/packages/plugins/apps/src/seams/artifact-store.ts +++ b/packages/plugins/apps/src/seams/artifact-store.ts @@ -64,4 +64,5 @@ export interface ArtifactStore { readonly forScope: ( address: ScopeAddress, ) => Effect.Effect; + readonly removeScope: (address: ScopeAddress) => Effect.Effect; } diff --git a/packages/plugins/apps/src/seams/scope-db.ts b/packages/plugins/apps/src/seams/scope-db.ts index bad96580d..22169fc49 100644 --- a/packages/plugins/apps/src/seams/scope-db.ts +++ b/packages/plugins/apps/src/seams/scope-db.ts @@ -39,5 +39,6 @@ export interface ScopeDbHandle { export interface ScopeDb { readonly forScope: (address: ScopeAddress) => Effect.Effect; + readonly removeScope: (address: ScopeAddress) => Effect.Effect; readonly close: () => Effect.Effect; } diff --git a/packages/plugins/apps/src/source/app-slug.ts b/packages/plugins/apps/src/source/app-slug.ts new file mode 100644 index 000000000..7da700111 --- /dev/null +++ b/packages/plugins/apps/src/source/app-slug.ts @@ -0,0 +1,17 @@ +export const CUSTOM_TOOLS_APP_SLUG_PATTERN = /^[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?$/; + +export const slugifyCustomToolsAppName = (input: string): string => { + const base = input + .trim() + .toLowerCase() + .replace(/[^a-z0-9]+/g, "-") + .replace(/^-+|-+$/g, ""); + return base.slice(0, 63).replace(/-+$/g, ""); +}; + +export const validateCustomToolsAppSlug = (slug: string): string | null => { + if (slug.length === 0) return "Enter a name with at least one letter or number."; + return CUSTOM_TOOLS_APP_SLUG_PATTERN.test(slug) + ? null + : "Use lowercase letters, numbers, and hyphens. Start and end with a letter or number."; +}; diff --git a/packages/plugins/apps/src/testing/index.ts b/packages/plugins/apps/src/testing/index.ts index e60cfbcba..9ac064e31 100644 --- a/packages/plugins/apps/src/testing/index.ts +++ b/packages/plugins/apps/src/testing/index.ts @@ -20,11 +20,6 @@ export const makeInMemoryAppsStore = (): AppsStore & { } => { const descriptors = new Map(); const publishedAt = new Map(); - const scopeConnections = new Map(); - const githubSourceTokens = new Map< - string, - { provider: string; itemId: string; updatedAt: number } - >(); const keyFor = (tenant: string, key: string): string => `${tenant}:${key}`; return { descriptors, @@ -36,6 +31,12 @@ export const makeInMemoryAppsStore = (): AppsStore & { }), getDescriptor: (tenant, scope) => Effect.sync(() => descriptors.get(keyFor(tenant, scope)) ?? null), + removeDescriptor: (tenant, scope) => + Effect.sync(() => { + const key = keyFor(tenant, scope); + descriptors.delete(key); + publishedAt.delete(key); + }), listDescriptors: (tenant) => Effect.sync(() => [...descriptors.entries()] @@ -45,14 +46,6 @@ export const makeInMemoryAppsStore = (): AppsStore & { publishedAt: publishedAt.get(key) ?? 0, })), ), - putScopeForConnection: (tenant, connectionName, scope) => - Effect.sync(() => void scopeConnections.set(keyFor(tenant, connectionName), scope)), - getScopeForConnection: (tenant, connectionName) => - Effect.sync(() => scopeConnections.get(keyFor(tenant, connectionName)) ?? null), - putGitHubSourceTokenRef: (tenant, scope, ref) => - Effect.sync(() => void githubSourceTokens.set(keyFor(tenant, scope), ref)), - getGitHubSourceTokenRef: (tenant, scope) => - Effect.sync(() => githubSourceTokens.get(keyFor(tenant, scope)) ?? null), }; }; @@ -94,7 +87,15 @@ export const makeInMemoryArtifactStore = (): ArtifactStore => { ), }; }; - return { forScope: (address) => Effect.succeed(forScope(scopeAddressStorageKey(address))) }; + return { + forScope: (address) => Effect.succeed(forScope(scopeAddressStorageKey(address))), + removeScope: (address) => + Effect.sync(() => { + const key = scopeAddressStorageKey(address); + scopes.delete(key); + order.delete(key); + }), + }; }; export type { SnapshotId }; diff --git a/packages/react/src/pages/integration-detail.tsx b/packages/react/src/pages/integration-detail.tsx index e331198e9..8ecb20db4 100644 --- a/packages/react/src/pages/integration-detail.tsx +++ b/packages/react/src/pages/integration-detail.tsx @@ -104,6 +104,7 @@ export function IntegrationDetailPage(props: { namespace: string }) { const integrationData = AsyncResult.isSuccess(integration) ? integration.value : null; useExecutorDocumentTitle(integrationData?.name || namespace); const isBuiltInIntegration = namespace === "executor" || integrationData?.kind === "built-in"; + const isAppsIntegration = integrationData?.kind === "apps"; const currentTab = isBuiltInIntegration ? "tools" : activeTab; const canRefresh = integrationData?.canRefresh ?? false; const canRemove = integrationData?.canRemove ?? false; @@ -153,9 +154,11 @@ export function IntegrationDetailPage(props: { namespace: string }) { // Find the plugin edit component based on integration kind const editPlugin = useMemo(() => { if (!integrationData) return null; - if (namespace === "apps") return integrationPlugins.find((p) => p.key === "apps") ?? null; + if (integrationData.kind === "apps") { + return integrationPlugins.find((p) => p.key === "apps") ?? null; + } return integrationPlugins.find((p) => p.key === integrationData.kind) ?? null; - }, [integrationData, integrationPlugins, namespace]); + }, [integrationData, integrationPlugins]); // Policies are pre-sorted by the server in evaluation order (owner rank, then // position ASC). The matcher walks the list and stops at the first hit per @@ -364,7 +367,7 @@ export function IntegrationDetailPage(props: { namespace: string }) {
- {!confirmDelete && !isBuiltInIntegration && integrationData && ( + {!confirmDelete && !isBuiltInIntegration && !isAppsIntegration && integrationData && ( @@ -382,6 +385,7 @@ export function IntegrationDetailPage(props: { namespace: string }) { )} {canRemove && + !isAppsIntegration && (confirmDelete ? (
-
-
- - - - - -
+
+
+

{model.lastSynced}

+

+ + {model.publishedTools.label} + {" "} + published +

+
{notice && ( {notice.message} - {(notice.added.length > 0 || notice.removed.length > 0 || notice.errors.length > 0) && ( + {noticeHasDetails && (
{notice.added.length > 0 &&

Added: {notice.added.join(", ")}

} @@ -250,6 +249,28 @@ function SourceDetail(props: { {notice.errors.map((error) => (

{error}

))} + {notice.skipped.length > 0 && ( +
+

Skipped:

+
    + {notice.skipped.map((entry) => ( +
  • + + {entry.path} + + + {entry.reason} + +
    • + ))} +
+
+ )} + {notice.upstreamSha && ( +

+ Commit {notice.upstreamSha} +

+ )}
)} @@ -257,68 +278,11 @@ function SourceDetail(props: { )} {props.removeError && } - -
- {source.tools.length > 0 ? ( -
- {source.tools.map((tool) => ( - - {tool} - - ))} -
- ) : ( -

No tools published.

- )} -
- -
- {source.skipped.length > 0 ? ( -
- {source.skipped.map((entry) => ( -
- - {entry.path} - - {entry.reason} -
- ))} -
- ) : ( -

Nothing skipped.

- )} -
); } -function LinkField(props: { readonly label: string; readonly value: string }) { - return ( -
-
{props.label}
-
- - {props.value} - -
-
- ); -} - -function Field(props: { readonly label: string; readonly value: string; readonly mono?: boolean }) { - return ( -
-
{props.label}
-
- {props.value} -
-
- ); -} - function FormErrorMessage(props: { readonly message: string }) { return (
@@ -326,12 +290,3 @@ function FormErrorMessage(props: { readonly message: string }) {
); } - -function Section(props: { readonly title: string; readonly children: ReactNode }) { - return ( -
-

{props.title}

- {props.children} -
- ); -} diff --git a/packages/plugins/apps/src/react/custom-tools-client.web.test.ts b/packages/plugins/apps/src/react/custom-tools-client.web.test.ts index c704e1891..c5a4e091b 100644 --- a/packages/plugins/apps/src/react/custom-tools-client.web.test.ts +++ b/packages/plugins/apps/src/react/custom-tools-client.web.test.ts @@ -16,6 +16,8 @@ import appsClientPlugin, { syncStatusLabel, validateGitHubSourceUrl, } from "./plugin-client"; +import { asSnapshotId } from "../seams/artifact-store"; +import { sourcePanelModel, syncNoticeFromResult } from "./source-panel-model"; const jsonResponse = (body: unknown, init?: ResponseInit): Response => new Response(JSON.stringify(body), { @@ -25,6 +27,21 @@ const jsonResponse = (body: unknown, init?: ResponseInit): Response => }); describe("custom tools console client", () => { + const demoSource = { + slug: "demo-tools", + name: "Demo tools", + scope: "demo-tools", + url: "https://github.com/RhysSullivan/executor-custom-tools-demo", + repo: "RhysSullivan/executor-custom-tools-demo", + ref: "main", + hasToken: true, + upstreamSha: "abc123", + snapshotId: "snap1", + publishedAt: "2026-07-06T12:00:00.000Z", + tools: ["repo-summary", "stale-issues"], + skipped: [{ path: "README.md", reason: "ignored" as const }], + }; + it("registers the manual Custom tools tile", () => { const manualTiles = [appsIntegrationPlugin].map((plugin) => ({ href: `/integrations/add/${plugin.key}`, @@ -90,6 +107,45 @@ describe("custom tools console client", () => { ).toBe(false); }); + it("models the custom-tools detail panel without default debug fields", () => { + const panel = sourcePanelModel(demoSource, { + now: Date.parse("2026-07-06T12:05:00.000Z"), + }); + + expect(panel.title).toBe("Demo tools"); + expect(panel.repository).toEqual({ + href: "https://github.com/RhysSullivan/executor-custom-tools-demo", + label: "github.com/RhysSullivan/executor-custom-tools-demo", + }); + expect(panel.lastSynced).toBe("Last synced 5m ago"); + expect(panel.publishedTools).toEqual({ + href: "/integrations/demo-tools?tab=tools", + label: "2 tools", + }); + expect(panel).not.toHaveProperty("toolNames"); + expect(panel).not.toHaveProperty("skipped"); + expect(panel).not.toHaveProperty("upstreamSha"); + expect(panel).not.toHaveProperty("hasToken"); + }); + + it("keeps skipped files and upstream SHA in the post-sync result details", () => { + const notice = syncNoticeFromResult( + { + status: "published", + snapshotId: asSnapshotId("snap2"), + upstreamSha: "def456", + tools: ["repo-summary", "stale-issues", "third-tool"], + skipped: [{ path: "workflows/x.ts", reason: "not supported yet" }], + }, + demoSource.tools, + ); + + expect(notice.message).toBe("Published 3 tools."); + expect(notice.added).toEqual(["third-tool"]); + expect(notice.skipped).toEqual([{ path: "workflows/x.ts", reason: "not supported yet" }]); + expect(notice.upstreamSha).toBe("def456"); + }); + it("surfaces successful sync and source detail data", async () => { let syncBody = ""; let removed = false; diff --git a/packages/plugins/apps/src/react/source-panel-model.ts b/packages/plugins/apps/src/react/source-panel-model.ts new file mode 100644 index 000000000..707703dae --- /dev/null +++ b/packages/plugins/apps/src/react/source-panel-model.ts @@ -0,0 +1,92 @@ +import type { GitHubCustomToolsSourceSummary, GitHubSyncResult } from "../api"; +import { + consoleIntegrationHref, + formatSyncErrors, + syncStatusLabel, + toolDiff, +} from "./custom-tools-client"; + +export interface SourcePanelModel { + readonly title: string; + readonly repository: { + readonly href: string; + readonly label: string; + }; + readonly lastSynced: string; + readonly publishedTools: { + readonly href: string; + readonly label: string; + }; +} + +export interface SyncNoticeModel { + readonly status: GitHubSyncResult["status"]; + readonly message: string; + readonly added: readonly string[]; + readonly removed: readonly string[]; + readonly errors: readonly string[]; + readonly skipped: GitHubSyncResult["skipped"]; + readonly upstreamSha?: string; +} + +export const formatRelativeSyncTime = (iso: string, now = Date.now()): string => { + const then = new Date(iso).getTime(); + if (Number.isNaN(then)) return iso; + const diffMs = Math.max(0, now - then); + const min = Math.floor(diffMs / 60_000); + if (min < 1) return "just now"; + if (min < 60) return `${min}m ago`; + const hrs = Math.floor(min / 60); + if (hrs < 24) return `${hrs}h ago`; + const days = Math.floor(hrs / 24); + if (days < 7) return `${days}d ago`; + const weeks = Math.floor(days / 7); + if (weeks < 5) return `${weeks}w ago`; + const months = Math.floor(days / 30); + if (months < 12) return `${months}mo ago`; + return `${Math.floor(days / 365)}y ago`; +}; + +export const sourceRepositoryDisplay = ( + source: GitHubCustomToolsSourceSummary, +): SourcePanelModel["repository"] => { + const explicitRef = /\/(tree|commit)\//.test(source.url); + const base = `github.com/${source.repo}`; + return { + href: source.url, + label: explicitRef ? `${base} @ ${source.ref}` : base, + }; +}; + +export const toolsCountLabel = (count: number): string => + `${count} ${count === 1 ? "tool" : "tools"}`; + +export const sourcePanelModel = ( + source: GitHubCustomToolsSourceSummary, + options?: { readonly now?: number }, +): SourcePanelModel => ({ + title: source.name, + repository: sourceRepositoryDisplay(source), + lastSynced: `Last synced ${formatRelativeSyncTime(source.publishedAt, options?.now)}`, + publishedTools: { + href: consoleIntegrationHref(`/integrations/${encodeURIComponent(source.slug)}?tab=tools`), + label: toolsCountLabel(source.tools.length), + }, +}); + +export const syncNoticeFromResult = ( + result: GitHubSyncResult, + beforeTools: readonly string[], +): SyncNoticeModel => { + const diff = + result.status === "failed" ? { added: [], removed: [] } : toolDiff(beforeTools, result.tools); + return { + status: result.status, + message: syncStatusLabel(result), + added: diff.added, + removed: diff.removed, + errors: formatSyncErrors(result), + skipped: result.skipped, + ...(result.upstreamSha ? { upstreamSha: result.upstreamSha } : {}), + }; +}; diff --git a/packages/react/src/lib/integration-detail-tabs.test.ts b/packages/react/src/lib/integration-detail-tabs.test.ts new file mode 100644 index 000000000..09096181b --- /dev/null +++ b/packages/react/src/lib/integration-detail-tabs.test.ts @@ -0,0 +1,23 @@ +import { describe, expect, it } from "@effect/vitest"; + +import { + integrationDetailInternalTabFromSearch, + integrationDetailSearchTabForInternal, + integrationDetailTabForAddCompletion, +} from "./integration-detail-tabs"; + +describe("integration detail tab routing", () => { + it("lands custom-tools add completion on the same source tab a reload reads", () => { + const addCompletionTab = integrationDetailTabForAddCompletion("apps"); + + expect(addCompletionTab).toBe("source"); + expect(integrationDetailInternalTabFromSearch(addCompletionTab)).toBe("accounts"); + expect(integrationDetailSearchTabForInternal("apps", "accounts")).toBe(addCompletionTab); + }); + + it("keeps non-app integrations on the accounts/tools tab vocabulary", () => { + expect(integrationDetailTabForAddCompletion("openapi")).toBeUndefined(); + expect(integrationDetailSearchTabForInternal("openapi", "accounts")).toBe("accounts"); + expect(integrationDetailSearchTabForInternal("openapi", "tools")).toBe("tools"); + }); +}); diff --git a/packages/react/src/lib/integration-detail-tabs.ts b/packages/react/src/lib/integration-detail-tabs.ts new file mode 100644 index 000000000..90495205d --- /dev/null +++ b/packages/react/src/lib/integration-detail-tabs.ts @@ -0,0 +1,16 @@ +export type IntegrationDetailSearchTab = "accounts" | "source" | "tools"; +export type IntegrationDetailInternalTab = "accounts" | "tools"; + +export const integrationDetailTabForAddCompletion = ( + pluginKey: string, +): IntegrationDetailSearchTab | undefined => (pluginKey === "apps" ? "source" : undefined); + +export const integrationDetailInternalTabFromSearch = ( + tab: IntegrationDetailSearchTab | undefined, +): IntegrationDetailInternalTab => (tab === "tools" ? "tools" : "accounts"); + +export const integrationDetailSearchTabForInternal = ( + integrationKind: string | undefined, + tab: IntegrationDetailInternalTab, +): IntegrationDetailSearchTab => + tab === "tools" ? "tools" : integrationKind === "apps" ? "source" : "accounts"; diff --git a/packages/react/src/pages/integration-add.tsx b/packages/react/src/pages/integration-add.tsx index df1faf752..1c6f74270 100644 --- a/packages/react/src/pages/integration-add.tsx +++ b/packages/react/src/pages/integration-add.tsx @@ -1,8 +1,11 @@ import { Suspense } from "react"; +import { useAtomRefresh } from "@effect/atom-react"; import { Link, useNavigate } from "@tanstack/react-router"; import { useIntegrationPlugins } from "@executor-js/sdk/client"; +import { integrationsOptimisticAtom } from "../api/atoms"; import { trackEvent } from "../api/analytics"; import { useExecutorDocumentTitle } from "../lib/document-title"; +import { integrationDetailTabForAddCompletion } from "../lib/integration-detail-tabs"; // --------------------------------------------------------------------------- // Page @@ -18,6 +21,7 @@ export function AddIntegrationPage(props: { const { pluginKey, url, preset, namespace } = props; const navigate = useNavigate(); const integrationPlugins = useIntegrationPlugins(); + const refreshIntegrations = useAtomRefresh(integrationsOptimisticAtom); const plugin = integrationPlugins.find((p) => p.key === pluginKey); @@ -59,9 +63,15 @@ export function AddIntegrationPage(props: { plugin_key: pluginKey, ...(slug ? { integration_slug: slug } : {}), }); + refreshIntegrations(); + const tab = integrationDetailTabForAddCompletion(pluginKey); void navigate( slug - ? { to: "/{-$orgSlug}/integrations/$namespace", params: { namespace: slug } } + ? { + to: "/{-$orgSlug}/integrations/$namespace", + params: { namespace: slug }, + search: tab ? { tab } : {}, + } : { to: "/{-$orgSlug}" }, ); }} diff --git a/packages/react/src/pages/integration-detail.tsx b/packages/react/src/pages/integration-detail.tsx index 8ecb20db4..dd35ba380 100644 --- a/packages/react/src/pages/integration-detail.tsx +++ b/packages/react/src/pages/integration-detail.tsx @@ -36,6 +36,12 @@ import { Skeleton } from "../components/skeleton"; import { useExecutorDocumentTitle } from "../lib/document-title"; import { ErrorState } from "../components/error-state"; import { isAsyncResultLoading } from "../lib/async-result"; +import { + integrationDetailInternalTabFromSearch, + integrationDetailSearchTabForInternal, + type IntegrationDetailInternalTab, + type IntegrationDetailSearchTab, +} from "../lib/integration-detail-tabs"; // v2: the route's `namespace` param is the integration slug. Tools belong to // the integration's per-owner connections; a tool's policy id is @@ -52,7 +58,10 @@ type ToolRow = { readonly static?: boolean; }; -export function IntegrationDetailPage(props: { namespace: string }) { +export function IntegrationDetailPage(props: { + namespace: string; + tab?: IntegrationDetailSearchTab; +}) { const { namespace } = props; const slug = IntegrationSlug.make(namespace); const integrationPlugins = useIntegrationPlugins(); @@ -89,7 +98,9 @@ export function IntegrationDetailPage(props: { namespace: string }) { const [deleting, setDeleting] = useState(false); const [refreshing, setRefreshing] = useState(false); const [editSheetOpen, setEditSheetOpen] = useState(false); - const [activeTab, setActiveTab] = useState<"accounts" | "tools">("accounts"); + const [activeTab, setActiveTab] = useState(() => + integrationDetailInternalTabFromSearch(props.tab), + ); const [manualAccountHandoff, setManualAccountHandoff] = useState(null); const [locationSearch] = useState(() => @@ -101,6 +112,10 @@ export function IntegrationDetailPage(props: { namespace: string }) { setEditSheetOpen(false); }, [namespace]); + useEffect(() => { + setActiveTab(integrationDetailInternalTabFromSearch(props.tab)); + }, [namespace, props.tab]); + const integrationData = AsyncResult.isSuccess(integration) ? integration.value : null; useExecutorDocumentTitle(integrationData?.name || namespace); const isBuiltInIntegration = namespace === "executor" || integrationData?.kind === "built-in"; @@ -351,6 +366,18 @@ export function IntegrationDetailPage(props: { namespace: string }) { setManualAccountHandoff({ key: `manual:${String(slug)}:${Date.now()}` }); }; + const handleTabChange = (value: string) => { + const nextTab = value === "tools" ? "tools" : "accounts"; + setActiveTab(nextTab); + void navigate({ + to: "/{-$orgSlug}/integrations/$namespace", + params: { namespace }, + search: { + tab: integrationDetailSearchTabForInternal(integrationData?.kind, nextTab), + }, + }); + }; + return (
{/* Header bar */} @@ -420,7 +447,7 @@ export function IntegrationDetailPage(props: { namespace: string }) { setActiveTab(value as "accounts" | "tools")} + onValueChange={handleTabChange} className="min-h-0 flex-1 gap-0 overflow-hidden" >
diff --git a/packages/react/src/routes/integrations.$namespace.tsx b/packages/react/src/routes/integrations.$namespace.tsx index 69f86603a..e8d9a3499 100644 --- a/packages/react/src/routes/integrations.$namespace.tsx +++ b/packages/react/src/routes/integrations.$namespace.tsx @@ -1,10 +1,19 @@ +import { Schema } from "effect"; import { createFileRoute } from "@tanstack/react-router"; import { IntegrationDetailPage } from "../pages/integration-detail"; +const SearchParams = Schema.toStandardSchemaV1( + Schema.Struct({ + tab: Schema.optional(Schema.Literals(["accounts", "source", "tools"])), + }), +); + export const Route = createFileRoute("/{-$orgSlug}/integrations/$namespace")({ + validateSearch: SearchParams, component: () => { const { namespace } = Route.useParams(); - return ; + const { tab } = Route.useSearch(); + return ; }, }); From ff362d24bb021e12d9aab699f17dac06a8a4dd7f Mon Sep 17 00:00:00 2001 From: Rhys Sullivan <39114868+RhysSullivan@users.noreply.github.com> Date: Tue, 7 Jul 2026 01:33:10 -0700 Subject: [PATCH 18/21] Remove custom tools storage --- apps/host-selfhost/src/apps-backings.ts | 11 +- packages/plugins/apps/src/api.ts | 2 +- packages/plugins/apps/src/authoring.ts | 3 - .../apps/src/backing/libsql-scope-db.test.ts | 4 - .../apps/src/backing/libsql-scope-db.ts | 187 ------------------ .../src/backing/quickjs-tool-sandbox.test.ts | 36 +++- .../apps/src/backing/quickjs-tool-sandbox.ts | 28 ++- .../apps/src/backing/scope-collision.test.ts | 40 +--- packages/plugins/apps/src/index.ts | 2 +- .../plugins/apps/src/pipeline/publish.test.ts | 28 +++ .../apps/src/plugin/apps-plugin.test.ts | 99 +++++----- .../plugins/apps/src/plugin/apps-plugin.ts | 58 ++++-- packages/plugins/apps/src/plugin/backings.ts | 3 - .../apps/src/plugin/bindings.strict.test.ts | 14 +- packages/plugins/apps/src/plugin/bindings.ts | 39 +--- .../plugins/apps/src/plugin/runtime.test.ts | 22 ++- packages/plugins/apps/src/plugin/runtime.ts | 16 -- .../apps/src/plugin/self-host-runtime.ts | 16 +- .../apps/src/plugin/tenant-isolation.test.ts | 25 +-- packages/plugins/apps/src/seams/index.ts | 1 - .../apps/src/seams/scope-db.conformance.ts | 70 ------- packages/plugins/apps/src/seams/scope-db.ts | 44 ----- .../src/seams/tool-sandbox.conformance.ts | 55 ------ .../plugins/apps/src/seams/tool-sandbox.ts | 10 +- .../plugins/apps/src/testing/daily-brief.ts | 25 +-- 25 files changed, 221 insertions(+), 617 deletions(-) delete mode 100644 packages/plugins/apps/src/backing/libsql-scope-db.test.ts delete mode 100644 packages/plugins/apps/src/backing/libsql-scope-db.ts delete mode 100644 packages/plugins/apps/src/seams/scope-db.conformance.ts delete mode 100644 packages/plugins/apps/src/seams/scope-db.ts diff --git a/apps/host-selfhost/src/apps-backings.ts b/apps/host-selfhost/src/apps-backings.ts index 62a9db595..6bebe52cb 100644 --- a/apps/host-selfhost/src/apps-backings.ts +++ b/apps/host-selfhost/src/apps-backings.ts @@ -1,10 +1,7 @@ import { join } from "node:path"; -import { Effect } from "effect"; - import { makeGitArtifactStore, - makeLibsqlScopeDb, makeQuickjsToolSandbox, makeSqliteAppsStore, type AppsBackings, @@ -19,19 +16,13 @@ interface SelfHostAppsBackings { const createBackings = (dataDir: string): SelfHostAppsBackings => { const appsDir = join(dataDir, "apps"); - const scopeDb = makeLibsqlScopeDb({ - root: join(appsDir, "scope-db"), - }); return { backings: { artifactStore: makeGitArtifactStore({ root: join(appsDir, "artifacts") }), - scopeDb, sandbox: makeQuickjsToolSandbox(), store: makeSqliteAppsStore({ path: join(appsDir, "store.sqlite") }), }, - close: async () => { - await Effect.runPromise(scopeDb.close().pipe(Effect.orElseSucceed(() => undefined))); - }, + close: async () => {}, }; }; diff --git a/packages/plugins/apps/src/api.ts b/packages/plugins/apps/src/api.ts index 11bae9160..b028b16e4 100644 --- a/packages/plugins/apps/src/api.ts +++ b/packages/plugins/apps/src/api.ts @@ -28,7 +28,6 @@ export { export { type AppsRuntime, type GitHubCustomToolsSourceSummary } from "./plugin/runtime"; export { makeAppsRuntimeFromBackings, type AppsBackings } from "./plugin/backings"; export { makeGitArtifactStore } from "./backing/git-artifact-store"; -export { makeLibsqlScopeDb } from "./backing/libsql-scope-db"; export { makeQuickjsToolSandbox } from "./backing/quickjs-tool-sandbox"; export { makeSqliteAppsStore } from "./backing/sqlite-apps-store"; export { @@ -38,6 +37,7 @@ export { type RoleBindings, } from "./plugin/bindings"; export { makePluginCtxAppsResolver } from "./plugin/resolver"; +export { SourceOriginError, assertSourceOrigin } from "./plugin/apps-plugin"; export { fetchGitHubSource, parseGitHubSourceUrl, diff --git a/packages/plugins/apps/src/authoring.ts b/packages/plugins/apps/src/authoring.ts index e60335d9f..67ee1262c 100644 --- a/packages/plugins/apps/src/authoring.ts +++ b/packages/plugins/apps/src/authoring.ts @@ -1,4 +1,3 @@ -import type { ScopeDbHandle } from "./seams/scope-db"; import type { StandardSchemaV1 } from "./standard-schema"; export type JsonPrimitive = string | number | boolean | null; @@ -16,8 +15,6 @@ export interface IntegrationDeclaration { export type ToolHandlerContext< TIntegrations extends Readonly> | undefined, > = { - readonly db: ScopeDbHandle; -} & { readonly [K in keyof NonNullable]: unknown; }; diff --git a/packages/plugins/apps/src/backing/libsql-scope-db.test.ts b/packages/plugins/apps/src/backing/libsql-scope-db.test.ts deleted file mode 100644 index 4ebbebd3a..000000000 --- a/packages/plugins/apps/src/backing/libsql-scope-db.test.ts +++ /dev/null @@ -1,4 +0,0 @@ -import { scopeDbConformance } from "../seams/scope-db.conformance"; -import { makeLibsqlScopeDb } from "./libsql-scope-db"; - -scopeDbConformance("libsql (in-memory)", () => makeLibsqlScopeDb({ root: ":memory:" })); diff --git a/packages/plugins/apps/src/backing/libsql-scope-db.ts b/packages/plugins/apps/src/backing/libsql-scope-db.ts deleted file mode 100644 index d38f8c692..000000000 --- a/packages/plugins/apps/src/backing/libsql-scope-db.ts +++ /dev/null @@ -1,187 +0,0 @@ -import { mkdirSync, rmSync } from "node:fs"; -import { join, resolve } from "node:path"; - -import { createClient, type Client } from "@libsql/client"; -import { Effect } from "effect"; - -import { ScopeDbError, type ScopeDb, type ScopeDbHandle } from "../seams/scope-db"; -import { scopeAddressStorageKey, type ScopeAddress } from "../seams/scope-address"; - -// --------------------------------------------------------------------------- -// libSQL-backed ScopeDb (self-hosted). One SQLite file per tenant/scope under -// ``. A control table `__versions` holds a monotonic counter per user -// table; every write statement bumps the counters for the tables it touched. -// Scope isolation is a separate file per tenant/scope, there is no cross-scope -// query path. -// --------------------------------------------------------------------------- - -const VERSION_TABLE = "__scope_versions"; - -// Statements that mutate data. Determines whether a `sql` call bumps versions. -const WRITE_RE = /^\s*(insert|update|delete|replace|create|drop|alter)\b/i; - -// Extract the table names a write statement targets (best-effort: covers the -// shapes authored tools produce — INSERT INTO x, UPDATE x, DELETE FROM x, -// CREATE TABLE x, REPLACE INTO x). -const targetsOf = (sql: string): string[] => { - const out = new Set(); - const patterns = [ - /\binsert\s+(?:or\s+\w+\s+)?into\s+["'`]?([a-zA-Z_][a-zA-Z0-9_]*)["'`]?/gi, - /\breplace\s+into\s+["'`]?([a-zA-Z_][a-zA-Z0-9_]*)["'`]?/gi, - /\bupdate\s+["'`]?([a-zA-Z_][a-zA-Z0-9_]*)["'`]?/gi, - /\bdelete\s+from\s+["'`]?([a-zA-Z_][a-zA-Z0-9_]*)["'`]?/gi, - /\bcreate\s+table\s+(?:if\s+not\s+exists\s+)?["'`]?([a-zA-Z_][a-zA-Z0-9_]*)["'`]?/gi, - /\bdrop\s+table\s+(?:if\s+exists\s+)?["'`]?([a-zA-Z_][a-zA-Z0-9_]*)["'`]?/gi, - /\balter\s+table\s+["'`]?([a-zA-Z_][a-zA-Z0-9_]*)["'`]?/gi, - ]; - for (const re of patterns) { - let m: RegExpExecArray | null; - while ((m = re.exec(sql)) !== null) { - const name = m[1]; - if (name && name !== VERSION_TABLE) out.add(name); - } - } - return [...out]; -}; - -const toArgs = (values: readonly unknown[]): unknown[] => - values.map((v) => (v === undefined ? null : v)); - -const makeHandle = (address: ScopeAddress, client: Client): ScopeDbHandle => { - const label = `${address.tenant}/${address.scope}`; - const ensureVersionTable = async () => { - await client.execute( - `CREATE TABLE IF NOT EXISTS ${VERSION_TABLE} (name TEXT PRIMARY KEY, version INTEGER NOT NULL DEFAULT 0)`, - ); - }; - - const bump = async (tables: readonly string[]): Promise<{ table: string; version: number }[]> => { - const bumped: { table: string; version: number }[] = []; - for (const table of tables) { - await client.execute({ - sql: `INSERT INTO ${VERSION_TABLE} (name, version) VALUES (?, 1) - ON CONFLICT(name) DO UPDATE SET version = version + 1`, - args: [table], - }); - const row = await client.execute({ - sql: `SELECT version FROM ${VERSION_TABLE} WHERE name = ?`, - args: [table], - }); - bumped.push({ table, version: Number(row.rows[0]?.version ?? 0) }); - } - return bumped; - }; - - const runStatement = ( - sql: string, - args: unknown[], - ): Effect.Effect => - Effect.tryPromise({ - try: async () => { - await ensureVersionTable(); - const result = await client.execute({ sql, args: args as never }); - if (WRITE_RE.test(sql)) { - const targets = targetsOf(sql); - if (targets.length > 0) { - await bump(targets); - } - } - return result.rows.map((row) => row as Row); - }, - catch: (cause) => - new ScopeDbError({ message: `scope-db statement failed for scope ${label}`, cause }), - }); - - return { - sql: >(strings: TemplateStringsArray, ...values: unknown[]) => { - const sql = strings.reduce((acc, part, i) => acc + part + (i < values.length ? "?" : ""), ""); - return runStatement(sql, toArgs(values)); - }, - exec: >(sql: string, params: readonly unknown[] = []) => - runStatement(sql, toArgs(params)), - tableVersion: (table: string) => - Effect.tryPromise({ - try: async () => { - await ensureVersionTable(); - const row = await client.execute({ - sql: `SELECT version FROM ${VERSION_TABLE} WHERE name = ?`, - args: [table], - }); - return Number(row.rows[0]?.version ?? 0); - }, - catch: (cause) => new ScopeDbError({ message: "tableVersion failed", cause }), - }), - versions: () => - Effect.tryPromise({ - try: async () => { - await ensureVersionTable(); - const rows = await client.execute(`SELECT name, version FROM ${VERSION_TABLE}`); - const out = new Map(); - for (const r of rows.rows) out.set(String(r.name), Number(r.version)); - return out as ReadonlyMap; - }, - catch: (cause) => new ScopeDbError({ message: "versions failed", cause }), - }), - }; -}; - -export interface LibsqlScopeDbOptions { - /** Directory holding one SQLite file per tenant/scope, or ":memory:" for tests. */ - readonly root: string; -} - -const toUrl = (path: string): string => (path === ":memory:" ? path : `file:${resolve(path)}`); - -export const makeLibsqlScopeDb = (options: LibsqlScopeDbOptions): ScopeDb => { - const clients = new Map(); - - const clientFor = (address: ScopeAddress): Client => { - const key = scopeAddressStorageKey(address); - let client = clients.get(key); - if (!client) { - if (options.root === ":memory:") { - client = createClient({ url: ":memory:" }); - } else { - mkdirSync(options.root, { recursive: true }); - client = createClient({ url: toUrl(join(options.root, `${key}.db`)) }); - } - clients.set(key, client); - } - return client; - }; - - return { - forScope: (address) => - Effect.try({ - try: () => makeHandle(address, clientFor(address)), - catch: (cause) => - new ScopeDbError({ - message: `failed to open scope db ${address.tenant}/${address.scope}`, - cause, - }), - }), - removeScope: (address) => - Effect.try({ - try: () => { - const key = scopeAddressStorageKey(address); - clients.get(key)?.close(); - clients.delete(key); - if (options.root === ":memory:") return; - const dbPath = join(options.root, `${key}.db`); - for (const path of [dbPath, `${dbPath}-shm`, `${dbPath}-wal`]) { - rmSync(path, { force: true }); - } - }, - catch: (cause) => - new ScopeDbError({ - message: `failed to remove scope db ${address.tenant}/${address.scope}`, - cause, - }), - }), - close: () => - Effect.sync(() => { - for (const client of clients.values()) client.close(); - clients.clear(); - }), - }; -}; diff --git a/packages/plugins/apps/src/backing/quickjs-tool-sandbox.test.ts b/packages/plugins/apps/src/backing/quickjs-tool-sandbox.test.ts index 01488e0cb..f1ed4bda1 100644 --- a/packages/plugins/apps/src/backing/quickjs-tool-sandbox.test.ts +++ b/packages/plugins/apps/src/backing/quickjs-tool-sandbox.test.ts @@ -31,7 +31,7 @@ describe("QuickJS ToolSandbox", () => { expect(descriptor.inputJsonSchema.properties).toHaveProperty("since"); }); - it("invokes a tool, routing injected-client + db calls through the bridge", async () => { + it("invokes a tool, routing injected-client calls through the bridge", async () => { const files = new Map([["tools/issues-sync.ts", ISSUES_SYNC_TS]]); const { code } = await run(bundleEntry({ files, entry: "tools/issues-sync.ts" })); @@ -56,7 +56,6 @@ describe("QuickJS ToolSandbox", () => { }, ]; } - // db.sql calls return empty rows return []; }), }; @@ -68,18 +67,43 @@ describe("QuickJS ToolSandbox", () => { artifact: "issues-sync", kind: "tool", input: {}, - roots: { github: { kind: "single" }, db: { kind: "single" } }, + roots: { github: { kind: "single" } }, }, bridge, ), ); - expect(result.output).toEqual({ synced: 1, repos: 1 }); - // The db write went through the bridge (root "db"). - expect(calls.some((c) => c.root === "db")).toBe(true); + expect(result.output).toEqual({ + synced: 1, + repos: 1, + issues: [{ repo: "acme/app", number: 1, title: "Bug" }], + }); expect(calls.some((c) => c.root === "github")).toBe(true); }); + it("reports storage as unavailable when an old handler calls db.sql", async () => { + const source = `import { defineTool } from "executor:app"; +import { z } from "zod"; +export default defineTool({ + description: "legacy storage", + input: z.object({}), + async handler(_input, { db }) { + await db.sql\`SELECT 1\`; + return { ok: true }; + }, +});`; + const files = new Map([["tools/storage.ts", source]]); + const { code } = await run(bundleEntry({ files, entry: "tools/storage.ts" })); + const bridge: HandleBridge = { call: () => Effect.succeed(null) }; + + const exit = await Effect.runPromiseExit( + sandbox.invoke(code, { artifact: "storage", kind: "tool", input: {}, roots: {} }, bridge), + ); + + expect(Exit.isFailure(exit)).toBe(true); + expect(JSON.stringify(exit)).toContain("storage is not available yet"); + }); + it("surfaces Standard Schema validation issues before the handler runs", async () => { const source = `import { defineTool } from "executor:app"; import { z } from "zod"; diff --git a/packages/plugins/apps/src/backing/quickjs-tool-sandbox.ts b/packages/plugins/apps/src/backing/quickjs-tool-sandbox.ts index 2402a8dd7..df87118cc 100644 --- a/packages/plugins/apps/src/backing/quickjs-tool-sandbox.ts +++ b/packages/plugins/apps/src/backing/quickjs-tool-sandbox.ts @@ -29,6 +29,8 @@ import { stableStringify } from "../pipeline/descriptor"; // `await __invokeTool("__handle__", { root, path, args })` — the ONE bridge the // QuickJS runtime already provides (`tools`/`__invokeTool`). Our // `SandboxToolInvoker` decodes that and forwards to the host `HandleBridge`. +// The old `db` handle is intentionally unavailable in v1; handlers that still +// try it get a clear error before any host storage call exists. // Everything crossing is JSON (the cloud version is RPC), so the interface // stays honest. // @@ -63,6 +65,16 @@ function __mkHandle(root, prefix) { } }); } +function __unavailableStorage() { + return new Proxy({}, { + get: function() { + return function() { throw new Error('storage is not available yet'); }; + }, + apply: function() { + throw new Error('storage is not available yet'); + } + }); +} var __executorApp = { integration: function(integration) { return { integration: integration }; }, defineTool: function(def) { __defs.tool = def; return def; }, @@ -167,6 +179,16 @@ return await (async () => { }); }; var def = __defs.tool || (globalThis.__artifact && (globalThis.__artifact.default || globalThis.__artifact)); + if (def && typeof def.handler === 'function') { + var handlerSource = Function.prototype.toString.call(def.handler); + if (/\\bdb\\b/.test(handlerSource)) { + fail('StorageUnavailableError', { + tool: artifactName, + field: 'handler', + message: 'tool "' + artifactName + '" uses storage, but storage is not available yet', + }); + } + } var integrations = {}; if (def && def.integrations) { for (var k in def.integrations) { @@ -226,6 +248,7 @@ return await (async () => { if (spec.kind !== 'single') throw new Error('unsupported handle root kind: ' + spec.kind); injected[name] = __mkHandle(name, []); } + injected.db = __unavailableStorage(); var input = await validateWithStandardSchema('input', def.input, ${inputLiteral}); var out = await def.handler(input, injected); out = await validateWithStandardSchema('output', def.output, out); @@ -267,7 +290,10 @@ export const makeQuickjsToolSandbox = (options: QuickjsToolSandboxOptions = {}): Effect.flatMap((result) => { if (result.error) { const marker = parseMarker(result.error); - if (isMarker(marker, "SchemaExportError")) { + if ( + isMarker(marker, "SchemaExportError") || + isMarker(marker, "StorageUnavailableError") + ) { return Effect.fail( new ToolSandboxError({ kind: "collect", diff --git a/packages/plugins/apps/src/backing/scope-collision.test.ts b/packages/plugins/apps/src/backing/scope-collision.test.ts index 9ed6b59be..72e08f3da 100644 --- a/packages/plugins/apps/src/backing/scope-collision.test.ts +++ b/packages/plugins/apps/src/backing/scope-collision.test.ts @@ -1,52 +1,18 @@ -import { mkdtempSync, readdirSync } from "node:fs"; -import { tmpdir } from "node:os"; -import { join } from "node:path"; - import { describe, expect, it } from "@effect/vitest"; import { Effect } from "effect"; -import { makeLibsqlScopeDb } from "./libsql-scope-db"; import { makeSqliteAppsStore } from "./sqlite-apps-store"; import { DESCRIPTOR_VERSION, type AppDescriptor } from "../pipeline/descriptor"; -import { scopeAddress } from "../seams/scope-address"; const run = (effect: Effect.Effect): Promise => Effect.runPromise(effect); // --------------------------------------------------------------------------- -// Finding 9 regression: scopes that differ only by a character the old naming -// collapsed ("my-scope" vs "my_scope") must NOT share a database, and the -// connection->scope mapping must route each to its own scope. Before the fix the -// scope-db filename collapsed `[^A-Za-z0-9._-]` to `_` (colliding two scopes onto -// one file) and the connection name was reverse-parsed (colliding two scopes onto -// one normalized identifier). +// Finding 9 regression: scopes that differ only by a character an old naming +// scheme collapsed ("my-scope" vs "my_scope") must not share descriptor +// storage. // --------------------------------------------------------------------------- describe("scope collision (Fix 9)", () => { - it("gives my-scope and my_scope DISTINCT scope-db files with isolated data", async () => { - const root = mkdtempSync(join(tmpdir(), "apps-scopecol-")); - const db = makeLibsqlScopeDb({ root }); - - const a = await run(db.forScope(scopeAddress("org", "my-scope"))); - await run(a.exec("CREATE TABLE t (v TEXT)")); - await run(a.exec("INSERT INTO t (v) VALUES ('a')")); - - const b = await run(db.forScope(scopeAddress("org", "my_scope"))); - await run(b.exec("CREATE TABLE t (v TEXT)")); - await run(b.exec("INSERT INTO t (v) VALUES ('b')")); - - // Each scope sees ONLY its own row (no shared file). - const aRows = await run(a.exec<{ v: string }>("SELECT v FROM t")); - const bRows = await run(b.exec<{ v: string }>("SELECT v FROM t")); - expect(aRows.map((r) => r.v)).toEqual(["a"]); - expect(bRows.map((r) => r.v)).toEqual(["b"]); - - // Two distinct .db files exist on disk. - const dbFiles = readdirSync(root).filter((f) => f.endsWith(".db")); - expect(dbFiles.length).toBe(2); - - await run(db.close()); - }); - it("stores distinct descriptor scopes independently", async () => { const store = makeSqliteAppsStore({ path: ":memory:" }); const descriptor = (scope: string): AppDescriptor => ({ diff --git a/packages/plugins/apps/src/index.ts b/packages/plugins/apps/src/index.ts index 56db164dc..8b82ff2b0 100644 --- a/packages/plugins/apps/src/index.ts +++ b/packages/plugins/apps/src/index.ts @@ -52,6 +52,7 @@ export { type BindingContext, } from "./plugin/bindings"; export { makePluginCtxAppsResolver } from "./plugin/resolver"; +export { SourceOriginError, assertSourceOrigin } from "./plugin/apps-plugin"; export { makeSelfHostAppsRuntime, type SelfHostAppsRuntime, @@ -60,6 +61,5 @@ export { // Self-host seam backings. export { makeGitArtifactStore } from "./backing/git-artifact-store"; -export { makeLibsqlScopeDb } from "./backing/libsql-scope-db"; export { makeQuickjsToolSandbox } from "./backing/quickjs-tool-sandbox"; export { makeSqliteAppsStore } from "./backing/sqlite-apps-store"; diff --git a/packages/plugins/apps/src/pipeline/publish.test.ts b/packages/plugins/apps/src/pipeline/publish.test.ts index 71fad6144..2f83f0d37 100644 --- a/packages/plugins/apps/src/pipeline/publish.test.ts +++ b/packages/plugins/apps/src/pipeline/publish.test.ts @@ -94,6 +94,34 @@ export default defineTool({ expect(JSON.stringify(exit)).toContain("crm"); }); + it("rejects tools that reference storage", async () => { + const deps = makeDeps(); + const files = new Map([ + [ + "tools/storage.ts", + `import { defineTool } from "executor:app"; +import { z } from "zod"; +export default defineTool({ + description: "storage", + input: z.object({}), + async handler(_input, { db }) { + await db.sql\`SELECT 1\`; + return { ok: true }; + }, +});`, + ], + ]); + + const exit = await Effect.runPromiseExit(publish(deps, { scope: "s", files })); + + expect(Exit.isFailure(exit)).toBe(true); + expect(JSON.stringify(exit)).toContain("storage is not available yet"); + const latest = await run( + Effect.flatMap(deps.artifactStore.forScope(scopeAddress("org", "s")), (s) => s.latest()), + ); + expect(latest).toBeNull(); + }); + it("rejects a Standard Schema vendor without JSON Schema export", async () => { const deps = makeDeps(); const files = new Map([ diff --git a/packages/plugins/apps/src/plugin/apps-plugin.test.ts b/packages/plugins/apps/src/plugin/apps-plugin.test.ts index e527c1de3..2e5d50a25 100644 --- a/packages/plugins/apps/src/plugin/apps-plugin.test.ts +++ b/packages/plugins/apps/src/plugin/apps-plugin.test.ts @@ -1,4 +1,4 @@ -import { mkdtempSync, readFileSync } from "node:fs"; +import { mkdtempSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; @@ -7,27 +7,13 @@ import { Effect } from "effect"; import { IntegrationSlug } from "@executor-js/sdk"; import { makeTestExecutor } from "@executor-js/sdk/testing"; -import { appsPlugin } from "./apps-plugin"; +import { SourceOriginError, appsPlugin, assertSourceOrigin } from "./apps-plugin"; import { makeSelfHostAppsRuntime } from "./self-host-runtime"; import { makeInMemoryAppsStore, makeTestResolver } from "../testing"; +import { dailyBriefFileSet } from "../testing/daily-brief"; const run = (effect: Effect.Effect): Promise => Effect.runPromise(effect); -const PROTOTYPE_ROOT = "/Users/rhyssullivan/agent-workspace/prototypes/custom-tools"; - -const prototypeFileSet = (): Map => - new Map([ - ["executor.json", readFileSync(join(PROTOTYPE_ROOT, "executor.json"), "utf8")], - [ - "tools/deal-pipeline-sync.ts", - readFileSync(join(PROTOTYPE_ROOT, "tools/deal-pipeline-sync.ts"), "utf8"), - ], - [ - "tools/find-deal-docs.ts", - readFileSync(join(PROTOTYPE_ROOT, "tools/find-deal-docs.ts"), "utf8"), - ], - ]); - describe("appsPlugin custom-tools contract", () => { it.effect("detects GitHub repo URLs for console auto-detect", () => Effect.gen(function* () { @@ -77,30 +63,41 @@ describe("appsPlugin custom-tools contract", () => { }), ); - it("round-trips prototype files through publish, resolveTools, and invokeTool", async () => { - let dealListArgs: unknown; + it.effect("rejects publishing through a different source door for a GitHub-managed app", () => + Effect.gen(function* () { + const failure = yield* Effect.flip(assertSourceOrigin("github", "mcp")); + + expect(failure).toBeInstanceOf(SourceOriginError); + expect(failure.message).toBe("this app is managed by its GitHub repo"); + expect(failure.existingOrigin).toBe("github"); + expect(failure.requestedOrigin).toBe("mcp"); + }), + ); + + it("round-trips custom tool files through publish, resolveTools, and invokeTool", async () => { + let issueListArgs: unknown; const resolver = makeTestResolver( { - dealcloud: { - "deals.list": (args) => { - dealListArgs = args[0]; - return []; + github: { + "issues.listForRepo": (args) => { + issueListArgs = args[0]; + return [{ number: 7, title: "Renewal diligence" }]; }, }, - "microsoft-sharepoint": { - "search.query": () => [], + gmail: { + "messages.search": () => ({ messages: [] }), }, }, [ { - address: "tools.dealcloud.user.crm-main", - integration: "dealcloud", - name: "crm-main", + address: "tools.github.user.github-main", + integration: "github", + name: "github-main", }, { - address: "tools.microsoft-sharepoint.user.sharepoint-main", - integration: "microsoft-sharepoint", - name: "sharepoint-main", + address: "tools.gmail.user.inbox-main", + integration: "gmail", + name: "inbox-main", }, ], ); @@ -114,6 +111,7 @@ describe("appsPlugin custom-tools contract", () => { const plugin = appsPlugin({ backings: host.backings }); const appIntegration = IntegrationSlug.make("rhys-tools"); const appConfig = { + origin: "github", kind: "github", repoUrl: "https://github.com/rhys/tools", repo: "rhys/tools", @@ -142,7 +140,7 @@ describe("appsPlugin custom-tools contract", () => { }, }; - await run(runtime.publish({ scope: "rhys", files: prototypeFileSet() })); + await run(runtime.publish({ scope: "rhys", files: dailyBriefFileSet() })); const resolved = await run( plugin.resolveTools!({ @@ -151,18 +149,18 @@ describe("appsPlugin custom-tools contract", () => { connection: { name: "main" }, } as never), ); - const syncTool = resolved.tools.find((tool) => String(tool.name) === "deal-pipeline-sync"); + const syncTool = resolved.tools.find((tool) => String(tool.name) === "issues-sync"); expect(syncTool).toBeTruthy(); const persistedInputSchema = syncTool!.inputSchema as { properties: Record; }; - expect(persistedInputSchema.properties.crm).toBeUndefined(); + expect(persistedInputSchema.properties.github).toBeUndefined(); const projected = await run( plugin.projectToolSchema!({ ctx, toolRow: { - name: "deal-pipeline-sync", + name: "issues-sync", integration: appIntegration, connection: "main", }, @@ -174,31 +172,38 @@ describe("appsPlugin custom-tools contract", () => { properties: Record; required?: string[]; }; - expect(inputSchema.properties.crm.enum).toEqual(["tools.dealcloud.user.crm-main"]); - expect(inputSchema.properties.crm.default).toBe("tools.dealcloud.user.crm-main"); - expect(inputSchema.properties.crm.description).toBe("Connection to use for crm (dealcloud)"); - expect(inputSchema.required ?? []).not.toContain("crm"); + expect(inputSchema.properties.github.enum).toEqual(["tools.github.user.github-main"]); + expect(inputSchema.properties.github.default).toBe("tools.github.user.github-main"); + expect(inputSchema.properties.github.description).toBe("Connection to use for github (github)"); + expect(inputSchema.required ?? []).not.toContain("github"); const output = await run( plugin.invokeTool!({ ctx, toolRow: { - name: "deal-pipeline-sync", + name: "issues-sync", integration: appIntegration, connection: "main", }, args: { - crm: "tools.dealcloud.user.crm-main", - updatedSince: "2026-01-01T00:00:00Z", + github: "tools.github.user.github-main", + repos: ["acme/tools"], + since: "2026-01-01T00:00:00Z", }, } as never), ); - expect(output).toEqual({ synced: 0 }); - expect(dealListArgs).toEqual({ - status: "active", - updatedSince: "2026-01-01T00:00:00Z", - pageSize: 200, + expect(output).toEqual({ + synced: 1, + repos: 1, + issues: [{ repo: "acme/tools", number: 7, title: "Renewal diligence" }], + }); + expect(issueListArgs).toEqual({ + owner: "acme", + repo: "tools", + state: "open", + since: "2026-01-01T00:00:00Z", + per_page: 100, }); await host.close(); }); diff --git a/packages/plugins/apps/src/plugin/apps-plugin.ts b/packages/plugins/apps/src/plugin/apps-plugin.ts index 403652c70..4108c478d 100644 --- a/packages/plugins/apps/src/plugin/apps-plugin.ts +++ b/packages/plugins/apps/src/plugin/apps-plugin.ts @@ -1,4 +1,4 @@ -import { Effect, Result } from "effect"; +import { Data, Effect, Result } from "effect"; import { AuthTemplateSlug, @@ -24,7 +24,6 @@ import { import type { IntegrationDecl, ToolDescriptor } from "../pipeline/descriptor"; import { PublishError } from "../pipeline/discover"; import { ArtifactStoreError } from "../seams/artifact-store"; -import { ScopeDbError } from "../seams/scope-db"; import { ToolSandboxError } from "../seams/tool-sandbox"; import { parseGitHubSourceUrl, @@ -45,6 +44,7 @@ export const APPS_PLUGIN_ID = "apps"; const APP_CONNECTION_NAME = connectionIdentifier("main"); interface AppsGitHubSourceConfig { + readonly origin: "github"; readonly kind: "github"; readonly repoUrl: string; readonly repo: string; @@ -59,6 +59,29 @@ type ResolverFactory = (input: { readonly tool: string; }) => ClientResolver; +export class SourceOriginError extends Data.TaggedError("SourceOriginError")<{ + readonly message: string; + readonly existingOrigin: string; + readonly requestedOrigin: string; +}> {} + +export const assertSourceOrigin = ( + existingOrigin: string, + requestedOrigin: string, +): Effect.Effect => + existingOrigin === requestedOrigin + ? Effect.void + : Effect.fail( + new SourceOriginError({ + existingOrigin, + requestedOrigin, + message: + existingOrigin === "github" + ? "this app is managed by its GitHub repo" + : `this app is managed by its ${existingOrigin} source`, + }), + ); + export type AppsPluginOptions = | { readonly backings: AppsBackings; @@ -171,21 +194,6 @@ const missingRuntime = (): AppsRuntime => ({ }), ), }, - scopeDb: { - forScope: () => - Effect.fail( - new ScopeDbError({ - message: missingBackingsMessage, - }), - ), - removeScope: () => - Effect.fail( - new ScopeDbError({ - message: missingBackingsMessage, - }), - ), - close: () => Effect.void, - }, sandbox: { collect: () => Effect.fail( @@ -221,7 +229,7 @@ const configBaseUrl = (config: unknown): string | undefined => : undefined; const decodeSourceConfig = (config: unknown): AppsGitHubSourceConfig | null => { - if (!isRecord(config) || config.kind !== "github") return null; + if (!isRecord(config) || config.origin !== "github" || config.kind !== "github") return null; const repoUrl = asString(config.repoUrl); const repo = asString(config.repo); const scope = asString(config.scope); @@ -235,6 +243,7 @@ const decodeSourceConfig = (config: unknown): AppsGitHubSourceConfig | null => { } : undefined; return { + origin: "github", kind: "github", repoUrl, repo, @@ -440,10 +449,18 @@ export const appsPlugin = definePlugin((options?: AppsPluginOptions) => { Effect.gen(function* () { const slug = IntegrationSlug.make(slugValue); const record = yield* ctx.core.integrations.get(slug).pipe(Effect.orElseSucceed(() => null)); - const config = record?.kind === APPS_PLUGIN_ID ? decodeSourceConfig(record.config) : null; - if (!record || !config) { + if (!record) { return syncFailure(`Custom tools source "${slugValue}" does not exist.`); } + if (record.kind === APPS_PLUGIN_ID && isRecord(record.config)) { + const origin = asString(record.config.origin); + if (origin) { + const originCheck = yield* assertSourceOrigin(origin, "github").pipe(Effect.result); + if (Result.isFailure(originCheck)) return syncFailure(originCheck.failure.message); + } + } + const config = record.kind === APPS_PLUGIN_ID ? decodeSourceConfig(record.config) : null; + if (!config) return syncFailure(`Custom tools source "${slugValue}" does not exist.`); const github = yield* ctx.core.integrations .get(IntegrationSlug.make("github")) .pipe(Effect.orElseSucceed(() => null)); @@ -542,6 +559,7 @@ export const appsPlugin = definePlugin((options?: AppsPluginOptions) => { const descriptor = yield* runtime.getDescriptor(tenant, scope); const source = descriptor?.source?.kind === "github" ? descriptor.source : null; const config: AppsGitHubSourceConfig = { + origin: "github", kind: "github", repoUrl: source?.url ?? input.url.trim(), repo: source?.repo ?? parsed.value.repo, diff --git a/packages/plugins/apps/src/plugin/backings.ts b/packages/plugins/apps/src/plugin/backings.ts index 84bb87650..5f2c0268e 100644 --- a/packages/plugins/apps/src/plugin/backings.ts +++ b/packages/plugins/apps/src/plugin/backings.ts @@ -1,5 +1,4 @@ import type { ArtifactStore } from "../seams/artifact-store"; -import type { ScopeDb } from "../seams/scope-db"; import type { ToolSandbox } from "../seams/tool-sandbox"; import { makeAppsRuntime, type AppsRuntime } from "./runtime"; import type { AppsStore } from "./store"; @@ -7,7 +6,6 @@ import type { ClientResolver } from "./bindings"; export interface AppsBackings { readonly artifactStore: ArtifactStore; - readonly scopeDb: ScopeDb; readonly sandbox: ToolSandbox; readonly store: AppsStore; readonly defaultTenant?: string; @@ -20,7 +18,6 @@ export const makeAppsRuntimeFromBackings = ( ): AppsRuntime => makeAppsRuntime({ artifactStore: backings.artifactStore, - scopeDb: backings.scopeDb, sandbox: backings.sandbox, store: backings.store, resolver: backings.resolver ?? fallbackResolver, diff --git a/packages/plugins/apps/src/plugin/bindings.strict.test.ts b/packages/plugins/apps/src/plugin/bindings.strict.test.ts index 8f2691df1..d27b1f5c8 100644 --- a/packages/plugins/apps/src/plugin/bindings.strict.test.ts +++ b/packages/plugins/apps/src/plugin/bindings.strict.test.ts @@ -3,7 +3,6 @@ import { Effect, Exit } from "effect"; import { buildBridge, resolveIntegrationBindings, type ClientResolver } from "./bindings"; import type { IntegrationDecl } from "../pipeline/descriptor"; -import type { ScopeDbHandle } from "../seams/scope-db"; // --------------------------------------------------------------------------- // Strict HandleBridge dispatch (Fix 4, grafted from build A). The bridge is the @@ -11,13 +10,6 @@ import type { ScopeDbHandle } from "../seams/scope-db"; // malformed, reserved, undeclared, or out-of-range rather than resolve it. // --------------------------------------------------------------------------- -const okDb: ScopeDbHandle = { - sql: () => Effect.succeed([]), - exec: () => Effect.succeed([]), - tableVersion: () => Effect.succeed(0), - versions: () => Effect.succeed(new Map()), -}; - const okResolver: ClientResolver = { listConnections: ({ integration }) => Effect.succeed([ @@ -55,7 +47,7 @@ describe("HandleBridge strict dispatch", () => { const bindings = { gh: "tools.github.user.main", }; - const bridge = buildBridge({ declared, bindings, db: okDb, resolver: okResolver }); + const bridge = buildBridge({ declared, bindings, resolver: okResolver }); it("rejects an empty method path", async () => { expect(await fails(bridge.call({ root: "gh", path: [], args: [] }))).toBe(true); @@ -75,10 +67,6 @@ describe("HandleBridge strict dispatch", () => { ); }); - it("rejects an unsupported db call", async () => { - expect(await fails(bridge.call({ root: "db", path: ["drop"], args: [] }))).toBe(true); - }); - it("routes a valid single-connection call through the resolver", async () => { const out = await Effect.runPromise( bridge.call({ root: "gh", path: ["repos", "list"], args: [{}] }), diff --git a/packages/plugins/apps/src/plugin/bindings.ts b/packages/plugins/apps/src/plugin/bindings.ts index c27cfa68a..c9fd7a17f 100644 --- a/packages/plugins/apps/src/plugin/bindings.ts +++ b/packages/plugins/apps/src/plugin/bindings.ts @@ -4,7 +4,6 @@ import type { InvokeOptions } from "@executor-js/sdk"; import type { HandleBridge, HandleRootSpec } from "../seams/tool-sandbox"; import { ToolSandboxError } from "../seams/tool-sandbox"; -import type { ScopeDbHandle } from "../seams/scope-db"; import type { IntegrationDecl } from "../pipeline/descriptor"; // --------------------------------------------------------------------------- @@ -16,7 +15,7 @@ import type { IntegrationDecl } from "../pipeline/descriptor"; // has exactly one connection for a role's integration, that connection is used // as the default. Missing, unknown, or wrong-integration choices are typed // BindingError failures. The handler receives clients under the declared role -// names and `db`. +// names. // --------------------------------------------------------------------------- export class BindingError extends Data.TaggedError("BindingError")<{ @@ -60,8 +59,6 @@ export interface BindingContext { readonly declared: Readonly>; /** Resolved role -> connection address choices for this invocation. */ readonly bindings: RoleBindings; - /** The invoking scope's app database (bound to the `db` root). */ - readonly db: ScopeDbHandle; /** Routes a bound method call to the real integration. */ readonly resolver: ClientResolver; /** Caller-supplied invoke options to preserve approval/elicitation context. */ @@ -166,12 +163,12 @@ export const resolveIntegrationBindings = ( return { input, bindings }; }); -/** Compute the sandbox handle roots: `db` plus one single root per declared - * integration role. */ +/** Compute the sandbox handle roots: one single root per declared integration + * role. */ export const rootsFor = ( declared: Readonly>, ): Readonly> => { - const roots: Record = { db: { kind: "single" } }; + const roots: Record = {}; for (const role of Object.keys(declared)) { roots[role] = { kind: "single" }; } @@ -192,12 +189,11 @@ const invokeErr = (message: string): ToolSandboxError => new ToolSandboxError({ kind: "invoke", message }); /** - * Build the HandleBridge the sandbox calls out through. `db` routes to the - * scope database; a declared role routes to its resolved connection through the - * `ClientResolver`. The dispatch is STRICT: an empty/malformed path, a reserved - * root, an undeclared root, or an indexed root is an error, never a silent - * success. A `.account` read on a client returns bound-connection metadata - * without a round-trip. + * Build the HandleBridge the sandbox calls out through. A declared role routes + * to its resolved connection through the `ClientResolver`. The dispatch is + * STRICT: an empty/malformed path, a reserved root, an undeclared root, or an + * indexed root is an error, never a silent success. A `.account` read on a + * client returns bound-connection metadata without a round-trip. */ export const buildBridge = (context: BindingContext): HandleBridge => ({ call: ({ root, path, args }) => { @@ -211,23 +207,6 @@ export const buildBridge = (context: BindingContext): HandleBridge => ({ return Effect.fail(invokeErr(`reserved handle root is not callable: ${root}`)); } - if (root === "db") { - if (path.length === 1 && path[0] === "sql") { - const [strings, ...values] = args as [TemplateStringsArray, ...unknown[]]; - return context.db.sql(strings, ...values).pipe( - Effect.mapError( - (cause) => - new ToolSandboxError({ - kind: "invoke", - message: "scope database call failed", - cause, - }), - ), - ); - } - return Effect.fail(invokeErr(`unsupported db call: ${path.join(".")}`)); - } - const { role, index } = parseRoot(root); if (index !== undefined) { return Effect.fail( diff --git a/packages/plugins/apps/src/plugin/runtime.test.ts b/packages/plugins/apps/src/plugin/runtime.test.ts index 0aacae294..d004dd0fd 100644 --- a/packages/plugins/apps/src/plugin/runtime.test.ts +++ b/packages/plugins/apps/src/plugin/runtime.test.ts @@ -7,7 +7,6 @@ import { Effect, Exit } from "effect"; import { makeSelfHostAppsRuntime } from "./self-host-runtime"; import { makeInMemoryAppsStore, makeTestResolver, dailyBriefFileSet } from "../testing"; -import { scopeAddress } from "../seams/scope-address"; const run = (effect: Effect.Effect): Promise => Effect.runPromise(effect); @@ -36,7 +35,7 @@ const githubHandlers = { }; describe("AppsRuntime end-to-end (publish -> invoke)", () => { - it("publishes daily-brief and invokes the tool into the scope db", async () => { + it("publishes daily-brief and invokes the tool through declared integrations", async () => { const store = makeInMemoryAppsStore(); const resolver = makeTestResolver(githubHandlers, [ { address: "tools.github.user.rhys-github", integration: "github", name: "rhys-github" }, @@ -61,12 +60,19 @@ describe("AppsRuntime end-to-end (publish -> invoke)", () => { tool: "issues-sync", args: { github: "tools.github.user.rhys-github" }, }), - )) as { synced: number; repos: number }; - expect(syncResult).toEqual({ synced: 2, repos: 1 }); - - const db = await run(host.scopeDb.forScope(scopeAddress("org", "rhys"))); - const rows = await run(db.exec<{ n: number }>("SELECT COUNT(*) AS n FROM issues")); - expect(Number(rows[0].n)).toBe(2); + )) as { + synced: number; + repos: number; + issues: readonly { repo: string; number: number; title: string }[]; + }; + expect(syncResult).toEqual({ + synced: 2, + repos: 1, + issues: [ + { repo: "acme/app", number: 1, title: "Fresh bug" }, + { repo: "acme/app", number: 2, title: "Old bug" }, + ], + }); await host.close(); }); diff --git a/packages/plugins/apps/src/plugin/runtime.ts b/packages/plugins/apps/src/plugin/runtime.ts index 8732df754..8f5ec6d9e 100644 --- a/packages/plugins/apps/src/plugin/runtime.ts +++ b/packages/plugins/apps/src/plugin/runtime.ts @@ -2,7 +2,6 @@ import { Effect, Predicate } from "effect"; import type { InvokeOptions } from "@executor-js/sdk"; import type { ArtifactStore } from "../seams/artifact-store"; -import type { ScopeDb } from "../seams/scope-db"; import { InputValidationError, OutputValidationError, @@ -33,7 +32,6 @@ import type { GitHubSkippedArtifact } from "../source/github-source"; export interface AppsRuntimeDeps { readonly artifactStore: ArtifactStore; - readonly scopeDb: ScopeDb; readonly sandbox: ToolSandbox; readonly store: AppsStore; /** Routes a bound integration method call to the real API (policy/audit). */ @@ -289,20 +287,9 @@ export const makeAppsRuntime = (deps: AppsRuntimeDeps): AppsRuntime => { ); const roots = rootsFor(toolDesc.integrations); const code = yield* bundleFor(descriptor, toolDesc.sourcePath); - const db = yield* deps.scopeDb.forScope(scopeAddress(descriptor.tenant, scope)).pipe( - Effect.mapError( - (c) => - new PublishError({ - message: c.message, - stage: "project", - diagnostics: [], - }), - ), - ); const bridge = buildBridge({ declared: toolDesc.integrations, bindings: resolved.bindings, - db, resolver: activeResolver, invokeOptions, }); @@ -434,9 +421,6 @@ export const makeAppsRuntime = (deps: AppsRuntimeDeps): AppsRuntime => { yield* deps.artifactStore .removeScope(address) .pipe(Effect.mapError(toPublishError("remove artifact scope failed"))); - yield* deps.scopeDb - .removeScope(address) - .pipe(Effect.mapError(toPublishError("remove scope db failed"))); }); }, diff --git a/packages/plugins/apps/src/plugin/self-host-runtime.ts b/packages/plugins/apps/src/plugin/self-host-runtime.ts index a7f897116..d2276ff4b 100644 --- a/packages/plugins/apps/src/plugin/self-host-runtime.ts +++ b/packages/plugins/apps/src/plugin/self-host-runtime.ts @@ -1,18 +1,14 @@ import { join } from "node:path"; -import { Effect } from "effect"; - import { makeGitArtifactStore } from "../backing/git-artifact-store"; -import { makeLibsqlScopeDb } from "../backing/libsql-scope-db"; import { makeQuickjsToolSandbox } from "../backing/quickjs-tool-sandbox"; -import type { ScopeDb } from "../seams/scope-db"; import { type AppsRuntime } from "./runtime"; import type { AppsStore } from "./store"; import type { ClientResolver } from "./bindings"; import { makeAppsRuntimeFromBackings, type AppsBackings } from "./backings"; export interface SelfHostAppsRuntimeOptions { - /** Data dir root; `/artifacts` and `/scope-db`. */ + /** Data dir root; `/artifacts`. */ readonly dataDir: string; /** Default tenant for direct runtime calls; request paths pass tenant explicitly. */ readonly tenant?: string; @@ -26,7 +22,6 @@ export interface SelfHostAppsRuntimeOptions { export interface SelfHostAppsRuntime { readonly runtime: AppsRuntime; readonly backings: AppsBackings; - readonly scopeDb: ScopeDb; readonly close: () => Promise; } @@ -37,13 +32,9 @@ export const makeSelfHostAppsRuntime = ( const artifactStore = makeGitArtifactStore({ root: inMem ? options.dataDir : join(options.dataDir, "artifacts"), }); - const scopeDb = makeLibsqlScopeDb({ - root: inMem ? ":memory:" : join(options.dataDir, "scope-db"), - }); const sandbox = makeQuickjsToolSandbox(); const backings: AppsBackings = { artifactStore, - scopeDb, sandbox, store: options.store, resolver: options.resolver, @@ -54,9 +45,6 @@ export const makeSelfHostAppsRuntime = ( return { runtime, backings, - scopeDb, - close: async () => { - await Effect.runPromise(scopeDb.close().pipe(Effect.orElseSucceed(() => undefined))); - }, + close: async () => {}, }; }; diff --git a/packages/plugins/apps/src/plugin/tenant-isolation.test.ts b/packages/plugins/apps/src/plugin/tenant-isolation.test.ts index 0cb43adc0..93befbe23 100644 --- a/packages/plugins/apps/src/plugin/tenant-isolation.test.ts +++ b/packages/plugins/apps/src/plugin/tenant-isolation.test.ts @@ -6,7 +6,6 @@ import { describe, expect, it } from "@effect/vitest"; import { Effect } from "effect"; import { makeSqliteAppsStore } from "../backing/sqlite-apps-store"; -import { scopeAddress } from "../seams/scope-address"; import { syncGitHubSource } from "../source/github-source"; import { makeTestResolver } from "../testing"; import { makeSelfHostAppsRuntime } from "./self-host-runtime"; @@ -18,11 +17,9 @@ import { z } from "zod"; export default defineTool({ description: "${name}", input: z.object({}), - output: z.object({ ok: z.boolean() }), - async handler(_input, { db }) { - await db.sql\`CREATE TABLE IF NOT EXISTS markers (name TEXT NOT NULL)\`; - await db.sql\`INSERT INTO markers (name) VALUES (${JSON.stringify(name)})\`; - return { ok: true }; + output: z.object({ ok: z.boolean(), name: z.string() }), + async handler() { + return { ok: true, name: ${JSON.stringify(name)} }; }, });`; @@ -119,22 +116,6 @@ describe("tenant scope isolation", () => { expect(descriptorA?.tools.map((tool) => tool.name)).toEqual(["alpha"]); expect(descriptorB?.tools.map((tool) => tool.name)).toEqual(["beta"]); - const dbA = await run(hostA.scopeDb.forScope(scopeAddress(tenantA, scope))); - const dbB = await run(hostB.scopeDb.forScope(scopeAddress(tenantB, scope))); - await run(dbA.exec("CREATE TABLE marker (value TEXT)")); - await run(dbA.exec("INSERT INTO marker (value) VALUES ('a')")); - await run(dbB.exec("CREATE TABLE marker (value TEXT)")); - await run(dbB.exec("INSERT INTO marker (value) VALUES ('b')")); - expect( - (await run(dbA.exec<{ value: string }>("SELECT value FROM marker"))).map((r) => r.value), - ).toEqual(["a"]); - expect( - (await run(dbB.exec<{ value: string }>("SELECT value FROM marker"))).map((r) => r.value), - ).toEqual(["b"]); - expect( - readdirSync(join(dataDir, "scope-db")).filter((name) => name.endsWith(".db")).length, - ).toBe(2); - await hostA.close(); await hostB.close(); }); diff --git a/packages/plugins/apps/src/seams/index.ts b/packages/plugins/apps/src/seams/index.ts index 024108bd8..02df3b451 100644 --- a/packages/plugins/apps/src/seams/index.ts +++ b/packages/plugins/apps/src/seams/index.ts @@ -2,5 +2,4 @@ // `src/backing/`; each seam has a conformance suite (`*.conformance.ts`) that a // backing must pass, keeping a future Cloudflare backing honest. export * from "./artifact-store"; -export * from "./scope-db"; export * from "./tool-sandbox"; diff --git a/packages/plugins/apps/src/seams/scope-db.conformance.ts b/packages/plugins/apps/src/seams/scope-db.conformance.ts deleted file mode 100644 index ee0df0502..000000000 --- a/packages/plugins/apps/src/seams/scope-db.conformance.ts +++ /dev/null @@ -1,70 +0,0 @@ -import { describe, expect, it } from "@effect/vitest"; -import { Effect } from "effect"; - -import type { ScopeDb } from "./scope-db"; -import { scopeAddress } from "./scope-address"; - -// --------------------------------------------------------------------------- -// ScopeDb conformance suite. Runs against the interface. Covers: scope -// isolation (a write in scope A is invisible in scope B), per-table version -// bumps on write, and tagged-template parameters. -// --------------------------------------------------------------------------- - -const run = (effect: Effect.Effect): Promise => Effect.runPromise(effect); - -export const scopeDbConformance = ( - name: string, - makeDb: () => Promise | ScopeDb, -): void => { - describe(`ScopeDb conformance: ${name}`, () => { - it("isolates data between scopes", async () => { - const db = await makeDb(); - const a = await run(db.forScope(scopeAddress("org", "scope-a"))); - const b = await run(db.forScope(scopeAddress("org", "scope-b"))); - await run(a.exec("CREATE TABLE items (id INTEGER PRIMARY KEY, name TEXT)")); - await run(a.exec("INSERT INTO items (name) VALUES ('a-only')")); - await run(b.exec("CREATE TABLE items (id INTEGER PRIMARY KEY, name TEXT)")); - - const aRows = await run(a.exec<{ name: string }>("SELECT name FROM items")); - const bRows = await run(b.exec<{ name: string }>("SELECT name FROM items")); - expect(aRows.map((r) => r.name)).toEqual(["a-only"]); - expect(bRows).toEqual([]); - await run(db.close()); - }); - - it("bumps a table's version on each write", async () => { - const db = await makeDb(); - const s = await run(db.forScope(scopeAddress("org", "ver"))); - await run(s.exec("CREATE TABLE t (id INTEGER PRIMARY KEY)")); - const afterCreate = await run(s.tableVersion("t")); - expect(afterCreate).toBeGreaterThanOrEqual(1); - - await run(s.exec("INSERT INTO t (id) VALUES (1)")); - const afterInsert = await run(s.tableVersion("t")); - expect(afterInsert).toBe(afterCreate + 1); - - await run(s.exec("UPDATE t SET id = 2 WHERE id = 1")); - const afterUpdate = await run(s.tableVersion("t")); - expect(afterUpdate).toBe(afterInsert + 1); - - // Reads do not bump. - await run(s.exec("SELECT * FROM t")); - expect(await run(s.tableVersion("t"))).toBe(afterUpdate); - await run(db.close()); - }); - - it("supports the author-facing tagged-template sql with parameters", async () => { - const db = await makeDb(); - const s = await run(db.forScope(scopeAddress("org", "tpl"))); - await run(s.exec("CREATE TABLE issues (repo TEXT, title TEXT)")); - const repo = "acme/app"; - const title = "Bug"; - await run(s.sql`INSERT INTO issues (repo, title) VALUES (${repo}, ${title})`); - const rows = await run( - s.sql<{ repo: string; title: string }>`SELECT * FROM issues WHERE repo = ${repo}`, - ); - expect(rows).toEqual([{ repo: "acme/app", title: "Bug" }]); - await run(db.close()); - }); - }); -}; diff --git a/packages/plugins/apps/src/seams/scope-db.ts b/packages/plugins/apps/src/seams/scope-db.ts deleted file mode 100644 index 22169fc49..000000000 --- a/packages/plugins/apps/src/seams/scope-db.ts +++ /dev/null @@ -1,44 +0,0 @@ -import type { Effect } from "effect"; -import { Data } from "effect"; -import type { ScopeAddress } from "./scope-address"; - -// --------------------------------------------------------------------------- -// ScopeDb — the per-scope app database (the shared primitive). -// -// One SQL database per scope: tools read and write it. The self-hosted backing -// is one libSQL (SQLite) file per scope; the cloud backing (future) is a -// Durable Object facet. The seam carries a template-tag `sql` (the author-facing -// shape) plus per-table version counters. -// -// Scope isolation is structural: `forScope(a)` and `forScope(b)` are distinct -// databases; there is no cross-scope query path. -// --------------------------------------------------------------------------- - -export class ScopeDbError extends Data.TaggedError("ScopeDbError")<{ - readonly message: string; - readonly cause?: unknown; -}> {} - -/** The author-facing db handle (matches `executor:app`'s `ScopeDb`): a tagged - * template returning rows. */ -export interface ScopeDbHandle { - readonly sql: >( - strings: TemplateStringsArray, - ...values: unknown[] - ) => Effect.Effect; - /** Run a raw statement (used by the runtime for probes/migrations). */ - readonly exec: >( - sql: string, - params?: readonly unknown[], - ) => Effect.Effect; - /** Current version counter for a table (0 if never written). */ - readonly tableVersion: (table: string) => Effect.Effect; - /** Snapshot of every tracked table's version. */ - readonly versions: () => Effect.Effect, ScopeDbError>; -} - -export interface ScopeDb { - readonly forScope: (address: ScopeAddress) => Effect.Effect; - readonly removeScope: (address: ScopeAddress) => Effect.Effect; - readonly close: () => Effect.Effect; -} diff --git a/packages/plugins/apps/src/seams/tool-sandbox.conformance.ts b/packages/plugins/apps/src/seams/tool-sandbox.conformance.ts index bb0e6ab14..2e647ead6 100644 --- a/packages/plugins/apps/src/seams/tool-sandbox.conformance.ts +++ b/packages/plugins/apps/src/seams/tool-sandbox.conformance.ts @@ -106,60 +106,5 @@ export default defineTool({ expect(seen[0].path).toEqual(["messages", "search"]); expect(seen[0].args[0]).toMatchObject({ q: "invoice" }); }); - - // --- richer assertions grafted from build A --------------------------- - - it("round-trips a db.sql write with its parameters intact", async () => { - const sandbox = makeSandbox(); - // A tool with a github client AND a db.sql write, asserting BOTH the - // connection call and the parameterized db statement cross the bridge with - // their args intact (the db.sql param round-trip A's suite covers). - const src = `import { defineTool, integration } from "executor:app"; -import { z } from "zod"; -export default defineTool({ - description: "bridge", - integrations: { gh: integration("github") }, - input: z.object({}), - async handler(_i, { gh, db }) { - const repos = await gh.repos.listForAuthenticatedUser({ per_page: 5 }); - await db.sql\`INSERT INTO log (n) VALUES (\${repos.length})\`; - return { count: repos.length }; - }, -});`; - const b = await bundle("tools/bridge.ts", src); - const calls: { root: string; path: readonly string[]; args: readonly unknown[] }[] = []; - const bridge: HandleBridge = { - call: ({ root, path, args }) => - Effect.sync(() => { - calls.push({ root, path, args }); - if (root === "gh" && path.join(".") === "repos.listForAuthenticatedUser") { - return [{ full_name: "a/b" }, { full_name: "c/d" }]; - } - return []; - }), - }; - const result = await run( - sandbox.invoke( - b, - { - artifact: "bridge", - kind: "tool", - input: {}, - roots: { gh: { kind: "single" }, db: { kind: "single" } }, - }, - bridge, - ), - ); - expect(result.output).toEqual({ count: 2 }); - // The github call crossed the bridge. - expect(calls.some((c) => c.root === "gh")).toBe(true); - // The db.sql write crossed as ["sql"] with the template strings + the - // interpolated parameter (2) intact. - const dbCall = calls.find((c) => c.root === "db"); - expect(dbCall?.path).toEqual(["sql"]); - const [strings, ...params] = dbCall!.args as [readonly string[], ...unknown[]]; - expect(strings.join("?")).toContain("INSERT INTO log"); - expect(params).toEqual([2]); - }); }); }; diff --git a/packages/plugins/apps/src/seams/tool-sandbox.ts b/packages/plugins/apps/src/seams/tool-sandbox.ts index c05ed9080..bc2f6c7da 100644 --- a/packages/plugins/apps/src/seams/tool-sandbox.ts +++ b/packages/plugins/apps/src/seams/tool-sandbox.ts @@ -49,7 +49,7 @@ export class OutputValidationError extends Data.TaggedError("OutputValidationErr /** * The serializable bridge the sandbox calls out through. `root` names an - * injected handle (a connection role, `db`, or one element of a fan-out set); + * injected handle (a connection role or one element of a fan-out set); * `path` is the method chain (`["events", "list"]`); `args` is the JSON call * arguments. The return value is JSON. This is the ONE way sandboxed code * reaches the host — nothing else is wired. @@ -62,9 +62,9 @@ export interface HandleBridge { }) => Effect.Effect; } -/** Which handle roots to inject. Each declared integration role and `db` are - * single roots. Everything the handler can see is enumerated here — - * undeclared roots are simply absent. */ +/** Which handle roots to inject. Each declared integration role is a single + * root. Everything the handler can see is enumerated here; undeclared roots + * are simply absent. */ export interface HandleRootSpec { readonly kind: "single"; } @@ -77,7 +77,7 @@ export interface InvokeRequest { /** JSON input passed to the handler. */ readonly input: unknown; /** The handle roots to inject, keyed by the name the handler destructures - * (`github`, `db`, `inboxes`). */ + * (`github`, `inboxes`). */ readonly roots: Readonly>; } diff --git a/packages/plugins/apps/src/testing/daily-brief.ts b/packages/plugins/apps/src/testing/daily-brief.ts index bda340065..3f7e5018f 100644 --- a/packages/plugins/apps/src/testing/daily-brief.ts +++ b/packages/plugins/apps/src/testing/daily-brief.ts @@ -11,7 +11,7 @@ import { defineTool, integration } from "executor:app"; export default defineTool({ description: - "Refresh the scope \\\`issues\\\` table from GitHub. Syncs open issues across the given repos (default: every repo the connection can see).", + "Summarize open GitHub issues across the given repos (default: every repo the connection can see).", integrations: { github: integration("github"), }, @@ -19,38 +19,25 @@ export default defineTool({ repos: z.array(z.string()).optional().describe("owner/repo entries; omit to sync all accessible repos"), since: z.string().optional().describe("ISO timestamp, only issues updated after this"), }), - output: z.object({ synced: z.number(), repos: z.number() }), + output: z.object({ synced: z.number(), repos: z.number(), issues: z.array(z.object({ repo: z.string(), number: z.number(), title: z.string() })) }), annotations: { readOnly: false, destructive: false }, - async handler({ repos, since }, { github, db }) { + async handler({ repos, since }, { github }) { const targets = repos ?? (await github.repos.listForAuthenticatedUser({ per_page: 100 })).map((r) => r.full_name); - await db.sql\` - CREATE TABLE IF NOT EXISTS issues ( - repo TEXT NOT NULL, number INTEGER NOT NULL, title TEXT NOT NULL, - labels TEXT NOT NULL DEFAULT '[]', assignee TEXT, updated_at TEXT NOT NULL, url TEXT NOT NULL, - PRIMARY KEY (repo, number) - )\`; - let synced = 0; + const collected = []; for (const target of targets) { const [owner, repo] = target.split("/"); const issues = await github.issues.listForRepo({ owner, repo, state: "open", since, per_page: 100 }); for (const issue of issues) { if (issue.pull_request) continue; - await db.sql\` - INSERT INTO issues (repo, number, title, labels, assignee, updated_at, url) - VALUES (\${target}, \${issue.number}, \${issue.title}, - \${JSON.stringify(issue.labels.map((l) => l.name))}, - \${issue.assignee?.login ?? null}, \${issue.updated_at}, \${issue.html_url}) - ON CONFLICT (repo, number) DO UPDATE SET - title = excluded.title, labels = excluded.labels, - assignee = excluded.assignee, updated_at = excluded.updated_at\`; + collected.push({ repo: target, number: issue.number, title: issue.title }); synced++; } } - return { synced, repos: targets.length }; + return { synced, repos: targets.length, issues: collected }; }, }); `; From 21aabdc2305ac09a92edd4d6fe99af403fe116ee Mon Sep 17 00:00:00 2001 From: Rhys Sullivan <39114868+RhysSullivan@users.noreply.github.com> Date: Tue, 7 Jul 2026 01:33:53 -0700 Subject: [PATCH 19/21] Trim custom tools CAS test --- .../backing/git-artifact-store.cas.test.ts | 69 ------------------- .../src/backing/git-artifact-store.test.ts | 27 +++++++- .../apps/src/backing/git-artifact-store.ts | 31 ++++++--- 3 files changed, 46 insertions(+), 81 deletions(-) delete mode 100644 packages/plugins/apps/src/backing/git-artifact-store.cas.test.ts diff --git a/packages/plugins/apps/src/backing/git-artifact-store.cas.test.ts b/packages/plugins/apps/src/backing/git-artifact-store.cas.test.ts deleted file mode 100644 index 20e28f8ba..000000000 --- a/packages/plugins/apps/src/backing/git-artifact-store.cas.test.ts +++ /dev/null @@ -1,69 +0,0 @@ -import { mkdtempSync } from "node:fs"; -import { tmpdir } from "node:os"; -import { join } from "node:path"; - -import { describe, expect, it } from "@effect/vitest"; -import { Effect } from "effect"; - -import { makeGitArtifactStore } from "./git-artifact-store"; -import { scopeAddress } from "../seams/scope-address"; -import type { ArtifactStoreError } from "../seams/artifact-store"; - -const run = (effect: Effect.Effect): Promise => Effect.runPromise(effect); - -// --------------------------------------------------------------------------- -// Finding 6 regression (git compare-and-swap): two commits built from the SAME -// parent cannot both advance the branch. The ref update is a CAS against the -// expected old value, so the second raced writer gets a typed conflict instead -// of silently clobbering the first. This is deterministic (no timing race): we -// force both to share a parent by committing them against a store whose head we -// pin. -// --------------------------------------------------------------------------- - -describe("git ArtifactStore ref CAS (Fix 6)", () => { - it("two concurrent commits from one parent: one wins, the other typed-conflicts", async () => { - const root = mkdtempSync(join(tmpdir(), "apps-cas-")); - const storeA = makeGitArtifactStore({ root }); - const scopeA = await run(storeA.forScope(scopeAddress("org", "s"))); - - // Seed a parent so racers commit ON TOP of the same head. - await run(scopeA.commit(new Map([["tools/base.ts", "// base"]]), "base")); - - // Many SEPARATE store instances over the SAME repo dir. The repo already - // exists (seeded above), so opening each scope is a re-`init --bare` (a no-op - // on an existing repo). Open them SEQUENTIALLY so the idempotent init doesn't - // itself race, then race only the COMMITS (the ref-CAS is what's under test). - const N = 12; - const openedScopes: { commit: typeof scopeA.commit; log: typeof scopeA.log }[] = []; - for (let i = 0; i < N; i++) { - const store = makeGitArtifactStore({ root }); - // eslint-disable-next-line no-await-in-loop - openedScopes.push(await run(store.forScope(scopeAddress("org", "s")))); - } - const scopeB = openedScopes[0]; - - // Fire many concurrent commits, ALL built against the seeded base parent. - const results = await Promise.allSettled( - openedScopes.map((sc, i) => - run(sc.commit(new Map([[`tools/c${i}.ts`, `// ${i}`]]), `c${i}`)), - ), - ); - - const fulfilled = results.filter((r) => r.status === "fulfilled").length; - const rejected = results.filter((r) => r.status === "rejected"); - - // Any failure MUST be a typed conflict (never a silent clobber). - for (const r of rejected) { - const err = (r as PromiseRejectedResult).reason as ArtifactStoreError; - expect(String(JSON.stringify(err))).toMatch(/conflict/i); - } - - // No lost commits: the git log length equals base(1) + every commit that - // reported success. Without the CAS, a raced writer clobbers the ref and an - // earlier successful commit vanishes from history, making the log SHORTER - // than base + fulfilled. The CAS guarantees every reported success is a real, - // reachable, non-clobbering advance. - const logEntries = await run(scopeB.log(1000)); - expect(logEntries.length).toBe(1 + fulfilled); - }, 60_000); -}); diff --git a/packages/plugins/apps/src/backing/git-artifact-store.test.ts b/packages/plugins/apps/src/backing/git-artifact-store.test.ts index e1a434fa4..1b0320891 100644 --- a/packages/plugins/apps/src/backing/git-artifact-store.test.ts +++ b/packages/plugins/apps/src/backing/git-artifact-store.test.ts @@ -2,9 +2,34 @@ import { mkdtempSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; +import { describe, expect, it } from "@effect/vitest"; +import { Effect, Exit } from "effect"; + import { artifactStoreConformance } from "../seams/artifact-store.conformance"; -import { makeGitArtifactStore } from "./git-artifact-store"; +import { scopeAddress, scopeAddressStorageKey } from "../seams/scope-address"; +import { compareAndSwapSnapshotRef, makeGitArtifactStore } from "./git-artifact-store"; artifactStoreConformance("git", () => makeGitArtifactStore({ root: mkdtempSync(join(tmpdir(), "apps-artifacts-")) }), ); + +const run = (effect: Effect.Effect): Promise => Effect.runPromise(effect); + +describe("git ArtifactStore CAS", () => { + it("returns a typed conflict when the expected head is stale", async () => { + const root = mkdtempSync(join(tmpdir(), "apps-artifacts-cas-")); + const address = scopeAddress("org", "s"); + const scope = await run(makeGitArtifactStore({ root }).forScope(address)); + const first = await run(scope.commit(new Map([["tools/a.ts", "a"]]), "first")); + await run(scope.commit(new Map([["tools/a.ts", "b"]]), "second")); + + const repoDir = join(root, `${scopeAddressStorageKey(address)}.git`); + const exit = await Effect.runPromiseExit( + compareAndSwapSnapshotRef(repoDir, String(first.id), String(first.id)), + ); + + expect(Exit.isFailure(exit)).toBe(true); + expect(JSON.stringify(exit)).toContain("publish conflict"); + expect(JSON.stringify(exit)).toContain('"conflict":true'); + }); +}); diff --git a/packages/plugins/apps/src/backing/git-artifact-store.ts b/packages/plugins/apps/src/backing/git-artifact-store.ts index a4af8518c..fbd5beb23 100644 --- a/packages/plugins/apps/src/backing/git-artifact-store.ts +++ b/packages/plugins/apps/src/backing/git-artifact-store.ts @@ -69,6 +69,25 @@ const run = ( } }); +export const compareAndSwapSnapshotRef = ( + repoDir: string, + commitHash: string, + expectedOld: string | null, +): Effect.Effect => + run(repoDir, ["update-ref", BRANCH, commitHash, expectedOld ?? EMPTY_OID]).pipe( + Effect.asVoid, + Effect.mapError( + (cause) => + new ArtifactStoreError({ + message: `publish conflict: ${BRANCH} moved concurrently (expected ${ + expectedOld ?? EMPTY_OID + }); retry from the new head`, + conflict: true, + cause, + }), + ), + ); + const makeScopeStore = (repoDir: string): ScopeArtifactStore => { // Environment forcing a deterministic author/committer so a snapshot's hash // depends only on its content + parent, not on wall-clock/identity drift. @@ -184,17 +203,7 @@ const makeScopeStore = (repoDir: string): ScopeArtifactStore => { // is no parent, so we assert the ref is absent (the empty-oid form). A CAS // failure surfaces as a typed conflict for the caller to retry from a // fresh parent. - const expectedOld = parent ?? EMPTY_OID; - yield* run(repoDir, ["update-ref", BRANCH, commitHash, expectedOld]).pipe( - Effect.mapError( - (cause) => - new ArtifactStoreError({ - message: `publish conflict: ${BRANCH} moved concurrently (expected ${expectedOld}); retry from the new head`, - conflict: true, - cause, - }), - ), - ); + yield* compareAndSwapSnapshotRef(repoDir, commitHash, parent); return yield* readMeta(commitHash); }), From 24629214ebebe0cfd6c3baf313b2190198e4d20f Mon Sep 17 00:00:00 2001 From: Rhys Sullivan <39114868+RhysSullivan@users.noreply.github.com> Date: Tue, 7 Jul 2026 01:34:20 -0700 Subject: [PATCH 20/21] Replace custom tools demo with e2e --- .../scripts/custom-tools-demo.ts | 442 ------- apps/host-selfhost/src/apps-wire.node.test.ts | 1020 ----------------- e2e/scenarios/custom-tools.test.ts | 613 ++++++++++ 3 files changed, 613 insertions(+), 1462 deletions(-) delete mode 100644 apps/host-selfhost/scripts/custom-tools-demo.ts delete mode 100644 apps/host-selfhost/src/apps-wire.node.test.ts create mode 100644 e2e/scenarios/custom-tools.test.ts diff --git a/apps/host-selfhost/scripts/custom-tools-demo.ts b/apps/host-selfhost/scripts/custom-tools-demo.ts deleted file mode 100644 index 18ea61999..000000000 --- a/apps/host-selfhost/scripts/custom-tools-demo.ts +++ /dev/null @@ -1,442 +0,0 @@ -import { mkdtempSync } from "node:fs"; -import { tmpdir } from "node:os"; -import { join } from "node:path"; - -const PORT = Number(process.env.PORT ?? "8791"); -const HOST = "127.0.0.1"; -const BASE_URL = `http://${HOST}:${PORT}`; -const ORIGIN = "http://custom-tools-demo.local"; -const DEMO_REPO = "RhysSullivan/executor-custom-tools-demo"; -const DEMO_APP = "demo-tools"; -const ADMIN_EMAIL = "admin@custom-tools-demo.local"; -const ADMIN_PASSWORD = "admin-pass-123456"; -const USER_EMAIL = "rhys@custom-tools-demo.local"; -const USER_PASSWORD = "password-12345678"; -const GITHUB_CONNECTION_NAME = "main"; - -process.env.EXECUTOR_DATA_DIR = mkdtempSync(join(tmpdir(), "eh-custom-tools-demo-")); -process.env.BETTER_AUTH_SECRET = "custom-tools-demo-better-auth-secret-0123456789"; -process.env.EXECUTOR_SECRET_KEY = "custom-tools-demo-secret-key-0123456789"; -process.env.EXECUTOR_BOOTSTRAP_ADMIN_EMAIL = ADMIN_EMAIL; -process.env.EXECUTOR_BOOTSTRAP_ADMIN_PASSWORD = ADMIN_PASSWORD; -process.env.EXECUTOR_BOOTSTRAP_ADMIN_NAME = "Demo Admin"; -process.env.EXECUTOR_ORG_NAME = "Custom Tools Demo"; -process.env.EXECUTOR_ORG_SLUG = "custom-tools-demo"; -process.env.EXECUTOR_WEB_BASE_URL = BASE_URL; -process.env.EXECUTOR_ALLOW_LOCAL_NETWORK = "true"; - -const log = (...args: unknown[]) => console.log("[custom-tools-demo]", ...args); - -const run = async (cmd: readonly string[]): Promise => { - const proc = Bun.spawn(cmd, { stdout: "pipe", stderr: "pipe" }); - const [stdout, stderr, code] = await Promise.all([ - new Response(proc.stdout).text(), - new Response(proc.stderr).text(), - proc.exited, - ]); - if (code !== 0) { - throw new Error(`${cmd.join(" ")} failed (${code}): ${stderr.trim() || stdout.trim()}`); - } - return stdout.trim(); -}; - -const githubToken = async (): Promise => { - const fromEnv = process.env.GITHUB_TOKEN?.trim(); - if (fromEnv) return fromEnv; - try { - return await run(["gh", "auth", "token"]); - } catch (error) { - throw new Error( - "Set GITHUB_TOKEN or run `gh auth login` before starting the custom-tools demo.", - { cause: error }, - ); - } -}; - -const bearerTemplate = { - slug: "bearer", - type: "apiKey", - label: "GitHub token", - headers: { Authorization: ["Bearer ", { type: "variable", name: "token" }] }, -}; - -const pathParam = (name: string, description: string) => ({ - name, - in: "path", - required: true, - description, - schema: { type: "string" }, -}); - -const queryParam = (name: string, schema: Record, description: string) => ({ - name, - in: "query", - required: false, - description, - schema, -}); - -const githubRestSpec = JSON.stringify({ - openapi: "3.0.3", - info: { - title: "GitHub REST", - version: "demo", - description: "Minimal GitHub REST surface for the custom-tools demo.", - }, - servers: [{ url: "https://api.github.com" }], - paths: { - "/repos/{owner}/{repo}": { - get: { - operationId: "repos/get", - "x-executor-toolPath": "repos.get", - tags: ["repos"], - summary: "Get repository metadata", - parameters: [pathParam("owner", "Repository owner"), pathParam("repo", "Repository name")], - responses: { - "200": { - description: "Repository", - content: { - "application/json": { - schema: { - type: "object", - properties: { - full_name: { type: "string" }, - description: { type: "string", nullable: true }, - html_url: { type: "string" }, - stargazers_count: { type: "number" }, - forks_count: { type: "number" }, - open_issues_count: { type: "number" }, - }, - }, - }, - }, - }, - }, - }, - }, - "/repos/{owner}/{repo}/issues": { - get: { - operationId: "issues/listForRepo", - "x-executor-toolPath": "issues.listForRepo", - tags: ["issues"], - summary: "List repository issues", - parameters: [ - pathParam("owner", "Repository owner"), - pathParam("repo", "Repository name"), - queryParam("state", { type: "string", enum: ["open", "closed", "all"] }, "Issue state"), - queryParam( - "sort", - { type: "string", enum: ["created", "updated", "comments"] }, - "Sort field", - ), - queryParam("direction", { type: "string", enum: ["asc", "desc"] }, "Sort direction"), - queryParam("since", { type: "string" }, "Only issues updated after this ISO timestamp"), - queryParam("per_page", { type: "number" }, "Page size"), - queryParam("page", { type: "number" }, "Page number"), - ], - responses: { - "200": { - description: "Issues", - content: { - "application/json": { - schema: { - type: "array", - items: { - type: "object", - properties: { - number: { type: "number" }, - title: { type: "string" }, - html_url: { type: "string" }, - updated_at: { type: "string" }, - state: { type: "string" }, - pull_request: { type: "object", nullable: true }, - labels: { - type: "array", - items: { - type: "object", - properties: { name: { type: "string" } }, - }, - }, - }, - }, - }, - }, - }, - }, - }, - }, - }, - "/repos/{owner}/{repo}/releases": { - get: { - operationId: "repos/listReleases", - "x-executor-toolPath": "repos.listReleases", - tags: ["repos"], - summary: "List repository releases", - parameters: [ - pathParam("owner", "Repository owner"), - pathParam("repo", "Repository name"), - queryParam("per_page", { type: "number" }, "Page size"), - queryParam("page", { type: "number" }, "Page number"), - ], - responses: { - "200": { - description: "Releases", - content: { - "application/json": { - schema: { - type: "array", - items: { - type: "object", - properties: { - tag_name: { type: "string" }, - name: { type: "string", nullable: true }, - html_url: { type: "string" }, - published_at: { type: "string", nullable: true }, - }, - }, - }, - }, - }, - }, - }, - }, - }, - }, -}); - -interface JsonResponse { - readonly status: number; - readonly body: unknown; -} - -const responseJson = async (response: Response): Promise => { - const text = await response.text(); - return { - status: response.status, - body: text.length > 0 ? JSON.parse(text) : null, - }; -}; - -const mustJson = async (response: Response, label: string): Promise => { - const parsed = await responseJson(response); - if (parsed.status < 200 || parsed.status >= 300) { - throw new Error(`${label} failed (${parsed.status}): ${JSON.stringify(parsed.body)}`); - } - return parsed.body as T; -}; - -const htmlEscape = (value: string): string => - value - .replaceAll("&", "&") - .replaceAll("<", "<") - .replaceAll(">", ">") - .replaceAll('"', """); - -const toolInvokeCode = (toolName: string, args: unknown): string => ` -const found = await tools.search({ namespace: ${JSON.stringify(DEMO_APP)}, query: ${JSON.stringify(toolName)}, limit: 20 }); -const item = found.items.find((candidate) => candidate.path.endsWith(${JSON.stringify(toolName)})); -if (!item) return { ok: false, missing: ${JSON.stringify(toolName)}, found }; -let fn = tools; -for (const segment of item.path.split(".")) fn = fn[segment]; -return await fn(${JSON.stringify(args)}); -`; - -const curlJson = (path: string, body: unknown): string => - `curl -sS -X POST ${BASE_URL}${path} -H 'content-type: application/json' -d '${JSON.stringify(body)}' | jq`; - -const buildBanner = (input: { - readonly repoUrl: string; - readonly connectionAddress: string; - readonly githubToken: string; - readonly token: string; - readonly consoleUi: string; -}): string => { - const sync = curlJson("/api/apps/sources/github/sync", { - name: DEMO_APP, - url: input.repoUrl, - token: input.githubToken, - }); - const list = `curl -sS '${BASE_URL}/api/tools?integration=${DEMO_APP}' | jq`; - const repoSummary = curlJson("/api/executions", { - code: toolInvokeCode("repo-summary", { - github: input.connectionAddress, - repo: DEMO_REPO, - }), - autoApprove: true, - }); - const staleIssues = curlJson("/api/executions", { - code: toolInvokeCode("stale-issues", { - github: input.connectionAddress, - repo: DEMO_REPO, - staleDays: 30, - }), - autoApprove: true, - }); - - return `Custom tools demo ready - -GitHub repo: - ${input.repoUrl} - -Local server: - ${BASE_URL} - -Console UI: - ${input.consoleUi} - -Auth: - The local wrapper injects the self-host bearer into forwarded requests. - Bearer for direct API debugging: - ${input.token} - -GitHub invocation connection: - ${input.connectionAddress} - -Custom tools app: - ${DEMO_APP} - -Sync the repo: - ${sync} - -List published custom tools: - ${list} - -Invoke repo-summary: - ${repoSummary} - -Invoke stale-issues: - ${staleIssues} - -MCP endpoint: - ${BASE_URL}/mcp -`; -}; - -const main = async () => { - const token = await githubToken(); - - const { makeSelfHostApiHandler } = await import("../src/app"); - const { mintInviteCode } = await import("../src/testing/mint-invite"); - const app = await makeSelfHostApiHandler(); - const handler = app.handler; - - const inviteCode = await mintInviteCode(handler, "admin"); - const signUp = await handler( - new Request(`${ORIGIN}/api/auth/sign-up/email`, { - method: "POST", - headers: { "content-type": "application/json" }, - body: JSON.stringify({ - email: USER_EMAIL, - password: USER_PASSWORD, - name: "Rhys", - inviteCode, - }), - }), - ); - const selfHostToken = signUp.headers.get("set-auth-token"); - if (!selfHostToken) throw new Error("sign-up produced no self-host bearer token"); - - const api = (path: string, init: RequestInit = {}): Promise => - handler( - new Request(`${ORIGIN}${path}`, { - ...init, - headers: { - authorization: `Bearer ${selfHostToken}`, - "content-type": "application/json", - ...(init.headers ?? {}), - }, - }), - ); - - const consoleCheck = await handler( - new Request(`${ORIGIN}/`, { headers: { authorization: `Bearer ${selfHostToken}` } }), - ); - const consoleUi = - consoleCheck.status === 404 - ? "not served by makeSelfHostApiHandler; use the curls below" - : `${BASE_URL}/`; - - const addSpec = await api("/api/openapi/specs", { - method: "POST", - body: JSON.stringify({ - spec: { kind: "blob", value: githubRestSpec }, - slug: "github", - name: "GitHub REST", - description: "Minimal GitHub REST surface for the custom-tools demo.", - baseUrl: "https://api.github.com", - authenticationTemplate: [bearerTemplate], - }), - }); - if (addSpec.status !== 200 && addSpec.status !== 409) { - const parsed = await responseJson(addSpec); - throw new Error( - `register github integration failed (${parsed.status}): ${JSON.stringify(parsed.body)}`, - ); - } - - const created = await mustJson<{ address: string }>( - await api("/api/connections", { - method: "POST", - body: JSON.stringify({ - owner: "user", - name: GITHUB_CONNECTION_NAME, - integration: "github", - template: "bearer", - value: token, - }), - }), - "create github connection", - ); - - const banner = buildBanner({ - repoUrl: `https://github.com/${DEMO_REPO}`, - connectionAddress: created.address, - githubToken: token, - token: selfHostToken, - consoleUi, - }); - - const server = Bun.serve({ - port: PORT, - hostname: HOST, - idleTimeout: 0, - async fetch(request) { - const url = new URL(request.url); - if (url.pathname === "/health") return new Response("ok"); - if (url.pathname === "/") { - return new Response( - `Custom tools demo
${htmlEscape(
-            banner,
-          )}
`, - { headers: { "content-type": "text/html; charset=utf-8" } }, - ); - } - const headers = new Headers(request.headers); - if (!headers.has("authorization")) headers.set("authorization", `Bearer ${selfHostToken}`); - return handler(new Request(request, { headers })); - }, - }); - - log(`serving at ${BASE_URL} (pid ${process.pid}, port ${server.port})`); - console.log(banner); - - const shutdown = async () => { - try { - server.stop(true); - } catch { - /* ignore */ - } - try { - await app.dispose(); - } catch { - /* ignore */ - } - process.exit(0); - }; - process.on("SIGINT", shutdown); - process.on("SIGTERM", shutdown); - - await new Promise(() => {}); -}; - -await main(); diff --git a/apps/host-selfhost/src/apps-wire.node.test.ts b/apps/host-selfhost/src/apps-wire.node.test.ts deleted file mode 100644 index 881016097..000000000 --- a/apps/host-selfhost/src/apps-wire.node.test.ts +++ /dev/null @@ -1,1020 +0,0 @@ -import { randomBytes } from "node:crypto"; -import { existsSync, mkdtempSync } from "node:fs"; -import { createServer, type IncomingMessage, type ServerResponse } from "node:http"; -import { createServer as createNetServer } from "node:net"; -import { tmpdir } from "node:os"; -import { join } from "node:path"; - -import { afterAll, beforeAll, expect, test } from "@effect/vitest"; -import { Effect, Option, Schema } from "effect"; -import { createEmulator, type Emulator } from "@executor-js/emulate"; -import { createClient } from "@libsql/client"; - -const dataDir = mkdtempSync(join(tmpdir(), "eh-apps-wire-")); -const dbPath = join(dataDir, "data.db"); -process.env.EXECUTOR_DATA_DIR = dataDir; -process.env.EXECUTOR_SECRET_KEY = "apps-wire-secret-key"; -process.env.EXECUTOR_ALLOW_LOCAL_NETWORK = "true"; - -const TEST_USER = "apps-user"; -const TEST_ORG = "apps-org"; -const OWNER = "syncer"; -const REPO = "custom-tools"; -const REPO_FULL_NAME = `${OWNER}/${REPO}`; -const GITHUB_CONNECTION = "tools.github.user.main"; -const APP_SLUG = "custom-tools-app"; -const SCHEMA_APP_SLUG = "schema-tools-app"; -const APPROVAL_APP_SLUG = "approval-tools-app"; - -interface TestHttpServer { - readonly baseUrl: string; - readonly close: () => Promise; -} - -interface SyncResult { - readonly status: "published" | "up-to-date" | "failed"; - readonly snapshotId?: string; - readonly upstreamSha?: string; - readonly tools: readonly string[]; - readonly skipped: readonly { readonly path: string; readonly reason: string }[]; - readonly errors?: readonly { readonly message?: string }[]; -} - -interface ToolRow { - readonly address: string; - readonly name: string; - readonly integration: string; -} - -interface ConnectionRow { - readonly address: string; -} - -interface IntegrationRow { - readonly slug: string; - readonly name: string; - readonly kind: string; - readonly canRemove: boolean; - readonly displayUrl?: string; -} - -interface ExecuteResponse { - readonly status: "completed" | "paused"; - readonly text: string; - readonly structured: { - readonly status?: string; - readonly result?: unknown; - readonly error?: unknown; - }; - readonly isError?: boolean; -} - -const bearerTemplate = { - slug: "bearer", - type: "apiKey", - label: "Bearer token", - headers: { Authorization: ["Bearer ", { type: "variable", name: "token" }] }, -}; - -const jsonHeaders = { - "content-type": "application/json", - "x-test-user": TEST_USER, - "x-test-org": TEST_ORG, -}; - -const decodeJsonText = Schema.decodeUnknownEffect(Schema.UnknownFromJsonString); - -const parseJsonText = async (text: string): Promise => { - if (text.length === 0) return null as T; - const value = await Effect.runPromise(decodeJsonText(text)); - return value as T; -}; - -const findFreePort = (): Promise => - Effect.runPromise( - Effect.callback((resume) => { - const probe = createNetServer(); - probe.once("error", () => resume(Effect.fail("failed to find a free port"))); - probe.listen(0, "127.0.0.1", () => { - const address = probe.address(); - const port = typeof address === "object" && address ? address.port : 0; - probe.close((error) => - error - ? resume(Effect.fail("failed to close free-port probe")) - : resume(Effect.succeed(port)), - ); - }); - }), - ); - -const readRequestBody = async (request: IncomingMessage): Promise => { - if (request.method === "GET" || request.method === "HEAD") return undefined; - const chunks: Buffer[] = []; - for await (const chunk of request) { - chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk)); - } - const body = Buffer.concat(chunks); - return body.length > 0 ? body : undefined; -}; - -const webRequestFromNode = async (request: IncomingMessage, port: number): Promise => { - const headers = new Headers(); - for (const [name, raw] of Object.entries(request.headers)) { - if (Array.isArray(raw)) { - for (const value of raw) headers.append(name, value); - } else if (raw !== undefined) { - headers.set(name, raw); - } - } - const body = await readRequestBody(request); - return new Request(`http://127.0.0.1:${port}${request.url ?? "/"}`, { - method: request.method, - headers, - body: body ? new Uint8Array(body) : undefined, - }); -}; - -const sendWebResponse = async (webResponse: Response, response: ServerResponse): Promise => { - response.statusCode = webResponse.status; - webResponse.headers.forEach((value, name) => response.setHeader(name, value)); - response.end(Buffer.from(await webResponse.arrayBuffer())); -}; - -const startHttpServer = async ( - handler: (request: Request) => Promise, -): Promise => { - let port = 0; - const server = createServer((request, response) => { - void Effect.runPromise( - Effect.tryPromise({ - try: async () => { - await sendWebResponse(await handler(await webRequestFromNode(request, port)), response); - }, - catch: () => "request failed", - }).pipe( - Effect.catch(() => - Effect.sync(() => { - response.statusCode = 500; - response.end("request failed"); - }), - ), - ), - ); - }); - await Effect.runPromise( - Effect.callback((resume) => { - server.once("error", () => resume(Effect.fail("failed to start test server"))); - server.listen(0, "127.0.0.1", () => { - const address = server.address(); - port = typeof address === "object" && address ? address.port : 0; - resume(Effect.void); - }); - }), - ); - return { - baseUrl: `http://127.0.0.1:${port}`, - close: () => - Effect.runPromise( - Effect.callback((resume) => { - server.close((error) => - error ? resume(Effect.fail("failed to close test server")) : resume(Effect.void), - ); - }), - ), - }; -}; - -const requestJson = async ( - path: string, - init: RequestInit = {}, - expectedStatus = 200, -): Promise => { - const response = await fetch(`${server.baseUrl}${path}`, init); - const text = await response.text(); - expect(response.status).toBe(expectedStatus); - return parseJsonText(text); -}; - -const postJson = (path: string, body: unknown, expectedStatus = 200): Promise => - requestJson( - path, - { - method: "POST", - headers: jsonHeaders, - body: JSON.stringify(body), - }, - expectedStatus, - ); - -const deleteJson = (path: string, expectedStatus = 200): Promise => - requestJson( - path, - { - method: "DELETE", - headers: jsonHeaders, - }, - expectedStatus, - ); - -const querySqliteRows = async ( - path: string, - sql: string, - args: readonly unknown[] = [], -): Promise[]> => { - const client = createClient({ url: `file:${path}` }); - const result = await client.execute({ sql, args: [...args] as never }); - client.close(); - return result.rows.map((row) => ({ ...row })); -}; - -const sourceStorageKey = (scope: string): string => - `v2-${Buffer.from(JSON.stringify([TEST_ORG, scope]), "utf8").toString("hex")}`; - -const sourceTokenItemId = (scope: string): string => - `apps:github-source:${TEST_ORG}:${scope}:token`; - -const DbJsonObject = Schema.Record(Schema.String, Schema.Unknown); -const DbJsonObjectFromString = Schema.fromJsonString(DbJsonObject); -const decodeDbJsonObject = Schema.decodeUnknownOption(DbJsonObject); -const decodeDbJsonObjectFromString = Schema.decodeUnknownOption(DbJsonObjectFromString); - -const decodeDbJson = (value: unknown): Record => { - if (value instanceof ArrayBuffer) { - return decodeDbJson(Buffer.from(value).toString("utf8")); - } - if (typeof value === "string") { - return Option.getOrElse(decodeDbJsonObjectFromString(value), () => ({})); - } - return Option.getOrElse(decodeDbJsonObject(value), () => ({})); -}; - -const githubFetch = async ( - emulator: Emulator, - token: string, - path: string, - init: RequestInit = {}, -): Promise => { - const headers = new Headers(init.headers); - headers.set("authorization", `Bearer ${token}`); - headers.set("accept", "application/vnd.github+json"); - if (init.body !== undefined) headers.set("content-type", "application/json"); - const response = await fetch(`${emulator.url}${path}`, { ...init, headers }); - const text = await response.text(); - expect(response.ok).toBe(true); - return parseJsonText(text); -}; - -const createIssue = ( - emulator: Emulator, - token: string, - title: string, - repo = REPO, -): Promise => - githubFetch(emulator, token, `/repos/${OWNER}/${repo}/issues`, { - method: "POST", - body: JSON.stringify({ title }), - }); - -const putRepoFiles = async ( - emulator: Emulator, - token: string, - files: Readonly>, - repo = REPO, -): Promise => { - const ref = await githubFetch<{ object: { sha: string } }>( - emulator, - token, - `/repos/${OWNER}/${repo}/git/ref/heads/main`, - ); - const parentSha = ref.object.sha; - const tree = await githubFetch<{ sha: string }>( - emulator, - token, - `/repos/${OWNER}/${repo}/git/trees`, - { - method: "POST", - body: JSON.stringify({ - tree: Object.entries(files).map(([path, content]) => ({ - path, - mode: "100644", - type: "blob", - content, - })), - }), - }, - ); - const commit = await githubFetch<{ sha: string }>( - emulator, - token, - `/repos/${OWNER}/${repo}/git/commits`, - { - method: "POST", - body: JSON.stringify({ - message: `Update custom tools ${randomBytes(3).toString("hex")}`, - tree: tree.sha, - parents: [parentSha], - }), - }, - ); - await githubFetch(emulator, token, `/repos/${OWNER}/${repo}/git/refs/heads/main`, { - method: "PATCH", - body: JSON.stringify({ sha: commit.sha }), - }); - return commit.sha; -}; - -const executorJson = JSON.stringify( - { - $schema: "https://executor.sh/schemas/executor.json", - description: "Curated tools wrapping our raw integration surface.", - }, - null, - 2, -); - -const dealPipelineSyncSource = `import { z } from "zod"; -import { defineTool, integration } from "executor:app"; - -/** - * The deterministic-program case: pull GitHub issues into the scope database - * so agents query a table instead of paging the raw API. - */ -export default defineTool({ - description: - "Refresh the deals table from GitHub issues. Syncs issue numbers and titles " + - "for pipeline questions.", - - integrations: { - github: integration("github"), - }, - - input: z.object({ - owner: z.string(), - repo: z.string(), - }), - - output: z.object({ synced: z.number() }), - - annotations: { readOnly: false, destructive: false }, - - async handler({ owner, repo }, { github, db }) { - await db.sql\` - CREATE TABLE IF NOT EXISTS deals ( - id TEXT PRIMARY KEY, - name TEXT NOT NULL, - stage TEXT NOT NULL - )\`; - - const issues = await github.repos.listIssues({ owner, repo }); - - let synced = 0; - for (const issue of issues) { - await db.sql\` - INSERT INTO deals (id, name, stage) - VALUES (\${String(issue.number)}, \${issue.title}, 'open') - ON CONFLICT (id) DO UPDATE SET - name = excluded.name, - stage = excluded.stage\`; - synced++; - } - - return { synced }; - }, -}); -`; - -const findDealDocsSource = `import { z } from "zod"; -import { defineTool, integration } from "executor:app"; - -/** - * The curation case: a read-only query over the scope database populated by - * the sync tool. - */ -export default defineTool({ - description: - "Find documents for a deal across synced pipeline records. Searches " + - "stored issue titles.", - - integrations: { - github: integration("github"), - }, - - input: z.object({ - limit: z.number().int().max(50).default(20), - }), - - output: z.object({ - documents: z.array( - z.object({ - name: z.string(), - stage: z.string(), - }), - ), - }), - - annotations: { readOnly: true, destructive: false }, - - async handler({ limit }, { db }) { - await db.sql\` - CREATE TABLE IF NOT EXISTS deals ( - id TEXT PRIMARY KEY, - name TEXT NOT NULL, - stage TEXT NOT NULL - )\`; - const rows = await db.sql\` - SELECT name, stage FROM deals ORDER BY id LIMIT \${limit}\`; - - return { - documents: rows.map((row) => ({ - name: String(row.name), - stage: String(row.stage), - })), - }; - }, -}); -`; - -const extraToolSource = `import { z } from "zod"; -import { defineTool } from "executor:app"; - -export default defineTool({ - description: "Return a static custom-tools health marker.", - input: z.object({}), - output: z.object({ ok: z.boolean() }), - annotations: { readOnly: true, destructive: false }, - async handler() { - return { ok: true }; - }, -}); -`; - -const approvalBridgeSource = `import { z } from "zod"; -import { defineTool, integration } from "executor:app"; - -export default defineTool({ - description: "List issues through a bridged GitHub client.", - integrations: { - github: integration("github"), - }, - input: z.object({ - owner: z.string(), - repo: z.string(), - }), - output: z.object({ count: z.number() }), - annotations: { readOnly: true, destructive: false }, - async handler({ owner, repo }, { github }) { - const issues = await github.repos.listIssues({ owner, repo }); - return { count: issues.length }; - }, -}); -`; - -const initialFiles = (): Record => ({ - "executor.json": executorJson, - "tools/deal-pipeline-sync.ts": dealPipelineSyncSource, - "tools/find-deal-docs.ts": findDealDocsSource, -}); - -const approvalFiles = (): Record => ({ - "executor.json": executorJson, - "tools/approval-bridge.ts": approvalBridgeSource, -}); - -const registerGitHubSpec = async (emulator: Emulator): Promise => { - const specResponse = await fetch(`${server.baseUrl}/api/openapi/specs`, { - method: "POST", - headers: jsonHeaders, - body: JSON.stringify({ - spec: { kind: "url", url: emulator.openapiUrl }, - slug: "github", - baseUrl: emulator.url, - authenticationTemplate: [bearerTemplate], - }), - }); - expect([200, 409]).toContain(specResponse.status); - await specResponse.text(); -}; - -const createGitHubConnection = async (token: string): Promise => { - const existing = await requestJson( - "/api/connections?integration=github", - { - headers: jsonHeaders, - }, - ); - if (existing.some((connection) => connection.address === GITHUB_CONNECTION)) { - return GITHUB_CONNECTION; - } - const created = await postJson<{ address: string }>("/api/connections", { - owner: "user", - name: "main", - integration: "github", - template: "bearer", - value: token, - }); - expect(created.address).toBe(GITHUB_CONNECTION); - return created.address; -}; - -const registerGitHubIntegration = async (emulator: Emulator, token: string): Promise => { - await registerGitHubSpec(emulator); - await createGitHubConnection(token); -}; - -const sourceUrl = (repo = REPO_FULL_NAME): string => `https://github.com/${repo}`; - -const addSource = ( - input: { readonly name?: string; readonly repo?: string; readonly token?: string } = {}, -): Promise => - postJson("/api/apps/sources/github/sync", { - name: input.name ?? APP_SLUG, - url: sourceUrl(input.repo), - ...(input.token ? { token: input.token } : {}), - }); - -const syncSource = ( - input: { readonly name?: string; readonly token?: string } = {}, -): Promise => - postJson("/api/apps/sources/github/sync", { - slug: input.name ?? APP_SLUG, - ...(input.token ? { token: input.token } : {}), - }); - -const listAppTools = (app = APP_SLUG): Promise => - requestJson(`/api/tools?integration=${encodeURIComponent(app)}`, { - headers: jsonHeaders, - }); - -const getSource = ( - app: string, -): Promise<{ readonly source: { readonly slug: string; readonly name: string } | null }> => - requestJson(`/api/apps/sources/github/${encodeURIComponent(app)}`, { headers: jsonHeaders }); - -const removeSource = (app: string): Promise<{ readonly removed: boolean }> => - deleteJson(`/api/apps/sources/github/${encodeURIComponent(app)}`); - -const execute = (code: string): Promise => - postJson("/api/executions", { code, autoApprove: true }); - -const executeWithApprovalPause = (code: string): Promise => - postJson("/api/executions", { code }); - -const executeResult = async (code: string): Promise => { - const response = await execute(code); - expect(response.status).toBe("completed"); - expect(response.isError).toBe(false); - return response.structured.result; -}; - -const callAppToolCode = (toolName: string, args: unknown, namespace = APP_SLUG): string => ` -const found = await tools.search({ namespace: ${JSON.stringify(namespace)}, query: ${JSON.stringify(toolName)}, limit: 20 }); -const item = found.items.find((candidate) => candidate.path.endsWith(${JSON.stringify(toolName)})); -if (!item) return { ok: false, missing: ${JSON.stringify(toolName)}, found }; -let fn = tools; -for (const segment of item.path.split(".")) fn = fn[segment]; -const result = await fn(${JSON.stringify(args)}); -return { path: item.path, result }; -`; - -let github!: Emulator; -let server!: TestHttpServer; -let disposeApp: () => Promise = async () => {}; - -beforeAll(async () => { - const emulatorPort = await findFreePort(); - github = await createEmulator({ - service: "github", - port: emulatorPort, - seed: { - github: { - users: [{ login: OWNER, name: "Syncer" }], - repos: [{ owner: OWNER, name: REPO, auto_init: true }], - }, - }, - }); - const { makeSelfHostTestApp, headerIdentityLayer } = await import("./testing/test-app"); - const app = await makeSelfHostTestApp({ identity: headerIdentityLayer, dbPath }); - disposeApp = app.dispose; - server = await startHttpServer(app.handler); -}); - -afterAll(async () => { - await server?.close(); - await disposeApp(); - await github?.close(); -}); - -test("custom tool connection schema updates when connections change after sync", async () => { - const credential = await github.credentials.mint({ - type: "api-key", - login: OWNER, - scopes: ["repo", "user"], - }); - const token = credential.token; - expect(token).toBeTruthy(); - await registerGitHubSpec(github); - const initialConnections = await requestJson( - "/api/connections?integration=github", - { headers: jsonHeaders }, - ); - expect(initialConnections).toEqual([]); - await putRepoFiles(github, token!, initialFiles()); - - const published = await addSource({ name: SCHEMA_APP_SLUG }); - expect(published.status).toBe("published"); - - const listed = await listAppTools(SCHEMA_APP_SLUG); - const dealSync = listed.find((tool) => tool.name === "deal-pipeline-sync"); - expect(dealSync).toBeTruthy(); - const beforeConnection = await requestJson<{ - inputSchema: { properties?: Record; required?: readonly string[] }; - }>(`/api/tools/schema?address=${encodeURIComponent(dealSync!.address)}`, { - headers: jsonHeaders, - }); - expect(beforeConnection.inputSchema.properties?.github).toMatchObject({ - type: "string", - enum: [], - }); - expect(beforeConnection.inputSchema.required ?? []).toContain("github"); - - await createGitHubConnection(token!); - const afterConnection = await requestJson<{ - inputSchema: { properties?: Record; required?: readonly string[] }; - }>(`/api/tools/schema?address=${encodeURIComponent(dealSync!.address)}`, { - headers: jsonHeaders, - }); - expect(afterConnection.inputSchema.properties?.github).toMatchObject({ - type: "string", - enum: [GITHUB_CONNECTION], - default: GITHUB_CONNECTION, - }); - expect(afterConnection.inputSchema.required ?? []).not.toContain("github"); - - const defaultInvoke = (await executeResult( - callAppToolCode( - "deal-pipeline-sync", - { - owner: OWNER, - repo: REPO, - }, - SCHEMA_APP_SLUG, - ), - )) as { result?: { ok?: boolean; data?: { synced?: number }; error?: unknown } }; - expect(defaultInvoke.result?.ok).toBe(true); - expect(defaultInvoke.result?.data?.synced).toBe(0); -}); - -test("GitHub source sync publishes and invokes custom tools through self-host HTTP", async () => { - const credential = await github.credentials.mint({ - type: "api-key", - login: OWNER, - scopes: ["repo", "user"], - }); - const token = credential.token; - expect(token).toBeTruthy(); - await registerGitHubIntegration(github, token!); - await createIssue(github, token!, "Acme renewal diligence"); - await createIssue(github, token!, "Beta pipeline memo"); - await putRepoFiles(github, token!, initialFiles()); - - const unauthorized = await fetch(`${server.baseUrl}/api/apps/sources/github/sync`, { - method: "POST", - headers: { "content-type": "application/json" }, - body: JSON.stringify({ url: sourceUrl(), token }), - }); - expect(unauthorized.status).toBe(401); - - const published = await addSource({ name: APP_SLUG, token: token! }); - expect(published.status).toBe("published"); - expect(JSON.stringify(published)).not.toContain(token!); - expect(published.tools).toEqual(["deal-pipeline-sync", "find-deal-docs"]); - expect(published.skipped).toEqual([]); - const firstSnapshot = published.snapshotId; - expect(firstSnapshot).toBeTruthy(); - - const tokenUpToDate = await syncSource({ token: token! }); - expect(tokenUpToDate.status).toBe("up-to-date"); - expect(JSON.stringify(tokenUpToDate)).not.toContain(token!); - - const sourcesList = await requestJson<{ - sources: readonly { readonly slug: string; readonly hasToken: boolean }[]; - }>("/api/apps/sources/github", { headers: jsonHeaders }); - expect(sourcesList.sources.find((source) => source.slug === APP_SLUG)?.hasToken).toBe(true); - expect(JSON.stringify(sourcesList)).not.toContain(token!); - - const listed = await listAppTools(); - expect(listed.map((tool) => tool.name).sort()).toEqual(["deal-pipeline-sync", "find-deal-docs"]); - - const searchResult = await executeResult( - `const found = await tools.search({ namespace: ${JSON.stringify(APP_SLUG)}, query: "pipeline", limit: 10 }); return found.items.map((item) => item.path);`, - ); - expect(JSON.stringify(searchResult)).toContain("deal-pipeline-sync"); - - const schema = await requestJson<{ - inputSchema: { properties?: Record; required?: readonly string[] }; - }>( - `/api/tools/schema?address=${encodeURIComponent(listed.find((tool) => tool.name === "deal-pipeline-sync")!.address)}`, - { headers: jsonHeaders }, - ); - expect(schema.inputSchema.properties?.github).toMatchObject({ - type: "string", - enum: [GITHUB_CONNECTION], - default: GITHUB_CONNECTION, - }); - expect(schema.inputSchema.required ?? []).not.toContain("github"); - - const defaultInvoke = (await executeResult( - callAppToolCode("deal-pipeline-sync", { - owner: OWNER, - repo: REPO, - }), - )) as { result?: { ok?: boolean; data?: { synced?: number }; error?: unknown } }; - expect(defaultInvoke.result?.ok).toBe(true); - expect(defaultInvoke.result?.data?.synced).toBe(2); - - const bareNameInvoke = await execute( - callAppToolCode("deal-pipeline-sync", { - github: "main", - owner: OWNER, - repo: REPO, - }), - ); - const bareNameStructured = bareNameInvoke.structured.result as { - readonly result?: { readonly error?: { readonly message?: string } }; - }; - expect(bareNameInvoke.status).toBe("completed"); - expect(bareNameStructured.result?.error?.message).toContain( - 'unknown connection "main" for role "github" (github)', - ); - expect(bareNameStructured.result?.error?.message).toContain(GITHUB_CONNECTION); - - const invoke = (await executeResult( - callAppToolCode("deal-pipeline-sync", { - github: GITHUB_CONNECTION, - owner: OWNER, - repo: REPO, - }), - )) as { result?: { ok?: boolean; data?: { synced?: number }; error?: unknown } }; - expect(invoke.result?.ok).toBe(true); - expect(invoke.result?.data?.synced).toBe(2); - - const bogusConnection = await execute( - callAppToolCode("deal-pipeline-sync", { - github: "garbage", - owner: OWNER, - repo: REPO, - }), - ); - const bogusBody = JSON.stringify(bogusConnection); - const bogusStructured = bogusConnection.structured.result as { - readonly result?: { readonly error?: { readonly message?: string } }; - }; - expect(bogusConnection.status).toBe("completed"); - expect(bogusStructured.result?.error?.message).toContain( - 'unknown connection "garbage" for role "github" (github)', - ); - expect(bogusBody).not.toContain("Internal tool error"); - - const ledger = await github.ledger.list(); - const sourceFetches = ledger.filter((entry) => entry.path === `/repos/${OWNER}/${REPO}`); - expect(sourceFetches.some((entry) => entry.identity.user?.login === OWNER)).toBe(true); - expect(sourceFetches.some((entry) => !entry.identity.user)).toBe(true); - const issueList = ledger.find( - (entry) => - entry.operationId === "issues/listForRepo" && entry.path === `/repos/${OWNER}/${REPO}/issues`, - ); - expect(issueList?.identity.user?.login).toBe(OWNER); - - const readDb = (await executeResult(callAppToolCode("find-deal-docs", { limit: 10 }))) as { - result?: { ok?: boolean; data?: { documents?: readonly { name: string }[] } }; - }; - expect(readDb.result?.ok).toBe(true); - expect(readDb.result?.data?.documents?.map((row) => row.name).sort()).toEqual([ - "Acme renewal diligence", - "Beta pipeline memo", - ]); - - const upToDate = await syncSource(); - expect(upToDate.status).toBe("up-to-date"); - expect("snapshotId" in upToDate).toBe(false); - - await putRepoFiles(github, token!, { - ...initialFiles(), - "tools/extra-tool.ts": extraToolSource, - }); - const withExtra = await syncSource(); - expect(withExtra.status).toBe("published"); - expect(withExtra.snapshotId).not.toBe(firstSnapshot); - expect((await listAppTools()).map((tool) => tool.name).sort()).toEqual([ - "deal-pipeline-sync", - "extra-tool", - "find-deal-docs", - ]); - - await putRepoFiles(github, token!, initialFiles()); - const removed = await syncSource(); - expect(removed.status).toBe("published"); - expect((await listAppTools()).map((tool) => tool.name).sort()).toEqual([ - "deal-pipeline-sync", - "find-deal-docs", - ]); - - await putRepoFiles(github, token!, { - ...initialFiles(), - "workflows/x.ts": "export default {};", - }); - const skipped = await syncSource(); - expect(skipped.status).toBe("published"); - expect(skipped.skipped).toEqual([{ path: "workflows/x.ts", reason: "not supported yet" }]); - const afterSkipped = await listAppTools(); - expect(afterSkipped.map((tool) => tool.name).sort()).toEqual([ - "deal-pipeline-sync", - "find-deal-docs", - ]); - expect(afterSkipped.some((tool) => tool.address.includes("workflow"))).toBe(false); -}); - -test("custom tools sources are per-integration and remove tears down owned state", async () => { - const credential = await github.credentials.mint({ - type: "api-key", - login: OWNER, - scopes: ["repo", "user"], - }); - const token = credential.token; - expect(token).toBeTruthy(); - await registerGitHubIntegration(github, token!); - await putRepoFiles(github, token!, initialFiles()); - - const firstApp = "side-tools-a"; - const secondApp = "side-tools-b"; - const first = await addSource({ name: firstApp, token: token! }); - const second = await addSource({ name: secondApp, token: token! }); - expect(first.status).toBe("published"); - expect(second.status).toBe("published"); - - const collision = await addSource({ name: firstApp, token: token! }); - expect(collision.status).toBe("failed"); - expect(collision.errors?.[0]?.message).toContain( - `Integration "${firstApp}" already exists. Choose another name.`, - ); - - const integrations = await requestJson("/api/integrations", { - headers: jsonHeaders, - }); - expect(integrations).toEqual( - expect.arrayContaining([ - expect.objectContaining({ - slug: firstApp, - name: firstApp, - kind: "apps", - canRemove: true, - displayUrl: sourceUrl(), - }), - expect.objectContaining({ - slug: secondApp, - name: secondApp, - kind: "apps", - canRemove: true, - displayUrl: sourceUrl(), - }), - ]), - ); - - const configRows = await querySqliteRows( - dbPath, - "SELECT slug, plugin_id, config FROM integration WHERE slug IN (?, ?) ORDER BY slug", - [firstApp, secondApp], - ); - expect(configRows).toHaveLength(2); - for (const row of configRows) { - expect(row.plugin_id).toBe("apps"); - const config = decodeDbJson(row.config); - expect(config).toMatchObject({ - kind: "github", - repoUrl: sourceUrl(), - repo: REPO_FULL_NAME, - scope: row.slug, - }); - expect(JSON.stringify(config)).not.toContain(token!); - } - - expect((await listAppTools(firstApp)).map((tool) => tool.address).sort()).toEqual([ - `tools.${firstApp}.user.main.deal-pipeline-sync`, - `tools.${firstApp}.user.main.find-deal-docs`, - ]); - expect((await listAppTools(secondApp)).map((tool) => tool.address).sort()).toEqual([ - `tools.${secondApp}.user.main.deal-pipeline-sync`, - `tools.${secondApp}.user.main.find-deal-docs`, - ]); - - const secondInvoke = (await executeResult( - callAppToolCode( - "deal-pipeline-sync", - { - owner: OWNER, - repo: REPO, - }, - secondApp, - ), - )) as { result?: { ok?: boolean; data?: { synced?: number }; error?: unknown } }; - expect(secondInvoke.result?.ok).toBe(true); - - const secondKey = sourceStorageKey(secondApp); - const secondArtifactDir = join(dataDir, "apps", "artifacts", `${secondKey}.git`); - const secondScopeDb = join(dataDir, "apps", "scope-db", `${secondKey}.db`); - expect(existsSync(secondArtifactDir)).toBe(true); - expect(existsSync(secondScopeDb)).toBe(true); - - const tokenRowsBefore = await querySqliteRows( - dbPath, - "SELECT key FROM plugin_storage WHERE plugin_id = ? AND collection = ? AND key = ?", - ["encryptedSecrets", "secrets", sourceTokenItemId(secondApp)], - ); - expect(tokenRowsBefore).toHaveLength(1); - - expect(await getSource(secondApp)).toMatchObject({ - source: { slug: secondApp, name: secondApp }, - }); - expect(await removeSource(secondApp)).toEqual({ removed: true }); - expect(await getSource(secondApp)).toEqual({ source: null }); - - const secondIntegration = await fetch( - `${server.baseUrl}/api/integrations/${encodeURIComponent(secondApp)}`, - { headers: jsonHeaders }, - ); - expect(secondIntegration.status).toBe(404); - await secondIntegration.text(); - - expect(await listAppTools(secondApp)).toEqual([]); - expect((await listAppTools(firstApp)).map((tool) => tool.address).sort()).toEqual([ - `tools.${firstApp}.user.main.deal-pipeline-sync`, - `tools.${firstApp}.user.main.find-deal-docs`, - ]); - - const descriptorRows = await querySqliteRows( - join(dataDir, "apps", "store.sqlite"), - "SELECT scope FROM descriptors WHERE tenant = ? AND scope = ?", - [TEST_ORG, secondApp], - ); - expect(descriptorRows).toEqual([]); - const tokenRowsAfter = await querySqliteRows( - dbPath, - "SELECT key FROM plugin_storage WHERE plugin_id = ? AND collection = ? AND key = ?", - ["encryptedSecrets", "secrets", sourceTokenItemId(secondApp)], - ); - expect(tokenRowsAfter).toEqual([]); - expect(existsSync(secondArtifactDir)).toBe(false); - expect(existsSync(secondScopeDb)).toBe(false); -}); - -test("bridged integration calls inherit the caller approval handler", async () => { - const credential = await github.credentials.mint({ - type: "api-key", - login: OWNER, - scopes: ["repo", "user"], - }); - const token = credential.token; - expect(token).toBeTruthy(); - await registerGitHubIntegration(github, token!); - await createIssue(github, token!, "Approval-gated issue"); - await putRepoFiles(github, token!, approvalFiles()); - const published = await addSource({ name: APPROVAL_APP_SLUG, token: token! }); - expect(published.status).toBe("published"); - - const githubTools = await requestJson("/api/tools?integration=github", { - headers: jsonHeaders, - }); - const listIssues = githubTools.find((tool) => tool.name === "repos.listIssues"); - expect(listIssues).toBeTruthy(); - await postJson("/api/policies", { - owner: "org", - pattern: listIssues!.address.replace(/^tools\./, ""), - action: "require_approval", - }); - - const beforeLedger = await github.ledger.list(); - const beforeCalls = beforeLedger.filter( - (entry) => - entry.operationId === "issues/listForRepo" && entry.path === `/repos/${OWNER}/${REPO}/issues`, - ).length; - - const response = await executeWithApprovalPause( - callAppToolCode( - "approval-bridge", - { - github: GITHUB_CONNECTION, - owner: OWNER, - repo: REPO, - }, - APPROVAL_APP_SLUG, - ), - ); - expect(response.status).toBe("paused"); - - const afterLedger = await github.ledger.list(); - const afterCalls = afterLedger.filter( - (entry) => - entry.operationId === "issues/listForRepo" && entry.path === `/repos/${OWNER}/${REPO}/issues`, - ).length; - expect(afterCalls).toBe(beforeCalls); -}); diff --git a/e2e/scenarios/custom-tools.test.ts b/e2e/scenarios/custom-tools.test.ts new file mode 100644 index 000000000..0647deafa --- /dev/null +++ b/e2e/scenarios/custom-tools.test.ts @@ -0,0 +1,613 @@ +import { randomBytes } from "node:crypto"; +import { createServer } from "node:net"; + +import { expect } from "@effect/vitest"; +import { Effect } from "effect"; +import { createEmulator, type Emulator } from "@executor-js/emulate"; + +import { scenario } from "../src/scenario"; +import { Api, Browser, Target } from "../src/services"; +import type { Identity, Target as TargetShape } from "../src/target"; +import type { BrowserSurface } from "../src/surfaces/browser"; + +const OWNER = "syncer"; +const GITHUB_CONNECTION = "tools.github.user.main"; + +const unique = (prefix: string) => `${prefix}-${randomBytes(4).toString("hex")}`; + +const bearerTemplate = { + slug: "bearer", + type: "apiKey", + label: "Bearer token", + headers: { Authorization: ["Bearer ", { type: "variable", name: "token" }] }, +}; + +const availablePort = Effect.callback((resume) => { + const server = createServer(); + server.listen(0, "127.0.0.1", () => { + const address = server.address(); + const port = typeof address === "object" && address ? address.port : 0; + server.close(() => resume(Effect.succeed(port))); + }); +}); + +const githubEmulator = (repo: string) => + Effect.acquireRelease( + Effect.gen(function* () { + const port = yield* availablePort; + return yield* Effect.promise(() => + createEmulator({ + service: "github", + port, + seed: { + github: { + users: [{ login: OWNER, name: "Custom Tools Syncer" }], + repos: [{ owner: OWNER, name: repo, auto_init: true }], + }, + }, + }), + ); + }), + (emulator: Emulator) => Effect.promise(() => emulator.close()).pipe(Effect.ignore), + ); + +interface SyncResult { + readonly status: "published" | "up-to-date" | "failed"; + readonly snapshotId?: string; + readonly upstreamSha?: string; + readonly tools: readonly string[]; + readonly skipped: readonly { readonly path: string; readonly reason: string }[]; + readonly errors?: readonly { readonly message?: string }[]; +} + +interface ToolRow { + readonly address: string; + readonly name: string; + readonly integration: string; +} + +interface ExecuteResponse { + readonly status: "completed" | "paused"; + readonly text: string; + readonly structured: { + readonly result?: unknown; + readonly error?: unknown; + }; + readonly isError?: boolean; +} + +const parseJson = async (response: Response): Promise => { + const text = await response.text(); + return (text.length > 0 ? JSON.parse(text) : null) as T; +}; + +const request = async ( + target: TargetShape, + identity: Identity, + path: string, + init: RequestInit = {}, + expectedStatus = 200, +): Promise<{ readonly body: T; readonly text: string }> => { + const headers = new Headers(init.headers); + headers.set("origin", new URL(target.baseUrl).origin); + for (const [name, value] of Object.entries(identity.headers ?? {})) { + headers.set(name, value); + } + if (init.body !== undefined && !headers.has("content-type")) { + headers.set("content-type", "application/json"); + } + const response = await fetch(new URL(path, target.baseUrl), { ...init, headers }); + const text = await response.text(); + expect(response.status, `${init.method ?? "GET"} ${path}: ${text}`).toBe(expectedStatus); + return { body: (text.length > 0 ? JSON.parse(text) : null) as T, text }; +}; + +const postJson = ( + target: TargetShape, + identity: Identity, + path: string, + body: unknown, + expectedStatus = 200, +): Promise<{ readonly body: T; readonly text: string }> => + request( + target, + identity, + path, + { + method: "POST", + body: JSON.stringify(body), + }, + expectedStatus, + ); + +const deletePath = ( + target: TargetShape, + identity: Identity, + path: string, +): Promise<{ readonly status: number; readonly text: string }> => + fetch(new URL(path, target.baseUrl), { + method: "DELETE", + headers: { + ...(identity.headers ?? {}), + origin: new URL(target.baseUrl).origin, + }, + }).then(async (response) => ({ status: response.status, text: await response.text() })); + +const githubFetch = async ( + emulator: Emulator, + token: string, + path: string, + init: RequestInit = {}, +): Promise => { + const headers = new Headers(init.headers); + headers.set("authorization", `Bearer ${token}`); + headers.set("accept", "application/vnd.github+json"); + if (init.body !== undefined) headers.set("content-type", "application/json"); + const response = await fetch(`${emulator.url}${path}`, { ...init, headers }); + expect(response.ok, `${init.method ?? "GET"} ${path}: ${await response.clone().text()}`).toBe( + true, + ); + return parseJson(response); +}; + +const createIssue = ( + emulator: Emulator, + token: string, + repo: string, + title: string, +): Promise => + githubFetch(emulator, token, `/repos/${OWNER}/${repo}/issues`, { + method: "POST", + body: JSON.stringify({ title }), + }); + +const putRepoFiles = async ( + emulator: Emulator, + token: string, + repo: string, + files: Readonly>, +): Promise => { + const ref = await githubFetch<{ object: { sha: string } }>( + emulator, + token, + `/repos/${OWNER}/${repo}/git/ref/heads/main`, + ); + const tree = await githubFetch<{ sha: string }>( + emulator, + token, + `/repos/${OWNER}/${repo}/git/trees`, + { + method: "POST", + body: JSON.stringify({ + tree: Object.entries(files).map(([path, content]) => ({ + path, + mode: "100644", + type: "blob", + content, + })), + }), + }, + ); + const commit = await githubFetch<{ sha: string }>( + emulator, + token, + `/repos/${OWNER}/${repo}/git/commits`, + { + method: "POST", + body: JSON.stringify({ + message: `Update custom tools ${randomBytes(3).toString("hex")}`, + tree: tree.sha, + parents: [ref.object.sha], + }), + }, + ); + await githubFetch(emulator, token, `/repos/${OWNER}/${repo}/git/refs/heads/main`, { + method: "PATCH", + body: JSON.stringify({ sha: commit.sha }), + }); + return commit.sha; +}; + +const executorJson = JSON.stringify( + { + $schema: "https://executor.sh/schemas/executor.json", + description: "Custom tools e2e fixture.", + }, + null, + 2, +); + +const dealPipelineSyncSource = `import { z } from "zod"; +import { defineTool, integration } from "executor:app"; + +export default defineTool({ + description: "Summarize open GitHub issues for pipeline review.", + integrations: { + github: integration("github"), + }, + input: z.object({ + owner: z.string(), + repo: z.string(), + }), + output: z.object({ + synced: z.number(), + issues: z.array(z.object({ number: z.number(), title: z.string() })), + }), + annotations: { readOnly: false, destructive: false }, + async handler({ owner, repo }, { github }) { + const issues = await github.repos.listIssues({ owner, repo, state: "open" }); + return { + synced: issues.length, + issues: issues.map((issue) => ({ number: issue.number, title: issue.title })), + }; + }, +}); +`; + +const findDealDocsSource = `import { z } from "zod"; +import { defineTool, integration } from "executor:app"; + +export default defineTool({ + description: "Return issue titles as deal-document search results.", + integrations: { + github: integration("github"), + }, + input: z.object({ + owner: z.string(), + repo: z.string(), + limit: z.number().int().max(50).default(20), + }), + output: z.object({ + documents: z.array(z.object({ name: z.string(), number: z.number() })), + }), + annotations: { readOnly: true, destructive: false }, + async handler({ owner, repo, limit }, { github }) { + const issues = await github.repos.listIssues({ owner, repo, state: "open" }); + return { + documents: issues.slice(0, limit).map((issue) => ({ + name: issue.title, + number: issue.number, + })), + }; + }, +}); +`; + +const extraToolSource = `import { z } from "zod"; +import { defineTool } from "executor:app"; + +export default defineTool({ + description: "Return a static custom-tools health marker.", + input: z.object({}), + output: z.object({ ok: z.boolean() }), + annotations: { readOnly: true, destructive: false }, + async handler() { + return { ok: true }; + }, +}); +`; + +const sourceFiles = (): Record => ({ + "executor.json": executorJson, + "tools/deal-pipeline-sync.ts": dealPipelineSyncSource, + "tools/find-deal-docs.ts": findDealDocsSource, +}); + +const sourceUrl = (repo: string): string => `https://github.com/${OWNER}/${repo}`; + +const registerGithubIntegration = async ( + target: TargetShape, + identity: Identity, + emulator: Emulator, + token: string, +): Promise => { + await deletePath(target, identity, "/api/connections/user/github/main"); + await deletePath(target, identity, "/api/openapi/integrations/github"); + const added = await postJson<{ slug?: string }>(target, identity, "/api/openapi/specs", { + spec: { kind: "url", url: emulator.openapiUrl }, + slug: "github", + baseUrl: emulator.url, + authenticationTemplate: [bearerTemplate], + }); + expect(added.body.slug).toBe("github"); + const connection = await postJson<{ address: string }>(target, identity, "/api/connections", { + owner: "user", + name: "main", + integration: "github", + template: "bearer", + value: token, + }); + expect(connection.body.address).toBe(GITHUB_CONNECTION); +}; + +const execute = (target: TargetShape, identity: Identity, code: string): Promise => + postJson(target, identity, "/api/executions", { + code, + autoApprove: true, + }).then((response) => response.body); + +const executeResult = async ( + target: TargetShape, + identity: Identity, + code: string, +): Promise => { + const response = await execute(target, identity, code); + expect(response.status, response.text).toBe("completed"); + expect(response.isError, response.text).toBe(false); + return response.structured.result; +}; + +const callAppToolCode = (namespace: string, toolName: string, args: unknown): string => ` +const found = await tools.search({ namespace: ${JSON.stringify(namespace)}, query: ${JSON.stringify( + toolName, +)}, limit: 20 }); +const item = found.items.find((candidate) => candidate.path.endsWith(${JSON.stringify(toolName)})); +if (!item) return { ok: false, missing: ${JSON.stringify(toolName)}, found }; +let fn = tools; +for (const segment of item.path.split(".")) fn = fn[segment]; +const result = await fn(${JSON.stringify(args)}); +return { path: item.path, result }; +`; + +const addSourceThroughConsole = (input: { + readonly target: TargetShape; + readonly browser: BrowserSurface; + readonly identity: Identity; + readonly repo: string; + readonly appSlug: string; + readonly token: string; +}) => + input.browser.session(input.identity, async ({ page, step }) => { + await step("Open the integrations page", async () => { + await page.goto(new URL("/integrations", input.target.baseUrl).toString(), { + waitUntil: "networkidle", + }); + }); + + await step("Detect the GitHub repository as custom tools", async () => { + await page.getByRole("button", { name: "Connect", exact: true }).click(); + const dialog = page.getByRole("dialog"); + await dialog.getByRole("textbox").fill(sourceUrl(input.repo)); + await dialog.getByRole("button", { name: "Detect" }).click(); + await page.getByRole("heading", { name: "Add custom tools" }).waitFor(); + }); + + await step("Sync the custom tools source", async () => { + await page.locator('input[type="password"]').fill(input.token); + await page.getByRole("button", { name: "Sync repo" }).click(); + await page.waitForURL(new RegExp(`/integrations/${input.appSlug}(?:\\?|$)`), { + timeout: 60_000, + }); + await page.getByRole("link", { name: "2 tools" }).waitFor({ timeout: 60_000 }); + }); + }); + +const syncSourceInConsole = (input: { + readonly target: TargetShape; + readonly browser: BrowserSurface; + readonly identity: Identity; + readonly appSlug: string; + readonly expectedNotice: string; + readonly expectedToolCount: string; +}) => + input.browser.session(input.identity, async ({ page, step }) => { + await step(`Sync ${input.appSlug}`, async () => { + await page.goto( + new URL(`/integrations/${input.appSlug}?tab=source`, input.target.baseUrl).toString(), + { + waitUntil: "networkidle", + }, + ); + await page.getByRole("button", { name: "Sync" }).click(); + await page.getByText(input.expectedNotice).waitFor({ timeout: 60_000 }); + await page.getByRole("link", { name: input.expectedToolCount }).waitFor({ timeout: 60_000 }); + }); + }); + +const removeSourceThroughConsole = (input: { + readonly target: TargetShape; + readonly browser: BrowserSurface; + readonly identity: Identity; + readonly appSlug: string; +}) => + input.browser.session(input.identity, async ({ page, step }) => { + await step("Remove the custom tools source", async () => { + await page.goto( + new URL(`/integrations/${input.appSlug}?tab=source`, input.target.baseUrl).toString(), + { + waitUntil: "networkidle", + }, + ); + await page.getByRole("button", { name: "Remove" }).click(); + await page.getByRole("button", { name: "Remove source" }).click(); + await page.waitForURL(/\/integrations(?:\?|$)/, { timeout: 60_000 }); + }); + }); + +scenario( + "Custom tools · GitHub source syncs, invokes, refreshes, and removes", + { timeout: 300_000 }, + Effect.scoped( + Effect.gen(function* () { + const target = yield* Target; + if (target.name !== "selfhost") return; + const browser = yield* Browser; + yield* Api; + const identity = yield* target.newIdentity(); + const repo = unique("custom-tools"); + const appSlug = repo; + const emulator = yield* githubEmulator(repo); + const credential = yield* Effect.promise(() => + emulator.credentials.mint({ type: "api-key", login: OWNER, scopes: ["repo", "user"] }), + ); + const token = credential.token; + if (!token) return yield* Effect.die("GitHub emulator did not mint a token."); + + yield* Effect.ensuring( + Effect.gen(function* () { + const unauthorized = yield* Effect.promise(() => + fetch(new URL("/api/apps/sources/github/sync", target.baseUrl), { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ url: sourceUrl(repo), token }), + }), + ); + expect(unauthorized.status, "sync requires authentication").toBe(401); + yield* Effect.promise(() => unauthorized.text()); + + yield* Effect.promise(() => registerGithubIntegration(target, identity, emulator, token)); + yield* Effect.promise(() => createIssue(emulator, token, repo, "Acme renewal diligence")); + yield* Effect.promise(() => createIssue(emulator, token, repo, "Beta pipeline memo")); + yield* Effect.promise(() => putRepoFiles(emulator, token, repo, sourceFiles())); + + yield* addSourceThroughConsole({ target, browser, identity, repo, appSlug, token }); + + const sources = yield* Effect.promise(() => + request<{ sources: readonly { readonly slug: string; readonly hasToken: boolean }[] }>( + target, + identity, + "/api/apps/sources/github", + ), + ); + expect(sources.text).not.toContain(token); + expect(sources.body.sources.find((source) => source.slug === appSlug)?.hasToken).toBe( + true, + ); + + const detail = yield* Effect.promise(() => + request<{ source: { readonly slug: string; readonly hasToken: boolean } | null }>( + target, + identity, + `/api/apps/sources/github/${encodeURIComponent(appSlug)}`, + ), + ); + expect(detail.text).not.toContain(token); + expect(detail.body.source).toMatchObject({ slug: appSlug, hasToken: true }); + + const tools = yield* Effect.promise(() => + request( + target, + identity, + `/api/tools?integration=${encodeURIComponent(appSlug)}`, + ), + ); + expect(tools.body.map((tool) => tool.name).sort()).toEqual([ + "deal-pipeline-sync", + "find-deal-docs", + ]); + const syncTool = tools.body.find((tool) => tool.name === "deal-pipeline-sync"); + expect(syncTool).toBeDefined(); + + const schema = yield* Effect.promise(() => + request<{ + inputSchema: { + readonly properties?: Record< + string, + { readonly enum?: readonly string[]; readonly default?: string } + >; + readonly required?: readonly string[]; + }; + }>( + target, + identity, + `/api/tools/schema?address=${encodeURIComponent(syncTool!.address)}`, + ), + ); + expect(schema.body.inputSchema.properties?.github).toMatchObject({ + enum: [GITHUB_CONNECTION], + default: GITHUB_CONNECTION, + }); + expect(schema.body.inputSchema.required ?? []).not.toContain("github"); + + const invoked = (yield* Effect.promise(() => + executeResult( + target, + identity, + callAppToolCode(appSlug, "deal-pipeline-sync", { owner: OWNER, repo }), + ), + )) as { + readonly result?: { + readonly ok?: boolean; + readonly data?: { readonly synced?: number }; + }; + }; + expect(invoked.result?.ok).toBe(true); + expect(invoked.result?.data?.synced).toBe(2); + + const ledger = yield* Effect.promise(() => emulator.ledger.list()); + const issueList = ledger.find( + (entry) => + entry.operationId === "issues/listForRepo" && + entry.path === `/repos/${OWNER}/${repo}/issues`, + ); + expect( + issueList?.identity.user?.login, + "custom tool called GitHub with the connection", + ).toBe(OWNER); + + const upToDate = yield* Effect.promise(() => + postJson(target, identity, "/api/apps/sources/github/sync", { + slug: appSlug, + }), + ); + expect(upToDate.text).not.toContain(token); + expect(upToDate.body.status).toBe("up-to-date"); + + yield* syncSourceInConsole({ + target, + browser, + identity, + appSlug, + expectedNotice: "Already up to date.", + expectedToolCount: "2 tools", + }); + + yield* Effect.promise(() => + putRepoFiles(emulator, token, repo, { + ...sourceFiles(), + "tools/extra-tool.ts": extraToolSource, + }), + ); + yield* syncSourceInConsole({ + target, + browser, + identity, + appSlug, + expectedNotice: "Added: extra-tool", + expectedToolCount: "3 tools", + }); + + yield* Effect.promise(() => putRepoFiles(emulator, token, repo, sourceFiles())); + yield* syncSourceInConsole({ + target, + browser, + identity, + appSlug, + expectedNotice: "Removed: extra-tool", + expectedToolCount: "2 tools", + }); + + yield* removeSourceThroughConsole({ target, browser, identity, appSlug }); + + const afterRemove = yield* Effect.promise(() => + request( + target, + identity, + `/api/tools?integration=${encodeURIComponent(appSlug)}`, + ), + ); + expect(afterRemove.body).toEqual([]); + }), + Effect.promise(async () => { + await deletePath( + target, + identity, + `/api/apps/sources/github/${encodeURIComponent(appSlug)}`, + ); + await deletePath(target, identity, "/api/connections/user/github/main"); + await deletePath(target, identity, "/api/openapi/integrations/github"); + }).pipe(Effect.ignore), + ); + }), + ), +); From 807d08cb0763fa39ca377f88168781bbf89fe292 Mon Sep 17 00:00:00 2001 From: Rhys Sullivan <39114868+RhysSullivan@users.noreply.github.com> Date: Tue, 7 Jul 2026 01:36:26 -0700 Subject: [PATCH 21/21] Cover custom tools resolver options --- .../plugins/apps/src/plugin/resolver.test.ts | 46 +++++++++++++++++++ packages/plugins/apps/src/plugin/resolver.ts | 9 +++- 2 files changed, 54 insertions(+), 1 deletion(-) create mode 100644 packages/plugins/apps/src/plugin/resolver.test.ts diff --git a/packages/plugins/apps/src/plugin/resolver.test.ts b/packages/plugins/apps/src/plugin/resolver.test.ts new file mode 100644 index 000000000..b675bc4ee --- /dev/null +++ b/packages/plugins/apps/src/plugin/resolver.test.ts @@ -0,0 +1,46 @@ +import { describe, expect, it } from "@effect/vitest"; +import { Effect } from "effect"; +import type { InvokeOptions } from "@executor-js/sdk"; + +import { makePluginCtxAppsResolver, type AppsResolverPluginCtx } from "./resolver"; + +const run = (effect: Effect.Effect): Promise => Effect.runPromise(effect); + +describe("makePluginCtxAppsResolver", () => { + it("passes caller invoke options through bridged integration calls", async () => { + const invokeOptions = { onElicitation: "accept-all" } satisfies InvokeOptions; + let seenAddress = ""; + let seenPayload: unknown; + let seenOptions: InvokeOptions | undefined; + + const ctx: AppsResolverPluginCtx = { + connections: { + list: () => Effect.succeed([]), + get: () => Effect.succeed(null), + }, + execute: (address: unknown, payload: unknown, options?: InvokeOptions) => + Effect.sync(() => { + seenAddress = String(address); + seenPayload = payload; + seenOptions = options; + return "called"; + }), + }; + + const resolver = makePluginCtxAppsResolver({ ctx }); + const result = await run( + resolver.call({ + integration: "github", + connection: "tools.github.user.main", + path: ["issues", "listForRepo"], + args: [{ owner: "acme", repo: "tools" }], + invokeOptions, + }), + ); + + expect(result).toBe("called"); + expect(seenAddress).toBe("tools.github.user.main.issues.listForRepo"); + expect(seenPayload).toEqual({ owner: "acme", repo: "tools" }); + expect(seenOptions).toBe(invokeOptions); + }); +}); diff --git a/packages/plugins/apps/src/plugin/resolver.ts b/packages/plugins/apps/src/plugin/resolver.ts index 27f5ca0c6..e4fd70c65 100644 --- a/packages/plugins/apps/src/plugin/resolver.ts +++ b/packages/plugins/apps/src/plugin/resolver.ts @@ -11,6 +11,11 @@ import { } from "@executor-js/sdk"; import { BindingError, type ClientResolver, type ConnectionCandidate } from "./bindings"; +export interface AppsResolverPluginCtx { + readonly connections: Pick; + readonly execute: PluginCtx["execute"]; +} + const parseConnectionAddress = (address: string): ConnectionRef | null => { const parts = address.split("."); if (parts.length !== 4 || parts[0] !== "tools") return null; @@ -36,7 +41,9 @@ const toCandidate = (connection: { ...(connection.owner !== undefined ? { owner: String(connection.owner) } : {}), }); -export const makePluginCtxAppsResolver = (input: { readonly ctx: PluginCtx }): ClientResolver => ({ +export const makePluginCtxAppsResolver = (input: { + readonly ctx: AppsResolverPluginCtx; +}): ClientResolver => ({ listConnections: ({ integration }) => input.ctx.connections.list({ integration: IntegrationSlug.make(integration) }).pipe( Effect.map((connections) => connections.map(toCandidate)),