Apps subsystem for self-hosted

apps/host-selfhost/executor.config.ts

@@ -8,6 +8,7 @@ import { encryptedSecretsPlugin } from "@executor-js/plugin-encrypted-secrets";
 import { toolkitsPlugin } from "@executor-js/plugin-toolkits/server";
 
 import { resolveSecretKey } from "./src/config";
+import { getSelfHostAppsSubsystem } from "./src/apps";
 
 // ---------------------------------------------------------------------------
 // Single source of truth for the self-hosted app's plugin list.
@@ -38,5 +39,10 @@ export default defineExecutorConfig({
       toolkitsPlugin({ activeToolkitSlug }),
       // First writable secret provider -> the default for `secrets.set`.
       encryptedSecretsPlugin({ key: resolveSecretKey() }),
+      // The apps source plugin: published custom tools become real catalog
+      // citizens (tools.list + execute through the same policy/audit path). The
+      // runtime is a boot-time singleton shared across per-request plugin
+      // instances; the plugin itself is thin.
+      getSelfHostAppsSubsystem().plugin,
     ] as const,
 });

apps/host-selfhost/package.json

@@ -26,6 +26,7 @@
     "@executor-js/execution": "workspace:*",
     "@executor-js/fumadb": "workspace:*",
     "@executor-js/host-mcp": "workspace:*",
+    "@executor-js/plugin-apps": "workspace:*",
     "@executor-js/plugin-encrypted-secrets": "workspace:*",
     "@executor-js/plugin-google": "workspace:*",
     "@executor-js/plugin-graphql": "workspace:*",
@@ -48,6 +49,7 @@
   },
   "devDependencies": {
     "@effect/vitest": "catalog:",
+    "@executor-js/emulate": "^0.13.2",
     "@executor-js/vite-plugin": "workspace:*",
     "@tailwindcss/vite": "catalog:",
     "@tanstack/router-plugin": "^1.167.12",

apps/host-selfhost/scripts/mcp-apps-serve.ts

@@ -0,0 +1,196 @@
+// Boot the REAL self-host app in-process, seed a published daily-brief app with
+// scope-db rows, then serve its `/mcp` endpoint on a fixed loopback port for the
+// sunpeak host simulation (e2e/mcp-apps) to connect to.
+//
+// The self-host MCP endpoint is Better-Auth-gated. Rather than teach sunpeak the
+// auth dance, this wrapper owns auth: it signs up once, holds the bearer, and
+// injects it on every forwarded request. sunpeak connects to `/mcp` with no
+// credentials; the wrapper adds the Authorization header.
+//
+// Lives here (under apps/host-selfhost) so its imports resolve through this
+// package's node_modules under Bun. Launched by e2e/mcp-apps/playwright.config.ts.
+//
+// Env in: PORT (wrapper port, default 8791). Health: GET /health -> 200 "ok".
+
+import { mkdtempSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+
+import { createEmulator } from "@executor-js/emulate";
+import { Client } from "@modelcontextprotocol/sdk/client/index.js";
+import { StreamableHTTPClientTransport } from "@modelcontextprotocol/sdk/client/streamableHttp.js";
+import { dailyBriefFileSet } from "@executor-js/plugin-apps/testing";
+
+import { mintInviteCode } from "../src/testing/mint-invite";
+
+const PORT = Number(process.env.PORT ?? "8791");
+const origin = "http://mcp-apps.internal";
+const log = (...args: unknown[]) => console.log("[mcp-apps-server]", ...args);
+
+process.env.EXECUTOR_DATA_DIR = mkdtempSync(join(tmpdir(), "eh-mcpapps-"));
+process.env.BETTER_AUTH_SECRET = "mcp-apps-secret-0123456789-abcdefghij-klmnop";
+process.env.EXECUTOR_BOOTSTRAP_ADMIN_EMAIL = "admin@mcp-apps.test";
+process.env.EXECUTOR_BOOTSTRAP_ADMIN_PASSWORD = "admin-pass-123456";
+process.env.EXECUTOR_ALLOW_LOCAL_NETWORK = "true";
+
+// --- real-shaped GitHub emulator + a seeded repo with issues ----------------
+const github = await createEmulator({ service: "github" });
+const cred = (await github.credentials.mint({
+  type: "api-key",
+})) as unknown as { token: string };
+const ghToken = cred.token;
+const ghHeaders = {
+  authorization: `Bearer ${ghToken}`,
+  accept: "application/vnd.github+json",
+  "content-type": "application/json",
+};
+const repoRes = await fetch(`${github.url}/user/repos`, {
+  method: "POST",
+  headers: ghHeaders,
+  body: JSON.stringify({ name: "app" }),
+});
+const owner = ((await repoRes.json()) as { owner: { login: string } }).owner.login;
+for (const title of ["Fresh bug", "Second bug"]) {
+  await fetch(`${github.url}/repos/${owner}/app/issues`, {
+    method: "POST",
+    headers: ghHeaders,
+    body: JSON.stringify({ title, labels: ["bug"] }),
+  });
+}
+log("emulator ready", github.url, "owner", owner);
+
+// --- boot the real self-host handler ---------------------------------------
+const { makeSelfHostApiHandler } = await import("../src/app");
+const app = await makeSelfHostApiHandler();
+const handler = app.handler;
+
+// --- sign up -> bearer token ------------------------------------------------
+const inviteCode = await mintInviteCode(handler);
+const su = await handler(
+  new Request(`${origin}/api/auth/sign-up/email`, {
+    method: "POST",
+    headers: { "content-type": "application/json" },
+    body: JSON.stringify({
+      email: "u@mcp-apps.test",
+      password: "password-12345678",
+      name: "U",
+      inviteCode,
+    }),
+  }),
+);
+const token = su.headers.get("set-auth-token");
+if (!token) throw new Error("sign-up produced no token");
+
+const api = (path: string, init: RequestInit = {}) =>
+  handler(
+    new Request(`${origin}${path}`, {
+      ...init,
+      headers: {
+        authorization: `Bearer ${token}`,
+        "content-type": "application/json",
+        ...(init.headers ?? {}),
+      },
+    }),
+  );
+
+// --- register the emulator as `github`, plus a connection to it -------------
+const spec = JSON.stringify({
+  openapi: "3.0.0",
+  info: { title: "GitHub (emulated)", version: "1.0.0" },
+  servers: [{ url: github.url }],
+  paths: {
+    "/user/repos": {
+      get: {
+        operationId: "listRepos",
+        responses: { 200: { description: "ok" } },
+      },
+    },
+  },
+});
+await api("/api/openapi/specs", {
+  method: "POST",
+  body: JSON.stringify({
+    spec: { kind: "blob", value: spec },
+    slug: "github",
+    baseUrl: github.url,
+  }),
+});
+await api("/api/connections", {
+  method: "POST",
+  body: JSON.stringify({
+    owner: "user",
+    // Name the connection after its integration: the apps default binding maps a
+    // role to a connection of the same name as its integration, and the resolver
+    // requires an exact match (no silent conns[0] fallback), so the catalog-invoke
+    // path only resolves when the connection is named "github".
+    name: "github",
+    integration: "github",
+    template: "bearer",
+    value: ghToken,
+  }),
+});
+
+// --- connect a real MCP client to publish + populate rows -------------------
+const wireFetch = (input: RequestInfo | URL, init?: RequestInit): Promise<Response> =>
+  handler(input instanceof Request ? input : new Request(input, init));
+const client = new Client({ name: "mcp-apps-setup", version: "1.0.0" });
+await client.connect(
+  new StreamableHTTPClientTransport(new URL(`${origin}/mcp`), {
+    fetch: wireFetch as unknown as typeof globalThis.fetch,
+    requestInit: { headers: { authorization: `Bearer ${token}` } },
+  }),
+);
+await client.callTool({
+  name: "apps_publish",
+  arguments: { files: Object.fromEntries(dailyBriefFileSet()) },
+});
+await client.callTool({
+  name: "execute",
+  arguments: {
+    code: "export default await tools.executor.apps.connect_catalog({});",
+  },
+});
+await client.callTool({
+  name: "execute",
+  arguments: {
+    code: `export default await tools.apps.user.appsdefault['issues-sync']({ repos: ["${owner}/app"] });`,
+  },
+});
+await client.close();
+log("published daily-brief + populated issues");
+
+// --- serve a wrapper that injects the bearer for sunpeak --------------------
+const server = Bun.serve({
+  port: PORT,
+  hostname: "127.0.0.1",
+  idleTimeout: 0,
+  async fetch(request) {
+    const url = new URL(request.url);
+    if (url.pathname === "/health") return new Response("ok");
+    const headers = new Headers(request.headers);
+    headers.set("authorization", `Bearer ${token}`);
+    return handler(new Request(request, { headers }));
+  },
+});
+log(`serving /mcp for sunpeak at http://127.0.0.1:${server.port}/mcp`);
+
+const shutdown = async () => {
+  try {
+    server.stop(true);
+  } catch {
+    /* ignore */
+  }
+  try {
+    await app.dispose();
+  } catch {
+    /* ignore */
+  }
+  try {
+    await github.close();
+  } catch {
+    /* ignore */
+  }
+  process.exit(0);
+};
+process.on("SIGINT", shutdown);
+process.on("SIGTERM", shutdown);

apps/host-selfhost/src/app.ts

@@ -19,6 +19,7 @@ import {
   SelfHostPluginsProvider,
 } from "./execution";
 import { makeSelfHostMcpSeams } from "./mcp";
+import { getSelfHostAppsSubsystem } from "./apps";
 import { selfHostPlugins } from "./plugins";
 import { ErrorCaptureLive } from "./observability";
 import { oauthCallbackSignInRedirectLocation } from "./auth/oauth-callback-login";
@@ -68,10 +69,52 @@ export const makeSelfHostApp = async (options: MakeSelfHostAppOptions = {}) => {
   // API + MCP OAuth seam, all over the shared libSQL handle.
   const { identityLayer, authHandler, betterAuth } = await resolveAuthProviders(dbHandle);
 
+  // ---- the apps subsystem (custom tools / workflows / ui / skills) -------
+  // Built over the five self-hosted seam backings rooted at the data dir (a
+  // boot-time singleton, same instance the source plugin in executor.config.ts
+  // closes over). Its HTTP surface mounts under /api/apps/*; published tools are
+  // catalog citizens through its source plugin; its extra MCP surface (publish
+  // door, skills, ui:// resources) is registered on the real MCP server below.
+  //
+  // The HTTP surface (publish/invoke/workflows/ui/SSE) mutates per-scope state
+  // and reaches real integrations, so it is authenticated with the SAME Better
+  // Auth credential the rest of `/api` requires. We resolve a session from the
+  // same three credential shapes the identity seam accepts (session cookie,
+  // Bearer session token, Bearer value retried as an x-api-key), matching
+  // `betterAuthIdentityLayer` so a token that works on `/api` works here too.
+  const appsAuthenticate = async (request: Request): Promise<boolean> => {
+    const auth = betterAuth.auth;
+    const session = await auth.api.getSession({ headers: request.headers }).catch(() => null);
+    if (session) return true;
+    const authorization = request.headers.get("authorization");
+    const token =
+      authorization && authorization.toLowerCase().startsWith("bearer ")
+        ? authorization.slice(7).trim()
+        : undefined;
+    if (!token) return false;
+    const apiKeySession = await auth.api
+      .getSession({ headers: new Headers({ "x-api-key": token }) })
+      .catch(() => null);
+    return apiKeySession != null;
+  };
+  const apps = getSelfHostAppsSubsystem({
+    authenticate: appsAuthenticate,
+    webBaseUrl: config.webBaseUrl,
+  });
+
   // ---- the in-process MCP serving seams (+ shutdown hook) ----------------
   // Pass the pinned public origin so browser-approval URLs are reachable behind
-  // a reverse proxy (not the internal 127.0.0.1 bind from the request URL).
-  const mcp = makeSelfHostMcpSeams(dbHandle, betterAuth, config.webBaseUrl);
+  // a reverse proxy (not the internal 127.0.0.1 bind from the request URL). The
+  // `onServer` hook registers the apps subsystem's non-catalog MCP surface
+  // (publish/skills/ui) on each per-session MCP server, so publish-over-MCP,
+  // skills list/read, and ui:// resources are LIVE on the real endpoint.
+  const mcp = makeSelfHostMcpSeams(dbHandle, betterAuth, config.webBaseUrl, (server) =>
+    // The real MCP SDK server registers with zod schema shapes; the apps
+    // registrar's structural `McpServerLike` uses a plain-object schema view. The
+    // shapes are runtime-compatible (the registrar passes zod raw shapes); the
+    // cast bridges the SDK's narrower generic signature.
+    apps.registerMcp(server as unknown as Parameters<typeof apps.registerMcp>[0]),
+  );
 
   // CLI device-login discovery (`executor login`). Points the CLI at Better
   // Auth's device endpoints; `requestFormat: "json"` because those endpoints
@@ -119,6 +162,9 @@ export const makeSelfHostApp = async (options: MakeSelfHostAppOptions = {}) => {
         makeSelfHostAdminApiLayer({ betterAuth, db: dbHandle, mountPrefix: "/api" }),
         // Public system API: /api/health + /api/setup-status (unauthenticated).
         makeSelfHostSystemApiLayer({ betterAuth, db: dbHandle, mountPrefix: "/api" }),
+        // The apps subsystem HTTP surface: publish / invoke / ui bundle / SSE /
+        // workflow lifecycle under /api/apps/*.
+        HttpRouter.add("*", "/api/apps/*", HttpEffect.fromWebHandler(apps.http.handler)),
         // Swagger UI at /docs, over the /api-prefixed spec (matches the served paths).
         HttpApiSwagger.layer(composePluginApi(selfHostPlugins).prefix("/api"), { path: "/docs" }),
       ],
@@ -139,6 +185,7 @@ export const makeSelfHostApp = async (options: MakeSelfHostAppOptions = {}) => {
     toWebHandler,
     betterAuth,
     closeDb: async () => {
+      await apps.close();
       await mcp.close();
       await dbHandle.close();
     },

apps/host-selfhost/src/apps-resolver.test.ts

@@ -0,0 +1,71 @@
+import { describe, expect, it } from "vitest";
+import { Effect, Layer } from "effect";
+import { HttpClient } from "effect/unstable/http";
+
+import { makeCtxResolver } from "./apps-resolver";
+
+// ---------------------------------------------------------------------------
+// Finding 4 regression: a missing/misnamed credential binding must fail with a
+// typed BindingError naming the role + surface, and must make NO upstream call.
+// Before the fix the resolver fell back to `conns[0]` and dispatched the request
+// with SOME OTHER connection's credential.
+//
+// We build a fake ctx whose connection list does NOT contain the requested name,
+// with an HttpClient that flags if it is ever invoked. The assertion: a
+// BindingError, and the http client was never touched (the "emulator ledger" is
+// empty).
+// ---------------------------------------------------------------------------
+
+describe("apps ClientResolver missing-binding (Fix 4)", () => {
+  it("fails typed and makes no upstream call when the bound connection is absent", async () => {
+    let httpCalled = false;
+    // A stub HttpClient layer that records any dispatch. If the resolver fell
+    // back to conns[0] it would build a request and call `execute`.
+    const httpLayer = Layer.succeed(HttpClient.HttpClient)({
+      execute: () => {
+        httpCalled = true;
+        return Effect.die("http should never be called for a missing binding");
+      },
+    } as unknown as HttpClient.HttpClient);
+
+    // The ctx exposes a DIFFERENT connection than the one the binding names.
+    const ctx = {
+      httpClientLayer: httpLayer,
+      connections: {
+        list: () =>
+          Effect.succeed([
+            {
+              owner: "user",
+              name: "some-other-connection",
+              integration: "github",
+              config: { baseUrl: "http://127.0.0.1:1/emulator" },
+            },
+          ]),
+        resolveValue: () => Effect.succeed("tok"),
+      },
+      core: { integrations: { get: () => Effect.succeed(null) } },
+    };
+
+    const resolver = makeCtxResolver(ctx);
+
+    const exit = await Effect.runPromiseExit(
+      resolver.call({
+        integration: "github",
+        // The binding names a connection that does NOT exist in the list.
+        connection: "the-bound-one",
+        path: ["repos", "listForAuthenticatedUser"],
+        args: [{}],
+      }),
+    );
+
+    expect(exit._tag).toBe("Failure");
+    // The typed BindingError names the role + surface and explains the refusal.
+    const serialized = JSON.stringify(exit);
+    expect(serialized).toContain("BindingError");
+    expect(serialized).toContain("the-bound-one");
+    expect(serialized).toContain("refusing to");
+
+    // No upstream call was made (the ledger stayed empty).
+    expect(httpCalled).toBe(false);
+  });
+});