Fix nectar dormancy: live-reload projects/credentials so post-boot bindings activate without restart

src/cli.ts

@@ -62,6 +62,7 @@ import type { HiveGraphStore, AsyncHiveGraphStore } from "./hive-graph/store.js"
 import { DeepLakeHiveGraphStore } from "./hive-graph/deeplake-store.js";
 import { InMemoryHiveGraphStore } from "./hive-graph/memory-store.js";
 import { loadDeepLakeCredentials, credentialsPath } from "./hive-graph/deeplake-credentials.js";
+import { LiveActiveProjects } from "./hive-graph/live-active-projects.js";
 import { broodPrereqsFromEnv, formatFirstRunGuidance } from "./brood-prereqs.js";
 import { resolveNectarTunables } from "./config-file.js";
 import { resolveProjectScope, type ProjectScopeSource } from "./hive-graph/project-scope.js";
@@ -1303,33 +1304,56 @@ async function runDaemon(): Promise<void> {
   // factory stands up one brood + watch context per active project, each rooted
   // at its own bound path and scoped to its own tenancy project id. Live brood
   // (describe) deps are attached only when Portkey is configured.
-  const controlOptions: ProjectsControlOptions =
-    creds !== undefined ? { expect: { org: creds.orgId, workspace: creds.workspaceId } } : {};
-  const broodDeps =
-    creds !== undefined && portkey.enabled
-      ? { portkey: portkey as PortkeyEnabled, embedProvider, embedModelId: embedModel }
-      : undefined;
-
-  const activeProjects =
-    durableStore !== undefined && creds !== undefined
-      ? {
-          resolve: () => readActiveProjects(controlOptions).resolution,
-          factory: (project: ResolvedProject): RunningContext =>
-            createProjectContext({
-              project,
-              tenancy: { orgId: creds.orgId, workspaceId: creds.workspaceId, projectId: project.projectId },
-              store: durableStore,
-              ...(broodDeps !== undefined ? { broodDeps } : {}),
-              log: (line) => process.stderr.write(`${JSON.stringify({ ts: new Date().toISOString(), ...line })}\n`),
-            }),
-        }
-      : undefined;
+  //
+  // W1-N remediation (live-reload of bindings/credentials without a restart):
+  // the seam is wired UNCONDITIONALLY (not gated on a boot-time credentials
+  // snapshot) and delegates to `LiveActiveProjects`, which re-resolves
+  // `~/.deeplake/credentials.json` on EVERY reconcile tick. So a daemon that
+  // booted BEFORE login, then had a project bound afterward, activates that
+  // project on its next tick (or immediately, when the credentials watch fires a
+  // reconcile the moment credentials appear) - no daemon restart. The tenancy
+  // `expect` guard is derived fresh from the currently-resolved credentials, so
+  // it is never frozen at boot. Absent creds -> an empty resolution (dormant).
+  const liveControlOptions: Omit<ProjectsControlOptions, "expect"> = {};
+  const liveActive = new LiveActiveProjects({
+    loadCredentials: () => {
+      try {
+        return loadDeepLakeCredentials();
+      } catch {
+        return undefined;
+      }
+    },
+    createStore: (credentials) => new DeepLakeHiveGraphStore({ credentials }),
+    buildContext: ({ project, credentials, store }): RunningContext =>
+      createProjectContext({
+        project,
+        tenancy: { orgId: credentials.orgId, workspaceId: credentials.workspaceId, projectId: project.projectId },
+        store,
+        ...(portkey.enabled
+          ? { broodDeps: { portkey: portkey as PortkeyEnabled, embedProvider, embedModelId: embedModel } }
+          : {}),
+        log: (line) => process.stderr.write(`${JSON.stringify({ ts: new Date().toISOString(), ...line })}\n`),
+      }),
+    controlOptions: liveControlOptions,
+    onError: (err) =>
+      process.stderr.write(
+        `nectar daemon: active-project resolve failed (non-fatal): ${err instanceof Error ? err.message : String(err)}\n`,
+      ),
+  });
+  const activeProjects = {
+    resolve: () => liveActive.resolve(),
+    factory: (project: ResolvedProject): RunningContext => liveActive.factory(project),
+  };
 
-  // The daemon-level enricher stays scoped to the PRIMARY active project at boot
-  // (the common single-project case); per-project enrichment for additional
-  // active projects is a documented follow-up. Absent creds/Portkey/active set
-  // -> the enricher stays dormant (honest).
-  const primary = durableStore !== undefined ? readActiveProjects(controlOptions).resolution.active[0] : undefined;
+  // The daemon-level enricher + hive-graph API stay scoped to the PRIMARY active
+  // project resolved at boot (the common single-project case); per-project
+  // enrichment for additional active projects is a documented follow-up. These
+  // read the BOOT-time tenancy snapshot (creds at boot); the multi-root brood +
+  // watch supervisor above is the credentials-live path that removes the dormancy
+  // bug. Absent creds/Portkey/active set -> the enricher stays dormant (honest).
+  const bootControlOptions: ProjectsControlOptions =
+    creds !== undefined ? { expect: { org: creds.orgId, workspace: creds.workspaceId } } : {};
+  const primary = durableStore !== undefined ? readActiveProjects(bootControlOptions).resolution.active[0] : undefined;
   const primaryTenancy: Tenancy | undefined =
     primary !== undefined && creds !== undefined
       ? { orgId: creds.orgId, workspaceId: creds.workspaceId, projectId: primary.projectId }
@@ -1348,13 +1372,17 @@ async function runDaemon(): Promise<void> {
     // pre-login fleet posture is honest; the watch below flips it healthy the
     // moment credentials appear (a-AC-2), no restart needed.
     //
-    // ACCEPTED DEVIATION (W1-N, ledger 2026-07-04): credentials appearing after
-    // boot flip /health healthy and reconcile active projects, but the
-    // creds-dependent subsystems assembled above from this boot-time snapshot
-    // (the durable store, brood wiring, enricher loop) are NOT hot-attached;
-    // they stay dormant until the next daemon restart. Same posture as
-    // honeycomb. If hot-attach ever lands, it belongs in the credentialsWatch
-    // onChange path in daemon.ts.
+    // W1-N remediation (live-reload without a daemon restart): the multi-root
+    // brood + watch supervisor (the `activeProjects` seam below) now re-resolves
+    // credentials + `projects.json` on EVERY reconcile tick via
+    // `LiveActiveProjects`, so credentials appearing after boot (and any project
+    // bound afterward) DO hot-attach - the daemon starts brooding/watching the
+    // newly-active project on its next tick, or immediately when the credentials
+    // watch fires `reconcileActiveProjects`. Remaining boot-snapshot scope: the
+    // PRIMARY-project enricher loop + hive-graph API assembled below still read
+    // the boot-time creds (a documented single-project follow-up), so those two
+    // subsystems alone stay dormant until the next restart when creds arrive
+    // late. The dormancy-until-restart bug (no brooding at all) is fixed.
     storageCredentialsPresent: creds !== undefined,
     credentialsWatch: {
       probe: () => {
@@ -1366,7 +1394,7 @@ async function runDaemon(): Promise<void> {
         }
       },
     },
-    ...(activeProjects !== undefined ? { activeProjects } : {}),
+    activeProjects,
     ...(live !== undefined && primaryTenancy !== undefined && primary !== undefined
       ? {
           tenancy: primaryTenancy,
@@ -1400,17 +1428,17 @@ async function runDaemon(): Promise<void> {
   // PRD-019b: the projects + brooding-control endpoints (consumed by Hive's
   // dashboard). Persisting a toggle triggers an immediate active-set reconcile.
   try {
-    if (activeProjects !== undefined) {
+    {
       mountProjectsApi(daemon, {
         view: () => {
-          const { resolution, cache } = readActiveProjects(controlOptions);
+          const { resolution, cache } = readActiveProjects(bootControlOptions);
           return buildProjectsView(resolution, cache, (id) => daemonWatcherState(daemon, id));
         },
         setProject: (projectId, brooding) => {
-          persistProjectBrooding(projectId, brooding, controlOptions);
+          persistProjectBrooding(projectId, brooding, bootControlOptions);
         },
         setGlobal: (global) => {
-          persistGlobalBrooding(global, controlOptions);
+          persistGlobalBrooding(global, bootControlOptions);
         },
         reconcile: () => daemon.reconcileActiveProjects(),
         // b-AC-6: the HTTP body carries only this path + a stable reason; the

test/live-active-projects.test.ts

@@ -0,0 +1,233 @@
+/**
+ * W1-N remediation: live-reload of project bindings + credentials WITHOUT a
+ * daemon restart. Proves the credentials-live active-project resolver
+ * (`LiveActiveProjects`) picks up a `projects.json` binding written AFTER "boot"
+ * on a subsequent resolve, and that the whole path stays dormant + fail-soft
+ * while credentials are absent.
+ *
+ * Hermetic: a temp `~/.deeplake` cache dir + a temp brooding-state dir + an
+ * injected (mutable) credentials seam. No real home, no network, no fs.watch,
+ * no real sleeps - the "next tick" is just a second `resolve()` call. Runs
+ * against the compiled `dist/` output.
+ */
+import { test } from "node:test";
+import assert from "node:assert/strict";
+import { mkdtempSync, rmSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { LiveActiveProjects } from "../dist/hive-graph/live-active-projects.js";
+import { assembleDaemon } from "../dist/index.js";
+import type { DeepLakeCredentials } from "../dist/hive-graph/deeplake-credentials.js";
+import type { AsyncHiveGraphStore } from "../dist/hive-graph/store.js";
+import type { ResolvedProject } from "../dist/hive-graph/active-projects.js";
+import type { RunningContext } from "../dist/project-supervisor.js";
+import { get } from "node:http";
+
+const CREDS: DeepLakeCredentials = {
+  apiUrl: "https://api.deeplake.test",
+  token: "tok-abcd",
+  orgId: "org-1",
+  workspaceId: "ws-1",
+};
+
+/** A trivial async store stand-in (the resolver never queries it; the supervisor holds it). */
+const fakeStore = {} as unknown as AsyncHiveGraphStore;
+
+/** Write a schema-v1 projects.json binding `path -> projectId` into `dir/projects.json`. */
+function writeProjectsCache(dir: string, org: string, workspace: string, bindings: Array<{ path: string; projectId: string }>): void {
+  writeFileSync(
+    join(dir, "projects.json"),
+    JSON.stringify({ schemaVersion: 1, org, workspace, bindings, projects: [] }),
+    "utf8",
+  );
+}
+
+function tempDir(prefix: string): string {
+  return mkdtempSync(join(tmpdir(), prefix));
+}
+
+test("W1-N: a projects.json binding written AFTER boot is picked up on the next resolve (no restart)", () => {
+  const cacheDir = tempDir("nectar-live-cache-");
+  const stateDir = tempDir("nectar-live-state-");
+  const builtFor: string[] = [];
+  // Credentials are PRESENT here (the daemon booted after login); the point of
+  // this test is that a BINDING added after boot activates without a restart.
+  const live = new LiveActiveProjects({
+    loadCredentials: () => CREDS,
+    createStore: () => fakeStore,
+    buildContext: ({ project }): RunningContext => {
+      builtFor.push(project.projectId);
+      return { projectId: project.projectId, path: project.path, watcherState: () => "running", start: async () => {}, stop: async () => {} };
+    },
+    controlOptions: { cacheDir, broodingState: { dir: stateDir } },
+  });
+
+  try {
+    // Boot: no bindings yet -> dormant.
+    writeProjectsCache(cacheDir, CREDS.orgId, CREDS.workspaceId, []);
+    assert.equal(live.resolve().active.length, 0, "no bindings => nothing active at boot");
+
+    // A project is bound AFTER boot (honeycomb writes projects.json).
+    writeProjectsCache(cacheDir, CREDS.orgId, CREDS.workspaceId, [{ path: "/work/repo-a", projectId: "proj-a" }]);
+
+    // The NEXT resolve (the reconcile loop's next tick) sees the new binding.
+    const after = live.resolve();
+    assert.equal(after.active.length, 1, "the post-boot binding is now active");
+    assert.equal(after.active[0].projectId, "proj-a");
+    assert.equal(after.active[0].path, "/work/repo-a");
+
+    // The factory builds a context for it against the live credentials.
+    const ctx = live.factory(after.active[0]);
+    assert.equal(ctx.projectId, "proj-a");
+    assert.deepEqual(builtFor, ["proj-a"]);
+  } finally {
+    rmSync(cacheDir, { recursive: true, force: true });
+    rmSync(stateDir, { recursive: true, force: true });
+  }
+});
+
+test("W1-N: with credentials absent the resolver is dormant and fail-soft (empty resolution, factory is a no-op)", async () => {
+  const cacheDir = tempDir("nectar-live-cache-");
+  const stateDir = tempDir("nectar-live-state-");
+  let present = false;
+  let buildContextCalls = 0;
+  const live = new LiveActiveProjects({
+    loadCredentials: () => (present ? CREDS : undefined),
+    createStore: () => fakeStore,
+    buildContext: (): RunningContext => {
+      buildContextCalls += 1;
+      throw new Error("buildContext must not be reached while credentials are absent");
+    },
+    controlOptions: { cacheDir, broodingState: { dir: stateDir } },
+  });
+
+  try {
+    // A binding exists on disk, but credentials are ABSENT -> dormant regardless.
+    writeProjectsCache(cacheDir, CREDS.orgId, CREDS.workspaceId, [{ path: "/work/repo-a", projectId: "proj-a" }]);
+    assert.equal(live.resolve().active.length, 0, "no credentials => dormant even with a binding present");
+
+    // A factory reached without creds (a race) yields a no-op context, never a throw.
+    const ctx = live.factory({ projectId: "proj-a", path: "/work/repo-a", brooding: "active" });
+    assert.equal(buildContextCalls, 0, "buildContext is skipped without credentials");
+    assert.equal(ctx.watcherState(), "stopped");
+    await ctx.start(); // must not throw
+    await ctx.stop();
+
+    // Credentials appear (login lands): the SAME resolver now activates the binding.
+    present = true;
+    const after = live.resolve();
+    assert.equal(after.active.length, 1, "credentials appearing activates the already-present binding");
+    assert.equal(after.active[0].projectId, "proj-a");
+  } finally {
+    rmSync(cacheDir, { recursive: true, force: true });
+    rmSync(stateDir, { recursive: true, force: true });
+  }
+});
+
+test("W1-N: a mismatched-tenancy cache reads empty (the expect guard is derived from the live credentials)", () => {
+  const cacheDir = tempDir("nectar-live-cache-");
+  const stateDir = tempDir("nectar-live-state-");
+  const live = new LiveActiveProjects({
+    loadCredentials: () => CREDS, // org-1 / ws-1
+    createStore: () => fakeStore,
+    buildContext: ({ project }): RunningContext => ({
+      projectId: project.projectId, path: project.path, watcherState: () => "running", start: async () => {}, stop: async () => {},
+    }),
+    controlOptions: { cacheDir, broodingState: { dir: stateDir } },
+  });
+  try {
+    // Cache synced for a DIFFERENT tenancy -> the guard drops it (wrong projects).
+    writeProjectsCache(cacheDir, "org-OTHER", "ws-OTHER", [{ path: "/work/repo-a", projectId: "proj-a" }]);
+    assert.equal(live.resolve().active.length, 0, "a tenancy-mismatched cache reads empty");
+
+    // Re-synced for the live tenancy -> activates (no restart, no reconstruction).
+    writeProjectsCache(cacheDir, CREDS.orgId, CREDS.workspaceId, [{ path: "/work/repo-a", projectId: "proj-a" }]);
+    assert.equal(live.resolve().active.length, 1, "the correctly-scoped cache activates");
+  } finally {
+    rmSync(cacheDir, { recursive: true, force: true });
+    rmSync(stateDir, { recursive: true, force: true });
+  }
+});
+
+// ── Integration: the daemon's reconcile loop drives LiveActiveProjects ────────
+
+function fetchHealth(port: number): Promise<{ status: number; body: any }> {
+  return new Promise((resolve, reject) => {
+    get({ host: "127.0.0.1", port, path: "/health" }, (res) => {
+      const chunks: Buffer[] = [];
+      res.on("data", (c: Buffer) => chunks.push(c));
+      res.on("end", () => resolve({ status: res.statusCode ?? 0, body: JSON.parse(Buffer.concat(chunks).toString("utf8")) }));
+    }).on("error", reject);
+  });
+}
+
+/** A timer that captures scheduled callbacks without firing them (reconcile is driven explicitly). */
+function inertTimer() {
+  return { set: (_fn: () => void, _ms: number) => 1, clear: (_h: unknown) => {} };
+}
+
+test("W1-N integration: a daemon booted dormant activates a post-boot binding on a driven reconcile tick", async () => {
+  const runtimeDir = tempDir("nectar-live-daemon-");
+  const cacheDir = tempDir("nectar-live-cache-");
+  const stateDir = tempDir("nectar-live-state-");
+  const events: string[] = [];
+  // Credentials appear AFTER boot (login lands late).
+  let present = false;
+
+  const live = new LiveActiveProjects({
+    loadCredentials: () => (present ? CREDS : undefined),
+    createStore: () => fakeStore,
+    buildContext: ({ project }): RunningContext => {
+      let state: "stopped" | "running" = "stopped";
+      return {
+        projectId: project.projectId,
+        path: project.path,
+        watcherState: () => state,
+        start: async () => {
+          events.push(`start:${project.projectId}`);
+          state = "running";
+        },
+        stop: async () => {
+          events.push(`stop:${project.projectId}`);
+          state = "stopped";
+        },
+      };
+    },
+    controlOptions: { cacheDir, broodingState: { dir: stateDir } },
+  });
+
+  const daemon = assembleDaemon({
+    port: 0,
+    runtimeDir,
+    log: () => {},
+    activeProjects: {
+      resolve: () => live.resolve(),
+      factory: (p: ResolvedProject) => live.factory(p),
+      timer: inertTimer(),
+    },
+  });
+
+  try {
+    const port = await daemon.start();
+    // Dormant at boot: no credentials on disk yet.
+    assert.equal((await fetchHealth(port)).body.activeProjects.count, 0);
+
+    // Login lands and honeycomb binds a project (both appear on disk).
+    present = true;
+    writeProjectsCache(cacheDir, CREDS.orgId, CREDS.workspaceId, [{ path: "/work/repo-a", projectId: "proj-a" }]);
+
+    // The reconcile loop's next tick (driven explicitly) activates the project -
+    // no daemon restart.
+    await daemon.reconcileActiveProjects();
+    assert.deepEqual(events, ["start:proj-a"]);
+    const health = await fetchHealth(port);
+    assert.equal(health.body.activeProjects.count, 1);
+    assert.equal(health.body.activeProjects.projects[0].projectId, "proj-a");
+    assert.equal(health.body.activeProjects.projects[0].watcher, "running");
+  } finally {
+    await daemon.shutdown();
+    rmSync(runtimeDir, { recursive: true, force: true });
+    rmSync(cacheDir, { recursive: true, force: true });
+    rmSync(stateDir, { recursive: true, force: true });
+  }
+});