Skip to content

Commit dc20f17

Browse files
ericallamericallam
committed
perf(webapp,run-engine,database): resolve the newest worker and deployment by createdAt
Ordering these lookups by id makes Postgres scan the primary key backwards betting on an early match for the target environment. When that bet loses the scan crosses a large part of the index. Ordering by createdAt then id keeps the lookup on a composite index instead. Adds (environmentId, createdAt) on WorkerDeployment, which had no matching index. BackgroundWorker already had one.
1 parent fc69101 commit dc20f17

9 files changed

Lines changed: 261 additions & 20 deletions

File tree

‎.server-changes/worker-deployment-lookup-ordering.md‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
---
2+
area: webapp
3+
type: improvement
4+
---
5+
6+
Speeds up resolving the latest worker version and deployment for an environment, removing an occasional stall when triggering runs in projects that have accumulated many deployed versions.

‎apps/webapp/app/services/runsRepository/clickhouseRunsRepository.server.ts‎

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -271,9 +271,6 @@ export class ClickHouseRunsRepository implements IRunsRepository {
271271
in: ids,
272272
},
273273
},
274-
orderBy: {
275-
id: "desc",
276-
},
277274
select: {
278275
id: true,
279276
friendlyId: true,

‎apps/webapp/app/v3/models/workerDeployment.server.ts‎

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -133,9 +133,7 @@ export async function findCurrentWorkerDeployment({
133133
environmentId,
134134
type,
135135
},
136-
orderBy: {
137-
id: "desc",
138-
},
136+
orderBy: [{ createdAt: "desc" }, { id: "desc" }],
139137
select: {
140138
id: true,
141139
imageReference: true,

‎apps/webapp/app/v3/runOpsMigration/controlPlaneResolver.server.ts‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -432,7 +432,7 @@ export class ControlPlaneResolver {
432432
// MANAGED deployment.
433433
const latestV2Deployment = await client.workerDeployment.findFirst({
434434
where: { environmentId, type: "MANAGED" },
435-
orderBy: { id: "desc" },
435+
orderBy: [{ createdAt: "desc" }, { id: "desc" }],
436436
include: { worker: { include: { tasks: true, queues: true } } },
437437
});
438438

@@ -455,7 +455,6 @@ export class ControlPlaneResolver {
455455
const worker = await client.backgroundWorker.findFirst({
456456
where: { id: workerId },
457457
include: { deployment: true, tasks: true, queues: true },
458-
orderBy: { id: "desc" },
459458
});
460459

461460
if (!worker) {
@@ -472,7 +471,7 @@ export class ControlPlaneResolver {
472471
const worker = await client.backgroundWorker.findFirst({
473472
where: { runtimeEnvironmentId: environmentId },
474473
include: { tasks: true, queues: true },
475-
orderBy: { id: "desc" },
474+
orderBy: [{ createdAt: "desc" }, { id: "desc" }],
476475
});
477476

478477
if (!worker) {

‎apps/webapp/test/v3/runOpsMigration/controlPlaneResolver.server.test.ts‎

Lines changed: 109 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -62,7 +62,12 @@ async function seedControlPlane(prisma: PrismaClient) {
6262
async function seedWorker(
6363
prisma: PrismaClient,
6464
ctx: { projectId: string; environmentId: string },
65-
opts?: { promote?: boolean }
65+
opts?: {
66+
promote?: boolean;
67+
createdAt?: Date;
68+
deploymentType?: "MANAGED" | "UNMANAGED" | "V1";
69+
deploymentCreatedAt?: Date;
70+
}
6671
) {
6772
const n = seedCounter++;
6873
const worker = await prisma.backgroundWorker.create({
@@ -74,6 +79,7 @@ async function seedWorker(
7479
version: `2024.1.${n}`,
7580
metadata: {},
7681
engine: "V2",
82+
...(opts?.createdAt ? { createdAt: opts.createdAt } : {}),
7783
},
7884
});
7985
const task = await prisma.backgroundWorkerTask.create({
@@ -104,11 +110,12 @@ async function seedWorker(
104110
contentHash: `hash_${n}`,
105111
version: worker.version,
106112
shortCode: `dep_${n}`,
107-
type: "MANAGED",
113+
type: opts?.deploymentType ?? "MANAGED",
108114
status: "DEPLOYED",
109115
projectId: ctx.projectId,
110116
environmentId: ctx.environmentId,
111117
workerId: worker.id,
118+
...(opts?.deploymentCreatedAt ? { createdAt: opts.deploymentCreatedAt } : {}),
112119
},
113120
});
114121
await prisma.workerDeploymentPromotion.create({
@@ -748,3 +755,103 @@ heteroPostgresTest(
748755
expect(reads()).toBe(readsAfterFirst * 2);
749756
}
750757
);
758+
759+
heteroPostgresTest(
760+
"resolveWorkerVersion (DEVELOPMENT) resolves the newest worker by createdAt, not by id",
761+
async ({ prisma14 }) => {
762+
const { environment, project } = await seedControlPlane(prisma14);
763+
const ctx = { projectId: project.id, environmentId: environment.id };
764+
765+
const newest = await seedWorker(prisma14, ctx, {
766+
createdAt: new Date("2026-07-31T12:00:00.000Z"),
767+
});
768+
const oldest = await seedWorker(prisma14, ctx, {
769+
createdAt: new Date("2026-07-30T12:00:00.000Z"),
770+
});
771+
772+
expect(oldest.worker.id > newest.worker.id).toBe(true);
773+
expect(oldest.worker.createdAt < newest.worker.createdAt).toBe(true);
774+
775+
const resolver = new ControlPlaneResolver({
776+
controlPlaneReplica: prisma14,
777+
controlPlanePrimary: prisma14,
778+
cache: new ControlPlaneCache(),
779+
splitEnabled: () => true,
780+
});
781+
782+
const resolved = await resolver.resolveWorkerVersion({
783+
environmentId: environment.id,
784+
type: "DEVELOPMENT",
785+
});
786+
787+
expect(resolved).not.toBeNull();
788+
expect(resolved!.worker.id).toBe(newest.worker.id);
789+
}
790+
);
791+
792+
heteroPostgresTest(
793+
"resolveWorkerVersion latest-MANAGED fallback resolves by createdAt, not by id",
794+
async ({ prisma14 }) => {
795+
const { environment, project } = await seedControlPlane(prisma14);
796+
const ctx = { projectId: project.id, environmentId: environment.id };
797+
798+
await seedWorker(prisma14, ctx, { promote: true, deploymentType: "V1" });
799+
800+
const newest = await seedWorker(prisma14, ctx, {
801+
promote: false,
802+
deploymentCreatedAt: new Date("2026-07-31T12:00:00.000Z"),
803+
});
804+
const oldest = await seedWorker(prisma14, ctx, {
805+
promote: false,
806+
deploymentCreatedAt: new Date("2026-07-30T12:00:00.000Z"),
807+
});
808+
809+
const newestDeployment = await prisma14.workerDeployment.create({
810+
data: {
811+
friendlyId: `deployment_newest_${environment.id}`,
812+
contentHash: "hash_newest",
813+
version: newest.worker.version,
814+
shortCode: "dep_newest",
815+
type: "MANAGED",
816+
status: "DEPLOYED",
817+
projectId: project.id,
818+
environmentId: environment.id,
819+
workerId: newest.worker.id,
820+
createdAt: new Date("2026-07-31T12:00:00.000Z"),
821+
},
822+
});
823+
const oldestDeployment = await prisma14.workerDeployment.create({
824+
data: {
825+
friendlyId: `deployment_oldest_${environment.id}`,
826+
contentHash: "hash_oldest",
827+
version: oldest.worker.version,
828+
shortCode: "dep_oldest",
829+
type: "MANAGED",
830+
status: "DEPLOYED",
831+
projectId: project.id,
832+
environmentId: environment.id,
833+
workerId: oldest.worker.id,
834+
createdAt: new Date("2026-07-30T12:00:00.000Z"),
835+
},
836+
});
837+
838+
expect(oldestDeployment.id > newestDeployment.id).toBe(true);
839+
expect(oldestDeployment.createdAt < newestDeployment.createdAt).toBe(true);
840+
841+
const resolver = new ControlPlaneResolver({
842+
controlPlaneReplica: prisma14,
843+
controlPlanePrimary: prisma14,
844+
cache: new ControlPlaneCache(),
845+
splitEnabled: () => true,
846+
});
847+
848+
const resolved = await resolver.resolveWorkerVersion({
849+
environmentId: environment.id,
850+
type: "PRODUCTION",
851+
});
852+
853+
expect(resolved).not.toBeNull();
854+
expect(resolved!.deployment!.id).toBe(newestDeployment.id);
855+
expect(resolved!.worker.id).toBe(newest.worker.id);
856+
}
857+
);

‎internal-packages/database/prisma/migrations/20260731160000_add_worker_deployment_environment_id_created_at_index/migration.sql‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
CREATE INDEX CONCURRENTLY IF NOT EXISTS "WorkerDeployment_environmentId_createdAt_idx" ON "public"."WorkerDeployment"("environmentId", "createdAt");

‎internal-packages/database/prisma/schema.prisma‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2208,6 +2208,7 @@ model WorkerDeployment {
22082208
@@unique([projectId, shortCode])
22092209
@@unique([environmentId, version])
22102210
@@index([commitSHA])
2211+
@@index([environmentId, createdAt])
22112212
}
22122213

22132214
enum WorkerDeploymentStatus {

‎internal-packages/run-engine/src/engine/controlPlaneResolver.ts‎

Lines changed: 2 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -234,9 +234,7 @@ export class PassthroughControlPlaneResolver implements ControlPlaneResolver {
234234
tasks: true,
235235
queues: true,
236236
},
237-
orderBy: {
238-
id: "desc",
239-
},
237+
orderBy: [{ createdAt: "desc" }, { id: "desc" }],
240238
});
241239

242240
if (!worker) {
@@ -256,9 +254,6 @@ export class PassthroughControlPlaneResolver implements ControlPlaneResolver {
256254
tasks: true,
257255
queues: true,
258256
},
259-
orderBy: {
260-
id: "desc",
261-
},
262257
});
263258

264259
if (!worker) {
@@ -315,9 +310,7 @@ export class PassthroughControlPlaneResolver implements ControlPlaneResolver {
315310
environmentId,
316311
type: "MANAGED",
317312
},
318-
orderBy: {
319-
id: "desc",
320-
},
313+
orderBy: [{ createdAt: "desc" }, { id: "desc" }],
321314
include: {
322315
worker: {
323316
include: {

‎internal-packages/run-engine/src/engine/tests/dequeueSystem.controlPlaneResolver.test.ts‎

Lines changed: 139 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -538,3 +538,142 @@ describe("DequeueSystem controlPlaneResolver (single-DB passthrough)", () => {
538538
}
539539
);
540540
});
541+
542+
heteroPostgresTest(
543+
"resolveWorkerVersion (DEVELOPMENT) resolves the newest worker by createdAt, not by id",
544+
async ({ prisma14 }) => {
545+
const cp = await seedControlPlane(
546+
prisma14 as unknown as PrismaClient,
547+
"cpord",
548+
"ordering-task"
549+
);
550+
551+
await prisma14.backgroundWorker.update({
552+
where: { id: cp.worker.id },
553+
data: { createdAt: new Date("2026-07-01T12:00:00.000Z") },
554+
});
555+
556+
const newest = await prisma14.backgroundWorker.create({
557+
data: {
558+
friendlyId: generateFriendlyId("worker"),
559+
contentHash: "hash_newest",
560+
projectId: cp.project.id,
561+
runtimeEnvironmentId: cp.environment.id,
562+
version: "20260731.1",
563+
metadata: {},
564+
engine: "V2",
565+
createdAt: new Date("2026-07-31T12:00:00.000Z"),
566+
},
567+
});
568+
const oldest = await prisma14.backgroundWorker.create({
569+
data: {
570+
friendlyId: generateFriendlyId("worker"),
571+
contentHash: "hash_oldest",
572+
projectId: cp.project.id,
573+
runtimeEnvironmentId: cp.environment.id,
574+
version: "20260730.1",
575+
metadata: {},
576+
engine: "V2",
577+
createdAt: new Date("2026-07-30T12:00:00.000Z"),
578+
},
579+
});
580+
581+
expect(oldest.id > newest.id).toBe(true);
582+
expect(oldest.createdAt < newest.createdAt).toBe(true);
583+
584+
const resolver = new PassthroughControlPlaneResolver({
585+
prisma: prisma14 as unknown as PrismaClient,
586+
});
587+
588+
const resolved = await resolver.resolveWorkerVersion({
589+
environmentId: cp.environment.id,
590+
type: "DEVELOPMENT",
591+
});
592+
593+
assertNonNullable(resolved);
594+
expect(resolved.worker.id).toBe(newest.id);
595+
}
596+
);
597+
598+
heteroPostgresTest(
599+
"resolveWorkerVersion latest-MANAGED fallback resolves by createdAt, not by id",
600+
async ({ prisma14 }) => {
601+
const cp = await seedControlPlane(
602+
prisma14 as unknown as PrismaClient,
603+
"cpfall",
604+
"fallback-task"
605+
);
606+
607+
await prisma14.workerDeployment.update({
608+
where: { id: cp.deployment.id },
609+
data: { type: "V1" },
610+
});
611+
612+
const newestWorker = await prisma14.backgroundWorker.create({
613+
data: {
614+
friendlyId: generateFriendlyId("worker"),
615+
contentHash: "hash_newest",
616+
projectId: cp.project.id,
617+
runtimeEnvironmentId: cp.environment.id,
618+
version: "20260731.1",
619+
metadata: {},
620+
engine: "V2",
621+
},
622+
});
623+
const oldestWorker = await prisma14.backgroundWorker.create({
624+
data: {
625+
friendlyId: generateFriendlyId("worker"),
626+
contentHash: "hash_oldest",
627+
projectId: cp.project.id,
628+
runtimeEnvironmentId: cp.environment.id,
629+
version: "20260730.1",
630+
metadata: {},
631+
engine: "V2",
632+
},
633+
});
634+
635+
const newestDeployment = await prisma14.workerDeployment.create({
636+
data: {
637+
friendlyId: generateFriendlyId("deployment"),
638+
contentHash: "hash_newest",
639+
version: "20260731.1",
640+
shortCode: "short_code_newest",
641+
status: "DEPLOYED",
642+
projectId: cp.project.id,
643+
environmentId: cp.environment.id,
644+
workerId: newestWorker.id,
645+
type: "MANAGED",
646+
createdAt: new Date("2026-07-31T12:00:00.000Z"),
647+
},
648+
});
649+
const oldestDeployment = await prisma14.workerDeployment.create({
650+
data: {
651+
friendlyId: generateFriendlyId("deployment"),
652+
contentHash: "hash_oldest",
653+
version: "20260730.1",
654+
shortCode: "short_code_oldest",
655+
status: "DEPLOYED",
656+
projectId: cp.project.id,
657+
environmentId: cp.environment.id,
658+
workerId: oldestWorker.id,
659+
type: "MANAGED",
660+
createdAt: new Date("2026-07-30T12:00:00.000Z"),
661+
},
662+
});
663+
664+
expect(oldestDeployment.id > newestDeployment.id).toBe(true);
665+
expect(oldestDeployment.createdAt < newestDeployment.createdAt).toBe(true);
666+
667+
const resolver = new PassthroughControlPlaneResolver({
668+
prisma: prisma14 as unknown as PrismaClient,
669+
});
670+
671+
const resolved = await resolver.resolveWorkerVersion({
672+
environmentId: cp.environment.id,
673+
type: "PRODUCTION",
674+
});
675+
676+
assertNonNullable(resolved);
677+
expect(resolved.deployment?.id).toBe(newestDeployment.id);
678+
}
679+
);

0 commit comments

Comments
 (0)