diff --git a/src/pages/api/v1/builds/[uuid]/deploy.ts b/src/pages/api/v1/builds/[uuid]/deploy.ts index e09c7b84..8e4c3204 100644 --- a/src/pages/api/v1/builds/[uuid]/deploy.ts +++ b/src/pages/api/v1/builds/[uuid]/deploy.ts @@ -107,10 +107,9 @@ export default async (req: NextApiRequest, res: NextApiResponse) => { const buildId = build.id; const runUUID = nanoid(); - await buildService.resolveAndDeployBuildQueue.add('resolve-deploy', { + await buildService.enqueueResolveAndDeployBuild({ buildId, runUUID, - correlationId, }); getLogger({ stage: LogStage.BUILD_QUEUED }).info('Build: redeploy queued'); diff --git a/src/pages/api/v1/builds/[uuid]/index.ts b/src/pages/api/v1/builds/[uuid]/index.ts index 5f6990c7..974c0ebe 100644 --- a/src/pages/api/v1/builds/[uuid]/index.ts +++ b/src/pages/api/v1/builds/[uuid]/index.ts @@ -101,10 +101,9 @@ async function updateBuild(req: NextApiRequest, res: NextApiResponse, correlatio if (build.pullRequest?.deployOnUpdate) { getLogger({ stage: LogStage.BUILD_QUEUED }).info(`Triggering redeploy after UUID update`); - await new BuildService().resolveAndDeployBuildQueue.add('resolve-deploy', { + await new BuildService().enqueueResolveAndDeployBuild({ buildId: build.id, runUUID: nanoid(), - correlationId, }); } diff --git a/src/pages/api/v1/builds/[uuid]/services/[name]/build.ts b/src/pages/api/v1/builds/[uuid]/services/[name]/build.ts index 5fa019e8..85f34993 100644 --- a/src/pages/api/v1/builds/[uuid]/services/[name]/build.ts +++ b/src/pages/api/v1/builds/[uuid]/services/[name]/build.ts @@ -15,11 +15,7 @@ */ import { NextApiRequest, NextApiResponse } from 'next/types'; -import { withLogContext, getLogger, extractContextForQueue, LogStage } from 'server/lib/logger'; -import GithubService from 'server/services/github'; -import { Build } from 'server/models'; -import DeployService from 'server/services/deploy'; -import { DeployStatus } from 'shared/constants'; +import { withLogContext, getLogger, LogStage } from 'server/lib/logger'; import { nanoid } from 'nanoid'; import BuildService from 'server/services/build'; @@ -102,55 +98,40 @@ export default async (req: NextApiRequest, res: NextApiResponse) => { return withLogContext({ correlationId, buildUuid: uuid as string }, async () => { try { - const githubService = new GithubService(); - const build: Build = await githubService.db.models.Build.query() + const buildService = new BuildService(); + const build = await buildService.db.models.Build.query() .findOne({ uuid, }) .withGraphFetched('deploys.deployable'); - const buildId = build.id; - if (!build) { getLogger().debug(`Build not found`); return res.status(404).json({ error: `Build not found for ${uuid}` }); } - const deploy = build.deploys.find((deploy) => deploy.deployable.name === name); + const deploy = build.deploys?.find((candidate) => candidate.deployable?.name === name); - if (!deploy) { + if (!deploy?.deployable) { getLogger().debug(`Deployable not found: service=${name}`); res.status(404).json({ error: `${name} service is not found in ${uuid} build.` }); return; } - const githubRepositoryId = deploy.deployable.repositoryId; + const githubRepositoryId = + deploy.deployable.resolvedFromRepositoryId ?? deploy.deployable.repositoryId ?? deploy.githubRepositoryId; + if (githubRepositoryId == null) { + throw new Error(`Cannot redeploy ${name}: source repository is unknown.`); + } const runUUID = nanoid(); - const buildService = new BuildService(); - await buildService.resolveAndDeployBuildQueue.add('resolve-deploy', { - buildId, - githubRepositoryId, + await buildService.enqueueResolveAndDeployBuild({ + buildId: build.id, + githubRepositoryId: Number(githubRepositoryId), runUUID, - ...extractContextForQueue(), }); getLogger({ stage: LogStage.BUILD_QUEUED }).info(`Build: service redeploy queued service=${name}`); - - const deployService = new DeployService(); - - await deploy.$query().patchAndFetch({ - runUUID, - }); - - await deployService.patchAndUpdateActivityFeed( - deploy, - { - status: DeployStatus.QUEUED, - }, - runUUID, - githubRepositoryId - ); return res.status(200).json({ status: 'success', message: `Redeploy for service ${name} in build ${uuid} has been queued`, diff --git a/src/server/db/migrations/001_seed.ts b/src/server/db/migrations/001_seed.ts index 692aa5c5..236a38a7 100644 --- a/src/server/db/migrations/001_seed.ts +++ b/src/server/db/migrations/001_seed.ts @@ -340,6 +340,7 @@ export async function up(knex: Knex): Promise { "ingressAnnotations" json DEFAULT '{}'::json, "appShort" varchar(255) DEFAULT NULL::character varying, helm json DEFAULT '{}'::json, + "requires" text[] NOT NULL DEFAULT ARRAY[]::text[], "deploymentDependsOn" text[] DEFAULT ARRAY[]::text[], builder jsonb DEFAULT '{}'::jsonb, "ecr" varchar(255) default NULL::character varying, diff --git a/src/server/db/migrations/031_add_deployment_reconciliation.ts b/src/server/db/migrations/031_add_deployment_reconciliation.ts new file mode 100644 index 00000000..6b6a3608 --- /dev/null +++ b/src/server/db/migrations/031_add_deployment_reconciliation.ts @@ -0,0 +1,45 @@ +/** + * Copyright 2026 GoodRx, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { Knex } from 'knex'; + +const PENDING_INDEX = 'builds_reconcile_pending'; + +export async function up(knex: Knex): Promise { + await knex.schema.alterTable('builds', (table) => { + table.bigInteger('desiredGeneration').notNullable().defaultTo(0); + table.bigInteger('observedGeneration').notNullable().defaultTo(0); + table.jsonb('acceptedRefs').notNullable().defaultTo('{}'); + }); + + await knex.raw( + ` + create index ?? on builds (id) + where "desiredGeneration" > "observedGeneration" and "deletedAt" is null + `, + [PENDING_INDEX] + ); +} + +export async function down(knex: Knex): Promise { + await knex.raw('drop index if exists ??', [PENDING_INDEX]); + + await knex.schema.alterTable('builds', (table) => { + table.dropColumn('acceptedRefs'); + table.dropColumn('observedGeneration'); + table.dropColumn('desiredGeneration'); + }); +} diff --git a/src/server/db/migrations/032_add_requires_to_deployables.ts b/src/server/db/migrations/032_add_requires_to_deployables.ts new file mode 100644 index 00000000..1ffe116d --- /dev/null +++ b/src/server/db/migrations/032_add_requires_to_deployables.ts @@ -0,0 +1,33 @@ +/** + * Copyright 2026 GoodRx, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { Knex } from 'knex'; + +export async function up(knex: Knex): Promise { + if (await knex.schema.hasColumn('deployables', 'requires')) return; + + await knex.schema.alterTable('deployables', (table) => { + table.specificType('requires', 'text[]').notNullable().defaultTo(knex.raw('ARRAY[]::text[]')); + }); +} + +export async function down(knex: Knex): Promise { + if (!(await knex.schema.hasColumn('deployables', 'requires'))) return; + + await knex.schema.alterTable('deployables', (table) => { + table.dropColumn('requires'); + }); +} diff --git a/src/server/jobs/__tests__/apiEnvironmentJobWiring.test.ts b/src/server/jobs/__tests__/apiEnvironmentJobWiring.test.ts index 9b74a463..d5c8440f 100644 --- a/src/server/jobs/__tests__/apiEnvironmentJobWiring.test.ts +++ b/src/server/jobs/__tests__/apiEnvironmentJobWiring.test.ts @@ -68,7 +68,9 @@ describe('bootstrapJobs API-environment wiring', () => { const handleApiEnvironmentCreateFailure = jest.fn(); const processApiEnvironmentCreateQueue = jest.fn(); const processApiEnvironmentExpiryQueue = jest.fn(); + const processDeploymentReconciliationQueue = jest.fn(); const setupApiEnvironmentExpiryJob = jest.fn().mockResolvedValue(undefined); + const setupDeploymentReconciliationSweep = jest.fn(); const services: any = { GithubService: { processWebhooks: jest.fn() }, @@ -78,12 +80,14 @@ describe('bootstrapJobs API-environment wiring', () => { SitesService: { setupSitesCleanupJob: jest.fn(), processSitesCleanupQueue: jest.fn() }, BuildService: { setupApiEnvironmentExpiryJob, + setupDeploymentReconciliationSweep, processApiEnvironmentCreateQueue, handleApiEnvironmentCreateFailure, processApiEnvironmentExpiryQueue, processDeleteQueue: jest.fn(), processResolveAndDeployBuildQueue: jest.fn(), processBuildQueue: jest.fn(), + processDeploymentReconciliationQueue, }, Webhook: { processWebhookQueue: jest.fn() }, Ingress: { createOrUpdateIngressForBuild: jest.fn(), ingressCleanupForBuild: jest.fn() }, @@ -95,11 +99,14 @@ describe('bootstrapJobs API-environment wiring', () => { const createWorker = registeredWorkers.find((w) => w.queue === QUEUE_NAMES.API_ENV_CREATE); const expiryWorker = registeredWorkers.find((w) => w.queue === QUEUE_NAMES.API_ENV_EXPIRY); + const reconciliationWorker = registeredWorkers.find((w) => w.queue === QUEUE_NAMES.DEPLOYMENT_RECONCILIATION); expect(createWorker?.handler).toBe(processApiEnvironmentCreateQueue); expect(expiryWorker?.handler).toBe(processApiEnvironmentExpiryQueue); + expect(reconciliationWorker?.handler).toBe(processDeploymentReconciliationQueue); expect(workerOnHandlers[QUEUE_NAMES.API_ENV_CREATE]?.failed).toBe(handleApiEnvironmentCreateFailure); expect(setupApiEnvironmentExpiryJob).toHaveBeenCalled(); + expect(setupDeploymentReconciliationSweep).toHaveBeenCalled(); const ownerSweepWorker = registeredWorkers.find((w) => w.queue === QUEUE_NAMES.API_TOKEN_OWNER_SWEEP); expect(ownerSweepWorker?.handler).toBe(processApiTokenOwnerSweep); diff --git a/src/server/jobs/index.ts b/src/server/jobs/index.ts index 8e0886d0..34e27997 100644 --- a/src/server/jobs/index.ts +++ b/src/server/jobs/index.ts @@ -76,6 +76,7 @@ export default function bootstrapJobs(services: IServices) { services.BuildService.setupApiEnvironmentExpiryJob().catch((error) => getLogger().error({ error }, 'Jobs: api environment expiry setup failed') ); + services.BuildService.setupDeploymentReconciliationSweep(); const apiEnvCreateWorker = queueManager.registerWorker( QUEUE_NAMES.API_ENV_CREATE, @@ -164,6 +165,15 @@ export default function bootstrapJobs(services: IServices) { concurrency: 125, }); + queueManager.registerWorker( + QUEUE_NAMES.DEPLOYMENT_RECONCILIATION, + services.BuildService.processDeploymentReconciliationQueue, + { + connection: redisClient.getConnection(), + concurrency: 125, + } + ); + queueManager.registerWorker(QUEUE_NAMES.GITHUB_DEPLOYMENT, services.GithubService.processGithubDeployment, { connection: redisClient.getConnection(), concurrency: 125, diff --git a/src/server/lib/__tests__/authorityLock.test.ts b/src/server/lib/__tests__/authorityLock.test.ts new file mode 100644 index 00000000..2822c91b --- /dev/null +++ b/src/server/lib/__tests__/authorityLock.test.ts @@ -0,0 +1,160 @@ +/** + * Copyright 2026 Lifecycle contributors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { AuthorityLockLostError, withAuthorityLock } from '../authorityLock'; + +describe('withAuthorityLock', () => { + afterEach(() => { + jest.useRealTimers(); + }); + + test('waits through contention and admits the still-current generation once', async () => { + jest.useFakeTimers(); + const acquired = { extend: jest.fn(), unlock: jest.fn().mockResolvedValue(undefined) }; + const lockWithOptions = jest.fn().mockRejectedValueOnce(new Error('busy')).mockResolvedValueOnce(acquired); + const action = jest.fn().mockResolvedValue('done'); + + const result = withAuthorityLock({ + redlock: { lock: jest.fn(), lockWithOptions } as any, + resource: 'build-promotion.1', + ttlMs: 60_000, + isCurrent: jest.fn().mockResolvedValue(true), + action, + }); + + await jest.advanceTimersByTimeAsync(250); + + await expect(result).resolves.toEqual({ admitted: true, value: 'done' }); + expect(lockWithOptions).toHaveBeenCalledTimes(2); + expect(action).toHaveBeenCalledTimes(1); + expect(acquired.unlock).toHaveBeenCalledTimes(1); + }); + + test('continues past the legacy 120-attempt contention budget while authority remains current', async () => { + jest.useFakeTimers(); + const acquired = { extend: jest.fn(), unlock: jest.fn().mockResolvedValue(undefined) }; + const lock = jest.fn(); + let attempts = 0; + const lockWithOptions = jest.fn(async () => { + attempts += 1; + if (attempts <= 121) throw new Error('busy'); + return acquired; + }); + const action = jest.fn().mockResolvedValue('done'); + + const result = withAuthorityLock({ + redlock: { lock, lockWithOptions } as any, + resource: 'build-deployment.1', + ttlMs: 60_000, + isCurrent: jest.fn().mockResolvedValue(true), + action, + }); + + await jest.advanceTimersByTimeAsync(121 * 250); + + await expect(result).resolves.toEqual({ admitted: true, value: 'done' }); + expect(lockWithOptions.mock.calls).toEqual( + Array.from({ length: 122 }, () => [ + 'build-deployment.1', + 60_000, + { retryCount: 4, retryDelay: 1000, retryJitter: 200 }, + ]) + ); + expect(lock).not.toHaveBeenCalled(); + expect(action).toHaveBeenCalledTimes(1); + expect(acquired.unlock).toHaveBeenCalledTimes(1); + }); + + test('leaves without executing when superseded while waiting', async () => { + jest.useFakeTimers(); + const isCurrent = jest.fn().mockResolvedValueOnce(true).mockResolvedValueOnce(true).mockResolvedValue(false); + const action = jest.fn(); + const result = withAuthorityLock({ + redlock: { + lock: jest.fn(), + lockWithOptions: jest.fn().mockRejectedValue(new Error('busy')), + } as any, + resource: 'build-promotion.1', + ttlMs: 60_000, + isCurrent, + action, + }); + + await jest.advanceTimersByTimeAsync(250); + + await expect(result).resolves.toEqual({ admitted: false }); + expect(action).not.toHaveBeenCalled(); + }); + + test('reports lost renewal so the generation remains pending for retry', async () => { + jest.useFakeTimers(); + let finish!: () => void; + const action = jest.fn( + () => + new Promise((resolve) => { + finish = resolve; + }) + ); + const acquired = { + extend: jest.fn().mockRejectedValue(new Error('redis unavailable')), + unlock: jest.fn().mockResolvedValue(undefined), + }; + const result = withAuthorityLock({ + redlock: { + lock: jest.fn(), + lockWithOptions: jest.fn().mockResolvedValue(acquired), + } as any, + resource: 'build-promotion.1', + ttlMs: 300, + isCurrent: jest.fn().mockResolvedValue(true), + action, + }); + + await Promise.resolve(); + await jest.advanceTimersByTimeAsync(100); + finish(); + + await expect(result).rejects.toBeInstanceOf(AuthorityLockLostError); + expect(acquired.unlock).toHaveBeenCalledTimes(1); + }); + + test('does not publish admission when authority changes during the action', async () => { + const acquired = { + extend: jest.fn().mockResolvedValue(undefined), + unlock: jest.fn().mockResolvedValue(undefined), + }; + const isCurrent = jest + .fn() + .mockResolvedValueOnce(true) + .mockResolvedValueOnce(true) + .mockResolvedValueOnce(true) + .mockResolvedValueOnce(false); + const action = jest.fn().mockResolvedValue('stale-value'); + + await expect( + withAuthorityLock({ + redlock: { lock: jest.fn(), lockWithOptions: jest.fn().mockResolvedValue(acquired) } as any, + resource: 'deploy-external-secrets.1', + ttlMs: 60_000, + isCurrent, + action, + }) + ).resolves.toEqual({ admitted: false }); + + expect(action).toHaveBeenCalledTimes(1); + expect(acquired.unlock).toHaveBeenCalledTimes(1); + }); +}); diff --git a/src/server/lib/authorityLock.ts b/src/server/lib/authorityLock.ts new file mode 100644 index 00000000..915959d1 --- /dev/null +++ b/src/server/lib/authorityLock.ts @@ -0,0 +1,115 @@ +/** + * Copyright 2026 Lifecycle contributors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import type Redlock from 'redlock'; + +export type AuthorityLockResult = { admitted: true; value: T } | { admitted: false }; + +/** The mutation finished, but lock ownership was not durable enough to publish it as terminal. */ +export class AuthorityLockLostError extends Error { + constructor(resource: string, options: { cause?: unknown } = {}) { + super(`Authority lock was lost for ${resource}`, options); + this.name = 'AuthorityLockLostError'; + } +} + +interface AuthorityLockOptions { + redlock: Redlock | null | undefined; + resource: string; + ttlMs: number; + isCurrent: () => Promise; + action: () => Promise; + onWait?: (error: unknown) => void; +} + +/** + * Wait for a mutation lock only while the caller remains authoritative. Lock + * contention is not a terminal deployment failure; a newer generation simply + * makes the waiter leave without being admitted. + */ +export async function withAuthorityLock({ + redlock, + resource, + ttlMs, + isCurrent, + action, + onWait, +}: AuthorityLockOptions): Promise> { + if (!(await isCurrent())) return { admitted: false }; + if (!redlock?.lock) { + if (!(await isCurrent())) return { admitted: false }; + const value = await action(); + if (!(await isCurrent())) return { admitted: false }; + return { admitted: true, value }; + } + + const lockWithOptions = (redlock as any).lockWithOptions; + let lastWaitLog = Date.now(); + let lock: any; + + while (await isCurrent()) { + try { + lock = + typeof lockWithOptions === 'function' + ? await lockWithOptions.call(redlock, resource, ttlMs, { + retryCount: 4, + retryDelay: 1000, + retryJitter: 200, + }) + : await redlock.lock(resource, ttlMs); + break; + } catch (error) { + if (Date.now() - lastWaitLog >= 60_000) { + onWait?.(error); + lastWaitLog = Date.now(); + } + // Production Redlock already waited between attempts. This prevents a + // hot loop in tests or alternate implementations that fail immediately. + await new Promise((resolve) => setTimeout(resolve, 250)); + } + } + + if (!lock) return { admitted: false }; + + let renewalError: unknown; + let renewal = Promise.resolve(); + const renewalTimer = setInterval(() => { + renewal = renewal + .then(async () => { + lock = await lock.extend(ttlMs); + }) + .catch((error) => { + renewalError = error; + }); + }, ttlMs / 3); + + try { + if (!(await isCurrent())) return { admitted: false }; + const value = await action(); + // Stop scheduling extensions before inspecting the final renewal. Without + // this ordering, the timer can enqueue one more extension after the check + // and make a lost lock look successful. + clearInterval(renewalTimer); + await renewal; + if (renewalError) throw new AuthorityLockLostError(resource, { cause: renewalError }); + if (!(await isCurrent())) return { admitted: false }; + return { admitted: true, value }; + } finally { + clearInterval(renewalTimer); + await renewal; + await lock.unlock().catch(() => undefined); + } +} diff --git a/src/server/lib/deploymentManager/deploymentManager.ts b/src/server/lib/deploymentManager/deploymentManager.ts index c2e5238f..da3d53da 100644 --- a/src/server/lib/deploymentManager/deploymentManager.ts +++ b/src/server/lib/deploymentManager/deploymentManager.ts @@ -16,6 +16,7 @@ import { Deploy } from 'server/models'; import { deployHelm } from '../helm'; +import { shouldUseNativeHelm } from '../nativeHelm'; import { DeployStatus, DeployTypes, CLIDeployTypes } from 'shared/constants'; import { createKubernetesApplyJob, monitorKubernetesJob } from '../kubernetesApply/applyManifest'; import { nanoid, customAlphabet } from 'nanoid'; @@ -26,17 +27,46 @@ import { waitForDeployPodReady } from '../kubernetes'; import { buildDeployJobName } from '../kubernetes/jobNames'; import GlobalConfigService from 'server/services/globalConfig'; import { getLogArchivalService } from 'server/services/logArchival'; +import { DeploymentSupersededError } from 'server/lib/deploymentReconciliation/errors'; +import type { HelmSecretMutationGate } from 'server/lib/nativeHelm/helm'; + +export { DeploymentSupersededError } from 'server/lib/deploymentReconciliation/errors'; const generateJobId = customAlphabet('abcdefghijklmnopqrstuvwxyz0123456789', 6); +export type NativeMutationGateResult = { admitted: true; value: T } | { admitted: false }; + +export type NativeMutationGate = (action: () => Promise) => Promise>; + +export interface DeploymentManagerOptions { + /** Returns false once this deployment generation is no longer authoritative. */ + isCurrent?: () => Promise; + /** + * Admits the native mutations for one dependency level after acquiring the + * caller's promotion lock and rechecking generation authority. + */ + nativeMutationGate?: NativeMutationGate; + /** Serializes only writers of one Deploy's ExternalSecret resources. */ + nativeSecretMutationGate?: HelmSecretMutationGate; +} + +interface KubernetesDeploymentContext { + deploy: Deploy; + deployService: DeployService; + runUUID: string; + cliDeploy: boolean; +} + export class DeploymentManager { private deploys: Map = new Map(); private deploymentLevels: Map = new Map(); // Deploys never placed in a level: members of a dependency cycle, or dependents of one. private unresolvedDeploys: Deploy[] = []; private dependencyCycleDescription = ''; + private readonly options: DeploymentManagerOptions; - constructor(deploys: Deploy[]) { + constructor(deploys: Deploy[], options: DeploymentManagerOptions = {}) { + this.options = options; deploys.forEach((deploy) => { this.deploys.set(deploy.deployable.name, deploy); }); @@ -136,39 +166,134 @@ export class DeploymentManager { } public async deploy(): Promise { + await this.assertCurrent(); + const unresolved = new Set(this.unresolvedDeploys); for (const value of this.deploys.values()) { if (!unresolved.has(value)) { - await value.$query().patch({ status: DeployStatus.QUEUED }); + await this.patchDeployIfCurrent(value, { status: DeployStatus.QUEUED }); } } + await this.assertCurrent(); + if (this.unresolvedDeploys.length > 0) { const statusMessage = `Dependency cycle detected: ${this.dependencyCycleDescription}; deploy order cannot be resolved`; for (const deploy of this.unresolvedDeploys) { getLogger().error(`Deploy: ${deploy.deployable.name} failed — ${statusMessage}`); - await deploy.$query().patch({ status: DeployStatus.DEPLOY_FAILED, statusMessage }); + await this.patchDeployIfCurrent(deploy, { status: DeployStatus.DEPLOY_FAILED, statusMessage }); } } for (let level = 0; level < this.deploymentLevels.size; level++) { + await this.assertCurrent(); + const deploysAtLevel = this.deploymentLevels.get(level); if (deploysAtLevel) { const helmDeploys = deploysAtLevel.filter((d) => this.shouldDeployWithHelm(d)); const githubDeploys = deploysAtLevel.filter((d) => this.shouldDeployWithKubernetes(d)); - const helmServices = helmDeploys.map((d) => d.deployable.name).join(','); + const helmMethods = await Promise.all( + helmDeploys.map(async (deploy) => ({ deploy, native: await shouldUseNativeHelm(deploy) })) + ); + await this.assertCurrent(); + + const nativeHelmDeploys = helmMethods.filter(({ native }) => native).map(({ deploy }) => deploy); + const codefreshHelmDeploys = helmMethods.filter(({ native }) => !native).map(({ deploy }) => deploy); + + const nativeHelmServices = nativeHelmDeploys.map((d) => d.deployable.name).join(','); + const codefreshHelmServices = codefreshHelmDeploys.map((d) => d.deployable.name).join(','); const k8sServices = githubDeploys.map((d) => d.deployable.name).join(','); - getLogger().info(`Deploy: level ${level} helm=[${helmServices}] k8s=[${k8sServices}]`); + getLogger().info( + `Deploy: level ${level} nativeHelm=[${nativeHelmServices}] codefreshHelm=[${codefreshHelmServices}] k8s=[${k8sServices}]` + ); - await Promise.all([ - helmDeploys.length > 0 ? deployHelm(helmDeploys) : Promise.resolve(), - ...githubDeploys.map((deploy) => this.deployManifests(deploy)), - ]); + // Codefresh is intentionally not part of native mutation admission. It + // may continue in the provider while a newer generation starts. + const codefreshDeploy = codefreshHelmDeploys.length > 0 ? deployHelm(codefreshHelmDeploys) : Promise.resolve(); + // Attach a handler immediately because supersession may deliberately + // detach this provider wait before we await it below. + void codefreshDeploy.catch(() => undefined); + + let kubernetesDeployments: KubernetesDeploymentContext[] = []; + let nativeFailure: unknown; + try { + if (nativeHelmDeploys.length > 0 || githubDeploys.length > 0) { + const nativeResult = await this.runNativeMutation(async () => { + // Run same-generation siblings in parallel, but do not release the + // promotion gate until every admitted native mutation is terminal. + const settled = await Promise.allSettled([ + ...nativeHelmDeploys.map((deploy) => + deployHelm([deploy], { secretMutationGate: this.options.nativeSecretMutationGate }) + ), + ...githubDeploys.map((deploy) => this.applyManifests(deploy)), + ]); + const failure = settled.find((result): result is PromiseRejectedResult => result.status === 'rejected'); + const successfulKubernetesDeployments = settled + .slice(nativeHelmDeploys.length) + .filter( + (result): result is PromiseFulfilledResult => + result.status === 'fulfilled' + ) + .map((result) => result.value); + return { kubernetesDeployments: successfulKubernetesDeployments, failure: failure?.reason }; + }); + kubernetesDeployments = nativeResult.kubernetesDeployments; + nativeFailure = nativeResult.failure; + } + + await this.assertCurrent(); + + // Pod readiness is observational and must not hold the native mutation + // gate. Settle every successful sibling even when another native + // mutation failed so no Deploy is left indefinitely at DEPLOYING. + const readiness = await Promise.allSettled( + kubernetesDeployments.map((deployment) => this.waitForManifestReadiness(deployment)) + ); + if (nativeFailure) throw nativeFailure; + const readinessFailure = readiness.find( + (result): result is PromiseRejectedResult => result.status === 'rejected' + ); + if (readinessFailure) throw readinessFailure.reason; + await codefreshDeploy; + } catch (error) { + if (error instanceof DeploymentSupersededError) { + getLogger().info(`Deploy: level ${level} stopped reason=superseded`); + } + throw error; + } } + + await this.assertCurrent(); + } + } + + private async assertCurrent(): Promise { + if (this.options.isCurrent && !(await this.options.isCurrent())) { + throw new DeploymentSupersededError(); } } + private async runNativeMutation(action: () => Promise): Promise { + if (!this.options.nativeMutationGate) return action(); + + const result = await this.options.nativeMutationGate(action); + if (!result.admitted) throw new DeploymentSupersededError(); + return result.value; + } + + private async patchDeployIfCurrent(deploy: Deploy, patch: Partial): Promise { + if (deploy.id == null || !deploy.runUUID) { + getLogger().info( + `Deploy: status patch skipped reason=missing_run_identity deployUuid=${deploy.uuid || 'unknown'}` + ); + return; + } + + const patched = await deploy.$query().patch(patch).where({ id: deploy.id, runUUID: deploy.runUUID }); + if (!patched) throw new DeploymentSupersededError(); + } + private shouldDeployWithHelm(deploy: Deploy): boolean { const deployType = deploy.deployable?.type; return deployType === DeployTypes.HELM; @@ -217,7 +342,7 @@ export class DeploymentManager { ); } - private async deployManifests(deploy: Deploy): Promise { + private async applyManifests(deploy: Deploy): Promise { return withLogContext({ deployUuid: deploy.uuid, serviceName: deploy.deployable?.name }, async () => { const jobId = generateJobId(); const deployService = new DeployService(); @@ -274,22 +399,44 @@ export class DeploymentManager { ); const cliDeploy = CLIDeployTypes.has(deploy.deployable.type); + return { deploy, deployService, runUUID, cliDeploy }; + } catch (error) { + if (error instanceof DeploymentSupersededError) throw error; + await deployService.recordDeployFailure(deploy, runUUID, { + status: DeployStatus.DEPLOY_FAILED, + error, + fallbackMessage: `Deployment failed for ${deploy.uuid}. Check deploy logs in Console > Deploy tab for details.`, + }); + throw error; + } + }); + } + + private async waitForManifestReadiness({ + deploy, + deployService, + runUUID, + cliDeploy, + }: KubernetesDeploymentContext): Promise { + return withLogContext({ deployUuid: deploy.uuid, serviceName: deploy.deployable?.name }, async () => { + try { const readiness = cliDeploy ? { ready: true } : await waitForDeployPodReady(deploy); - if (readiness.ready) { - await deployService.patchAndUpdateActivityFeed( - deploy, - { - status: DeployStatus.READY, - statusMessage: cliDeploy ? 'CLI Deploy completed' : 'Kubernetes pods are ready', - }, - runUUID - ); - } else { + if (!readiness.ready) { const cause = readiness.causeSummary ? `: ${readiness.causeSummary.slice(0, 350)}` : ''; throw new Error(`Pods failed to become ready within timeout${cause}`); } + + await deployService.patchAndUpdateActivityFeed( + deploy, + { + status: DeployStatus.READY, + statusMessage: cliDeploy ? 'CLI Deploy completed' : 'Kubernetes pods are ready', + }, + runUUID + ); } catch (error) { + if (error instanceof DeploymentSupersededError) throw error; await deployService.recordDeployFailure(deploy, runUUID, { status: DeployStatus.DEPLOY_FAILED, error, diff --git a/src/server/lib/deploymentReconciliation/__tests__/mailbox.test.ts b/src/server/lib/deploymentReconciliation/__tests__/mailbox.test.ts new file mode 100644 index 00000000..a44f7e50 --- /dev/null +++ b/src/server/lib/deploymentReconciliation/__tests__/mailbox.test.ts @@ -0,0 +1,353 @@ +/** + * Copyright 2026 GoodRx, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +jest.mock('server/models/Build', () => ({ + __esModule: true, + default: { + query: jest.fn(), + transact: jest.fn(), + }, +})); + +import Build from 'server/models/Build'; +import { + acceptDeploymentIntent, + AcceptedDeploymentRefs, + deploymentIntentScopeKey, + dirtyDeploymentIntents, +} from '../mailbox'; + +const TRX = { transaction: true } as any; + +function readQuery(row: any) { + const query: any = { + select: jest.fn().mockReturnThis(), + findById: jest.fn().mockReturnThis(), + whereNull: jest.fn().mockReturnThis(), + forUpdate: jest.fn().mockResolvedValue(row), + }; + return query; +} + +function writeQuery() { + const query: any = { + findById: jest.fn().mockReturnThis(), + patch: jest.fn().mockResolvedValue(1), + }; + return query; +} + +beforeEach(() => { + jest.clearAllMocks(); + (Build.transact as jest.Mock).mockImplementation(async (callback: any) => callback(TRX)); +}); + +describe('deployment intent scope keys', () => { + it('uses stable, non-overlapping keys for source, repository, and whole-environment work', () => { + expect( + deploymentIntentScopeKey({ + type: 'source', + requestId: 'request-source', + target: 'repository', + githubRepositoryId: 123, + branch: 'Feature/A B', + sha: 'abc', + }) + ).toBe('source:123:Feature%2FA%20B'); + expect( + deploymentIntentScopeKey({ type: 'repository', requestId: 'request-repository', githubRepositoryId: 123 }) + ).toBe('repository:123'); + expect(deploymentIntentScopeKey({ type: 'all', requestId: 'request-all' })).toBe('all'); + }); +}); + +describe('acceptDeploymentIntent', () => { + it('coalesces A/B/C for one source to only C while retaining monotonic authority', async () => { + const row: any = { desiredGeneration: 0, acceptedRefs: {}, runUUID: 'teardown-owner' }; + const query: any = { + select: jest.fn().mockReturnThis(), + findById: jest.fn().mockReturnThis(), + whereNull: jest.fn().mockReturnThis(), + forUpdate: jest.fn(async () => row), + patch: jest.fn(async (patch: any) => { + Object.assign(row, patch); + return 1; + }), + }; + (Build.query as jest.Mock).mockImplementation(() => query); + + for (const [requestId, sha] of [ + ['request-a', 'sha-a'], + ['request-b', 'sha-b'], + ['request-c', 'sha-c'], + ]) { + await acceptDeploymentIntent(42, { + type: 'source', + requestId, + target: 'repository', + githubRepositoryId: 123, + branch: 'main', + sha, + }); + } + + expect(row.desiredGeneration).toBe(3); + expect(row.runUUID).toBe('teardown-owner'); + expect(row.acceptedRefs).toEqual({ + 'source:123:main': { + type: 'source', + requestId: 'request-c', + target: 'repository', + githubRepositoryId: 123, + branch: 'main', + sha: 'sha-c', + gen: 3, + }, + }); + }); + + it('locks the Build row, increments its generation, and preserves other scopes', async () => { + const existing: AcceptedDeploymentRefs = { + 'repository:9': { type: 'repository', requestId: 'request-2', githubRepositoryId: 9, gen: 2 }, + }; + const read = readQuery({ desiredGeneration: '2', acceptedRefs: existing }); + const write = writeQuery(); + (Build.query as jest.Mock).mockReturnValueOnce(read).mockReturnValueOnce(write); + + const result = await acceptDeploymentIntent(42, { + type: 'source', + requestId: 'request-3', + target: 'repository', + githubRepositoryId: 123, + branch: 'main', + sha: 'commit-c', + }); + + expect(result).toEqual({ accepted: true, generation: 3, scopeKey: 'source:123:main' }); + expect(read.select).toHaveBeenCalledWith('id', 'desiredGeneration', 'acceptedRefs'); + expect(read.findById).toHaveBeenCalledWith(42); + expect(read.whereNull).toHaveBeenCalledWith('deletedAt'); + expect(read.forUpdate).toHaveBeenCalledTimes(1); + expect(write.patch).toHaveBeenCalledWith({ + desiredGeneration: 3, + acceptedRefs: { + ...existing, + 'source:123:main': { + type: 'source', + requestId: 'request-3', + target: 'repository', + githubRepositoryId: 123, + branch: 'main', + sha: 'commit-c', + gen: 3, + }, + }, + }); + }); + + it('does not bump for a redelivery of the same source SHA', async () => { + const read = readQuery({ + desiredGeneration: 7, + acceptedRefs: { + 'source:123:main': { + type: 'source', + requestId: 'request-7', + target: 'repository', + githubRepositoryId: 123, + branch: 'main', + sha: 'same-sha', + gen: 7, + }, + }, + }); + (Build.query as jest.Mock).mockReturnValueOnce(read); + + await expect( + acceptDeploymentIntent(42, { + type: 'source', + requestId: 'request-redelivery', + target: 'repository', + githubRepositoryId: 123, + branch: 'main', + sha: 'same-sha', + }) + ).resolves.toEqual({ accepted: false, generation: 7, scopeKey: 'source:123:main' }); + expect(Build.query).toHaveBeenCalledTimes(1); + }); + + it('does not let a delayed predecessor push replace the newer accepted push', async () => { + const read = readQuery({ + desiredGeneration: 8, + acceptedRefs: { + 'source:123:main': { + type: 'source', + requestId: 'request-c', + target: 'repository', + githubRepositoryId: 123, + branch: 'main', + sha: 'commit-c', + beforeSha: 'commit-b', + gen: 8, + }, + }, + }); + (Build.query as jest.Mock).mockReturnValueOnce(read); + + await expect( + acceptDeploymentIntent(42, { + type: 'source', + requestId: 'request-delayed-b', + target: 'repository', + githubRepositoryId: 123, + branch: 'main', + sha: 'commit-b', + beforeSha: 'commit-a', + }) + ).resolves.toEqual({ accepted: false, generation: 8, scopeKey: 'source:123:main' }); + expect(Build.query).toHaveBeenCalledTimes(1); + }); + + it('accepts an intentional rollback to the previous SHA', async () => { + const read = readQuery({ + desiredGeneration: 8, + acceptedRefs: { + 'source:123:main': { + type: 'source', + requestId: 'request-c', + target: 'repository', + githubRepositoryId: 123, + branch: 'main', + sha: 'commit-c', + beforeSha: 'commit-b', + gen: 8, + }, + }, + }); + const write = writeQuery(); + (Build.query as jest.Mock).mockReturnValueOnce(read).mockReturnValueOnce(write); + + await expect( + acceptDeploymentIntent(42, { + type: 'source', + requestId: 'request-rollback-b', + target: 'repository', + githubRepositoryId: 123, + branch: 'main', + sha: 'commit-b', + beforeSha: 'commit-c', + }) + ).resolves.toEqual({ accepted: true, generation: 9, scopeKey: 'source:123:main' }); + expect(write.patch).toHaveBeenCalledWith( + expect.objectContaining({ + desiredGeneration: 9, + acceptedRefs: expect.objectContaining({ + 'source:123:main': expect.objectContaining({ sha: 'commit-b', beforeSha: 'commit-c', gen: 9 }), + }), + }) + ); + }); + + it('does not bump when legacy queues redeliver the same explicit request', async () => { + const read = readQuery({ + desiredGeneration: 7, + acceptedRefs: { + 'repository:123': { type: 'repository', requestId: 'request-7', githubRepositoryId: 123, gen: 7 }, + }, + }); + (Build.query as jest.Mock).mockReturnValueOnce(read); + + await expect( + acceptDeploymentIntent(42, { + type: 'repository', + requestId: 'request-7', + githubRepositoryId: 123, + }) + ).resolves.toEqual({ accepted: false, generation: 7, scopeKey: 'repository:123' }); + expect(Build.query).toHaveBeenCalledTimes(1); + }); + + it('always bumps an explicit repository redeploy and replaces only its key', async () => { + const read = readQuery({ + desiredGeneration: 5, + acceptedRefs: { + 'repository:123': { type: 'repository', requestId: 'request-4', githubRepositoryId: 123, gen: 4 }, + 'source:456:main': { + type: 'source', + requestId: 'request-5', + target: 'repository', + githubRepositoryId: 456, + branch: 'main', + sha: 'sha-y', + gen: 5, + }, + }, + }); + const write = writeQuery(); + (Build.query as jest.Mock).mockReturnValueOnce(read).mockReturnValueOnce(write); + + await acceptDeploymentIntent(42, { + type: 'repository', + requestId: 'request-6', + githubRepositoryId: 123, + }); + + expect(write.patch).toHaveBeenCalledWith({ + desiredGeneration: 6, + acceptedRefs: { + 'repository:123': { type: 'repository', requestId: 'request-6', githubRepositoryId: 123, gen: 6 }, + 'source:456:main': { + type: 'source', + requestId: 'request-5', + target: 'repository', + githubRepositoryId: 456, + branch: 'main', + sha: 'sha-y', + gen: 5, + }, + }, + }); + }); + + it('returns null without writing when the Build does not exist', async () => { + (Build.query as jest.Mock).mockReturnValueOnce(readQuery(undefined)); + + await expect(acceptDeploymentIntent(404, { type: 'all', requestId: 'request-missing' })).resolves.toBeNull(); + expect(Build.query).toHaveBeenCalledTimes(1); + }); +}); + +describe('dirtyDeploymentIntents', () => { + it('extracts only unobserved latest intents in generation order', () => { + const acceptedRefs: AcceptedDeploymentRefs = { + all: { type: 'all', requestId: 'request-5', gen: 5 }, + 'source:123:main': { + type: 'source', + requestId: 'request-2', + target: 'repository', + githubRepositoryId: 123, + branch: 'main', + sha: 'old', + gen: 2, + }, + 'repository:456': { type: 'repository', requestId: 'request-3', githubRepositoryId: 456, gen: 3 }, + }; + + expect(dirtyDeploymentIntents(acceptedRefs, 2)).toEqual([ + { scopeKey: 'repository:456', intent: acceptedRefs['repository:456'] }, + { scopeKey: 'all', intent: acceptedRefs.all }, + ]); + }); +}); diff --git a/src/server/lib/deploymentReconciliation/errors.ts b/src/server/lib/deploymentReconciliation/errors.ts new file mode 100644 index 00000000..33cf3469 --- /dev/null +++ b/src/server/lib/deploymentReconciliation/errors.ts @@ -0,0 +1,7 @@ +/** Expected control flow when a newer deployment generation takes ownership. */ +export class DeploymentSupersededError extends Error { + constructor() { + super('Deployment generation was superseded'); + this.name = 'DeploymentSupersededError'; + } +} diff --git a/src/server/lib/deploymentReconciliation/mailbox.ts b/src/server/lib/deploymentReconciliation/mailbox.ts new file mode 100644 index 00000000..e608ea7e --- /dev/null +++ b/src/server/lib/deploymentReconciliation/mailbox.ts @@ -0,0 +1,158 @@ +/** + * Copyright 2026 GoodRx, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import type { JobDataWithContext } from 'server/lib/logger'; +import Build from 'server/models/Build'; + +export interface SourcePushIntent { + type: 'source'; + requestId: string; + /** A root/config source can intentionally request a full Environment pass. */ + target: 'repository' | 'all'; + githubRepositoryId: number; + branch: string; + sha: string; + /** Delivered push predecessor; used to recognize a temporarily stale branch-head read. */ + beforeSha?: string; +} + +export interface RepositoryRedeployIntent { + type: 'repository'; + requestId: string; + githubRepositoryId: number; +} + +export interface EnvironmentRedeployIntent { + type: 'all'; + requestId: string; +} + +export type DeploymentIntent = SourcePushIntent | RepositoryRedeployIntent | EnvironmentRedeployIntent; +export type AcceptedDeploymentIntent = DeploymentIntent & { gen: number }; +export type AcceptedDeploymentRefs = Record; + +export interface DeploymentReconciliationJobData extends JobDataWithContext { + buildId: number; + /** The exact desired generation that caused this reconciliation signal. */ + generation: number; +} + +export interface DirtyDeploymentIntent { + scopeKey: string; + intent: AcceptedDeploymentIntent; +} + +export interface AcceptDeploymentIntentResult { + accepted: boolean; + generation: number; + scopeKey: string; +} + +/** A stable key lets a newer intent replace only earlier work for the same scope. */ +export function deploymentIntentScopeKey(intent: DeploymentIntent): string { + switch (intent.type) { + case 'source': + return `source:${intent.githubRepositoryId}:${encodeURIComponent(intent.branch)}`; + case 'repository': + return `repository:${intent.githubRepositoryId}`; + case 'all': + return 'all'; + } +} + +/** Returns the latest intent for every scope that has not yet been observed. */ +export function dirtyDeploymentIntents( + acceptedRefs: AcceptedDeploymentRefs | null | undefined, + observedGeneration: number +): DirtyDeploymentIntent[] { + if (!acceptedRefs || typeof acceptedRefs !== 'object' || Array.isArray(acceptedRefs)) return []; + + return Object.entries(acceptedRefs) + .filter(([, intent]) => Number.isSafeInteger(intent?.gen) && intent.gen > observedGeneration) + .map(([scopeKey, intent]) => ({ scopeKey, intent })) + .sort((left, right) => left.intent.gen - right.intent.gen || left.scopeKey.localeCompare(right.scopeKey)); +} + +/** + * Atomically records desired work. The Build row lock serializes concurrent + * accepters; the JSON map coalesces repeated work without becoming a queue. + */ +export async function acceptDeploymentIntent( + buildId: number, + intent: DeploymentIntent +): Promise { + return Build.transact(async (trx) => { + const build = await Build.query(trx) + .select('id', 'desiredGeneration', 'acceptedRefs') + .findById(buildId) + .whereNull('deletedAt') + .forUpdate(); + + if (!build) return null; + + const scopeKey = deploymentIntentScopeKey(intent); + const acceptedRefs = + build.acceptedRefs && typeof build.acceptedRefs === 'object' && !Array.isArray(build.acceptedRefs) + ? build.acceptedRefs + : {}; + const previous = acceptedRefs[scopeKey]; + const desiredGeneration = Number(build.desiredGeneration); + + if (!Number.isSafeInteger(desiredGeneration) || desiredGeneration < 0) { + throw new Error(`Build ${buildId} has an invalid desired generation`); + } + + if ( + previous?.requestId === intent.requestId || + (intent.type === 'source' && + previous?.type === 'source' && + previous.sha === intent.sha && + previous.target === intent.target) + ) { + return { accepted: false, generation: desiredGeneration, scopeKey }; + } + + // GitHub can deliver adjacent pushes out of order. If C is already desired, + // a delayed B identifies itself as C's predecessor and must not replace C. + // A real rollback from C to B remains valid because its `before` is C. + if ( + intent.type === 'source' && + previous?.type === 'source' && + previous.beforeSha && + intent.sha === previous.beforeSha && + intent.beforeSha !== previous.sha + ) { + return { accepted: false, generation: desiredGeneration, scopeKey }; + } + + const generation = desiredGeneration + 1; + if (!Number.isSafeInteger(generation)) { + throw new Error(`Build ${buildId} exhausted the safe generation range`); + } + + const nextRefs: AcceptedDeploymentRefs = { + ...acceptedRefs, + [scopeKey]: { ...intent, gen: generation }, + }; + + await Build.query(trx).findById(buildId).patch({ + desiredGeneration: generation, + acceptedRefs: nextRefs, + }); + + return { accepted: true, generation, scopeKey }; + }); +} diff --git a/src/server/lib/github/index.ts b/src/server/lib/github/index.ts index b515c976..83ffb846 100644 --- a/src/server/lib/github/index.ts +++ b/src/server/lib/github/index.ts @@ -36,6 +36,8 @@ export interface ChangedFilesForPushResult { reason?: string; } +export type CommitComparisonStatus = 'ahead' | 'behind' | 'diverged' | 'identical'; + interface PushPayloadCommitFiles { added?: string[]; removed?: string[]; @@ -281,6 +283,23 @@ export async function getSHAForBranch(branchName: string, owner: string, name: s } } +export async function compareCommits({ + fullName, + base, + head, +}: { + fullName: string; + base: string; + head: string; +}): Promise { + const response = await cacheRequest(`GET /repos/${fullName}/compare/${base}...${head}`); + const status = response?.data?.status; + if (!['ahead', 'behind', 'diverged', 'identical'].includes(status)) { + throw new Error(`GitHub compare returned an invalid status for ${fullName}`); + } + return status as CommitComparisonStatus; +} + export async function getChangedFilesForPush({ fullName, before, diff --git a/src/server/lib/helm/__tests__/helm.test.ts b/src/server/lib/helm/__tests__/helm.test.ts index 50219dc4..6c416d37 100644 --- a/src/server/lib/helm/__tests__/helm.test.ts +++ b/src/server/lib/helm/__tests__/helm.test.ts @@ -124,6 +124,33 @@ describe('Helm tests', () => { expect(metadata).toEqual(expectedMetadata); }); + test('constructHelmDeploysBuildMetaData hydrates an unloaded pull request on an existing build', async () => { + const deploy = { + build: { + uuid: '123', + pullRequestId: 42, + }, + $fetchGraph: jest.fn(async () => { + deploy.build.pullRequest = { + branchName: 'feature/branch', + fullName: 'user/repo', + latestCommit: 'abc123', + }; + }), + } as any; + + const metadata = await constructHelmDeploysBuildMetaData([deploy]); + + expect(deploy.$fetchGraph).toHaveBeenCalledWith('build.pullRequest'); + expect(metadata).toEqual({ + uuid: '123', + branchName: 'feature/branch', + fullName: 'user/repo', + sha: 'abc123', + error: '', + }); + }); + test('constructHelmDeploysBuildMetaData should handle missing build or pull request', async () => { const deploys = [ { diff --git a/src/server/lib/helm/helm.ts b/src/server/lib/helm/helm.ts index 4968cad2..58f3a35d 100644 --- a/src/server/lib/helm/helm.ts +++ b/src/server/lib/helm/helm.ts @@ -256,9 +256,12 @@ export async function helmDeployStep(deploy: Deploy): Promise { try { const deploy = deploys?.[0]; let build = deploy?.build; - if (!build) { + if (!build || (build.pullRequestId != null && build.pullRequest == null)) { await deploy?.$fetchGraph('build.pullRequest'); } build = deploy?.build; diff --git a/src/server/lib/kubernetes/externalSecret.ts b/src/server/lib/kubernetes/externalSecret.ts index da6a1dcc..5089b839 100644 --- a/src/server/lib/kubernetes/externalSecret.ts +++ b/src/server/lib/kubernetes/externalSecret.ts @@ -168,14 +168,18 @@ export async function applyExternalSecret(manifest: ExternalSecretManifest, name getLogger().info(`ExternalSecret: applying name=${manifest.metadata.name} namespace=${namespace}`); - await shellPromise(`kubectl apply -f ${localPath} --namespace ${namespace}`); + await shellPromise(`kubectl apply -f ${localPath} --namespace ${namespace}`, { + timeout: 90_000, + }); } export async function deleteExternalSecret(name: string, namespace: string): Promise { getLogger().info(`ExternalSecret: deleting name=${name} namespace=${namespace}`); try { - await shellPromise(`kubectl delete externalsecret ${name} --namespace ${namespace} --ignore-not-found`); + await shellPromise(`kubectl delete externalsecret ${name} --namespace ${namespace} --ignore-not-found`, { + timeout: 90_000, + }); } catch (error) { getLogger().warn({ error }, `ExternalSecret: delete failed name=${name}`); } diff --git a/src/server/lib/kubernetesApply/applyManifest.ts b/src/server/lib/kubernetesApply/applyManifest.ts index 7fbec39e..097fa3d5 100644 --- a/src/server/lib/kubernetesApply/applyManifest.ts +++ b/src/server/lib/kubernetesApply/applyManifest.ts @@ -23,6 +23,8 @@ import { buildDeployJobName } from 'server/lib/kubernetes/jobNames'; import { JobMonitor } from 'server/lib/kubernetes/JobMonitor'; import { buildLifecycleLabels } from 'server/lib/kubernetes/labels'; +const APPLY_JOB_DEADLINE_SECONDS = 9 * 60; + export interface KubernetesApplyJobConfig { deploy: Deploy; namespace: string; @@ -74,6 +76,10 @@ export async function createKubernetesApplyJob({ }, spec: { ttlSecondsAfterFinished: 86400, + // JobMonitor's terminal-condition loop is intentionally generic. Give + // this admitted mutation a Kubernetes-enforced bound below its 10-minute + // monitor budget so a wedged kubectl cannot hold promotion forever. + activeDeadlineSeconds: APPLY_JOB_DEADLINE_SECONDS, backoffLimit: 0, template: { metadata: { @@ -99,7 +105,6 @@ export async function createKubernetesApplyJob({ if kubectl get deployment ${deploy.uuid} -n ${namespace} &>/dev/null; then kubectl rollout restart deployment/${deploy.uuid} -n ${namespace} - kubectl rollout status deployment/${deploy.uuid} -n ${namespace} --timeout=300s fi `, ], diff --git a/src/server/lib/nativeBuild/__tests__/index.test.ts b/src/server/lib/nativeBuild/__tests__/index.test.ts index 320ab3cc..fde8fad1 100644 --- a/src/server/lib/nativeBuild/__tests__/index.test.ts +++ b/src/server/lib/nativeBuild/__tests__/index.test.ts @@ -113,4 +113,36 @@ describe('buildWithNative', () => { 'buildkit' ); }); + + it('uses reconciliation-prepared service account without repeating namespace setup', async () => { + const deploy = { + deployable: { name: 'sample-service', builder: { engine: 'buildkit' } }, + $fetchGraph: jest.fn().mockResolvedValue(undefined), + }; + const options = { + ecrRepo: 'sample-repo', + ecrDomain: 'registry.example.com', + envVars: {}, + dockerfilePath: 'Dockerfile', + tag: 'sample-tag', + revision: 'abcdef1234567890', + repo: 'example-org/example-repo', + branch: 'main', + namespace: 'env-build123', + buildId: '1', + buildUuid: 'build123', + deployUuid: 'deploy123', + serviceAccount: 'prepared-build-sa', + }; + + await buildWithNative(deploy as any, options); + + expect(mockCreateOrUpdateNamespace).not.toHaveBeenCalled(); + expect(mockEnsureServiceAccountForJob).not.toHaveBeenCalled(); + expect(mockBuildWithEngine).toHaveBeenCalledWith( + deploy, + expect.objectContaining({ serviceAccount: 'prepared-build-sa' }), + 'buildkit' + ); + }); }); diff --git a/src/server/lib/nativeBuild/index.ts b/src/server/lib/nativeBuild/index.ts index 159c56f3..96e257bf 100644 --- a/src/server/lib/nativeBuild/index.ts +++ b/src/server/lib/nativeBuild/index.ts @@ -38,21 +38,27 @@ export async function buildWithNative(deploy: Deploy, options: NativeBuildOption getLogger().info('Build: starting (native)'); try { - await deploy.$fetchGraph('[build.[pullRequest]]'); + let serviceAccountName = options.serviceAccount; + if (!serviceAccountName) { + // Direct/legacy callers retain the old self-contained setup path. + // Reconciliation prepares this once per scope under its short + // native-mutation gate and passes the resulting account here. + await deploy.$fetchGraph('[build.[pullRequest]]'); - if (!deploy.build) { - throw new Error('Build: namespace setup requires build metadata'); - } + if (!deploy.build) { + throw new Error('Build: namespace setup requires build metadata'); + } - await createOrUpdateNamespace({ - name: options.namespace, - buildUUID: deploy.build.uuid, - staticEnv: deploy.build.isStatic, - pullRequest: deploy.build.pullRequest, - waitForReady: true, - }); + await createOrUpdateNamespace({ + name: options.namespace, + buildUUID: deploy.build.uuid, + staticEnv: deploy.build.isStatic, + pullRequest: deploy.build.pullRequest, + waitForReady: true, + }); - const serviceAccountName = await ensureServiceAccountForJob(options.namespace, 'build'); + serviceAccountName = await ensureServiceAccountForJob(options.namespace, 'build'); + } const buildOptions = { ...options, diff --git a/src/server/lib/nativeHelm/__tests__/helm.test.ts b/src/server/lib/nativeHelm/__tests__/helm.test.ts index 4e07ba8a..8973e93a 100644 --- a/src/server/lib/nativeHelm/__tests__/helm.test.ts +++ b/src/server/lib/nativeHelm/__tests__/helm.test.ts @@ -14,7 +14,7 @@ * limitations under the License. */ -import { shouldUseNativeHelm, createHelmContainer, nativeHelmDeploy, generateHelmManifest } from '../helm'; +import { shouldUseNativeHelm, createHelmContainer, nativeHelmDeploy, generateHelmManifest, deployHelm } from '../helm'; import * as nativeHelmUtils from '../utils'; import yaml from 'js-yaml'; import { @@ -35,6 +35,11 @@ import { shellPromise } from 'server/lib/shell'; import { getLogArchivalService } from 'server/services/logArchival'; import { YamlConfigParser } from 'server/lib/yamlConfigParser'; import * as YamlService from 'server/models/yaml'; +import { SecretProcessor } from 'server/services/secretProcessor'; +import DeployService from 'server/services/deploy'; +import { Metrics } from 'server/lib/metrics'; +import { AuthorityLockLostError } from 'server/lib/authorityLock'; +import { DeployStatus } from 'shared/constants'; jest.mock('server/services/globalConfig'); jest.mock('../utils', () => { @@ -44,8 +49,15 @@ jest.mock('../utils', () => { determineChartType: jest.fn(originalModule.determineChartType), getHelmConfiguration: jest.fn(originalModule.getHelmConfiguration), mergeHelmConfigWithGlobal: jest.fn(originalModule.mergeHelmConfigWithGlobal), + validateHelmConfiguration: jest.fn(originalModule.validateHelmConfiguration), }; }); +jest.mock('server/lib/metrics', () => ({ + Metrics: jest.fn().mockImplementation(() => ({ + increment: jest.fn(), + event: jest.fn(), + })), +})); jest.mock('server/lib/kubernetes'); jest.mock('server/lib/random', () => ({ randomAlphanumeric: jest.fn().mockReturnValue('k4hlde'), @@ -1702,11 +1714,16 @@ describe('Native Helm', () => { describe('nativeHelmDeploy', () => { it('uses the canonical deploy job name for monitoring and archival', async () => { + const outputPatch: any = { + patch: jest.fn(() => outputPatch), + where: jest.fn().mockResolvedValue(1), + }; const deploy = { uuid: 'sample-cosmos-emulator-preview-build-123456', sha: 'abcdef1234567890', branchName: 'main', id: 42, + runUUID: 'run-c', deployableId: 99, deployable: { name: 'sample-cosmos-emulator', @@ -1719,9 +1736,7 @@ describe('Native Helm', () => { pullRequest: { repository: { fullName: 'example-org/example-chart' } }, }, $fetchGraph: jest.fn().mockResolvedValue(undefined), - $query: jest.fn().mockReturnValue({ - patch: jest.fn().mockResolvedValue(undefined), - }), + $query: jest.fn().mockReturnValue(outputPatch), } as unknown as Deploy; const archiveLogs = jest.fn().mockResolvedValue(undefined); @@ -1733,6 +1748,7 @@ describe('Native Helm', () => { (nativeHelmUtils.getHelmConfiguration as jest.Mock).mockResolvedValue({ chartType: ChartType.PUBLIC, customValues: [], + helmSecretRefs: [{ provider: 'aws', path: 'apps/sample', key: 'token', envKey: 'TOKEN' }], valuesFiles: [], chartPath: 'prometheus-community/prometheus', releaseName: deploy.uuid, @@ -1756,6 +1772,17 @@ describe('Native Helm', () => { (getLogArchivalService as jest.Mock).mockReturnValue({ archiveLogs, }); + const processSecretRefs = jest.spyOn(SecretProcessor.prototype, 'processSecretRefs').mockResolvedValue({ + secretRefs: [], + expectedKeysPerSecret: { 'sample-secret': ['TOKEN'] }, + syncTokensPerSecret: { 'sample-secret': 'sync-c' }, + warnings: [], + }); + const waitForSecretSync = jest.spyOn(SecretProcessor.prototype, 'waitForSecretSync').mockResolvedValue(); + const secretMutationGate = jest.fn(async (_deploy: Deploy, action: () => Promise) => ({ + admitted: true as const, + value: await action(), + })); const setTimeoutSpy = jest.spyOn(global, 'setTimeout').mockImplementation(((fn: TimerHandler) => { if (typeof fn === 'function') { @@ -1764,7 +1791,7 @@ describe('Native Helm', () => { return 0 as any; }) as typeof setTimeout); - const result = await nativeHelmDeploy(deploy, { namespace: 'testns' }); + const result = await nativeHelmDeploy(deploy, { namespace: 'testns', secretMutationGate }); const expectedJobName = buildDeployJobName({ deployUuid: deploy.uuid, jobId: 'k4hlde', @@ -1772,6 +1799,16 @@ describe('Native Helm', () => { }); expect(waitForJobAndGetLogs).toHaveBeenCalledWith(expectedJobName, 'testns', `[HELM ${deploy.uuid}]`); + expect(secretMutationGate).toHaveBeenCalledWith(deploy, expect.any(Function)); + expect(processSecretRefs).toHaveBeenCalled(); + expect(waitForSecretSync).toHaveBeenCalledWith({ 'sample-secret': ['TOKEN'] }, 'testns', 60000, { + 'sample-secret': 'sync-c', + }); + expect(shellPromise).toHaveBeenCalledWith(expect.stringContaining('kubectl apply -f '), { + timeout: 90_000, + }); + expect(shellPromise).not.toHaveBeenCalledWith(expect.stringContaining('--request-timeout'), expect.anything()); + expect(outputPatch.where).toHaveBeenCalledWith({ id: 42, runUUID: 'run-c' }); expect(archiveLogs).toHaveBeenCalledWith(expect.objectContaining({ jobName: expectedJobName }), 'helm logs'); expect(result).toEqual({ completed: true, @@ -1783,6 +1820,53 @@ describe('Native Helm', () => { }); }); + describe('deployHelm', () => { + it('rethrows authority lock loss without publishing a deployment failure', async () => { + const deploy = { + uuid: 'sample-service-preview-build-123456', + runUUID: 'run-c', + deployable: { + name: 'sample-service', + helm: { deploymentMethod: 'native' }, + }, + build: { namespace: 'testns' }, + $fetchGraph: jest.fn().mockResolvedValue(undefined), + } as unknown as Deploy; + const lockError = new AuthorityLockLostError('deploy-external-secrets.42'); + const secretMutationGate = jest.fn().mockRejectedValue(lockError); + const patchAndUpdateActivityFeed = jest + .spyOn(DeployService.prototype, 'patchAndUpdateActivityFeed') + .mockResolvedValue(undefined); + const recordDeployFailure = jest.spyOn(DeployService.prototype, 'recordDeployFailure').mockResolvedValue(false); + + (nativeHelmUtils.validateHelmConfiguration as jest.Mock).mockResolvedValueOnce([]); + (nativeHelmUtils.getHelmConfiguration as jest.Mock).mockResolvedValueOnce({ + helmSecretRefs: [{ provider: 'aws', path: 'apps/sample', key: 'token', envKey: 'TOKEN' }], + }); + mockGetAllConfigs.mockResolvedValue({}); + + try { + await expect(deployHelm([deploy], { secretMutationGate })).rejects.toBe(lockError); + + expect(secretMutationGate).toHaveBeenCalledWith(deploy, expect.any(Function)); + expect(Metrics).not.toHaveBeenCalled(); + expect(recordDeployFailure).not.toHaveBeenCalled(); + expect(patchAndUpdateActivityFeed).toHaveBeenCalledTimes(1); + expect(patchAndUpdateActivityFeed).toHaveBeenCalledWith( + deploy, + { + status: DeployStatus.DEPLOYING, + statusMessage: 'Deploying via Native Helm', + }, + 'run-c' + ); + } finally { + patchAndUpdateActivityFeed.mockRestore(); + recordDeployFailure.mockRestore(); + } + }); + }); + describe('generateHelmManifest', () => { it('builds the expected manifest for plain native Helm without a renderer', async () => { mockGetAllConfigs.mockResolvedValue({ diff --git a/src/server/lib/nativeHelm/helm.ts b/src/server/lib/nativeHelm/helm.ts index 0a52cfdc..78e0f843 100644 --- a/src/server/lib/nativeHelm/helm.ts +++ b/src/server/lib/nativeHelm/helm.ts @@ -59,6 +59,14 @@ import { buildHelmSecretVolumeMounts, HelmSecretSetFile, } from 'server/lib/helm/secretValueRefs'; +import { AuthorityLockLostError, type AuthorityLockResult } from 'server/lib/authorityLock'; +import { DeploymentSupersededError } from 'server/lib/deploymentReconciliation/errors'; + +export type HelmSecretMutationGate = (deploy: Deploy, action: () => Promise) => Promise>; + +export interface HelmDeploymentExecutionOptions { + secretMutationGate?: HelmSecretMutationGate; +} export interface JobResult { completed: boolean; @@ -278,7 +286,8 @@ export async function generateHelmManifest( async function processNativeHelmSecrets( deploy: Deploy, namespace: string, - helmConfig: HelmConfiguration + helmConfig: HelmConfiguration, + secretMutationGate?: HelmSecretMutationGate ): Promise { const deployable = requireDeployable(deploy); const globalConfig = await GlobalConfigService.getInstance().getAllConfigs(); @@ -293,13 +302,19 @@ async function processNativeHelmSecrets( } const secretProcessor = new SecretProcessor(globalConfig.secretProviders); - const secretResult = await secretProcessor.processSecretRefs({ - secretRefs, - serviceName: deployable.name, - namespace, - buildUuid: deploy.uuid, - strict: true, - }); + const processSecrets = () => + secretProcessor.processSecretRefs({ + secretRefs, + serviceName: deployable.name, + namespace, + buildUuid: deploy.uuid, + strict: true, + }); + const mutation = secretMutationGate + ? await secretMutationGate(deploy, processSecrets) + : ({ admitted: true, value: await processSecrets() } as const); + if (!mutation.admitted) throw new DeploymentSupersededError(); + const secretResult = mutation.value; const secretNames = Object.keys(secretResult.expectedKeysPerSecret); if (secretNames.length === 0) { @@ -320,7 +335,10 @@ async function processNativeHelmSecrets( ); } -export async function nativeHelmDeploy(deploy: Deploy, options: HelmDeployOptions): Promise { +export async function nativeHelmDeploy( + deploy: Deploy, + options: HelmDeployOptions & HelmDeploymentExecutionOptions +): Promise { await deploy.$fetchGraph('build.pullRequest.repository'); await deploy.$fetchGraph('deployable.repository'); @@ -340,18 +358,22 @@ export async function nativeHelmDeploy(deploy: Deploy, options: HelmDeployOption }); const helmConfig = await getHelmConfiguration(deploy); - await processNativeHelmSecrets(deploy, namespace, helmConfig); + await processNativeHelmSecrets(deploy, namespace, helmConfig, options.secretMutationGate); const manifest = await generateHelmManifest(deploy, jobName, options, helmConfig); const localPath = `${MANIFEST_PATH}/helm/${deploy.uuid}-helm-${shortSha}`; await fs.promises.mkdir(`${MANIFEST_PATH}/helm/`, { recursive: true }); await fs.promises.writeFile(localPath, manifest, 'utf8'); - await shellPromise(`kubectl apply -f ${localPath}`); + await shellPromise(`kubectl apply -f ${localPath}`, { timeout: 90_000 }); const jobResult = await waitForJobAndGetLogs(jobName, namespace, `[HELM ${deploy.uuid}]`); - await deploy.$query().patch({ buildOutput: jobResult.logs }); + let outputPatch = deploy.$query().patch({ buildOutput: jobResult.logs }); + if (deploy.id != null && deploy.runUUID) { + outputPatch = outputPatch.where({ id: deploy.id, runUUID: deploy.runUUID }); + } + await outputPatch; const globalConfig = await GlobalConfigService.getInstance().getAllConfigs(); if (globalConfig.logArchival?.enabled) { @@ -403,7 +425,7 @@ export async function shouldUseNativeHelm(deploy: Deploy): Promise { return false; } -export async function deployNativeHelm(deploy: Deploy): Promise { +export async function deployNativeHelm(deploy: Deploy, options: HelmDeploymentExecutionOptions = {}): Promise { const deployable = requireDeployable(deploy); const { build } = deploy; @@ -416,6 +438,7 @@ export async function deployNativeHelm(deploy: Deploy): Promise { const jobResult = await nativeHelmDeploy(deploy, { namespace: build.namespace, + ...options, }); if (jobResult.status !== 'succeeded') { @@ -485,7 +508,7 @@ async function deployCodefreshHelm(deploy: Deploy, deployService: DeployService, } } -export async function deployHelm(deploys: Deploy[]): Promise { +export async function deployHelm(deploys: Deploy[], options: HelmDeploymentExecutionOptions = {}): Promise { if (deploys?.length === 0) return; getLogger().info(`Helm: deploying services=${deploys.map((d) => d.deployable?.name || d.uuid).join(',')}`); @@ -520,7 +543,7 @@ export async function deployHelm(deploys: Deploy[]): Promise { ); if (useNative) { - await deployNativeHelm(deploy); + await deployNativeHelm(deploy, options); } else { await deployCodefreshHelm(deploy, deployService, runUUID); } @@ -536,6 +559,7 @@ export async function deployHelm(deploys: Deploy[]): Promise { await trackHelmDeploymentMetrics(deploy, 'success', Date.now() - startTime); } catch (error) { + if (error instanceof DeploymentSupersededError || error instanceof AuthorityLockLostError) throw error; const errorMessage = error instanceof Error ? error.message : String(error); await trackHelmDeploymentMetrics(deploy, 'failure', Date.now() - startTime, errorMessage); diff --git a/src/server/lib/tests/deploymentManager.test.ts b/src/server/lib/tests/deploymentManager.test.ts index bc4661b8..89eac1c8 100644 --- a/src/server/lib/tests/deploymentManager.test.ts +++ b/src/server/lib/tests/deploymentManager.test.ts @@ -14,14 +14,18 @@ * limitations under the License. */ -import { DeploymentManager } from '../deploymentManager/deploymentManager'; +import { DeploymentManager, DeploymentSupersededError } from '../deploymentManager/deploymentManager'; import { Deploy } from 'server/models'; import { buildDeployJobName } from '../kubernetes/jobNames'; -// import { deployHelm } from '../helm'; +import { deployHelm } from '../helm'; +import { shouldUseNativeHelm } from '../nativeHelm'; jest.mock('../helm', () => ({ deployHelm: jest.fn().mockResolvedValue(void 0), })); +jest.mock('../nativeHelm', () => ({ + shouldUseNativeHelm: jest.fn().mockResolvedValue(false), +})); jest.mock('../kubernetesApply/applyManifest', () => ({ createKubernetesApplyJob: jest.fn().mockResolvedValue(void 0), monitorKubernetesJob: jest.fn().mockResolvedValue({ success: true, message: 'ok', logs: 'apply logs' }), @@ -69,6 +73,13 @@ describe('DeploymentManager', () => { beforeEach(() => { jest.clearAllMocks(); + (deployHelm as jest.Mock).mockReset().mockResolvedValue(undefined); + (shouldUseNativeHelm as jest.Mock).mockReset().mockResolvedValue(false); + (createKubernetesApplyJob as jest.Mock).mockReset().mockResolvedValue(undefined); + (monitorKubernetesJob as jest.Mock) + .mockReset() + .mockResolvedValue({ success: true, message: 'ok', logs: 'apply logs' }); + (waitForDeployPodReady as jest.Mock).mockReset().mockResolvedValue({ ready: true }); (GlobalConfigService.getInstance as jest.Mock).mockReturnValue({ getAllConfigs: mockGetAllConfigs, }); @@ -220,6 +231,9 @@ describe('DeploymentManager', () => { describe('dependency cycles', () => { function cyclicDeploy(name: string, dependsOn: string[], patch: jest.Mock) { return { + id: name.split('').reduce((total, character) => total + character.charCodeAt(0), 0), + uuid: `${name}-uuid`, + runUUID: `run-${name}`, deployable: { name, deploymentDependsOn: [...dependsOn], type: 'helm' }, service: { type: 'helm' }, $query: () => ({ patch }), @@ -243,7 +257,7 @@ describe('DeploymentManager', () => { it('fails cycle members and their dependents with a cycle message instead of leaving them queued', async () => { const patchByName = new Map(); const make = (name: string, dependsOn: string[]) => { - const patch = jest.fn().mockResolvedValue(undefined); + const patch = jest.fn().mockReturnValue({ where: jest.fn().mockResolvedValue(1) }); patchByName.set(name, patch); return cyclicDeploy(name, dependsOn, patch); }; @@ -275,6 +289,175 @@ describe('DeploymentManager', () => { }); }); + describe('provider-aware promotion', () => { + function runnableDeploy(name: string, type: string, deploymentDependsOn: string[] = []): Deploy { + return { + id: name.split('').reduce((total, character) => total + character.charCodeAt(0), 0), + uuid: `${name}-preview-build-123456`, + sha: 'abcdef1234567890', + manifest: 'apiVersion: v1\nkind: ConfigMap', + runUUID: `run-${name}`, + build: { namespace: 'testns' }, + deployable: { name, type, deploymentDependsOn: [...deploymentDependsOn] }, + service: { type }, + $query: () => ({ + patch: jest.fn().mockReturnValue({ where: jest.fn().mockResolvedValue(1) }), + }), + $fetchGraph: jest.fn().mockResolvedValue(undefined), + } as unknown as Deploy; + } + + it('admits all native siblings for a level together while Codefresh and pod readiness stay outside', async () => { + const nativeHelm = runnableDeploy('native-chart', 'helm'); + const codefreshHelm = runnableDeploy('codefresh-chart', 'helm'); + const kubernetes = runnableDeploy('web', 'github'); + const events: string[] = []; + let insideGate = false; + + (shouldUseNativeHelm as jest.Mock).mockImplementation(async (deploy: Deploy) => { + return deploy.deployable.name === 'native-chart'; + }); + (deployHelm as jest.Mock).mockImplementation(async (deploysToRun: Deploy[]) => { + events.push(`${deploysToRun[0].deployable.name}:${insideGate}`); + }); + (createKubernetesApplyJob as jest.Mock).mockImplementation(async () => { + events.push(`apply:${insideGate}`); + }); + (monitorKubernetesJob as jest.Mock).mockImplementation(async () => { + events.push(`monitor:${insideGate}`); + return { success: true, message: 'ok', logs: 'apply logs' }; + }); + (waitForDeployPodReady as jest.Mock).mockImplementation(async () => { + events.push(`ready:${insideGate}`); + return { ready: true }; + }); + + const nativeMutationGate = jest.fn(async (action: () => Promise) => { + insideGate = true; + try { + return { admitted: true as const, value: await action() }; + } finally { + insideGate = false; + } + }); + const nativeSecretMutationGate = jest.fn(); + + const manager = new DeploymentManager([nativeHelm, codefreshHelm, kubernetes], { + isCurrent: async () => true, + nativeMutationGate: nativeMutationGate as any, + nativeSecretMutationGate: nativeSecretMutationGate as any, + }); + + await manager.deploy(); + + expect(nativeMutationGate).toHaveBeenCalledTimes(1); + expect(deployHelm).toHaveBeenCalledWith([nativeHelm], { + secretMutationGate: nativeSecretMutationGate, + }); + expect(events).toEqual( + expect.arrayContaining([ + 'codefresh-chart:false', + 'native-chart:true', + 'apply:true', + 'monitor:true', + 'ready:false', + ]) + ); + }); + + it('keeps the promotion gate until every admitted native sibling is terminal', async () => { + const nativeHelm = runnableDeploy('native-chart', 'helm'); + const kubernetes = runnableDeploy('web', 'github'); + let insideGate = false; + let releaseMonitor!: () => void; + let markMonitorStarted!: () => void; + const monitorStarted = new Promise((resolve) => { + markMonitorStarted = resolve; + }); + + (shouldUseNativeHelm as jest.Mock).mockResolvedValue(true); + (deployHelm as jest.Mock).mockRejectedValue(new Error('helm failed')); + (monitorKubernetesJob as jest.Mock).mockImplementation(async () => { + markMonitorStarted(); + await new Promise((resolve) => { + releaseMonitor = resolve; + }); + return { success: true, message: 'ok', logs: 'apply logs' }; + }); + + const nativeMutationGate = jest.fn(async (action: () => Promise) => { + insideGate = true; + try { + return { admitted: true as const, value: await action() }; + } finally { + insideGate = false; + } + }); + const manager = new DeploymentManager([nativeHelm, kubernetes], { + nativeMutationGate: nativeMutationGate as any, + }); + + const deployment = manager.deploy(); + await monitorStarted; + expect(insideGate).toBe(true); + + releaseMonitor(); + await expect(deployment).rejects.toThrow('helm failed'); + expect(insideGate).toBe(false); + expect(waitForDeployPodReady).toHaveBeenCalledWith(kubernetes); + }); + + it('treats denied native admission as supersession without creating a job or recording failure', async () => { + const deploy = runnableDeploy('web', 'github'); + const manager = new DeploymentManager([deploy], { + nativeMutationGate: (async () => ({ admitted: false as const })) as any, + }); + + await expect(manager.deploy()).rejects.toBeInstanceOf(DeploymentSupersededError); + + expect(createKubernetesApplyJob).not.toHaveBeenCalled(); + expect(monitorKubernetesJob).not.toHaveBeenCalled(); + expect(mockRecordDeployFailure).not.toHaveBeenCalled(); + }); + + it('stops as superseded when the run-fenced queued patch affects no row', async () => { + const deploy = runnableDeploy('web', 'github'); + const where = jest.fn().mockResolvedValue(0); + deploy.$query = () => ({ patch: jest.fn().mockReturnValue({ where }) } as any); + const manager = new DeploymentManager([deploy]); + + await expect(manager.deploy()).rejects.toBeInstanceOf(DeploymentSupersededError); + + expect(where).toHaveBeenCalledWith({ id: deploy.id, runUUID: deploy.runUUID }); + expect(createKubernetesApplyJob).not.toHaveBeenCalled(); + expect(mockRecordDeployFailure).not.toHaveBeenCalled(); + }); + + it('checks authority before advancing to the next dependency level', async () => { + const first = runnableDeploy('first', 'github'); + const second = runnableDeploy('second', 'github', ['first']); + let current = true; + + (waitForDeployPodReady as jest.Mock).mockImplementation(async () => { + current = false; + return { ready: true }; + }); + + const manager = new DeploymentManager([first, second], { + isCurrent: async () => current, + nativeMutationGate: (async (action: () => Promise) => ({ + admitted: true as const, + value: await action(), + })) as any, + }); + + await expect(manager.deploy()).rejects.toBeInstanceOf(DeploymentSupersededError); + + expect(createKubernetesApplyJob).toHaveBeenCalledTimes(1); + expect(mockRecordDeployFailure).not.toHaveBeenCalled(); + }); + }); + describe('deployManifests', () => { it('monitors the canonical truncated deploy job name for long deploy uuids', async () => { const deploy = { @@ -298,7 +481,8 @@ describe('DeploymentManager', () => { deploymentManager = new DeploymentManager([deploy]); - await deploymentManager['deployManifests'](deploy); + const deployment = await deploymentManager['applyManifests'](deploy); + await deploymentManager['waitForManifestReadiness'](deployment); expect(createKubernetesApplyJob).toHaveBeenCalledWith({ deploy, @@ -335,7 +519,8 @@ describe('DeploymentManager', () => { deploymentManager = new DeploymentManager([deploy]); - await expect(deploymentManager['deployManifests'](deploy)).rejects.toThrow( + const deployment = await deploymentManager['applyManifests'](deploy); + await expect(deploymentManager['waitForManifestReadiness'](deployment)).rejects.toThrow( 'Pods failed to become ready within timeout: pod web-1: web waiting=ImagePullBackOff (Back-off pulling image "x") restarts=0' ); expect(mockRecordDeployFailure).toHaveBeenCalledWith( @@ -378,7 +563,8 @@ describe('DeploymentManager', () => { deploymentManager = new DeploymentManager([deploy]); - await deploymentManager['deployManifests'](deploy); + const deployment = await deploymentManager['applyManifests'](deploy); + await deploymentManager['waitForManifestReadiness'](deployment); expect(mockArchiveLogs).toHaveBeenCalledWith( expect.objectContaining({ diff --git a/src/server/lib/tests/prlessSourceResolution.test.ts b/src/server/lib/tests/prlessSourceResolution.test.ts index d90ddbbf..bf5d9c86 100644 --- a/src/server/lib/tests/prlessSourceResolution.test.ts +++ b/src/server/lib/tests/prlessSourceResolution.test.ts @@ -132,4 +132,29 @@ describe('webhook yaml source resolution for PR-less builds', () => { expect(mockFetchLifecycleConfigByRepository).toHaveBeenCalledWith(repository, 'push-sha'); }); + + it('imports PR webhook configuration from the immutable delivered source ref', async () => { + const repository = { fullName: 'org/repo' }; + mockFetchLifecycleConfigByRepository.mockResolvedValue({ environment: { webhooks: [] } }); + const service = new WebhookService( + {} as any, + {} as any, + {} as any, + { registerQueue: jest.fn(() => ({ add: jest.fn() })) } as any + ); + const build: any = { + uuid: 'pr-env-222222', + triggerType: 'github_pr', + $query: jest.fn(() => ({ patch: jest.fn() })), + }; + const pullRequest: any = { + branchName: 'feature', + repository, + $fetchGraph: jest.fn().mockResolvedValue(undefined), + }; + + await service.upsertWebhooksWithYaml(build, pullRequest, 'push-sha'); + + expect(mockFetchLifecycleConfigByRepository).toHaveBeenCalledWith(repository, 'push-sha'); + }); }); diff --git a/src/server/models/Build.ts b/src/server/models/Build.ts index 0abf38bf..c10587a8 100644 --- a/src/server/models/Build.ts +++ b/src/server/models/Build.ts @@ -15,6 +15,7 @@ */ import { BuildKind, DeployStatus } from 'shared/constants'; +import type { AcceptedDeploymentRefs } from 'server/lib/deploymentReconciliation/mailbox'; import { Deploy, Deployable, Environment, PullRequest, Service } from '.'; import Model from './_Model'; @@ -82,6 +83,10 @@ export default class Build extends Model { dependencyGraph: Record; namespace: string; + desiredGeneration!: number; + observedGeneration!: number; + acceptedRefs!: AcceptedDeploymentRefs; + static tableName = 'builds'; static timestamps = true; @@ -159,6 +164,6 @@ export default class Build extends Model { }; static get jsonAttributes() { - return ['commentInitEnv', 'commentRuntimeEnv']; + return ['commentInitEnv', 'commentRuntimeEnv', 'acceptedRefs']; } } diff --git a/src/server/models/Deployable.ts b/src/server/models/Deployable.ts index 42aa119f..da11489a 100644 --- a/src/server/models/Deployable.ts +++ b/src/server/models/Deployable.ts @@ -35,6 +35,7 @@ export default class Deployable extends Service { appShort?: string; ecr?: string; helm: Helm; + requires: string[]; deploymentDependsOn: string[]; builder: Builder; envLens?: boolean; diff --git a/src/server/services/__tests__/apiEnvironment.test.ts b/src/server/services/__tests__/apiEnvironment.test.ts index 2476b84c..278587dc 100644 --- a/src/server/services/__tests__/apiEnvironment.test.ts +++ b/src/server/services/__tests__/apiEnvironment.test.ts @@ -142,6 +142,10 @@ function makeService({ } = {}) { const buildCreate = jest.fn(async (attrs: any) => createdBuild ?? { id: 99, ...attrs }); const buildFindOneChain = { whereNull: jest.fn().mockResolvedValue(existingIdempotent) }; + const deployQueryChain: any = { + patch: jest.fn(() => deployQueryChain), + where: jest.fn(() => deployQueryChain), + }; const models = { Build: { create: buildCreate, @@ -168,7 +172,7 @@ function makeService({ })), }, Environment: { query: jest.fn(() => ({ findById: jest.fn().mockResolvedValue(environment) })) }, - Deploy: { query: jest.fn() }, + Deploy: { query: jest.fn(() => deployQueryChain) }, Deployable: { query: jest.fn() }, }; const services = { @@ -1017,12 +1021,46 @@ describe('sweepExpiredApiEnvironments', () => { whereNotIn: jest.fn().mockReturnThis(), whereNull: jest.fn().mockResolvedValue(expired), }; - const stuckChain: any = { + const apiTeardownScope: any = { where: jest.fn().mockReturnThis(), whereIn: jest.fn().mockReturnThis(), + }; + const pullRequestTeardownState: any = { + where: jest.fn().mockReturnThis(), + orWhereRaw: jest.fn().mockReturnThis(), + }; + const pullRequestTeardownScope: any = { + where: jest.fn((condition: unknown) => { + if (typeof condition === 'function') condition(pullRequestTeardownState); + return pullRequestTeardownScope; + }), + whereNotNull: jest.fn().mockReturnThis(), + whereNot: jest.fn().mockReturnThis(), + }; + const teardownScope: any = { + where: jest.fn((callback: (builder: any) => void) => { + callback(apiTeardownScope); + return teardownScope; + }), + orWhere: jest.fn((callback: (builder: any) => void) => { + callback(pullRequestTeardownScope); + return teardownScope; + }), + }; + const stuckChain: any = { + where: jest.fn((condition: unknown) => { + if (typeof condition === 'function') condition(teardownScope); + return stuckChain; + }), whereNull: jest.fn().mockResolvedValue(stuck), }; - return { expiredChain, stuckChain }; + return { + expiredChain, + stuckChain, + apiTeardownScope, + pullRequestTeardownScope, + pullRequestTeardownState, + }; }; it('enqueues deletion only for expired live API builds', async () => { @@ -1067,26 +1105,41 @@ describe('sweepExpiredApiEnvironments', () => { expect(result).toEqual({ expired: 2, stuckTeardowns: 0, enqueued: 1 }); }); - it('re-enqueues teardown for API builds stuck in tearing_down beyond the grace window', async () => { - const stuck = [{ id: 9, uuid: 'stuck-env-999999' }]; - const { expiredChain, stuckChain } = sweepChains([], stuck); + it('re-enqueues stuck API teardown and both phases of PR teardown beyond the grace window', async () => { + const stuck = [ + { id: 9, uuid: 'stuck-env-999999' }, + { id: 10, uuid: 'stuck-pr-101010', pullRequestId: 44 }, + { id: 11, uuid: 'claimed-pr-111111', pullRequestId: 45, runUUID: 'build-teardown-11' }, + ]; + const { expiredChain, stuckChain, apiTeardownScope, pullRequestTeardownScope, pullRequestTeardownState } = + sweepChains([], stuck); const { service, models } = makeService(); (models.Build.query as jest.Mock).mockReturnValueOnce(expiredChain).mockReturnValueOnce(stuckChain); const enqueue = jest.spyOn(service, 'enqueueBuildDeletion').mockResolvedValue(); const result = await service.sweepExpiredApiEnvironments(); - expect(stuckChain.where).toHaveBeenCalledWith('triggerType', 'api'); - expect(stuckChain.whereIn).toHaveBeenCalledWith('status', [BuildStatus.TEARING_DOWN, BuildStatus.TORN_DOWN]); + expect(apiTeardownScope.where).toHaveBeenCalledWith('triggerType', 'api'); + expect(apiTeardownScope.whereIn).toHaveBeenCalledWith('status', [BuildStatus.TEARING_DOWN, BuildStatus.TORN_DOWN]); + expect(pullRequestTeardownScope.whereNotNull).toHaveBeenCalledWith('pullRequestId'); + expect(pullRequestTeardownScope.whereNot).toHaveBeenCalledWith('status', BuildStatus.TORN_DOWN); + expect(pullRequestTeardownState.where).toHaveBeenCalledWith('status', BuildStatus.TEARING_DOWN); + expect(pullRequestTeardownState.orWhereRaw).toHaveBeenCalledWith('?? = concat(?, ??)', [ + 'runUUID', + 'build-teardown-', + 'id', + ]); expect(stuckChain.where).toHaveBeenCalledWith( 'updatedAt', '<=', new Date(NOW.getTime() - 15 * 60 * 1000).toISOString() ); expect(stuckChain.whereNull).toHaveBeenCalledWith('deletedAt'); - expect(enqueue).toHaveBeenCalledTimes(1); + expect(enqueue).toHaveBeenCalledTimes(3); expect(enqueue).toHaveBeenCalledWith(stuck[0], 'teardown_stuck'); - expect(result).toEqual({ expired: 0, stuckTeardowns: 1, enqueued: 1 }); + expect(enqueue).toHaveBeenCalledWith(stuck[1], 'teardown_stuck'); + expect(enqueue).toHaveBeenCalledWith(stuck[2], 'teardown_stuck'); + expect(result).toEqual({ expired: 0, stuckTeardowns: 3, enqueued: 3 }); }); }); @@ -1144,6 +1197,25 @@ describe('enqueueBuildDeletion', () => { ]); }); + it('does not coalesce PR teardown recovery with an authoritative destroy', async () => { + const { service, queueAdd } = makeService(); + const build = { + id: 7, + uuid: 'pr-env-123456', + status: BuildStatus.DEPLOYED, + pullRequestId: 55, + runUUID: 'build-teardown-7', + } as any; + + await service.enqueueBuildDeletion(build, 'teardown_stuck'); + await service.enqueueBuildDeletion(build, 'manual_destroy'); + + expect(queueAdd.mock.calls.map((call) => call[2].jobId)).toEqual([ + expect.stringMatching(/^build-delete-7-conditional-/), + 'build-delete-7-authoritative', + ]); + }); + it('never reuses a conditional jobId so a fresh close always enqueues teardown', async () => { const { service, queueAdd } = makeService(); const build = { id: 7, uuid: 'pr-env-123456', status: BuildStatus.DEPLOYED } as any; @@ -1450,7 +1522,6 @@ describe('requestApiEnvironmentDeletion', () => { expect.objectContaining({ buildId: current.id, runUUID: expect.any(String), - triggerRef: expect.any(String), }) ); @@ -1608,9 +1679,7 @@ describe('processApiEnvironmentCreateQueue', () => { }) ); expect(updateStatus).toHaveBeenCalledWith(build, BuildStatus.PENDING, expect.any(String), true, true); - expect(enqueueResolve).toHaveBeenCalledWith( - expect.objectContaining({ buildId: 7, triggerRef: expect.any(String) }) - ); + expect(enqueueResolve).toHaveBeenCalledWith(expect.objectContaining({ buildId: 7, runUUID: expect.any(String) })); }); it('skips the deploy chain when deploys are paused', async () => { @@ -2076,7 +2145,6 @@ describe('applyApiEnvironmentPatch', () => { expect(enqueue).toHaveBeenCalledWith({ buildId: 1, runUUID: result.deployId, - triggerRef: result.deployId, }); }); @@ -2767,7 +2835,6 @@ describe('redeploy and destroy uuid liveness', () => { expect.objectContaining({ buildId: 4, runUUID: result.deployId, - triggerRef: result.deployId, }) ); }); @@ -2800,12 +2867,10 @@ describe('redeploy and destroy uuid liveness', () => { expect.objectContaining({ buildId: 4, runUUID: first.deployId, - triggerRef: first.deployId, }), expect.objectContaining({ buildId: 4, runUUID: second.deployId, - triggerRef: second.deployId, }), ]); }); @@ -3029,6 +3094,52 @@ describe('processDeleteQueue', () => { ); }); + it('holds deployment authority while acquiring promotion and running cleanup', async () => { + const build = { id: 3, uuid: 'x', expiresAt: null }; + const { service, deleteBuild } = makeDeleteQueueService(build); + const events: string[] = []; + const deploymentLock: any = { + extend: jest.fn().mockResolvedValue(undefined), + unlock: jest.fn(async () => events.push('unlock:deployment')), + }; + const promotionLock: any = { + extend: jest.fn().mockResolvedValue(undefined), + unlock: jest.fn(async () => events.push('unlock:promotion')), + }; + const plainLock = jest.fn(); + const lockWithOptions = jest.fn(async (resource: string) => { + events.push(`lock:${resource}`); + return resource === 'build-deployment.3' ? deploymentLock : promotionLock; + }); + (service as any).redlock = { lock: plainLock, lockWithOptions }; + deleteBuild.mockImplementation(async () => { + events.push('cleanup'); + }); + + await service.processDeleteQueue(deleteJob('api_delete')); + + expect(events).toEqual([ + 'lock:build-deployment.3', + 'lock:build-promotion.3', + 'cleanup', + 'unlock:promotion', + 'unlock:deployment', + ]); + expect(plainLock).not.toHaveBeenCalled(); + expect(lockWithOptions).toHaveBeenNthCalledWith( + 1, + 'build-deployment.3', + 15 * 60 * 1000, + expect.objectContaining({ retryCount: 4 }) + ); + expect(lockWithOptions).toHaveBeenNthCalledWith( + 2, + 'build-promotion.3', + 15 * 60 * 1000, + expect.objectContaining({ retryCount: 4 }) + ); + }); + it('serializes an expiry claim ahead of a concurrent extension so teardown wins cleanly', async () => { const sharedBuild: any = { id: 3, @@ -3077,7 +3188,7 @@ describe('processDeleteQueue', () => { }; (models.Build.query as jest.Mock).mockImplementation(() => ({ findById: jest.fn(() => deleteLock), - findOne: jest.fn(() => extensionLock), + findOne: jest.fn((identity: any) => (identity.id === sharedBuild.id ? sharedBuild : extensionLock)), patchAndFetchById: jest.fn(), })); const deleteBuild = jest.fn().mockResolvedValue(undefined); @@ -3120,11 +3231,70 @@ describe('processDeleteQueue', () => { await service.processDeleteQueue(deleteJob('pull_request_closed')); - expect(build.$fetchGraph).toHaveBeenCalledWith('pullRequest', expect.objectContaining({ transaction: {} })); + expect(build.$fetchGraph).toHaveBeenCalledWith('pullRequest'); expect(claimPatch).not.toHaveBeenCalled(); expect(deleteBuild).not.toHaveBeenCalled(); }); + it('cancels a claimed PR deletion when the PR reopens while promotion is busy', async () => { + const build = { + id: 3, + uuid: 'x', + pullRequestId: 44, + pullRequest: { status: 'closed', deployOnUpdate: true }, + $fetchGraph: jest.fn().mockResolvedValue(undefined), + }; + const { service, deleteBuild, claimPatch } = makeDeleteQueueService(build); + const deploymentLock = { + extend: jest.fn().mockResolvedValue(undefined), + unlock: jest.fn().mockResolvedValue(undefined), + }; + let promotionAttempted!: () => void; + const promotionWasAttempted = new Promise((resolve) => { + promotionAttempted = resolve; + }); + (service as any).redlock = { + lock: jest.fn(), + lockWithOptions: jest.fn(async (resource: string) => { + if (resource === 'build-deployment.3') return deploymentLock; + build.pullRequest.status = 'open'; + promotionAttempted(); + throw new Error('promotion busy'); + }), + }; + + const processing = service.processDeleteQueue(deleteJob('pull_request_closed')); + await promotionWasAttempted; + await jest.advanceTimersByTimeAsync(250); + await expect(processing).resolves.toBeUndefined(); + + expect(claimPatch).toHaveBeenCalledWith({ runUUID: 'build-teardown-3' }); + expect(build.status).toBe(BuildStatus.DEPLOYED); + expect(deleteBuild).not.toHaveBeenCalled(); + expect(deploymentLock.unlock).toHaveBeenCalledTimes(1); + }); + + it('finishes a same-owner PR teardown retry after cleanup was admitted and the PR reopened', async () => { + const build = { + id: 3, + uuid: 'x', + status: BuildStatus.TEARING_DOWN, + runUUID: 'build-teardown-3', + pullRequestId: 44, + pullRequest: { status: 'open', deployOnUpdate: true }, + $fetchGraph: jest.fn().mockResolvedValue(undefined), + }; + const { service, deleteBuild, claimPatch } = makeDeleteQueueService(build); + + await service.processDeleteQueue(deleteJob('pull_request_closed')); + + expect(claimPatch).not.toHaveBeenCalled(); + expect(deleteBuild).toHaveBeenCalledWith( + build, + expect.objectContaining({ reason: 'pull_request_closed', runUUID: 'build-teardown-3' }) + ); + }); + it('keeps a manual destroy authoritative after a PR is re-enabled', async () => { const build = { id: 3, @@ -3165,6 +3335,55 @@ describe('processDeleteQueue', () => { expect(deleteBuild).not.toHaveBeenCalled(); }); + it('does not let a stale stuck-teardown job claim a live build', async () => { + const build = { id: 3, uuid: 'x', status: BuildStatus.DEPLOYED, runUUID: 'run-c', expiresAt: null }; + const { service, deleteBuild, expiryLock, claimPatch } = makeDeleteQueueService(build); + + await service.processDeleteQueue(deleteJob('teardown_stuck')); + + expect(expiryLock.forUpdate).not.toHaveBeenCalled(); + expect(claimPatch).not.toHaveBeenCalled(); + expect(deleteBuild).not.toHaveBeenCalled(); + }); + + it('resumes a stuck PR teardown claim that failed before publishing tearing_down', async () => { + const build = { + id: 3, + uuid: 'x', + status: BuildStatus.DEPLOYED, + runUUID: 'build-teardown-3', + pullRequestId: 44, + pullRequest: { status: 'closed', deployOnUpdate: true }, + $fetchGraph: jest.fn().mockResolvedValue(undefined), + }; + const { service, deleteBuild } = makeDeleteQueueService(build); + + await service.processDeleteQueue(deleteJob('teardown_stuck')); + + expect(deleteBuild).toHaveBeenCalledWith( + build, + expect.objectContaining({ reason: 'teardown_stuck', runUUID: 'build-teardown-3' }) + ); + }); + + it('cancels a stuck pre-status PR teardown claim after deploy authority returns', async () => { + const build = { + id: 3, + uuid: 'x', + status: BuildStatus.DEPLOYED, + runUUID: 'build-teardown-3', + pullRequestId: 44, + pullRequest: { status: 'open', deployOnUpdate: true }, + $fetchGraph: jest.fn().mockResolvedValue(undefined), + }; + const { service, deleteBuild, expiryLock } = makeDeleteQueueService(build); + + await service.processDeleteQueue(deleteJob('teardown_stuck')); + + expect(expiryLock.forUpdate).not.toHaveBeenCalled(); + expect(deleteBuild).not.toHaveBeenCalled(); + }); + it('lets only one concurrent delete job clean up before the vanity name can be reused', async () => { const sharedBuild: any = { id: 3, @@ -3194,7 +3413,10 @@ describe('processDeleteQueue', () => { whereNull: jest.fn().mockReturnThis(), forUpdate: jest.fn(async () => (sharedBuild.deletedAt ? undefined : sharedBuild)), }; - (models.Build.query as jest.Mock).mockImplementation(() => ({ findById: jest.fn(() => lock) })); + (models.Build.query as jest.Mock).mockImplementation(() => ({ + findById: jest.fn(() => lock), + findOne: jest.fn().mockResolvedValue(sharedBuild), + })); let finishCleanup!: () => void; const cleanupCanFinish = new Promise((resolve) => { @@ -3357,6 +3579,50 @@ describe('deleteBuild teardown ownership', () => { expect(updateStatus).toHaveBeenCalledWith(build, BuildStatus.TEARING_DOWN, 'run-before', true, true); }); + it.each(['pull_request_closed', 'teardown_stuck'])( + 'cancels %s PR cleanup when authority returns before teardown starts', + async (reason) => { + const { service, updateStatus } = makeTeardownService(); + const { build } = makeTeardownBuild({ + pullRequestId: 55, + pullRequest: { status: 'open', deployOnUpdate: true }, + runUUID: 'build-teardown-7', + idempotencyKey: null, + }); + const kubernetes = jest.requireMock('server/lib/kubernetes'); + + await service.deleteBuild(build, { + reason, + runUUID: 'build-teardown-7', + deploymentLockAlreadyHeld: true, + }); + + expect(updateStatus).not.toHaveBeenCalled(); + expect(kubernetes.deleteNamespace).not.toHaveBeenCalled(); + } + ); + + it('finishes conditional PR cleanup after the same owner published tearing_down', async () => { + const { service, updateStatus } = makeTeardownService(); + const { build } = makeTeardownBuild({ + status: BuildStatus.TEARING_DOWN, + pullRequestId: 55, + pullRequest: { status: 'open', deployOnUpdate: true }, + runUUID: 'build-teardown-7', + idempotencyKey: null, + }); + const kubernetes = jest.requireMock('server/lib/kubernetes'); + + await service.deleteBuild(build, { + reason: 'pull_request_closed', + runUUID: 'build-teardown-7', + deploymentLockAlreadyHeld: true, + }); + + expect(updateStatus).toHaveBeenCalledWith(build, BuildStatus.TEARING_DOWN, 'build-teardown-7', true, true); + expect(kubernetes.deleteNamespace).toHaveBeenCalledWith('env-api-env-123456'); + }); + it('soft-deletes torn-down API environments so the vanity uuid is released', async () => { const { service } = makeTeardownService(); const { build, patch } = makeTeardownBuild({ kind: 'environment', triggerType: 'api' }); @@ -3424,272 +3690,45 @@ describe('deleteBuild teardown ownership', () => { }); }); -describe('deploy pipeline torn-down guards', () => { - const pipelineBuild = (overrides: any = {}) => ({ - id: 7, - uuid: 'api-env-123456', - status: BuildStatus.TORN_DOWN, - deletedAt: null, - deployEnabled: true, - pullRequest: null, - $fetchGraph: jest.fn().mockResolvedValue(undefined), - ...overrides, - }); - - function makePipelineService(build: any) { - const { service, models } = makeService(); - (models.Build.query as jest.Mock).mockReturnValue({ findOne: jest.fn().mockResolvedValue(build) }); - return { service }; - } - - it('processResolveAndDeployBuildQueue skips torn-down PR-less builds', async () => { - const build = pipelineBuild(); - const { service } = makePipelineService(build); - const enqueueBuildJob = jest.spyOn(service as any, 'enqueueBuildJob').mockResolvedValue(undefined); - - await service.processResolveAndDeployBuildQueue({ data: { buildId: 7 } }); - - expect(enqueueBuildJob).not.toHaveBeenCalled(); - }); - - it('processResolveAndDeployBuildQueue lets a re-enabled open PR recreate a torn-down build', async () => { - const build = pipelineBuild({ - pullRequestId: 55, - pullRequest: { status: 'open', deployOnUpdate: true, $fetchGraph: jest.fn() }, - }); - const { service } = makePipelineService(build); - const enqueueBuildJob = jest.spyOn(service as any, 'enqueueBuildJob').mockResolvedValue(undefined); - - await service.processResolveAndDeployBuildQueue({ data: { buildId: 7 } }); - - expect(enqueueBuildJob).toHaveBeenCalled(); - }); - - it('processBuildQueue skips torn-down PR-less builds before importing yaml', async () => { - const build = pipelineBuild({ status: BuildStatus.TEARING_DOWN }); - const { service } = makePipelineService(build); - const importYaml = jest.spyOn(service as any, 'importYamlConfigFile').mockResolvedValue(undefined); - const resolveAndDeploy = jest.fn().mockResolvedValue(undefined); - (service.db.services as any).BuildService = { resolveAndDeployBuild: resolveAndDeploy }; - - await service.processBuildQueue({ data: { buildId: 7 } }); - - expect(importYaml).not.toHaveBeenCalled(); - expect(resolveAndDeploy).not.toHaveBeenCalled(); - }); - - it('processBuildQueue lets a re-enabled open PR reclaim and recreate a torn-down build', async () => { - const build = pipelineBuild({ - pullRequestId: 55, - pullRequest: { status: 'open', deployOnUpdate: true, $fetchGraph: jest.fn() }, - }); - const { service } = makePipelineService(build); - jest.spyOn(service as any, 'importYamlConfigFile').mockResolvedValue(undefined); - jest.spyOn(service as any, 'claimDeploymentRun').mockResolvedValue('recreate-run'); - jest.spyOn(service as any, 'isDeploymentRunCurrent').mockResolvedValue(true); - const resolveAndDeploy = jest.spyOn(service, 'resolveAndDeployBuild').mockResolvedValue(build); - - await service.processBuildQueue({ data: { buildId: 7 } }); - - expect(resolveAndDeploy).toHaveBeenCalledWith( - build, - true, - undefined, - undefined, - expect.objectContaining({ - deploymentLockAlreadyHeld: true, - runAlreadyClaimed: true, - runUUID: 'recreate-run', - }) - ); - }); - - it.each([ - { pullRequest: { status: 'open', deployOnUpdate: false }, label: 'label-disabled' }, - { pullRequest: { status: 'closed', deployOnUpdate: true }, label: 'closed' }, - ])('processBuildQueue fails closed for a $label PR', async ({ pullRequest }) => { - const build = pipelineBuild({ - status: BuildStatus.DEPLOYED, - pullRequestId: 55, - pullRequest: { ...pullRequest, $fetchGraph: jest.fn() }, - }); - const { service } = makePipelineService(build); - const importYaml = jest.spyOn(service as any, 'importYamlConfigFile').mockResolvedValue(undefined); - - await service.processBuildQueue({ data: { buildId: 7 } }); - - expect(importYaml).not.toHaveBeenCalled(); - }); -}); - -describe('serialized build setup and deploy workers', () => { - it('serializes YAML import through deploy completion for distinct triggers on one build', async () => { +describe('build setup authority', () => { + it('re-reads PR authority under the setup lock before importing YAML', async () => { const { service } = makeService(); const build: any = { id: 7, uuid: 'pr-env-123456', - status: BuildStatus.DEPLOYED, + status: BuildStatus.QUEUED, pullRequestId: 55, - pullRequest: { status: 'open', deployOnUpdate: true }, - environment: { id: 5 }, - $fetchGraph: jest.fn().mockResolvedValue(undefined), + pullRequest: { status: 'closed', deployOnUpdate: true }, }; + jest.spyOn(service as any, 'findOrCreateBuild').mockResolvedValue(build); jest.spyOn(service as any, 'loadBuildDeploymentAuthority').mockResolvedValue(build); - jest - .spyOn(service as any, 'claimDeploymentRun') - .mockImplementation(async (_build: any, requested: string) => requested); - jest.spyOn(service as any, 'isDeploymentRunCurrent').mockResolvedValue(true); + const lock = jest.spyOn(service as any, 'withCurrentBuildDeploymentLock'); const importYaml = jest.spyOn(service as any, 'importYamlConfigFile').mockResolvedValue(undefined); - let releaseFirst!: () => void; - const firstCanFinish = new Promise((resolve) => { - releaseFirst = resolve; - }); - let firstStarted!: () => void; - const firstDidStart = new Promise((resolve) => { - firstStarted = resolve; - }); - const resolveAndDeploy = jest - .spyOn(service, 'resolveAndDeployBuild') - .mockImplementation(async (_build, _enabled, _repo, sourceRef) => { - if (sourceRef === 'sha-a') { - firstStarted(); - await firstCanFinish; - } - return build; - }); - - let lockTail = Promise.resolve(undefined); - jest.spyOn(service, 'withBuildDeploymentLock').mockImplementation((_buildId, action) => { - const run = lockTail.then(action); - lockTail = run.then( - () => undefined, - () => undefined - ); - return run; - }); - - const first = service.processBuildQueue({ - data: { buildId: 7, runUUID: 'run-a', triggerRef: 'sha-a', sourceRef: 'sha-a' }, - }); - await firstDidStart; - const second = service.processBuildQueue({ - data: { buildId: 7, runUUID: 'run-b', triggerRef: 'sha-b', sourceRef: 'sha-b' }, - }); - - await Promise.resolve(); - expect(importYaml).toHaveBeenCalledTimes(1); - expect(resolveAndDeploy).toHaveBeenCalledTimes(1); - - releaseFirst(); - await Promise.all([first, second]); - - expect(importYaml.mock.calls.map((call) => (call[3] as { sourceRef?: string } | undefined)?.sourceRef)).toEqual([ - 'sha-a', - 'sha-b', - ]); - expect(resolveAndDeploy.mock.calls.map((call) => call[3])).toEqual(['sha-a', 'sha-b']); - }); - - it('skips a delayed older trigger before YAML import after a newer sequence claimed the scope', async () => { - const { service } = makeService(); - const build: any = { - id: 7, - uuid: 'pr-env-123456', - status: BuildStatus.DEPLOYED, - pullRequestId: 55, - pullRequest: { status: 'open', deployOnUpdate: true }, - environment: { id: 5 }, - $fetchGraph: jest.fn().mockResolvedValue(undefined), - }; - jest.spyOn(service as any, 'loadBuildDeploymentAuthority').mockResolvedValue(build); - jest.spyOn(service as any, 'resolveEffectiveSourceRef').mockResolvedValue('old-sha'); - jest.spyOn(service as any, 'claimTriggerSequence').mockResolvedValue(false); - const claimRun = jest.spyOn(service as any, 'claimDeploymentRun'); - const importYaml = jest.spyOn(service as any, 'importYamlConfigFile'); - - await service.processBuildQueue({ - data: { - buildId: 7, - githubRepositoryId: 42, - sourceGithubRepositoryId: 42, - sourceBranch: 'main', - triggerSequence: '101', - triggerRef: 'old-sha', - sourceRef: 'old-sha', - }, - }); + await service.createBuild({ id: 5 } as any, { pullRequestId: 55, repositoryId: 1 }, {} as any); - expect(claimRun).not.toHaveBeenCalled(); + expect(lock).toHaveBeenCalledWith(7, expect.any(Function), expect.any(Function)); expect(importYaml).not.toHaveBeenCalled(); }); - it('converges a stale pushed source to the live branch head instead of dropping the push', async () => { - const { service } = makeService(); - const build: any = { - id: 7, - uuid: 'api-env-123456', - status: BuildStatus.DEPLOYED, - deployEnabled: true, - deletedAt: null, - pullRequest: null, - pullRequestId: null, - environment: { id: 5 }, - $fetchGraph: jest.fn().mockResolvedValue(undefined), - }; - jest.spyOn(service as any, 'loadBuildDeploymentAuthority').mockResolvedValue(build); - const resolveRef = jest.spyOn(service as any, 'resolveEffectiveSourceRef').mockResolvedValue('newest-sha'); - const claimSequence = jest.spyOn(service as any, 'claimTriggerSequence').mockResolvedValue(true); - jest.spyOn(service as any, 'claimDeploymentRun').mockResolvedValue('run-1'); - jest.spyOn(service as any, 'isDeploymentRunCurrent').mockResolvedValue(true); - const importYaml = jest.spyOn(service as any, 'importYamlConfigFile').mockResolvedValue(undefined); - const resolveAndDeploy = jest.spyOn(service, 'resolveAndDeployBuild').mockResolvedValue(build); - - await service.processBuildQueue({ - data: { - buildId: 7, - sourceGithubRepositoryId: 42, - sourceBranch: 'Main', - sourceRef: 'older-sha', - triggerSequence: '101', - }, - }); - - expect(resolveRef).toHaveBeenCalledWith(42, 'Main', 'older-sha'); - expect(claimSequence).toHaveBeenCalled(); - expect(importYaml).toHaveBeenCalledWith( - build.environment, - build, - undefined, - expect.objectContaining({ sourceRef: 'newest-sha', sourceBranch: 'Main' }) - ); - expect(resolveAndDeploy).toHaveBeenCalledWith(build, true, undefined, 'newest-sha', expect.any(Object)); - }); - - it('rethrows lock acquisition failures so BullMQ retries instead of dropping the trigger', async () => { - const { service } = makeService(); - jest.spyOn(service, 'withBuildDeploymentLock').mockRejectedValue(new Error('lock acquisition timed out')); - - await expect(service.processBuildQueue({ data: { buildId: 7 } })).rejects.toThrow('lock acquisition timed out'); - }); - - it('re-reads PR authority under the setup lock before importing YAML', async () => { + it('does not enter the legacy lock retry budget while PR teardown is active', async () => { const { service } = makeService(); const build: any = { id: 7, uuid: 'pr-env-123456', - status: BuildStatus.QUEUED, + status: BuildStatus.TEARING_DOWN, pullRequestId: 55, - pullRequest: { status: 'closed', deployOnUpdate: true }, + pullRequest: { status: 'open', deployOnUpdate: true }, }; jest.spyOn(service as any, 'findOrCreateBuild').mockResolvedValue(build); jest.spyOn(service as any, 'loadBuildDeploymentAuthority').mockResolvedValue(build); - const lock = jest.spyOn(service, 'withBuildDeploymentLock'); + const lockWithOptions = jest.fn(); + (service as any).redlock = { lock: jest.fn(), lockWithOptions }; const importYaml = jest.spyOn(service as any, 'importYamlConfigFile').mockResolvedValue(undefined); await service.createBuild({ id: 5 } as any, { pullRequestId: 55, repositoryId: 1 }, {} as any); - expect(lock).toHaveBeenCalledWith(7, expect.any(Function)); + expect(lockWithOptions).not.toHaveBeenCalled(); expect(importYaml).not.toHaveBeenCalled(); }); @@ -3710,215 +3749,23 @@ describe('serialized build setup and deploy workers', () => { }) ).toBe('deploy_disabled'); }); -}); - -describe('resolveAndDeployBuild teardown-ownership abort', () => { - function makeDeployRunService( - currentRow: (patchedRunUUID: string | null) => any, - { claimResult = 1 }: { claimResult?: number } = {} - ) { - const { service, models, services } = makeService(); - let patchedRunUUID: string | null = null; - const build: any = { - id: 7, - uuid: 'api-env-123456', - namespace: 'env-api-env-123456', - branchName: 'main', - configSha: 'abc', - githubRepositoryId: 42, - pullRequest: null, - pullRequestId: null, - status: BuildStatus.DEPLOYED, - deletedAt: null, - deployEnabled: true, - environment: { id: 5 }, - $query: jest.fn(() => ({ - patch: jest.fn(async (p: any) => { - if (p?.runUUID) patchedRunUUID = p.runUUID; - }), - })), - $fetchGraph: jest.fn().mockResolvedValue(undefined), - $setRelated: jest.fn(), - }; - const claimChain: any = { - where: jest.fn(() => claimChain), - whereNotIn: jest.fn(() => claimChain), - whereNull: jest.fn(() => claimChain), - then: (resolve: any, reject: any) => Promise.resolve(claimResult).then(resolve, reject), - }; - const claimPatch = jest.fn((p: any) => { - if (p?.runUUID && claimResult > 0) patchedRunUUID = p.runUUID; - return claimChain; - }); - (models.Build.query as jest.Mock).mockReturnValue({ - findById: jest.fn(() => ({ - select: jest.fn(async () => { - const current: any = { - id: build.id, - pullRequestId: build.pullRequestId, - status: build.status, - deletedAt: build.deletedAt, - deployEnabled: build.deployEnabled, - ...currentRow(patchedRunUUID), - }; - current.$fetchGraph = jest.fn(async () => { - current.pullRequest = build.pullRequest; - }); - return current; - }), - })), - patch: claimPatch, - }); - const { Repository: RepositoryMock } = jest.requireMock('server/models'); - (RepositoryMock as any).query = jest.fn(() => ({ - findOne: jest.fn(() => ({ - whereNull: jest.fn().mockResolvedValue({ fullName: 'org/repo' }), - })), - })); - (services.Deploy.findOrCreateDeploys as jest.Mock).mockResolvedValue([{ id: 1 }]); - (service.db.services as any).BuildService = service; - jest.spyOn(service as any, 'markConfigurationsAsBuilt').mockResolvedValue(undefined); - const updateStatus = jest - .spyOn(service as any, 'updateStatusAndComment') - .mockResolvedValue(undefined) as jest.SpyInstance; - const buildImages = jest.spyOn(service as any, 'buildImages').mockResolvedValue(true) as jest.SpyInstance; - jest.spyOn(service as any, 'deployCLIServices').mockResolvedValue(true); - const applyManifests = jest - .spyOn(service as any, 'generateAndApplyManifests') - .mockResolvedValue(true) as jest.SpyInstance; - const recordFailure = jest - .spyOn(service as any, 'recordBuildFailure') - .mockResolvedValue(undefined) as jest.SpyInstance; - return { - service, - models, - build, - claimChain, - claimPatch, - buildImages, - applyManifests, - updateStatus, - recordFailure, - }; - } - - it('aborts the manifest apply when teardown took runUUID ownership mid-flight', async () => { - const { service, build, applyManifests, recordFailure, updateStatus } = makeDeployRunService(() => ({ - runUUID: 'teardown-owner', - status: BuildStatus.TEARING_DOWN, - })); - - await service.resolveAndDeployBuild(build, true); - - expect(applyManifests).not.toHaveBeenCalled(); - expect(recordFailure).not.toHaveBeenCalled(); - expect(updateStatus).not.toHaveBeenCalledWith(build, BuildStatus.DEPLOYED, expect.any(String), true, true); - }); - - it('aborts the manifest apply when an API environment is paused mid-run', async () => { - const { service, build, applyManifests, recordFailure } = makeDeployRunService((patchedRunUUID) => ({ - runUUID: patchedRunUUID, - status: BuildStatus.DEPLOYING, - deployEnabled: false, - })); - - await service.resolveAndDeployBuild(build, true); - - expect(applyManifests).not.toHaveBeenCalled(); - expect(recordFailure).not.toHaveBeenCalled(); - }); - - it('applies manifests when the run still owns the build', async () => { - const { service, build, applyManifests, updateStatus } = makeDeployRunService((patchedRunUUID) => ({ - runUUID: patchedRunUUID, - status: 'deploying', - })); - - await service.resolveAndDeployBuild(build, true); - - expect(applyManifests).toHaveBeenCalled(); - expect(updateStatus).toHaveBeenCalledWith(build, BuildStatus.DEPLOYED, expect.any(String), true, true); - }); - - it('passes a concrete repository id through deploy and environment-variable resolution', async () => { - const { service, build } = makeDeployRunService((patchedRunUUID) => ({ - runUUID: patchedRunUUID, - status: BuildStatus.BUILDING, - })); - const findOrCreateDeploys = service.db.services.Deploy.findOrCreateDeploys as jest.Mock; - - await service.resolveAndDeployBuild(build, false, 42, 'commit-ref'); - - expect(findOrCreateDeploys).toHaveBeenCalledWith(build.environment, build, 42, 'commit-ref', undefined); - }); - - it('rechecks ownership inside the apply lock when teardown starts during manifest deployment', async () => { - let ownershipReads = 0; - const { service, build, applyManifests, updateStatus, recordFailure } = makeDeployRunService((patchedRunUUID) => { - ownershipReads++; - return ownershipReads >= 5 - ? { runUUID: 'teardown-owner', status: BuildStatus.TEARING_DOWN, deployEnabled: false } - : { runUUID: patchedRunUUID, status: BuildStatus.DEPLOYING, deployEnabled: true, deletedAt: null }; - }); - const unlock = jest.fn().mockResolvedValue(undefined); - (service as any).redlock = { - lock: jest.fn().mockResolvedValue({ unlock, extend: jest.fn() }), - }; - applyManifests.mockResolvedValue(true); - - await service.resolveAndDeployBuild(build, true); - - expect(applyManifests).toHaveBeenCalledTimes(1); - expect((service as any).redlock.lock).toHaveBeenCalledWith('build-deployment.7', 15 * 60 * 1000); - expect(unlock).toHaveBeenCalledTimes(1); - expect(updateStatus).not.toHaveBeenCalledWith(build, BuildStatus.DEPLOYED, expect.any(String), true, true); - expect(recordFailure).not.toHaveBeenCalled(); - }); - - it('aborts the whole run when teardown completed before the ownership claim', async () => { - const { service, build, claimChain, buildImages, applyManifests, updateStatus, recordFailure } = - makeDeployRunService(() => ({ runUUID: 'irrelevant', status: BuildStatus.TORN_DOWN }), { claimResult: 0 }); - - await service.resolveAndDeployBuild(build, true); - - expect(claimChain.whereNotIn).toHaveBeenCalledWith('status', [BuildStatus.TEARING_DOWN, BuildStatus.TORN_DOWN]); - expect(claimChain.whereNull).toHaveBeenCalledWith('deletedAt'); - expect(claimChain.where).toHaveBeenCalledWith('deployEnabled', true); - expect(buildImages).not.toHaveBeenCalled(); - expect(applyManifests).not.toHaveBeenCalled(); - expect(updateStatus).not.toHaveBeenCalled(); - expect(recordFailure).not.toHaveBeenCalled(); - }); - - it('claims PR builds conditionally and honors current open/label authority', async () => { - const { service, models, build, claimChain, claimPatch, applyManifests } = makeDeployRunService( - (patchedRunUUID) => ({ - runUUID: patchedRunUUID, - status: BuildStatus.DEPLOYING, - }) - ); - build.pullRequestId = 55; - build.pullRequest = { - branchName: 'feature-1', - fullName: 'org/repo', - latestCommit: 'sha-1', - status: 'open', - deployOnUpdate: true, - $fetchGraph: jest.fn().mockResolvedValue(undefined), - }; - build.branchName = null; - build.githubRepositoryId = null; - await service.resolveAndDeployBuild(build, true); + it('blocks PR setup and deployment only while teardown is in progress', () => { + const { service } = makeService(); + const pullRequest = { status: 'open', deployOnUpdate: true }; - expect(claimPatch).toHaveBeenCalledWith({ - runUUID: expect.any(String), - status: BuildStatus.PENDING, - statusMessage: null, - }); - expect(claimChain.where).not.toHaveBeenCalledWith('deployEnabled', true); - expect(models.Deploy.query).not.toHaveBeenCalled(); - expect(applyManifests).toHaveBeenCalled(); + expect( + (service as any).buildSetupBlockReason({ status: BuildStatus.TEARING_DOWN, pullRequestId: 55, pullRequest }) + ).toBe('tearing_down'); + expect( + (service as any).deploymentBlockReason({ status: BuildStatus.TEARING_DOWN, pullRequestId: 55, pullRequest }) + ).toBe('tearing_down'); + expect( + (service as any).buildSetupBlockReason({ status: BuildStatus.TORN_DOWN, pullRequestId: 55, pullRequest }) + ).toBeNull(); + expect( + (service as any).deploymentBlockReason({ status: BuildStatus.TORN_DOWN, pullRequestId: 55, pullRequest }) + ).toBeNull(); }); }); @@ -3963,7 +3810,15 @@ describe('recordBuildFailure teardown ownership', () => { await (service as any).recordBuildFailure(build, BuildStatus.ERROR, 'new-run', new Error('boom'), 'fallback'); - expect(updateStatus).toHaveBeenCalledWith(build, BuildStatus.ERROR, 'new-run', true, true, expect.any(Error)); + expect(updateStatus).toHaveBeenCalledWith( + build, + BuildStatus.ERROR, + 'new-run', + true, + true, + expect.any(Error), + undefined + ); }); it('re-stamps PR builds unconditionally', async () => { diff --git a/src/server/services/__tests__/build.test.ts b/src/server/services/__tests__/build.test.ts index 70bdcd2d..1a638992 100644 --- a/src/server/services/__tests__/build.test.ts +++ b/src/server/services/__tests__/build.test.ts @@ -24,6 +24,12 @@ const mockQueueAdd = jest.fn(); const mockCleanupDeploy = jest.fn(); const mockDeleteServiceRows = jest.fn(); const mockGetServiceOverrideStates = jest.fn(); +const mockGenerateGraph = jest.fn().mockResolvedValue({}); +const mockAcceptDeploymentIntent = jest.fn().mockResolvedValue({ + accepted: true, + generation: 1, + scopeKey: 'all', +}); jest.mock('server/lib/dependencies', () => ({ defaultDb: {}, @@ -43,6 +49,14 @@ jest.mock('server/lib/tracer', () => ({ }, })); +jest.mock('server/lib/deploymentReconciliation/mailbox', () => { + const actual = jest.requireActual('server/lib/deploymentReconciliation/mailbox'); + return { + ...actual, + acceptDeploymentIntent: (...args: any[]) => mockAcceptDeploymentIntent(...args), + }; +}); + jest.mock('server/lib/logger', () => ({ getLogger: jest.fn(() => ({ error: jest.fn(), @@ -61,6 +75,7 @@ jest.mock('shared/config', () => ({ QUEUE_NAMES: { DELETE_QUEUE: 'delete_queue_test', BUILD_QUEUE: 'build_queue_test', + DEPLOYMENT_RECONCILIATION: 'deployment_reconciliation_test', RESOLVE_AND_DEPLOY: 'resolve_and_deploy_test', BUILD_CLEANUP_QUEUE: 'build_cleanup_test', BUILD_REQUEST_QUEUE: 'build_request_test', @@ -96,6 +111,7 @@ jest.mock('server/lib/github', () => ({ updateGitDeploymentStatus: jest.fn(), getPullRequest: jest.fn(), getSHAForBranch: jest.fn(), + compareCommits: jest.fn(), getYamlFileContentFromBranch: jest.fn(), })); @@ -113,6 +129,10 @@ jest.mock('server/lib/buildEnvVariables', () => ({ })), })); +jest.mock('server/lib/dependencyGraph', () => ({ + generateGraph: (...args: any[]) => mockGenerateGraph(...args), +})); + jest.mock('server/services/globalConfig', () => ({ __esModule: true, default: { @@ -440,10 +460,22 @@ describe('BuildService build response queries', () => { }); describe('BuildService status updates', () => { + const statusQuery = (affectedRows = 1) => { + const query: any = { + patch: jest.fn(() => query), + where: jest.fn(() => query), + whereNull: jest.fn(() => query), + then: (resolve: (value: number) => void, reject: (reason: unknown) => void) => + Promise.resolve(affectedRows).then(resolve, reject), + }; + return query; + }; + test('updates only build status fields', async () => { - const patch = jest.fn().mockResolvedValue(undefined); + const query = statusQuery(); const buildService = new BuildService( { + models: { Build: { query: jest.fn(() => query) } }, services: { Webhook: { webhookQueue: { @@ -470,23 +502,24 @@ describe('BuildService status updates', () => { deploys: undefined, reload: jest.fn().mockResolvedValue(undefined), $fetchGraph: jest.fn().mockResolvedValue(undefined), - $query: jest.fn(() => ({ patch })), }; await buildService.updateStatusAndComment(build as any, BuildStatus.DEPLOYED, 'run-1', true, true); - expect(patch).toHaveBeenCalledTimes(1); - expect(patch).toHaveBeenCalledWith({ + expect(query.patch).toHaveBeenCalledTimes(1); + expect(query.patch).toHaveBeenCalledWith({ status: BuildStatus.DEPLOYED, statusMessage: '', }); + expect(query.where).toHaveBeenCalledWith({ id: 1, runUUID: 'run-1' }); }); test('does not abort teardown status progress when webhook notification enqueue fails', async () => { - const patch = jest.fn().mockResolvedValue(undefined); + const query = statusQuery(); const webhookAdd = jest.fn().mockRejectedValue(new Error('redis unavailable')); const buildService = new BuildService( { + models: { Build: { query: jest.fn(() => query) } }, services: { Webhook: { webhookQueue: { add: webhookAdd } }, }, @@ -510,19 +543,56 @@ describe('BuildService status updates', () => { pullRequest: null, reload: jest.fn().mockResolvedValue(undefined), $fetchGraph: jest.fn().mockResolvedValue(undefined), - $query: jest.fn(() => ({ patch })), }; await expect( buildService.updateStatusAndComment(build as any, BuildStatus.TEARING_DOWN, 'run-1', true, true) ).resolves.toBeUndefined(); - expect(patch).toHaveBeenCalledWith({ + expect(query.patch).toHaveBeenCalledWith({ status: BuildStatus.TEARING_DOWN, statusMessage: '', }); expect(webhookAdd).toHaveBeenCalledTimes(1); }); + + test('does not publish after a newer desired generation takes ownership', async () => { + const query = statusQuery(0); + const webhookAdd = jest.fn(); + const activityUpdate = jest.fn(); + const buildService = new BuildService( + { + models: { Build: { query: jest.fn(() => query) } }, + services: { + ActivityStream: { updatePullRequestActivityStream: activityUpdate }, + Webhook: { webhookQueue: { add: webhookAdd } }, + }, + } as any, + {} as any, + {} as any, + { + registerQueue: jest.fn(() => ({ add: mockQueueAdd, process: jest.fn(), on: jest.fn() })), + } as any + ); + const build = { + id: 1, + uuid: 'sample-build', + runUUID: 'run-a', + status: BuildStatus.DEPLOYING, + kind: BuildKind.ENVIRONMENT, + deploys: [], + pullRequest: { repository: {} }, + reload: jest.fn().mockResolvedValue(undefined), + $fetchGraph: jest.fn().mockResolvedValue(undefined), + }; + + await buildService.updateStatusAndComment(build as any, BuildStatus.DEPLOYED, 'run-a', true, true, null, 2); + + expect(query.where).toHaveBeenCalledWith('desiredGeneration', 2); + expect(build.status).toBe(BuildStatus.DEPLOYING); + expect(activityUpdate).not.toHaveBeenCalled(); + expect(webhookAdd).not.toHaveBeenCalled(); + }); }); describe('BuildService destroyBuildEnvironment', () => { @@ -942,6 +1012,66 @@ describe('BuildService stale deploy reconciliation', () => { expect(build.$fetchGraph).toHaveBeenCalledWith('[deployables, deploys]'); }); + test('a false cleanup result retains database rows for a retry', async () => { + createService([{ id: 1, name: 'old-api' }], [{ id: 77, uuid: 'old-api-build-1', deployableId: 1 }]); + mockCleanupDeploy.mockResolvedValue(false); + + await (buildService as any).reconcileDeletedDeployables(createBuild(), { + canReconcile: true, + deployables: [], + reconcileEligibleDeployables: [], + }); + + expect(mockDeleteServiceRows).not.toHaveBeenCalled(); + }); + + test('runs only stale native teardown through the current generation promotion gate', async () => { + const staleDeploy = { id: 77, uuid: 'old-api-build-1', deployableId: 1 }; + createService([{ id: 1, name: 'old-api' }], [staleDeploy]); + const nativeAction = jest.fn().mockResolvedValue(['native-clean']); + mockCleanupDeploy.mockImplementation(async (_deploy: any, options: any) => { + await options.nativeMutationGate(nativeAction); + return true; + }); + const promotion = jest + .spyOn(buildService, 'withCurrentBuildPromotionLock') + .mockImplementation(async (_buildId, _isCurrent, action) => ({ admitted: true, value: await action() })); + jest.spyOn(buildService as any, 'isDeploymentRunCurrent').mockResolvedValue(true); + + await (buildService as any).reconcileDeletedDeployables( + createBuild(), + { canReconcile: true, deployables: [], reconcileEligibleDeployables: [] }, + undefined, + undefined, + 'run-c', + 3 + ); + + expect(promotion).toHaveBeenCalledWith(10, expect.any(Function), nativeAction); + expect(mockDeleteServiceRows).toHaveBeenCalledWith({ buildId: 10, deployableIds: [1] }); + }); + + test('does not delete stale rows when native teardown loses generation authority', async () => { + createService([{ id: 1, name: 'old-api' }], [{ id: 77, uuid: 'old-api-build-1', deployableId: 1 }]); + mockCleanupDeploy.mockImplementation(async (_deploy: any, options: any) => { + await options.nativeMutationGate(async () => true); + return true; + }); + jest.spyOn(buildService, 'withCurrentBuildPromotionLock').mockResolvedValue({ admitted: false }); + + await expect( + (buildService as any).reconcileDeletedDeployables( + createBuild(), + { canReconcile: true, deployables: [], reconcileEligibleDeployables: [] }, + undefined, + undefined, + 'run-c', + 3 + ) + ).rejects.toThrow('Deployment generation was superseded'); + expect(mockDeleteServiceRows).not.toHaveBeenCalled(); + }); + test('a partial cleanup failure deletes only the successfully cleaned rows', async () => { createService( [ @@ -1012,462 +1142,755 @@ describe('BuildService stale deploy reconciliation', () => { build, targetRepoId, undefined, - undefined + undefined, + targetRepoId ); expect(reconcileDeletedDeployables).not.toHaveBeenCalled(); expect(upsertWebhooksWithYaml).toHaveBeenCalledWith(build, build.pullRequest, null); }); }); - -describe('BuildService queue fingerprinting', () => { - let buildService: BuildService; - let mockBuildQuery: any; - let mockBuildQueueAdd: jest.Mock; - let mockResolveQueueAdd: jest.Mock; - let mockBuildQueueGetJob: jest.Mock; - - const createMockBuild = (overrides: any = {}) => - ({ - id: 1, - commentRuntimeEnv: { FEATURE_FLAG: 'on' }, - commentInitEnv: {}, - pullRequest: { latestCommit: 'abcdef123456', status: 'open', deployOnUpdate: true }, - deploys: [ - { - id: 11, - uuid: 'api-deploy', - githubRepositoryId: 100, - branchName: 'feature-branch', - active: true, - publicUrl: 'https://example.test/api', - env: { API_URL: 'https://api.test' }, - initEnv: { INIT_MODE: 'warm' }, - deployable: { name: 'api', commentBranchName: null }, - }, - { - id: 22, - uuid: 'worker-deploy', - githubRepositoryId: 200, - branchName: 'feature-branch', - active: true, - publicUrl: 'https://example.test/worker', - env: { QUEUE: 'jobs' }, - initEnv: {}, - deployable: { name: 'worker', commentBranchName: 'worker-override' }, - }, - ], - $fetchGraph: jest.fn().mockResolvedValue(undefined), - ...overrides, - } as any); - - beforeEach(() => { - jest.clearAllMocks(); - - mockBuildQueueAdd = jest.fn().mockResolvedValue(undefined); - mockResolveQueueAdd = jest.fn().mockResolvedValue(undefined); - mockBuildQueueGetJob = jest.fn().mockResolvedValue(undefined); - - mockBuildQuery = { - findOne: jest.fn().mockReturnThis(), - whereNull: jest.fn().mockReturnThis(), +describe('BuildService deployment reconciliation', () => { + const queueManager = () => ({ + registerQueue: jest.fn(() => ({ add: jest.fn(), process: jest.fn(), on: jest.fn() })), + }); + + const createBuild = (overrides: Record = {}) => ({ + id: 1, + uuid: 'sample-build', + status: BuildStatus.DEPLOYED, + deployEnabled: true, + pullRequest: { latestCommit: 'abcdef123456', status: 'open', deployOnUpdate: true }, + deploys: [], + ...overrides, + }); + + const serviceHarness = () => { + const buildQuery: any = { + findOne: jest.fn(() => buildQuery), + findById: jest.fn(() => buildQuery), + select: jest.fn(() => buildQuery), + whereRaw: jest.fn(() => buildQuery), + where: jest.fn(() => buildQuery), + whereNull: jest.fn(() => buildQuery), + orderBy: jest.fn(() => buildQuery), + limit: jest.fn().mockResolvedValue([]), withGraphFetched: jest.fn(), }; - - const queueManager = { - registerQueue: jest.fn(() => ({ - add: jest.fn(), - process: jest.fn(), - on: jest.fn(), - })), - }; - - buildService = new BuildService( - { - models: { - Build: { - query: jest.fn(() => mockBuildQuery), - }, - }, - services: {}, - } as any, + const service = new BuildService( + { models: { Build: { query: jest.fn(() => buildQuery) } }, services: {} } as any, {} as any, {} as any, - queueManager as any + queueManager() as any ); - (buildService as any).buildQueue = { add: mockBuildQueueAdd, getJob: mockBuildQueueGetJob }; - (buildService as any).resolveAndDeployBuildQueue = { add: mockResolveQueueAdd }; - }); + const add = jest.fn().mockResolvedValue(undefined); + (service as any).deploymentReconciliationQueue = { add }; + return { service, buildQuery, add }; + }; - test('changes fingerprint when comment runtime env changes', async () => { - const baseBuild = createMockBuild(); - const changedBuild = createMockBuild({ - commentRuntimeEnv: { FEATURE_FLAG: 'off' }, + const reconciliationWorkerHarness = () => { + const { service } = serviceHarness(); + const failure = new Error('reconciliation infrastructure failed'); + const claim = { + generation: 7, + token: 'run-current', + dirty: [{ scopeKey: 'all', intent: { type: 'all', requestId: 'run-current', gen: 7 } }], + }; + const build = createBuild({ id: 1, runUUID: claim.token }); + + jest + .spyOn(service as any, 'tryWithDeploymentGenerationLock') + .mockImplementation(async (_buildId, _generation, action) => action()); + const claimReconciliation = jest.spyOn(service as any, 'claimDeploymentReconciliation').mockResolvedValue(claim); + const withDeploymentLock = jest + .spyOn(service as any, 'withCurrentBuildDeploymentLock') + .mockImplementation(async (_buildId, _isCurrent, action) => ({ admitted: true, value: await action() })); + const loadBuild = jest.spyOn(service as any, 'loadBuildDeploymentAuthority').mockResolvedValue(build); + jest.spyOn(service as any, 'claimDeploymentRun').mockResolvedValue(claim.token); + jest.spyOn(service as any, 'deploymentReconciliationScopes').mockImplementation(() => { + throw failure; + }); + const isCurrent = jest.spyOn(service as any, 'isDeploymentRunCurrent').mockResolvedValue(true); + const recordFailure = jest.spyOn(service as any, 'recordBuildFailure').mockResolvedValue(undefined); + const markObserved = jest.spyOn(service as any, 'markDeploymentReconciliationObserved').mockResolvedValue(true); + + const job = (attemptsMade: number) => ({ + data: { buildId: 1, generation: claim.generation }, + attemptsMade, + opts: { attempts: 10 }, }); - const baseFingerprint = await buildService.computeBuildRequestFingerprint(baseBuild); - const changedFingerprint = await buildService.computeBuildRequestFingerprint(changedBuild); + return { + service, + failure, + claim, + build, + claimReconciliation, + withDeploymentLock, + loadBuild, + isCurrent, + recordFailure, + markObserved, + job, + }; + }; - expect(baseFingerprint).not.toEqual(changedFingerprint); + beforeEach(() => { + jest.clearAllMocks(); + mockAcceptDeploymentIntent.mockResolvedValue({ accepted: true, generation: 1, scopeKey: 'all' }); }); - test('changes fingerprint when static mode changes', async () => { - const previewBuild = createMockBuild({ - isStatic: false, - }); - const staticBuild = createMockBuild({ - isStatic: true, - }); - - const previewFingerprint = await buildService.computeBuildRequestFingerprint(previewBuild); - const staticFingerprint = await buildService.computeBuildRequestFingerprint(staticBuild); + test.each([ + ['a newer live PR generation', createBuild({ desiredGeneration: 8, runUUID: 'run-c', pullRequestId: 11 }), 7, true], + [ + 'a newer live static generation', + createBuild({ + desiredGeneration: 8, + runUUID: 'run-c', + pullRequest: null, + pullRequestId: null, + deployEnabled: true, + isStatic: true, + }), + 7, + true, + ], + ['the same generation', createBuild({ desiredGeneration: 7, runUUID: 'other-run' }), 7, false], + [ + 'a closed PR', + createBuild({ + desiredGeneration: 8, + runUUID: 'run-c', + pullRequestId: 11, + pullRequest: { status: 'closed', deployOnUpdate: true }, + }), + 7, + false, + ], + [ + 'a deploy-disabled PR', + createBuild({ + desiredGeneration: 8, + runUUID: 'run-c', + pullRequestId: 11, + pullRequest: { status: 'open', deployOnUpdate: false }, + }), + 7, + false, + ], + [ + 'an API teardown', + createBuild({ + desiredGeneration: 8, + runUUID: 'build-teardown-1', + status: BuildStatus.TEARING_DOWN, + pullRequest: null, + pullRequestId: null, + deployEnabled: false, + }), + 7, + false, + ], + [ + 'a PR teardown owner before status publication', + createBuild({ + desiredGeneration: 8, + runUUID: 'build-teardown-1', + pullRequestId: 11, + }), + 7, + false, + ], + ['a deleted Build', createBuild({ desiredGeneration: 8, deletedAt: '2026-08-06T00:00:00Z' }), 7, false], + ['a stale run without a generation', createBuild({ desiredGeneration: 8 }), undefined, false], + ])('permits stale after-build only for %s', async (_case, build, expectedGeneration, expected) => { + const { service } = serviceHarness(); + jest.spyOn(service as any, 'loadBuildDeploymentAuthority').mockResolvedValue(build); - expect(previewFingerprint).not.toEqual(staticFingerprint); + await expect( + service.isSupersededByNewerLiveDeploymentGeneration(1, expectedGeneration as number | undefined) + ).resolves.toBe(expected); }); - test('changes fingerprint when repository filter changes', async () => { - const build = createMockBuild(); + test('signals only the exact durable generation accepted by the mailbox', async () => { + const { service, add } = serviceHarness(); - const apiFingerprint = await buildService.computeBuildRequestFingerprint(build, 100); - const workerFingerprint = await buildService.computeBuildRequestFingerprint(build, 200); + await service.enqueueResolveAndDeployBuild({ buildId: 1, githubRepositoryId: 100 }); - expect(apiFingerprint).not.toEqual(workerFingerprint); + expect(add).toHaveBeenCalledWith('reconcile', { buildId: 1, generation: 1 }, { jobId: 'reconcile-1-1' }); }); - test('enqueues resolve queue with deduplication derived from the current build fingerprint', async () => { - const build = createMockBuild(); - mockBuildQuery.withGraphFetched.mockResolvedValue(build); + test('uses different queue identities for successive desired generations', async () => { + const { service, add } = serviceHarness(); + mockAcceptDeploymentIntent + .mockResolvedValueOnce({ accepted: true, generation: 8, scopeKey: 'repository:100' }) + .mockResolvedValueOnce({ accepted: true, generation: 9, scopeKey: 'repository:100' }); - const expectedFingerprint = await buildService.computeBuildRequestFingerprint(build, 100); + await service.enqueueResolveAndDeployBuild({ buildId: 1, githubRepositoryId: 100 }); + await service.enqueueResolveAndDeployBuild({ buildId: 1, githubRepositoryId: 100 }); - await buildService.enqueueResolveAndDeployBuild({ - buildId: 1, - githubRepositoryId: 100, - correlationId: 'corr-1', - }); - - expect(mockResolveQueueAdd).toHaveBeenCalledWith( - 'resolve-deploy', - expect.objectContaining({ - buildId: 1, - githubRepositoryId: 100, - correlationId: 'corr-1', - }), - expect.objectContaining({ - deduplication: { - id: `resolve:1:${expectedFingerprint}`, - ttl: 30000, - }, - }) - ); + expect(add.mock.calls[0][2]).toEqual({ jobId: 'reconcile-1-8' }); + expect(add.mock.calls[1][2]).toEqual({ jobId: 'reconcile-1-9' }); }); - test('enqueues build queue with a deterministic job id derived from the current build fingerprint', async () => { - const build = createMockBuild(); - mockBuildQuery.withGraphFetched.mockResolvedValue(build); - - const expectedFingerprint = await buildService.computeBuildRequestFingerprint(build, 100); + test('accepts a tracked push as repository-selective work with its delivered SHA floor', async () => { + const { service } = serviceHarness(); - await buildService.enqueueBuildJob({ + await service.enqueueResolveAndDeployBuild({ buildId: 1, githubRepositoryId: 100, - correlationId: 'corr-2', + sourceGithubRepositoryId: 100, + sourceBranch: 'main', + sourceRef: 'commit-c', + sourceBeforeRef: 'commit-b', + runUUID: 'request-c', }); - expect(mockBuildQueueAdd).toHaveBeenCalledWith( - 'build', - expect.objectContaining({ - buildId: 1, - githubRepositoryId: 100, - correlationId: 'corr-2', - }), - expect.objectContaining({ - jobId: `build:1:${expectedFingerprint}`, - }) - ); + expect(mockAcceptDeploymentIntent).toHaveBeenCalledWith(1, { + type: 'source', + requestId: 'request-c', + target: 'repository', + githubRepositoryId: 100, + branch: 'main', + sha: 'commit-c', + beforeSha: 'commit-b', + }); }); - test('appends triggerRef to the resolve dedupe key so distinct triggers get distinct keys', async () => { - const build = createMockBuild(); - mockBuildQuery.withGraphFetched.mockResolvedValue(build); - - const expectedFingerprint = await buildService.computeBuildRequestFingerprint(build, 100); + test('uses distinct execution tokens when two source scopes reference the same SHA', async () => { + const { service } = serviceHarness(); + mockAcceptDeploymentIntent + .mockResolvedValueOnce({ accepted: true, generation: 1, scopeKey: 'source:100:main' }) + .mockResolvedValueOnce({ accepted: true, generation: 2, scopeKey: 'source:100:release' }); - await buildService.enqueueResolveAndDeployBuild({ + await service.enqueueResolveAndDeployBuild({ buildId: 1, githubRepositoryId: 100, - triggerRef: 'commit-a', + sourceGithubRepositoryId: 100, + sourceBranch: 'main', + sourceRef: 'shared-sha', }); - await buildService.enqueueResolveAndDeployBuild({ + await service.enqueueResolveAndDeployBuild({ buildId: 1, githubRepositoryId: 100, - triggerRef: 'commit-b', + sourceGithubRepositoryId: 100, + sourceBranch: 'release', + sourceRef: 'shared-sha', }); - const firstKey = mockResolveQueueAdd.mock.calls[0][2].deduplication.id; - const secondKey = mockResolveQueueAdd.mock.calls[1][2].deduplication.id; - - expect(firstKey).toBe(`resolve:1:${expectedFingerprint}:commit-a`); - expect(secondKey).toBe(`resolve:1:${expectedFingerprint}:commit-b`); - expect(firstKey).not.toBe(secondKey); - // The trigger is forwarded into the job payload so the resolve step can hand it to the build step. - expect(mockResolveQueueAdd.mock.calls[0][1]).toEqual(expect.objectContaining({ triggerRef: 'commit-a' })); + const first = mockAcceptDeploymentIntent.mock.calls[0][1]; + const second = mockAcceptDeploymentIntent.mock.calls[1][1]; + expect(first.sha).toBe('shared-sha'); + expect(second.sha).toBe('shared-sha'); + expect(first.requestId).not.toBe(second.requestId); }); - test('keeps the exact source repository, branch, and ref in the resolve queue payload', async () => { - const build = createMockBuild(); - mockBuildQuery.withGraphFetched.mockResolvedValue(build); + test('keeps root-source provenance while requesting a full-scope pass', async () => { + const { service } = serviceHarness(); - await buildService.enqueueResolveAndDeployBuild({ + await service.enqueueResolveAndDeployBuild({ buildId: 1, - githubRepositoryId: 100, sourceGithubRepositoryId: 100, - sourceBranch: 'Feature/X', - sourceRef: 'commit-a', - triggerRef: 'commit-a', + sourceBranch: 'main', + sourceRef: 'commit-c', + runUUID: 'request-c', }); - expect(mockResolveQueueAdd).toHaveBeenCalledWith( - 'resolve-deploy', + expect(mockAcceptDeploymentIntent).toHaveBeenCalledWith( + 1, expect.objectContaining({ + type: 'source', + requestId: 'request-c', + target: 'all', githubRepositoryId: 100, - sourceGithubRepositoryId: 100, - sourceBranch: 'Feature/X', - sourceRef: 'commit-a', - }), - expect.any(Object) + sha: 'commit-c', + }) ); }); - test('keeps the dedupe key commit-agnostic when no triggerRef is provided', async () => { - const build = createMockBuild(); - mockBuildQuery.withGraphFetched.mockResolvedValue(build); - - const expectedFingerprint = await buildService.computeBuildRequestFingerprint(build, 100); - - await buildService.enqueueResolveAndDeployBuild({ buildId: 1, githubRepositoryId: 100 }); - - expect(mockResolveQueueAdd.mock.calls[0][2].deduplication.id).toBe(`resolve:1:${expectedFingerprint}`); - expect(mockResolveQueueAdd.mock.calls[0][1]).not.toHaveProperty('triggerRef'); - }); - - test('the same triggerRef yields the same build job id so genuine duplicates still coalesce', async () => { - const build = createMockBuild(); - mockBuildQuery.withGraphFetched.mockResolvedValue(build); - - const expectedFingerprint = await buildService.computeBuildRequestFingerprint(build, 100); + test('does not signal a source SHA already present in the mailbox', async () => { + const { service, add } = serviceHarness(); + mockAcceptDeploymentIntent.mockResolvedValueOnce({ + accepted: false, + generation: 7, + scopeKey: 'source:100:main', + }); - await buildService.enqueueBuildJob({ buildId: 1, githubRepositoryId: 100, triggerRef: 'commit-a' }); - await buildService.enqueueBuildJob({ buildId: 1, githubRepositoryId: 100, triggerRef: 'commit-a' }); + await service.enqueueResolveAndDeployBuild({ + buildId: 1, + githubRepositoryId: 100, + sourceGithubRepositoryId: 100, + sourceBranch: 'main', + sourceRef: 'commit-c', + }); - expect(mockBuildQueueAdd.mock.calls[0][2].jobId).toBe(`build:1:${expectedFingerprint}:commit-a`); - expect(mockBuildQueueAdd.mock.calls[1][2].jobId).toBe(mockBuildQueueAdd.mock.calls[0][2].jobId); + expect(add).not.toHaveBeenCalled(); }); - test('logs a dedupe skip when a matching build job already exists', async () => { - const build = createMockBuild(); - mockBuildQuery.withGraphFetched.mockResolvedValue(build); - mockBuildQueueGetJob.mockResolvedValue({ id: 'existing' }); - - const expectedFingerprint = await buildService.computeBuildRequestFingerprint(build, 100); + test('the recovery sweep re-signals durable pending work', async () => { + const { service, buildQuery, add } = serviceHarness(); + buildQuery.limit.mockResolvedValueOnce([{ id: 7, desiredGeneration: '3' }]); - await buildService.enqueueBuildJob({ buildId: 1, githubRepositoryId: 100, triggerRef: 'commit-a' }); + await service.enqueuePendingDeploymentReconciliations(); - expect(mockBuildQueueGetJob).toHaveBeenCalledWith(`build:1:${expectedFingerprint}:commit-a`); - // add() is still invoked; it is a no-op when the job already exists. - expect(mockBuildQueueAdd).toHaveBeenCalled(); + expect(add).toHaveBeenCalledWith('reconcile', { buildId: 7, generation: 3 }, { jobId: 'reconcile-7-3' }); }); - test('service redeploy queues scoped build without deleted-service reconciliation', async () => { - const patchAndFetch = jest.fn().mockResolvedValue(undefined); - const deploy = { + test('service redeploy uses the effective source repository and leaves row ownership to the worker', async () => { + const { service, buildQuery, add } = serviceHarness(); + const directPatch = jest.fn(); + const clicked = { id: 33, - uuid: 'pdm-db-good-dev-0', + uuid: 'pdm-db-sample-build', githubRepositoryId: 425935548, deployable: { name: 'pdm-db', - repositoryId: 425935548, + resolvedFromRepositoryId: 425935548, + repositoryId: 100, }, - $query: jest.fn(() => ({ patchAndFetch })), + $query: directPatch, }; - const build = createMockBuild({ - id: 1449, - uuid: 'good-dev-0', - deploys: [deploy], - }); - mockBuildQuery.withGraphFetched.mockResolvedValue(build); + buildQuery.withGraphFetched.mockResolvedValue( + createBuild({ + id: 1449, + uuid: 'good-dev-0', + deploys: [ + clicked, + { + id: 34, + githubRepositoryId: 425935548, + deployable: { name: 'pdm-api', resolvedFromRepositoryId: 425935548 }, + }, + { id: 35, githubRepositoryId: 999, deployable: { name: 'unrelated', resolvedFromRepositoryId: 999 } }, + ], + }) + ); - await buildService.redeployServiceFromBuild('good-dev-0', 'pdm-db'); + await service.redeployServiceFromBuild('good-dev-0', 'pdm-db'); - expect(mockResolveQueueAdd).toHaveBeenCalledWith( - 'resolve-deploy', - expect.objectContaining({ - buildId: 1449, - githubRepositoryId: 425935548, - skipDeletedServiceReconciliation: true, - }), - expect.any(Object) + expect(mockAcceptDeploymentIntent).toHaveBeenCalledWith( + 1449, + expect.objectContaining({ type: 'repository', githubRepositoryId: 425935548 }) ); - expect(patchAndFetch).toHaveBeenCalledWith({ - runUUID: expect.any(String), - }); + expect(add).toHaveBeenCalledWith('reconcile', { buildId: 1449, generation: 1 }, { jobId: 'reconcile-1449-1' }); + expect(directPatch).not.toHaveBeenCalled(); + }); + + test('service redeploy rejects a service without a source repository identity', async () => { + const { service, buildQuery, add } = serviceHarness(); + buildQuery.withGraphFetched.mockResolvedValue( + createBuild({ + deploys: [ + { + id: 33, + githubRepositoryId: null, + deployable: { name: 'pdm-db', resolvedFromRepositoryId: null, repositoryId: null }, + }, + ], + }) + ); + + await expect(service.redeployServiceFromBuild('sample-build', 'pdm-db')).rejects.toThrow( + 'Cannot redeploy pdm-db: source repository is unknown.' + ); + expect(add).not.toHaveBeenCalled(); }); - test('resolve queue preserves deleted-service reconciliation skip flag for service redeploys', async () => { - const build = createMockBuild({ - id: 1449, - uuid: 'good-dev-0', + test('repository redeploy bulk-claims every Deploy from that repository without a service-id predicate', async () => { + const deployClaim: any = { + patch: jest.fn(() => deployClaim), + where: jest.fn(() => deployClaim), + then: (resolve: (value: number) => void, reject: (reason: unknown) => void) => + Promise.resolve(2).then(resolve, reject), + }; + const deploys = [ + { id: 11, githubRepositoryId: 42, branchName: 'main' }, + { id: 12, githubRepositoryId: 42, branchName: 'release' }, + { id: 13, githubRepositoryId: 99, branchName: 'main' }, + ]; + const findOrCreateDeploys = jest.fn().mockResolvedValue(deploys); + const service = new BuildService( + { + models: { Deploy: { query: jest.fn(() => deployClaim) } }, + services: { Deploy: { findOrCreateDeploys } }, + } as any, + {} as any, + {} as any, + queueManager() as any + ); + jest.spyOn(service as any, 'isDeploymentRunCurrent').mockResolvedValue(true); + jest.spyOn(service, 'markConfigurationsAsBuilt').mockResolvedValue(undefined); + jest.spyOn(service, 'updateStatusAndComment').mockResolvedValue(undefined); + mockGenerateGraph.mockRejectedValueOnce(new Error('graph omitted from scope assertion')); + const build: any = { + id: 4, + uuid: 'large-static', + runUUID: 'run-c', + environment: { id: 7 }, pullRequest: { - latestCommit: 'abcdef123456', - status: 'open', - deployOnUpdate: true, + fullName: 'org/root', + branchName: 'main', + latestCommit: 'root-sha', + repository: { githubRepositoryId: 1 }, $fetchGraph: jest.fn().mockResolvedValue(undefined), }, $fetchGraph: jest.fn().mockResolvedValue(undefined), + $setRelated: jest.fn(), + }; + + await (service as any).prepareDeploymentScope(build, 42, undefined, { + runUUID: 'run-c', }); - mockBuildQuery.findOne.mockResolvedValue(build); - const enqueueBuildJob = jest.spyOn(buildService, 'enqueueBuildJob').mockResolvedValue(undefined as any); - - await buildService.processResolveAndDeployBuildQueue({ - data: { - buildId: 1449, - githubRepositoryId: 425935548, - skipDeletedServiceReconciliation: true, + + expect(deployClaim.patch).toHaveBeenCalledWith({ runUUID: 'run-c' }); + expect(deployClaim.where).toHaveBeenCalledWith({ buildId: 4 }); + expect(deployClaim.where).toHaveBeenCalledWith('githubRepositoryId', 42); + expect(deployClaim.where).not.toHaveBeenCalledWith('branchName', expect.anything()); + expect(deployClaim.where).not.toHaveBeenCalledWith('id', expect.anything()); + expect(deploys.map((deploy: any) => deploy.runUUID)).toEqual(['run-c', 'run-c', undefined]); + }); + + test('a stale B signal cannot claim or execute C mailbox contents', async () => { + const row = { + desiredGeneration: 3, + observedGeneration: 0, + acceptedRefs: { + 'source:100:main': { + type: 'source', + requestId: 'request-c', + target: 'repository', + githubRepositoryId: 100, + branch: 'main', + sha: 'commit-c', + gen: 3, + }, }, + }; + const query: any = { + findById: jest.fn(() => query), + whereNull: jest.fn().mockResolvedValue(row), + }; + const service = new BuildService( + { models: { Build: { query: jest.fn(() => query) } } } as any, + {} as any, + {} as any, + queueManager() as any + ); + + await expect((service as any).claimDeploymentReconciliation(1, 2)).resolves.toBeNull(); + await expect((service as any).claimDeploymentReconciliation(1, 3)).resolves.toEqual({ + generation: 3, + token: 'request-c', + dirty: [ + { + scopeKey: 'source:100:main', + intent: row.acceptedRefs['source:100:main'], + }, + ], }); + }); - expect(enqueueBuildJob).toHaveBeenCalledWith( - expect.objectContaining({ - buildId: 1449, - githubRepositoryId: 425935548, - skipDeletedServiceReconciliation: true, - }) - ); + test('repository-scoped work stays selective while a root-source intent remains full-scope', () => { + const { service } = serviceHarness(); + const scopes = (service as any).deploymentReconciliationScopes([ + { + scopeKey: 'source:100:main', + intent: { + type: 'source', + requestId: 'repo-c', + target: 'repository', + githubRepositoryId: 100, + branch: 'main', + sha: 'commit-c', + gen: 1, + }, + }, + { + scopeKey: 'source:200:main', + intent: { + type: 'source', + requestId: 'root-d', + target: 'all', + githubRepositoryId: 200, + branch: 'main', + sha: 'commit-d', + gen: 2, + }, + }, + ]); + + // The full pass runs first, then the delivered repo SHA is re-applied as a + // selective floor so a lagging live-head read cannot roll C back to B. + expect(scopes).toEqual([ + { + githubRepositoryId: null, + sourceGithubRepositoryId: 200, + sourceRef: 'commit-d', + sourceBeforeRef: undefined, + sourceBranch: 'main', + }, + { + githubRepositoryId: 100, + sourceGithubRepositoryId: 100, + sourceRef: 'commit-c', + sourceBeforeRef: undefined, + sourceBranch: 'main', + }, + ]); }); - test('forwards queue identity and immutable source refs from resolve to build', async () => { - const build = createMockBuild({ - id: 1449, - uuid: 'good-dev-0', - pullRequest: { - latestCommit: 'abcdef123456', - status: 'open', - deployOnUpdate: true, - $fetchGraph: jest.fn().mockResolvedValue(undefined), + test('manual full redeploy cannot erase an older delivered repository SHA', () => { + const { service } = serviceHarness(); + const scopes = (service as any).deploymentReconciliationScopes([ + { + scopeKey: 'source:100:main', + intent: { + type: 'source', + requestId: 'repo-c', + target: 'repository', + githubRepositoryId: 100, + branch: 'main', + sha: 'commit-c', + gen: 1, + }, }, - $fetchGraph: jest.fn().mockResolvedValue(undefined), - }); - mockBuildQuery.findOne.mockResolvedValue(build); - const enqueueBuildJob = jest.spyOn(buildService, 'enqueueBuildJob').mockResolvedValue(undefined as any); - - await buildService.processResolveAndDeployBuildQueue({ - id: '42', - data: { - buildId: 1449, - githubRepositoryId: 425935548, - triggerRef: 'head-commit-sha', - sourceRef: 'head-commit-sha', - sourceGithubRepositoryId: 425935548, - sourceBranch: 'Main', + { scopeKey: 'all', intent: { type: 'all', requestId: 'manual-all', gen: 2 } }, + ]); + + expect(scopes).toEqual([ + { githubRepositoryId: null }, + { + githubRepositoryId: 100, + sourceGithubRepositoryId: 100, + sourceRef: 'commit-c', + sourceBeforeRef: undefined, + sourceBranch: 'main', }, + ]); + }); + + test('starts the newest repository source before retained work for another repository', () => { + const { service } = serviceHarness(); + const scopes = (service as any).deploymentReconciliationScopes([ + { + scopeKey: 'source:100:main', + intent: { + type: 'source', + requestId: 'repo-a', + target: 'repository', + githubRepositoryId: 100, + branch: 'main', + sha: 'commit-a', + gen: 1, + }, + }, + { + scopeKey: 'source:200:main', + intent: { + type: 'source', + requestId: 'repo-b', + target: 'repository', + githubRepositoryId: 200, + branch: 'main', + sha: 'commit-b', + gen: 2, + }, + }, + ]); + + expect(scopes.map((scope) => scope.githubRepositoryId)).toEqual([200, 100]); + }); + + test('different generations use different execution locks so C does not wait for A', async () => { + let releaseA!: () => void; + let signalAStarted!: () => void; + const aHeld = new Promise((resolve) => { + releaseA = resolve; + }); + const aStarted = new Promise((resolve) => { + signalAStarted = resolve; }); + const lock = jest.fn(async () => ({ unlock: jest.fn().mockResolvedValue(undefined), extend: jest.fn() })); + const service = new BuildService({} as any, {} as any, { lock } as any, queueManager() as any); - expect(enqueueBuildJob).toHaveBeenCalledWith( - expect.objectContaining({ - triggerRef: 'head-commit-sha', - sourceRef: 'head-commit-sha', - sourceGithubRepositoryId: 425935548, - sourceBranch: 'Main', - triggerSequence: '42', - }) + const a = (service as any).tryWithDeploymentGenerationLock(1, 1, async () => { + signalAStarted(); + await aHeld; + }); + await aStarted; + + let cRan = false; + await (service as any).tryWithDeploymentGenerationLock(1, 3, async () => { + cRan = true; + }); + + expect(cRan).toBe(true); + expect(lock.mock.calls.map(([resource]) => resource)).toEqual(['build-reconcile.1.1', 'build-reconcile.1.3']); + releaseA(); + await a; + }); + + test('a superseded image phase cannot enter deployment rollout', async () => { + const { service } = serviceHarness(); + const build = createBuild({ id: 1, runUUID: 'run-a' }); + const isCurrent = jest + .spyOn(service as any, 'isDeploymentRunCurrent') + .mockResolvedValueOnce(true) + .mockResolvedValueOnce(false); + jest.spyOn(service, 'buildImages').mockResolvedValue(true); + jest.spyOn(service, 'deployCLIServices').mockResolvedValue(true); + const updateStatus = jest.spyOn(service, 'updateStatusAndComment').mockResolvedValue(undefined); + const applyManifests = jest.spyOn(service, 'generateAndApplyManifests').mockResolvedValue(true); + + const result = await (service as any).executeDeploymentScope( + { + build, + runUUID: 'run-a', + githubRepositoryId: 100, + sourceGithubRepositoryId: 100, + sourceRef: 'commit-a', + sourceBranch: 'main', + }, + 7 ); + + expect(result).toBeNull(); + expect(isCurrent).toHaveBeenCalledTimes(2); + expect(updateStatus).not.toHaveBeenCalled(); + expect(applyManifests).not.toHaveBeenCalled(); }); - test('rethrows build enqueue failures so the resolve job retry budget is used', async () => { - const build = createMockBuild({ id: 1449, uuid: 'good-dev-0' }); - mockBuildQuery.findOne.mockResolvedValue(build); - jest.spyOn(buildService, 'enqueueBuildJob').mockRejectedValue(new Error('redis unavailable')); + test('leaves the latest generation pending while PR teardown is in progress', async () => { + const { service, build, markObserved, job } = reconciliationWorkerHarness(); + build.status = BuildStatus.TEARING_DOWN; - await expect( - buildService.processResolveAndDeployBuildQueue({ id: '43', data: { buildId: 1449, triggerRef: 'sha' } }) - ).rejects.toThrow('redis unavailable'); + await expect(service.processDeploymentReconciliationQueue(job(0))).resolves.toBeUndefined(); + + expect((service as any).claimDeploymentRun).not.toHaveBeenCalled(); + expect((service as any).deploymentReconciliationScopes).not.toHaveBeenCalled(); + expect(markObserved).not.toHaveBeenCalled(); }); - test('rejects an out-of-order trigger after a newer sequence claimed the same build/source scope', async () => { - const values = new Map(); - (buildService as any).redis = { - get: jest.fn(async (key: string) => values.get(key) ?? null), - set: jest.fn(async (key: string, value: string) => { - values.set(key, value); - return 'OK'; - }), - }; + test('retries an intermediate generic reconciliation failure without publishing a terminal error', async () => { + const { service, failure, recordFailure, markObserved, job } = reconciliationWorkerHarness(); - await expect((buildService as any).claimTriggerSequence(7, 100, 'main', '102')).resolves.toBe(true); - await expect((buildService as any).claimTriggerSequence(7, 100, 'main', '101')).resolves.toBe(false); - await expect((buildService as any).claimTriggerSequence(7, 100, 'stable', '101')).resolves.toBe(true); - await expect((buildService as any).claimTriggerSequence(7, 200, 'main', '101')).resolves.toBe(true); - await expect((buildService as any).claimTriggerSequence(7, 100, 'main', '102')).resolves.toBe(true); - - expect(values).toEqual( - new Map([ - ['build-deployment-sequence.resolve_and_deploy_test.7.100.main', '102'], - ['build-deployment-sequence.resolve_and_deploy_test.7.100.stable', '101'], - ['build-deployment-sequence.resolve_and_deploy_test.7.200.main', '101'], - ]) - ); - for (const call of ((buildService as any).redis.set as jest.Mock).mock.calls) { - expect(call.slice(2)).toEqual(['EX', 7 * 24 * 60 * 60]); - } + await expect(service.processDeploymentReconciliationQueue(job(0))).rejects.toBe(failure); + + expect(recordFailure).not.toHaveBeenCalled(); + expect(markObserved).not.toHaveBeenCalled(); }); - test('keeps the delivered ref when it is still the live branch head', async () => { - const whereNull = jest.fn().mockResolvedValue({ githubRepositoryId: 100, fullName: 'org/repo' }); - const findOne = jest.fn(() => ({ whereNull })); - (buildService as any).db.models.Repository = { query: jest.fn(() => ({ findOne })) }; - (github.getSHAForBranch as jest.Mock).mockResolvedValue('newest-sha'); + test('publishes and observes one fenced generic failure on the final queue attempt', async () => { + const { service, failure, build, claim, recordFailure, markObserved, job } = reconciliationWorkerHarness(); + + await expect(service.processDeploymentReconciliationQueue(job(9))).resolves.toBeUndefined(); - await expect((buildService as any).resolveEffectiveSourceRef(100, 'Feature/X', 'newest-sha')).resolves.toBe( - 'newest-sha' + expect(recordFailure).toHaveBeenCalledTimes(1); + expect(recordFailure).toHaveBeenCalledWith( + build, + BuildStatus.ERROR, + claim.token, + failure, + 'Build queue processing failed.', + claim.generation ); + expect(markObserved).toHaveBeenCalledTimes(1); + expect(markObserved).toHaveBeenCalledWith(1, claim.generation, claim.token); + }); - expect(findOne).toHaveBeenCalledWith({ githubRepositoryId: 100 }); - expect(whereNull).toHaveBeenCalledWith('deletedAt'); - expect(github.getSHAForBranch).toHaveBeenCalledWith('Feature/X', 'org', 'repo'); + test('rethrows a still-current generic failure that happens before the run becomes active', async () => { + const { service, failure, loadBuild, recordFailure, markObserved, job } = reconciliationWorkerHarness(); + loadBuild.mockRejectedValueOnce(failure); + + await expect(service.processDeploymentReconciliationQueue(job(0))).rejects.toBe(failure); + + expect(recordFailure).not.toHaveBeenCalled(); + expect(markObserved).not.toHaveBeenCalled(); }); - test('converges a stale delivered ref to the live branch head instead of dropping the push', async () => { - const whereNull = jest.fn().mockResolvedValue({ githubRepositoryId: 100, fullName: 'org/repo' }); - (buildService as any).db.models.Repository = { - query: jest.fn(() => ({ findOne: jest.fn(() => ({ whereNull })) })), - }; - (github.getSHAForBranch as jest.Mock).mockResolvedValue('newest-sha'); + test('claims the token to publish and observe a pre-active failure only on its final attempt', async () => { + const { service, failure, build, claim, withDeploymentLock, loadBuild, recordFailure, markObserved, job } = + reconciliationWorkerHarness(); + loadBuild.mockRejectedValueOnce(failure); - await expect((buildService as any).resolveEffectiveSourceRef(100, 'Feature/X', 'older-sha')).resolves.toBe( - 'newest-sha' + await expect(service.processDeploymentReconciliationQueue(job(9))).resolves.toBeUndefined(); + + expect(withDeploymentLock).toHaveBeenCalledTimes(2); + expect(recordFailure).toHaveBeenCalledTimes(1); + expect(recordFailure).toHaveBeenCalledWith( + build, + BuildStatus.ERROR, + claim.token, + failure, + 'Build queue processing failed.', + claim.generation ); + expect(markObserved).toHaveBeenCalledWith(1, claim.generation, claim.token); + }); + + test('does not mistake an authority-read failure for proof that a failed pass is stale', async () => { + const { service, failure, isCurrent, recordFailure, markObserved, job } = reconciliationWorkerHarness(); + const authorityError = new Error('authority read unavailable'); + isCurrent.mockRejectedValueOnce(authorityError); + + await expect(service.processDeploymentReconciliationQueue(job(0))).rejects.toBe(authorityError); + + expect(recordFailure).not.toHaveBeenCalled(); + expect(markObserved).not.toHaveBeenCalled(); + expect(failure).not.toBe(authorityError); }); - test('fails open to the delivered ref on head-check, repository, or lookup failures', async () => { - const whereNull = jest - .fn() - .mockResolvedValueOnce({ githubRepositoryId: 100, fullName: 'org/repo' }) - .mockResolvedValueOnce(undefined); - (buildService as any).db.models.Repository = { - query: jest.fn(() => ({ findOne: jest.fn(() => ({ whereNull })) })), + test('never regresses an accepted SHA to a lagging or divergent live branch head', async () => { + const repositoryQuery: any = { + findOne: jest.fn(() => repositoryQuery), + whereNull: jest.fn().mockResolvedValue({ fullName: 'org/service' }), }; - (github.getSHAForBranch as jest.Mock).mockRejectedValue(new Error('GitHub unavailable')); + const service = new BuildService( + { models: { Repository: { query: jest.fn(() => repositoryQuery) } } } as any, + {} as any, + {} as any, + queueManager() as any + ); + (github.getSHAForBranch as jest.Mock).mockResolvedValue('older-head'); + (github.compareCommits as jest.Mock).mockResolvedValue('behind'); - await expect((buildService as any).resolveEffectiveSourceRef(100, 'main', 'sha')).resolves.toBe('sha'); - await expect((buildService as any).resolveEffectiveSourceRef(100, 'main', 'sha')).resolves.toBe('sha'); - await expect((buildService as any).resolveEffectiveSourceRef(100, undefined, 'sha')).resolves.toBe('sha'); + await expect( + (service as any).resolveCurrentSourceRef({ + githubRepositoryId: 42, + sourceGithubRepositoryId: 42, + sourceBranch: 'main', + sourceRef: 'commit-c', + sourceBeforeRef: 'commit-b', + }) + ).resolves.toBe('commit-c'); + }); - const databaseError = new Error('database unavailable'); - (buildService as any).db.models.Repository = { - query: jest.fn(() => ({ - findOne: jest.fn(() => ({ whereNull: jest.fn().mockRejectedValue(databaseError) })), - })), + test('may advance an accepted SHA only when GitHub proves the live head is its descendant', async () => { + const repositoryQuery: any = { + findOne: jest.fn(() => repositoryQuery), + whereNull: jest.fn().mockResolvedValue({ fullName: 'org/service' }), }; - await expect((buildService as any).resolveEffectiveSourceRef(100, 'main', 'sha')).resolves.toBe('sha'); + const service = new BuildService( + { models: { Repository: { query: jest.fn(() => repositoryQuery) } } } as any, + {} as any, + {} as any, + queueManager() as any + ); + (github.getSHAForBranch as jest.Mock).mockResolvedValue('commit-d'); + (github.compareCommits as jest.Mock).mockResolvedValue('ahead'); + + await expect( + (service as any).resolveCurrentSourceRef({ + githubRepositoryId: 42, + sourceGithubRepositoryId: 42, + sourceBranch: 'main', + sourceRef: 'commit-c', + }) + ).resolves.toBe('commit-d'); + expect(github.compareCommits).toHaveBeenCalledWith({ + fullName: 'org/service', + base: 'commit-c', + head: 'commit-d', + }); }); }); @@ -1609,16 +2032,6 @@ describe('BuildService focused changed-line coverage', () => { mockGetAllConfigs.mockResolvedValue({ serviceAccount: { name: 'builder' } }); }); - test('returns null when the queue fingerprint build lookup misses', async () => { - const query: any = { - findOne: jest.fn(() => query), - withGraphFetched: jest.fn().mockResolvedValue(undefined), - }; - const service = serviceWith({ models: { Build: { query: jest.fn(() => query) } } }); - - await expect((service as any).getBuildForQueueFingerprint(41)).resolves.toBeNull(); - }); - test('uses default pagination when a legacy caller omits pagination', async () => { const query: any = { select: jest.fn(() => query), @@ -1694,10 +2107,23 @@ describe('BuildService focused changed-line coverage', () => { 'env', 'initEnv' ); + expect(graphSelect).toHaveBeenCalledWith( + 'name', + 'type', + 'dockerfilePath', + 'requires', + 'deploymentDependsOn', + 'dependsOnDeployableName', + 'builder', + 'ecr', + 'grpc', + 'hostPortMapping' + ); }); test('builds every supported image type and ignores inactive and unsupported deploys', async () => { const makeDeploy = (uuid: string, type: DeployTypes, active = true) => ({ + id: uuid, uuid, active, deployable: { type }, @@ -1712,21 +2138,75 @@ describe('BuildService focused changed-line coverage', () => { ]; mockDeployQuery.mockReturnValue(deployQuery(deploys)); const buildImage = jest.fn().mockResolvedValue(true); - const service = serviceWith({ services: { Deploy: { buildImage } } }); + const mutationQuery: any = { + patch: jest.fn(() => mutationQuery), + where: jest.fn().mockResolvedValue(1), + }; + const service = serviceWith({ + models: { Deploy: { query: jest.fn(() => mutationQuery) } }, + services: { Deploy: { buildImage } }, + }); - await expect(service.buildImages({ id: 4 } as any)).resolves.toBe(true); + await expect(service.buildImages({ id: 4 } as any, 'build-run')).resolves.toBe(true); expect(buildImage.mock.calls.map(([deploy]) => deploy.uuid)).toEqual(['docker', 'github', 'helm']); }); + test('keeps every static execution lane scoped to the changed repository and branch', async () => { + const imageQuery = deployQuery([]); + const cliQuery = deployQuery([]); + const manifestQuery = deployQuery([]); + mockDeployQuery.mockReturnValueOnce(imageQuery).mockReturnValueOnce(cliQuery).mockReturnValueOnce(manifestQuery); + const service = serviceWith({ services: { Deploy: { buildImage: jest.fn(), deployCLI: jest.fn() } } }); + jest.spyOn(service as any, 'isDeploymentRunCurrent').mockResolvedValue(true); + jest.spyOn(service as any, 'updateDeploysImageDetails').mockResolvedValue(undefined); + const build = { + id: 4, + uuid: 'large-static', + namespace: 'env-large-static', + kind: BuildKind.SANDBOX, + isStatic: true, + deploys: [], + $fetchGraph: jest.fn().mockResolvedValue(undefined), + } as any; + + await service.buildImages(build, 'run-c', 42, 'sha-c', 'main'); + await service.deployCLIServices(build, 'run-c', 42, 'sha-c', 'main'); + await service.generateAndApplyManifests({ + build, + runUUID: 'run-c', + expectedGeneration: 3, + githubRepositoryId: 42, + sourceBranch: 'main', + namespace: build.namespace, + }); + + for (const query of [imageQuery, cliQuery, manifestQuery]) { + expect(query.where).toHaveBeenCalledWith({ + buildId: 4, + runUUID: 'run-c', + githubRepositoryId: 42, + branchName: 'main', + }); + } + }); + + test('treats a retained targeted scope with no remaining Deploy rows as a successful no-op', async () => { + mockDeployQuery.mockReturnValue(deployQuery([])); + const service = serviceWith({ services: { Deploy: { deployCLI: jest.fn() } } }); + const build = { id: 4, $fetchGraph: jest.fn().mockResolvedValue(undefined) }; + + await expect(service.deployCLIServices(build as any, 'run-c', 42, 'sha-c', 'main')).resolves.toBe(true); + }); + test('fails image and CLI processing cleanly for a loaded deploy missing its deployable', async () => { const missing = { uuid: 'missing', active: true }; mockDeployQuery.mockReturnValue(deployQuery([{ uuid: 'inactive', active: false }, missing])); const service = serviceWith({ services: { Deploy: { buildImage: jest.fn(), deployCLI: jest.fn() } } }); - await expect(service.buildImages({ id: 4 } as any)).resolves.toBe(false); + await expect(service.buildImages({ id: 4 } as any, 'build-run')).resolves.toBe(false); await expect( - service.deployCLIServices({ id: 4, $fetchGraph: jest.fn().mockResolvedValue(undefined) } as any) + service.deployCLIServices({ id: 4, $fetchGraph: jest.fn().mockResolvedValue(undefined) } as any, 'build-run') ).resolves.toBe(false); }); @@ -1755,7 +2235,7 @@ describe('BuildService focused changed-line coverage', () => { const service = serviceWith({ services: { Deploy: { deployCLI, recordDeployFailure } } }); const build = { id: 4, runUUID: 'build-run', $fetchGraph: jest.fn().mockResolvedValue(undefined) }; - await expect(service.deployCLIServices(build as any)).resolves.toBe(false); + await expect(service.deployCLIServices(build as any, 'build-run')).resolves.toBe(false); expect(deployCLI).toHaveBeenCalledTimes(2); expect(recordDeployFailure).toHaveBeenCalledWith(cliFailure, 'build-run', { @@ -1800,6 +2280,7 @@ describe('BuildService focused changed-line coverage', () => { mockDeployQuery.mockReturnValue(deployQuery([deploy])); const service = serviceWith({}); jest.spyOn(service as any, 'updateDeploysImageDetails').mockResolvedValue(undefined); + const enqueueIngress = jest.spyOn(service as any, 'enqueueIngressManifest').mockResolvedValue(undefined); const build = { id: 4, uuid: 'manifest', @@ -1815,6 +2296,18 @@ describe('BuildService focused changed-line coverage', () => { namespace: build.namespace, }) ).resolves.toBe(true); + expect(enqueueIngress).toHaveBeenCalledWith(4, undefined, undefined); + + enqueueIngress.mockClear(); + await expect( + service.generateAndApplyManifests({ + build: build as any, + githubRepositoryId: null, + namespace: build.namespace, + enqueueIngress: false, + }) + ).resolves.toBe(true); + expect(enqueueIngress).not.toHaveBeenCalled(); }); test('covers empty and scoped running-image updates', async () => { @@ -1826,8 +2319,14 @@ describe('BuildService focused changed-line coverage', () => { }) ).resolves.toBeUndefined(); - const patch = jest.fn().mockResolvedValue(undefined); - await (service as any).updateDeploysImageDetails( + const deployUpdate: any = { + patch: jest.fn(() => deployUpdate), + where: jest.fn(() => deployUpdate), + then: (resolve: (value: number) => void, reject: (reason: unknown) => void) => + Promise.resolve(1).then(resolve, reject), + }; + const scopedService = serviceWith({ models: { Deploy: { query: jest.fn(() => deployUpdate) } } }); + await (scopedService as any).updateDeploysImageDetails( { $fetchGraph: jest.fn().mockResolvedValue(undefined), deploys: [ @@ -1835,14 +2334,17 @@ describe('BuildService focused changed-line coverage', () => { githubRepositoryId: 42, branchName: 'main', dockerImage: 'image:v1', - $query: jest.fn(() => ({ patch })), + id: 9, }, ], }, + 'build-run', 42, 'main' ); - expect(patch).toHaveBeenCalledWith({ isRunningLatest: true, runningImage: 'image:v1' }); + expect(deployUpdate.patch).toHaveBeenCalledWith({ isRunningLatest: true, runningImage: 'image:v1' }); + expect(deployUpdate.where).toHaveBeenCalledWith({ id: 9 }); + expect(deployUpdate.where).toHaveBeenCalledWith('runUUID', 'build-run'); }); test('resolves direct and repository-derived build environments', async () => { diff --git a/src/server/services/__tests__/deploy.test.ts b/src/server/services/__tests__/deploy.test.ts index 8ce98694..a46afc97 100644 --- a/src/server/services/__tests__/deploy.test.ts +++ b/src/server/services/__tests__/deploy.test.ts @@ -29,20 +29,25 @@ const mockCodefreshBuildImage = jest.fn(); const mockCodefreshGetLogs = jest.fn(); const mockCodefreshGetRepositoryTag = jest.fn(); const mockCodefreshTagExists = jest.fn(); +const mockCodefreshTriggerPipeline = jest.fn(); const mockCodefreshWaitForImage = jest.fn(); const mockBuildWithNative = jest.fn(); const mockGlobalConfigGetAllConfigs = jest.fn(); const mockGlobalConfigGetOrgChartName = jest.fn(); const mockCreateOrUpdateNamespace = jest.fn(); +const mockIsSupersededByNewerLiveDeploymentGeneration = jest.fn(); +const mockLoggerInfo = jest.fn(); +const mockLoggerWarn = jest.fn(); +const mockGetLogger = jest.fn(() => ({ + error: jest.fn(), + info: mockLoggerInfo, + warn: mockLoggerWarn, + debug: jest.fn(), + child: jest.fn().mockReturnThis(), +})); jest.mock('server/lib/logger', () => ({ - getLogger: jest.fn(() => ({ - error: jest.fn(), - info: jest.fn(), - warn: jest.fn(), - debug: jest.fn(), - child: jest.fn().mockReturnThis(), - })), + getLogger: (...args: any[]) => mockGetLogger(...args), withLogContext: jest.fn((ctx, fn) => fn()), extractContextForQueue: jest.fn(() => ({})), LogStage: {}, @@ -63,6 +68,7 @@ jest.mock('server/lib/codefresh', () => ({ getLogs: (...args: any[]) => mockCodefreshGetLogs(...args), getRepositoryTag: (...args: any[]) => mockCodefreshGetRepositoryTag(...args), tagExists: (...args: any[]) => mockCodefreshTagExists(...args), + triggerPipeline: (...args: any[]) => mockCodefreshTriggerPipeline(...args), waitForImage: (...args: any[]) => mockCodefreshWaitForImage(...args), })); @@ -97,6 +103,10 @@ describe('DeployService - shouldTriggerGithubDeployment', () => { let mockRedis: any; let mockRedlock: any; let mockQueueManager: any; + let conditionalDeployPatch: jest.Mock; + let conditionalDeployWhere: jest.Mock; + let currentDeploySelect: jest.Mock; + let currentDeployFindOne: jest.Mock; const createMockDeploy = (overrides: any = {}) => ({ id: 1, @@ -122,9 +132,12 @@ describe('DeployService - shouldTriggerGithubDeployment', () => { mockCodefreshGetLogs.mockReset(); mockCodefreshGetRepositoryTag.mockReset(); mockCodefreshTagExists.mockReset(); + mockCodefreshTriggerPipeline.mockReset(); mockCodefreshWaitForImage.mockReset(); mockBuildWithNative.mockReset(); mockCreateOrUpdateNamespace.mockReset(); + mockIsSupersededByNewerLiveDeploymentGeneration.mockReset(); + mockIsSupersededByNewerLiveDeploymentGeneration.mockResolvedValue(false); mockGlobalConfigGetOrgChartName.mockResolvedValue('org-chart'); mockGlobalConfigGetAllConfigs.mockResolvedValue({ lifecycleDefaults: { @@ -140,9 +153,22 @@ describe('DeployService - shouldTriggerGithubDeployment', () => { }); mockDetermineChartType.mockResolvedValue(ChartType.PUBLIC); + conditionalDeployPatch = jest.fn().mockResolvedValue(1); + conditionalDeployWhere = jest.fn().mockReturnValue({ patch: conditionalDeployPatch }); + currentDeploySelect = jest.fn().mockResolvedValue({ id: 1 }); + currentDeployFindOne = jest.fn().mockReturnValue({ select: currentDeploySelect }); mockDb = { - models: {}, - services: {}, + models: { + Deploy: { + query: jest.fn(() => ({ where: conditionalDeployWhere, findOne: currentDeployFindOne })), + }, + }, + services: { + BuildService: { + isSupersededByNewerLiveDeploymentGeneration: (...args: any[]) => + mockIsSupersededByNewerLiveDeploymentGeneration(...args), + }, + }, }; mockRedis = {}; @@ -294,7 +320,7 @@ describe('DeployService - shouldTriggerGithubDeployment', () => { }); }); - describe('API environment source pinning', () => { + describe('deployment source pinning', () => { test('updates only deploys matching the targeted repository and exact effective branch', async () => { const mainPatch = jest.fn().mockResolvedValue(undefined); const stablePatch = jest.fn().mockResolvedValue(undefined); @@ -323,8 +349,8 @@ describe('DeployService - shouldTriggerGithubDeployment', () => { mockDb.services.Deploy = { hostForDeployableDeploy: jest.fn(() => 'service.example.test') }; const build = { id: 7, - uuid: 'api-env-123456', - triggerType: 'api', + uuid: 'pr-env-123456', + triggerType: 'github_pr', githubRepositoryId: 42, branchName: 'main', configSha: null, @@ -357,6 +383,50 @@ describe('DeployService - shouldTriggerGithubDeployment', () => { expect(github.getShaForDeploy).not.toHaveBeenCalled(); }); + test('keeps source identity when a root push intentionally targets the whole environment', async () => { + const rootPatch = jest.fn().mockResolvedValue(undefined); + const dependencyPatch = jest.fn().mockResolvedValue(undefined); + const rootDeploy = { + id: 1, + deployableId: 11, + githubRepositoryId: 42, + branchName: 'main', + $query: jest.fn(() => ({ patch: rootPatch })), + }; + const dependencyDeploy = { + id: 2, + deployableId: 22, + githubRepositoryId: 99, + branchName: 'main', + $query: jest.fn(() => ({ patch: dependencyPatch })), + }; + const deployQuery: any = { + where: jest.fn().mockReturnThis(), + withGraphFetched: jest.fn().mockResolvedValue([rootDeploy, dependencyDeploy]), + }; + mockDb.models.Deploy = { query: jest.fn(() => deployQuery), findOne: jest.fn() }; + mockDb.services.Deploy = { hostForDeployableDeploy: jest.fn(() => 'service.example.test') }; + (github.getShaForDeploy as jest.Mock).mockResolvedValue('dependency-head'); + const build = { + id: 7, + uuid: 'static-env-123456', + triggerType: 'github_pr', + githubRepositoryId: 42, + branchName: 'main', + deployables: [ + { id: 11, name: 'root', repositoryId: 42, branchName: 'main', type: DeployTypes.GITHUB }, + { id: 22, name: 'dependency', repositoryId: 99, branchName: 'main', type: DeployTypes.GITHUB }, + ], + deploys: [rootDeploy, dependencyDeploy], + $fetchGraph: jest.fn().mockResolvedValue(undefined), + }; + + await deployService.findOrCreateDeploys({} as any, build as any, undefined, 'root-push-sha', 'main', 42); + + expect(rootPatch).toHaveBeenCalledWith(expect.objectContaining({ sha: 'root-push-sha' })); + expect(dependencyPatch).toHaveBeenCalledWith(expect.objectContaining({ sha: 'dependency-head' })); + }); + test('backfills a missing deploy row outside the targeted source without resolving its SHA', async () => { const createdPatch = jest.fn().mockResolvedValue(undefined); const createdDeploy = { @@ -462,7 +532,7 @@ describe('DeployService - shouldTriggerGithubDeployment', () => { $query: jest.fn(() => ({ patch: jest.fn().mockResolvedValue(undefined) })), }; - await deployService.deployCodefresh(deploy as any, 'push-sha', 42, 'main'); + await deployService.deployCodefresh(deploy as any, 'old-run', 'push-sha', 42, 'main'); expect(mockCodefreshDeploy).toHaveBeenCalledWith(deploy, deploy.build, deploy.deployable, 'push-sha'); expect(github.getSHAForBranch).not.toHaveBeenCalled(); @@ -498,7 +568,7 @@ describe('DeployService - shouldTriggerGithubDeployment', () => { $query: jest.fn(() => ({ patch: jest.fn().mockResolvedValue(undefined) })), }; - await deployService.deployCodefresh(deploy as any, 'push-sha', 42); + await deployService.deployCodefresh(deploy as any, 'old-run', 'push-sha', 42); expect(mockCodefreshDeploy).toHaveBeenCalledWith(deploy, deploy.build, deploy.deployable, null); expect(github.getSHAForBranch).toHaveBeenCalledWith('feature-branch', 'org', 'repo'); @@ -526,11 +596,11 @@ describe('DeployService - shouldTriggerGithubDeployment', () => { expect(github.getSHAForBranch).toHaveBeenNthCalledWith(2, 'stable', 'org', 'dependency'); }); - test('pins the pushed dependency SHA only for an API dependency-tracking run', async () => { + test('pins the delivered dependency SHA for a non-API tracked-source run', async () => { const dependencyDeploy = { githubRepositoryId: 99, branchName: 'main', - build: { triggerType: 'api', githubRepositoryId: 42, branchName: 'main', configSha: null }, + build: { triggerType: 'github_pr', githubRepositoryId: 42, branchName: 'main', configSha: null }, }; await expect( @@ -562,6 +632,135 @@ describe('DeployService - shouldTriggerGithubDeployment', () => { }); describe('failure boundaries', () => { + const createNativeAfterBuildDeploy = () => ({ + id: 17, + buildId: 91, + uuid: 'sample-service-build', + branchName: 'feature-branch', + env: { + FEATURE_FLAG: 'enabled', + }, + initEnv: {}, + dockerImage: 'old-image', + build: { + id: 91, + uuid: 'sample-build', + namespace: 'env-sample', + isStatic: false, + commentRuntimeEnv: {}, + enabledFeatures: [], + pullRequest: { + githubLogin: 'sample-user', + }, + $fetchGraph: jest.fn().mockResolvedValue(undefined), + }, + deployable: { + name: 'sample-service', + type: DeployTypes.GITHUB, + dockerfilePath: './Dockerfile', + initDockerfilePath: null, + ecr: 'sample/app-images', + afterBuildPipelineId: 'sample/after-build', + builder: { + engine: 'buildkit', + }, + repository: { + fullName: 'example-org/example-repo', + }, + $fetchGraph: jest.fn().mockResolvedValue(undefined), + }, + reload: jest.fn().mockResolvedValue(undefined), + $fetchGraph: jest.fn().mockResolvedValue(undefined), + }); + + const prepareNativeAfterBuildTest = (isCurrent: () => boolean) => { + (github.getSHAForBranch as jest.Mock).mockResolvedValue('abcdef1234567890'); + mockCodefreshTagExists.mockResolvedValue(false); + mockCodefreshTriggerPipeline.mockResolvedValue('after-build-run'); + mockCodefreshWaitForImage.mockResolvedValue(true); + jest.spyOn(deployService as any, 'isDeploymentRunCurrent').mockImplementation(async () => isCurrent()); + jest.spyOn(deployService as any, 'waitAndResolveForBuildDependentEnvVars').mockResolvedValue(undefined); + jest.spyOn(deployService as any, 'syncServiceExternalSecrets').mockResolvedValue({ + secretNames: [], + buildSecretEnvKeys: new Set(), + }); + jest.spyOn(deployService, 'patchAndUpdateActivityFeed').mockResolvedValue(undefined); + return jest.spyOn(deployService as any, 'patchDeployWithTag').mockResolvedValue(undefined); + }; + + test('buildImage treats a stale run as a superseded no-op before starting work', async () => { + currentDeploySelect.mockResolvedValue(undefined); + const deploy = { + id: 17, + uuid: 'sample-service-build', + $fetchGraph: jest.fn().mockResolvedValue(undefined), + }; + + await expect(deployService.buildImage(deploy as any, 0, 'stale-run')).resolves.toBe(true); + + expect(currentDeployFindOne).toHaveBeenCalledWith({ id: 17, runUUID: 'stale-run' }); + expect(deploy.$fetchGraph).not.toHaveBeenCalled(); + expect(mockCodefreshBuildImage).not.toHaveBeenCalled(); + expect(mockBuildWithNative).not.toHaveBeenCalled(); + }); + + test('generation check uses buildId before the Build relation is loaded', async () => { + const currentBuildWhere = jest.fn().mockResolvedValue({ id: 91 }); + mockDb.models.Build = { + query: jest.fn(() => ({ + findOne: jest.fn(() => ({ + whereNull: jest.fn(() => ({ where: currentBuildWhere })), + })), + })), + }; + const patchSpy = jest.spyOn(deployService, 'patchAndUpdateActivityFeed').mockResolvedValue(undefined); + const deploy = { + id: 17, + buildId: 91, + uuid: 'sample-service-build', + deployable: { type: DeployTypes.DOCKER, dockerImage: 'nginx' }, + tag: 'latest', + $fetchGraph: jest.fn().mockResolvedValue(undefined), + }; + + await expect( + deployService.buildImage(deploy as any, 0, 'run-c', undefined, undefined, undefined, 7) + ).resolves.toBe(true); + + expect(currentBuildWhere).toHaveBeenCalledWith('desiredGeneration', 7); + expect(deploy.$fetchGraph).toHaveBeenCalled(); + expect(patchSpy).toHaveBeenCalledWith( + deploy, + expect.objectContaining({ status: DeployStatus.BUILT, dockerImage: 'nginx:latest' }), + 'run-c' + ); + }); + + test('patchAndUpdateActivityFeed skips stale writes and side effects when runUUID no longer owns the deploy', async () => { + conditionalDeployPatch.mockResolvedValue(0); + const deploy = { + id: 17, + uuid: 'sample-service-build', + // Deliberately stale in-memory state: fencing must use the affected-row count, + // not the model instance that the old worker already holds. + runUUID: 'stale-run', + $fetchGraph: jest.fn().mockResolvedValue(undefined), + }; + + await deployService.patchAndUpdateActivityFeed( + deploy as any, + { status: DeployStatus.BUILT, dockerImage: 'stale-image' }, + 'stale-run' + ); + + expect(conditionalDeployWhere).toHaveBeenCalledWith({ id: 17, runUUID: 'stale-run' }); + expect(conditionalDeployPatch).toHaveBeenCalledWith({ + status: DeployStatus.BUILT, + dockerImage: 'stale-image', + }); + expect(deploy.$fetchGraph).not.toHaveBeenCalled(); + }); + test('recordDeployFailure writes a terminal status with the original error message', async () => { const patchSpy = jest.spyOn(deployService, 'patchAndUpdateActivityFeed').mockResolvedValue(undefined); const deploy = { @@ -623,7 +822,7 @@ describe('DeployService - shouldTriggerGithubDeployment', () => { }, }; - const result = await deployService.buildImage(deploy as any, 0); + const result = await deployService.buildImage(deploy as any, 0, 'run-1'); expect(result).toBe(false); expect(github.getSHAForBranch).toHaveBeenCalledWith('missing-branch', 'example-org', 'example-repo'); @@ -702,11 +901,234 @@ describe('DeployService - shouldTriggerGithubDeployment', () => { repo: 'example-org/example-repo', }) ); - expect(deployPatch).toHaveBeenCalledWith({ buildPipelineId: 'codefresh-build-123' }); - expect(deployPatch).toHaveBeenCalledWith({ buildOutput: 'codefresh logs' }); + expect(conditionalDeployPatch).toHaveBeenCalledWith({ buildPipelineId: 'codefresh-build-123' }); + expect(conditionalDeployPatch).toHaveBeenCalledWith({ buildOutput: 'codefresh logs' }); expect(patchSpy).toHaveBeenLastCalledWith(deploy, { status: DeployStatus.BUILD_FAILED }, 'run-1'); }); + test('native builds invoke and wait for their configured after-build pipeline while current', async () => { + const deploy = createNativeAfterBuildDeploy(); + const patchTagSpy = prepareNativeAfterBuildTest(() => true); + mockBuildWithNative.mockResolvedValue({ success: true }); + + const result = await deployService.buildImageForHelmAndGithub( + deploy as any, + 'run-1', + undefined, + undefined, + undefined, + 7 + ); + + expect(result).toBe(true); + expect(patchTagSpy).toHaveBeenCalledTimes(1); + expect(mockCodefreshTriggerPipeline).toHaveBeenCalledTimes(1); + expect(mockCodefreshTriggerPipeline).toHaveBeenCalledWith( + 'sample/after-build', + 'cli', + expect.objectContaining({ + FEATURE_FLAG: 'enabled', + TAG: expect.stringMatching( + /^123456789012\.dkr\.ecr\.us-west-2\.amazonaws\.com\/sample\/app-images:lfc-abcdef1-/ + ), + branch: 'feature-branch', + }) + ); + expect(mockCodefreshWaitForImage).toHaveBeenCalledWith('after-build-run'); + expect(mockGetLogger).toHaveBeenCalledWith( + expect.objectContaining({ + afterBuildPipelineId: 'sample/after-build', + pipelineId: 'after-build-run', + imageTag: expect.stringContaining('/sample/app-images:lfc-abcdef1-'), + }) + ); + expect(mockLoggerInfo).toHaveBeenCalledWith( + expect.stringMatching( + /^Codefresh: after-build pipeline completed result=success afterBuildPipelineId=sample\/after-build pipelineId=after-build-run imageTag=.*\/sample\/app-images:lfc-abcdef1-/ + ) + ); + }); + + test('a superseded native build still invokes its after-build pipeline without publishing stale image state', async () => { + let current = true; + const deploy = createNativeAfterBuildDeploy(); + const patchTagSpy = prepareNativeAfterBuildTest(() => current); + mockIsSupersededByNewerLiveDeploymentGeneration.mockResolvedValue(true); + mockBuildWithNative.mockImplementation(async () => { + current = false; + return { success: true, logs: 'native build completed' }; + }); + + const result = await deployService.buildImageForHelmAndGithub( + deploy as any, + 'run-a', + undefined, + undefined, + undefined, + 7 + ); + + expect(result).toBe(true); + expect(mockCodefreshTriggerPipeline).toHaveBeenCalledTimes(1); + expect(mockIsSupersededByNewerLiveDeploymentGeneration).toHaveBeenCalledWith(91, 7); + expect(mockCodefreshWaitForImage).not.toHaveBeenCalled(); + expect(patchTagSpy).not.toHaveBeenCalled(); + expect(conditionalDeployPatch).not.toHaveBeenCalled(); + expect(mockGetLogger).toHaveBeenCalledWith( + expect.objectContaining({ + afterBuildPipelineId: 'sample/after-build', + pipelineId: 'after-build-run', + }) + ); + expect(mockLoggerInfo).toHaveBeenCalledWith( + expect.stringMatching( + /^Codefresh: after-build pipeline detached reason=superseded afterBuildPipelineId=sample\/after-build pipelineId=after-build-run imageTag=/ + ) + ); + }); + + test('a native build superseded by teardown does not invoke its after-build pipeline', async () => { + let current = true; + const deploy = createNativeAfterBuildDeploy(); + const patchTagSpy = prepareNativeAfterBuildTest(() => current); + mockBuildWithNative.mockImplementation(async () => { + current = false; + return { success: true, logs: 'native build completed' }; + }); + + const result = await deployService.buildImageForHelmAndGithub( + deploy as any, + 'run-a', + undefined, + undefined, + undefined, + 7 + ); + + expect(result).toBe(true); + expect(mockIsSupersededByNewerLiveDeploymentGeneration).toHaveBeenCalledWith(91, 7); + expect(mockCodefreshTriggerPipeline).not.toHaveBeenCalled(); + expect(mockCodefreshWaitForImage).not.toHaveBeenCalled(); + expect(patchTagSpy).not.toHaveBeenCalled(); + expect(conditionalDeployPatch).not.toHaveBeenCalled(); + expect(mockLoggerInfo).toHaveBeenCalledWith( + expect.stringMatching( + /^Codefresh: after-build pipeline skipped reason=no_live_successor afterBuildPipelineId=sample\/after-build imageTag=/ + ) + ); + }); + + test('a stale native build fails closed when live-successor authority cannot be read', async () => { + let current = true; + const deploy = createNativeAfterBuildDeploy(); + prepareNativeAfterBuildTest(() => current); + mockIsSupersededByNewerLiveDeploymentGeneration.mockRejectedValue(new Error('database unavailable')); + mockBuildWithNative.mockImplementation(async () => { + current = false; + return { success: true }; + }); + + const result = await deployService.buildImageForHelmAndGithub( + deploy as any, + 'run-a', + undefined, + undefined, + undefined, + 7 + ); + + expect(result).toBe(true); + expect(mockCodefreshTriggerPipeline).not.toHaveBeenCalled(); + expect(mockCodefreshWaitForImage).not.toHaveBeenCalled(); + expect(mockLoggerWarn).toHaveBeenCalledWith('Codefresh: after-build successor check failed'); + }); + + test('a native after-build pipeline is left detached when superseded during its trigger', async () => { + let current = true; + const deploy = createNativeAfterBuildDeploy(); + const patchTagSpy = prepareNativeAfterBuildTest(() => current); + mockBuildWithNative.mockResolvedValue({ success: true }); + mockCodefreshTriggerPipeline.mockImplementation(async () => { + current = false; + return 'after-build-run'; + }); + + const result = await deployService.buildImageForHelmAndGithub( + deploy as any, + 'run-a', + undefined, + undefined, + undefined, + 7 + ); + + expect(result).toBe(true); + expect(patchTagSpy).toHaveBeenCalledTimes(1); + expect(mockCodefreshTriggerPipeline).toHaveBeenCalledTimes(1); + expect(mockCodefreshWaitForImage).not.toHaveBeenCalled(); + }); + + test('a failed superseded native build neither invokes the after-build pipeline nor publishes failure', async () => { + let current = true; + const deploy = createNativeAfterBuildDeploy(); + const patchTagSpy = prepareNativeAfterBuildTest(() => current); + mockBuildWithNative.mockImplementation(async () => { + current = false; + return { success: false, logs: 'native build failed' }; + }); + + const result = await deployService.buildImageForHelmAndGithub( + deploy as any, + 'run-a', + undefined, + undefined, + undefined, + 7 + ); + + expect(result).toBe(true); + expect(mockCodefreshTriggerPipeline).not.toHaveBeenCalled(); + expect(mockCodefreshWaitForImage).not.toHaveBeenCalled(); + expect(patchTagSpy).not.toHaveBeenCalled(); + expect(deployService.patchAndUpdateActivityFeed).not.toHaveBeenCalledWith( + deploy, + { status: DeployStatus.BUILD_FAILED }, + 'run-a' + ); + }); + + test('a current native after-build pipeline failure still fails the image phase', async () => { + const deploy = createNativeAfterBuildDeploy(); + const patchTagSpy = prepareNativeAfterBuildTest(() => true); + mockBuildWithNative.mockResolvedValue({ success: true }); + mockCodefreshWaitForImage.mockResolvedValue(false); + + const result = await deployService.buildImageForHelmAndGithub( + deploy as any, + 'run-1', + undefined, + undefined, + undefined, + 7 + ); + + expect(result).toBe(false); + expect(patchTagSpy).toHaveBeenCalledTimes(1); + expect(mockCodefreshTriggerPipeline).toHaveBeenCalledTimes(1); + expect(mockCodefreshWaitForImage).toHaveBeenCalledWith('after-build-run'); + expect(mockGetLogger).toHaveBeenCalledWith( + expect.objectContaining({ + afterBuildPipelineId: 'sample/after-build', + pipelineId: 'after-build-run', + }) + ); + expect(mockLoggerWarn).toHaveBeenCalledWith( + expect.stringMatching( + /^Codefresh: after-build pipeline completed result=failure afterBuildPipelineId=sample\/after-build pipelineId=after-build-run imageTag=/ + ) + ); + }); + test('buildImageForHelmAndGithub syncs external secrets when native image tag already exists', async () => { (github.getSHAForBranch as jest.Mock).mockResolvedValue('abcdef1234567890'); mockCodefreshTagExists.mockResolvedValue(true); @@ -837,7 +1259,7 @@ describe('DeployService - shouldTriggerGithubDeployment', () => { 'sample-service-aws-secrets': 'sync-token', } ); - expect(deployPatch).toHaveBeenCalledWith( + expect(conditionalDeployPatch).toHaveBeenCalledWith( expect.objectContaining({ status: DeployStatus.BUILT, dockerImage: '123456789012.dkr.ecr.us-west-2.amazonaws.com/sample/app-images:lfc-abcdef1', @@ -977,12 +1399,11 @@ describe('DeployService - shouldTriggerGithubDeployment', () => { } }); - test('deployAurora records failures with the newly assigned runUUID', async () => { + test('deployAurora records failures with its expected runUUID', async () => { const patchSpy = jest.spyOn(deployService, 'patchAndUpdateActivityFeed').mockResolvedValue(undefined); jest.spyOn(deployService as any, 'findExistingAuroraDatabase').mockResolvedValue(null); mockCliDeploy.mockRejectedValue(new Error('restore command failed')); - const patches: any[] = []; const deploy = { uuid: 'sample-aurora-restore', runUUID: 'old-run', @@ -997,28 +1418,22 @@ describe('DeployService - shouldTriggerGithubDeployment', () => { }, reload: jest.fn().mockResolvedValue(undefined), $fetchGraph: jest.fn().mockResolvedValue(undefined), - $query: jest.fn(() => ({ - patch: jest.fn((params) => { - patches.push(params); - return Promise.resolve(undefined); - }), - })), + $query: jest.fn(() => ({ patch: jest.fn().mockResolvedValue(undefined) })), }; - const result = await deployService.deployAurora(deploy as any); - const assignedRunUUID = patches.find((params) => params.status === DeployStatus.BUILDING)?.runUUID; + const result = await deployService.deployAurora(deploy as any, 'expected-run'); expect(result).toBe(false); - expect(assignedRunUUID).toBeDefined(); - expect(assignedRunUUID).not.toBe('old-run'); - expect(deploy.runUUID).toBe(assignedRunUUID); + expect(conditionalDeployPatch).toHaveBeenCalledWith(expect.objectContaining({ status: DeployStatus.BUILDING })); + expect(conditionalDeployPatch).not.toHaveBeenCalledWith(expect.objectContaining({ runUUID: expect.anything() })); + expect(deploy.runUUID).toBe('old-run'); expect(patchSpy).toHaveBeenLastCalledWith( deploy, { status: DeployStatus.ERROR, statusMessage: 'restore command failed', }, - assignedRunUUID + 'expected-run' ); }); }); diff --git a/src/server/services/__tests__/deployCleanup.test.ts b/src/server/services/__tests__/deployCleanup.test.ts index 31ea27e2..c0ae5259 100644 --- a/src/server/services/__tests__/deployCleanup.test.ts +++ b/src/server/services/__tests__/deployCleanup.test.ts @@ -135,7 +135,7 @@ describe('DeployCleanupService', () => { expect(commands).toContain( "kubectl delete pvc 'old-api-build-1-data-claim' --namespace 'env-build-1' --ignore-not-found" ); - expect(commands).toContain("helm uninstall 'old-api-build-1' --namespace 'env-build-1'"); + expect(commands).toContain("helm uninstall 'old-api-build-1' --namespace 'env-build-1' --timeout 5m"); expect(joinedCommands).toContain('deploy_uuid=old-api-build-1'); expect(joinedCommands).toContain('deploy-id=77'); expect(joinedCommands).toContain('deployable-id=9'); @@ -146,6 +146,12 @@ describe('DeployCleanupService', () => { expect(joinedCommands).toContain( "kubectl delete secret 'old-api-aws-secrets' --namespace 'env-build-1' --ignore-not-found" ); + expect(joinedCommands).not.toContain('--request-timeout'); + + for (const [command, options] of mockShellPromise.mock.calls) { + if (String(command).startsWith('kubectl ')) expect(options).toEqual({ timeout: 90_000 }); + if (String(command).startsWith('helm ')) expect(options).toEqual({ timeout: 360_000 }); + } expect(joinedCommands).not.toContain('delete namespace'); expect(joinedCommands).not.toContain('delete all,pvc'); @@ -189,6 +195,38 @@ describe('DeployCleanupService', () => { expect(mockDeleteDeploy).not.toHaveBeenCalled(); }); + test('starts provider cleanup without holding or waiting for the native mutation gate', async () => { + const deploy = createDeploy({ + deployable: { name: 'old-codefresh', type: DeployTypes.CODEFRESH, serviceDisksYaml: null }, + env: {}, + }); + const service = createService(); + let releaseNative!: () => void; + const nativeReleased = new Promise((resolve) => { + releaseNative = resolve; + }); + let providerStarted!: () => void; + const providerStart = new Promise((resolve) => { + providerStarted = resolve; + }); + mockCodefreshDestroy.mockImplementation(async () => providerStarted()); + const nativeMutationGate = jest.fn(async (action: () => Promise) => { + await nativeReleased; + return action(); + }); + + const cleanup = service.cleanupDeploy(deploy, { mode: 'service', nativeMutationGate }); + await providerStart; + + expect(mockCodefreshDestroy).toHaveBeenCalledWith(deploy); + expect(mockShellPromise).not.toHaveBeenCalled(); + + releaseNative(); + await expect(cleanup).resolves.toBe(true); + expect(nativeMutationGate).toHaveBeenCalledTimes(1); + expect(mockShellPromise).toHaveBeenCalled(); + }); + test('service mode continues after a targeted teardown failure', async () => { mockShellPromise.mockImplementation((command: string) => { if (command.includes('kubectl delete deployment')) { @@ -202,7 +240,7 @@ describe('DeployCleanupService', () => { await expect(service.cleanupDeploy(deploy, { mode: 'service' })).resolves.toBe(false); const commands = mockShellPromise.mock.calls.map(([command]) => command as string); - expect(commands).toContain("helm uninstall 'old-api-build-1' --namespace 'env-build-1'"); + expect(commands).toContain("helm uninstall 'old-api-build-1' --namespace 'env-build-1' --timeout 5m"); expect(mockMetricsIncrement).toHaveBeenCalledWith( 'task', expect.objectContaining({ result: 'error', resourceType: 'deployment' }) diff --git a/src/server/services/__tests__/deployable.test.ts b/src/server/services/__tests__/deployable.test.ts index 9de48e62..2b4bf32d 100644 --- a/src/server/services/__tests__/deployable.test.ts +++ b/src/server/services/__tests__/deployable.test.ts @@ -254,6 +254,7 @@ describe('Deployable Service', () => { active: undefined, defaultBranchName: 'unit-test', dependsOnDeployableName: undefined, + requires: ['github-db', 'test-db'], deploymentDependsOn: [], helm: undefined, }); @@ -419,6 +420,7 @@ describe('Deployable Service', () => { active: undefined, defaultBranchName: 'unit-test', dependsOnDeployableName: undefined, + requires: ['github-db', 'test-db'], deploymentDependsOn: [], helm: undefined, }); diff --git a/src/server/services/__tests__/deployableSourceSeam.test.ts b/src/server/services/__tests__/deployableSourceSeam.test.ts index a5668a31..90f9972f 100644 --- a/src/server/services/__tests__/deployableSourceSeam.test.ts +++ b/src/server/services/__tests__/deployableSourceSeam.test.ts @@ -241,7 +241,7 @@ describe('deployable source seam (PR vs API build)', () => { jest.spyOn(service, 'updateOrCreateDeployableAttributesUsingYAMLConfig').mockResolvedValue(undefined); const build: any = { id: 9, - triggerType: 'api', + triggerType: 'github_pr', githubRepositoryId: 42, branchName: 'main', configSha: 'root-config-sha', @@ -276,6 +276,41 @@ describe('deployable source seam (PR vs API build)', () => { ); }); + it('pins a root full-scope import to the delivered source SHA independently of its target selector', async () => { + const service = makeService(); + const repository = { githubRepositoryId: 42, fullName: 'org/root' }; + mockFetchLifecycleConfigByRepository.mockResolvedValue(null); + const build: any = { + id: 9, + triggerType: 'github_pr', + githubRepositoryId: 42, + branchName: 'main', + deploys: [], + environment: { id: 5 }, + $fetchGraph: jest.fn().mockResolvedValue(undefined), + }; + const pullRequest: any = { + branchName: 'main', + repository, + build, + $fetchGraph: jest.fn().mockResolvedValue(undefined), + }; + + await (service as any).updateOrCreateDeployableUsingYamlConfig( + new Map(), + 9, + 'uuid-9', + pullRequest, + build, + undefined, + 'root-push-sha', + 'main', + 42 + ); + + expect(mockFetchLifecycleConfigByRepository).toHaveBeenCalledWith(repository, 'root-push-sha'); + }); + it('targets a same-repository dependency by its exact effective branch and preserves the configured branch', async () => { const service = makeService(); const repository = { githubRepositoryId: 42, fullName: 'org/repo' }; diff --git a/src/server/services/__tests__/github.test.ts b/src/server/services/__tests__/github.test.ts index 2a02f02d..89cb63f8 100644 --- a/src/server/services/__tests__/github.test.ts +++ b/src/server/services/__tests__/github.test.ts @@ -1048,7 +1048,26 @@ describe('Github Service - handlePushWebhook', () => { ); }); - test('forwards the pushed commit as triggerRef so distinct commits are not coalesced', async () => { + test('keeps a tracked static service push scoped to the changed repository', async () => { + const buildId = 100; + const mockDeploy = createMockDeploy(buildId, DeployStatus.READY); + (mockDeploy.build as any).isStatic = true; + mockDb.models.Deploy.query.mockReturnValue(createAllDeploysQuery([mockDeploy])); + + await githubService.handlePushWebhook(createMockPushEvent()); + + expect(mockDb.services.BuildService.enqueueResolveAndDeployBuild).toHaveBeenCalledWith( + expect.objectContaining({ + buildId, + githubRepositoryId: 12345, + sourceGithubRepositoryId: 12345, + sourceBranch: 'main', + sourceRef: 'abc123def456', + }) + ); + }); + + test('forwards the pushed commit as the immutable source ref', async () => { const buildId = 100; const mockDeploy = createMockDeploy(buildId); @@ -1058,8 +1077,7 @@ describe('Github Service - handlePushWebhook', () => { return queryCount === 1 ? createAllDeploysQuery([mockDeploy]) : createFailedDeploysQuery([]); }); - // Keep the default void `before` so the deploy routes directly without changed-files setup; triggerRef - // derives only from the pushed head commit (`after`). + // Keep the default void `before` so the deploy routes directly without changed-files setup. const pushEvent = { ...createMockPushEvent(), after: 'head-commit-sha', @@ -1069,11 +1087,11 @@ describe('Github Service - handlePushWebhook', () => { expect(mockDb.services.BuildService.resolveAndDeployBuildQueue.add).toHaveBeenCalledWith( 'resolve-deploy', - expect.objectContaining({ buildId, triggerRef: 'head-commit-sha' }) + expect.objectContaining({ buildId, sourceRef: 'head-commit-sha' }) ); }); - test('omits triggerRef when the pushed head commit is void', async () => { + test('omits sourceRef when the pushed head commit is void', async () => { const buildId = 100; const mockDeploy = createMockDeploy(buildId); @@ -1092,10 +1110,10 @@ describe('Github Service - handlePushWebhook', () => { await githubService.handlePushWebhook(pushEvent); const addCall = mockDb.services.BuildService.resolveAndDeployBuildQueue.add.mock.calls[0]; - expect(addCall[1]).not.toHaveProperty('triggerRef'); + expect(addCall[1]).not.toHaveProperty('sourceRef'); }); - test('forwards the pushed commit as triggerRef for static environments', async () => { + test('forwards the pushed commit as sourceRef for static environments', async () => { const buildId = 701; const repoBuilder = { from: jest.fn().mockReturnThis(), @@ -1133,9 +1151,20 @@ describe('Github Service - handlePushWebhook', () => { await githubService.handlePushWebhook(pushEvent); + const request = mockDb.services.BuildService.enqueueResolveAndDeployBuild.mock.calls[0][0]; + expect(request).toEqual( + expect.objectContaining({ + buildId, + sourceGithubRepositoryId: 12345, + sourceBranch: 'main', + sourceRef: 'static-head-sha', + sourceBeforeRef: 'previous-commit', + }) + ); + expect(request).not.toHaveProperty('githubRepositoryId'); expect(mockDb.services.BuildService.resolveAndDeployBuildQueue.add).toHaveBeenCalledWith( 'resolve-deploy', - expect.objectContaining({ buildId, triggerRef: 'static-head-sha' }) + expect.objectContaining({ buildId, sourceRef: 'static-head-sha' }) ); }); diff --git a/src/server/services/__tests__/githubAutoTrack.test.ts b/src/server/services/__tests__/githubAutoTrack.test.ts index d1b26425..715f7908 100644 --- a/src/server/services/__tests__/githubAutoTrack.test.ts +++ b/src/server/services/__tests__/githubAutoTrack.test.ts @@ -82,10 +82,10 @@ describe('enqueueAutoTrackedApiBuilds', () => { expect(enqueue).toHaveBeenCalledTimes(2); expect(enqueue).toHaveBeenCalledWith( - expect.objectContaining({ buildId: 1, triggerRef: 'sha123', sourceBranch: 'Main' }) + expect.objectContaining({ buildId: 1, sourceRef: 'sha123', sourceBranch: 'Main' }) ); expect(enqueue).toHaveBeenCalledWith( - expect.objectContaining({ buildId: 2, triggerRef: 'sha123', sourceBranch: 'Main' }) + expect.objectContaining({ buildId: 2, sourceRef: 'sha123', sourceBranch: 'Main' }) ); }); @@ -129,7 +129,7 @@ describe('handlePushWebhook auto-track wiring', () => { repository: { id: 42, full_name: 'org/repo' }, } as any); - expect(autoTrack).toHaveBeenCalledWith(42, 'main', 'sha123'); + expect(autoTrack).toHaveBeenCalledWith(42, 'main', 'sha123', 'aaaa111'); }); it('continues the existing PR redeploy flow when the API auto-track lookup fails', async () => { @@ -184,7 +184,6 @@ describe('handlePushWebhook auto-track wiring', () => { expect(enqueueResolveAndDeployBuild).toHaveBeenCalledWith( expect.objectContaining({ buildId: 7, - triggerRef: 'sha123', sourceRef: 'sha123', sourceGithubRepositoryId: 42, sourceBranch: 'main', @@ -251,7 +250,6 @@ describe('handlePushWebhook auto-track wiring', () => { buildId: 8, githubRepositoryId: 99, sourceGithubRepositoryId: 99, - triggerRef: 'dependency-sha', sourceRef: 'dependency-sha', sourceBranch: 'main', }) @@ -406,7 +404,7 @@ describe('handlePushWebhook auto-track wiring', () => { repository: { id: 42, full_name: 'org/repo' }, } as any); - expect(autoTrack).toHaveBeenCalledWith(42, 'main', 'sha123'); + expect(autoTrack).toHaveBeenCalledWith(42, 'main', 'sha123', 'aaaa111'); expect(db.services.BuildService.enqueueResolveAndDeployBuild).not.toHaveBeenCalled(); }); @@ -441,7 +439,6 @@ describe('handlePushWebhook auto-track wiring', () => { expect(enqueue).toHaveBeenCalledWith( expect.objectContaining({ buildId: 1, - triggerRef: 'sha123', sourceRef: 'sha123', sourceGithubRepositoryId: 42, sourceBranch: 'main', diff --git a/src/server/services/__tests__/ingress.test.ts b/src/server/services/__tests__/ingress.test.ts new file mode 100644 index 00000000..cbbafb63 --- /dev/null +++ b/src/server/services/__tests__/ingress.test.ts @@ -0,0 +1,152 @@ +/** + * Copyright 2026 Lifecycle contributors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +jest.mock('server/lib/dependencies', () => ({ + defaultDb: {}, + defaultRedis: {}, + defaultRedlock: {}, + defaultQueueManager: {}, + redisClient: { getConnection: jest.fn() }, +})); + +jest.mock('server/lib/logger', () => ({ + getLogger: jest.fn(() => ({ info: jest.fn(), warn: jest.fn(), error: jest.fn() })), + withLogContext: jest.fn((_context, action) => action()), + LogStage: {}, +})); + +jest.mock('server/lib/shell', () => ({ shellPromise: jest.fn() })); + +jest.mock('server/services/globalConfig', () => ({ + __esModule: true, + default: { + getInstance: jest.fn(() => ({ + getAllConfigs: jest.fn().mockResolvedValue({ + lifecycleDefaults: {}, + domainDefaults: { altHttp: [] }, + }), + })), + }, +})); + +import IngressService from '../ingress'; + +const queueManager = { + registerQueue: jest.fn(() => ({ add: jest.fn() })), +}; + +function authorityQuery(result: any) { + const query: any = { + findOne: jest.fn(() => query), + findById: jest.fn(() => query), + whereNull: jest.fn(() => query), + where: jest.fn(() => query), + then: (resolve: (value: any) => void, reject: (reason: unknown) => void) => + Promise.resolve(result).then(resolve, reject), + }; + return query; +} + +describe('IngressService generation fencing', () => { + test('applies ingress through the shared native promotion gate', async () => { + const configuration = { + deployUUID: 'deploy-a', + host: 'a.example.test', + pathPortMapping: { '/': 8080 }, + serviceHost: 'service-a', + ingressAnnotations: {}, + ipWhitelist: [], + }; + const gate = jest.fn(async (_buildId, isCurrent, action) => { + expect(await isCurrent()).toBe(true); + return { admitted: true, value: await action() }; + }); + const db = { + models: { Build: { query: jest.fn(() => authorityQuery({ id: 7 })) } }, + services: { + BuildService: { + configurationsForBuildId: jest.fn().mockResolvedValue([configuration]), + getNamespace: jest.fn().mockResolvedValue('env-test'), + withCurrentBuildPromotionLock: gate, + }, + }, + }; + const service = new IngressService(db as any, {} as any, {} as any, queueManager as any); + const apply = jest.spyOn(service as any, 'applyManifests').mockResolvedValue(undefined); + + await service.createOrUpdateIngressForBuild({ + data: { buildId: 7, runUUID: 'run-c', expectedGeneration: 3 }, + }); + + expect(gate).toHaveBeenCalledWith(7, expect.any(Function), expect.any(Function)); + expect(apply).toHaveBeenCalledWith(expect.any(String), '7-0-nginx', 'env-test', 7, 'run-c', 3); + }); + + test('does not apply when authority is lost before promotion admission', async () => { + const gate = jest.fn().mockResolvedValue({ admitted: false }); + const db = { + models: { Build: { query: jest.fn(() => authorityQuery({ id: 7 })) } }, + services: { + BuildService: { + configurationsForBuildId: jest.fn().mockResolvedValue([ + { + deployUUID: 'deploy-a', + host: 'a.example.test', + pathPortMapping: { '/': 8080 }, + serviceHost: 'service-a', + ingressAnnotations: {}, + ipWhitelist: [], + }, + ]), + getNamespace: jest.fn().mockResolvedValue('env-test'), + withCurrentBuildPromotionLock: gate, + }, + }, + }; + const service = new IngressService(db as any, {} as any, {} as any, queueManager as any); + const apply = jest.spyOn(service as any, 'applyManifests'); + + await service.createOrUpdateIngressForBuild({ + data: { buildId: 7, runUUID: 'run-a', expectedGeneration: 1 }, + }); + + expect(apply).not.toHaveBeenCalled(); + }); + + test('fences a late ingress failure note by run token and generation', async () => { + const read = authorityQuery({ id: 7, statusMessage: 'deployed' }); + const patch: any = { + patch: jest.fn(() => patch), + where: jest.fn(() => patch), + then: (resolve: (value: number) => void, reject: (reason: unknown) => void) => + Promise.resolve(1).then(resolve, reject), + }; + const db = { + models: { Build: { query: jest.fn().mockReturnValueOnce(read).mockReturnValueOnce(patch) } }, + services: {}, + }; + const service = new IngressService(db as any, {} as any, {} as any, queueManager as any); + + await (service as any).recordIngressFailureOnBuild(7, new Error('bad route'), 'run-c', 3); + + expect(read.where).toHaveBeenCalledWith('runUUID', 'run-c'); + expect(read.where).toHaveBeenCalledWith('desiredGeneration', 3); + expect(patch.patch).toHaveBeenCalledWith({ statusMessage: 'deployed | Ingress apply failed: bad route' }); + expect(patch.where).toHaveBeenCalledWith({ id: 7 }); + expect(patch.where).toHaveBeenCalledWith('runUUID', 'run-c'); + expect(patch.where).toHaveBeenCalledWith('desiredGeneration', 3); + }); +}); diff --git a/src/server/services/__tests__/override.test.ts b/src/server/services/__tests__/override.test.ts index a0909d97..4313aa2c 100644 --- a/src/server/services/__tests__/override.test.ts +++ b/src/server/services/__tests__/override.test.ts @@ -203,8 +203,6 @@ describe('OverrideService.applyBuildOverrides', () => { expect(enqueueResolveAndDeployBuild).toHaveBeenCalledWith({ buildId: 42, runUUID: 'run-uuid', - triggerRef: 'run-uuid', - correlationId: 'test-correlation', }); }); @@ -349,8 +347,6 @@ describe('OverrideService.applyBuildOverrides', () => { expect(enqueueResolveAndDeployBuild).toHaveBeenCalledWith({ buildId: 42, runUUID: 'run-uuid', - triggerRef: 'run-uuid', - correlationId: 'test-correlation', }); }); @@ -381,8 +377,6 @@ describe('OverrideService.applyBuildOverrides', () => { expect(enqueueResolveAndDeployBuild).toHaveBeenCalledWith({ buildId: 42, runUUID: 'run-uuid', - triggerRef: 'run-uuid', - correlationId: 'test-correlation', }); expect(result).toEqual({ buildUuid: 'current-build', @@ -781,8 +775,6 @@ describe('OverrideService.applyBuildConfigPatch', () => { expect(enqueueResolveAndDeployBuild).toHaveBeenCalledWith({ buildId: 42, runUUID: 'run-uuid', - triggerRef: 'run-uuid', - correlationId: 'test-correlation', }); expect(result).toMatchObject({ uuid: 'current-build', @@ -814,8 +806,6 @@ describe('OverrideService.applyBuildConfigPatch', () => { expect(enqueueResolveAndDeployBuild).toHaveBeenCalledWith({ buildId: 42, runUUID: 'run-uuid', - triggerRef: 'run-uuid', - correlationId: 'test-correlation', }); expect(result).toMatchObject({ uuid: 'current-build', @@ -918,8 +908,6 @@ describe('OverrideService.applyBuildConfigPatch', () => { expect(enqueueResolveAndDeployBuild).toHaveBeenCalledWith({ buildId: 42, runUUID: 'run-uuid', - triggerRef: 'run-uuid', - correlationId: 'test-correlation', }); expect(result).toMatchObject({ id: 42, @@ -972,8 +960,6 @@ describe('OverrideService.applyBuildConfigPatch', () => { expect(mockFallbackEnqueueResolveAndDeployBuild).toHaveBeenCalledWith({ buildId: 42, runUUID: 'run-uuid', - triggerRef: 'run-uuid', - correlationId: 'test-correlation', }); }); }); diff --git a/src/server/services/__tests__/overrideDeployGate.test.ts b/src/server/services/__tests__/overrideDeployGate.test.ts index 970a91ef..73f0a589 100644 --- a/src/server/services/__tests__/overrideDeployGate.test.ts +++ b/src/server/services/__tests__/overrideDeployGate.test.ts @@ -43,7 +43,7 @@ describe('enqueueRedeployIfEnabled PR-less gate (F1 site)', () => { expect(queued).toBe(true); expect(enqueueResolveAndDeployBuild).toHaveBeenCalledWith( - expect.objectContaining({ buildId: 9, triggerRef: 'run-1' }) + expect.objectContaining({ buildId: 9, runUUID: 'run-1' }) ); }); diff --git a/src/server/services/activityStream.ts b/src/server/services/activityStream.ts index 0c14709d..8e6cf6a6 100644 --- a/src/server/services/activityStream.ts +++ b/src/server/services/activityStream.ts @@ -150,9 +150,6 @@ export default class ActivityStream extends BaseService { await this.db.services.BuildService.enqueueResolveAndDeployBuild({ buildId, runUUID: runUuid, - // Use the unique run id as the trigger so an explicit redeploy is never coalesced into a prior deploy. - triggerRef: runUuid, - ...extractContextForQueue(), }); return; } diff --git a/src/server/services/agent/tools/lifecycle/__tests__/triggerRedeploy.test.ts b/src/server/services/agent/tools/lifecycle/__tests__/triggerRedeploy.test.ts index 50a23d65..3a559fc8 100644 --- a/src/server/services/agent/tools/lifecycle/__tests__/triggerRedeploy.test.ts +++ b/src/server/services/agent/tools/lifecycle/__tests__/triggerRedeploy.test.ts @@ -28,7 +28,7 @@ jest.mock('server/models/Build', () => ({ jest.mock('server/services/build', () => ({ __esModule: true, default: class MockBuildService { - resolveAndDeployBuildQueue = { add: mockQueueAdd }; + enqueueResolveAndDeployBuild = (...args: unknown[]) => mockQueueAdd(...args); }, })); @@ -67,6 +67,7 @@ describe('TriggerRedeployTool', () => { const result = await tool.execute({ reason: 'transient failure' }); expect(result.success).toBe(true); + expect(mockQueueAdd).toHaveBeenCalledWith(expect.objectContaining({ buildId: 11, runUUID: expect.any(String) })); expect(mockScheduleEnvironmentWatch).toHaveBeenCalledWith( expect.objectContaining({ buildUuid: 'build-1', diff --git a/src/server/services/agent/tools/lifecycle/triggerRedeploy.ts b/src/server/services/agent/tools/lifecycle/triggerRedeploy.ts index 30503984..120e31b8 100644 --- a/src/server/services/agent/tools/lifecycle/triggerRedeploy.ts +++ b/src/server/services/agent/tools/lifecycle/triggerRedeploy.ts @@ -73,11 +73,9 @@ export class TriggerRedeployTool extends BaseTool { return this.createErrorResult(`Build not found for ${buildUuid}`, 'BUILD_NOT_FOUND'); } - const correlationId = `agent-redeploy-${Date.now()}-${nanoid(8)}`; - await new BuildService().resolveAndDeployBuildQueue.add('resolve-deploy', { + await new BuildService().enqueueResolveAndDeployBuild({ buildId: build.id, runUUID: nanoid(), - correlationId, }); const { default: EnvironmentWatchService } = await import('server/services/agent/EnvironmentWatchService'); diff --git a/src/server/services/build.ts b/src/server/services/build.ts index 516f8ca3..69637e60 100644 --- a/src/server/services/build.ts +++ b/src/server/services/build.ts @@ -28,6 +28,7 @@ import { computeExtendedExpiry, computeInitialExpiry, isExpired } from 'server/l import { getUtcTimestamp } from 'server/lib/time'; import { AppError, BadRequestError } from 'server/lib/appError'; import { containsSecretRefTemplate } from 'server/lib/secretRefs'; +import { isNativeBuilderEngine } from 'server/lib/buildEngines'; import { validateBuildUuidFormat } from 'server/lib/validation/buildUuidValidator'; import { normalizeRepoFullName } from 'server/lib/normalizeRepoFullName'; import { toPublicHref } from 'server/lib/publicHref'; @@ -36,9 +37,7 @@ import { UniqueViolationError, type AnyQueryBuilder, type Transaction } from 'ob import { Build, Deploy, Deployable, Environment, Repository } from 'server/models'; import { BuildKind, BuildStatus, CLIDeployTypes, DeployStatus, DeployTypes, PullRequestStatus } from 'shared/constants'; import { type DeployOptions } from './deploy'; -import DeployService from './deploy'; import BaseService from './_service'; -import hash from 'object-hash'; import _ from 'lodash'; import { QUEUE_NAMES } from 'shared/config'; import { LifecycleError } from 'server/lib/errors'; @@ -48,7 +47,8 @@ import { ValidationError, YamlConfigValidator } from 'server/lib/yamlConfigValid import { type LifecycleYamlConfigOptions } from 'server/models/yaml/types'; import type { DeployableReconciliationResult } from 'server/services/deployable'; -import { DeploymentManager } from 'server/lib/deploymentManager/deploymentManager'; +import { DeploymentManager, DeploymentSupersededError } from 'server/lib/deploymentManager/deploymentManager'; +import { AuthorityLockLostError, type AuthorityLockResult, withAuthorityLock } from 'server/lib/authorityLock'; import { ensureServiceAccountForJob } from 'server/lib/kubernetes/common/serviceAccount'; import { Tracer } from 'server/lib/tracer'; import { redisClient } from 'server/lib/dependencies'; @@ -67,14 +67,20 @@ import { getBranchName, getDeployType, getRepositoryName, type Service } from 's import type { LifecycleConfig } from 'server/models/yaml/Config'; import * as YamlService from 'server/models/yaml'; import { getEnvironmentPhaseFromState, isActiveServiceReady } from 'server/lib/environments/readiness'; +import { + acceptDeploymentIntent, + dirtyDeploymentIntents, + type DeploymentIntent, + type DeploymentReconciliationJobData, + type DirtyDeploymentIntent, +} from 'server/lib/deploymentReconciliation/mailbox'; const tracer = Tracer.getInstance(); tracer.initialize('build-service'); -const RESOLVE_QUEUE_DEDUP_TTL_MS = 30000; -// Far beyond any queue retry window, so a delayed worker still sees the marker; keys then expire instead of accumulating. -const TRIGGER_SEQUENCE_TTL_SECONDS = 7 * 24 * 60 * 60; const TEARDOWN_RETRY_GRACE_MS = 15 * 60 * 1000; const BUILD_DEPLOYMENT_LOCK_TTL_MS = 15 * 60 * 1000; +const DEPLOY_SECRET_MUTATION_LOCK_TTL_MS = 2 * 60 * 1000; +const DEPLOYMENT_RECONCILIATION_SWEEP_MS = 5_000; const HOUR_MS = 60 * 60 * 1000; const PR_AUTHORITY_REVALIDATED_DELETE_REASONS = new Set([ 'pull_request_closed', @@ -83,6 +89,10 @@ const PR_AUTHORITY_REVALIDATED_DELETE_REASONS = new Set([ 'ttl_closed_pull_request', ]); +function requiresPrDeletionAuthorityRevalidation(reason?: string): boolean { + return reason === 'teardown_stuck' || PR_AUTHORITY_REVALIDATED_DELETE_REASONS.has(reason ?? ''); +} + /** * Every delete reason for one Build converges on the same durable ownership * token. BullMQ keeps the first payload when a deterministic jobId is already @@ -108,15 +118,61 @@ type DeployServiceOverrideState = Pick< 'name' | 'branchOrExternalUrl' | 'group' | 'editable' >; -interface ResolveAndDeployBuildOptions { - /** The build worker already holds the per-build deployment lock. */ - deploymentLockAlreadyHeld?: boolean; - /** The build worker claimed this run before importing lifecycle.yaml. */ - runAlreadyClaimed?: boolean; - /** Preserve an enqueue-time run token for explicit redeploys/API create hand-off. */ - runUUID?: string | null; +interface DeploymentScopeOptions { + runUUID: string; /** Exact case-sensitive branch paired with an immutable push source ref. */ sourceBranch?: string | null; + /** Repository that owns sourceRef; independent from the execution target selector. */ + sourceGithubRepositoryId?: number | null; + /** Exact mailbox generation owned by this pass. */ + expectedGeneration?: number; +} + +interface DeploymentReconciliationClaim { + generation: number; + token: string; + dirty: DirtyDeploymentIntent[]; +} + +interface DeploymentReconciliationScope { + githubRepositoryId: number | null; + sourceGithubRepositoryId?: number; + sourceRef?: string; + sourceBeforeRef?: string; + sourceBranch?: string; + skipDeletedServiceReconciliation?: boolean; +} + +interface DeploymentScopePreparation { + build: Build; + runUUID: string; + githubRepositoryId: number | null; + sourceGithubRepositoryId?: number | null; + sourceRef?: string | null; + sourceBranch?: string | null; +} + +interface DeploymentScopeOutcome { + status: BuildStatus; + error?: unknown; +} + +interface DeploymentReconciliationRequest { + buildId: number; + githubRepositoryId?: number | null; + triggerRef?: string | null; + sourceBranch?: string | null; + sourceGithubRepositoryId?: number | null; + sourceRef?: string | null; + sourceBeforeRef?: string | null; + runUUID?: string | null; +} + +interface DeploymentReconciliationQueueJob { + data: DeploymentReconciliationJobData; + /** BullMQ counts attempts completed before the currently running attempt. */ + attemptsMade?: number; + opts?: { attempts?: number }; } interface DeleteBuildOptions { @@ -168,6 +224,7 @@ export interface LockedApiEnvironmentDeletionGuard { export default class BuildService extends BaseService { ingressService = new IngressService(this.db, this.redis, this.redlock, this.queueManager); + private deploymentReconciliationSweepCursor = 0; /** * For every build that is not closed @@ -203,230 +260,57 @@ export default class BuildService extends BaseService { } } - private async getBuildForQueueFingerprint(buildId: number): Promise { - const build = await this.db.models.Build.query() - .findOne({ id: buildId }) - .withGraphFetched('[pullRequest, deploys.[deployable]]'); - return build ?? null; - } - - private getBuildFingerprintDeployKey(deploy: Deploy): string { - return deploy.deployable?.name || deploy.uuid || String(deploy.id || ''); - } - - private buildFingerprintPayload(build: Build, githubRepositoryId?: number, sourceBranch?: string | null) { - const deploys = (build.deploys || []) - .filter( - (deploy) => - (!githubRepositoryId || deploy.githubRepositoryId === githubRepositoryId) && - (!githubRepositoryId || !sourceBranch || deploy.branchName === sourceBranch) - ) - .map((deploy) => ({ - key: this.getBuildFingerprintDeployKey(deploy), - githubRepositoryId: deploy.githubRepositoryId ?? null, - branchName: deploy.branchName ?? null, - active: deploy.active ?? true, - publicUrl: deploy.publicUrl ?? null, - env: deploy.env || {}, - initEnv: deploy.initEnv || {}, - commentBranchName: deploy.deployable?.commentBranchName ?? null, - })); - - return { - buildId: build.id, - githubRepositoryId: githubRepositoryId ?? null, - sourceBranch: sourceBranch ?? null, - latestCommit: build.pullRequest?.latestCommit ?? null, - commentRuntimeEnv: build.commentRuntimeEnv || {}, - commentInitEnv: build.commentInitEnv || {}, - isStatic: build.isStatic ?? false, - deploys: _.sortBy(deploys, 'key'), - }; - } - - async computeBuildRequestFingerprint( - buildOrId: Build | number, - githubRepositoryId?: number, - sourceBranch?: string | null - ): Promise { - const build = - typeof buildOrId === 'number' ? await this.getBuildForQueueFingerprint(buildOrId) : (buildOrId as Build); - - if (!build) { - throw new Error(`Build not found for fingerprint`); + private deploymentIntentForRequest( + { + githubRepositoryId, + triggerRef, + sourceBranch, + sourceGithubRepositoryId, + sourceRef, + sourceBeforeRef, + }: DeploymentReconciliationRequest, + requestId: string + ): DeploymentIntent { + const immutableSourceRef = sourceRef || (sourceGithubRepositoryId != null ? triggerRef : null); + if (sourceGithubRepositoryId != null && sourceBranch && immutableSourceRef) { + return { + type: 'source', + requestId, + target: githubRepositoryId == null ? 'all' : 'repository', + githubRepositoryId: Number(sourceGithubRepositoryId), + branch: sourceBranch, + sha: immutableSourceRef, + ...(sourceBeforeRef ? { beforeSha: sourceBeforeRef } : {}), + }; } - - if (!build.pullRequest || !build.deploys) { - await build.$fetchGraph('[pullRequest, deploys.[deployable]]'); + if (githubRepositoryId != null) { + return { type: 'repository', requestId, githubRepositoryId: Number(githubRepositoryId) }; } - - return hash(this.buildFingerprintPayload(build, githubRepositoryId, sourceBranch)); + return { type: 'all', requestId }; } - async enqueueResolveAndDeployBuild({ - buildId, - githubRepositoryId, - triggerRef, - sourceBranch, - ...jobData - }: { - buildId: number; - githubRepositoryId?: number | null; - triggerRef?: string | null; - sourceBranch?: string | null; - [key: string]: any; - }) { - const fingerprint = await this.computeBuildRequestFingerprint( - buildId, - githubRepositoryId ?? undefined, - sourceBranch - ); - // The fingerprint only captures build configuration, not the commit being deployed. Without a per-trigger - // suffix, two deploys of the same build (e.g. two pushes to a tracked branch landing close together) collapse - // onto one dedupe key, and the later one is silently dropped. triggerRef (the pushed commit, or a redeploy id) - // makes each distinct trigger its own key while still coalescing genuine duplicates of the same trigger. - const suffix = triggerRef ? `:${triggerRef}` : ''; - const dedupeId = `resolve:${buildId}:${fingerprint}${suffix}`; - getLogger({ stage: LogStage.BUILD_QUEUED }).info( - `Build queue: name=resolve-deploy buildId=${buildId} scope=${githubRepositoryId || 'all'}:${ - sourceBranch || 'all' - } dedupeKey=${dedupeId}` - ); - return this.resolveAndDeployBuildQueue.add( - 'resolve-deploy', - { - buildId, - ...(githubRepositoryId ? { githubRepositoryId } : {}), - ...(triggerRef ? { triggerRef } : {}), - ...(sourceBranch ? { sourceBranch } : {}), - ...jobData, - }, - { - deduplication: { - id: dedupeId, - ttl: RESOLVE_QUEUE_DEDUP_TTL_MS, - }, - } - ); - } + async enqueueResolveAndDeployBuild(request: DeploymentReconciliationRequest) { + // The source SHA is desired content, not execution ownership. A unique token + // prevents two scopes that happen to reference the same commit from accepting + // each other's late Deploy writes. + const requestId = request.runUUID || nanoid(); + const intent = this.deploymentIntentForRequest(request, requestId); + const accepted = await acceptDeploymentIntent(request.buildId, intent); - async enqueueBuildJob({ - buildId, - githubRepositoryId, - triggerRef, - sourceBranch, - ...jobData - }: { - buildId: number; - githubRepositoryId?: number | null; - triggerRef?: string | null; - sourceBranch?: string | null; - [key: string]: any; - }) { - const fingerprint = await this.computeBuildRequestFingerprint( - buildId, - githubRepositoryId ?? undefined, - sourceBranch - ); - // Mirror the suffix used by the resolve step so both queue layers agree on identity. A build job is keyed by - // jobId, which makes add() idempotent: an enqueue whose jobId matches an existing job is a no-op rather than new - // work. Including triggerRef ensures a distinct trigger yields a distinct job instead of being dropped. - const suffix = triggerRef ? `:${triggerRef}` : ''; - const jobId = `build:${buildId}:${fingerprint}${suffix}`; - getLogger({ stage: LogStage.BUILD_QUEUED }).info( - `Build queue: name=build buildId=${buildId} scope=${githubRepositoryId || 'all'}:${ - sourceBranch || 'all' - } jobId=${jobId}` - ); - // Best-effort visibility: a matching job here means this enqueue will be coalesced into existing work rather - // than building. Without this log the drop is invisible, since the dedupe happens inside the queue. - const existing = await this.buildQueue.getJob(jobId); - if (existing) { - getLogger({ stage: LogStage.BUILD_QUEUED }).info( - `Build queue: skipped reason=deduped buildId=${buildId} jobId=${jobId}` - ); + if (!accepted) throw new Error(`Build ${request.buildId} was not found while accepting deployment work`); + if (!accepted.accepted) { + getLogger({ + buildId: request.buildId, + scopeKey: accepted.scopeKey, + generation: accepted.generation, + }).info('Build reconciliation: duplicate deployment intent ignored'); + return null; } - return this.buildQueue.add( - 'build', - { - buildId, - ...(githubRepositoryId ? { githubRepositoryId } : {}), - ...(triggerRef ? { triggerRef } : {}), - ...(sourceBranch ? { sourceBranch } : {}), - ...jobData, - }, - { - jobId, - } - ); - } - - private normalizeTriggerSequence(value: unknown): string | null { - if (value == null) return null; - const raw = String(value); - if (!/^\d+$/.test(raw)) return null; - return raw.replace(/^0+(?=\d)/, ''); - } - - private compareTriggerSequences(left: string, right: string): number { - if (left.length !== right.length) return left.length < right.length ? -1 : 1; - return left === right ? 0 : left < right ? -1 : 1; - } - - /** - * BullMQ assigns an atomic monotonic id to each accepted resolve job; a - * deduplicated redelivery receives the existing id. Persist the latest id per - * build/source scope so a delayed worker cannot roll a newer sourceRef back. - * The per-build deployment lock makes this read/compare/write atomic. - */ - private async claimTriggerSequence( - buildId: number, - githubRepositoryId: number | null | undefined, - sourceBranch: string | null | undefined, - triggerSequence: unknown - ): Promise { - const sequence = this.normalizeTriggerSequence(triggerSequence); - // Legacy/internal jobs created before sequencing have no resolve job id. - if (!sequence) return true; - - const scope = - githubRepositoryId == null ? 'all' : `${githubRepositoryId}.${encodeURIComponent(sourceBranch ?? 'all')}`; - // Include the versioned resolve queue name: BullMQ's auto-id counter can - // restart when JOB_VERSION creates a new queue, while older Redis markers remain. - const key = `build-deployment-sequence.${QUEUE_NAMES.RESOLVE_AND_DEPLOY}.${buildId}.${scope}`; - const current = this.normalizeTriggerSequence(await this.redis.get(key)); - if (current && this.compareTriggerSequences(sequence, current) < 0) return false; - if (current !== sequence) await this.redis.set(key, sequence, 'EX', TRIGGER_SEQUENCE_TTL_SECONDS); - return true; - } - /** Best-effort convergence: a delivered ref behind the live branch head deploys the head instead; any lookup failure keeps the delivered ref. */ - private async resolveEffectiveSourceRef( - githubRepositoryId: number | null | undefined, - sourceBranch: string | null | undefined, - sourceRef: string | null | undefined - ): Promise { - if (githubRepositoryId == null || !sourceBranch || !sourceRef) return sourceRef; - - try { - const repository: Repository | undefined = await this.db.models.Repository.query() - .findOne({ githubRepositoryId: Number(githubRepositoryId) }) - .whereNull('deletedAt'); - const parts = repository?.fullName?.split('/') ?? []; - if (parts.length !== 2 || !parts[0] || !parts[1]) return sourceRef; - - const currentHead = await github.getSHAForBranch(sourceBranch, parts[0], parts[1]); - if (!currentHead || currentHead === sourceRef) return sourceRef; - getLogger({ githubRepositoryId, sourceBranch, sourceRef, currentHead }).info( - 'Build: deploying reason=source_ref_behind_branch_head' - ); - return currentHead; - } catch (error) { - getLogger({ error, githubRepositoryId, sourceBranch }).warn( - 'Build: branch head check failed; deploying delivered ref' - ); - return sourceRef; - } + getLogger({ stage: LogStage.BUILD_QUEUED, buildId: request.buildId, generation: accepted.generation }).info( + 'Build reconciliation: accepted' + ); + return this.signalPendingDeploymentReconciliation(request.buildId, accepted.generation); } /** @@ -659,7 +543,18 @@ export default class BuildService extends BaseService { ); }) .modifyGraph('deploys.deployable', (b) => { - b.select('name', 'type', 'dockerfilePath', 'deploymentDependsOn', 'builder', 'ecr', 'grpc', 'hostPortMapping'); + b.select( + 'name', + 'type', + 'dockerfilePath', + 'requires', + 'deploymentDependsOn', + 'dependsOnDeployableName', + 'builder', + 'ecr', + 'grpc', + 'hostPortMapping' + ); }) .modifyGraph('deploys.repository', (b) => { b.select('fullName'); @@ -1073,81 +968,61 @@ export default class BuildService extends BaseService { } async redeployServiceFromBuild(buildUuid: string, serviceName: string, expectedBuildId?: number) { - const enqueue = async () => { - const identity = expectedBuildId == null ? { uuid: buildUuid } : { uuid: buildUuid, id: expectedBuildId }; - const build = await this.db.models.Build.query() - .findOne(identity) - .whereNull('deletedAt') - .withGraphFetched('[deploys.deployable, pullRequest]'); - - if (!build) { - getLogger().debug(`Build not found for ${buildUuid}.`); - return { - status: 'not_found', - message: `Build not found for ${buildUuid}.`, - }; - } - const blocked = this.deploymentBlockReason(build); - if (blocked === 'torn_down') { - return { - status: 'tearing_down', - message: `Build ${buildUuid} is being (or has been) torn down and cannot be redeployed.`, - }; - } - if (blocked) { - return { - status: 'deploy_disabled', - message: `Deploys are disabled for build ${buildUuid}; enable deploys before redeploying.`, - }; - } - - const buildId = build.id; - - const deploy = build.deploys?.find((deploy) => deploy.deployable?.name === serviceName); - - if (!deploy || !deploy.deployable) { - getLogger().debug(`Deployable ${serviceName} not found for ${buildUuid}.`); - throw new Error(`Deployable ${serviceName} not found for ${buildUuid}.`); - } - - const githubRepositoryId = Number(deploy.deployable.repositoryId); - - const runUUID = nanoid(); - - await this.enqueueResolveAndDeployBuild({ - buildId, - githubRepositoryId, - skipDeletedServiceReconciliation: true, - runUUID, - // Use the unique run id as the trigger so an explicit redeploy is never coalesced into a prior deploy. - triggerRef: runUUID, - ...extractContextForQueue(), - }); - - getLogger({ stage: LogStage.BUILD_QUEUED }).info(`Build: service redeploy queued service=${serviceName}`); + const identity = expectedBuildId == null ? { uuid: buildUuid } : { uuid: buildUuid, id: expectedBuildId }; + const build = await this.db.models.Build.query() + .findOne(identity) + .whereNull('deletedAt') + .withGraphFetched('[deploys.deployable, pullRequest]'); - const deployService = new DeployService(); + if (!build) { + getLogger().debug(`Build not found for ${buildUuid}.`); + return { + status: 'not_found', + message: `Build not found for ${buildUuid}.`, + }; + } + const blocked = this.deploymentBlockReason(build); + if (blocked === 'torn_down') { + return { + status: 'tearing_down', + message: `Build ${buildUuid} is being (or has been) torn down and cannot be redeployed.`, + }; + } + if (blocked) { + return { + status: 'deploy_disabled', + message: `Deploys are disabled for build ${buildUuid}; enable deploys before redeploying.`, + }; + } - await deploy.$query().patchAndFetch({ - runUUID, - }); + const deploy = build.deploys?.find((candidate) => candidate.deployable?.name === serviceName); + if (!deploy?.deployable) { + getLogger().debug(`Deployable ${serviceName} not found for ${buildUuid}.`); + throw new Error(`Deployable ${serviceName} not found for ${buildUuid}.`); + } - await deployService.patchAndUpdateActivityFeed( - deploy, - { - status: DeployStatus.QUEUED, - }, - runUUID, - githubRepositoryId + const sourceRepositoryId = + deploy.deployable.resolvedFromRepositoryId ?? deploy.deployable.repositoryId ?? deploy.githubRepositoryId; + if (sourceRepositoryId == null) { + getLogger({ buildUuid, serviceName, deployId: deploy.id }).error( + 'Build: service redeploy has no source repository identity' ); + throw new Error(`Cannot redeploy ${serviceName}: source repository is unknown.`); + } - return { - status: 'success', - message: `Redeploy for service ${serviceName} in environment ${buildUuid} has been queued`, - }; - }; + const runUUID = nanoid(); + await this.enqueueResolveAndDeployBuild({ + buildId: build.id, + githubRepositoryId: Number(sourceRepositoryId), + runUUID, + }); - return expectedBuildId == null ? enqueue() : this.withBuildDeploymentLock(expectedBuildId, enqueue); + // The mailbox worker owns status/token changes for every selected Deploy. + getLogger({ stage: LogStage.BUILD_QUEUED }).info(`Build: service redeploy queued service=${serviceName}`); + return { + status: 'success', + message: `Redeploy for service ${serviceName} in environment ${buildUuid} has been queued`, + }; } async redeployBuild( @@ -1198,9 +1073,6 @@ export default class BuildService extends BaseService { await this.enqueueResolveAndDeployBuild({ buildId, runUUID, - // Use the unique run id as the trigger so an explicit redeploy is never coalesced into a prior deploy. - triggerRef: runUUID, - correlationId, }); getLogger({ stage: LogStage.BUILD_QUEUED }).info('Build: redeploy queued'); @@ -1729,8 +1601,6 @@ export default class BuildService extends BaseService { await this.enqueueResolveAndDeployBuild({ buildId: build.id, runUUID, - triggerRef: runUUID, - ...extractContextForQueue(), }); } else { getLogger().info('Environment: api create complete deploy=paused'); @@ -1996,8 +1866,6 @@ export default class BuildService extends BaseService { await this.enqueueResolveAndDeployBuild({ buildId: build.id, runUUID: runUuid, - triggerRef: runUuid, - ...extractContextForQueue(), }); return { mode: 'redeploy_queued', changed: true, deployId: runUuid, build: persisted.build }; } @@ -2014,7 +1882,10 @@ export default class BuildService extends BaseService { : buildTeardownRunUUID(build.id); // Conditional cleanup never reuses a jobId: BullMQ would swallow a fresh close enqueued while a // prior job (about to decide authority_restored) still exists; claim-time revalidation absorbs duplicates. - const jobId = PR_AUTHORITY_REVALIDATED_DELETE_REASONS.has(reason) + const usesConditionalJobId = + PR_AUTHORITY_REVALIDATED_DELETE_REASONS.has(reason) || + (reason === 'teardown_stuck' && build.pullRequestId != null); + const jobId = usesConditionalJobId ? `build-delete-${build.id}-conditional-${nanoid()}` : `build-delete-${build.id}-authoritative`; await this.deleteQueue.add( @@ -2118,11 +1989,25 @@ export default class BuildService extends BaseService { .whereNotIn('status', [BuildStatus.TORN_DOWN, BuildStatus.TEARING_DOWN]) .whereNull('deletedAt'); - // A teardown whose delete job exhausted its retries would otherwise strand cleanup or identity release, - // leaking the namespace or locking the vanity uuid; updatedAt bumps on every teardown attempt. + // A teardown whose delete job exhausted its retries would otherwise strand cleanup or identity release; + // updatedAt bumps on every teardown attempt. const stuckTeardowns = await this.db.models.Build.query() - .where('triggerType', 'api') - .whereIn('status', [BuildStatus.TEARING_DOWN, BuildStatus.TORN_DOWN]) + .where((builder) => { + builder + .where((apiBuilds) => { + apiBuilds.where('triggerType', 'api').whereIn('status', [BuildStatus.TEARING_DOWN, BuildStatus.TORN_DOWN]); + }) + .orWhere((pullRequestBuilds) => { + pullRequestBuilds + .whereNotNull('pullRequestId') + .whereNot('status', BuildStatus.TORN_DOWN) + .where((teardown) => { + teardown + .where('status', BuildStatus.TEARING_DOWN) + .orWhereRaw('?? = concat(?, ??)', ['runUUID', 'build-teardown-', 'id']); + }); + }); + }) .where('updatedAt', '<=', new Date(now.getTime() - TEARDOWN_RETRY_GRACE_MS).toISOString()) .whereNull('deletedAt'); @@ -2403,15 +2288,19 @@ export default class BuildService extends BaseService { skipDeletedServiceReconciliation?: boolean; sourceRef?: string | null; sourceBranch?: string | null; + sourceGithubRepositoryId?: number | null; + runUUID?: string; + expectedGeneration?: number; } = {} ) { const buildSource = getBuildSource(build); + const sourceGithubRepositoryId = options.sourceGithubRepositoryId ?? filterGithubRepositoryId; const sourceRefTargetsRoot = options.sourceRef != null && options.sourceBranch != null && - filterGithubRepositoryId != null && + sourceGithubRepositoryId != null && buildSource.githubRepositoryId != null && - Number(filterGithubRepositoryId) === Number(buildSource.githubRepositoryId) && + Number(sourceGithubRepositoryId) === Number(buildSource.githubRepositoryId) && options.sourceBranch === buildSource.branchName; const rootSourceRef = sourceRefTargetsRoot ? options.sourceRef : null; // Write the deployables here for now and not going to use them yet. @@ -2425,7 +2314,8 @@ export default class BuildService extends BaseService { build, filterGithubRepositoryId, options.sourceRef, - options.sourceBranch + options.sourceBranch, + sourceGithubRepositoryId ); if (options.skipDeletedServiceReconciliation) { @@ -2438,7 +2328,9 @@ export default class BuildService extends BaseService { build, reconciliationResult, filterGithubRepositoryId, - options.sourceBranch + options.sourceBranch, + options.runUUID, + options.expectedGeneration ); } } catch (error) { @@ -2465,7 +2357,9 @@ export default class BuildService extends BaseService { build: Build, reconciliationResult: DeployableReconciliationResult, filterGithubRepositoryId?: number, - sourceBranch?: string | null + sourceBranch?: string | null, + runUUID?: string, + expectedGeneration?: number ) { if (!reconciliationResult?.canReconcile) { return; @@ -2527,56 +2421,77 @@ export default class BuildService extends BaseService { return; } - const staleDeployableIds = staleDeployables.map((deployable) => deployable.id); - const staleDeploys = await this.db.models.Deploy.query() - .where({ buildId }) - .whereIn('deployableId', staleDeployableIds) - .withGraphFetched('[build, deployable]'); - const cleanupService = - this.db.services?.DeployCleanupService || - new DeployCleanupService(this.db, this.redis, this.redlock, this.queueManager); - - getLogger({ - buildUuid: build.uuid, - filterGithubRepositoryId, - sourceBranch, - staleDeployableNames: staleDeployables.map((deployable) => deployable.name), - staleDeployCount: staleDeploys.length, - }).warn('Stale deploy reconciliation: cleaning deleted deployables'); + const cleanup = async () => { + const staleDeployableIds = staleDeployables.map((deployable) => deployable.id); + const staleDeploys = await this.db.models.Deploy.query() + .where({ buildId }) + .whereIn('deployableId', staleDeployableIds) + .withGraphFetched('[build, deployable]'); + const cleanupService = + this.db.services?.DeployCleanupService || + new DeployCleanupService(this.db, this.redis, this.redlock, this.queueManager); - // Retained rows are the retry handle for failed external cleanup; a failure must not fail the deploy run. - const failedCleanupDeployableIds = new Set(); - await Promise.all( - staleDeploys.map(async (deploy) => { - try { - await cleanupService.cleanupDeploy(deploy, { mode: 'service' }); - } catch (error) { - if (deploy.deployableId != null) failedCleanupDeployableIds.add(deploy.deployableId); - getLogger({ - error, - buildUuid: build.uuid, - deployUuid: deploy.uuid, - deployableId: deploy.deployableId, - }).error('Stale deploy reconciliation: cleanup failed rows_retained=true continuing=true'); - } - }) - ); + getLogger({ + buildUuid: build.uuid, + filterGithubRepositoryId, + sourceBranch, + staleDeployableNames: staleDeployables.map((deployable) => deployable.name), + staleDeployCount: staleDeploys.length, + }).warn('Stale deploy reconciliation: cleaning deleted deployables'); - const cleanedDeployableIds = staleDeployableIds.filter((id) => !failedCleanupDeployableIds.has(id)); - if (cleanedDeployableIds.length > 0) { - await cleanupService.deleteServiceRows({ buildId, deployableIds: cleanedDeployableIds }); - } - await build.$fetchGraph('[deployables, deploys]'); + // Retained rows are the retry handle for failed external cleanup; a failure must not fail the deploy run. + const failedCleanupDeployableIds = new Set(); + await Promise.all( + staleDeploys.map(async (deploy) => { + try { + const cleaned = await cleanupService.cleanupDeploy(deploy, { + mode: 'service', + ...(runUUID + ? { + nativeMutationGate: async (action: () => Promise) => { + const result = await this.withCurrentBuildPromotionLock( + buildId, + () => this.isDeploymentRunCurrent(buildId, runUUID, expectedGeneration), + action + ); + if (!result.admitted) throw new DeploymentSupersededError(); + return result.value as T; + }, + } + : {}), + }); + if (!cleaned && deploy.deployableId != null) failedCleanupDeployableIds.add(deploy.deployableId); + } catch (error) { + if (error instanceof DeploymentSupersededError) throw error; + if (deploy.deployableId != null) failedCleanupDeployableIds.add(deploy.deployableId); + getLogger({ + error, + buildUuid: build.uuid, + deployUuid: deploy.uuid, + deployableId: deploy.deployableId, + }).error('Stale deploy reconciliation: cleanup failed rows_retained=true continuing=true'); + } + }) + ); - getLogger({ - buildUuid: build.uuid, - filterGithubRepositoryId, - sourceBranch, - staleDeployableNames: staleDeployables - .filter((deployable) => cleanedDeployableIds.includes(deployable.id)) - .map((deployable) => deployable.name), - retainedDeployableCount: failedCleanupDeployableIds.size, - }).warn('Stale deploy reconciliation: deleted stale deploy database rows'); + const cleanedDeployableIds = staleDeployableIds.filter((id) => !failedCleanupDeployableIds.has(id)); + if (cleanedDeployableIds.length > 0) { + await cleanupService.deleteServiceRows({ buildId, deployableIds: cleanedDeployableIds }); + } + await build.$fetchGraph('[deployables, deploys]'); + + getLogger({ + buildUuid: build.uuid, + filterGithubRepositoryId, + sourceBranch, + staleDeployableNames: staleDeployables + .filter((deployable) => cleanedDeployableIds.includes(deployable.id)) + .map((deployable) => deployable.name), + retainedDeployableCount: failedCleanupDeployableIds.size, + }).warn('Stale deploy reconciliation: deleted stale deploy database rows'); + }; + + await cleanup(); } public async createBuild( @@ -2594,7 +2509,7 @@ export default class BuildService extends BaseService { // After a build is susccessfully created or retrieved, // we need to create or update the deployables to be used for build and deploy. if (build && options != null) { - await this.withBuildDeploymentLock(build.id, async () => { + const prepareBuild = async () => { // A close webhook can queue teardown while the open/reopen handler is // still importing YAML. Re-read after the shared lock so setup cannot // steal run ownership from deletion or revive a closed PR. @@ -2647,7 +2562,13 @@ export default class BuildService extends BaseService { : 'Build setup failed before deploys could be created.' ); } - }); + }; + + await this.withCurrentBuildDeploymentLock( + build.id, + async () => this.buildSetupBlockReason(await this.loadBuildDeploymentAuthority(build.id)) == null, + prepareBuild + ); } else { throw new Error('Missing build or deployment options from environment.'); } @@ -2668,12 +2589,11 @@ export default class BuildService extends BaseService { if (build.deletedAt != null) return 'build_deleted'; if (build.pullRequest || build.pullRequestId != null) { + if (build.status === BuildStatus.TEARING_DOWN) return 'tearing_down'; if (!build.pullRequest) return 'pull_request_missing'; if (build.pullRequest.status !== PullRequestStatus.OPEN) return 'pull_request_closed'; if (!isDeployEnabled(build)) return 'deploy_disabled'; - // Re-adding the deploy label to an open PR historically recreates an - // environment after teardown. The current PR authority may reclaim that - // row; API environments remain terminal below. + // Re-adding the deploy label to an open PR may reclaim the row after teardown completes. return null; } @@ -2681,10 +2601,11 @@ export default class BuildService extends BaseService { return build.deployEnabled === true ? null : 'deploy_disabled'; } - /** PR setup is allowed without a deploy label, but never after close/teardown. */ + /** PR setup is allowed without a deploy label, but not while teardown is active or the PR is closed. */ private buildSetupBlockReason(build: Build | null | undefined): string | null { if (!build) return 'build_missing'; if (build.deletedAt != null) return 'build_deleted'; + if (build.status === BuildStatus.TEARING_DOWN) return 'tearing_down'; if (!build.pullRequest) return 'pull_request_missing'; return build.pullRequest.status === PullRequestStatus.OPEN ? null : 'pull_request_closed'; } @@ -2695,6 +2616,26 @@ export default class BuildService extends BaseService { return build ?? null; } + /** + * Prove that stale work lost authority to a newer live deployment intent, + * rather than teardown, deletion, or another non-deployment owner. + */ + async isSupersededByNewerLiveDeploymentGeneration(buildId: number, expectedGeneration?: number): Promise { + if (expectedGeneration == null || !Number.isSafeInteger(expectedGeneration)) return false; + + const current = await this.loadBuildDeploymentAuthority(buildId); + const desiredGeneration = Number(current?.desiredGeneration); + return Boolean( + current && + Number.isSafeInteger(desiredGeneration) && + desiredGeneration > expectedGeneration && + current.status !== BuildStatus.TEARING_DOWN && + current.status !== BuildStatus.TORN_DOWN && + current.runUUID !== buildTeardownRunUUID(buildId) && + this.deploymentBlockReason(current) == null + ); + } + private async isBuildSetupRunCurrent(buildId: number, runUUID: string): Promise { const current = await this.loadBuildDeploymentAuthority(buildId); return Boolean(current && current.runUUID === runUUID && this.buildSetupBlockReason(current) == null); @@ -2705,7 +2646,11 @@ export default class BuildService extends BaseService { * authority read closes the race with a PR label/close update, which lives on * a separate row and therefore cannot be guarded by the Build patch alone. */ - private async claimDeploymentRun(build: Build, requestedRunUUID?: string | null): Promise { + private async claimDeploymentRun( + build: Build, + requestedRunUUID?: string | null, + expectedGeneration?: number + ): Promise { const blocked = this.deploymentBlockReason(build); if (blocked) { getLogger().info(`Deploy: skipping reason=${blocked}`); @@ -2723,6 +2668,10 @@ export default class BuildService extends BaseService { .where({ id: build.id }) .whereNull('deletedAt'); + if (expectedGeneration != null) { + claim = claim.where('desiredGeneration', expectedGeneration); + } + // API-created environments keep their kill switch on the Build row, so make // that half of the claim atomic. PR authority is re-read immediately below. if (!build.pullRequest && build.pullRequestId == null) { @@ -2740,7 +2689,7 @@ export default class BuildService extends BaseService { build.runUUID = runUUID; build.status = BuildStatus.PENDING; build.statusMessage = null; - if (!(await this.isDeploymentRunCurrent(build.id, runUUID))) { + if (!(await this.isDeploymentRunCurrent(build.id, runUUID, expectedGeneration))) { getLogger().info('Deploy: aborting run reason=authority_changed'); return null; } @@ -2748,186 +2697,273 @@ export default class BuildService extends BaseService { } /** - * Deploy an existing build/PR (usually happens when adding the lifecycle-deploy! label) - * @param build Build associates to a PR - * @param deploy deploy on changed? + * Reconcile shared Build/Deploy rows while the short configuration lock is held. + * The expensive image/provider phase deliberately runs after this returns. */ - public async resolveAndDeployBuild( + private async prepareDeploymentScope( build: Build, - isDeploy: boolean, - githubRepositoryId: number | null = null, - sourceRef?: string | null, - options: ResolveAndDeployBuildOptions = {} - ) { - if (!options.deploymentLockAlreadyHeld) { - return this.withBuildDeploymentLock(build.id, () => - this.resolveAndDeployBuild(build, isDeploy, githubRepositoryId, sourceRef, { - ...options, - deploymentLockAlreadyHeld: true, - }) - ); + githubRepositoryId: number | null, + sourceRef: string | null | undefined, + options: DeploymentScopeOptions + ): Promise { + const runUUID = options.runUUID; + const expectedGeneration = options.expectedGeneration; + + if (build.uuid) updateLogContext({ buildUuid: build.uuid }); + await build.$fetchGraph('[environment, pullRequest.[repository]]'); + + if (!(await this.isDeploymentRunCurrent(build.id, runUUID, expectedGeneration))) { + getLogger().info('Deploy: preparation skipped reason=ownership_lost'); + return null; } + build.runUUID = runUUID; - // We have to always assume there may be no service entry into the database - // since the service config exists only in the YAML file. - /* Set populate deploys */ - let runUUID = options.runUUID || nanoid(); - /* We own the build for as long as we see this UUID and live deploy authority remains enabled. */ - const uuid = build?.uuid; + const deploys = await this.db.services.Deploy.findOrCreateDeploys( + build.environment!, + build, + githubRepositoryId ?? undefined, + sourceRef, + options.sourceBranch, + options.sourceGithubRepositoryId ?? githubRepositoryId + ); + build.$setRelated('deploys', deploys); + await build.$fetchGraph('pullRequest'); + await new BuildEnvironmentVariables(this.db).resolve(build, githubRepositoryId ?? undefined, options.sourceBranch); - if (uuid) { - updateLogContext({ buildUuid: uuid }); + if (!(await this.isDeploymentRunCurrent(build.id, runUUID, expectedGeneration))) { + getLogger().info('Deploy: preparation skipped after config reason=ownership_lost'); + return null; } - try { - await build?.$fetchGraph('[environment, pullRequest.[repository]]'); - if (options.runAlreadyClaimed) { - if (!runUUID || !(await this.isDeploymentRunCurrent(build.id, runUUID))) { - getLogger().info('Deploy: aborting run reason=ownership_lost'); - return build; - } - build.runUUID = runUUID; - } else { - const claimedRunUUID = await this.claimDeploymentRun(build, runUUID); - if (!claimedRunUUID) return build; - runUUID = claimedRunUUID; - } - const source = getBuildSource(build); - let fullName = source.fullName; - const branchName = source.branchName; - let latestCommit = source.pullRequest?.latestCommit; - const environment = build?.environment; - if (!fullName) { - fullName = (await resolveBuildSourceRepository(build))?.fullName ?? null; - } - if (!fullName) { - throw new Error(`Build ${build?.uuid} has no source repository to resolve`); - } - const [owner, name] = fullName.split('/'); - if (!latestCommit) { - latestCommit = source.configSha ?? undefined; - if (!latestCommit) { - if (!branchName) { - throw new Error(`Build ${build?.uuid} has no branch or pinned sha to resolve a commit from`); - } - latestCommit = await github.getSHAForBranch(branchName, owner, name); - } + let deployClaim = this.db.models.Deploy.query().patch({ runUUID }).where({ buildId: build.id }); + if (githubRepositoryId != null) { + deployClaim = deployClaim.where('githubRepositoryId', githubRepositoryId); + if (options.sourceBranch) deployClaim = deployClaim.where('branchName', options.sourceBranch); + } + await deployClaim; + for (const deploy of deploys ?? []) { + if ( + (githubRepositoryId == null || deploy.githubRepositoryId === githubRepositoryId) && + (githubRepositoryId == null || !options.sourceBranch || deploy.branchName === options.sourceBranch) + ) { + deploy.runUUID = runUUID; } - const deploys = await this.db.services.Deploy.findOrCreateDeploys( - environment!, - build, - githubRepositoryId ?? undefined, - sourceRef, - options.sourceBranch - ); - build?.$setRelated('deploys', deploys); - await build?.$fetchGraph('pullRequest'); - await new BuildEnvironmentVariables(this.db).resolve( - build, - githubRepositoryId ?? undefined, - options.sourceBranch - ); + } - // Source/config resolution can take long enough for a PR close/label removal - // or API pause to land. Never begin shared build/CLI work on stale authority. - if (!(await this.isDeploymentRunCurrent(build.id, runUUID))) { - getLogger().info('Deploy: aborting build reason=authority_changed'); - return build; - } + await this.markConfigurationsAsBuilt(build, runUUID, githubRepositoryId, options.sourceBranch); + await this.updateStatusAndComment(build, BuildStatus.BUILDING, runUUID, true, true, null, expectedGeneration); + await build.pullRequest?.$fetchGraph('repository'); - await this.markConfigurationsAsBuilt(build, githubRepositoryId, options.sourceBranch); - await this.updateStatusAndComment(build, BuildStatus.BUILDING, runUUID, true, true); - await build?.pullRequest?.$fetchGraph('repository'); + try { + const dependencyGraph = await generateGraph(build, 'TB'); + let graphPatch = this.db.models.Build.query().patch({ dependencyGraph }).where({ id: build.id, runUUID }); + if (expectedGeneration != null) graphPatch = graphPatch.where('desiredGeneration', expectedGeneration); + await graphPatch; + } catch (error) { + getLogger().warn({ error }, 'Graph: generation failed'); + } - try { - const dependencyGraph = await generateGraph(build, 'TB'); - await build.$query().patch({ - dependencyGraph, - }); - } catch (error) { - getLogger().warn({ error }, 'Graph: generation failed'); - } + return { + build, + runUUID, + githubRepositoryId, + sourceGithubRepositoryId: options.sourceGithubRepositoryId ?? githubRepositoryId, + sourceRef, + sourceBranch: options.sourceBranch, + }; + } + + private async executeDeploymentScope( + preparation: DeploymentScopePreparation, + expectedGeneration?: number + ): Promise { + const { build, runUUID, githubRepositoryId, sourceGithubRepositoryId, sourceRef, sourceBranch } = preparation; + + try { + if (!(await this.isDeploymentRunCurrent(build.id, runUUID, expectedGeneration))) return null; - // Build Docker Images & Deploy CLI Based Infra At the Same Time const results = await Promise.all([ - this.buildImages(build, githubRepositoryId, sourceRef, options.sourceBranch), - this.deployCLIServices(build, githubRepositoryId, sourceRef, options.sourceBranch), + this.buildImages( + build, + runUUID, + githubRepositoryId, + sourceRef, + sourceBranch, + expectedGeneration, + sourceGithubRepositoryId + ), + this.deployCLIServices(build, runUUID, githubRepositoryId, sourceRef, sourceBranch, sourceGithubRepositoryId), ]); getLogger().debug(`Build results: buildImages=${results[0]} deployCLIServices=${results[1]}`); - // Label/close/pause can change while external builders are running. Cleanup - // owns the next mutation once that happens, even though it is waiting on our lock. - if (!(await this.isDeploymentRunCurrent(build.id, runUUID))) { - getLogger().info('Deploy: aborting rollout reason=authority_changed'); - return build; + // A/B may finish the expensive operation they already entered, but may not + // enter native promotion after C has become authoritative. + if (!(await this.isDeploymentRunCurrent(build.id, runUUID, expectedGeneration))) { + getLogger().info('Deploy: rollout skipped reason=ownership_lost'); + return null; } - const success = _.every(results); - /* Verify that all deploys are successfully built that are active */ - if (success) { - await this.db.services.BuildService.updateStatusAndComment(build, BuildStatus.BUILT, runUUID, true, true); + if (!_.every(results)) { + getLogger({ buildId: build.id, githubRepositoryId, sourceBranch }).warn('Build: errored skipping=rollout'); + return { status: BuildStatus.ERROR }; + } - if (isDeploy) { - await this.updateStatusAndComment(build, BuildStatus.DEPLOYING, runUUID, true, true); + await this.updateStatusAndComment(build, BuildStatus.DEPLOYING, runUUID, true, true, null, expectedGeneration); + if (!(await this.isDeploymentRunCurrent(build.id, runUUID, expectedGeneration))) return null; - // A teardown, pause, PR close/label removal, or newer run must win: - // never recreate the namespace after live authority changed. - if (!(await this.isDeploymentRunCurrent(build.id, runUUID))) { - getLogger().info('Deploy: aborting manifest apply reason=ownership_lost'); - return build; - } + const applySuccess = await this.generateAndApplyManifests({ + build, + runUUID, + expectedGeneration, + githubRepositoryId, + sourceBranch, + namespace: build.namespace, + enqueueIngress: false, + }); + if (!(await this.isDeploymentRunCurrent(build.id, runUUID, expectedGeneration))) return null; - const applyManifests = () => - this.generateAndApplyManifests({ - build, - githubRepositoryId, - sourceBranch: options.sourceBranch, - namespace: build.namespace, - }); - const applySuccess = await applyManifests(); - if (!(await this.isDeploymentRunCurrent(build.id, runUUID))) { - getLogger().info('Deploy: manifest apply finished after ownership loss'); - return build; - } - if (applySuccess) { - await this.updateStatusAndComment(build, BuildStatus.DEPLOYED, runUUID, true, true); - } else { - await this.updateStatusAndComment(build, BuildStatus.ERROR, runUUID, true, true); - } - } - } else { - getLogger().warn( - `Build: errored skipping=rollout fullName=${fullName} branchName=${branchName} latestCommit=${latestCommit}` - ); - await this.updateStatusAndComment(build, BuildStatus.ERROR, runUUID, true, true); - } + return { status: applySuccess ? BuildStatus.DEPLOYED : BuildStatus.ERROR }; } catch (error) { + if (error instanceof DeploymentSupersededError) { + getLogger().info('Build: deployment phase stopped reason=superseded'); + return null; + } + if (error instanceof AuthorityLockLostError) throw error; getLogger().error({ error }, 'Build: deploy failed'); - if (await this.isDeploymentRunCurrent(build.id, runUUID).catch(() => false)) { - await this.recordBuildFailure(build, BuildStatus.ERROR, runUUID, error, 'Build failed unexpectedly.'); - } else { + if (!(await this.isDeploymentRunCurrent(build.id, runUUID, expectedGeneration).catch(() => false))) { getLogger().info('Build: failure ignored reason=ownership_lost'); + return null; } - } - return build; + return { status: BuildStatus.ERROR, error }; + } } - private async isDeploymentRunCurrent(buildId: number, runUUID: string): Promise { + private async isDeploymentRunCurrent( + buildId: number, + runUUID: string, + expectedGeneration?: number + ): Promise { const current = await this.db.models.Build.query() .findById(buildId) - .select('id', 'runUUID', 'status', 'deployEnabled', 'deletedAt', 'pullRequestId'); + .select('id', 'runUUID', 'status', 'deployEnabled', 'deletedAt', 'pullRequestId', 'desiredGeneration'); if (current?.pullRequestId != null) { await current.$fetchGraph('pullRequest'); } - return Boolean(current && current.runUUID === runUUID && this.deploymentBlockReason(current) == null); + return Boolean( + current && + current.runUUID === runUUID && + (expectedGeneration == null || Number(current.desiredGeneration) === expectedGeneration) && + this.deploymentBlockReason(current) == null + ); } async withBuildDeploymentLock(buildId: number, action: () => Promise): Promise { if (!this.redlock?.lock) return action(); const resource = `build-deployment.${buildId}`; - let lock = await this.redlock.lock(resource, BUILD_DEPLOYMENT_LOCK_TTL_MS); + const lock = await this.redlock.lock(resource, BUILD_DEPLOYMENT_LOCK_TTL_MS); + return this.runWithRenewableLock(resource, lock, action); + } + + /** + * Waits only at the irreducible native-mutation boundary. Unlike the global + * Redlock retry budget, this wait ends when the caller loses generation + * authority and therefore can never consume the latest generation as a + * terminal lock-acquisition failure. + */ + async withCurrentBuildPromotionLock( + buildId: number, + isCurrent: () => Promise, + action: () => Promise + ): Promise> { + return withAuthorityLock({ + redlock: this.redlock, + resource: `build-promotion.${buildId}`, + ttlMs: BUILD_DEPLOYMENT_LOCK_TTL_MS, + isCurrent, + action, + onWait: (error) => getLogger({ error, buildId }).warn('Build promotion: waiting for admitted native mutation'), + }); + } + + async withCurrentDeploySecretMutationLock( + deployId: number, + isCurrent: () => Promise, + action: () => Promise + ): Promise> { + return withAuthorityLock({ + redlock: this.redlock, + resource: `deploy-external-secrets.${deployId}`, + ttlMs: DEPLOY_SECRET_MUTATION_LOCK_TTL_MS, + isCurrent, + action, + onWait: (error) => getLogger({ error, deployId }).warn('Deploy secrets: waiting for current resource writer'), + }); + } + + private async withCurrentBuildDeploymentLock( + buildId: number, + isCurrent: () => Promise, + action: () => Promise + ): Promise> { + return withAuthorityLock({ + redlock: this.redlock, + resource: `build-deployment.${buildId}`, + ttlMs: BUILD_DEPLOYMENT_LOCK_TTL_MS, + isCurrent, + action, + onWait: (error) => getLogger({ error, buildId }).warn('Build reconciliation: waiting for configuration mutation'), + }); + } + + private async withCurrentBuildDeletionLock( + buildId: number, + isCurrent: () => Promise, + action: () => Promise + ): Promise> { + return withAuthorityLock({ + redlock: this.redlock, + resource: `build-deployment.${buildId}`, + ttlMs: BUILD_DEPLOYMENT_LOCK_TTL_MS, + isCurrent, + action, + onWait: (error) => getLogger({ error, buildId }).warn('Build deletion: waiting for deployment mutation'), + }); + } + + /** A duplicate delivery of one generation coalesces; newer generations use a different resource. */ + private async tryWithDeploymentGenerationLock( + buildId: number, + generation: number, + action: () => Promise + ): Promise { + if (!this.redlock?.lock) { + await action(); + return; + } + + const resource = `build-reconcile.${buildId}.${generation}`; + const lockWithOptions = (this.redlock as any).lockWithOptions; + let lock: any; + try { + lock = + typeof lockWithOptions === 'function' + ? await lockWithOptions.call(this.redlock, resource, BUILD_DEPLOYMENT_LOCK_TTL_MS, { + retryCount: 1, + retryDelay: 1, + }) + : await this.redlock.lock(resource, BUILD_DEPLOYMENT_LOCK_TTL_MS); + } catch { + getLogger({ buildId, generation }).info('Build reconciliation: duplicate generation signal coalesced'); + return; + } + + await this.runWithRenewableLock(resource, lock, action); + } + + private async runWithRenewableLock(resource: string, acquiredLock: any, action: () => Promise): Promise { + let lock = acquiredLock; if (!lock?.unlock) return action(); let renewalError: unknown; let renewal = Promise.resolve(); @@ -2943,13 +2979,15 @@ export default class BuildService extends BaseService { try { const result = await action(); + clearInterval(renewalTimer); await renewal; if (renewalError) throw renewalError; return result; } finally { clearInterval(renewalTimer); + await renewal; await lock.unlock().catch((error) => { - getLogger().warn({ error, buildId }, 'Build: deployment lock release failed'); + getLogger().warn({ error, resource }, 'Build: lock release failed'); }); } } @@ -3022,18 +3060,6 @@ export default class BuildService extends BaseService { updateLogContext({ buildUuid: build.uuid }); } - if (PR_AUTHORITY_REVALIDATED_DELETE_REASONS.has(options.reason ?? '') && build.pullRequestId != null) { - await build.$fetchGraph('pullRequest'); - if ( - build.pullRequest != null && - build.pullRequest.status === PullRequestStatus.OPEN && - isDeployEnabled(build) - ) { - getLogger().info('Build: delete skipped reason=pr_authority_restored'); - return; - } - } - // PR-less teardown closes the deploy gate and takes runUUID ownership so queued or in-flight // deploy runs abort instead of resurrecting the environment (PR builds get this from the label kill-switch). let teardownRunUUID = options.runUUID ?? build.runUUID; @@ -3059,9 +3085,31 @@ export default class BuildService extends BaseService { return; } + if ( + build.status !== BuildStatus.TEARING_DOWN && + requiresPrDeletionAuthorityRevalidation(options.reason) && + build.pullRequestId != null + ) { + await build.$fetchGraph('pullRequest'); + if ( + build.pullRequest != null && + build.pullRequest.status === PullRequestStatus.OPEN && + isDeployEnabled(build) + ) { + getLogger().info('Build: delete skipped reason=pr_authority_restored'); + return; + } + } + await build.$fetchGraph('[services, deploys.[build]]'); getLogger().debug('Build: triggering cleanup'); await this.updateStatusAndComment(build, BuildStatus.TEARING_DOWN, teardownRunUUID, true, true); + if (teardownRunUUID) { + // Revoke every in-flight Deploy writer before cleanup starts. This one + // bulk fence prevents detached readiness/provider waits from restoring + // READY or failure state after teardown records TORN_DOWN. + await this.db.models.Deploy.query().where({ buildId: build.id }).patch({ runUUID: teardownRunUUID }); + } // A failing cleanup step must never block namespace deletion below; retries pick up the rest. const cleanupResults = await Promise.allSettled([ k8s.deleteBuild(build), @@ -3134,9 +3182,11 @@ export default class BuildService extends BaseService { runUUID: string, updateMissionControl: boolean, updateStatus: boolean, - error: Error | null = null + error: Error | null = null, + expectedGeneration?: number ) { return withLogContext({ buildUuid: build.uuid }, async () => { + let published = false; try { await build.reload(); await build?.$fetchGraph('[deploys.[deployable], pullRequest.[repository]]'); @@ -3146,33 +3196,34 @@ export default class BuildService extends BaseService { const isSandboxBuild = build.kind === BuildKind.SANDBOX; const repository = pullRequest?.repository; - if (build.runUUID !== runUUID) { - return; - } else { - const statusMessage = this.resolveBuildStatusMessage(status, deploys || [], error); - await build.$query().patch({ - status, - statusMessage, + const statusMessage = this.resolveBuildStatusMessage(status, deploys || [], error); + let patch = this.db.models.Build.query() + .patch({ status, statusMessage }) + .where({ id: build.id, runUUID }) + .whereNull('deletedAt'); + if (expectedGeneration != null) patch = patch.where('desiredGeneration', expectedGeneration); + if ((await patch) !== 1) return; + + published = true; + build.status = status; + build.statusMessage = statusMessage; + if (!isSandboxBuild && pullRequest && repository) { + await this.db.services.ActivityStream.updatePullRequestActivityStream( + build, + deploys, + pullRequest, + repository, + updateMissionControl, + updateStatus, + error + ).catch((e) => { + getLogger().error({ error: e }, 'ActivityStream: update failed'); }); - - if (!isSandboxBuild && pullRequest && repository) { - await this.db.services.ActivityStream.updatePullRequestActivityStream( - build, - deploys, - pullRequest, - repository, - updateMissionControl, - updateStatus, - error - ).catch((e) => { - getLogger().error({ error: e }, 'ActivityStream: update failed'); - }); - } } } finally { getLogger().debug(`Build status changed: status=${build.status}`); - if (build.kind !== BuildKind.SANDBOX) { + if (published && build.kind !== BuildKind.SANDBOX) { await this.db.services.Webhook.webhookQueue .add('webhook', { buildId: build.id, @@ -3189,10 +3240,13 @@ export default class BuildService extends BaseService { status: BuildStatus, runUUID: string | null | undefined, error: unknown, - fallbackMessage: string + fallbackMessage: string, + expectedGeneration?: number ): Promise { const activeRunUUID = runUUID || build.runUUID || nanoid(); - if (build.runUUID !== activeRunUUID) { + if (expectedGeneration != null) { + if (!(await this.isDeploymentRunCurrent(build.id, activeRunUUID, expectedGeneration))) return; + } else if (build.runUUID !== activeRunUUID) { if (build.pullRequestId != null) { await build.$query().patch({ runUUID: activeRunUUID }); } else { @@ -3211,7 +3265,7 @@ export default class BuildService extends BaseService { } const statusError = error instanceof Error ? error : new Error(statusMessageFromError(error, fallbackMessage)); - await this.updateStatusAndComment(build, status, activeRunUUID, true, true, statusError); + await this.updateStatusAndComment(build, status, activeRunUUID, true, true, statusError, expectedGeneration); } private resolveBuildStatusMessage(status: BuildStatus, deploys: Deploy[], error: Error | null): string { @@ -3247,7 +3301,12 @@ export default class BuildService extends BaseService { return compactStatusMessage(`Build failed because ${failedDeployMessages.slice(0, 3).join('; ')}`); } - async markConfigurationsAsBuilt(build: Build, githubRepositoryId?: number | null, sourceBranch?: string | null) { + async markConfigurationsAsBuilt( + build: Build, + runUUID: string, + githubRepositoryId?: number | null, + sourceBranch?: string | null + ) { try { await build?.$fetchGraph({ deploys: { @@ -3267,7 +3326,7 @@ export default class BuildService extends BaseService { return; } for (const deploy of configDeploys) { - await deploy.$query().patch({ status: DeployStatus.BUILT }); + await this.db.models.Deploy.query().patch({ status: DeployStatus.BUILT }).where({ id: deploy.id, runUUID }); } const configUUIDs = configDeploys.map((deploy) => deploy?.uuid).join(','); getLogger().info(`Build: config deploys marked built uuids=${configUUIDs}`); @@ -3278,9 +3337,11 @@ export default class BuildService extends BaseService { async deployCLIServices( build: Build, + runUUID: string, githubRepositoryId: number | null = null, sourceRef?: string | null, - sourceBranch?: string | null + sourceBranch?: string | null, + sourceGithubRepositoryId: number | null | undefined = githubRepositoryId ): Promise { await build?.$fetchGraph({ deploys: { @@ -3294,11 +3355,11 @@ export default class BuildService extends BaseService { const deploys = await Deploy.query() .where({ buildId, + runUUID, ...(githubRepositoryId ? { githubRepositoryId } : {}), ...(githubRepositoryId && sourceBranch ? { branchName: sourceBranch } : {}), }) .withGraphFetched({ deployable: true }); - if (!deploys || deploys.length === 0) return false; try { return _.every( await Promise.all( @@ -3314,14 +3375,15 @@ export default class BuildService extends BaseService { try { const result = await this.db.services.Deploy.deployCLI( deploy, + runUUID, sourceRef, - githubRepositoryId, + sourceGithubRepositoryId, sourceBranch ); return result; } catch (err) { getLogger().error({ error: err }, `CLI: deploy failed uuid=${deploy?.uuid}`); - return this.db.services.Deploy.recordDeployFailure(deploy, deploy.runUUID || build.runUUID, { + return this.db.services.Deploy.recordDeployFailure(deploy, runUUID, { status: DeployStatus.ERROR, error: err, fallbackMessage: 'CLI deploy failed.', @@ -3343,9 +3405,12 @@ export default class BuildService extends BaseService { */ async buildImages( build: Build, + runUUID: string, githubRepositoryId: number | null = null, sourceRef?: string | null, - sourceBranch?: string | null + sourceBranch?: string | null, + expectedGeneration?: number, + sourceGithubRepositoryId: number | null | undefined = githubRepositoryId ): Promise { const buildId = build?.id; if (!buildId) { @@ -3355,6 +3420,7 @@ export default class BuildService extends BaseService { const deploys = await Deploy.query() .where({ buildId, + runUUID, ...(githubRepositoryId ? { githubRepositoryId } : {}), ...(githubRepositoryId && sourceBranch ? { branchName: sourceBranch } : {}), }) @@ -3380,18 +3446,41 @@ export default class BuildService extends BaseService { .join(',')}` ); + let nativeServiceAccount: string | undefined; + if (deploysToBuild.some((deploy) => isNativeBuilderEngine(deploy.deployable.builder?.engine))) { + await build.$fetchGraph('pullRequest'); + const setup = await this.withCurrentBuildDeploymentLock( + build.id, + () => this.isDeploymentRunCurrent(build.id, runUUID, expectedGeneration), + async () => { + await k8s.createOrUpdateNamespace({ + name: build.namespace, + buildUUID: build.uuid, + staticEnv: build.isStatic, + pullRequest: build.pullRequest, + waitForReady: true, + }); + return ensureServiceAccountForJob(build.namespace, 'build'); + } + ); + if (!setup.admitted) throw new DeploymentSupersededError(); + nativeServiceAccount = setup.value; + } + const results = await Promise.all( deploysToBuild.map(async (deploy, index) => { - await deploy.$query().patchAndFetch({ - deployPipelineId: null, - deployOutput: null, - } as unknown as Partial); + await this.db.models.Deploy.query() + .patch({ deployPipelineId: null, deployOutput: null } as unknown as Partial) + .where({ id: deploy.id, runUUID }); const result = await this.db.services.Deploy.buildImage( deploy, index, + runUUID, sourceRef, - githubRepositoryId, - sourceBranch + sourceGithubRepositoryId, + sourceBranch, + expectedGeneration, + nativeServiceAccount ); getLogger().debug(`buildImage completed: deployUuid=${deploy.uuid} result=${result}`); return result; @@ -3401,6 +3490,7 @@ export default class BuildService extends BaseService { getLogger().debug(`Build results: results=${results.join(',')} final=${finalResult}`); return finalResult; } catch (error) { + if (error instanceof AuthorityLockLostError || error instanceof DeploymentSupersededError) throw error; getLogger().error({ error }, 'Docker: build error'); return false; } @@ -3412,30 +3502,44 @@ export default class BuildService extends BaseService { */ async generateAndApplyManifests({ build, + runUUID, + expectedGeneration, githubRepositoryId = null, sourceBranch, namespace, + enqueueIngress = true, }: { build: Build; + runUUID?: string; + expectedGeneration?: number; githubRepositoryId: number | null; sourceBranch?: string | null; namespace: string; + enqueueIngress?: boolean; }): Promise { try { const buildId = build?.id; const { serviceAccount } = await GlobalConfigService.getInstance().getAllConfigs(); const serviceAccountName = serviceAccount?.name || 'default'; - // create namespace and annotate the service account - await k8s.createOrUpdateNamespace({ - name: build.namespace, - buildUUID: build.uuid, - staticEnv: build.isStatic, - pullRequest: build.pullRequest, - // API-created environments are reaped by the expiresAt sweep, never by namespace-label TTL. - ...(build.triggerType === 'api' ? { ttl: false } : {}), - }); - await ensureServiceAccountForJob(build.namespace, 'deploy'); + const prepareNativeDeployment = async () => { + await k8s.createOrUpdateNamespace({ + name: build.namespace, + buildUUID: build.uuid, + staticEnv: build.isStatic, + pullRequest: build.pullRequest, + // API-created environments are reaped by the expiresAt sweep, never by namespace-label TTL. + ...(build.triggerType === 'api' ? { ttl: false } : {}), + }); + await ensureServiceAccountForJob(build.namespace, 'deploy'); + }; + if (runUUID) { + const isCurrent = () => this.isDeploymentRunCurrent(build.id, runUUID, expectedGeneration); + const setup = await this.withCurrentBuildDeploymentLock(build.id, isCurrent, prepareNativeDeployment); + if (!setup.admitted) throw new DeploymentSupersededError(); + } else { + await prepareNativeDeployment(); + } if (build.kind === BuildKind.ENVIRONMENT && build.uuid) { await new AgentPrewarmService(this.db, this.redis, this.redlock, this.queueManager) .queueBuildPrewarm(build.uuid) @@ -3450,6 +3554,7 @@ export default class BuildService extends BaseService { const allDeploys = await Deploy.query() .where({ buildId, + ...(runUUID ? { runUUID } : {}), ...(githubRepositoryId ? { githubRepositoryId } : {}), ...(githubRepositoryId && sourceBranch ? { branchName: sourceBranch } : {}), }) @@ -3479,7 +3584,10 @@ export default class BuildService extends BaseService { // Store manifest in deploy record if (manifest && manifest.trim().length > 0) { - await deploy.$query().patch({ manifest }); + let manifestPatch = this.db.models.Deploy.query().patch({ manifest }).where({ id: deploy.id }); + if (runUUID) manifestPatch = manifestPatch.where('runUUID', runUUID); + await manifestPatch; + deploy.manifest = manifest; } } } @@ -3490,15 +3598,28 @@ export default class BuildService extends BaseService { const managedDeploys = activeDeploys.filter( (d) => d.deployable.type !== DeployTypes.CODEFRESH && d.deployable.type !== DeployTypes.CONFIGURATION ); - const deploymentManager = new DeploymentManager(managedDeploys); + const isCurrent = runUUID + ? () => this.isDeploymentRunCurrent(build.id, runUUID, expectedGeneration) + : undefined; + const deploymentManager = new DeploymentManager(managedDeploys, { + isCurrent, + nativeMutationGate: runUUID + ? (action) => this.withCurrentBuildPromotionLock(build.id, isCurrent!, action) + : undefined, + nativeSecretMutationGate: runUUID + ? (deploy, action) => this.withCurrentDeploySecretMutationLock(deploy.id, isCurrent!, action) + : undefined, + }); await deploymentManager.deploy(); } - // Queue ingress creation after all deployments - await this.ingressService.ingressManifestQueue.add('manifest', { - buildId, - ...extractContextForQueue(), - }); + if (runUUID && !(await this.isDeploymentRunCurrent(build.id, runUUID, expectedGeneration))) { + throw new DeploymentSupersededError(); + } + + if (enqueueIngress) { + await this.enqueueIngressManifest(buildId, runUUID, expectedGeneration); + } // Legacy manifest generation for backwards compatibility const githubTypeDeploys = activeDeploys.filter( @@ -3517,17 +3638,33 @@ export default class BuildService extends BaseService { serviceAccountName, }); if (legacyManifest && legacyManifest.replace(/---/g, '').trim().length > 0) { - await build.$query().patch({ manifest: legacyManifest }); + let manifestPatch = this.db.models.Build.query().patch({ manifest: legacyManifest }).where({ id: build.id }); + if (runUUID) manifestPatch = manifestPatch.where('runUUID', runUUID); + if (expectedGeneration != null) manifestPatch = manifestPatch.where('desiredGeneration', expectedGeneration); + await manifestPatch; } } - await this.updateDeploysImageDetails(build, githubRepositoryId, sourceBranch); + await this.updateDeploysImageDetails(build, runUUID, githubRepositoryId, sourceBranch); return true; } catch (e) { + if (e instanceof DeploymentSupersededError) { + getLogger().info('K8s: deployment stopped reason=superseded'); + throw e; + } getLogger().warn({ error: e }, 'K8s: deploy failed'); throw e; } } + private async enqueueIngressManifest(buildId: number, runUUID?: string, expectedGeneration?: number) { + await this.ingressService.ingressManifestQueue.add('manifest', { + buildId, + ...(runUUID ? { runUUID } : {}), + ...(expectedGeneration != null ? { expectedGeneration } : {}), + ...extractContextForQueue(), + }); + } + /** * Returns an array of environments to build. * @param environmentId the default environmentId (if one exists) @@ -3548,6 +3685,7 @@ export default class BuildService extends BaseService { private async updateDeploysImageDetails( build: Build, + runUUID?: string, githubRepositoryId?: number | null, sourceBranch?: string | null ) { @@ -3558,7 +3696,13 @@ export default class BuildService extends BaseService { (!githubRepositoryId || !sourceBranch || deploy.branchName === sourceBranch) ); await Promise.all( - deploys.map((deploy) => deploy.$query().patch({ isRunningLatest: true, runningImage: deploy?.dockerImage })) + deploys.map((deploy) => { + let update = this.db.models.Deploy.query() + .patch({ isRunningLatest: true, runningImage: deploy?.dockerImage }) + .where({ id: deploy.id }); + if (runUUID) update = update.where('runUUID', runUUID); + return update; + }) ); getLogger().debug('Deploy: updated running image and status'); } @@ -3575,29 +3719,10 @@ export default class BuildService extends BaseService { }, }); - /** - * A queue entrypoint for the purpose of deleting builds - */ - buildQueue = this.queueManager.registerQueue(QUEUE_NAMES.BUILD_QUEUE, { + deploymentReconciliationQueue = this.queueManager.registerQueue(QUEUE_NAMES.DEPLOYMENT_RECONCILIATION, { connection: redisClient.getConnection(), defaultJobOptions: { - // A different run can legitimately hold the per-build lock longer than - // Redlock's acquisition retry window. Retry instead of dropping this - // distinct trigger when lock contention times out. attempts: 10, - backoff: { type: 'fixed', delay: 15000 }, - removeOnComplete: true, - removeOnFail: true, - }, - }); - - /** - * A queue specifically for the purpose of performing builds and deploying to K8 - */ - resolveAndDeployBuildQueue = this.queueManager.registerQueue(QUEUE_NAMES.RESOLVE_AND_DEPLOY, { - connection: redisClient.getConnection(), - defaultJobOptions: { - attempts: 3, backoff: { type: 'exponential', delay: 5000 }, removeOnComplete: true, removeOnFail: true, @@ -3623,20 +3748,87 @@ export default class BuildService extends BaseService { }, }); + private async loadBuildDeletionAuthority(buildId: number): Promise { + const build = await this.db.models.Build.query().findOne({ id: buildId }); + if (build?.pullRequestId != null) { + await build.$fetchGraph('pullRequest'); + } + return build ?? null; + } + + private async isBuildDeletionRequestCurrent( + buildId: number, + reason: string | undefined, + teardownRunUUID: string + ): Promise { + const build = await this.loadBuildDeletionAuthority(buildId); + if (!build || build.deletedAt != null) return false; + + if (build.status === BuildStatus.TEARING_DOWN || build.status === BuildStatus.TORN_DOWN) { + return build.runUUID === teardownRunUUID; + } + if (reason === 'teardown_stuck' && (build.pullRequestId == null || build.runUUID !== teardownRunUUID)) return false; + if (reason === 'lease_expired') { + return ( + build.kind === BuildKind.ENVIRONMENT && build.triggerType === 'api' && isExpired(new Date(), build.expiresAt) + ); + } + if ( + requiresPrDeletionAuthorityRevalidation(reason) && + build.pullRequestId != null && + build.pullRequest?.status === PullRequestStatus.OPEN && + isDeployEnabled(build) + ) { + return false; + } + return true; + } + + private async isClaimedBuildDeletionCurrent( + buildId: number, + reason: string | undefined, + teardownRunUUID: string + ): Promise { + const build = await this.loadBuildDeletionAuthority(buildId); + if (!build || build.deletedAt != null || build.runUUID !== teardownRunUUID) return false; + if (build.status === BuildStatus.TEARING_DOWN || build.status === BuildStatus.TORN_DOWN) return true; + if (reason === 'teardown_stuck' && build.pullRequestId == null) return false; + return !( + requiresPrDeletionAuthorityRevalidation(reason) && + build.pullRequestId != null && + build.pullRequest?.status === PullRequestStatus.OPEN && + isDeployEnabled(build) + ); + } + private async claimBuildForDeletion( buildId: number, reason: string | undefined, teardownRunUUID: string ): Promise<{ build: Build | null; - outcome: 'claimed' | 'already_claimed' | 'lease_extended' | 'authority_restored' | 'missing'; + outcome: 'claimed' | 'already_claimed' | 'lease_extended' | 'authority_restored' | 'stale' | 'missing'; }> { return this.db.models.Build.transact(async (trx) => { const build = await this.db.models.Build.query(trx).findById(buildId).whereNull('deletedAt').forUpdate(); if (!build) return { build: null, outcome: 'missing' }; - if (PR_AUTHORITY_REVALIDATED_DELETE_REASONS.has(reason ?? '') && build.pullRequestId != null) { + if (reason === 'lease_expired' && (build.kind !== BuildKind.ENVIRONMENT || build.triggerType !== 'api')) { + return { build: null, outcome: 'missing' }; + } + + // Once cleanup has published TEARING_DOWN, the same owner must finish even if PR authority returns. + if (build.status === BuildStatus.TEARING_DOWN || build.status === BuildStatus.TORN_DOWN) { + return build.runUUID === teardownRunUUID + ? { build, outcome: 'claimed' } + : { build: null, outcome: 'already_claimed' }; + } + if (reason === 'teardown_stuck' && (build.pullRequestId == null || build.runUUID !== teardownRunUUID)) { + return { build: null, outcome: 'stale' }; + } + + if (requiresPrDeletionAuthorityRevalidation(reason) && build.pullRequestId != null) { await build.$fetchGraph('pullRequest', { transaction: trx } as any); // Close/disable cleanup is asynchronous. A later reopen/re-enable event // is authoritative, even when BullMQ coalesced multiple reasons onto the @@ -3650,17 +3842,6 @@ export default class BuildService extends BaseService { } } - if (reason === 'lease_expired' && (build.kind !== BuildKind.ENVIRONMENT || build.triggerType !== 'api')) { - return { build: null, outcome: 'missing' }; - } - - // A retry must be able to finish cleanup (or identity release) once teardown owns the row. - if (build.status === BuildStatus.TEARING_DOWN || build.status === BuildStatus.TORN_DOWN) { - return build.runUUID === teardownRunUUID - ? { build, outcome: 'claimed' } - : { build: null, outcome: 'already_claimed' }; - } - // Extension takes the same row lock, so whichever transaction commits first wins deterministically. if (reason === 'lease_expired' && !isExpired(new Date(), build.expiresAt)) { return { build: null, outcome: 'lease_extended' }; @@ -3689,40 +3870,76 @@ export default class BuildService extends BaseService { return withLogContext({ correlationId, buildUuid, sender, _ddTraceContext }, async () => { try { - await this.withBuildDeploymentLock(buildId, async () => { - const claim = await this.claimBuildForDeletion(buildId, reason, activeTeardownRunUUID); - if (claim.outcome === 'lease_extended') { - getLogger().info('Build: delete skipped reason=lease_extended'); - return; - } - if (claim.outcome === 'already_claimed') { - getLogger().info('Build: delete skipped reason=teardown_owned'); - return; - } - if (claim.outcome === 'authority_restored') { - getLogger().info('Build: delete skipped reason=pr_authority_restored'); - return; - } - const build = claim.build; + let deploymentActionFinished = false; + const deployment = await this.withCurrentBuildDeletionLock( + buildId, + () => + deploymentActionFinished + ? Promise.resolve(true) + : this.isBuildDeletionRequestCurrent(buildId, reason, activeTeardownRunUUID), + async () => { + try { + const claim = await this.claimBuildForDeletion(buildId, reason, activeTeardownRunUUID); + if (claim.outcome === 'lease_extended') { + getLogger().info('Build: delete skipped reason=lease_extended'); + return; + } + if (claim.outcome === 'already_claimed') { + getLogger().info('Build: delete skipped reason=teardown_owned'); + return; + } + if (claim.outcome === 'authority_restored') { + getLogger().info('Build: delete skipped reason=pr_authority_restored'); + return; + } + if (claim.outcome === 'stale') { + getLogger().info('Build: delete skipped reason=teardown_no_longer_active'); + return; + } + const build = claim.build; - if (build?.uuid) { - updateLogContext({ buildUuid: build.uuid }); - } + if (build?.uuid) { + updateLogContext({ buildUuid: build.uuid }); + } - if (!build) { - getLogger({ stage: LogStage.CLEANUP_FAILED }).warn(`Build: not found for deletion buildId=${buildId}`); - return; - } + if (!build) { + getLogger({ stage: LogStage.CLEANUP_FAILED }).warn(`Build: not found for deletion buildId=${buildId}`); + return; + } - getLogger({ stage: LogStage.CLEANUP_STARTING }).info('Build: deleting'); - await this.db.services.BuildService.deleteBuild(build, { - rethrow: true, - runUUID: activeTeardownRunUUID, - reason, - deploymentLockAlreadyHeld: true, - }); - getLogger({ stage: LogStage.CLEANUP_COMPLETE }).info('Build: deleted'); - }); + let cleanupActionFinished = false; + const promotion = await this.withCurrentBuildPromotionLock( + buildId, + () => + cleanupActionFinished + ? Promise.resolve(true) + : this.isClaimedBuildDeletionCurrent(buildId, reason, activeTeardownRunUUID), + async () => { + try { + getLogger({ stage: LogStage.CLEANUP_STARTING }).info('Build: deleting'); + await this.db.services.BuildService.deleteBuild(build, { + rethrow: true, + runUUID: activeTeardownRunUUID, + reason, + deploymentLockAlreadyHeld: true, + }); + getLogger({ stage: LogStage.CLEANUP_COMPLETE }).info('Build: deleted'); + } finally { + cleanupActionFinished = true; + } + } + ); + if (!promotion.admitted && !cleanupActionFinished) { + getLogger().info('Build: delete skipped reason=teardown_authority_lost'); + } + } finally { + deploymentActionFinished = true; + } + } + ); + if (!deployment.admitted && !deploymentActionFinished) { + getLogger().info('Build: delete skipped reason=request_no_longer_current'); + } } catch (error) { getLogger({ stage: LogStage.CLEANUP_FAILED }).error( { error }, @@ -3734,191 +3951,425 @@ export default class BuildService extends BaseService { }); }; + private async claimDeploymentReconciliation( + buildId: number, + generation: number + ): Promise { + const build = await this.db.models.Build.query().findById(buildId).whereNull('deletedAt'); + if (!build) return null; + + const desiredGeneration = Number(build.desiredGeneration); + const observedGeneration = Number(build.observedGeneration); + if (!Number.isSafeInteger(desiredGeneration) || !Number.isSafeInteger(observedGeneration)) { + throw new Error(`Build ${buildId} has invalid deployment generations`); + } + // One signal owns exactly one generation. B must never wake up later and + // read/execute C's mailbox contents. + if (desiredGeneration !== generation || generation <= observedGeneration) return null; + + const dirty = dirtyDeploymentIntents(build.acceptedRefs, observedGeneration).filter( + ({ intent }) => intent.gen <= generation + ); + const latest = dirty.find(({ intent }) => intent.gen === generation)?.intent; + if (!latest?.requestId) throw new Error(`Build ${buildId} generation ${generation} has no request token`); + + return { generation, token: latest.requestId, dirty }; + } + + private deploymentReconciliationScopes(dirty: DirtyDeploymentIntent[]): DeploymentReconciliationScope[] { + const newestBroadIndex = _.findLastIndex( + dirty, + ({ intent }) => intent.type === 'all' || (intent.type === 'source' && intent.target === 'all') + ); + + // A full pass subsumes older unpinned requests, but it must not erase a + // delivered source SHA. Run unpinned/live-head work first and immutable + // source floors last so a temporarily stale provider read cannot roll C + // back to B. A source-targeted full pass still precedes selective pins. + const retained = dirty.filter( + ({ intent }, index) => newestBroadIndex < 0 || index >= newestBroadIndex || intent.type === 'source' + ); + const ordered = _.orderBy( + retained, + [ + ({ intent }) => { + if (intent.type === 'all') return 0; + if (intent.type === 'repository') return 1; + return intent.target === 'all' ? 2 : 3; + }, + ({ intent }) => intent.gen, + ], + ['asc', 'desc'] + ); + + return ordered.map(({ intent }) => { + if (intent.type === 'all') return { githubRepositoryId: null }; + if (intent.type === 'source') { + return { + githubRepositoryId: intent.target === 'all' ? null : intent.githubRepositoryId, + sourceGithubRepositoryId: intent.githubRepositoryId, + sourceRef: intent.sha, + sourceBeforeRef: intent.beforeSha, + sourceBranch: intent.branch, + }; + } + return { + githubRepositoryId: intent.githubRepositoryId, + skipDeletedServiceReconciliation: true, + }; + }); + } + /** - * Kicks off the process of actually deploying a build to the kubernetes cluster - * @param job the BullMQ job with the buildID + * The delivered SHA is the acceptance floor. A live head may advance it, but + * a lagging GitHub read must never move accepted C back to A/B. */ - processBuildQueue = async (job) => { - const { + private async resolveCurrentSourceRef(scope: DeploymentReconciliationScope): Promise { + if (!scope.sourceBranch || !scope.sourceRef || scope.sourceGithubRepositoryId == null) return scope.sourceRef; + + const repository: Repository | undefined = await this.db.models.Repository.query() + .findOne({ githubRepositoryId: scope.sourceGithubRepositoryId }) + .whereNull('deletedAt'); + const repositoryFullName = repository?.fullName; + const [owner, name] = repositoryFullName?.split('/') ?? []; + if (!repositoryFullName || !owner || !name) return scope.sourceRef; + + let currentRef: string; + try { + currentRef = await github.getSHAForBranch(scope.sourceBranch, owner, name); + } catch (error) { + getLogger({ error, repositoryFullName, branch: scope.sourceBranch }).warn( + 'Build reconciliation: branch head lookup failed; using accepted source' + ); + return scope.sourceRef; + } + if (!currentRef || currentRef === scope.sourceRef || currentRef === scope.sourceBeforeRef) return scope.sourceRef; + + try { + const comparison = await github.compareCommits({ + fullName: repositoryFullName, + base: scope.sourceRef, + head: currentRef, + }); + if (comparison === 'ahead') return currentRef; + } catch (error) { + getLogger({ error, repositoryFullName, branch: scope.sourceBranch }).warn( + 'Build reconciliation: commit comparison failed; using accepted source' + ); + } + return scope.sourceRef; + } + + private async markDeploymentReconciliationObserved( + buildId: number, + generation: number, + token?: string + ): Promise { + let update = this.db.models.Build.query() + .patch({ observedGeneration: generation }) + .where({ id: buildId, desiredGeneration: generation }) + .whereNull('deletedAt'); + if (token) update = update.where('runUUID', token); + return (await update) === 1; + } + + private async signalPendingDeploymentReconciliation(buildId: number, generation: number): Promise { + const data: DeploymentReconciliationJobData = { + ...extractContextForQueue(), buildId, - githubRepositoryId, - sourceGithubRepositoryId, - sender, - correlationId, - skipDeletedServiceReconciliation, - sourceRef, - sourceBranch, - runUUID: requestedRunUUID, - triggerSequence, - _ddTraceContext, - } = job.data; + generation, + }; + await this.deploymentReconciliationQueue.add('reconcile', data, { + jobId: `reconcile-${buildId}-${generation}`, + }); + } + + private async adoptLegacyDeploymentJob(request: DeploymentReconciliationRequest): Promise { + const requestId = request.runUUID || nanoid(); + const accepted = await acceptDeploymentIntent(request.buildId, this.deploymentIntentForRequest(request, requestId)); + if (!accepted) return; + await this.signalPendingDeploymentReconciliation(request.buildId, accepted.generation); + } + + async enqueuePendingDeploymentReconciliations(): Promise { + const pendingAfter = (id: number) => + this.db.models.Build.query() + .select('id', 'desiredGeneration') + .whereRaw('?? > ??', ['desiredGeneration', 'observedGeneration']) + .whereNull('deletedAt') + .where('id', '>', id) + .orderBy('id') + .limit(500); + + let builds = await pendingAfter(this.deploymentReconciliationSweepCursor); + if (builds.length === 0 && this.deploymentReconciliationSweepCursor !== 0) { + this.deploymentReconciliationSweepCursor = 0; + builds = await pendingAfter(0); + } + if (builds.length > 0) this.deploymentReconciliationSweepCursor = Number(builds[builds.length - 1].id); + + await Promise.all( + builds.map((build) => + this.signalPendingDeploymentReconciliation(build.id, Number(build.desiredGeneration)).catch((error) => { + getLogger({ error, buildId: build.id }).warn('Build reconciliation: sweep signal failed'); + }) + ) + ); + } + + setupDeploymentReconciliationSweep(intervalMs = DEPLOYMENT_RECONCILIATION_SWEEP_MS): NodeJS.Timeout { + void this.enqueuePendingDeploymentReconciliations().catch((error) => + getLogger({ error }).warn('Build reconciliation: initial sweep failed') + ); + const timer = setInterval(() => { + void this.enqueuePendingDeploymentReconciliations().catch((error) => + getLogger({ error }).warn('Build reconciliation: sweep failed') + ); + }, intervalMs); + timer.unref?.(); + return timer; + } + + private isFinalDeploymentReconciliationAttempt(job: DeploymentReconciliationQueueJob): boolean { + const attempts = Math.max(1, Number(job.opts?.attempts ?? 1)); + return Number(job.attemptsMade ?? 0) + 1 >= attempts; + } + + /** + * A failure can happen before the normal run claim finishes. On the final + * BullMQ attempt, reacquire the short mutation lock and establish the same + * generation token before publishing one terminal failure. + */ + private async recordFinalDeploymentReconciliationFailure( + buildId: number, + claim: DeploymentReconciliationClaim, + build: Build | null, + runUUID: string | null, + error: unknown + ): Promise { + let failureBuild = build; + let failureRunUUID = runUUID; + if (!failureBuild || !failureRunUUID) { + const terminalClaim = await this.withCurrentBuildDeploymentLock( + buildId, + async () => Boolean(await this.claimDeploymentReconciliation(buildId, claim.generation)), + async () => { + const currentClaim = await this.claimDeploymentReconciliation(buildId, claim.generation); + if (!currentClaim || currentClaim.token !== claim.token) return null; + + const currentBuild = await this.loadBuildDeploymentAuthority(buildId); + if (!currentBuild) return null; + const claimedRunUUID = await this.claimDeploymentRun(currentBuild, claim.token, claim.generation); + return claimedRunUUID ? { build: currentBuild, runUUID: claimedRunUUID } : null; + } + ); + if (!terminalClaim.admitted || !terminalClaim.value) return false; + failureBuild = terminalClaim.value.build; + failureRunUUID = terminalClaim.value.runUUID; + } + + if (!(await this.isDeploymentRunCurrent(buildId, failureRunUUID, claim.generation))) return false; + await this.recordBuildFailure( + failureBuild, + BuildStatus.ERROR, + failureRunUUID, + error, + 'Build queue processing failed.', + claim.generation + ); + return this.markDeploymentReconciliationObserved(buildId, claim.generation, failureRunUUID); + } + + processDeploymentReconciliationQueue = async (job: DeploymentReconciliationQueueJob) => { + const { buildId, generation, sender, correlationId, _ddTraceContext } = job.data; return withLogContext({ correlationId, sender, _ddTraceContext }, async () => { - // Assigned in the lock closure; the wide initializer keeps CFA from narrowing reads in the catch to null. - let build: Build | null = null as Build | null; - let activeRunUUID: string | null = null; - try { - await this.withBuildDeploymentLock(buildId, async () => { - // Workers can wait on the lock for minutes. Always load authority after - // acquiring it; the enqueue-time build/label state is not authoritative. - build = await this.loadBuildDeploymentAuthority(buildId); - if (!build) { - getLogger().info('Build: skipping reason=build_missing'); - return; - } + if (!buildId || !Number.isSafeInteger(Number(generation))) { + throw new Error('buildId and generation are required'); + } - if (build.uuid) { - updateLogContext({ buildUuid: build.uuid }); - } + await this.tryWithDeploymentGenerationLock(buildId, Number(generation), async () => { + const claim = await this.claimDeploymentReconciliation(buildId, Number(generation)); + if (!claim) return; + + let build: Build | null = null; + let activeBuild: Build | null = null; + let activeRunUUID: string | null = null; + try { + const initialClaim = await this.withCurrentBuildDeploymentLock( + buildId, + async () => Boolean(await this.claimDeploymentReconciliation(buildId, claim.generation)), + async () => { + const currentClaim = await this.claimDeploymentReconciliation(buildId, claim.generation); + if (!currentClaim) return; + build = await this.loadBuildDeploymentAuthority(buildId); + if (!build) return; + if (build.uuid) updateLogContext({ buildUuid: build.uuid }); + + const blocked = this.deploymentBlockReason(build); + if (blocked) { + if (blocked === 'tearing_down') { + getLogger({ buildId, generation: claim.generation }).info( + 'Build reconciliation: deferred reason=tearing_down' + ); + return; + } + getLogger({ buildId, generation: claim.generation }).info( + `Build reconciliation: observed without execution reason=${blocked}` + ); + await this.markDeploymentReconciliationObserved(buildId, claim.generation); + return; + } + activeRunUUID = await this.claimDeploymentRun(build, claim.token, claim.generation); + } + ); + + if (!initialClaim.admitted || !build || !activeRunUUID) return; + const reconciliationBuild = build; + activeBuild = reconciliationBuild; + const terminalOutcomes: Array<{ status: BuildStatus; error?: unknown }> = []; + const scopes = this.deploymentReconciliationScopes(claim.dirty); + getLogger({ stage: LogStage.BUILD_STARTING, buildId, generation: claim.generation }).info( + `Build reconciliation: started scopes=${scopes.length}` + ); - const triggerRepositoryId = sourceGithubRepositoryId ?? githubRepositoryId; - // A stale delivered ref must still deploy (converged to the live head), never drop the push. - const effectiveSourceRef = await this.resolveEffectiveSourceRef(triggerRepositoryId, sourceBranch, sourceRef); - - // Claim ordering before the live deploy-authority gate. Even if this newer - // trigger is paused or closed, an older queued run must not resume afterward. - if (!(await this.claimTriggerSequence(build.id, triggerRepositoryId, sourceBranch, triggerSequence))) { - getLogger().info( - `Build: skipping reason=stale_trigger sequence=${triggerSequence} scope=${triggerRepositoryId ?? 'all'}:${ - sourceBranch ?? 'all' - }` + for (const scope of scopes) { + const sourceRef = await this.resolveCurrentSourceRef(scope); + let preparation: DeploymentScopePreparation | null = null; + const configuration = await this.withCurrentBuildDeploymentLock( + buildId, + () => this.isDeploymentRunCurrent(buildId, claim.token, claim.generation), + async () => { + if (!(await this.isDeploymentRunCurrent(buildId, claim.token, claim.generation))) return; + + await this.importYamlConfigFile( + reconciliationBuild.environment!, + reconciliationBuild, + scope.githubRepositoryId ?? undefined, + { + skipDeletedServiceReconciliation: scope.skipDeletedServiceReconciliation, + sourceRef, + sourceBranch: scope.sourceBranch, + sourceGithubRepositoryId: scope.sourceGithubRepositoryId, + runUUID: claim.token, + expectedGeneration: claim.generation, + } + ); + preparation = await this.prepareDeploymentScope( + reconciliationBuild, + scope.githubRepositoryId, + sourceRef, + { + runUUID: claim.token, + sourceBranch: scope.sourceBranch, + sourceGithubRepositoryId: scope.sourceGithubRepositoryId, + expectedGeneration: claim.generation, + } + ); + } ); - return; - } - const blocked = this.deploymentBlockReason(build); - if (blocked) { - getLogger().info(`Build: skipping reason=${blocked}`); - return; + if (!configuration.admitted || !preparation) return; + const outcome = await this.executeDeploymentScope(preparation, claim.generation); + if (!outcome) return; + terminalOutcomes.push(outcome); + if (!(await this.isDeploymentRunCurrent(buildId, claim.token, claim.generation))) return; } - getLogger({ stage: LogStage.BUILD_STARTING }).info('Build: started'); - activeRunUUID = await this.claimDeploymentRun(build, requestedRunUUID); - if (!activeRunUUID) return; + // Scope workers mutate one shared ingress snapshot. Publish it once, + // after every selected scope has settled, so an earlier scope cannot + // overwrite the final configuration for the same generation. + if (terminalOutcomes.some(({ status }) => status === BuildStatus.DEPLOYED)) { + await this.enqueueIngressManifest(buildId, claim.token, claim.generation); + } - // YAML import, deployable/deploy reconciliation, image builds, CLI - // deploys, and manifest application all mutate shared per-build rows. - // Keep the one lock for this entire sequence. - await this.importYamlConfigFile(build.environment!, build, githubRepositoryId, { - skipDeletedServiceReconciliation, - sourceRef: effectiveSourceRef, - sourceBranch, - }); + const failedOutcome = terminalOutcomes.find(({ status }) => status === BuildStatus.ERROR); + const finalStatus = failedOutcome + ? BuildStatus.ERROR + : isDeployEnabled(reconciliationBuild) + ? BuildStatus.DEPLOYED + : BuildStatus.BUILT; + await this.updateStatusAndComment( + reconciliationBuild, + finalStatus, + claim.token, + true, + true, + failedOutcome?.error instanceof Error ? failedOutcome.error : null, + claim.generation + ); - if (!(await this.isDeploymentRunCurrent(build.id, activeRunUUID))) { - getLogger().info('Build: skipping after import reason=authority_changed'); - return; + if (await this.markDeploymentReconciliationObserved(buildId, claim.generation, claim.token)) { + getLogger({ stage: LogStage.BUILD_COMPLETE, buildId, generation: claim.generation }).info( + 'Build reconciliation: completed' + ); + } + } catch (error) { + if (error instanceof AuthorityLockLostError) { + const stillCurrent = activeRunUUID + ? await this.isDeploymentRunCurrent(buildId, activeRunUUID, claim.generation) + : Boolean(await this.claimDeploymentReconciliation(buildId, claim.generation)); + if (stillCurrent) { + getLogger({ error, buildId, generation: claim.generation }).warn( + 'Build reconciliation: mutation lock lost; leaving generation pending for retry' + ); + throw error; + } } + const stillCurrent = activeRunUUID + ? await this.isDeploymentRunCurrent(buildId, activeRunUUID, claim.generation) + : Boolean(await this.claimDeploymentReconciliation(buildId, claim.generation)); + if (stillCurrent) { + const configError = error instanceof ParsingError || error instanceof ValidationError; + if (configError && activeBuild && activeRunUUID) { + await this.recordBuildFailure( + activeBuild, + BuildStatus.CONFIG_ERROR, + activeRunUUID, + error, + 'Lifecycle configuration failed validation.', + claim.generation + ); + await this.markDeploymentReconciliationObserved(buildId, claim.generation, activeRunUUID); + return; + } - await this.resolveAndDeployBuild(build, isDeployEnabled(build), githubRepositoryId, effectiveSourceRef, { - deploymentLockAlreadyHeld: true, - runAlreadyClaimed: true, - runUUID: activeRunUUID, - sourceBranch, - }); + if (!this.isFinalDeploymentReconciliationAttempt(job)) { + getLogger({ error, buildId, generation: claim.generation }).warn( + 'Build reconciliation: transient failure; leaving generation pending for queue retry' + ); + throw error; + } - getLogger({ stage: LogStage.BUILD_COMPLETE }).info('Build: completed'); - }); - } catch (error) { - if (!build) { - getLogger({ stage: LogStage.BUILD_FAILED }).fatal({ error }, `Build: queue failed buildId=${buildId}`); - throw error; - } else if ( - activeRunUUID && - (await this.isDeploymentRunCurrent(build.id, activeRunUUID).catch(() => false)) && - (error instanceof ParsingError || error instanceof ValidationError) - ) { - await this.recordBuildFailure( - build, - BuildStatus.CONFIG_ERROR, - activeRunUUID, - error, - 'Lifecycle configuration failed validation.' - ); - } else if (activeRunUUID && (await this.isDeploymentRunCurrent(build.id, activeRunUUID).catch(() => false))) { - getLogger({ stage: LogStage.BUILD_FAILED }).fatal({ error }, 'Build: uncaught exception'); - await this.recordBuildFailure( - build, - BuildStatus.ERROR, - activeRunUUID, - error, - 'Build queue processing failed.' - ); - throw error; - } else if (!activeRunUUID) { - getLogger({ stage: LogStage.BUILD_FAILED }).fatal({ error }, 'Build: queue failed before run claim'); - throw error; - } else { - getLogger({ stage: LogStage.BUILD_FAILED }).info('Build: queue failure ignored reason=ownership_lost'); + if ( + await this.recordFinalDeploymentReconciliationFailure(buildId, claim, activeBuild, activeRunUUID, error) + ) { + getLogger({ stage: LogStage.BUILD_FAILED, buildId, generation: claim.generation }).error( + { error }, + 'Build reconciliation: failed after final queue attempt' + ); + return; + } + + // Authority may have changed while the final failure was being + // recorded. Stale failures are intentionally silent; a still-current + // failure must remain retryable if its terminal write could not land. + if (!(await this.claimDeploymentReconciliation(buildId, claim.generation))) return; + throw error; + } + getLogger({ buildId, generation: claim.generation }).info('Build reconciliation: stale failure ignored'); } - } + }); }); }; - /** - * Initial step in routing a build into the build queue. A job will either get enqueue in the build queue - * after this job - * @param job the Bull job with the buildID - * @param done the Bull callback to invoke when we're done - */ - processResolveAndDeployBuildQueue = async (job) => { - const { - sender, - correlationId, - skipDeletedServiceReconciliation, - triggerRef, - sourceRef, - sourceGithubRepositoryId, - sourceBranch, - runUUID, - _ddTraceContext, - } = job.data; - - return withLogContext({ correlationId, sender, _ddTraceContext }, async () => { - let jobId; - let buildId: number | undefined; - try { - jobId = job?.data?.buildId; - const githubRepositoryId = job?.data?.githubRepositoryId; - const triggerSequence = this.normalizeTriggerSequence(job?.id); - if (!jobId) throw new Error('jobId is required but undefined'); - const build = await this.loadBuildDeploymentAuthority(jobId); - buildId = build?.id; - if (!buildId) throw new Error('buildId is required but undefined'); - - if (build?.uuid) { - updateLogContext({ buildUuid: build.uuid }); - } - - getLogger({ stage: LogStage.BUILD_QUEUED }).info('Build: processing'); + /** Mixed-version rollout adapters; remove after no old queue payloads can remain. */ + processBuildQueue = async (job) => { + await this.adoptLegacyDeploymentJob(job.data); + }; - const blocked = this.deploymentBlockReason(build); - if (blocked && !triggerSequence) { - getLogger().info(`Deploy: skipping reason=${blocked}`); - return; - } - if (blocked) { - getLogger().info(`Deploy: forwarding blocked sequenced trigger reason=${blocked}`); - } - // Enqueue a standard resolve build. Forward triggerRef so the build job shares the resolve step's dedupe - // identity; otherwise the two layers would disagree and idempotent coalescing of genuine duplicates breaks. - await this.enqueueBuildJob({ - buildId, - githubRepositoryId, - skipDeletedServiceReconciliation, - triggerRef, - sourceRef, - sourceGithubRepositoryId, - sourceBranch, - ...(runUUID ? { runUUID } : {}), - ...(triggerSequence ? { triggerSequence } : {}), - ...extractContextForQueue(), - }); - } catch (error) { - getLogger().error({ error }, `Queue: processing failed buildId=${buildId} jobId=${jobId}`); - throw error; - } - }); + processResolveAndDeployBuildQueue = async (job) => { + await this.adoptLegacyDeploymentJob(job.data); }; } diff --git a/src/server/services/deploy.ts b/src/server/services/deploy.ts index 34c14ed4..d6bce34e 100644 --- a/src/server/services/deploy.ts +++ b/src/server/services/deploy.ts @@ -42,6 +42,7 @@ import { fallbackDeployStatusMessage, statusMessageFromError } from 'server/lib/ import { isNativeBuilderEngine } from 'server/lib/buildEngines'; import { SecretProvidersConfig } from 'server/services/types/globalConfig'; import { createOrUpdateNamespace } from 'server/lib/kubernetes'; +import { AuthorityLockLostError, type AuthorityLockResult } from 'server/lib/authorityLock'; export interface DeployOptions { ownerId?: number; @@ -78,7 +79,8 @@ export default class DeployService extends BaseService { build: Build, githubRepositoryId?: number, sourceRef?: string | null, - sourceBranch?: string | null + sourceBranch?: string | null, + sourceGithubRepositoryId: number | null | undefined = githubRepositoryId ): Promise { await build?.$fetchGraph('[deployables.[repository]]'); @@ -147,12 +149,12 @@ export default class DeployService extends BaseService { if (isTargetSource && [DeployTypes.HELM, DeployTypes.GITHUB, DeployTypes.CODEFRESH].includes(deployable.type)) { try { const sha = - this.getApiBuildSourceRef( + this.getPinnedBuildSourceRef( build, deployableRepositoryId, effectiveBranch, sourceRef, - githubRepositoryId, + sourceGithubRepositoryId, sourceBranch ) ?? (await getShaForDeploy(deploy)); patchFields.sha = sha; @@ -246,13 +248,11 @@ export default class DeployService extends BaseService { } } - async deployAurora(deploy: Deploy): Promise { + async deployAurora(deploy: Deploy, runUUID: string): Promise { return withLogContext({ deployUuid: deploy.uuid, serviceName: deploy.deployable?.name }, async () => { - let runUUID = deploy.runUUID; try { await deploy.reload(); await deploy.$fetchGraph('[build, deployable]'); - runUUID = deploy.runUUID; if (!deploy.deployable) { getLogger().error('Aurora: deployable missing for=restore'); @@ -267,7 +267,7 @@ export default class DeployService extends BaseService { const existingDbEndpoint = await this.findExistingAuroraDatabase(deploy.build.uuid, deploy.deployable.name); if (existingDbEndpoint) { getLogger().info('Aurora: skipped reason=exists'); - await deploy.$query().patch({ + await this.patchDeployForRun(deploy, runUUID, { cname: existingDbEndpoint, status: DeployStatus.BUILT, }); @@ -275,34 +275,32 @@ export default class DeployService extends BaseService { } const uuid = nanoid(); - runUUID = nanoid(); - await deploy.$query().patch({ + const current = await this.patchDeployForRun(deploy, runUUID, { status: DeployStatus.BUILDING, buildLogs: uuid, - runUUID, }); - deploy.runUUID = runUUID; + if (!current) { + getLogger().info('Aurora: skipped reason=superseded'); + return true; + } getLogger().info('Aurora: restoring'); await cli.cliDeploy(deploy); const dbEndpoint = await this.findExistingAuroraDatabase(deploy.build.uuid, deploy.deployable.name); if (dbEndpoint) { - await deploy.$query().patch({ + await this.patchDeployForRun(deploy, runUUID, { cname: dbEndpoint, }); } - await deploy.reload(); - if (deploy.buildLogs === uuid) { - await deploy.$query().patch({ - status: DeployStatus.BUILT, - }); - } + await this.patchDeployForRun(deploy, runUUID, { + status: DeployStatus.BUILT, + }); getLogger().info('Aurora: restored'); return true; } catch (e) { getLogger().error({ error: e }, 'Aurora: cluster restore failed'); - return this.recordDeployFailure(deploy, runUUID || deploy.runUUID, { + return this.recordDeployFailure(deploy, runUUID, { status: DeployStatus.ERROR, error: e, fallbackMessage: 'Aurora restore failed.', @@ -313,6 +311,7 @@ export default class DeployService extends BaseService { async deployCodefresh( deploy: Deploy, + runUUID: string, sourceRef?: string | null, sourceGithubRepositoryId?: number | null, sourceBranch?: string | null @@ -320,12 +319,6 @@ export default class DeployService extends BaseService { return withLogContext({ deployUuid: deploy.uuid, serviceName: deploy.deployable?.name }, async () => { let result: boolean = false; - const runUUID = nanoid(); - await deploy.$query().patch({ - runUUID, - }); - deploy.runUUID = runUUID; - await deploy.reload(); await deploy.$fetchGraph('[deployable.[repository], build]'); const { build, deployable } = deploy; @@ -357,15 +350,19 @@ export default class DeployService extends BaseService { let buildLogs: string; let codefreshBuildId: string; try { - await deploy.$query().patch({ + const current = await this.patchDeployForRun(deploy, runUUID, { buildLogs: null, buildPipelineId: null, buildOutput: null, deployPipelineId: null, deployOutput: null, }); + if (!current) { + getLogger().info('Codefresh: skipped reason=superseded'); + return true; + } - const pinnedSourceRef = this.getApiBuildSourceRef( + const pinnedSourceRef = this.getPinnedBuildSourceRef( build, deploy.githubRepositoryId, deploy.branchName, @@ -439,15 +436,16 @@ export default class DeployService extends BaseService { async deployCLI( deploy: Deploy, + runUUID: string, sourceRef?: string | null, sourceGithubRepositoryId?: number | null, sourceBranch?: string | null ): Promise { if (deploy.deployable != null) { if (deploy.deployable.type === DeployTypes.AURORA_RESTORE) { - return this.deployAurora(deploy); + return this.deployAurora(deploy, runUUID); } else if (deploy.deployable.type === DeployTypes.CODEFRESH) { - return this.deployCodefresh(deploy, sourceRef, sourceGithubRepositoryId, sourceBranch); + return this.deployCodefresh(deploy, runUUID, sourceRef, sourceGithubRepositoryId, sourceBranch); } } } @@ -459,17 +457,19 @@ export default class DeployService extends BaseService { async buildImage( deploy: Deploy, _index: number, + runUUID: string, sourceRef?: string | null, sourceGithubRepositoryId?: number | null, - sourceBranch?: string | null + sourceBranch?: string | null, + expectedGeneration?: number, + nativeServiceAccount?: string ): Promise { return withLogContext({ deployUuid: deploy.uuid, serviceName: deploy.deployable?.name }, async () => { - const runUUID = deploy.runUUID ?? nanoid(); try { - await deploy.$query().patch({ - runUUID, - }); - deploy.runUUID = runUUID; + if (!(await this.isDeploymentRunCurrent(deploy, runUUID, expectedGeneration))) { + getLogger().info('Image: skipped reason=superseded'); + return true; + } await deploy.$fetchGraph('[build.[environment], deployable]'); const { deployable } = deploy; @@ -481,7 +481,9 @@ export default class DeployService extends BaseService { runUUID, sourceRef, sourceGithubRepositoryId, - sourceBranch + sourceBranch, + expectedGeneration, + nativeServiceAccount ); case DeployTypes.DOCKER: await this.patchAndUpdateActivityFeed( @@ -504,7 +506,9 @@ export default class DeployService extends BaseService { runUUID, sourceRef, sourceGithubRepositoryId, - sourceBranch + sourceBranch, + expectedGeneration, + nativeServiceAccount ); } @@ -539,6 +543,7 @@ export default class DeployService extends BaseService { ); return true; } catch (error) { + if (error instanceof AuthorityLockLostError) throw error; getLogger().warn({ error }, 'Helm: deployment processing failed'); return false; } @@ -548,7 +553,15 @@ export default class DeployService extends BaseService { return false; } } catch (e) { + if (e instanceof AuthorityLockLostError) throw e; getLogger().error({ error: e }, 'Docker: build error'); + if ( + expectedGeneration != null && + !(await this.isDeploymentRunCurrent(deploy, runUUID, expectedGeneration).catch(() => false)) + ) { + getLogger().info('Docker: failure ignored reason=superseded'); + return true; + } return this.recordDeployFailure(deploy, runUUID, { status: DeployStatus.BUILD_FAILED, error: e, @@ -560,7 +573,7 @@ export default class DeployService extends BaseService { async recordDeployFailure( deploy: Deploy, - runUUID: string | null | undefined, + runUUID: string, { status, error, @@ -571,26 +584,19 @@ export default class DeployService extends BaseService { fallbackMessage: string; } ): Promise { - const activeRunUUID = runUUID || deploy.runUUID || nanoid(); - - if (deploy.runUUID !== activeRunUUID) { - await deploy.$query().patch({ runUUID: activeRunUUID }); - deploy.runUUID = activeRunUUID; - } - await this.patchAndUpdateActivityFeed( deploy, { status, statusMessage: statusMessageFromError(error, fallbackMessage), }, - activeRunUUID + runUUID ); return false; } - private getApiBuildSourceRef( + private getPinnedBuildSourceRef( build: Build | null | undefined, deployGithubRepositoryId: number | null | undefined, deployBranchName: string | null | undefined, @@ -599,7 +605,6 @@ export default class DeployService extends BaseService { sourceBranch?: string | null ): string | null { if ( - build?.triggerType === 'api' && sourceRef && sourceGithubRepositoryId != null && deployGithubRepositoryId != null && @@ -635,7 +640,7 @@ export default class DeployService extends BaseService { sourceGithubRepositoryId?: number | null, sourceBranch?: string | null ): Promise { - const pinned = this.getApiBuildSourceRef( + const pinned = this.getPinnedBuildSourceRef( deploy.build, deploy.githubRepositoryId, deploy.branchName, @@ -669,6 +674,55 @@ export default class DeployService extends BaseService { ); } + private async patchDeployForRun( + deploy: Deploy, + runUUID: string, + params: Objection.PartialModelObject + ): Promise { + const updated = await this.db.models.Deploy.query().where({ id: deploy.id, runUUID }).patch(params); + if (!updated) { + getLogger().debug(`Deploy: stale write skipped deployId=${deploy.id} runUUID=${runUUID}`); + return false; + } + return true; + } + + private async isDeployRunCurrent(deploy: Deploy, runUUID: string): Promise { + const current = await this.db.models.Deploy.query().findOne({ id: deploy.id, runUUID }).select('id'); + return Boolean(current); + } + + private async isBuildRunCurrent( + buildId: number | null | undefined, + runUUID: string, + expectedGeneration?: number + ): Promise { + if (!buildId) return false; + let current = this.db.models.Build.query().findOne({ id: buildId, runUUID }).whereNull('deletedAt'); + if (expectedGeneration != null) current = current.where('desiredGeneration', expectedGeneration); + return Boolean(await current); + } + + private async isDeploymentRunCurrent(deploy: Deploy, runUUID: string, expectedGeneration?: number): Promise { + if (!(await this.isDeployRunCurrent(deploy, runUUID))) return false; + return expectedGeneration == null || this.isBuildRunCurrent(deploy.buildId, runUUID, expectedGeneration); + } + + private async withDeploySecretMutation( + deploy: Deploy, + runUUID: string, + expectedGeneration: number | undefined, + action: () => Promise + ): Promise> { + const isCurrent = () => this.isDeploymentRunCurrent(deploy, runUUID, expectedGeneration); + const gate = this.db.services?.BuildService?.withCurrentDeploySecretMutationLock; + if (typeof gate === 'function') { + return gate.call(this.db.services.BuildService, deploy.id, isCurrent, action); + } + if (!(await isCurrent())) return { admitted: false }; + return { admitted: true, value: await action() }; + } + public async patchAndUpdateActivityFeed( deploy: Deploy, params: Objection.PartialModelObject, @@ -677,18 +731,14 @@ export default class DeployService extends BaseService { ) { let build: Build; try { - const id = deploy?.id; const failureStatuses = [DeployStatus.ERROR, DeployStatus.BUILD_FAILED, DeployStatus.DEPLOY_FAILED]; const status = params.status as DeployStatus; const fallbackStatusMessage = !params.statusMessage && failureStatuses.includes(status) ? fallbackDeployStatusMessage(status) : ''; const patchParams = fallbackStatusMessage ? { ...params, statusMessage: fallbackStatusMessage } : params; - await this.db.models.Deploy.query().where({ id, runUUID }).patch(patchParams); - if (deploy.runUUID !== runUUID) { - getLogger().debug(`runUUID mismatch: deployRunUUID=${deploy.runUUID} providedRunUUID=${runUUID}`); - return; - } + const updated = await this.patchDeployForRun(deploy, runUUID, patchParams); + if (!updated) return; await deploy.$fetchGraph('build.pullRequest'); build = deploy?.build; @@ -766,7 +816,19 @@ export default class DeployService extends BaseService { ); } - private async patchDeployWithTag({ tag, deploy, initTag, ecrDomain }) { + private async patchDeployWithTag({ + tag, + deploy, + initTag, + ecrDomain, + runUUID, + }: { + tag: string; + deploy: Deploy; + initTag: string; + ecrDomain: string; + runUUID: string; + }) { await deploy.$fetchGraph('[build, deployable]'); const { deployable } = deploy; let ecrRepo = deployable?.ecr as string; @@ -776,22 +838,16 @@ export default class DeployService extends BaseService { const dockerImage = codefresh.getRepositoryTag({ tag, ecrRepo, ecrDomain }); - if (deployable?.initDockerfilePath) { - const initDockerImage = codefresh.getRepositoryTag({ tag: initTag, ecrRepo, ecrDomain }); - await deploy - .$query() - .patch({ - initDockerImage, - }) - .catch((error) => { - getLogger().warn({ error }, 'Deploy: tag patch failed'); - }); - } - - await deploy.$query().patch({ + const initDockerImage = deployable?.initDockerfilePath + ? codefresh.getRepositoryTag({ tag: initTag, ecrRepo, ecrDomain }) + : undefined; + await this.patchDeployForRun(deploy, runUUID, { status: DeployStatus.BUILT, dockerImage, statusMessage: 'Successfully built image', + ...(initDockerImage ? { initDockerImage } : {}), + }).catch((error) => { + getLogger().warn({ error }, 'Deploy: tag patch failed'); }); } @@ -800,11 +856,13 @@ export default class DeployService extends BaseService { serviceName, secretProviders, runUUID, + expectedGeneration, }: { deploy: Deploy; serviceName: string; secretProviders: SecretProvidersConfig | undefined; runUUID: string; + expectedGeneration?: number; }): Promise { const emptyResult = { secretNames: [], buildSecretEnvKeys: new Set() }; @@ -844,7 +902,9 @@ export default class DeployService extends BaseService { getLogger().error( `Build: secret env conflict service=${serviceName} keys=[${Array.from(conflictingSecretEnvKeys).join(', ')}]` ); - await this.patchAndUpdateActivityFeed(deploy, { status: DeployStatus.BUILD_FAILED }, runUUID); + if (await this.isDeploymentRunCurrent(deploy, runUUID, expectedGeneration)) { + await this.patchAndUpdateActivityFeed(deploy, { status: DeployStatus.BUILD_FAILED }, runUUID); + } return false; } @@ -856,22 +916,36 @@ export default class DeployService extends BaseService { await deploy.$fetchGraph('[build.[pullRequest]]'); - await createOrUpdateNamespace({ - name: deploy.build.namespace, - buildUUID: deploy.build.uuid, - staticEnv: deploy.build.isStatic, - pullRequest: deploy.build.pullRequest, - waitForReady: true, - }); - const secretProcessor = new SecretProcessor(secretProviders); - const secretResult = await secretProcessor.processEnvSecrets({ - env: envToProcess, - serviceName, - namespace: deploy.build.namespace, - buildUuid: deploy.uuid, + // Reconciliation prepared the namespace under its short configuration + // lock. Keep the legacy/direct path self-contained without putting C's + // build behind an older generation's long native promotion. + if (expectedGeneration == null) { + await createOrUpdateNamespace({ + name: deploy.build.namespace, + buildUUID: deploy.build.uuid, + staticEnv: deploy.build.isStatic, + pullRequest: deploy.build.pullRequest, + waitForReady: true, + }); + } + + // Native build and native Helm write the same per-service ExternalSecrets. + // Serialize only that short write; provider convergence waits outside it. + const mutation = await this.withDeploySecretMutation(deploy, runUUID, expectedGeneration, () => { + return secretProcessor.processEnvSecrets({ + env: envToProcess, + serviceName, + namespace: deploy.build.namespace, + buildUuid: deploy.uuid, + }); }); + if (!mutation.admitted) { + getLogger({ deployId: deploy.id }).info('Build: external-secret mutation skipped reason=superseded'); + return false; + } + const secretResult = mutation.value; const buildSecretEnvKeys = new Set( secretResult.secretRefs.filter((ref) => buildSecretRefKeys.has(ref.envKey)).map((ref) => ref.envKey) @@ -907,7 +981,9 @@ export default class DeployService extends BaseService { return { secretNames, buildSecretEnvKeys }; } catch (error) { getLogger().error({ error }, `Build: secret sync failed service=${serviceName}`); - await this.patchAndUpdateActivityFeed(deploy, { status: DeployStatus.BUILD_FAILED }, runUUID); + if (await this.isDeploymentRunCurrent(deploy, runUUID, expectedGeneration)) { + await this.patchAndUpdateActivityFeed(deploy, { status: DeployStatus.BUILD_FAILED }, runUUID); + } return false; } } @@ -931,7 +1007,9 @@ export default class DeployService extends BaseService { runUUID: string, sourceRef?: string | null, sourceGithubRepositoryId?: number | null, - sourceBranch?: string | null + sourceBranch?: string | null, + expectedGeneration?: number, + nativeServiceAccount?: string ) { const { build, deployable } = deploy; const uuid = build?.uuid; @@ -1004,6 +1082,10 @@ export default class DeployService extends BaseService { // if this deploy has any env vars that depend on other builds, we need to wait for those builds to finish // and update the env vars in this deploy before we can build the image await this.waitAndResolveForBuildDependentEnvVars(deploy, envVariables, runUUID); + if (!(await this.isDeploymentRunCurrent(deploy, runUUID, expectedGeneration))) { + getLogger().info('Image: stopped after dependency wait reason=superseded'); + return true; + } await deploy.reload(); await this.patchAndUpdateActivityFeed( @@ -1044,11 +1126,16 @@ export default class DeployService extends BaseService { serviceName: deployable.name, secretProviders, runUUID, + expectedGeneration, }); if (!syncedExternalSecrets) { return false; } + if (!(await this.isDeploymentRunCurrent(deploy, runUUID, expectedGeneration))) { + getLogger().info('Image: stopped after secret sync reason=superseded'); + return true; + } const filteredEnvVars = syncedExternalSecrets.buildSecretEnvKeys.size > 0 @@ -1071,6 +1158,7 @@ export default class DeployService extends BaseService { podAnnotations: deployable.builder?.podAnnotations, secretRefs: syncedExternalSecrets.secretNames, secretEnvKeys: Array.from(syncedExternalSecrets.buildSecretEnvKeys), + ...(nativeServiceAccount ? { serviceAccount: nativeServiceAccount } : {}), }; if (!initDockerfilePath) { @@ -1078,27 +1166,95 @@ export default class DeployService extends BaseService { } const result = await buildWithNative(deploy, nativeOptions); + let runIsCurrent = await this.isDeploymentRunCurrent(deploy, runUUID, expectedGeneration); + const nativeBuildResult = result.success ? 'success' : 'failure'; + if (!runIsCurrent) { + getLogger().info(`Image: native result publication skipped reason=superseded result=${nativeBuildResult}`); + } // Persist build logs so failures stay diagnosable after the build job pod is gone. - if (result.logs) { - await deploy.$query().patch({ buildOutput: result.logs.slice(-65536) }); + if (runIsCurrent && result.logs) { + await this.patchDeployForRun(deploy, runUUID, { buildOutput: result.logs.slice(-65536) }); + } + if (runIsCurrent) { + runIsCurrent = await this.isDeploymentRunCurrent(deploy, runUUID, expectedGeneration); + if (!runIsCurrent) { + getLogger().info( + `Image: native result publication skipped reason=superseded result=${nativeBuildResult}` + ); + } } if (result.success) { - await this.patchDeployWithTag({ tag, initTag, deploy, ecrDomain }); + if (runIsCurrent) { + await this.patchDeployWithTag({ tag, initTag, deploy, ecrDomain, runUUID }); + runIsCurrent = await this.isDeploymentRunCurrent(deploy, runUUID, expectedGeneration); + } + if (buildOptions?.afterBuildPipelineId) { + // This pipeline belongs to the produced image. A newer live generation may detach it, + // while teardown or deletion must suppress new external work. const ecrRepoTag = constructEcrTag({ repo: ecrRepo, tag, ecrDomain }); + if (!runIsCurrent) { + let hasLiveSuccessor = false; + try { + hasLiveSuccessor = await this.db.services.BuildService.isSupersededByNewerLiveDeploymentGeneration( + deploy.buildId, + expectedGeneration + ); + } catch (error) { + getLogger({ error }).warn('Codefresh: after-build successor check failed'); + } + + if (!hasLiveSuccessor) { + const skippedLog = { + afterBuildPipelineId: buildOptions.afterBuildPipelineId, + imageTag: ecrRepoTag, + }; + getLogger(skippedLog).info( + `Codefresh: after-build pipeline skipped reason=no_live_successor ` + + `afterBuildPipelineId=${skippedLog.afterBuildPipelineId} imageTag=${skippedLog.imageTag}` + ); + return true; + } + } + const afterbuildPipeline = await codefresh.triggerPipeline(buildOptions.afterBuildPipelineId, 'cli', { ...deploy.env, ...{ TAG: ecrRepoTag }, ...{ branch: branchName }, }); + const afterBuildLog = { + afterBuildPipelineId: buildOptions.afterBuildPipelineId, + pipelineId: afterbuildPipeline, + imageTag: ecrRepoTag, + }; + const afterBuildLogDetails = + `afterBuildPipelineId=${afterBuildLog.afterBuildPipelineId} ` + + `pipelineId=${afterBuildLog.pipelineId} imageTag=${afterBuildLog.imageTag}`; + const afterBuildLogger = getLogger(afterBuildLog); + afterBuildLogger.info(`Codefresh: after-build pipeline triggered ${afterBuildLogDetails}`); + + if (!runIsCurrent || !(await this.isDeploymentRunCurrent(deploy, runUUID, expectedGeneration))) { + afterBuildLogger.info( + `Codefresh: after-build pipeline detached reason=superseded ${afterBuildLogDetails}` + ); + return true; + } + const completed = await codefresh.waitForImage(afterbuildPipeline); - if (!completed) return false; + if (!completed) { + afterBuildLogger.warn( + `Codefresh: after-build pipeline completed result=failure ${afterBuildLogDetails}` + ); + return false; + } + afterBuildLogger.info(`Codefresh: after-build pipeline completed result=success ${afterBuildLogDetails}`); } return true; } else { + if (!runIsCurrent) return true; await this.patchAndUpdateActivityFeed(deploy, { status: DeployStatus.BUILD_FAILED }, runUUID); return false; } @@ -1109,13 +1265,17 @@ export default class DeployService extends BaseService { const buildPipelineId = await codefresh.buildImage(buildOptions); const buildLogs = `https://g.codefresh.io/build/${buildPipelineId}`; await this.patchAndUpdateActivityFeed(deploy, { buildLogs }, runUUID); - await deploy.$query().patch({ buildPipelineId }); + await this.patchDeployForRun(deploy, runUUID, { buildPipelineId }); const buildSuccess = await codefresh.waitForImage(buildPipelineId); const buildOutput = await codefresh.getLogs(buildPipelineId); - await deploy.$query().patch({ buildOutput }); + if (!(await this.isDeploymentRunCurrent(deploy, runUUID, expectedGeneration))) { + getLogger().info('Image: Codefresh result ignored reason=superseded'); + return true; + } + await this.patchDeployForRun(deploy, runUUID, { buildOutput }); if (buildSuccess) { - await this.patchDeployWithTag({ tag, initTag, deploy, ecrDomain }); + await this.patchDeployWithTag({ tag, initTag, deploy, ecrDomain, runUUID }); getLogger().info('Image: built'); return true; } else { @@ -1131,13 +1291,18 @@ export default class DeployService extends BaseService { serviceName: deployable.name, secretProviders, runUUID, + expectedGeneration, }); if (!syncedExternalSecrets) { return false; } + if (!(await this.isDeploymentRunCurrent(deploy, runUUID, expectedGeneration))) { + getLogger().info('Image: stopped after secret sync reason=superseded'); + return true; + } } - await this.patchDeployWithTag({ tag, initTag, deploy, ecrDomain }); + await this.patchDeployWithTag({ tag, initTag, deploy, ecrDomain, runUUID }); await this.patchAndUpdateActivityFeed(deploy, { status: DeployStatus.BUILT }, runUUID); return true; } @@ -1223,7 +1388,7 @@ export default class DeployService extends BaseService { await Promise.all(pipelinePromises); - await deploy.$query().patch({ + await this.patchDeployForRun(deploy, runUUID, { env: { ...envVariables, ...extractedValues, diff --git a/src/server/services/deployCleanup.ts b/src/server/services/deployCleanup.ts index d9ae1e14..9bef4760 100644 --- a/src/server/services/deployCleanup.ts +++ b/src/server/services/deployCleanup.ts @@ -28,6 +28,7 @@ import { QUEUE_NAMES } from 'shared/config'; import { redisClient } from 'server/lib/dependencies'; export type DeployCleanupMode = 'infra' | 'service'; +type NativeMutationGate = (action: () => Promise) => Promise; interface CleanupTask { name: string; @@ -163,7 +164,10 @@ export default class DeployCleanupService extends BaseService { }); } - async cleanupDeploy(deployOrId: Deploy | number, { mode }: { mode: DeployCleanupMode }): Promise { + async cleanupDeploy( + deployOrId: Deploy | number, + { mode, nativeMutationGate }: { mode: DeployCleanupMode; nativeMutationGate?: NativeMutationGate } + ): Promise { const deploy = await this.resolveDeploy(deployOrId); if (!deploy) { getLogger({ deployId: deployOrId, mode }).warn('Deploy cleanup: deploy not found'); @@ -183,19 +187,24 @@ export default class DeployCleanupService extends BaseService { const { namespace, serviceName, deployType } = metadata; return withLogContext({ deployUuid: deploy.uuid, serviceName }, async () => { const metrics = this.metricsFor(deploy, mode); - const tasks = [ + const nativeTasks = [ ...this.buildKubernetesTasks(deploy, namespace), ...this.buildSecretTasks(deploy, namespace, serviceName), this.buildHelmTask(deploy, namespace, deployType), - this.buildCliTask(deploy, deployType), ].filter(Boolean) as CleanupTask[]; + const providerTasks = [this.buildCliTask(deploy, deployType)].filter(Boolean) as CleanupTask[]; + const tasks = [...nativeTasks, ...providerTasks]; getLogger({ mode, taskCount: tasks.length }).info('Deploy cleanup: targeted teardown started'); - const results: boolean[] = []; - for (const task of tasks) { - results.push(await this.runTask(task, deploy, metrics, mode)); - } + const runTasks = async (selectedTasks: CleanupTask[]) => { + const results: boolean[] = []; + for (const task of selectedTasks) results.push(await this.runTask(task, deploy, metrics, mode)); + return results; + }; + const results = nativeMutationGate + ? (await Promise.all([nativeMutationGate(() => runTasks(nativeTasks)), runTasks(providerTasks)])).flat() + : await runTasks(tasks); const success = results.every(Boolean); metrics.increment('deploy', { mode, result: success ? 'complete' : 'partial_failure' }); @@ -271,7 +280,8 @@ export default class DeployCleanupService extends BaseService { shellPromise( `kubectl delete ${resourceType} ${filteredNames.map(shellQuote).join(' ')} --namespace ${shellQuote( namespace - )} --ignore-not-found` + )} --ignore-not-found`, + { timeout: 90_000 } ), }; } @@ -284,7 +294,8 @@ export default class DeployCleanupService extends BaseService { shellPromise( `kubectl delete ${resourceType} --namespace ${shellQuote(namespace)} -l ${shellQuote( selector - )} --ignore-not-found` + )} --ignore-not-found`, + { timeout: 90_000 } ), }; } @@ -341,7 +352,8 @@ export default class DeployCleanupService extends BaseService { shellPromise( `kubectl delete externalsecret ${shellQuote(secretName)} --namespace ${shellQuote( namespace - )} --ignore-not-found` + )} --ignore-not-found`, + { timeout: 90_000 } ), }, { @@ -349,7 +361,8 @@ export default class DeployCleanupService extends BaseService { resourceType: 'secret', run: () => shellPromise( - `kubectl delete secret ${shellQuote(secretName)} --namespace ${shellQuote(namespace)} --ignore-not-found` + `kubectl delete secret ${shellQuote(secretName)} --namespace ${shellQuote(namespace)} --ignore-not-found`, + { timeout: 90_000 } ), }, ]; @@ -364,7 +377,10 @@ export default class DeployCleanupService extends BaseService { return { name: 'helm-uninstall', resourceType: 'helm-release', - run: () => shellPromise(`helm uninstall ${shellQuote(deploy.uuid)} --namespace ${shellQuote(namespace)}`), + run: () => + shellPromise(`helm uninstall ${shellQuote(deploy.uuid)} --namespace ${shellQuote(namespace)} --timeout 5m`, { + timeout: 360_000, + }), }; } diff --git a/src/server/services/deployable.ts b/src/server/services/deployable.ts index 53054d3a..bfc9d06f 100644 --- a/src/server/services/deployable.ts +++ b/src/server/services/deployable.ts @@ -108,6 +108,7 @@ export interface DeployableAttributes { commentBranchName?: string; ingressAnnotations?: Record; helm?: Helm; + requires?: string[]; deploymentDependsOn?: string[]; builder?: Builder; envLens?: boolean; @@ -278,6 +279,13 @@ export default class DeployableService extends BaseService { active, dependsOnDeployableName, helm: await YamlService.getHelmConfigFromYaml(service), + requires: [ + ...new Set( + (service.requires ?? []) + .map((requiredService) => requiredService.name?.trim()) + .filter((name): name is string => Boolean(name)) + ), + ], deploymentDependsOn: service.deploymentDependsOn || [], builder: YamlService.getEffectiveBuilder(service, buildDefaults?.engine) ?? {}, envLens: await YamlService.getEnvLens(service), @@ -423,7 +431,8 @@ export default class DeployableService extends BaseService { build?: Build, filterGithubRepositoryId?: number, sourceRef?: string | null, - sourceBranch?: string | null + sourceBranch?: string | null, + sourceGithubRepositoryId: number | null | undefined = filterGithubRepositoryId ): Promise { try { let allReferencedYamlConfigsResolved = true; @@ -458,6 +467,15 @@ export default class DeployableService extends BaseService { (repository?.githubRepositoryId != null && Number(repository.githubRepositoryId) === Number(filterGithubRepositoryId) && (sourceBranch == null || branchName === sourceBranch)); + const matchesDeliveredSource = ( + repository: Repository | null | undefined, + branchName: string | null | undefined + ) => + sourceGithubRepositoryId != null && + repository?.githubRepositoryId != null && + Number(repository.githubRepositoryId) === Number(sourceGithubRepositoryId) && + sourceBranch != null && + branchName === sourceBranch; const attribution = async ( services: YamlService.DependencyService[], @@ -525,11 +543,7 @@ export default class DeployableService extends BaseService { } const sourceRefTargetsDependency = - sourceBuild?.triggerType === 'api' && - sourceRef != null && - filterGithubRepositoryId != null && - sourceBranch != null && - targetsSource(repository, branchName); + sourceRef != null && matchesDeliveredSource(repository, branchName); if (filterGithubRepositoryId != null && !targetsSource(repository, branchName)) { getLogger({ buildUUID, service: yamlEnvService.name, branchName }).debug( 'Skipping remote YAML fetch outside filtered source branch' @@ -636,12 +650,8 @@ export default class DeployableService extends BaseService { rootBranch != null && rootBaseConfigRef != null ) { - const sourceRefTargetsRoot = - filterGithubRepositoryId != null && sourceBranch != null && targetsSource(sourceRepository, rootBranch); - const rootConfigRef = - sourceBuild?.triggerType === 'api' && sourceRefTargetsRoot - ? sourceRef ?? rootBaseConfigRef - : rootBaseConfigRef; + const sourceRefTargetsRoot = sourceRef != null && matchesDeliveredSource(sourceRepository, rootBranch); + const rootConfigRef = sourceRefTargetsRoot ? sourceRef : rootBaseConfigRef; const yamlConfig: YamlService.LifecycleConfig = await YamlService.fetchLifecycleConfigByRepository( sourceRepository, rootConfigRef @@ -739,7 +749,8 @@ export default class DeployableService extends BaseService { build?: Build, filterGithubRepositoryId?: number, sourceRef?: string | null, - sourceBranch?: string | null + sourceBranch?: string | null, + sourceGithubRepositoryId: number | null | undefined = filterGithubRepositoryId ): Promise { // We are going to ingest all the database and yaml configuration and process in the memory before writes into the database let deployables: Deployable[] = []; @@ -765,7 +776,8 @@ export default class DeployableService extends BaseService { build, filterGithubRepositoryId, sourceRef, - sourceBranch + sourceBranch, + sourceGithubRepositoryId ); // Finally, Upsert the deployables into the database diff --git a/src/server/services/github.ts b/src/server/services/github.ts index 9da5ac81..968f6ab3 100644 --- a/src/server/services/github.ts +++ b/src/server/services/github.ts @@ -247,7 +247,6 @@ export default class GithubService extends Service { } else { await this.db.services.BuildService.enqueueResolveAndDeployBuild({ buildId: build.id, - ...extractContextForQueue(), }); } } else if (pullRequestState?.deployOnUpdate) { @@ -371,7 +370,6 @@ export default class GithubService extends Service { } await this.db.services.BuildService.enqueueResolveAndDeployBuild({ buildId, - ...extractContextForQueue(), }); } catch (error) { getLogger().error({ error }, `Label: webhook processing failed`); @@ -544,7 +542,12 @@ export default class GithubService extends Service { .whereNot('status', 'torn_down') .withGraphFetched('[build.[pullRequest], deployable]'); - await this.enqueueAutoTrackedApiBuilds(githubRepositoryId, branchName, deployTriggerRef).catch((error) => { + await this.enqueueAutoTrackedApiBuilds( + githubRepositoryId, + branchName, + deployTriggerRef, + !this.isVoidCommit(previousCommit) ? previousCommit : null + ).catch((error) => { getLogger({ error, githubRepositoryId, branchName }).error('Push: API auto-track processing failed'); }); @@ -554,6 +557,7 @@ export default class GithubService extends Service { githubRepositoryId, branchName, headCommit: deployTriggerRef ?? null, + beforeCommit: !this.isVoidCommit(previousCommit) ? previousCommit : null, }); return; } @@ -678,14 +682,10 @@ export default class GithubService extends Service { await this.db.services.BuildService.enqueueResolveAndDeployBuild({ buildId, ...(hasFailedDeploys && build.pullRequest != null ? {} : { githubRepositoryId }), - // The pushed commit is the deploy trigger. It keeps back-to-back pushes to the same branch from collapsing - // onto one dedupe key (which would silently drop the later commit), while a redelivered webhook for the - // same commit still coalesces. - ...(deployTriggerRef ? { triggerRef: deployTriggerRef } : {}), ...(deployTriggerRef ? { sourceRef: deployTriggerRef } : {}), + ...(!this.isVoidCommit(previousCommit) ? { sourceBeforeRef: previousCommit } : {}), sourceGithubRepositoryId: githubRepositoryId, sourceBranch: branchName, - ...extractContextForQueue(), }); } } catch (error) { @@ -701,7 +701,8 @@ export default class GithubService extends Service { private enqueueAutoTrackedApiBuilds = async ( githubRepositoryId: number, branchName: string, - deployTriggerRef?: string | null + deployTriggerRef?: string | null, + sourceBeforeRef?: string | null ) => { const autoTrackedBuilds = await this.db.models.Build.query() .where('triggerType', 'api') @@ -719,11 +720,10 @@ export default class GithubService extends Service { getLogger().info(`Push: deploying api environment uuid=${build.uuid} branch=${branchName}`); await this.db.services.BuildService.enqueueResolveAndDeployBuild({ buildId: build.id, - ...(deployTriggerRef ? { triggerRef: deployTriggerRef } : {}), ...(deployTriggerRef ? { sourceRef: deployTriggerRef } : {}), + ...(sourceBeforeRef ? { sourceBeforeRef } : {}), sourceGithubRepositoryId: githubRepositoryId, sourceBranch: branchName, - ...extractContextForQueue(), }); } }; @@ -738,10 +738,12 @@ export default class GithubService extends Service { githubRepositoryId, branchName, headCommit, + beforeCommit, }: { githubRepositoryId: number; branchName: string; headCommit?: string | null; + beforeCommit?: string | null; }): Promise => { try { const build = await this.db.models.Build.query() @@ -766,11 +768,10 @@ export default class GithubService extends Service { getLogger().info(`Push: redeploying reason=staticEnv`); await this.db.services.BuildService.enqueueResolveAndDeployBuild({ buildId: build?.id, - // The pushed commit is the deploy trigger, so a later push to the tracked branch is not coalesced onto an - // in-flight or recent deploy of an earlier commit. Static envs rebuild on every tracked-branch push by - // design, so this only prevents wrongly dropping a distinct commit. - ...(headCommit ? { triggerRef: headCommit } : {}), - ...extractContextForQueue(), + ...(headCommit ? { sourceRef: headCommit } : {}), + ...(beforeCommit ? { sourceBeforeRef: beforeCommit } : {}), + sourceGithubRepositoryId: githubRepositoryId, + sourceBranch: branchName, }); } catch (error) { getLogger({}).error( diff --git a/src/server/services/ingress.ts b/src/server/services/ingress.ts index cbff2bc4..02512298 100644 --- a/src/server/services/ingress.ts +++ b/src/server/services/ingress.ts @@ -72,14 +72,16 @@ export default class IngressService extends BaseService { const configurations = await this.db.services.BuildService.configurationsForBuildId(buildId, true); const namespace = await this.db.services.BuildService.getNamespace({ id: buildId }); try { - configurations.forEach(async (configuration) => { - await shellPromise( - `kubectl delete ingress ingress-${configuration.deployUUID} --namespace ${namespace}` - ).catch((error) => { - getLogger({ stage: LogStage.INGRESS_PROCESSING }).warn(`${error}`); - return null; - }); - }); + await Promise.all( + configurations.map((configuration) => + shellPromise(`kubectl delete ingress ingress-${configuration.deployUUID} --namespace ${namespace}`).catch( + (error) => { + getLogger({ stage: LogStage.INGRESS_PROCESSING }).warn(`${error}`); + return null; + } + ) + ) + ); getLogger({ stage: LogStage.INGRESS_COMPLETE }).info('Ingress: cleaned up'); } catch (e) { getLogger({ stage: LogStage.INGRESS_FAILED }).warn({ error: e }, 'Ingress: cleanup failed'); @@ -88,9 +90,19 @@ export default class IngressService extends BaseService { }; createOrUpdateIngressForBuild = async (job) => { - const { buildId, buildUuid, sender, correlationId, _ddTraceContext } = job.data; + const { buildId, buildUuid, runUUID, expectedGeneration, sender, correlationId, _ddTraceContext } = job.data; return withLogContext({ correlationId, buildUuid, sender, _ddTraceContext }, async () => { + const isCurrent = async () => { + if (!runUUID) return true; + let authority = this.db.models.Build.query().findOne({ id: buildId, runUUID }).whereNull('deletedAt'); + if (expectedGeneration != null) authority = authority.where('desiredGeneration', expectedGeneration); + return Boolean(await authority); + }; + if (!(await isCurrent())) { + getLogger({ buildId }).info('Ingress: skipped reason=superseded'); + return; + } getLogger({ stage: LogStage.INGRESS_PROCESSING }).info('Ingress: creating'); // We just want to create/update ingress for active services only @@ -109,9 +121,22 @@ export default class IngressService extends BaseService { } ); }); - manifests.forEach(async (manifest, idx) => { - await this.applyManifests(manifest, `${buildId}-${idx}-nginx`, namespace, buildId); - }); + const apply = () => + Promise.all( + manifests.map((manifest, idx) => + this.applyManifests(manifest, `${buildId}-${idx}-nginx`, namespace, buildId, runUUID, expectedGeneration) + ) + ); + + if (runUUID && this.db.services.BuildService.withCurrentBuildPromotionLock) { + const result = await this.db.services.BuildService.withCurrentBuildPromotionLock(buildId, isCurrent, apply); + if (!result.admitted) { + getLogger({ buildId }).info('Ingress: skipped at promotion reason=superseded'); + return; + } + } else { + await apply(); + } getLogger({ stage: LogStage.INGRESS_COMPLETE }).info('Ingress: created'); }); @@ -192,32 +217,52 @@ export default class IngressService extends BaseService { * @param manifest the manifest to apply * @param ingressName a name for the manifest for tmp directory namespacing */ - private applyManifests = async (manifest, ingressName, namespace: string, buildId?: number) => { + private applyManifests = async ( + manifest, + ingressName, + namespace: string, + buildId?: number, + runUUID?: string, + expectedGeneration?: number + ) => { try { const localPath = `${MANIFEST_PATH}/global-ingress/${ingressName}-ingress.yaml`; await fs.promises.mkdir(`${MANIFEST_PATH}/global-ingress/`, { recursive: true, }); await fs.promises.writeFile(localPath, manifest, 'utf8'); - await shellPromise(`kubectl apply -f ${localPath} --namespace ${namespace}`); + await shellPromise(`kubectl apply -f ${localPath} --namespace ${namespace}`, { + timeout: 90_000, + }); } catch (error) { getLogger({ stage: LogStage.INGRESS_FAILED }).warn({ error }, 'Ingress: manifest apply failed'); if (buildId !== undefined) { - await this.recordIngressFailureOnBuild(buildId, error).catch(() => undefined); + await this.recordIngressFailureOnBuild(buildId, error, runUUID, expectedGeneration).catch(() => undefined); } } }; // Surface broken routing in the build row; the build otherwise reports deployed with no DB trace of the failure. - private recordIngressFailureOnBuild = async (buildId: number, error: unknown) => { + private recordIngressFailureOnBuild = async ( + buildId: number, + error: unknown, + runUUID?: string, + expectedGeneration?: number + ) => { const note = `Ingress apply failed: ${(error as Error)?.message || String(error)}` .replace(/\s+/g, ' ') .slice(0, 300); - const build = await this.db.models.Build.query().findById(buildId); + let buildRead = this.db.models.Build.query().findById(buildId).whereNull('deletedAt'); + if (runUUID) buildRead = buildRead.where('runUUID', runUUID); + if (expectedGeneration != null) buildRead = buildRead.where('desiredGeneration', expectedGeneration); + const build = await buildRead; if (!build || build.statusMessage?.includes(note)) { return; } const statusMessage = [build.statusMessage, note].filter(Boolean).join(' | ').slice(-500); - await build.$query().patch({ statusMessage }); + let buildPatch = this.db.models.Build.query().patch({ statusMessage }).where({ id: buildId }); + if (runUUID) buildPatch = buildPatch.where('runUUID', runUUID); + if (expectedGeneration != null) buildPatch = buildPatch.where('desiredGeneration', expectedGeneration); + await buildPatch; }; } diff --git a/src/server/services/override.ts b/src/server/services/override.ts index 6bcf90f8..a0403501 100644 --- a/src/server/services/override.ts +++ b/src/server/services/override.ts @@ -15,7 +15,7 @@ */ import BaseService from './_service'; -import { extractContextForQueue, getLogger, updateLogContext } from 'server/lib/logger'; +import { getLogger, updateLogContext } from 'server/lib/logger'; import { isDeployEnabled } from 'server/lib/buildSource'; import { validateBuildUuidFormat } from 'server/lib/validation/buildUuidValidator'; import { Build, Deploy, PullRequest } from 'server/models'; @@ -528,9 +528,6 @@ export default class OverrideService extends BaseService { await buildService.enqueueResolveAndDeployBuild({ buildId: build.id, runUUID: runUuid, - // Use the unique run id as the trigger so an explicit redeploy is never coalesced into a prior deploy. - triggerRef: runUuid, - ...extractContextForQueue(), }); return true; } diff --git a/src/server/services/webhook.ts b/src/server/services/webhook.ts index 402401ca..2c8e3624 100644 --- a/src/server/services/webhook.ts +++ b/src/server/services/webhook.ts @@ -69,8 +69,7 @@ export default class WebhookService extends BaseService { } if (sourceRepository != null && sourceBranchName != null) { - const sourceConfigRef = - build.triggerType === 'api' ? sourceRef ?? build.configSha ?? sourceBranchName : sourceBranchName; + const sourceConfigRef = sourceRef ?? (build.triggerType === 'api' ? build.configSha : null) ?? sourceBranchName; const yamlConfig: YamlService.LifecycleConfig = await YamlService.fetchLifecycleConfigByRepository( sourceRepository, sourceConfigRef diff --git a/src/shared/config.ts b/src/shared/config.ts index 37d5e60e..9fcf93b4 100644 --- a/src/shared/config.ts +++ b/src/shared/config.ts @@ -109,6 +109,7 @@ export const QUEUE_NAMES = { API_TOKEN_OWNER_SWEEP: 'api_token_owner_sweep', RESOLVE_AND_DEPLOY: `resolve_and_deploy_${JOB_VERSION}`, BUILD_QUEUE: `build_queue_${JOB_VERSION}`, + DEPLOYMENT_RECONCILIATION: `deployment_reconciliation_${JOB_VERSION}`, GITHUB_DEPLOYMENT: `github_deployment_${JOB_VERSION}`, DEPLOY_CLEANUP: `deploy_cleanup_${JOB_VERSION}`, LABEL: `label_${JOB_VERSION}`, diff --git a/src/shared/openApiSpec.ts b/src/shared/openApiSpec.ts index 4060cd48..99af817e 100644 --- a/src/shared/openApiSpec.ts +++ b/src/shared/openApiSpec.ts @@ -4970,7 +4970,9 @@ export const openApiSpecificationForV2Api: OAS3Options = { name: { type: 'string', example: 'web' }, type: { type: 'string', enum: Object.values(DeployTypes) }, dockerfilePath: { type: 'string', example: 'Dockerfile' }, - deploymentDependsOn: { type: 'string', example: '{redis}' }, + requires: { type: 'array', items: { type: 'string' }, example: ['postgres', 'redis'] }, + deploymentDependsOn: { type: 'array', items: { type: 'string' }, example: ['redis'] }, + dependsOnDeployableName: { type: 'string', nullable: true, example: 'web' }, builder: { type: 'object', properties: { @@ -4981,7 +4983,7 @@ export const openApiSpecificationForV2Api: OAS3Options = { grpc: { type: 'boolean', example: true }, hostPortMapping: { type: 'object', example: { '80': 8080 } }, }, - required: ['name', 'type', 'dockerfilePath', 'deploymentDependsOn', 'builder', 'ecr'], + required: ['name', 'type', 'dockerfilePath', 'requires', 'deploymentDependsOn', 'builder', 'ecr'], }, /**