feat(ci): promote Trigger.dev tasks in lockstep with the ECS traffic cutover

.github/scripts/wait-for-ecs-cutover.sh

@@ -0,0 +1,146 @@
+#!/usr/bin/env bash
+# Waits for the ECS blue/green deploy triggered by a specific app image push to
+# reach its traffic cutover (CodeDeploy AllowTraffic == Succeeded on every ECS
+# target), then exits 0.
+#
+# ECR app images use a floating tag (latest/staging) with no git SHA, so the
+# only durable key linking this CI push to its ECS deploy is the image DIGEST.
+# Correlation: image digest -> CodePipeline execution (ECR_Source revision) ->
+# Deploy action externalExecutionId (== CodeDeploy deployment id) -> AllowTraffic.
+#
+# The digest alone is ambiguous: a prior run with the same image could match an
+# older, already-cutover execution and promote too early. SINCE_EPOCH (the time
+# the deploy tag was retagged, i.e. when THIS push's pipeline was triggered)
+# disambiguates — only an execution that started at/after the retag is ours.
+#
+# Usage: wait-for-ecs-cutover.sh <pipeline-name> <image-digest> <since-epoch>
+# Requires: awscli v2, python3, credentials with codedeploy + codepipeline read.
+set -euo pipefail
+
+PIPELINE="${1:?pipeline name required}"
+DIGEST="${2:?image digest required}"
+SINCE_EPOCH="${3:?since-epoch (retag time) required}"
+
+POLL_INTERVAL="${POLL_INTERVAL:-15}"
+# 70 min covers a prod deploy whose Deploy stage is queued behind a prior
+# deploy's ~50-min termination bake before its own traffic shift begins.
+OVERALL_TIMEOUT="${OVERALL_TIMEOUT:-4200}"
+# Tolerate minor clock skew between the runner (retag time) and CodePipeline.
+SINCE_SKEW="${SINCE_SKEW:-120}"
+
+deadline=$(( $(date +%s) + OVERALL_TIMEOUT ))
+remaining() { echo $(( deadline - $(date +%s) )); }
+log() { echo "[wait-for-ecs-cutover] $*"; }
+fail_if_expired() {
+  if [ "$(remaining)" -le 0 ]; then
+    log "ERROR: timed out after ${OVERALL_TIMEOUT}s waiting for: $1"
+    exit 1
+  fi
+}
+
+log "Pipeline: $PIPELINE"
+log "Target app image digest: $DIGEST"
+log "Requiring execution started at/after epoch $SINCE_EPOCH (minus ${SINCE_SKEW}s skew)"
+
+# Phase A: find the newest pipeline execution whose ECR source revision matches
+# our digest AND that started at/after the retag. The since filter rejects a
+# stale historical execution reusing the same digest. --max-items bounds the
+# fetch (the CLI otherwise auto-paginates the whole history).
+EXECUTION_ID=""
+while [ -z "$EXECUTION_ID" ]; do
+  fail_if_expired "pipeline execution matching digest since retag"
+  matches=$(aws codepipeline list-pipeline-executions \
+        --pipeline-name "$PIPELINE" --max-items 30 \
+        --query "pipelineExecutionSummaries[?sourceRevisions[?actionName=='ECR_Source' && revisionId=='$DIGEST']].[startTime, pipelineExecutionId]" \
+        --output text 2>/dev/null || true)
+  EXECUTION_ID=$(printf '%s\n' "$matches" | SINCE="$SINCE_EPOCH" SKEW="$SINCE_SKEW" python3 -c '
+import sys, os, datetime
+since = float(os.environ["SINCE"]) - float(os.environ["SKEW"])
+best_epoch = None
+best_id = None
+for line in sys.stdin:
+    parts = line.rstrip("\n").split("\t")
+    if len(parts) < 2:
+        continue
+    ts, eid = parts[0].strip(), parts[1].strip()
+    if not ts or not eid or "-" not in eid:
+        continue
+    try:
+        epoch = float(ts)
+    except ValueError:
+        try:
+            epoch = datetime.datetime.fromisoformat(ts.replace("Z", "+00:00")).timestamp()
+        except ValueError:
+            continue
+    if epoch >= since and (best_epoch is None or epoch > best_epoch):
+        best_epoch, best_id = epoch, eid
+print(best_id or "")
+' 2>/dev/null || true)
+  if [ -z "$EXECUTION_ID" ]; then
+    log "No matching post-retag pipeline execution yet; retry in ${POLL_INTERVAL}s (remaining $(remaining)s)"
+    sleep "$POLL_INTERVAL"
+  fi
+done
+log "Matched pipeline execution: $EXECUTION_ID"
+
+# Phase B: resolve the CodeDeploy deployment id from the Deploy action. This may
+# stay empty for a while if the Deploy stage is queued behind a prior deploy.
+DEPLOYMENT_ID=""
+while [ -z "$DEPLOYMENT_ID" ] || [ "$DEPLOYMENT_ID" = "None" ]; do
+  fail_if_expired "CodeDeploy deployment id (Deploy stage may be queued behind a prior deploy's bake)"
+  status=$(aws codepipeline get-pipeline-execution \
+        --pipeline-name "$PIPELINE" --pipeline-execution-id "$EXECUTION_ID" \
+        --query 'pipelineExecution.status' --output text 2>/dev/null || true)
+  case "$status" in
+    Failed|Stopped|Superseded)
+      log "ERROR: pipeline execution $EXECUTION_ID ended in status $status before deploy"
+      exit 1
+      ;;
+  esac
+  DEPLOYMENT_ID=$(aws codepipeline list-action-executions \
+        --pipeline-name "$PIPELINE" \
+        --filter pipelineExecutionId="$EXECUTION_ID" \
+        --query "actionExecutionDetails[?stageName=='Deploy'].output.executionResult.externalExecutionId | [0]" \
+        --output text 2>/dev/null || true)
+  if [ -z "$DEPLOYMENT_ID" ] || [ "$DEPLOYMENT_ID" = "None" ]; then
+    log "Deploy stage not started yet (pipeline status: $status); retry in ${POLL_INTERVAL}s (remaining $(remaining)s)"
+    sleep "$POLL_INTERVAL"
+  fi
+done
+log "CodeDeploy deployment: $DEPLOYMENT_ID"
+
+# Phase C: wait for the traffic cutover. Require AllowTraffic == Succeeded on
+# EVERY ECS target, so a multi-target deploy can't promote while one target is
+# still mid-cutover or failed.
+while true; do
+  fail_if_expired "AllowTraffic (traffic cutover) on all targets"
+  dstatus=$(aws deploy get-deployment --deployment-id "$DEPLOYMENT_ID" \
+        --query 'deploymentInfo.status' --output text 2>/dev/null || true)
+  case "$dstatus" in
+    Failed|Stopped)
+      log "ERROR: CodeDeploy deployment $DEPLOYMENT_ID ended in status $dstatus; not promoting"
+      exit 1
+      ;;
+  esac
+  target_ids=$(aws deploy list-deployment-targets --deployment-id "$DEPLOYMENT_ID" \
+        --query 'targetIds' --output text 2>/dev/null || true)
+  if [ -n "$target_ids" ] && [ "$target_ids" != "None" ]; then
+    all_ok=1
+    ntargets=0
+    for tid in $target_ids; do
+      ntargets=$((ntargets + 1))
+      at_status=$(aws deploy get-deployment-target --deployment-id "$DEPLOYMENT_ID" --target-id "$tid" \
+            --query "deploymentTarget.ecsTarget.lifecycleEvents[?lifecycleEventName=='AllowTraffic'].status | [0]" \
+            --output text 2>/dev/null || true)
+      if [ "$at_status" != "Succeeded" ]; then
+        all_ok=0
+      fi
+    done
+    if [ "$ntargets" -gt 0 ] && [ "$all_ok" = "1" ]; then
+      log "Traffic cutover complete (AllowTraffic Succeeded on all $ntargets target(s)) for $DEPLOYMENT_ID"
+      exit 0
+    fi
+  fi
+  log "Deployment $DEPLOYMENT_ID status=$dstatus; not all targets past AllowTraffic; wait ${POLL_INTERVAL}s (remaining $(remaining)s)"
+  sleep "$POLL_INTERVAL"
+done

.github/workflows/ci.yml

@@ -182,6 +182,69 @@ jobs:
           fi
           bunx trigger.dev@4.4.3 deploy --env preview --branch dev-sim
 
+  # Main/staging: build & upload the Trigger.dev task version WITHOUT promoting it
+  # (--skip-promotion). New runs keep executing the OLD promoted version until
+  # promote-trigger flips it at the ECS traffic cutover — so the app cutting over
+  # never changes which task version runs until promote-trigger (which depends on
+  # this job) promotes the version uploaded here. Runs in parallel with the build;
+  # intentionally NOT gating the app deploy on it, to avoid coupling every app /
+  # realtime / pii / migration deploy to trigger.dev availability.
+  deploy-trigger:
+    name: Deploy Trigger.dev
+    needs: [migrate]
+    if: >-
+      github.event_name == 'push' &&
+      (github.ref == 'refs/heads/main' || github.ref == 'refs/heads/staging')
+    runs-on: blacksmith-4vcpu-ubuntu-2404
+    timeout-minutes: 15
+    outputs:
+      version: ${{ steps.deploy.outputs.version }}
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6
+
+      - name: Setup Bun
+        uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2
+        with:
+          bun-version: 1.3.13
+
+      - name: Cache Bun dependencies
+        uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5
+        with:
+          path: |
+            ~/.bun/install/cache
+            node_modules
+            **/node_modules
+          key: ${{ runner.os }}-bun-${{ hashFiles('**/bun.lock') }}
+          restore-keys: |
+            ${{ runner.os }}-bun-
+
+      - name: Install dependencies
+        run: bun install --frozen-lockfile
+
+      - name: Deploy to Trigger.dev (skip promotion)
+        id: deploy
+        working-directory: ./apps/sim
+        env:
+          TRIGGER_ACCESS_TOKEN: ${{ secrets.TRIGGER_ACCESS_TOKEN }}
+          TRIGGER_PROJECT_ID: ${{ secrets.TRIGGER_PROJECT_ID }}
+          TRIGGER_ENV: ${{ github.ref == 'refs/heads/main' && 'prod' || 'staging' }}
+        run: |
+          set -eo pipefail
+          if [ -z "$TRIGGER_ACCESS_TOKEN" ] || [ -z "$TRIGGER_PROJECT_ID" ]; then
+            echo "ERROR: TRIGGER_ACCESS_TOKEN and TRIGGER_PROJECT_ID repo secrets must both be set" >&2
+            exit 1
+          fi
+          bunx trigger.dev@4.4.3 deploy --env "$TRIGGER_ENV" --skip-promotion 2>&1 | tee deploy.log
+          # Extract the deployed version (e.g. 20260715.2) tied to THIS invocation.
+          VERSION=$(sed -E 's/\x1b\[[0-9;]*m//g' deploy.log | grep -oE '20[0-9]{6}\.[0-9]+' | tail -n1 || true)
+          if [ -z "$VERSION" ]; then
+            echo "ERROR: could not parse deployed version from deploy output" >&2
+            exit 1
+          fi
+          echo "Captured deployed version: $VERSION"
+          echo "version=$VERSION" >> "$GITHUB_OUTPUT"
+
   # Main/staging: build AMD64 images and push sha-tagged images to ECR + GHCR.
   # Runs in parallel with tests — only immutable sha tags are pushed here, and
   # the CodePipeline EventBridge triggers filter on exactly the
@@ -270,6 +333,7 @@ jobs:
           echo "tags=${TAGS}" >> $GITHUB_OUTPUT
 
       - name: Build and push images
+        id: build
         uses: useblacksmith/build-push-action@fb9e3e6a9299c78462bfadd0d93352c316adc9b8 # v2
         with:
           context: .
@@ -280,6 +344,25 @@ jobs:
           provenance: false
           sbom: false
 
+      # Publish the app image digest so promote-trigger-* can correlate this push
+      # to its ECS CodePipeline execution. promote-images retags this same sha
+      # image to latest/staging (preserving the digest), so the pipeline's ECR
+      # source revision equals this digest — the only durable key (the deploy tag
+      # is floating). App leg only.
+      - name: Publish app image digest
+        if: matrix.ecr_repo_secret == 'ECR_APP'
+        run: |
+          mkdir -p digest
+          echo "${{ steps.build.outputs.digest }}" > digest/app-image-digest.txt
+
+      - name: Upload app image digest
+        if: matrix.ecr_repo_secret == 'ECR_APP'
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
+        with:
+          name: app-image-digest
+          path: digest/app-image-digest.txt
+          retention-days: 1
+
   # Promote the sha-tagged ECR images to the deploy tags once tests and
   # migrations pass. Pushing the ECR latest/staging tag is what triggers
   # CodePipeline, so this seconds-long manifest retag is the deploy gate —
@@ -297,6 +380,15 @@ jobs:
     permissions:
       contents: read
       id-token: write
+    outputs:
+      # Whether the deploy tag was actually moved (false on a stale-run guard
+      # skip). promote-trigger keys off this so tasks are never promoted when
+      # the app itself wasn't.
+      promoted: ${{ steps.guard.outputs.fresh }}
+      # Epoch when the deploy tag was retagged (this push's ECS pipeline trigger).
+      # promote-trigger passes it to the poll script so a stale pipeline execution
+      # reusing the same image digest can't satisfy the cutover gate.
+      retag_epoch: ${{ steps.promote.outputs.retag_epoch }}
     steps:
       - name: Configure AWS credentials
         uses: aws-actions/configure-aws-credentials@e7f100cf4c008499ea8adda475de1042d6975c7b # v6
@@ -326,6 +418,7 @@ jobs:
           fi
 
       - name: Promote images to deploy tags
+        id: promote
         if: steps.guard.outputs.fresh == 'true'
         env:
           ECR_REPOS: >-
@@ -334,6 +427,11 @@ jobs:
             ${{ secrets.ECR_REALTIME }}
             ${{ secrets.ECR_PII }}
         run: |
+          # Record the retag time BEFORE moving any tag — this is when the ECS
+          # pipeline for this push is triggered. promote-trigger uses it to
+          # reject an older pipeline execution reusing the same image digest.
+          echo "retag_epoch=$(date +%s)" >> "$GITHUB_OUTPUT"
+
           REGISTRY="${{ steps.login-ecr.outputs.registry }}"
 
           if [ "${{ github.ref }}" = "refs/heads/main" ]; then
@@ -356,6 +454,93 @@ jobs:
               "${REGISTRY}/${repo}:${{ github.sha }}"
           done
 
+  # Main/staging: promote the skip-promoted Trigger.dev version at the exact moment
+  # the ECS app deploy shifts traffic (CodeDeploy AllowTraffic on every target), so
+  # tasks and app cut over in lockstep. The promote-images retag is what triggers
+  # the ECS pipeline; this job correlates it via the app image digest + retag epoch
+  # (rejecting a stale execution reusing the digest) and promotes at cutover.
+  # Skipped when promote-images skipped the tag move (stale run) — tasks then
+  # correctly stay on the old version. If the app deploy fails or never cuts over,
+  # promote never fires and this job fails visibly.
+  promote-trigger:
+    name: Promote Trigger.dev
+    needs: [promote-images, deploy-trigger]
+    if: >-
+      github.event_name == 'push' &&
+      (github.ref == 'refs/heads/main' || github.ref == 'refs/heads/staging') &&
+      needs.promote-images.outputs.promoted == 'true'
+    runs-on: blacksmith-4vcpu-ubuntu-2404
+    # Must exceed the poll script's OVERALL_TIMEOUT (70 min, covering a prod deploy
+    # queued behind a ~50-min bake) PLUS runner setup + the final promote step, so
+    # the Actions timeout never kills the job before the script's own deadline.
+    timeout-minutes: 90
+    permissions:
+      contents: read
+      id-token: write
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6
+
+      - name: Setup Bun
+        uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2
+        with:
+          bun-version: 1.3.13
+
+      - name: Cache Bun dependencies
+        uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5
+        with:
+          path: |
+            ~/.bun/install/cache
+            node_modules
+            **/node_modules
+          key: ${{ runner.os }}-bun-${{ hashFiles('**/bun.lock') }}
+          restore-keys: |
+            ${{ runner.os }}-bun-
+
+      - name: Install dependencies
+        run: bun install --frozen-lockfile
+
+      - name: Download app image digest
+        uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4
+        with:
+          name: app-image-digest
+          path: digest
+
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@e7f100cf4c008499ea8adda475de1042d6975c7b # v6
+        with:
+          role-to-assume: ${{ github.ref == 'refs/heads/main' && secrets.AWS_ROLE_TO_ASSUME || secrets.STAGING_AWS_ROLE_TO_ASSUME }}
+          aws-region: ${{ github.ref == 'refs/heads/main' && secrets.AWS_REGION || secrets.STAGING_AWS_REGION }}
+
+      - name: Wait for ECS traffic cutover
+        env:
+          PIPELINE: sim-${{ github.ref == 'refs/heads/main' && 'production' || 'staging' }}-us-east-1-app-deployment
+          RETAG_EPOCH: ${{ needs.promote-images.outputs.retag_epoch }}
+        run: |
+          set -eo pipefail
+          DIGEST=$(cat digest/app-image-digest.txt)
+          bash .github/scripts/wait-for-ecs-cutover.sh "$PIPELINE" "$DIGEST" "$RETAG_EPOCH"
+
+      - name: Promote Trigger.dev version
+        working-directory: ./apps/sim
+        env:
+          TRIGGER_ACCESS_TOKEN: ${{ secrets.TRIGGER_ACCESS_TOKEN }}
+          TRIGGER_PROJECT_ID: ${{ secrets.TRIGGER_PROJECT_ID }}
+          TRIGGER_ENV: ${{ github.ref == 'refs/heads/main' && 'prod' || 'staging' }}
+          VERSION: ${{ needs.deploy-trigger.outputs.version }}
+        run: |
+          set -eo pipefail
+          if [ -z "$TRIGGER_ACCESS_TOKEN" ] || [ -z "$TRIGGER_PROJECT_ID" ]; then
+            echo "ERROR: TRIGGER_ACCESS_TOKEN and TRIGGER_PROJECT_ID repo secrets must both be set" >&2
+            exit 1
+          fi
+          if [ -z "$VERSION" ]; then
+            echo "ERROR: no deployed version passed from deploy-trigger" >&2
+            exit 1
+          fi
+          echo "Promoting Trigger.dev version $VERSION ($TRIGGER_ENV) at ECS cutover"
+          bunx trigger.dev@4.4.3 promote "$VERSION" --env "$TRIGGER_ENV"
+
   # Build ARM64 images for GHCR (main branch only, runs in parallel with
   # tests). Pushes only the immutable sha tag — latest-arm64/version-arm64
   # are applied by create-ghcr-manifests after the gate, so a failing run