From 5894a60d18e72cd238b3d5e7cd652ddf04812d81 Mon Sep 17 00:00:00 2001 From: Milan Garnier Date: Wed, 24 Jun 2026 14:57:42 +0200 Subject: [PATCH 01/12] add new job to publish CI builds to ghcr.io --- .gitlab/generate-package.php | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/.gitlab/generate-package.php b/.gitlab/generate-package.php index 2272b5fc7f..1582eefb6e 100644 --- a/.gitlab/generate-package.php +++ b/.gitlab/generate-package.php @@ -1515,6 +1515,39 @@ paths: - packages/datadog-setup.php +"publish docker image for system tests": + stage: release + image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/docker:29.4.0-noble + tags: [ "docker-in-docker:amd64" ] + rules: + - when: on_success + needs: + - job: "datadog-setup.php" + artifacts: true + - job: "package extension: [x86_64, x86_64-unknown-linux-gnu]" + artifacts: true + id_tokens: + DDOCTOSTS_ID_TOKEN: + aud: dd-octo-sts + variables: + GIT_STRATEGY: none + script: | + set -e + NORMALIZED_BRANCH=$(echo "$CI_COMMIT_REF_NAME" | sed 's/\//_/g') + IMAGE="ghcr.io/datadog/dd-trace-php/dd-library-php:${NORMALIZED_BRANCH}" + + GITHUB_TOKEN=$(dd-octo-sts token \ + --scope DataDog/dd-trace-php \ + --policy gitlab-ci-publish-packages) + echo "$GITHUB_TOKEN" | docker login ghcr.io \ + -u DataDog --password-stdin + + printf 'FROM scratch\nCOPY packages/dd-library-php-*-x86_64-linux-gnu.tar.gz /\nCOPY packages/datadog-setup.php /\n' \ + > Dockerfile.system-tests + docker build -f Dockerfile.system-tests -t "$IMAGE" . + docker push "$IMAGE" + echo "Pushed $IMAGE" + "bundle for reliability env": stage: shared-pipeline image: registry.ddbuild.io/ci/libdatadog-build/ci_docker_base:67145216 From 8631e32041a118f441fad12ed09014ec087ef4eb Mon Sep 17 00:00:00 2001 From: Milan Garnier Date: Wed, 24 Jun 2026 15:43:21 +0200 Subject: [PATCH 02/12] fix base image name --- .gitlab/generate-package.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab/generate-package.php b/.gitlab/generate-package.php index 1582eefb6e..b571cdae0e 100644 --- a/.gitlab/generate-package.php +++ b/.gitlab/generate-package.php @@ -1524,7 +1524,7 @@ needs: - job: "datadog-setup.php" artifacts: true - - job: "package extension: [x86_64, x86_64-unknown-linux-gnu]" + - job: "package extension: [amd64, x86_64-unknown-linux-gnu]" artifacts: true id_tokens: DDOCTOSTS_ID_TOKEN: From 1032cd9dbd623f8184f287b0c6ece0ecf1ca3339 Mon Sep 17 00:00:00 2001 From: Milan Garnier Date: Fri, 26 Jun 2026 10:46:16 +0200 Subject: [PATCH 03/12] publish in two steps --- .gitlab/generate-package.php | 31 ++++++++++++++++++++++++------- 1 file changed, 24 insertions(+), 7 deletions(-) diff --git a/.gitlab/generate-package.php b/.gitlab/generate-package.php index 3aca3c4f78..8adbac1ba8 100644 --- a/.gitlab/generate-package.php +++ b/.gitlab/generate-package.php @@ -1518,6 +1518,27 @@ paths: - packages/datadog-setup.php +"publish docker image for system tests (token)": + stage: release + image: registry.ddbuild.io/images/dd-octo-sts-ci-base:2025.06-1 + tags: [ "arch:amd64" ] + rules: + - when: on_success + id_tokens: + DDOCTOSTS_ID_TOKEN: + aud: dd-octo-sts + variables: + GIT_STRATEGY: none + script: + - dd-octo-sts token --scope DataDog/dd-trace-php --policy gitlab-ci-publish-packages > github_token_system_tests.txt + after_script: + - if [[ -f github_token_system_tests.txt ]]; then dd-octo-sts revoke -t "$(cat github_token_system_tests.txt)" || true; fi + artifacts: + paths: + - github_token_system_tests.txt + expire_in: 1 hour + when: on_success + "publish docker image for system tests": stage: release image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/docker:29.4.0-noble @@ -1525,13 +1546,12 @@ rules: - when: on_success needs: + - job: "publish docker image for system tests (token)" + artifacts: true - job: "datadog-setup.php" artifacts: true - job: "package extension: [amd64, x86_64-unknown-linux-gnu]" artifacts: true - id_tokens: - DDOCTOSTS_ID_TOKEN: - aud: dd-octo-sts variables: GIT_STRATEGY: none script: | @@ -1539,10 +1559,7 @@ NORMALIZED_BRANCH=$(echo "$CI_COMMIT_REF_NAME" | sed 's/\//_/g') IMAGE="ghcr.io/datadog/dd-trace-php/dd-library-php:${NORMALIZED_BRANCH}" - GITHUB_TOKEN=$(dd-octo-sts token \ - --scope DataDog/dd-trace-php \ - --policy gitlab-ci-publish-packages) - echo "$GITHUB_TOKEN" | docker login ghcr.io \ + cat github_token_system_tests.txt | docker login ghcr.io \ -u DataDog --password-stdin printf 'FROM scratch\nCOPY packages/dd-library-php-*-x86_64-linux-gnu.tar.gz /\nCOPY packages/datadog-setup.php /\n' \ From e7401c8d7647b4184544a77d16f8822ec813ef3d Mon Sep 17 00:00:00 2001 From: Milan Garnier Date: Fri, 26 Jun 2026 11:17:27 +0200 Subject: [PATCH 04/12] fix(ci): fix token lifecycle and mask github token in system tests publish jobs - Remove after_script revocation from the token job: GitLab uploads artifacts after after_script, so the token was being revoked before the downstream job could use it. The 1-hour artifact expiry is sufficient, matching the pattern in "generate github token". - Add GITHUB_TOKEN: "[MASKED]" to prevent the token value from appearing in CI logs. - Add comment documenting that the jobs intentionally run on every branch (for ad-hoc system test runs against in-progress branches). Co-Authored-By: Claude Sonnet 4.6 --- .gitlab/generate-package.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab/generate-package.php b/.gitlab/generate-package.php index 8adbac1ba8..5d102ae25c 100644 --- a/.gitlab/generate-package.php +++ b/.gitlab/generate-package.php @@ -1518,6 +1518,7 @@ paths: - packages/datadog-setup.php +# Runs on every branch so system tests can be run against any in-progress branch. "publish docker image for system tests (token)": stage: release image: registry.ddbuild.io/images/dd-octo-sts-ci-base:2025.06-1 @@ -1529,10 +1530,9 @@ aud: dd-octo-sts variables: GIT_STRATEGY: none + GITHUB_TOKEN: "[MASKED]" script: - dd-octo-sts token --scope DataDog/dd-trace-php --policy gitlab-ci-publish-packages > github_token_system_tests.txt - after_script: - - if [[ -f github_token_system_tests.txt ]]; then dd-octo-sts revoke -t "$(cat github_token_system_tests.txt)" || true; fi artifacts: paths: - github_token_system_tests.txt From 34c8a7f215e3cc9db5fb9003100a59191c2778d3 Mon Sep 17 00:00:00 2001 From: Milan Garnier Date: Fri, 26 Jun 2026 11:36:03 +0200 Subject: [PATCH 05/12] restrict CI uploads to Datadog/dd-trace-php --- .gitlab/generate-package.php | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/.gitlab/generate-package.php b/.gitlab/generate-package.php index 5d102ae25c..db16063a4b 100644 --- a/.gitlab/generate-package.php +++ b/.gitlab/generate-package.php @@ -1523,8 +1523,6 @@ stage: release image: registry.ddbuild.io/images/dd-octo-sts-ci-base:2025.06-1 tags: [ "arch:amd64" ] - rules: - - when: on_success id_tokens: DDOCTOSTS_ID_TOKEN: aud: dd-octo-sts @@ -1543,8 +1541,6 @@ stage: release image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/docker:29.4.0-noble tags: [ "docker-in-docker:amd64" ] - rules: - - when: on_success needs: - job: "publish docker image for system tests (token)" artifacts: true @@ -1567,6 +1563,8 @@ docker build -f Dockerfile.system-tests -t "$IMAGE" . docker push "$IMAGE" echo "Pushed $IMAGE" + after_script: + - rm -f github_token_system_tests.txt "bundle for reliability env": stage: shared-pipeline From f87131d0f851be56d6a303d27d5e412ed6de288c Mon Sep 17 00:00:00 2001 From: Milan Garnier Date: Fri, 26 Jun 2026 11:37:58 +0200 Subject: [PATCH 06/12] cleanup github token handling --- .gitlab/generate-package.php | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/.gitlab/generate-package.php b/.gitlab/generate-package.php index db16063a4b..e287084474 100644 --- a/.gitlab/generate-package.php +++ b/.gitlab/generate-package.php @@ -1528,7 +1528,6 @@ aud: dd-octo-sts variables: GIT_STRATEGY: none - GITHUB_TOKEN: "[MASKED]" script: - dd-octo-sts token --scope DataDog/dd-trace-php --policy gitlab-ci-publish-packages > github_token_system_tests.txt artifacts: @@ -1552,11 +1551,9 @@ GIT_STRATEGY: none script: | set -e - NORMALIZED_BRANCH=$(echo "$CI_COMMIT_REF_NAME" | sed 's/\//_/g') - IMAGE="ghcr.io/datadog/dd-trace-php/dd-library-php:${NORMALIZED_BRANCH}" + IMAGE="ghcr.io/datadog/dd-trace-php/dd-library-php:${CI_COMMIT_REF_SLUG}" - cat github_token_system_tests.txt | docker login ghcr.io \ - -u DataDog --password-stdin + docker login ghcr.io -u DataDog --password-stdin < github_token_system_tests.txt printf 'FROM scratch\nCOPY packages/dd-library-php-*-x86_64-linux-gnu.tar.gz /\nCOPY packages/datadog-setup.php /\n' \ > Dockerfile.system-tests From 495dcd14b8645a6059f1ff2d66a56b22ee5a4cb5 Mon Sep 17 00:00:00 2001 From: Milan Garnier Date: Tue, 30 Jun 2026 13:44:41 +0200 Subject: [PATCH 07/12] add policy file --- .github/chainguard/gitlab-ci-publish-packages.sts.yaml | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 .github/chainguard/gitlab-ci-publish-packages.sts.yaml diff --git a/.github/chainguard/gitlab-ci-publish-packages.sts.yaml b/.github/chainguard/gitlab-ci-publish-packages.sts.yaml new file mode 100644 index 0000000000..04173d825b --- /dev/null +++ b/.github/chainguard/gitlab-ci-publish-packages.sts.yaml @@ -0,0 +1,6 @@ +issuer: https://gitlab.ddbuild.io + +subject_pattern: "project_path:DataDog/apm-reliability/dd-trace-php:ref_type:(branch|tag):ref:.*" + +permissions: + packages: write From 1cbb0107c1dd429fce6aac108e14e30ed2d03b2a Mon Sep 17 00:00:00 2001 From: Milan Garnier Date: Thu, 2 Jul 2026 17:46:14 +0200 Subject: [PATCH 08/12] add a workflow to delete pr images when the pr is closed --- .github/workflows/delete-pr-image.yml | 31 +++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) create mode 100644 .github/workflows/delete-pr-image.yml diff --git a/.github/workflows/delete-pr-image.yml b/.github/workflows/delete-pr-image.yml new file mode 100644 index 0000000000..230c9983ea --- /dev/null +++ b/.github/workflows/delete-pr-image.yml @@ -0,0 +1,31 @@ +name: "Delete Docker image on PR close" + +on: + pull_request: + types: [closed] + +jobs: + delete-pr-image: + runs-on: ubuntu-latest + permissions: + packages: write # required to delete GHCR container versions + contents: read + + steps: + - name: Sanitize branch name to match image tag (CI_COMMIT_REF_SLUG from gitlab) + id: sanitize + env: + PR_REF: ${{ github.event.pull_request.head.ref }} + run: | # Match GitLab's CI_COMMIT_REF_SLUG: lowercase, non-alphanumeric → '-', collapse and trim + TAG=$(echo -n "$PR_REF" | tr '[:upper:]' '[:lower:]' | sed -E 's/[^a-z0-9]+/-/g' | cut -c1-63 | sed -E 's/^-+//; s/-+$//') + echo "tag=$TAG" >> "$GITHUB_OUTPUT" + + - name: Delete image by tag from GHCR + env: + CI_COMMIT_REF_SLUG: ${{ steps.sanitize.outputs.tag }} + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + gh api "orgs/DataDog/packages/container/dd-trace-php%2Fdd-library-php/versions" \ + --jq ".[] | select(.metadata.container.tags[]? == \"${CI_COMMIT_REF_SLUG}\") | .id" \ + | xargs -r -I {} gh api "orgs/DataDog/packages/container/dd-trace-php%2Fdd-library-php/versions/{}" -X DELETE + echo "Deleted version with tag: ${CI_COMMIT_REF_SLUG}" From 2a6c818558fef606b6c64510ada24cc19f587ff3 Mon Sep 17 00:00:00 2001 From: Milan Garnier Date: Fri, 3 Jul 2026 10:28:14 +0200 Subject: [PATCH 09/12] delete image with same tag before pushing --- .gitlab/generate-package.php | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/.gitlab/generate-package.php b/.gitlab/generate-package.php index a55fd648ad..0cb58123b5 100644 --- a/.gitlab/generate-package.php +++ b/.gitlab/generate-package.php @@ -1570,9 +1570,40 @@ script: | set -e IMAGE="ghcr.io/datadog/dd-trace-php/dd-library-php:${CI_COMMIT_REF_SLUG}" + VERSIONS_URL="https://api.github.com/orgs/DataDog/packages/container/dd-trace-php%2Fdd-library-php/versions" + GITHUB_TOKEN=$( /dev/null docker login ghcr.io -u DataDog --password-stdin < github_token_system_tests.txt + echo "Docker login to ghcr.io succeeded" + + gh_api() { curl -s -o /tmp/resp.json -w "%{http_code}" -X "$1" \ + -H "Authorization: Bearer ${GITHUB_TOKEN}" -H "Accept: application/vnd.github+json" "$2"; } + + # delete any pre-existing image with the same tag so pushing with the same tag doesn't pile up untagged versions. + echo "== Checking token and existing image tagged ${CI_COMMIT_REF_SLUG} ==" + STATUS=$(gh_api GET "$VERSIONS_URL") + if [ "$STATUS" != "200" ]; then + echo "WARNING: token check failed (HTTP ${STATUS}), skipping pre-delete:"; cat /tmp/resp.json + else + echo "Token OK (HTTP 200): listed package versions for dd-trace-php/dd-library-php" + VERSION_ID=$(jq -r --arg TAG "${CI_COMMIT_REF_SLUG}" \ + '.[] | select(.metadata.container.tags[]? == $TAG) | .id' /tmp/resp.json | head -n1) + if [ -z "$VERSION_ID" ]; then + echo "No pre-existing image tagged ${CI_COMMIT_REF_SLUG} found, nothing to delete" + else + STATUS=$(gh_api DELETE "${VERSIONS_URL}/${VERSION_ID}") + if [ "$STATUS" = "204" ]; then + echo "Deleted existing image version ${VERSION_ID} (HTTP 204)" + else + echo "WARNING: failed to delete existing image version ${VERSION_ID} (HTTP ${STATUS}), continuing anyway:" + cat /tmp/resp.json + fi + fi + fi + echo "== Building and pushing ${IMAGE} ==" printf 'FROM scratch\nCOPY packages/dd-library-php-*-x86_64-linux-gnu.tar.gz /\nCOPY packages/datadog-setup.php /\n' \ > Dockerfile.system-tests docker build -f Dockerfile.system-tests -t "$IMAGE" . From 7c267ecd4b84cb7569898e89cc7485719bd8fd83 Mon Sep 17 00:00:00 2001 From: Milan Garnier Date: Fri, 3 Jul 2026 11:03:28 +0200 Subject: [PATCH 10/12] prevent two uploading jobs from running concurrently, skip uploading images them in the default branch --- .gitlab/generate-package.php | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/.gitlab/generate-package.php b/.gitlab/generate-package.php index 0cb58123b5..ec83c775c4 100644 --- a/.gitlab/generate-package.php +++ b/.gitlab/generate-package.php @@ -1536,7 +1536,7 @@ paths: - packages/datadog-setup.php -# Runs on every branch so system tests can be run against any in-progress branch. +# Runs on every non-default branch so system tests can be run against any (non-default) in-progress branch. "publish docker image for system tests (token)": stage: release image: registry.ddbuild.io/images/dd-octo-sts-ci-base:2025.06-1 @@ -1544,6 +1544,10 @@ id_tokens: DDOCTOSTS_ID_TOKEN: aud: dd-octo-sts + rules: + - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH + when: never + - when: on_success variables: GIT_STRATEGY: none script: @@ -1558,6 +1562,12 @@ stage: release image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/docker:29.4.0-noble tags: [ "docker-in-docker:amd64" ] + resource_group: "publish-system-tests-image-${CI_COMMIT_REF_SLUG}" + interruptible: true + rules: + - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH + when: never + - when: on_success needs: - job: "publish docker image for system tests (token)" artifacts: true From 9b94ded241dc00d6cd33b4cd808bc4765ccb8454 Mon Sep 17 00:00:00 2001 From: Milan Garnier Date: Fri, 3 Jul 2026 15:11:59 +0200 Subject: [PATCH 11/12] upload image with builds for all archs --- .gitlab/generate-package.php | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/.gitlab/generate-package.php b/.gitlab/generate-package.php index 1dd8ace098..d23ec0f53d 100644 --- a/.gitlab/generate-package.php +++ b/.gitlab/generate-package.php @@ -1575,6 +1575,8 @@ artifacts: true - job: "package extension: [amd64, x86_64-unknown-linux-gnu]" artifacts: true + - job: "package extension: [arm64, aarch64-unknown-linux-gnu]" + artifacts: true variables: GIT_STRATEGY: none script: | @@ -1613,14 +1615,24 @@ fi fi - echo "== Building and pushing ${IMAGE} ==" - printf 'FROM scratch\nCOPY packages/dd-library-php-*-x86_64-linux-gnu.tar.gz /\nCOPY packages/datadog-setup.php /\n' \ + echo "== Building and pushing ${IMAGE} (linux/amd64, linux/arm64) ==" + # stage each arch's tarball under its own dir so the Dockerfile can pick the right one via buildx's TARGETARCH arg. + mkdir -p packages/amd64 packages/arm64 + cp packages/dd-library-php-*-x86_64-linux-gnu.tar.gz packages/amd64/ + cp packages/dd-library-php-*-aarch64-linux-gnu.tar.gz packages/arm64/ + cp packages/datadog-setup.php packages/amd64/ + cp packages/datadog-setup.php packages/arm64/ + + printf 'FROM scratch\nARG TARGETARCH\nCOPY packages/${TARGETARCH}/dd-library-php-*-linux-gnu.tar.gz /\nCOPY packages/${TARGETARCH}/datadog-setup.php /\n' \ > Dockerfile.system-tests - docker build -f Dockerfile.system-tests -t "$IMAGE" . - docker push "$IMAGE" + BUILDER="system-tests-builder-${CI_JOB_ID}" + docker buildx create --name "$BUILDER" --driver docker-container + docker buildx build --builder "$BUILDER" --platform linux/amd64,linux/arm64 \ + -f Dockerfile.system-tests -t "$IMAGE" --push . echo "Pushed $IMAGE" after_script: - rm -f github_token_system_tests.txt + - docker buildx rm "system-tests-builder-${CI_JOB_ID}" || true "bundle for reliability env": stage: shared-pipeline From 47446346670a9480b79f690b9f9b06c883912998 Mon Sep 17 00:00:00 2001 From: Milan Garnier Date: Fri, 3 Jul 2026 15:28:24 +0200 Subject: [PATCH 12/12] Gate upload to having a PR open --- .../gitlab-ci-publish-packages.sts.yaml | 1 + .gitlab/generate-package.php | 15 +++++++++++++++ 2 files changed, 16 insertions(+) diff --git a/.github/chainguard/gitlab-ci-publish-packages.sts.yaml b/.github/chainguard/gitlab-ci-publish-packages.sts.yaml index 04173d825b..ede956febc 100644 --- a/.github/chainguard/gitlab-ci-publish-packages.sts.yaml +++ b/.github/chainguard/gitlab-ci-publish-packages.sts.yaml @@ -4,3 +4,4 @@ subject_pattern: "project_path:DataDog/apm-reliability/dd-trace-php:ref_type:(br permissions: packages: write + pull_requests: read diff --git a/.gitlab/generate-package.php b/.gitlab/generate-package.php index d23ec0f53d..682d072367 100644 --- a/.gitlab/generate-package.php +++ b/.gitlab/generate-package.php @@ -1579,6 +1579,8 @@ artifacts: true variables: GIT_STRATEGY: none + allow_failure: + exit_codes: 3 script: | set -e IMAGE="ghcr.io/datadog/dd-trace-php/dd-library-php:${CI_COMMIT_REF_SLUG}" @@ -1587,6 +1589,19 @@ apt-get update -qq && apt-get install -y -qq curl jq > /dev/null + STATUS=$(curl -s -o /tmp/pulls.json -w "%{http_code}" -H "Authorization: Bearer ${GITHUB_TOKEN}" -H "Accept: application/vnd.github+json" \ + "https://api.github.com/repos/DataDog/dd-trace-php/pulls?head=DataDog:${CI_COMMIT_BRANCH}&state=open") + if [ "${STATUS}" != "200" ]; then + echo "ERROR: failed to check for an open PR on branch ${CI_COMMIT_BRANCH} (HTTP ${STATUS}):"; cat /tmp/pulls.json + exit 1 + fi + PR_COUNT=$(jq 'length' /tmp/pulls.json) + if [ "${PR_COUNT}" -eq 0 ]; then + echo "No open PR for branch ${CI_COMMIT_BRANCH}; skipping system-tests image publish." + exit 3 + fi + echo "Found open PR(s) on branch ${CI_COMMIT_BRANCH}, continuing" + docker login ghcr.io -u DataDog --password-stdin < github_token_system_tests.txt echo "Docker login to ghcr.io succeeded"