diff --git a/Cargo.lock b/Cargo.lock index cc71da65..0ff54926 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -538,7 +538,7 @@ version = "1.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" dependencies = [ - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -549,7 +549,7 @@ checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" dependencies = [ "anstyle", "once_cell_polyfill", - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -1058,6 +1058,12 @@ dependencies = [ "hex-conservative", ] +[[package]] +name = "bitfield" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c821a6e124197eb56d907ccc2188eab1038fb919c914f47976e64dd8dbc855d1" + [[package]] name = "bitflags" version = "1.3.2" @@ -1513,6 +1519,12 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "codicon" +version = "3.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12170080f3533d6f09a19f81596f836854d0fa4867dc32c8172b8474b4e9de61" + [[package]] name = "colorchoice" version = "1.0.4" @@ -2168,13 +2180,34 @@ dependencies = [ "subtle", ] +[[package]] +name = "dirs" +version = "5.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44c45a9d03d6676652bcb5e724c7e988de1acad23a711b5217ab9cbecbec2225" +dependencies = [ + "dirs-sys 0.4.1", +] + [[package]] name = "dirs" version = "6.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c3e8aa94d75141228480295a7d0e7feb620b1a5ad9f12bc40be62411e38cce4e" dependencies = [ - "dirs-sys", + "dirs-sys 0.5.0", +] + +[[package]] +name = "dirs-sys" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "520f05a5cbd335fae5a99ff7a6ab8627577660ee5cfd6a94a6a929b52ff0321c" +dependencies = [ + "libc", + "option-ext", + "redox_users 0.4.6", + "windows-sys 0.48.0", ] [[package]] @@ -2185,8 +2218,8 @@ checksum = "e01a3366d27ee9890022452ee61b2b63a67e6f13f58900b651ff5665f0bb1fab" dependencies = [ "libc", "option-ext", - "redox_users", - "windows-sys 0.59.0", + "redox_users 0.5.2", + "windows-sys 0.61.2", ] [[package]] @@ -2248,6 +2281,7 @@ name = "dstack-attest" version = "0.5.11" dependencies = [ "anyhow", + "base64 0.22.1", "cc-eventlog", "dcap-qvl", "dstack-types", @@ -2258,12 +2292,15 @@ dependencies = [ "hex", "hex_fmt", "insta", + "libc", "or-panic", "parity-scale-codec", + "reqwest", "rmp-serde", "serde", "serde-human-bytes", "serde_json", + "sev", "sha2 0.10.9", "sha3", "tdx-attest", @@ -2433,6 +2470,7 @@ dependencies = [ "anyhow", "chrono", "clap", + "dstack-attest", "dstack-guest-agent-rpc", "dstack-kms-rpc", "dstack-mr", @@ -2666,7 +2704,7 @@ dependencies = [ "base64 0.22.1", "bon", "clap", - "dirs", + "dirs 6.0.0", "dstack-kms-rpc", "dstack-port-forward", "dstack-types", @@ -2811,6 +2849,7 @@ dependencies = [ "ff", "generic-array", "group", + "hkdf", "pem-rfc7468", "pkcs8", "rand_core 0.6.4", @@ -2933,7 +2972,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -4124,6 +4163,12 @@ dependencies = [ "memoffset", ] +[[package]] +name = "iocuddle" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d8972d5be69940353d5347a1344cb375d9b457d6809b428b05bb1ca2fb9ce007" + [[package]] name = "iohash" version = "0.5.11" @@ -4176,7 +4221,7 @@ checksum = "3640c1c38b8e4e43584d8df18be5fc6b0aa314ce6ebf51b53313d4306cca8e46" dependencies = [ "hermit-abi", "libc", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -4474,7 +4519,7 @@ dependencies = [ "rust-argon2", "secrecy", "serde", - "serde-big-array", + "serde-big-array 0.3.3", "serde_json", "sha2 0.9.9", "thiserror 1.0.69", @@ -4810,7 +4855,7 @@ version = "0.50.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" dependencies = [ - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -5006,7 +5051,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7d8fae84b431384b68627d0f9b3b1245fcf9f46f6c0e3dc902e9dce64edd1967" dependencies = [ "libc", - "windows-sys 0.48.0", + "windows-sys 0.61.2", ] [[package]] @@ -6010,6 +6055,17 @@ dependencies = [ "bitflags 2.11.0", ] +[[package]] +name = "redox_users" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba009ff324d1fc1b900bd1fdb31564febe58a8ccc8a6fdbb93b543d33b13ca43" +dependencies = [ + "getrandom 0.2.17", + "libredox", + "thiserror 1.0.69", +] + [[package]] name = "redox_users" version = "0.5.2" @@ -6463,7 +6519,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys 0.12.1", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -6924,6 +6980,15 @@ dependencies = [ "serde", ] +[[package]] +name = "serde-big-array" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11fc7cc2c76d73e0f27ee52abbd64eec84d46f370c88371120433196934e4b7f" +dependencies = [ + "serde", +] + [[package]] name = "serde-duration" version = "0.5.11" @@ -7081,6 +7146,34 @@ dependencies = [ "serde", ] +[[package]] +name = "sev" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "20ac277517d8fffdf3c41096323ed705b3a7c75e397129c072fb448339839d0f" +dependencies = [ + "base64 0.22.1", + "bincode 1.3.3", + "bitfield", + "bitflags 1.3.2", + "byteorder", + "codicon", + "dirs 5.0.1", + "hex", + "iocuddle", + "lazy_static", + "libc", + "p384", + "rsa", + "serde", + "serde-big-array 0.5.1", + "serde_bytes", + "sha2 0.10.9", + "static_assertions", + "uuid", + "x509-cert", +] + [[package]] name = "sha1" version = "0.10.6" @@ -7596,7 +7689,7 @@ dependencies = [ "getrandom 0.4.2", "once_cell", "rustix 1.1.4", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -7744,6 +7837,27 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" +[[package]] +name = "tls_codec" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0de2e01245e2bb89d6f05801c564fa27624dbd7b1846859876c7dad82e90bf6b" +dependencies = [ + "tls_codec_derive", + "zeroize", +] + +[[package]] +name = "tls_codec_derive" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d2e76690929402faae40aebdda620a2c0e25dd6d3b9afe48867dfd95991f4bd" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + [[package]] name = "tokio" version = "1.50.0" @@ -8153,6 +8267,7 @@ checksum = "a68d3c8f01c0cfa54a75291d83601161799e4a89a39e0929f4b0354d88757a37" dependencies = [ "getrandom 0.4.2", "js-sys", + "serde_core", "wasm-bindgen", ] @@ -8454,7 +8569,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.48.0", + "windows-sys 0.61.2", ] [[package]] @@ -9000,6 +9115,7 @@ dependencies = [ "const-oid", "der", "spki", + "tls_codec", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 2b144b46..384e77cc 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -123,6 +123,7 @@ hex_fmt = "0.3.0" hex-literal = "1.0.0" prost = "0.13.5" prost-types = "0.13.5" +sev = { version = "=6.0.0", default-features = false, features = ["snp", "crypto_nossl"] } scale = { version = "3.7.4", package = "parity-scale-codec", features = [ "derive", ] } diff --git a/basefiles/dstack-guest-agent.service b/basefiles/dstack-guest-agent.service index a88d395d..91853789 100644 --- a/basefiles/dstack-guest-agent.service +++ b/basefiles/dstack-guest-agent.service @@ -15,6 +15,7 @@ WatchdogSec=30s StandardOutput=journal+console StandardError=journal+console Environment=RUST_LOG=warn +EnvironmentFile=-/run/dstack/environment [Install] WantedBy=multi-user.target diff --git a/basefiles/dstack-prepare.sh b/basefiles/dstack-prepare.sh index 7b6ac2c3..b27219a7 100755 --- a/basefiles/dstack-prepare.sh +++ b/basefiles/dstack-prepare.sh @@ -80,27 +80,41 @@ WORK_DIR="/var/volatile/dstack" DATA_MNT="$WORK_DIR/persistent" OVERLAY_TMP="/var/volatile/overlay" -OVERLAY_PERSIST="$DATA_MNT/overlay" # Prepare volatile dirs mount_overlay() { - local src=$1 - local dst=$2/$1 - mkdir -p $dst/upper $dst/work - mount -t overlay overlay -o lowerdir=$src,upperdir=$dst/upper,workdir=$dst/work $src + local src="$1" + local dst="$2/$1" + local overlay_opts="lowerdir=$src,upperdir=$dst/upper,workdir=$dst/work" + mkdir -p "$dst/upper" "$dst/work" + mount -t overlay overlay -o "$overlay_opts" "$src" } -mount_overlay /etc $OVERLAY_TMP -mount_overlay /usr $OVERLAY_TMP -mount_overlay /bin $OVERLAY_TMP -mount_overlay /home $OVERLAY_TMP +mount_overlay /etc "$OVERLAY_TMP" +mount_overlay /usr "$OVERLAY_TMP" +mount_overlay /bin "$OVERLAY_TMP" +mount_overlay /home "$OVERLAY_TMP" + +# systemd-resolved may be unavailable in minimal smoke/debug boots; keep DNS usable for dockerd pulls. +if ! [[ -s /etc/resolv.conf ]] || grep -Eq 'nameserver[[:space:]]+(127\.|::1)' /etc/resolv.conf; then + printf 'nameserver 1.1.1.1\nnameserver 8.8.8.8\n' >/etc/resolv.conf +fi # Make sure the system time is synchronized log "Syncing system time..." -# Let the chronyd correct the system time immediately -chronyc makestep - -if ! [[ -e /dev/tdx_guest ]]; then - modprobe tdx-guest +# Let the chronyd correct the system time immediately; keep booting if chronyd is not ready yet. +chronyc makestep || log "Warning: chronyc makestep failed; continuing" + +if [[ -e /dev/sev-guest ]] || grep -qw sev_guest /sys/kernel/config/tsm/report/*/provider 2>/dev/null; then + log "SEV-SNP guest device/TSM provider detected" +elif [[ -e /dev/tdx_guest ]]; then + log "TDX guest device detected" +elif modprobe sev-guest 2>/dev/null; then + log "Loaded sev-guest module" +elif modprobe tdx-guest 2>/dev/null; then + log "Loaded tdx-guest module" +else + log "Error: neither sev-guest nor tdx-guest module is available" + exit 1 fi # Setup configfs and TSM for TDX attestation @@ -125,9 +139,10 @@ log "Preparing dstack system..." has_partition_table() { local disk="$1" - local disk_name=$(basename "$disk") + local disk_name + disk_name=$(basename "$disk") # Check sysfs for any child partitions - for entry in /sys/class/block/${disk_name}/${disk_name}*; do + for entry in "/sys/class/block/${disk_name}/${disk_name}"*; do [ -e "$entry/partition" ] || continue return 0 done @@ -259,6 +274,14 @@ if [ -f "/sys/class/block/${device_name}/partition" ]; then fi fi +AMD_KDS_PROXY_URL="$(tr ' ' '\n' /run/dstack/environment +fi + dstack-util setup --work-dir $WORK_DIR --device "$DATA_DEVICE" --mount-point $DATA_MNT log "Mounting container runtime dirs to persistent storage" @@ -279,9 +302,10 @@ echo "============================" cd /dstack -if [ $(jq 'has("init_script")' app-compose.json) == true ]; then +if [ "$(jq 'has("init_script")' app-compose.json)" == true ]; then log "Running init script" dstack-util notify-host -e "boot.progress" -d "init-script" || true + # shellcheck disable=SC1090 source <(jq -r '.init_script' app-compose.json) fi diff --git a/docs/amd-sev-snp-review-readiness.md b/docs/amd-sev-snp-review-readiness.md new file mode 100644 index 00000000..419b15aa --- /dev/null +++ b/docs/amd-sev-snp-review-readiness.md @@ -0,0 +1,198 @@ +# AMD SEV-SNP Review Readiness + +This branch adds AMD SEV-SNP support and now includes a controlled, explicitly opt-in KMS key/cert release gate for SNP. + +## Current review boundary + +Implemented and intended for review: + +- AMD SEV-SNP evidence plumbing in the v1 attestation format. +- SNP report verification with AMD Genoa ARK/ASK/VCEK chain verification. +- Report-data challenge binding and fail-closed report policy checks. +- SNP launch-measurement recomputation from OVMF/kernel/initrd/cmdline inputs. +- KMS SNP `BootInfo` construction from verified report measurement, chip id, launch inputs, TCB status, and advisory ids. +- Auth-policy evaluation through the existing KMS auth flow. +- Controlled SNP key/cert release guarded by both external auth policy and local KMS config. +- VMM-provided SNP launch inputs in `.sys-config.json` so KMS self/app auth can recompute the same launch measurement used by QEMU. +- Onboarding attestation-info reporting for SNP identity fields. +- VMM explicit `platform = "amd-sev-snp"` launch path. + +Default posture: + +- SNP app key release, KMS/root/temp CA key release, and app certificate release are still disabled by default. +- Operators must explicitly set `[core.sev_snp_key_release].enabled = true` before any SNP `BootInfo` can release sensitive material. +- KMS startup rejects `enabled = true` unless `enforce_self_authorization = true`, so the self-authorized `GetTempCaCert` path cannot silently bypass the SNP release gate in production config. +- Even with the local KMS gate enabled, the existing auth API must first allow the verified SNP `BootInfo` for the app/KMS identity. + +## Fail-closed policy summary + +- `platform = "auto"` remains conservative while SNP is experimental; operators must explicitly set `platform = "amd-sev-snp"` to launch an SNP guest. +- SNP launch measurement is recomputed from trusted KMS config/input and compared to the hardware-verified report measurement. +- SNP `BootInfo.tcb_status` is verifier-derived from signed AMD SNP report TCB fields: + - `UpToDate` only when current/reported/committed/launch TCB versions all match. + - `OutOfDate` otherwise. +- SNP advisory ids are propagated from verifier output into `BootInfo`; currently this list is explicit and empty because the AMD report/VCEK evidence used here does not carry a direct advisory-list field. +- `auth-simple` defaults remain strict: only `UpToDate` is accepted and any advisory id is denied unless explicitly allowlisted. +- The local KMS release gate mirrors that strict default: + - `[core.sev_snp_key_release].enabled = false` by default. + - `allowed_tcb_statuses = ["UpToDate"]` by default. + - `allowed_advisory_ids = []` by default, so any advisory remains fail-closed unless explicitly allowlisted. + +Example opt-in gate: + +```toml +[core.sev_snp_key_release] +enabled = true +allowed_tcb_statuses = ["UpToDate"] +allowed_advisory_ids = [] +``` + +Sensitive release surfaces using this gate: + +- `GetAppKey`: app disk/env/k256 key material. +- `GetKmsKey`: temp CA key plus root CA/k256 key material for authorized KMS transfer. +- `SignCert`: app certificate chain signing. +- `GetTempCaCert`: temp CA material for self-authorized KMS instances. + +## Live golden-vector proof + +The ignored live regression test cross-checks dstack's pure Rust SNP measurement recomputation against `sev-snp-measure` on the SNP-capable host. + +Command: + +```bash +cargo test -p dstack-kms --all-features recomputation_matches_sev_snp_measure_live_golden_vector -- --ignored --nocapture +``` + +Latest local proof: + +```text +DSTACK_SEV_SNP_MEASURE_GOLDEN_VECTOR_BEGIN +utc=2026-06-02T19:49:14Z +host=dedicated-m24-fork +uname=Linux dedicated-m24-fork 6.11.0-rc3-snp-host-85ef1ac03941 #2 SMP Sat May 3 11:42:34 EDT 2025 x86_64 GNU/Linux +sev_snp_measure=/usr/local/bin/sev-snp-measure +sev_snp_measure_version=sev-snp-measure 0.0.10 +ovmf_path=/opt/AMDSEV/usr/local/share/qemu/OVMF.fd +ovmf_sha256=67e7a7027437823e9c166a60d00666d5d5391e13050488cad5cc2acd913fab4a +kernel_fixture_sha256=3f73f96a321b35a4c5561b05cfa6e9b5c573159380d37abe76f9a8ebe113a72e +initrd_fixture_sha256=e8790816224329cd76675c2aba4e62e885b5a4e0ec056227da70e775191d6d56 +vcpus=2 +vcpu_type=EPYC-v4 +guest_features=0x1 +append=console=ttyS0 loglevel=7 docker_compose_hash=2222222222222222222222222222222222222222222222222222222222222222 rootfs_hash=3333333333333333333333333333333333333333333333333333333333333333 app_id=1111111111111111111111111111111111111111 +sev_snp_measurement=6497fb9f90dc4a322228a8a5eb14742e09067bc44c184c2068d583ef628b5bae8c6cf15d91fe1bc0b7a8cbcc575be370 +cargo_live_test=cargo test -p dstack-kms --all-features recomputation_matches_sev_snp_measure_live_golden_vector -- --ignored --nocapture +cargo_live_test_result=passed locally on this host at 2026-06-02T19:49:14Z +DSTACK_SEV_SNP_MEASURE_GOLDEN_VECTOR_END +``` + +## Guest attestation proof + +A prior SNP guest smoke proof confirmed the guest kernel exposed SEV-SNP report support and could produce a report containing the expected challenge bytes. + +```text +Memory Encryption Features active: AMD SEV SEV-ES SEV-SNP +SEV: SNP running at VMPL0. +sev-guest sev-guest: Initialized SEV guest driver (using vmpck_id 0) +DSTACK_SEV_SNP_ATTESTATION_PROOF_BEGIN +source=configfs-tsm +report_size=1184 +report_data_offset=80 +report_contains_expected_report_data=true +DSTACK_SEV_SNP_ATTESTATION_PROOF_END +``` + +## Manual dstack E2E smoke status + +An additional manual smoke was attempted on the SNP host (`chris@173.234.27.162`) using the PR branch, release-built `dstack-vmm`/`supervisor`/`dstack-kms`, QEMU 10.0.2, and the SNP-capable OVMF at `/opt/AMDSEV/usr/local/share/qemu/OVMF.fd`. The reusable version of that smoke is checked in at `test-scripts/snp-e2e-smoke.sh` for follow-up debugging on SNP hosts. + +That smoke exposed and fixed several VMM/KMS-auth integration issues before the guest reached KMS: + +- `.sys-config.json` did not include the `sev_snp_measurement` launch input object needed by KMS SNP `BootInfo` recomputation. +- The VMM launch path required `metadata.json.rootfs_hash`, while the released `dstack-0.5.11` images carry the rootfs hash in `dstack.rootfs_hash=...` on the kernel cmdline. +- The VMM SNP QEMU path now uses the SNP measurement CPU model (`EPYC-v4`) and confidential virtio PCI options (`disable-legacy=on,iommu_platform=true`) for SNP-launched virtio devices, matching the host's working SNP launch posture more closely. + +After those fixes, the manual smoke progressed through full dstack-managed SNP guest boot and KMS self-bootstrap on the known-good remote host. Additional smoke/debug fixes made the host/KMS side reach the app-key boundary: + +- Minimal guest boot now keeps DNS usable when `systemd-resolved`/`chronyd` are unavailable early in smoke boots and detects `sev-guest` before trying the TDX guest module. +- SNP guests skip TDX-only `mr_config_id` and app-info RTMR decoding while still preserving non-SNP behavior. +- Configfs TSM report collection falls back to the SEV-SNP extended-report ioctl when configfs does not carry certificate collateral. +- If verifier-side evidence still lacks ASK/VCEK collateral, the verifier can fetch AMD KDS ARK/ASK/VCEK using the report `chip_id` and reported TCB, then verify the signed report fail-closed. +- KMS measurement recomputation now uses the image's original kernel cmdline as the measurement base before appending `docker_compose_hash`, `rootfs_hash`, and `app_id`, matching the VMM QEMU `-append` path. + +Latest sanitized remote smoke result with PR-built host binaries and a coherent `MACHINE = "sev-snp"` guest image: + +```text +remote_host=chris@173.234.27.162 +host_kernel=Linux 6.11.0-rc3-snp-host-85ef1ac03941 +qemu_version=10.0.2 +ovmf_sha256=67e7a7027437823e9c166a60d00666d5d5391e13050488cad5cc2acd913fab4a +image=dstack-dev-0.6.0 +platform=amd-sev-snp +image_kernel=Linux 6.18.24-dstack with CONFIG_AMD_MEM_ENCRYPT=y, CONFIG_SEV_GUEST=y, CONFIG_TSM_REPORTS=y +kms_guest=booted SNP Linux/userspace and started dstack-kms +kms_marker=SNP_KMS_CONTAINER_STARTED / KMS runtime ready +kds_proxy=enabled for smoke via DSTACK_AMD_KDS_PROXY_URL=https://cors.litgateway.com/ +strict_tcb_probe=denied_as_expected with tcb_status is not allowed +success_probe=GetTempCaCert HTTP 200; GetAppKey HTTP 200; SignCert HTTP 200; app container started +smoke_result=SNP E2E smoke success +no_secret_material_logged=true +``` + +This means the PR has live SNP report proof, live golden-vector measurement proof, release-gate unit/integration coverage, and hardware smoke proof through dstack-managed SNP KMS boot, strict TCB denial, app guest key release, and app container startup. The fresh-box smoke now reaches Linux/userspace, `SNP_KMS_CONTAINER_STARTED`, `GetTempCaCert`, `GetAppKey`, `SignCert`, and app container startup when using a coherent **SNP** `meta-dstack` image. During the smoke, AMD KDS throttling was worked around by explicitly routing AMD KDS collateral fetches through `DSTACK_AMD_KDS_PROXY_URL`; the proxy value is carried in the measured guest cmdline for smoke runs and mirrored in KMS measurement recomputation to avoid measurement drift. Host/KMS binaries must match PR #703, guest-side `dstack-util`/`dstack-attest` must include the PR cert-chain/KDS fallback, and the Yocto image must be built with `MACHINE = "sev-snp"` so the guest kernel includes AMD memory-encryption/SNP support. A coherent PR image built with the default `tdx` machine produced a `6.18.24-dstack` kernel with `# CONFIG_AMD_MEM_ENCRYPT is not set`; controlled QEMU tests showed that kernel resets immediately after OVMF loads kernel/initrd, while SNP-capable kernels boot the same QEMU/OVMF path to Linux/SNP markers. + +### Fresh SNP host / image requirements + +The checked-in smoke is enough to reproduce the current boundary on a compatible SNP host, but reviewers should treat the guest image/kernel/userspace as part of the test matrix: + +- Known-good host for reaching KMS and app `dstack-prepare.sh`: `chris@173.234.27.162` with QEMU 10.0.2, the SNP-capable OVMF above, and a coherent `dstack-dev-0.6.0` guest image built with `MACHINE = "sev-snp"`. +- Released images that do not carry PR #703 guest-side `dstack-util`/`dstack-attest` may reject SNP evidence before the newer PR fallback paths can help. +- A coherent PR #703 image must be built as an SNP image, not with `meta-dstack`'s default `tdx` machine. The default TDX build can emit a kernel without `CONFIG_AMD_MEM_ENCRYPT`, which fails before Linux serial output under SNP. +- On the same remote host/QEMU/OVMF, a minimal SNP initramfs booted SNP-capable kernels (`6.11.0-rc3-snp-host`, `6.9.0-rc7-snp-host`, and the `MACHINE = "sev-snp"` `6.18.24-dstack` kernel) to Linux/SNP markers, while the default-TDX `6.18.24-dstack` kernel reset immediately after OVMF loaded kernel/initrd. This isolates that failure to the guest kernel config, not PSP firmware, KMS/auth policy, author-key, command line, virtio wiring, or basic host SNP enablement. + +Practical implication for reviewers/testers on a fresh box: + +1. Install/use an AMDSEV QEMU 10.x build and the matching SNP-capable OVMF. +2. Build the PR binaries with `cargo build --release -p dstack-vmm -p supervisor -p dstack-kms`. +3. Run `test-scripts/snp-e2e-smoke.sh` unchanged and first confirm it reaches `SNP_KMS_CONTAINER_STARTED`; if AMD KDS throttles the lab host, set `DSTACK_SNP_SMOKE_KDS_PROXY_URL` to a trusted AMD-KDS passthrough/cache endpoint and rerun. +4. For full `SNP_APP_CONTAINER_STARTED` / `GetAppKey` success, use or publish a coherent `meta-dstack` guest image whose kernel, modules, initramfs, rootfs, verity metadata, and guest userspace include the same PR #703 `dstack-util`/`dstack-attest` SNP cert-chain/KDS fallback code. The reproducible path is to build `meta-dstack` with its `dstack` submodule checked out to this PR branch, for example: + + ```bash + git clone https://github.com/Dstack-TEE/meta-dstack.git + cd meta-dstack + git submodule update --init --recursive --depth 1 + cd dstack + git fetch https://github.com/clawdbot-glitch003/dstack.git feat/amd-sev-snp-conversion + git checkout -B feat/amd-sev-snp-conversion FETCH_HEAD + cd .. + source dev-setup ./bb-build + sed -i 's/^MACHINE ??= .*/MACHINE = "sev-snp"/' ./bb-build/conf/local.conf + FLAVORS=dev make dist DIST_DIR=$PWD/images BB_BUILD_DIR=$PWD/bb-build + # Use the resulting dstack-dev image directory with: + # DSTACK_SNP_SMOKE_IMAGE_NAME= + ``` + + Do not try to inject only a replacement `dstack-util` into the stock image; that experiment changed the initramfs/measurement enough to regress boot. +5. Only after the baseline smoke reaches the app success marker should testers swap the simple app workload for Chipotle. + +If the smoke stops after `EFI stub: Loaded initrd ...` with `cpus are not resettable`, use a host/image/kernel that is known to boot dstack under SNP before debugging app-level behavior. If it reaches `Requesting app keys from KMS` and fails with AMD KDS `HTTP 429`, use the smoke proxy hook above; if it fails with missing cert-chain/collateral without KDS proxy evidence, rebuild/use a coherent PR guest image rather than changing KMS release policy. + +## Validation commands + +Run locally for this review-ready staging branch: + +```bash +cargo fmt --all +cargo test -p dstack-kms --all-features +cargo test -p dstack-attest --all-features +cargo test -p dstack-vmm --all-features +cargo check --workspace --all-features +cargo clippy --workspace --all-features -- -D warnings --allow unused_variables +git diff --check +cd kms/auth-simple && npx oxlint . && npx vitest run +``` + +## Remaining production follow-up + +The release gate is controlled and production-oriented, but AMD advisory/revocation collateral is still limited by the evidence source available here: SNP reports/VCEKs do not directly carry an advisory list, so `advisory_ids` currently propagates as an explicit empty list. Future collateral fetchers can populate this field and will be denied by both auth-simple and the local KMS release gate unless each advisory is explicitly allowlisted. diff --git a/dstack-attest/Cargo.toml b/dstack-attest/Cargo.toml index d17bdc9f..e8fc1dc2 100644 --- a/dstack-attest/Cargo.toml +++ b/dstack-attest/Cargo.toml @@ -11,19 +11,23 @@ license.workspace = true [dependencies] anyhow.workspace = true +base64.workspace = true cc-eventlog.workspace = true rmp-serde.workspace = true +reqwest = { workspace = true, features = ["blocking"] } dcap-qvl.workspace = true dstack-types.workspace = true ez-hash.workspace = true fs-err.workspace = true hex.workspace = true hex_fmt.workspace = true +libc.workspace = true or-panic.workspace = true scale = { workspace = true, features = ["derive"] } serde.workspace = true serde-human-bytes.workspace = true serde_json.workspace = true +sev.workspace = true sha2.workspace = true sha3.workspace = true tdx-attest.workspace = true diff --git a/dstack-attest/src/amd_sev_snp.rs b/dstack-attest/src/amd_sev_snp.rs new file mode 100644 index 00000000..ad305886 --- /dev/null +++ b/dstack-attest/src/amd_sev_snp.rs @@ -0,0 +1,716 @@ +// SPDX-FileCopyrightText: © 2025 Phala Network +// +// SPDX-License-Identifier: Apache-2.0 + +//! AMD SEV-SNP attestation verification helpers. +//! +//! This module implements the hardware report verification slice: certificate +//! normalization, AMD ARK/ASK/VCEK chain verification, report signature checks, +//! report_data binding, and invariant SNP policy checks. KMS/app authorization +//! must still bind the verified measurement to app/config identity before +//! production key release. + +use anyhow::{bail, Context, Result}; +use base64::engine::general_purpose::STANDARD; +use base64::Engine as _; +use sev::certs::snp::{ca, Certificate, Chain, Verifiable}; +use sev::firmware::{guest::AttestationReport, host::TcbVersion}; + +/// AMD Genoa ARK certificate (DER, base64-encoded). +/// Source: https://kdsintf.amd.com/vcek/v1/Genoa/cert_chain +const GENOA_ARK_DER_B64: &str = "MIIGYzCCBBKgAwIBAgIDAgAAMEYGCSqGSIb3DQEBCjA5oA8wDQYJYIZIAWUDBAICBQChHDAaBgkqhkiG9w0BAQgwDQYJYIZIAWUDBAICBQCiAwIBMKMDAgEBMHsxFDASBgNVBAsMC0VuZ2luZWVyaW5nMQswCQYDVQQGEwJVUzEUMBIGA1UEBwwLU2FudGEgQ2xhcmExCzAJBgNVBAgMAkNBMR8wHQYDVQQKDBZBZHZhbmNlZCBNaWNybyBEZXZpY2VzMRIwEAYDVQQDDAlBUkstR2Vub2EwHhcNMjIwMTI2MTUzNDM3WhcNNDcwMTI2MTUzNDM3WjB7MRQwEgYDVQQLDAtFbmdpbmVlcmluZzELMAkGA1UEBhMCVVMxFDASBgNVBAcMC1NhbnRhIENsYXJhMQswCQYDVQQIDAJDQTEfMB0GA1UECgwWQWR2YW5jZWQgTWljcm8gRGV2aWNlczESMBAGA1UEAwwJQVJLLUdlbm9hMIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEA3Cd95S/uFOuRIskW9vz9VDBF69NDQF79oRhL/L2PVQGhK3YdfEBgpF/JiwWFBsT/fXDhzA01p3LkcT/7LdjcRfKXjHl+0Qq/M4dZkh6QDoUeKzNBLDcBKDDGWo3v35NyrxbA1DnkYwUKU5AAk4P94tKXLp80oxt84ahyHoLmc/LqsGsp+oq1Bz4PPsYLwTG4iMKVaaT90/oZ4I8oibSru92vJhlqWO27d/Rxc3iUMyhNeGToOvgx/iUo4gGpG61NDpkEUvIzuKcaMx8IdTpWg2DF6SwF0IgVMffnvtJmA68BwJNWo1E4PLJdaPfBifcJpuBFwNVQIPQEVX3aP89HJSp8YbY9lySS6PlVEqTBBtaQmi4ATGmMR+n2K/e+JAhU2Gj7jIpJhOkdH9firQDnmlA2SFfJ/Cc0mGNzW9RmIhyOUnNFoclmkRhl3/AQU5Ys9Qsan1jT/EiyT+pCpmnA+y9edvhDCbOG8F2oxHGRdTBkylungrkXJGYiwGrR8kaiqv7NN8QhOBMqYjcbrkEr0f8QMKklIS5ruOfqlLMCBw8JLB3LkjpWgtD7OpxkzSsohN47Uom86RY6lp72g8eXHP1qYrnvhzaG1S70vw6OkbaaC9EjiH/uHgAJQGxon7u0Q7xgoREWA/e7JcBQwLg80Hq/sbRuqesxz7wBWSY254cCAwEAAaN+MHwwDgYDVR0PAQH/BAQDAgEGMB0GA1UdDgQWBBSfXfn+DdjzWtAzGiXvgSlPvjGoWzAPBgNVHRMBAf8EBTADAQH/MDoGA1UdHwQzMDEwL6AtoCuGKWh0dHBzOi8va2RzaW50Zi5hbWQuY29tL3ZjZWsvdjEvR2Vub2EvY3JsMEYGCSqGSIb3DQEBCjA5oA8wDQYJYIZIAWUDBAICBQChHDAaBgkqhkiG9w0BAQgwDQYJYIZIAWUDBAICBQCiAwIBMKMDAgEBA4ICAQAdIlPBC7DQmvH7kjlOznFx3i21SzOPDs5L7SgFjMC9rR07292GQCA7Z7Ulq97JQaWeD2ofGGse5swj4OQfKfVv/zaJUFjvosZOnfZ63epu8MjWgBSXJg5QE/Al0zRsZsp53DBTdA+Uv/s33fexdenT1mpKYzhIg/cKtz4oMxq8JKWJ8Po1CXLzKcfrTphjlbkh8AVKMXeBd2SpM33B1YP4g1BOdk013kqb7bRHZ1iB2JHG5cMKKbwRCSAAGHLTzASgDcXr9Fp7Z3liDhGu/ci1opGmkp12QNiJuBbkTU+xDZHm5X8Jm99BX7NEpzlOwIVR8ClgBDyuBkBC2ljtr3ZSaUIYj2xuyWN95KFY49nWxcz90CFa3Hzmy4zMQmBe9dVyls5eL5p9bkXcgRMDTbgmVZiAf4afe8DLdmQcYcMFQbHhgVzMiyZHGJgcCrQmA7MkTwEIds1wx/HzMcwU4qqNBAoZV7oeIIPxdqFXfPqHqiRlEbRDfX1TG5NFVaeByX0GyH6jzYVuezETzruaky6fp2bl2bczxPE8HdS38ijiJmm9vl50RGUeOAXjSuInGR4bsRufeGPB9peTa9BcBOeTWzstqTUB/F/qaZCIZKr4X6TyfUuSDz/1JDAGl+lxdM0P9+lLaP9NahQjHCVf0zf1c1salVuGFk2w/wMz1R1BHg=="; + +const ASK_CERT_GUID: [u8; 16] = [ + 0x4a, 0xb7, 0xb3, 0x79, 0xbb, 0xac, 0x4f, 0xe4, 0xa0, 0x2f, 0x05, 0xae, 0xf3, 0x27, 0xc7, 0x82, +]; +const VCEK_CERT_GUID: [u8; 16] = [ + 0x63, 0xda, 0x75, 0x8d, 0xe6, 0x64, 0x45, 0x64, 0xad, 0xc5, 0xf4, 0xb9, 0x3b, 0xe8, 0xac, 0xcd, +]; +const VLEK_CERT_GUID: [u8; 16] = [ + 0xa8, 0x07, 0x4b, 0xc2, 0xa2, 0x5a, 0x48, 0x3e, 0xaa, 0xe6, 0x39, 0xc0, 0x45, 0xa0, 0xb8, 0xa1, +]; +const CERT_TABLE_ENTRY_SIZE: usize = 24; + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +pub struct AmdSnpTcbVersion { + pub bootloader: u8, + pub tee: u8, + pub snp: u8, + pub microcode: u8, +} + +impl From for AmdSnpTcbVersion { + fn from(value: TcbVersion) -> Self { + Self { + bootloader: value.bootloader, + tee: value.tee, + snp: value.snp, + microcode: value.microcode, + } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +pub struct AmdSnpTcbInfo { + pub current: AmdSnpTcbVersion, + pub reported: AmdSnpTcbVersion, + pub committed: AmdSnpTcbVersion, + pub launch: AmdSnpTcbVersion, +} + +impl AmdSnpTcbInfo { + pub fn from_report(report: &AttestationReport) -> Self { + Self { + current: report.current_tcb.into(), + reported: report.reported_tcb.into(), + committed: report.committed_tcb.into(), + launch: report.launch_tcb.into(), + } + } + + pub fn tcb_status(&self) -> &'static str { + if self.current == self.reported + && self.committed == self.reported + && self.launch == self.reported + { + "UpToDate" + } else { + "OutOfDate" + } + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct VerifiedAmdSnpReport { + pub measurement: [u8; 48], + pub report_data: [u8; 64], + pub chip_id: [u8; 64], + pub tcb_info: AmdSnpTcbInfo, + pub advisory_ids: Vec, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum CertEncoding { + Pem, + Der, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +struct CertBytes { + bytes: Vec, + encoding: CertEncoding, +} + +pub struct AmdSnpAttestationInput<'a> { + pub report: &'a [u8], + pub ask_pem: &'a [u8], + pub vcek_pem: &'a [u8], +} + +#[derive(Debug, Clone, PartialEq, Eq)] +struct AmdKdsCollateral { + ark: CertBytes, + ask: CertBytes, + vcek: CertBytes, +} + +pub fn verify_amd_snp_attestation( + input: &AmdSnpAttestationInput<'_>, +) -> Result { + verify_amd_snp_attestation_with_certs( + input.report, + CertBytes { + bytes: input.ask_pem.to_vec(), + encoding: CertEncoding::Pem, + }, + CertBytes { + bytes: input.vcek_pem.to_vec(), + encoding: CertEncoding::Pem, + }, + ) +} + +fn verify_amd_snp_attestation_with_certs( + report_bytes: &[u8], + ask_bytes: CertBytes, + vcek_bytes: CertBytes, +) -> Result { + let ark_der = STANDARD + .decode(GENOA_ARK_DER_B64) + .context("failed to decode amd genoa ark")?; + verify_amd_snp_attestation_with_cert_chain( + report_bytes, + CertBytes { + bytes: ark_der, + encoding: CertEncoding::Der, + }, + ask_bytes, + vcek_bytes, + ) +} + +fn verify_amd_snp_attestation_with_cert_chain( + report_bytes: &[u8], + ark_bytes: CertBytes, + ask_bytes: CertBytes, + vcek_bytes: CertBytes, +) -> Result { + if report_bytes.len() != 1184 { + bail!( + "invalid amd sev-snp report length: expected 1184 bytes, got {}", + report_bytes.len() + ); + } + let report = AttestationReport::from_bytes(report_bytes) + .map_err(|err| anyhow::anyhow!("failed to parse amd sev-snp report: {err}"))?; + + let ark = parse_certificate(&ark_bytes, "ark")?; + let ask = parse_certificate(&ask_bytes, "ask")?; + let vcek = parse_certificate(&vcek_bytes, "vcek")?; + + let chain = Chain { + ca: ca::Chain { ark, ask }, + vek: vcek.clone(), + }; + chain + .verify() + .map_err(|err| anyhow::anyhow!("amd cert chain verification failed: {err:?}"))?; + (&vcek, &report).verify().map_err(|err| { + anyhow::anyhow!("amd sev-snp report signature verification failed: {err:?}") + })?; + validate_amd_snp_report_policy(&report)?; + + let mut measurement = [0u8; 48]; + measurement.copy_from_slice( + report + .measurement + .as_ref() + .get(..48) + .context("amd sev-snp measurement too short")?, + ); + let mut report_data = [0u8; 64]; + report_data.copy_from_slice( + report + .report_data + .as_ref() + .get(..64) + .context("amd sev-snp report_data too short")?, + ); + let mut chip_id = [0u8; 64]; + chip_id.copy_from_slice( + report + .chip_id + .as_ref() + .get(..64) + .context("amd sev-snp chip_id too short")?, + ); + + Ok(VerifiedAmdSnpReport { + measurement, + report_data, + chip_id, + tcb_info: AmdSnpTcbInfo::from_report(&report), + // AMD SEV-SNP attestation reports and VCEKs do not carry a direct + // advisory list. Keep this explicit and empty so downstream auth stays + // fail-closed if a future verifier adds advisories from revocation or + // external policy collateral. + advisory_ids: Vec::new(), + }) +} + +pub fn verify_amd_snp_evidence( + report: &[u8], + cert_chain: &[Vec], + expected_report_data: &[u8; 64], +) -> Result { + let (ask, vcek) = normalize_ask_vcek_certs(cert_chain)?; + let verified = verify_amd_snp_attestation_with_certs(report, ask, vcek)?; + if &verified.report_data != expected_report_data { + bail!("amd sev-snp report_data mismatch"); + } + Ok(verified) +} + +pub fn verify_amd_snp_evidence_with_kds_fallback( + report: &[u8], + cert_chain: &[Vec], + expected_report_data: &[u8; 64], +) -> Result { + if !cert_chain.is_empty() { + return verify_amd_snp_evidence(report, cert_chain, expected_report_data); + } + let report_obj = AttestationReport::from_bytes(report) + .map_err(|err| anyhow::anyhow!("failed to parse amd sev-snp report: {err}"))?; + let collateral = fetch_amd_kds_collateral_for_report(&report_obj) + .context("failed to fetch amd sev-snp KDS collateral for empty cert_chain")?; + let verified = verify_amd_snp_attestation_with_cert_chain( + report, + collateral.ark, + collateral.ask, + collateral.vcek, + )?; + if &verified.report_data != expected_report_data { + bail!("amd sev-snp report_data mismatch"); + } + Ok(verified) +} + +fn fetch_amd_kds_collateral_for_report(report: &AttestationReport) -> Result { + let mut errors = Vec::new(); + for product in ["Genoa", "Milan", "Bergamo", "Siena", "Turin"] { + match fetch_amd_kds_collateral_for_product(product, report) { + Ok(collateral) => return Ok(collateral), + Err(err) => errors.push(format!("{product}: {err:#}")), + } + } + bail!( + "amd sev-snp KDS collateral unavailable for supported products: {}", + errors.join("; ") + ) +} + +fn fetch_amd_kds_collateral_for_product( + product: &str, + report: &AttestationReport, +) -> Result { + let (ark, ask) = fetch_amd_kds_ca_chain(product)?; + let mut chip_id = [0u8; 64]; + chip_id.copy_from_slice( + report + .chip_id + .as_ref() + .get(..64) + .context("amd sev-snp chip_id too short")?, + ); + let vcek_url = amd_kds_vcek_url(product, &chip_id, report.reported_tcb.into()); + let vcek_request_url = amd_kds_request_url(&vcek_url); + let vcek = reqwest::blocking::Client::new() + .get(&vcek_request_url) + .send() + .with_context(|| format!("failed to request amd sev-snp vcek from {vcek_request_url}"))? + .error_for_status() + .with_context(|| { + format!("amd sev-snp vcek request failed for {vcek_url} via {vcek_request_url}") + })? + .bytes() + .context("failed to read amd sev-snp vcek response")? + .to_vec(); + Ok(AmdKdsCollateral { + ark, + ask, + vcek: CertBytes { + bytes: vcek, + encoding: CertEncoding::Der, + }, + }) +} + +fn fetch_amd_kds_ca_chain(product: &str) -> Result<(CertBytes, CertBytes)> { + let url = format!("https://kdsintf.amd.com/vcek/v1/{product}/cert_chain"); + let request_url = amd_kds_request_url(&url); + let chain = reqwest::blocking::Client::new() + .get(&request_url) + .send() + .with_context(|| format!("failed to request amd sev-snp cert_chain from {request_url}"))? + .error_for_status() + .with_context(|| format!("amd sev-snp cert_chain request failed for {request_url}"))? + .bytes() + .context("failed to read amd sev-snp cert_chain response")?; + extract_ark_ask_from_amd_kds_cert_chain(&chain) +} + +fn amd_kds_request_url(amd_url: &str) -> String { + match std::env::var("DSTACK_AMD_KDS_PROXY_URL") { + Ok(proxy) if !proxy.trim().is_empty() => format!("{}{}", proxy.trim(), amd_url), + _ => amd_url.to_string(), + } +} + +fn amd_kds_vcek_url(product: &str, chip_id: &[u8; 64], tcb: AmdSnpTcbVersion) -> String { + format!( + "https://kdsintf.amd.com/vcek/v1/{}/{}?blSPL={}&teeSPL={}&snpSPL={}&ucodeSPL={}", + product, + hex::encode(chip_id), + tcb.bootloader, + tcb.tee, + tcb.snp, + tcb.microcode + ) +} + +fn extract_ark_ask_from_amd_kds_cert_chain(chain: &[u8]) -> Result<(CertBytes, CertBytes)> { + let certs = extract_pem_certs(chain)?; + if certs.len() < 2 { + bail!("amd sev-snp cert_chain must contain ASK and ARK certificates"); + } + Ok(( + CertBytes { + bytes: certs[1].clone(), + encoding: CertEncoding::Pem, + }, + CertBytes { + bytes: certs[0].clone(), + encoding: CertEncoding::Pem, + }, + )) +} + +fn extract_pem_certs(chain: &[u8]) -> Result>> { + let chain = std::str::from_utf8(chain).context("amd sev-snp cert_chain is not utf-8 pem")?; + let begin = "-----BEGIN CERTIFICATE-----"; + let end = "-----END CERTIFICATE-----"; + let mut rest = chain; + let mut certs = Vec::new(); + while let Some(start) = rest.find(begin) { + let after_start = &rest[start..]; + let cert_end = after_start + .find(end) + .map(|idx| idx + end.len()) + .context("amd sev-snp cert_chain has unterminated certificate")?; + let mut cert = after_start.as_bytes()[..cert_end].to_vec(); + cert.push(b'\n'); + certs.push(cert); + rest = &after_start[cert_end..]; + } + if certs.is_empty() { + bail!("amd sev-snp cert_chain missing certificates"); + } + Ok(certs) +} + +fn parse_certificate(cert: &CertBytes, name: &str) -> Result { + match cert.encoding { + CertEncoding::Pem => Certificate::from_pem(&cert.bytes) + .map_err(|err| anyhow::anyhow!("failed to parse amd {name} certificate: {err:?}")), + CertEncoding::Der => Certificate::from_der(&cert.bytes) + .map_err(|err| anyhow::anyhow!("failed to parse amd {name} certificate: {err:?}")), + } +} + +fn validate_amd_snp_report_policy(report: &AttestationReport) -> Result<()> { + if !matches!(report.version, 2 | 3) { + bail!("unsupported amd sev-snp report version: {}", report.version); + } + if report.vmpl != 0 { + bail!("amd sev-snp report must be generated at vmpl0"); + } + if report.policy.debug_allowed() { + bail!("amd sev-snp guest policy allows debug"); + } + if report.policy.migrate_ma_allowed() { + bail!("amd sev-snp guest policy allows migration agent"); + } + if report.key_info.mask_chip_key() { + bail!("amd sev-snp report masks the chip signing key"); + } + if report.key_info.signing_key() != 0 { + bail!( + "unsupported amd sev-snp signing key: expected vcek, got {}", + report.key_info.signing_key() + ); + } + if !report.policy.smt_allowed() && report.plat_info.smt_enabled() { + bail!("amd sev-snp platform has smt enabled but guest policy does not allow smt"); + } + if report.policy.rapl_dis() && !report.plat_info.rapl_disabled() { + bail!("amd sev-snp guest policy requires rapl disabled, but platform reports rapl enabled"); + } + if report.policy.ciphertext_hiding() && !report.plat_info.ciphertext_hiding_enabled() { + bail!( + "amd sev-snp guest policy requires ciphertext hiding, but platform does not report it" + ); + } + Ok(()) +} + +fn normalize_ask_vcek_certs(cert_chain: &[Vec]) -> Result<(CertBytes, CertBytes)> { + match cert_chain { + [ask, vcek] => Ok((cert_bytes_from_blob(ask), cert_bytes_from_blob(vcek))), + [auxblob] => normalize_kernel_cert_table(auxblob), + _ => bail!( + "amd sev-snp cert_chain must contain either ASK and VCEK certificates or one kernel certificate table auxblob" + ), + } +} + +fn cert_bytes_from_blob(blob: &[u8]) -> CertBytes { + let encoding = if blob.starts_with(b"-----BEGIN CERTIFICATE-----") { + CertEncoding::Pem + } else { + CertEncoding::Der + }; + CertBytes { + bytes: blob.to_vec(), + encoding, + } +} + +fn normalize_kernel_cert_table(auxblob: &[u8]) -> Result<(CertBytes, CertBytes)> { + let mut ask = None; + let mut vcek = None; + for (guid, data) in parse_kernel_cert_table(auxblob)? { + match guid { + ASK_CERT_GUID => ask = Some(data), + VCEK_CERT_GUID => vcek = Some(data), + VLEK_CERT_GUID => bail!("amd sev-snp vlek certificates are not supported yet"), + _ => {} + } + } + let ask = ask.context("amd sev-snp certificate table missing ASK certificate")?; + let vcek = vcek.context("amd sev-snp certificate table missing VCEK certificate")?; + Ok(( + CertBytes { + bytes: ask, + encoding: CertEncoding::Der, + }, + CertBytes { + bytes: vcek, + encoding: CertEncoding::Der, + }, + )) +} + +fn parse_kernel_cert_table(auxblob: &[u8]) -> Result)>> { + if auxblob.len() < CERT_TABLE_ENTRY_SIZE { + bail!("amd sev-snp certificate table is too short"); + } + let mut entries = Vec::new(); + let mut pos = 0usize; + loop { + let entry = auxblob + .get(pos..pos + CERT_TABLE_ENTRY_SIZE) + .context("amd sev-snp certificate table is missing terminator")?; + let guid: [u8; 16] = entry[..16] + .try_into() + .context("amd sev-snp certificate table entry guid has invalid length")?; + let offset = u32::from_le_bytes( + entry[16..20] + .try_into() + .context("amd sev-snp certificate table entry offset has invalid length")?, + ) as usize; + let length = u32::from_le_bytes( + entry[20..24] + .try_into() + .context("amd sev-snp certificate table entry length has invalid length")?, + ) as usize; + if guid == [0u8; 16] && offset == 0 && length == 0 { + break; + } + let end = offset + .checked_add(length) + .context("amd sev-snp certificate table entry length overflows")?; + if offset < CERT_TABLE_ENTRY_SIZE || end > auxblob.len() || length == 0 { + bail!("amd sev-snp certificate table entry has invalid bounds"); + } + entries.push((guid, auxblob[offset..end].to_vec())); + pos = pos + .checked_add(CERT_TABLE_ENTRY_SIZE) + .context("amd sev-snp certificate table entry count overflows")?; + if pos >= auxblob.len() { + bail!("amd sev-snp certificate table is missing terminator"); + } + } + Ok(entries) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn tcb(bootloader: u8, tee: u8, snp: u8, microcode: u8) -> AmdSnpTcbVersion { + AmdSnpTcbVersion { + bootloader, + tee, + snp, + microcode, + } + } + + #[test] + fn tcb_status_is_up_to_date_only_when_all_reported_versions_match() { + let up_to_date = AmdSnpTcbInfo { + current: tcb(1, 2, 3, 4), + reported: tcb(1, 2, 3, 4), + committed: tcb(1, 2, 3, 4), + launch: tcb(1, 2, 3, 4), + }; + assert_eq!(up_to_date.tcb_status(), "UpToDate"); + + let stale_launch = AmdSnpTcbInfo { + launch: tcb(1, 2, 3, 3), + ..up_to_date + }; + assert_eq!(stale_launch.tcb_status(), "OutOfDate"); + + let stale_vcek_reported = AmdSnpTcbInfo { + reported: tcb(1, 2, 3, 3), + ..up_to_date + }; + assert_eq!(stale_vcek_reported.tcb_status(), "OutOfDate"); + } + + #[test] + fn missing_cert_chain_fails_closed() { + let report = vec![0u8; 1184]; + let expected_report_data = [0u8; 64]; + let err = verify_amd_snp_evidence(&report, &[], &expected_report_data).unwrap_err(); + assert!( + err.to_string() + .contains("cert_chain must contain either ASK and VCEK certificates or one kernel certificate table auxblob"), + "unexpected error: {err:#}" + ); + } + + #[test] + fn amd_kds_vcek_url_binds_chip_id_and_reported_tcb() { + let chip_id = [0xab; 64]; + let tcb = AmdSnpTcbVersion { + bootloader: 1, + tee: 2, + snp: 3, + microcode: 4, + }; + + let url = amd_kds_vcek_url("Genoa", &chip_id, tcb); + + assert_eq!( + url, + format!( + "https://kdsintf.amd.com/vcek/v1/Genoa/{}?blSPL=1&teeSPL=2&snpSPL=3&ucodeSPL=4", + hex::encode(chip_id) + ) + ); + } + + #[test] + fn amd_kds_proxy_url_wraps_amd_urls_when_configured() { + const ENV_KEY: &str = "DSTACK_AMD_KDS_PROXY_URL"; + let old = std::env::var(ENV_KEY).ok(); + std::env::set_var(ENV_KEY, "https://cors.litgateway.com/"); + + let wrapped = amd_kds_request_url("https://kdsintf.amd.com/vcek/v1/Genoa/cert_chain"); + + assert_eq!( + wrapped, + "https://cors.litgateway.com/https://kdsintf.amd.com/vcek/v1/Genoa/cert_chain" + ); + if let Some(old) = old { + std::env::set_var(ENV_KEY, old); + } else { + std::env::remove_var(ENV_KEY); + } + } + + #[test] + fn amd_kds_cert_chain_extracts_ask_pem_and_ark_pem() { + let chain = b"-----BEGIN CERTIFICATE-----\nASK\n-----END CERTIFICATE-----\n-----BEGIN CERTIFICATE-----\nARK\n-----END CERTIFICATE-----\n"; + + let (ark_cert, ask_cert) = extract_ark_ask_from_amd_kds_cert_chain(chain).unwrap(); + + assert_eq!( + ask_cert.bytes, + b"-----BEGIN CERTIFICATE-----\nASK\n-----END CERTIFICATE-----\n".to_vec() + ); + assert_eq!(ask_cert.encoding, CertEncoding::Pem); + assert_eq!( + ark_cert.bytes, + b"-----BEGIN CERTIFICATE-----\nARK\n-----END CERTIFICATE-----\n".to_vec() + ); + assert_eq!(ark_cert.encoding, CertEncoding::Pem); + } + + #[test] + fn malformed_report_fails_closed_before_success() { + let cert_chain = vec![b"not ask".to_vec(), b"not vcek".to_vec()]; + let expected_report_data = [0u8; 64]; + let err = + verify_amd_snp_evidence(b"too short", &cert_chain, &expected_report_data).unwrap_err(); + assert!( + err.to_string() + .contains("invalid amd sev-snp report length"), + "unexpected error: {err:#}" + ); + } + + #[test] + fn normalizes_kernel_cert_table_auxblob_to_ask_and_vcek_der() { + use sev::firmware::host::{CertTableEntry, CertType}; + + let auxblob = CertTableEntry::cert_table_to_vec_bytes(&[ + CertTableEntry::new(CertType::VCEK, b"vcek-der".to_vec()), + CertTableEntry::new(CertType::ASK, b"ask-der".to_vec()), + ]) + .unwrap(); + + let (ask, vcek) = normalize_ask_vcek_certs(&[auxblob]).unwrap(); + + assert_eq!(ask.bytes, b"ask-der"); + assert_eq!(ask.encoding, CertEncoding::Der); + assert_eq!(vcek.bytes, b"vcek-der"); + assert_eq!(vcek.encoding, CertEncoding::Der); + } + + #[test] + fn malformed_single_auxblob_fails_closed_without_panic() { + let err = normalize_ask_vcek_certs(&[vec![0xff; 23]]).unwrap_err(); + + assert!( + err.to_string().contains("certificate table"), + "unexpected error: {err:#}" + ); + } + + #[test] + fn normalizes_existing_two_item_pem_chain_without_reordering() { + let ask = b"-----BEGIN CERTIFICATE-----\nask\n-----END CERTIFICATE-----\n".to_vec(); + let vcek = b"-----BEGIN CERTIFICATE-----\nvcek\n-----END CERTIFICATE-----\n".to_vec(); + + let (normalized_ask, normalized_vcek) = + normalize_ask_vcek_certs(&[ask.clone(), vcek.clone()]).unwrap(); + + assert_eq!(normalized_ask.bytes, ask); + assert_eq!(normalized_ask.encoding, CertEncoding::Pem); + assert_eq!(normalized_vcek.bytes, vcek); + assert_eq!(normalized_vcek.encoding, CertEncoding::Pem); + } + + #[test] + fn report_policy_rejects_debug_allowed() { + let mut report = base_report(); + report.policy.set_debug_allowed(true); + + let err = validate_amd_snp_report_policy(&report).unwrap_err(); + + assert!( + err.to_string().contains("debug"), + "unexpected error: {err:#}" + ); + } + + #[test] + fn report_policy_rejects_non_vmpl0() { + let mut report = base_report(); + report.vmpl = 1; + + let err = validate_amd_snp_report_policy(&report).unwrap_err(); + + assert!( + err.to_string().contains("vmpl0"), + "unexpected error: {err:#}" + ); + } + + #[test] + fn report_policy_accepts_strict_vcek_vmpl0_report() { + let report = base_report(); + + validate_amd_snp_report_policy(&report).unwrap(); + } + + fn base_report() -> AttestationReport { + AttestationReport { + version: 2, + ..Default::default() + } + } +} diff --git a/dstack-attest/src/attestation.rs b/dstack-attest/src/attestation.rs index 55611b5f..6b9b0870 100644 --- a/dstack-attest/src/attestation.rs +++ b/dstack-attest/src/attestation.rs @@ -29,7 +29,10 @@ use sha2::Digest as _; pub use crate::v1::{Attestation as AttestationV1, PlatformEvidence, StackEvidence}; +pub const SNP_REPORT_DATA_RANGE: std::ops::Range = 0x50..0x90; + const DSTACK_TDX: &str = "dstack-tdx"; +const DSTACK_AMD_SEV_SNP: &str = "dstack-amd-sev-snp"; const DSTACK_GCP_TDX: &str = "dstack-gcp-tdx"; const DSTACK_NITRO_ENCLAVE: &str = "dstack-nitro-enclave"; #[cfg(feature = "quote")] @@ -82,6 +85,9 @@ fn platform_from_legacy_quote(quote: AttestationQuote) -> PlatformEvidence { AttestationQuote::DstackTdx(TdxQuote { quote, event_log }) => { PlatformEvidence::Tdx { quote, event_log } } + AttestationQuote::DstackAmdSevSnp(SnpQuote { report, cert_chain }) => { + PlatformEvidence::SevSnp { report, cert_chain } + } AttestationQuote::DstackGcpTdx => PlatformEvidence::GcpTdx, AttestationQuote::DstackNitroEnclave => PlatformEvidence::NitroEnclave, } @@ -92,6 +98,9 @@ fn platform_into_legacy_quote(platform: PlatformEvidence) -> AttestationQuote { PlatformEvidence::Tdx { quote, event_log } => { AttestationQuote::DstackTdx(TdxQuote { quote, event_log }) } + PlatformEvidence::SevSnp { report, cert_chain } => { + AttestationQuote::DstackAmdSevSnp(SnpQuote { report, cert_chain }) + } PlatformEvidence::GcpTdx => AttestationQuote::DstackGcpTdx, PlatformEvidence::NitroEnclave => AttestationQuote::DstackNitroEnclave, } @@ -100,6 +109,7 @@ fn platform_into_legacy_quote(platform: PlatformEvidence) -> AttestationQuote { fn platform_attestation_mode(platform: &PlatformEvidence) -> AttestationMode { match platform { PlatformEvidence::Tdx { .. } => AttestationMode::DstackTdx, + PlatformEvidence::SevSnp { .. } => AttestationMode::DstackAmdSevSnp, PlatformEvidence::GcpTdx => AttestationMode::DstackGcpTdx, PlatformEvidence::NitroEnclave => AttestationMode::DstackNitroEnclave, } @@ -151,22 +161,43 @@ pub enum AttestationMode { /// Dstack attestation SDK in AWS Nitro Enclave #[serde(rename = "dstack-nitro-enclave")] DstackNitroEnclave, + /// AMD SEV-SNP report generated by the dstack attestation SDK. + /// Keep this last to preserve SCALE discriminants for existing variants. + #[serde(rename = "dstack-amd-sev-snp")] + DstackAmdSevSnp, +} + +#[cfg(feature = "quote")] +fn has_sev_snp_tsm_provider() -> bool { + crate::sev_snp::has_sev_snp_tsm_provider(std::path::Path::new("/sys/kernel/config/tsm/report")) +} + +#[cfg(not(feature = "quote"))] +fn has_sev_snp_tsm_provider() -> bool { + false +} + +fn choose_dstack_attestation_mode(has_tdx: bool, has_sev_snp: bool) -> Result { + if has_tdx { + return Ok(AttestationMode::DstackTdx); + } + if has_sev_snp { + return Ok(AttestationMode::DstackAmdSevSnp); + } + bail!("Unsupported platform: Dstack(-tdx/-amd-sev-snp)"); } impl AttestationMode { /// Detect attestation mode from system pub fn detect() -> Result { let has_tdx = std::path::Path::new("/dev/tdx_guest").exists(); + let has_sev_snp = + std::path::Path::new("/dev/sev-guest").exists() || has_sev_snp_tsm_provider(); // First, try to detect platform from DMI product name let platform = Platform::detect_or_dstack(); match platform { - Platform::Dstack => { - if has_tdx { - return Ok(Self::DstackTdx); - } - bail!("Unsupported platform: Dstack(-tdx)"); - } + Platform::Dstack => choose_dstack_attestation_mode(has_tdx, has_sev_snp), Platform::Gcp => { // GCP platform: TDX + TPM dual mode if has_tdx { @@ -182,6 +213,7 @@ impl AttestationMode { pub fn has_tdx(&self) -> bool { match self { Self::DstackTdx => true, + Self::DstackAmdSevSnp => false, Self::DstackGcpTdx => true, Self::DstackNitroEnclave => false, } @@ -192,6 +224,7 @@ impl AttestationMode { match self { Self::DstackGcpTdx => Some(14), Self::DstackTdx => None, + Self::DstackAmdSevSnp => None, Self::DstackNitroEnclave => None, } } @@ -200,6 +233,7 @@ impl AttestationMode { pub fn as_str(&self) -> &'static str { match self { Self::DstackTdx => DSTACK_TDX, + Self::DstackAmdSevSnp => DSTACK_AMD_SEV_SNP, Self::DstackGcpTdx => DSTACK_GCP_TDX, Self::DstackNitroEnclave => DSTACK_NITRO_ENCLAVE, } @@ -209,6 +243,10 @@ impl AttestationMode { pub fn is_composable(&self) -> bool { match self { Self::DstackTdx => true, + // SEV-SNP launch measurement does not provide a TDX RTMR3-equivalent + // runtime event extension path yet, so runtime events are + // informational until an SNP-specific app binding is added. + Self::DstackAmdSevSnp => false, Self::DstackGcpTdx => true, Self::DstackNitroEnclave => false, } @@ -292,6 +330,7 @@ impl QuoteContentType<'_> { #[derive(Clone)] pub enum DstackVerifiedReport { DstackTdx(TdxVerifiedReport), + DstackAmdSevSnp(crate::amd_sev_snp::VerifiedAmdSnpReport), DstackGcpTdx, DstackNitroEnclave, } @@ -300,10 +339,20 @@ impl DstackVerifiedReport { pub fn tdx_report(&self) -> Option<&TdxVerifiedReport> { match self { DstackVerifiedReport::DstackTdx(report) => Some(report), + DstackVerifiedReport::DstackAmdSevSnp(_) => None, DstackVerifiedReport::DstackGcpTdx => None, DstackVerifiedReport::DstackNitroEnclave => None, } } + + pub fn amd_snp_report(&self) -> Option<&crate::amd_sev_snp::VerifiedAmdSnpReport> { + match self { + DstackVerifiedReport::DstackAmdSevSnp(report) => Some(report), + DstackVerifiedReport::DstackTdx(_) + | DstackVerifiedReport::DstackGcpTdx + | DstackVerifiedReport::DstackNitroEnclave => None, + } + } } /// Represents a verified attestation @@ -318,6 +367,15 @@ pub struct TdxQuote { pub event_log: Vec, } +/// Represents an AMD SEV-SNP attestation report. +#[derive(Clone, Encode, Decode)] +pub struct SnpQuote { + /// Raw SNP report bytes. + pub report: Vec, + /// Optional certificate chain blobs, when exposed by the kernel/firmware path. + pub cert_chain: Vec>, +} + /// Represents an NSM (Nitro Security Module) attestation document #[derive(Clone, Encode, Decode)] pub struct NsmQuote { @@ -535,7 +593,9 @@ impl AttestationV1 { PlatformEvidence::Tdx { quote, .. } => { decode_mr_tdx_from_quote(boottime_mr, &mr_key_provider, quote, runtime_events)? } - PlatformEvidence::GcpTdx | PlatformEvidence::NitroEnclave => { + PlatformEvidence::SevSnp { .. } + | PlatformEvidence::GcpTdx + | PlatformEvidence::NitroEnclave => { bail!("Unsupported attestation quote"); } }; @@ -601,6 +661,15 @@ impl AttestationV1 { verify_tdx_quote_with_events(pccs_url, quote, &runtime_events, &report_data) .await?, ), + PlatformEvidence::SevSnp { report, cert_chain } => { + DstackVerifiedReport::DstackAmdSevSnp( + crate::amd_sev_snp::verify_amd_snp_evidence_with_kds_fallback( + report, + cert_chain, + &report_data, + )?, + ) + } PlatformEvidence::GcpTdx | PlatformEvidence::NitroEnclave => { bail!( "Unsupported attestation mode: {:?}", @@ -642,18 +711,62 @@ pub enum AttestationQuote { DstackTdx(TdxQuote), DstackGcpTdx, DstackNitroEnclave, + /// Keep this last to preserve SCALE discriminants for existing variants. + DstackAmdSevSnp(SnpQuote), } impl AttestationQuote { pub fn mode(&self) -> AttestationMode { match self { AttestationQuote::DstackTdx { .. } => AttestationMode::DstackTdx, + AttestationQuote::DstackAmdSevSnp { .. } => AttestationMode::DstackAmdSevSnp, AttestationQuote::DstackGcpTdx => AttestationMode::DstackGcpTdx, AttestationQuote::DstackNitroEnclave => AttestationMode::DstackNitroEnclave, } } } +#[cfg(test)] +mod compatibility_tests { + use super::*; + use scale::Encode; + + #[test] + fn attestation_mode_scale_discriminants_preserve_existing_wire_values() { + assert_eq!(AttestationMode::DstackTdx.encode(), vec![0]); + assert_eq!(AttestationMode::DstackGcpTdx.encode(), vec![1]); + assert_eq!(AttestationMode::DstackNitroEnclave.encode(), vec![2]); + assert_eq!(AttestationMode::DstackAmdSevSnp.encode(), vec![3]); + } + + #[test] + fn attestation_quote_scale_discriminants_preserve_existing_wire_values() { + assert_eq!(AttestationQuote::DstackGcpTdx.encode(), vec![1]); + assert_eq!(AttestationQuote::DstackNitroEnclave.encode(), vec![2]); + let quote = AttestationQuote::DstackAmdSevSnp(SnpQuote { + report: Vec::new(), + cert_chain: Vec::new(), + }); + assert_eq!(quote.encode()[0], 3); + } + + #[test] + fn dstack_attestation_mode_prefers_tdx_when_both_tdx_and_tsm_exist() { + assert_eq!( + choose_dstack_attestation_mode(true, true).unwrap(), + AttestationMode::DstackTdx + ); + } + + #[test] + fn dstack_attestation_mode_uses_snp_when_only_snp_exists() { + assert_eq!( + choose_dstack_attestation_mode(false, true).unwrap(), + AttestationMode::DstackAmdSevSnp + ); + } +} + /// Attestation data #[derive(Clone, Encode, Decode)] pub struct Attestation { @@ -681,6 +794,7 @@ impl Attestation { pub fn tdx_quote_mut(&mut self) -> Option<&mut TdxQuote> { match &mut self.quote { AttestationQuote::DstackTdx(quote) => Some(quote), + AttestationQuote::DstackAmdSevSnp(_) => None, AttestationQuote::DstackGcpTdx => None, AttestationQuote::DstackNitroEnclave => None, } @@ -689,6 +803,7 @@ impl Attestation { pub fn tdx_quote(&self) -> Option<&TdxQuote> { match &self.quote { AttestationQuote::DstackTdx(quote) => Some(quote), + AttestationQuote::DstackAmdSevSnp(_) => None, AttestationQuote::DstackGcpTdx => None, AttestationQuote::DstackNitroEnclave => None, } @@ -735,6 +850,7 @@ impl GetDeviceId for DstackVerifiedReport { fn get_devide_id(&self) -> Vec { match self { DstackVerifiedReport::DstackTdx(tdx_report) => tdx_report.ppid.to_vec(), + DstackVerifiedReport::DstackAmdSevSnp(report) => report.chip_id.to_vec(), DstackVerifiedReport::DstackGcpTdx => Vec::new(), DstackVerifiedReport::DstackNitroEnclave => Vec::new(), } @@ -909,8 +1025,10 @@ impl Attestation { AttestationQuote::DstackTdx(q) => { self.decode_mr_tdx(boottime_mr, &mr_key_provider, q)? } - AttestationQuote::DstackGcpTdx | AttestationQuote::DstackNitroEnclave => { - bail!("Unsupported attestation quote"); + AttestationQuote::DstackAmdSevSnp(_) + | AttestationQuote::DstackGcpTdx + | AttestationQuote::DstackNitroEnclave => { + bail!("unsupported attestation quote for app info decoding"); } }; let compose_hash = if self.quote.mode().is_composable() { @@ -1048,12 +1166,17 @@ impl Attestation { cc_eventlog::tdx::read_event_log().context("Failed to read event log")?; AttestationQuote::DstackTdx(TdxQuote { quote, event_log }) } + AttestationMode::DstackAmdSevSnp => { + let quote = crate::sev_snp::get_report(*report_data) + .context("Failed to get SEV-SNP report")?; + AttestationQuote::DstackAmdSevSnp(quote) + } AttestationMode::DstackGcpTdx | AttestationMode::DstackNitroEnclave => { bail!("Unsupported attestation mode: {mode:?}"); } }; let config = match "e { - AttestationQuote::DstackTdx(_) => { + AttestationQuote::DstackAmdSevSnp(_) | AttestationQuote::DstackTdx(_) => { read_vm_config().context("Failed to read VM config")? } AttestationQuote::DstackGcpTdx | AttestationQuote::DstackNitroEnclave => { @@ -1087,6 +1210,9 @@ impl Attestation { let report = self.verify_tdx(pccs_url, &q.quote).await?; DstackVerifiedReport::DstackTdx(report) } + AttestationQuote::DstackAmdSevSnp(_) => { + bail!("Unsupported attestation mode: {:?}", self.quote.mode()); + } AttestationQuote::DstackGcpTdx | AttestationQuote::DstackNitroEnclave => { bail!("Unsupported attestation mode: {:?}", self.quote.mode()); } diff --git a/dstack-attest/src/lib.rs b/dstack-attest/src/lib.rs index 1f7bc814..27f322d9 100644 --- a/dstack-attest/src/lib.rs +++ b/dstack-attest/src/lib.rs @@ -10,7 +10,10 @@ pub use tdx_attest as tdx; use crate::attestation::AttestationMode; +pub mod amd_sev_snp; pub mod attestation; +#[cfg(feature = "quote")] +mod sev_snp; mod v1; /// Emit a runtime event that extends RTMR3 and logs the event. diff --git a/dstack-attest/src/sev_snp.rs b/dstack-attest/src/sev_snp.rs new file mode 100644 index 00000000..cc1f6b09 --- /dev/null +++ b/dstack-attest/src/sev_snp.rs @@ -0,0 +1,283 @@ +// SPDX-FileCopyrightText: © 2025 Phala Network +// +// SPDX-License-Identifier: Apache-2.0 + +//! Minimal AMD SEV-SNP guest report support. + +use std::path::Path; + +use anyhow::{bail, Context, Result}; +use sev::firmware::{guest::Firmware, host::CertTableEntry}; + +use crate::attestation::{SnpQuote, SNP_REPORT_DATA_RANGE}; + +const TSM_REPORT_ROOT: &str = "/sys/kernel/config/tsm/report"; +const SEV_GUEST_DEVICE: &str = "/dev/sev-guest"; +const SNP_REPORT_SIZE: usize = 1184; + +pub fn get_report(report_data: [u8; 64]) -> Result { + if has_sev_snp_tsm_provider(Path::new(TSM_REPORT_ROOT)) { + match get_report_configfs(report_data) { + Ok(quote) => { + if configfs_report_needs_ioctl_cert_chain_fallback( + "e, + Path::new(SEV_GUEST_DEVICE).exists(), + ) { + tracing::debug!( + "sev-snp configfs tsm report did not include a certificate chain; falling back to ioctl extended report" + ); + match get_report_ioctl(report_data) { + Ok(ioctl_quote) if !ioctl_quote.cert_chain.is_empty() => { + return Ok(ioctl_quote) + } + Ok(_) => return Ok(quote), + Err(err) => tracing::debug!( + "failed to get sev-snp report from ioctl fallback: {err:#}" + ), + } + } + return Ok(quote); + } + Err(err) => tracing::debug!("failed to get sev-snp report from configfs tsm: {err:#}"), + } + } + if Path::new(SEV_GUEST_DEVICE).exists() { + return get_report_ioctl(report_data); + } + bail!("sev-snp report is unavailable: neither {TSM_REPORT_ROOT} nor {SEV_GUEST_DEVICE} exists") +} + +fn configfs_report_needs_ioctl_cert_chain_fallback( + quote: &SnpQuote, + sev_guest_device_available: bool, +) -> bool { + sev_guest_device_available && quote.cert_chain.is_empty() +} + +pub(crate) fn has_sev_snp_tsm_provider(root: &Path) -> bool { + if !root.exists() { + return false; + } + + if provider_file_is_sev_guest(&root.join("provider")) { + return true; + } + + let probe = root.join(format!("dstack-probe-{}", std::process::id())); + if fs_err::create_dir(&probe).is_ok() { + let is_sev_snp = provider_file_is_sev_guest(&probe.join("provider")); + let _ = fs_err::remove_dir(&probe); + if is_sev_snp { + return true; + } + } + + let Ok(entries) = fs_err::read_dir(root) else { + return false; + }; + entries.flatten().any(|entry| { + let Ok(file_type) = entry.file_type() else { + return false; + }; + file_type.is_dir() && provider_file_is_sev_guest(&entry.path().join("provider")) + }) +} + +fn provider_file_is_sev_guest(path: &Path) -> bool { + fs_err::read_to_string(path) + .map(|provider| matches!(provider.trim(), "sev_guest" | "sev-guest")) + .unwrap_or(false) +} + +fn get_report_configfs(report_data: [u8; 64]) -> Result { + let root = Path::new(TSM_REPORT_ROOT); + let dir = root.join(format!("dstack-{}", std::process::id())); + if !dir.exists() { + fs_err::create_dir(&dir).with_context(|| format!("failed to create {}", dir.display()))?; + } + + let hex_report_data = hex::encode(report_data); + write_first_existing( + &[ + dir.join("inblob"), + dir.join("reportdata"), + dir.join("report_data"), + ], + &report_data, + hex_report_data.as_bytes(), + )?; + + let report = read_first_existing(&[dir.join("outblob"), dir.join("report")])?; + if report.is_empty() { + bail!("sev-snp configfs tsm returned an empty report"); + } + ensure_report_data_matches(&report, &report_data)?; + Ok(SnpQuote { + report, + cert_chain: read_cert_chain_configfs(&dir), + }) +} + +fn write_first_existing(paths: &[std::path::PathBuf], binary: &[u8], hex: &[u8]) -> Result<()> { + let mut last_err = None; + for path in paths { + if !path.exists() { + continue; + } + match fs_err::write(path, binary).or_else(|_| fs_err::write(path, hex)) { + Ok(()) => return Ok(()), + Err(err) => last_err = Some(err), + } + } + match last_err { + Some(err) => Err(err).context("failed to write sev-snp tsm report data"), + None => bail!("failed to find sev-snp tsm report input file"), + } +} + +fn read_first_existing(paths: &[std::path::PathBuf]) -> Result> { + for path in paths { + if path.exists() { + return fs_err::read(path) + .with_context(|| format!("failed to read {}", path.display())); + } + } + bail!("failed to find sev-snp tsm report output file") +} + +fn read_cert_chain_configfs(dir: &Path) -> Vec> { + for name in ["certs", "cert_chain", "auxblob"] { + let Ok(bytes) = fs_err::read(dir.join(name)) else { + continue; + }; + if !bytes.is_empty() { + return vec![bytes]; + } + } + Vec::new() +} + +fn get_report_ioctl(report_data: [u8; 64]) -> Result { + let mut firmware = + Firmware::open().with_context(|| format!("failed to open {SEV_GUEST_DEVICE}"))?; + let (report, cert_entries) = firmware + .get_ext_report(Some(1), Some(report_data), Some(0)) + .map_err(|err| anyhow::anyhow!("sev-snp get extended report ioctl failed: {err}"))?; + ensure_report_data_matches(&report, &report_data)?; + let cert_chain = match cert_entries { + Some(entries) if !entries.is_empty() => { + vec![CertTableEntry::cert_table_to_vec_bytes(&entries) + .context("failed to encode sev-snp certificate table")?] + } + _ => Vec::new(), + }; + Ok(SnpQuote { report, cert_chain }) +} + +fn ensure_report_data_matches(report: &[u8], report_data: &[u8; 64]) -> Result<()> { + if report.len() != SNP_REPORT_SIZE { + bail!( + "sev-snp report has invalid length: expected {} bytes, got {}", + SNP_REPORT_SIZE, + report.len() + ); + } + if &report[SNP_REPORT_DATA_RANGE] != report_data { + bail!("sev-snp report_data mismatch"); + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + use std::time::{SystemTime, UNIX_EPOCH}; + + #[test] + fn rejects_report_with_wrong_report_data() { + let expected = [0x42; 64]; + let mut report = vec![0u8; SNP_REPORT_SIZE]; + report[SNP_REPORT_DATA_RANGE].copy_from_slice(&[0x24; 64]); + assert!(ensure_report_data_matches(&report, &expected).is_err()); + } + + #[test] + fn accepts_report_with_matching_report_data() { + let expected = [0x42; 64]; + let mut report = vec![0u8; SNP_REPORT_SIZE]; + report[SNP_REPORT_DATA_RANGE].copy_from_slice(&expected); + ensure_report_data_matches(&report, &expected).unwrap(); + } + + #[test] + fn tsm_provider_detection_accepts_only_sev_guest_provider() { + let root = test_dir("sev-guest"); + fs_err::create_dir_all(root.join("entry")).unwrap(); + fs_err::write(root.join("entry/provider"), "sev_guest\n").unwrap(); + + assert!(has_sev_snp_tsm_provider(&root)); + + let _ = fs_err::remove_dir_all(root); + } + + #[test] + fn tsm_provider_detection_accepts_legacy_hyphenated_sev_guest_provider() { + let root = test_dir("sev-guest-hyphen"); + fs_err::create_dir_all(root.join("entry")).unwrap(); + fs_err::write(root.join("entry/provider"), "sev-guest\n").unwrap(); + + assert!(has_sev_snp_tsm_provider(&root)); + + let _ = fs_err::remove_dir_all(root); + } + + #[test] + fn tsm_provider_detection_rejects_tdx_guest_provider() { + let root = test_dir("tdx-guest"); + fs_err::create_dir_all(root.join("entry")).unwrap(); + fs_err::write(root.join("entry/provider"), "tdx-guest\n").unwrap(); + + assert!(!has_sev_snp_tsm_provider(&root)); + + let _ = fs_err::remove_dir_all(root); + } + + #[test] + fn configfs_cert_chain_uses_first_supported_nonempty_blob() { + let root = test_dir("cert-chain"); + fs_err::create_dir_all(&root).unwrap(); + fs_err::write(root.join("certs"), []).unwrap(); + fs_err::write(root.join("cert_chain"), b"chain").unwrap(); + fs_err::write(root.join("auxblob"), b"auxblob").unwrap(); + + assert_eq!(read_cert_chain_configfs(&root), vec![b"chain".to_vec()]); + + let _ = fs_err::remove_dir_all(root); + } + + #[test] + fn configfs_report_without_cert_chain_requires_ioctl_fallback_when_available() { + let quote = SnpQuote { + report: vec![0u8; SNP_REPORT_SIZE], + cert_chain: vec![], + }; + + assert!(configfs_report_needs_ioctl_cert_chain_fallback( + "e, true + )); + assert!(!configfs_report_needs_ioctl_cert_chain_fallback( + "e, false + )); + } + + fn test_dir(name: &str) -> std::path::PathBuf { + let nanos = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_nanos(); + std::env::temp_dir().join(format!( + "dstack-sev-snp-test-{name}-{}-{nanos}", + std::process::id() + )) + } +} diff --git a/dstack-attest/src/v1.rs b/dstack-attest/src/v1.rs index 900e7aed..62206b36 100644 --- a/dstack-attest/src/v1.rs +++ b/dstack-attest/src/v1.rs @@ -16,6 +16,11 @@ pub enum PlatformEvidence { quote: Vec, event_log: Vec, }, + #[serde(rename = "sev-snp")] + SevSnp { + report: Vec, + cert_chain: Vec>, + }, #[serde(rename = "gcp-tdx")] GcpTdx, #[serde(rename = "nitro-enclave")] @@ -37,6 +42,20 @@ impl PlatformEvidence { } } + pub fn sev_snp_report(&self) -> Option<&[u8]> { + match self { + Self::SevSnp { report, .. } => Some(report.as_slice()), + _ => None, + } + } + + pub fn sev_snp_cert_chain(&self) -> Option<&[Vec]> { + match self { + Self::SevSnp { cert_chain, .. } => Some(cert_chain.as_slice()), + _ => None, + } + } + pub fn into_stripped(self) -> Self { match self { Self::Tdx { quote, event_log } => Self::Tdx { @@ -192,7 +211,7 @@ impl Attestation { /// Return a new attestation with the report_data patched in both platform quote and stack. pub fn with_report_data(self, report_data: [u8; 64]) -> Self { - use crate::attestation::TDX_QUOTE_REPORT_DATA_RANGE; + use crate::attestation::{SNP_REPORT_DATA_RANGE, TDX_QUOTE_REPORT_DATA_RANGE}; let platform = match self.platform { PlatformEvidence::Tdx { @@ -204,6 +223,15 @@ impl Attestation { } PlatformEvidence::Tdx { quote, event_log } } + PlatformEvidence::SevSnp { + mut report, + cert_chain, + } => { + if report.len() >= SNP_REPORT_DATA_RANGE.end { + report[SNP_REPORT_DATA_RANGE].copy_from_slice(&report_data); + } + PlatformEvidence::SevSnp { report, cert_chain } + } other => other, }; let stack = match self.stack { @@ -294,4 +322,55 @@ mod tests { _ => panic!("expected dstack-pod stack evidence"), } } + + #[test] + fn sev_snp_msgpack_roundtrip_preserves_evidence() { + let attestation = Attestation::new( + PlatformEvidence::SevSnp { + report: vec![0x11; 1184], + cert_chain: vec![vec![0x22, 0x33]], + }, + StackEvidence::Dstack { + report_data: vec![9u8; 64], + runtime_events: vec![], + config: "{}".into(), + }, + ); + + let encoded = attestation.to_msgpack().expect("encode msgpack"); + let decoded = Attestation::from_msgpack(&encoded).expect("decode msgpack"); + assert_eq!( + decoded.platform.sev_snp_report(), + Some(vec![0x11; 1184].as_slice()) + ); + assert_eq!( + decoded.platform.sev_snp_cert_chain(), + Some(vec![vec![0x22, 0x33]].as_slice()) + ); + } + + #[test] + fn sev_snp_with_report_data_patches_report_and_stack() { + let mut report = vec![0x11; 1184]; + report[crate::attestation::SNP_REPORT_DATA_RANGE].copy_from_slice(&[0x22; 64]); + let attestation = Attestation::new( + PlatformEvidence::SevSnp { + report, + cert_chain: vec![], + }, + StackEvidence::Dstack { + report_data: vec![0x22; 64], + runtime_events: vec![], + config: "{}".into(), + }, + ); + + let patched = attestation.with_report_data([0x33; 64]); + assert_eq!(patched.report_data().unwrap(), [0x33; 64]); + let report = patched.platform.sev_snp_report().unwrap(); + assert_eq!( + &report[crate::attestation::SNP_REPORT_DATA_RANGE], + &[0x33; 64] + ); + } } diff --git a/dstack-util/src/main.rs b/dstack-util/src/main.rs index 05ad9ad6..2302be16 100644 --- a/dstack-util/src/main.rs +++ b/dstack-util/src/main.rs @@ -4,7 +4,7 @@ use anyhow::{Context, Result}; use clap::{Parser, Subcommand}; -use dstack_attest::emit_runtime_event; +use dstack_attest::{attestation::AttestationMode, emit_runtime_event}; use dstack_types::{KeyProvider, KeyProviderKind}; use fs_err as fs; use getrandom::fill as getrandom; @@ -240,6 +240,21 @@ fn cmd_rand(rand_args: RandArgs) -> Result<()> { } fn cmd_show_mrs() -> Result<()> { + if AttestationMode::detect()? == AttestationMode::DstackAmdSevSnp { + serde_json::to_writer_pretty( + io::stdout(), + &serde_json::json!({ + "attestation_mode": AttestationMode::DstackAmdSevSnp.as_str(), + "mr_system": null, + "mr_aggregated": null, + "note": "app-info MRs are TDX RTMR-derived and unavailable for AMD SEV-SNP", + }), + ) + .context("Failed to write app info")?; + println!(); + return Ok(()); + } + let attestation = ra_tls::attestation::Attestation::local().context("Failed to get attestation")?; let app_info = attestation diff --git a/dstack-util/src/system_setup.rs b/dstack-util/src/system_setup.rs index 2276ff93..f34e5333 100644 --- a/dstack-util/src/system_setup.rs +++ b/dstack-util/src/system_setup.rs @@ -85,6 +85,12 @@ async fn sign_cert_request( mod config_id_verifier; +fn is_unsupported_app_info_quote(err: &anyhow::Error) -> bool { + let message = format!("{err:#}"); + message.contains("Unsupported attestation quote") + || message.contains("unsupported attestation quote for app info decoding") +} + #[derive(clap::Parser)] /// Prepare full disk encryption pub struct SetupArgs { @@ -852,11 +858,14 @@ impl<'a> Stage0<'a> { bail!("Invalid server cert usage: {usage}"); } if let Some(att) = &cert.attestation { - let kms_info = att - .decode_app_info(false) - .context("Failed to decode app_info")?; - emit_runtime_event("mr-kms", &kms_info.mr_aggregated) - .context("Failed to extend mr-kms to RTMR3")?; + match att.decode_app_info(false) { + Ok(kms_info) => emit_runtime_event("mr-kms", &kms_info.mr_aggregated) + .context("Failed to extend mr-kms to RTMR3")?, + Err(err) if is_unsupported_app_info_quote(&err) => { + warn!("Skipping mr-kms runtime event for unsupported attestation quote: {err:#}"); + } + Err(err) => return Err(err).context("Failed to decode app_info"), + } } Ok(()) })) diff --git a/dstack-util/src/system_setup/config_id_verifier.rs b/dstack-util/src/system_setup/config_id_verifier.rs index c62f665c..d4ffdb2a 100644 --- a/dstack-util/src/system_setup/config_id_verifier.rs +++ b/dstack-util/src/system_setup/config_id_verifier.rs @@ -3,6 +3,7 @@ // SPDX-License-Identifier: Apache-2.0 use anyhow::{bail, Context, Result}; +use dstack_attest::attestation::AttestationMode; use dstack_types::{mr_config::MrConfig, KeyProviderKind}; use tracing::info; @@ -35,6 +36,22 @@ pub fn verify_mr_config_id( key_provider: KeyProviderKind, key_provider_id: &[u8], ) -> Result<()> { + let mode = AttestationMode::detect().context("Failed to detect attestation mode")?; + verify_mr_config_id_for_mode(mode, compose_hash, app_id, key_provider, key_provider_id) +} + +fn verify_mr_config_id_for_mode( + mode: AttestationMode, + compose_hash: &[u8; 32], + app_id: &[u8; 20], + key_provider: KeyProviderKind, + key_provider_id: &[u8], +) -> Result<()> { + if mode == AttestationMode::DstackAmdSevSnp { + info!("Skipping TDX mr_config_id verification for AMD SEV-SNP guest"); + return Ok(()); + } + let read_mr_config_id = read_mr_config_id().context("Failed to read mr_config_id")?; info!("mr_config_id: {}", hex::encode(read_mr_config_id)); if read_mr_config_id == [0u8; 48] { @@ -55,3 +72,20 @@ pub fn verify_mr_config_id( } Ok(()) } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn amd_sev_snp_skips_tdx_mr_config_id_quote_verification() { + verify_mr_config_id_for_mode( + AttestationMode::DstackAmdSevSnp, + &[0u8; 32], + &[0u8; 20], + KeyProviderKind::None, + &[], + ) + .unwrap(); + } +} diff --git a/kms/Cargo.toml b/kms/Cargo.toml index bc33bc6a..b59d3111 100644 --- a/kms/Cargo.toml +++ b/kms/Cargo.toml @@ -48,5 +48,8 @@ serde-duration.workspace = true dstack-verifier = { workspace = true, default-features = false } dstack-mr.workspace = true +[dev-dependencies] +dstack-attest.workspace = true + [features] default = [] diff --git a/kms/auth-simple/README.md b/kms/auth-simple/README.md index dbb425aa..58360895 100644 --- a/kms/auth-simple/README.md +++ b/kms/auth-simple/README.md @@ -38,6 +38,8 @@ Add more fields as you deploy Gateway and apps: ```json { "osImages": ["0x..."], + "allowedTcbStatuses": ["UpToDate"], + "allowedAdvisoryIds": [], "gatewayAppId": "0x...", "kms": { "mrAggregated": ["0x..."], @@ -60,6 +62,8 @@ Add more fields as you deploy Gateway and apps: |-------|----------|-------------| | `osImages` | Yes | Allowed OS image hashes (from `digest.txt`) | | `gatewayAppId` | No | Gateway app ID (add after Gateway deployment) | +| `allowedTcbStatuses` | No | Allowed verifier-derived TCB status strings. Defaults to `["UpToDate"]`; non-up-to-date SNP/TDX statuses remain fail-closed unless explicitly allowlisted for testing. | +| `allowedAdvisoryIds` | No | Advisory IDs permitted in `advisoryIds`. Defaults to `[]`, which rejects any advisory. | | `kms.mrAggregated` | Yes for KMS authorization | Allowed KMS aggregated MR values. An empty array denies all KMS boots. | | `kms.devices` | No | Allowed KMS device IDs | | `kms.allowAnyDevice` | No | If true, skip device ID check for KMS | @@ -67,6 +71,8 @@ Add more fields as you deploy Gateway and apps: | `apps..devices` | No | Allowed device IDs for this app | | `apps..allowAnyDevice` | No | If true, skip device ID check for this app | +For experimental AMD SEV-SNP dry-run authorization, keep the default fail-closed TCB policy unless you intentionally want the auth webhook to accept non-up-to-date verifier-derived SNP `BootInfo`. To exercise the dry-run path without enabling key release, allowlist the recomputed SNP `mrAggregated`, `osImageHash`, app/compose identity, device/chip identity, and any non-default `allowedTcbStatuses`/`allowedAdvisoryIds` values explicitly. KMS still rejects SNP before returning app keys, KMS keys, or app certificates. + ### Getting Hash Values **OS Image Hash:** @@ -128,13 +134,15 @@ App boot authorization. **Request:** ```json { + "attestationMode": "DstackTdx", "mrAggregated": "0x...", "osImageHash": "0x...", "appId": "0x...", "composeHash": "0x...", "instanceId": "0x...", "deviceId": "0x...", - "tcbStatus": "UpToDate" + "tcbStatus": "UpToDate", + "advisoryIds": [] } ``` @@ -159,18 +167,20 @@ KMS boot authorization. ### KMS Boot Validation -1. `tcbStatus` must be "UpToDate" -2. `osImageHash` must be in `osImages` array -3. `mrAggregated` must be in `kms.mrAggregated` -4. `deviceId` must be in `kms.devices` (unless `allowAnyDevice` is true) +1. `tcbStatus` must be listed in `allowedTcbStatuses` (default: only `"UpToDate"`) +2. Every `advisoryIds` entry must be listed in `allowedAdvisoryIds` (default: none allowed) +3. `osImageHash` must be in `osImages` array +4. `mrAggregated` must be in `kms.mrAggregated` +5. `deviceId` must be in `kms.devices` (unless `allowAnyDevice` is true) ### App Boot Validation -1. `tcbStatus` must be "UpToDate" -2. `osImageHash` must be in `osImages` array -3. `appId` must exist in `apps` object -4. `composeHash` must be in app's `composeHashes` array -5. `deviceId` must be in app's `devices` (unless `allowAnyDevice` is true) +1. `tcbStatus` must be listed in `allowedTcbStatuses` (default: only `"UpToDate"`) +2. Every `advisoryIds` entry must be listed in `allowedAdvisoryIds` (default: none allowed) +3. `osImageHash` must be in `osImages` array +4. `appId` must exist in `apps` object +5. `composeHash` must be in app's `composeHashes` array +6. `deviceId` must be in app's `devices` (unless `allowAnyDevice` is true) ## Hot Reload diff --git a/kms/auth-simple/index.test.ts b/kms/auth-simple/index.test.ts index 856a0ded..584d0f9e 100644 --- a/kms/auth-simple/index.test.ts +++ b/kms/auth-simple/index.test.ts @@ -92,6 +92,91 @@ describe('auth-simple', () => { expect(json.reason).toContain('TCB status'); }); + it('requires explicit opt-in for non-UpToDate SEV-SNP TCB status', async () => { + writeTestConfig({ + gatewayAppId: '0xgateway', + osImages: ['0x1fbb0cf9cc6cfbf23d6b779776fabad2c5403d643badb9e5e238615e4960a78a'], + kms: { + mrAggregated: ['0xabc123'], + devices: ['0xdevice999'], + allowAnyDevice: false + } + }); + + const sevSnpBootInfo = { + ...baseBootInfo, + attestationMode: 'DstackAmdSevSnp', + tcbStatus: 'OutOfDate' + }; + const denied = await app.fetch(new Request('http://localhost/bootAuth/kms', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify(sevSnpBootInfo) + })); + expect((await denied.json()).isAllowed).toBe(false); + + writeTestConfig({ + gatewayAppId: '0xgateway', + osImages: ['0x1fbb0cf9cc6cfbf23d6b779776fabad2c5403d643badb9e5e238615e4960a78a'], + allowedTcbStatuses: ['OutOfDate'], + kms: { + mrAggregated: ['0xabc123'], + devices: ['0xdevice999'], + allowAnyDevice: false + } + }); + + const allowed = await app.fetch(new Request('http://localhost/bootAuth/kms', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify(sevSnpBootInfo) + })); + const allowedJson = await allowed.json(); + + expect(allowedJson.isAllowed).toBe(true); + expect(allowedJson.reason).toBe(''); + }); + + it('rejects unallowlisted advisory IDs by default', async () => { + writeTestConfig({ + gatewayAppId: '0xgateway', + osImages: ['0x1fbb0cf9cc6cfbf23d6b779776fabad2c5403d643badb9e5e238615e4960a78a'], + kms: { + mrAggregated: ['0xabc123'], + allowAnyDevice: true + } + }); + + const denied = await app.fetch(new Request('http://localhost/bootAuth/kms', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ ...baseBootInfo, advisoryIds: ['INTEL-SA-TEST'] }) + })); + const deniedJson = await denied.json(); + + expect(deniedJson.isAllowed).toBe(false); + expect(deniedJson.reason).toContain('advisory'); + + writeTestConfig({ + gatewayAppId: '0xgateway', + osImages: ['0x1fbb0cf9cc6cfbf23d6b779776fabad2c5403d643badb9e5e238615e4960a78a'], + allowedAdvisoryIds: ['INTEL-SA-TEST'], + kms: { + mrAggregated: ['0xabc123'], + allowAnyDevice: true + } + }); + + const allowed = await app.fetch(new Request('http://localhost/bootAuth/kms', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ ...baseBootInfo, advisoryIds: ['INTEL-SA-TEST'] }) + })); + const allowedJson = await allowed.json(); + + expect(allowedJson.isAllowed).toBe(true); + }); + it('rejects KMS boot with invalid OS image', async () => { writeTestConfig({ gatewayAppId: '0xgateway', diff --git a/kms/auth-simple/index.ts b/kms/auth-simple/index.ts index 7307f49c..b8d8c86c 100644 --- a/kms/auth-simple/index.ts +++ b/kms/auth-simple/index.ts @@ -9,6 +9,7 @@ import { readFileSync, existsSync } from 'fs'; // zod schemas for validation - compatible with auth-eth implementation const BootInfoSchema = z.object({ + attestationMode: z.string().optional().default(''), mrAggregated: z.string().describe('aggregated MR measurement'), osImageHash: z.string().describe('OS Image hash'), appId: z.string().describe('application ID'), @@ -46,6 +47,10 @@ const AuthConfigSchema = z.object({ chainId: z.number().default(0), appImplementation: z.string().default('0x0000000000000000000000000000000000000000'), osImages: z.array(z.string()).default([]), + // TDX and SEV-SNP production defaults remain strict: only UpToDate is + // accepted unless operators explicitly allow another verifier-derived status. + allowedTcbStatuses: z.array(z.string()).default(['UpToDate']), + allowedAdvisoryIds: z.array(z.string()).default([]), kms: KmsConfigSchema.default({}), apps: z.record(z.string(), AppConfigSchema).default({}) }); @@ -92,14 +97,25 @@ class ConfigBackend { const deviceId = normalizeHex(bootInfo.deviceId); // check TCB status - if (bootInfo.tcbStatus !== 'UpToDate') { + const allowedTcbStatuses = config.allowedTcbStatuses; + if (!allowedTcbStatuses.includes(bootInfo.tcbStatus)) { return { isAllowed: false, - reason: 'TCB status is not up to date', + reason: 'TCB status is not allowed', gatewayAppId: config.gatewayAppId }; } + for (const advisoryId of bootInfo.advisoryIds) { + if (!config.allowedAdvisoryIds.includes(advisoryId)) { + return { + isAllowed: false, + reason: 'advisory ID is not allowed', + gatewayAppId: config.gatewayAppId + }; + } + } + // check OS image const allowedOsImages = config.osImages.map(normalizeHex); if (!allowedOsImages.includes(osImageHash)) { diff --git a/kms/kms.toml b/kms/kms.toml index d3d171b1..83bccde4 100644 --- a/kms/kms.toml +++ b/kms/kms.toml @@ -33,6 +33,15 @@ site_name = "" # is unavailable. enforce_self_authorization = true +# AMD SEV-SNP key/cert release remains disabled unless this local KMS gate is +# explicitly enabled. External auth policy must still allow the verified +# BootInfo before any sensitive material is returned. Enabling this also +# requires enforce_self_authorization = true. +[core.sev_snp_key_release] +enabled = false +allowed_tcb_statuses = ["UpToDate"] +allowed_advisory_ids = [] + [core.image] verify = true cache_dir = "/usr/share/dstack/images" diff --git a/kms/src/config.rs b/kms/src/config.rs index ecdfb9ae..934f2a7c 100644 --- a/kms/src/config.rs +++ b/kms/src/config.rs @@ -31,6 +31,59 @@ pub(crate) struct ImageConfig { pub download_timeout: Duration, } +/// Configuration for AMD SEV-SNP measurement/app binding validation. +#[derive(Debug, Clone, Deserialize)] +pub(crate) struct SevSnpMeasureConfig { + /// Path to the AMD SEV-SNP OVMF binary used for this VM image. + /// + /// Optional when callers provide OVMF section metadata with the request. + pub ovmf_path: Option, + /// Optional diagnostic/cache proxy used for AMD KDS collateral requests. + /// + /// Empty by default. When set, the KMS process exports the same proxy env + /// used by dstack-attest before any attestation verification happens. + #[serde(default)] + pub amd_kds_proxy_url: Option, + /// SNP guest features bitmask used at launch. Defaults to SNP with kernel + /// hashes enabled. + #[serde(default = "default_guest_features")] + pub guest_features: u64, +} + +fn default_guest_features() -> u64 { + 0x1 +} + +#[derive(Debug, Clone, Deserialize)] +pub(crate) struct SevSnpKeyReleaseConfig { + /// Enable AMD SEV-SNP key/cert release after attestation, measurement + /// binding, and external auth-policy checks have all succeeded. + #[serde(default)] + pub enabled: bool, + /// Verifier-derived TCB statuses that are acceptable for releasing + /// sensitive key/cert material. Defaults to the strict production value. + #[serde(default = "default_allowed_tcb_statuses")] + pub allowed_tcb_statuses: Vec, + /// Advisory IDs that are acceptable for releasing sensitive key/cert + /// material. Defaults to empty, which rejects any advisory. + #[serde(default)] + pub allowed_advisory_ids: Vec, +} + +impl Default for SevSnpKeyReleaseConfig { + fn default() -> Self { + Self { + enabled: false, + allowed_tcb_statuses: default_allowed_tcb_statuses(), + allowed_advisory_ids: Vec::new(), + } + } +} + +fn default_allowed_tcb_statuses() -> Vec { + vec!["UpToDate".to_string()] +} + #[derive(Debug, Clone, Deserialize)] pub(crate) struct KmsConfig { pub cert_dir: PathBuf, @@ -38,6 +91,16 @@ pub(crate) struct KmsConfig { pub auth_api: AuthApi, pub onboard: OnboardConfig, pub image: ImageConfig, + /// AMD SEV-SNP measurement verification configuration. Optional at config + /// load time for non-SNP/dev deployments; SNP binding helpers require it. + #[serde(default)] + #[allow(dead_code)] + pub sev_snp: Option, + /// Additional local release gate for AMD SEV-SNP key/cert material. This is + /// separate from the auth API so production deployments need an explicit KMS + /// opt-in as well as a successful external policy decision. + #[serde(default)] + pub sev_snp_key_release: SevSnpKeyReleaseConfig, #[serde(with = "serde_human_bytes")] pub admin_token_hash: Vec, #[serde(default)] diff --git a/kms/src/main.rs b/kms/src/main.rs index 1ab9b568..7945963d 100644 --- a/kms/src/main.rs +++ b/kms/src/main.rs @@ -105,6 +105,20 @@ fn record_attestation_metrics(req: &rocket::Request<'_>, res: &rocket::Response< .record_attestation_request(res.status().code >= 400); } +fn configure_amd_kds_proxy_from_config(config: &KmsConfig) { + let Some(proxy_url) = config + .sev_snp + .as_ref() + .and_then(|sev_snp| sev_snp.amd_kds_proxy_url.as_deref()) + .map(str::trim) + .filter(|proxy_url| !proxy_url.is_empty()) + else { + return; + }; + std::env::set_var("DSTACK_AMD_KDS_PROXY_URL", proxy_url); + info!("AMD SEV-SNP KDS proxy configured"); +} + #[rocket::main] async fn main() -> Result<()> { { @@ -116,6 +130,7 @@ async fn main() -> Result<()> { let figment = config::load_config_figment(args.config.as_deref()); let config: KmsConfig = figment.focus("core").extract()?; + configure_amd_kds_proxy_from_config(&config); if config.onboard.enabled && !config.keys_exists() { info!("Onboarding"); @@ -137,6 +152,10 @@ async fn main() -> Result<()> { } let pccs_url = config.pccs_url.clone(); + let amd_kds_proxy_url = config + .sev_snp + .as_ref() + .and_then(|sev_snp| sev_snp.amd_kds_proxy_url.clone()); let metrics_enabled = config.metrics.enabled; let state = main_service::KmsState::new(config).context("Failed to initialize KMS state")?; let figment = figment @@ -164,7 +183,7 @@ async fn main() -> Result<()> { .mount("/", rocket::routes![metrics]); } - let verifier = QuoteVerifier::new(pccs_url); + let verifier = QuoteVerifier::new_with_amd_kds_proxy(pccs_url, amd_kds_proxy_url); rocket = rocket.manage(verifier); rocket diff --git a/kms/src/main_service.rs b/kms/src/main_service.rs index 00723566..cf5e60ae 100644 --- a/kms/src/main_service.rs +++ b/kms/src/main_service.rs @@ -22,7 +22,7 @@ use fs_err as fs; use k256::ecdsa::SigningKey; use ra_rpc::{CallContext, RpcCall}; use ra_tls::{ - attestation::VerifiedAttestation, + attestation::{AttestationMode, VerifiedAttestation}, cert::{CaCert, CertRequest, CertSigningRequestV1, CertSigningRequestV2, Csr}, kdf, }; @@ -33,10 +33,11 @@ use tracing::{info, warn}; use upgrade_authority::{build_boot_info, local_kms_boot_info, BootInfo}; use crate::{ - config::KmsConfig, + config::{KmsConfig, SevSnpKeyReleaseConfig, SevSnpMeasureConfig}, crypto::{derive_k256_key, sign_message, sign_message_with_timestamp}, }; +pub(crate) mod amd_attest; pub(crate) mod upgrade_authority; #[derive(Clone)] @@ -116,6 +117,10 @@ impl KmsState { "self-authorization is disabled; trusted RPCs will not be gated by KMS self-attestation - do not use in production TEE deployments" ); } + ensure_snp_key_release_config_safe( + config.enforce_self_authorization, + &config.sev_snp_key_release, + )?; Ok(Self { inner: Arc::new(KmsStateInner { config, @@ -145,15 +150,92 @@ struct BootConfig { gateway_app_id: String, } +pub(crate) fn build_boot_info_for_attestation( + sev_snp_config: Option<&SevSnpMeasureConfig>, + att: &VerifiedAttestation, + use_boottime_mr: bool, + vm_config_str: &str, +) -> Result { + if att.report.amd_snp_report().is_some() { + let config = sev_snp_config + .ok_or_else(|| anyhow::anyhow!("sev_snp config is required for amd sev-snp"))?; + let vm_config_str = if vm_config_str.is_empty() { + att.config.as_str() + } else { + vm_config_str + }; + return amd_attest::build_amd_snp_boot_info_from_verified_attestation_and_vm_config( + config, + att, + vm_config_str, + ); + } + build_boot_info(att, use_boottime_mr, vm_config_str) +} + +fn ensure_snp_key_release_allowed( + boot_info: &BootInfo, + policy: &SevSnpKeyReleaseConfig, +) -> Result<()> { + if boot_info.attestation_mode != AttestationMode::DstackAmdSevSnp { + return Ok(()); + } + if !policy.enabled { + bail!("amd sev-snp key release is not enabled"); + } + if !policy + .allowed_tcb_statuses + .iter() + .any(|allowed| allowed == &boot_info.tcb_status) + { + bail!("tcb_status is not allowed"); + } + for advisory_id in &boot_info.advisory_ids { + if !policy + .allowed_advisory_ids + .iter() + .any(|allowed| allowed == advisory_id) + { + bail!("advisory_id is not allowed"); + } + } + Ok(()) +} + +fn ensure_snp_key_release_config_safe( + enforce_self_authorization: bool, + policy: &SevSnpKeyReleaseConfig, +) -> Result<()> { + if policy.enabled && !enforce_self_authorization { + bail!("self-authorization is required for amd sev-snp key release"); + } + Ok(()) +} + +fn ensure_self_key_release_allowed( + self_boot_info: Option<&BootInfo>, + policy: &SevSnpKeyReleaseConfig, +) -> Result<()> { + if let Some(boot_info) = self_boot_info { + ensure_snp_key_release_allowed(boot_info, policy)?; + } + Ok(()) +} + impl RpcHandler { - async fn ensure_self_allowed(&self) -> Result<()> { + async fn ensure_self_allowed(&self) -> Result> { if !self.state.config.enforce_self_authorization { - return Ok(()); + return Ok(None); } let boot_info = self .state .self_boot_info - .get_or_try_init(|| local_kms_boot_info(self.state.config.pccs_url.as_deref())) + .get_or_try_init(|| { + local_kms_boot_info( + self.state.config.pccs_url.as_deref(), + self.state.config.sev_snp.as_ref(), + ) + }) .await .context("Failed to load cached self boot info")?; let response = self @@ -166,7 +248,7 @@ impl RpcHandler { if !response.is_allowed { bail!("KMS is not allowed: {}", response.reason); } - Ok(()) + Ok(Some(boot_info)) } fn ensure_attested(&self) -> Result<&VerifiedAttestation> { @@ -251,7 +333,12 @@ impl RpcHandler { use_boottime_mr: bool, vm_config_str: &str, ) -> Result { - let boot_info = build_boot_info(att, use_boottime_mr, vm_config_str)?; + let boot_info = build_boot_info_for_attestation( + self.state.config.sev_snp.as_ref(), + att, + use_boottime_mr, + vm_config_str, + )?; let response = self .state .config @@ -261,9 +348,14 @@ impl RpcHandler { if !response.is_allowed { bail!("Boot denied: {}", response.reason); } - self.verify_os_image_hash(vm_config_str.into(), att) - .await - .context("Failed to verify os image hash")?; + // SNP rootfs/app/config binding is handled by the SNP launch-measurement + // helper above. The legacy OS-image verifier is TDX-oriented and still + // rejects SNP quotes; keep SNP on the explicit fail-closed helper path. + if boot_info.attestation_mode != AttestationMode::DstackAmdSevSnp { + self.verify_os_image_hash(vm_config_str.into(), att) + .await + .context("Failed to verify os image hash")?; + } Ok(BootConfig { boot_info, gateway_app_id: response.gateway_app_id, @@ -306,8 +398,10 @@ impl KmsRpc for RpcHandler { .ensure_app_boot_allowed(&request.vm_config) .await .context("App not allowed")?; + ensure_snp_key_release_allowed(&boot_info, &self.state.config.sev_snp_key_release)?; let app_id = boot_info.app_id; let instance_id = boot_info.instance_id; + let os_image_hash = boot_info.os_image_hash; let context_data = vec![&app_id[..], &instance_id[..], b"app-disk-crypt-key"]; let app_disk_key = kdf::derive_dh_secret(&self.state.root_ca.key, &context_data) @@ -334,7 +428,7 @@ impl KmsRpc for RpcHandler { k256_signature, tproxy_app_id: gateway_app_id.clone(), gateway_app_id, - os_image_hash: boot_info.os_image_hash, + os_image_hash, }) } @@ -411,7 +505,8 @@ impl KmsRpc for RpcHandler { self.ensure_self_allowed() .await .context("KMS self authorization failed")?; - let _info = self.ensure_kms_allowed(&request.vm_config).await?; + let info = self.ensure_kms_allowed(&request.vm_config).await?; + ensure_snp_key_release_allowed(&info, &self.state.config.sev_snp_key_release)?; Ok(KmsKeyResponse { temp_ca_key: self.state.inner.temp_ca_key.clone(), keys: vec![KmsKeys { @@ -422,9 +517,11 @@ impl KmsRpc for RpcHandler { } async fn get_temp_ca_cert(self) -> Result { - self.ensure_self_allowed() + let self_boot_info = self + .ensure_self_allowed() .await .context("KMS self authorization failed")?; + ensure_self_key_release_allowed(self_boot_info, &self.state.config.sev_snp_key_release)?; Ok(GetTempCaCertResponse { temp_ca_cert: self.state.inner.temp_ca_cert.clone(), temp_ca_key: self.state.inner.temp_ca_key.clone(), @@ -463,6 +560,10 @@ impl KmsRpc for RpcHandler { let app_info = self .ensure_app_attestation_allowed(&attestation, false, true, &request.vm_config) .await?; + ensure_snp_key_release_allowed( + &app_info.boot_info, + &self.state.config.sev_snp_key_release, + )?; let app_ca = self.derive_app_ca(&app_info.boot_info.app_id)?; let cert = app_ca .sign_csr(&csr, Some(&app_info.boot_info.app_id), "app:custom") @@ -502,3 +603,245 @@ impl RpcCall for RpcHandler { pub fn rpc_methods() -> &'static [&'static str] { >::supported_methods() } + +#[cfg(test)] +mod tests { + use super::*; + use crate::main_service::amd_attest::{ + compute_expected_measurement, MeasurementInput, OvmfSectionParam, + }; + + fn sev_snp_config() -> SevSnpMeasureConfig { + SevSnpMeasureConfig { + ovmf_path: None, + amd_kds_proxy_url: None, + guest_features: 1, + } + } + + fn hex_of(byte: u8, len: usize) -> String { + hex::encode(vec![byte; len]) + } + + fn valid_snp_measurement_input() -> MeasurementInput { + MeasurementInput { + app_id: hex_of(0x11, 20), + compose_hash: hex_of(0x22, 32), + rootfs_hash: hex_of(0x33, 32), + base_cmdline: None, + docker_files_hash: Some(hex_of(0x77, 32)), + ovmf_hash: hex_of(0x44, 48), + kernel_hash: hex_of(0x55, 32), + initrd_hash: hex_of(0x66, 32), + sev_hashes_table_gpa: 0x80_1000, + sev_es_reset_eip: 0xffff_fff0, + vcpus: 2, + vcpu_type: Some("epyc-v4".to_string()), + ovmf_sections: vec![ + OvmfSectionParam { + gpa: 0x100000, + size: 0x2000, + section_type: 1, + }, + OvmfSectionParam { + gpa: 0x80_0000, + size: 0x1000, + section_type: 0x10, + }, + OvmfSectionParam { + gpa: 0x81_0000, + size: 0x1000, + section_type: 2, + }, + OvmfSectionParam { + gpa: 0x82_0000, + size: 0x1000, + section_type: 3, + }, + ], + } + } + + fn verified_snp_attestation(measurement: [u8; 48], chip_id: [u8; 64]) -> VerifiedAttestation { + verified_snp_attestation_with_config(measurement, chip_id, String::new()) + } + + fn verified_snp_attestation_with_config( + measurement: [u8; 48], + chip_id: [u8; 64], + config: String, + ) -> VerifiedAttestation { + VerifiedAttestation { + quote: ra_tls::attestation::AttestationQuote::DstackAmdSevSnp( + ra_tls::attestation::SnpQuote { + report: Vec::new(), + cert_chain: Vec::new(), + }, + ), + runtime_events: Vec::new(), + report_data: [0x42; 64], + config, + report: ra_tls::attestation::DstackVerifiedReport::DstackAmdSevSnp( + dstack_attest::amd_sev_snp::VerifiedAmdSnpReport { + measurement, + report_data: [0x42; 64], + chip_id, + tcb_info: dstack_attest::amd_sev_snp::AmdSnpTcbInfo::default(), + advisory_ids: Vec::new(), + }, + ), + } + } + + #[test] + fn build_boot_info_for_attestation_accepts_snp_vm_config_path() { + let input = valid_snp_measurement_input(); + let measurement = compute_expected_measurement(&sev_snp_config(), &input).unwrap(); + let attestation = verified_snp_attestation(measurement, [0xab; 64]); + let vm_config = serde_json::json!({ + "sev_snp_measurement": input, + }) + .to_string(); + + let boot_info = build_boot_info_for_attestation( + Some(&sev_snp_config()), + &attestation, + false, + &vm_config, + ) + .expect("snp attestation should build boot info through vm_config path"); + + assert_eq!(boot_info.attestation_mode, AttestationMode::DstackAmdSevSnp); + assert_eq!(boot_info.mr_aggregated, measurement.to_vec()); + assert_eq!(boot_info.device_id, vec![0xab; 64]); + assert_eq!(boot_info.app_id, vec![0x11; 20]); + } + + #[test] + fn build_boot_info_for_attestation_uses_embedded_snp_vm_config_when_external_is_empty() { + let input = valid_snp_measurement_input(); + let measurement = compute_expected_measurement(&sev_snp_config(), &input).unwrap(); + let embedded_config = serde_json::json!({ + "sev_snp_measurement": input, + }) + .to_string(); + let attestation = + verified_snp_attestation_with_config(measurement, [0xab; 64], embedded_config); + + let boot_info = + build_boot_info_for_attestation(Some(&sev_snp_config()), &attestation, false, "") + .expect("snp local KMS attestation should use embedded vm_config"); + + assert_eq!(boot_info.attestation_mode, AttestationMode::DstackAmdSevSnp); + assert_eq!(boot_info.mr_aggregated, measurement.to_vec()); + assert_eq!(boot_info.app_id, vec![0x11; 20]); + } + + #[test] + fn build_boot_info_for_attestation_requires_snp_config_for_snp() { + let input = valid_snp_measurement_input(); + let measurement = compute_expected_measurement(&sev_snp_config(), &input).unwrap(); + let attestation = verified_snp_attestation(measurement, [0xab; 64]); + let vm_config = serde_json::json!({ + "sev_snp_measurement": input, + }) + .to_string(); + + let err = build_boot_info_for_attestation(None, &attestation, false, &vm_config) + .expect_err("snp attestation must require sev_snp config"); + assert!( + err.to_string().contains("sev_snp config is required"), + "unexpected error: {err:?}" + ); + } + + fn snp_boot_info() -> BootInfo { + let input = valid_snp_measurement_input(); + let measurement = compute_expected_measurement(&sev_snp_config(), &input).unwrap(); + let attestation = verified_snp_attestation(measurement, [0xab; 64]); + let vm_config = serde_json::json!({ + "sev_snp_measurement": input, + }) + .to_string(); + build_boot_info_for_attestation(Some(&sev_snp_config()), &attestation, false, &vm_config) + .unwrap() + } + + #[test] + fn snp_key_release_requires_explicit_enablement() { + let boot_info = snp_boot_info(); + let policy = SevSnpKeyReleaseConfig::default(); + + let err = ensure_snp_key_release_allowed(&boot_info, &policy) + .expect_err("snp boot info must not be key-release enabled by default"); + assert!( + err.to_string() + .contains("amd sev-snp key release is not enabled"), + "unexpected error: {err:?}" + ); + } + + #[test] + fn snp_key_release_accepts_clean_tcb_when_explicitly_enabled() { + let boot_info = snp_boot_info(); + let policy = SevSnpKeyReleaseConfig { + enabled: true, + ..Default::default() + }; + + ensure_snp_key_release_allowed(&boot_info, &policy).expect( + "explicitly enabled SNP key release should allow UpToDate/no-advisory boot info", + ); + } + + #[test] + fn snp_key_release_rejects_bad_tcb_or_unallowed_advisory() { + let mut boot_info = snp_boot_info(); + let policy = SevSnpKeyReleaseConfig { + enabled: true, + ..Default::default() + }; + + boot_info.tcb_status = "OutOfDate".to_string(); + let err = ensure_snp_key_release_allowed(&boot_info, &policy) + .expect_err("OutOfDate SNP TCB must not release keys by default"); + assert!(err.to_string().contains("tcb_status is not allowed")); + + let mut boot_info = snp_boot_info(); + boot_info.advisory_ids.push("SNP-TEST-ADVISORY".to_string()); + let err = ensure_snp_key_release_allowed(&boot_info, &policy) + .expect_err("unallowlisted SNP advisory must not release keys"); + assert!(err.to_string().contains("advisory_id is not allowed")); + } + + #[test] + fn snp_release_config_requires_self_authorization_when_enabled() { + let policy = SevSnpKeyReleaseConfig { + enabled: true, + ..Default::default() + }; + + let err = ensure_snp_key_release_config_safe(false, &policy) + .expect_err("enabled SNP release must require KMS self-authorization"); + assert!(err + .to_string() + .contains("self-authorization is required for amd sev-snp key release")); + ensure_snp_key_release_config_safe(true, &policy) + .expect("enabled SNP release is safe only with self-authorization enforced"); + } + + #[test] + fn snp_self_boot_info_uses_same_release_policy_for_temp_ca() { + let boot_info = snp_boot_info(); + let disabled = SevSnpKeyReleaseConfig::default(); + let enabled = SevSnpKeyReleaseConfig { + enabled: true, + ..Default::default() + }; + + ensure_self_key_release_allowed(Some(&boot_info), &disabled) + .expect_err("disabled SNP self boot info must not receive temp CA key material"); + ensure_self_key_release_allowed(Some(&boot_info), &enabled) + .expect("enabled clean SNP self boot info should pass the temp CA release gate"); + } +} diff --git a/kms/src/main_service/amd_attest.rs b/kms/src/main_service/amd_attest.rs new file mode 100644 index 00000000..625b1c14 --- /dev/null +++ b/kms/src/main_service/amd_attest.rs @@ -0,0 +1,1864 @@ +// SPDX-FileCopyrightText: © 2024-2025 Phala Network +// +// SPDX-License-Identifier: Apache-2.0 + +//! Fail-closed AMD SEV-SNP measurement/app binding validation. +//! +//! This module does not release keys by itself. It recomputes the expected SNP +//! MEASUREMENT from validated KMS configuration and launch inputs, then compares +//! the recomputed value to the hardware-verified report measurement. KMS release +//! paths must apply their own explicit local release gate after auth succeeds. +//! +//! Important: this is launch measurement binding, not a complete authorization +//! decision. `app_id`, compose hash, and rootfs hash are included in the SNP +//! measured kernel command line so the recomputed launch `MEASUREMENT` changes +//! with app identity, matching dstack's TDX measured-identity semantics. Do not +//! use this helper by itself to release app keys. + +#![allow(dead_code)] + +use anyhow::{bail, Context, Result}; +use ra_tls::attestation::{AttestationMode, VerifiedAttestation}; +use sha2::{Digest, Sha256, Sha384}; +use std::fs; + +use crate::config::SevSnpMeasureConfig; + +use super::upgrade_authority::BootInfo; + +const LD_BYTES: usize = 48; +const ZEROS_LD: [u8; LD_BYTES] = [0u8; LD_BYTES]; +const MAX_VCPUS: u32 = 512; +const MAX_OVMF_SECTIONS: usize = 64; +/// 64 GiB worth of 4 KiB pages. +const MAX_OVMF_METADATA_PAGES: u64 = 16_777_216; +// VMSA page GPA: (u64)(-1) page-aligned, bits >51 cleared. +const VMSA_GPA: u64 = 0x0000_FFFF_FFFF_F000; + +#[derive(Debug, Clone, PartialEq, Eq, serde::Deserialize)] +#[cfg_attr(test, derive(serde::Serialize))] +#[serde(deny_unknown_fields)] +pub(crate) struct OvmfSectionParam { + pub gpa: u64, + pub size: u64, + /// Raw OVMF SEV metadata section type: + /// 1=SNP_SEC_MEMORY, 2=SNP_SECRETS, 3=CPUID, 4=SVSM_CAA, + /// 0x10=SNP_KERNEL_HASHES. + pub section_type: u32, +} + +#[derive(Debug, Clone, PartialEq, Eq, serde::Deserialize)] +#[cfg_attr(test, derive(serde::Serialize))] +#[serde(deny_unknown_fields)] +pub(crate) struct MeasurementInput { + /// 20-byte app identity included in the measured kernel cmdline for SNP, + /// matching TDX's app-id-in-measured-identity semantics. + pub app_id: String, + /// 32-byte docker compose hash included in the measured kernel cmdline. + pub compose_hash: String, + /// 32-byte rootfs hash included in the measured kernel cmdline. + pub rootfs_hash: String, + /// Original image kernel cmdline used as the base for SNP measured launch + /// before app identity fields are appended. + pub base_cmdline: Option, + /// Optional 32-byte additional docker files hash included in the measured + /// kernel cmdline when present. + pub docker_files_hash: Option, + /// 48-byte OVMF GCTX launch digest seed. Required when OVMF sections are + /// supplied by the request; optional only when KMS can load ovmf_path. + pub ovmf_hash: String, + /// 32-byte kernel SHA-256 hash. + pub kernel_hash: String, + /// 32-byte initrd SHA-256 hash. An empty string is treated as the SHA-256 of + /// an empty initrd, matching QEMU/sev-snp-measure behavior. + pub initrd_hash: String, + /// GPA of the SevHashTable, from OVMF footer metadata. + pub sev_hashes_table_gpa: u64, + /// AP reset EIP, from OVMF footer metadata. + pub sev_es_reset_eip: u32, + pub vcpus: u32, + pub vcpu_type: Option, + #[serde(deserialize_with = "deserialize_ovmf_sections_bounded")] + pub ovmf_sections: Vec, +} + +fn deserialize_ovmf_sections_bounded<'de, D>( + deserializer: D, +) -> std::result::Result, D::Error> +where + D: serde::Deserializer<'de>, +{ + struct BoundedOvmfSections; + + impl<'de> serde::de::Visitor<'de> for BoundedOvmfSections { + type Value = Vec; + + fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!( + formatter, + "at most {MAX_OVMF_SECTIONS} OVMF metadata sections" + ) + } + + fn visit_seq(self, mut seq: A) -> std::result::Result, A::Error> + where + A: serde::de::SeqAccess<'de>, + { + let mut sections = + Vec::with_capacity(seq.size_hint().unwrap_or(0).min(MAX_OVMF_SECTIONS)); + while let Some(section) = seq.next_element()? { + if sections.len() >= MAX_OVMF_SECTIONS { + return Err(serde::de::Error::custom(format!( + "ovmf section count must not exceed {MAX_OVMF_SECTIONS}" + ))); + } + sections.push(section); + } + Ok(sections) + } + } + + deserializer.deserialize_seq(BoundedOvmfSections) +} + +pub(crate) fn validate_amd_snp_measurement_binding( + config: Option<&SevSnpMeasureConfig>, + verified_measurement: &[u8; 48], + input: &MeasurementInput, +) -> Result<()> { + let config = config.ok_or_else(|| anyhow::anyhow!("sev-snp measurement config is required"))?; + validate_measurement_input(config, input)?; + + let expected_measurement = compute_expected_measurement(config, input)?; + if expected_measurement.as_slice() != verified_measurement { + bail!("amd sev-snp measurement mismatch"); + } + + Ok(()) +} + +/// Builds a deterministic authorization `BootInfo` for an already-verified AMD +/// SEV-SNP report without releasing KMS key material by itself. +/// +/// This helper first recomputes and validates the QEMU SNP launch measurement. +/// `mr_aggregated` is the hardware-verified 48-byte SNP `MEASUREMENT`, and +/// `device_id` is the hardware-verified 64-byte SNP `chip_id`. `app_id`, +/// `compose_hash`, and `rootfs_hash` are bound through the measured kernel +/// command line; `os_image_hash` is therefore represented by `rootfs_hash`. +/// +/// Authorization-specific digests are domain separated and deterministic: +/// * `mr_system = sha256("dstack-amd-sev-snp:mr-system:v1" || launch/system inputs)` +/// * `key_provider_info = sha256("dstack-amd-sev-snp:app-binding:v1" || mr_system || app_id || compose_hash || chip_id)` +/// * `instance_id = sha256("dstack-amd-sev-snp:instance-id:v1" || chip_id || measurement || app_id || compose_hash)` +/// +/// Keeping these values explicit lets authorization/release policy inspect +/// exactly which SNP-specific inputs were bound before any sensitive output path +/// returns key material. +#[cfg(test)] +pub(crate) fn build_amd_snp_boot_info( + config: &SevSnpMeasureConfig, + verified_measurement: &[u8; 48], + verified_chip_id: &[u8; 64], + input: &MeasurementInput, +) -> Result { + build_amd_snp_boot_info_with_tcb_status( + config, + verified_measurement, + verified_chip_id, + "UpToDate", + &[], + input, + ) +} + +fn build_amd_snp_boot_info_with_tcb_status( + config: &SevSnpMeasureConfig, + verified_measurement: &[u8; 48], + verified_chip_id: &[u8; 64], + tcb_status: &str, + advisory_ids: &[String], + input: &MeasurementInput, +) -> Result { + validate_amd_snp_measurement_binding(Some(config), verified_measurement, input)?; + + let app_id = decode_required_hex("app_id", &input.app_id, 20)?; + let compose_hash = decode_required_hex("compose_hash", &input.compose_hash, 32)?; + let rootfs_hash = decode_required_hex("rootfs_hash", &input.rootfs_hash, 32)?; + let mr_system = snp_mr_system_digest(config, verified_measurement, input)?; + let key_provider_info = snp_app_binding_digest( + &mr_system, + &app_id, + &compose_hash, + verified_chip_id.as_slice(), + ); + let instance_id = snp_instance_id_digest( + verified_chip_id.as_slice(), + verified_measurement, + &app_id, + &compose_hash, + ); + + Ok(BootInfo { + attestation_mode: AttestationMode::DstackAmdSevSnp, + mr_aggregated: verified_measurement.to_vec(), + os_image_hash: rootfs_hash, + mr_system, + app_id, + compose_hash, + instance_id, + device_id: verified_chip_id.to_vec(), + key_provider_info, + tcb_status: tcb_status.to_string(), + advisory_ids: advisory_ids.to_vec(), + }) +} + +/// Extracts the verified AMD SEV-SNP report from a verified attestation and +/// materializes the helper-only SNP `BootInfo` used by future authorization. +/// +/// This is the safe integration seam: the attestation verifier has already +/// checked the report signature/collateral/report_data, while this KMS helper +/// recomputes the launch measurement from trusted config and request inputs. +/// It still does not release keys by itself. +pub(crate) fn build_amd_snp_boot_info_from_verified_attestation( + config: &SevSnpMeasureConfig, + attestation: &VerifiedAttestation, + input: &MeasurementInput, +) -> Result { + let verified = attestation + .report + .amd_snp_report() + .ok_or_else(|| anyhow::anyhow!("verified attestation is not amd sev-snp"))?; + build_amd_snp_boot_info_with_tcb_status( + config, + &verified.measurement, + &verified.chip_id, + verified.tcb_info.tcb_status(), + &verified.advisory_ids, + input, + ) +} + +#[derive(Debug, serde::Deserialize)] +struct SevSnpMeasurementVmConfig { + sev_snp_measurement: Option, +} + +/// Parses SNP launch-measurement inputs from the KMS request `vm_config` and +/// builds helper-only SNP `BootInfo` from an already verified attestation. +/// +/// The field is intentionally explicit (`sev_snp_measurement`) so missing SNP +/// launch inputs fail closed instead of falling back to TDX event-log decoding. +pub(crate) fn build_amd_snp_boot_info_from_verified_attestation_and_vm_config( + config: &SevSnpMeasureConfig, + attestation: &VerifiedAttestation, + vm_config: &str, +) -> Result { + let input = parse_measurement_input_from_vm_config(vm_config)?; + build_amd_snp_boot_info_from_verified_attestation(config, attestation, &input) +} + +fn parse_measurement_input_from_vm_config(vm_config: &str) -> Result { + let parsed: SevSnpMeasurementVmConfig = serde_json::from_str(vm_config) + .context("failed to parse vm_config for amd sev-snp measurement")?; + parsed + .sev_snp_measurement + .ok_or_else(|| anyhow::anyhow!("sev_snp_measurement is required for amd sev-snp")) +} + +/// Explicit helper-only AMD SEV-SNP authorization policy. +/// +/// Explicit AMD SEV-SNP authorization policy: an SNP `BootInfo` must match +/// allowlisted hardware measurement, app/config identity, device identity, and +/// TCB/advisory policy. Empty allowlists fail closed. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct AmdSnpAuthPolicy { + pub allowed_measurements: Vec>, + pub allowed_app_ids: Vec>, + pub allowed_compose_hashes: Vec>, + pub allowed_os_image_hashes: Vec>, + pub allowed_device_ids: Vec>, + pub allowed_tcb_statuses: Vec, + pub allowed_advisory_ids: Vec, +} + +impl AmdSnpAuthPolicy { + /// Build a narrow exact-match policy from an already verified SNP boot + /// identity. This is useful for tests and for future allowlist materializing + /// logic, but still does not release keys by itself. + pub(crate) fn from_boot_info(boot_info: &BootInfo) -> Result { + ensure_snp_boot_info_shape(boot_info)?; + Ok(Self { + allowed_measurements: vec![boot_info.mr_aggregated.clone()], + allowed_app_ids: vec![boot_info.app_id.clone()], + allowed_compose_hashes: vec![boot_info.compose_hash.clone()], + allowed_os_image_hashes: vec![boot_info.os_image_hash.clone()], + allowed_device_ids: vec![boot_info.device_id.clone()], + allowed_tcb_statuses: vec![boot_info.tcb_status.clone()], + allowed_advisory_ids: boot_info.advisory_ids.clone(), + }) + } +} + +pub(crate) fn validate_amd_snp_auth_policy( + boot_info: &BootInfo, + policy: &AmdSnpAuthPolicy, +) -> Result<()> { + ensure_snp_boot_info_shape(boot_info)?; + ensure_allowed_bytes( + "measurement", + &boot_info.mr_aggregated, + &policy.allowed_measurements, + )?; + ensure_allowed_bytes("app_id", &boot_info.app_id, &policy.allowed_app_ids)?; + ensure_allowed_bytes( + "compose_hash", + &boot_info.compose_hash, + &policy.allowed_compose_hashes, + )?; + ensure_allowed_bytes( + "os_image_hash", + &boot_info.os_image_hash, + &policy.allowed_os_image_hashes, + )?; + ensure_allowed_bytes( + "device_id", + &boot_info.device_id, + &policy.allowed_device_ids, + )?; + ensure_allowed_string( + "tcb_status", + &boot_info.tcb_status, + &policy.allowed_tcb_statuses, + )?; + for advisory_id in &boot_info.advisory_ids { + ensure_allowed_string("advisory_id", advisory_id, &policy.allowed_advisory_ids)?; + } + Ok(()) +} + +fn ensure_snp_boot_info_shape(boot_info: &BootInfo) -> Result<()> { + if boot_info.attestation_mode != AttestationMode::DstackAmdSevSnp { + bail!("attestation mode is not amd sev-snp"); + } + ensure_len("measurement", &boot_info.mr_aggregated, 48)?; + ensure_len("app_id", &boot_info.app_id, 20)?; + ensure_len("compose_hash", &boot_info.compose_hash, 32)?; + ensure_len("os_image_hash", &boot_info.os_image_hash, 32)?; + ensure_len("device_id", &boot_info.device_id, 64)?; + ensure_len("mr_system", &boot_info.mr_system, 32)?; + ensure_len("key_provider_info", &boot_info.key_provider_info, 32)?; + ensure_len("instance_id", &boot_info.instance_id, 32)?; + if boot_info.tcb_status.trim().is_empty() { + bail!("tcb_status is not allowed"); + } + Ok(()) +} + +fn ensure_len(name: &str, value: &[u8], expected_len: usize) -> Result<()> { + if value.len() != expected_len { + bail!("{name} must be {expected_len} bytes"); + } + Ok(()) +} + +fn ensure_allowed_bytes(name: &str, value: &[u8], allowed: &[Vec]) -> Result<()> { + if allowed + .iter() + .any(|candidate| candidate.as_slice() == value) + { + return Ok(()); + } + bail!("{name} is not allowed") +} + +fn ensure_allowed_string(name: &str, value: &str, allowed: &[String]) -> Result<()> { + if allowed.iter().any(|candidate| candidate == value) { + return Ok(()); + } + bail!("{name} is not allowed") +} + +fn snp_mr_system_digest( + config: &SevSnpMeasureConfig, + verified_measurement: &[u8; 48], + input: &MeasurementInput, +) -> Result> { + let ovmf_hash = decode_optional_hex("ovmf_hash", &input.ovmf_hash, 48)?; + let kernel_hash = decode_required_hex("kernel_hash", &input.kernel_hash, 32)?; + let initrd_hash = if input.initrd_hash.is_empty() { + Sha256::digest(b"").to_vec() + } else { + decode_required_hex("initrd_hash", &input.initrd_hash, 32)? + }; + let rootfs_hash = decode_required_hex("rootfs_hash", &input.rootfs_hash, 32)?; + let docker_files_hash = input + .docker_files_hash + .as_deref() + .map(|value| decode_required_hex("docker_files_hash", value, 32)) + .transpose()?; + let vcpu_type = input + .vcpu_type + .as_deref() + .ok_or_else(|| anyhow::anyhow!("vcpu_type is required"))?; + + let mut h = Sha256::new(); + h.update(b"dstack-amd-sev-snp:mr-system:v1"); + h.update(verified_measurement); + h.update(len_prefixed(&ovmf_hash)); + h.update(kernel_hash); + h.update(initrd_hash); + h.update(rootfs_hash); + h.update(input.vcpus.to_le_bytes()); + h.update(len_prefixed(vcpu_type.as_bytes())); + h.update(config.guest_features.to_le_bytes()); + match docker_files_hash { + Some(value) => { + h.update([1]); + h.update(value); + } + None => h.update([0]), + } + h.update(input.sev_hashes_table_gpa.to_le_bytes()); + h.update(input.sev_es_reset_eip.to_le_bytes()); + h.update((input.ovmf_sections.len() as u64).to_le_bytes()); + for section in &input.ovmf_sections { + h.update(section.gpa.to_le_bytes()); + h.update(section.size.to_le_bytes()); + h.update(section.section_type.to_le_bytes()); + } + Ok(h.finalize().to_vec()) +} + +fn snp_app_binding_digest( + mr_system: &[u8], + app_id: &[u8], + compose_hash: &[u8], + chip_id: &[u8], +) -> Vec { + let mut h = Sha256::new(); + h.update(b"dstack-amd-sev-snp:app-binding:v1"); + h.update(mr_system); + h.update(app_id); + h.update(compose_hash); + h.update(chip_id); + h.finalize().to_vec() +} + +fn snp_instance_id_digest( + chip_id: &[u8], + measurement: &[u8], + app_id: &[u8], + compose_hash: &[u8], +) -> Vec { + let mut h = Sha256::new(); + h.update(b"dstack-amd-sev-snp:instance-id:v1"); + h.update(chip_id); + h.update(measurement); + h.update(app_id); + h.update(compose_hash); + h.finalize().to_vec() +} + +fn len_prefixed(bytes: &[u8]) -> Vec { + let mut out = Vec::with_capacity(8 + bytes.len()); + out.extend_from_slice(&(bytes.len() as u64).to_le_bytes()); + out.extend_from_slice(bytes); + out +} + +fn validate_measurement_input( + config: &SevSnpMeasureConfig, + input: &MeasurementInput, +) -> Result<()> { + if config.guest_features == 0 { + bail!("guest_features must be non-zero"); + } + + let app_id = decode_required_hex("app_id", &input.app_id, 20)?; + if app_id.iter().all(|&b| b == 0) { + bail!("app_id must not be all-zeros"); + } + decode_required_hex("compose_hash", &input.compose_hash, 32)?; + decode_required_hex("rootfs_hash", &input.rootfs_hash, 32)?; + decode_required_hex("kernel_hash", &input.kernel_hash, 32)?; + decode_optional_hex("initrd_hash", &input.initrd_hash, 32)?; + if let Some(docker_files_hash) = &input.docker_files_hash { + decode_required_hex("docker_files_hash", docker_files_hash, 32)?; + } + + if input.vcpus == 0 { + bail!("vcpus must be greater than zero"); + } + if input.vcpus > MAX_VCPUS { + bail!("vcpus must not exceed {MAX_VCPUS}"); + } + match input.vcpu_type.as_deref() { + Some(vcpu_type) if !vcpu_type.trim().is_empty() => { + vcpu_sig_from_type(vcpu_type)?; + } + _ => bail!("vcpu_type is required"), + } + + if input.ovmf_sections.is_empty() { + if config + .ovmf_path + .as_deref() + .unwrap_or_default() + .trim() + .is_empty() + { + bail!("ovmf_sections are required when ovmf_path is not configured"); + } + if !input.ovmf_hash.is_empty() { + bail!("ovmf_hash must be empty when ovmf_path is used"); + } + return Ok(()); + } + + decode_required_hex("ovmf_hash", &input.ovmf_hash, 48)?; + if input.ovmf_sections.len() > MAX_OVMF_SECTIONS { + bail!("ovmf section count must not exceed {MAX_OVMF_SECTIONS}"); + } + if input.sev_hashes_table_gpa == 0 { + bail!("sev_hashes_table_gpa must be non-zero"); + } + if input.sev_es_reset_eip == 0 { + bail!("sev_es_reset_eip must be non-zero"); + } + + let mut has_kernel_hashes_section = false; + let mut measured_pages = 0u64; + for section in &input.ovmf_sections { + if section.size == 0 { + bail!("ovmf section size must be greater than zero"); + } + let pages = section.size.div_ceil(4096); + measured_pages = measured_pages + .checked_add(pages) + .ok_or_else(|| anyhow::anyhow!("ovmf metadata page count overflow"))?; + if measured_pages > MAX_OVMF_METADATA_PAGES { + bail!("ovmf metadata page count must not exceed {MAX_OVMF_METADATA_PAGES}"); + } + let section_type = SectionType::from_u32(section.section_type).ok_or_else(|| { + anyhow::anyhow!("unknown ovmf section_type {:#x}", section.section_type) + })?; + has_kernel_hashes_section |= section_type == SectionType::SnpKernelHashes; + } + if !has_kernel_hashes_section { + bail!("ovmf metadata does not include a snp_kernel_hashes section"); + } + + Ok(()) +} + +fn decode_required_hex(name: &str, value: &str, expected_len: usize) -> Result> { + if value.is_empty() { + bail!("{name} must not be empty"); + } + decode_optional_hex(name, value, expected_len) +} + +fn decode_optional_hex(name: &str, value: &str, expected_len: usize) -> Result> { + if value.is_empty() { + return Ok(Vec::new()); + } + let bytes = hex::decode(value).map_err(|_| anyhow::anyhow!("{name} must be valid hex"))?; + if bytes.len() != expected_len { + bail!("{name} must be {expected_len} bytes"); + } + Ok(bytes) +} + +struct Gctx { + ld: [u8; LD_BYTES], +} + +impl Gctx { + fn new() -> Self { + Self { ld: ZEROS_LD } + } + + fn from_ovmf_hash(hex_value: &str) -> Result { + let raw = hex::decode(hex_value).context("ovmf_hash must be valid hex")?; + let ld: [u8; LD_BYTES] = raw + .try_into() + .map_err(|_| anyhow::anyhow!("ovmf_hash must be 48 bytes"))?; + Ok(Self { ld }) + } + + /// SNP spec §8.17.2 PAGE_INFO layout (112 bytes): current digest, + /// contents digest, length, page type, permissions/reserved, and GPA. + fn update(&mut self, page_type: u8, gpa: u64, contents: &[u8; LD_BYTES]) { + let mut buf = [0u8; 0x70]; + buf[..LD_BYTES].copy_from_slice(&self.ld); + buf[48..96].copy_from_slice(contents); + buf[96..98].copy_from_slice(&0x70u16.to_le_bytes()); + buf[98] = page_type; + buf[104..112].copy_from_slice(&gpa.to_le_bytes()); + let mut digest = [0u8; LD_BYTES]; + digest.copy_from_slice(&Sha384::digest(buf)); + self.ld = digest; + } + + fn sha384(data: &[u8]) -> [u8; LD_BYTES] { + let mut out = [0u8; LD_BYTES]; + out.copy_from_slice(&Sha384::digest(data)); + out + } + + fn update_normal_pages(&mut self, start_gpa: u64, data: &[u8]) { + for (i, chunk) in data.chunks(4096).enumerate() { + self.update(0x01, start_gpa + (i * 4096) as u64, &Self::sha384(chunk)); + } + } + + fn update_zero_pages(&mut self, gpa: u64, len: usize) { + for i in (0..len).step_by(4096) { + self.update(0x03, gpa + i as u64, &ZEROS_LD); + } + } + + fn update_secrets_page(&mut self, gpa: u64) { + self.update(0x05, gpa, &ZEROS_LD); + } + + fn update_cpuid_page(&mut self, gpa: u64) { + self.update(0x06, gpa, &ZEROS_LD); + } + + fn update_vmsa_page(&mut self, page: &[u8]) { + self.update(0x02, VMSA_GPA, &Self::sha384(page)); + } +} + +const GUID_LE_HASH_TABLE_HEADER: [u8; 16] = [ + 0x06, 0xd6, 0x38, 0x94, 0x22, 0x4f, 0xc9, 0x4c, 0xb4, 0x79, 0xa7, 0x93, 0xd4, 0x11, 0xfd, 0x21, +]; +const GUID_LE_KERNEL_ENTRY: [u8; 16] = [ + 0x37, 0x94, 0xe7, 0x4d, 0xd2, 0xab, 0x7f, 0x42, 0xb8, 0x35, 0xd5, 0xb1, 0x72, 0xd2, 0x04, 0x5b, +]; +const GUID_LE_INITRD_ENTRY: [u8; 16] = [ + 0x31, 0xf7, 0xba, 0x44, 0x2f, 0x3a, 0xd7, 0x4b, 0x9a, 0xf1, 0x41, 0xe2, 0x91, 0x69, 0x78, 0x1d, +]; +const GUID_LE_CMDLINE_ENTRY: [u8; 16] = [ + 0xd8, 0x2d, 0xd0, 0x97, 0x20, 0xbd, 0x94, 0x4c, 0xaa, 0x78, 0xe7, 0x71, 0x4d, 0x36, 0xab, 0x2a, +]; + +fn sev_entry(guid: &[u8; 16], hash: &[u8; 32]) -> [u8; 50] { + let mut entry = [0u8; 50]; + entry[..16].copy_from_slice(guid); + entry[16..18].copy_from_slice(&50u16.to_le_bytes()); + entry[18..].copy_from_slice(hash); + entry +} + +fn build_sev_hashes_page( + kernel_hash_hex: &str, + initrd_hash_hex: &str, + append: &str, + page_offset: usize, +) -> Result<[u8; 4096]> { + let kernel_hash: [u8; 32] = hex::decode(kernel_hash_hex) + .context("kernel_hash must be valid hex")? + .try_into() + .map_err(|_| anyhow::anyhow!("kernel_hash must be 32 bytes"))?; + + let initrd_hash: [u8; 32] = if initrd_hash_hex.is_empty() { + let mut h = [0u8; 32]; + h.copy_from_slice(&Sha256::digest(b"")); + h + } else { + hex::decode(initrd_hash_hex) + .context("initrd_hash must be valid hex")? + .try_into() + .map_err(|_| anyhow::anyhow!("initrd_hash must be 32 bytes"))? + }; + + let mut cmdline_bytes = append.as_bytes().to_vec(); + cmdline_bytes.push(0); + let mut cmdline_hash = [0u8; 32]; + cmdline_hash.copy_from_slice(&Sha256::digest(&cmdline_bytes)); + + let cmdline_entry = sev_entry(&GUID_LE_CMDLINE_ENTRY, &cmdline_hash); + let initrd_entry = sev_entry(&GUID_LE_INITRD_ENTRY, &initrd_hash); + let kernel_entry = sev_entry(&GUID_LE_KERNEL_ENTRY, &kernel_hash); + + const TABLE_SIZE: usize = 16 + 2 + 50 + 50 + 50; + let mut table = [0u8; TABLE_SIZE]; + table[..16].copy_from_slice(&GUID_LE_HASH_TABLE_HEADER); + table[16..18].copy_from_slice(&(TABLE_SIZE as u16).to_le_bytes()); + table[18..68].copy_from_slice(&cmdline_entry); + table[68..118].copy_from_slice(&initrd_entry); + table[118..168].copy_from_slice(&kernel_entry); + + const PADDED: usize = (TABLE_SIZE + 15) & !(15usize); + if page_offset + PADDED > 4096 { + bail!("sev hash table overflows 4096-byte page"); + } + let mut page = [0u8; 4096]; + page[page_offset..page_offset + TABLE_SIZE].copy_from_slice(&table); + Ok(page) +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum SectionType { + SnpSecMemory = 1, + SnpSecrets = 2, + Cpuid = 3, + SvsmCaa = 4, + SnpKernelHashes = 0x10, +} + +impl SectionType { + fn from_u32(value: u32) -> Option { + match value { + 1 => Some(Self::SnpSecMemory), + 2 => Some(Self::SnpSecrets), + 3 => Some(Self::Cpuid), + 4 => Some(Self::SvsmCaa), + 0x10 => Some(Self::SnpKernelHashes), + _ => None, + } + } +} + +struct MetadataSection { + gpa: u64, + size: u64, + section_type: SectionType, +} + +struct OvmfInfo { + data: Vec, + gpa: u64, + sections: Vec, + sev_hashes_table_gpa: u64, + sev_es_reset_eip: u32, +} + +const GUID_FOOTER_TABLE: [u8; 16] = [ + 0xde, 0x82, 0xb5, 0x96, 0xb2, 0x1f, 0xf7, 0x45, 0xba, 0xea, 0xa3, 0x66, 0xc5, 0x5a, 0x08, 0x2d, +]; +const GUID_SEV_HASH_TABLE_RV: [u8; 16] = [ + 0x1f, 0x37, 0x55, 0x72, 0x3b, 0x3a, 0x04, 0x4b, 0x92, 0x7b, 0x1d, 0xa6, 0xef, 0xa8, 0xd4, 0x54, +]; +const GUID_SEV_ES_RESET_BLK: [u8; 16] = [ + 0xde, 0x71, 0xf7, 0x00, 0x7e, 0x1a, 0xcb, 0x4f, 0x89, 0x0e, 0x68, 0xc7, 0x7e, 0x2f, 0xb4, 0x4e, +]; +const GUID_SEV_META_DATA: [u8; 16] = [ + 0x66, 0x65, 0x88, 0xdc, 0x4a, 0x98, 0x98, 0x47, 0xa7, 0x5e, 0x55, 0x85, 0xa7, 0xbf, 0x67, 0xcc, +]; + +fn read_u16_le(buf: &[u8], off: usize) -> u16 { + u16::from_le_bytes([buf[off], buf[off + 1]]) +} + +fn read_u32_le(buf: &[u8], off: usize) -> u32 { + u32::from_le_bytes([buf[off], buf[off + 1], buf[off + 2], buf[off + 3]]) +} + +impl OvmfInfo { + fn load(path: &str) -> Result { + let data = fs::read(path).with_context(|| format!("cannot read ovmf binary '{path}'"))?; + let size = data.len(); + let gpa = (0x1_0000_0000u64) + .checked_sub(size as u64) + .context("ovmf binary is larger than 4 gib")?; + + const ENTRY_HDR: usize = 18; + let footer_off = size.saturating_sub(32 + ENTRY_HDR); + if footer_off + ENTRY_HDR > size { + bail!("ovmf binary too small to contain footer table"); + } + if data[footer_off + 2..footer_off + 18] != GUID_FOOTER_TABLE { + bail!("ovmf footer guid not found"); + } + let footer_total_size = read_u16_le(&data, footer_off) as usize; + if footer_total_size < ENTRY_HDR { + bail!("ovmf footer table has invalid total size"); + } + let table_size = footer_total_size - ENTRY_HDR; + if table_size > footer_off { + bail!("ovmf footer table is out of bounds"); + } + let table_start = footer_off - table_size; + let table_bytes = &data[table_start..footer_off]; + + let mut sev_hashes_table_gpa = 0u64; + let mut sev_es_reset_eip = 0u32; + let mut meta_offset_from_end = None; + + let mut pos = table_bytes.len(); + while pos >= ENTRY_HDR { + let entry_off = pos - ENTRY_HDR; + let entry_size = read_u16_le(table_bytes, entry_off) as usize; + if entry_size < ENTRY_HDR || entry_size > pos { + bail!("ovmf footer table has invalid entry size"); + } + let guid = &table_bytes[entry_off + 2..entry_off + 18]; + let data_start = pos - entry_size; + let data_end = pos - ENTRY_HDR; + let entry_data = &table_bytes[data_start..data_end]; + + if guid == GUID_SEV_HASH_TABLE_RV && entry_data.len() >= 4 { + sev_hashes_table_gpa = read_u32_le(entry_data, 0) as u64; + } else if guid == GUID_SEV_ES_RESET_BLK && entry_data.len() >= 4 { + sev_es_reset_eip = read_u32_le(entry_data, 0); + } else if guid == GUID_SEV_META_DATA && entry_data.len() >= 4 { + meta_offset_from_end = Some(read_u32_le(entry_data, 0) as usize); + } + pos -= entry_size; + } + + if sev_hashes_table_gpa == 0 { + bail!("ovmf sev hash table entry not found in footer table"); + } + if sev_es_reset_eip == 0 { + bail!("ovmf sev_es_reset_block entry not found in footer table"); + } + + let mut sections = Vec::new(); + let off_from_end = meta_offset_from_end + .ok_or_else(|| anyhow::anyhow!("ovmf sev metadata entry not found in footer table"))?; + if off_from_end > size { + bail!("ovmf sev metadata offset exceeds file size"); + } + let meta_start = size - off_from_end; + if meta_start + 16 > size { + bail!("ovmf sev metadata header out of bounds"); + } + if &data[meta_start..meta_start + 4] != b"ASEV" { + bail!("ovmf sev metadata has bad signature"); + } + let meta_version = read_u32_le(&data, meta_start + 8); + if meta_version != 1 { + bail!("ovmf sev metadata has unsupported version {meta_version}"); + } + let num_items = read_u32_le(&data, meta_start + 12) as usize; + let items_start = meta_start + 16; + if items_start + num_items * 12 > size { + bail!("ovmf sev metadata sections out of bounds"); + } + for i in 0..num_items { + let off = items_start + i * 12; + let section_type_value = read_u32_le(&data, off + 8); + let section_type = SectionType::from_u32(section_type_value).ok_or_else(|| { + anyhow::anyhow!("unknown ovmf section_type {section_type_value:#x}") + })?; + sections.push(MetadataSection { + gpa: read_u32_le(&data, off) as u64, + size: read_u32_le(&data, off + 4) as u64, + section_type, + }); + } + + Ok(Self { + data, + gpa, + sections, + sev_hashes_table_gpa, + sev_es_reset_eip, + }) + } +} + +fn write_u16_le_at(buf: &mut [u8], off: usize, value: u16) { + buf[off..off + 2].copy_from_slice(&value.to_le_bytes()); +} + +fn write_u32_le_at(buf: &mut [u8], off: usize, value: u32) { + buf[off..off + 4].copy_from_slice(&value.to_le_bytes()); +} + +fn write_u64_le_at(buf: &mut [u8], off: usize, value: u64) { + buf[off..off + 8].copy_from_slice(&value.to_le_bytes()); +} + +fn write_vmcb_seg(buf: &mut [u8], off: usize, selector: u16, attrib: u16, limit: u32, base: u64) { + write_u16_le_at(buf, off, selector); + write_u16_le_at(buf, off + 2, attrib); + write_u32_le_at(buf, off + 4, limit); + write_u64_le_at(buf, off + 8, base); +} + +fn amd_cpu_sig(family: u32, model: u32, stepping: u32) -> u32 { + let (family_low, family_high) = if family > 0xf { + (0xf, (family - 0xf) & 0xff) + } else { + (family, 0) + }; + let model_low = model & 0xf; + let model_high = (model >> 4) & 0xf; + (family_high << 20) + | (model_high << 16) + | (family_low << 8) + | (model_low << 4) + | (stepping & 0xf) +} + +fn vcpu_sig_from_type(vcpu_type: &str) -> Result { + match vcpu_type.trim().to_lowercase().as_str() { + "epyc" | "epyc-v1" | "epyc-v2" | "epyc-ibpb" | "epyc-v3" | "epyc-v4" => { + Ok(amd_cpu_sig(23, 1, 2)) + } + "epyc-rome" | "epyc-rome-v1" | "epyc-rome-v2" | "epyc-rome-v3" => { + Ok(amd_cpu_sig(23, 49, 0)) + } + "epyc-milan" | "epyc-milan-v1" | "epyc-milan-v2" => Ok(amd_cpu_sig(25, 1, 1)), + "epyc-genoa" | "epyc-genoa-v1" => Ok(amd_cpu_sig(25, 17, 0)), + other => bail!("unknown vcpu_type {other:?}"), + } +} + +fn build_vmsa_page(eip: u32, vcpu_sig: u32, sev_features: u64) -> Box<[u8; 4096]> { + let mut page = Box::new([0u8; 4096]); + let p = page.as_mut_slice(); + + let cs_base = (eip as u64) & 0xffff_0000; + let rip = (eip as u64) & 0x0000_ffff; + + write_vmcb_seg(p, 0x000, 0, 0x0093, 0xffff, 0); + write_vmcb_seg(p, 0x010, 0xf000, 0x009b, 0xffff, cs_base); + write_vmcb_seg(p, 0x020, 0, 0x0093, 0xffff, 0); + write_vmcb_seg(p, 0x030, 0, 0x0093, 0xffff, 0); + write_vmcb_seg(p, 0x040, 0, 0x0093, 0xffff, 0); + write_vmcb_seg(p, 0x050, 0, 0x0093, 0xffff, 0); + write_vmcb_seg(p, 0x060, 0, 0x0000, 0xffff, 0); + write_vmcb_seg(p, 0x070, 0, 0x0082, 0xffff, 0); + write_vmcb_seg(p, 0x080, 0, 0x0000, 0xffff, 0); + write_vmcb_seg(p, 0x090, 0, 0x008b, 0xffff, 0); + + write_u64_le_at(p, 0x0D0, 0x1000); + write_u64_le_at(p, 0x148, 0x40); + write_u64_le_at(p, 0x158, 0x10); + write_u64_le_at(p, 0x160, 0x400); + write_u64_le_at(p, 0x168, 0xffff_0ff0); + write_u64_le_at(p, 0x170, 0x2); + write_u64_le_at(p, 0x178, rip); + write_u64_le_at(p, 0x268, 0x0007_0406_0007_0406); + write_u64_le_at(p, 0x310, vcpu_sig as u64); + write_u64_le_at(p, 0x3B0, sev_features); + write_u64_le_at(p, 0x3E8, 0x1); + write_u32_le_at(p, 0x408, 0x1f80); + write_u16_le_at(p, 0x410, 0x037f); + + page +} + +pub(crate) fn compute_expected_measurement( + config: &SevSnpMeasureConfig, + input: &MeasurementInput, +) -> Result<[u8; 48]> { + let vcpu_type = input + .vcpu_type + .as_deref() + .ok_or_else(|| anyhow::anyhow!("vcpu_type is required"))?; + + let mut cmdline = match input.base_cmdline.as_deref() { + Some(base) if !base.trim().is_empty() => format!( + "{} docker_compose_hash={} rootfs_hash={} app_id={}", + base.trim(), + input.compose_hash, + input.rootfs_hash, + input.app_id + ), + _ => format!( + "console=ttyS0 loglevel=7 docker_compose_hash={} rootfs_hash={} app_id={}", + input.compose_hash, input.rootfs_hash, input.app_id + ), + }; + if let Some(docker_files_hash) = input.docker_files_hash.as_deref() { + cmdline.push_str(&format!( + " docker_additional_files_hash={docker_files_hash}" + )); + } + + let (mut gctx, effective_hashes_gpa, effective_reset_eip, resolved_sections) = + if !input.ovmf_sections.is_empty() { + let sections = input + .ovmf_sections + .iter() + .map(|section| { + let section_type = + SectionType::from_u32(section.section_type).ok_or_else(|| { + anyhow::anyhow!("unknown ovmf section_type {:#x}", section.section_type) + })?; + Ok(MetadataSection { + gpa: section.gpa, + size: section.size, + section_type, + }) + }) + .collect::>>()?; + ( + Gctx::from_ovmf_hash(&input.ovmf_hash)?, + input.sev_hashes_table_gpa, + input.sev_es_reset_eip, + sections, + ) + } else { + let path = config.ovmf_path.as_deref().ok_or_else(|| { + anyhow::anyhow!("ovmf_sections are required when ovmf_path is not configured") + })?; + let ovmf = OvmfInfo::load(path)?; + let gctx = if input.ovmf_hash.is_empty() { + let mut g = Gctx::new(); + g.update_normal_pages(ovmf.gpa, &ovmf.data); + g + } else { + Gctx::from_ovmf_hash(&input.ovmf_hash)? + }; + ( + gctx, + ovmf.sev_hashes_table_gpa, + ovmf.sev_es_reset_eip, + ovmf.sections, + ) + }; + + let mut has_kernel_hashes_section = false; + for section in &resolved_sections { + let gpa = section.gpa; + let size = usize::try_from(section.size) + .map_err(|_| anyhow::anyhow!("ovmf section size is too large"))?; + match section.section_type { + SectionType::SnpSecMemory => gctx.update_zero_pages(gpa, size), + SectionType::SnpSecrets => gctx.update_secrets_page(gpa), + SectionType::Cpuid => gctx.update_cpuid_page(gpa), + SectionType::SvsmCaa => gctx.update_zero_pages(gpa, size), + SectionType::SnpKernelHashes => { + has_kernel_hashes_section = true; + if effective_hashes_gpa == 0 { + bail!("snp_kernel_hashes section present but sev_hashes_table_gpa is 0"); + } + let page_offset = (effective_hashes_gpa & 0xfff) as usize; + let page = build_sev_hashes_page( + &input.kernel_hash, + &input.initrd_hash, + &cmdline, + page_offset, + )?; + gctx.update_normal_pages(gpa, &page); + } + } + } + if !has_kernel_hashes_section { + bail!("ovmf metadata does not include a snp_kernel_hashes section"); + } + + let vcpu_sig = vcpu_sig_from_type(vcpu_type)?; + let bsp_vmsa = build_vmsa_page(0xffff_fff0, vcpu_sig, config.guest_features); + let ap_vmsa = build_vmsa_page(effective_reset_eip, vcpu_sig, config.guest_features); + + for i in 0..input.vcpus as usize { + let vmsa_page = if i == 0 { + bsp_vmsa.as_ref() + } else { + ap_vmsa.as_ref() + }; + gctx.update_vmsa_page(vmsa_page); + } + + Ok(gctx.ld) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn config() -> SevSnpMeasureConfig { + SevSnpMeasureConfig { + ovmf_path: None, + amd_kds_proxy_url: None, + guest_features: 1, + } + } + + fn config_with_path(path: &str) -> SevSnpMeasureConfig { + SevSnpMeasureConfig { + ovmf_path: Some(path.to_string()), + amd_kds_proxy_url: None, + guest_features: 1, + } + } + + fn hex_of(byte: u8, len: usize) -> String { + hex::encode(vec![byte; len]) + } + + fn valid_input() -> MeasurementInput { + MeasurementInput { + app_id: hex_of(0x11, 20), + compose_hash: hex_of(0x22, 32), + rootfs_hash: hex_of(0x33, 32), + base_cmdline: None, + docker_files_hash: Some(hex_of(0x77, 32)), + ovmf_hash: hex_of(0x44, 48), + kernel_hash: hex_of(0x55, 32), + initrd_hash: hex_of(0x66, 32), + sev_hashes_table_gpa: 0x80_1000, + sev_es_reset_eip: 0xffff_fff0, + vcpus: 2, + vcpu_type: Some("epyc-v4".to_string()), + ovmf_sections: vec![ + OvmfSectionParam { + gpa: 0x100000, + size: 0x2000, + section_type: 1, + }, + OvmfSectionParam { + gpa: 0x80_0000, + size: 0x1000, + section_type: 0x10, + }, + OvmfSectionParam { + gpa: 0x81_0000, + size: 0x1000, + section_type: 2, + }, + OvmfSectionParam { + gpa: 0x82_0000, + size: 0x1000, + section_type: 3, + }, + ], + } + } + + fn assert_rejects(input: MeasurementInput, msg: &str) { + let verified = [0xaa; 48]; + let err = validate_amd_snp_measurement_binding(Some(&config()), &verified, &input) + .expect_err("binding should reject invalid input"); + assert!( + err.to_string().contains(msg), + "expected error containing {msg:?}, got {err:?}" + ); + } + + #[test] + fn gctx_update_is_deterministic_and_order_sensitive() { + let contents = Gctx::sha384(b"page"); + let mut first = Gctx::new(); + first.update(0x01, 0x1000, &contents); + assert_eq!( + hex::encode(first.ld), + "3ebc1a70acc0bae5ae2788fae29a0371f983b19a68faf9843064f36040f58571ce5bb6bcdc9c361087073f8cffd92635" + ); + + let mut second = Gctx::new(); + second.update(0x01, 0x2000, &contents); + assert_ne!(first.ld, second.ld); + } + + #[test] + fn builds_sev_hashes_page_at_requested_offset() { + let page = build_sev_hashes_page(&hex_of(0x55, 32), "", "console=ttyS0", 0x80) + .expect("sev hashes page should build"); + assert_eq!(&page[..0x80], &[0u8; 0x80]); + assert_eq!(&page[0x80..0x90], &GUID_LE_HASH_TABLE_HEADER); + assert_eq!(u16::from_le_bytes([page[0x90], page[0x91]]), 168); + assert_eq!( + &page[0x92..0xa2], + &GUID_LE_CMDLINE_ENTRY, + "cmdline entry must be first" + ); + let empty_hash = Sha256::digest(b""); + assert_eq!(&page[0x80 + 68 + 18..0x80 + 68 + 50], empty_hash.as_slice()); + } + + #[test] + fn vcpu_type_mapping_is_strict() { + assert_eq!( + vcpu_sig_from_type("EPYC-v4").unwrap(), + amd_cpu_sig(23, 1, 2) + ); + assert_eq!( + vcpu_sig_from_type("epyc-genoa-v1").unwrap(), + amd_cpu_sig(25, 17, 0) + ); + let err = vcpu_sig_from_type("not-a-cpu").expect_err("unknown vcpu should reject"); + assert!(err.to_string().contains("unknown vcpu_type")); + } + + #[test] + fn accepts_recomputed_matching_measurement_and_rejects_mismatch() { + let input = valid_input(); + let expected = compute_expected_measurement(&config(), &input).unwrap(); + assert_eq!( + hex::encode(expected), + "4753950048f296ea9cc36be3ba3e26f9cb014411188134d2ea40580a76edf277268cc46b67dfd213d1a7dfc9a9006e0f", + "synthetic measurement vector should not drift silently" + ); + validate_amd_snp_measurement_binding(Some(&config()), &expected, &input) + .expect("matching recomputed binding should be accepted"); + + let mut mismatched = expected; + mismatched[0] ^= 0xff; + let err = validate_amd_snp_measurement_binding(Some(&config()), &mismatched, &input) + .expect_err("mismatched measurement must reject"); + assert!(err.to_string().contains("amd sev-snp measurement mismatch")); + } + + #[test] + fn builds_snp_boot_info_for_matching_measurement_only() { + let input = valid_input(); + let verified = compute_expected_measurement(&config(), &input).unwrap(); + let chip_id = [0xab; 64]; + + let boot_info = build_amd_snp_boot_info(&config(), &verified, &chip_id, &input) + .expect("matching measurement should build snp boot info"); + assert_eq!(boot_info.attestation_mode, AttestationMode::DstackAmdSevSnp); + assert_eq!(boot_info.mr_aggregated, verified.to_vec()); + assert_eq!(boot_info.device_id, chip_id.to_vec()); + assert_eq!(boot_info.app_id, vec![0x11; 20]); + assert_eq!(boot_info.compose_hash, vec![0x22; 32]); + assert_eq!(boot_info.os_image_hash, vec![0x33; 32]); + assert_eq!(boot_info.mr_system.len(), 32); + assert_eq!(boot_info.key_provider_info.len(), 32); + assert_eq!(boot_info.instance_id.len(), 32); + assert_eq!(boot_info.tcb_status, "UpToDate"); + assert_ne!(boot_info.tcb_status, "snp-verified-basic-policy"); + assert!(boot_info.advisory_ids.is_empty()); + + let mut mismatched = verified; + mismatched[0] ^= 0xff; + let err = build_amd_snp_boot_info(&config(), &mismatched, &chip_id, &input) + .expect_err("mismatched measurement must not build boot info"); + assert!(err.to_string().contains("amd sev-snp measurement mismatch")); + } + + #[test] + fn builds_snp_boot_info_from_verified_attestation_report() { + let input = valid_input(); + let verified = compute_expected_measurement(&config(), &input).unwrap(); + let chip_id = [0xab; 64]; + let attestation = ra_tls::attestation::VerifiedAttestation { + quote: ra_tls::attestation::AttestationQuote::DstackAmdSevSnp( + ra_tls::attestation::SnpQuote { + report: Vec::new(), + cert_chain: Vec::new(), + }, + ), + runtime_events: Vec::new(), + report_data: [0x42; 64], + config: String::new(), + report: ra_tls::attestation::DstackVerifiedReport::DstackAmdSevSnp( + dstack_attest::amd_sev_snp::VerifiedAmdSnpReport { + measurement: verified, + report_data: [0x42; 64], + chip_id, + tcb_info: dstack_attest::amd_sev_snp::AmdSnpTcbInfo::default(), + advisory_ids: Vec::new(), + }, + ), + }; + + let boot_info = + build_amd_snp_boot_info_from_verified_attestation(&config(), &attestation, &input) + .expect("verified snp attestation should feed boot info helper"); + + assert_eq!(boot_info.mr_aggregated, verified.to_vec()); + assert_eq!(boot_info.device_id, chip_id.to_vec()); + assert_eq!(boot_info.app_id, vec![0x11; 20]); + assert_eq!(boot_info.tcb_status, "UpToDate"); + } + + #[test] + fn verified_attestation_tcb_status_replaces_snp_placeholder() { + let input = valid_input(); + let verified = compute_expected_measurement(&config(), &input).unwrap(); + let chip_id = [0xbc; 64]; + let tcb = dstack_attest::amd_sev_snp::AmdSnpTcbVersion { + bootloader: 1, + tee: 2, + snp: 3, + microcode: 4, + }; + let stale_tcb = dstack_attest::amd_sev_snp::AmdSnpTcbVersion { + microcode: 3, + ..tcb + }; + let attestation = ra_tls::attestation::VerifiedAttestation { + quote: ra_tls::attestation::AttestationQuote::DstackAmdSevSnp( + ra_tls::attestation::SnpQuote { + report: Vec::new(), + cert_chain: Vec::new(), + }, + ), + runtime_events: Vec::new(), + report_data: [0x42; 64], + config: String::new(), + report: ra_tls::attestation::DstackVerifiedReport::DstackAmdSevSnp( + dstack_attest::amd_sev_snp::VerifiedAmdSnpReport { + measurement: verified, + report_data: [0x42; 64], + chip_id, + tcb_info: dstack_attest::amd_sev_snp::AmdSnpTcbInfo { + current: tcb, + reported: tcb, + committed: tcb, + launch: stale_tcb, + }, + advisory_ids: vec!["SNP-TEST-ADVISORY".to_string()], + }, + ), + }; + + let boot_info = + build_amd_snp_boot_info_from_verified_attestation(&config(), &attestation, &input) + .expect("verified snp attestation should feed boot info helper"); + + assert_eq!(boot_info.tcb_status, "OutOfDate"); + assert_eq!(boot_info.advisory_ids, vec!["SNP-TEST-ADVISORY"]); + assert_ne!(boot_info.tcb_status, "snp-verified-basic-policy"); + let policy = AmdSnpAuthPolicy::from_boot_info(&boot_info).unwrap(); + let mut up_to_date_only = policy.clone(); + up_to_date_only.allowed_tcb_statuses = vec!["UpToDate".to_string()]; + let err = validate_amd_snp_auth_policy(&boot_info, &up_to_date_only) + .expect_err("out-of-date snp tcb must not satisfy up-to-date policy"); + assert!( + err.to_string().contains("tcb_status is not allowed"), + "unexpected error: {err:?}" + ); + } + + #[test] + fn builds_snp_boot_info_from_verified_attestation_and_vm_config_json() { + let input = valid_input(); + let verified = compute_expected_measurement(&config(), &input).unwrap(); + let chip_id = [0xab; 64]; + let attestation = ra_tls::attestation::VerifiedAttestation { + quote: ra_tls::attestation::AttestationQuote::DstackAmdSevSnp( + ra_tls::attestation::SnpQuote { + report: Vec::new(), + cert_chain: Vec::new(), + }, + ), + runtime_events: Vec::new(), + report_data: [0x42; 64], + config: String::new(), + report: ra_tls::attestation::DstackVerifiedReport::DstackAmdSevSnp( + dstack_attest::amd_sev_snp::VerifiedAmdSnpReport { + measurement: verified, + report_data: [0x42; 64], + chip_id, + tcb_info: dstack_attest::amd_sev_snp::AmdSnpTcbInfo::default(), + advisory_ids: Vec::new(), + }, + ), + }; + let vm_config = serde_json::json!({ + "sev_snp_measurement": input, + }) + .to_string(); + + let boot_info = build_amd_snp_boot_info_from_verified_attestation_and_vm_config( + &config(), + &attestation, + &vm_config, + ) + .expect("vm_config-carried snp measurement inputs should build boot info"); + + assert_eq!(boot_info.mr_aggregated, verified.to_vec()); + assert_eq!(boot_info.device_id, chip_id.to_vec()); + assert_eq!(boot_info.app_id, vec![0x11; 20]); + } + + #[test] + fn verified_attestation_vm_config_helper_requires_snp_measurement_input() { + let input = valid_input(); + let verified = compute_expected_measurement(&config(), &input).unwrap(); + let attestation = ra_tls::attestation::VerifiedAttestation { + quote: ra_tls::attestation::AttestationQuote::DstackAmdSevSnp( + ra_tls::attestation::SnpQuote { + report: Vec::new(), + cert_chain: Vec::new(), + }, + ), + runtime_events: Vec::new(), + report_data: [0x42; 64], + config: String::new(), + report: ra_tls::attestation::DstackVerifiedReport::DstackAmdSevSnp( + dstack_attest::amd_sev_snp::VerifiedAmdSnpReport { + measurement: verified, + report_data: [0x42; 64], + chip_id: [0xab; 64], + tcb_info: dstack_attest::amd_sev_snp::AmdSnpTcbInfo::default(), + advisory_ids: Vec::new(), + }, + ), + }; + + let err = build_amd_snp_boot_info_from_verified_attestation_and_vm_config( + &config(), + &attestation, + r#"{"os_image_hash":"0x00"}"#, + ) + .expect_err("missing sev_snp_measurement must fail closed"); + assert!( + err.to_string().contains("sev_snp_measurement is required"), + "unexpected error: {err:?}" + ); + } + + #[test] + fn vm_config_measurement_parser_rejects_unknown_measurement_fields() { + let mut measurement = serde_json::to_value(valid_input()).unwrap(); + measurement["unexpected"] = serde_json::json!(true); + let vm_config = serde_json::json!({ + "sev_snp_measurement": measurement, + }) + .to_string(); + + let err = parse_measurement_input_from_vm_config(&vm_config) + .expect_err("unknown measurement fields must reject"); + assert!( + format!("{err:?}").contains("unknown field"), + "unexpected error: {err:?}" + ); + } + + #[test] + fn vm_config_measurement_parser_bounds_ovmf_sections_during_deserialization() { + let mut measurement = serde_json::to_value(valid_input()).unwrap(); + measurement["ovmf_sections"] = serde_json::Value::Array( + (0..=MAX_OVMF_SECTIONS) + .map(|_| { + serde_json::json!({ + "gpa": 0x100000u64, + "size": 0x1000u64, + "section_type": 1u32, + }) + }) + .collect(), + ); + let vm_config = serde_json::json!({ + "sev_snp_measurement": measurement, + }) + .to_string(); + + let err = parse_measurement_input_from_vm_config(&vm_config) + .expect_err("oversized ovmf_sections must reject during parse"); + assert!( + format!("{err:?}").contains("ovmf section count must not exceed"), + "unexpected error: {err:?}" + ); + } + + #[test] + fn verified_attestation_helper_rejects_non_snp_reports() { + let input = valid_input(); + let attestation = ra_tls::attestation::VerifiedAttestation { + quote: ra_tls::attestation::AttestationQuote::DstackGcpTdx, + runtime_events: Vec::new(), + report_data: [0x42; 64], + config: String::new(), + report: ra_tls::attestation::DstackVerifiedReport::DstackGcpTdx, + }; + + let err = + build_amd_snp_boot_info_from_verified_attestation(&config(), &attestation, &input) + .expect_err("non-snp verified attestation must reject"); + assert!( + err.to_string() + .contains("verified attestation is not amd sev-snp"), + "unexpected error: {err:?}" + ); + } + + #[test] + fn app_id_changes_launch_measurement_and_authorization_binding() { + let input = valid_input(); + let verified = compute_expected_measurement(&config(), &input).unwrap(); + let chip_id = [0xcd; 64]; + let boot_info = build_amd_snp_boot_info(&config(), &verified, &chip_id, &input).unwrap(); + + let mut changed = input.clone(); + changed.app_id = hex_of(0x12, 20); + let changed_measurement = compute_expected_measurement(&config(), &changed).unwrap(); + assert_ne!( + changed_measurement, verified, + "app_id must be launch-measured for SNP to match TDX app identity semantics" + ); + let err = build_amd_snp_boot_info(&config(), &verified, &chip_id, &changed) + .expect_err("stale measurement must reject changed app_id"); + assert!(err.to_string().contains("amd sev-snp measurement mismatch")); + + let changed_boot_info = + build_amd_snp_boot_info(&config(), &changed_measurement, &chip_id, &changed).unwrap(); + + assert_ne!(boot_info.app_id, changed_boot_info.app_id); + assert_ne!(boot_info.instance_id, changed_boot_info.instance_id); + assert_ne!( + boot_info.key_provider_info, + changed_boot_info.key_provider_info + ); + assert_ne!(boot_info.mr_aggregated, changed_boot_info.mr_aggregated); + assert_ne!(boot_info.mr_system, changed_boot_info.mr_system); + } + + #[test] + fn measured_input_changes_reject_until_measurement_is_recomputed() { + let input = valid_input(); + let verified = compute_expected_measurement(&config(), &input).unwrap(); + let chip_id = [0xef; 64]; + let boot_info = build_amd_snp_boot_info(&config(), &verified, &chip_id, &input).unwrap(); + + for mutate in [ + |i: &mut MeasurementInput| i.compose_hash = hex_of(0x23, 32), + |i: &mut MeasurementInput| i.rootfs_hash = hex_of(0x34, 32), + |i: &mut MeasurementInput| i.kernel_hash = hex_of(0x56, 32), + |i: &mut MeasurementInput| i.vcpus = 3, + ] { + let mut changed = input.clone(); + mutate(&mut changed); + let err = build_amd_snp_boot_info(&config(), &verified, &chip_id, &changed) + .expect_err("stale verified measurement must reject changed measured input"); + assert!(err.to_string().contains("amd sev-snp measurement mismatch")); + + let changed_verified = compute_expected_measurement(&config(), &changed).unwrap(); + let changed_boot_info = + build_amd_snp_boot_info(&config(), &changed_verified, &chip_id, &changed) + .expect("recomputed measurement should build boot info"); + assert_ne!(boot_info.mr_aggregated, changed_boot_info.mr_aggregated); + assert_ne!(boot_info.mr_system, changed_boot_info.mr_system); + } + } + + #[test] + fn chip_id_maps_to_device_id_and_changes_chip_bound_digests() { + let input = valid_input(); + let verified = compute_expected_measurement(&config(), &input).unwrap(); + let boot_info = build_amd_snp_boot_info(&config(), &verified, &[0x01; 64], &input).unwrap(); + let changed_boot_info = + build_amd_snp_boot_info(&config(), &verified, &[0x02; 64], &input).unwrap(); + + assert_eq!(boot_info.device_id, vec![0x01; 64]); + assert_eq!(changed_boot_info.device_id, vec![0x02; 64]); + assert_ne!(boot_info.device_id, changed_boot_info.device_id); + assert_ne!(boot_info.instance_id, changed_boot_info.instance_id); + assert_ne!( + boot_info.key_provider_info, + changed_boot_info.key_provider_info + ); + assert_eq!(boot_info.mr_aggregated, changed_boot_info.mr_aggregated); + assert_eq!(boot_info.mr_system, changed_boot_info.mr_system); + } + + #[test] + #[ignore = "requires sev-snp-measure and an SNP-capable OVMF binary"] + fn recomputation_matches_sev_snp_measure_live_golden_vector() { + let ovmf_path = std::env::var("DSTACK_SEV_SNP_GOLDEN_OVMF") + .unwrap_or_else(|_| "/opt/AMDSEV/usr/local/share/qemu/OVMF.fd".to_string()); + assert!( + std::path::Path::new(&ovmf_path).exists(), + "set DSTACK_SEV_SNP_GOLDEN_OVMF to an SNP-capable OVMF binary" + ); + + let dir = tempfile::tempdir().expect("tempdir should be available"); + let kernel_path = dir.path().join("kernel.bin"); + let initrd_path = dir.path().join("initrd.bin"); + let kernel_bytes = b"golden-kernel-for-dstack-sev-snp-measure\n"; + let initrd_bytes = b"golden-initrd-for-dstack-sev-snp-measure\n"; + std::fs::write(&kernel_path, kernel_bytes).expect("kernel fixture should be written"); + std::fs::write(&initrd_path, initrd_bytes).expect("initrd fixture should be written"); + + let kernel_hash = hex::encode(Sha256::digest(kernel_bytes)); + let initrd_hash = hex::encode(Sha256::digest(initrd_bytes)); + let mut input = valid_input(); + input.docker_files_hash = None; + input.ovmf_hash.clear(); + input.ovmf_sections.clear(); + input.kernel_hash = kernel_hash; + input.initrd_hash = initrd_hash; + input.vcpus = 2; + input.vcpu_type = Some("EPYC-v4".to_string()); + + let config = config_with_path(&ovmf_path); + let recomputed = compute_expected_measurement(&config, &input) + .expect("dstack recomputation should succeed"); + + let append = format!( + "console=ttyS0 loglevel=7 docker_compose_hash={} rootfs_hash={} app_id={}", + input.compose_hash, input.rootfs_hash, input.app_id + ); + let output = std::process::Command::new("sev-snp-measure") + .args([ + "--mode", + "snp", + "--vcpus", + "2", + "--vcpu-type", + "EPYC-v4", + "--ovmf", + &ovmf_path, + "--kernel", + kernel_path.to_str().unwrap(), + "--initrd", + initrd_path.to_str().unwrap(), + "--append", + &append, + "--guest-features", + "0x1", + "--output-format", + "hex", + ]) + .output() + .expect("sev-snp-measure should be installed"); + assert!( + output.status.success(), + "sev-snp-measure failed: {}", + String::from_utf8_lossy(&output.stderr) + ); + let tool_measurement = String::from_utf8(output.stdout) + .expect("sev-snp-measure output should be utf8") + .trim() + .to_string(); + + assert_eq!(hex::encode(recomputed), tool_measurement); + assert_eq!( + tool_measurement, + "6497fb9f90dc4a322228a8a5eb14742e09067bc44c184c2068d583ef628b5bae8c6cf15d91fe1bc0b7a8cbcc575be370", + "live sev-snp-measure golden vector should not drift silently" + ); + } + + #[test] + fn explicit_snp_auth_policy_accepts_only_exact_verified_identity() { + let input = valid_input(); + let verified = compute_expected_measurement(&config(), &input).unwrap(); + let chip_id = [0x42; 64]; + let boot_info = build_amd_snp_boot_info(&config(), &verified, &chip_id, &input).unwrap(); + let policy = AmdSnpAuthPolicy::from_boot_info(&boot_info) + .expect("boot info should produce an exact SNP auth policy"); + + validate_amd_snp_auth_policy(&boot_info, &policy) + .expect("exact verified SNP identity should satisfy policy"); + + let mut changed = boot_info; + changed.compose_hash[0] ^= 0xff; + let err = validate_amd_snp_auth_policy(&changed, &policy) + .expect_err("compose hash mismatch must reject"); + assert!(err.to_string().contains("compose_hash is not allowed")); + } + + #[test] + fn explicit_snp_auth_policy_rejects_incomplete_or_unsafe_tcb() { + let input = valid_input(); + let verified = compute_expected_measurement(&config(), &input).unwrap(); + let chip_id = [0x24; 64]; + let boot_info = build_amd_snp_boot_info(&config(), &verified, &chip_id, &input).unwrap(); + let policy = AmdSnpAuthPolicy::from_boot_info(&boot_info).unwrap(); + + let mut wrong_mode = boot_info.clone(); + wrong_mode.attestation_mode = AttestationMode::DstackTdx; + let err = validate_amd_snp_auth_policy(&wrong_mode, &policy) + .expect_err("non-SNP mode must reject"); + assert!(err + .to_string() + .contains("attestation mode is not amd sev-snp")); + + let mut wrong_status = boot_info.clone(); + wrong_status.tcb_status = "OutOfDate".to_string(); + let err = validate_amd_snp_auth_policy(&wrong_status, &policy) + .expect_err("unexpected tcb status must reject"); + assert!(err.to_string().contains("tcb_status is not allowed")); + + let mut advisory = boot_info.clone(); + advisory.advisory_ids.push("SNP-TEST-ADVISORY".to_string()); + let err = validate_amd_snp_auth_policy(&advisory, &policy) + .expect_err("unexpected advisory must reject by default"); + assert!(err.to_string().contains("advisory_id is not allowed")); + } + + #[test] + fn explicit_snp_auth_policy_rejects_partial_allowlists() { + let input = valid_input(); + let verified = compute_expected_measurement(&config(), &input).unwrap(); + let chip_id = [0x35; 64]; + let boot_info = build_amd_snp_boot_info(&config(), &verified, &chip_id, &input).unwrap(); + + for mutate in [ + |p: &mut AmdSnpAuthPolicy| p.allowed_measurements.clear(), + |p: &mut AmdSnpAuthPolicy| p.allowed_app_ids.clear(), + |p: &mut AmdSnpAuthPolicy| p.allowed_compose_hashes.clear(), + |p: &mut AmdSnpAuthPolicy| p.allowed_os_image_hashes.clear(), + |p: &mut AmdSnpAuthPolicy| p.allowed_device_ids.clear(), + |p: &mut AmdSnpAuthPolicy| p.allowed_tcb_statuses.clear(), + ] { + let mut policy = AmdSnpAuthPolicy::from_boot_info(&boot_info).unwrap(); + mutate(&mut policy); + let err = validate_amd_snp_auth_policy(&boot_info, &policy) + .expect_err("partial SNP policy allowlist must reject"); + assert!( + err.to_string().contains("is not allowed"), + "unexpected error: {err:?}" + ); + } + } + + #[test] + fn rejects_missing_config() { + let verified = [0xaa; 48]; + let err = validate_amd_snp_measurement_binding(None, &verified, &valid_input()) + .expect_err("missing config must fail closed"); + assert!(err + .to_string() + .contains("sev-snp measurement config is required")); + } + + #[test] + fn rejects_empty_or_malformed_binding_hashes() { + let mut input = valid_input(); + input.app_id.clear(); + assert_rejects(input, "app_id must not be empty"); + + let mut input = valid_input(); + input.app_id = hex_of(0x00, 20); + assert_rejects(input, "app_id must not be all-zeros"); + + let mut input = valid_input(); + input.compose_hash = "not hex".to_string(); + assert_rejects(input, "compose_hash must be valid hex"); + + let mut input = valid_input(); + input.rootfs_hash = hex_of(0x33, 31); + assert_rejects(input, "rootfs_hash must be 32 bytes"); + + let mut input = valid_input(); + input.ovmf_hash = hex_of(0x44, 47); + assert_rejects(input, "ovmf_hash must be 48 bytes"); + + let mut input = valid_input(); + input.kernel_hash = hex_of(0x55, 31); + assert_rejects(input, "kernel_hash must be 32 bytes"); + + let mut input = valid_input(); + input.initrd_hash = hex_of(0x66, 31); + assert_rejects(input, "initrd_hash must be 32 bytes"); + + let mut input = valid_input(); + input.initrd_hash.clear(); + let expected = compute_expected_measurement(&config(), &input).unwrap(); + validate_amd_snp_measurement_binding(Some(&config()), &expected, &input) + .expect("empty initrd hash should mean empty initrd"); + + let mut input = valid_input(); + input.docker_files_hash = Some(String::new()); + assert_rejects(input, "docker_files_hash must not be empty"); + } + + #[test] + fn rejects_missing_machine_binding_inputs() { + let mut input = valid_input(); + input.vcpus = 0; + assert_rejects(input, "vcpus must be greater than zero"); + + let mut input = valid_input(); + input.vcpus = MAX_VCPUS + 1; + assert_rejects(input, "vcpus must not exceed"); + + let mut input = valid_input(); + input.vcpu_type = None; + assert_rejects(input, "vcpu_type is required"); + + let mut input = valid_input(); + input.vcpu_type = Some("mystery".to_string()); + assert_rejects(input, "unknown vcpu_type"); + + let mut input = valid_input(); + input.ovmf_sections.clear(); + assert_rejects( + input, + "ovmf_sections are required when ovmf_path is not configured", + ); + + let mut input = valid_input(); + input.ovmf_sections.clear(); + input.ovmf_hash.clear(); + let verified = [0xaa; 48]; + let err = validate_amd_snp_measurement_binding( + Some(&config_with_path("/path/that/does/not/exist/ovmf.fd")), + &verified, + &input, + ) + .expect_err("configured ovmf_path should be used for recomputation"); + assert!(err.to_string().contains("cannot read ovmf binary")); + + let mut input = valid_input(); + input.ovmf_sections.clear(); + let err = validate_amd_snp_measurement_binding( + Some(&config_with_path("/opt/amd/ovmf.fd")), + &verified, + &input, + ) + .expect_err("request ovmf_hash must not override configured ovmf_path"); + assert!(err + .to_string() + .contains("ovmf_hash must be empty when ovmf_path is used")); + } + + #[test] + fn rejects_unsafe_machine_config() { + let verified = [0xaa; 48]; + let err = validate_amd_snp_measurement_binding( + Some(&SevSnpMeasureConfig { + ovmf_path: None, + amd_kds_proxy_url: None, + guest_features: 0, + }), + &verified, + &valid_input(), + ) + .expect_err("zero guest_features must fail closed"); + assert!(err.to_string().contains("guest_features must be non-zero")); + + let mut input = valid_input(); + input.ovmf_sections[0].size = 0; + assert_rejects(input, "ovmf section size must be greater than zero"); + + let mut input = valid_input(); + input.ovmf_sections = vec![ + OvmfSectionParam { + gpa: 0x1000, + size: 0x1000, + section_type: 1, + }; + MAX_OVMF_SECTIONS + 1 + ]; + assert_rejects(input, "ovmf section count must not exceed"); + + let mut input = valid_input(); + input.ovmf_sections[0].size = (MAX_OVMF_METADATA_PAGES + 1) * 4096; + assert_rejects(input, "ovmf metadata page count must not exceed"); + + let mut input = valid_input(); + input.ovmf_sections[0].section_type = 0xff; + assert_rejects(input, "unknown ovmf section_type 0xff"); + + let mut input = valid_input(); + input.ovmf_sections.retain(|s| s.section_type != 0x10); + assert_rejects( + input, + "ovmf metadata does not include a snp_kernel_hashes section", + ); + + let mut input = valid_input(); + input.sev_hashes_table_gpa = 0; + assert_rejects(input, "sev_hashes_table_gpa must be non-zero"); + + let mut input = valid_input(); + input.sev_es_reset_eip = 0; + assert_rejects(input, "sev_es_reset_eip must be non-zero"); + + let mut input = valid_input(); + input.ovmf_sections.clear(); + let err = + validate_amd_snp_measurement_binding(Some(&config_with_path(" ")), &verified, &input) + .expect_err("blank ovmf_path must not bypass section metadata requirement"); + assert!(err + .to_string() + .contains("ovmf_sections are required when ovmf_path is not configured")); + } +} diff --git a/kms/src/main_service/upgrade_authority.rs b/kms/src/main_service/upgrade_authority.rs index 9e164998..9f164cb6 100644 --- a/kms/src/main_service/upgrade_authority.rs +++ b/kms/src/main_service/upgrade_authority.rs @@ -2,7 +2,8 @@ // // SPDX-License-Identifier: Apache-2.0 -use crate::config::{AuthApi, KmsConfig}; +use super::build_boot_info_for_attestation; +use crate::config::{AuthApi, KmsConfig, SevSnpMeasureConfig}; use anyhow::{bail, Context, Result}; use dstack_guest_agent_rpc::{ dstack_guest_client::DstackGuestClient, AttestResponse, RawQuoteArgs, @@ -15,7 +16,7 @@ use serde::de::DeserializeOwned; use serde::{Deserialize, Serialize}; use serde_human_bytes as hex_bytes; -#[derive(Debug, Serialize, Deserialize)] +#[derive(Debug, Clone, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub(crate) struct BootInfo { pub attestation_mode: AttestationMode, @@ -72,7 +73,10 @@ pub(crate) fn build_boot_info( }) } -pub(crate) async fn local_kms_boot_info(pccs_url: Option<&str>) -> Result { +pub(crate) async fn local_kms_boot_info( + pccs_url: Option<&str>, + sev_snp_config: Option<&SevSnpMeasureConfig>, +) -> Result { let response = app_attest(pad64([0u8; 32])) .await .context("Failed to get local KMS attestation")?; @@ -83,7 +87,7 @@ pub(crate) async fn local_kms_boot_info(pccs_url: Option<&str>) -> Result Result<()> { if !cfg.enforce_self_authorization { return Ok(()); } - let boot_info = local_kms_boot_info(cfg.pccs_url.as_deref()) + let boot_info = local_kms_boot_info(cfg.pccs_url.as_deref(), cfg.sev_snp.as_ref()) .await .context("failed to build local KMS boot info")?; let response = cfg @@ -237,15 +241,16 @@ pub(crate) async fn ensure_kms_allowed( cfg: &KmsConfig, attestation: &VerifiedAttestation, ) -> Result<()> { - let mut boot_info = build_boot_info(attestation, false, "") - .context("failed to build KMS boot info from attestation")?; + let mut boot_info = + build_boot_info_for_attestation(cfg.sev_snp.as_ref(), attestation, false, "") + .context("failed to build KMS boot info from attestation")?; // Workaround: old source KMS instances use the legacy cert format (separate TDX_QUOTE + // EVENT_LOG OIDs) which lacks vm_config, resulting in an empty os_image_hash. // Fill it from the local KMS's own value. This is safe because mrAggregated already // validates OS image integrity transitively through the RTMR measurement chain. // TODO: remove once all source KMS instances use the unified PHALA_RATLS_ATTESTATION format. if boot_info.os_image_hash.is_empty() { - let local_info = local_kms_boot_info(cfg.pccs_url.as_deref()) + let local_info = local_kms_boot_info(cfg.pccs_url.as_deref(), cfg.sev_snp.as_ref()) .await .context("failed to get local KMS boot info for os_image_hash fallback")?; boot_info.os_image_hash = local_info.os_image_hash; diff --git a/kms/src/onboard_service.rs b/kms/src/onboard_service.rs index bef924b3..a7f45ddb 100644 --- a/kms/src/onboard_service.rs +++ b/kms/src/onboard_service.rs @@ -18,16 +18,22 @@ use ra_rpc::{ CallContext, RpcCall, }; use ra_tls::{ - attestation::{PlatformEvidence, QuoteContentType, VerifiedAttestation, VersionedAttestation}, + attestation::{ + GetDeviceId, PlatformEvidence, QuoteContentType, VerifiedAttestation, VersionedAttestation, + }, cert::{CaCert, CertRequest}, rcgen::{Certificate, KeyPair, PKCS_ECDSA_P256_SHA256}, }; use safe_write::safe_write; +use sha2::Digest; use crate::{ - config::KmsConfig, - main_service::upgrade_authority::{ - app_attest, dstack_client, ensure_kms_allowed, ensure_self_kms_allowed, pad64, + config::{KmsConfig, SevSnpMeasureConfig}, + main_service::{ + build_boot_info_for_attestation, + upgrade_authority::{ + app_attest, dstack_client, ensure_kms_allowed, ensure_self_kms_allowed, pad64, + }, }, }; @@ -116,6 +122,7 @@ impl OnboardRpc for OnboardHandler { .context("Failed to decode attestation")?; let attestation_mode = match &attestation.clone().into_v1().platform { PlatformEvidence::Tdx { .. } => "dstack-tdx", + PlatformEvidence::SevSnp { .. } => "dstack-amd-sev-snp", PlatformEvidence::GcpTdx => "dstack-gcp-tdx", PlatformEvidence::NitroEnclave => "dstack-nitro-enclave", } @@ -132,16 +139,6 @@ impl OnboardRpc for OnboardHandler { .await .context("Failed to get VM info")?; - // Decode app info to get device_id, mr_aggregated, os_image_hash, mr_system - let app_info = verified - .decode_app_info_ex(false, &info.vm_config) - .context("Failed to decode app info")?; - let ppid = verified - .report - .tdx_report() - .map(|report| report.ppid.to_vec()) - .unwrap_or_default(); - let (eth_rpc_url, kms_contract_address) = match self.state.config.auth_api.get_info().await { Ok(info) => ( @@ -154,16 +151,15 @@ impl OnboardRpc for OnboardHandler { } }; - Ok(AttestationInfoResponse { - device_id: app_info.device_id, - mr_aggregated: app_info.mr_aggregated.to_vec(), - os_image_hash: app_info.os_image_hash, + build_attestation_info_response( + self.state.config.sev_snp.as_ref(), + &verified, attestation_mode, - site_name: self.state.config.site_name.clone(), + &info.vm_config, + self.state.config.site_name.clone(), eth_rpc_url, kms_contract_address, - ppid, - }) + ) } async fn finish(self) -> anyhow::Result<()> { @@ -171,6 +167,150 @@ impl OnboardRpc for OnboardHandler { } } +fn build_attestation_info_response( + sev_snp_config: Option<&SevSnpMeasureConfig>, + verified: &VerifiedAttestation, + attestation_mode: String, + vm_config: &str, + site_name: String, + eth_rpc_url: String, + kms_contract_address: String, +) -> Result { + let boot_info = build_boot_info_for_attestation(sev_snp_config, verified, false, vm_config) + .context("Failed to decode app info")?; + let raw_device_id = verified.report.get_devide_id(); + Ok(AttestationInfoResponse { + device_id: sha2::Sha256::digest(&raw_device_id).to_vec(), + mr_aggregated: boot_info.mr_aggregated, + os_image_hash: boot_info.os_image_hash, + attestation_mode, + site_name, + eth_rpc_url, + kms_contract_address, + ppid: raw_device_id, + }) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::{ + config::SevSnpMeasureConfig, + main_service::amd_attest::{ + compute_expected_measurement, MeasurementInput, OvmfSectionParam, + }, + }; + use sha2::Digest; + + fn sev_snp_config() -> SevSnpMeasureConfig { + SevSnpMeasureConfig { + ovmf_path: None, + amd_kds_proxy_url: None, + guest_features: 1, + } + } + + fn hex_of(byte: u8, len: usize) -> String { + hex::encode(vec![byte; len]) + } + + fn valid_snp_measurement_input() -> MeasurementInput { + MeasurementInput { + app_id: hex_of(0x11, 20), + compose_hash: hex_of(0x22, 32), + rootfs_hash: hex_of(0x33, 32), + base_cmdline: None, + docker_files_hash: Some(hex_of(0x77, 32)), + ovmf_hash: hex_of(0x44, 48), + kernel_hash: hex_of(0x55, 32), + initrd_hash: hex_of(0x66, 32), + sev_hashes_table_gpa: 0x80_1000, + sev_es_reset_eip: 0xffff_fff0, + vcpus: 2, + vcpu_type: Some("epyc-v4".to_string()), + ovmf_sections: vec![ + OvmfSectionParam { + gpa: 0x100000, + size: 0x2000, + section_type: 1, + }, + OvmfSectionParam { + gpa: 0x80_0000, + size: 0x1000, + section_type: 0x10, + }, + OvmfSectionParam { + gpa: 0x81_0000, + size: 0x1000, + section_type: 2, + }, + OvmfSectionParam { + gpa: 0x82_0000, + size: 0x1000, + section_type: 3, + }, + ], + } + } + + fn verified_snp_attestation(measurement: [u8; 48], chip_id: [u8; 64]) -> VerifiedAttestation { + VerifiedAttestation { + quote: ra_tls::attestation::AttestationQuote::DstackAmdSevSnp( + ra_tls::attestation::SnpQuote { + report: Vec::new(), + cert_chain: Vec::new(), + }, + ), + runtime_events: Vec::new(), + report_data: [0x42; 64], + config: String::new(), + report: ra_tls::attestation::DstackVerifiedReport::DstackAmdSevSnp( + dstack_attest::amd_sev_snp::VerifiedAmdSnpReport { + measurement, + report_data: [0x42; 64], + chip_id, + tcb_info: dstack_attest::amd_sev_snp::AmdSnpTcbInfo::default(), + advisory_ids: Vec::new(), + }, + ), + } + } + + #[test] + fn attestation_info_response_uses_snp_boot_info_and_chip_id() { + let input = valid_snp_measurement_input(); + let measurement = compute_expected_measurement(&sev_snp_config(), &input).unwrap(); + let attestation = verified_snp_attestation(measurement, [0xab; 64]); + let vm_config = serde_json::json!({ + "sev_snp_measurement": input, + }) + .to_string(); + + let response = build_attestation_info_response( + Some(&sev_snp_config()), + &attestation, + "dstack-amd-sev-snp".to_string(), + &vm_config, + "test-site".to_string(), + "https://rpc.example".to_string(), + "0x1234".to_string(), + ) + .expect("snp attestation info should be derived from snp boot info"); + + assert_eq!( + response.device_id, + sha2::Sha256::digest([0xab; 64]).to_vec() + ); + assert_eq!(response.ppid, vec![0xab; 64]); + assert_eq!(response.mr_aggregated, measurement.to_vec()); + assert_eq!(response.os_image_hash, vec![0x33; 32]); + assert_eq!(response.attestation_mode, "dstack-amd-sev-snp"); + assert_eq!(response.site_name, "test-site"); + assert_eq!(response.eth_rpc_url, "https://rpc.example"); + assert_eq!(response.kms_contract_address, "0x1234"); + } +} + struct Keys { k256_key: SigningKey, tmp_ca_key: KeyPair, diff --git a/ra-rpc/src/rocket_helper.rs b/ra-rpc/src/rocket_helper.rs index 87e6872e..68bb813b 100644 --- a/ra-rpc/src/rocket_helper.rs +++ b/ra-rpc/src/rocket_helper.rs @@ -184,6 +184,7 @@ fn unix_peer_cred(stream: &UnixStream) -> Option { #[derive(Debug, Clone)] pub struct QuoteVerifier { pccs_url: Option, + amd_kds_proxy_url: Option, } pub mod deps { @@ -316,7 +317,25 @@ impl<'r> FromRequest<'r> for &'r QuoteVerifier { impl QuoteVerifier { pub fn new(pccs_url: Option) -> Self { - Self { pccs_url } + Self::new_with_amd_kds_proxy(pccs_url, None) + } + + pub fn new_with_amd_kds_proxy( + pccs_url: Option, + amd_kds_proxy_url: Option, + ) -> Self { + Self { + pccs_url, + amd_kds_proxy_url: amd_kds_proxy_url + .map(|url| url.trim().to_string()) + .filter(|url| !url.is_empty()), + } + } + + fn configure_amd_kds_proxy_for_request(&self) { + if let Some(proxy_url) = &self.amd_kds_proxy_url { + std::env::set_var("DSTACK_AMD_KDS_PROXY_URL", proxy_url); + } } } @@ -440,6 +459,21 @@ mod tests { use rocket::tokio; use std::time::{SystemTime, UNIX_EPOCH}; + #[test] + fn quote_verifier_carries_trimmed_amd_kds_proxy_url() { + let verifier = QuoteVerifier::new_with_amd_kds_proxy( + None, + Some(" https://cors.litgateway.com/ ".to_string()), + ); + assert_eq!( + verifier.amd_kds_proxy_url.as_deref(), + Some("https://cors.litgateway.com/") + ); + + let verifier = QuoteVerifier::new_with_amd_kds_proxy(None, Some(" ".to_string())); + assert!(verifier.amd_kds_proxy_url.is_none()); + } + #[test] fn custom_unix_endpoint_maps_to_remote_endpoint() { let endpoint = Endpoint::new(UnixPeerEndpoint { @@ -533,6 +567,7 @@ pub async fn handle_prpc_impl>( .flatten(); let attestation = match (request.quote_verifier, attestation) { (Some(quote_verifier), Some(attestation)) => { + quote_verifier.configure_amd_kds_proxy_for_request(); let pubkey = request .certificate .context("certificate is missing")? diff --git a/supervisor/client/src/main.rs b/supervisor/client/src/main.rs index c3b13abd..4f50793e 100644 --- a/supervisor/client/src/main.rs +++ b/supervisor/client/src/main.rs @@ -71,36 +71,37 @@ async fn main() -> Result<()> { cid: None, note: String::new(), }; - print_json(&client.deploy(&config).await?); + print_json(&client.deploy(&config).await?)?; } Commands::Start { id } => { - print_json(&client.start(&id).await?); + print_json(&client.start(&id).await?)?; } Commands::Stop { id } => { - print_json(&client.stop(&id).await?); + print_json(&client.stop(&id).await?)?; } Commands::Remove { id } => { - print_json(&client.remove(&id).await?); + print_json(&client.remove(&id).await?)?; } Commands::List => { - print_json(&client.list().await?); + print_json(&client.list().await?)?; } Commands::Info { id } => { - print_json(&client.info(&id).await?); + print_json(&client.info(&id).await?)?; } Commands::Ping => { - print_json(&client.ping().await?); + print_json(&client.ping().await?)?; } Commands::Clear => { - print_json(&client.clear().await?); + print_json(&client.clear().await?)?; } Commands::Shutdown => { - print_json(&client.shutdown().await?); + print_json(&client.shutdown().await?)?; } } Ok(()) } -fn print_json(value: &T) { - println!("{}", serde_json::to_string(value).unwrap()); +fn print_json(value: &T) -> Result<()> { + println!("{}", serde_json::to_string(value)?); + Ok(()) } diff --git a/test-scripts/snp-e2e-smoke.sh b/test-scripts/snp-e2e-smoke.sh new file mode 100755 index 00000000..a91b2801 --- /dev/null +++ b/test-scripts/snp-e2e-smoke.sh @@ -0,0 +1,502 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: © 2026 Phala Network +# +# SPDX-License-Identifier: Apache-2.0 +# +# Manual AMD SEV-SNP hardware smoke for dstack-managed KMS/app key release. +# +# This is intentionally not a CI script. It requires an SNP-capable host with the +# AMDSEV QEMU/OVMF build used by the PR smoke, sudo for QEMU/KVM, and locally +# built release binaries. +# +# Minimal setup used by the original smoke: +# cargo build --release -p dstack-vmm -p supervisor -p dstack-kms +# export DSTACK_SNP_SMOKE_BIN_DIR=$PWD/target/release +# export DSTACK_SNP_SMOKE_ALLOW_OUT_OF_DATE_TCB=1 # lab hosts only +# test-scripts/snp-e2e-smoke.sh +# +# Useful overrides: +# DSTACK_SNP_SMOKE_BASE=$HOME/dstack-snp-e2e +# DSTACK_SNP_SMOKE_REPO=$PWD +# DSTACK_SNP_SMOKE_QEMU=/opt/AMDSEV/usr/local/bin/qemu-system-x86_64 +# DSTACK_SNP_SMOKE_OVMF=/opt/AMDSEV/usr/local/share/qemu/OVMF.fd +# DSTACK_SNP_SMOKE_IMAGE_URL=https://github.com/Dstack-TEE/meta-dstack/releases/download/v0.5.11/dstack-dev-0.5.11.tar.gz +# DSTACK_SNP_SMOKE_IMAGE_NAME=dstack-dev-0.5.11-snp-dnsfix +# DSTACK_SNP_SMOKE_ALLOW_OLD_QEMU=1 # bypasses the QEMU >= 10 preflight +# +# Host/image caveat: QEMU >= 10 is necessary but not sufficient. One local SNP +# host could boot a newer Lit SNP guest kernel but reset before Linux serial +# output with the stock meta-dstack v0.5.11 6.9.0-dstack kernel. If this smoke +# stops after `EFI stub: Loaded initrd ...` with `cpus are not resettable`, first +# validate the guest image/kernel on that host before debugging KMS or apps. +# +# Guest userspace caveat: rebuilding the host-side PR binaries is not enough for +# full app-key success if the downloaded meta-dstack image still embeds an older +# dstack-util/dstack-attest. On that skewed image the app guest can reach +# dstack-prepare.sh and fail at GetTempCaCert/GetAppKey with: +# amd sev-snp cert_chain must contain either ASK and VCEK certificates or one +# kernel certificate table auxblob +# For full SNP_APP_CONTAINER_STARTED / GetAppKey success, use a coherent +# meta-dstack guest image that includes the same PR cert-chain/KDS fallback code. +# The smoke may still stop at the app GetAppKey boundary if AMD KDS throttles +# VCEK/cert-chain retrieval (for example HTTP 429 from kdsintf.amd.com); that is +# an external collateral-fetch boundary, not a guest boot or KMS startup failure. +# One reproducible way is to build meta-dstack with its dstack submodule checked +# out to this PR branch, set the Yocto build MACHINE to `sev-snp` (not the +# default `tdx`, otherwise the guest kernel can miss AMD memory-encryption +# support and reset immediately after OVMF loads the kernel/initrd), then point +# DSTACK_SNP_SMOKE_IMAGE_NAME at the resulting dstack-dev image directory. + +set -euo pipefail + +BASE="${DSTACK_SNP_SMOKE_BASE:-$HOME/dstack-snp-e2e}" +REPO="${DSTACK_SNP_SMOKE_REPO:-$(pwd)}" +BIN="${DSTACK_SNP_SMOKE_BIN_DIR:-$REPO/target/release}" +ART="$BASE/artifacts" +LOG="$ART/snp-e2e-smoke.log" +IMAGE_NAME="${DSTACK_SNP_SMOKE_IMAGE_NAME:-dstack-dev-0.5.11-snp-dnsfix}" +IMAGE_URL="${DSTACK_SNP_SMOKE_IMAGE_URL:-https://github.com/Dstack-TEE/meta-dstack/releases/download/v0.5.11/dstack-dev-0.5.11.tar.gz}" +QEMU_PATH="${DSTACK_SNP_SMOKE_QEMU:-/opt/AMDSEV/usr/local/bin/qemu-system-x86_64}" +OVMF_PATH="${DSTACK_SNP_SMOKE_OVMF:-/opt/AMDSEV/usr/local/share/qemu/OVMF.fd}" +HOST_ART_PORT="${DSTACK_SNP_SMOKE_HOST_ART_PORT:-18080}" +AUTH_PORT="${DSTACK_SNP_SMOKE_AUTH_PORT:-18081}" +KMS_HOST_PORT="${DSTACK_SNP_SMOKE_KMS_HOST_PORT:-15443}" +STRICT_KMS_HOST_PORT="${DSTACK_SNP_SMOKE_STRICT_KMS_HOST_PORT:-15444}" +APP_HOST_PORT="${DSTACK_SNP_SMOKE_APP_HOST_PORT:-15543}" +STRICT_APP_HOST_PORT="${DSTACK_SNP_SMOKE_STRICT_APP_HOST_PORT:-15544}" +VMM_PORT="${DSTACK_SNP_SMOKE_VMM_PORT:-18082}" +VMM_URL="${DSTACK_SNP_SMOKE_VMM_URL:-http://127.0.0.1:$VMM_PORT}" +ALLOW_OUT_OF_DATE_TCB="${DSTACK_SNP_SMOKE_ALLOW_OUT_OF_DATE_TCB:-0}" +RUN_STRICT_TCB_PROBE="${DSTACK_SNP_SMOKE_STRICT_TCB_PROBE:-1}" +ALLOW_OLD_QEMU="${DSTACK_SNP_SMOKE_ALLOW_OLD_QEMU:-0}" + +need() { + if ! command -v "$1" >/dev/null 2>&1; then + echo "missing required command: $1" >&2 + exit 1 + fi +} + +need curl +need jq +need python3 +need sudo + +test -x "$BIN/dstack-vmm" || { echo "missing $BIN/dstack-vmm; run cargo build --release -p dstack-vmm" >&2; exit 1; } +test -x "$BIN/supervisor" || { echo "missing $BIN/supervisor; run cargo build --release -p supervisor" >&2; exit 1; } +test -x "$BIN/dstack-kms" || { echo "missing $BIN/dstack-kms; run cargo build --release -p dstack-kms" >&2; exit 1; } +test -x "$QEMU_PATH" || { echo "missing SNP QEMU: $QEMU_PATH" >&2; exit 1; } +test -r "$OVMF_PATH" || { echo "missing SNP OVMF: $OVMF_PATH" >&2; exit 1; } +test -f "$REPO/vmm/src/vmm-cli.py" || { echo "missing vmm-cli.py; set DSTACK_SNP_SMOKE_REPO" >&2; exit 1; } + +qemu_version_output=$("$QEMU_PATH" --version | head -1) +qemu_version=$(printf '%s\n' "$qemu_version_output" | sed -n 's/.*version \([0-9][0-9]*\)\.\([0-9][0-9]*\).*/\1.\2/p') +qemu_major=${qemu_version%%.*} +if [[ -z "$qemu_version" ]]; then + echo "Warning: could not parse QEMU version from: $qemu_version_output" >&2 +elif (( qemu_major < 10 )) && [[ "$ALLOW_OLD_QEMU" != "1" ]]; then + cat >&2 <= 10 build, or set DSTACK_SNP_SMOKE_ALLOW_OLD_QEMU=1 +if you intentionally want to reproduce/debug the older-QEMU failure. +EOF + exit 1 +fi + +mkdir -p "$ART" "$BASE/images" "$BASE/run" "$BASE/http-root" +exec > >(tee "$LOG") 2>&1 + +echo "== SNP E2E smoke start: $(date -Is) ==" +echo "repo=$REPO" +echo "repo_head=$(git -C "$REPO" rev-parse --short=16 HEAD 2>/dev/null || echo unknown)" +echo "qemu=$QEMU_PATH" +echo "qemu_version=$qemu_version_output" +echo "ovmf_sha256=$(sha256sum "$OVMF_PATH" | awk '{print $1}')" +echo "image=$IMAGE_NAME" +if [[ -n "${DSTACK_SNP_SMOKE_KDS_PROXY_URL:-}" ]]; then + echo "amd_kds_proxy_url=${DSTACK_SNP_SMOKE_KDS_PROXY_URL}" +fi + +cleanup() { + set +e + if [[ -f "$BASE/vmm.pid" ]]; then sudo kill "$(cat "$BASE/vmm.pid")" 2>/dev/null || true; fi + if [[ -f "$BASE/artifacts-http.pid" ]]; then kill "$(cat "$BASE/artifacts-http.pid")" 2>/dev/null || true; fi + if [[ -f "$BASE/auth.pid" ]]; then kill "$(cat "$BASE/auth.pid")" 2>/dev/null || true; fi + sudo pkill -f "$BIN/dstack-vmm" 2>/dev/null || true + sudo pkill -f "qemu-system-x86_64.*$BASE" 2>/dev/null || true + sudo pkill -f "$BASE/images" 2>/dev/null || true + if command -v fuser >/dev/null 2>&1; then + fuser -k "${HOST_ART_PORT}/tcp" "${AUTH_PORT}/tcp" "${KMS_HOST_PORT}/tcp" "${STRICT_KMS_HOST_PORT}/tcp" "${APP_HOST_PORT}/tcp" "${STRICT_APP_HOST_PORT}/tcp" "${VMM_PORT}/tcp" 2>/dev/null || true + fi +} +trap cleanup EXIT +cleanup +sudo pkill -f "$BIN/supervisor" 2>/dev/null || true +sudo rm -rf "$BASE/run"/* "$BASE/tmp"/* + +cp "$BIN/dstack-kms" "$BASE/http-root/dstack-kms" +cp "$OVMF_PATH" "$BASE/http-root/OVMF.fd" +chmod +x "$BASE/http-root/dstack-kms" + +if [[ ! -d "$BASE/images/$IMAGE_NAME" ]]; then + echo "== Downloading/extracting $IMAGE_NAME ==" + curl -L "$IMAGE_URL" -o "$BASE/$IMAGE_NAME.tar.gz" + mkdir -p "$BASE/images/$IMAGE_NAME" + tar -xzf "$BASE/$IMAGE_NAME.tar.gz" -C "$BASE/images/$IMAGE_NAME" --strip-components=1 +fi +cp "$OVMF_PATH" "$BASE/images/$IMAGE_NAME/ovmf.fd" +jq . "$BASE/images/$IMAGE_NAME/metadata.json" | tee "$ART/image-metadata.json" + +cat >"$BASE/auth-server.py" <<'PY' +from http.server import BaseHTTPRequestHandler, HTTPServer +import json +import os +import time + + +class H(BaseHTTPRequestHandler): + def _send(self, obj, status=200): + body = json.dumps(obj).encode() + self.send_response(status) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + def log_message(self, fmt, *args): + print(time.strftime("%Y-%m-%dT%H:%M:%S"), self.path, fmt % args, flush=True) + + def do_GET(self): + self._send({ + "status": "ok", + "kmsContractAddr": "0x0000000000000000000000000000000000000000", + "ethRpcUrl": "", + "gatewayAppId": "", + "chainId": 1, + "appImplementation": "0x0000000000000000000000000000000000000000", + }) + + def do_POST(self): + length = int(self.headers.get("Content-Length", "0") or 0) + body = self.rfile.read(length) + try: + data = json.loads(body or b"{}") + except Exception: + data = {} + summary = {k: data.get(k) for k in ["attestationMode", "tcbStatus", "advisoryIds"] if k in data} + for key in ["appId", "mrAggregated", "osImageHash", "composeHash", "instanceId"]: + if key in data: + summary[key] = str(data[key])[:96] + print(json.dumps({"path": self.path, "summary": summary}), flush=True) + self._send({"isAllowed": True, "gatewayAppId": "", "reason": "snp smoke permissive auth"}) + + +HTTPServer(("0.0.0.0", int(os.environ["AUTH_PORT"])), H).serve_forever() +PY + +(cd "$BASE/http-root" && python3 -m http.server "$HOST_ART_PORT" >"$ART/artifacts-http.log" 2>&1 & echo $! >"$BASE/artifacts-http.pid") +AUTH_PORT="$AUTH_PORT" python3 "$BASE/auth-server.py" >"$ART/auth-server.log" 2>&1 & echo $! >"$BASE/auth.pid" +sleep 1 +curl -fsS "http://127.0.0.1:$HOST_ART_PORT/dstack-kms" -o /dev/null +curl -fsS "http://127.0.0.1:$AUTH_PORT/" | jq . | tee "$ART/auth-info.json" + +cat >"$BASE/vmm.toml" <"$ART/vmm.log" 2>&1 & echo $! >"$BASE/vmm.pid" +for i in $(seq 1 60); do + if python3 "$REPO/vmm/src/vmm-cli.py" --url "$VMM_URL" lsvm --json >/dev/null 2>&1; then break; fi + sleep 1 + if [[ $i -eq 60 ]]; then echo "VMM did not become ready"; tail -80 "$ART/vmm.log"; exit 1; fi +done +echo "== VMM ready ==" + +allowed_tcb_statuses='["UpToDate"]' +if [[ "$ALLOW_OUT_OF_DATE_TCB" = "1" ]]; then + allowed_tcb_statuses='["UpToDate", "OutOfDate"]' +fi + +write_kms_config() { + local tcb_statuses="$1" + cat >"$BASE/http-root/kms.toml" </etc/docker/daemon.json <<'JSON' +{"dns":["10.0.2.3","1.1.1.1","8.8.8.8"]} +JSON +rm -f /etc/resolv.conf +printf 'nameserver 10.0.2.3\nnameserver 1.1.1.1\nnameserver 8.8.8.8\noptions timeout:2 attempts:3\n' >/etc/resolv.conf +if command -v systemctl >/dev/null 2>&1 && systemctl is-active docker >/dev/null 2>&1; then + systemctl restart docker +fi +SH +) + +DNS_INIT_SCRIPT=${DNS_INIT_SCRIPT/__DSTACK_AMD_KDS_PROXY_URL__/${DSTACK_SNP_SMOKE_KDS_PROXY_URL:-}} + +KMS_BASH_SCRIPT=$(cat <<'SH' +set -eux +mkdir -p /dstack/kms-certs /dstack/kms-images +curl -fsS http://10.0.2.2:__DSTACK_HOST_ART_PORT__/dstack-kms -o /dstack/dstack-kms +curl -fsS http://10.0.2.2:__DSTACK_HOST_ART_PORT__/OVMF.fd -o /dstack/OVMF.fd +curl -fsS http://10.0.2.2:__DSTACK_HOST_ART_PORT__/kms.toml -o /dstack/kms.toml +chmod +x /dstack/dstack-kms +echo SNP_KMS_CONTAINER_STARTED +export DSTACK_AMD_KDS_PROXY_URL="__DSTACK_AMD_KDS_PROXY_URL__" +RUST_LOG=info /dstack/dstack-kms -c /dstack/kms.toml +SH +) +KMS_BASH_SCRIPT=${KMS_BASH_SCRIPT/__DSTACK_HOST_ART_PORT__/$HOST_ART_PORT} +KMS_BASH_SCRIPT=${KMS_BASH_SCRIPT//__DSTACK_HOST_ART_PORT__/$HOST_ART_PORT} +KMS_BASH_SCRIPT=${KMS_BASH_SCRIPT/__DSTACK_AMD_KDS_PROXY_URL__/${DSTACK_SNP_SMOKE_KDS_PROXY_URL:-}} + +deploy_kms() { + local name="$1" + local statuses="$2" + local host_port="$3" + write_kms_config "$statuses" + cat >"$BASE/kms-compose.yaml" <<'YAML' +services: + kms: + image: debian:bookworm-slim + command: sh -c 'echo unused-container-compose; sleep 300' +YAML + python3 "$REPO/vmm/src/vmm-cli.py" --url "$VMM_URL" compose --docker-compose "$BASE/kms-compose.yaml" --name "$name" --public-logs --public-sysinfo --no-instance-id --output "$BASE/$name.app-compose.json" | tee "$ART/$name-compose-create.txt" >&2 + jq --arg init_script "$DNS_INIT_SCRIPT" --arg bash_script "$KMS_BASH_SCRIPT" '.storage_fs="ext4" | .init_script=$init_script | .runner="bash" | .bash_script=$bash_script | del(.docker_compose_file)' "$BASE/$name.app-compose.json" >"$BASE/$name.app-compose.json.tmp" + mv "$BASE/$name.app-compose.json.tmp" "$BASE/$name.app-compose.json" + python3 "$REPO/vmm/src/vmm-cli.py" --url "$VMM_URL" deploy --name "$name" --compose "$BASE/$name.app-compose.json" --image "$IMAGE_NAME" --port "tcp:127.0.0.1:$host_port:8000" --vcpu 2 --memory 4096 --disk 20G | tee "$ART/$name-deploy.txt" >&2 + sed -n 's/Created VM with ID: //p' "$ART/$name-deploy.txt" | tail -1 +} + +wait_for_kms_metrics() { + local vm_id="$1" + local host_port="$2" + local label="$3" + for i in $(seq 1 240); do + if curl -kfsS "https://127.0.0.1:$host_port/metrics" >/dev/null 2>&1; then echo "$label KMS runtime ready after ${i}s"; break; fi + sleep 2 + if [[ $((i % 30)) -eq 0 ]]; then echo "waiting for $label KMS..."; python3 "$REPO/vmm/src/vmm-cli.py" --url "$VMM_URL" logs "$vm_id" -n 30 || true; fi + if [[ $i -eq 240 ]]; then echo "$label KMS did not become ready"; python3 "$REPO/vmm/src/vmm-cli.py" --url "$VMM_URL" logs "$vm_id" -n 200 || true; exit 1; fi + done +} + +deploy_app() { + local name="$1" + local kms_port="$2" + local app_port="$3" + cat >"$BASE/$name-compose.yaml" <<'YAML' +services: + smoke: + image: debian:bookworm-slim + command: sh -c 'echo SNP_APP_CONTAINER_STARTED; sleep 300' +YAML + python3 "$REPO/vmm/src/vmm-cli.py" --url "$VMM_URL" compose --docker-compose "$BASE/$name-compose.yaml" --name "$name" --kms --public-logs --public-sysinfo --no-instance-id --output "$BASE/$name.app-compose.json" | tee "$ART/$name-compose-create.txt" >&2 + jq --arg init_script "$DNS_INIT_SCRIPT" '.storage_fs="ext4" | .init_script=$init_script' "$BASE/$name.app-compose.json" >"$BASE/$name.app-compose.json.tmp" + mv "$BASE/$name.app-compose.json.tmp" "$BASE/$name.app-compose.json" + python3 "$REPO/vmm/src/vmm-cli.py" --url "$VMM_URL" deploy --name "$name" --compose "$BASE/$name.app-compose.json" --image "$IMAGE_NAME" --kms-url "https://10.0.2.2:$kms_port" --port "tcp:127.0.0.1:$app_port:8000" --vcpu 2 --memory 4096 --disk 20G | tee "$ART/$name-deploy.txt" >&2 + sed -n 's/Created VM with ID: //p' "$ART/$name-deploy.txt" | tail -1 +} + +if [[ "$RUN_STRICT_TCB_PROBE" = "1" && "$ALLOW_OUT_OF_DATE_TCB" = "1" ]]; then + echo "== Strict TCB probe: expect app GetAppKey denial on lab OutOfDate host ==" + STRICT_KMS_VM_ID=$(deploy_kms snp-smoke-kms-strict '["UpToDate"]' "$STRICT_KMS_HOST_PORT") + echo "STRICT_KMS_VM_ID=$STRICT_KMS_VM_ID" + wait_for_kms_metrics "$STRICT_KMS_VM_ID" "$STRICT_KMS_HOST_PORT" strict + STRICT_APP_VM_ID=$(deploy_app snp-smoke-app-strict "$STRICT_KMS_HOST_PORT" "$STRICT_APP_HOST_PORT") + echo "STRICT_APP_VM_ID=$STRICT_APP_VM_ID" + for i in $(seq 1 240); do + logs=$(python3 "$REPO/vmm/src/vmm-cli.py" --url "$VMM_URL" logs "$STRICT_APP_VM_ID" -n 180 2>/dev/null || true) + kms_logs=$(python3 "$REPO/vmm/src/vmm-cli.py" --url "$VMM_URL" logs "$STRICT_KMS_VM_ID" -n 220 2>/dev/null || true) + if { echo "$logs"; echo "$kms_logs"; } | grep -q "tcb_status is not allowed"; then + { echo "$logs"; echo "$kms_logs"; } | tee "$ART/strict-tcb-denial-log.txt" + echo "strict_tcb_probe=denied_as_expected" + break + fi + if { echo "$logs"; echo "$kms_logs"; } | grep -q "KDS collateral unavailable\|HTTP status client error"; then + { echo "$logs"; echo "$kms_logs"; } | tee "$ART/strict-tcb-kds-blocked-log.txt" + echo "strict_tcb_probe=blocked_by_kds_collateral" + break + fi + if echo "$logs" | grep -Eq "SNP_APP_CONTAINER_STARTED|Container dstack-smoke-1 Started"; then echo "$logs" | tee "$ART/strict-tcb-unexpected-success-log.txt"; echo "strict TCB probe unexpectedly reached app container"; exit 1; fi + sleep 2 + if [[ $((i % 30)) -eq 0 ]]; then echo "waiting for strict APP denial..."; echo "$logs" | tail -60; echo "$kms_logs" | tail -60; fi + if [[ $i -eq 240 ]]; then echo "strict TCB probe did not reach expected denial"; { echo "$logs"; echo "$kms_logs"; } | tee "$ART/strict-tcb-timeout-log.txt"; exit 1; fi + done +fi + +echo "== KMS success run ==" +KMS_VM_ID=$(deploy_kms snp-smoke-kms "$allowed_tcb_statuses" "$KMS_HOST_PORT") +echo "KMS_VM_ID=$KMS_VM_ID" +wait_for_kms_metrics "$KMS_VM_ID" "$KMS_HOST_PORT" success +curl -kfsS "https://127.0.0.1:$KMS_HOST_PORT/metrics" | tee "$ART/kms-metrics-before-app.txt" + +APP_VM_ID=$(deploy_app snp-smoke-app "$KMS_HOST_PORT" "$APP_HOST_PORT") +echo "APP_VM_ID=$APP_VM_ID" + +for i in $(seq 1 240); do + logs=$(python3 "$REPO/vmm/src/vmm-cli.py" --url "$VMM_URL" logs "$APP_VM_ID" -n 160 2>/dev/null || true) + if echo "$logs" | grep -Eq "SNP_APP_CONTAINER_STARTED|Container dstack-smoke-1 Started"; then echo "$logs" | tee "$ART/app-ready-log.txt"; echo "APP ready after ${i}s"; break; fi + if echo "$logs" | grep -q "Failed to get app key\|amd sev-snp key release\|measurement mismatch\|App not allowed\|KMS self authorization failed\|KDS collateral unavailable\|HTTP status client error"; then echo "$logs" | tee "$ART/app-failure-log.txt"; exit 2; fi + sleep 2 + if [[ $((i % 30)) -eq 0 ]]; then echo "waiting for APP..."; echo "$logs" | tail -60; fi + if [[ $i -eq 240 ]]; then echo "APP did not become ready"; echo "$logs" | tee "$ART/app-timeout-log.txt"; exit 1; fi +done + +curl -kfsS "https://127.0.0.1:$KMS_HOST_PORT/metrics" | tee "$ART/kms-metrics-after-app.txt" +python3 "$REPO/vmm/src/vmm-cli.py" --url "$VMM_URL" info "$KMS_VM_ID" --json | tee "$ART/kms-info.json" +python3 "$REPO/vmm/src/vmm-cli.py" --url "$VMM_URL" info "$APP_VM_ID" --json | tee "$ART/app-info.json" +python3 "$REPO/vmm/src/vmm-cli.py" --url "$VMM_URL" logs "$KMS_VM_ID" -n 200 | tee "$ART/kms-final-log.txt" || true +python3 "$REPO/vmm/src/vmm-cli.py" --url "$VMM_URL" logs "$APP_VM_ID" -n 200 | tee "$ART/app-final-log.txt" || true + +echo "== SNP E2E smoke success: $(date -Is) ==" +echo "Artifacts: $ART" diff --git a/verifier/src/verification.rs b/verifier/src/verification.rs index 0cc91bda..9d1f805a 100644 --- a/verifier/src/verification.rs +++ b/verifier/src/verification.rs @@ -502,7 +502,9 @@ impl CvmVerifier { self.verify_os_image_hash_for_dstack_tdx(&vm_config, attestation, debug, details) .await?; } - AttestationQuote::DstackGcpTdx | AttestationQuote::DstackNitroEnclave => { + AttestationQuote::DstackAmdSevSnp(_) + | AttestationQuote::DstackGcpTdx + | AttestationQuote::DstackNitroEnclave => { bail!( "Unsupported attestation quote: {:?}", attestation.quote.mode() diff --git a/vmm/src/app.rs b/vmm/src/app.rs index 8b9714cb..0d1bea0e 100644 --- a/vmm/src/app.rs +++ b/vmm/src/app.rs @@ -21,6 +21,7 @@ use or_panic::ResultOrPanic; use ra_rpc::client::RaClient; use serde::{Deserialize, Serialize}; use serde_json::json; +use sha2::{Digest, Sha256}; use std::collections::{BTreeSet, HashMap, HashSet, VecDeque}; use std::net::IpAddr; use std::path::{Path, PathBuf}; @@ -998,7 +999,8 @@ impl App { let shared_dir = self.shared_dir(id); let manifest = work_dir.manifest().context("Failed to read manifest")?; let cfg = &self.config; - let sys_config_str = make_sys_config(cfg, &manifest)?; + let compose_hash = sha256_file(shared_dir.join(APP_COMPOSE))?; + let sys_config_str = make_sys_config(cfg, &manifest, &hex::encode(compose_hash))?; fs::write(shared_dir.join(SYS_CONFIG), sys_config_str) .context("Failed to write vm config")?; Ok(()) @@ -1136,7 +1138,11 @@ fn rotate_serial_log(work_dir: &VmWorkDir, max_bytes: u64) { } } -pub(crate) fn make_sys_config(cfg: &Config, manifest: &Manifest) -> Result { +pub(crate) fn make_sys_config( + cfg: &Config, + manifest: &Manifest, + compose_hash: &str, +) -> Result { let image_path = cfg.image.path.join(&manifest.image); let image = Image::load(image_path).context("Failed to load image info")?; let img_ver = image.info.version_tuple().unwrap_or((0, 0, 0)); @@ -1160,14 +1166,61 @@ pub(crate) fn make_sys_config(cfg: &Config, manifest: &Manifest) -> Result Result { +fn file_sha256_hex(path: &Path) -> Result { + Ok(hex::encode(sha256_file(path)?)) +} + +fn image_rootfs_hash(image: &Image) -> Result<&str> { + if let Some(rootfs_hash) = image.info.rootfs_hash.as_deref() { + return Ok(rootfs_hash); + } + let cmdline = image.info.cmdline.as_deref().unwrap_or_default(); + cmdline + .split_whitespace() + .find_map(|param| param.strip_prefix("dstack.rootfs_hash=")) + .ok_or_else(|| anyhow::anyhow!("rootfs_hash is required for amd sev-snp")) +} + +fn amd_sev_snp_base_cmdline_with_kds_proxy( + base_cmdline: Option<&str>, + proxy_url: Option<&str>, +) -> Option { + let base_cmdline = base_cmdline?; + let mut cmdline = base_cmdline.trim().to_string(); + if let Some(proxy_url) = proxy_url.map(str::trim).filter(|url| !url.is_empty()) { + cmdline.push_str(" dstack.amd_kds_proxy_url="); + cmdline.push_str(proxy_url); + } + Some(cmdline) +} + +fn amd_sev_snp_measurement_base_cmdline(base_cmdline: Option<&str>) -> Option { + amd_sev_snp_base_cmdline_with_kds_proxy( + base_cmdline, + std::env::var("DSTACK_AMD_KDS_PROXY_URL").ok().as_deref(), + ) +} + +fn sha256_file(path: impl AsRef) -> Result<[u8; 32]> { + let data = fs::read(path).context("Failed to read file for sha256")?; + let mut out = [0u8; 32]; + out.copy_from_slice(&Sha256::digest(data)); + Ok(out) +} + +fn make_vm_config( + cfg: &Config, + manifest: &Manifest, + image: &Image, + compose_hash: &str, +) -> Result { let os_image_hash = image .digest .as_ref() @@ -1192,9 +1245,133 @@ fn make_vm_config(cfg: &Config, manifest: &Manifest, image: &Image) -> Result String { + hex::encode(vec![byte; len]) + } + + #[test] + fn amd_sev_snp_measurement_base_cmdline_can_carry_kds_proxy_for_smoke() { + assert_eq!( + amd_sev_snp_base_cmdline_with_kds_proxy( + Some("console=ttyS0 loglevel=7"), + Some("https://cors.litgateway.com/"), + ), + Some( + "console=ttyS0 loglevel=7 dstack.amd_kds_proxy_url=https://cors.litgateway.com/" + .to_string() + ) + ); + } + + #[test] + fn amd_sev_snp_sys_config_includes_measurement_input_for_kms_auth() { + let temp = std::env::temp_dir().join(format!( + "dstack-vmm-snp-test-{}", + SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_nanos() + )); + let temp = temp.as_path(); + let image_root = temp.join("images"); + let image_dir = image_root.join("dstack-test"); + fs::create_dir_all(&image_dir).unwrap(); + fs::write(image_dir.join("kernel"), b"snp-test-kernel").unwrap(); + fs::write(image_dir.join("initrd"), b"snp-test-initrd").unwrap(); + fs::write(image_dir.join("rootfs"), b"snp-test-rootfs").unwrap(); + fs::write( + image_dir.join("metadata.json"), + serde_json::json!({ + "cmdline": format!("console=ttyS0 dstack.rootfs_hash={}", hex_of(0x33, 32)), + "kernel": "kernel", + "initrd": "initrd", + "rootfs": "rootfs", + "version": "0.5.11" + }) + .to_string(), + ) + .unwrap(); + + let mut config: Config = Figment::from(load_config_figment(None)).extract().unwrap(); + config.image.path = image_root; + config.cvm.platform = TeePlatform::AmdSevSnp; + let compose_hash = hex_of(0x22, 32); + let manifest = Manifest { + id: "snp-test".to_string(), + name: "snp-test".to_string(), + app_id: hex_of(0x11, 20), + vcpu: 2, + memory: 1024, + disk_size: 1024, + image: "dstack-test".to_string(), + port_map: vec![], + created_at_ms: 0, + hugepages: false, + pin_numa: false, + gpus: None, + kms_urls: vec![], + gateway_urls: vec![], + no_tee: false, + networking: None, + }; + + let sys_config: serde_json::Value = + serde_json::from_str(&make_sys_config(&config, &manifest, &compose_hash).unwrap()) + .unwrap(); + let vm_config: serde_json::Value = + serde_json::from_str(sys_config["vm_config"].as_str().unwrap()).unwrap(); + let measurement = &vm_config["sev_snp_measurement"]; + + assert_eq!(measurement["app_id"], manifest.app_id); + assert_eq!(measurement["compose_hash"], compose_hash); + assert_eq!(measurement["rootfs_hash"], hex_of(0x33, 32)); + assert_eq!( + measurement["base_cmdline"], + format!("console=ttyS0 dstack.rootfs_hash={}", hex_of(0x33, 32)) + ); + assert_eq!( + measurement["kernel_hash"], + hex::encode(Sha256::digest(b"snp-test-kernel")) + ); + assert_eq!( + measurement["initrd_hash"], + hex::encode(Sha256::digest(b"snp-test-initrd")) + ); + assert_eq!(measurement["vcpus"], 2); + assert_eq!(measurement["vcpu_type"], "EPYC-v4"); + assert_eq!(measurement["ovmf_hash"], ""); + assert!(measurement["ovmf_sections"].as_array().unwrap().is_empty()); + } +} + fn paginate(items: Vec, page: u32, page_size: u32) -> impl Iterator { let skip; let take; diff --git a/vmm/src/app/qemu.rs b/vmm/src/app/qemu.rs index f47a987b..d4ee854b 100644 --- a/vmm/src/app/qemu.rs +++ b/vmm/src/app/qemu.rs @@ -5,7 +5,9 @@ //! QEMU related code use crate::{ app::Manifest, - config::{CvmConfig, GatewayConfig, Networking, NetworkingMode, ProcessAnnotation}, + config::{ + CvmConfig, GatewayConfig, Networking, NetworkingMode, ProcessAnnotation, TeePlatform, + }, }; use std::{collections::HashMap, os::unix::fs::PermissionsExt}; use std::{ @@ -16,7 +18,7 @@ use std::{ time::{Duration, SystemTime}, }; -use super::{image::Image, GpuConfig, VmState}; +use super::{image::Image, GpuConfig, ImageInfo, VmState}; use anyhow::{bail, Context, Result}; use base64::prelude::*; use bon::Builder; @@ -362,7 +364,10 @@ impl VmState { #[cfg(test)] mod tests { - use super::sanitize_optional; + use super::{ + amd_sev_snp_measured_cmdline, amd_sev_snp_memory_backend_arg, amd_sev_snp_rootfs_hash, + sanitize_optional, virtio_pci_device, ImageInfo, + }; #[test] fn sanitize_optional_filters_empty_owned_values() { @@ -383,6 +388,93 @@ mod tests { Some("instance-123") ); } + + #[test] + fn amd_sev_snp_memory_backend_arg_uses_passed_final_memory_size() { + assert_eq!( + amd_sev_snp_memory_backend_arg(4096), + "memory-backend-memfd,id=ram1,size=4096M,share=true,prealloc=false" + ); + } + + #[test] + fn amd_sev_snp_measured_cmdline_binds_app_identity() { + assert_eq!( + amd_sev_snp_measured_cmdline( + "console=ttyS0 loglevel=7", + "22", + "33", + "1111111111111111111111111111111111111111", + None, + ), + "console=ttyS0 loglevel=7 docker_compose_hash=22 rootfs_hash=33 app_id=1111111111111111111111111111111111111111" + ); + } + + #[test] + fn amd_sev_snp_measured_cmdline_can_carry_kds_proxy_for_smoke() { + assert_eq!( + amd_sev_snp_measured_cmdline( + "console=ttyS0 loglevel=7", + "22", + "33", + "1111111111111111111111111111111111111111", + Some("https://cors.litgateway.com/"), + ), + "console=ttyS0 loglevel=7 dstack.amd_kds_proxy_url=https://cors.litgateway.com/ docker_compose_hash=22 rootfs_hash=33 app_id=1111111111111111111111111111111111111111" + ); + } + + #[test] + fn amd_sev_snp_rootfs_hash_falls_back_to_dstack_cmdline() { + let info = ImageInfo { + cmdline: Some("console=ttyS0 dstack.rootfs_hash=abc123 dstack.rootfs_size=100".into()), + kernel: "kernel".into(), + initrd: "initrd".into(), + hda: None, + rootfs: None, + bios: None, + rootfs_hash: None, + shared_ro: false, + version: "0.5.11".into(), + is_dev: false, + ovmf_variant: None, + }; + + assert_eq!(amd_sev_snp_rootfs_hash(&info).unwrap(), "abc123"); + } + + #[test] + fn amd_sev_snp_uses_confidential_virtio_pci_options() { + assert_eq!( + virtio_pci_device("virtio-blk-pci,drive=hd0", true), + "virtio-blk-pci,drive=hd0,disable-legacy=on,iommu_platform=true" + ); + assert_eq!( + virtio_pci_device("virtio-blk-pci,drive=hd0", false), + "virtio-blk-pci,drive=hd0" + ); + } +} + +fn amd_sev_snp_rootfs_hash(info: &ImageInfo) -> Result<&str> { + if let Some(rootfs_hash) = info.rootfs_hash.as_deref() { + return Ok(rootfs_hash); + } + info.cmdline + .as_deref() + .unwrap_or_default() + .split_whitespace() + .find_map(|param| param.strip_prefix("dstack.rootfs_hash=")) + .ok_or_else(|| anyhow::anyhow!("rootfs_hash is required for amd sev-snp")) +} + +fn virtio_pci_device(device: &str, snp: bool) -> String { + if snp { + format!("{device},disable-legacy=on,iommu_platform=true") + } else { + device.to_string() + } } impl VmConfig { @@ -410,11 +502,14 @@ impl VmConfig { } let app_compose = workdir.app_compose().context("Failed to get app compose")?; let qemu = &cfg.qemu_path; + let is_amd_sev_snp = + cfg.platform.resolve() == TeePlatform::AmdSevSnp && !self.manifest.no_tee; let mut smp = self.manifest.vcpu.max(1); let mut mem = self.manifest.memory; let mut command = Command::new(qemu); command.arg("-accel").arg("kvm"); - command.arg("-cpu").arg("host"); + let cpu = if is_amd_sev_snp { "EPYC-v4" } else { "host" }; + command.arg("-cpu").arg(cpu); command.arg("-nographic"); command.arg("-nodefaults"); command.arg("-chardev").arg(format!( @@ -470,7 +565,10 @@ impl VmConfig { "file={},if=none,id=hd0,format=raw,readonly=on", rootfs.display() )); - command.arg("-device").arg("virtio-blk-pci,drive=hd0"); + command.arg("-device").arg(virtio_pci_device( + "virtio-blk-pci,drive=hd0", + is_amd_sev_snp, + )); } _ => { bail!("Unsupported rootfs type: {ext}"); @@ -482,7 +580,10 @@ impl VmConfig { .arg("-drive") .arg(format!("file={},if=none,id=hd1", hda_path.display())) .arg("-device") - .arg("virtio-blk-pci,drive=hd1"); + .arg(virtio_pci_device( + "virtio-blk-pci,drive=hd1", + is_amd_sev_snp, + )); // Resolve per-VM networking override against global config. // Per-VM only sets mode; shared fields (bridge name, mac_prefix, etc.) // are merged from global config. @@ -506,7 +607,10 @@ impl VmConfig { // Generate deterministic MAC for all networking modes let prefix = networking.mac_prefix_bytes(); let mac = mac_address_for_vm(&self.manifest.id, &prefix); - let net_device = format!("virtio-net-pci,netdev=net0,mac={mac}"); + let net_device = virtio_pci_device( + &format!("virtio-net-pci,netdev=net0,mac={mac}"), + is_amd_sev_snp, + ); let netdev = match networking.mode { NetworkingMode::User => { let mut netdev = format!( @@ -535,7 +639,6 @@ impl VmConfig { command.arg("-netdev").arg(netdev); command.arg("-device").arg(net_device); - self.configure_machine(&mut command, &workdir, cfg, &app_compose)?; self.configure_smbios(&mut command, cfg); if matches!(app_compose.key_provider(), KeyProviderKind::Tpm) { @@ -553,9 +656,10 @@ impl VmConfig { .arg("tpm-tis,tpmdev=tpm0"); } - command - .arg("-device") - .arg(format!("vhost-vsock-pci,guest-cid={}", self.cid)); + command.arg("-device").arg(virtio_pci_device( + &format!("vhost-vsock-pci,guest-cid={}", self.cid), + is_amd_sev_snp, + )); // Configure shared files delivery: either via disk or 9p match cfg.host_share_mode.as_str() { @@ -580,7 +684,10 @@ impl VmConfig { HOST_SHARED_DISK_LABEL )) .arg("-device") - .arg("virtio-blk-pci,drive=vvfat0"); + .arg(virtio_pci_device( + "virtio-blk-pci,drive=vvfat0", + is_amd_sev_snp, + )); } "vhd" => { // Use a second virtual disk (hd2) to share files @@ -597,7 +704,10 @@ impl VmConfig { shared_disk_path.display() )) .arg("-device") - .arg("virtio-blk-pci,drive=hd2"); + .arg(virtio_pci_device( + "virtio-blk-pci,drive=hd2", + is_amd_sev_snp, + )); } _ => { bail!("Invalid host sharing mode: {}", cfg.host_share_mode); @@ -658,6 +768,8 @@ impl VmConfig { } } + self.configure_machine(&mut command, &workdir, cfg, &app_compose, mem)?; + // Configure GPU devices if !gpus.gpus.is_empty() { // Add iommufd object @@ -721,8 +833,30 @@ impl VmConfig { } } - // Add kernel command line - if let Some(cmdline) = &self.image.info.cmdline { + // Add kernel command line. SNP launch measurement includes app identity + // through the measured QEMU kernel command line, matching TDX's + // app-id-in-measured-identity semantics without relying on post-launch + // RTMRs (which SNP does not have). + let cmdline = match (&self.image.info.cmdline, cfg.platform.resolve()) { + (Some(cmdline), TeePlatform::AmdSevSnp) if !self.manifest.no_tee => { + let compose_hash = hex::encode( + workdir + .app_compose_hash() + .context("Failed to get compose hash")?, + ); + let rootfs_hash = amd_sev_snp_rootfs_hash(&self.image.info)?; + Some(amd_sev_snp_measured_cmdline( + cmdline, + &compose_hash, + rootfs_hash, + &self.manifest.app_id, + std::env::var("DSTACK_AMD_KDS_PROXY_URL").ok().as_deref(), + )) + } + (Some(cmdline), _) => Some(cmdline.clone()), + (None, _) => None, + }; + if let Some(cmdline) = cmdline { command.arg("-append").arg(cmdline); } @@ -783,6 +917,7 @@ impl VmConfig { workdir: &VmWorkDir, cfg: &CvmConfig, app_compose: &AppCompose, + mem: u32, ) -> Result<()> { if self.manifest.no_tee { command @@ -791,10 +926,27 @@ impl VmConfig { return Ok(()); } - command - .arg("-machine") - .arg("q35,kernel-irqchip=split,confidential-guest-support=tdx,hpet=off"); + match cfg.platform.resolve() { + TeePlatform::Tdx | TeePlatform::Auto => { + command + .arg("-machine") + .arg("q35,kernel-irqchip=split,confidential-guest-support=tdx,hpet=off"); + self.configure_tdx_guest(command, workdir, cfg, app_compose)?; + } + TeePlatform::AmdSevSnp => { + self.configure_amd_sev_snp_guest(command, cfg, mem); + } + } + Ok(()) + } + fn configure_tdx_guest( + &self, + command: &mut Command, + workdir: &VmWorkDir, + cfg: &CvmConfig, + app_compose: &AppCompose, + ) -> Result<()> { let img_ver = self.image.info.version_tuple().unwrap_or_default(); let support_mr_config_id = img_ver >= (0, 5, 2); @@ -871,6 +1023,21 @@ impl VmConfig { Ok(()) } + fn configure_amd_sev_snp_guest(&self, command: &mut Command, cfg: &CvmConfig, mem: u32) { + command + .arg("-object") + .arg(amd_sev_snp_memory_backend_arg(mem)); + command + .arg("-object") + .arg("sev-snp-guest,id=sev0,policy=0x30000,sev-device=/dev/sev,kernel-hashes=on,author-key-enabled=on,cbitpos=51,reduced-phys-bits=1"); + command.arg("-machine").arg( + "q35,kernel-irqchip=split,confidential-guest-support=sev0,memory-backend=ram1,hpet=off", + ); + if cfg.qgs_port.is_some() { + tracing::warn!("qgs_port is ignored for amd sev-snp guests"); + } + } + fn configure_smbios(&self, command: &mut Command, cfg: &CvmConfig) { let p = &cfg.product; @@ -916,6 +1083,41 @@ impl VmConfig { } } +fn amd_sev_snp_memory_backend_arg(mem: u32) -> String { + format!("memory-backend-memfd,id=ram1,size={mem}M,share=true,prealloc=false") +} + +fn amd_sev_snp_base_cmdline_with_kds_proxy( + base_cmdline: &str, + amd_kds_proxy_url: Option<&str>, +) -> String { + let mut cmdline = base_cmdline.trim().to_string(); + if let Some(proxy_url) = amd_kds_proxy_url + .map(str::trim) + .filter(|url| !url.is_empty()) + { + cmdline.push_str(" dstack.amd_kds_proxy_url="); + cmdline.push_str(proxy_url); + } + cmdline +} + +fn amd_sev_snp_measured_cmdline( + base_cmdline: &str, + compose_hash: &str, + rootfs_hash: &str, + app_id: &str, + amd_kds_proxy_url: Option<&str>, +) -> String { + format!( + "{} docker_compose_hash={} rootfs_hash={} app_id={}", + amd_sev_snp_base_cmdline_with_kds_proxy(base_cmdline, amd_kds_proxy_url), + compose_hash, + rootfs_hash, + app_id + ) +} + /// Round up a value to the nearest multiple of another value. /// If the value is already a multiple, it remains unchanged. fn round_up(value: u32, multiple: u32) -> u32 { diff --git a/vmm/src/config.rs b/vmm/src/config.rs index cdb58745..ce2194d0 100644 --- a/vmm/src/config.rs +++ b/vmm/src/config.rs @@ -106,6 +106,33 @@ impl Protocol { } } +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Deserialize, Serialize)] +#[serde(rename_all = "kebab-case")] +pub enum TeePlatform { + #[default] + Auto, + Tdx, + AmdSevSnp, +} + +impl TeePlatform { + pub fn resolve(self) -> Self { + match self { + Self::Auto => Self::resolve_from_cpuinfo( + &fs_err::read_to_string("/proc/cpuinfo").unwrap_or_default(), + ), + platform => platform, + } + } + + pub fn resolve_from_cpuinfo(_cpuinfo: &str) -> Self { + // Keep `auto` conservative while AMD SEV-SNP support is experimental and + // verifier/KMS/app binding are not production-ready. Operators must opt + // into SNP explicitly with `platform = "amd-sev-snp"`. + Self::Tdx + } +} + #[derive(Debug, Clone, Deserialize, Serialize)] pub struct PortRange { pub protocol: Protocol, @@ -143,6 +170,9 @@ impl PortMappingConfig { #[derive(Debug, Clone, Deserialize)] pub struct CvmConfig { + /// TEE platform to use when launching CVMs. + #[serde(default)] + pub platform: TeePlatform, pub qemu_path: PathBuf, /// The URL of the KMS server pub kms_urls: Vec, @@ -605,4 +635,22 @@ mod tests { let result = parse_qemu_version_from_output(output); assert!(result.is_err()); } + + #[test] + fn tee_platform_deserializes_amd_sev_snp() { + let platform: TeePlatform = serde_json::from_str("\"amd-sev-snp\"").unwrap(); + assert_eq!(platform, TeePlatform::AmdSevSnp); + } + + #[test] + fn tee_platform_auto_stays_tdx_even_with_amd_snp_flag_while_experimental() { + let cpuinfo = "flags : fpu svm sev sev_es sev_snp debug_swap"; + assert_eq!(TeePlatform::resolve_from_cpuinfo(cpuinfo), TeePlatform::Tdx); + } + + #[test] + fn tee_platform_auto_falls_back_to_tdx_without_amd_snp_flag() { + let cpuinfo = "flags : fpu vmx tdx_guest"; + assert_eq!(TeePlatform::resolve_from_cpuinfo(cpuinfo), TeePlatform::Tdx); + } } diff --git a/vmm/src/one_shot.rs b/vmm/src/one_shot.rs index 39f9d68f..0736e1a7 100644 --- a/vmm/src/one_shot.rs +++ b/vmm/src/one_shot.rs @@ -235,7 +235,7 @@ Compose file content (first 200 chars): // 2. Create .sys-config.json (critical for 0.5.x VMs) // Use manifest URLs if available, fallback to config URLs (matching VMM's sync_dynamic_config logic) - let sys_config_str = make_sys_config(&config, &manifest)?; + let sys_config_str = make_sys_config(&config, &manifest, &compose_hash)?; let sys_config_path = vm_work_dir.shared_dir().join(".sys-config.json"); fs_err::write(&sys_config_path, sys_config_str).context("Failed to write sys config")?; diff --git a/vmm/vmm.toml b/vmm/vmm.toml index ba8e2a88..73d8c124 100644 --- a/vmm/vmm.toml +++ b/vmm/vmm.toml @@ -21,6 +21,8 @@ node_name = "" registry = "" [cvm] +# TEE platform: "auto", "tdx", or "amd-sev-snp". Auto selects AMD SEV-SNP when host CPU flags include sev_snp, otherwise TDX. +platform = "auto" qemu_path = "" kms_urls = ["http://127.0.0.1:8081"] gateway_urls = ["http://127.0.0.1:8082"]