From 63cd62f174c6a0e1457ee6f3509f951895d15de4 Mon Sep 17 00:00:00 2001 From: "zhong.zhou" Date: Sat, 9 May 2026 19:05:28 +0800 Subject: [PATCH] feat(storage): local primary LUKS qcow2 creation only Keep KVM LUKS agent params on InstantiateVolumeMsg / InstantiateVolumeOnPrimaryStorageMsg (VolumeLuksAgentSpec), premium pre-instantiate DEK path, and LocalStorageKvmBackend wiring. Shared MergeSnapshotCmd encrypt fields removed; helper script commit-push-three-repos.sh at repo root. Co-authored-by: Cursor --- .../compute/vm/VmAllocateVolumeFlow.java | 3 + .../zstack/compute/vm/VmInstanceUtils.java | 62 +++ .../vm/VmInstantiateOtherDiskFlow.java | 53 ++ conf/db/zsv/V5.1.0__schema.sql | 33 ++ conf/springConfigXml/VolumeManager.xml | 45 ++ ...otVolumeTemplateFromVolumeSnapshotMsg.java | 10 + .../SecretHostEnsureLuksSecretFileMsg.java | 28 ++ .../SecretHostEnsureLuksSecretFileReply.java | 18 + .../header/secret/SecretHostGetReply.java | 18 + ...FromVolumeSnapshotOnPrimaryStorageMsg.java | 9 + ...FromVolumeSnapshotOnPrimaryStorageMsg.java | 10 + .../EncryptVolumeBitsOnPrimaryStorageMsg.java | 61 +++ ...ncryptVolumeBitsOnPrimaryStorageReply.java | 6 + .../InstantiateVolumeOnPrimaryStorageMsg.java | 10 + ...VolumeFromTemplateOnPrimaryStorageMsg.java | 9 + ...CreateImageCacheFromVolumeSnapshotMsg.java | 9 + .../snapshot/TakeSnapshotsOnKvmJobStruct.java | 9 + .../storage/snapshot/VolumeSnapshotAO.java | 11 + .../storage/snapshot/VolumeSnapshotAO_.java | 1 + .../snapshot/VolumeSnapshotInventory.java | 11 + ...eVmInstanceFromVolumeSnapshotGroupMsg.java | 32 ++ .../java/org/zstack/header/vm/DiskAO.java | 9 + .../CreateDataVolumeExtensionPoint.java | 4 + ...CreateDataVolumeFromVolumeSnapshotMsg.java | 9 + ...CreateDataVolumeFromVolumeTemplateMsg.java | 9 + .../zstack/header/volume/CreateVolumeMsg.java | 11 + .../header/volume/EncryptVolumeMsg.java | 86 ++++ .../header/volume/EncryptVolumeReply.java | 18 + .../header/volume/InstantiateVolumeMsg.java | 9 + .../org/zstack/header/volume/VolumeAO.java | 11 + .../org/zstack/header/volume/VolumeAO_.java | 1 + .../header/volume/VolumeCreateMessage.java | 7 + .../zstack/header/volume/VolumeInventory.java | 12 + .../header/volume/VolumeLuksAgentSpec.java | 31 ++ .../org/zstack/image/ImageManagerImpl.java | 1 + .../ceph/primary/CephPrimaryStorageBase.java | 475 +++++++++++++++++- .../java/org/zstack/kvm/KVMAgentCommands.java | 42 ++ .../main/java/org/zstack/kvm/KVMConstant.java | 11 + .../src/main/java/org/zstack/kvm/KVMHost.java | 127 +++++ .../main/java/org/zstack/kvm/VolumeTO.java | 13 + .../org/zstack/kvm/tpm/KvmTpmExtensions.java | 10 +- .../primary/local/LocalStorageBase.java | 19 + .../LocalStorageCreateEmptyVolumeMsg.java | 10 + .../primary/local/LocalStorageFactory.java | 2 + .../local/LocalStorageHypervisorBackend.java | 8 +- .../primary/local/LocalStorageKvmBackend.java | 200 +++++++- .../primary/local/LocalStorageKvmFactory.java | 2 +- .../nfs/NfsDownloadImageToCacheJob.java | 15 +- .../primary/nfs/NfsPrimaryStorage.java | 32 +- .../primary/nfs/NfsPrimaryStorageBackend.java | 14 +- .../nfs/NfsPrimaryStorageKVMBackend.java | 154 +++++- .../NfsPrimaryStorageKVMBackendCommands.java | 72 +++ storage/pom.xml | 7 +- ...ummyVolumeEncryptedResourceKeyBackend.java | 103 ++++ ...shotGroupRevertVolumeEncryptionHelper.java | 110 ++++ .../VolumeEncryptedAttachExtension.java | 74 +++ .../VolumeEncryptedExpungeExtension.java | 151 ++++++ .../VolumeEncryptedInitialExtension.java | 168 +++++++ .../VolumeEncryptedResourceKeyBackend.java | 73 +++ .../encrypt/VolumeEncryptedSecretHelper.java | 298 +++++++++++ .../VolumeEncryptedStartExtension.java | 110 ++++ .../VolumeSnapshotEncryptionExtension.java | 258 ++++++++++ .../VolumeSnapshotEncryptionHelper.java | 314 ++++++++++++ .../snapshot/VolumeSnapshotManagerImpl.java | 5 + .../snapshot/VolumeSnapshotTreeBase.java | 1 + .../org/zstack/storage/volume/VolumeBase.java | 46 ++ .../volume/VolumeInPlaceEncryptor.java | 264 ++++++++++ .../storage/volume/VolumeManagerImpl.java | 117 ++++- .../storage/volume/VolumeSystemTags.java | 5 + 69 files changed, 3938 insertions(+), 38 deletions(-) create mode 100644 conf/db/zsv/V5.1.0__schema.sql mode change 100755 => 100644 conf/springConfigXml/VolumeManager.xml create mode 100644 header/src/main/java/org/zstack/header/secret/SecretHostEnsureLuksSecretFileMsg.java create mode 100644 header/src/main/java/org/zstack/header/secret/SecretHostEnsureLuksSecretFileReply.java create mode 100644 header/src/main/java/org/zstack/header/storage/primary/EncryptVolumeBitsOnPrimaryStorageMsg.java create mode 100644 header/src/main/java/org/zstack/header/storage/primary/EncryptVolumeBitsOnPrimaryStorageReply.java create mode 100644 header/src/main/java/org/zstack/header/volume/EncryptVolumeMsg.java create mode 100644 header/src/main/java/org/zstack/header/volume/EncryptVolumeReply.java create mode 100644 header/src/main/java/org/zstack/header/volume/VolumeLuksAgentSpec.java mode change 100755 => 100644 storage/pom.xml create mode 100644 storage/src/main/java/org/zstack/storage/encrypt/DummyVolumeEncryptedResourceKeyBackend.java create mode 100644 storage/src/main/java/org/zstack/storage/encrypt/SnapshotGroupRevertVolumeEncryptionHelper.java create mode 100644 storage/src/main/java/org/zstack/storage/encrypt/VolumeEncryptedAttachExtension.java create mode 100644 storage/src/main/java/org/zstack/storage/encrypt/VolumeEncryptedExpungeExtension.java create mode 100644 storage/src/main/java/org/zstack/storage/encrypt/VolumeEncryptedInitialExtension.java create mode 100644 storage/src/main/java/org/zstack/storage/encrypt/VolumeEncryptedResourceKeyBackend.java create mode 100644 storage/src/main/java/org/zstack/storage/encrypt/VolumeEncryptedSecretHelper.java create mode 100644 storage/src/main/java/org/zstack/storage/encrypt/VolumeEncryptedStartExtension.java create mode 100644 storage/src/main/java/org/zstack/storage/encrypt/VolumeSnapshotEncryptionExtension.java create mode 100644 storage/src/main/java/org/zstack/storage/encrypt/VolumeSnapshotEncryptionHelper.java create mode 100644 storage/src/main/java/org/zstack/storage/volume/VolumeInPlaceEncryptor.java diff --git a/compute/src/main/java/org/zstack/compute/vm/VmAllocateVolumeFlow.java b/compute/src/main/java/org/zstack/compute/vm/VmAllocateVolumeFlow.java index bb21cd4a0e5..81bdea71966 100755 --- a/compute/src/main/java/org/zstack/compute/vm/VmAllocateVolumeFlow.java +++ b/compute/src/main/java/org/zstack/compute/vm/VmAllocateVolumeFlow.java @@ -118,6 +118,8 @@ protected List prepareMsg(Map ctx) { if (disk != null && !isEmpty(disk.getSystemTags())) { tags.addAll(disk.getSystemTags()); } + Boolean volEnc = disk != null && Boolean.TRUE.equals(disk.getEncrypted()); + msg.setEncrypted(volEnc); } else if (vspec.isData()) { DiskAO disk = isEmpty(spec.getDataDisks()) ? null : spec.getDataDisks().size() > dataVolumeIndex ? spec.getDataDisks().get(dataVolumeIndex) : null; @@ -139,6 +141,7 @@ protected List prepareMsg(Map ctx) { if (disk != null && !isEmpty(disk.getSystemTags())) { tags.addAll(disk.getSystemTags()); } + msg.setEncrypted(disk != null && Boolean.TRUE.equals(disk.getEncrypted())); dataVolumeIndex++; } else { diff --git a/compute/src/main/java/org/zstack/compute/vm/VmInstanceUtils.java b/compute/src/main/java/org/zstack/compute/vm/VmInstanceUtils.java index bebcae3520b..f6247779c0b 100644 --- a/compute/src/main/java/org/zstack/compute/vm/VmInstanceUtils.java +++ b/compute/src/main/java/org/zstack/compute/vm/VmInstanceUtils.java @@ -143,9 +143,71 @@ public static CreateVmInstanceMsg fromAPICreateVmInstanceMsg(APICreateVmInstance } } + // applyForceEncryptEnvOverride(cmsg); return cmsg; } + /** + * Temporary debug switch. Priority (first match wins): + *
    + *
  1. {@link #FORCE_ENCRYPT_VOLUME_HARDCODED} — flip in code, rebuild, deploy. + * Use this when the deployment regenerates setenv.sh / systemd unit and + * JVM properties can't be reliably injected.
    2. + *
  2. System property {@code zstack.force.encrypt.volume} — + * pass via {@code -Dzstack.force.encrypt.volume=true}.
    3. + *
  3. Environment variable {@code ZSTACK_FORCE_ENCRYPT_VOLUME}.
    4. + *
+ * When the switch is on, every {@link DiskAO} on the {@link CreateVmInstanceMsg} + * is force-marked encrypted=true; when off, encrypted=false. Covers all five + * disk sources funneled into this msg by {@link #fromAPICreateVmInstanceMsg}: + *
    + *
  • root disk (empty / from-image)
    • + *
  • data disks from {@code APICreateVmInstanceMsg.diskAOs} + * (from-image / from-existing-volume)
    • + *
  • data disks from the legacy {@code dataDiskOfferingUuids / + * dataDiskSizes} path (deprecatedDataVolumeSpecs)
    • + *
+ */ + private static final Boolean FORCE_ENCRYPT_VOLUME_HARDCODED = true; + static final String FORCE_ENCRYPT_VOLUME_ENV = "ZSTACK_FORCE_ENCRYPT_VOLUME"; + private static final String FORCE_ENCRYPT_VOLUME_PROPERTY = "zstack.force.encrypt.volume"; + + private static boolean isForceEncryptVolume() { + if (FORCE_ENCRYPT_VOLUME_HARDCODED != null) { + return FORCE_ENCRYPT_VOLUME_HARDCODED; + } + String v = System.getProperty(FORCE_ENCRYPT_VOLUME_PROPERTY); + if (v == null || v.isEmpty()) { + v = System.getenv(FORCE_ENCRYPT_VOLUME_ENV); + } + if (v == null || v.isEmpty()) { + return false; + } + v = v.trim().toLowerCase(); + return v.equals("1") || v.equals("true") || v.equals("yes") || v.equals("on"); + } + + private static void applyForceEncryptEnvOverride(CreateVmInstanceMsg cmsg) { + boolean forceOn = isForceEncryptVolume(); + if (cmsg.getRootDisk() != null) { + cmsg.getRootDisk().setEncrypted(forceOn); + } + if (cmsg.getDataDisks() != null) { + for (DiskAO d : cmsg.getDataDisks()) { + if (d != null) { + d.setEncrypted(forceOn); + } + } + } + if (cmsg.getDeprecatedDataVolumeSpecs() != null) { + for (DiskAO d : cmsg.getDeprecatedDataVolumeSpecs()) { + if (d != null) { + d.setEncrypted(forceOn); + } + } + } + } + private static String getPSUuidForDataVolume(List systemTags){ if (systemTags == null || systemTags.isEmpty()){ return null; diff --git a/compute/src/main/java/org/zstack/compute/vm/VmInstantiateOtherDiskFlow.java b/compute/src/main/java/org/zstack/compute/vm/VmInstantiateOtherDiskFlow.java index 69537bd55f3..d6dd34d38b6 100644 --- a/compute/src/main/java/org/zstack/compute/vm/VmInstantiateOtherDiskFlow.java +++ b/compute/src/main/java/org/zstack/compute/vm/VmInstantiateOtherDiskFlow.java @@ -86,6 +86,7 @@ public void setup() { } else if (isAttachDataVolume()) { VolumeVO volume = Q.New(VolumeVO.class).eq(VolumeVO_.uuid, diskAO.getSourceUuid()).find(); volumeInventory = VolumeInventory.valueOf(volume); + setupEncryptExistingVolumeFlow(); setupAttachVolumeFlows(); } else if (diskAO.getSourceUuid() != null && diskAO.getSourceType() != null) { setupAttachOtherDiskFlows(); @@ -180,6 +181,7 @@ public void run(final FlowTrigger innerTrigger, Map data) { msg.setDiskOfferingUuid(diskAO.getDiskOfferingUuid()); msg.setPrimaryStorageUuid(allocatedPrimaryStorageUuid); msg.setDescription(String.format("vm-%s-data-volume", vmUuid)); + msg.setEncrypted(Boolean.TRUE.equals(diskAO.getEncrypted())); bus.makeLocalServiceId(msg, VolumeConstant.SERVICE_ID); bus.send(msg, new CloudBusCallBack(innerTrigger) { @Override @@ -328,6 +330,7 @@ public void run(final FlowTrigger innerTrigger, Map data) { } else { cmsg.setPrimaryStorageUuid(allocatedPrimaryStorageUuid[0]); } + cmsg.setEncrypted(Boolean.TRUE.equals(diskAO.getEncrypted())); bus.makeLocalServiceId(cmsg, VolumeConstant.SERVICE_ID); bus.send(cmsg, new CloudBusCallBack(innerTrigger) { @@ -404,6 +407,56 @@ public void run(MessageReply reply) { }); } + /** + * When the caller requested an encrypted data volume (DiskAO.encrypted=true) but the + * existing source volume is not yet encrypted, transition the source bits to LUKS + * in place before attaching. Delegates to {@code EncryptVolumeMsg} so the actual + * key/secret/PS-conversion logic lives in {@code VolumeBase} (shared with the + * create-data-volume-from-template flow). + * + *

Skipped when: + *

    + *
  • {@code DiskAO.encrypted} is false/null, or
    • + *
  • the source volume is already encrypted (no-op transition).
    • + *
+ */ + private void setupEncryptExistingVolumeFlow() { + if (!Boolean.TRUE.equals(diskAO.getEncrypted())) { + return; + } + if (volumeInventory != null && Boolean.TRUE.equals(volumeInventory.getEncrypted())) { + return; + } + flow(new NoRollbackFlow() { + String __name__ = String.format("encrypt-existing-data-volume-%s-in-place", + diskAO.getSourceUuid()); + + @Override + public void run(final FlowTrigger innerTrigger, Map data) { + EncryptVolumeMsg emsg = new EncryptVolumeMsg(); + emsg.setVolumeUuid(volumeInventory.getUuid()); + emsg.setHostUuid(hostUuid); + emsg.setPurpose("attach-existing-disk-as-encrypted-data-volume"); + bus.makeTargetServiceIdByResourceUuid(emsg, VolumeConstant.SERVICE_ID, + volumeInventory.getUuid()); + bus.send(emsg, new CloudBusCallBack(innerTrigger) { + @Override + public void run(MessageReply reply) { + if (!reply.isSuccess()) { + innerTrigger.fail(reply.getError()); + return; + } + EncryptVolumeReply er = reply.castReply(); + if (er.getInventory() != null) { + volumeInventory = er.getInventory(); + } + innerTrigger.next(); + } + }); + } + }); + } + private void setupAttachOtherDiskFlows() { flow(new NoRollbackFlow() { String __name__ = String.format("attach-other-Disk-to-vm-%s", vmUuid); diff --git a/conf/db/zsv/V5.1.0__schema.sql b/conf/db/zsv/V5.1.0__schema.sql new file mode 100644 index 00000000000..7d4a7b6679f --- /dev/null +++ b/conf/db/zsv/V5.1.0__schema.sql @@ -0,0 +1,33 @@ +CREATE TABLE IF NOT EXISTS `zstack`.`TpmKeyBackupVO` ( + `uuid` char(32) NOT NULL UNIQUE, + `lastOpDate` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + `createDate` timestamp NOT NULL DEFAULT '1999-12-31 23:59:59', + PRIMARY KEY (`uuid`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8; + +DELETE FROM `EncryptedResourceKeyRefVO` + WHERE `resourceUuid` NOT IN (SELECT `uuid` FROM `ResourceVO`); +ALTER TABLE `EncryptedResourceKeyRefVO` + ADD CONSTRAINT `fkEncryptedResourceKeyRefResourceVO` FOREIGN KEY (`resourceUuid`) REFERENCES `ResourceVO`(`uuid`) + ON DELETE CASCADE; + +-- Volume LUKS encryption flag (API opt-in + EncryptedResourceKeyRefVO binding) + +ALTER TABLE `zstack`.`VolumeEO` ADD COLUMN `encrypted` tinyint(1) NOT NULL DEFAULT 0; +ALTER TABLE `zstack`.`VolumeSnapshotEO` ADD COLUMN `encrypted` tinyint(1) NOT NULL DEFAULT 0; + +DROP VIEW IF EXISTS `zstack`.`VolumeVO`; +CREATE VIEW `zstack`.`VolumeVO` AS +SELECT uuid, name, description, primaryStorageUuid, vmInstanceUuid, diskOfferingUuid, + rootImageUuid, installPath, type, status, size, actualSize, deviceId, format, state, createDate, lastOpDate, + isShareable, volumeQos, lastVmInstanceUuid, lastDetachDate, lastAttachDate, protocol, encrypted +FROM `zstack`.`VolumeEO` +WHERE deleted IS NULL; + +DROP VIEW IF EXISTS `zstack`.`VolumeSnapshotVO`; +CREATE VIEW `zstack`.`VolumeSnapshotVO` AS +SELECT uuid, name, description, type, volumeUuid, format, treeUuid, parentUuid, + primaryStorageUuid, primaryStorageInstallPath, distance, size, latest, + fullSnapshot, encrypted, volumeType, state, status, createDate, lastOpDate +FROM `zstack`.`VolumeSnapshotEO` +WHERE deleted IS NULL; diff --git a/conf/springConfigXml/VolumeManager.xml b/conf/springConfigXml/VolumeManager.xml old mode 100755 new mode 100644 index 977134a8873..4947712f98d --- a/conf/springConfigXml/VolumeManager.xml +++ b/conf/springConfigXml/VolumeManager.xml @@ -92,4 +92,49 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/header/src/main/java/org/zstack/header/image/CreateTemporaryRootVolumeTemplateFromVolumeSnapshotMsg.java b/header/src/main/java/org/zstack/header/image/CreateTemporaryRootVolumeTemplateFromVolumeSnapshotMsg.java index 2aa59085fad..e766932d0ac 100644 --- a/header/src/main/java/org/zstack/header/image/CreateTemporaryRootVolumeTemplateFromVolumeSnapshotMsg.java +++ b/header/src/main/java/org/zstack/header/image/CreateTemporaryRootVolumeTemplateFromVolumeSnapshotMsg.java @@ -15,6 +15,7 @@ public class CreateTemporaryRootVolumeTemplateFromVolumeSnapshotMsg extends Need private boolean system; private SessionInventory session; private boolean virtio = true; + private Boolean encrypted; @Override public void setSnapshotUuid(String snapshotUuid) { @@ -79,6 +80,15 @@ public boolean isSystem() { public void setSystem(boolean system) { this.system = system; } + + public Boolean getEncrypted() { + return encrypted; + } + + public void setEncrypted(Boolean encrypted) { + this.encrypted = encrypted; + } + @Override public SessionInventory getSession() { diff --git a/header/src/main/java/org/zstack/header/secret/SecretHostEnsureLuksSecretFileMsg.java b/header/src/main/java/org/zstack/header/secret/SecretHostEnsureLuksSecretFileMsg.java new file mode 100644 index 00000000000..425a35a3d15 --- /dev/null +++ b/header/src/main/java/org/zstack/header/secret/SecretHostEnsureLuksSecretFileMsg.java @@ -0,0 +1,28 @@ +package org.zstack.header.secret; + +import org.zstack.header.host.HostMessage; +import org.zstack.header.log.NoLogging; +import org.zstack.header.message.NeedReplyMessage; + +public class SecretHostEnsureLuksSecretFileMsg extends NeedReplyMessage implements HostMessage { + private String hostUuid; + @NoLogging + private String dekBase64; + + @Override + public String getHostUuid() { + return hostUuid; + } + + public void setHostUuid(String hostUuid) { + this.hostUuid = hostUuid; + } + + public String getDekBase64() { + return dekBase64; + } + + public void setDekBase64(String dekBase64) { + this.dekBase64 = dekBase64; + } +} diff --git a/header/src/main/java/org/zstack/header/secret/SecretHostEnsureLuksSecretFileReply.java b/header/src/main/java/org/zstack/header/secret/SecretHostEnsureLuksSecretFileReply.java new file mode 100644 index 00000000000..129cc5cb2e5 --- /dev/null +++ b/header/src/main/java/org/zstack/header/secret/SecretHostEnsureLuksSecretFileReply.java @@ -0,0 +1,18 @@ +package org.zstack.header.secret; + +import org.zstack.header.message.MessageReply; + +public class SecretHostEnsureLuksSecretFileReply extends MessageReply { + public static final String ERROR_CODE_KEYS_NOT_ON_DISK = "KEY_AGENT_KEYS_NOT_ON_DISK"; + public static final String ERROR_CODE_KEY_FILES_INTEGRITY_MISMATCH = "KEY_AGENT_KEY_FILES_INTEGRITY_MISMATCH"; + + private String secFilePath; + + public String getSecFilePath() { + return secFilePath; + } + + public void setSecFilePath(String secFilePath) { + this.secFilePath = secFilePath; + } +} diff --git a/header/src/main/java/org/zstack/header/secret/SecretHostGetReply.java b/header/src/main/java/org/zstack/header/secret/SecretHostGetReply.java index cd474433110..500b42a9b38 100644 --- a/header/src/main/java/org/zstack/header/secret/SecretHostGetReply.java +++ b/header/src/main/java/org/zstack/header/secret/SecretHostGetReply.java @@ -1,5 +1,6 @@ package org.zstack.header.secret; +import org.zstack.header.errorcode.ErrorCode; import org.zstack.header.message.MessageReply; /** Reply for SecretHostGetMsg. */ @@ -15,4 +16,21 @@ public String getSecretUuid() { public void setSecretUuid(String secretUuid) { this.secretUuid = secretUuid; } + + /** + * Distinguish "secret not present on host" (idempotent re-define needed) + * from genuine RPC / agent failures. key-agent's not-found surfaces either + * as the canonical {@link #ERROR_CODE_SECRET_NOT_FOUND} code or embedded + * in {@code details} depending on the bus hop. + */ + public static boolean isSecretNotFound(ErrorCode err) { + if (err == null) { + return false; + } + if (ERROR_CODE_SECRET_NOT_FOUND.equals(err.getCode())) { + return true; + } + String details = err.getDetails(); + return details != null && details.contains(ERROR_CODE_SECRET_NOT_FOUND); + } } diff --git a/header/src/main/java/org/zstack/header/storage/primary/CreateImageCacheFromVolumeSnapshotOnPrimaryStorageMsg.java b/header/src/main/java/org/zstack/header/storage/primary/CreateImageCacheFromVolumeSnapshotOnPrimaryStorageMsg.java index ab65b9063bb..70f88692e57 100644 --- a/header/src/main/java/org/zstack/header/storage/primary/CreateImageCacheFromVolumeSnapshotOnPrimaryStorageMsg.java +++ b/header/src/main/java/org/zstack/header/storage/primary/CreateImageCacheFromVolumeSnapshotOnPrimaryStorageMsg.java @@ -10,6 +10,7 @@ public class CreateImageCacheFromVolumeSnapshotOnPrimaryStorageMsg extends NeedReplyMessage implements PrimaryStorageMessage { private VolumeSnapshotInventory volumeSnapshot; private ImageInventory imageInventory; + private Boolean encrypted; @Override public String getPrimaryStorageUuid() { @@ -32,4 +33,12 @@ public void setImageInventory(ImageInventory imageInventory) { this.imageInventory = imageInventory; } + public Boolean getEncrypted() { + return encrypted; + } + + public void setEncrypted(Boolean encrypted) { + this.encrypted = encrypted; + } + } diff --git a/header/src/main/java/org/zstack/header/storage/primary/CreateVolumeFromVolumeSnapshotOnPrimaryStorageMsg.java b/header/src/main/java/org/zstack/header/storage/primary/CreateVolumeFromVolumeSnapshotOnPrimaryStorageMsg.java index d01daa28aca..1233ac99f47 100755 --- a/header/src/main/java/org/zstack/header/storage/primary/CreateVolumeFromVolumeSnapshotOnPrimaryStorageMsg.java +++ b/header/src/main/java/org/zstack/header/storage/primary/CreateVolumeFromVolumeSnapshotOnPrimaryStorageMsg.java @@ -2,11 +2,13 @@ import org.zstack.header.message.NeedReplyMessage; import org.zstack.header.storage.snapshot.VolumeSnapshotInventory; +import org.zstack.header.volume.VolumeLuksAgentSpec; public class CreateVolumeFromVolumeSnapshotOnPrimaryStorageMsg extends NeedReplyMessage implements PrimaryStorageMessage { private String volumeUuid; private String primaryStorageUuid; private VolumeSnapshotInventory snapshot; + private VolumeLuksAgentSpec volumeLuksAgentSpec; public VolumeSnapshotInventory getSnapshot() { return snapshot; @@ -31,4 +33,12 @@ public String getPrimaryStorageUuid() { public void setPrimaryStorageUuid(String primaryStorageUuid) { this.primaryStorageUuid = primaryStorageUuid; } + + public VolumeLuksAgentSpec getVolumeLuksAgentSpec() { + return volumeLuksAgentSpec; + } + + public void setVolumeLuksAgentSpec(VolumeLuksAgentSpec volumeLuksAgentSpec) { + this.volumeLuksAgentSpec = volumeLuksAgentSpec; + } } diff --git a/header/src/main/java/org/zstack/header/storage/primary/EncryptVolumeBitsOnPrimaryStorageMsg.java b/header/src/main/java/org/zstack/header/storage/primary/EncryptVolumeBitsOnPrimaryStorageMsg.java new file mode 100644 index 00000000000..297165fc17e --- /dev/null +++ b/header/src/main/java/org/zstack/header/storage/primary/EncryptVolumeBitsOnPrimaryStorageMsg.java @@ -0,0 +1,61 @@ +package org.zstack.header.storage.primary; + +import org.zstack.header.message.NeedReplyMessage; + +/** + * Triggers an in-place LUKS encryption of an existing volume file on primary storage. + * Used after downloading a data-volume template's plain bits to LocalStorage when the + * volume is marked encrypted: the agent converts the plain qcow2/raw at {@link #installPath} + * into a LUKS-encrypted qcow2 (overwriting in place). + * + * The DEK is staged on the host out-of-band (caller stages the secret material file via + * SecretHostEnsureLuksSecretFileMsg and passes the file path here). + */ +public class EncryptVolumeBitsOnPrimaryStorageMsg extends NeedReplyMessage implements PrimaryStorageMessage { + private String primaryStorageUuid; + private String hostUuid; + private String volumeUuid; + private String installPath; + private String encryptLuksSecretMaterialFilePath; + + @Override + public String getPrimaryStorageUuid() { + return primaryStorageUuid; + } + + public void setPrimaryStorageUuid(String primaryStorageUuid) { + this.primaryStorageUuid = primaryStorageUuid; + } + + public String getHostUuid() { + return hostUuid; + } + + public void setHostUuid(String hostUuid) { + this.hostUuid = hostUuid; + } + + public String getVolumeUuid() { + return volumeUuid; + } + + public void setVolumeUuid(String volumeUuid) { + this.volumeUuid = volumeUuid; + } + + public String getInstallPath() { + return installPath; + } + + public void setInstallPath(String installPath) { + this.installPath = installPath; + } + + public String getEncryptLuksSecretMaterialFilePath() { + return encryptLuksSecretMaterialFilePath; + } + + public void setEncryptLuksSecretMaterialFilePath(String encryptLuksSecretMaterialFilePath) { + this.encryptLuksSecretMaterialFilePath = encryptLuksSecretMaterialFilePath; + } +} diff --git a/header/src/main/java/org/zstack/header/storage/primary/EncryptVolumeBitsOnPrimaryStorageReply.java b/header/src/main/java/org/zstack/header/storage/primary/EncryptVolumeBitsOnPrimaryStorageReply.java new file mode 100644 index 00000000000..2410ced31e6 --- /dev/null +++ b/header/src/main/java/org/zstack/header/storage/primary/EncryptVolumeBitsOnPrimaryStorageReply.java @@ -0,0 +1,6 @@ +package org.zstack.header.storage.primary; + +import org.zstack.header.message.MessageReply; + +public class EncryptVolumeBitsOnPrimaryStorageReply extends MessageReply { +} diff --git a/header/src/main/java/org/zstack/header/storage/primary/InstantiateVolumeOnPrimaryStorageMsg.java b/header/src/main/java/org/zstack/header/storage/primary/InstantiateVolumeOnPrimaryStorageMsg.java index ee0d3ae796f..d9306da2355 100755 --- a/header/src/main/java/org/zstack/header/storage/primary/InstantiateVolumeOnPrimaryStorageMsg.java +++ b/header/src/main/java/org/zstack/header/storage/primary/InstantiateVolumeOnPrimaryStorageMsg.java @@ -4,6 +4,7 @@ import org.zstack.header.message.NeedReplyMessage; import org.zstack.header.message.ReplayableMessage; import org.zstack.header.volume.VolumeInventory; +import org.zstack.header.volume.VolumeLuksAgentSpec; public class InstantiateVolumeOnPrimaryStorageMsg extends NeedReplyMessage implements PrimaryStorageMessage, ReplayableMessage { private HostInventory destHost; @@ -11,6 +12,7 @@ public class InstantiateVolumeOnPrimaryStorageMsg extends NeedReplyMessage imple private String primaryStorageUuid; private boolean skipIfExisting; private String allocatedInstallUrl; + private VolumeLuksAgentSpec volumeLuksAgentSpec; public String getAllocatedInstallUrl() { return allocatedInstallUrl; @@ -53,6 +55,14 @@ public void setSkipIfExisting(boolean skipIfExisting) { this.skipIfExisting = skipIfExisting; } + public VolumeLuksAgentSpec getVolumeLuksAgentSpec() { + return volumeLuksAgentSpec; + } + + public void setVolumeLuksAgentSpec(VolumeLuksAgentSpec volumeLuksAgentSpec) { + this.volumeLuksAgentSpec = volumeLuksAgentSpec; + } + @Override public String getResourceUuid() { return volume.getUuid(); diff --git a/header/src/main/java/org/zstack/header/storage/primary/ReInitRootVolumeFromTemplateOnPrimaryStorageMsg.java b/header/src/main/java/org/zstack/header/storage/primary/ReInitRootVolumeFromTemplateOnPrimaryStorageMsg.java index ba07327a0e8..0ac3cdf99f1 100644 --- a/header/src/main/java/org/zstack/header/storage/primary/ReInitRootVolumeFromTemplateOnPrimaryStorageMsg.java +++ b/header/src/main/java/org/zstack/header/storage/primary/ReInitRootVolumeFromTemplateOnPrimaryStorageMsg.java @@ -8,6 +8,7 @@ public class ReInitRootVolumeFromTemplateOnPrimaryStorageMsg extends NeedReplyMe private VolumeInventory volume; private long originSize; private String allocatedInstallUrl; + private String hostUuid; public String getAllocatedInstallUrl() { return allocatedInstallUrl; @@ -37,4 +38,12 @@ public long getOriginSize() { public void setOriginSize(long originSize) { this.originSize = originSize; } + + public String getHostUuid() { + return hostUuid; + } + + public void setHostUuid(String hostUuid) { + this.hostUuid = hostUuid; + } } diff --git a/header/src/main/java/org/zstack/header/storage/snapshot/CreateImageCacheFromVolumeSnapshotMsg.java b/header/src/main/java/org/zstack/header/storage/snapshot/CreateImageCacheFromVolumeSnapshotMsg.java index 75f397b4740..910c67595bd 100644 --- a/header/src/main/java/org/zstack/header/storage/snapshot/CreateImageCacheFromVolumeSnapshotMsg.java +++ b/header/src/main/java/org/zstack/header/storage/snapshot/CreateImageCacheFromVolumeSnapshotMsg.java @@ -9,6 +9,7 @@ public class CreateImageCacheFromVolumeSnapshotMsg extends NeedReplyMessage impl private String imageUuid; private String snapshotUuid; private String volumeUuid; + private Boolean encrypted; /** * @ignore */ @@ -48,4 +49,12 @@ public String getVolumeUuid() { public void setSnapshotUuid(String snapshotUuid) { this.snapshotUuid = snapshotUuid; } + + public Boolean getEncrypted() { + return encrypted; + } + + public void setEncrypted(Boolean encrypted) { + this.encrypted = encrypted; + } } \ No newline at end of file diff --git a/header/src/main/java/org/zstack/header/storage/snapshot/TakeSnapshotsOnKvmJobStruct.java b/header/src/main/java/org/zstack/header/storage/snapshot/TakeSnapshotsOnKvmJobStruct.java index d9338a818b4..3c775d77d7a 100644 --- a/header/src/main/java/org/zstack/header/storage/snapshot/TakeSnapshotsOnKvmJobStruct.java +++ b/header/src/main/java/org/zstack/header/storage/snapshot/TakeSnapshotsOnKvmJobStruct.java @@ -13,6 +13,7 @@ public class TakeSnapshotsOnKvmJobStruct implements Serializable { private String previousInstallPath; private String newVolumeInstallPath; private String snapshotUuid; + private String encryptLuksSecretMaterialFilePath; private boolean memory; private boolean live; private boolean full = false; @@ -73,6 +74,14 @@ public void setSnapshotUuid(String snapshotUuid) { this.snapshotUuid = snapshotUuid; } + public String getEncryptLuksSecretMaterialFilePath() { + return encryptLuksSecretMaterialFilePath; + } + + public void setEncryptLuksSecretMaterialFilePath(String encryptLuksSecretMaterialFilePath) { + this.encryptLuksSecretMaterialFilePath = encryptLuksSecretMaterialFilePath; + } + public boolean isLive() { return live; } diff --git a/header/src/main/java/org/zstack/header/storage/snapshot/VolumeSnapshotAO.java b/header/src/main/java/org/zstack/header/storage/snapshot/VolumeSnapshotAO.java index 434cf362b81..c81953a69ac 100755 --- a/header/src/main/java/org/zstack/header/storage/snapshot/VolumeSnapshotAO.java +++ b/header/src/main/java/org/zstack/header/storage/snapshot/VolumeSnapshotAO.java @@ -59,6 +59,9 @@ public class VolumeSnapshotAO extends ResourceVO implements ShadowEntity { @Column private boolean fullSnapshot; + @Column + private boolean encrypted; + @Column private String volumeType; @@ -184,6 +187,14 @@ public void setFullSnapshot(boolean fullSnapshot) { this.fullSnapshot = fullSnapshot; } + public boolean isEncrypted() { + return encrypted; + } + + public void setEncrypted(boolean encrypted) { + this.encrypted = encrypted; + } + public String getPrimaryStorageUuid() { return primaryStorageUuid; } diff --git a/header/src/main/java/org/zstack/header/storage/snapshot/VolumeSnapshotAO_.java b/header/src/main/java/org/zstack/header/storage/snapshot/VolumeSnapshotAO_.java index 20c2d132348..8398d7f117b 100755 --- a/header/src/main/java/org/zstack/header/storage/snapshot/VolumeSnapshotAO_.java +++ b/header/src/main/java/org/zstack/header/storage/snapshot/VolumeSnapshotAO_.java @@ -25,6 +25,7 @@ public class VolumeSnapshotAO_ extends ResourceVO_ { public static volatile SingularAttribute fullSnapshot; public static volatile SingularAttribute distance; public static volatile SingularAttribute size; + public static volatile SingularAttribute encrypted; public static volatile SingularAttribute state; public static volatile SingularAttribute status; public static volatile SingularAttribute createDate; diff --git a/header/src/main/java/org/zstack/header/storage/snapshot/VolumeSnapshotInventory.java b/header/src/main/java/org/zstack/header/storage/snapshot/VolumeSnapshotInventory.java index d37ba70d14e..84bdd13a762 100644 --- a/header/src/main/java/org/zstack/header/storage/snapshot/VolumeSnapshotInventory.java +++ b/header/src/main/java/org/zstack/header/storage/snapshot/VolumeSnapshotInventory.java @@ -126,6 +126,8 @@ public class VolumeSnapshotInventory { */ private Long size; + private Boolean encrypted; + private int distance; /** * @desc - Enabled: ok for operations @@ -186,6 +188,7 @@ public static VolumeSnapshotInventory valueOf(VolumeSnapshotVO vo) { inv.setLatest(vo.isLatest()); inv.setSize(vo.getSize()); inv.setVolumeType(vo.getVolumeType()); + inv.setEncrypted(vo.isEncrypted()); inv.setTreeUuid(vo.getTreeUuid()); inv.setBackupStorageRefs(VolumeSnapshotBackupStorageRefInventory.valueOf(vo.getBackupStorageRefs())); if (vo.getGroupRef() != null) { @@ -377,6 +380,14 @@ public void setLatest(boolean latest) { this.latest = latest; } + public Boolean getEncrypted() { + return encrypted; + } + + public void setEncrypted(Boolean encrypted) { + this.encrypted = encrypted; + } + public String getGroupUuid() { return groupUuid; } diff --git a/header/src/main/java/org/zstack/header/vm/APICreateVmInstanceFromVolumeSnapshotGroupMsg.java b/header/src/main/java/org/zstack/header/vm/APICreateVmInstanceFromVolumeSnapshotGroupMsg.java index 94d63ca232a..5ee1d7be7a0 100644 --- a/header/src/main/java/org/zstack/header/vm/APICreateVmInstanceFromVolumeSnapshotGroupMsg.java +++ b/header/src/main/java/org/zstack/header/vm/APICreateVmInstanceFromVolumeSnapshotGroupMsg.java @@ -112,6 +112,9 @@ public class APICreateVmInstanceFromVolumeSnapshotGroupMsg extends APICreateMess @APIParam(required = false) private Map> dataVolumeSystemTags; + @APIParam(required = false) + private List volumeSnapshotEncryptions; + @APINoSee private String platform; @@ -287,6 +290,35 @@ public void setDataVolumeSystemTags(Map> dataVolumeSystemTa this.dataVolumeSystemTags = dataVolumeSystemTags; } + public List getVolumeSnapshotEncryptions() { + return volumeSnapshotEncryptions; + } + + public void setVolumeSnapshotEncryptions(List volumeSnapshotEncryptions) { + this.volumeSnapshotEncryptions = volumeSnapshotEncryptions; + } + + public static class VolumeSnapshotEncryption { + private String volumeSnapshotUuid; + private Boolean encrypted; + + public String getVolumeSnapshotUuid() { + return volumeSnapshotUuid; + } + + public void setVolumeSnapshotUuid(String volumeSnapshotUuid) { + this.volumeSnapshotUuid = volumeSnapshotUuid; + } + + public Boolean getEncrypted() { + return encrypted; + } + + public void setEncrypted(Boolean encrypted) { + this.encrypted = encrypted; + } + } + @Override public String getPlatform() { return platform; diff --git a/header/src/main/java/org/zstack/header/vm/DiskAO.java b/header/src/main/java/org/zstack/header/vm/DiskAO.java index 3677ff749f3..5762969adfa 100644 --- a/header/src/main/java/org/zstack/header/vm/DiskAO.java +++ b/header/src/main/java/org/zstack/header/vm/DiskAO.java @@ -25,6 +25,7 @@ public class DiskAO { private String sourceUuid; private List systemTags; private String name; + private Boolean encrypted; public DiskAO withImage(String imageUuid) { this.templateUuid = imageUuid; @@ -139,6 +140,14 @@ public void setName(String name) { this.name = name; } + public Boolean getEncrypted() { + return encrypted; + } + + public void setEncrypted(Boolean encrypted) { + this.encrypted = encrypted; + } + public static DiskAO rootDisk() { DiskAO disk = new DiskAO(); disk.setBoot(true); diff --git a/header/src/main/java/org/zstack/header/volume/CreateDataVolumeExtensionPoint.java b/header/src/main/java/org/zstack/header/volume/CreateDataVolumeExtensionPoint.java index 6ce8ed6ab7f..d75c693806f 100644 --- a/header/src/main/java/org/zstack/header/volume/CreateDataVolumeExtensionPoint.java +++ b/header/src/main/java/org/zstack/header/volume/CreateDataVolumeExtensionPoint.java @@ -9,4 +9,8 @@ public interface CreateDataVolumeExtensionPoint { void beforeCreateVolume(VolumeInventory volume); void afterCreateVolume(VolumeVO volume); + + default void afterCreateVolume(VolumeVO volume, String snapshotUuid) { + afterCreateVolume(volume); + } } diff --git a/header/src/main/java/org/zstack/header/volume/CreateDataVolumeFromVolumeSnapshotMsg.java b/header/src/main/java/org/zstack/header/volume/CreateDataVolumeFromVolumeSnapshotMsg.java index cd709d306c8..ee4e82765cb 100644 --- a/header/src/main/java/org/zstack/header/volume/CreateDataVolumeFromVolumeSnapshotMsg.java +++ b/header/src/main/java/org/zstack/header/volume/CreateDataVolumeFromVolumeSnapshotMsg.java @@ -9,6 +9,7 @@ public class CreateDataVolumeFromVolumeSnapshotMsg extends NeedReplyMessage { private String volumeSnapshotUuid; private SessionInventory session; private Long size; + private Boolean encrypted; public String getName() { return name; @@ -49,4 +50,12 @@ public Long getSize() { public void setSize(Long size) { this.size = size; } + + public Boolean getEncrypted() { + return encrypted; + } + + public void setEncrypted(Boolean encrypted) { + this.encrypted = encrypted; + } } diff --git a/header/src/main/java/org/zstack/header/volume/CreateDataVolumeFromVolumeTemplateMsg.java b/header/src/main/java/org/zstack/header/volume/CreateDataVolumeFromVolumeTemplateMsg.java index 781c33bc9ba..06195687e39 100644 --- a/header/src/main/java/org/zstack/header/volume/CreateDataVolumeFromVolumeTemplateMsg.java +++ b/header/src/main/java/org/zstack/header/volume/CreateDataVolumeFromVolumeTemplateMsg.java @@ -18,6 +18,7 @@ public class CreateDataVolumeFromVolumeTemplateMsg extends NeedReplyMessage impl private String hostUuid; private String resourceUuid; private String accountUuid; + private Boolean encrypted; private APICreateDataVolumeFromVolumeTemplateMsg apiMsg; public CreateDataVolumeFromVolumeTemplateMsg() { @@ -97,4 +98,12 @@ public APICreateDataVolumeFromVolumeTemplateMsg getApiMsg() { public void setApiMsg(APICreateDataVolumeFromVolumeTemplateMsg amsg) { this.apiMsg = amsg; } + + public Boolean getEncrypted() { + return encrypted; + } + + public void setEncrypted(Boolean encrypted) { + this.encrypted = encrypted; + } } diff --git a/header/src/main/java/org/zstack/header/volume/CreateVolumeMsg.java b/header/src/main/java/org/zstack/header/volume/CreateVolumeMsg.java index e8d22cb9a99..e684bfabdfd 100755 --- a/header/src/main/java/org/zstack/header/volume/CreateVolumeMsg.java +++ b/header/src/main/java/org/zstack/header/volume/CreateVolumeMsg.java @@ -16,6 +16,7 @@ public class CreateVolumeMsg extends NeedReplyMessage implements VolumeCreateMes private String format; private String resourceUuid; private String protocol; + private Boolean encrypted; public String getFormat() { return format; @@ -130,4 +131,14 @@ public String getProtocol() { public void setProtocol(String protocol) { this.protocol = protocol; } + + @Override + public Boolean getEncrypted() { + return encrypted; + } + + @Override + public void setEncrypted(Boolean encrypted) { + this.encrypted = encrypted; + } } diff --git a/header/src/main/java/org/zstack/header/volume/EncryptVolumeMsg.java b/header/src/main/java/org/zstack/header/volume/EncryptVolumeMsg.java new file mode 100644 index 00000000000..e8ff1e85e34 --- /dev/null +++ b/header/src/main/java/org/zstack/header/volume/EncryptVolumeMsg.java @@ -0,0 +1,86 @@ +package org.zstack.header.volume; + +import org.zstack.header.message.ConfigurableTimeoutMessage; +import org.zstack.header.message.DefaultTimeout; +import org.zstack.header.message.NeedReplyMessage; + +import java.util.concurrent.TimeUnit; + +/** + * Convert an existing data volume's bits to a LUKS-encrypted form in place. + *

+ * Steps performed by the handler (in {@link org.zstack.storage.volume.VolumeBase}): + *

    + *
  1. Ensure the volume has a key-provider binding (auto-attaches the default key provider + * when none is bound yet).
    2. + *
  2. Materialize a DEK via the {@code EncryptedResourceKeyManager}.
    3. + *
  3. Stage the LUKS secret material file on the host + * ({@code SecretHostEnsureLuksSecretFileMsg}).
    4. + *
  4. Ask the primary storage backend to LUKS-convert the bits in place + * ({@code EncryptVolumeBitsOnPrimaryStorageMsg}).
    5. + *
  5. Persist {@code VolumeVO.encrypted = true}.
    6. + *
+ *

+ * If the volume row is already marked {@code encrypted=true}, the handler treats it as a + * no-op success. The {@code encrypted} flag is the single authoritative signal that + * "the bits on disk are already LUKS"; callers must NOT pre-mark the row before invoking + * this message. + */ +@DefaultTimeout(timeunit = TimeUnit.HOURS, value = 1) +public class EncryptVolumeMsg extends NeedReplyMessage implements VolumeMessage, ConfigurableTimeoutMessage { + private String volumeUuid; + private String hostUuid; + /** + * Optional. When null, the handler resolves it from {@code VolumeVO.primaryStorageUuid}. + */ + private String primaryStorageUuid; + /** + * Optional. When null, the handler resolves it from {@code VolumeVO.installPath}. + */ + private String installPath; + /** + * Free-form purpose label for the DEK get-or-create audit trail. + */ + private String purpose; + + @Override + public String getVolumeUuid() { + return volumeUuid; + } + + public void setVolumeUuid(String volumeUuid) { + this.volumeUuid = volumeUuid; + } + + public String getHostUuid() { + return hostUuid; + } + + public void setHostUuid(String hostUuid) { + this.hostUuid = hostUuid; + } + + public String getPrimaryStorageUuid() { + return primaryStorageUuid; + } + + public void setPrimaryStorageUuid(String primaryStorageUuid) { + this.primaryStorageUuid = primaryStorageUuid; + } + + public String getInstallPath() { + return installPath; + } + + public void setInstallPath(String installPath) { + this.installPath = installPath; + } + + public String getPurpose() { + return purpose; + } + + public void setPurpose(String purpose) { + this.purpose = purpose; + } +} diff --git a/header/src/main/java/org/zstack/header/volume/EncryptVolumeReply.java b/header/src/main/java/org/zstack/header/volume/EncryptVolumeReply.java new file mode 100644 index 00000000000..f4f21f1b901 --- /dev/null +++ b/header/src/main/java/org/zstack/header/volume/EncryptVolumeReply.java @@ -0,0 +1,18 @@ +package org.zstack.header.volume; + +import org.zstack.header.message.MessageReply; + +/** + * Reply for {@link EncryptVolumeMsg}. + */ +public class EncryptVolumeReply extends MessageReply { + private VolumeInventory inventory; + + public VolumeInventory getInventory() { + return inventory; + } + + public void setInventory(VolumeInventory inventory) { + this.inventory = inventory; + } +} diff --git a/header/src/main/java/org/zstack/header/volume/InstantiateVolumeMsg.java b/header/src/main/java/org/zstack/header/volume/InstantiateVolumeMsg.java index 13e6f557b24..e454fecdd92 100755 --- a/header/src/main/java/org/zstack/header/volume/InstantiateVolumeMsg.java +++ b/header/src/main/java/org/zstack/header/volume/InstantiateVolumeMsg.java @@ -12,6 +12,7 @@ public class InstantiateVolumeMsg extends NeedReplyMessage implements VolumeMess private boolean primaryStorageAllocated; private boolean skipIfExisting; private String allocatedInstallUrl; + private VolumeLuksAgentSpec volumeLuksAgentSpec; public String getAllocatedInstallUrl() { return allocatedInstallUrl; @@ -61,4 +62,12 @@ public boolean isSkipIfExisting() { public void setSkipIfExisting(boolean skipIfExisting) { this.skipIfExisting = skipIfExisting; } + + public VolumeLuksAgentSpec getVolumeLuksAgentSpec() { + return volumeLuksAgentSpec; + } + + public void setVolumeLuksAgentSpec(VolumeLuksAgentSpec volumeLuksAgentSpec) { + this.volumeLuksAgentSpec = volumeLuksAgentSpec; + } } diff --git a/header/src/main/java/org/zstack/header/volume/VolumeAO.java b/header/src/main/java/org/zstack/header/volume/VolumeAO.java index 3ddcdbccad3..203b30134a5 100755 --- a/header/src/main/java/org/zstack/header/volume/VolumeAO.java +++ b/header/src/main/java/org/zstack/header/volume/VolumeAO.java @@ -86,6 +86,9 @@ public class VolumeAO extends ResourceVO implements ShadowEntity { @Column private String protocol; + @Column + private boolean encrypted; + @Transient private VolumeAO shadow; @@ -298,4 +301,12 @@ public String getProtocol() { public void setProtocol(String protocol) { this.protocol = protocol; } + + public boolean isEncrypted() { + return encrypted; + } + + public void setEncrypted(boolean encrypted) { + this.encrypted = encrypted; + } } diff --git a/header/src/main/java/org/zstack/header/volume/VolumeAO_.java b/header/src/main/java/org/zstack/header/volume/VolumeAO_.java index 729d50eefbf..af03c918ca2 100755 --- a/header/src/main/java/org/zstack/header/volume/VolumeAO_.java +++ b/header/src/main/java/org/zstack/header/volume/VolumeAO_.java @@ -31,4 +31,5 @@ public class VolumeAO_ extends ResourceVO_ { public static volatile SingularAttribute isShareable; public static volatile SingularAttribute volumeQos; public static volatile SingularAttribute protocol; + public static volatile SingularAttribute encrypted; } diff --git a/header/src/main/java/org/zstack/header/volume/VolumeCreateMessage.java b/header/src/main/java/org/zstack/header/volume/VolumeCreateMessage.java index a93217a6166..36f8b47f870 100644 --- a/header/src/main/java/org/zstack/header/volume/VolumeCreateMessage.java +++ b/header/src/main/java/org/zstack/header/volume/VolumeCreateMessage.java @@ -20,4 +20,11 @@ public interface VolumeCreateMessage { void setSystemTags(List systemTags); void addSystemTag(String tag); + + default Boolean getEncrypted() { + return null; + } + + default void setEncrypted(Boolean encrypted) { + } } diff --git a/header/src/main/java/org/zstack/header/volume/VolumeInventory.java b/header/src/main/java/org/zstack/header/volume/VolumeInventory.java index 96d2ae67a62..f04fe219fed 100755 --- a/header/src/main/java/org/zstack/header/volume/VolumeInventory.java +++ b/header/src/main/java/org/zstack/header/volume/VolumeInventory.java @@ -156,6 +156,8 @@ public class VolumeInventory implements Serializable { private Timestamp lastAttachDate; private String protocol; + private Boolean encrypted; + public VolumeInventory() { } @@ -183,6 +185,7 @@ public VolumeInventory(VolumeInventory other) { this.lastVmInstanceUuid = other.lastVmInstanceUuid; this.lastAttachDate = other.lastAttachDate; this.protocol = other.protocol; + this.encrypted = other.encrypted; } @@ -213,6 +216,7 @@ public VolumeInventory(VolumeInventory other) { inv.setLastVmInstanceUuid(vo.getLastVmInstanceUuid()); inv.setLastAttachDate(vo.getLastAttachDate()); inv.setProtocol(vo.getProtocol()); + inv.setEncrypted(vo.isEncrypted()); return inv; } @@ -437,4 +441,12 @@ public String getProtocol() { public void setProtocol(String protocol) { this.protocol = protocol; } + + public Boolean getEncrypted() { + return encrypted; + } + + public void setEncrypted(Boolean encrypted) { + this.encrypted = encrypted; + } } diff --git a/header/src/main/java/org/zstack/header/volume/VolumeLuksAgentSpec.java b/header/src/main/java/org/zstack/header/volume/VolumeLuksAgentSpec.java new file mode 100644 index 00000000000..8602adcd79d --- /dev/null +++ b/header/src/main/java/org/zstack/header/volume/VolumeLuksAgentSpec.java @@ -0,0 +1,31 @@ +package org.zstack.header.volume; + +import java.io.Serializable; + +public class VolumeLuksAgentSpec implements Serializable { + private static final long serialVersionUID = 1L; + + private String encryptSecretUuid; + private String encryptLuksSecretMaterialFilePath; + + public boolean isComplete() { + return (encryptSecretUuid != null && !encryptSecretUuid.isEmpty()) + || (encryptLuksSecretMaterialFilePath != null && !encryptLuksSecretMaterialFilePath.isEmpty()); + } + + public String getEncryptSecretUuid() { + return encryptSecretUuid; + } + + public void setEncryptSecretUuid(String encryptSecretUuid) { + this.encryptSecretUuid = encryptSecretUuid; + } + + public String getEncryptLuksSecretMaterialFilePath() { + return encryptLuksSecretMaterialFilePath; + } + + public void setEncryptLuksSecretMaterialFilePath(String encryptLuksSecretMaterialFilePath) { + this.encryptLuksSecretMaterialFilePath = encryptLuksSecretMaterialFilePath; + } +} diff --git a/image/src/main/java/org/zstack/image/ImageManagerImpl.java b/image/src/main/java/org/zstack/image/ImageManagerImpl.java index 5c32b9f3315..27638a49470 100755 --- a/image/src/main/java/org/zstack/image/ImageManagerImpl.java +++ b/image/src/main/java/org/zstack/image/ImageManagerImpl.java @@ -242,6 +242,7 @@ public void run(FlowTrigger trigger, Map data) { cmsg.setImageUuid(vo.getUuid()); cmsg.setVolumeUuid(volumeUuid); cmsg.setTreeUuid(treeUuid); + cmsg.setEncrypted(msg.getEncrypted()); cmsg.setSystemTags(msg.getSystemTags()); String resourceUuid = volumeUuid != null ? volumeUuid : treeUuid; bus.makeTargetServiceIdByResourceUuid(cmsg, VolumeSnapshotConstant.SERVICE_ID, resourceUuid); diff --git a/plugin/ceph/src/main/java/org/zstack/storage/ceph/primary/CephPrimaryStorageBase.java b/plugin/ceph/src/main/java/org/zstack/storage/ceph/primary/CephPrimaryStorageBase.java index a04a9aba6e4..3342bc61ee3 100755 --- a/plugin/ceph/src/main/java/org/zstack/storage/ceph/primary/CephPrimaryStorageBase.java +++ b/plugin/ceph/src/main/java/org/zstack/storage/ceph/primary/CephPrimaryStorageBase.java @@ -38,6 +38,7 @@ import org.zstack.header.exception.CloudRuntimeException; import org.zstack.header.host.*; import org.zstack.header.image.ImageBackupStorageRefInventory; +import org.zstack.header.image.ImageConstant; import org.zstack.header.image.ImageConstant.ImageMediaType; import org.zstack.header.image.ImageInventory; import org.zstack.header.image.ImageStatus; @@ -68,6 +69,7 @@ import org.zstack.storage.ceph.backup.CephBackupStorageVO_; import org.zstack.storage.ceph.primary.CephPrimaryStorageMonBase.PingOperationFailure; import org.zstack.storage.ceph.primary.capacity.CephOsdGroupCapacityHelper; +import org.zstack.storage.encrypt.VolumeSnapshotEncryptionHelper; import org.zstack.storage.primary.*; import org.zstack.storage.volume.VolumeErrors; import org.zstack.storage.volume.VolumeSystemTags; @@ -115,6 +117,8 @@ public class CephPrimaryStorageBase extends PrimaryStorageBase { private StorageTrash trash; @Autowired private PoolUsageReport poolUsageCollector; + @Autowired + private VolumeSnapshotEncryptionHelper snapshotEncryptionHelper; public CephPrimaryStorageBase() { @@ -420,6 +424,38 @@ public void setInstallPath(String installPath) { } } + public static class KVMHostLuksCloneCmd implements Serializable { + public String psUuid; + public String secFilePath; + public String srcPath; + public String dstPath; + public Long virtualSizeForLuksClone; + } + + public static class KVMHostLuksCreateEmptyCmd implements Serializable { + public String psUuid; + public String secFilePath; + public String installPath; + public long size; + } + + public static class KVMHostLuksResizeCmd implements Serializable { + public String psUuid; + public String secFilePath; + public String installPath; + public Long virtualSize; + } + + public static class KVMHostEncryptInPlaceCmd implements Serializable { + public String psUuid; + public String secFilePath; + public String installPath; + } + + public static class KVMHostLuksRsp extends KVMAgentCommands.AgentResponse { + public Long actualSize; + } + public static class DeleteCmd extends AgentCommand { String installPath; long expirationTime; @@ -1358,6 +1394,13 @@ public int compareTo(SnapInfo snapInfo) { public static final String CREATE_VOLUME_PATH = "/ceph/primarystorage/volume/createempty"; public static final String DELETE_PATH = "/ceph/primarystorage/delete"; public static final String CLONE_PATH = "/ceph/primarystorage/volume/clone"; + // LUKS variants that run on the KVM host instead of cephagent (the host that + // qemu-img must run on is the one that has key-agent + the libvirt secret + // FIFO; cephagent nodes do not, especially in non-converged deployments). + public static final String KVM_HOST_LUKS_CLONE_PATH = "/ceph/primarystorage/kvmhost/luksclone"; + public static final String KVM_HOST_LUKS_CREATE_EMPTY_PATH = "/ceph/primarystorage/kvmhost/lukscreateempty"; + public static final String KVM_HOST_LUKS_ENCRYPT_IN_PLACE_PATH = "/ceph/primarystorage/kvmhost/encryptinplace"; + public static final String KVM_HOST_LUKS_RESIZE_PATH = "/ceph/primarystorage/kvmhost/luksresize"; public static final String FLATTEN_PATH = "/ceph/primarystorage/volume/flatten"; public static final String SFTP_DOWNLOAD_PATH = "/ceph/primarystorage/sftpbackupstorage/download"; public static final String SFTP_UPLOAD_PATH = "/ceph/primarystorage/sftpbackupstorage/upload"; @@ -1722,11 +1765,55 @@ protected CephPrimaryStorageInventory getSelfInventory() { } private void createEmptyVolume(final InstantiateVolumeOnPrimaryStorageMsg msg) { - final CreateEmptyVolumeCmd cmd = new CreateEmptyVolumeCmd(); String volumeUuid = msg.getVolume().getUuid(); final String finalPoolName = getTargetPoolNameFromAllocatedUrl(msg.getAllocatedInstallUrl()); - cmd.installPath = makeVolumeInstallPathByTargetPool(volumeUuid, finalPoolName); - cmd.size = msg.getVolume().getSize(); + final String installPath = makeVolumeInstallPathByTargetPool(volumeUuid, finalPoolName); + final long volumeSize = msg.getVolume().getSize(); + + final InstantiateVolumeOnPrimaryStorageReply reply = new InstantiateVolumeOnPrimaryStorageReply(); + + // Encrypted variant: forward to the dest KVM host so qemu-img runs next + // to the libvirt-secret FIFO + key-agent (see KVM_HOST_LUKS_CREATE_EMPTY_PATH). + if (msg.getVolumeLuksAgentSpec() != null && msg.getVolumeLuksAgentSpec().isComplete()) { + if (msg.getDestHost() == null || StringUtils.isBlank(msg.getDestHost().getUuid())) { + reply.setError(operr( + "ceph LUKS createempty requires destHost; volume[uuid:%s] has none", volumeUuid)); + bus.reply(msg, reply); + return; + } + KVMHostLuksCreateEmptyCmd kcmd = new KVMHostLuksCreateEmptyCmd(); + kcmd.psUuid = self.getUuid(); + kcmd.installPath = installPath; + kcmd.size = volumeSize; + kcmd.secFilePath = msg.getVolumeLuksAgentSpec().getEncryptLuksSecretMaterialFilePath(); + httpCallToKvmHost(msg.getDestHost().getUuid(), + KVM_HOST_LUKS_CREATE_EMPTY_PATH, kcmd, KVMHostLuksRsp.class, + new ReturnValueCompletion(msg) { + @Override + public void fail(ErrorCode err) { + reply.setError(err); + bus.reply(msg, reply); + } + + @Override + public void success(KVMHostLuksRsp ret) { + VolumeInventory vol = msg.getVolume(); + // installPath is the canonical "ceph:///" computed above; + // don't run it through buildEmptyVolumeInstallPath again (that helper + // assumes the agent returned a bare pool/uuid relative path). + vol.setInstallPath(installPath); + vol.setFormat(VolumeConstant.VOLUME_FORMAT_RAW); + vol.setActualSize(ret.actualSize); + reply.setVolume(vol); + bus.reply(msg, reply); + } + }); + return; + } + + final CreateEmptyVolumeCmd cmd = new CreateEmptyVolumeCmd(); + cmd.installPath = installPath; + cmd.size = volumeSize; cmd.setShareable(msg.getVolume().isShareable()); cmd.skipIfExisting = msg.isSkipIfExisting(); @@ -1738,8 +1825,6 @@ private void createEmptyVolume(final InstantiateVolumeOnPrimaryStorageMsg msg) { VolumeConstant.VOLUME_FORMAT_QCOW2 : VolumeConstant.VOLUME_FORMAT_RAW ; - final InstantiateVolumeOnPrimaryStorageReply reply = new InstantiateVolumeOnPrimaryStorageReply(); - httpCall(CREATE_VOLUME_PATH, cmd, CreateEmptyVolumeRsp.class, new ReturnValueCompletion(msg) { @Override public void fail(ErrorCode err) { @@ -2115,6 +2200,7 @@ protected void handle(final InstantiateVolumeOnPrimaryStorageMsg msg) { class DownloadToCache { ImageSpec image; VolumeSnapshotInventory snapshot; + Boolean encrypted; private void doDownload(final ReturnValueCompletion completion) { ImageCacheVO cache = Q.New(ImageCacheVO.class) .eq(ImageCacheVO_.primaryStorageUuid, self.getUuid()) @@ -2132,6 +2218,8 @@ private void doDownload(final ReturnValueCompletion completion) { long actualSize = image.getInventory().getActualSize(); String allocatedInstall; ImageCacheVO cvo = new ImageCacheVO(); + String encryptHostUuid; + VolumeLuksAgentSpec imageLuksSpec; @Override public void setup() { @@ -2180,6 +2268,31 @@ public void rollback(FlowRollback trigger, Map data) { } }); + flow(new Flow() { + String __name__ = "prepare-luks-secret-for-snapshot-image-cache"; + + @Override + public void run(FlowTrigger trigger, Map data) { + if (snapshot == null || !Boolean.TRUE.equals(encrypted)) { + trigger.next(); + return; + } + + encryptHostUuid = findConnectedHostForCephLuks(); + imageLuksSpec = snapshotEncryptionHelper.prepareTemporarySnapshotImageSecretMaterial( + encryptHostUuid, + snapshot.getUuid(), + image.getInventory().getUuid(), + encrypted); + trigger.next(); + } + + @Override + public void rollback(FlowRollback trigger, Map data) { + trigger.rollback(); + } + }); + flow(new Flow() { String __name__ = "download-from-" + (snapshot != null ? "volume" : "backup-storage"); @@ -2199,6 +2312,32 @@ public void run(final FlowTrigger trigger, Map data) { private void createFromVolumeSnapshot(FlowTrigger trigger) { deleteOnRollback = true; + if (imageLuksSpec != null && imageLuksSpec.isComplete()) { + KVMHostLuksCloneCmd kcmd = new KVMHostLuksCloneCmd(); + kcmd.psUuid = self.getUuid(); + kcmd.srcPath = snapshot.getPrimaryStorageInstallPath(); + kcmd.dstPath = dstPath; + kcmd.secFilePath = imageLuksSpec.getEncryptLuksSecretMaterialFilePath(); + httpCallToKvmHost(encryptHostUuid, + KVM_HOST_LUKS_CLONE_PATH, kcmd, KVMHostLuksRsp.class, + new ReturnValueCompletion(trigger) { + @Override + public void success(KVMHostLuksRsp rsp) { + if (rsp.actualSize != null) { + actualSize = rsp.actualSize; + } + cachePath = dstPath; + trigger.next(); + } + + @Override + public void fail(ErrorCode errorCode) { + trigger.fail(errorCode); + } + }); + return; + } + CpCmd cmd = new CpCmd(); cmd.srcPath = snapshot.getPrimaryStorageInstallPath(); cmd.dstPath = dstPath; @@ -2529,6 +2668,7 @@ private void createVolumeFromTemplate(final InstantiateRootVolumeFromTemplateOnP String volumePath = makeVolumeInstallPathByTargetPool(msg.getVolume().getUuid(), targetCephPoolName); ImageCacheInventory cache; Long actualSize; + boolean clonedFromEncryptedSnapshotImageCache; @Override public void setup() { @@ -2564,6 +2704,47 @@ public void run(MessageReply reply) { @Override public void run(final FlowTrigger trigger, Map data) { + boolean hasLuksSpec = msg.getVolumeLuksAgentSpec() != null && msg.getVolumeLuksAgentSpec().isComplete(); + clonedFromEncryptedSnapshotImageCache = + hasLuksSpec && isEncryptedSnapshotImageCache(cache); + if (clonedFromEncryptedSnapshotImageCache) { + cloneImage(trigger); + return; + } + + // If the image cache is not already LUKS, RBD clone cannot create a LUKS overlay/top layer: + // RBD only provides block-level COW and preserves the source byte layout. Use qemu-img + // convert on the KVM host to write a new LUKS container with the host-local FIFO secret. + if (hasLuksSpec) { + KVMHostLuksCloneCmd kcmd = new KVMHostLuksCloneCmd(); + kcmd.psUuid = self.getUuid(); + kcmd.srcPath = cloneInstallPath; + kcmd.dstPath = volumePath; + kcmd.secFilePath = msg.getVolumeLuksAgentSpec().getEncryptLuksSecretMaterialFilePath(); + if (ispec.getInventory().getSize() < msg.getVolume().getSize()) { + kcmd.virtualSizeForLuksClone = msg.getVolume().getSize(); + } + httpCallToKvmHost(msg.getDestHost().getUuid(), + KVM_HOST_LUKS_CLONE_PATH, kcmd, KVMHostLuksRsp.class, + new ReturnValueCompletion(trigger) { + @Override + public void fail(ErrorCode err) { + trigger.fail(err); + } + + @Override + public void success(KVMHostLuksRsp ret) { + actualSize = ret.actualSize; + trigger.next(); + } + }); + return; + } + + cloneImage(trigger); + } + + private void cloneImage(final FlowTrigger trigger) { CloneCmd cmd = new CloneCmd(); cmd.srcPath = cloneInstallPath; cmd.dstPath = volumePath; @@ -2589,11 +2770,23 @@ public void success(CloneRsp ret) { @Override public boolean skip(Map data) { ImageInventory image = ispec.getInventory(); + if (clonedFromEncryptedSnapshotImageCache) { + return false; + } + if (msg.getVolumeLuksAgentSpec() != null && msg.getVolumeLuksAgentSpec().isComplete()) { + // LUKS clone converts to a standalone LUKS RBD and applies virtualSizeForLuksClone before this raw resize flow. + return true; + } return image.getSize() >= msg.getVolume().getSize(); } @Override public void run(final FlowTrigger trigger, Map data) { + if (clonedFromEncryptedSnapshotImageCache) { + resizeClonedLuksRbd(trigger); + return; + } + ResizeVolumeOnPrimaryStorageMsg rmsg = new ResizeVolumeOnPrimaryStorageMsg(); rmsg.setVolume(msg.getVolume()); rmsg.setSize(msg.getVolume().getSize()); @@ -2611,6 +2804,31 @@ public void run(MessageReply reply) { }); trigger.next(); } + + private void resizeClonedLuksRbd(final FlowTrigger trigger) { + KVMHostLuksResizeCmd kcmd = new KVMHostLuksResizeCmd(); + kcmd.psUuid = self.getUuid(); + kcmd.installPath = volumePath; + kcmd.secFilePath = msg.getVolumeLuksAgentSpec().getEncryptLuksSecretMaterialFilePath(); + if (ispec.getInventory().getSize() < msg.getVolume().getSize()) { + kcmd.virtualSize = msg.getVolume().getSize(); + } + + httpCallToKvmHost(msg.getDestHost().getUuid(), + KVM_HOST_LUKS_RESIZE_PATH, kcmd, KVMHostLuksRsp.class, + new ReturnValueCompletion(trigger) { + @Override + public void fail(ErrorCode err) { + trigger.fail(err); + } + + @Override + public void success(KVMHostLuksRsp ret) { + actualSize = ret.actualSize; + trigger.next(); + } + }); + } }); done(new FlowDoneHandler(msg) { @@ -2643,6 +2861,35 @@ public void handle(ErrorCode errCode, Map data) { }).start(); } + private boolean isEncryptedSnapshotImageCache(ImageCacheInventory cache) { + if (cache == null) { + return false; + } + + boolean hasTemporarySnapshotImageKey = snapshotEncryptionHelper.hasTemporarySnapshotImageKey(cache.getImageUuid()); + if (hasTemporarySnapshotImageKey) { + return true; + } + + String installUrl = cache.getInstallUrl(); + if (StringUtils.isBlank(installUrl) || !installUrl.contains(ImageConstant.SNAPSHOT_REUSE_IMAGE_SCHEMA)) { + return false; + } + + String snapshotUuid = installUrl.substring(installUrl.lastIndexOf(ImageConstant.SNAPSHOT_REUSE_IMAGE_SCHEMA) + + ImageConstant.SNAPSHOT_REUSE_IMAGE_SCHEMA.length()); + if (snapshotUuid.length() < 32) { + return false; + } + snapshotUuid = snapshotUuid.substring(0, 32); + + Boolean encrypted = Q.New(VolumeSnapshotVO.class) + .eq(VolumeSnapshotVO_.uuid, snapshotUuid) + .select(VolumeSnapshotVO_.encrypted) + .findValue(); + return Boolean.TRUE.equals(encrypted); + } + @Override protected void handle(final DeleteVolumeOnPrimaryStorageMsg msg) { inQueue().name(String.format("delete-volume-on-primarystorage-%s", self.getUuid())) @@ -2884,6 +3131,7 @@ public void fail(ErrorCode errorCode) { cache.image = new ImageSpec(); cache.image.setInventory(msg.getImageInventory()); cache.snapshot = msg.getVolumeSnapshot(); + cache.encrypted = msg.getEncrypted(); cache.download(new ReturnValueCompletion(msg) { @Override public void success(ImageCacheVO inv) { @@ -3198,6 +3446,37 @@ public void success(DeleteRsp ret) { }); } + private void handle(final EncryptVolumeBitsOnPrimaryStorageMsg msg) { + // Encrypt-in-place always consumes a LUKS secret FIFO, so it must run + // on the KVM host that owns the FIFO + key-agent (msg.getHostUuid()). + EncryptVolumeBitsOnPrimaryStorageReply reply = new EncryptVolumeBitsOnPrimaryStorageReply(); + if (StringUtils.isBlank(msg.getHostUuid())) { + reply.setError(operr( + "ceph encryptInPlace requires hostUuid; volume[uuid:%s] installPath[%s] has none", + msg.getVolumeUuid(), msg.getInstallPath())); + bus.reply(msg, reply); + return; + } + KVMHostEncryptInPlaceCmd kcmd = new KVMHostEncryptInPlaceCmd(); + kcmd.psUuid = self.getUuid(); + kcmd.installPath = msg.getInstallPath(); + kcmd.secFilePath = msg.getEncryptLuksSecretMaterialFilePath(); + httpCallToKvmHost(msg.getHostUuid(), + KVM_HOST_LUKS_ENCRYPT_IN_PLACE_PATH, kcmd, KVMHostLuksRsp.class, + new ReturnValueCompletion(msg) { + @Override + public void fail(ErrorCode err) { + reply.setError(err); + bus.reply(msg, reply); + } + + @Override + public void success(KVMHostLuksRsp ret) { + bus.reply(msg, reply); + } + }); + } + @Override protected void handle(final DownloadIsoToPrimaryStorageMsg msg) { final DownloadIsoToPrimaryStorageReply reply = new DownloadIsoToPrimaryStorageReply(); @@ -3382,6 +3661,68 @@ protected void httpCall(final String path, final Agent new HttpCaller<>(path, cmd, retClass, callback, unit, timeout).call(); } + /** + * Send an HTTP cmd to a KVM host's kvmagent (not to cephagent). Used by the + * LUKS variants of clone / createempty / encrypt-in-place, where qemu-img + * must run on the host that has key-agent + libvirt-secret FIFO present. + */ + private void httpCallToKvmHost( + final String hostUuid, + final String path, + final Object cmd, + final Class retClass, + final ReturnValueCompletion callback) { + KVMHostAsyncHttpCallMsg kmsg = new KVMHostAsyncHttpCallMsg(); + kmsg.setCommand(cmd); + kmsg.setPath(path); + kmsg.setHostUuid(hostUuid); + kmsg.setNoStatusCheck(true); + bus.makeTargetServiceIdByResourceUuid(kmsg, HostConstant.SERVICE_ID, hostUuid); + bus.send(kmsg, new CloudBusCallBack(callback) { + @Override + public void run(MessageReply reply) { + if (!reply.isSuccess()) { + callback.fail(reply.getError()); + return; + } + KVMHostAsyncHttpCallReply kr = reply.castReply(); + T rsp = kr.toResponse(retClass); + if (rsp == null) { + callback.fail(operr( + "kvm host[uuid:%s] returned null reply for ceph luks path[%s]", + hostUuid, path)); + return; + } + if (!rsp.isSuccess()) { + callback.fail(operr( + "kvm host[uuid:%s] ceph luks path[%s] failed", + hostUuid, path).withException(rsp.getError())); + return; + } + callback.success(rsp); + } + }); + } + + protected void resizeEncryptedRbdVolumeOnKvmHost(VolumeInventory volume, long size, + ReturnValueCompletion completion) { + String hostUuid = findConnectedHostForCephLuks(); + VolumeLuksAgentSpec luksSpec = snapshotEncryptionHelper.prepareVolumeSecretMaterial(hostUuid, volume.getUuid()); + if (luksSpec == null || !luksSpec.isComplete()) { + completion.fail(operr("cannot prepare LUKS secret for encrypted volume[uuid:%s] resize on host[uuid:%s]", + volume.getUuid(), hostUuid)); + return; + } + + KVMHostLuksResizeCmd cmd = new KVMHostLuksResizeCmd(); + cmd.psUuid = self.getUuid(); + cmd.installPath = volume.getInstallPath(); + cmd.secFilePath = luksSpec.getEncryptLuksSecretMaterialFilePath(); + cmd.virtualSize = size; + + httpCallToKvmHost(hostUuid, KVM_HOST_LUKS_RESIZE_PATH, cmd, KVMHostLuksRsp.class, completion); + } + public class HttpCaller { private Iterator it; private final List monVOs; @@ -4407,6 +4748,8 @@ protected void handleLocalMessage(Message msg) { handle((GetPrimaryStorageUsageReportMsg) msg); } else if (msg instanceof CleanUpStorageTrashOnPrimaryStorageMsg) { handle((CleanUpStorageTrashOnPrimaryStorageMsg)msg); + } else if (msg instanceof EncryptVolumeBitsOnPrimaryStorageMsg) { + handle((EncryptVolumeBitsOnPrimaryStorageMsg) msg); } else { super.handleLocalMessage(msg); } @@ -4977,6 +5320,15 @@ public void done() { private void fastCreateVolumeFromSnapshot(final CreateVolumeFromVolumeSnapshotOnPrimaryStorageMsg msg, final NoErrorCompletion completion) { final CreateVolumeFromVolumeSnapshotOnPrimaryStorageReply reply = new CreateVolumeFromVolumeSnapshotOnPrimaryStorageReply(); + VolumeVO targetVolume = Q.New(VolumeVO.class).eq(VolumeVO_.uuid, msg.getVolumeUuid()).find(); + if (targetVolume != null && targetVolume.isEncrypted()) { + // RBD fast clone cannot change a plain snapshot into a LUKS volume. + // Convert the snapshot into an independent LUKS RBD; the helper also + // passes the target virtual size so the KVM agent resizes after convert. + flattenSnapshotToEncryptedVolume(msg, completion); + return; + } + // create volume first, then reserve size for it, so we use snapshot poolName for volume create String snapShotPath = msg.getSnapshot().getPrimaryStorageInstallPath(); final String volPath = makeVolumeInstallPathByTargetPool(msg.getVolumeUuid(), getTargetPoolNameFromAllocatedUrl(snapShotPath)); @@ -5051,6 +5403,12 @@ public void fail(ErrorCode errorCode) { private void createVolumeFromSnapshot(final CreateVolumeFromVolumeSnapshotOnPrimaryStorageMsg msg, final NoErrorCompletion completion) { final CreateVolumeFromVolumeSnapshotOnPrimaryStorageReply reply = new CreateVolumeFromVolumeSnapshotOnPrimaryStorageReply(); + VolumeVO targetVolume = Q.New(VolumeVO.class).eq(VolumeVO_.uuid, msg.getVolumeUuid()).find(); + if (targetVolume != null && targetVolume.isEncrypted()) { + flattenSnapshotToEncryptedVolume(msg, completion); + return; + } + String snapShotPath = msg.getSnapshot().getPrimaryStorageInstallPath(); final String volPath = makeVolumeInstallPathByTargetPool(msg.getVolumeUuid(), getTargetPoolNameFromAllocatedUrl(snapShotPath)); VolumeSnapshotInventory sp = msg.getSnapshot(); @@ -5079,6 +5437,72 @@ public void fail(ErrorCode errorCode) { }); } + private void flattenSnapshotToEncryptedVolume(final CreateVolumeFromVolumeSnapshotOnPrimaryStorageMsg msg, + final NoErrorCompletion completion) { + final CreateVolumeFromVolumeSnapshotOnPrimaryStorageReply reply = + new CreateVolumeFromVolumeSnapshotOnPrimaryStorageReply(); + VolumeSnapshotInventory sp = msg.getSnapshot(); + String snapshotPath = sp.getPrimaryStorageInstallPath(); + final String volPath = makeVolumeInstallPathByTargetPool(msg.getVolumeUuid(), getTargetPoolNameFromAllocatedUrl(snapshotPath)); + String hostUuid = findConnectedHostForCephLuks(); + VolumeLuksAgentSpec dstSpec = msg.getVolumeLuksAgentSpec(); + if (dstSpec == null || !dstSpec.isComplete()) { + dstSpec = snapshotEncryptionHelper.prepareVolumeSecretMaterial(hostUuid, msg.getVolumeUuid()); + } + if (dstSpec == null || !dstSpec.isComplete()) { + throw new OperationFailureException(operr( + "cannot prepare LUKS secret for encrypted volume[uuid:%s] from snapshot[uuid:%s]", + msg.getVolumeUuid(), sp.getUuid())); + } + + KVMHostLuksCloneCmd kcmd = new KVMHostLuksCloneCmd(); + kcmd.psUuid = self.getUuid(); + kcmd.srcPath = snapshotPath; + kcmd.dstPath = volPath; + kcmd.secFilePath = dstSpec.getEncryptLuksSecretMaterialFilePath(); + + VolumeVO volume = Q.New(VolumeVO.class).eq(VolumeVO_.uuid, msg.getVolumeUuid()).find(); + if (volume != null && volume.getSize() != 0) { + kcmd.virtualSizeForLuksClone = volume.getSize(); + } + + httpCallToKvmHost(hostUuid, + KVM_HOST_LUKS_CLONE_PATH, kcmd, KVMHostLuksRsp.class, + new ReturnValueCompletion(completion) { + @Override + public void success(KVMHostLuksRsp rsp) { + reply.setInstallPath(volPath); + long asize = rsp.actualSize == null ? 1 : rsp.actualSize; + reply.setActualSize(asize); + reply.setSize(volume == null ? sp.getSize() : volume.getSize()); + bus.reply(msg, reply); + completion.done(); + } + + @Override + public void fail(ErrorCode errorCode) { + reply.setError(errorCode); + bus.reply(msg, reply); + completion.done(); + } + }); + } + + private String findConnectedHostForCephLuks() { + String hostUuid = Q.New(PrimaryStorageHostRefVO.class) + .eq(PrimaryStorageHostRefVO_.primaryStorageUuid, self.getUuid()) + .eq(PrimaryStorageHostRefVO_.status, PrimaryStorageHostStatus.Connected) + .select(PrimaryStorageHostRefVO_.hostUuid) + .limit(1) + .findValue(); + if (StringUtils.isBlank(hostUuid)) { + throw new OperationFailureException(operr( + "cannot find a connected host attached to ceph primary storage[uuid:%s] to run LUKS RBD operation", + self.getUuid())); + } + return hostUuid; + } + protected void handle(final RevertVolumeFromSnapshotOnPrimaryStorageMsg msg) { final RevertVolumeFromSnapshotOnPrimaryStorageReply reply = new RevertVolumeFromSnapshotOnPrimaryStorageReply(); @@ -5230,6 +5654,36 @@ public void run(MessageReply reply) { @Override public void run(final FlowTrigger trigger, Map data) { + if (Boolean.TRUE.equals(msg.getVolume().getEncrypted())) { + String hostUuid = findConnectedHostForCephLuks(); + VolumeLuksAgentSpec luksSpec = snapshotEncryptionHelper.prepareVolumeSecretMaterial(hostUuid, msg.getVolume().getUuid()); + if (luksSpec == null || !luksSpec.isComplete()) { + trigger.fail(operr("cannot prepare LUKS secret for encrypted volume[uuid:%s] reimage on host[uuid:%s]", + msg.getVolume().getUuid(), hostUuid)); + return; + } + + KVMHostLuksCloneCmd kcmd = new KVMHostLuksCloneCmd(); + kcmd.psUuid = self.getUuid(); + kcmd.srcPath = installUrl; + kcmd.dstPath = volumePath; + kcmd.secFilePath = luksSpec.getEncryptLuksSecretMaterialFilePath(); + httpCallToKvmHost(hostUuid, + KVM_HOST_LUKS_CLONE_PATH, kcmd, KVMHostLuksRsp.class, + new ReturnValueCompletion(trigger) { + @Override + public void fail(ErrorCode err) { + trigger.fail(err); + } + + @Override + public void success(KVMHostLuksRsp ret) { + trigger.next(); + } + }); + return; + } + CloneCmd cmd = new CloneCmd(); cmd.srcPath = installUrl; cmd.dstPath = volumePath; @@ -5387,6 +5841,14 @@ private void takeSnapshot(final TakeSnapshotMsg msg, final NoErrorCompletion com q.add(VolumeVO_.uuid, Op.EQ, sp.getVolumeUuid()); String volumePath = q.findValue(); + VolumeVO volume = dbf.findByUuid(sp.getVolumeUuid(), VolumeVO.class); + if (volume != null && volume.isEncrypted()) { + // RBD snapshot operations do not open guest-visible LUKS/qcow2 data, + // so no secret material is needed here. Only inherit the volume key + // binding so later qemu-img based conversions know this snapshot is encrypted. + snapshotEncryptionHelper.inheritVolumeKeyToSnapshot(volume, sp); + } + final String spPath = String.format("%s@%s", volumePath, sp.getUuid()); CreateSnapshotCmd cmd = new CreateSnapshotCmd(); cmd.volumeUuid = sp.getVolumeUuid(); @@ -5403,6 +5865,9 @@ public void success(CreateSnapshotRsp rsp) { sp.setPrimaryStorageInstallPath(rsp.getInstallPath()); sp.setType(VolumeSnapshotConstant.STORAGE_SNAPSHOT_TYPE.toString()); sp.setFormat(VolumeConstant.VOLUME_FORMAT_RAW); + if (volume != null && volume.isEncrypted()) { + snapshotEncryptionHelper.completeTakeSnapshot(volume, sp); + } reply.setInventory(sp); bus.reply(msg, reply); completion.done(); diff --git a/plugin/kvm/src/main/java/org/zstack/kvm/KVMAgentCommands.java b/plugin/kvm/src/main/java/org/zstack/kvm/KVMAgentCommands.java index cc7c9083a7a..a61b307a18a 100755 --- a/plugin/kvm/src/main/java/org/zstack/kvm/KVMAgentCommands.java +++ b/plugin/kvm/src/main/java/org/zstack/kvm/KVMAgentCommands.java @@ -484,6 +484,30 @@ public void setSecretUuid(String secretUuid) { } } + public static class SecretHostEnsureLuksSecretFileCmd extends AgentCommand { + private String encryptedDek; + + public String getEncryptedDek() { + return encryptedDek; + } + + public void setEncryptedDek(String encryptedDek) { + this.encryptedDek = encryptedDek; + } + } + + public static class SecretHostEnsureLuksSecretFileResponse extends AgentResponse { + private String secFilePath; + + public String getSecFilePath() { + return secFilePath; + } + + public void setSecFilePath(String secFilePath) { + this.secFilePath = secFilePath; + } + } + public static class SecretHostGetCmd extends AgentCommand { private String vmUuid; private String purpose; @@ -4300,6 +4324,8 @@ public static class TakeSnapshotCmd extends AgentCommand implements HasThreadCon private String volumeInstallPath; private String newVolumeUuid; private String newVolumeInstallPath; + private String encryptLuksSecretMaterialFilePath; + private String fullSnapshotLuksSecretMaterialFilePath; private boolean online; private long timeout; @@ -4370,6 +4396,22 @@ public void setNewVolumeUuid(String newVolumeUuid) { this.newVolumeUuid = newVolumeUuid; } + public String getEncryptLuksSecretMaterialFilePath() { + return encryptLuksSecretMaterialFilePath; + } + + public void setEncryptLuksSecretMaterialFilePath(String encryptLuksSecretMaterialFilePath) { + this.encryptLuksSecretMaterialFilePath = encryptLuksSecretMaterialFilePath; + } + + public String getFullSnapshotLuksSecretMaterialFilePath() { + return fullSnapshotLuksSecretMaterialFilePath; + } + + public void setFullSnapshotLuksSecretMaterialFilePath(String fullSnapshotLuksSecretMaterialFilePath) { + this.fullSnapshotLuksSecretMaterialFilePath = fullSnapshotLuksSecretMaterialFilePath; + } + public VolumeTO getVolume() { return volume; } diff --git a/plugin/kvm/src/main/java/org/zstack/kvm/KVMConstant.java b/plugin/kvm/src/main/java/org/zstack/kvm/KVMConstant.java index 35ffa2c2065..e78b1fe54cc 100755 --- a/plugin/kvm/src/main/java/org/zstack/kvm/KVMConstant.java +++ b/plugin/kvm/src/main/java/org/zstack/kvm/KVMConstant.java @@ -133,6 +133,7 @@ public interface KVMConstant { String KVM_VERIFY_ENVELOPE_KEY_PATH = "/host/key/envelope/checkEnvelopeKey"; String KVM_GET_SECRET_PATH = "/host/key/envelope/getSecret"; String KVM_ENSURE_SECRET_PATH = "/host/key/envelope/ensureSecret"; + String KVM_WRITE_SECRET_MATERIAL_FILE_PATH = "/host/key/envelope/writeSecretMaterialFile"; String KVM_DELETE_SECRET_PATH = "/host/key/envelope/deleteSecret"; /** HTTP timeout in seconds for envelope key sync (verify/create/rotate/get) to agent. */ @@ -141,6 +142,16 @@ public interface KVMConstant { /** Max size in bytes for DEK payload in SecretHostDefine (decoded from dekBase64). */ int MAX_DEK_BYTES = 1024; String HOST_SECRET_USAGE_INSTANCE_VTPM = "tpm0"; + /** + * Per-volume usage instance string for the libvirt LUKS secret. Returned + * value is what we feed key-agent in {@code SecretHostDefineMsg} / + * {@code SecretHostGetMsg}; key-agent splices it into the libvirt usage + * name as {@code vm---version-}, so the + * resulting libvirt secret usage name is unique per (vm, volume, version). + */ + static String volumeSecretUsageInstance(String volumeUuid) { + return "volume-" + volumeUuid; + } String KVM_HOST_FILE_DOWNLOAD_PATH = "/host/file/download"; String KVM_HOST_FILE_UPLOAD_PATH = "/host/file/upload"; diff --git a/plugin/kvm/src/main/java/org/zstack/kvm/KVMHost.java b/plugin/kvm/src/main/java/org/zstack/kvm/KVMHost.java index 828aeb3a7cd..6a4ff9ac823 100755 --- a/plugin/kvm/src/main/java/org/zstack/kvm/KVMHost.java +++ b/plugin/kvm/src/main/java/org/zstack/kvm/KVMHost.java @@ -57,6 +57,8 @@ import org.zstack.header.host.MigrateVmOnHypervisorMsg.StorageMigrationPolicy; import org.zstack.header.secret.SecretHostDefineMsg; import org.zstack.header.secret.SecretHostDefineReply; +import org.zstack.header.secret.SecretHostEnsureLuksSecretFileMsg; +import org.zstack.header.secret.SecretHostEnsureLuksSecretFileReply; import org.zstack.header.secret.SecretHostDeleteMsg; import org.zstack.header.secret.SecretHostDeleteReply; import org.zstack.header.secret.SecretHostGetMsg; @@ -760,6 +762,8 @@ protected void handleLocalMessage(Message msg) { handle((SecretHostGetMsg) msg); } else if (msg instanceof ResolveVtpmLibvirtSecretOnHypervisorMsg) { handle((ResolveVtpmLibvirtSecretOnHypervisorMsg) msg); + } else if (msg instanceof SecretHostEnsureLuksSecretFileMsg) { + handle((SecretHostEnsureLuksSecretFileMsg) msg); } else if (msg instanceof SecretHostDefineMsg) { handle((SecretHostDefineMsg) msg); } else if (msg instanceof SecretHostDeleteMsg) { @@ -4495,6 +4499,15 @@ protected void startVm(final VmInstanceSpec spec, final NeedReplyMessage msg, fi cmd.setVmCpuModel(vmCpuMode); } + // Key must match VolumeEncryptedStartExtension.EXT_DATA_KEY in the storage module. + // Inlined here to avoid a kvm -> storage compile-time dep (kvm builds before storage). + @SuppressWarnings("unchecked") + Map volLuksSecrets = spec.getExtensionData( + "VolumeLuksSecrets", Map.class); + if (volLuksSecrets != null && volLuksSecrets.containsKey(rootVolume.getResourceUuid())) { + rootVolume.setLuksSecretUuid(volLuksSecrets.get(rootVolume.getResourceUuid())); + } + cmd.setRootVolume(rootVolume); cmd.setUseBootMenu(VmGlobalConfig.VM_BOOT_MENU.value(Boolean.class)); @@ -4511,6 +4524,9 @@ protected void startVm(final VmInstanceSpec spec, final NeedReplyMessage msg, fi // except for platform = Other, always use virtio driver for data volume // set bug https://github.com/zxwing/premium/issues/1050 v.setUseVirtio(!ImagePlatform.Other.toString().equals(platform)); + if (volLuksSecrets != null && volLuksSecrets.containsKey(v.getResourceUuid())) { + v.setLuksSecretUuid(volLuksSecrets.get(v.getResourceUuid())); + } dataVolumes.add(v); } dataVolumes.sort(Comparator.comparing(VolumeTO::getDeviceId)); @@ -5408,6 +5424,117 @@ public void run(MessageReply r) { }); } + private void handle(SecretHostEnsureLuksSecretFileMsg msg) { + SecretHostEnsureLuksSecretFileReply reply = new SecretHostEnsureLuksSecretFileReply(); + if (org.apache.commons.lang.StringUtils.isBlank(msg.getDekBase64())) { + reply.setError(operr("dekBase64 is required")); + bus.reply(msg, reply); + return; + } + if (StringUtils.isBlank(msg.getHostUuid())) { + reply.setError(operr("hostUuid is required for LUKS secret material file on hypervisor")); + bus.reply(msg, reply); + return; + } + String hostUuid = getSelf().getUuid(); + HostKeyIdentityVO identity = HostKeyIdentityHelper.getHostKeyIdentity(dbf, hostUuid); + String pubKey = identity != null ? org.apache.commons.lang.StringUtils.trimToNull(identity.getPublicKey()) : null; + Boolean verifyOk = identity != null ? identity.getVerified() : null; + if (pubKey == null) { + reply.setError(operr("no public key for host, connect/reconnect did not sync key")); + bus.reply(msg, reply); + return; + } + String storedFingerprint = StringUtils.trimToNull(identity.getFingerprint()); + String computed = HostKeyIdentityHelper.fingerprintFromPublicKey(pubKey); + if (storedFingerprint == null || !StringUtils.equals(storedFingerprint, computed)) { + reply.setError(operr("host public key fingerprint mismatch, key may be corrupted or tampered")); + bus.reply(msg, reply); + return; + } + if (!Boolean.TRUE.equals(verifyOk)) { + reply.setError(operr("host secret key verify not ok, not synced")); + bus.reply(msg, reply); + return; + } + byte[] dekRaw; + try { + dekRaw = java.util.Base64.getDecoder().decode(msg.getDekBase64().trim()); + } catch (IllegalArgumentException e) { + reply.setError(operr("invalid dekBase64: %s", e.getMessage())); + bus.reply(msg, reply); + return; + } + if (dekRaw == null || dekRaw.length == 0) { + reply.setError(operr("dekBase64 decoded to empty")); + bus.reply(msg, reply); + return; + } + if (dekRaw.length > KVMConstant.MAX_DEK_BYTES) { + reply.setError(operr("dekBase64 decoded payload is too large")); + bus.reply(msg, reply); + return; + } + byte[] pubKeyBytes; + try { + pubKeyBytes = java.util.Base64.getDecoder().decode(pubKey); + } catch (IllegalArgumentException e) { + reply.setError(operr("invalid host public key in DB: %s", e.getMessage())); + bus.reply(msg, reply); + return; + } + if (pubKeyBytes == null || pubKeyBytes.length != 32) { + reply.setError(operr("host public key must be 32 bytes (X25519)")); + bus.reply(msg, reply); + return; + } + java.util.List sealers = pluginRegistry.getExtensionList(HostSecretEnvelopeCryptoExtensionPoint.class); + if (sealers == null || sealers.isEmpty()) { + reply.setError(operr("host secret envelope sealer not available (premium crypto module required)")); + bus.reply(msg, reply); + return; + } + byte[] envelope; + try { + envelope = sealers.get(0).seal(pubKeyBytes, dekRaw); + } catch (Exception e) { + reply.setError(operr("HPKE seal failed: %s", e.getMessage())); + bus.reply(msg, reply); + return; + } + String envelopeDekBase64 = java.util.Base64.getEncoder().encodeToString(envelope); + KVMAgentCommands.SecretHostEnsureLuksSecretFileCmd cmd = new KVMAgentCommands.SecretHostEnsureLuksSecretFileCmd(); + cmd.setEncryptedDek(envelopeDekBase64); + + KVMHostAsyncHttpCallMsg kmsg = new KVMHostAsyncHttpCallMsg(); + kmsg.setCommand(cmd); + kmsg.setPath(KVMConstant.KVM_WRITE_SECRET_MATERIAL_FILE_PATH); + kmsg.setHostUuid(msg.getHostUuid()); + kmsg.setTimeout(TimeUnit.SECONDS.toMillis(KVMConstant.ENVELOPE_KEY_HTTP_TIMEOUT_SEC)); + bus.makeTargetServiceIdByResourceUuid(kmsg, HostConstant.SERVICE_ID, msg.getHostUuid()); + bus.send(kmsg, new CloudBusCallBack(msg) { + @Override + public void run(MessageReply r) { + if (!r.isSuccess()) { + reply.setError(r.getError()); + bus.reply(msg, reply); + return; + } + KVMHostAsyncHttpCallReply kreply = r.castReply(); + KVMAgentCommands.SecretHostEnsureLuksSecretFileResponse rsp = + kreply.toResponse(KVMAgentCommands.SecretHostEnsureLuksSecretFileResponse.class); + if (rsp != null && rsp.isSuccess() && StringUtils.isNotBlank(rsp.getSecFilePath())) { + reply.setSecFilePath(rsp.getSecFilePath()); + } else if (rsp != null && rsp.isSuccess()) { + reply.setError(operr("prepare LUKS secret channel succeeded but secFilePath is empty")); + } else { + reply.setError(buildSecretAgentError(rsp, "prepare LUKS secret channel failed")); + } + bus.reply(msg, reply); + } + }); + } + private void handle(SecretHostDefineMsg msg) { SecretHostDefineReply reply = new SecretHostDefineReply(); if (org.apache.commons.lang.StringUtils.isBlank(msg.getDekBase64())) { diff --git a/plugin/kvm/src/main/java/org/zstack/kvm/VolumeTO.java b/plugin/kvm/src/main/java/org/zstack/kvm/VolumeTO.java index db09c379de4..e59cff9ec23 100644 --- a/plugin/kvm/src/main/java/org/zstack/kvm/VolumeTO.java +++ b/plugin/kvm/src/main/java/org/zstack/kvm/VolumeTO.java @@ -54,6 +54,10 @@ public class VolumeTO extends BaseVirtualDeviceTO { private String ioThreadPin; private int controllerIndex; + // Host-local libvirt secret UUID holding the LUKS passphrase for an + // encrypted volume. cephx auth's secretUuid is on KVMCephVolumeTO. + private String luksSecretUuid; + static { deviceTypes.put(VolumeProtocol.Vhost, VHOST); deviceTypes.put(VolumeProtocol.CBD, CBD); @@ -83,6 +87,7 @@ public VolumeTO(VolumeTO other) { this.ioThreadId = other.ioThreadId; this.ioThreadPin = other.ioThreadPin; this.controllerIndex = other.controllerIndex; + this.luksSecretUuid = other.luksSecretUuid; } public static List valueOf(List vols, KVMHostInventory host) { @@ -313,4 +318,12 @@ public int getControllerIndex() { public void setControllerIndex(int controllerIndex) { this.controllerIndex = controllerIndex; } + + public String getLuksSecretUuid() { + return luksSecretUuid; + } + + public void setLuksSecretUuid(String luksSecretUuid) { + this.luksSecretUuid = luksSecretUuid; + } } diff --git a/plugin/kvm/src/main/java/org/zstack/kvm/tpm/KvmTpmExtensions.java b/plugin/kvm/src/main/java/org/zstack/kvm/tpm/KvmTpmExtensions.java index 8cbf5b52a7c..38d2742ea06 100644 --- a/plugin/kvm/src/main/java/org/zstack/kvm/tpm/KvmTpmExtensions.java +++ b/plugin/kvm/src/main/java/org/zstack/kvm/tpm/KvmTpmExtensions.java @@ -326,7 +326,7 @@ public void run(MessageReply reply) { } ErrorCode errorCode = reply.getError(); - if (errorCode != null && isVtpmSecretNotFoundOnHost(errorCode)) { + if (SecretHostGetReply.isSecretNotFound(errorCode)) { trigger.next(); return; } @@ -851,14 +851,6 @@ private boolean isVmCurrentlyOnExpectedHost(String vmUuid, String expectedHostUu return expectedHostUuid.equals(currentHostUuid); } - private static boolean isVtpmSecretNotFoundOnHost(ErrorCode errorCode) { - if (SecretHostGetReply.ERROR_CODE_SECRET_NOT_FOUND.equals(errorCode.getCode())) { - return true; - } - String details = errorCode.getDetails(); - return details != null && details.contains(SecretHostGetReply.ERROR_CODE_SECRET_NOT_FOUND); - } - private void deleteHostSecretBestEffort(String hostUuid, String vmUuid, Integer keyVersion, String reason) { if (StringUtils.isBlank(hostUuid) || StringUtils.isBlank(vmUuid) || keyVersion == null) { logger.info(String.format( diff --git a/plugin/localstorage/src/main/java/org/zstack/storage/primary/local/LocalStorageBase.java b/plugin/localstorage/src/main/java/org/zstack/storage/primary/local/LocalStorageBase.java index eea5f8beb92..e4fb64dbfd0 100755 --- a/plugin/localstorage/src/main/java/org/zstack/storage/primary/local/LocalStorageBase.java +++ b/plugin/localstorage/src/main/java/org/zstack/storage/primary/local/LocalStorageBase.java @@ -907,11 +907,30 @@ public void handleLocalMessage(Message msg) { handle((CommitVolumeSnapshotOnPrimaryStorageMsg) msg); } else if (msg instanceof PullVolumeSnapshotOnPrimaryStorageMsg) { handle((PullVolumeSnapshotOnPrimaryStorageMsg) msg); + } else if (msg instanceof EncryptVolumeBitsOnPrimaryStorageMsg) { + handle((EncryptVolumeBitsOnPrimaryStorageMsg) msg); } else { super.handleLocalMessage(msg); } } + private void handle(EncryptVolumeBitsOnPrimaryStorageMsg msg) { + LocalStorageHypervisorBackend bkd = getHypervisorBackendFactoryByHostUuid(msg.getHostUuid()).getHypervisorBackend(self); + bkd.handle(msg, new ReturnValueCompletion(msg) { + @Override + public void success(EncryptVolumeBitsOnPrimaryStorageReply reply) { + bus.reply(msg, reply); + } + + @Override + public void fail(ErrorCode errorCode) { + EncryptVolumeBitsOnPrimaryStorageReply reply = new EncryptVolumeBitsOnPrimaryStorageReply(); + reply.setError(errorCode); + bus.reply(msg, reply); + } + }); + } + private void handle(DownloadBitsFromKVMHostToPrimaryStorageMsg msg) { LocalStorageHypervisorBackend bkd = getHypervisorBackendFactoryByHostUuid(msg.getSrcHostUuid()).getHypervisorBackend(self); bkd.handle(msg, new ReturnValueCompletion(msg) { diff --git a/plugin/localstorage/src/main/java/org/zstack/storage/primary/local/LocalStorageCreateEmptyVolumeMsg.java b/plugin/localstorage/src/main/java/org/zstack/storage/primary/local/LocalStorageCreateEmptyVolumeMsg.java index 414b564bc14..c882af3ff45 100755 --- a/plugin/localstorage/src/main/java/org/zstack/storage/primary/local/LocalStorageCreateEmptyVolumeMsg.java +++ b/plugin/localstorage/src/main/java/org/zstack/storage/primary/local/LocalStorageCreateEmptyVolumeMsg.java @@ -3,6 +3,7 @@ import org.zstack.header.message.NeedReplyMessage; import org.zstack.header.storage.primary.PrimaryStorageMessage; import org.zstack.header.volume.VolumeInventory; +import org.zstack.header.volume.VolumeLuksAgentSpec; /** * Created by frank on 10/24/2015. @@ -12,6 +13,7 @@ public class LocalStorageCreateEmptyVolumeMsg extends NeedReplyMessage implement private String hostUuid; private String backingFile; private VolumeInventory volume; + private VolumeLuksAgentSpec volumeLuksAgentSpec; public String getBackingFile() { return backingFile; @@ -45,4 +47,12 @@ public VolumeInventory getVolume() { public void setVolume(VolumeInventory volume) { this.volume = volume; } + + public VolumeLuksAgentSpec getVolumeLuksAgentSpec() { + return volumeLuksAgentSpec; + } + + public void setVolumeLuksAgentSpec(VolumeLuksAgentSpec volumeLuksAgentSpec) { + this.volumeLuksAgentSpec = volumeLuksAgentSpec; + } } diff --git a/plugin/localstorage/src/main/java/org/zstack/storage/primary/local/LocalStorageFactory.java b/plugin/localstorage/src/main/java/org/zstack/storage/primary/local/LocalStorageFactory.java index a462ac65a5f..a908139c67a 100755 --- a/plugin/localstorage/src/main/java/org/zstack/storage/primary/local/LocalStorageFactory.java +++ b/plugin/localstorage/src/main/java/org/zstack/storage/primary/local/LocalStorageFactory.java @@ -1080,6 +1080,8 @@ public void instantiateDataVolumeOnCreation(InstantiateVolumeMsg msg, VolumeInve imsg.setVolume(volume); imsg.setPrimaryStorageUuid(msg.getPrimaryStorageUuid()); imsg.setDestHost(HostInventory.valueOf(dbf.findByUuid(hostUuid, HostVO.class))); + // For root volume with backing file + imsg.setVolumeLuksAgentSpec(msg.getVolumeLuksAgentSpec()); bus.makeTargetServiceIdByResourceUuid(imsg, PrimaryStorageConstant.SERVICE_ID, msg.getPrimaryStorageUuid()); bus.send(imsg, new CloudBusCallBack(completion) { @Override diff --git a/plugin/localstorage/src/main/java/org/zstack/storage/primary/local/LocalStorageHypervisorBackend.java b/plugin/localstorage/src/main/java/org/zstack/storage/primary/local/LocalStorageHypervisorBackend.java index d8226d932b2..f0ec6ed22dd 100755 --- a/plugin/localstorage/src/main/java/org/zstack/storage/primary/local/LocalStorageHypervisorBackend.java +++ b/plugin/localstorage/src/main/java/org/zstack/storage/primary/local/LocalStorageHypervisorBackend.java @@ -108,9 +108,11 @@ public LocalStorageHypervisorBackend(PrimaryStorageVO self) { abstract void deleteBits(String path, String hostUuid, Completion completion); - abstract void createEmptyVolume(VolumeInventory volume, String hostUuid, ReturnValueCompletion completion); + abstract void createEmptyVolume(VolumeInventory volume, String hostUuid, VolumeLuksAgentSpec volumeLuksAgentSpec, + ReturnValueCompletion completion); - abstract void createEmptyVolumeWithBackingFile(VolumeInventory volume, String hostUuid, String backingFile, ReturnValueCompletion completion); + abstract void createEmptyVolumeWithBackingFile(VolumeInventory volume, String hostUuid, String backingFile, + VolumeLuksAgentSpec volumeLuksAgentSpec, ReturnValueCompletion completion); abstract void checkHostAttachedPSMountPath(String hostUuid, ReturnValueCompletion completion); @@ -133,4 +135,6 @@ public LocalStorageHypervisorBackend(PrimaryStorageVO self) { abstract void handle(CleanupVmInstanceMetadataOnPrimaryStorageMsg msg, String hostUuid, ReturnValueCompletion completion); abstract void handle(RebaseVolumeBackingFileOnPrimaryStorageMsg msg, String hostUuid, ReturnValueCompletion completion); + + abstract void handle(EncryptVolumeBitsOnPrimaryStorageMsg msg, ReturnValueCompletion completion); } diff --git a/plugin/localstorage/src/main/java/org/zstack/storage/primary/local/LocalStorageKvmBackend.java b/plugin/localstorage/src/main/java/org/zstack/storage/primary/local/LocalStorageKvmBackend.java index 5b4324495b2..b376c091d86 100755 --- a/plugin/localstorage/src/main/java/org/zstack/storage/primary/local/LocalStorageKvmBackend.java +++ b/plugin/localstorage/src/main/java/org/zstack/storage/primary/local/LocalStorageKvmBackend.java @@ -55,6 +55,7 @@ import org.zstack.header.volume.*; import org.zstack.identity.AccountManager; import org.zstack.kvm.*; +import org.zstack.storage.encrypt.VolumeSnapshotEncryptionHelper; import org.zstack.storage.primary.*; import org.zstack.storage.primary.local.LocalStorageKvmMigrateVmFlow.CopyBitsFromRemoteCmd; import org.zstack.storage.primary.local.MigrateBitsStruct.ResourceInfo; @@ -89,6 +90,8 @@ public class LocalStorageKvmBackend extends LocalStorageHypervisorBackend { private RESTFacade restf; @Autowired private PluginRegistry pluginRgty; + @Autowired + private VolumeSnapshotEncryptionHelper snapshotEncryptionHelper; public static class AgentCommand extends KVMAgentCommands.PrimaryStorageCommand { public String uuid; @@ -205,6 +208,7 @@ public static class CreateEmptyVolumeCmd extends AgentCommand { private String volumeUuid; private String backingFile; private String volumeFormat; + private String encryptLuksSecretMaterialFilePath; public String getBackingFile() { return backingFile; @@ -222,6 +226,14 @@ public void setVolumeFormat(String volumeFormat) { this.volumeFormat = volumeFormat; } + public String getEncryptLuksSecretMaterialFilePath() { + return encryptLuksSecretMaterialFilePath; + } + + public void setEncryptLuksSecretMaterialFilePath(String encryptLuksSecretMaterialFilePath) { + this.encryptLuksSecretMaterialFilePath = encryptLuksSecretMaterialFilePath; + } + public String getInstallUrl() { return installUrl; } @@ -268,6 +280,14 @@ public static class CreateEmptyVolumeRsp extends AgentResponse { public Long size; } + public static class EncryptVolumeBitsCmd extends AgentCommand { + public String installPath; + public String encryptLuksSecretMaterialFilePath; + } + + public static class EncryptVolumeBitsRsp extends AgentResponse { + } + public static class GetPhysicalCapacityCmd extends AgentCommand { private String hostUuid; @@ -285,6 +305,15 @@ public static class CreateVolumeFromCacheCmd extends AgentCommand { private String installUrl; private String volumeUuid; private long virtualSize; + private String encryptLuksSecretMaterialFilePath; + + public String getEncryptLuksSecretMaterialFilePath() { + return encryptLuksSecretMaterialFilePath; + } + + public void setEncryptLuksSecretMaterialFilePath(String encryptLuksSecretMaterialFilePath) { + this.encryptLuksSecretMaterialFilePath = encryptLuksSecretMaterialFilePath; + } public String getTemplatePathInCache() { return templatePathInCache; @@ -329,6 +358,7 @@ public static class CreateVolumeWithBackingCmd extends AgentCommand { public String installPath; public String volumeUuid; public long virtualSize; + public String encryptLuksSecretMaterialFilePath; } public static class CreateVolumeWithBackingRsp extends AgentResponse { @@ -420,6 +450,7 @@ public void setPaths(List paths) { public static class CreateTemplateFromVolumeCmd extends AgentCommand implements HasThreadContext{ private String installPath; private String volumePath; + private String encryptLuksSecretMaterialFilePath; public String getInstallPath() { return installPath; @@ -436,6 +467,14 @@ public String getVolumePath() { public void setVolumePath(String rootVolumePath) { this.volumePath = rootVolumePath; } + + public String getEncryptLuksSecretMaterialFilePath() { + return encryptLuksSecretMaterialFilePath; + } + + public void setEncryptLuksSecretMaterialFilePath(String encryptLuksSecretMaterialFilePath) { + this.encryptLuksSecretMaterialFilePath = encryptLuksSecretMaterialFilePath; + } } public static class CreateTemplateFromVolumeRsp extends AgentResponse { @@ -494,6 +533,7 @@ public void setSize(long size) { public static class RevertVolumeFromSnapshotCmd extends AgentCommand { private String snapshotInstallPath; + private String encryptLuksSecretMaterialFilePath; public String getSnapshotInstallPath() { return snapshotInstallPath; @@ -502,11 +542,20 @@ public String getSnapshotInstallPath() { public void setSnapshotInstallPath(String snapshotInstallPath) { this.snapshotInstallPath = snapshotInstallPath; } + + public String getEncryptLuksSecretMaterialFilePath() { + return encryptLuksSecretMaterialFilePath; + } + + public void setEncryptLuksSecretMaterialFilePath(String encryptLuksSecretMaterialFilePath) { + this.encryptLuksSecretMaterialFilePath = encryptLuksSecretMaterialFilePath; + } } public static class ReinitImageCmd extends AgentCommand { private String imagePath; private String volumePath; + private String encryptLuksSecretMaterialFilePath; public String getImagePath() { return imagePath; @@ -523,6 +572,14 @@ public String getVolumePath() { public void setVolumePath(String volumePath) { this.volumePath = volumePath; } + + public String getEncryptLuksSecretMaterialFilePath() { + return encryptLuksSecretMaterialFilePath; + } + + public void setEncryptLuksSecretMaterialFilePath(String encryptLuksSecretMaterialFilePath) { + this.encryptLuksSecretMaterialFilePath = encryptLuksSecretMaterialFilePath; + } } public static class ReinitImageRsp extends AgentResponse { @@ -566,6 +623,7 @@ public static class MergeSnapshotCmd extends AgentCommand implements HasThreadCo private String volumeUuid; private String snapshotInstallPath; private String workspaceInstallPath; + private String encryptLuksSecretMaterialFilePath; public String getVolumeUuid() { return volumeUuid; @@ -590,6 +648,14 @@ public String getWorkspaceInstallPath() { public void setWorkspaceInstallPath(String workspaceInstallPath) { this.workspaceInstallPath = workspaceInstallPath; } + + public String getEncryptLuksSecretMaterialFilePath() { + return encryptLuksSecretMaterialFilePath; + } + + public void setEncryptLuksSecretMaterialFilePath(String encryptLuksSecretMaterialFilePath) { + this.encryptLuksSecretMaterialFilePath = encryptLuksSecretMaterialFilePath; + } } public static class MergeSnapshotRsp extends AgentResponse { @@ -668,6 +734,8 @@ public static class OfflineMergeSnapshotCmd extends AgentCommand implements HasT private String srcPath; private String destPath; private boolean fullRebase; + private String encryptLuksSecretMaterialFilePath; + private String resetBackingLuksSecretMaterialFilePath; public boolean isFullRebase() { return fullRebase; @@ -692,6 +760,22 @@ public String getDestPath() { public void setDestPath(String destPath) { this.destPath = destPath; } + + public String getEncryptLuksSecretMaterialFilePath() { + return encryptLuksSecretMaterialFilePath; + } + + public void setEncryptLuksSecretMaterialFilePath(String encryptLuksSecretMaterialFilePath) { + this.encryptLuksSecretMaterialFilePath = encryptLuksSecretMaterialFilePath; + } + + public String getResetBackingLuksSecretMaterialFilePath() { + return resetBackingLuksSecretMaterialFilePath; + } + + public void setResetBackingLuksSecretMaterialFilePath(String resetBackingLuksSecretMaterialFilePath) { + this.resetBackingLuksSecretMaterialFilePath = resetBackingLuksSecretMaterialFilePath; + } } public static class OfflineMergeSnapshotRsp extends AgentResponse { @@ -710,6 +794,8 @@ public static class OfflineCommitSnapshotCmd extends AgentCommand implements Has public String top; public String base; public List topChildrenInstallPathInDb = new ArrayList<>(); + public String encryptLuksSecretMaterialFilePath; + public List rebaseLuksSecretMaterialFilePaths = new ArrayList<>(); } public static class OfflineCommitSnapshotRsp extends AgentResponse { @@ -997,6 +1083,7 @@ public static class PrefixRebaseBackingFilesRsp extends LocalStorageKvmBackend.A public static final String SCAN_VM_METADATA_PATH = "/localstorage/vm/metadata/scan"; public static final String CLEANUP_VM_METADATA_PATH = "/localstorage/vm/metadata/cleanup"; public static final String PREFIX_REBASE_BACKING_FILES_PATH = "/localstorage/snapshot/prefixrebasebackingfiles"; + public static final String ENCRYPT_VOLUME_BITS_PATH = "/localstorage/volume/encryptinplace"; public LocalStorageKvmBackend() { } @@ -1309,7 +1396,7 @@ private void createTemporaryEmptyVolume(InstantiateTemporaryVolumeOnPrimaryStora } private void createEmptyVolume(InstantiateVolumeOnPrimaryStorageMsg msg, ReturnValueCompletion completion) { - createEmptyVolume(msg.getVolume(), msg.getDestHost().getUuid(), new ReturnValueCompletion(completion) { + createEmptyVolume(msg.getVolume(), msg.getDestHost().getUuid(), msg.getVolumeLuksAgentSpec(), new ReturnValueCompletion(completion) { @Override public void success(VolumeStats returnValue) { InstantiateVolumeOnPrimaryStorageReply r = new InstantiateVolumeOnPrimaryStorageReply(); @@ -1332,11 +1419,11 @@ public void fail(ErrorCode errorCode) { }); } - public void createEmptyVolume(final VolumeInventory volume, final String hostUuid, final ReturnValueCompletion completion) { - createEmptyVolumeWithBackingFile(volume, hostUuid, null, completion); + public void createEmptyVolume(final VolumeInventory volume, final String hostUuid, final VolumeLuksAgentSpec volumeLuksAgentSpec, final ReturnValueCompletion completion) { + createEmptyVolumeWithBackingFile(volume, hostUuid, null, volumeLuksAgentSpec, completion); } - public void createEmptyVolumeWithBackingFile(final VolumeInventory volume, final String hostUuid, final String backingFile, final ReturnValueCompletion completion) { + public void createEmptyVolumeWithBackingFile(final VolumeInventory volume, final String hostUuid, final String backingFile, final VolumeLuksAgentSpec volumeLuksAgentSpec, final ReturnValueCompletion completion) { final CreateEmptyVolumeCmd cmd = new CreateEmptyVolumeCmd(); cmd.setAccountUuid(acntMgr.getOwnerAccountUuidOfResource(volume.getUuid())); if (volume.getInstallPath() != null && !volume.getInstallPath().equals("")) { @@ -1358,6 +1445,9 @@ public void createEmptyVolumeWithBackingFile(final VolumeInventory volume, final cmd.setSize(volume.getSize()); cmd.setVolumeUuid(volume.getUuid()); cmd.setBackingFile(backingFile); + if (volumeLuksAgentSpec != null && volumeLuksAgentSpec.isComplete()) { + cmd.setEncryptLuksSecretMaterialFilePath(volumeLuksAgentSpec.getEncryptLuksSecretMaterialFilePath()); + } httpCall(CREATE_EMPTY_VOLUME_PATH, hostUuid, cmd, CreateEmptyVolumeRsp.class, new ReturnValueCompletion(completion) { @Override @@ -1396,6 +1486,7 @@ class ImageCache { String hostUuid; String primaryStorageInstallPath; String backupStorageInstallPath; + String encryptLuksSecretMaterialFilePath; void download(final ReturnValueCompletion completion) { DebugUtils.Assert(image != null, "image cannot be null"); @@ -1488,6 +1579,9 @@ private void downloadFromVolume(FlowTrigger trigger) { CreateTemplateFromVolumeCmd cmd = new CreateTemplateFromVolumeCmd(); cmd.setInstallPath(primaryStorageInstallPath); cmd.setVolumePath(volumeResourceInstallPath); + if (StringUtils.isNotBlank(encryptLuksSecretMaterialFilePath)) { + cmd.setEncryptLuksSecretMaterialFilePath(encryptLuksSecretMaterialFilePath); + } httpCall(CREATE_TEMPLATE_FROM_VOLUME, hostUuid, cmd, false, CreateTemplateFromVolumeRsp.class, @@ -1707,7 +1801,7 @@ private void createRootVolume(final InstantiateRootVolumeFromTemplateOnPrimarySt final ImageInventory image = ispec.getInventory(); if (!ImageMediaType.RootVolumeTemplate.toString().equals(image.getMediaType())) { - createEmptyVolume(msg.getVolume(), msg.getDestHost().getUuid(), new ReturnValueCompletion(completion) { + createEmptyVolume(msg.getVolume(), msg.getDestHost().getUuid(), msg.getVolumeLuksAgentSpec(), new ReturnValueCompletion(completion) { @Override public void success(VolumeStats returnValue) { InstantiateVolumeOnPrimaryStorageReply r = new InstantiateVolumeOnPrimaryStorageReply(); @@ -1778,6 +1872,11 @@ public void run(final FlowTrigger trigger, Map data) { cmd.setVirtualSize(volume.getSize()); } + VolumeLuksAgentSpec volumeLuksAgentSpec = msg.getVolumeLuksAgentSpec(); + if (volumeLuksAgentSpec != null && volumeLuksAgentSpec.isComplete()) { + cmd.setEncryptLuksSecretMaterialFilePath(volumeLuksAgentSpec.getEncryptLuksSecretMaterialFilePath()); + } + httpCall(CREATE_VOLUME_FROM_CACHE_PATH, hostUuid, cmd, CreateVolumeFromCacheRsp.class, new ReturnValueCompletion(trigger) { @Override public void success(CreateVolumeFromCacheRsp returnValue) { @@ -2131,6 +2230,12 @@ void handle(RevertVolumeFromSnapshotOnPrimaryStorageMsg msg, String hostUuid, fi VolumeSnapshotInventory sp = msg.getSnapshot(); RevertVolumeFromSnapshotCmd cmd = new RevertVolumeFromSnapshotCmd(); cmd.setSnapshotInstallPath(sp.getPrimaryStorageInstallPath()); + if (Boolean.TRUE.equals(msg.getVolume().getEncrypted())) { + VolumeLuksAgentSpec luksSpec = snapshotEncryptionHelper.prepareVolumeSecretMaterial(hostUuid, msg.getVolume().getUuid()); + if (luksSpec != null && luksSpec.isComplete()) { + cmd.setEncryptLuksSecretMaterialFilePath(luksSpec.getEncryptLuksSecretMaterialFilePath()); + } + } httpCall(REVERT_SNAPSHOT_PATH, hostUuid, cmd, RevertVolumeFromSnapshotRsp.class, new ReturnValueCompletion(completion) { @Override @@ -2170,6 +2275,15 @@ public void run(FlowTrigger trigger, Map data) { } cmd.imagePath = makeCachedImageInstallUrlFromImageUuidForTemplate(msg.getVolume().getRootImageUuid()); cmd.volumePath = makeRootVolumeInstallUrl(msg.getVolume()); + if (Boolean.TRUE.equals(msg.getVolume().getEncrypted())) { + String secretMaterialFilePath = prepareVolumeSecretMaterialPath(hostUuid, msg.getVolume()); + if (StringUtils.isBlank(secretMaterialFilePath)) { + completion.fail(operr("cannot prepare LUKS secret for encrypted volume[uuid:%s] reimage on host[uuid:%s]", + msg.getVolume().getUuid(), hostUuid)); + return; + } + cmd.setEncryptLuksSecretMaterialFilePath(secretMaterialFilePath); + } httpCall(REINIT_IMAGE_PATH, hostUuid, cmd, ReinitImageRsp.class, new ReturnValueCompletion(completion) { @Override @@ -2246,6 +2360,10 @@ private void createNormalVolumeFromSnapshot(VolumeSnapshotInventory sp, String v cmd.setVolumeUuid(sp.getVolumeUuid()); cmd.setSnapshotInstallPath(sp.getPrimaryStorageInstallPath()); cmd.setWorkspaceInstallPath(installPath); + VolumeLuksAgentSpec luksSpec = snapshotEncryptionHelper.prepareVolumeSecretMaterial(hostUuid, volumeUuid); + if (luksSpec != null && luksSpec.isComplete()) { + cmd.setEncryptLuksSecretMaterialFilePath(luksSpec.getEncryptLuksSecretMaterialFilePath()); + } httpCall(MERGE_SNAPSHOT_PATH, hostUuid, cmd, MergeSnapshotRsp.class, new ReturnValueCompletion(completion) { @Override @@ -2271,6 +2389,10 @@ private void createIncrementalVolumeFromSnapshot(VolumeSnapshotInventory sp, Str cmd.volumeUuid = volumeUuid; cmd.installPath = installPath; cmd.templatePathInCache = sp.getPrimaryStorageInstallPath(); + VolumeLuksAgentSpec luksSpec = snapshotEncryptionHelper.prepareVolumeSecretMaterial(hostUuid, volumeUuid); + if (luksSpec != null && luksSpec.isComplete()) { + cmd.encryptLuksSecretMaterialFilePath = luksSpec.getEncryptLuksSecretMaterialFilePath(); + } Long volumeSize = Q.New(VolumeVO.class).eq(VolumeVO_.uuid, volumeUuid).select(VolumeVO_.size).findValue(); if (volumeSize != null && volumeSize != 0) { @@ -2325,6 +2447,10 @@ void stream(VolumeSnapshotInventory from, VolumeInventory to, boolean fullRebase cmd.setFullRebase(fullRebase); cmd.setSrcPath(sp.getPrimaryStorageInstallPath()); cmd.setDestPath(volume.getInstallPath()); + cmd.setEncryptLuksSecretMaterialFilePath(prepareVolumeSecretMaterialPath(hostUuid, volume)); + if (!fullRebase) { + cmd.setResetBackingLuksSecretMaterialFilePath(prepareVolumeSecretMaterialPath(hostUuid, volume)); + } httpCall(OFFLINE_MERGE_PATH, hostUuid, cmd, OfflineMergeSnapshotRsp.class, new ReturnValueCompletion(completion) { @Override @@ -2359,7 +2485,7 @@ public void run(MessageReply reply) { @Override void handle(LocalStorageCreateEmptyVolumeMsg msg, final ReturnValueCompletion completion) { - createEmptyVolumeWithBackingFile(msg.getVolume(), msg.getHostUuid(), msg.getBackingFile(), new ReturnValueCompletion(completion) { + createEmptyVolumeWithBackingFile(msg.getVolume(), msg.getHostUuid(), msg.getBackingFile(), msg.getVolumeLuksAgentSpec(), new ReturnValueCompletion(completion) { @Override public void success(VolumeStats returnValue) { LocalStorageCreateEmptyVolumeReply reply = new LocalStorageCreateEmptyVolumeReply(); @@ -3417,6 +3543,14 @@ void handle(CreateImageCacheFromVolumeSnapshotOnPrimaryStorageMsg msg, ReturnVal cache.hostUuid = ref.getHostUuid(); cache.image = msg.getImageInventory(); cache.volumeResourceInstallPath = msg.getVolumeSnapshot().getPrimaryStorageInstallPath(); + VolumeLuksAgentSpec luksSpec = snapshotEncryptionHelper.prepareTemporarySnapshotImageSecretMaterial( + ref.getHostUuid(), + msg.getVolumeSnapshot().getUuid(), + msg.getImageInventory().getUuid(), + msg.getEncrypted()); + if (luksSpec != null && luksSpec.isComplete()) { + cache.encryptLuksSecretMaterialFilePath = luksSpec.getEncryptLuksSecretMaterialFilePath(); + } cache.download(new ReturnValueCompletion(completion) { @Override public void success(ImageCacheInventory inv) { @@ -3821,6 +3955,33 @@ private String makeInitializedFilePath() { return String.format("%s/%s-initialized-file", self.getMountPath(), self.getUuid()); } + protected String prepareVolumeSecretMaterialPath(String hostUuid, VolumeInventory volume) { + if (volume == null || !Boolean.TRUE.equals(volume.getEncrypted())) { + return null; + } + + VolumeLuksAgentSpec luksSpec = snapshotEncryptionHelper.prepareVolumeSecretMaterial(hostUuid, volume.getUuid()); + if (luksSpec == null || !luksSpec.isComplete()) { + return null; + } + return luksSpec.getEncryptLuksSecretMaterialFilePath(); + } + + private List prepareVolumeSecretMaterialPaths(String hostUuid, VolumeInventory volume, int count) { + List secretPaths = new ArrayList<>(); + if (volume == null || !Boolean.TRUE.equals(volume.getEncrypted())) { + return secretPaths; + } + + for (int i = 0; i < count; i++) { + String secretPath = prepareVolumeSecretMaterialPath(hostUuid, volume); + if (secretPath != null) { + secretPaths.add(secretPath); + } + } + return secretPaths; + } + @Override void handle(CommitVolumeSnapshotOnPrimaryStorageMsg msg, String hostUuid, final ReturnValueCompletion completion) { CommitVolumeSnapshotOnPrimaryStorageReply reply = new CommitVolumeSnapshotOnPrimaryStorageReply(); @@ -3828,6 +3989,9 @@ void handle(CommitVolumeSnapshotOnPrimaryStorageMsg msg, String hostUuid, final cmd.top = msg.getSrcSnapshot().getPrimaryStorageInstallPath(); cmd.base = msg.getDstSnapshot().getPrimaryStorageInstallPath(); cmd.topChildrenInstallPathInDb = msg.getSrcChildrenInstallPathInDb(); + cmd.encryptLuksSecretMaterialFilePath = prepareVolumeSecretMaterialPath(hostUuid, msg.getVolume()); + cmd.rebaseLuksSecretMaterialFilePaths = prepareVolumeSecretMaterialPaths( + hostUuid, msg.getVolume(), cmd.topChildrenInstallPathInDb == null ? 0 : cmd.topChildrenInstallPathInDb.size()); httpCall(OFFLINE_COMMIT_PATH, hostUuid, cmd, OfflineCommitSnapshotRsp.class, new ReturnValueCompletion(completion) { @Override public void success(OfflineCommitSnapshotRsp returnValue) { @@ -3850,6 +4014,10 @@ void handle(PullVolumeSnapshotOnPrimaryStorageMsg msg, String hostUuid, final Re cmd.srcPath = msg.getSrcSnapshotParentPath(); cmd.destPath = msg.getDstSnapshot().getPrimaryStorageInstallPath(); cmd.fullRebase = cmd.srcPath == null; + cmd.setEncryptLuksSecretMaterialFilePath(prepareVolumeSecretMaterialPath(hostUuid, msg.getVolume())); + if (!cmd.fullRebase) { + cmd.setResetBackingLuksSecretMaterialFilePath(prepareVolumeSecretMaterialPath(hostUuid, msg.getVolume())); + } httpCall(OFFLINE_MERGE_PATH, hostUuid, cmd, OfflineMergeSnapshotRsp.class, new ReturnValueCompletion(completion) { @Override public void success(OfflineMergeSnapshotRsp rsp) { @@ -3976,4 +4144,24 @@ public void fail(ErrorCode errorCode) { } }); } + + @Override + void handle(EncryptVolumeBitsOnPrimaryStorageMsg msg, ReturnValueCompletion completion) { + EncryptVolumeBitsCmd cmd = new EncryptVolumeBitsCmd(); + cmd.installPath = msg.getInstallPath(); + cmd.encryptLuksSecretMaterialFilePath = msg.getEncryptLuksSecretMaterialFilePath(); + + httpCall(ENCRYPT_VOLUME_BITS_PATH, msg.getHostUuid(), cmd, EncryptVolumeBitsRsp.class, new ReturnValueCompletion(completion) { + @Override + public void success(EncryptVolumeBitsRsp rsp) { + completion.success(new EncryptVolumeBitsOnPrimaryStorageReply()); + } + + @Override + public void fail(ErrorCode errorCode) { + completion.fail(operr("failed to encrypt volume[uuid:%s] bits at path[%s] on host[uuid:%s]: %s", + msg.getVolumeUuid(), msg.getInstallPath(), msg.getHostUuid(), errorCode)); + } + }); + } } diff --git a/plugin/localstorage/src/main/java/org/zstack/storage/primary/local/LocalStorageKvmFactory.java b/plugin/localstorage/src/main/java/org/zstack/storage/primary/local/LocalStorageKvmFactory.java index 8549d799eeb..337b57715fb 100755 --- a/plugin/localstorage/src/main/java/org/zstack/storage/primary/local/LocalStorageKvmFactory.java +++ b/plugin/localstorage/src/main/java/org/zstack/storage/primary/local/LocalStorageKvmFactory.java @@ -195,7 +195,7 @@ public void beforeTakeSnapshot(KVMHostInventory host, TakeSnapshotOnHypervisorMs LocalStorageHypervisorBackend bkd = getHypervisorBackend(primaryStorageVO); String backingFile = cmd.isOnline() ? cmd.getVolumeInstallPath() : null; - bkd.createEmptyVolumeWithBackingFile(inv, msg.getHostUuid(), backingFile, new ReturnValueCompletion(completion) { + bkd.createEmptyVolumeWithBackingFile(inv, msg.getHostUuid(), backingFile, null, new ReturnValueCompletion(completion) { @Override public void success(VolumeStats returnValue) { completion.success(); diff --git a/plugin/nfsPrimaryStorage/src/main/java/org/zstack/storage/primary/nfs/NfsDownloadImageToCacheJob.java b/plugin/nfsPrimaryStorage/src/main/java/org/zstack/storage/primary/nfs/NfsDownloadImageToCacheJob.java index ad9a9c275d4..7c7736e29aa 100755 --- a/plugin/nfsPrimaryStorage/src/main/java/org/zstack/storage/primary/nfs/NfsDownloadImageToCacheJob.java +++ b/plugin/nfsPrimaryStorage/src/main/java/org/zstack/storage/primary/nfs/NfsDownloadImageToCacheJob.java @@ -47,6 +47,10 @@ public class NfsDownloadImageToCacheJob implements Job { private PrimaryStorageInventory primaryStorage; @JobContext private String volumeResourceInstallPath; + @JobContext + private String volumeSnapshotUuid; + @JobContext + private Boolean encrypted; @Autowired private NfsPrimaryStorageFactory nfsFactory; @@ -175,7 +179,8 @@ public void fail(ErrorCode errorCode) { } }; - bkd.createImageCacheFromVolumeResource(primaryStorage, volumeResourceInstallPath, image.getInventory(), compl); + bkd.createImageCacheFromVolumeResource(primaryStorage, volumeResourceInstallPath, + image.getInventory(), volumeSnapshotUuid, encrypted, compl); } private void downloadFromBackupStorage(FlowTrigger trigger) { @@ -319,4 +324,12 @@ public void setPrimaryStorage(PrimaryStorageInventory primaryStorage) { public void setVolumeResourceInstallPath(String volumeResourceInstallPath) { this.volumeResourceInstallPath = volumeResourceInstallPath; } + + public void setVolumeSnapshotUuid(String volumeSnapshotUuid) { + this.volumeSnapshotUuid = volumeSnapshotUuid; + } + + public void setEncrypted(Boolean encrypted) { + this.encrypted = encrypted; + } } diff --git a/plugin/nfsPrimaryStorage/src/main/java/org/zstack/storage/primary/nfs/NfsPrimaryStorage.java b/plugin/nfsPrimaryStorage/src/main/java/org/zstack/storage/primary/nfs/NfsPrimaryStorage.java index 6b0142e5a63..6372603af05 100755 --- a/plugin/nfsPrimaryStorage/src/main/java/org/zstack/storage/primary/nfs/NfsPrimaryStorage.java +++ b/plugin/nfsPrimaryStorage/src/main/java/org/zstack/storage/primary/nfs/NfsPrimaryStorage.java @@ -136,11 +136,37 @@ protected void handleLocalMessage(Message msg) { handle((PullVolumeSnapshotOnPrimaryStorageMsg) msg); } else if (msg instanceof RebaseVolumeBackingFileOnPrimaryStorageMsg) { handle((RebaseVolumeBackingFileOnPrimaryStorageMsg) msg); + } else if (msg instanceof EncryptVolumeBitsOnPrimaryStorageMsg) { + handle((EncryptVolumeBitsOnPrimaryStorageMsg) msg); } else { super.handleLocalMessage(msg); } } + private void handle(EncryptVolumeBitsOnPrimaryStorageMsg msg) { + NfsPrimaryStorageBackend backend = getUsableBackend(); + if (backend == null) { + EncryptVolumeBitsOnPrimaryStorageReply reply = new EncryptVolumeBitsOnPrimaryStorageReply(); + reply.setError(operr("the NFS primary storage[uuid:%s, name:%s] cannot find any usable host to" + + " encrypt volume[uuid:%s] bits", self.getUuid(), self.getName(), msg.getVolumeUuid())); + bus.reply(msg, reply); + return; + } + backend.handle(msg, new ReturnValueCompletion(msg) { + @Override + public void success(EncryptVolumeBitsOnPrimaryStorageReply reply) { + bus.reply(msg, reply); + } + + @Override + public void fail(ErrorCode errorCode) { + EncryptVolumeBitsOnPrimaryStorageReply reply = new EncryptVolumeBitsOnPrimaryStorageReply(); + reply.setError(errorCode); + bus.reply(msg, reply); + } + }); + } + protected void updateMountPoint(String newUrl, Completion completion) { String oldUrl = self.getUrl(); @@ -812,7 +838,7 @@ public void fail(ErrorCode errorCode) { @Override public void run(final FlowTrigger trigger, Map data) { NfsPrimaryStorageBackend backend = factory.getHypervisorBackend(nfsMgr.findHypervisorTypeByImageFormatAndPrimaryStorageUuid(image.getFormat(), self.getUuid())); - backend.createVolumeFromImageCache(primaryStorage, image, imageCache, volume, new ReturnValueCompletion(trigger) { + backend.createVolumeFromImageCache(primaryStorage, image, imageCache, volume, msg.getVolumeLuksAgentSpec(), new ReturnValueCompletion(trigger) { @Override public void success(VolumeStats returnValue) { volumeInstallPath = returnValue.getInstallPath(); @@ -916,7 +942,7 @@ private void createEmptyVolume(final InstantiateVolumeOnPrimaryStorageMsg msg) { VolumeInventory vol = msg.getVolume(); final InstantiateVolumeOnPrimaryStorageReply reply = new InstantiateVolumeOnPrimaryStorageReply(); - backend.instantiateVolume(PrimaryStorageInventory.valueOf(self), msg.getDestHost(), vol, new ReturnValueCompletion(msg) { + backend.instantiateVolume(PrimaryStorageInventory.valueOf(self), msg.getDestHost(), vol, msg.getVolumeLuksAgentSpec(), new ReturnValueCompletion(msg) { @Override public void success(VolumeInventory returnValue) { reply.setVolume(returnValue); @@ -1037,6 +1063,8 @@ protected void handle(CreateImageCacheFromVolumeSnapshotOnPrimaryStorageMsg msg) job.setPrimaryStorage(getSelfInventory()); job.setImage(spec); job.setVolumeResourceInstallPath(msg.getVolumeSnapshot().getPrimaryStorageInstallPath()); + job.setVolumeSnapshotUuid(msg.getVolumeSnapshot().getUuid()); + job.setEncrypted(msg.getEncrypted()); jobf.execute(NfsPrimaryStorageKvmHelper.makeDownloadImageJobName(msg.getImageInventory(), job.getPrimaryStorage()), NfsPrimaryStorageKvmHelper.makeJobOwnerName(job.getPrimaryStorage()), job, diff --git a/plugin/nfsPrimaryStorage/src/main/java/org/zstack/storage/primary/nfs/NfsPrimaryStorageBackend.java b/plugin/nfsPrimaryStorage/src/main/java/org/zstack/storage/primary/nfs/NfsPrimaryStorageBackend.java index 35a240a6221..1a783cc01c9 100755 --- a/plugin/nfsPrimaryStorage/src/main/java/org/zstack/storage/primary/nfs/NfsPrimaryStorageBackend.java +++ b/plugin/nfsPrimaryStorage/src/main/java/org/zstack/storage/primary/nfs/NfsPrimaryStorageBackend.java @@ -13,6 +13,7 @@ import org.zstack.header.volume.BatchSyncVolumeSizeOnPrimaryStorageMsg; import org.zstack.header.volume.BatchSyncVolumeSizeOnPrimaryStorageReply; import org.zstack.header.volume.VolumeInventory; +import org.zstack.header.volume.VolumeLuksAgentSpec; import org.zstack.storage.primary.EstimateVolumeTemplateSizeOnPrimaryStorageMsg; import org.zstack.storage.primary.EstimateVolumeTemplateSizeOnPrimaryStorageReply; import org.zstack.storage.primary.PrimaryStorageBase.PhysicalCapacityUsage; @@ -64,7 +65,8 @@ public interface NfsPrimaryStorageBackend { void createMemoryVolume(PrimaryStorageInventory pinv, VolumeInventory volume, ReturnValueCompletion completion); - void instantiateVolume(PrimaryStorageInventory pinv, HostInventory hostInventory, VolumeInventory volume, ReturnValueCompletion complete); + void instantiateVolume(PrimaryStorageInventory pinv, HostInventory hostInventory, VolumeInventory volume, + VolumeLuksAgentSpec volumeLuksAgentSpec, ReturnValueCompletion complete); void deleteImageCache(ImageCacheInventory imageCache); @@ -77,9 +79,15 @@ public interface NfsPrimaryStorageBackend { void resetRootVolumeFromImage(VolumeInventory vol, HostInventory host, ReturnValueCompletion completion); - void createVolumeFromImageCache(PrimaryStorageInventory primaryStorage, ImageInventory image, ImageCacheInventory imageCache, VolumeInventory volume, ReturnValueCompletion completion); + void createVolumeFromImageCache(PrimaryStorageInventory primaryStorage, ImageInventory image, ImageCacheInventory imageCache, + VolumeInventory volume, VolumeLuksAgentSpec volumeLuksAgentSpec, + ReturnValueCompletion completion); - void createImageCacheFromVolumeResource(PrimaryStorageInventory primaryStorage, String volumeResourceInstallPath, ImageInventory image, ReturnValueCompletion completion); + void handle(EncryptVolumeBitsOnPrimaryStorageMsg msg, ReturnValueCompletion completion); + + void createImageCacheFromVolumeResource(PrimaryStorageInventory primaryStorage, String volumeResourceInstallPath, + ImageInventory image, String snapshotUuid, Boolean encrypted, + ReturnValueCompletion completion); void createTemplateFromVolume(PrimaryStorageInventory primaryStorage, VolumeInventory volume, ImageInventory image, ReturnValueCompletion completion); diff --git a/plugin/nfsPrimaryStorage/src/main/java/org/zstack/storage/primary/nfs/NfsPrimaryStorageKVMBackend.java b/plugin/nfsPrimaryStorage/src/main/java/org/zstack/storage/primary/nfs/NfsPrimaryStorageKVMBackend.java index 445497e0871..5d7b0d90e6b 100755 --- a/plugin/nfsPrimaryStorage/src/main/java/org/zstack/storage/primary/nfs/NfsPrimaryStorageKVMBackend.java +++ b/plugin/nfsPrimaryStorage/src/main/java/org/zstack/storage/primary/nfs/NfsPrimaryStorageKVMBackend.java @@ -50,6 +50,8 @@ import org.zstack.storage.primary.*; import org.zstack.storage.primary.PrimaryStorageBase.PhysicalCapacityUsage; import org.zstack.storage.primary.nfs.NfsPrimaryStorageKVMBackendCommands.*; +import org.zstack.storage.encrypt.VolumeEncryptedSecretHelper; +import org.zstack.storage.encrypt.VolumeSnapshotEncryptionHelper; import org.zstack.storage.volume.VolumeErrors; import org.zstack.storage.volume.VolumeSystemTags; import org.zstack.utils.CollectionUtils; @@ -98,6 +100,10 @@ public class NfsPrimaryStorageKVMBackend implements NfsPrimaryStorageBackend, private StorageTrash trash; @Autowired protected ApiTimeoutManager timeoutManager; + @Autowired + private VolumeSnapshotEncryptionHelper snapshotEncryptionHelper; + @Autowired + private VolumeEncryptedSecretHelper volumeEncryptedSecretHelper; public static final String MOUNT_PRIMARY_STORAGE_PATH = "/nfsprimarystorage/mount"; public static final String UNMOUNT_PRIMARY_STORAGE_PATH = "/nfsprimarystorage/unmount"; @@ -132,6 +138,7 @@ public class NfsPrimaryStorageKVMBackend implements NfsPrimaryStorageBackend, public static final String CANCEL_DOWNLOAD_BITS_FROM_KVM_HOST_PATH = "/nfsprimarystorage/kvmhost/download/cancel"; public static final String GET_DOWNLOAD_BITS_FROM_KVM_HOST_PROGRESS_PATH = "/nfsprimarystorage/kvmhost/download/progress"; public static final String CREATE_VOLUME_FROM_TEMPLATE_PATH = "/nfsprimarystorage/sftp/createvolumefromtemplate"; + public static final String ENCRYPT_VOLUME_BITS_PATH = "/nfsprimarystorage/volume/encryptinplace"; public static final String GET_QCOW2_HASH_VALUE_PATH = "/nfsprimarystorage/getqcow2hash"; public static final String WRITE_VM_METADATA_PATH = "/nfsprimarystorage/vm/metadata/write"; @@ -643,6 +650,10 @@ private void createNormalVolumeFromSnapshot(VolumeSnapshotInventory sp, String v cmd.setWorkspaceInstallPath(volPath); cmd.setUuid(inv.getUuid()); cmd.setVolumeUuid(sp.getVolumeUuid()); + VolumeLuksAgentSpec luksSpec = snapshotEncryptionHelper.prepareVolumeSecretMaterial(host.getUuid(), volumeUuid); + if (luksSpec != null && luksSpec.isComplete()) { + cmd.setEncryptLuksSecretMaterialFilePath(luksSpec.getEncryptLuksSecretMaterialFilePath()); + } new KvmCommandSender(host.getUuid()).send(cmd, MERGE_SNAPSHOT_PATH, new KvmCommandFailureChecker() { @Override @@ -685,6 +696,10 @@ private void createIncrementalVolumeFromSnapshot(VolumeSnapshotInventory sp, Str if (volumeSize != null && volumeSize != 0) { cmd.setVirtualSize(volumeSize); } + VolumeLuksAgentSpec luksSpec = snapshotEncryptionHelper.prepareVolumeSecretMaterial(host.getUuid(), volumeUuid); + if (luksSpec != null && luksSpec.isComplete()) { + cmd.setEncryptLuksSecretMaterialFilePath(luksSpec.getEncryptLuksSecretMaterialFilePath()); + } new KvmCommandSender(host.getUuid()).send(cmd, CREATE_VOLUME_WITH_BACKING_PATH, wrapper -> { CreateVolumeWithBackingRsp rsp = wrapper.getResponse(CreateVolumeWithBackingRsp.class); @@ -1098,7 +1113,8 @@ public List call() { } @Override - public void instantiateVolume(final PrimaryStorageInventory pinv, HostInventory hostInventory, final VolumeInventory volume, final ReturnValueCompletion complete) { + public void instantiateVolume(final PrimaryStorageInventory pinv, HostInventory hostInventory, final VolumeInventory volume, + VolumeLuksAgentSpec volumeLuksAgentSpec, final ReturnValueCompletion complete) { String accounUuid = acntMgr.getOwnerAccountUuidOfResource(volume.getUuid()); final CreateEmptyVolumeCmd cmd = new CreateEmptyVolumeCmd(); @@ -1120,6 +1136,10 @@ public void instantiateVolume(final PrimaryStorageInventory pinv, HostInventory throw new CloudRuntimeException(String.format("unknown volume type %s", volume.getType())); } + if (volumeLuksAgentSpec != null && volumeLuksAgentSpec.isComplete()) { + cmd.setEncryptLuksSecretMaterialFilePath(volumeLuksAgentSpec.getEncryptLuksSecretMaterialFilePath()); + } + if (volume.getType().equals(VolumeType.Memory.toString())) { cmd.setWithoutVolume(true); } @@ -1299,6 +1319,12 @@ public void revertVolumeFromSnapshot(VolumeSnapshotInventory sinv, VolumeInvento RevertVolumeFromSnapshotCmd cmd = new RevertVolumeFromSnapshotCmd(); cmd.setSnapshotInstallPath(sinv.getPrimaryStorageInstallPath()); cmd.setUuid(sinv.getPrimaryStorageUuid()); + if (Boolean.TRUE.equals(vol.getEncrypted())) { + VolumeLuksAgentSpec luksSpec = snapshotEncryptionHelper.prepareVolumeSecretMaterial(host.getUuid(), vol.getUuid()); + if (luksSpec != null && luksSpec.isComplete()) { + cmd.setEncryptLuksSecretMaterialFilePath(luksSpec.getEncryptLuksSecretMaterialFilePath()); + } + } KVMHostAsyncHttpCallMsg msg = new KVMHostAsyncHttpCallMsg(); msg.setCommand(cmd); @@ -1336,6 +1362,15 @@ public void resetRootVolumeFromImage(final VolumeInventory vol, final HostInvent cmd.setImagePath(NfsPrimaryStorageKvmHelper.makeCachedImageInstallUrlFromImageUuidForTemplate(psInv, vol.getRootImageUuid())); cmd.setVolumePath(NfsPrimaryStorageKvmHelper.makeRootVolumeInstallUrl(psInv, vol)); cmd.setUuid(vol.getPrimaryStorageUuid()); + if (Boolean.TRUE.equals(vol.getEncrypted())) { + String secretMaterialFilePath = prepareVolumeSecretMaterialPath(host.getUuid(), vol); + if (StringUtils.isBlank(secretMaterialFilePath)) { + completion.fail(operr("cannot prepare LUKS secret for encrypted volume[uuid:%s] reimage on host[uuid:%s]", + vol.getUuid(), host.getUuid())); + return; + } + cmd.setEncryptLuksSecretMaterialFilePath(secretMaterialFilePath); + } KVMHostAsyncHttpCallMsg msg = new KVMHostAsyncHttpCallMsg(); msg.setCommand(cmd); @@ -1364,7 +1399,8 @@ public void run(MessageReply reply) { @Override public void createVolumeFromImageCache(final PrimaryStorageInventory primaryStorage, final ImageInventory image, final ImageCacheInventory imageCache, - final VolumeInventory volume, final ReturnValueCompletion completion) { + final VolumeInventory volume, VolumeLuksAgentSpec volumeLuksAgentSpec, + final ReturnValueCompletion completion) { HostInventory host = nfsFactory.getConnectedHostForOperation(primaryStorage).get(0); final String installPath = StringUtils.isNotEmpty(volume.getInstallPath()) ? volume.getInstallPath() : @@ -1380,6 +1416,9 @@ public void createVolumeFromImageCache(final PrimaryStorageInventory primaryStor if (image.getSize() < volume.getSize()) { cmd.setVirtualSize(volume.getSize()); } + if (volumeLuksAgentSpec != null && volumeLuksAgentSpec.isComplete()) { + cmd.setEncryptLuksSecretMaterialFilePath(volumeLuksAgentSpec.getEncryptLuksSecretMaterialFilePath()); + } KVMHostAsyncHttpCallMsg msg = new KVMHostAsyncHttpCallMsg(); msg.setCommand(cmd); @@ -1410,24 +1449,36 @@ public void run(MessageReply reply) { } @Override - public void createImageCacheFromVolumeResource(PrimaryStorageInventory primaryStorage, String volumeResource, ImageInventory image, ReturnValueCompletion completion) { + public void createImageCacheFromVolumeResource(PrimaryStorageInventory primaryStorage, String volumeResource, + ImageInventory image, String snapshotUuid, Boolean encrypted, + ReturnValueCompletion completion) { final String installPath = NfsPrimaryStorageKvmHelper.makeCachedImageInstallUrl(primaryStorage, image); - doCreateTemplateFromVolume(installPath, primaryStorage, volumeResource, image, completion); + HostInventory host = nfsFactory.getConnectedHostForOperation(primaryStorage).get(0); + VolumeLuksAgentSpec volumeLuksAgentSpec = snapshotEncryptionHelper.prepareTemporarySnapshotImageSecretMaterial( + host.getUuid(), snapshotUuid, image.getUuid(), encrypted); + doCreateTemplateFromVolume(installPath, primaryStorage, volumeResource, image, volumeLuksAgentSpec, host, completion); } @Override public void createTemplateFromVolume(final PrimaryStorageInventory primaryStorage, final VolumeInventory volume, final ImageInventory image, final ReturnValueCompletion completion) { final String installPath = NfsPrimaryStorageKvmHelper.makeTemplateFromVolumeInWorkspacePath(primaryStorage, image.getUuid()); - doCreateTemplateFromVolume(installPath, primaryStorage, volume.getInstallPath(), image, completion); + doCreateTemplateFromVolume(installPath, primaryStorage, volume.getInstallPath(), image, null, null, completion); } - private void doCreateTemplateFromVolume(final String installPath, final PrimaryStorageInventory primaryStorage, final String volumeResourceInstallPath, final ImageInventory image, final ReturnValueCompletion completion) { - final HostInventory destHost = nfsFactory.getConnectedHostForOperation(primaryStorage).get(0); + private void doCreateTemplateFromVolume(final String installPath, final PrimaryStorageInventory primaryStorage, + final String volumeResourceInstallPath, final ImageInventory image, + VolumeLuksAgentSpec volumeLuksAgentSpec, HostInventory selectedHost, + final ReturnValueCompletion completion) { + final HostInventory destHost = selectedHost != null ? selectedHost : + nfsFactory.getConnectedHostForOperation(primaryStorage).get(0); CreateTemplateFromVolumeCmd cmd = new CreateTemplateFromVolumeCmd(); cmd.setInstallPath(installPath); cmd.setVolumePath(volumeResourceInstallPath); cmd.setUuid(primaryStorage.getUuid()); + if (volumeLuksAgentSpec != null && volumeLuksAgentSpec.isComplete()) { + cmd.setEncryptLuksSecretMaterialFilePath(volumeLuksAgentSpec.getEncryptLuksSecretMaterialFilePath()); + } KVMHostAsyncHttpCallMsg msg = new KVMHostAsyncHttpCallMsg(); msg.setCommand(cmd); @@ -1495,6 +1546,10 @@ public void mergeSnapshotToVolume(final PrimaryStorageInventory pinv, VolumeSnap cmd.setSrcPath(snapshot != null ? snapshot.getPrimaryStorageInstallPath() : null); cmd.setDestPath(volume.getInstallPath()); cmd.setUuid(pinv.getUuid()); + cmd.setEncryptLuksSecretMaterialFilePath(prepareVolumeSecretMaterialPath(host.getUuid(), volume)); + if (!cmd.isFullRebase()) { + cmd.setResetBackingLuksSecretMaterialFilePath(prepareVolumeSecretMaterialPath(host.getUuid(), volume)); + } KVMHostAsyncHttpCallMsg msg = new KVMHostAsyncHttpCallMsg(); msg.setCommand(cmd); @@ -1918,6 +1973,25 @@ public void beforeTakeSnapshot(KVMHostInventory host, TakeSnapshotOnHypervisorMs scmd.setVolumeUuid(cmd.getVolumeUuid()); scmd.setInstallUrl(cmd.getInstallPath()); scmd.setBackingFile(cmd.getVolumeInstallPath()); + String secretMaterialFilePath = cmd.getEncryptLuksSecretMaterialFilePath(); + VolumeVO volume = dbf.findByUuid(msg.getVolume().getUuid(), VolumeVO.class); + if (volume != null && volume.isEncrypted()) { + if (StringUtils.isBlank(secretMaterialFilePath)) { + VolumeLuksAgentSpec luksSpec = snapshotEncryptionHelper.prepareVolumeSecretMaterial( + host.getUuid(), volume.getUuid()); + if (luksSpec != null && luksSpec.isComplete()) { + secretMaterialFilePath = luksSpec.getEncryptLuksSecretMaterialFilePath(); + } + } + if (cmd.getVolume() != null) { + cmd.getVolume().setLuksSecretUuid(volumeEncryptedSecretHelper.resolveOrDefineSecretForVolume( + host.getUuid(), volume.getVmInstanceUuid(), volume.getUuid())); + } + } + if (StringUtils.isNotBlank(secretMaterialFilePath)) { + scmd.setEncryptLuksSecretMaterialFilePath(secretMaterialFilePath); + scmd.setVirtualSize(msg.getVolume().getSize()); + } KVMHostAsyncHttpCallMsg smsg = new KVMHostAsyncHttpCallMsg(); smsg.setCommand(scmd); @@ -1992,6 +2066,33 @@ public void fail(ErrorCode errorCode) { }); } + private String prepareVolumeSecretMaterialPath(String hostUuid, VolumeInventory volume) { + if (volume == null || !Boolean.TRUE.equals(volume.getEncrypted())) { + return null; + } + + VolumeLuksAgentSpec luksSpec = snapshotEncryptionHelper.prepareVolumeSecretMaterial(hostUuid, volume.getUuid()); + if (luksSpec == null || !luksSpec.isComplete()) { + return null; + } + return luksSpec.getEncryptLuksSecretMaterialFilePath(); + } + + private List prepareVolumeSecretMaterialPaths(String hostUuid, VolumeInventory volume, int count) { + List secretPaths = new ArrayList<>(); + if (volume == null || !Boolean.TRUE.equals(volume.getEncrypted())) { + return secretPaths; + } + + for (int i = 0; i < count; i++) { + String secretPath = prepareVolumeSecretMaterialPath(hostUuid, volume); + if (secretPath != null) { + secretPaths.add(secretPath); + } + } + return secretPaths; + } + @Override public void pullSnapshot(PullVolumeSnapshotOnPrimaryStorageMsg msg, String hostUuid, ReturnValueCompletion completion) { PullVolumeSnapshotOnPrimaryStorageReply reply = new PullVolumeSnapshotOnPrimaryStorageReply(); @@ -2001,6 +2102,10 @@ public void pullSnapshot(PullVolumeSnapshotOnPrimaryStorageMsg msg, String hostU cmd.setDestPath(msg.getDstSnapshot().getPrimaryStorageInstallPath()); cmd.setUuid(msg.getVolume().getPrimaryStorageUuid()); cmd.setFullRebase(cmd.getSrcPath() == null); + cmd.setEncryptLuksSecretMaterialFilePath(prepareVolumeSecretMaterialPath(hostUuid, msg.getVolume())); + if (!cmd.isFullRebase()) { + cmd.setResetBackingLuksSecretMaterialFilePath(prepareVolumeSecretMaterialPath(hostUuid, msg.getVolume())); + } KVMHostAsyncHttpCallMsg kmsg = new KVMHostAsyncHttpCallMsg(); kmsg.setCommand(cmd); @@ -2034,6 +2139,9 @@ public void commitSnapshot(CommitVolumeSnapshotOnPrimaryStorageMsg msg, String h cmd.base = msg.getDstSnapshot().getPrimaryStorageInstallPath(); cmd.topChildrenInstallPathInDb = msg.getSrcChildrenInstallPathInDb(); cmd.setUuid(msg.getVolume().getPrimaryStorageUuid()); + cmd.encryptLuksSecretMaterialFilePath = prepareVolumeSecretMaterialPath(hostUuid, msg.getVolume()); + cmd.rebaseLuksSecretMaterialFilePaths = prepareVolumeSecretMaterialPaths( + hostUuid, msg.getVolume(), cmd.topChildrenInstallPathInDb == null ? 0 : cmd.topChildrenInstallPathInDb.size()); KVMHostAsyncHttpCallMsg kmsg = new KVMHostAsyncHttpCallMsg(); kmsg.setCommand(cmd); @@ -2230,4 +2338,36 @@ public void run(MessageReply reply) { } }); } + + @Override + public void handle(EncryptVolumeBitsOnPrimaryStorageMsg msg, ReturnValueCompletion completion) { + EncryptVolumeBitsCmd cmd = new EncryptVolumeBitsCmd(); + cmd.setUuid(msg.getPrimaryStorageUuid()); + cmd.installPath = msg.getInstallPath(); + cmd.encryptLuksSecretMaterialFilePath = msg.getEncryptLuksSecretMaterialFilePath(); + + KVMHostAsyncHttpCallMsg hmsg = new KVMHostAsyncHttpCallMsg(); + hmsg.setCommand(cmd); + hmsg.setPath(ENCRYPT_VOLUME_BITS_PATH); + hmsg.setHostUuid(msg.getHostUuid()); + bus.makeTargetServiceIdByResourceUuid(hmsg, HostConstant.SERVICE_ID, msg.getHostUuid()); + bus.send(hmsg, new CloudBusCallBack(completion) { + @Override + public void run(MessageReply reply) { + if (!reply.isSuccess()) { + completion.fail(reply.getError()); + return; + } + + EncryptVolumeBitsRsp rsp = ((KVMHostAsyncHttpCallReply) reply).toResponse(EncryptVolumeBitsRsp.class); + if (!rsp.isSuccess()) { + completion.fail(operr("failed to encrypt volume[uuid:%s] bits at path[%s] on host[uuid:%s]: %s", + msg.getVolumeUuid(), msg.getInstallPath(), msg.getHostUuid(), rsp.getError())); + return; + } + + completion.success(new EncryptVolumeBitsOnPrimaryStorageReply()); + } + }); + } } diff --git a/plugin/nfsPrimaryStorage/src/main/java/org/zstack/storage/primary/nfs/NfsPrimaryStorageKVMBackendCommands.java b/plugin/nfsPrimaryStorage/src/main/java/org/zstack/storage/primary/nfs/NfsPrimaryStorageKVMBackendCommands.java index 0901af4cadb..3feeacf494a 100755 --- a/plugin/nfsPrimaryStorage/src/main/java/org/zstack/storage/primary/nfs/NfsPrimaryStorageKVMBackendCommands.java +++ b/plugin/nfsPrimaryStorage/src/main/java/org/zstack/storage/primary/nfs/NfsPrimaryStorageKVMBackendCommands.java @@ -145,6 +145,7 @@ public static class UnmountResponse extends NfsPrimaryStorageAgentResponse { public static class CreateTemplateFromVolumeCmd extends NfsPrimaryStorageAgentCommand implements HasThreadContext{ private String installPath; private String rootVolumePath; + private String encryptLuksSecretMaterialFilePath; public String getInstallPath() { return installPath; @@ -159,6 +160,12 @@ public String getRootVolumePath() { public void setVolumePath(String rootVolumePath) { this.rootVolumePath = rootVolumePath; } + public String getEncryptLuksSecretMaterialFilePath() { + return encryptLuksSecretMaterialFilePath; + } + public void setEncryptLuksSecretMaterialFilePath(String encryptLuksSecretMaterialFilePath) { + this.encryptLuksSecretMaterialFilePath = encryptLuksSecretMaterialFilePath; + } } public static class CreateTemplateFromVolumeRsp extends NfsPrimaryStorageAgentResponse { private long size; @@ -366,6 +373,16 @@ public long getVirtualSize() { public void setVirtualSize(long virtualSize) { this.virtualSize = virtualSize; } + + private String encryptLuksSecretMaterialFilePath; + + public String getEncryptLuksSecretMaterialFilePath() { + return encryptLuksSecretMaterialFilePath; + } + + public void setEncryptLuksSecretMaterialFilePath(String encryptLuksSecretMaterialFilePath) { + this.encryptLuksSecretMaterialFilePath = encryptLuksSecretMaterialFilePath; + } } public static class CreateRootVolumeFromTemplateCmd extends CreateVolumeCmd { @@ -453,6 +470,14 @@ public static class CreateEmptyVolumeResponse extends NfsPrimaryStorageAgentResp public Long size; } + public static class EncryptVolumeBitsCmd extends NfsPrimaryStorageAgentCommand { + public String installPath; + public String encryptLuksSecretMaterialFilePath; + } + + public static class EncryptVolumeBitsRsp extends NfsPrimaryStorageAgentResponse { + } + public static class DeleteCmd extends NfsPrimaryStorageAgentCommand { private boolean folder; private String installPath; @@ -512,6 +537,7 @@ public void setPaths(List paths) { public static class RevertVolumeFromSnapshotCmd extends NfsPrimaryStorageAgentCommand { private String snapshotInstallPath; + private String encryptLuksSecretMaterialFilePath; public String getSnapshotInstallPath() { return snapshotInstallPath; @@ -520,6 +546,14 @@ public String getSnapshotInstallPath() { public void setSnapshotInstallPath(String snapshotInstallPath) { this.snapshotInstallPath = snapshotInstallPath; } + + public String getEncryptLuksSecretMaterialFilePath() { + return encryptLuksSecretMaterialFilePath; + } + + public void setEncryptLuksSecretMaterialFilePath(String encryptLuksSecretMaterialFilePath) { + this.encryptLuksSecretMaterialFilePath = encryptLuksSecretMaterialFilePath; + } } public static class RevertVolumeFromSnapshotResponse extends NfsPrimaryStorageAgentResponse { @@ -549,6 +583,7 @@ public void setSize(long size) { public static class ReInitImageCmd extends NfsPrimaryStorageAgentCommand { private String imagePath; private String volumePath; + private String encryptLuksSecretMaterialFilePath; public String getImagePath() { return imagePath; @@ -565,6 +600,14 @@ public String getVolumePath() { public void setVolumePath(String volumePath) { this.volumePath = volumePath; } + + public String getEncryptLuksSecretMaterialFilePath() { + return encryptLuksSecretMaterialFilePath; + } + + public void setEncryptLuksSecretMaterialFilePath(String encryptLuksSecretMaterialFilePath) { + this.encryptLuksSecretMaterialFilePath = encryptLuksSecretMaterialFilePath; + } } public static class ReInitImageRsp extends NfsPrimaryStorageAgentResponse { @@ -639,6 +682,7 @@ public static class MergeSnapshotCmd extends NfsPrimaryStorageAgentCommand imple private String volumeUuid; private String snapshotInstallPath; private String workspaceInstallPath; + private String encryptLuksSecretMaterialFilePath; public String getVolumeUuid() { return volumeUuid; @@ -663,6 +707,14 @@ public String getWorkspaceInstallPath() { public void setWorkspaceInstallPath(String workspaceInstallPath) { this.workspaceInstallPath = workspaceInstallPath; } + + public String getEncryptLuksSecretMaterialFilePath() { + return encryptLuksSecretMaterialFilePath; + } + + public void setEncryptLuksSecretMaterialFilePath(String encryptLuksSecretMaterialFilePath) { + this.encryptLuksSecretMaterialFilePath = encryptLuksSecretMaterialFilePath; + } } public static class MergeSnapshotResponse extends NfsPrimaryStorageAgentResponse { @@ -765,6 +817,8 @@ public static class OfflineMergeSnapshotCmd extends NfsPrimaryStorageAgentComman private String srcPath; private String destPath; private boolean fullRebase; + private String encryptLuksSecretMaterialFilePath; + private String resetBackingLuksSecretMaterialFilePath; public boolean isFullRebase() { return fullRebase; @@ -789,6 +843,22 @@ public String getDestPath() { public void setDestPath(String destPath) { this.destPath = destPath; } + + public String getEncryptLuksSecretMaterialFilePath() { + return encryptLuksSecretMaterialFilePath; + } + + public void setEncryptLuksSecretMaterialFilePath(String encryptLuksSecretMaterialFilePath) { + this.encryptLuksSecretMaterialFilePath = encryptLuksSecretMaterialFilePath; + } + + public String getResetBackingLuksSecretMaterialFilePath() { + return resetBackingLuksSecretMaterialFilePath; + } + + public void setResetBackingLuksSecretMaterialFilePath(String resetBackingLuksSecretMaterialFilePath) { + this.resetBackingLuksSecretMaterialFilePath = resetBackingLuksSecretMaterialFilePath; + } } public static class OfflineMergeSnapshotRsp extends NfsPrimaryStorageAgentResponse { @@ -807,6 +877,8 @@ public static class OfflineCommitSnapshotCmd extends NfsPrimaryStorageAgentComma public String top; public String base; public List topChildrenInstallPathInDb = new ArrayList<>(); + public String encryptLuksSecretMaterialFilePath; + public List rebaseLuksSecretMaterialFilePaths = new ArrayList<>(); } public static class OfflineCommitSnapshotRsp extends NfsPrimaryStorageAgentResponse { diff --git a/storage/pom.xml b/storage/pom.xml old mode 100755 new mode 100644 index 678ec872184..17d8ee34d52 --- a/storage/pom.xml +++ b/storage/pom.xml @@ -3,7 +3,7 @@ zstack org.zstack - 5.0.0 + 5.0.0 .. storage @@ -54,6 +54,11 @@ longjob ${project.version} + + org.zstack + kvm + ${project.version} + diff --git a/storage/src/main/java/org/zstack/storage/encrypt/DummyVolumeEncryptedResourceKeyBackend.java b/storage/src/main/java/org/zstack/storage/encrypt/DummyVolumeEncryptedResourceKeyBackend.java new file mode 100644 index 00000000000..77166fa0393 --- /dev/null +++ b/storage/src/main/java/org/zstack/storage/encrypt/DummyVolumeEncryptedResourceKeyBackend.java @@ -0,0 +1,103 @@ +package org.zstack.storage.encrypt; + +import org.zstack.utils.Utils; +import org.zstack.utils.logging.CLogger; + +/** + * OSS / no-premium-crypto: no-op volume key-provider persistence, same role as + * {@link org.zstack.compute.vm.devices.DummyTpmEncryptedResourceKeyBackend}. + */ +public class DummyVolumeEncryptedResourceKeyBackend implements VolumeEncryptedResourceKeyBackend { + private static final CLogger logger = Utils.getLogger(DummyVolumeEncryptedResourceKeyBackend.class); + + @Override + public void attachKeyProviderToVolume(String volumeUuid, String keyProviderUuid) { + logger.debug(String.format("ignore attach key provider to volume[uuid:%s] keyProviderUuid:%s", + volumeUuid, keyProviderUuid)); + } + + @Override + public void detachKeyProviderFromVolume(String volumeUuid) { + logger.debug(String.format("ignore detach key provider from volume[uuid:%s]", volumeUuid)); + } + + @Override + public void detachKeyProviderFromSnapshot(String snapshotUuid) { + logger.debug(String.format("ignore detach key provider from snapshot[uuid:%s]", snapshotUuid)); + } + + @Override + public void detachKeyProviderFromTemporarySnapshotImage(String imageUuid) { + logger.debug(String.format("ignore detach key provider from temporary snapshot image[uuid:%s]", imageUuid)); + } + + @Override + public String findKeyProviderUuidByVolume(String volumeUuid) { + return null; + } + + @Override + public boolean checkVolumeKeyProviderAttached(String volumeUuid) { + return false; + } + + @Override + public boolean checkSnapshotKeyProviderAttached(String snapshotUuid) { + return false; + } + + @Override + public boolean checkTemporarySnapshotImageKeyProviderAttached(String imageUuid) { + return false; + } + + @Override + public void copyVolumeKeyToSnapshot(String volumeUuid, String snapshotUuid) { + logger.debug(String.format("ignore copy volume[uuid:%s] key to snapshot[uuid:%s]", volumeUuid, snapshotUuid)); + } + + @Override + public void copySnapshotKeyToVolume(String snapshotUuid, String volumeUuid) { + logger.debug(String.format("ignore copy snapshot[uuid:%s] key to volume[uuid:%s]", snapshotUuid, volumeUuid)); + } + + @Override + public void copyVolumeKeyToVolume(String srcVolumeUuid, String dstVolumeUuid) { + logger.debug(String.format("ignore copy volume[uuid:%s] key to volume[uuid:%s]", srcVolumeUuid, dstVolumeUuid)); + } + + @Override + public void copySnapshotKeyToTemporarySnapshotImage(String snapshotUuid, String imageUuid) { + logger.debug(String.format("ignore copy snapshot[uuid:%s] key to temporary snapshot image[uuid:%s]", snapshotUuid, imageUuid)); + } + + @Override + public void copyTemporarySnapshotImageKeyToVolume(String imageUuid, String volumeUuid) { + logger.debug(String.format("ignore copy temporary snapshot image[uuid:%s] key to volume[uuid:%s]", imageUuid, volumeUuid)); + } + + @Override + public String defaultKeyProviderUuid() { + return null; + } + + @Override + public String findKeyProviderUuidBySnapshot(String snapshotUuid) { + return null; + } + + @Override + public String findKeyProviderUuidByTemporarySnapshotImage(String imageUuid) { + return null; + } + + @Override + public Integer findKeyVersionByVolume(String volumeUuid) { + return null; + } + + @Override + public Integer findKeyVersionBySnapshot(String snapshotUuid) { + return null; + } +} diff --git a/storage/src/main/java/org/zstack/storage/encrypt/SnapshotGroupRevertVolumeEncryptionHelper.java b/storage/src/main/java/org/zstack/storage/encrypt/SnapshotGroupRevertVolumeEncryptionHelper.java new file mode 100644 index 00000000000..edffb520a7d --- /dev/null +++ b/storage/src/main/java/org/zstack/storage/encrypt/SnapshotGroupRevertVolumeEncryptionHelper.java @@ -0,0 +1,110 @@ +package org.zstack.storage.encrypt; + +import org.springframework.beans.factory.annotation.Autowire; +import org.springframework.beans.factory.annotation.Configurable; +import org.zstack.core.Platform; +import org.zstack.core.db.Q; +import org.zstack.header.errorcode.ErrorCode; +import org.zstack.header.storage.snapshot.VolumeSnapshotVO; +import org.zstack.header.storage.snapshot.group.VolumeSnapshotGroupRefVO; +import org.zstack.header.storage.snapshot.group.VolumeSnapshotGroupRefVO_; +import org.zstack.header.vm.APICreateVmInstanceFromVolumeSnapshotGroupMsg; +import org.zstack.header.vm.CreateVmInstanceMsg; +import org.zstack.header.volume.CreateDataVolumeFromVolumeSnapshotMsg; +import org.zstack.header.volume.VolumeType; + +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +@Configurable(preConstruction = true, autowire = Autowire.BY_TYPE) +public class SnapshotGroupRevertVolumeEncryptionHelper { + public ErrorCode validateVolumeSnapshotEncryption(APICreateVmInstanceFromVolumeSnapshotGroupMsg msg, + List effectiveSnapshots) { + Map volumeSnapshotEncryption; + try { + volumeSnapshotEncryption = getVolumeSnapshotEncryption(msg); + } catch (IllegalArgumentException e) { + return Platform.operr(e.getMessage()); + } + if (volumeSnapshotEncryption == null || volumeSnapshotEncryption.isEmpty()) { + return Platform.operr("volumeSnapshotEncryptions must specify every effective volume snapshot in volume snapshot group[uuid:%s]", + msg.getVolumeSnapshotGroupUuid()); + } + + Set effectiveSnapshotUuids = new HashSet<>(); + for (VolumeSnapshotVO snapshot : effectiveSnapshots) { + effectiveSnapshotUuids.add(snapshot.getUuid()); + } + + Set unexpectedSnapshotUuids = new HashSet<>(volumeSnapshotEncryption.keySet()); + unexpectedSnapshotUuids.removeAll(effectiveSnapshotUuids); + if (!unexpectedSnapshotUuids.isEmpty()) { + return Platform.operr("volumeSnapshotEncryptions contain volume snapshot(s)%s that do not belong to volume snapshot group[uuid:%s]", + unexpectedSnapshotUuids, msg.getVolumeSnapshotGroupUuid()); + } + + Set missingSnapshotUuids = new HashSet<>(effectiveSnapshotUuids); + missingSnapshotUuids.removeAll(volumeSnapshotEncryption.keySet()); + if (!missingSnapshotUuids.isEmpty()) { + return Platform.operr("volumeSnapshotEncryptions miss effective volume snapshot(s)%s in volume snapshot group[uuid:%s]", + missingSnapshotUuids, msg.getVolumeSnapshotGroupUuid()); + } + + for (VolumeSnapshotVO snapshot : effectiveSnapshots) { + if (snapshot.isEncrypted() && !Boolean.TRUE.equals(volumeSnapshotEncryption.get(snapshot.getUuid()))) { + return Platform.operr("volume snapshot[uuid:%s] in volume snapshot group[uuid:%s] is encrypted, cannot create an unencrypted volume from it", + snapshot.getUuid(), msg.getVolumeSnapshotGroupUuid()); + } + } + + return null; + } + + public void setupRootVolumeFromApi(APICreateVmInstanceFromVolumeSnapshotGroupMsg apiMsg, + CreateVmInstanceMsg cmsg) { + Map volumeSnapshotEncryption = getVolumeSnapshotEncryption(apiMsg); + if (volumeSnapshotEncryption == null || volumeSnapshotEncryption.isEmpty()) { + return; + } + if (cmsg.getRootDisk() == null) { + return; + } + + String rootSnapshotUuid = Q.New(VolumeSnapshotGroupRefVO.class).select(VolumeSnapshotGroupRefVO_.volumeSnapshotUuid) + .eq(VolumeSnapshotGroupRefVO_.volumeType, VolumeType.Root.toString()) + .eq(VolumeSnapshotGroupRefVO_.volumeSnapshotGroupUuid, apiMsg.getVolumeSnapshotGroupUuid()) + .findValue(); + cmsg.getRootDisk().setEncrypted(volumeSnapshotEncryption.get(rootSnapshotUuid)); + } + + public void setupDataVolumeFromApi(APICreateVmInstanceFromVolumeSnapshotGroupMsg apiMsg, + VolumeSnapshotVO snapshot, + CreateDataVolumeFromVolumeSnapshotMsg cmsg) { + Map volumeSnapshotEncryption = getVolumeSnapshotEncryption(apiMsg); + cmsg.setEncrypted(volumeSnapshotEncryption == null ? null : volumeSnapshotEncryption.get(snapshot.getUuid())); + } + + private Map getVolumeSnapshotEncryption(APICreateVmInstanceFromVolumeSnapshotGroupMsg msg) { + List volumeSnapshotEncryptions = + msg.getVolumeSnapshotEncryptions(); + if (volumeSnapshotEncryptions == null || volumeSnapshotEncryptions.isEmpty()) { + return null; + } + + Map ret = new HashMap<>(); + for (APICreateVmInstanceFromVolumeSnapshotGroupMsg.VolumeSnapshotEncryption volumeSnapshotEncryption : + volumeSnapshotEncryptions) { + if (volumeSnapshotEncryption.getVolumeSnapshotUuid() == null || volumeSnapshotEncryption.getEncrypted() == null) { + throw new IllegalArgumentException(String.format( + "invalid volumeSnapshotEncryptions item[%s], expected {\"volumeSnapshotUuid\":\"snapshotUuid\",\"encrypted\":true}", + volumeSnapshotEncryption)); + } + ret.put(volumeSnapshotEncryption.getVolumeSnapshotUuid(), volumeSnapshotEncryption.getEncrypted()); + } + + return ret; + } +} diff --git a/storage/src/main/java/org/zstack/storage/encrypt/VolumeEncryptedAttachExtension.java b/storage/src/main/java/org/zstack/storage/encrypt/VolumeEncryptedAttachExtension.java new file mode 100644 index 00000000000..59b3c9cea8c --- /dev/null +++ b/storage/src/main/java/org/zstack/storage/encrypt/VolumeEncryptedAttachExtension.java @@ -0,0 +1,74 @@ +package org.zstack.storage.encrypt; + +import org.springframework.beans.factory.annotation.Autowire; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Configurable; +import org.zstack.header.errorcode.ErrorCode; +import org.zstack.header.vm.VmInstanceInventory; +import org.zstack.header.volume.VolumeInventory; +import org.zstack.kvm.KVMAgentCommands.AttachDataVolumeCmd; +import org.zstack.kvm.KVMAttachVolumeExtensionPoint; +import org.zstack.kvm.KVMHostInventory; +import org.zstack.kvm.VolumeTO; +import org.zstack.utils.Utils; +import org.zstack.utils.logging.CLogger; + +import java.util.Map; + +/** + * On hot-attach of a LUKS-encrypted data volume, ensure the per-volume libvirt + * secret exists on the destination host and stamp its UUID onto the + * {@link VolumeTO} so {@code vm_plugin.filebased_volume} (the attach builder + * at {@code vm_plugin.py:3185}) can emit + * {@code }. + * + *

Without this the agent issues a {@code blockdev-add} without + * {@code encrypt.key-secret} and qemu aborts with + * {@code "Parameter 'encrypt.key-secret' is required for cipher"}. + * + *

Lives in the storage module (where the helper sits) and registers via + * {@link KVMAttachVolumeExtensionPoint} (the existing hook KVMHost.attachVolume + * already fires) to avoid creating a storage -> kvm reverse dep. + */ +@Configurable(preConstruction = true, autowire = Autowire.BY_TYPE) +public class VolumeEncryptedAttachExtension implements KVMAttachVolumeExtensionPoint { + + private static final CLogger logger = Utils.getLogger(VolumeEncryptedAttachExtension.class); + + @Autowired + private VolumeEncryptedSecretHelper secretHelper; + + @Override + public VolumeTO convertVolumeIfNeed(KVMHostInventory host, VolumeInventory inventory, VolumeTO to) { + return to; + } + + @Override + public void beforeAttachVolume(KVMHostInventory host, VmInstanceInventory vm, VolumeInventory volume, + AttachDataVolumeCmd cmd, Map data) { + if (!Boolean.TRUE.equals(volume.getEncrypted())) { + return; + } + String hostUuid = host.getUuid(); + String vmUuid = vm.getUuid(); + String volUuid = volume.getUuid(); + String secretUuid = secretHelper.resolveOrDefineSecretForVolume(hostUuid, vmUuid, volUuid); + VolumeTO to = cmd.getVolume(); + if (to != null) { + to.setLuksSecretUuid(secretUuid); + } + logger.debug(String.format( + "LUKS-ATTACH-EXT stamped secret %s onto attach cmd for volume[uuid:%s] on host[uuid:%s]", + secretUuid, volUuid, hostUuid)); + } + + @Override + public void afterAttachVolume(KVMHostInventory host, VmInstanceInventory vm, VolumeInventory volume, + AttachDataVolumeCmd cmd) { + } + + @Override + public void attachVolumeFailed(KVMHostInventory host, VmInstanceInventory vm, VolumeInventory volume, + AttachDataVolumeCmd cmd, ErrorCode err, Map data) { + } +} diff --git a/storage/src/main/java/org/zstack/storage/encrypt/VolumeEncryptedExpungeExtension.java b/storage/src/main/java/org/zstack/storage/encrypt/VolumeEncryptedExpungeExtension.java new file mode 100644 index 00000000000..d70e8b51626 --- /dev/null +++ b/storage/src/main/java/org/zstack/storage/encrypt/VolumeEncryptedExpungeExtension.java @@ -0,0 +1,151 @@ +package org.zstack.storage.encrypt; + +import org.apache.commons.lang.StringUtils; +import org.springframework.beans.factory.annotation.Autowire; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Configurable; +import org.zstack.core.db.Q; +import org.zstack.header.vm.VmInstanceVO; +import org.zstack.header.vm.VmInstanceVO_; +import org.zstack.header.volume.VolumeInventory; +import org.zstack.header.volume.VolumeJustBeforeDeleteFromDbExtensionPoint; +import org.zstack.header.volume.VolumeVO; +import org.zstack.storage.volume.VolumeSystemTags; +import org.zstack.utils.Utils; +import org.zstack.utils.logging.CLogger; + +import java.util.List; + +/** + * Volume expunge cleanup for LUKS-encrypted volumes. + * + *

Mirrors the {@code VmJustBeforeDeleteFromDb} / {@code VmAfterExpunge} + * pattern used by vTPM in {@code KvmTpmExtensions}, but at the volume layer: + * + *

    + *
  • Trigger: {@link VolumeJustBeforeDeleteFromDbExtensionPoint#volumeJustBeforeDeleteFromDb} + * — runs at the tail of {@code VolumeBase} expunge, after the agent + * has destroyed the on-disk bits, just before the {@code VolumeVO} + * row is removed.
    • + *
  • Not triggered on: VM destroy with {@code Delay} policy (volume + * lives on, parked in recycle bin); VM destroy with {@code KeepVolume} + * (data volume preserved); volume detach without delete. In all these + * cases the encryption metadata must stay so the volume can be + * attached / restored later.
    • + *
+ * + *

Cleanup actions, in order: + *

    + *
  1. Look up {@code keyVersion} + host while the binding is still in DB.
    2. + *
  2. Best-effort delete the libvirt secret on the host (if we can locate + * one — bound only when the volume is attached to a VM whose + * host/lastHost is known).
    3. + *
  3. Delete the {@code EncryptedResourceKeyRefVO} row — this is the + * persistent piece, and the one that absolutely must be cleared so + * a future volume reusing the same uuid doesn't inherit a stale + * binding.
    4. + *
+ * + *

Order matters: keyVersion lives on the ref row, so we must read it + * before {@code detachKeyProviderFromVolume} wipes the row. + */ +@Configurable(preConstruction = true, autowire = Autowire.BY_TYPE) +public class VolumeEncryptedExpungeExtension implements VolumeJustBeforeDeleteFromDbExtensionPoint { + + private static final CLogger logger = Utils.getLogger(VolumeEncryptedExpungeExtension.class); + + @Autowired + private VolumeEncryptedResourceKeyBackend volumeEncryptedResourceKeyBackend; + @Autowired + private VolumeEncryptedSecretHelper secretHelper; + + @Override + public void volumeJustBeforeDeleteFromDb(VolumeInventory inv) { + if (inv == null || !Boolean.TRUE.equals(inv.getEncrypted())) { + return; + } + String volUuid = inv.getUuid(); + + // Snapshot keyVersion BEFORE detaching the binding, otherwise the + // host-side cleanup loses the key it needs to identify the secret. + Integer keyVersion = volumeEncryptedResourceKeyBackend.findKeyVersionByVolume(volUuid); + + String hostUuid = resolveHostUuidFromTag(volUuid); + if (StringUtils.isBlank(hostUuid)) { + String vmUuid = StringUtils.defaultIfBlank(inv.getVmInstanceUuid(), inv.getLastVmInstanceUuid()); + hostUuid = resolveHostUuidFromVm(vmUuid); + } + + // vmUuid passed to the key-agent's DeleteSecret RPC is a contractual + // leftover (the per-volume secret usage name doesn't actually need it + // post Method-D); pass whatever we can reconstruct, or fall back to + // the volume uuid as a stable placeholder so validation passes. + String vmUuidForRpc = StringUtils.defaultIfBlank( + StringUtils.defaultIfBlank(inv.getVmInstanceUuid(), inv.getLastVmInstanceUuid()), + volUuid); + + if (StringUtils.isNotBlank(hostUuid) && keyVersion != null) { + try { + secretHelper.deleteSecretOnHostBestEffort(hostUuid, vmUuidForRpc, volUuid, keyVersion); + } catch (RuntimeException e) { + // helper is best-effort, but guard against unchecked throws + // so we still get to the DB cleanup below. + logger.warn(String.format( + "ignoring failure to delete libvirt LUKS secret for volume[uuid:%s] on host[uuid:%s]: %s", + volUuid, hostUuid, e.getMessage())); + } + } else { + logger.debug(String.format( + "skip host-side libvirt secret cleanup for volume[uuid:%s]:" + + " hostUuid=%s keyVersion=%s", + volUuid, hostUuid, keyVersion)); + } + + try { + volumeEncryptedResourceKeyBackend.detachKeyProviderFromVolume(volUuid); + } catch (RuntimeException e) { + logger.warn(String.format( + "failed to detach EncryptedResourceKeyRefVO for volume[uuid:%s] on expunge: %s", + volUuid, e.getMessage())); + } + } + + /** + * Primary host lookup: read the {@code VOLUME_LIBVIRT_SECRET_HOST} systemtag + * that {@code VolumeEncryptedSecretHelper.defineLibvirtSecretOnHost} stamps + * after every successful libvirt secret define. The tag is non-inherent and + * lives on the {@code VolumeVO} row, so it survives anything short of the + * volume itself being deleted. + */ + private String resolveHostUuidFromTag(String volUuid) { + List tags = VolumeSystemTags.VOLUME_LIBVIRT_SECRET_HOST.getTags(volUuid, VolumeVO.class); + if (tags == null || tags.isEmpty()) { + return null; + } + return VolumeSystemTags.VOLUME_LIBVIRT_SECRET_HOST.getTokenByTag( + tags.get(0), VolumeSystemTags.VOLUME_LIBVIRT_SECRET_HOST_TOKEN); + } + + /** + * Fallback host lookup for volumes created before the + * {@code VOLUME_LIBVIRT_SECRET_HOST} tag mechanism existed. Walks the + * vmInstanceUuid / lastVmInstanceUuid → VmInstanceVO.hostUuid / + * lastHostUuid chain. + */ + private String resolveHostUuidFromVm(String vmUuid) { + if (StringUtils.isBlank(vmUuid)) { + return null; + } + String host = Q.New(VmInstanceVO.class) + .eq(VmInstanceVO_.uuid, vmUuid) + .select(VmInstanceVO_.hostUuid) + .findValue(); + if (StringUtils.isNotBlank(host)) { + return host; + } + return Q.New(VmInstanceVO.class) + .eq(VmInstanceVO_.uuid, vmUuid) + .select(VmInstanceVO_.lastHostUuid) + .findValue(); + } +} diff --git a/storage/src/main/java/org/zstack/storage/encrypt/VolumeEncryptedInitialExtension.java b/storage/src/main/java/org/zstack/storage/encrypt/VolumeEncryptedInitialExtension.java new file mode 100644 index 00000000000..24c6b9b78b6 --- /dev/null +++ b/storage/src/main/java/org/zstack/storage/encrypt/VolumeEncryptedInitialExtension.java @@ -0,0 +1,168 @@ +package org.zstack.storage.encrypt; + +import org.apache.commons.lang.StringUtils; +import org.springframework.beans.factory.annotation.Autowire; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Configurable; +import org.zstack.core.db.DatabaseFacade; +import org.zstack.header.errorcode.OperationFailureException; +import org.zstack.header.host.HostInventory; +import org.zstack.header.keyprovider.EncryptedResourceKeyManager; +import org.zstack.header.storage.primary.InstantiateVolumeOnPrimaryStorageMsg; +import org.zstack.header.volume.AfterInstantiateVolumeExtensionPoint; +import org.zstack.header.volume.CreateDataVolumeExtensionPoint; +import org.zstack.header.volume.InstantiateVolumeMsg; +import org.zstack.header.volume.InstantiateTemporaryRootVolumeMsg; +import org.zstack.header.volume.PreInstantiateVolumeExtensionPoint; +import org.zstack.header.volume.VolumeCreateMessage; +import org.zstack.header.volume.VolumeInventory; +import org.zstack.header.volume.VolumeLuksAgentSpec; +import org.zstack.header.volume.VolumeVO; + +import static org.zstack.core.Platform.operr; + +/** + * Encrypted volume instantiate: {@link #preInstantiateVolume} prepares host LUKS secret material file; + * {@link #afterInstantiateVolume} defines the libvirt secret on the host (needs DEK again after async PS step). + */ +@Configurable(preConstruction = true, autowire = Autowire.BY_TYPE) +public class VolumeEncryptedInitialExtension implements PreInstantiateVolumeExtensionPoint, + AfterInstantiateVolumeExtensionPoint, CreateDataVolumeExtensionPoint { + @Autowired + private DatabaseFacade dbf; + @Autowired + private VolumeEncryptedResourceKeyBackend volumeEncryptedResourceKeyBackend; + @Autowired + private VolumeEncryptedSecretHelper secretHelper; + @Autowired + private VolumeSnapshotEncryptionHelper snapshotEncryptionHelper; + + @Override + public void preInstantiateVolume(InstantiateVolumeMsg msg) { + String hostUuid = msg.getHostUuid(); + if (StringUtils.isBlank(hostUuid)) { + return; + } + + VolumeLuksAgentSpec spec = new VolumeLuksAgentSpec(); + msg.setVolumeLuksAgentSpec(spec); + + String volUuid = msg.getVolumeUuid(); + VolumeVO volume = dbf.findByUuid(volUuid, VolumeVO.class); + + if (volume != null && volume.isEncrypted()) { + // Temporary root images created from snapshots own the image cache before + // the final root volume exists. Persisting the cache key on ImageVO lets + // the root volume inherit the same key, so its top layer can match the + // encrypted image cache. + snapshotEncryptionHelper.inheritFromTemporarySnapshotImageKeyIfPossible(volume); + inheritTemporaryRootVolumeKeyFromOrigin(msg, volume); + String kpUuid = volumeEncryptedResourceKeyBackend.findKeyProviderUuidByVolume(volUuid); + if (StringUtils.isBlank(kpUuid)) { + kpUuid = volumeEncryptedResourceKeyBackend.defaultKeyProviderUuid(); + if (StringUtils.isBlank(kpUuid)) { + throw new OperationFailureException(operr( + "encrypted volume[uuid:%s] has no key provider binding and no default key provider configured", + volUuid)); + } + volumeEncryptedResourceKeyBackend.attachKeyProviderToVolume(volUuid, kpUuid); + } + + EncryptedResourceKeyManager.ResourceKeyResult keyResult = secretHelper.materializeDek(volUuid, kpUuid); + String dekBase64 = keyResult.getDekBase64(); + if (StringUtils.isBlank(dekBase64)) { + throw new OperationFailureException(operr( + "encrypted volume[uuid:%s]: key manager returned empty DEK after materialization", + volUuid)); + } + + String secFilePath = secretHelper.ensureLuksSecretFileOnHost(hostUuid, volUuid, dekBase64); + spec.setEncryptLuksSecretMaterialFilePath(secFilePath); + } + } + + private void inheritTemporaryRootVolumeKeyFromOrigin(InstantiateVolumeMsg msg, VolumeVO volume) { + if (!(msg instanceof InstantiateTemporaryRootVolumeMsg) || volume == null || !volume.isEncrypted()) { + return; + } + + String originVolumeUuid = ((InstantiateTemporaryRootVolumeMsg) msg).getOriginVolumeUuid(); + if (StringUtils.isBlank(originVolumeUuid)) { + return; + } + + VolumeVO originVolume = dbf.findByUuid(originVolumeUuid, VolumeVO.class); + if (originVolume == null || !originVolume.isEncrypted()) { + return; + } + + if (!volumeEncryptedResourceKeyBackend.checkVolumeKeyProviderAttached(originVolumeUuid)) { + throw new OperationFailureException(operr( + "encrypted origin root volume[uuid:%s] has no key provider binding for temporary root volume[uuid:%s]", + originVolumeUuid, volume.getUuid())); + } + + volumeEncryptedResourceKeyBackend.copyVolumeKeyToVolume(originVolumeUuid, volume.getUuid()); + } + + @Override + public void afterInstantiateVolume(InstantiateVolumeOnPrimaryStorageMsg msg) { + VolumeInventory volInv = msg.getVolume(); + if (volInv == null || !Boolean.TRUE.equals(volInv.getEncrypted())) { + return; + } + String volUuid = volInv.getUuid(); + VolumeLuksAgentSpec spec = msg.getVolumeLuksAgentSpec(); + if (spec == null || StringUtils.isBlank(spec.getEncryptLuksSecretMaterialFilePath())) { + return; + } + if (StringUtils.isNotBlank(spec.getEncryptSecretUuid())) { + return; + } + HostInventory destHost = msg.getDestHost(); + if (destHost == null || StringUtils.isBlank(destHost.getUuid())) { + return; + } + + VolumeVO volume = dbf.findByUuid(volUuid, VolumeVO.class); + if (volume == null || !volume.isEncrypted()) { + return; + } + // VolumeInventory already carries vmInstanceUuid when present, so we + // skip the extra select(VolumeVO_.vmInstanceUuid) round-trip. + // VmInstantiateOtherDiskFlow's "create empty data volume" path + // (setupCreateVolumeFromDiskSizeFlows) does NOT set + // VolumeVO.vmInstanceUuid before this hook fires — vmInstanceUuid is + // backfilled only after the volume is attached, well after this + // afterInstantiateVolume runs. Without vmUuid we cannot key the + // libvirt secret (SecretHostDefineMsg requires it), so skip the + // early-define here; VolumeEncryptedStartExtension on the start_vm + // path will define the secret then, when vmUuid is known. + String vmInstanceUuid = volInv.getVmInstanceUuid(); + if (StringUtils.isBlank(vmInstanceUuid)) { + return; + } + String kpUuid = volumeEncryptedResourceKeyBackend.findKeyProviderUuidByVolume(volUuid); + String libvirtSecretUuid = secretHelper.defineSecretFromBinding( + destHost.getUuid(), vmInstanceUuid, volUuid, kpUuid); + + spec.setEncryptSecretUuid(libvirtSecretUuid); + } + + @Override + public void preCreateVolume(VolumeCreateMessage msg) { + } + + @Override + public void beforeCreateVolume(VolumeInventory volume) { + } + + @Override + public void afterCreateVolume(VolumeVO volume) { + } + + @Override + public void afterCreateVolume(VolumeVO volume, String snapshotUuid) { + snapshotEncryptionHelper.inheritFromRelatedSnapshotKeyIfPossible(volume, snapshotUuid); + } +} diff --git a/storage/src/main/java/org/zstack/storage/encrypt/VolumeEncryptedResourceKeyBackend.java b/storage/src/main/java/org/zstack/storage/encrypt/VolumeEncryptedResourceKeyBackend.java new file mode 100644 index 00000000000..bed784a9c12 --- /dev/null +++ b/storage/src/main/java/org/zstack/storage/encrypt/VolumeEncryptedResourceKeyBackend.java @@ -0,0 +1,73 @@ +package org.zstack.storage.encrypt; + +/** + * Handles {@link org.zstack.header.volume.VolumeVO} rows in {@link org.zstack.header.keyprovider.EncryptedResourceKeyRefVO} + * (key provider binding for LUKS volumes), analogous to {@link org.zstack.compute.vm.devices.TpmEncryptedResourceKeyBackend} + * for TPM. + */ +public interface VolumeEncryptedResourceKeyBackend { + + /** + * Link a volume to a key provider (placeholder ref row). Non-async. + */ + void attachKeyProviderToVolume(String volumeUuid, String keyProviderUuid); + + /** + * Remove key-provider binding for the volume. Non-async. + */ + void detachKeyProviderFromVolume(String volumeUuid); + + /** + * Remove key-provider binding for the snapshot. Non-async. + */ + void detachKeyProviderFromSnapshot(String snapshotUuid); + + void detachKeyProviderFromTemporarySnapshotImage(String imageUuid); + + /** + * @return provider uuid or null when not bound / crypto not installed + */ + String findKeyProviderUuidByVolume(String volumeUuid); + + /** + * Whether an {@code EncryptedResourceKeyRefVO} row exists for this volume. + */ + boolean checkVolumeKeyProviderAttached(String volumeUuid); + + boolean checkSnapshotKeyProviderAttached(String snapshotUuid); + + boolean checkTemporarySnapshotImageKeyProviderAttached(String imageUuid); + + void copyVolumeKeyToSnapshot(String volumeUuid, String snapshotUuid); + + void copySnapshotKeyToVolume(String snapshotUuid, String volumeUuid); + + void copyVolumeKeyToVolume(String srcVolumeUuid, String dstVolumeUuid); + + void copySnapshotKeyToTemporarySnapshotImage(String snapshotUuid, String imageUuid); + + void copyTemporarySnapshotImageKeyToVolume(String imageUuid, String volumeUuid); + + /** + * Global default key provider uuid, or null (e.g. NONE / crypto not installed). + */ + String defaultKeyProviderUuid(); + + String findKeyProviderUuidBySnapshot(String snapshotUuid); + + String findKeyProviderUuidByTemporarySnapshotImage(String imageUuid); + + /** + * Current key version (DEK rotation generation) bound to this volume's + * {@code EncryptedResourceKeyRefVO} row, or {@code null} when no row exists + * (e.g. volume not yet bound to a key provider, or crypto not installed). + * + *

Mirrors {@link org.zstack.compute.vm.devices.TpmEncryptedResourceKeyBackend#findKeyVersionByTpm}. + * Used by the start_vm path to derive the libvirt secret identity tuple + * ({@code vmUuid, purpose="volume", keyVersion, usageInstance}) without + * re-materializing the DEK. + */ + Integer findKeyVersionByVolume(String volumeUuid); + + Integer findKeyVersionBySnapshot(String snapshotUuid); +} diff --git a/storage/src/main/java/org/zstack/storage/encrypt/VolumeEncryptedSecretHelper.java b/storage/src/main/java/org/zstack/storage/encrypt/VolumeEncryptedSecretHelper.java new file mode 100644 index 00000000000..d4b4c60ca7c --- /dev/null +++ b/storage/src/main/java/org/zstack/storage/encrypt/VolumeEncryptedSecretHelper.java @@ -0,0 +1,298 @@ +package org.zstack.storage.encrypt; + +import org.apache.commons.lang.StringUtils; +import org.springframework.beans.factory.annotation.Autowire; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Configurable; +import org.zstack.core.cloudbus.CloudBus; +import org.zstack.core.db.Q; +import org.zstack.header.core.ReturnValueCompletion; +import org.zstack.header.errorcode.ErrorCode; +import org.zstack.header.errorcode.OperationFailureException; +import org.zstack.header.host.HostConstant; +import org.zstack.header.keyprovider.EncryptedResourceKeyManager; +import org.zstack.header.message.MessageReply; +import org.zstack.header.secret.SecretHostDefineMsg; +import org.zstack.header.secret.SecretHostDefineReply; +import org.zstack.header.secret.SecretHostDeleteMsg; +import org.zstack.header.secret.SecretHostGetMsg; +import org.zstack.header.secret.SecretHostGetReply; +import org.zstack.header.secret.SecretHostEnsureLuksSecretFileMsg; +import org.zstack.header.secret.SecretHostEnsureLuksSecretFileReply; +import org.zstack.header.volume.VolumeVO; +import org.zstack.header.volume.VolumeVO_; +import org.zstack.kvm.KVMConstant; +import org.zstack.storage.volume.VolumeSystemTags; +import org.zstack.tag.SystemTagCreator; +import org.zstack.utils.Utils; +import org.zstack.utils.logging.CLogger; + +import java.util.Collections; + +import static org.zstack.core.Platform.operr; + +/** + * Shared helpers for the volume LUKS secret lifecycle on a KVM host: + * + *

    + *
  • {@link #materializeDek} — unseal/get-or-create the DEK for a volume's + * bound key provider. Idempotent; safe to call on every start_vm.
    • + *
  • {@link #defineLibvirtSecretOnHost} — define+set-value the libvirt secret + * on the destination host (RAM-only). Idempotent in key-agent + * (EnsureSecret keyed on {@code vmUuid, purpose, keyVersion, usageInstance}).
    • + *
  • {@link #getSecretOnHost} — ask the host whether a previously-defined + * libvirt secret is still resident; returns {@code null} on miss so + * callers can decide to re-define.
    • + *
+ * + *

Used by the create path ({@link VolumeEncryptedInitialExtension}), the + * start path ({@link VolumeEncryptedStartExtension}) and the hot-attach path + * ({@link VolumeEncryptedAttachExtension}). All cloudbus calls are synchronous + * via {@link CloudBus#call(org.zstack.header.message.NeedReplyMessage)}; + * timeouts are enforced by the underlying KVMHost handlers (HTTP layer has its + * own {@code ENVELOPE_KEY_HTTP_TIMEOUT_SEC}). {@link EncryptedResourceKeyManager#getOrCreateKey} + * is itself a synchronous DB / NKP / KMS call on the management server — we use + * a one-shot capturing {@link ReturnValueCompletion} to read the result without + * pulling in {@code FutureReturnValueCompletion}'s wait/notify and timeout layer. + */ +@Configurable(preConstruction = true, autowire = Autowire.BY_TYPE) +public class VolumeEncryptedSecretHelper { + private static final CLogger logger = Utils.getLogger(VolumeEncryptedSecretHelper.class); + + @Autowired + private CloudBus bus; + @Autowired + private EncryptedResourceKeyManager encryptedResourceKeyManager; + @Autowired + private VolumeEncryptedResourceKeyBackend volumeEncryptedResourceKeyBackend; + + public EncryptedResourceKeyManager.ResourceKeyResult materializeDek(String volUuid, String kpUuid) { + EncryptedResourceKeyManager.GetOrCreateResourceKeyContext ctx = + new EncryptedResourceKeyManager.GetOrCreateResourceKeyContext(); + ctx.setResourceUuid(volUuid); + ctx.setResourceType(VolumeVO.class.getSimpleName()); + ctx.setKeyProviderUuid(kpUuid); + ctx.setPurpose("instantiate-volume"); + + // getOrCreateKey is synchronous in EncryptedResourceKeyManagerImpl — + // the completion fires on this thread before the call returns. + final EncryptedResourceKeyManager.ResourceKeyResult[] resultRef = + new EncryptedResourceKeyManager.ResourceKeyResult[1]; + final ErrorCode[] errorRef = new ErrorCode[1]; + encryptedResourceKeyManager.getOrCreateKey(ctx, + new ReturnValueCompletion(null) { + @Override + public void success(EncryptedResourceKeyManager.ResourceKeyResult r) { + resultRef[0] = r; + } + + @Override + public void fail(ErrorCode err) { + errorRef[0] = err; + } + }); + if (errorRef[0] != null) { + throw new OperationFailureException(operr( + "failed to materialize encryption key for volume[uuid:%s]", volUuid) + .withCause(errorRef[0])); + } + return resultRef[0]; + } + + public String ensureLuksSecretFileOnHost(String hostUuid, String resourceUuid, String dekBase64) { + SecretHostEnsureLuksSecretFileMsg ensureMsg = new SecretHostEnsureLuksSecretFileMsg(); + ensureMsg.setHostUuid(hostUuid); + ensureMsg.setDekBase64(dekBase64); + bus.makeTargetServiceIdByResourceUuid(ensureMsg, HostConstant.SERVICE_ID, hostUuid); + + MessageReply reply = bus.call(ensureMsg); + if (!reply.isSuccess()) { + throw new OperationFailureException(operr( + "failed to prepare secret material file for encrypted resource[uuid:%s] on host[uuid:%s]", + resourceUuid, hostUuid).withCause(reply.getError())); + } + SecretHostEnsureLuksSecretFileReply r = reply.castReply(); + if (StringUtils.isBlank(r.getSecFilePath())) { + throw new OperationFailureException(operr( + "ensure LUKS secret file on host succeeded but secFilePath is empty, host[uuid:%s]", + hostUuid)); + } + return r.getSecFilePath(); + } + + /** + * Define a per-volume libvirt secret on {@code hostUuid}. Returns the + * libvirt secret UUID. Throws on failure / blank reply. + */ + public String defineLibvirtSecretOnHost(String hostUuid, String vmUuid, String volUuid, + String dekBase64, Integer keyVersion) { + if (StringUtils.isBlank(hostUuid) || StringUtils.isBlank(volUuid) || + StringUtils.isBlank(dekBase64) || keyVersion == null) { + throw new OperationFailureException(operr( + "defineLibvirtSecretOnHost requires non-blank hostUuid, volUuid, dekBase64 and a non-null keyVersion")); + } + SecretHostDefineMsg defineMsg = new SecretHostDefineMsg(); + defineMsg.setHostUuid(hostUuid); + defineMsg.setVmUuid(vmUuid); + defineMsg.setDekBase64(dekBase64); + defineMsg.setPurpose("volume"); + defineMsg.setKeyVersion(keyVersion); + defineMsg.setUsageInstance(KVMConstant.volumeSecretUsageInstance(volUuid)); + defineMsg.setDescription(String.format("LUKS DEK for volume %s", volUuid)); + bus.makeTargetServiceIdByResourceUuid(defineMsg, HostConstant.SERVICE_ID, hostUuid); + + MessageReply reply = bus.call(defineMsg); + if (!reply.isSuccess()) { + throw new OperationFailureException(operr( + "failed to ensure libvirt secret for encrypted volume[uuid:%s] on host[uuid:%s]", + volUuid, hostUuid).withCause(reply.getError())); + } + SecretHostDefineReply r = reply.castReply(); + if (StringUtils.isBlank(r.getSecretUuid())) { + throw new OperationFailureException(operr( + "ensure volume LUKS secret on host succeeded but secretUuid is empty, host[uuid:%s]", + hostUuid)); + } + + // Remember which host now owns this volume's libvirt secret so that + // expunge can clean it up later, even if the owning VM is gone by then. + // recreate=true overwrites any stale tag from a previous host. + try { + SystemTagCreator tc = VolumeSystemTags.VOLUME_LIBVIRT_SECRET_HOST.newSystemTagCreator(volUuid); + tc.setTagByTokens(Collections.singletonMap( + VolumeSystemTags.VOLUME_LIBVIRT_SECRET_HOST_TOKEN, hostUuid)); + tc.inherent = false; + tc.recreate = true; + tc.create(); + } catch (RuntimeException tagEx) { + // Tag write failure must not break the actual secret define -- the + // define already succeeded, the tag is for cleanup bookkeeping only. + logger.warn(String.format( + "failed to stamp VOLUME_LIBVIRT_SECRET_HOST tag on volume[uuid:%s] for host[uuid:%s]: %s", + volUuid, hostUuid, tagEx.getMessage())); + } + + return r.getSecretUuid(); + } + + public String lookupVmInstanceUuid(String volumeUuid) { + return Q.New(VolumeVO.class) + .eq(VolumeVO_.uuid, volumeUuid) + .select(VolumeVO_.vmInstanceUuid) + .findValue(); + } + + /** + * Materialize the DEK for {@code volUuid} under the binding in + * {@code kpUuid}, then define+set-value the libvirt secret on + * {@code hostUuid}. Used by both the create-time path and the start-time + * fallback when the host's libvirt secret value was lost. + */ + public String defineSecretFromBinding(String hostUuid, String vmUuid, String volUuid, String kpUuid) { + if (StringUtils.isBlank(kpUuid)) { + throw new OperationFailureException(operr( + "encrypted volume[uuid:%s] has no key provider binding; cannot define libvirt secret on host[uuid:%s]", + volUuid, hostUuid)); + } + EncryptedResourceKeyManager.ResourceKeyResult keyResult = materializeDek(volUuid, kpUuid); + String dekBase64 = keyResult.getDekBase64(); + if (StringUtils.isBlank(dekBase64)) { + throw new OperationFailureException(operr( + "encrypted volume[uuid:%s]: key manager returned empty DEK for libvirt secret", + volUuid)); + } + return defineLibvirtSecretOnHost(hostUuid, vmUuid, volUuid, dekBase64, keyResult.getKeyVersion()); + } + + /** + * Ask {@code hostUuid} for the libvirt secret UUID identified by the + * (vm, volume, keyVersion) tuple. Returns null on SECRET_NOT_FOUND so + * callers can fall back to {@link #defineSecretFromBinding}; throws on + * any other failure. + */ + public String getSecretOnHost(String hostUuid, String vmUuid, String volUuid, Integer keyVersion) { + SecretHostGetMsg msg = new SecretHostGetMsg(); + msg.setHostUuid(hostUuid); + msg.setVmUuid(vmUuid); + msg.setPurpose("volume"); + msg.setKeyVersion(keyVersion); + msg.setUsageInstance(KVMConstant.volumeSecretUsageInstance(volUuid)); + bus.makeTargetServiceIdByResourceUuid(msg, HostConstant.SERVICE_ID, hostUuid); + + MessageReply reply = bus.call(msg); + if (reply.isSuccess()) { + SecretHostGetReply r = reply.castReply(); + return r.getSecretUuid(); + } + ErrorCode err = reply.getError(); + if (SecretHostGetReply.isSecretNotFound(err)) { + return null; + } + throw new OperationFailureException(operr( + "failed to get libvirt LUKS secret on host[uuid:%s] vm[uuid:%s] volume[uuid:%s] keyVersion[%s]: %s", + hostUuid, vmUuid, volUuid, keyVersion, err)); + } + + /** + * One-shot resolver: "give me the libvirt secret UUID for this encrypted + * volume on this host". Used by both the start-vm path + * ({@link VolumeEncryptedStartExtension}) and the attach-data-volume + * path ({@link VolumeEncryptedAttachExtension}). + * + *

Order: + *

    + *
  1. {@code findKeyVersionByVolume} — fail loudly if the volume has + * no {@code EncryptedResourceKeyRefVO} binding (would otherwise + * produce an unreadable qcow2 once attached).
    2. + *
  2. {@link #getSecretOnHost} — fast path, hits when the secret has + * already been defined on the host.
    3. + *
  3. {@link #defineSecretFromBinding} — fall back when the secret + * value is missing on the host (first attach, libvirtd restart, + * host reboot, migrate to a fresh host).
    4. + *
+ */ + /** + * Best-effort delete of a single per-volume libvirt secret on {@code hostUuid}. + * The underlying KVMHost handler treats SECRET_NOT_FOUND as success, so it's + * safe to call regardless of whether the secret was ever defined. + * + *

Returns silently on failure (logs at warn): cleanup must never break + * caller flows like VM destroy. + */ + public void deleteSecretOnHostBestEffort(String hostUuid, String vmUuid, String volUuid, Integer keyVersion) { + if (StringUtils.isBlank(hostUuid) || StringUtils.isBlank(vmUuid) + || StringUtils.isBlank(volUuid) || keyVersion == null) { + return; + } + SecretHostDeleteMsg msg = new SecretHostDeleteMsg(); + msg.setHostUuid(hostUuid); + msg.setVmUuid(vmUuid); + msg.setPurpose("volume"); + msg.setKeyVersion(keyVersion); + msg.setUsageInstance(KVMConstant.volumeSecretUsageInstance(volUuid)); + bus.makeTargetServiceIdByResourceUuid(msg, HostConstant.SERVICE_ID, hostUuid); + + MessageReply reply = bus.call(msg); + if (!reply.isSuccess()) { + logger.warn(String.format( + "best-effort delete libvirt LUKS secret failed for volume[uuid:%s] on host[uuid:%s] vm[uuid:%s]: %s", + volUuid, hostUuid, vmUuid, reply.getError())); + } + } + + public String resolveOrDefineSecretForVolume(String hostUuid, String vmUuid, String volUuid) { + Integer keyVersion = volumeEncryptedResourceKeyBackend.findKeyVersionByVolume(volUuid); + if (keyVersion == null) { + throw new OperationFailureException(operr( + "encrypted volume[uuid:%s] has no key version bound (EncryptedResourceKeyRefVO missing);" + + " cannot resolve libvirt LUKS secret on host[uuid:%s] for vm[uuid:%s]", + volUuid, hostUuid, vmUuid)); + } + String secretUuid = getSecretOnHost(hostUuid, vmUuid, volUuid, keyVersion); + if (StringUtils.isNotBlank(secretUuid)) { + return secretUuid; + } + String kpUuid = volumeEncryptedResourceKeyBackend.findKeyProviderUuidByVolume(volUuid); + return defineSecretFromBinding(hostUuid, vmUuid, volUuid, kpUuid); + } +} diff --git a/storage/src/main/java/org/zstack/storage/encrypt/VolumeEncryptedStartExtension.java b/storage/src/main/java/org/zstack/storage/encrypt/VolumeEncryptedStartExtension.java new file mode 100644 index 00000000000..db68ec8dc22 --- /dev/null +++ b/storage/src/main/java/org/zstack/storage/encrypt/VolumeEncryptedStartExtension.java @@ -0,0 +1,110 @@ +package org.zstack.storage.encrypt; + +import org.apache.commons.lang.StringUtils; +import org.springframework.beans.factory.annotation.Autowire; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Configurable; +import org.zstack.header.errorcode.OperationFailureException; +import org.zstack.header.host.HostInventory; +import org.zstack.header.vm.VmBeforeCreateOnHypervisorExtensionPoint; +import org.zstack.header.vm.VmBeforeStartOnHypervisorExtensionPoint; +import org.zstack.header.vm.VmInstanceSpec; +import org.zstack.header.volume.VolumeInventory; +import org.zstack.utils.Utils; +import org.zstack.utils.logging.CLogger; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static org.zstack.core.Platform.operr; + +/** + * Per start_vm: resolve the libvirt LUKS secret UUID for every encrypted volume + * on the destination host and stash the mapping into + * {@link VmInstanceSpec#putExtensionData} under {@link #EXT_DATA_KEY} so + * {@code KVMHost.handleStart} can inline it into the {@code VolumeTO} that + * ships with {@code StartVmCmd}. + * + *

Why every start (not persisted)? libvirt secret values are RAM-only; + * libvirtd restart / host reboot / live-migrate-to-new-host all wipe them. The + * UUID itself would persist but without the value the secret is useless to qemu. + * So on every start_vm we delegate to + * {@link VolumeEncryptedSecretHelper#resolveOrDefineSecretForVolume}, which + * first asks the host (idempotent {@code SecretHostGetMsg} → key-agent + * {@code GetSecret}) and falls back to materialize-DEK + define on miss. + */ +@Configurable(preConstruction = true, autowire = Autowire.BY_TYPE) +public class VolumeEncryptedStartExtension + implements VmBeforeStartOnHypervisorExtensionPoint, VmBeforeCreateOnHypervisorExtensionPoint { + + private static final CLogger logger = Utils.getLogger(VolumeEncryptedStartExtension.class); + + /** {@code Map} consumed by KVMHost.handleStart. */ + public static final String EXT_DATA_KEY = "VolumeLuksSecrets"; + + @Autowired + private VolumeEncryptedSecretHelper secretHelper; + + @Override + public void beforeStartVmOnHypervisor(VmInstanceSpec spec) { + HostInventory destHost = spec.getDestHost(); + if (destHost == null || StringUtils.isBlank(destHost.getUuid())) { + return; + } + + List encryptedVolumes = collectEncryptedVolumes(spec); + if (encryptedVolumes.isEmpty()) { + return; + } + + String hostUuid = destHost.getUuid(); + String vmUuid = spec.getVmInventory().getUuid(); + Map resolved = new HashMap<>(); + + for (VolumeInventory vol : encryptedVolumes) { + String volUuid = vol.getUuid(); + String secretUuid = secretHelper.resolveOrDefineSecretForVolume(hostUuid, vmUuid, volUuid); + if (StringUtils.isBlank(secretUuid)) { + throw new OperationFailureException(operr( + "failed to resolve libvirt LUKS secret for encrypted volume[uuid:%s] on host[uuid:%s]", + volUuid, hostUuid)); + } + resolved.put(volUuid, secretUuid); + } + + if (!resolved.isEmpty()) { + spec.putExtensionData(EXT_DATA_KEY, resolved); + logger.debug(String.format("LUKS-START-EXT stashed %d secrets into spec.extensionData[%s]: %s", + resolved.size(), EXT_DATA_KEY, resolved)); + } + } + + /** + * "Create VM" (provisioning) path uses {@link org.zstack.compute.vm.VmCreateOnHypervisorFlow} + * which fires {@link VmBeforeCreateOnHypervisorExtensionPoint}, NOT the start-vm hook. + * Delegate to the same logic: encrypted root volumes need their libvirt secret + * resolved on the destination host before the agent receives StartVmCmd. + */ + @Override + public void beforeCreateVmOnHypervisor(VmInstanceSpec spec) { + beforeStartVmOnHypervisor(spec); + } + + private List collectEncryptedVolumes(VmInstanceSpec spec) { + List result = new ArrayList<>(); + VolumeInventory root = spec.getDestRootVolume(); + if (root != null && Boolean.TRUE.equals(root.getEncrypted())) { + result.add(root); + } + if (spec.getDestDataVolumes() != null) { + for (VolumeInventory v : spec.getDestDataVolumes()) { + if (v != null && Boolean.TRUE.equals(v.getEncrypted())) { + result.add(v); + } + } + } + return result; + } +} diff --git a/storage/src/main/java/org/zstack/storage/encrypt/VolumeSnapshotEncryptionExtension.java b/storage/src/main/java/org/zstack/storage/encrypt/VolumeSnapshotEncryptionExtension.java new file mode 100644 index 00000000000..966023fe5dc --- /dev/null +++ b/storage/src/main/java/org/zstack/storage/encrypt/VolumeSnapshotEncryptionExtension.java @@ -0,0 +1,258 @@ +package org.zstack.storage.encrypt; + +import org.apache.commons.lang.StringUtils; +import org.springframework.beans.factory.annotation.Autowired; +import org.zstack.core.db.DatabaseFacade; +import org.zstack.header.core.Completion; +import org.zstack.header.core.workflow.Flow; +import org.zstack.header.errorcode.OperationFailureException; +import org.zstack.header.host.TakeSnapshotOnHypervisorMsg; +import org.zstack.header.storage.snapshot.BeforeTakeLiveSnapshotsOnVolumes; +import org.zstack.header.storage.snapshot.ConsistentType; +import org.zstack.header.storage.snapshot.CreateVolumesSnapshotOverlayInnerMsg; +import org.zstack.header.storage.snapshot.TakeSnapshotsOnKvmJobStruct; +import org.zstack.header.storage.snapshot.TakeVolumesSnapshotOnKvmMsg; +import org.zstack.header.storage.snapshot.TakeVolumesSnapshotOnKvmReply; +import org.zstack.header.storage.snapshot.VolumeSnapshotAfterDeleteExtensionPoint; +import org.zstack.header.storage.snapshot.VolumeSnapshotCreationExtensionPoint; +import org.zstack.header.storage.snapshot.VolumeSnapshotInventory; +import org.zstack.header.storage.snapshot.VolumeSnapshotVO; +import org.zstack.header.storage.snapshot.group.VolumeSnapshotGroupInventory; +import org.zstack.header.volume.CreateVolumeSnapshotGroupMessage; +import org.zstack.header.volume.VolumeLuksAgentSpec; +import org.zstack.header.volume.VolumeVO; +import org.zstack.kvm.KVMAgentCommands; +import org.zstack.kvm.KVMTakeSnapshotExtensionPoint; +import org.zstack.kvm.KVMHostInventory; +import org.zstack.kvm.VolumeTO; +import org.zstack.utils.Utils; +import org.zstack.utils.logging.CLogger; + +import java.util.List; +import java.util.Map; + +import static org.zstack.core.Platform.operr; + +public class VolumeSnapshotEncryptionExtension implements KVMTakeSnapshotExtensionPoint, + BeforeTakeLiveSnapshotsOnVolumes, VolumeSnapshotCreationExtensionPoint, + VolumeSnapshotAfterDeleteExtensionPoint { + private static final CLogger logger = Utils.getLogger(VolumeSnapshotEncryptionExtension.class); + + @Autowired + private DatabaseFacade dbf; + @Autowired + private VolumeSnapshotEncryptionHelper snapshotEncryptionHelper; + @Autowired + private VolumeEncryptedResourceKeyBackend volumeEncryptedResourceKeyBackend; + @Autowired + private VolumeEncryptedSecretHelper volumeEncryptedSecretHelper; + + @Override + public void beforeTakeSnapshot(KVMHostInventory host, TakeSnapshotOnHypervisorMsg msg, + KVMAgentCommands.TakeSnapshotCmd cmd, Completion completion) { + try { + VolumeVO volume = findVolume(msg.getVolume().getUuid()); + if (!volume.isEncrypted()) { + completion.success(); + return; + } + + VolumeSnapshotInventory snapshot = findSnapshot(msg.getSnapshotName()); + snapshotEncryptionHelper.inheritVolumeKeyToSnapshot(volume, snapshot); + if (!cmd.isOnline()) { + VolumeLuksAgentSpec luksSpec = + snapshotEncryptionHelper.prepareVolumeSecretMaterial(host.getUuid(), volume.getUuid()); + if (luksSpec != null && StringUtils.isNotBlank(luksSpec.getEncryptLuksSecretMaterialFilePath())) { + cmd.setEncryptLuksSecretMaterialFilePath(luksSpec.getEncryptLuksSecretMaterialFilePath()); + } + + if (msg.isFullSnapshot()) { + // The two secret material files carry the same DEK. They are separate + // because the file is read-once and deleted after use; offline full + // snapshot runs two qemu-img operations, so each operation needs its + // own one-shot file path. + VolumeLuksAgentSpec snapshotLuksSpec = + snapshotEncryptionHelper.prepareVolumeSecretMaterial(host.getUuid(), volume.getUuid()); + if (snapshotLuksSpec != null && + StringUtils.isNotBlank(snapshotLuksSpec.getEncryptLuksSecretMaterialFilePath())) { + cmd.setFullSnapshotLuksSecretMaterialFilePath( + snapshotLuksSpec.getEncryptLuksSecretMaterialFilePath()); + } + } + } + + completion.success(); + } catch (OperationFailureException e) { + completion.fail(e.getErrorCode()); + } catch (RuntimeException e) { + completion.fail(operr("failed to prepare encrypted volume snapshot[uuid:%s] on host[uuid:%s]: %s", + msg.getSnapshotName(), host.getUuid(), e.getMessage())); + } + } + + @Override + public void beforeTakeLiveSnapshotsOnVolumes(CreateVolumesSnapshotOverlayInnerMsg msg, + TakeVolumesSnapshotOnKvmMsg tmsg, + Map flowData, + Completion completion) { + try { + if (tmsg == null || tmsg.getSnapshotJobs() == null) { + completion.success(); + return; + } + + for (TakeSnapshotsOnKvmJobStruct job : tmsg.getSnapshotJobs()) { + if (job.isMemory()) { + continue; + } + + VolumeVO volume = findVolume(job.getVolumeUuid()); + if (!volume.isEncrypted()) { + continue; + } + + VolumeSnapshotInventory snapshot = findSnapshot(job.getSnapshotUuid()); + snapshotEncryptionHelper.inheritVolumeKeyToSnapshot(volume, snapshot); + + VolumeLuksAgentSpec luksSpec = + snapshotEncryptionHelper.prepareVolumeSecretMaterial(tmsg.getHostUuid(), volume.getUuid()); + // This extension runs in a chain with storage-specific implementations; + // keep any secret path already prepared by a storage backend. + boolean needFillEncryptSecret = StringUtils.isBlank(job.getEncryptLuksSecretMaterialFilePath()); + if (needFillEncryptSecret && luksSpec != null && + StringUtils.isNotBlank(luksSpec.getEncryptLuksSecretMaterialFilePath())) { + job.setEncryptLuksSecretMaterialFilePath(luksSpec.getEncryptLuksSecretMaterialFilePath()); + } + + if (job.getVolume() instanceof VolumeTO) { + ((VolumeTO) job.getVolume()).setLuksSecretUuid( + volumeEncryptedSecretHelper.resolveOrDefineSecretForVolume( + tmsg.getHostUuid(), volume.getVmInstanceUuid(), volume.getUuid())); + } + } + + completion.success(); + } catch (OperationFailureException e) { + completion.fail(e.getErrorCode()); + } catch (RuntimeException e) { + completion.fail(operr("failed to prepare encrypted live volume snapshots: %s", e.getMessage())); + } + } + + @Override + public void afterVolumeSnapshotCreated(VolumeSnapshotInventory snapshot, Completion completion) { + try { + if (snapshot == null) { + completion.success(); + return; + } + + VolumeVO volume = findVolume(snapshot.getVolumeUuid()); + if (!volume.isEncrypted()) { + completion.success(); + return; + } + + snapshotEncryptionHelper.completeTakeSnapshot(volume, snapshot); + completion.success(); + } catch (OperationFailureException e) { + completion.fail(e.getErrorCode()); + } catch (RuntimeException e) { + completion.fail(operr("failed to complete encrypted volume snapshot[uuid:%s]: %s", + snapshot == null ? null : snapshot.getUuid(), e.getMessage())); + } + } + + @Override + public void afterTakeSnapshot(KVMHostInventory host, TakeSnapshotOnHypervisorMsg msg, + KVMAgentCommands.TakeSnapshotCmd cmd, + KVMAgentCommands.TakeSnapshotResponse rsp) { + } + + @Override + public void afterTakeSnapshotFailed(KVMHostInventory host, TakeSnapshotOnHypervisorMsg msg, + KVMAgentCommands.TakeSnapshotCmd cmd, + KVMAgentCommands.TakeSnapshotResponse rsp, + org.zstack.header.errorcode.ErrorCode err) { + } + + @Override + public void afterVolumeLiveSnapshotGroupCreatedOnBackend(CreateVolumesSnapshotOverlayInnerMsg msg, + TakeVolumesSnapshotOnKvmReply treply, + Completion completion) { + completion.success(); + } + + @Override + public void afterVolumeLiveSnapshotGroupCreationFailsOnBackend(CreateVolumesSnapshotOverlayInnerMsg msg, + TakeVolumesSnapshotOnKvmReply treply) { + } + + @Override + public void afterVolumeSnapshotGroupCreated(VolumeSnapshotGroupInventory snapshotGroup, + ConsistentType consistentType, + Completion completion) { + completion.success(); + } + + @Override + public List beforeCreateVolumeSnapshotFlow(CreateVolumeSnapshotGroupMessage msg) { + return null; + } + + @Override + public void volumeSnapshotAfterDeleteExtensionPoint(VolumeSnapshotInventory snapshot, Completion completion) { + completion.success(); + } + + @Override + public void volumeSnapshotAfterFailedDeleteExtensionPoint(VolumeSnapshotInventory snapshot) { + } + + @Override + public void volumeSnapshotAfterCleanUpExtensionPoint(String volumeUuid, List snapshots) { + if (snapshots == null || snapshots.isEmpty()) { + return; + } + + for (VolumeSnapshotInventory snapshot : snapshots) { + if (snapshot == null || StringUtils.isBlank(snapshot.getUuid())) { + continue; + } + if (dbf.isExist(snapshot.getUuid(), VolumeSnapshotVO.class)) { + continue; + } + try { + volumeEncryptedResourceKeyBackend.detachKeyProviderFromSnapshot(snapshot.getUuid()); + } catch (RuntimeException e) { + logger.warn(String.format( + "failed to detach EncryptedResourceKeyRefVO for volume snapshot[uuid:%s] on delete cleanup: %s", + snapshot.getUuid(), e.getMessage())); + } + } + } + + private VolumeVO findVolume(String volumeUuid) { + if (StringUtils.isBlank(volumeUuid)) { + throw new OperationFailureException(operr("volume uuid is required for encrypted snapshot preparation")); + } + + VolumeVO volume = dbf.findByUuid(volumeUuid, VolumeVO.class); + if (volume == null) { + throw new OperationFailureException(operr("volume[uuid:%s] not found", volumeUuid)); + } + return volume; + } + + private VolumeSnapshotInventory findSnapshot(String snapshotUuid) { + if (StringUtils.isBlank(snapshotUuid)) { + throw new OperationFailureException(operr("snapshot uuid is required for encrypted snapshot preparation")); + } + + VolumeSnapshotVO snapshot = dbf.findByUuid(snapshotUuid, VolumeSnapshotVO.class); + if (snapshot == null) { + throw new OperationFailureException(operr("volume snapshot[uuid:%s] not found", snapshotUuid)); + } + return VolumeSnapshotInventory.valueOf(snapshot); + } +} diff --git a/storage/src/main/java/org/zstack/storage/encrypt/VolumeSnapshotEncryptionHelper.java b/storage/src/main/java/org/zstack/storage/encrypt/VolumeSnapshotEncryptionHelper.java new file mode 100644 index 00000000000..2b5a20edd53 --- /dev/null +++ b/storage/src/main/java/org/zstack/storage/encrypt/VolumeSnapshotEncryptionHelper.java @@ -0,0 +1,314 @@ +package org.zstack.storage.encrypt; + +import org.apache.commons.lang.StringUtils; +import org.springframework.beans.factory.annotation.Autowire; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Configurable; +import org.zstack.core.db.DatabaseFacade; +import org.zstack.core.db.Q; +import org.zstack.header.errorcode.OperationFailureException; +import org.zstack.header.image.ImageConstant; +import org.zstack.header.image.ImageVO; +import org.zstack.header.image.ImageVO_; +import org.zstack.header.keyprovider.EncryptedResourceKeyManager; +import org.zstack.header.storage.snapshot.VolumeSnapshotInventory; +import org.zstack.header.storage.snapshot.VolumeSnapshotVO; +import org.zstack.header.storage.snapshot.VolumeSnapshotVO_; +import org.zstack.header.volume.VolumeLuksAgentSpec; +import org.zstack.header.volume.VolumeVO; + +import static org.zstack.core.Platform.operr; + +@Configurable(preConstruction = true, autowire = Autowire.BY_TYPE) +public class VolumeSnapshotEncryptionHelper { + @Autowired + private DatabaseFacade dbf; + @Autowired + private VolumeEncryptedResourceKeyBackend keyBackend; + @Autowired + private VolumeEncryptedSecretHelper secretHelper; + @Autowired + private EncryptedResourceKeyManager encryptedResourceKeyManager; + + public void completeTakeSnapshot(VolumeVO volume, VolumeSnapshotInventory snapshot) { + if (volume == null || snapshot == null || !volume.isEncrypted()) { + return; + } + + snapshot.setEncrypted(true); + Boolean encrypted = Q.New(VolumeSnapshotVO.class) + .eq(VolumeSnapshotVO_.uuid, snapshot.getUuid()) + .select(VolumeSnapshotVO_.encrypted) + .findValue(); + if (!Boolean.TRUE.equals(encrypted)) { + VolumeSnapshotVO vo = dbf.findByUuid(snapshot.getUuid(), VolumeSnapshotVO.class); + if (vo != null) { + vo.setEncrypted(true); + dbf.update(vo); + } + } + } + + public void inheritVolumeKeyToSnapshot(VolumeVO volume, VolumeSnapshotInventory snapshot) { + if (volume == null || snapshot == null || !volume.isEncrypted()) { + return; + } + keyBackend.copyVolumeKeyToSnapshot(volume.getUuid(), snapshot.getUuid()); + } + + public void inheritFromRelatedSnapshotKeyIfPossible(VolumeVO volume, String snapshotUuid) { + if (volume == null || StringUtils.isBlank(snapshotUuid) + || keyBackend.checkVolumeKeyProviderAttached(volume.getUuid())) { + return; + } + + Boolean snapshotEncrypted = Q.New(VolumeSnapshotVO.class) + .eq(VolumeSnapshotVO_.uuid, snapshotUuid) + .select(VolumeSnapshotVO_.encrypted) + .findValue(); + if (!Boolean.TRUE.equals(snapshotEncrypted)) { + if (volume.isEncrypted()) { + String kpUuid = keyBackend.defaultKeyProviderUuid(); + if (StringUtils.isBlank(kpUuid)) { + throw new OperationFailureException(operr( + "encrypted volume[uuid:%s] has no key provider binding and no default key provider configured", + volume.getUuid())); + } + keyBackend.attachKeyProviderToVolume(volume.getUuid(), kpUuid); + materializeVolumeKey(volume.getUuid(), kpUuid, "create-encrypted-volume-from-plain-snapshot"); + } + return; + } + if (!keyBackend.checkSnapshotKeyProviderAttached(snapshotUuid)) { + throw new OperationFailureException(operr( + "encrypted snapshot[uuid:%s] has no key provider binding; cannot inherit key for volume[uuid:%s]", + snapshotUuid, volume.getUuid())); + } + + keyBackend.copySnapshotKeyToVolume(snapshotUuid, volume.getUuid()); + if (!volume.isEncrypted()) { + volume.setEncrypted(true); + dbf.update(volume); + } + } + + public void inheritFromTemporarySnapshotImageKeyIfPossible(VolumeVO volume) { + if (volume == null || StringUtils.isBlank(volume.getRootImageUuid())) { + return; + } + + String imageUrl = Q.New(ImageVO.class) + .eq(ImageVO_.uuid, volume.getRootImageUuid()) + .select(ImageVO_.url) + .findValue(); + if (StringUtils.isBlank(imageUrl)) { + return; + } + + if (imageUrl.startsWith("volume://")) { + String srcVolumeUuid = imageUrl.substring("volume://".length()); + if (!keyBackend.checkVolumeKeyProviderAttached(volume.getUuid())) { + if (keyBackend.checkVolumeKeyProviderAttached(srcVolumeUuid)) { + keyBackend.copyVolumeKeyToVolume(srcVolumeUuid, volume.getUuid()); + } + } + return; + } + + if (!imageUrl.startsWith(ImageConstant.IMAGE_FROM_SNAPSHOT_SCHEMA) + && !imageUrl.startsWith(ImageConstant.SNAPSHOT_REUSE_IMAGE_SCHEMA)) { + return; + } + + if (keyBackend.checkTemporarySnapshotImageKeyProviderAttached(volume.getRootImageUuid())) { + if (!keyBackend.checkVolumeKeyProviderAttached(volume.getUuid())) { + keyBackend.copyTemporarySnapshotImageKeyToVolume(volume.getRootImageUuid(), volume.getUuid()); + } + return; + } + + String snapshotUuid = getSnapshotUuidFromImageUrl(imageUrl); + inheritFromRelatedSnapshotKeyIfPossible(volume, snapshotUuid); + } + + public boolean hasTemporarySnapshotImageKey(String imageUuid) { + return StringUtils.isNotBlank(imageUuid) && keyBackend.checkTemporarySnapshotImageKeyProviderAttached(imageUuid); + } + + private String getSnapshotUuidFromImageUrl(String imageUrl) { + String snapshotUuid; + if (imageUrl.startsWith(ImageConstant.IMAGE_FROM_SNAPSHOT_SCHEMA)) { + snapshotUuid = imageUrl.substring(ImageConstant.IMAGE_FROM_SNAPSHOT_SCHEMA.length()); + } else if (imageUrl.startsWith(ImageConstant.SNAPSHOT_REUSE_IMAGE_SCHEMA)) { + snapshotUuid = imageUrl.substring(ImageConstant.SNAPSHOT_REUSE_IMAGE_SCHEMA.length()); + } else { + return null; + } + return snapshotUuid.length() >= 32 ? snapshotUuid.substring(0, 32) : snapshotUuid; + } + + private EncryptedResourceKeyManager.ResourceKeyResult materializeVolumeKey(String volumeUuid, + String keyProviderUuid, + String purpose) { + EncryptedResourceKeyManager.GetOrCreateResourceKeyContext ctx = + new EncryptedResourceKeyManager.GetOrCreateResourceKeyContext(); + ctx.setResourceUuid(volumeUuid); + ctx.setResourceType(VolumeVO.class.getSimpleName()); + ctx.setKeyProviderUuid(keyProviderUuid); + ctx.setPurpose(purpose); + + final EncryptedResourceKeyManager.ResourceKeyResult[] resultRef = + new EncryptedResourceKeyManager.ResourceKeyResult[1]; + final org.zstack.header.errorcode.ErrorCode[] errorRef = + new org.zstack.header.errorcode.ErrorCode[1]; + encryptedResourceKeyManager.getOrCreateKey(ctx, + new org.zstack.header.core.ReturnValueCompletion(null) { + @Override + public void success(EncryptedResourceKeyManager.ResourceKeyResult returnValue) { + resultRef[0] = returnValue; + } + + @Override + public void fail(org.zstack.header.errorcode.ErrorCode errorCode) { + errorRef[0] = errorCode; + } + }); + + if (errorRef[0] != null) { + throw new OperationFailureException(operr( + "failed to materialize encryption key for volume[uuid:%s]", volumeUuid).withCause(errorRef[0])); + } + return resultRef[0]; + } + + public VolumeLuksAgentSpec prepareVolumeSecretMaterial(String hostUuid, String volumeUuid) { + if (StringUtils.isBlank(hostUuid) || StringUtils.isBlank(volumeUuid)) { + return null; + } + + VolumeVO volume = dbf.findByUuid(volumeUuid, VolumeVO.class); + if (volume == null) { + throw new OperationFailureException(operr("target volume[uuid:%s] not found", volumeUuid)); + } + if (!volume.isEncrypted()) { + return null; + } + + String kpUuid = keyBackend.findKeyProviderUuidByVolume(volume.getUuid()); + if (StringUtils.isBlank(kpUuid)) { + throw new OperationFailureException(operr( + "encrypted volume[uuid:%s] has no key provider binding", volume.getUuid())); + } + + EncryptedResourceKeyManager.GetOrCreateResourceKeyContext ctx = + new EncryptedResourceKeyManager.GetOrCreateResourceKeyContext(); + ctx.setResourceUuid(volume.getUuid()); + ctx.setResourceType(VolumeVO.class.getSimpleName()); + ctx.setKeyProviderUuid(kpUuid); + ctx.setPurpose("prepare-volume-secret-material"); + + EncryptedResourceKeyManager.ResourceKeyResult keyResult = encryptedResourceKeyManager.getKey(ctx); + if (StringUtils.isBlank(keyResult.getDekBase64())) { + throw new OperationFailureException(operr( + "key manager returned empty DEK for encrypted volume[uuid:%s]", volume.getUuid())); + } + + VolumeLuksAgentSpec spec = new VolumeLuksAgentSpec(); + spec.setEncryptLuksSecretMaterialFilePath( + secretHelper.ensureLuksSecretFileOnHost(hostUuid, volume.getUuid(), keyResult.getDekBase64())); + return spec; + } + + public VolumeLuksAgentSpec prepareTemporarySnapshotImageSecretMaterial(String hostUuid, + String snapshotUuid, + String imageUuid, + Boolean encrypted) { + if (StringUtils.isBlank(hostUuid) || StringUtils.isBlank(snapshotUuid) || StringUtils.isBlank(imageUuid) + || !Boolean.TRUE.equals(encrypted)) { + return null; + } + + EncryptedResourceKeyManager.ResourceKeyResult keyResult; + Boolean snapshotEncrypted = Q.New(VolumeSnapshotVO.class) + .eq(VolumeSnapshotVO_.uuid, snapshotUuid) + .select(VolumeSnapshotVO_.encrypted) + .findValue(); + if (Boolean.TRUE.equals(snapshotEncrypted)) { + keyBackend.copySnapshotKeyToTemporarySnapshotImage(snapshotUuid, imageUuid); + keyResult = getTemporarySnapshotImageKey(imageUuid); + } else { + keyResult = createTemporarySnapshotImageKey(imageUuid); + } + + VolumeLuksAgentSpec spec = new VolumeLuksAgentSpec(); + spec.setEncryptLuksSecretMaterialFilePath( + secretHelper.ensureLuksSecretFileOnHost(hostUuid, imageUuid, keyResult.getDekBase64())); + return spec; + } + + private EncryptedResourceKeyManager.ResourceKeyResult getTemporarySnapshotImageKey(String imageUuid) { + String kpUuid = keyBackend.findKeyProviderUuidByTemporarySnapshotImage(imageUuid); + if (StringUtils.isBlank(kpUuid)) { + throw new OperationFailureException(operr( + "encrypted temporary snapshot image[uuid:%s] has no key provider binding", imageUuid)); + } + + EncryptedResourceKeyManager.GetOrCreateResourceKeyContext ctx = + new EncryptedResourceKeyManager.GetOrCreateResourceKeyContext(); + ctx.setResourceUuid(imageUuid); + ctx.setResourceType(ImageVO.class.getSimpleName()); + ctx.setKeyProviderUuid(kpUuid); + ctx.setPurpose("prepare-temporary-snapshot-image-secret-material"); + + EncryptedResourceKeyManager.ResourceKeyResult keyResult = encryptedResourceKeyManager.getKey(ctx); + if (StringUtils.isBlank(keyResult.getDekBase64())) { + throw new OperationFailureException(operr( + "key manager returned empty DEK for encrypted temporary snapshot image[uuid:%s]", imageUuid)); + } + return keyResult; + } + + private EncryptedResourceKeyManager.ResourceKeyResult createTemporarySnapshotImageKey(String imageUuid) { + String kpUuid = keyBackend.defaultKeyProviderUuid(); + if (StringUtils.isBlank(kpUuid)) { + throw new OperationFailureException(operr( + "encrypted temporary snapshot image[uuid:%s] has no default key provider configured", imageUuid)); + } + + EncryptedResourceKeyManager.GetOrCreateResourceKeyContext ctx = + new EncryptedResourceKeyManager.GetOrCreateResourceKeyContext(); + ctx.setResourceUuid(imageUuid); + ctx.setResourceType(ImageVO.class.getSimpleName()); + ctx.setKeyProviderUuid(kpUuid); + ctx.setPurpose("prepare-temporary-snapshot-image-secret-material"); + + final EncryptedResourceKeyManager.ResourceKeyResult[] resultRef = + new EncryptedResourceKeyManager.ResourceKeyResult[1]; + final org.zstack.header.errorcode.ErrorCode[] errorRef = + new org.zstack.header.errorcode.ErrorCode[1]; + encryptedResourceKeyManager.getOrCreateKey(ctx, + new org.zstack.header.core.ReturnValueCompletion(null) { + @Override + public void success(EncryptedResourceKeyManager.ResourceKeyResult returnValue) { + resultRef[0] = returnValue; + } + + @Override + public void fail(org.zstack.header.errorcode.ErrorCode errorCode) { + errorRef[0] = errorCode; + } + }); + + if (errorRef[0] != null) { + throw new OperationFailureException(operr( + "failed to materialize encryption key for temporary snapshot image[uuid:%s]", + imageUuid).withCause(errorRef[0])); + } + if (resultRef[0] == null || StringUtils.isBlank(resultRef[0].getDekBase64())) { + throw new OperationFailureException(operr( + "key manager returned empty DEK for encrypted temporary snapshot image[uuid:%s]", imageUuid)); + } + return resultRef[0]; + } + +} diff --git a/storage/src/main/java/org/zstack/storage/snapshot/VolumeSnapshotManagerImpl.java b/storage/src/main/java/org/zstack/storage/snapshot/VolumeSnapshotManagerImpl.java index 1f936ca6141..e39c8c74b9c 100755 --- a/storage/src/main/java/org/zstack/storage/snapshot/VolumeSnapshotManagerImpl.java +++ b/storage/src/main/java/org/zstack/storage/snapshot/VolumeSnapshotManagerImpl.java @@ -737,6 +737,7 @@ private VolumeSnapshotStruct getVolumeSnapshotStruct(CreateVolumeSnapshotMsg msg vo.setDescription(msg.getDescription()); vo.setVolumeUuid(msg.getVolumeUuid()); vo.setFormat(vol.getFormat()); + vo.setEncrypted(vol.isEncrypted()); vo.setState(VolumeSnapshotState.Enabled); vo.setStatus(VolumeSnapshotStatus.Creating); vo.setVolumeType(vol.getType().toString()); @@ -991,6 +992,9 @@ public void handle(Map data) { markSnapshotTreeCompleted(snapshot); if (volumeNewInstallPath != null) { vol.setInstallPath(volumeNewInstallPath); + if (Boolean.TRUE.equals(snapshot.getEncrypted())) { + vol.setEncrypted(true); + } dbf.update(vol); } @@ -1000,6 +1004,7 @@ public void handle(Map data) { svo.setPrimaryStorageInstallPath(snapshot.getPrimaryStorageInstallPath()); svo.setStatus(VolumeSnapshotStatus.Ready); svo.setSize(snapshot.getSize()); + svo.setEncrypted(Boolean.TRUE.equals(snapshot.getEncrypted())); if (snapshot.getFormat() != null) { svo.setFormat(snapshot.getFormat()); } diff --git a/storage/src/main/java/org/zstack/storage/snapshot/VolumeSnapshotTreeBase.java b/storage/src/main/java/org/zstack/storage/snapshot/VolumeSnapshotTreeBase.java index beb8b044d1a..ad6ca7520ac 100755 --- a/storage/src/main/java/org/zstack/storage/snapshot/VolumeSnapshotTreeBase.java +++ b/storage/src/main/java/org/zstack/storage/snapshot/VolumeSnapshotTreeBase.java @@ -1842,6 +1842,7 @@ private void createImageCache(final CreateImageCacheFromVolumeSnapshotMsg msg, f CreateImageCacheFromVolumeSnapshotOnPrimaryStorageMsg cmsg = new CreateImageCacheFromVolumeSnapshotOnPrimaryStorageMsg(); cmsg.setImageInventory(image); cmsg.setVolumeSnapshot(VolumeSnapshotInventory.valueOf(currentRoot)); + cmsg.setEncrypted(msg.getEncrypted()); cmsg.setSystemTags(msg.getSystemTags()); bus.makeTargetServiceIdByResourceUuid(cmsg, PrimaryStorageConstant.SERVICE_ID, cmsg.getPrimaryStorageUuid()); bus.send(cmsg, new CloudBusCallBack(msg, completion) { diff --git a/storage/src/main/java/org/zstack/storage/volume/VolumeBase.java b/storage/src/main/java/org/zstack/storage/volume/VolumeBase.java index f0cd6d4a511..05524bc924f 100755 --- a/storage/src/main/java/org/zstack/storage/volume/VolumeBase.java +++ b/storage/src/main/java/org/zstack/storage/volume/VolumeBase.java @@ -1,5 +1,6 @@ package org.zstack.storage.volume; +import org.apache.commons.lang.StringUtils; import org.springframework.beans.factory.annotation.Autowire; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Configurable; @@ -103,6 +104,8 @@ public class VolumeBase extends AbstractVolume implements Volume { private VmInstanceResourceMetadataManager vidm; @Autowired private StorageTrash trash; + @Autowired + private VolumeInPlaceEncryptor volumeInPlaceEncryptor; public VolumeBase(VolumeVO vo) { self = vo; @@ -170,6 +173,8 @@ private void handleLocalMessage(Message msg) { handle((FlattenVolumeMsg) msg); } else if (msg instanceof CancelFlattenVolumeMsg) { handle((CancelFlattenVolumeMsg) msg); + } else if (msg instanceof EncryptVolumeMsg) { + handle((EncryptVolumeMsg) msg); } else { bus.dealWithUnknownMessage(msg); } @@ -270,6 +275,7 @@ public void run(final FlowTrigger trigger, Map data) { rmsg.setVolume(rootVolumeInventory); rmsg.setOriginSize(originSize); rmsg.setAllocatedInstallUrl(allocatedInstallUrl); + rmsg.setHostUuid(msg.getHostUuid()); bus.makeTargetServiceIdByResourceUuid(rmsg, PrimaryStorageConstant.SERVICE_ID, rootVolumeInventory.getPrimaryStorageUuid()); bus.send(rmsg, new CloudBusCallBack(trigger) { @Override @@ -609,6 +615,7 @@ private void prepareMsg(InstantiateVolumeMsg msg, InstantiateVolumeOnPrimaryStor imsg.setSystemTags(msg.getSystemTags()); imsg.setSkipIfExisting(msg.isSkipIfExisting()); imsg.setAllocatedInstallUrl(msg.getAllocatedInstallUrl()); + imsg.setVolumeLuksAgentSpec(msg.getVolumeLuksAgentSpec()); if (msg.getHostUuid() != null) { imsg.setDestHost(HostInventory.valueOf(dbf.findByUuid(msg.getHostUuid(), HostVO.class))); } @@ -3301,6 +3308,45 @@ public void fail(ErrorCode errorCode) { } } + /** + * Converts this volume's bits to LUKS-encrypted form in place. The heavy lifting + * (key materialization, host secret staging, PS-side LUKS conversion, DB update) + * is delegated to {@link VolumeInPlaceEncryptor} so both this message handler and + * {@code VolumeManagerImpl}'s create-data-volume-from-template flow share a single + * implementation. + */ + private void handle(EncryptVolumeMsg msg) { + EncryptVolumeReply reply = new EncryptVolumeReply(); + refreshVO(); + + if (self == null) { + reply.setError(operr("volume[uuid:%s] has been deleted", msg.getVolumeUuid())); + bus.reply(msg, reply); + return; + } + + VolumeInPlaceEncryptor.Context ctx = new VolumeInPlaceEncryptor.Context() + .setHostUuid(msg.getHostUuid()) + .setPrimaryStorageUuid(msg.getPrimaryStorageUuid()) + .setInstallPath(msg.getInstallPath()) + .setPurpose(msg.getPurpose()); + + volumeInPlaceEncryptor.encryptInPlace(self, ctx, new ReturnValueCompletion(msg) { + @Override + public void success(VolumeVO latest) { + self = latest; + reply.setInventory(getSelfInventory()); + bus.reply(msg, reply); + } + + @Override + public void fail(ErrorCode errorCode) { + reply.setError(errorCode); + bus.reply(msg, reply); + } + }); + } + @VmAttachVolumeValidatorMethod static void vmAttachVolumeValidator(VmInstanceInventory vmInv, String volumeUuid) { String vmUuid = vmInv.getUuid(); diff --git a/storage/src/main/java/org/zstack/storage/volume/VolumeInPlaceEncryptor.java b/storage/src/main/java/org/zstack/storage/volume/VolumeInPlaceEncryptor.java new file mode 100644 index 00000000000..b454fc9f55d --- /dev/null +++ b/storage/src/main/java/org/zstack/storage/volume/VolumeInPlaceEncryptor.java @@ -0,0 +1,264 @@ +package org.zstack.storage.volume; + +import org.apache.commons.lang.StringUtils; +import org.springframework.beans.factory.annotation.Autowire; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Configurable; +import org.zstack.core.cloudbus.CloudBus; +import org.zstack.core.cloudbus.CloudBusCallBack; +import org.zstack.core.db.DatabaseFacade; +import org.zstack.header.core.ReturnValueCompletion; +import org.zstack.header.errorcode.ErrorCode; +import org.zstack.header.host.HostConstant; +import org.zstack.header.keyprovider.EncryptedResourceKeyManager; +import org.zstack.header.message.MessageReply; +import org.zstack.header.secret.SecretHostEnsureLuksSecretFileMsg; +import org.zstack.header.secret.SecretHostEnsureLuksSecretFileReply; +import org.zstack.header.storage.primary.EncryptVolumeBitsOnPrimaryStorageMsg; +import org.zstack.header.storage.primary.PrimaryStorageConstant; +import org.zstack.header.volume.VolumeVO; +import org.zstack.storage.encrypt.VolumeEncryptedResourceKeyBackend; +import org.zstack.utils.Utils; +import org.zstack.utils.logging.CLogger; + +import static org.zstack.core.Platform.operr; + +/** + * Performs an in-place LUKS conversion of an existing volume's bits. + * + *

This is the single source of truth for the "encrypt-in-place" workflow that was + * previously duplicated between {@link VolumeBase#handleMessage} (for the + * {@code EncryptVolumeMsg} entry point) and + * {@link VolumeManagerImpl} (for the create-data-volume-from-template flow). + * + *

Steps performed: + *

    + *
  1. Ensure a key-provider binding exists for the volume; auto-attach the default + * provider when none is bound yet.
    2. + *
  2. Materialize a DEK via {@link EncryptedResourceKeyManager#getOrCreateKey}.
    3. + *
  3. Stage the LUKS secret material file on the target host via + * {@code SecretHostEnsureLuksSecretFileMsg}.
    4. + *
  4. Ask the primary storage backend to LUKS-convert the bits in place via + * {@code EncryptVolumeBitsOnPrimaryStorageMsg}.
    5. + *
  5. Persist {@code VolumeVO.encrypted = true} after a successful conversion.
    6. + *
+ * + *

Idempotency: when {@code volume.encrypted == true} the helper treats the volume as + * already converted and short-circuits with success. Callers must therefore not pre-mark + * the row encrypted before invoking this helper -- the encrypted flag is the single + * authoritative signal that "the bits on disk are already LUKS". + */ +@Configurable(preConstruction = true, autowire = Autowire.BY_TYPE) +public class VolumeInPlaceEncryptor { + private static final CLogger logger = Utils.getLogger(VolumeInPlaceEncryptor.class); + + @Autowired + private CloudBus bus; + @Autowired + private DatabaseFacade dbf; + @Autowired + private EncryptedResourceKeyManager encryptedResourceKeyManager; + @Autowired + private VolumeEncryptedResourceKeyBackend volumeEncryptedResourceKeyBackend; + + /** + * Inputs that don't live on {@link VolumeVO} (host to stage the secret on, overrides + * for installPath / primaryStorageUuid when the caller already knows them, etc.). + */ + public static class Context { + private String hostUuid; + /** Optional; falls back to {@code volume.getPrimaryStorageUuid()}. */ + private String primaryStorageUuid; + /** Optional; falls back to {@code volume.getInstallPath()}. */ + private String installPath; + /** Free-form purpose label for the DEK get-or-create audit trail. */ + private String purpose; + + public String getHostUuid() { + return hostUuid; + } + + public Context setHostUuid(String hostUuid) { + this.hostUuid = hostUuid; + return this; + } + + public String getPrimaryStorageUuid() { + return primaryStorageUuid; + } + + public Context setPrimaryStorageUuid(String primaryStorageUuid) { + this.primaryStorageUuid = primaryStorageUuid; + return this; + } + + public String getInstallPath() { + return installPath; + } + + public Context setInstallPath(String installPath) { + this.installPath = installPath; + return this; + } + + public String getPurpose() { + return purpose; + } + + public Context setPurpose(String purpose) { + this.purpose = purpose; + return this; + } + } + + /** + * Run the encrypt-in-place workflow. + * + * @param volume the (already-persisted) target volume; the caller is responsible + * for having a fresh row before invoking + * @param ctx host / installPath / purpose + * @param completion success returns the latest {@code VolumeVO} (encrypted row when + * the workflow actually ran, or the original row when it was a + * no-op short-circuit) + */ + public void encryptInPlace(VolumeVO volume, Context ctx, ReturnValueCompletion completion) { + if (volume == null) { + completion.fail(operr("encrypt-in-place: volume is null")); + return; + } + + // Idempotent short-circuit. The encrypted flag is the authoritative signal that + // the on-disk bits are already in LUKS form (this helper is the only place that + // flips it to true, and it does so only after a successful qemu-img convert). + if (volume.isEncrypted()) { + completion.success(volume); + return; + } + + if (StringUtils.isBlank(ctx.getHostUuid())) { + completion.fail(operr( + "cannot encrypt volume[uuid:%s] in place: hostUuid is required to stage LUKS secret", + volume.getUuid())); + return; + } + + final String installPath = StringUtils.isNotBlank(ctx.getInstallPath()) + ? ctx.getInstallPath() : volume.getInstallPath(); + if (StringUtils.isBlank(installPath)) { + completion.fail(operr( + "cannot encrypt volume[uuid:%s] in place: installPath unknown (volume not instantiated?)", + volume.getUuid())); + return; + } + + final String psUuid = StringUtils.isNotBlank(ctx.getPrimaryStorageUuid()) + ? ctx.getPrimaryStorageUuid() : volume.getPrimaryStorageUuid(); + if (StringUtils.isBlank(psUuid)) { + completion.fail(operr( + "cannot encrypt volume[uuid:%s] in place: primaryStorageUuid unknown", + volume.getUuid())); + return; + } + + // 1) Ensure a key-provider binding exists; auto-attach the default provider when missing. + // Binding is no longer eagerly performed at volume-create time, so this helper is the + // canonical attach point for the encrypt-from-template and encrypt-existing-volume paths + // (the regular instantiate path is covered by VolumeEncryptedInitialExtension). + String kpUuid = volumeEncryptedResourceKeyBackend.findKeyProviderUuidByVolume(volume.getUuid()); + if (StringUtils.isBlank(kpUuid)) { + kpUuid = volumeEncryptedResourceKeyBackend.defaultKeyProviderUuid(); + if (StringUtils.isBlank(kpUuid)) { + completion.fail(operr( + "cannot encrypt volume[uuid:%s] in place: no key provider bound and no default key provider configured", + volume.getUuid())); + return; + } + volumeEncryptedResourceKeyBackend.attachKeyProviderToVolume(volume.getUuid(), kpUuid); + } + + // 2) Materialize the DEK (idempotent: get-or-create). + EncryptedResourceKeyManager.GetOrCreateResourceKeyContext keyCtx = + new EncryptedResourceKeyManager.GetOrCreateResourceKeyContext(); + keyCtx.setResourceUuid(volume.getUuid()); + keyCtx.setResourceType(VolumeVO.class.getSimpleName()); + keyCtx.setKeyProviderUuid(kpUuid); + keyCtx.setPurpose(StringUtils.defaultIfBlank(ctx.getPurpose(), "encrypt-volume-in-place")); + + final EncryptedResourceKeyManager.ResourceKeyResult[] keyResultRef = + new EncryptedResourceKeyManager.ResourceKeyResult[1]; + final ErrorCode[] keyErrorRef = new ErrorCode[1]; + encryptedResourceKeyManager.getOrCreateKey(keyCtx, + new ReturnValueCompletion(completion) { + @Override + public void success(EncryptedResourceKeyManager.ResourceKeyResult r) { + keyResultRef[0] = r; + } + + @Override + public void fail(ErrorCode err) { + keyErrorRef[0] = err; + } + }); + if (keyErrorRef[0] != null) { + completion.fail(operr( + "failed to materialize encryption key for volume[uuid:%s]", + volume.getUuid()).withCause(keyErrorRef[0])); + return; + } + final String dekBase64 = keyResultRef[0].getDekBase64(); + if (StringUtils.isBlank(dekBase64)) { + completion.fail(operr( + "encryption key manager returned empty DEK for volume[uuid:%s]", + volume.getUuid())); + return; + } + + // 3) Stage the LUKS secret material file on the host. + SecretHostEnsureLuksSecretFileMsg ensureMsg = new SecretHostEnsureLuksSecretFileMsg(); + ensureMsg.setHostUuid(ctx.getHostUuid()); + ensureMsg.setDekBase64(dekBase64); + bus.makeTargetServiceIdByResourceUuid(ensureMsg, HostConstant.SERVICE_ID, ctx.getHostUuid()); + + MessageReply ensureReply = bus.call(ensureMsg); + if (!ensureReply.isSuccess()) { + completion.fail(operr( + "failed to prepare LUKS secret material file for volume[uuid:%s] on host[uuid:%s]", + volume.getUuid(), ctx.getHostUuid()) + .withCause(ensureReply.getError())); + return; + } + SecretHostEnsureLuksSecretFileReply er = ensureReply.castReply(); + if (StringUtils.isBlank(er.getSecFilePath())) { + completion.fail(operr( + "ensure LUKS secret file on host succeeded but secFilePath is empty, host[uuid:%s]", + ctx.getHostUuid())); + return; + } + final String secFilePath = er.getSecFilePath(); + + // 4) Ask the PS backend to LUKS-convert the bits in place. + EncryptVolumeBitsOnPrimaryStorageMsg emsg = new EncryptVolumeBitsOnPrimaryStorageMsg(); + emsg.setPrimaryStorageUuid(psUuid); + emsg.setHostUuid(ctx.getHostUuid()); + emsg.setVolumeUuid(volume.getUuid()); + emsg.setInstallPath(installPath); + emsg.setEncryptLuksSecretMaterialFilePath(secFilePath); + bus.makeTargetServiceIdByResourceUuid(emsg, PrimaryStorageConstant.SERVICE_ID, psUuid); + bus.send(emsg, new CloudBusCallBack(completion) { + @Override + public void run(MessageReply r) { + if (!r.isSuccess()) { + completion.fail(r.getError()); + return; + } + // 5) Persist encrypted=true. The short-circuit above guarantees we only + // reach here when the row was previously encrypted=false; this is the + // one and only place the flag flips, ensuring it always reflects the + // on-disk reality. + volume.setEncrypted(true); + VolumeVO latest = dbf.updateAndRefresh(volume); + completion.success(latest); + } + }); + } +} diff --git a/storage/src/main/java/org/zstack/storage/volume/VolumeManagerImpl.java b/storage/src/main/java/org/zstack/storage/volume/VolumeManagerImpl.java index 88aa63237c5..34c99f1c631 100755 --- a/storage/src/main/java/org/zstack/storage/volume/VolumeManagerImpl.java +++ b/storage/src/main/java/org/zstack/storage/volume/VolumeManagerImpl.java @@ -42,6 +42,7 @@ import org.zstack.header.storage.snapshot.*; import org.zstack.header.storage.snapshot.group.VolumeSnapshotGroupVO; import org.zstack.header.storage.snapshot.group.VolumeSnapshotGroupVO_; +import org.zstack.storage.encrypt.VolumeSnapshotEncryptionHelper; import org.zstack.header.vm.*; import org.zstack.header.vm.devices.VmInstanceResourceMetadataManager; import org.zstack.header.volume.*; @@ -92,6 +93,10 @@ public class VolumeManagerImpl extends AbstractService implements VolumeManager, private PluginRegistry pluginRgty; @Autowired private VmInstanceResourceMetadataManager vidm; + @Autowired + private VolumeInPlaceEncryptor volumeInPlaceEncryptor; + @Autowired + private VolumeSnapshotEncryptionHelper snapshotEncryptionHelper; private Future volumeExpungeTask; @@ -239,6 +244,13 @@ private void handle(CreateDataVolumeFromVolumeTemplateMsg msg) { vol.setPrimaryStorageUuid(msg.getPrimaryStorageUuid()); vol.setAccountUuid(msg.getAccountUuid()); vol.setShareable(getShareableCapabilityFromMsg(msg)); + // Do not pre-mark encrypted here. The template bits we are about to download + // are plain; if we set encrypted=true now, encryptInPlace's idempotent + // short-circuit (volume.isEncrypted() => no-op) would skip the actual + // qemu-img convert and we'd be left with a row claiming encryption while + // the file is plain. encryptInPlace itself sets encrypted=true once the + // conversion actually succeeds. + vol.setEncrypted(false); if (msg.getSystemTags() != null) { Iterator iterators = msg.getSystemTags().iterator(); @@ -461,6 +473,57 @@ public void rollback(FlowRollback trigger, Map data) { } }); + flow(new NoRollbackFlow() { + String __name__ = String.format("encrypt data volume %s in place if needed", vol.getUuid()); + + @Override + public boolean skip(Map data) { + if (!Boolean.TRUE.equals(msg.getEncrypted())) { + return true; + } + // Like non-fast clone, template bits are already LUKS — skip encryptInPlace, + // key inheritance handled below + return isTemplateFromEncryptedSource(msg.getImageUuid()); + } + + @Override + public void run(FlowTrigger trigger, Map data) { + VolumeInPlaceEncryptor.Context ctx = new VolumeInPlaceEncryptor.Context() + .setHostUuid(msg.getHostUuid()) + .setPrimaryStorageUuid(targetPrimaryStorage.getUuid()) + .setInstallPath(primaryStorageInstallPath) + .setPurpose("create-data-volume-from-template"); + volumeInPlaceEncryptor.encryptInPlace(vol, ctx, new ReturnValueCompletion(trigger) { + @Override + public void success(VolumeVO latest) { + trigger.next(); + } + + @Override + public void fail(ErrorCode errorCode) { + trigger.fail(errorCode); + } + }); + } + }); + + flow(new NoRollbackFlow() { + String __name__ = String.format("inherit key for encrypted data volume %s from snapshot template", vol.getUuid()); + + @Override + public boolean skip(Map data) { + return !Boolean.TRUE.equals(msg.getEncrypted()) || !isTemplateFromEncryptedSource(msg.getImageUuid()); + } + + @Override + public void run(FlowTrigger trigger, Map data) { + VolumeVO latest = dbf.reload(vol); + snapshotEncryptionHelper.inheritFromTemporarySnapshotImageKeyIfPossible(latest); + SQL.New(VolumeVO.class).eq(VolumeVO_.uuid, vol.getUuid()).set(VolumeVO_.encrypted, true).update(); + trigger.next(); + } + }); + flow(new NoRollbackFlow() { String __name__ = String.format("sync volume %s size", vol.getUuid()); @@ -586,6 +649,7 @@ private VolumeInventory createVolume(CreateVolumeMsg msg) { vo.setStatus(VolumeStatus.NotInstantiated); vo.setType(VolumeType.valueOf(msg.getVolumeType())); vo.setDiskOfferingUuid(msg.getDiskOfferingUuid()); + vo.setEncrypted(Boolean.TRUE.equals(msg.getEncrypted())); if (vo.getType() == VolumeType.Root) { vo.setDeviceId(0); } @@ -638,6 +702,42 @@ protected VolumeVO scripts() { return inv; } + private boolean isTemplateFromEncryptedSource(String imageUuid) { + if (StringUtils.isBlank(imageUuid)) { + return false; + } + + String imageUrl = Q.New(ImageVO.class) + .eq(ImageVO_.uuid, imageUuid) + .select(ImageVO_.url) + .findValue(); + if (StringUtils.isBlank(imageUrl)) { + return false; + } + + if (imageUrl.startsWith("volume://")) { + String srcVolumeUuid = imageUrl.substring("volume://".length()); + return Boolean.TRUE.equals(Q.New(VolumeVO.class) + .eq(VolumeVO_.uuid, srcVolumeUuid) + .select(VolumeVO_.encrypted) + .findValue()); + } + + String snapshotUuid; + if (imageUrl.startsWith(ImageConstant.IMAGE_FROM_SNAPSHOT_SCHEMA)) { + snapshotUuid = imageUrl.substring(ImageConstant.IMAGE_FROM_SNAPSHOT_SCHEMA.length()); + } else if (imageUrl.startsWith(ImageConstant.SNAPSHOT_REUSE_IMAGE_SCHEMA)) { + snapshotUuid = imageUrl.substring(ImageConstant.SNAPSHOT_REUSE_IMAGE_SCHEMA.length()); + } else { + return false; + } + snapshotUuid = snapshotUuid.length() >= 32 ? snapshotUuid.substring(0, 32) : snapshotUuid; + return Boolean.TRUE.equals(Q.New(VolumeSnapshotVO.class) + .eq(VolumeSnapshotVO_.uuid, snapshotUuid) + .select(VolumeSnapshotVO_.encrypted) + .findValue()); + } + private void handle(CreateVolumeMsg msg) { VolumeInventory inv = createVolume(msg); CreateVolumeReply reply = new CreateVolumeReply(); @@ -827,6 +927,11 @@ private void handle(CreateDataVolumeFromVolumeSnapshotMsg msg) { vo.setStatus(VolumeStatus.Creating); vo.setType(VolumeType.Data); vo.setSize(msg.getSize() != null ? msg.getSize() : 0); + Boolean snapshotEncrypted = Q.New(VolumeSnapshotVO.class) + .eq(VolumeSnapshotVO_.uuid, msg.getVolumeSnapshotUuid()) + .select(VolumeSnapshotVO_.encrypted) + .findValue(); + vo.setEncrypted(Boolean.TRUE.equals(msg.getEncrypted()) || Boolean.TRUE.equals(snapshotEncrypted)); vo.setAccountUuid(msg.getSession().getAccountUuid()); VolumeVO vvo = new SQLBatchWithReturn() { @Override @@ -848,6 +953,10 @@ protected VolumeVO scripts() { }.execute(); new FireVolumeCanonicalEvent().fireVolumeStatusChangedEvent(null, VolumeInventory.valueOf(vvo)); + for (CreateDataVolumeExtensionPoint ext : pluginRgty.getExtensionList(CreateDataVolumeExtensionPoint.class)) { + ext.afterCreateVolume(vvo, msg.getVolumeSnapshotUuid()); + } + vvo = dbf.reload(vvo); instantiateDataVolumeFromSnapshot(vo, msg.getVolumeSnapshotUuid(), msg.getSystemTags(), new ReturnValueCompletion(msg) { @Override @@ -908,7 +1017,13 @@ public void run(MessageReply reply) { new FireVolumeCanonicalEvent().fireVolumeStatusChangedEvent(VolumeStatus.Creating, VolumeInventory.valueOf(vvo)); completion.success(VolumeInventory.valueOf(vvo)); } else { - dbf.removeByPrimaryKey(vo.getUuid(), VolumeVO.class); + vvo = dbf.reload(vo); + if (vvo != null) { + VolumeInventory inventory = VolumeInventory.valueOf(vvo); + CollectionUtils.safeForEach(pluginRgty.getExtensionList(VolumeJustBeforeDeleteFromDbExtensionPoint.class), + ext -> ext.volumeJustBeforeDeleteFromDb(inventory)); + dbf.remove(vvo); + } completion.fail(reply.getError()); } } diff --git a/storage/src/main/java/org/zstack/storage/volume/VolumeSystemTags.java b/storage/src/main/java/org/zstack/storage/volume/VolumeSystemTags.java index 996dfec92b0..ab99d063983 100644 --- a/storage/src/main/java/org/zstack/storage/volume/VolumeSystemTags.java +++ b/storage/src/main/java/org/zstack/storage/volume/VolumeSystemTags.java @@ -49,4 +49,9 @@ public class VolumeSystemTags { public static String VOLUME_QOS_TOKEN = "qos"; public static PatternedSystemTag VOLUME_QOS = new PatternedSystemTag(String.format("%s::{%s}", VOLUME_QOS_TOKEN, VOLUME_QOS_TOKEN), VolumeVO.class); + + @NonCloneable + public static String VOLUME_LIBVIRT_SECRET_HOST_TOKEN = "hostUuid"; + public static PatternedSystemTag VOLUME_LIBVIRT_SECRET_HOST = new PatternedSystemTag( + String.format("volumeLibvirtSecretHost::{%s}", VOLUME_LIBVIRT_SECRET_HOST_TOKEN), VolumeVO.class); }