Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 12 additions & 0 deletions PendingReleaseNotes
Original file line number Diff line number Diff line change
Expand Up @@ -39,3 +39,15 @@ example.ver.1 > example.ver.2:
which can now be attached to Instances. This is to prevent the Secondary
Storage to grow to enormous sizes as Linux Distributions keep growing in
size while a stripped down Linux should fit on a 2.88MB floppy.


4.22.0 > 4.23.0:
* Added NVMe-over-Fabrics (TCP) support to the adaptive storage framework
and the Pure Storage FlashArray plugin. Volumes on a FlashArray primary
pool can now be delivered to KVM hypervisors over NVMe-TCP instead of
Fibre Channel by setting transport=nvme-tcp on the pool's provider URL.
Volumes are identified on the host via EUI-128 NGUIDs and attached to
guests as plain block devices through the native NVMe multipath layer;
no device-mapper multipath configuration is required. A new
Storage.StoragePoolType.NVMeTCP + MultipathNVMeOFAdapterBase /
NVMeTCPAdapter on the KVM side back the new pool type.
3 changes: 2 additions & 1 deletion api/src/main/java/com/cloud/storage/Storage.java
Original file line number Diff line number Diff line change
Expand Up @@ -185,7 +185,8 @@ public static enum StoragePoolType {
Linstor(true, true, EncryptionSupport.Storage),
DatastoreCluster(true, true, EncryptionSupport.Unsupported), // for VMware, to abstract pool of clusters
StorPool(true, true, EncryptionSupport.Hypervisor),
FiberChannel(true, true, EncryptionSupport.Unsupported); // Fiber Channel Pool for KVM hypervisors is used to find the volume by WWN value (/dev/disk/by-id/wwn-<wwnvalue>)
FiberChannel(true, true, EncryptionSupport.Unsupported), // Fiber Channel Pool for KVM hypervisors is used to find the volume by WWN value (/dev/disk/by-id/wwn-<wwnvalue>)
NVMeTCP(true, true, EncryptionSupport.Unsupported); // NVMe over TCP (NVMe-oF/TCP) Pool for KVM hypervisors; volumes are identified by EUI-128 NGUID (/dev/disk/by-id/nvme-eui.<eui>)

private final boolean shared;
private final boolean overProvisioning;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -415,7 +415,8 @@ public Answer copyTemplateToPrimaryStorage(final CopyCommand cmd) {
StoragePoolType.PowerFlex,
StoragePoolType.Linstor,
StoragePoolType.FiberChannel,
StoragePoolType.CLVM).contains(primaryPool.getType())) {
StoragePoolType.CLVM,
StoragePoolType.NVMeTCP).contains(primaryPool.getType())) {
newTemplate.setFormat(ImageFormat.RAW);
} else {
newTemplate.setFormat(ImageFormat.QCOW2);
Expand Down Expand Up @@ -448,7 +449,8 @@ public Answer copyTemplateToPrimaryStorage(final CopyCommand cmd) {

public static String derivePath(PrimaryDataStoreTO primaryStore, DataTO destData, Map<String, String> details) {
String path = null;
if (primaryStore.getPoolType() == StoragePoolType.FiberChannel) {
if (primaryStore.getPoolType() == StoragePoolType.FiberChannel
|| primaryStore.getPoolType() == StoragePoolType.NVMeTCP) {
path = destData.getPath();
} else {
path = details != null ? details.get("managedStoreTarget") : null;
Expand Down Expand Up @@ -3431,7 +3433,8 @@ private Storage.ImageFormat getFormat(StoragePoolType poolType) {
StoragePoolType.PowerFlex,
StoragePoolType.Linstor,
StoragePoolType.FiberChannel,
StoragePoolType.CLVM).contains(poolType)) {
StoragePoolType.CLVM,
StoragePoolType.NVMeTCP).contains(poolType)) {
return ImageFormat.RAW;
} else {
return ImageFormat.QCOW2;
Expand Down

Large diffs are not rendered by default.

Original file line number Diff line number Diff line change
@@ -0,0 +1,157 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.

package com.cloud.hypervisor.kvm.storage;

import java.util.HashMap;
import java.util.List;
import java.util.Map;

import org.apache.cloudstack.utils.qemu.QemuImg;
import org.apache.cloudstack.utils.qemu.QemuImg.PhysicalDiskFormat;
import org.joda.time.Duration;

import com.cloud.agent.api.to.HostTO;
import com.cloud.hypervisor.kvm.resource.KVMHABase.HAStoragePool;
import com.cloud.storage.Storage;
import com.cloud.storage.Storage.ProvisioningType;

/**
* KVMStoragePool for NVMe-over-Fabrics pools. Mirror of
* {@link MultipathSCSIPool} for adapters based on
* {@link MultipathNVMeOFAdapterBase}. Every data operation is delegated
* back to the adapter; the pool itself only tracks addressing/identity.
*/
public class MultipathNVMeOFPool implements KVMStoragePool {
private final String uuid;
private final String sourceHost;
private final int sourcePort;
private final String sourceDir;
private final Storage.StoragePoolType storagePoolType;
private final StorageAdaptor storageAdaptor;
private final Map<String, String> details;
private long capacity;
private long used;
private long available;

public MultipathNVMeOFPool(String uuid, String host, int port, String path,
Storage.StoragePoolType poolType, Map<String, String> poolDetails, StorageAdaptor adaptor) {
this.uuid = uuid;
this.sourceHost = host;
this.sourcePort = port;
this.sourceDir = path;
this.storagePoolType = poolType;
this.storageAdaptor = adaptor;
this.details = poolDetails;
this.capacity = 0;
this.used = 0;
this.available = 0;
}

public MultipathNVMeOFPool(String uuid, StorageAdaptor adaptor) {
this.uuid = uuid;
this.sourceHost = null;
this.sourcePort = -1;
this.sourceDir = null;
this.storagePoolType = Storage.StoragePoolType.NVMeTCP;
this.storageAdaptor = adaptor;
this.details = new HashMap<>();
this.capacity = 0;
this.used = 0;
this.available = 0;
}

@Override
public KVMPhysicalDisk createPhysicalDisk(String volumeUuid, ProvisioningType provisioningType, long size, byte[] passphrase) {
return null;
}

@Override
public KVMPhysicalDisk createPhysicalDisk(String volumeUuid, PhysicalDiskFormat format, ProvisioningType provisioningType, long size, byte[] passphrase) {
return null;
}
Comment thread
DaanHoogland marked this conversation as resolved.

@Override
public boolean connectPhysicalDisk(String volumeUuid, Map<String, String> details) {
return storageAdaptor.connectPhysicalDisk(volumeUuid, this, details, false);
}

@Override
public KVMPhysicalDisk getPhysicalDisk(String volumeId) {
return storageAdaptor.getPhysicalDisk(volumeId, this);
}

@Override
public boolean disconnectPhysicalDisk(String volumeUuid) {
return storageAdaptor.disconnectPhysicalDisk(volumeUuid, this);
}

@Override
public boolean deletePhysicalDisk(String volumeUuid, Storage.ImageFormat format) {
return true;
}
Comment on lines +103 to +106

@genegr genegr Aug 4, 2026

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'd like to keep this one as return true, for two reasons.

It matches the sibling adapter. MultipathSCSIPool.deletePhysicalDisk (the FC/iSCSI class this one is modelled on) is byte-identical — return true; — as is its listPhysicalDisks() { return null; }. Changing only the NVMe class would leave the two out of step.

Nothing is actually leaked here. For managed storage the namespace is deleted on the provider side: the management server drives FlashArrayAdapter.delete(), which is what issues the destroy against the array. The KVM agent's pool-level deletePhysicalDisk exists for hypervisor-local disk files (a qcow2 on NFS, an LV on CLVM); an NVMe-TCP namespace is not a host-local object, so there is genuinely nothing for this method to do. It isn't "reporting success for work it skipped" so much as "no host-side artifact to remove".

Worth noting too that all three in-tree callers ignore the return value — KVMStorageProcessor.deleteVolume (twice) and deleteBackup call it for side effects only and catch just CloudRuntimeException. So true vs false changes nothing observable, while throwing UnsupportedOperationException would change behaviour: it would turn a harmless no-op into a hard failure on the volume-delete and backup-delete paths.

Happy to switch both classes together if a maintainer would rather have the stricter contract — I just don't want to introduce a new failure mode in this PR for a path I can't exercise.


@Override
public List<KVMPhysicalDisk> listPhysicalDisks() {
return null;
}
Comment thread
DaanHoogland marked this conversation as resolved.

@Override
public String getUuid() {
return uuid;
}

public void setCapacity(long capacity) { this.capacity = capacity; }
@Override public long getCapacity() { return this.capacity; }
public void setUsed(long used) { this.used = used; }
@Override public long getUsed() { return this.used; }
public void setAvailable(long available) { this.available = available; }
@Override public long getAvailable() { return this.available; }

@Override public boolean refresh() { return false; }
@Override public boolean isExternalSnapshot() { return true; }
@Override public String getLocalPath() { return null; }
@Override public String getSourceHost() { return this.sourceHost; }
@Override public String getSourceDir() { return this.sourceDir; }
@Override public int getSourcePort() { return this.sourcePort; }
@Override public String getAuthUserName() { return null; }
@Override public String getAuthSecret() { return null; }
@Override public Storage.StoragePoolType getType() { return storagePoolType; }
@Override public boolean delete() { return false; }
@Override public QemuImg.PhysicalDiskFormat getDefaultFormat() { return QemuImg.PhysicalDiskFormat.RAW; }
@Override public boolean createFolder(String path) { return false; }
@Override public boolean supportsConfigDriveIso() { return false; }
@Override public Map<String, String> getDetails() { return this.details; }
@Override public boolean isPoolSupportHA() { return false; }
@Override public String getHearthBeatPath() { return null; }

@Override
public String createHeartBeatCommand(HAStoragePool primaryStoragePool, String hostPrivateIp, boolean hostValidation) {
return null;
}

@Override public String getStorageNodeId() { return null; }

@Override
public Boolean hasHeartBeat(HAStoragePool pool, HostTO host) { return null; }

@Override
public Boolean hasVmActivity(HAStoragePool pool, HostTO host, Duration activityScriptTimeout,
String volumeUUIDListString, String vmActivityCheckPath, long duration) {
return null;
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,51 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.

package com.cloud.hypervisor.kvm.storage;

import com.cloud.storage.Storage;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;

/**
* StorageAdaptor for the {@link Storage.StoragePoolType#NVMeTCP} pool type.
* All operational logic lives in {@link MultipathNVMeOFAdapterBase}; this
* class just binds that logic to a pool type so
* {@link KVMStoragePoolManager} can find it via reflection.
*/
public class NVMeTCPAdapter extends MultipathNVMeOFAdapterBase {
private static final Logger LOGGER = LogManager.getLogger(NVMeTCPAdapter.class);

public NVMeTCPAdapter() {
LOGGER.info("Loaded NVMeTCPAdapter for StorageLayer");
}

@Override
public String getName() {
return "NVMeTCPAdapter";
}

@Override
public Storage.StoragePoolType getStoragePoolType() {
return Storage.StoragePoolType.NVMeTCP;
}

@Override
public boolean isStoragePoolTypeSupported(Storage.StoragePoolType type) {
return Storage.StoragePoolType.NVMeTCP.equals(type);
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@ public interface ProviderVolume {
public String getExternalName();
public String getExternalConnectionId();
public enum AddressType {
FIBERWWN
FIBERWWN,
NVMETCP
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -179,7 +179,7 @@ public DataStore initialize(Map<String, Object> dsInfos) {
parameters.setHost(uri.getHost());
parameters.setPort(uri.getPort());
parameters.setPath(uri.getPath() + "?" + uri.getQuery());
parameters.setType(StoragePoolType.FiberChannel);
parameters.setType(pickPoolType(uri));
parameters.setZoneId(zoneId);
parameters.setPodId(podId);
parameters.setClusterId(clusterId);
Expand Down Expand Up @@ -402,4 +402,26 @@ public void disableStoragePool(DataStore store) {
logger.info("Disabling storage pool {}", store);
_dataStoreHelper.disable(store);
}

/**
* Resolve the CloudStack StoragePoolType from the provider URL. Adaptive
* plugins advertise the underlying fabric via a {@code transport=} query
* parameter on the URL; when absent we keep the legacy FiberChannel
* default for backwards compatibility with adapters that still assume it.
*/
private static StoragePoolType pickPoolType(java.net.URL uri) {
String query = uri.getQuery();
if (query != null) {
for (String tok : query.split("&")) {
int i = tok.indexOf('=');
if (i > 0 && "transport".equalsIgnoreCase(tok.substring(0, i))) {
String value = tok.substring(i + 1);
if ("nvme-tcp".equalsIgnoreCase(value)) {
return StoragePoolType.NVMeTCP;
}
}
}
}
return StoragePoolType.FiberChannel;
}
}
Loading