KVM: DPDK live migrations

api/src/main/java/com/cloud/agent/api/to/DpdkTO.java

@@ -0,0 +1,45 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package com.cloud.agent.api.to;
+
+public class DpdkTO {
+
+    private String path;
+    private String port;
+    private String mode;
+
+    public DpdkTO() {
+    }
+
+    public DpdkTO(String path, String port, String mode) {
+        this.path = path;
+        this.port = port;
+        this.mode = mode;
+    }
+
+    public String getPath() {
+        return path;
+    }
+
+    public String getPort() {
+        return port;
+    }
+
+    public String getMode() {
+        return mode;
+    }
+}

api/src/main/java/com/cloud/agent/api/to/NicTO.java

@@ -30,7 +30,7 @@ public class NicTO extends NetworkTO {
     String nicUuid;
     List<String> nicSecIps;
     Map<NetworkOffering.Detail, String> details;
-    boolean dpdkDisabled;
+    boolean dpdkEnabled;
 
     public NicTO() {
         super();
@@ -111,11 +111,11 @@ public void setDetails(final Map<NetworkOffering.Detail, String> details) {
         this.details = details;
     }
 
-    public boolean isDpdkDisabled() {
-        return dpdkDisabled;
+    public boolean isDpdkEnabled() {
+        return dpdkEnabled;
     }
 
-    public void setDpdkDisabled(boolean dpdkDisabled) {
-        this.dpdkDisabled = dpdkDisabled;
+    public void setDpdkEnabled(boolean dpdkEnabled) {
+        this.dpdkEnabled = dpdkEnabled;
     }
 }

core/src/main/java/com/cloud/agent/api/MigrateCommand.java

@@ -24,6 +24,7 @@
 import java.util.List;
 import java.util.Map;
 
+import com.cloud.agent.api.to.DpdkTO;
 import com.cloud.agent.api.to.VirtualMachineTO;
 
 public class MigrateCommand extends Command {
@@ -37,6 +38,15 @@ public class MigrateCommand extends Command {
     private VirtualMachineTO vmTO;
     private boolean executeInSequence = false;
     private List<MigrateDiskInfo> migrateDiskInfoList = new ArrayList<>();
+    private Map<String, DpdkTO> dpdkInterfaceMapping = new HashMap<>();
+
+    public Map<String, DpdkTO> getDpdkInterfaceMapping() {
+        return dpdkInterfaceMapping;
+    }
+
+    public void setDpdkInterfaceMapping(Map<String, DpdkTO> dpdkInterfaceMapping) {
+        this.dpdkInterfaceMapping = dpdkInterfaceMapping;
+    }
 
     protected MigrateCommand() {
     }

core/src/main/java/com/cloud/agent/api/PrepareForMigrationAnswer.java

@@ -19,7 +19,15 @@
 
 package com.cloud.agent.api;
 
+import com.cloud.agent.api.to.DpdkTO;
+
+import java.util.HashMap;
+import java.util.Map;
+
 public class PrepareForMigrationAnswer extends Answer {
+
+    private Map<String, DpdkTO> dpdkInterfaceMapping = new HashMap<>();
+
     protected PrepareForMigrationAnswer() {
     }
 
@@ -34,4 +42,12 @@ public PrepareForMigrationAnswer(PrepareForMigrationCommand cmd, Exception ex) {
     public PrepareForMigrationAnswer(PrepareForMigrationCommand cmd) {
         super(cmd, true, null);
     }
+
+    public void setDpdkInterfaceMapping(Map<String, DpdkTO> mapping) {
+        this.dpdkInterfaceMapping = mapping;
+    }
+
+    public Map<String, DpdkTO> getDpdkInterfaceMapping() {
+        return this.dpdkInterfaceMapping;
+    }
 }

core/src/main/java/com/cloud/agent/api/StopCommand.java

@@ -19,6 +19,7 @@
 
 package com.cloud.agent.api;
 
+import com.cloud.agent.api.to.DpdkTO;
 import com.cloud.agent.api.to.GPUDeviceTO;
 import com.cloud.vm.VirtualMachine;
 
@@ -34,6 +35,15 @@ public class StopCommand extends RebootCommand {
     boolean checkBeforeCleanup = false;
     String controlIp = null;
     boolean forceStop = false;
+    private Map<String, DpdkTO> dpdkInterfaceMapping;
+
+    public Map<String, DpdkTO> getDpdkInterfaceMapping() {
+        return dpdkInterfaceMapping;
+    }
+
+    public void setDpdkInterfaceMapping(Map<String, DpdkTO> dpdkInterfaceMapping) {
+        this.dpdkInterfaceMapping = dpdkInterfaceMapping;
+    }
     /**
      * On KVM when using iSCSI-based managed storage, if the user shuts a VM down from the guest OS (as opposed to doing so from CloudStack),
      * we need to pass to the KVM agent a list of applicable iSCSI volumes that need to be disconnected.

engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java

@@ -39,7 +39,8 @@
 import javax.inject.Inject;
 import javax.naming.ConfigurationException;
 
-import org.apache.cloudstack.api.ApiConstants;
+import com.cloud.agent.api.PrepareForMigrationAnswer;
+import com.cloud.agent.api.to.DpdkTO;
 import org.apache.cloudstack.affinity.dao.AffinityGroupVMMapDao;
 import org.apache.cloudstack.api.command.admin.vm.MigrateVMCmd;
 import org.apache.cloudstack.api.command.admin.volume.MigrateVolumeCmdByAdmin;
@@ -1118,8 +1119,6 @@ public void orchestrateStart(final String vmUuid, final Map<VirtualMachineProfil
 
                     vmGuru.finalizeDeployment(cmds, vmProfile, dest, ctx);
 
-                    addExtraConfig(vmTO);
-
                     work = _workDao.findById(work.getId());
                     if (work == null || work.getStep() != Step.Prepare) {
                         throw new ConcurrentOperationException("Work steps have been changed: " + work);
@@ -1284,15 +1283,6 @@ public void orchestrateStart(final String vmUuid, final Map<VirtualMachineProfil
         }
     }
 
-    private void addExtraConfig(VirtualMachineTO vmTO) {
-        Map<String, String> details = vmTO.getDetails();
-        for (String key : details.keySet()) {
-            if (key.startsWith(ApiConstants.EXTRA_CONFIG)) {
-                vmTO.addExtraConfig(key, details.get(key));
-            }
-        }
-    }
-
     // for managed storage on KVM, need to make sure the path field of the volume in question is populated with the IQN
     private void handlePath(final DiskTO[] disks, final HypervisorType hypervisorType) {
         if (hypervisorType != HypervisorType.KVM) {
@@ -2362,6 +2352,7 @@ protected void migrate(final VMInstanceVO vm, final long srcHostId, final Deploy
         }
 
         boolean migrated = false;
+        Map<String, DpdkTO> dpdkInterfaceMapping = null;
         try {
             final boolean isWindows = _guestOsCategoryDao.findById(_guestOsDao.findById(vm.getGuestOSId()).getCategoryId()).getName().equalsIgnoreCase("Windows");
             final MigrateCommand mc = new MigrateCommand(vm.getInstanceName(), dest.getHost().getPrivateIpAddress(), isWindows, to, getExecuteInSequence(vm.getHypervisorType()));
@@ -2370,6 +2361,11 @@ protected void migrate(final VMInstanceVO vm, final long srcHostId, final Deploy
             mc.setAutoConvergence(kvmAutoConvergence);
             mc.setHostGuid(dest.getHost().getGuid());
 
+            dpdkInterfaceMapping = ((PrepareForMigrationAnswer) pfma).getDpdkInterfaceMapping();
+            if (MapUtils.isNotEmpty(dpdkInterfaceMapping)) {
+                mc.setDpdkInterfaceMapping(dpdkInterfaceMapping);
+            }
+
             try {
                 final Answer ma = _agentMgr.send(vm.getLastHostId(), mc);
                 if (ma == null || !ma.getResult()) {
@@ -2396,7 +2392,7 @@ protected void migrate(final VMInstanceVO vm, final long srcHostId, final Deploy
                 if (!checkVmOnHost(vm, dstHostId)) {
                     s_logger.error("Unable to complete migration for " + vm);
                     try {
-                        _agentMgr.send(srcHostId, new Commands(cleanup(vm)), null);
+                        _agentMgr.send(srcHostId, new Commands(cleanup(vm, dpdkInterfaceMapping)), null);
                     } catch (final AgentUnavailableException e) {
                         s_logger.error("AgentUnavailableException while cleanup on source host: " + srcHostId);
                     }
@@ -2417,7 +2413,7 @@ protected void migrate(final VMInstanceVO vm, final long srcHostId, final Deploy
                         "Unable to migrate vm " + vm.getInstanceName() + " from host " + fromHost.getName() + " in zone " + dest.getDataCenter().getName() + " and pod " +
                                 dest.getPod().getName(), "Migrate Command failed.  Please check logs.");
                 try {
-                    _agentMgr.send(dstHostId, new Commands(cleanup(vm)), null);
+                    _agentMgr.send(dstHostId, new Commands(cleanup(vm, dpdkInterfaceMapping)), null);
                 } catch (final AgentUnavailableException ae) {
                     s_logger.info("Looks like the destination Host is unavailable for cleanup");
                 }
@@ -3106,9 +3102,12 @@ private void orchestrateReboot(final String vmUuid, final Map<VirtualMachineProf
         }
     }
 
-    public Command cleanup(final VirtualMachine vm) {
+    public Command cleanup(final VirtualMachine vm, Map<String, DpdkTO> dpdkInterfaceMapping) {
         StopCommand cmd = new StopCommand(vm, getExecuteInSequence(vm.getHypervisorType()), false);
         cmd.setControlIp(getControlNicIpForVM(vm));
+        if (MapUtils.isNotEmpty(dpdkInterfaceMapping)) {
+            cmd.setDpdkInterfaceMapping(dpdkInterfaceMapping);
+        }
         return cmd;
     }
 

plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/dpdk/DPDKDriver.java → plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/dpdk/DpdkDriver.java

@@ -22,7 +22,7 @@
 
 import java.util.Map;
 
-public interface DPDKDriver extends Adapter {
+public interface DpdkDriver extends Adapter {
 
     /**
      * Get the next DPDK port name to be created
@@ -37,17 +37,17 @@ public interface DPDKDriver extends Adapter {
     /**
      * Add OVS port (if it does not exist) to bridge with DPDK support
      */
-    void addDpdkPort(String bridgeName, String port, String vlan, DPDKHelper.VHostUserMode vHostUserMode, String dpdkOvsPath);
+    void addDpdkPort(String bridgeName, String port, String vlan, DpdkHelper.VHostUserMode vHostUserMode, String dpdkOvsPath);
 
     /**
      * Since DPDK user client/server mode, retrieve the guest interfaces mode from the DPDK vHost User mode
      */
-    String getGuestInterfacesModeFromDPDKVhostUserMode(DPDKHelper.VHostUserMode dpdKvHostUserMode);
+    String getGuestInterfacesModeFromDpdkVhostUserMode(DpdkHelper.VHostUserMode dpdKvHostUserMode);
 
     /**
      * Get DPDK vHost User mode from extra config. If it is not present, server is returned as default
      */
-    DPDKHelper.VHostUserMode getDPDKvHostUserMode(Map<String, String> extraConfig);
+    DpdkHelper.VHostUserMode getDpdkvHostUserMode(Map<String, String> extraConfig);
 
     /**
      * Check for additional extra 'dpdk-interface' configurations, return them appended

plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/dpdk/DPDKDriverImpl.java → plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/dpdk/DpdkDriverImpl.java

@@ -25,15 +25,15 @@
 
 import java.util.Map;
 
-public class DPDKDriverImpl extends AdapterBase implements DPDKDriver {
+public class DpdkDriverImpl extends AdapterBase implements DpdkDriver {
     static final String DPDK_PORT_PREFIX = "csdpdk-";
 
     private final String dpdkPortVhostUserType = "dpdkvhostuser";
     private final String dpdkPortVhostUserClientType = "dpdkvhostuserclient";
 
-    private static final Logger s_logger = Logger.getLogger(DPDKDriver.class);
+    private static final Logger s_logger = Logger.getLogger(DpdkDriver.class);
 
-    public DPDKDriverImpl() {
+    public DpdkDriverImpl() {
     }
 
     /**
@@ -64,8 +64,8 @@ public int getDpdkLatestPortNumberUsed() {
     /**
      * Add OVS port (if it does not exist) to bridge with DPDK support
      */
-    public void addDpdkPort(String bridgeName, String port, String vlan, DPDKHelper.VHostUserMode vHostUserMode, String dpdkOvsPath) {
-        String type = vHostUserMode == DPDKHelper.VHostUserMode.SERVER ?
+    public void addDpdkPort(String bridgeName, String port, String vlan, DpdkHelper.VHostUserMode vHostUserMode, String dpdkOvsPath) {
+        String type = vHostUserMode == DpdkHelper.VHostUserMode.SERVER ?
                 dpdkPortVhostUserType :
                 dpdkPortVhostUserClientType;
 
@@ -74,7 +74,7 @@ public void addDpdkPort(String bridgeName, String port, String vlan, DPDKHelper.
                 "vlan_mode=access tag=%s " +
                 "-- set Interface %s type=%s", bridgeName, port, vlan, port, type));
 
-        if (vHostUserMode == DPDKHelper.VHostUserMode.CLIENT) {
+        if (vHostUserMode == DpdkHelper.VHostUserMode.CLIENT) {
             stringBuilder.append(String.format(" options:vhost-server-path=%s/%s",
                     dpdkOvsPath, port));
         }
@@ -87,17 +87,17 @@ public void addDpdkPort(String bridgeName, String port, String vlan, DPDKHelper.
     /**
      * Since DPDK user client/server mode, retrieve the guest interfaces mode from the DPDK vHost User mode
      */
-    public String getGuestInterfacesModeFromDPDKVhostUserMode(DPDKHelper.VHostUserMode dpdKvHostUserMode) {
-        return dpdKvHostUserMode == DPDKHelper.VHostUserMode.CLIENT ? "server" : "client";
+    public String getGuestInterfacesModeFromDpdkVhostUserMode(DpdkHelper.VHostUserMode dpdKvHostUserMode) {
+        return dpdKvHostUserMode == DpdkHelper.VHostUserMode.CLIENT ? "server" : "client";
     }
 
     /**
      * Get DPDK vHost User mode from extra config. If it is not present, server is returned as default
      */
-    public DPDKHelper.VHostUserMode getDPDKvHostUserMode(Map<String, String> extraConfig) {
-        return extraConfig.containsKey(DPDKHelper.DPDK_VHOST_USER_MODE) ?
-                DPDKHelper.VHostUserMode.fromValue(extraConfig.get(DPDKHelper.DPDK_VHOST_USER_MODE)) :
-                DPDKHelper.VHostUserMode.SERVER;
+    public DpdkHelper.VHostUserMode getDpdkvHostUserMode(Map<String, String> extraConfig) {
+        return extraConfig.containsKey(DpdkHelper.DPDK_VHOST_USER_MODE) ?
+                DpdkHelper.VHostUserMode.fromValue(extraConfig.get(DpdkHelper.DPDK_VHOST_USER_MODE)) :
+                DpdkHelper.VHostUserMode.SERVER;
     }
 
     /**
@@ -106,7 +106,7 @@ public DPDKHelper.VHostUserMode getDPDKvHostUserMode(Map<String, String> extraCo
     public String getExtraDpdkProperties(Map<String, String> extraConfig) {
         StringBuilder stringBuilder = new StringBuilder();
         for (String key : extraConfig.keySet()) {
-            if (key.startsWith(DPDKHelper.DPDK_INTERFACE_PREFIX)) {
+            if (key.startsWith(DpdkHelper.DPDK_INTERFACE_PREFIX)) {
                 stringBuilder.append(extraConfig.get(key));
             }
         }

plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java

@@ -46,7 +46,7 @@
 import javax.xml.parsers.DocumentBuilderFactory;
 import javax.xml.parsers.ParserConfigurationException;
 
-import com.cloud.hypervisor.kvm.dpdk.DPDKHelper;
+import com.cloud.hypervisor.kvm.dpdk.DpdkHelper;
 import com.cloud.resource.RequestWrapper;
 import org.apache.cloudstack.storage.to.PrimaryDataStoreTO;
 import org.apache.cloudstack.storage.to.TemplateObjectTO;
@@ -2070,7 +2070,7 @@ public LibvirtVMDef createVMFromSpec(final VirtualMachineTO vmTO) {
         vm.setPlatformEmulator(vmTO.getPlatformEmulator());
 
         Map<String, String> extraConfig = vmTO.getExtraConfig();
-        if (dpdkSupport && (!extraConfig.containsKey(DPDKHelper.DPDK_NUMA) || !extraConfig.containsKey(DPDKHelper.DPDK_HUGE_PAGES))) {
+        if (dpdkSupport && (!extraConfig.containsKey(DpdkHelper.DPDK_NUMA) || !extraConfig.containsKey(DpdkHelper.DPDK_HUGE_PAGES))) {
             s_logger.info("DPDK is enabled but it needs extra configurations for CPU NUMA and Huge Pages for VM deployment");
         }
 
@@ -2107,7 +2107,7 @@ public LibvirtVMDef createVMFromSpec(final VirtualMachineTO vmTO) {
         grd.setVcpuNum(vcpus);
         vm.addComp(grd);
 
-        if (!extraConfig.containsKey(DPDKHelper.DPDK_NUMA)) {
+        if (!extraConfig.containsKey(DpdkHelper.DPDK_NUMA)) {
             final CpuModeDef cmd = new CpuModeDef();
             cmd.setMode(_guestCpuMode);
             cmd.setModel(_guestCpuModel);
@@ -2235,7 +2235,7 @@ protected void addExtraConfigComponent(Map<String, String> extraConfig, LibvirtV
         if (MapUtils.isNotEmpty(extraConfig)) {
             StringBuilder extraConfigBuilder = new StringBuilder();
             for (String key : extraConfig.keySet()) {
-                if (!key.startsWith(DPDKHelper.DPDK_INTERFACE_PREFIX) && !key.equals(DPDKHelper.DPDK_VHOST_USER_MODE)) {
+                if (!key.startsWith(DpdkHelper.DPDK_INTERFACE_PREFIX) && !key.equals(DpdkHelper.DPDK_VHOST_USER_MODE)) {
                     extraConfigBuilder.append(extraConfig.get(key));
                 }
             }
@@ -2706,7 +2706,10 @@ public StartupCommand[] initialize() {
 
         final KVMHostInfo info = new KVMHostInfo(_dom0MinMem, _dom0OvercommitMem);
 
-        final String capabilities = String.join(",", info.getCapabilities());
+        String capabilities = String.join(",", info.getCapabilities());
+        if (dpdkSupport) {
+            capabilities += ",dpdk";
+        }
 
         final StartupRoutingCommand cmd =
                 new StartupRoutingCommand(info.getCpus(), info.getCpuSpeed(), info.getTotalMemory(), info.getReservedMemory(), capabilities, _hypervisorType,