CLOUDSTACK-9641 In KVM SSVM and CPVM may use the old cmdline data, if we fail to fetch the new cmdline in the first pass.

Author: bvbharat

core/src/com/cloud/agent/api/StopCommand.java

@@ -28,6 +28,7 @@ public class StopCommand extends RebootCommand {
     private String publicConsoleProxyIpAddress = null;
     private GPUDeviceTO gpuDevice;
     boolean checkBeforeCleanup = false;
+    String controlIp = null;
 
     protected StopCommand() {
     }
@@ -82,4 +83,12 @@ public void setGpuDevice(GPUDeviceTO gpuDevice) {
     public boolean checkBeforeCleanup() {
         return this.checkBeforeCleanup;
     }
+
+    public String getControlIp(){
+        return controlIp;
+    }
+
+    public void setControlIp(String controlIp){
+        this.controlIp =controlIp;
+    }
 }

engine/orchestration/src/com/cloud/vm/VirtualMachineManagerImpl.java

@@ -1072,8 +1072,9 @@ public void orchestrateStart(final String vmUuid, final Map<VirtualMachineProfil
                             if (s_logger.isDebugEnabled()) {
                                 s_logger.info("The guru did not like the answers so stopping " + vm);
                             }
-
-                            final StopCommand cmd = new StopCommand(vm, getExecuteInSequence(vm.getHypervisorType()), false);
+                            StopCommand stopCmd = new StopCommand(vm, getExecuteInSequence(vm.getHypervisorType()), false);
+                            stopCmd.setControlIp(getControlNicIpForVM(vm));
+                            final StopCommand cmd = stopCmd;
                             final Answer answer = _agentMgr.easySend(destHostId, cmd);
                             if (answer != null && answer instanceof StopAnswer) {
                                 final StopAnswer stopAns = (StopAnswer)answer;
@@ -1278,7 +1279,9 @@ public boolean getExecuteInSequence(final HypervisorType hypervisorType) {
 
     protected boolean sendStop(final VirtualMachineGuru guru, final VirtualMachineProfile profile, final boolean force, final boolean checkBeforeCleanup) {
         final VirtualMachine vm = profile.getVirtualMachine();
-        final StopCommand stop = new StopCommand(vm, getExecuteInSequence(vm.getHypervisorType()), checkBeforeCleanup);
+        StopCommand stpCmd = new StopCommand(vm, getExecuteInSequence(vm.getHypervisorType()), checkBeforeCleanup);
+        stpCmd.setControlIp(getControlNicIpForVM(vm));
+        final StopCommand stop = stpCmd;
         try {
             final Answer answer = _agentMgr.send(vm.getHostId(), stop);
             if (answer != null && answer instanceof StopAnswer) {
@@ -1541,8 +1544,9 @@ private void advanceStop(final VMInstanceVO vm, final boolean cleanUpEvenIfUnabl
         }
 
         vmGuru.prepareStop(profile);
-
-        final StopCommand stop = new StopCommand(vm, getExecuteInSequence(vm.getHypervisorType()), false);
+        StopCommand stpCmd = new StopCommand(vm, getExecuteInSequence(vm.getHypervisorType()), false);
+        stpCmd.setControlIp(getControlNicIpForVM(vm));
+        final StopCommand stop = stpCmd;
 
         boolean stopped = false;
         Answer answer = null;
@@ -2669,12 +2673,24 @@ private void orchestrateReboot(final String vmUuid, final Map<VirtualMachineProf
     }
 
     public Command cleanup(final VirtualMachine vm) {
-        return new StopCommand(vm, getExecuteInSequence(vm.getHypervisorType()), false);
+        StopCommand cmd = new StopCommand(vm, getExecuteInSequence(vm.getHypervisorType()), false);
+        cmd.setControlIp(getControlNicIpForVM(vm));
+        return cmd;
     }
 
+    private String getControlNicIpForVM(VirtualMachine vm) {
+        if (vm.getType() == VirtualMachine.Type.ConsoleProxy || vm.getType() == VirtualMachine.Type.SecondaryStorageVm) {
+            NicVO nic = _nicsDao.getControlNicForVM(vm.getId());
+            return nic.getIPv4Address();
+        } else if (vm.getType() == VirtualMachine.Type.DomainRouter) return vm.getPrivateIpAddress();
+        else return null;
+    }
     public Command cleanup(final String vmName) {
-        return new StopCommand(vmName, getExecuteInSequence(null), false);
+        VirtualMachine vm = _vmDao.findVMByInstanceName(vmName);
 
+        StopCommand cmd = new StopCommand(vmName, getExecuteInSequence(null), false);
+        cmd.setControlIp(getControlNicIpForVM(vm));
+        return cmd;
     }
 
 

engine/schema/src/com/cloud/vm/dao/NicDao.java

@@ -74,4 +74,6 @@ public interface NicDao extends GenericDao<NicVO, Long> {
     List<NicVO> listByNetworkIdTypeAndGatewayAndBroadcastUri(long networkId, VirtualMachine.Type vmType, String gateway, URI broadcastUri);
 
     int countNicsForStartingVms(long networkId);
+
+    NicVO getControlNicForVM(long vmId);
 }

engine/schema/src/com/cloud/vm/dao/NicDaoImpl.java

@@ -66,6 +66,7 @@ protected void init() {
         AllFieldsSearch.and("secondaryip", AllFieldsSearch.entity().getSecondaryIp(), Op.EQ);
         AllFieldsSearch.and("nicid", AllFieldsSearch.entity().getId(), Op.EQ);
         AllFieldsSearch.and("strategy", AllFieldsSearch.entity().getReservationStrategy(), Op.EQ);
+        AllFieldsSearch.and("reserverName",AllFieldsSearch.entity().getReserver(),Op.EQ);
         AllFieldsSearch.done();
 
         IpSearch = createSearchBuilder(String.class);
@@ -198,6 +199,14 @@ public NicVO findDefaultNicForVM(long instanceId) {
         return findOneBy(sc);
     }
 
+    @Override
+    public NicVO getControlNicForVM(long vmId){
+        SearchCriteria<NicVO> sc = AllFieldsSearch.create();
+        sc.setParameters("instance", vmId);
+        sc.setParameters("reserverName", "ControlNetworkGuru");
+        return findOneBy(sc);
+    }
+
     @Override
     public NicVO findNonReleasedByInstanceIdAndNetworkId(long networkId, long instanceId) {
         SearchCriteria<NicVO> sc = NonReleasedSearch.create();

plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtStopCommandWrapper.java

@@ -20,12 +20,14 @@
 package com.cloud.hypervisor.kvm.resource.wrapper;
 
 import java.util.List;
+import java.io.File;
 
+import com.cloud.utils.Pair;
+import com.cloud.utils.ssh.SshHelper;
 import org.apache.log4j.Logger;
 import org.libvirt.Connect;
 import org.libvirt.Domain;
 import org.libvirt.DomainInfo.DomainState;
-import org.libvirt.LibvirtException;
 
 import com.cloud.agent.api.Answer;
 import com.cloud.agent.api.StopAnswer;
@@ -36,11 +38,14 @@
 import com.cloud.hypervisor.kvm.resource.VifDriver;
 import com.cloud.resource.CommandWrapper;
 import com.cloud.resource.ResourceWrapper;
+import org.libvirt.LibvirtException;
 
 @ResourceWrapper(handles =  StopCommand.class)
 public final class LibvirtStopCommandWrapper extends CommandWrapper<StopCommand, Answer, LibvirtComputingResource> {
 
     private static final Logger s_logger = Logger.getLogger(LibvirtStopCommandWrapper.class);
+    private static final String CMDLINE_PATH = "/var/cache/cloud/cmdline";
+    private static final String CMDLINE_BACKUP_PATH = "/var/cache/cloud/cmdline.backup";
 
     @Override
     public Answer execute(final StopCommand command, final LibvirtComputingResource libvirtComputingResource) {
@@ -59,8 +64,21 @@ public Answer execute(final StopCommand command, final LibvirtComputingResource
                 s_logger.debug("Failed to get vm status in case of checkboforecleanup is true", e);
             }
         }
-
+        File pemFile = new File(LibvirtComputingResource.SSHPRVKEYPATH);
         try {
+            if(vmName.startsWith("s-") || vmName.startsWith("v-")){
+                //move the command line file to backup.
+                s_logger.debug("backing up the cmdline");
+                try{
+                    Pair<Boolean, String> ret = SshHelper.sshExecute(command.getControlIp(), 3922, "root", pemFile, null,"mv -f "+CMDLINE_PATH+" "+CMDLINE_BACKUP_PATH);
+                    if(!ret.first()){
+                        s_logger.debug("Failed to backup cmdline file due to "+ret.second());
+                    }
+                } catch (Exception e){
+                    s_logger.debug("Failed to backup cmdline file due to "+e.getMessage());
+                }
+            }
+
             final Connect conn = libvirtUtilitiesHelper.getConnectionByVmName(vmName);
 
             final List<DiskDef> disks = libvirtComputingResource.getDisks(conn, vmName);
@@ -83,7 +101,18 @@ public Answer execute(final StopCommand command, final LibvirtComputingResource
 
             return new StopAnswer(command, result, true);
         } catch (final LibvirtException e) {
+            s_logger.debug("unable to stop VM:"+vmName+" due to"+e.getMessage());
+            try{
+                if(vmName.startsWith("s-") || vmName.startsWith("v-"))
+                    s_logger.debug("restoring cmdline file from backup");
+                Pair<Boolean, String> ret = SshHelper.sshExecute(command.getControlIp(), 3922, "root", pemFile, null, "mv "+CMDLINE_BACKUP_PATH+" "+CMDLINE_PATH);
+                if(!ret.first()){
+                    s_logger.debug("unable to restore cmdline due to "+ret.second());
+                }
+            }catch (final Exception ex){
+                s_logger.debug("unable to restore cmdline due to:"+ex.getMessage());
+            }
             return new StopAnswer(command, e.getMessage(), false);
         }
     }
-}
+}