From 1f1340d5f4455efaae8af2c6ce1567f49aaa3088 Mon Sep 17 00:00:00 2001 From: nvazquez Date: Mon, 18 Mar 2019 20:11:19 -0300 Subject: [PATCH 1/6] Keep connection alive when on maintenance --- agent/src/com/cloud/agent/Agent.java | 5 ++- .../com/cloud/agent/manager/AgentAttache.java | 3 +- .../cloud/resource/ResourceManagerImpl.java | 35 ++++++++++--------- 3 files changed, 22 insertions(+), 21 deletions(-) diff --git a/agent/src/com/cloud/agent/Agent.java b/agent/src/com/cloud/agent/Agent.java index df0448dab224..bc5e0f62d904 100644 --- a/agent/src/com/cloud/agent/Agent.java +++ b/agent/src/com/cloud/agent/Agent.java @@ -606,9 +606,8 @@ protected void processRequest(final Request request, final Link link) { System.exit(1); return; } else if (cmd instanceof MaintainCommand) { - s_logger.debug("Received maintainCommand"); - cancelTasks(); - _reconnectAllowed = false; + s_logger.debug("Received maintainCommand, do not cancel current tasks"); + _reconnectAllowed = true; answer = new MaintainAnswer((MaintainCommand)cmd); } else if (cmd instanceof AgentControlCommand) { answer = null; diff --git a/engine/orchestration/src/com/cloud/agent/manager/AgentAttache.java b/engine/orchestration/src/com/cloud/agent/manager/AgentAttache.java index a3838e1e6efe..4f03acd19fdf 100644 --- a/engine/orchestration/src/com/cloud/agent/manager/AgentAttache.java +++ b/engine/orchestration/src/com/cloud/agent/manager/AgentAttache.java @@ -32,6 +32,7 @@ import java.util.concurrent.TimeUnit; import com.cloud.agent.api.ModifySshKeysCommand; +import com.cloud.agent.api.ModifyStoragePoolCommand; import org.apache.cloudstack.managed.context.ManagedContextRunnable; import org.apache.log4j.Logger; @@ -115,7 +116,7 @@ public int compare(final Object o1, final Object o2) { StopCommand.class.toString(), CheckVirtualMachineCommand.class.toString(), PingTestCommand.class.toString(), CheckHealthCommand.class.toString(), ReadyCommand.class.toString(), ShutdownCommand.class.toString(), SetupCommand.class.toString(), CleanupNetworkRulesCmd.class.toString(), CheckNetworkCommand.class.toString(), PvlanSetupCommand.class.toString(), CheckOnHostCommand.class.toString(), - ModifyTargetsCommand.class.toString(), ModifySshKeysCommand.class.toString()}; + ModifyTargetsCommand.class.toString(), ModifySshKeysCommand.class.toString(), ModifyStoragePoolCommand.class.toString()}; protected final static String[] s_commandsNotAllowedInConnectingMode = new String[] { StartCommand.class.toString(), CreateCommand.class.toString() }; static { Arrays.sort(s_commandsAllowedInMaintenanceMode); diff --git a/server/src/com/cloud/resource/ResourceManagerImpl.java b/server/src/com/cloud/resource/ResourceManagerImpl.java index 59a7fa85d7a9..9e5a9b294b73 100755 --- a/server/src/com/cloud/resource/ResourceManagerImpl.java +++ b/server/src/com/cloud/resource/ResourceManagerImpl.java @@ -2351,13 +2351,6 @@ private boolean doCancelMaintenance(final long hostId) { // for kvm, need to log into kvm host, restart cloudstack-agent if ((host.getHypervisorType() == HypervisorType.KVM && !vms_migrating) || host.getHypervisorType() == HypervisorType.LXC) { - - final boolean sshToAgent = Boolean.parseBoolean(_configDao.getValue(Config.KvmSshToAgentEnabled.key())); - if (!sshToAgent) { - s_logger.info("Configuration tells us not to SSH into Agents. Please restart the Agent (" + hostId + ") manually"); - return true; - } - _hostDao.loadDetails(host); final String password = host.getDetail("password"); final String username = host.getDetail("username"); @@ -2365,17 +2358,25 @@ private boolean doCancelMaintenance(final long hostId) { s_logger.debug("Can't find password/username"); return false; } - final com.trilead.ssh2.Connection connection = SSHCmdHelper.acquireAuthorizedConnection(host.getPrivateIpAddress(), 22, username, password); - if (connection == null) { - s_logger.debug("Failed to connect to host: " + host.getPrivateIpAddress()); - return false; - } - try { - SSHCmdHelper.SSHCmdResult result = SSHCmdHelper.sshExecuteCmdOneShot(connection, "service cloudstack-agent restart"); - s_logger.debug("cloudstack-agent restart result: " + result.toString()); - } catch (final SshException e) { - return false; + if (host.getStatus() != Status.Up) { + final boolean sshToAgent = Boolean.parseBoolean(_configDao.getValue(Config.KvmSshToAgentEnabled.key())); + if (sshToAgent) { + final com.trilead.ssh2.Connection connection = SSHCmdHelper.acquireAuthorizedConnection(host.getPrivateIpAddress(), 22, username, password); + if (connection == null) { + s_logger.debug("Failed to connect to host: " + host.getPrivateIpAddress()); + return false; + } + try { + SSHCmdHelper.SSHCmdResult result = SSHCmdHelper.sshExecuteCmdOneShot(connection, "service cloudstack-agent restart"); + s_logger.debug("cloudstack-agent restart result: " + result.toString()); + } catch (final SshException e) { + return false; + } + } else { + s_logger.debug("Host must be connected before cancelling maintenance mode"); + return false; + } } } From 44249fa97d34cc565b729d736a626e9fc5a94513 Mon Sep 17 00:00:00 2001 From: nvazquez Date: Tue, 19 Mar 2019 19:15:58 -0300 Subject: [PATCH 2/6] Refactor cancel maintenance and unit tests --- .../cloud/resource/ResourceManagerImpl.java | 91 ++++++++++----- .../resource/ResourceManagerImplTest.java | 106 +++++++++++++++++- 2 files changed, 164 insertions(+), 33 deletions(-) diff --git a/server/src/com/cloud/resource/ResourceManagerImpl.java b/server/src/com/cloud/resource/ResourceManagerImpl.java index 9e5a9b294b73..b841591beae3 100755 --- a/server/src/com/cloud/resource/ResourceManagerImpl.java +++ b/server/src/com/cloud/resource/ResourceManagerImpl.java @@ -31,6 +31,7 @@ import javax.inject.Inject; import javax.naming.ConfigurationException; +import com.cloud.utils.Pair; import com.cloud.vm.dao.UserVmDetailsDao; import org.apache.cloudstack.framework.config.ConfigKey; import org.apache.commons.lang.ObjectUtils; @@ -2344,46 +2345,74 @@ private boolean doCancelMaintenance(final long hostId) { } } + handleAgentIfNotConnected(host, vms_migrating); + try { resourceStateTransitTo(host, ResourceState.Event.AdminCancelMaintenance, _nodeId); _agentMgr.pullAgentOutMaintenance(hostId); retryHostMaintenance.remove(hostId); + } catch (final NoTransitionException e) { + s_logger.debug("Cannot transmit host " + host.getId() + "to Enabled state", e); + return false; + } - // for kvm, need to log into kvm host, restart cloudstack-agent - if ((host.getHypervisorType() == HypervisorType.KVM && !vms_migrating) || host.getHypervisorType() == HypervisorType.LXC) { - _hostDao.loadDetails(host); - final String password = host.getDetail("password"); - final String username = host.getDetail("username"); - if (password == null || username == null) { - s_logger.debug("Can't find password/username"); - return false; - } + return true; - if (host.getStatus() != Status.Up) { - final boolean sshToAgent = Boolean.parseBoolean(_configDao.getValue(Config.KvmSshToAgentEnabled.key())); - if (sshToAgent) { - final com.trilead.ssh2.Connection connection = SSHCmdHelper.acquireAuthorizedConnection(host.getPrivateIpAddress(), 22, username, password); - if (connection == null) { - s_logger.debug("Failed to connect to host: " + host.getPrivateIpAddress()); - return false; - } - try { - SSHCmdHelper.SSHCmdResult result = SSHCmdHelper.sshExecuteCmdOneShot(connection, "service cloudstack-agent restart"); - s_logger.debug("cloudstack-agent restart result: " + result.toString()); - } catch (final SshException e) { - return false; - } - } else { - s_logger.debug("Host must be connected before cancelling maintenance mode"); - return false; - } + } + + /** + * Handle agent (if available) if its not connected before cancelling maintenance. + * Agent must be connected before cancelling maintenance. + * If the host status is not Up: + * - If kvm.ssh.to.agent is true, then SSH into the host and restart the agent. + * - If kvm.shh.to.agent is false, then fail cancelling maintenance + */ + protected void handleAgentIfNotConnected(HostVO host, boolean vmsMigrating) { + final boolean isAgentOnHost = host.getHypervisorType() == HypervisorType.KVM || + host.getHypervisorType() == HypervisorType.LXC; + if (isAgentOnHost) { + final boolean sshToAgent = Boolean.parseBoolean(_configDao.getValue(Config.KvmSshToAgentEnabled.key())); + if (!vmsMigrating && host.getStatus() != Status.Up) { + if (sshToAgent) { + Pair credentials = getHostCredentials(host); + connectAndRestartAgentOnHost(host, credentials.first(), credentials.second()); + } else { + throw new CloudRuntimeException("SSH access is disabled, cannot cancel maintenance mode as " + + "host agent is not connected"); } } + } + } - return true; - } catch (final NoTransitionException e) { - s_logger.debug("Cannot transmit host " + host.getId() + "to Enabled state", e); - return false; + /** + * Get host credentials + * @throws CloudRuntimeException if username or password are not found + */ + protected Pair getHostCredentials(HostVO host) { + _hostDao.loadDetails(host); + final String password = host.getDetail("password"); + final String username = host.getDetail("username"); + if (password == null || username == null) { + throw new CloudRuntimeException("SSH to agent is enabled, but username/password credentials are not found"); + } + return new Pair<>(username, password); + } + + /** + * True if agent is restarted via SSH. Assumes kvm.ssh.to.agent = true and host status is not Up + */ + protected void connectAndRestartAgentOnHost(HostVO host, String username, String password) { + final com.trilead.ssh2.Connection connection = SSHCmdHelper.acquireAuthorizedConnection( + host.getPrivateIpAddress(), 22, username, password); + if (connection == null) { + throw new CloudRuntimeException("SSH to agent is enabled, but failed to connect to host: " + host.getPrivateIpAddress()); + } + try { + SSHCmdHelper.SSHCmdResult result = SSHCmdHelper.sshExecuteCmdOneShot( + connection, "service cloudstack-agent restart"); + s_logger.debug("cloudstack-agent restart result: " + result.toString()); + } catch (final SshException e) { + throw new CloudRuntimeException("SSH to agent is enabled, but agent restart failed", e); } } diff --git a/server/test/com/cloud/resource/ResourceManagerImplTest.java b/server/test/com/cloud/resource/ResourceManagerImplTest.java index 6d14410fc1c2..e02d5bb69794 100644 --- a/server/test/com/cloud/resource/ResourceManagerImplTest.java +++ b/server/test/com/cloud/resource/ResourceManagerImplTest.java @@ -21,17 +21,25 @@ import com.cloud.agent.api.GetVncPortAnswer; import com.cloud.agent.api.GetVncPortCommand; import com.cloud.capacity.dao.CapacityDao; +import com.cloud.configuration.Config; import com.cloud.event.ActionEventUtils; import com.cloud.ha.HighAvailabilityManager; import com.cloud.host.Host; import com.cloud.host.HostVO; +import com.cloud.host.Status; import com.cloud.host.dao.HostDao; import com.cloud.hypervisor.Hypervisor; import com.cloud.storage.StorageManager; +import com.cloud.utils.Pair; +import com.cloud.utils.exception.CloudRuntimeException; import com.cloud.utils.fsm.NoTransitionException; +import com.cloud.utils.ssh.SSHCmdHelper; +import com.cloud.utils.ssh.SshException; import com.cloud.vm.VMInstanceVO; import com.cloud.vm.dao.UserVmDetailsDao; import com.cloud.vm.dao.VMInstanceDao; +import com.trilead.ssh2.Connection; +import org.apache.cloudstack.framework.config.dao.ConfigurationDao; import org.junit.Assert; import org.junit.Before; import org.junit.Test; @@ -56,12 +64,13 @@ import static org.mockito.Matchers.anyLong; import static org.mockito.Matchers.anyString; import static org.mockito.Matchers.eq; +import static org.mockito.Mockito.never; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; @RunWith(PowerMockRunner.class) -@PrepareForTest({ActionEventUtils.class, ResourceManagerImpl.class}) +@PrepareForTest({ActionEventUtils.class, ResourceManagerImpl.class, SSHCmdHelper.class}) public class ResourceManagerImplTest { @Mock @@ -78,6 +87,8 @@ public class ResourceManagerImplTest { private HostDao hostDao; @Mock private VMInstanceDao vmInstanceDao; + @Mock + private ConfigurationDao configurationDao; @Spy @InjectMocks @@ -99,7 +110,13 @@ public class ResourceManagerImplTest { @Mock private GetVncPortCommand getVncPortCommandVm2; + @Mock + private Connection sshConnection; + private static long hostId = 1L; + private static final String hostUsername = "user"; + private static final String hostPassword = "password"; + private static final String hostPrivateIp = "192.168.1.10"; private static long vm1Id = 1L; private static String vm1InstanceName = "i-1-VM"; @@ -117,9 +134,13 @@ public void setup() throws Exception { when(host.getType()).thenReturn(Host.Type.Routing); when(host.getId()).thenReturn(hostId); when(host.getResourceState()).thenReturn(ResourceState.Enabled); - when(host.getHypervisorType()).thenReturn(Hypervisor.HypervisorType.VMware); + when(host.getHypervisorType()).thenReturn(Hypervisor.HypervisorType.KVM); when(host.getClusterId()).thenReturn(1L); when(hostDao.findById(hostId)).thenReturn(host); + when(host.getDetail("username")).thenReturn(hostUsername); + when(host.getDetail("password")).thenReturn(hostPassword); + when(host.getStatus()).thenReturn(Status.Up); + when(host.getPrivateIpAddress()).thenReturn(hostPrivateIp); when(vm1.getId()).thenReturn(vm1Id); when(vm2.getId()).thenReturn(vm2Id); when(vm1.getInstanceName()).thenReturn(vm1InstanceName); @@ -138,6 +159,15 @@ public void setup() throws Exception { PowerMockito.whenNew(GetVncPortCommand.class).withArguments(vm2Id, vm2InstanceName).thenReturn(getVncPortCommandVm2); when(agentManager.easySend(eq(hostId), eq(getVncPortCommandVm1))).thenReturn(getVncPortAnswerVm1); when(agentManager.easySend(eq(hostId), eq(getVncPortCommandVm2))).thenReturn(getVncPortAnswerVm2); + + PowerMockito.mockStatic(SSHCmdHelper.class); + BDDMockito.given(SSHCmdHelper.acquireAuthorizedConnection(eq(hostPrivateIp), eq(22), + eq(hostUsername), eq(hostPassword))).willReturn(sshConnection); + BDDMockito.given(SSHCmdHelper.sshExecuteCmdOneShot(eq(sshConnection), + eq("service cloudstack-agent restart"))). + willReturn(new SSHCmdHelper.SSHCmdResult(0,"","")); + + when(configurationDao.getValue(Config.KvmSshToAgentEnabled.key())).thenReturn("true"); } @Test @@ -206,4 +236,76 @@ public void testCheckAndMaintainErrorInMaintenanceRetries() throws NoTransitionE verify(resourceManager, times(retries + 1)).isHostInMaintenance(host, failedMigrations, new ArrayList<>(), failedMigrations); verify(resourceManager).setHostIntoErrorInMaintenance(host, failedMigrations); } + + @Test(expected = CloudRuntimeException.class) + public void testGetHostCredentialsMissingParameter() { + when(host.getDetail("password")).thenReturn(null); + resourceManager.getHostCredentials(host); + } + + @Test + public void testGetHostCredentials() { + Pair credentials = resourceManager.getHostCredentials(host); + Assert.assertNotNull(credentials); + Assert.assertEquals(hostUsername, credentials.first()); + Assert.assertEquals(hostPassword, credentials.second()); + } + + @Test(expected = CloudRuntimeException.class) + public void testConnectAndRestartAgentOnHostCannotConnect() { + BDDMockito.given(SSHCmdHelper.acquireAuthorizedConnection(eq(hostPrivateIp), eq(22), + eq(hostUsername), eq(hostPassword))).willReturn(null); + resourceManager.connectAndRestartAgentOnHost(host, hostUsername, hostPassword); + } + + @Test(expected = CloudRuntimeException.class) + public void testConnectAndRestartAgentOnHostCannotRestart() throws Exception { + BDDMockito.given(SSHCmdHelper.sshExecuteCmdOneShot(eq(sshConnection), + eq("service cloudstack-agent restart"))).willThrow(new SshException("exception")); + resourceManager.connectAndRestartAgentOnHost(host, hostUsername, hostPassword); + } + + @Test + public void testConnectAndRestartAgentOnHost() { + resourceManager.connectAndRestartAgentOnHost(host, hostUsername, hostPassword); + } + + @Test + public void testHandleAgentSSHEnabledNotConnectedAgent() { + when(host.getStatus()).thenReturn(Status.Disconnected); + resourceManager.handleAgentIfNotConnected(host, false); + verify(resourceManager).getHostCredentials(eq(host)); + verify(resourceManager).connectAndRestartAgentOnHost(eq(host), eq(hostUsername), eq(hostPassword)); + } + + @Test + public void testHandleAgentSSHEnabledConnectedAgent() { + when(host.getStatus()).thenReturn(Status.Up); + resourceManager.handleAgentIfNotConnected(host, false); + verify(resourceManager, never()).getHostCredentials(eq(host)); + verify(resourceManager, never()).connectAndRestartAgentOnHost(eq(host), eq(hostUsername), eq(hostPassword)); + } + + @Test(expected = CloudRuntimeException.class) + public void testHandleAgentSSHDisabledNotConnectedAgent() { + when(host.getStatus()).thenReturn(Status.Disconnected); + when(configurationDao.getValue(Config.KvmSshToAgentEnabled.key())).thenReturn("false"); + resourceManager.handleAgentIfNotConnected(host, false); + } + + @Test + public void testHandleAgentSSHDisabledConnectedAgent() { + when(host.getStatus()).thenReturn(Status.Up); + when(configurationDao.getValue(Config.KvmSshToAgentEnabled.key())).thenReturn("false"); + resourceManager.handleAgentIfNotConnected(host, false); + verify(resourceManager, never()).getHostCredentials(eq(host)); + verify(resourceManager, never()).connectAndRestartAgentOnHost(eq(host), eq(hostUsername), eq(hostPassword)); + } + + @Test + public void testHandleAgentVMsMigrating() { + resourceManager.handleAgentIfNotConnected(host, true); + verify(resourceManager, never()).getHostCredentials(eq(host)); + verify(resourceManager, never()).connectAndRestartAgentOnHost(eq(host), eq(hostUsername), eq(hostPassword)); + } } From bf1a6d50cfd705a0d529d786b95a2165d81ede53 Mon Sep 17 00:00:00 2001 From: nvazquez Date: Thu, 21 Mar 2019 18:49:50 -0300 Subject: [PATCH 3/6] Add marvin tests --- .../smoke/test_host_maintenance.py | 394 ++++++++++++++++-- 1 file changed, 350 insertions(+), 44 deletions(-) diff --git a/test/integration/smoke/test_host_maintenance.py b/test/integration/smoke/test_host_maintenance.py index 7fc2139e3bbd..b3530a14c1ff 100644 --- a/test/integration/smoke/test_host_maintenance.py +++ b/test/integration/smoke/test_host_maintenance.py @@ -18,15 +18,14 @@ """ # Import Local Modules -from marvin.codes import FAILED from marvin.cloudstackTestCase import * -from marvin.cloudstackAPI import * from marvin.lib.utils import * from marvin.lib.base import * -from marvin.lib.common import * +from marvin.lib.common import (get_zone, get_pod, get_template) from nose.plugins.attrib import attr - -from time import sleep +from marvin.lib.decoratorGenerators import skipTestIf +from distutils.util import strtobool +from marvin.sshClient import SshClient _multiprocess_shared_ = False @@ -45,37 +44,6 @@ def setUp(self): self.zone = get_zone(self.apiclient, self.testClient.getZoneForTests()) self.pod = get_pod(self.apiclient, self.zone.id) self.cleanup = [] - self.services = { - "service_offering": { - "name": "Ultra Tiny Instance", - "displaytext": "Ultra Tiny Instance", - "cpunumber": 1, - "cpuspeed": 100, - "memory": 128, - }, - "vm": { - "username": "root", - "password": "password", - "ssh_port": 22, - # Hypervisor type should be same as - # hypervisor type of cluster - "privateport": 22, - "publicport": 22, - "protocol": 'TCP', - }, - "natrule": { - "privateport": 22, - "publicport": 22, - "startport": 22, - "endport": 22, - "protocol": "TCP", - "cidrlist": '0.0.0.0/0', - }, - "ostype": 'CentOS 5.3 (64-bit)', - "sleep": 60, - "timeout": 10, - } - def tearDown(self): try: @@ -89,14 +57,14 @@ def tearDown(self): def createVMs(self, hostId, number): - self.template = get_test_template( + self.template = get_template( self.apiclient, self.zone.id, self.hypervisor ) if self.template == FAILED: - assert False, "get_test_template() failed to return template" + assert False, "get_template() failed to return template" self.logger.debug("Using template %s " % self.template.id) @@ -105,22 +73,38 @@ def createVMs(self, hostId, number): self.services["service_offering"] ) self.logger.debug("Using service offering %s " % self.service_offering.id) - + self.network_offering = NetworkOffering.create( + self.apiclient, + self.services["l2-network_offering"], + ) + self.network_offering.update(self.apiclient, state='Enabled') + self.services["network"]["networkoffering"] = self.network_offering.id + self.l2_network = Network.create( + self.apiclient, + self.services["l2-network"], + zoneid=self.zone.id, + networkofferingid=self.network_offering.id + ) + vms=[] for i in range(0, number): - self.services["vm"]["zoneid"] = self.zone.id - self.services["vm"]["template"] = self.template.id - self.services["vm"]["displayname"] = 'vm' + str(i) - self.services["vm"]["hypervisor"] = self.hypervisor + self.services["virtual_machine"]["zoneid"] = self.zone.id + self.services["virtual_machine"]["template"] = self.template.id + self.services["virtual_machine"]["displayname"] = 'vm' + str(i) + self.services["virtual_machine"]["hypervisor"] = self.hypervisor vm = VirtualMachine.create( self.apiclient, - self.services["vm"], + self.services["virtual_machine"], serviceofferingid=self.service_offering.id, + networkids=self.l2_network.id, hostid=hostId ) vms.append(vm) self.cleanup.append(vm) self.logger.debug("VM create = {}".format(vm.id)) + self.cleanup.append(self.l2_network) + self.cleanup.append(self.network_offering) + self.cleanup.append(self.service_offering) return vms def checkVmMigratingOnHost(self, hostId): @@ -290,3 +274,325 @@ def test_02_cancel_host_maintenace_with_migration_jobs(self): return +class TestHostMaintenanceAgents(cloudstackTestCase): + + @classmethod + def setUpClass(cls): + cls.testClient = super(TestHostMaintenanceAgents, cls).getClsTestClient() + cls.apiclient = cls.testClient.getApiClient() + cls.hypervisor = cls.testClient.getHypervisorInfo() + cls.dbclient = cls.testClient.getDbConnection() + cls.zone = get_zone(cls.apiclient, cls.testClient.getZoneForTests()) + cls.pod = get_pod(cls.apiclient, cls.zone.id) + cls.services = cls.testClient.getParsedTestDataConfig() + + cls.logger = logging.getLogger('TestHMAgents') + cls.stream_handler = logging.StreamHandler() + cls.logger.setLevel(logging.DEBUG) + cls.logger.addHandler(cls.stream_handler) + + cls._cleanup = [] + cls.hypervisorNotSupported = False + if cls.hypervisor.lower() not in ['kvm', 'lxc']: + cls.hypervisorNotSupported = True + + if not cls.hypervisorNotSupported: + cls.initialsshvalue = cls.is_ssh_enabled() + cls.template = get_template( + cls.apiclient, + cls.zone.id, + cls.hypervisor + ) + cls.services["virtual_machine"]["zoneid"] = cls.zone.id + cls.services["virtual_machine"]["template"] = cls.template.id + cls.services["virtual_machine"]["hypervisor"] = cls.hypervisor + cls.service_offering = ServiceOffering.create( + cls.apiclient, + cls.services["service_offering"] + ) + cls._cleanup.append(cls.service_offering) + cls.network_offering = NetworkOffering.create( + cls.apiclient, + cls.services["l2-network_offering"], + ) + cls.network_offering.update(cls.apiclient, state='Enabled') + cls.services["network"]["networkoffering"] = cls.network_offering.id + cls.l2_network = Network.create( + cls.apiclient, + cls.services["l2-network"], + zoneid=cls.zone.id, + networkofferingid=cls.network_offering.id + ) + cls._cleanup.append(cls.l2_network) + cls._cleanup.append(cls.network_offering) + + @classmethod + def tearDownClass(cls): + try: + if not cls.hypervisorNotSupported: + # Revert setting value to the original + cls.set_ssh_enabled(cls.initialsshvalue) + cleanup_resources(cls.apiclient, cls._cleanup) + except Exception as e: + raise Exception("Warning: Exception during cleanup : %s" % e) + + def setUp(self): + if not self.hypervisorNotSupported: + self.host = self.get_enabled_host_connected_agent() + self.cleanup = [] + + def tearDown(self): + try: + cleanup_resources(self.apiclient, self.cleanup) + except Exception as e: + raise Exception("Warning: Exception during cleanup : %s" % e) + + + @classmethod + def is_ssh_enabled(cls): + conf = Configurations.list(cls.apiclient, name="kvm.ssh.to.agent") + if not conf: + return False + else: + return bool(strtobool(conf[0].value)) if conf[0].value else False + + @classmethod + def set_ssh_enabled(cls, on): + value = "true" if on else "false" + sql = "update configuration set value = '%s' where name = 'kvm.ssh.to.agent';" % value + cls.dbclient.execute(sql) + + def prepare_host_for_maintenance(self, hostid): + cmd = prepareHostForMaintenance.prepareHostForMaintenanceCmd() + cmd.id = hostid + self.apiclient.prepareHostForMaintenance(cmd) + self.logger.debug('Host with id %s is in prepareHostForMaintenance' % hostid) + + def wait_until_host_is_in_state(self, hostid, resourcestate, interval=3, retries=20): + def check_resource_state(): + response = Host.list( + self.apiclient, + id=hostid + ) + if isinstance(response, list): + if response[0].resourcestate == resourcestate: + self.logger.debug('Host with id %s is in resource state = %s' % (hostid, resourcestate)) + return True, None + return False, None + + done, _ = wait_until(interval, retries, check_resource_state) + if not done: + raise Exception("Failed to wait for host %s to be on resource state %s" % (hostid, resourcestate)) + return True + + def wait_until_agent_is_in_state(self, hostid, state, interval=3, retries=20): + def check_agent_state(): + response = Host.list( + self.apiclient, + id=hostid + ) + if isinstance(response, list): + if response[0].state == state: + self.logger.debug('Host agent with id %s is in state = %s' % (hostid, state)) + return True, None + return False, None + + done, _ = wait_until(interval, retries, check_agent_state) + if not done: + raise Exception("Failed to wait for host agent %s to be on state %s" % (hostid, state)) + return True + + def cancel_host_maintenance(self, hostid): + cmd = cancelHostMaintenance.cancelHostMaintenanceCmd() + cmd.id = hostid + self.apiclient.cancelHostMaintenance(cmd) + self.logger.debug('Host with id %s is cancelling maintenance' % hostid) + + def get_enabled_host_connected_agent(self): + hosts = Host.list( + self.apiclient, + type='Routing', + zoneid=self.zone.id, + podid=self.pod.id, + hypervisor=self.hypervisor, + resourcestate='Enabled', + state='Up' + ) + if len(hosts) < 2: + raise unittest.SkipTest("Cancel host maintenance must be tested for 2 or more hosts") + return hosts[0] + + def deploy_vm_on_host(self, hostid): + return VirtualMachine.create( + self.apiclient, + self.services["virtual_machine"], + serviceofferingid=self.service_offering.id, + networkids=self.l2_network.id, + hostid=hostid + ) + + def assert_host_is_functional_after_cancelling_maintenance(self, hostid): + self.wait_until_agent_is_in_state(hostid, "Up") + self.logger.debug('Deploying VM on host %s' % hostid) + vm = self.deploy_vm_on_host(hostid) + self.assertEqual( + vm.state, + "Running", + "Check VM is running on the host" + ) + self.cleanup.append(vm) + + @skipTestIf("hypervisorNotSupported") + @attr(tags=["advanced", "advancedns", "smoke", "basic", "eip", "sg"], required_hardware="true") + def test_01_cancel_host_maintenance_ssh_enabled_agent_connected(self): + """ + Test cancel maintenance when: 'kvm.ssh.to.agent' = true, agent state = 'Up' + + 1) Put host on Maintenance + 2) Cancel maintenance on host + 4) Assert agent is still connected after cancelling maintenance + 3) Deploy VM on the host after cancelling maintenance + """ + + if not self.is_ssh_enabled(): + self.set_ssh_enabled(True) + + try: + self.prepare_host_for_maintenance(self.host.id) + self.wait_until_host_is_in_state(self.host.id, "Maintenance") + self.cancel_host_maintenance(self.host.id) + self.wait_until_host_is_in_state(self.host.id, "Enabled") + self.assert_host_is_functional_after_cancelling_maintenance(self.host.id) + except Exception as e: + self.fail(e) + + @classmethod + def get_host_credentials(cls, hostid): + sql = "select id from host where uuid = '%s';" % hostid + internal_id = cls.dbclient.execute(sql)[0] + sql = "select name, value from host_details where host_id = '%s' and name in ('username', 'password');" \ + % internal_id + rows = cls.dbclient.execute(sql) + username = None + password = None + for row in rows: + if row[0] == "username": + username = row[1] + elif row[0] == "password": + password = row[1] + if not username or not password: + raise unittest.SkipTest("Incomplete credentials for host %s" % hostid) + return username, password + + def get_ssh_client(self, ip, username, password, retries=10): + """ Setup ssh client connection and return connection """ + + try: + ssh_client = SshClient(ip, 22, username, password, retries) + except Exception as e: + raise unittest.SkipTest("Unable to create ssh connection: " % e) + + self.assertIsNotNone( + ssh_client, "Failed to setup ssh connection to ip=%s" % ip) + + return ssh_client + + @skipTestIf("hypervisorNotSupported") + @attr(tags=["advanced", "advancedns", "smoke", "basic", "eip", "sg", "NICO"], required_hardware="true") + def test_02_cancel_host_maintenance_ssh_enabled_agent_disconnected(self): + """ + Test cancel maintenance when: 'kvm.ssh.to.agent' = true, agent state != 'Up' + + 1) Put host on maintenance + 2) SSH into host and stop cloudstack-agent service - host gets Disconnected + 3) Cancel maintenance on host + 4) Assert agent is connected after cancelling maintenance + 5) Deploy VM on the host + """ + + if not self.is_ssh_enabled(): + self.set_ssh_enabled(True) + username, password = self.get_host_credentials(self.host.id) + + try: + self.prepare_host_for_maintenance(self.host.id) + self.wait_until_host_is_in_state(self.host.id, "Maintenance") + + ssh_client = self.get_ssh_client(self.host.ipaddress, username, password) + ssh_client.execute("service cloudstack-agent stop") + self.wait_until_agent_is_in_state(self.host.id, "Disconnected") + + self.cancel_host_maintenance(self.host.id) + self.wait_until_host_is_in_state(self.host.id, "Enabled") + + self.assert_host_is_functional_after_cancelling_maintenance(self.host.id) + except Exception as e: + self.fail(e) + + @skipTestIf("hypervisorNotSupported") + @attr(tags=["advanced", "advancedns", "smoke", "basic", "eip", "sg"], required_hardware="true") + def test_03_cancel_host_maintenance_ssh_disabled_agent_connected(self): + """ + Test cancel maintenance when: 'kvm.ssh.to.agent' = false, agent state = 'Up' + + 1) Put host on Maintenance + 2) Cancel maintenance on host + 4) Assert agent is still connected after cancelling maintenance + 3) Deploy VM on the host after cancelling maintenance + """ + + if self.is_ssh_enabled(): + self.set_ssh_enabled(False) + + try: + self.prepare_host_for_maintenance(self.host.id) + self.wait_until_host_is_in_state(self.host.id, "Maintenance") + self.cancel_host_maintenance(self.host.id) + self.wait_until_host_is_in_state(self.host.id, "Enabled") + self.assert_host_is_functional_after_cancelling_maintenance(self.host.id) + except Exception as e: + self.fail(e) + + @skipTestIf("hypervisorNotSupported") + @attr(tags=["advanced", "advancedns", "smoke", "basic", "eip", "sg"], required_hardware="true") + def test_04_cancel_host_maintenance_ssh_disabled_agent_disconnected(self): + """ + Test cancel maintenance when: 'kvm.ssh.to.agent' = false, agent state != 'Up' + + 1) Put host on maintenance + 2) SSH into host (if possible) and stop cloudstack-agent service - host gets Disconnected. + Skip test if not possible to SSH into host + 3) Cancel maintenance on host - assert cannot cancel maintenance on disconnected host (exception thwown) + 4( SSH into host and start cloudstack-agent service - host gets connected + 5) Cancel maintenance on host + 4) Assert agent is connected after cancelling maintenance + 5) Deploy VM on the host + """ + + if self.is_ssh_enabled(): + self.set_ssh_enabled(False) + + username, password = self.get_host_credentials(self.host.id) + + try: + self.prepare_host_for_maintenance(self.host.id) + self.wait_until_host_is_in_state(self.host.id, "Maintenance") + + ssh_client = self.get_ssh_client(self.host.ipaddress, username, password) + ssh_client.execute("service cloudstack-agent stop") + self.wait_until_agent_is_in_state(self.host.id, "Disconnected") + except Exception as e: + self.fail(e) + + self.assertRaises(Exception, self.cancel_host_maintenance, self.host.id) + + try: + ssh_client = self.get_ssh_client(self.host.ipaddress, username, password) + ssh_client.execute("service cloudstack-agent start") + self.wait_until_agent_is_in_state(self.host.id, "Up") + + self.cancel_host_maintenance(self.host.id) + self.wait_until_host_is_in_state(self.host.id, "Enabled") + self.assert_host_is_functional_after_cancelling_maintenance(self.host.id) + except Exception as e: + self.fail(e) From 237ebbc84441bf1ee36f0b9471fd8248b0dc3a7e Mon Sep 17 00:00:00 2001 From: nvazquez Date: Wed, 27 Mar 2019 11:12:20 -0300 Subject: [PATCH 4/6] Refactor --- agent/src/com/cloud/agent/Agent.java | 1 - .../com/cloud/resource/ResourceManager.java | 5 +++++ .../src/com/cloud/configuration/Config.java | 8 ------- .../cloud/resource/ResourceManagerImpl.java | 21 +++++++++---------- .../resource/ResourceManagerImplTest.java | 7 +++---- .../smoke/test_host_maintenance.py | 6 +++--- 6 files changed, 21 insertions(+), 27 deletions(-) diff --git a/agent/src/com/cloud/agent/Agent.java b/agent/src/com/cloud/agent/Agent.java index bc5e0f62d904..500724dd5a36 100644 --- a/agent/src/com/cloud/agent/Agent.java +++ b/agent/src/com/cloud/agent/Agent.java @@ -607,7 +607,6 @@ protected void processRequest(final Request request, final Link link) { return; } else if (cmd instanceof MaintainCommand) { s_logger.debug("Received maintainCommand, do not cancel current tasks"); - _reconnectAllowed = true; answer = new MaintainAnswer((MaintainCommand)cmd); } else if (cmd instanceof AgentControlCommand) { answer = null; diff --git a/engine/components-api/src/com/cloud/resource/ResourceManager.java b/engine/components-api/src/com/cloud/resource/ResourceManager.java index 720a980f4e72..b66f7923b4da 100755 --- a/engine/components-api/src/com/cloud/resource/ResourceManager.java +++ b/engine/components-api/src/com/cloud/resource/ResourceManager.java @@ -52,6 +52,11 @@ public interface ResourceManager extends ResourceService, Configurable { "Number of retries when preparing a host into Maintenance Mode is faulty before failing", true, ConfigKey.Scope.Cluster); + ConfigKey KvmSshToAgentEnabled = new ConfigKey<>("Advanced", Boolean.class, + "kvm.ssh.to.agent","true", + "Number of retries when preparing a host into Maintenance Mode is faulty before failing", + false); + /** * Register a listener for different types of resource life cycle events. * There can only be one type of listener per type of host. diff --git a/server/src/com/cloud/configuration/Config.java b/server/src/com/cloud/configuration/Config.java index 98bacf2b9075..098d5d7701f7 100644 --- a/server/src/com/cloud/configuration/Config.java +++ b/server/src/com/cloud/configuration/Config.java @@ -1211,14 +1211,6 @@ public enum Config { KvmPublicNetwork("Hidden", ManagementServer.class, String.class, "kvm.public.network.device", null, "Specify the public bridge on host for public network", null), KvmPrivateNetwork("Hidden", ManagementServer.class, String.class, "kvm.private.network.device", null, "Specify the private bridge on host for private network", null), KvmGuestNetwork("Hidden", ManagementServer.class, String.class, "kvm.guest.network.device", null, "Specify the private bridge on host for private network", null), - KvmSshToAgentEnabled( - "Advanced", - ManagementServer.class, - Boolean.class, - "kvm.ssh.to.agent", - "true", - "Specify whether or not the management server is allowed to SSH into KVM Agents", - null), // Hyperv HypervPublicNetwork( diff --git a/server/src/com/cloud/resource/ResourceManagerImpl.java b/server/src/com/cloud/resource/ResourceManagerImpl.java index b841591beae3..c259eb666f38 100755 --- a/server/src/com/cloud/resource/ResourceManagerImpl.java +++ b/server/src/com/cloud/resource/ResourceManagerImpl.java @@ -2370,17 +2370,16 @@ private boolean doCancelMaintenance(final long hostId) { protected void handleAgentIfNotConnected(HostVO host, boolean vmsMigrating) { final boolean isAgentOnHost = host.getHypervisorType() == HypervisorType.KVM || host.getHypervisorType() == HypervisorType.LXC; - if (isAgentOnHost) { - final boolean sshToAgent = Boolean.parseBoolean(_configDao.getValue(Config.KvmSshToAgentEnabled.key())); - if (!vmsMigrating && host.getStatus() != Status.Up) { - if (sshToAgent) { - Pair credentials = getHostCredentials(host); - connectAndRestartAgentOnHost(host, credentials.first(), credentials.second()); - } else { - throw new CloudRuntimeException("SSH access is disabled, cannot cancel maintenance mode as " + - "host agent is not connected"); - } - } + if (!isAgentOnHost || vmsMigrating || host.getStatus() == Status.Up) { + return; + } + final boolean sshToAgent = Boolean.parseBoolean(_configDao.getValue(KvmSshToAgentEnabled.key())); + if (sshToAgent) { + Pair credentials = getHostCredentials(host); + connectAndRestartAgentOnHost(host, credentials.first(), credentials.second()); + } else { + throw new CloudRuntimeException("SSH access is disabled, cannot cancel maintenance mode as " + + "host agent is not connected"); } } diff --git a/server/test/com/cloud/resource/ResourceManagerImplTest.java b/server/test/com/cloud/resource/ResourceManagerImplTest.java index e02d5bb69794..7d1a0fe0163e 100644 --- a/server/test/com/cloud/resource/ResourceManagerImplTest.java +++ b/server/test/com/cloud/resource/ResourceManagerImplTest.java @@ -21,7 +21,6 @@ import com.cloud.agent.api.GetVncPortAnswer; import com.cloud.agent.api.GetVncPortCommand; import com.cloud.capacity.dao.CapacityDao; -import com.cloud.configuration.Config; import com.cloud.event.ActionEventUtils; import com.cloud.ha.HighAvailabilityManager; import com.cloud.host.Host; @@ -167,7 +166,7 @@ public void setup() throws Exception { eq("service cloudstack-agent restart"))). willReturn(new SSHCmdHelper.SSHCmdResult(0,"","")); - when(configurationDao.getValue(Config.KvmSshToAgentEnabled.key())).thenReturn("true"); + when(configurationDao.getValue(ResourceManager.KvmSshToAgentEnabled.key())).thenReturn("true"); } @Test @@ -289,14 +288,14 @@ public void testHandleAgentSSHEnabledConnectedAgent() { @Test(expected = CloudRuntimeException.class) public void testHandleAgentSSHDisabledNotConnectedAgent() { when(host.getStatus()).thenReturn(Status.Disconnected); - when(configurationDao.getValue(Config.KvmSshToAgentEnabled.key())).thenReturn("false"); + when(configurationDao.getValue(ResourceManager.KvmSshToAgentEnabled.key())).thenReturn("false"); resourceManager.handleAgentIfNotConnected(host, false); } @Test public void testHandleAgentSSHDisabledConnectedAgent() { when(host.getStatus()).thenReturn(Status.Up); - when(configurationDao.getValue(Config.KvmSshToAgentEnabled.key())).thenReturn("false"); + when(configurationDao.getValue(ResourceManager.KvmSshToAgentEnabled.key())).thenReturn("false"); resourceManager.handleAgentIfNotConnected(host, false); verify(resourceManager, never()).getHostCredentials(eq(host)); verify(resourceManager, never()).connectAndRestartAgentOnHost(eq(host), eq(hostUsername), eq(hostPassword)); diff --git a/test/integration/smoke/test_host_maintenance.py b/test/integration/smoke/test_host_maintenance.py index b3530a14c1ff..ca5dffdbcd9c 100644 --- a/test/integration/smoke/test_host_maintenance.py +++ b/test/integration/smoke/test_host_maintenance.py @@ -70,7 +70,7 @@ def createVMs(self, hostId, number): self.service_offering = ServiceOffering.create( self.apiclient, - self.services["service_offering"] + self.services["service_offerings"]["tiny"] ) self.logger.debug("Using service offering %s " % self.service_offering.id) self.network_offering = NetworkOffering.create( @@ -308,7 +308,7 @@ def setUpClass(cls): cls.services["virtual_machine"]["hypervisor"] = cls.hypervisor cls.service_offering = ServiceOffering.create( cls.apiclient, - cls.services["service_offering"] + cls.services["service_offerings"]["tiny"] ) cls._cleanup.append(cls.service_offering) cls.network_offering = NetworkOffering.create( @@ -498,7 +498,7 @@ def get_ssh_client(self, ip, username, password, retries=10): return ssh_client @skipTestIf("hypervisorNotSupported") - @attr(tags=["advanced", "advancedns", "smoke", "basic", "eip", "sg", "NICO"], required_hardware="true") + @attr(tags=["advanced", "advancedns", "smoke", "basic", "eip", "sg"], required_hardware="true") def test_02_cancel_host_maintenance_ssh_enabled_agent_disconnected(self): """ Test cancel maintenance when: 'kvm.ssh.to.agent' = true, agent state != 'Up' From f79456670dddbbcd20cd2d7cff8dc4c60321fb0d Mon Sep 17 00:00:00 2001 From: Boris Date: Fri, 12 Apr 2019 11:13:15 +0300 Subject: [PATCH 5/6] Changing the way we get ssh credentials --- .../smoke/test_host_maintenance.py | 38 +++++++------------ 1 file changed, 13 insertions(+), 25 deletions(-) diff --git a/test/integration/smoke/test_host_maintenance.py b/test/integration/smoke/test_host_maintenance.py index ca5dffdbcd9c..645aa427cad4 100644 --- a/test/integration/smoke/test_host_maintenance.py +++ b/test/integration/smoke/test_host_maintenance.py @@ -326,6 +326,9 @@ def setUpClass(cls): cls._cleanup.append(cls.l2_network) cls._cleanup.append(cls.network_offering) + cls.hostConfig = cls.config.__dict__["zones"][0].__dict__["pods"][0].__dict__["clusters"][0].__dict__["hosts"][0].__dict__ + + @classmethod def tearDownClass(cls): try: @@ -466,24 +469,6 @@ def test_01_cancel_host_maintenance_ssh_enabled_agent_connected(self): except Exception as e: self.fail(e) - @classmethod - def get_host_credentials(cls, hostid): - sql = "select id from host where uuid = '%s';" % hostid - internal_id = cls.dbclient.execute(sql)[0] - sql = "select name, value from host_details where host_id = '%s' and name in ('username', 'password');" \ - % internal_id - rows = cls.dbclient.execute(sql) - username = None - password = None - for row in rows: - if row[0] == "username": - username = row[1] - elif row[0] == "password": - password = row[1] - if not username or not password: - raise unittest.SkipTest("Incomplete credentials for host %s" % hostid) - return username, password - def get_ssh_client(self, ip, username, password, retries=10): """ Setup ssh client connection and return connection """ @@ -498,7 +483,7 @@ def get_ssh_client(self, ip, username, password, retries=10): return ssh_client @skipTestIf("hypervisorNotSupported") - @attr(tags=["advanced", "advancedns", "smoke", "basic", "eip", "sg"], required_hardware="true") + @attr(tags=["boris", "advancedns", "smoke", "basic", "eip", "sg"], required_hardware="true") def test_02_cancel_host_maintenance_ssh_enabled_agent_disconnected(self): """ Test cancel maintenance when: 'kvm.ssh.to.agent' = true, agent state != 'Up' @@ -512,13 +497,16 @@ def test_02_cancel_host_maintenance_ssh_enabled_agent_disconnected(self): if not self.is_ssh_enabled(): self.set_ssh_enabled(True) - username, password = self.get_host_credentials(self.host.id) + # username, password = self.get_host_credentials(self.host.id) + username = self.hostConfig["username"] + password = self.hostConfig["password"] try: self.prepare_host_for_maintenance(self.host.id) self.wait_until_host_is_in_state(self.host.id, "Maintenance") - ssh_client = self.get_ssh_client(self.host.ipaddress, username, password) + ssh_client = self.get_ssh_client(self.host.ipaddress, self.hostConfig["username"], + self.hostConfig["password"]) ssh_client.execute("service cloudstack-agent stop") self.wait_until_agent_is_in_state(self.host.id, "Disconnected") @@ -572,13 +560,12 @@ def test_04_cancel_host_maintenance_ssh_disabled_agent_disconnected(self): if self.is_ssh_enabled(): self.set_ssh_enabled(False) - username, password = self.get_host_credentials(self.host.id) - try: self.prepare_host_for_maintenance(self.host.id) self.wait_until_host_is_in_state(self.host.id, "Maintenance") - ssh_client = self.get_ssh_client(self.host.ipaddress, username, password) + ssh_client = self.get_ssh_client(self.host.ipaddress, self.hostConfig["username"], + self.hostConfig["password"]) ssh_client.execute("service cloudstack-agent stop") self.wait_until_agent_is_in_state(self.host.id, "Disconnected") except Exception as e: @@ -587,7 +574,8 @@ def test_04_cancel_host_maintenance_ssh_disabled_agent_disconnected(self): self.assertRaises(Exception, self.cancel_host_maintenance, self.host.id) try: - ssh_client = self.get_ssh_client(self.host.ipaddress, username, password) + ssh_client = self.get_ssh_client(self.host.ipaddress, self.hostConfig["username"], + self.hostConfig["password"]) ssh_client.execute("service cloudstack-agent start") self.wait_until_agent_is_in_state(self.host.id, "Up") From a5e39edd50363ec2a9683f8ffc437e900f199cd5 Mon Sep 17 00:00:00 2001 From: nvazquez Date: Thu, 25 Apr 2019 11:57:33 -0300 Subject: [PATCH 6/6] Add check on SSH restart and improve marvin tests --- .../cloud/resource/ResourceManagerImpl.java | 3 +++ .../smoke/test_host_maintenance.py | 22 +++++++++++++++++-- 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/server/src/com/cloud/resource/ResourceManagerImpl.java b/server/src/com/cloud/resource/ResourceManagerImpl.java index c259eb666f38..34a5196d8397 100755 --- a/server/src/com/cloud/resource/ResourceManagerImpl.java +++ b/server/src/com/cloud/resource/ResourceManagerImpl.java @@ -2409,6 +2409,9 @@ protected void connectAndRestartAgentOnHost(HostVO host, String username, String try { SSHCmdHelper.SSHCmdResult result = SSHCmdHelper.sshExecuteCmdOneShot( connection, "service cloudstack-agent restart"); + if (result.getReturnCode() != 0) { + throw new CloudRuntimeException("Could not restart agent on host " + host.getId() + " due to: " + result.getStdErr()); + } s_logger.debug("cloudstack-agent restart result: " + result.toString()); } catch (final SshException e) { throw new CloudRuntimeException("SSH to agent is enabled, but agent restart failed", e); diff --git a/test/integration/smoke/test_host_maintenance.py b/test/integration/smoke/test_host_maintenance.py index 645aa427cad4..c7cd9d3998f5 100644 --- a/test/integration/smoke/test_host_maintenance.py +++ b/test/integration/smoke/test_host_maintenance.py @@ -359,11 +359,17 @@ def is_ssh_enabled(cls): else: return bool(strtobool(conf[0].value)) if conf[0].value else False + @classmethod + def updateConfiguration(self, name, value): + cmd = updateConfiguration.updateConfigurationCmd() + cmd.name = name + cmd.value = value + self.apiclient.updateConfiguration(cmd) + @classmethod def set_ssh_enabled(cls, on): value = "true" if on else "false" - sql = "update configuration set value = '%s' where name = 'kvm.ssh.to.agent';" % value - cls.dbclient.execute(sql) + cls.updateConfiguration('kvm.ssh.to.agent', value) def prepare_host_for_maintenance(self, hostid): cmd = prepareHostForMaintenance.prepareHostForMaintenanceCmd() @@ -445,6 +451,13 @@ def assert_host_is_functional_after_cancelling_maintenance(self, hostid): ) self.cleanup.append(vm) + def revert_host_state_on_failure(self, host): + cmd = updateHost.updateHostCmd() + cmd.id = host.id + cmd.allocationstate = "Enable" + response = self.apiclient.updateHost(cmd) + self.assertEqual(response.resourcestate, "Enabled") + @skipTestIf("hypervisorNotSupported") @attr(tags=["advanced", "advancedns", "smoke", "basic", "eip", "sg"], required_hardware="true") def test_01_cancel_host_maintenance_ssh_enabled_agent_connected(self): @@ -467,6 +480,7 @@ def test_01_cancel_host_maintenance_ssh_enabled_agent_connected(self): self.wait_until_host_is_in_state(self.host.id, "Enabled") self.assert_host_is_functional_after_cancelling_maintenance(self.host.id) except Exception as e: + self.revert_host_state_on_failure(self.host) self.fail(e) def get_ssh_client(self, ip, username, password, retries=10): @@ -515,6 +529,7 @@ def test_02_cancel_host_maintenance_ssh_enabled_agent_disconnected(self): self.assert_host_is_functional_after_cancelling_maintenance(self.host.id) except Exception as e: + self.revert_host_state_on_failure(self.host) self.fail(e) @skipTestIf("hypervisorNotSupported") @@ -539,6 +554,7 @@ def test_03_cancel_host_maintenance_ssh_disabled_agent_connected(self): self.wait_until_host_is_in_state(self.host.id, "Enabled") self.assert_host_is_functional_after_cancelling_maintenance(self.host.id) except Exception as e: + self.revert_host_state_on_failure(self.host) self.fail(e) @skipTestIf("hypervisorNotSupported") @@ -569,6 +585,7 @@ def test_04_cancel_host_maintenance_ssh_disabled_agent_disconnected(self): ssh_client.execute("service cloudstack-agent stop") self.wait_until_agent_is_in_state(self.host.id, "Disconnected") except Exception as e: + self.revert_host_state_on_failure(self.host) self.fail(e) self.assertRaises(Exception, self.cancel_host_maintenance, self.host.id) @@ -583,4 +600,5 @@ def test_04_cancel_host_maintenance_ssh_disabled_agent_disconnected(self): self.wait_until_host_is_in_state(self.host.id, "Enabled") self.assert_host_is_functional_after_cancelling_maintenance(self.host.id) except Exception as e: + self.revert_host_state_on_failure(self.host) self.fail(e)