From f498394a23f7ce419f2a32ffd1fae3d16a50b352 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Wed, 3 Feb 2021 13:23:55 +0530 Subject: [PATCH 01/10] server: destroy ssvm, cpvm on last host maintenance When a single or last UP host enters into maintenance just stopping SSVM and CPVM will leave behind VMs on hypervisor side. As these system vms will be recreated they can be destroyed. Fixes #3719 Signed-off-by: Abhishek Kumar --- .../java/com/cloud/resource/ResourceManagerImpl.java | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/server/src/main/java/com/cloud/resource/ResourceManagerImpl.java b/server/src/main/java/com/cloud/resource/ResourceManagerImpl.java index 6945d6f43685..4e286f259ee7 100755 --- a/server/src/main/java/com/cloud/resource/ResourceManagerImpl.java +++ b/server/src/main/java/com/cloud/resource/ResourceManagerImpl.java @@ -1249,9 +1249,15 @@ private boolean doMaintain(final long hostId) { if (hosts == null || hosts.isEmpty() || !answer.getMigrate() || _serviceOfferingDetailsDao.findDetail(vm.getServiceOfferingId(), GPU.Keys.vgpuType.toString()) != null) { // Migration is not supported for VGPU Vms so stop them. - // for the last host in this cluster, stop all the VMs - s_logger.error("Maintenance: No hosts available for migrations. Scheduling shutdown instead of migrations."); - _haMgr.scheduleStop(vm, hostId, WorkType.ForceStop); + // for the last host in this cluster, destroy SSVM/CPVM and stop all other VMs + if (VirtualMachine.Type.SecondaryStorageVm.equals(vm.getType()) + || VirtualMachine.Type.ConsoleProxy.equals(vm.getType())) { + s_logger.error(String.format("Maintenance: No hosts available for migrations. Scheduling destroy for VM %s instead of migration.", vm.getUuid())); + _haMgr.scheduleDestroy(vm, hostId); + } else { + s_logger.error(String.format("Maintenance: No hosts available for migrations. Scheduling shutdown for VM %s instead of migration.", vm.getUuid())); + _haMgr.scheduleStop(vm, hostId, WorkType.ForceStop); + } } else if (HypervisorType.LXC.equals(host.getHypervisorType()) && VirtualMachine.Type.User.equals(vm.getType())){ //Migration is not supported for LXC Vms. Schedule restart instead. _haMgr.scheduleRestart(vm, false); From b1f74bf1c8877f31208cb0c093d847f2a898b2c7 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Thu, 4 Feb 2021 12:07:41 +0530 Subject: [PATCH 02/10] fix methods Signed-off-by: Abhishek Kumar --- .../cloud/ha/HighAvailabilityManagerImpl.java | 77 ++++++++++--------- .../cloud/resource/ResourceManagerImpl.java | 8 +- 2 files changed, 44 insertions(+), 41 deletions(-) diff --git a/server/src/main/java/com/cloud/ha/HighAvailabilityManagerImpl.java b/server/src/main/java/com/cloud/ha/HighAvailabilityManagerImpl.java index b05e008546be..c70ea02a0866 100644 --- a/server/src/main/java/com/cloud/ha/HighAvailabilityManagerImpl.java +++ b/server/src/main/java/com/cloud/ha/HighAvailabilityManagerImpl.java @@ -16,6 +16,28 @@ // under the License. package com.cloud.ha; +import java.util.ArrayList; +import java.util.Date; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; + +import javax.inject.Inject; +import javax.naming.ConfigurationException; + +import org.apache.cloudstack.engine.orchestration.service.VolumeOrchestrationService; +import org.apache.cloudstack.framework.config.ConfigKey; +import org.apache.cloudstack.framework.config.Configurable; +import org.apache.cloudstack.framework.config.dao.ConfigurationDao; +import org.apache.cloudstack.managed.context.ManagedContext; +import org.apache.cloudstack.managed.context.ManagedContextRunnable; +import org.apache.cloudstack.management.ManagementServerHost; +import org.apache.log4j.Logger; +import org.apache.log4j.NDC; + import com.cloud.agent.AgentManager; import com.cloud.alert.AlertManager; import com.cloud.cluster.ClusterManagerListener; @@ -52,31 +74,9 @@ import com.cloud.utils.exception.CloudRuntimeException; import com.cloud.vm.VMInstanceVO; import com.cloud.vm.VirtualMachine; -import com.cloud.vm.VirtualMachine.State; import com.cloud.vm.VirtualMachineManager; import com.cloud.vm.VirtualMachineProfile; import com.cloud.vm.dao.VMInstanceDao; -import org.apache.cloudstack.engine.orchestration.service.VolumeOrchestrationService; -import org.apache.cloudstack.framework.config.ConfigKey; -import org.apache.cloudstack.framework.config.Configurable; -import org.apache.cloudstack.framework.config.dao.ConfigurationDao; -import org.apache.cloudstack.managed.context.ManagedContext; -import org.apache.cloudstack.managed.context.ManagedContextRunnable; -import org.apache.cloudstack.management.ManagementServerHost; -import org.apache.log4j.Logger; -import org.apache.log4j.NDC; - -import java.util.ArrayList; -import java.util.Date; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.concurrent.Executors; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.TimeUnit; - -import javax.inject.Inject; -import javax.naming.ConfigurationException; /** * HighAvailabilityManagerImpl coordinates the HA process. VMs are registered with the HA Manager for HA. The request is stored @@ -682,25 +682,28 @@ public void cancelDestroy(VMInstanceVO vm, Long hostId) { protected Long destroyVM(final HaWorkVO work) { final VirtualMachine vm = _itMgr.findById(work.getInstanceId()); + if (vm == null) { + s_logger.info("No longer can find VM " + work.getInstanceId() + ". Throwing away " + work); + work.setStep(Step.Done); + return null; + } + boolean expunge = VirtualMachine.Type.SecondaryStorageVm.equals(vm.getType()) + || VirtualMachine.Type.ConsoleProxy.equals(vm.getType()); + if (!expunge && VirtualMachine.State.Destroyed.equals(work.getPreviousState())) { + s_logger.info("VM " + vm.getUuid() + " already in " + vm.getState() + " state. Throwing away " + work); + work.setStep(Step.Done); + return null; + } s_logger.info("Destroying " + vm.toString()); try { - if (vm.getState() != State.Destroyed) { - s_logger.info("VM is no longer in Destroyed state " + vm.toString()); - return null; - } - - if (vm.getHostId() != null) { - _itMgr.destroy(vm.getUuid(), false); - s_logger.info("Successfully destroy " + vm); - return null; + if (!VirtualMachine.State.Expunging.equals(work.getPreviousState())) { + s_logger.info("Destroying " + vm.getUuid()); + _itMgr.destroy(vm.getUuid(), expunge); } else { - if (s_logger.isDebugEnabled()) { - s_logger.debug(vm + " has already been stopped"); - } - return null; + s_logger.info("VM " + vm.getUuid() + " still in " + vm.getState() + " state."); } } catch (final AgentUnavailableException e) { - s_logger.debug("Agnet is not available" + e.getMessage()); + s_logger.debug("Agent is not available" + e.getMessage()); } catch (OperationTimedoutException e) { s_logger.debug("operation timed out: " + e.getMessage()); } catch (ConcurrentOperationException e) { @@ -795,7 +798,7 @@ private long getRescheduleTime(WorkType workType) { case ForceStop: return ((System.currentTimeMillis() >> 10) + _stopRetryInterval); case Destroy: - return ((System.currentTimeMillis() >> 10) + _restartRetryInterval); + return ((System.currentTimeMillis() >> 10) + _stopRetryInterval); } return 0; } diff --git a/server/src/main/java/com/cloud/resource/ResourceManagerImpl.java b/server/src/main/java/com/cloud/resource/ResourceManagerImpl.java index 4e286f259ee7..5460438fad77 100755 --- a/server/src/main/java/com/cloud/resource/ResourceManagerImpl.java +++ b/server/src/main/java/com/cloud/resource/ResourceManagerImpl.java @@ -1252,12 +1252,12 @@ private boolean doMaintain(final long hostId) { // for the last host in this cluster, destroy SSVM/CPVM and stop all other VMs if (VirtualMachine.Type.SecondaryStorageVm.equals(vm.getType()) || VirtualMachine.Type.ConsoleProxy.equals(vm.getType())) { - s_logger.error(String.format("Maintenance: No hosts available for migrations. Scheduling destroy for VM %s instead of migration.", vm.getUuid())); + s_logger.error(String.format("Maintenance: VM is of type %s. Scheduling destroy for VM %s instead of migration.", vm.getType().toString(), vm.getUuid())); _haMgr.scheduleDestroy(vm, hostId); - } else { - s_logger.error(String.format("Maintenance: No hosts available for migrations. Scheduling shutdown for VM %s instead of migration.", vm.getUuid())); - _haMgr.scheduleStop(vm, hostId, WorkType.ForceStop); + continue; } + s_logger.error(String.format("Maintenance: No hosts available for migrations. Scheduling shutdown for VM %s instead of migration.", vm.getUuid())); + _haMgr.scheduleStop(vm, hostId, WorkType.ForceStop); } else if (HypervisorType.LXC.equals(host.getHypervisorType()) && VirtualMachine.Type.User.equals(vm.getType())){ //Migration is not supported for LXC Vms. Schedule restart instead. _haMgr.scheduleRestart(vm, false); From 620c9c54a50af6c6d7a6f83bbf40ac2d632aab75 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Thu, 25 Mar 2021 12:33:31 +0530 Subject: [PATCH 03/10] immediately destroy systemvms Signed-off-by: Abhishek Kumar --- .../java/com/cloud/resource/ResourceManagerImpl.java | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/server/src/main/java/com/cloud/resource/ResourceManagerImpl.java b/server/src/main/java/com/cloud/resource/ResourceManagerImpl.java index 5460438fad77..f1cde0357ee7 100755 --- a/server/src/main/java/com/cloud/resource/ResourceManagerImpl.java +++ b/server/src/main/java/com/cloud/resource/ResourceManagerImpl.java @@ -103,6 +103,7 @@ import com.cloud.exception.AgentUnavailableException; import com.cloud.exception.DiscoveryException; import com.cloud.exception.InvalidParameterValueException; +import com.cloud.exception.OperationTimedoutException; import com.cloud.exception.PermissionDeniedException; import com.cloud.exception.ResourceInUseException; import com.cloud.gpu.GPU; @@ -1252,8 +1253,13 @@ private boolean doMaintain(final long hostId) { // for the last host in this cluster, destroy SSVM/CPVM and stop all other VMs if (VirtualMachine.Type.SecondaryStorageVm.equals(vm.getType()) || VirtualMachine.Type.ConsoleProxy.equals(vm.getType())) { - s_logger.error(String.format("Maintenance: VM is of type %s. Scheduling destroy for VM %s instead of migration.", vm.getType().toString(), vm.getUuid())); - _haMgr.scheduleDestroy(vm, hostId); + s_logger.error(String.format("Maintenance: VM is of type %s. Destroying for VM %s immediately instead of migration.", vm.getType().toString(), vm.getUuid())); + // Scheduled destroy may not work as HAManager won't find agent + try { + _vmMgr.destroy(vm.getUuid(), true); + } catch (AgentUnavailableException | OperationTimedoutException e) { + s_logger.error(String.format("Failed to destroy VM %s (ID: %s) ", vm.getInstanceName(), vm.getUuid()), e); + } continue; } s_logger.error(String.format("Maintenance: No hosts available for migrations. Scheduling shutdown for VM %s instead of migration.", vm.getUuid())); From f6158080dd9feac044b815a7f21590c9f76c89a8 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Thu, 22 Apr 2021 21:32:01 +0530 Subject: [PATCH 04/10] fix destroy Added bypassHostMaintenance flag in Comma.java class to allow command to be handled by host agent even when host is in maintenace. Flag is set true only for delete commands for ssvm and cpvm. Signed-off-by: Abhishek Kumar --- .../java/com/cloud/agent/api/Command.java | 9 +++++++++ .../com/cloud/agent/manager/AgentAttache.java | 8 ++++---- .../cloud/vm/VirtualMachineManagerImpl.java | 12 +++++++++++ .../CloudStackPrimaryDataStoreDriverImpl.java | 20 +++++++++++++++---- .../consoleproxy/ConsoleProxyManagerImpl.java | 7 +++++++ .../cloud/ha/HighAvailabilityManagerImpl.java | 15 ++++++++++++-- .../cloud/resource/ResourceManagerImpl.java | 15 ++++---------- .../SecondaryStorageManagerImpl.java | 7 +++++++ 8 files changed, 72 insertions(+), 21 deletions(-) diff --git a/api/src/main/java/com/cloud/agent/api/Command.java b/api/src/main/java/com/cloud/agent/api/Command.java index 4a698e46bdec..b3c6120462a4 100644 --- a/api/src/main/java/com/cloud/agent/api/Command.java +++ b/api/src/main/java/com/cloud/agent/api/Command.java @@ -37,6 +37,7 @@ public static enum OnError { @LogLevel(Log4jLevel.Trace) protected Map contextMap = new HashMap(); private int wait; //in second + private boolean bypassHostMaintenance = false; protected Command() { this.wait = 0; @@ -74,6 +75,14 @@ public boolean allowCaching() { return true; } + public boolean isBypassHostMaintenance() { + return bypassHostMaintenance; + } + + public void setBypassHostMaintenance(boolean bypassHostMaintenance) { + this.bypassHostMaintenance = bypassHostMaintenance; + } + @Override public boolean equals(Object o) { if (this == o) return true; diff --git a/engine/orchestration/src/main/java/com/cloud/agent/manager/AgentAttache.java b/engine/orchestration/src/main/java/com/cloud/agent/manager/AgentAttache.java index 45df2311f3a6..8810465d794d 100644 --- a/engine/orchestration/src/main/java/com/cloud/agent/manager/AgentAttache.java +++ b/engine/orchestration/src/main/java/com/cloud/agent/manager/AgentAttache.java @@ -31,10 +31,7 @@ import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; -import com.cloud.agent.api.ModifySshKeysCommand; -import com.cloud.agent.api.ModifyStoragePoolCommand; import org.apache.cloudstack.agent.lb.SetupMSListCommand; -import com.cloud.agent.api.RollingMaintenanceCommand; import org.apache.cloudstack.managed.context.ManagedContextRunnable; import org.apache.log4j.Logger; @@ -48,10 +45,13 @@ import com.cloud.agent.api.Command; import com.cloud.agent.api.MaintainCommand; import com.cloud.agent.api.MigrateCommand; +import com.cloud.agent.api.ModifySshKeysCommand; +import com.cloud.agent.api.ModifyStoragePoolCommand; import com.cloud.agent.api.ModifyTargetsCommand; import com.cloud.agent.api.PingTestCommand; import com.cloud.agent.api.PvlanSetupCommand; import com.cloud.agent.api.ReadyCommand; +import com.cloud.agent.api.RollingMaintenanceCommand; import com.cloud.agent.api.SetupCommand; import com.cloud.agent.api.ShutdownCommand; import com.cloud.agent.api.StartCommand; @@ -167,7 +167,7 @@ protected void checkAvailability(final Command[] cmds) throws AgentUnavailableEx if (_maintenance) { for (final Command cmd : cmds) { - if (Arrays.binarySearch(s_commandsAllowedInMaintenanceMode, cmd.getClass().toString()) < 0) { + if (Arrays.binarySearch(s_commandsAllowedInMaintenanceMode, cmd.getClass().toString()) < 0 && !cmd.isBypassHostMaintenance()) { throw new AgentUnavailableException("Unable to send " + cmd.getClass().toString() + " because agent " + _name + " is in maintenance mode", _id); } } diff --git a/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java b/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java index 6faeeb5f4666..10c4c1b7b9b9 100755 --- a/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java +++ b/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java @@ -565,6 +565,10 @@ protected void advanceExpunge(VMInstanceVO vm) throws ResourceUnavailableExcepti final Commands cmds = new Commands(Command.OnError.Stop); for (final Command volumeExpungeCommand : volumeExpungeCommands) { + if (VirtualMachine.Type.SecondaryStorageVm.equals(vm.getType()) || + VirtualMachine.Type.ConsoleProxy.equals(vm.getType())) { + volumeExpungeCommand.setBypassHostMaintenance(true); + } cmds.addCommand(volumeExpungeCommand); } @@ -606,10 +610,18 @@ protected void advanceExpunge(VMInstanceVO vm) throws ResourceUnavailableExcepti if (hostId != null) { final Commands cmds = new Commands(Command.OnError.Stop); for (final Command command : finalizeExpungeCommands) { + if (VirtualMachine.Type.SecondaryStorageVm.equals(vm.getType()) || + VirtualMachine.Type.ConsoleProxy.equals(vm.getType())) { + command.setBypassHostMaintenance(true); + } cmds.addCommand(command); } if (nicExpungeCommands != null) { for (final Command command : nicExpungeCommands) { + if (VirtualMachine.Type.SecondaryStorageVm.equals(vm.getType()) || + VirtualMachine.Type.ConsoleProxy.equals(vm.getType())) { + command.setBypassHostMaintenance(true); + } cmds.addCommand(command); } } diff --git a/plugins/storage/volume/default/src/main/java/org/apache/cloudstack/storage/datastore/driver/CloudStackPrimaryDataStoreDriverImpl.java b/plugins/storage/volume/default/src/main/java/org/apache/cloudstack/storage/datastore/driver/CloudStackPrimaryDataStoreDriverImpl.java index 6ce874107b32..b726ec3acf59 100644 --- a/plugins/storage/volume/default/src/main/java/org/apache/cloudstack/storage/datastore/driver/CloudStackPrimaryDataStoreDriverImpl.java +++ b/plugins/storage/volume/default/src/main/java/org/apache/cloudstack/storage/datastore/driver/CloudStackPrimaryDataStoreDriverImpl.java @@ -18,14 +18,14 @@ */ package org.apache.cloudstack.storage.datastore.driver; +import static com.cloud.utils.NumbersUtil.toHumanReadableSize; + import java.util.HashMap; import java.util.Map; import java.util.UUID; import javax.inject.Inject; -import org.apache.log4j.Logger; - import org.apache.cloudstack.engine.subsystem.api.storage.ChapInfo; import org.apache.cloudstack.engine.subsystem.api.storage.CopyCommandResult; import org.apache.cloudstack.engine.subsystem.api.storage.CreateCmdResult; @@ -53,6 +53,7 @@ import org.apache.cloudstack.storage.to.SnapshotObjectTO; import org.apache.cloudstack.storage.to.TemplateObjectTO; import org.apache.cloudstack.storage.volume.VolumeObject; +import org.apache.log4j.Logger; import com.cloud.agent.api.Answer; import com.cloud.agent.api.storage.ResizeVolumeAnswer; @@ -70,16 +71,17 @@ import com.cloud.storage.Storage; import com.cloud.storage.StorageManager; import com.cloud.storage.StoragePool; +import com.cloud.storage.Volume; import com.cloud.storage.dao.DiskOfferingDao; import com.cloud.storage.dao.SnapshotDao; import com.cloud.storage.dao.VMTemplateDao; import com.cloud.storage.dao.VolumeDao; import com.cloud.storage.snapshot.SnapshotManager; import com.cloud.template.TemplateManager; +import com.cloud.vm.VMInstanceVO; +import com.cloud.vm.VirtualMachine; import com.cloud.vm.dao.VMInstanceDao; -import static com.cloud.utils.NumbersUtil.toHumanReadableSize; - public class CloudStackPrimaryDataStoreDriverImpl implements PrimaryDataStoreDriver { @Override public Map getCapabilities() { @@ -214,6 +216,16 @@ public void createAsync(DataStore dataStore, DataObject data, AsyncCompletionCal @Override public void deleteAsync(DataStore dataStore, DataObject data, AsyncCompletionCallback callback) { DeleteCommand cmd = new DeleteCommand(data.getTO()); + if (DataObjectType.VOLUME.equals(data.getType())) { + Volume volume = (Volume)data; + if (volume.getInstanceId() != null) { + VMInstanceVO vm = vmDao.findById(volume.getInstanceId()); + if (vm != null && (VirtualMachine.Type.SecondaryStorageVm.equals(vm.getType()) || + VirtualMachine.Type.ConsoleProxy.equals(vm.getType()))) { + cmd.setBypassHostMaintenance(true); + } + } + } CommandResult result = new CommandResult(); try { diff --git a/server/src/main/java/com/cloud/consoleproxy/ConsoleProxyManagerImpl.java b/server/src/main/java/com/cloud/consoleproxy/ConsoleProxyManagerImpl.java index 8dfa1f67db0a..3a89d9641c57 100644 --- a/server/src/main/java/com/cloud/consoleproxy/ConsoleProxyManagerImpl.java +++ b/server/src/main/java/com/cloud/consoleproxy/ConsoleProxyManagerImpl.java @@ -1006,6 +1006,13 @@ private void allocCapacity(long dataCenterId) { } public boolean isZoneReady(Map zoneHostInfoMap, long dataCenterId) { + List hosts = _hostDao.listByDataCenterId(dataCenterId); + if (CollectionUtils.isEmpty(hosts)) { + if (s_logger.isDebugEnabled()) { + s_logger.debug("Zone " + dataCenterId + " has no host available which is enabled and in Up state"); + } + return false; + } ZoneHostInfo zoneHostInfo = zoneHostInfoMap.get(dataCenterId); if (zoneHostInfo != null && isZoneHostReady(zoneHostInfo)) { VMTemplateVO template = _templateDao.findSystemVMReadyTemplate(dataCenterId, HypervisorType.Any); diff --git a/server/src/main/java/com/cloud/ha/HighAvailabilityManagerImpl.java b/server/src/main/java/com/cloud/ha/HighAvailabilityManagerImpl.java index c70ea02a0866..49047fc30a04 100644 --- a/server/src/main/java/com/cloud/ha/HighAvailabilityManagerImpl.java +++ b/server/src/main/java/com/cloud/ha/HighAvailabilityManagerImpl.java @@ -41,6 +41,7 @@ import com.cloud.agent.AgentManager; import com.cloud.alert.AlertManager; import com.cloud.cluster.ClusterManagerListener; +import com.cloud.consoleproxy.ConsoleProxyManager; import com.cloud.dc.ClusterDetailsDao; import com.cloud.dc.DataCenterVO; import com.cloud.dc.HostPodVO; @@ -68,6 +69,7 @@ import com.cloud.storage.StorageManager; import com.cloud.storage.dao.GuestOSCategoryDao; import com.cloud.storage.dao.GuestOSDao; +import com.cloud.storage.secondary.SecondaryStorageVmManager; import com.cloud.user.AccountManager; import com.cloud.utils.component.ManagerBase; import com.cloud.utils.concurrency.NamedThreadFactory; @@ -125,9 +127,12 @@ public class HighAvailabilityManagerImpl extends ManagerBase implements Configur HostPodDao _podDao; @Inject ClusterDetailsDao _clusterDetailsDao; - @Inject ServiceOfferingDao _serviceOfferingDao; + @Inject + private ConsoleProxyManager consoleProxyMgr; + @Inject + private SecondaryStorageVmManager secondaryStorageVmManager; long _serverId; @@ -698,7 +703,13 @@ protected Long destroyVM(final HaWorkVO work) { try { if (!VirtualMachine.State.Expunging.equals(work.getPreviousState())) { s_logger.info("Destroying " + vm.getUuid()); - _itMgr.destroy(vm.getUuid(), expunge); + if (VirtualMachine.Type.ConsoleProxy.equals(vm.getType())) { + consoleProxyMgr.destroyProxy(vm.getId()); + } else if (VirtualMachine.Type.SecondaryStorageVm.equals(vm.getType())) { + secondaryStorageVmManager.destroySecStorageVm(vm.getId()); + } else { + _itMgr.destroy(vm.getUuid(), expunge); + } } else { s_logger.info("VM " + vm.getUuid() + " still in " + vm.getState() + " state."); } diff --git a/server/src/main/java/com/cloud/resource/ResourceManagerImpl.java b/server/src/main/java/com/cloud/resource/ResourceManagerImpl.java index 77577e9eb49f..19c1e2ca9381 100755 --- a/server/src/main/java/com/cloud/resource/ResourceManagerImpl.java +++ b/server/src/main/java/com/cloud/resource/ResourceManagerImpl.java @@ -16,6 +16,8 @@ // under the License. package com.cloud.resource; +import static com.cloud.configuration.ConfigurationManagerImpl.SET_HOST_DOWN_TO_MAINTENANCE; + import java.net.URI; import java.net.URISyntaxException; import java.net.URLDecoder; @@ -104,7 +106,6 @@ import com.cloud.exception.AgentUnavailableException; import com.cloud.exception.DiscoveryException; import com.cloud.exception.InvalidParameterValueException; -import com.cloud.exception.OperationTimedoutException; import com.cloud.exception.PermissionDeniedException; import com.cloud.exception.ResourceInUseException; import com.cloud.gpu.GPU; @@ -181,9 +182,6 @@ import com.cloud.vm.dao.VMInstanceDao; import com.google.gson.Gson; - -import static com.cloud.configuration.ConfigurationManagerImpl.SET_HOST_DOWN_TO_MAINTENANCE; - @Component public class ResourceManagerImpl extends ManagerBase implements ResourceManager, ResourceService, Manager { private static final Logger s_logger = Logger.getLogger(ResourceManagerImpl.class); @@ -1271,13 +1269,8 @@ private boolean doMaintain(final long hostId) { // for the last host in this cluster, destroy SSVM/CPVM and stop all other VMs if (VirtualMachine.Type.SecondaryStorageVm.equals(vm.getType()) || VirtualMachine.Type.ConsoleProxy.equals(vm.getType())) { - s_logger.error(String.format("Maintenance: VM is of type %s. Destroying for VM %s immediately instead of migration.", vm.getType().toString(), vm.getUuid())); - // Scheduled destroy may not work as HAManager won't find agent - try { - _vmMgr.destroy(vm.getUuid(), true); - } catch (AgentUnavailableException | OperationTimedoutException e) { - s_logger.error(String.format("Failed to destroy VM %s (ID: %s) ", vm.getInstanceName(), vm.getUuid()), e); - } + s_logger.error(String.format("Maintenance: VM is of type %s. Destroying VM %s (ID: %s) immediately instead of migration.", vm.getType().toString(), vm.getInstanceName(), vm.getUuid())); + _haMgr.scheduleDestroy(vm, host.getId()); continue; } s_logger.error(String.format("Maintenance: No hosts available for migrations. Scheduling shutdown for VM %s instead of migration.", vm.getUuid())); diff --git a/services/secondary-storage/controller/src/main/java/org/apache/cloudstack/secondarystorage/SecondaryStorageManagerImpl.java b/services/secondary-storage/controller/src/main/java/org/apache/cloudstack/secondarystorage/SecondaryStorageManagerImpl.java index 2fc1eeda53c4..f0b3c9172f93 100644 --- a/services/secondary-storage/controller/src/main/java/org/apache/cloudstack/secondarystorage/SecondaryStorageManagerImpl.java +++ b/services/secondary-storage/controller/src/main/java/org/apache/cloudstack/secondarystorage/SecondaryStorageManagerImpl.java @@ -811,6 +811,13 @@ public void allocCapacity(long dataCenterId, SecondaryStorageVm.Role role) { } public boolean isZoneReady(Map zoneHostInfoMap, long dataCenterId) { + List hosts = _hostDao.listByDataCenterId(dataCenterId); + if (CollectionUtils.isEmpty(hosts)) { + if (s_logger.isDebugEnabled()) { + s_logger.debug("Zone " + dataCenterId + " has no host available which is enabled and in Up state"); + } + return false; + } ZoneHostInfo zoneHostInfo = zoneHostInfoMap.get(dataCenterId); if (zoneHostInfo != null && (zoneHostInfo.getFlags() & RunningHostInfoAgregator.ZoneHostInfo.ROUTING_HOST_MASK) != 0) { VMTemplateVO template = _templateDao.findSystemVMReadyTemplate(dataCenterId, HypervisorType.Any); From 3e1b992b8d691d4b4dba51cda484dbf4af5cea39 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Fri, 23 Apr 2021 01:40:04 +0530 Subject: [PATCH 05/10] unit test fix Signed-off-by: Abhishek Kumar --- .../main/java/com/cloud/ha/HighAvailabilityManagerImpl.java | 4 ++-- .../java/com/cloud/ha/HighAvailabilityManagerImplTest.java | 6 ++++++ 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/server/src/main/java/com/cloud/ha/HighAvailabilityManagerImpl.java b/server/src/main/java/com/cloud/ha/HighAvailabilityManagerImpl.java index 49047fc30a04..363acc7a1e1a 100644 --- a/server/src/main/java/com/cloud/ha/HighAvailabilityManagerImpl.java +++ b/server/src/main/java/com/cloud/ha/HighAvailabilityManagerImpl.java @@ -130,7 +130,7 @@ public class HighAvailabilityManagerImpl extends ManagerBase implements Configur @Inject ServiceOfferingDao _serviceOfferingDao; @Inject - private ConsoleProxyManager consoleProxyMgr; + private ConsoleProxyManager consoleProxyManager; @Inject private SecondaryStorageVmManager secondaryStorageVmManager; @@ -704,7 +704,7 @@ protected Long destroyVM(final HaWorkVO work) { if (!VirtualMachine.State.Expunging.equals(work.getPreviousState())) { s_logger.info("Destroying " + vm.getUuid()); if (VirtualMachine.Type.ConsoleProxy.equals(vm.getType())) { - consoleProxyMgr.destroyProxy(vm.getId()); + consoleProxyManager.destroyProxy(vm.getId()); } else if (VirtualMachine.Type.SecondaryStorageVm.equals(vm.getType())) { secondaryStorageVmManager.destroySecStorageVm(vm.getId()); } else { diff --git a/server/src/test/java/com/cloud/ha/HighAvailabilityManagerImplTest.java b/server/src/test/java/com/cloud/ha/HighAvailabilityManagerImplTest.java index 7410f1e6e03b..39fa6bb2d673 100644 --- a/server/src/test/java/com/cloud/ha/HighAvailabilityManagerImplTest.java +++ b/server/src/test/java/com/cloud/ha/HighAvailabilityManagerImplTest.java @@ -44,6 +44,7 @@ import com.cloud.agent.AgentManager; import com.cloud.alert.AlertManager; +import com.cloud.consoleproxy.ConsoleProxyManager; import com.cloud.dc.ClusterDetailsDao; import com.cloud.dc.DataCenterVO; import com.cloud.dc.HostPodVO; @@ -64,6 +65,7 @@ import com.cloud.storage.StorageManager; import com.cloud.storage.dao.GuestOSCategoryDao; import com.cloud.storage.dao.GuestOSDao; +import com.cloud.storage.secondary.SecondaryStorageVmManager; import com.cloud.user.AccountManager; import com.cloud.vm.VMInstanceVO; import com.cloud.vm.VirtualMachine; @@ -112,6 +114,10 @@ public class HighAvailabilityManagerImplTest { @Mock VolumeOrchestrationService volumeMgr; @Mock + ConsoleProxyManager consoleProxyManager; + @Mock + SecondaryStorageVmManager secondaryStorageVmManager; + @Mock HostVO hostVO; HighAvailabilityManagerImpl highAvailabilityManager; From da674943c9b635010d008e3a83bcfa212639bbf8 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Thu, 29 Apr 2021 17:36:40 +0530 Subject: [PATCH 06/10] fix missing return statement Signed-off-by: Abhishek Kumar --- .../src/main/java/com/cloud/ha/HighAvailabilityManagerImpl.java | 1 + 1 file changed, 1 insertion(+) diff --git a/server/src/main/java/com/cloud/ha/HighAvailabilityManagerImpl.java b/server/src/main/java/com/cloud/ha/HighAvailabilityManagerImpl.java index 363acc7a1e1a..d0c9a90d8ecf 100644 --- a/server/src/main/java/com/cloud/ha/HighAvailabilityManagerImpl.java +++ b/server/src/main/java/com/cloud/ha/HighAvailabilityManagerImpl.java @@ -710,6 +710,7 @@ protected Long destroyVM(final HaWorkVO work) { } else { _itMgr.destroy(vm.getUuid(), expunge); } + return null; } else { s_logger.info("VM " + vm.getUuid() + " still in " + vm.getState() + " state."); } From 87c96ae2bd86c34c1ed8d46600fda47062699766 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Fri, 30 Apr 2021 00:32:27 +0530 Subject: [PATCH 07/10] fix VM should be stopped with cleanup before calling expunge else it server may through error with host in PrepareForMaintenance state. Signed-off-by: Abhishek Kumar --- .../java/com/cloud/ha/HighAvailabilityManagerImpl.java | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/server/src/main/java/com/cloud/ha/HighAvailabilityManagerImpl.java b/server/src/main/java/com/cloud/ha/HighAvailabilityManagerImpl.java index d0c9a90d8ecf..645233a69725 100644 --- a/server/src/main/java/com/cloud/ha/HighAvailabilityManagerImpl.java +++ b/server/src/main/java/com/cloud/ha/HighAvailabilityManagerImpl.java @@ -699,8 +699,11 @@ protected Long destroyVM(final HaWorkVO work) { work.setStep(Step.Done); return null; } - s_logger.info("Destroying " + vm.toString()); try { + if (VirtualMachine.State.Running.equals(work.getPreviousState())) { + _itMgr.advanceStop(vm.getUuid(), true); + } + s_logger.info("Destroying " + vm.toString()); if (!VirtualMachine.State.Expunging.equals(work.getPreviousState())) { s_logger.info("Destroying " + vm.getUuid()); if (VirtualMachine.Type.ConsoleProxy.equals(vm.getType())) { @@ -720,6 +723,8 @@ protected Long destroyVM(final HaWorkVO work) { s_logger.debug("operation timed out: " + e.getMessage()); } catch (ConcurrentOperationException e) { s_logger.debug("concurrent operation: " + e.getMessage()); + } catch (ResourceUnavailableException e) { + s_logger.debug("Resource unavailable: " + e.getMessage()); } return (System.currentTimeMillis() >> 10) + _stopRetryInterval; From 17edecbcdda966c16df39f4f2eb8c0698b2b4744 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Wed, 5 May 2021 15:07:34 +0530 Subject: [PATCH 08/10] refactor Signed-off-by: Abhishek Kumar --- .../cloud/vm/VirtualMachineManagerImpl.java | 20 +++++------ .../CloudStackPrimaryDataStoreDriverImpl.java | 16 +++++---- .../cloud/ha/HighAvailabilityManagerImpl.java | 33 +++++++++++-------- .../cloud/resource/ResourceManagerImpl.java | 26 ++++++++------- 4 files changed, 51 insertions(+), 44 deletions(-) diff --git a/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java b/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java index 648ec80c08dd..9a1bd5431973 100755 --- a/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java +++ b/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java @@ -519,6 +519,11 @@ public void advanceExpunge(final String vmUuid) throws ResourceUnavailableExcept advanceExpunge(vm); } + private boolean expungeCommandsCanBypassHostMaintenance(VirtualMachine vm) { + return VirtualMachine.Type.SecondaryStorageVm.equals(vm.getType()) || + VirtualMachine.Type.ConsoleProxy.equals(vm.getType()); + } + protected void advanceExpunge(VMInstanceVO vm) throws ResourceUnavailableException, OperationTimedoutException, ConcurrentOperationException { if (vm == null || vm.getRemoved() != null) { if (s_logger.isDebugEnabled()) { @@ -565,10 +570,7 @@ protected void advanceExpunge(VMInstanceVO vm) throws ResourceUnavailableExcepti final Commands cmds = new Commands(Command.OnError.Stop); for (final Command volumeExpungeCommand : volumeExpungeCommands) { - if (VirtualMachine.Type.SecondaryStorageVm.equals(vm.getType()) || - VirtualMachine.Type.ConsoleProxy.equals(vm.getType())) { - volumeExpungeCommand.setBypassHostMaintenance(true); - } + volumeExpungeCommand.setBypassHostMaintenance(expungeCommandsCanBypassHostMaintenance(vm)); cmds.addCommand(volumeExpungeCommand); } @@ -610,18 +612,12 @@ protected void advanceExpunge(VMInstanceVO vm) throws ResourceUnavailableExcepti if (hostId != null) { final Commands cmds = new Commands(Command.OnError.Stop); for (final Command command : finalizeExpungeCommands) { - if (VirtualMachine.Type.SecondaryStorageVm.equals(vm.getType()) || - VirtualMachine.Type.ConsoleProxy.equals(vm.getType())) { - command.setBypassHostMaintenance(true); - } + command.setBypassHostMaintenance(expungeCommandsCanBypassHostMaintenance(vm)); cmds.addCommand(command); } if (nicExpungeCommands != null) { for (final Command command : nicExpungeCommands) { - if (VirtualMachine.Type.SecondaryStorageVm.equals(vm.getType()) || - VirtualMachine.Type.ConsoleProxy.equals(vm.getType())) { - command.setBypassHostMaintenance(true); - } + command.setBypassHostMaintenance(expungeCommandsCanBypassHostMaintenance(vm)); cmds.addCommand(command); } } diff --git a/plugins/storage/volume/default/src/main/java/org/apache/cloudstack/storage/datastore/driver/CloudStackPrimaryDataStoreDriverImpl.java b/plugins/storage/volume/default/src/main/java/org/apache/cloudstack/storage/datastore/driver/CloudStackPrimaryDataStoreDriverImpl.java index b726ec3acf59..65b55f72c39a 100644 --- a/plugins/storage/volume/default/src/main/java/org/apache/cloudstack/storage/datastore/driver/CloudStackPrimaryDataStoreDriverImpl.java +++ b/plugins/storage/volume/default/src/main/java/org/apache/cloudstack/storage/datastore/driver/CloudStackPrimaryDataStoreDriverImpl.java @@ -213,20 +213,22 @@ public void createAsync(DataStore dataStore, DataObject data, AsyncCompletionCal } } - @Override - public void deleteAsync(DataStore dataStore, DataObject data, AsyncCompletionCallback callback) { - DeleteCommand cmd = new DeleteCommand(data.getTO()); + private boolean commandCanBypassHostMaintenance(DataObject data) { if (DataObjectType.VOLUME.equals(data.getType())) { Volume volume = (Volume)data; if (volume.getInstanceId() != null) { VMInstanceVO vm = vmDao.findById(volume.getInstanceId()); - if (vm != null && (VirtualMachine.Type.SecondaryStorageVm.equals(vm.getType()) || - VirtualMachine.Type.ConsoleProxy.equals(vm.getType()))) { - cmd.setBypassHostMaintenance(true); - } + return vm != null && (VirtualMachine.Type.SecondaryStorageVm.equals(vm.getType()) || + VirtualMachine.Type.ConsoleProxy.equals(vm.getType())); } } + return false; + } + @Override + public void deleteAsync(DataStore dataStore, DataObject data, AsyncCompletionCallback callback) { + DeleteCommand cmd = new DeleteCommand(data.getTO()); + cmd.setBypassHostMaintenance(commandCanBypassHostMaintenance(data)); CommandResult result = new CommandResult(); try { EndPoint ep = null; diff --git a/server/src/main/java/com/cloud/ha/HighAvailabilityManagerImpl.java b/server/src/main/java/com/cloud/ha/HighAvailabilityManagerImpl.java index 645233a69725..23d935e3a01e 100644 --- a/server/src/main/java/com/cloud/ha/HighAvailabilityManagerImpl.java +++ b/server/src/main/java/com/cloud/ha/HighAvailabilityManagerImpl.java @@ -685,34 +685,39 @@ public void cancelDestroy(VMInstanceVO vm, Long hostId) { _haDao.delete(vm.getId(), WorkType.Destroy); } + private void stopVMWithCleanup(VirtualMachine vm, VirtualMachine.State state) throws OperationTimedoutException, ResourceUnavailableException { + if (VirtualMachine.State.Running.equals(state)) { + _itMgr.advanceStop(vm.getUuid(), true); + } + } + + private void destroyVM(VirtualMachine vm, boolean expunge) throws OperationTimedoutException, AgentUnavailableException { + s_logger.info("Destroying " + vm.toString()); + if (VirtualMachine.Type.ConsoleProxy.equals(vm.getType())) { + consoleProxyManager.destroyProxy(vm.getId()); + } else if (VirtualMachine.Type.SecondaryStorageVm.equals(vm.getType())) { + secondaryStorageVmManager.destroySecStorageVm(vm.getId()); + } else { + _itMgr.destroy(vm.getUuid(), expunge); + } + } + protected Long destroyVM(final HaWorkVO work) { final VirtualMachine vm = _itMgr.findById(work.getInstanceId()); if (vm == null) { s_logger.info("No longer can find VM " + work.getInstanceId() + ". Throwing away " + work); - work.setStep(Step.Done); return null; } boolean expunge = VirtualMachine.Type.SecondaryStorageVm.equals(vm.getType()) || VirtualMachine.Type.ConsoleProxy.equals(vm.getType()); if (!expunge && VirtualMachine.State.Destroyed.equals(work.getPreviousState())) { s_logger.info("VM " + vm.getUuid() + " already in " + vm.getState() + " state. Throwing away " + work); - work.setStep(Step.Done); return null; } try { - if (VirtualMachine.State.Running.equals(work.getPreviousState())) { - _itMgr.advanceStop(vm.getUuid(), true); - } - s_logger.info("Destroying " + vm.toString()); + stopVMWithCleanup(vm, work.getPreviousState()); if (!VirtualMachine.State.Expunging.equals(work.getPreviousState())) { - s_logger.info("Destroying " + vm.getUuid()); - if (VirtualMachine.Type.ConsoleProxy.equals(vm.getType())) { - consoleProxyManager.destroyProxy(vm.getId()); - } else if (VirtualMachine.Type.SecondaryStorageVm.equals(vm.getType())) { - secondaryStorageVmManager.destroySecStorageVm(vm.getId()); - } else { - _itMgr.destroy(vm.getUuid(), expunge); - } + destroyVM(vm, expunge); return null; } else { s_logger.info("VM " + vm.getUuid() + " still in " + vm.getState() + " state."); diff --git a/server/src/main/java/com/cloud/resource/ResourceManagerImpl.java b/server/src/main/java/com/cloud/resource/ResourceManagerImpl.java index 19c1e2ca9381..17c84c71d23e 100755 --- a/server/src/main/java/com/cloud/resource/ResourceManagerImpl.java +++ b/server/src/main/java/com/cloud/resource/ResourceManagerImpl.java @@ -1228,6 +1228,19 @@ public boolean resourceStateTransitTo(final Host host, final ResourceState.Event return _hostDao.updateResourceState(currentState, event, nextState, host); } + private void handleVmForLastHostOrWithVGpu(final HostVO host, final VMInstanceVO vm) { + // Migration is not supported for VGPU Vms so stop them. + // for the last host in this cluster, destroy SSVM/CPVM and stop all other VMs + if (VirtualMachine.Type.SecondaryStorageVm.equals(vm.getType()) + || VirtualMachine.Type.ConsoleProxy.equals(vm.getType())) { + s_logger.error(String.format("Maintenance: VM is of type %s. Destroying VM %s (ID: %s) immediately instead of migration.", vm.getType().toString(), vm.getInstanceName(), vm.getUuid())); + _haMgr.scheduleDestroy(vm, host.getId()); + return; + } + s_logger.error(String.format("Maintenance: No hosts available for migrations. Scheduling shutdown for VM %s instead of migration.", vm.getUuid())); + _haMgr.scheduleStop(vm, host.getId(), WorkType.ForceStop); + } + private boolean doMaintain(final long hostId) { final HostVO host = _hostDao.findById(hostId); s_logger.info("Maintenance: attempting maintenance of host " + host.getUuid()); @@ -1265,16 +1278,7 @@ private boolean doMaintain(final long hostId) { for (final VMInstanceVO vm : vms) { if (hosts == null || hosts.isEmpty() || !answer.getMigrate() || _serviceOfferingDetailsDao.findDetail(vm.getServiceOfferingId(), GPU.Keys.vgpuType.toString()) != null) { - // Migration is not supported for VGPU Vms so stop them. - // for the last host in this cluster, destroy SSVM/CPVM and stop all other VMs - if (VirtualMachine.Type.SecondaryStorageVm.equals(vm.getType()) - || VirtualMachine.Type.ConsoleProxy.equals(vm.getType())) { - s_logger.error(String.format("Maintenance: VM is of type %s. Destroying VM %s (ID: %s) immediately instead of migration.", vm.getType().toString(), vm.getInstanceName(), vm.getUuid())); - _haMgr.scheduleDestroy(vm, host.getId()); - continue; - } - s_logger.error(String.format("Maintenance: No hosts available for migrations. Scheduling shutdown for VM %s instead of migration.", vm.getUuid())); - _haMgr.scheduleStop(vm, hostId, WorkType.ForceStop); + handleVmForLastHostOrWithVGpu(host, vm); } else if (HypervisorType.LXC.equals(host.getHypervisorType()) && VirtualMachine.Type.User.equals(vm.getType())){ //Migration is not supported for LXC Vms. Schedule restart instead. _haMgr.scheduleRestart(vm, false); @@ -1422,7 +1426,7 @@ protected boolean setHostIntoPrepareForMaintenanceAfterErrorsFixed(HostVO host) * on a host. We need to track the various VM states on each run and accordingly transit to the * appropriate state. * - * We change states as follws - + * We change states as follows - * 1. If there are no VMs in running, migrating, starting, stopping, error, unknown states we can move * to maintenance state. Note that there cannot be incoming migrations as the API Call prepare for * maintenance checks incoming migrations before starting. From 092ee8fe3df67a37c63da91e21c85dec8e0cca26 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Wed, 5 May 2021 15:10:07 +0530 Subject: [PATCH 09/10] rename Signed-off-by: Abhishek Kumar --- .../main/java/com/cloud/vm/VirtualMachineManagerImpl.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java b/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java index 9a1bd5431973..f697a171792b 100755 --- a/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java +++ b/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java @@ -519,7 +519,7 @@ public void advanceExpunge(final String vmUuid) throws ResourceUnavailableExcept advanceExpunge(vm); } - private boolean expungeCommandsCanBypassHostMaintenance(VirtualMachine vm) { + private boolean expungeCommandCanBypassHostMaintenance(VirtualMachine vm) { return VirtualMachine.Type.SecondaryStorageVm.equals(vm.getType()) || VirtualMachine.Type.ConsoleProxy.equals(vm.getType()); } @@ -570,7 +570,7 @@ protected void advanceExpunge(VMInstanceVO vm) throws ResourceUnavailableExcepti final Commands cmds = new Commands(Command.OnError.Stop); for (final Command volumeExpungeCommand : volumeExpungeCommands) { - volumeExpungeCommand.setBypassHostMaintenance(expungeCommandsCanBypassHostMaintenance(vm)); + volumeExpungeCommand.setBypassHostMaintenance(expungeCommandCanBypassHostMaintenance(vm)); cmds.addCommand(volumeExpungeCommand); } @@ -612,12 +612,12 @@ protected void advanceExpunge(VMInstanceVO vm) throws ResourceUnavailableExcepti if (hostId != null) { final Commands cmds = new Commands(Command.OnError.Stop); for (final Command command : finalizeExpungeCommands) { - command.setBypassHostMaintenance(expungeCommandsCanBypassHostMaintenance(vm)); + command.setBypassHostMaintenance(expungeCommandCanBypassHostMaintenance(vm)); cmds.addCommand(command); } if (nicExpungeCommands != null) { for (final Command command : nicExpungeCommands) { - command.setBypassHostMaintenance(expungeCommandsCanBypassHostMaintenance(vm)); + command.setBypassHostMaintenance(expungeCommandCanBypassHostMaintenance(vm)); cmds.addCommand(command); } } From 438f4e4aa8b0d0476885e36f668f09d2ece71a83 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Thu, 6 May 2021 12:13:11 +0530 Subject: [PATCH 10/10] refactor Signed-off-by: Abhishek Kumar --- .../src/main/java/com/cloud/ha/HighAvailabilityManagerImpl.java | 1 - 1 file changed, 1 deletion(-) diff --git a/server/src/main/java/com/cloud/ha/HighAvailabilityManagerImpl.java b/server/src/main/java/com/cloud/ha/HighAvailabilityManagerImpl.java index 23d935e3a01e..6991ab4039db 100644 --- a/server/src/main/java/com/cloud/ha/HighAvailabilityManagerImpl.java +++ b/server/src/main/java/com/cloud/ha/HighAvailabilityManagerImpl.java @@ -818,7 +818,6 @@ private long getRescheduleTime(WorkType workType) { case Stop: case CheckStop: case ForceStop: - return ((System.currentTimeMillis() >> 10) + _stopRetryInterval); case Destroy: return ((System.currentTimeMillis() >> 10) + _stopRetryInterval); }