CLOUDSTACK-9782: Host HA and KVM HA provider

Host-HA offers investigation, fencing and recovery mechanisms for host that for
any reason are malfunctioning. It uses Activity and Health checks to determine
current host state based on which it may degrade a host or try to recover it. On
failing to recover it, it may try to fence the host.

The core feature is implemented in a hypervisor agnostic way, with two separate
implementations of the driver/provider for Simulator and KVM hypervisors. The
framework also allows for implementation of other hypervisor specific provider
implementation in future.

The Host-HA provider implementation for KVM hypervisor uses the out-of-band
management sub-system to issue IPMI calls to reset (recover) or poweroff (fence)
a host.

The Host-HA provider implementation for Simulator provides a means of testing
and validating the core framework implementation.

Signed-off-by: Abhinandan Prateek <abhinandan.prateek@shapeblue.com>
Signed-off-by: Rohit Yadav <rohit.yadav@shapeblue.com>

Author: yadvr

.travis.yml

@@ -42,13 +42,18 @@ env:
              smoke/test_dynamicroles
              smoke/test_global_settings
              smoke/test_guest_vlan_range
+             smoke/test_ha_for_host
+             smoke/test_ha_kvm_agent
+             smoke/test_ha_kvm
+             smoke/test_hostha_simulator
              smoke/test_hosts
              smoke/test_internal_lb
              smoke/test_iso
              smoke/test_list_ids_parameter
              smoke/test_loadbalance
              smoke/test_login
              smoke/test_multipleips_per_nic
+             smoke/test_nested_virtualization
              smoke/test_network
              smoke/test_network_acl
              smoke/test_nic

api/src/com/cloud/dc/DataCenter.java

@@ -18,15 +18,14 @@
 
 import com.cloud.org.Grouping;
 import org.apache.cloudstack.acl.InfrastructureEntity;
-import org.apache.cloudstack.api.Identity;
-import org.apache.cloudstack.api.InternalIdentity;
+import org.apache.cloudstack.kernel.Partition;
 
 import java.util.Map;
 
 /**
  *
  */
-public interface DataCenter extends InfrastructureEntity, Grouping, Identity, InternalIdentity {
+public interface DataCenter extends InfrastructureEntity, Grouping, Partition {
 
     public enum NetworkType {
         Basic, Advanced,

api/src/com/cloud/event/EventTypes.java

@@ -69,6 +69,7 @@
 import org.apache.cloudstack.acl.Role;
 import org.apache.cloudstack.acl.RolePermission;
 import org.apache.cloudstack.config.Configuration;
+import org.apache.cloudstack.ha.HAConfig;
 import org.apache.cloudstack.usage.Usage;
 
 import java.util.HashMap;
@@ -317,6 +318,12 @@ public class EventTypes {
     public static final String EVENT_HOST_OUTOFBAND_MANAGEMENT_CHANGE_PASSWORD = "HOST.OOBM.CHANGEPASSWORD";
     public static final String EVENT_HOST_OUTOFBAND_MANAGEMENT_POWERSTATE_TRANSITION = "HOST.OOBM.POWERSTATE.TRANSITION";
 
+    // HA
+    public static final String EVENT_HA_RESOURCE_ENABLE = "HA.RESOURCE.ENABLE";
+    public static final String EVENT_HA_RESOURCE_DISABLE = "HA.RESOURCE.DISABLE";
+    public static final String EVENT_HA_RESOURCE_CONFIGURE = "HA.RESOURCE.CONFIGURE";
+    public static final String EVENT_HA_STATE_TRANSITION = "HA.STATE.TRANSITION";
+
     // Maintenance
     public static final String EVENT_MAINTENANCE_CANCEL = "MAINT.CANCEL";
     public static final String EVENT_MAINTENANCE_CANCEL_PRIMARY_STORAGE = "MAINT.CANCEL.PS";
@@ -761,6 +768,12 @@ public class EventTypes {
         entityEventDetails.put(EVENT_HOST_OUTOFBAND_MANAGEMENT_CHANGE_PASSWORD, Host.class);
         entityEventDetails.put(EVENT_HOST_OUTOFBAND_MANAGEMENT_POWERSTATE_TRANSITION, Host.class);
 
+        // HA
+        entityEventDetails.put(EVENT_HA_RESOURCE_ENABLE, HAConfig.class);
+        entityEventDetails.put(EVENT_HA_RESOURCE_DISABLE, HAConfig.class);
+        entityEventDetails.put(EVENT_HA_RESOURCE_CONFIGURE, HAConfig.class);
+        entityEventDetails.put(EVENT_HA_STATE_TRANSITION, HAConfig.class);
+
         // Maintenance
         entityEventDetails.put(EVENT_MAINTENANCE_CANCEL, Host.class);
         entityEventDetails.put(EVENT_MAINTENANCE_CANCEL_PRIMARY_STORAGE, Host.class);

api/src/com/cloud/host/Host.java

@@ -16,19 +16,19 @@
 // under the License.
 package com.cloud.host;
 
-import java.util.Date;
-
-import org.apache.cloudstack.api.Identity;
-import org.apache.cloudstack.api.InternalIdentity;
-
 import com.cloud.hypervisor.Hypervisor.HypervisorType;
 import com.cloud.resource.ResourceState;
 import com.cloud.utils.fsm.StateObject;
+import org.apache.cloudstack.api.Identity;
+import org.apache.cloudstack.ha.HAResource;
+import org.apache.cloudstack.kernel.Partition;
+
+import java.util.Date;
 
 /**
  *  Host represents one particular host server.
  */
-public interface Host extends StateObject<Status>, Identity, InternalIdentity {
+public interface Host extends StateObject<Status>, Identity, Partition, HAResource {
     public enum Type {
         Storage(false), Routing(false), SecondaryStorage(false), SecondaryStorageCmdExecutor(false), ConsoleProxy(true), ExternalFirewall(false), ExternalLoadBalancer(
                 false), ExternalVirtualSwitchSupervisor(false), PxeServer(false), BaremetalPxe(false), BaremetalDhcp(false), TrafficMonitor(false),
@@ -202,5 +202,7 @@ public static String[] toStrings(Host.Type... types) {
 
     boolean isInMaintenanceStates();
 
+    boolean isDisabled();
+
     ResourceState getResourceState();
 }

api/src/com/cloud/host/Status.java

@@ -150,12 +150,14 @@ public static String[] toStrings(Status... states) {
         s_fsm.addTransition(Status.Down, Event.ManagementServerDown, Status.Down);
         s_fsm.addTransition(Status.Down, Event.AgentDisconnected, Status.Down);
         s_fsm.addTransition(Status.Down, Event.PingTimeout, Status.Down);
+        s_fsm.addTransition(Status.Down, Event.HostDown, Status.Down);
         s_fsm.addTransition(Status.Alert, Event.AgentConnected, Status.Connecting);
         s_fsm.addTransition(Status.Alert, Event.Ping, Status.Up);
         s_fsm.addTransition(Status.Alert, Event.Remove, Status.Removed);
         s_fsm.addTransition(Status.Alert, Event.ManagementServerDown, Status.Alert);
         s_fsm.addTransition(Status.Alert, Event.AgentDisconnected, Status.Alert);
         s_fsm.addTransition(Status.Alert, Event.ShutdownRequested, Status.Disconnected);
+        s_fsm.addTransition(Status.Alert, Event.HostDown, Status.Down);
         s_fsm.addTransition(Status.Rebalancing, Event.RebalanceFailed, Status.Disconnected);
         s_fsm.addTransition(Status.Rebalancing, Event.RebalanceCompleted, Status.Connecting);
         s_fsm.addTransition(Status.Rebalancing, Event.ManagementServerDown, Status.Disconnected);

api/src/com/cloud/org/Cluster.java

@@ -16,13 +16,11 @@
 // under the License.
 package com.cloud.org;
 
-import org.apache.cloudstack.api.Identity;
-import org.apache.cloudstack.api.InternalIdentity;
-
 import com.cloud.hypervisor.Hypervisor.HypervisorType;
 import com.cloud.org.Managed.ManagedState;
+import org.apache.cloudstack.kernel.Partition;
 
-public interface Cluster extends Grouping, InternalIdentity, Identity {
+public interface Cluster extends Grouping, Partition {
     public static enum ClusterType {
         CloudManaged, ExternalManaged;
     };

api/src/com/cloud/resource/ResourceState.java

@@ -93,6 +93,7 @@ public static String[] toString(ResourceState... states) {
         s_fsm.addTransition(ResourceState.Enabled, Event.InternalCreated, ResourceState.Enabled);
         s_fsm.addTransition(ResourceState.Enabled, Event.Disable, ResourceState.Disabled);
         s_fsm.addTransition(ResourceState.Enabled, Event.AdminAskMaintenace, ResourceState.PrepareForMaintenance);
+        s_fsm.addTransition(ResourceState.Enabled, Event.InternalEnterMaintenance, ResourceState.Maintenance);
         s_fsm.addTransition(ResourceState.Disabled, Event.Enable, ResourceState.Enabled);
         s_fsm.addTransition(ResourceState.Disabled, Event.Disable, ResourceState.Disabled);
         s_fsm.addTransition(ResourceState.Disabled, Event.InternalCreated, ResourceState.Disabled);
@@ -109,5 +110,7 @@ public static String[] toString(ResourceState... states) {
         s_fsm.addTransition(ResourceState.ErrorInMaintenance, Event.InternalEnterMaintenance, ResourceState.Maintenance);
         s_fsm.addTransition(ResourceState.ErrorInMaintenance, Event.AdminCancelMaintenance, ResourceState.Enabled);
         s_fsm.addTransition(ResourceState.Error, Event.InternalCreated, ResourceState.Error);
+        s_fsm.addTransition(ResourceState.Disabled, Event.DeleteHost, ResourceState.Disabled);
+
     }
 }

api/src/com/cloud/vm/VirtualMachine.java

@@ -16,26 +16,24 @@
 // under the License.
 package com.cloud.vm;
 
-import java.util.Arrays;
-import java.util.Date;
-import java.util.Map;
-
-import org.apache.cloudstack.acl.ControlledEntity;
-import org.apache.cloudstack.api.Displayable;
-import org.apache.cloudstack.api.Identity;
-import org.apache.cloudstack.api.InternalIdentity;
-
 import com.cloud.hypervisor.Hypervisor.HypervisorType;
 import com.cloud.utils.fsm.StateMachine2;
 import com.cloud.utils.fsm.StateMachine2.Transition;
 import com.cloud.utils.fsm.StateMachine2.Transition.Impact;
 import com.cloud.utils.fsm.StateObject;
+import org.apache.cloudstack.acl.ControlledEntity;
+import org.apache.cloudstack.api.Displayable;
+import org.apache.cloudstack.kernel.Partition;
+
+import java.util.Arrays;
+import java.util.Date;
+import java.util.Map;
 
 /**
  * VirtualMachine describes the properties held by a virtual machine
  *
  */
-public interface VirtualMachine extends RunningOn, ControlledEntity, Identity, InternalIdentity, Displayable, StateObject<VirtualMachine.State> {
+public interface VirtualMachine extends RunningOn, ControlledEntity, Partition, Displayable, StateObject<VirtualMachine.State> {
 
     public enum PowerState {
         PowerUnknown,

api/src/org/apache/cloudstack/alert/AlertService.java

@@ -16,12 +16,12 @@
 // under the License.
 package org.apache.cloudstack.alert;
 
-import java.util.HashSet;
-import java.util.Set;
-
 import com.cloud.capacity.Capacity;
 import com.cloud.exception.InvalidParameterValueException;
 
+import java.util.HashSet;
+import java.util.Set;
+
 public interface AlertService {
     public static class AlertType {
         private static Set<AlertType> defaultAlertTypes = new HashSet<AlertType>();
@@ -67,6 +67,7 @@ private AlertType(short type, String name, boolean isDefault) {
         public static final AlertType ALERT_TYPE_SYNC = new AlertType((short)27, "ALERT.TYPE.SYNC", true);
         public static final AlertType ALERT_TYPE_UPLOAD_FAILED = new AlertType((short)28, "ALERT.UPLOAD.FAILED", true);
         public static final AlertType ALERT_TYPE_OOBM_AUTH_ERROR = new AlertType((short)29, "ALERT.OOBM.AUTHERROR", true);
+        public static final AlertType ALERT_TYPE_HA_ACTION = new AlertType((short)30, "ALERT.HA.ACTION", true);
 
         public short getType() {
             return type;

api/src/org/apache/cloudstack/api/ApiConstants.java

@@ -21,6 +21,7 @@ public class ApiConstants {
     public static final String ACCOUNTS = "accounts";
     public static final String ACCOUNT_TYPE = "accounttype";
     public static final String ACCOUNT_ID = "accountid";
+    public static final String ACTIVITY = "activity";
     public static final String ADDRESS = "address";
     public static final String ALGORITHM = "algorithm";
     public static final String ALLOCATED_ONLY = "allocatedonly";
@@ -94,13 +95,15 @@ public class ApiConstants {
     public static final String DOMAIN_ID = "domainid";
     public static final String DOMAIN__ID = "domainId";
     public static final String DURATION = "duration";
+    public static final String ELIGIBLE = "eligible";
     public static final String EMAIL = "email";
     public static final String END_DATE = "enddate";
     public static final String END_IP = "endip";
     public static final String END_IPV6 = "endipv6";
     public static final String END_PORT = "endport";
     public static final String ENTRY_TIME = "entrytime";
     public static final String EXPIRES = "expires";
+    public static final String FENCE = "fence";
     public static final String FETCH_LATEST = "fetchlatest";
     public static final String FIRSTNAME = "firstname";
     public static final String FORCED = "forced";
@@ -119,6 +122,9 @@ public class ApiConstants {
     public static final String GUEST_CIDR_ADDRESS = "guestcidraddress";
     public static final String GUEST_VLAN_RANGE = "guestvlanrange";
     public static final String HA_ENABLE = "haenable";
+    public static final String HA_PROVIDER = "haprovider";
+    public static final String HA_STATE = "hastate";
+    public static final String HEALTH = "health";
     public static final String HOST_ID = "hostid";
     public static final String HOST_NAME = "hostname";
     public static final String HYPERVISOR = "hypervisor";
@@ -215,6 +221,7 @@ public class ApiConstants {
     public static final String PUBLIC_END_PORT = "publicendport";
     public static final String PUBLIC_ZONE = "publiczone";
     public static final String RECEIVED_BYTES = "receivedbytes";
+    public static final String RECOVER = "recover";
     public static final String REQUIRES_HVM = "requireshvm";
     public static final String RESOURCE_TYPE = "resourcetype";
     public static final String RESPONSE = "response";