Skip to content

Commit c558681

Browse files
Boris Stoyanovyadvr
Boris Stoyanov
authored and
yadvr
committed
CLOUDSTACK-9782: Improve host HA tests
- All tests should pass on KVM, Simulator - Add test cases covering FSM state transitions and actions Signed-off-by: Rohit Yadav <rohit.yadav@shapeblue.com>
1 parent 957b905 commit c558681

File tree

12 files changed

+409
-1107
lines changed

12 files changed

+409
-1107
lines changed

‎.travis.yml‎

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -44,9 +44,7 @@ env:
4444
smoke/test_dynamicroles
4545
smoke/test_global_settings
4646
smoke/test_guest_vlan_range
47-
smoke/test_ha_for_host
48-
smoke/test_ha_kvm_agent
49-
smoke/test_ha_kvm
47+
smoke/test_hostha_kvm
5048
smoke/test_hostha_simulator
5149
smoke/test_hosts
5250
smoke/test_internal_lb

‎api/src/org/apache/cloudstack/api/command/admin/host/PrepareForMaintenanceCmd.java‎

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -108,8 +108,4 @@ public void execute() {
108108
throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Failed to prepare host for maintenance");
109109
}
110110
}
111-
112-
public void setHostId(final Long hostId) {
113-
id = hostId;
114-
}
115111
}

‎plugins/hypervisors/simulator/src/org/apache/cloudstack/ha/SimulatorHAProvider.java‎

Lines changed: 15 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -133,15 +133,8 @@ public Object getConfigValue(final HAProvider.HAProviderConfig name, final Host
133133
}
134134
}
135135

136-
@Override
137-
public boolean preStateTransitionEvent(final HAConfig.HAState oldState, final HAConfig.Event event,
138-
final HAConfig.HAState newState, final HAConfig vo, final boolean status, final Object opaque) {
139-
return false;
140-
}
141-
142-
@Override
143-
public boolean postStateTransitionEvent(final StateMachine2.Transition<HAConfig.HAState, HAConfig.Event> transition,
144-
final HAConfig vo, final boolean status, final Object opaque) {
136+
private boolean addStateTransition(final HAConfig vo, final boolean status,
137+
final HAConfig.HAState oldState, final HAConfig.HAState newState, final HAConfig.Event event) {
145138
if (vo.getResourceType() != HAResource.ResourceType.Host) {
146139
return false;
147140
}
@@ -150,6 +143,18 @@ public boolean postStateTransitionEvent(final StateMachine2.Transition<HAConfig.
150143
return false;
151144
}
152145
final HAResourceCounter counter = haManager.getHACounter(vo.getResourceId(), vo.getResourceType());
153-
return haState.addStateTransition(transition.getToState(), transition.getCurrentState(), transition.getEvent(), counter);
146+
return haState.addStateTransition(newState, oldState, event, counter);
147+
}
148+
149+
@Override
150+
public boolean preStateTransitionEvent(final HAConfig.HAState oldState, final HAConfig.Event event,
151+
final HAConfig.HAState newState, final HAConfig vo, final boolean status, final Object opaque) {
152+
return addStateTransition(vo, status, oldState, newState, event);
153+
}
154+
155+
@Override
156+
public boolean postStateTransitionEvent(final StateMachine2.Transition<HAConfig.HAState, HAConfig.Event> transition,
157+
final HAConfig vo, final boolean status, final Object opaque) {
158+
return addStateTransition(vo, status, transition.getCurrentState(), transition.getToState(), transition.getEvent());
154159
}
155160
}

‎scripts/vm/hypervisor/kvm/kvmvmactivity.sh‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -116,7 +116,8 @@ else
116116
lastUpdateTime=${arrTime[1]}
117117
echo "$SuspectTime:$latestUpdateTime:$MSTime" > $acFile
118118

119-
if [[ $lastSuspectTime -ne $SuspectTime ]]; then
119+
suspectTimeDiff=$(expr $SuspectTime - $lastSuspectTime)
120+
if [[ $suspectTimeDiff -lt 0 ]]; then
120121
if [[ $latestUpdateTime -gt $SuspectTime ]]; then
121122
echo "=====> ALIVE <====="
122123
else

‎server/src/com/cloud/resource/ResourceManagerImpl.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2272,7 +2272,7 @@ private boolean doCancelMaintenance(final long hostId) {
22722272
}
22732273

22742274
try {
2275-
SSHCmdHelper.SSHCmdResult result = SSHCmdHelper.sshExecuteCmdOneShot(connection, "service cloudstack-agent restart");
2275+
SSHCmdHelper.SSHCmdResult result = SSHCmdHelper.sshExecuteCmdOneShot(connection, "service cloudstack-agent restart || systemctl restart cloudstack-agent");
22762276
s_logger.debug("cloudstack-agent restart result: " + result.toString());
22772277
} catch (final SshException e) {
22782278
return false;

‎server/src/org/apache/cloudstack/ha/HAManagerImpl.java‎

Lines changed: 20 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -156,7 +156,9 @@ public boolean transitionHAState(final HAConfig.Event event, final HAConfig haCo
156156
if (result) {
157157
final String message = String.format("Transitioned host HA state from:%s to:%s due to event:%s for the host id:%d",
158158
currentHAState, nextState, event, haConfig.getResourceId());
159-
LOG.debug(message);
159+
if (LOG.isTraceEnabled()) {
160+
LOG.trace(message);
161+
}
160162
if (nextState == HAConfig.HAState.Recovering || nextState == HAConfig.HAState.Fencing || nextState == HAConfig.HAState.Fenced) {
161163
ActionEventUtils.onActionEvent(CallContext.current().getCallingUserId(), CallContext.current().getCallingAccountId(),
162164
Domain.ROOT_DOMAIN, EventTypes.EVENT_HA_STATE_TRANSITION, message);
@@ -475,7 +477,7 @@ public void onManagementNodeLeft(List<? extends ManagementServerHost> nodeList,
475477
public void onManagementNodeIsolated() {
476478
}
477479

478-
private boolean processHAStateChange(final HAConfig haConfig, final boolean status) {
480+
private boolean processHAStateChange(final HAConfig haConfig, final HAConfig.HAState newState, final boolean status) {
479481
if (!status || !checkHAOwnership(haConfig)) {
480482
return false;
481483
}
@@ -493,14 +495,14 @@ private boolean processHAStateChange(final HAConfig haConfig, final boolean stat
493495
final HAResourceCounter counter = getHACounter(haConfig.getResourceId(), haConfig.getResourceType());
494496

495497
// Perform activity checks
496-
if (haConfig.getState() == HAConfig.HAState.Checking) {
498+
if (newState == HAConfig.HAState.Checking) {
497499
final ActivityCheckTask job = ComponentContext.inject(new ActivityCheckTask(resource, haProvider, haConfig,
498500
HAProviderConfig.ActivityCheckTimeout, activityCheckExecutor, counter.getSuspectTimeStamp()));
499501
activityCheckExecutor.submit(job);
500502
}
501503

502504
// Attempt recovery
503-
if (haConfig.getState() == HAConfig.HAState.Recovering) {
505+
if (newState == HAConfig.HAState.Recovering) {
504506
if (counter.getRecoveryCounter() >= (Long) (haProvider.getConfigValue(HAProviderConfig.MaxRecoveryAttempts, resource))) {
505507
return false;
506508
}
@@ -511,7 +513,7 @@ private boolean processHAStateChange(final HAConfig haConfig, final boolean stat
511513
}
512514

513515
// Fencing
514-
if (haConfig.getState() == HAConfig.HAState.Fencing) {
516+
if (newState == HAConfig.HAState.Fencing) {
515517
final FenceTask task = ComponentContext.inject(new FenceTask(resource, haProvider, haConfig,
516518
HAProviderConfig.FenceTimeout, fenceExecutor));
517519
final Future<Boolean> fenceFuture = fenceExecutor.submit(task);
@@ -528,15 +530,21 @@ public boolean preStateTransitionEvent(final HAConfig.HAState oldState, final HA
528530
if (LOG.isTraceEnabled()) {
529531
LOG.trace("HA state pre-transition:: new state=" + newState + ", old state=" + oldState + ", for resource id=" + haConfig.getResourceId() + ", status=" + status + ", ha config state=" + haConfig.getState());
530532
}
531-
return processHAStateChange(haConfig, status);
533+
if (status && haConfig.getState() != newState) {
534+
LOG.warn("HA state pre-transition:: HA state is not equal to transition state, HA state=" + haConfig.getState() + ", new state=" + newState);
535+
}
536+
return processHAStateChange(haConfig, newState, status);
532537
}
533538

534539
@Override
535540
public boolean postStateTransitionEvent(final StateMachine2.Transition<HAConfig.HAState, HAConfig.Event> transition, final HAConfig haConfig, final boolean status, final Object opaque) {
536541
if (LOG.isTraceEnabled()) {
537542
LOG.trace("HA state post-transition:: new state=" + transition.getToState() + ", old state=" + transition.getCurrentState() + ", for resource id=" + haConfig.getResourceId() + ", status=" + status + ", ha config state=" + haConfig.getState());
538543
}
539-
return processHAStateChange(haConfig, status);
544+
if (status && haConfig.getState() != transition.getToState()) {
545+
LOG.warn("HA state post-transition:: HA state is not equal to transition state, HA state=" + haConfig.getState() + ", new state=" + transition.getToState());
546+
}
547+
return processHAStateChange(haConfig, transition.getToState(), status);
540548
}
541549

542550
///////////////////////////////////////////////////
@@ -697,5 +705,10 @@ protected void runInContext() {
697705
LOG.error("Error trying to perform health checks in HA manager", t);
698706
}
699707
}
708+
709+
@Override
710+
public Long getDelay() {
711+
return null;
712+
}
700713
}
701714
}

0 commit comments

Comments
 (0)