<fix>[vm]: 修复迁移失败锁回滚

compute/src/main/java/org/zstack/compute/vm/VmInstanceBase.java

@@ -7179,25 +7179,30 @@ public void done() {
             @Override
             public void handle(final ErrorCode errCode, Map data) {
                 String destHostUuid = spec.getDestHost().getUuid().equals(lastHostUuid) ? null : spec.getDestHost().getUuid();
-                extEmitter.failedToMigrateVm(VmInstanceInventory.valueOf(self), destHostUuid, errCode, new NoErrorCompletion(completion) {
+                rollbackFailedMigrateVm(originState, destHostUuid, errCode, completion);
+            }
+        }).start();
+    }
+
+    private void rollbackFailedMigrateVm(final VmInstanceState originState, final String destHostUuid,
+                                         final ErrorCode errCode, final Completion completion) {
+        extEmitter.failedToMigrateVm(VmInstanceInventory.valueOf(self), destHostUuid, errCode, new NoErrorCompletion(completion) {
+            @Override
+            public void done() {
+                if (!HostErrors.FAILED_TO_MIGRATE_VM_ON_HYPERVISOR.isEqual(errCode.getCode())) {
+                    changeVmStateInDb(originState.getDrivenEvent());
+                    completion.fail(errCode);
+                    return;
+                }
+
+                checkState(originalCopy.getHostUuid(), new NoErrorCompletion(completion) {
                     @Override
                     public void done() {
-                        if (!HostErrors.FAILED_TO_MIGRATE_VM_ON_HYPERVISOR.isEqual(errCode.getCode())) {
-                            changeVmStateInDb(originState.getDrivenEvent());
-                            completion.fail(errCode);
-                            return;
-                        }
-
-                        checkState(originalCopy.getHostUuid(), new NoErrorCompletion(completion) {
-                            @Override
-                            public void done() {
-                                completion.fail(errCode);
-                            }
-                        });
+                        completion.fail(errCode);
                     }
                 });
             }
-        }).start();
+        });
     }
 
     protected void handle(CancelMigrateVmMsg msg) {
@@ -9272,4 +9277,3 @@ public void run(MessageReply reply) {
         });
     }
 }
-

compute/src/main/java/org/zstack/compute/vm/VmMigrateOnHypervisorFlow.java

@@ -10,6 +10,8 @@
 import org.zstack.header.core.workflow.FlowRollback;
 import org.zstack.header.core.workflow.FlowTrigger;
 import org.zstack.header.errorcode.ErrorCode;
+import org.zstack.header.host.CheckVmStateOnHypervisorMsg;
+import org.zstack.header.host.CheckVmStateOnHypervisorReply;
 import org.zstack.header.host.HostConstant;
 import org.zstack.header.message.MessageReply;
 import org.zstack.header.host.MigrateVmOnHypervisorMsg;
@@ -18,6 +20,8 @@
 
 import java.util.Map;
 
+import static org.zstack.utils.CollectionDSL.list;
+
 @Configurable(preConstruction = true, autowire = Autowire.BY_TYPE)
 public class VmMigrateOnHypervisorFlow implements Flow {
     @Autowired
@@ -60,8 +64,34 @@ public void run(MessageReply reply) {
                 if (reply.isSuccess()) {
                     chain.next();
                 } else {
-                    chain.fail(reply.getError());
+                    checkVmStateOnDestinationHost(spec, reply.getError(), chain);
+                }
+            }
+        });
+    }
+
+    private void checkVmStateOnDestinationHost(final VmInstanceSpec spec, final ErrorCode migrateError,
+                                               final FlowTrigger chain) {
+        CheckVmStateOnHypervisorMsg msg = new CheckVmStateOnHypervisorMsg();
+        msg.setVmInstanceUuids(list(spec.getVmInventory().getUuid()));
+        msg.setHostUuid(spec.getDestHost().getUuid());
+        bus.makeTargetServiceIdByResourceUuid(msg, HostConstant.SERVICE_ID, msg.getHostUuid());
+        bus.send(msg, new CloudBusCallBack(chain) {
+            @Override
+            public void run(MessageReply reply) {
+                if (!reply.isSuccess()) {
+                    chain.fail(migrateError);
+                    return;
                 }
+
+                CheckVmStateOnHypervisorReply r = reply.castReply();
+                String state = r.getStates().get(spec.getVmInventory().getUuid());
+                if (VmInstanceState.Running.toString().equals(state)) {
+                    chain.next();
+                    return;
+                }
+
+                chain.fail(migrateError);
             }
         });
     }

test/src/test/groovy/org/zstack/test/integration/kvm/vm/migrate/MigrateVmFailureCheckTargetHostCase.groovy

@@ -0,0 +1,213 @@
+package org.zstack.test.integration.kvm.vm.migrate
+
+import org.zstack.core.cloudbus.CloudBus
+import org.zstack.header.host.CheckVmStateOnHypervisorMsg
+import org.zstack.header.host.CheckVmStateOnHypervisorReply
+import org.zstack.header.network.service.NetworkServiceType
+import org.zstack.header.vm.VmInstanceState
+import org.zstack.kvm.KVMAgentCommands
+import org.zstack.kvm.KVMConstant
+import org.zstack.network.securitygroup.SecurityGroupConstant
+import org.zstack.network.service.flat.FlatNetworkServiceConstant
+import org.zstack.network.service.userdata.UserdataConstant
+import org.zstack.sdk.HostInventory
+import org.zstack.sdk.MigrateVmAction
+import org.zstack.sdk.VmInstanceInventory
+import org.zstack.test.integration.kvm.KvmTest
+import org.zstack.testlib.EnvSpec
+import org.zstack.testlib.SubCase
+import org.zstack.utils.data.SizeUnit
+
+class MigrateVmFailureCheckTargetHostCase extends SubCase {
+    EnvSpec env
+
+    @Override
+    void setup() {
+        useSpring(KvmTest.springSpec)
+    }
+
+    @Override
+    void environment() {
+        env = env {
+            instanceOffering {
+                name = "instanceOffering"
+                memory = SizeUnit.GIGABYTE.toByte(1)
+                cpu = 1
+            }
+
+            cephBackupStorage {
+                name = "ceph-bk"
+                fsid = "7ff218d9-f525-435f-8a40-3618d1772a64"
+                monUrls = ["root:password@localhost:23", "root:password@127.0.0.1:23"]
+
+                image {
+                    name = "image1"
+                    url = "http://zstack.org/download/test.qcow2"
+                }
+            }
+
+            zone {
+                name = "zone"
+
+                cluster {
+                    name = "cluster"
+                    hypervisorType = "KVM"
+
+                    kvm {
+                        name = "kvm1"
+                        managementIp = "127.0.0.1"
+                        username = "root"
+                        password = "password"
+                    }
+
+                    kvm {
+                        name = "kvm2"
+                        managementIp = "127.0.0.2"
+                        username = "root"
+                        password = "password"
+                    }
+
+                    attachPrimaryStorage("ceph-pri")
+                    attachL2Network("l2")
+                }
+
+                cephPrimaryStorage {
+                    name = "ceph-pri"
+                    fsid = "7ff218d9-f525-435f-8a40-3618d1772a64"
+                    monUrls = ["root:password@localhost/?monPort=7777", "root:password@127.0.0.1/?monPort=7777"]
+                }
+
+                l2NoVlanNetwork {
+                    name = "l2"
+                    physicalInterface = "eth0"
+
+                    l3Network {
+                        name = "l3"
+
+                        service {
+                            provider = FlatNetworkServiceConstant.FLAT_NETWORK_SERVICE_TYPE_STRING
+                            types = [NetworkServiceType.DHCP.toString(), UserdataConstant.USERDATA_TYPE_STRING]
+                        }
+
+                        service {
+                            provider = SecurityGroupConstant.SECURITY_GROUP_PROVIDER_TYPE
+                            types = [SecurityGroupConstant.SECURITY_GROUP_NETWORK_SERVICE_TYPE]
+                        }
+
+                        ip {
+                            startIp = "192.168.100.10"
+                            endIp = "192.168.100.100"
+                            netmask = "255.255.255.0"
+                            gateway = "192.168.100.1"
+                        }
+                    }
+                }
+
+                attachBackupStorage("ceph-bk")
+            }
+
+            vm {
+                name = "vm"
+                useInstanceOffering("instanceOffering")
+                useImage("image1")
+                useL3Networks("l3")
+            }
+        }
+    }
+
+    @Override
+    void test() {
+        env.create {
+            testRollbackWhenTargetHostReportsVmNotRunning()
+            testMigrationSuccessWhenTargetHostReportsVmRunning()
+        }
+    }
+
+    @Override
+    void clean() {
+        env.delete()
+    }
+
+    void testRollbackWhenTargetHostReportsVmNotRunning() {
+        VmInstanceInventory vm = env.inventoryByName("vm") as VmInstanceInventory
+        HostInventory destHost = findAnotherHost(vm.hostUuid)
+
+        assertRollbackWhenTargetReports(vm, destHost, VmInstanceState.Stopped.toString())
+        assertRollbackWhenTargetReports(vm, destHost, VmInstanceState.Paused.toString())
+    }
+
+    void assertRollbackWhenTargetReports(VmInstanceInventory vm, HostInventory destHost, String targetHostState) {
+        List<String> checkedHosts = []
+
+        mockMigrateVmFailure()
+        mockVmState(vm.uuid, destHost.uuid, targetHostState, checkedHosts)
+
+        MigrateVmAction.Result result = migrateVmAction(vm.uuid, destHost.uuid).call()
+
+        assert result.error != null
+        VmInstanceInventory after = queryVmInstance {
+            conditions = ["uuid=${vm.uuid}".toString()]
+        }[0] as VmInstanceInventory
+        assert after.hostUuid == vm.hostUuid
+        assert after.state == VmInstanceState.Running.toString()
+        assert checkedHosts[0] == destHost.uuid
+    }
+
+    void testMigrationSuccessWhenTargetHostReportsVmRunning() {
+        VmInstanceInventory vm = queryVmInstance {
+            conditions = ["name=vm"]
+        }[0] as VmInstanceInventory
+        HostInventory destHost = findAnotherHost(vm.hostUuid)
+        List<String> checkedHosts = []
+
+        mockMigrateVmFailure()
+        mockVmState(vm.uuid, destHost.uuid, VmInstanceState.Running.toString(), checkedHosts)
+
+        MigrateVmAction.Result result = migrateVmAction(vm.uuid, destHost.uuid).call()
+
+        assert result.error == null
+        VmInstanceInventory after = queryVmInstance {
+            conditions = ["uuid=${vm.uuid}".toString()]
+        }[0] as VmInstanceInventory
+        assert after.hostUuid == destHost.uuid
+        assert after.lastHostUuid == vm.hostUuid
+        assert checkedHosts
+        assert checkedHosts.every { it == destHost.uuid }
+    }
+
+    HostInventory findAnotherHost(String hostUuid) {
+        return queryHost {
+            conditions = ["uuid!=${hostUuid}".toString()]
+        }[0] as HostInventory
+    }
+
+    void mockMigrateVmFailure() {
+        env.simulator(KVMConstant.KVM_MIGRATE_VM_PATH) {
+            KVMAgentCommands.MigrateVmResponse rsp = new KVMAgentCommands.MigrateVmResponse()
+            rsp.setError("mock migration API failure")
+            return rsp
+        }
+    }
+
+    void mockVmState(String vmUuid, String hostUuid, String targetHostState, List<String> checkedHosts) {
+        env.revokeMessage(CheckVmStateOnHypervisorMsg.class, null)
+        env.message(CheckVmStateOnHypervisorMsg.class) { CheckVmStateOnHypervisorMsg msg, CloudBus bus ->
+            CheckVmStateOnHypervisorReply reply = new CheckVmStateOnHypervisorReply()
+            Map<String, String> states = new HashMap<>()
+            checkedHosts.add(msg.hostUuid)
+            msg.vmInstanceUuids.each {
+                states.put(it, it == vmUuid && msg.hostUuid == hostUuid ? targetHostState : VmInstanceState.Running.toString())
+            }
+            reply.setStates(states)
+            bus.reply(msg, reply)
+        }
+    }
+
+    MigrateVmAction migrateVmAction(String vmUuid, String destHostUuid) {
+        MigrateVmAction action = new MigrateVmAction()
+        action.sessionId = adminSession()
+        action.vmInstanceUuid = vmUuid
+        action.hostUuid = destHostUuid
+        return action
+    }
+}