<fix>[kvm]: extend reconnect echo timeout after libvirtd restart

Resolves: ZSTAC-84691

Change-Id: I6668736975707575756d7665647867686e6b6b76

Author: Ezreal-07

plugin/kvm/src/main/java/org/zstack/kvm/KVMHost.java

@@ -5316,6 +5316,21 @@ private boolean noStorageAccessible(){
         return inaccessiblePsCount == attachedPsCount && attachedPsCount > 0;
     }
 
+    @Transactional(readOnly = true)
+    private long getKvmAgentEchoTimeoutAfterDeploy(boolean deployed, boolean restartLibvirtdDuringDeploy) {
+        if (!deployed || !restartLibvirtdDuringDeploy) {
+            return TimeUnit.SECONDS.toMillis(CoreGlobalProperty.REST_FACADE_ECHO_TIMEOUT);
+        }
+
+        long vmCount = Q.New(VmInstanceVO.class)
+                .eq(VmInstanceVO_.hostUuid, self.getUuid())
+                .count();
+        long timeout = KVMHostUtils.calculateLibvirtRestartEchoTimeoutMillis(vmCount);
+        logger.info(String.format("extend kvmagent echo timeout after libvirtd restart on host[uuid:%s, vmCount:%s] to %sms",
+                self.getUuid(), vmCount, timeout));
+        return timeout;
+    }
+
     private void updateHostOsInformation(String distro, String release, String version) {
         final KVMHostVO kvmHostVO = getSelf();
         kvmHostVO.setOsDistribution(distro);
@@ -5586,6 +5601,7 @@ public void connectHook(final ConnectHostInfo info, final Completion complete) {
         chain.allowWatch();
         chain.then(new ShareFlow() {
             boolean deployed = false;
+            boolean restartLibvirtdDuringDeploy = false;
             @Override
             public void setup() {
                 flow(new NoRollbackFlow() {
@@ -5935,6 +5951,8 @@ public void run(final FlowTrigger trigger, Map data) {
                         if (deployArguments.isForceRun()) {
                             runner.setForceRun(true);
                         }
+                        restartLibvirtdDuringDeploy = KVMHostUtils.shouldRestartLibvirtdDuringDeploy(
+                                deployArguments.getInit(), deployArguments.getRestartLibvirtd());
 
                         UriComponentsBuilder ub = UriComponentsBuilder.fromHttpUrl(restf.getBaseUrl());
                         ub.path(new StringBind(KVMConstant.KVM_ANSIBLE_LOG_PATH_FROMAT).bind("uuid", self.getUuid()).toString());
@@ -6023,6 +6041,7 @@ public boolean skip(Map data) {
 
                     @Override
                     public void run(final FlowTrigger trigger, Map data) {
+                        final long echoTimeout = getKvmAgentEchoTimeoutAfterDeploy(deployed, restartLibvirtdDuringDeploy);
                         restf.echo(echoPath, new Completion(trigger) {
                             @Override
                             public void success() {
@@ -6054,7 +6073,7 @@ public void success() {
                                                 public void fail(ErrorCode errorCode) {
                                                     trigger.fail(errorCode);
                                                 }
-                                            });
+                                            }, TimeUnit.SECONDS.toMillis(1), echoTimeout);
                                         }
 
                                         @Override
@@ -6066,7 +6085,7 @@ public void fail(ErrorCode errorCode) {
                                     trigger.fail(errorCode);
                                 }
                             }
-                        });
+                        }, TimeUnit.SECONDS.toMillis(1), echoTimeout);
                     }
                 });
 

plugin/kvm/src/main/java/org/zstack/kvm/KVMHostUtils.java

@@ -22,6 +22,7 @@
 import java.util.List;
 import java.util.Locale;
 import java.util.Set;
+import java.util.concurrent.TimeUnit;
 
 /**
  * Created by GuoYi on 4/16/20.
@@ -33,6 +34,9 @@ public class KVMHostUtils {
     // to keep check-flow and deploy-flow IP lists identical.
     public static final Set<String> EXCLUDED_INTERNAL_IPS = Collections.unmodifiableSet(
             new LinkedHashSet<>(Collections.singletonList("169.254.64.1")));
+    public static final long LIBVIRT_RESTART_ECHO_TIMEOUT_VM_THRESHOLD = 100;
+    public static final long LIBVIRT_RESTART_ECHO_TIMEOUT_PER_VM_SECONDS = 1;
+    public static final long LIBVIRT_RESTART_ECHO_TIMEOUT_MAX_SECONDS = 180;
 
     // Collect host IPv4 addresses; mirrors host_plugin.fact() filters and
     // applies EXCLUDED_INTERNAL_IPS so check and deploy share one source.
@@ -147,6 +151,22 @@ public static boolean shouldForceTlsRedeploy(boolean needDeployTlsCert,
         return allowRestartLibvirtd || isNewAdded;
     }
 
+    public static boolean shouldRestartLibvirtdDuringDeploy(String init, String restartLibvirtd) {
+        return "true".equalsIgnoreCase(init) || "true".equalsIgnoreCase(restartLibvirtd);
+    }
+
+    public static long calculateLibvirtRestartEchoTimeoutMillis(long vmCount) {
+        long defaultTimeoutSeconds = CoreGlobalProperty.REST_FACADE_ECHO_TIMEOUT;
+        if (vmCount <= LIBVIRT_RESTART_ECHO_TIMEOUT_VM_THRESHOLD) {
+            return TimeUnit.SECONDS.toMillis(defaultTimeoutSeconds);
+        }
+
+        long maxExtraSeconds = Math.max(0, LIBVIRT_RESTART_ECHO_TIMEOUT_MAX_SECONDS - defaultTimeoutSeconds);
+        long extraVmCount = vmCount - LIBVIRT_RESTART_ECHO_TIMEOUT_VM_THRESHOLD;
+        long extraSeconds = Math.min(extraVmCount * LIBVIRT_RESTART_ECHO_TIMEOUT_PER_VM_SECONDS, maxExtraSeconds);
+        return TimeUnit.SECONDS.toMillis(defaultTimeoutSeconds + extraSeconds);
+    }
+
     public static boolean shouldContinueReconnectOnAnsibleFailure(boolean isNewAdded, ErrorCode errorCode) {
         return !isNewAdded && isLibvirtSocketMaskSystemdTimeout(errorCode);
     }