Merge branch '4.20'

Author: Daan Hoogland

engine/components-api/src/main/java/com/cloud/agent/AgentManager.java

@@ -37,7 +37,7 @@
  * AgentManager manages hosts. It directly coordinates between the DAOs and the connections it manages.
  */
 public interface AgentManager {
-    static final ConfigKey<Integer> Wait = new ConfigKey<Integer>("Advanced", Integer.class, "wait", "1800", "Time in seconds to wait for control commands to return",
+    ConfigKey<Integer> Wait = new ConfigKey<Integer>("Advanced", Integer.class, "wait", "1800", "Time in seconds to wait for control commands to return",
             true);
     ConfigKey<Boolean> EnableKVMAutoEnableDisable = new ConfigKey<>(Boolean.class,
                     "enable.kvm.host.auto.enable.disable",
@@ -54,7 +54,7 @@ public interface AgentManager {
             "This timeout overrides the wait global config. This holds a comma separated key value pairs containing timeout (in seconds) for specific commands. " +
                     "For example: DhcpEntryCommand=600, SavePasswordCommand=300, VmDataCommand=300", false);
 
-    public enum TapAgentsAction {
+    enum TapAgentsAction {
         Add, Del, Contains,
     }
 

engine/orchestration/src/main/java/com/cloud/agent/manager/AgentManagerImpl.java

@@ -55,8 +55,8 @@
 import org.apache.cloudstack.management.ManagementServerHost;
 import org.apache.cloudstack.outofbandmanagement.dao.OutOfBandManagementDao;
 import org.apache.cloudstack.utils.identity.ManagementServerNode;
-import org.apache.commons.collections.MapUtils;
 import org.apache.cloudstack.utils.reflectiontostringbuilderutils.ReflectionToStringBuilderUtils;
+import org.apache.commons.collections.MapUtils;
 import org.apache.commons.lang3.BooleanUtils;
 import org.apache.commons.lang3.StringUtils;
 import org.apache.logging.log4j.ThreadContext;
@@ -223,6 +223,8 @@ public class AgentManagerImpl extends ManagerBase implements AgentManager, Handl
             "Number of maximum concurrent new connections server allows for remote (indirect) agents. " +
                     "If set to zero (default value) then no limit will be enforced on concurrent new connections",
             false);
+    protected final ConfigKey<Integer> RemoteAgentNewConnectionsMonitorInterval = new ConfigKey<>("Advanced", Integer.class, "agent.connections.monitor.interval", "1800",
+            "Time in seconds to monitor the new agent connections and cleanup the expired connections.", false);
     protected final ConfigKey<Integer> AlertWait = new ConfigKey<>("Advanced", Integer.class, "alert.wait", "1800",
             "Seconds to wait before alerting on a disconnected agent", true);
     protected final ConfigKey<Integer> DirectAgentLoadSize = new ConfigKey<>("Advanced", Integer.class, "direct.agent.load.size", "16",
@@ -1945,7 +1947,7 @@ protected class AgentNewConnectionsMonitorTask extends ManagedContextRunnable {
         @Override
         protected void runInContext() {
             logger.trace("Agent New Connections Monitor is started.");
-            final int cleanupTime = Wait.value();
+            final int cleanupTime = RemoteAgentNewConnectionsMonitorInterval.value();
             Set<Map.Entry<String, Long>> entrySet = newAgentConnections.entrySet();
             long cutOff = System.currentTimeMillis() - (cleanupTime * 1000L);
             List<String> expiredConnections = newAgentConnections.entrySet()
@@ -2040,7 +2042,8 @@ public String getConfigComponentName() {
     public ConfigKey<?>[] getConfigKeys() {
         return new ConfigKey<?>[] { CheckTxnBeforeSending, Workers, Port, Wait, AlertWait, DirectAgentLoadSize,
                 DirectAgentPoolSize, DirectAgentThreadCap, EnableKVMAutoEnableDisable, ReadyCommandWait,
-                GranularWaitTimeForCommands, RemoteAgentSslHandshakeTimeout, RemoteAgentMaxConcurrentNewConnections };
+                GranularWaitTimeForCommands, RemoteAgentSslHandshakeTimeout, RemoteAgentMaxConcurrentNewConnections,
+                RemoteAgentNewConnectionsMonitorInterval };
     }
 
     protected class SetHostParamsListener implements Listener {

engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java

@@ -427,7 +427,7 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac
     static final ConfigKey<Long> VmOpCleanupInterval = new ConfigKey<Long>("Advanced", Long.class, "vm.op.cleanup.interval", "86400",
             "Interval to run the thread that cleans up the vm operations (in seconds)", false);
     static final ConfigKey<Long> VmOpCleanupWait = new ConfigKey<Long>("Advanced", Long.class, "vm.op.cleanup.wait", "3600",
-            "Time (in seconds) to wait before cleanuping up any vm work items", true);
+            "Time (in seconds) to wait before cleaning up any vm work items", true);
     static final ConfigKey<Long> VmOpCancelInterval = new ConfigKey<Long>("Advanced", Long.class, "vm.op.cancel.interval", "3600",
             "Time (in seconds) to wait before cancelling a operation", false);
     static final ConfigKey<Boolean> VmDestroyForcestop = new ConfigKey<Boolean>("Advanced", Boolean.class, "vm.destroy.forcestop", "false",

plugins/hypervisors/xenserver/src/main/java/com/cloud/hypervisor/xenserver/resource/XenServerStorageProcessor.java

@@ -23,6 +23,7 @@
 import static com.google.common.collect.Lists.newArrayList;
 
 import java.io.File;
+import java.io.UnsupportedEncodingException;
 import java.net.URI;
 import java.util.ArrayList;
 import java.util.Arrays;
@@ -810,7 +811,7 @@ public Answer createVolume(final CreateObjectCommand cmd) {
             final SR poolSr = hypervisorResource.getStorageRepository(conn,
                     CitrixHelper.getSRNameLabel(primaryStore.getUuid(), primaryStore.getPoolType(), primaryStore.getPath()));
             VDI.Record vdir = new VDI.Record();
-            vdir.nameLabel = volume.getName();
+            vdir.nameLabel = getEncodedVolumeName(volume.getName());
             vdir.SR = poolSr;
             vdir.type = Types.VdiType.USER;
 
@@ -831,6 +832,26 @@ public Answer createVolume(final CreateObjectCommand cmd) {
         }
     }
 
+    private String getEncodedVolumeName(String volumeName) throws UnsupportedEncodingException {
+        byte[] utf8Bytes = volumeName.getBytes("UTF-8");
+        // Decode UTF-8 into a Java String (UTF-16)
+        String decoded = new String(utf8Bytes, "UTF-8");
+        // Print each code unit as a Unicode escape
+        StringBuilder unicodeEscaped = new StringBuilder();
+        for (int i = 0; i < decoded.length(); i++) {
+            char ch = decoded.charAt(i);
+            if (ch <= 127 && Character.isLetterOrDigit(ch)) {
+                // Keep ASCII alphanumerics as-is
+                unicodeEscaped.append(ch);
+            } else {
+                // Escape non-ASCII characters
+                unicodeEscaped.append(String.format("\\u%04X", (int) ch));
+            }
+        }
+
+        return unicodeEscaped.toString();
+    }
+
     @Override
     public Answer cloneVolumeFromBaseTemplate(final CopyCommand cmd) {
         final Connection conn = hypervisorResource.getConnection();

plugins/storage/object/ceph/src/main/java/org/apache/cloudstack/storage/datastore/driver/CephObjectStoreDriverImpl.java

@@ -95,8 +95,8 @@ public Bucket createBucket(Bucket bucket, boolean objectLock) {
         AmazonS3 s3client = getS3Client(storeId, accountId);
 
         try {
-            if (s3client.getBucketAcl(bucketName) != null) {
-                throw new CloudRuntimeException("Bucket already exists with name " + bucketName);
+            if (s3client.doesBucketExistV2(bucketName)) {
+                throw new CloudRuntimeException("Bucket already exists with the name: " + bucketName);
             }
         } catch (AmazonS3Exception e) {
             if (e.getStatusCode() != 404) {
@@ -221,9 +221,11 @@ public boolean createUser(long accountId, long storeId) {
             if (user.isPresent()) {
                 logger.info("User already exists in Ceph RGW: " + username);
                 return true;
+            } else {
+                logger.debug("User does not exist. Creating user in Ceph RGW: " + username);
             }
         } catch (Exception e) {
-            logger.debug("User does not exist. Creating user in Ceph RGW: " + username);
+            logger.debug("Get user info failed for user {} with exception {}. Proceeding with user creation.",  username, e.getMessage());
         }
 
         try {
@@ -348,7 +350,7 @@ protected AmazonS3 getS3Client(String url, String accessKey, String secretKey) {
                         new AWSStaticCredentialsProvider(
                                 new BasicAWSCredentials(accessKey, secretKey)))
                 .withEndpointConfiguration(
-                        new AwsClientBuilder.EndpointConfiguration(url, "auto"))
+                        new AwsClientBuilder.EndpointConfiguration(url, null))
                 .build();
 
         if (client == null) {

plugins/storage/object/ceph/src/test/java/org/apache/cloudstack/storage/datastore/driver/CephObjectStoreDriverImplTest.java

@@ -96,7 +96,7 @@ public void testCreateBucket() throws Exception {
         when(bucketDao.findById(anyLong())).thenReturn(new BucketVO(bucket.getName()));
         Bucket bucketRet = cephObjectStoreDriverImpl.createBucket(bucket, false);
         assertEquals(bucketRet.getName(), bucket.getName());
-        verify(rgwClient, times(1)).getBucketAcl(anyString());
+        verify(rgwClient, times(1)).doesBucketExistV2(anyString());
         verify(rgwClient, times(1)).createBucket(anyString());
     }
 

python/lib/cloudutils/networkConfig.py

@@ -41,7 +41,7 @@ def listNetworks():
         return devs
     @staticmethod
     def getDefaultNetwork():
-        cmd = bash("route -n|awk \'/^0.0.0.0/ {print $2,$8}\'")
+        cmd = bash("ip route show default | awk \'{print $3,$5}\'")
         if not cmd.isSuccess():
             logging.debug("Failed to get default route")
             raise CloudRuntimeException("Failed to get default route")

server/src/main/java/org/apache/cloudstack/network/topology/AdvancedNetworkTopology.java

@@ -229,11 +229,6 @@ public boolean associatePublicIP(final Network network, final List<? extends Pub
     public boolean applyNetworkACLs(final Network network, final List<? extends NetworkACLItem> rules, final VirtualRouter router, final boolean isPrivateGateway)
             throws ResourceUnavailableException {
 
-        if (rules == null || rules.isEmpty()) {
-            logger.debug("No network ACLs to be applied for network {}", network);
-            return true;
-        }
-
         logger.debug("APPLYING NETWORK ACLs RULES");
 
         final String typeString = "network acls";

server/src/test/java/org/apache/cloudstack/resource/ResourceCleanupServiceImplTest.java

@@ -17,6 +17,11 @@
 package org.apache.cloudstack.resource;
 
 import java.lang.reflect.Field;
+import java.time.Duration;
+import java.time.Instant;
+import java.time.ZoneId;
+import java.time.ZonedDateTime;
+import java.time.zone.ZoneRules;
 import java.util.ArrayList;
 import java.util.Calendar;
 import java.util.Date;
@@ -629,9 +634,23 @@ public void testCalculatePastDateFromConfig() {
         Date result = resourceCleanupService.calculatePastDateFromConfig(
                 ResourceCleanupService.ExpungedResourcesPurgeKeepPastDays.key(),
                 days);
-        Date today = new Date();
-        long diff = today.getTime() - result.getTime();
-        Assert.assertEquals(days, TimeUnit.DAYS.convert(diff, TimeUnit.MILLISECONDS));
+        Instant resultInstant = result.toInstant();
+        ZoneId systemZone = ZoneId.systemDefault();
+        ZoneRules rules = systemZone.getRules();
+        ZonedDateTime resultDateTime = resultInstant.atZone(systemZone);
+        boolean isDSTInResultDateTime = rules.isDaylightSavings(resultDateTime.toInstant());
+
+        ZonedDateTime todayDateTime = ZonedDateTime.now(systemZone);
+        boolean isDSTInTodayDateTime = rules.isDaylightSavings(todayDateTime.toInstant());
+
+        Duration duration = Duration.between(resultDateTime, todayDateTime);
+        long actualDays = TimeUnit.DAYS.convert(duration.toMillis(), TimeUnit.MILLISECONDS);
+
+        if (!isDSTInResultDateTime && isDSTInTodayDateTime) {
+            Assert.assertEquals(days - 1, actualDays);
+        } else {
+            Assert.assertEquals(days, actualDays);
+        }
     }
 
     @Test

ui/src/config/section/infra/hosts.js

@@ -32,7 +32,12 @@ export default {
   },
   params: { type: 'routing' },
   columns: () => {
-    const fields = ['name', 'state', 'resourcestate', 'ipaddress', 'arch', 'hypervisor', 'instances', 'powerstate', 'version']
+    const fields = [
+      'name', 'state', 'resourcestate', 'ipaddress',
+      'arch', 'hypervisor', 'instances',
+      { field: 'systeminstances', customTitle: 'system.vms' },
+      'powerstate', 'version'
+    ]
     const metricsFields = ['cpunumber', 'cputotalghz', 'cpuusedghz', 'cpuallocatedghz', 'memorytotalgb', 'memoryusedgb', 'memoryallocatedgb', 'networkread', 'networkwrite']
     if (store.getters.metrics) {
       fields.push(...metricsFields)