Support vm dynamic scaling with kvm (#4878)

* Create utility to centralize byte convertions

* Add/change toString definitions

* Create Libvirt handler to ScaleVmCommand

* Enable dynamic scalling VM with KVM

* Move config from interface to class and rename it

As every variable declared in interfaces are already final,
this moving will be needed to mock tests in nexts commits

* Configure VM max memory and cpu cores

The values are according to service offering or global configs

* Extract dpdk configuration to a method and test it

* Extract OS desc config to a method and test it

* Extract guest resource def to a method and test it

Improve libvirt def

* Refactor LibvirtVMDef.GuestResourceDef

* Refactor ScaleVmCommand

* Improve VMInstaVO toString()

* Refactor upgradeRunningVirtualMachine method

* Turn int variables into long on utility

* Verify if VM is scalable on KVMGuru

* Rename some KVMGuruTest's methods

* Change vm's xml to work with max memory

* Verify if service offering is dynamic before scale

* Create methods to retrieve data from domain

* Create def to hotplug memory

* Adjust the way command was scaling the VM

* Fix database persistence before executing command

* Send more info to host to improve log

* Fix var name

* Fix missing "}"

* Undo unnecessary changes

* Address review

* Fix scale validation

* Add VM prepared for dynamic scaling validation

* Refactor LibvirtScaleVmCommandWrapper and improve unit tests

* Remove duplicated method

* Add RuntimeException check

* Remove copyright from header

* Remove copyright from header

* Remove copyright from header

* Remove copyright from header

* Remove copyright from header

* Update ByteScaleUtilsTest.java

Co-authored-by: Daniel Augusto Veronezi Salvador <daniel@scclouds.com.br>

Author: GutoVeronezi

api/src/main/java/com/cloud/agent/api/to/VirtualMachineTO.java

@@ -146,6 +146,10 @@ public Type getType() {
         return type;
     }
 
+    public void setType(Type type) {
+        this.type = type;
+    }
+
     public BootloaderType getBootloader() {
         return bootloader;
     }

core/src/main/java/com/cloud/agent/api/ScaleVmCommand.java

@@ -52,9 +52,6 @@ public ScaleVmCommand(String vmName, int cpus, Integer minSpeed, Integer maxSpee
         this.minRam = minRam;
         this.maxRam = maxRam;
         this.vm = new VirtualMachineTO(1L, vmName, null, cpus, minSpeed, maxSpeed, minRam, maxRam, null, null, false, limitCpuUse, null);
-        /*vm.setName(vmName);
-        vm.setCpus(cpus);
-        vm.setRam(minRam, maxRam);*/
     }
 
     public void setCpus(int cpus) {

core/src/main/java/com/cloud/resource/CommandWrapper.java

@@ -21,8 +21,10 @@
 
 import com.cloud.agent.api.Answer;
 import com.cloud.agent.api.Command;
+import org.apache.log4j.Logger;
 
 public abstract class CommandWrapper<T extends Command, A extends Answer, R extends ServerResource> {
+    protected Logger logger = Logger.getLogger(getClass());
 
     /**
      * @param T is the command to be used.

engine/api/src/main/java/com/cloud/vm/VirtualMachineManager.java

@@ -76,22 +76,6 @@ public interface VirtualMachineManager extends Manager {
     ConfigKey<Boolean> AllowExposeHypervisorHostname = new ConfigKey<Boolean>("Advanced", Boolean.class, "global.allow.expose.host.hostname",
             "false", "If set to true, it allows the hypervisor host name on which the VM is spawned on to be exposed to the VM", true, ConfigKey.Scope.Global);
 
-    static final ConfigKey<Integer> VmServiceOfferingMaxCPUCores = new ConfigKey<Integer>("Advanced",
-            Integer.class,
-            "vm.serviceoffering.cpu.cores.max",
-            "0",
-            "Maximum CPU cores for vm service offering. If 0 - no limitation",
-            true
-    );
-
-    static final ConfigKey<Integer> VmServiceOfferingMaxRAMSize = new ConfigKey<Integer>("Advanced",
-            Integer.class,
-            "vm.serviceoffering.ram.size.max",
-            "0",
-            "Maximum RAM size in MB for vm service offering. If 0 - no limitation",
-            true
-    );
-
     interface Topics {
         String VM_POWER_STATE = "vm.powerstate";
     }

engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java

@@ -4665,6 +4665,8 @@ public VMInstanceVO reConfigureVm(final String vmUuid, final ServiceOffering old
                     throw (ConcurrentOperationException)jobResult;
                 } else if (jobResult instanceof InsufficientServerCapacityException) {
                     throw (InsufficientServerCapacityException)jobResult;
+                } else if (jobResult instanceof RuntimeException) {
+                    throw (RuntimeException)jobResult;
                 } else if (jobResult instanceof Throwable) {
                     s_logger.error("Unhandled exception", (Throwable)jobResult);
                     throw new RuntimeException("Unhandled exception", (Throwable)jobResult);
@@ -4677,8 +4679,7 @@ public VMInstanceVO reConfigureVm(final String vmUuid, final ServiceOffering old
 
     private VMInstanceVO orchestrateReConfigureVm(String vmUuid, ServiceOffering oldServiceOffering, ServiceOffering newServiceOffering,
                                                   boolean reconfiguringOnExistingHost) throws ResourceUnavailableException, ConcurrentOperationException {
-        VMInstanceVO vm = _vmDao.findByUuid(vmUuid);
-        upgradeVmDb(vm.getId(), newServiceOffering, oldServiceOffering);
+        final VMInstanceVO vm = _vmDao.findByUuid(vmUuid);
 
         HostVO hostVo = _hostDao.findById(vm.getHostId());
 
@@ -4695,6 +4696,10 @@ private VMInstanceVO orchestrateReConfigureVm(String vmUuid, ServiceOffering old
                 new ScaleVmCommand(vm.getInstanceName(), newServiceOffering.getCpu(), minSpeed,
                         newServiceOffering.getSpeed(), minMemory * 1024L * 1024L, newServiceOffering.getRamSize() * 1024L * 1024L, newServiceOffering.getLimitCpuUse());
 
+        scaleVmCommand.getVirtualMachine().setId(vm.getId());
+        scaleVmCommand.getVirtualMachine().setUuid(vm.getUuid());
+        scaleVmCommand.getVirtualMachine().setType(vm.getType());
+
         Long dstHostId = vm.getHostId();
 
         if (vm.getHypervisorType().equals(HypervisorType.VMware)) {
@@ -4710,36 +4715,31 @@ private VMInstanceVO orchestrateReConfigureVm(String vmUuid, ServiceOffering old
         work.setResourceId(vm.getHostId());
         _workDao.persist(work);
 
-        boolean success = false;
-
         try {
+            Answer reconfigureAnswer = _agentMgr.send(vm.getHostId(), scaleVmCommand);
+
+            if (reconfigureAnswer == null || !reconfigureAnswer.getResult()) {
+                s_logger.error("Unable to scale vm due to " + (reconfigureAnswer == null ? "" : reconfigureAnswer.getDetails()));
+                throw new CloudRuntimeException("Unable to scale vm due to " + (reconfigureAnswer == null ? "" : reconfigureAnswer.getDetails()));
+            }
+
+            if (vm.getType().equals(VirtualMachine.Type.User)) {
+                _userVmMgr.generateUsageEvent(vm, vm.isDisplayVm(), EventTypes.EVENT_VM_DYNAMIC_SCALE);
+            }
+
+            upgradeVmDb(vm.getId(), newServiceOffering, oldServiceOffering);
+
             if (reconfiguringOnExistingHost) {
                 vm.setServiceOfferingId(oldServiceOffering.getId());
                 _capacityMgr.releaseVmCapacity(vm, false, false, vm.getHostId()); //release the old capacity
                 vm.setServiceOfferingId(newServiceOffering.getId());
                 _capacityMgr.allocateVmCapacity(vm, false); // lock the new capacity
             }
 
-            Answer scaleVmAnswer = _agentMgr.send(vm.getHostId(), scaleVmCommand);
-            if (scaleVmAnswer == null || !scaleVmAnswer.getResult()) {
-                String msg = String.format("Unable to scale %s due to [%s].", vm.toString(), (scaleVmAnswer == null ? "" : scaleVmAnswer.getDetails()));
-                s_logger.error(msg);
-                throw new CloudRuntimeException(msg);
-            }
-            if (vm.getType().equals(VirtualMachine.Type.User)) {
-                _userVmMgr.generateUsageEvent(vm, vm.isDisplayVm(), EventTypes.EVENT_VM_DYNAMIC_SCALE);
-            }
-            success = true;
-        } catch (OperationTimedoutException e) {
-            throw new AgentUnavailableException(String.format("Unable to scale %s due to [%s].", vm.toString(), e.getMessage()), dstHostId, e);
+        } catch (final OperationTimedoutException e) {
+            throw new AgentUnavailableException("Operation timed out on reconfiguring " + vm, dstHostId);
         } catch (final AgentUnavailableException e) {
             throw e;
-        } finally {
-            if (!success) {
-                _capacityMgr.releaseVmCapacity(vm, false, false, vm.getHostId()); // release the new capacity
-                upgradeVmDb(vm.getId(), oldServiceOffering, newServiceOffering); // rollback
-                _capacityMgr.allocateVmCapacity(vm, false); // allocate the old capacity
-            }
         }
 
         return vm;
@@ -4783,8 +4783,7 @@ public ConfigKey<?>[] getConfigKeys() {
         return new ConfigKey<?>[] { ClusterDeltaSyncInterval, StartRetry, VmDestroyForcestop, VmOpCancelInterval, VmOpCleanupInterval, VmOpCleanupWait,
                 VmOpLockStateRetry, VmOpWaitInterval, ExecuteInSequence, VmJobCheckInterval, VmJobTimeout, VmJobStateReportInterval,
                 VmConfigDriveLabel, VmConfigDriveOnPrimaryPool, VmConfigDriveForceHostCacheUse, VmConfigDriveUseHostCacheOnUnsupportedPool,
-                HaVmRestartHostUp, ResoureCountRunningVMsonly, AllowExposeHypervisorHostname, AllowExposeHypervisorHostnameAccountLevel,
-                VmServiceOfferingMaxCPUCores, VmServiceOfferingMaxRAMSize };
+                HaVmRestartHostUp, ResoureCountRunningVMsonly, AllowExposeHypervisorHostname, AllowExposeHypervisorHostnameAccountLevel };
     }
 
     public List<StoragePoolAllocator> getStoragePoolAllocators() {

engine/schema/src/main/java/com/cloud/host/HostVO.java

@@ -680,7 +680,7 @@ public boolean equals(Object obj) {
 
     @Override
     public String toString() {
-        return String.format("Host [{id: \"%s\", name: \"%s\", uuid: \"%s\", type=\"%s\"}]", id, name, uuid, type);
+        return String.format("Host {\"id\": \"%s\", \"name\": \"%s\", \"uuid\": \"%s\", \"type\"=\"%s\"}", id, name, uuid, type);
     }
 
     public void setHypervisorType(HypervisorType hypervisorType) {

engine/schema/src/main/java/com/cloud/service/ServiceOfferingVO.java

@@ -298,6 +298,10 @@ public boolean isCustomCpuSpeedSupported() {
     }
 
     @Override
+    public String toString() {
+        return String.format("Service offering {\"id\": %s, \"name\": \"%s\", \"uuid\": \"%s\"}", getId(), getName(), getUuid());
+    }
+
     public boolean isDynamicScalingEnabled() {
         return dynamicScalingEnabled;
     }

plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java

@@ -185,6 +185,8 @@
 import com.cloud.vm.VirtualMachine.PowerState;
 import com.cloud.vm.VmDetailConstants;
 import com.google.common.base.Strings;
+import org.apache.cloudstack.utils.bytescale.ByteScaleUtils;
+import org.libvirt.VcpuInfo;
 
 /**
  * LibvirtComputingResource execute requests on the computing/routing host using
@@ -2534,21 +2536,6 @@ private CpuModeDef createCpuModeDef(VirtualMachineTO vmTO, int vcpus) {
         return cmd;
     }
 
-    /**
-     * Creates guest resources based in VM specification.
-     */
-    protected GuestResourceDef createGuestResourceDef(VirtualMachineTO vmTO) {
-        GuestResourceDef grd = new GuestResourceDef();
-
-        grd.setMemorySize(vmTO.getMaxRam() / 1024);
-        if (vmTO.getMinRam() != vmTO.getMaxRam() && !_noMemBalloon) {
-            grd.setMemBalloning(true);
-            grd.setCurrentMem(vmTO.getMinRam() / 1024);
-        }
-        grd.setVcpuNum(vmTO.getCpus());
-        return grd;
-    }
-
     private void configureGuestIfUefiEnabled(boolean isSecureBoot, String bootMode, GuestDef guest) {
         setGuestLoader(bootMode, SECURE, guest, GuestDef.GUEST_LOADER_SECURE);
         setGuestLoader(bootMode, LEGACY, guest, GuestDef.GUEST_LOADER_LEGACY);
@@ -2628,6 +2615,37 @@ private void configureGuestAndUserVMToUseLXC(LibvirtVMDef vm, GuestDef guest) {
         vm.setHvsType(HypervisorType.LXC.toString().toLowerCase());
     }
 
+    /**
+     * Creates guest resources based in VM specification.
+     */
+    protected GuestResourceDef createGuestResourceDef(VirtualMachineTO vmTO){
+        GuestResourceDef grd = new GuestResourceDef();
+
+        grd.setMemBalloning(!_noMemBalloon);
+
+        Long maxRam = ByteScaleUtils.bytesToKib(vmTO.getMaxRam());
+
+        grd.setMemorySize(maxRam);
+        grd.setCurrentMem(getCurrentMemAccordingToMemBallooning(vmTO, maxRam));
+
+        int vcpus = vmTO.getCpus();
+        Integer maxVcpus = vmTO.getVcpuMaxLimit();
+
+        grd.setVcpuNum(vcpus);
+        grd.setMaxVcpuNum(maxVcpus == null ? vcpus : maxVcpus);
+
+        return grd;
+    }
+
+    protected long getCurrentMemAccordingToMemBallooning(VirtualMachineTO vmTO, long maxRam) {
+        if (_noMemBalloon) {
+            s_logger.warn(String.format("Setting VM's [%s] current memory as max memory [%s] due to memory ballooning is disabled. If you are using a custom service offering, verify if memory ballooning really should be disabled.", vmTO.toString(), maxRam));
+            return maxRam;
+        } else {
+            return ByteScaleUtils.bytesToKib(vmTO.getMinRam());
+        }
+    }
+
     /**
      * Adds extra configurations (if any) as a String component to the domain XML
      */
@@ -4556,4 +4574,25 @@ public void setBackingFileFormat(String volPath) {
         }
     }
 
+    /**
+     * Retrieves the memory of the running VM. <br/>
+     * The libvirt (see <a href="https://github.com/libvirt/libvirt/blob/master/src/conf/domain_conf.c">https://github.com/libvirt/libvirt/blob/master/src/conf/domain_conf.c</a>, function <b>virDomainDefParseMemory</b>) uses <b>total memory</b> as the tag <b>memory</b>, in VM's XML.
+     * @param dm domain of the VM.
+     * @return the memory of the VM.
+     * @throws org.libvirt.LibvirtException
+     **/
+    public static long getDomainMemory(Domain dm) throws LibvirtException {
+        return dm.getMaxMemory();
+    }
+
+    /**
+     * Retrieves the quantity of running VCPUs of the running VM. <br/>
+     * @param dm domain of the VM.
+     * @return the quantity of running VCPUs of the running VM.
+     * @throws org.libvirt.LibvirtException
+     **/
+    public static long countDomainRunningVcpus(Domain dm) throws LibvirtException {
+        VcpuInfo vcpus[] = dm.getVcpusInfo();
+        return Arrays.stream(vcpus).filter(vcpu -> vcpu.state.equals(VcpuInfo.VcpuState.VIR_VCPU_RUNNING)).count();
+    }
 }

plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtVMDef.java

@@ -230,51 +230,54 @@ public String toString() {
     }
 
     public static class GuestResourceDef {
-        private long _mem;
-        private long _currentMem = -1;
-        private String _memBacking;
-        private int _vcpu = -1;
-        private boolean _memBalloning = false;
+        private long memory;
+        private long currentMemory = -1;
+        private int vcpu = -1;
+        private int maxVcpu = -1;
+        private boolean memoryBalloning = false;
 
         public void setMemorySize(long mem) {
-            _mem = mem;
+            this.memory = mem;
         }
 
         public void setCurrentMem(long currMem) {
-            _currentMem = currMem;
+            this.currentMemory = currMem;
         }
 
-        public void setMemBacking(String memBacking) {
-            _memBacking = memBacking;
+        public void setVcpuNum(int vcpu) {
+            this.vcpu = vcpu;
         }
 
-        public void setVcpuNum(int vcpu) {
-            _vcpu = vcpu;
+        public void setMaxVcpuNum(int maxVcpu) {
+            this.maxVcpu = maxVcpu;
+        }
+
+        public int getVcpu() {
+            return vcpu;
+        }
+
+        public int getMaxVcpu() {
+            return maxVcpu;
         }
 
-        public void setMemBalloning(boolean turnon) {
-            _memBalloning = turnon;
+        public void setMemBalloning(boolean memoryBalloning) {
+            this.memoryBalloning = memoryBalloning;
         }
 
         @Override
         public String toString() {
-            StringBuilder resBuidler = new StringBuilder();
-            resBuidler.append("<memory>" + _mem + "</memory>\n");
-            if (_currentMem != -1) {
-                resBuidler.append("<currentMemory>" + _currentMem + "</currentMemory>\n");
-            }
-            if (_memBacking != null) {
-                resBuidler.append("<memoryBacking>" + "<" + _memBacking + "/>" + "</memoryBacking>\n");
-            }
-            if (_memBalloning) {
-                resBuidler.append("<devices>\n" + "<memballoon model='virtio'/>\n" + "</devices>\n");
-            } else {
-                resBuidler.append("<devices>\n" + "<memballoon model='none'/>\n" + "</devices>\n");
-            }
-            if (_vcpu != -1) {
-                resBuidler.append("<vcpu>" + _vcpu + "</vcpu>\n");
+            StringBuilder response = new StringBuilder();
+            response.append(String.format("<memory>%s</memory>\n", this.currentMemory));
+            response.append(String.format("<currentMemory>%s</currentMemory>\n", this.currentMemory));
+
+            if (this.memory > this.currentMemory) {
+                response.append(String.format("<maxMemory slots='16' unit='KiB'>%s</maxMemory>\n", this.memory));
+                response.append(String.format("<cpu> <numa> <cell id='0' cpus='0-%s' memory='%s' unit='KiB'/> </numa> </cpu>\n", this.maxVcpu - 1, this.currentMemory));
             }
-            return resBuidler.toString();
+
+            response.append(String.format("<devices>\n<memballoon model='%s'/>\n</devices>\n", this.memoryBalloning ? "virtio" : "none"));
+            response.append(String.format("<vcpu current=\"%s\">%s</vcpu>\n", this.vcpu, this.maxVcpu));
+            return response.toString();
         }
     }
 

plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtVmMemoryDeviceDef.java

@@ -0,0 +1,43 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.cloud.hypervisor.kvm.resource;
+
+/**
+ * Provides the XML definition to a memory device which can be hotpluged to the VM.<br/>
+ * Memory is provided in KiB.
+ *
+ */
+public class LibvirtVmMemoryDeviceDef {
+
+    private final long memorySize;
+
+    public LibvirtVmMemoryDeviceDef(long memorySize) {
+        this.memorySize = memorySize;
+    }
+
+    @Override
+    public String toString() {
+        StringBuilder response = new StringBuilder();
+        response.append("<memory model='dimm'>");
+        response.append("<target>");
+        response.append(String.format("<size unit='KiB'>%s</size>", memorySize));
+        response.append("<node>0</node>");
+        response.append("</target>");
+        response.append("</memory>");
+
+        return response.toString();
+    }
+
+}