diff --git a/core/src/main/java/org/apache/cloudstack/backup/CommvaultRestoreBackupCommand.java b/core/src/main/java/org/apache/cloudstack/backup/CommvaultRestoreBackupCommand.java new file mode 100644 index 000000000000..fbcff2070801 --- /dev/null +++ b/core/src/main/java/org/apache/cloudstack/backup/CommvaultRestoreBackupCommand.java @@ -0,0 +1,150 @@ +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// + +package org.apache.cloudstack.backup; + +import com.cloud.agent.api.Command; +import com.cloud.agent.api.LogLevel; +import com.cloud.vm.VirtualMachine; +import org.apache.cloudstack.storage.to.PrimaryDataStoreTO; + +import java.util.List; + +public class CommvaultRestoreBackupCommand extends Command { + private String vmName; + private String backupPath; + private List backupVolumesUUIDs; + private List restoreVolumePools; + private List restoreVolumePaths; + private String diskType; + private Boolean vmExists; + private String restoreVolumeUUID; + private VirtualMachine.State vmState; + private Integer timeout; + private String cacheMode; + private String hostName; + + protected CommvaultRestoreBackupCommand() { + super(); + } + + public String getVmName() { + return vmName; + } + + public void setVmName(String vmName) { + this.vmName = vmName; + } + + public String getBackupPath() { + return backupPath; + } + + public void setBackupPath(String backupPath) { + this.backupPath = backupPath; + } + + public List getRestoreVolumePools() { + return restoreVolumePools; + } + + public void setRestoreVolumePools(List restoreVolumePools) { + this.restoreVolumePools = restoreVolumePools; + } + + public List getRestoreVolumePaths() { + return restoreVolumePaths; + } + + public void setRestoreVolumePaths(List restoreVolumePaths) { + this.restoreVolumePaths = restoreVolumePaths; + } + + public Boolean isVmExists() { + return vmExists; + } + + public void setVmExists(Boolean vmExists) { + this.vmExists = vmExists; + } + + public String getDiskType() { + return diskType; + } + + public void setDiskType(String diskType) { + this.diskType = diskType; + } + + public String getRestoreVolumeUUID() { + return restoreVolumeUUID; + } + + public void setRestoreVolumeUUID(String restoreVolumeUUID) { + this.restoreVolumeUUID = restoreVolumeUUID; + } + + public VirtualMachine.State getVmState() { + return vmState; + } + + public void setVmState(VirtualMachine.State vmState) { + this.vmState = vmState; + } + + @LogLevel(LogLevel.Log4jLevel.Off) + private String mountOptions; + @Override + + public boolean executeInSequence() { + return true; + } + + public List getBackupVolumesUUIDs() { + return backupVolumesUUIDs; + } + + public void setBackupVolumesUUIDs(List backupVolumesUUIDs) { + this.backupVolumesUUIDs = backupVolumesUUIDs; + } + + public Integer getTimeout() { + return this.timeout == null ? 0 : this.timeout; + } + + public void setTimeout(Integer timeout) { + this.timeout = timeout; + } + + public String getCacheMode() { + return cacheMode; + } + + public void setCacheMode(String cacheMode) { + this.cacheMode = cacheMode; + } + + public String getHostName() { + return hostName; + } + + public void setHostName(String hostName) { + this.hostName = hostName; + } +} diff --git a/core/src/main/java/org/apache/cloudstack/backup/CommvaultTakeBackupCommand.java b/core/src/main/java/org/apache/cloudstack/backup/CommvaultTakeBackupCommand.java new file mode 100644 index 000000000000..f24f41d98675 --- /dev/null +++ b/core/src/main/java/org/apache/cloudstack/backup/CommvaultTakeBackupCommand.java @@ -0,0 +1,84 @@ +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// + +package org.apache.cloudstack.backup; + +import com.cloud.agent.api.Command; +import org.apache.cloudstack.storage.to.PrimaryDataStoreTO; + +import java.util.List; + +public class CommvaultTakeBackupCommand extends Command { + private String vmName; + private String backupPath; + private List volumePools; + private List volumePaths; + private Boolean quiesce; + + public CommvaultTakeBackupCommand(String vmName, String backupPath) { + super(); + this.vmName = vmName; + this.backupPath = backupPath; + } + + public String getVmName() { + return vmName; + } + + public void setVmName(String vmName) { + this.vmName = vmName; + } + + public String getBackupPath() { + return backupPath; + } + + public void setBackupPath(String backupPath) { + this.backupPath = backupPath; + } + + public List getVolumePools() { + return volumePools; + } + + public void setVolumePools(List volumePools) { + this.volumePools = volumePools; + } + + public List getVolumePaths() { + return volumePaths; + } + + public void setVolumePaths(List volumePaths) { + this.volumePaths = volumePaths; + } + + public Boolean getQuiesce() { + return quiesce; + } + + public void setQuiesce(Boolean quiesce) { + this.quiesce = quiesce; + } + + @Override + public boolean executeInSequence() { + return true; + } +} diff --git a/plugins/backup/commvault/src/main/java/org/apache/cloudstack/backup/CommvaultBackupProvider.java b/plugins/backup/commvault/src/main/java/org/apache/cloudstack/backup/CommvaultBackupProvider.java index d024ef1d8292..a5e0f1e00a7c 100644 --- a/plugins/backup/commvault/src/main/java/org/apache/cloudstack/backup/CommvaultBackupProvider.java +++ b/plugins/backup/commvault/src/main/java/org/apache/cloudstack/backup/CommvaultBackupProvider.java @@ -16,75 +16,70 @@ // under the License. package org.apache.cloudstack.backup; -import com.cloud.api.query.vo.UserVmJoinVO; -import com.cloud.api.query.dao.UserVmJoinDao; -import com.cloud.cluster.ManagementServerHostVO; -import com.cloud.cluster.dao.ManagementServerHostDao; +import com.cloud.agent.AgentManager; +import com.cloud.exception.AgentUnavailableException; +import com.cloud.exception.OperationTimedoutException; import com.cloud.dc.dao.ClusterDao; import com.cloud.domain.Domain; +import com.cloud.host.Host; import com.cloud.host.HostVO; import com.cloud.host.Status; import com.cloud.host.dao.HostDao; import com.cloud.hypervisor.Hypervisor; import com.cloud.offering.DiskOffering; +import com.cloud.resource.ResourceManager; import com.cloud.storage.DataStoreRole; +import com.cloud.storage.ScopeType; +import com.cloud.storage.Storage; import com.cloud.storage.Volume; +import com.cloud.storage.Volume.Type; import com.cloud.storage.VolumeVO; import com.cloud.storage.dao.DiskOfferingDao; -import com.cloud.storage.SnapshotVO; -import com.cloud.storage.dao.SnapshotDao; import com.cloud.storage.dao.StoragePoolHostDao; import com.cloud.storage.dao.VolumeDao; import com.cloud.user.User; -import com.cloud.user.UserAccount; import com.cloud.user.Account; import com.cloud.user.AccountService; import com.cloud.utils.NumbersUtil; -import com.cloud.utils.PropertiesUtil; -import com.cloud.utils.server.ServerProperties; import com.cloud.utils.Pair; import com.cloud.utils.Ternary; import com.cloud.utils.ssh.SshHelper; import com.cloud.utils.component.AdapterBase; import com.cloud.utils.exception.CloudRuntimeException; -import com.cloud.utils.nio.TrustAllManager; import com.cloud.event.ActionEventUtils; import com.cloud.event.EventTypes; import com.cloud.vm.VMInstanceVO; import com.cloud.vm.VirtualMachine; import com.cloud.vm.dao.VMInstanceDao; +import com.cloud.vm.snapshot.VMSnapshot; +import com.cloud.vm.snapshot.dao.VMSnapshotDao; import org.apache.cloudstack.api.ApiCommandResourceType; import org.apache.cloudstack.storage.datastore.db.StoragePoolVO; -import org.apache.cloudstack.storage.datastore.db.SnapshotDataStoreVO; import org.apache.cloudstack.storage.datastore.db.SnapshotDataStoreDao; import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao; +import org.apache.cloudstack.storage.to.PrimaryDataStoreTO; import org.apache.cloudstack.backup.commvault.CommvaultClient; import org.apache.cloudstack.backup.dao.BackupDao; import org.apache.cloudstack.backup.dao.BackupOfferingDao; import org.apache.cloudstack.backup.dao.BackupOfferingDaoImpl; +import org.apache.cloudstack.engine.subsystem.api.storage.DataStore; +import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager; import org.apache.cloudstack.framework.config.ConfigKey; import org.apache.cloudstack.framework.config.Configurable; import org.apache.cloudstack.framework.config.dao.ConfigurationDao; -import org.apache.cloudstack.utils.security.SSLUtils; -import org.apache.cloudstack.utils.identity.ManagementServerNode; import org.apache.commons.collections.CollectionUtils; -import org.apache.commons.codec.binary.Base64; import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.LogManager; import org.apache.xml.utils.URI; import org.json.JSONObject; -import org.json.XML; -import java.net.HttpURLConnection; import java.net.URISyntaxException; -import java.net.URL; -import java.net.URLEncoder; import java.security.KeyManagementException; import java.security.NoSuchAlgorithmException; -import java.security.SecureRandom; import java.text.ParseException; import java.text.SimpleDateFormat; import java.util.ArrayList; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.HashMap; import java.util.Date; @@ -92,48 +87,41 @@ import java.util.UUID; import java.util.Optional; import java.util.stream.Collectors; -import java.util.StringTokenizer; -import java.util.StringJoiner; -import java.util.Properties; import java.util.Collections; import java.util.Comparator; import java.util.regex.Matcher; import java.util.regex.Pattern; -import java.io.File; -import java.io.InputStream; -import java.io.FileInputStream; -import java.io.IOException; -import java.io.BufferedReader; -import java.io.InputStreamReader; -import java.io.UnsupportedEncodingException; import javax.inject.Inject; -import javax.net.ssl.SSLContext; -import javax.net.ssl.TrustManager; -import javax.net.ssl.HttpsURLConnection; -import javax.crypto.spec.SecretKeySpec; -import javax.crypto.Mac; + +import static org.apache.cloudstack.backup.BackupManager.BackupFrameworkEnabled; public class CommvaultBackupProvider extends AdapterBase implements BackupProvider, Configurable { private static final Logger LOG = LogManager.getLogger(CommvaultBackupProvider.class); + private static final String RM_COMMAND = "rm -rf %s"; + private static final int BASE_MAJOR = 11; + private static final int BASE_FR = 32; + private static final int BASE_MT = 89; + private static final Pattern VERSION_PATTERN = Pattern.compile("^(\\d+)\\s*SP\\s*(\\d+)(?:\\.(\\d+))?$", Pattern.CASE_INSENSITIVE); + private static final String COMMVAULT_DIRECTORY = "/tmp/mold/backup"; public ConfigKey CommvaultUrl = new ConfigKey<>("Advanced", String.class, "backup.plugin.commvault.url", "https://localhost/commandcenter/api", "Commvault Command Center API URL.", true, ConfigKey.Scope.Zone); - private final ConfigKey CommvaultUsername = new ConfigKey<>("Advanced", String.class, + private ConfigKey CommvaultUsername = new ConfigKey<>("Advanced", String.class, "backup.plugin.commvault.username", "admin", "Commvault Command Center API username.", true, ConfigKey.Scope.Zone); - private final ConfigKey CommvaultPassword = new ConfigKey<>("Secure", String.class, + private ConfigKey CommvaultPassword = new ConfigKey<>("Secure", String.class, "backup.plugin.commvault.password", "password", "Commvault Command Center API password.", true, ConfigKey.Scope.Zone); - private final ConfigKey CommvaultValidateSSLSecurity = new ConfigKey<>("Advanced", Boolean.class, + private ConfigKey CommvaultValidateSSLSecurity = new ConfigKey<>("Advanced", Boolean.class, "backup.plugin.commvault.validate.ssl", "false", "Validate the SSL certificate when connecting to Commvault Command Center API service.", true, ConfigKey.Scope.Zone); - private final ConfigKey CommvaultApiRequestTimeout = new ConfigKey<>("Advanced", Integer.class, + private ConfigKey CommvaultApiRequestTimeout = new ConfigKey<>("Advanced", Integer.class, "backup.plugin.commvault.request.timeout", "300", "Commvault Command Center API request timeout in seconds.", true, ConfigKey.Scope.Zone); @@ -149,18 +137,16 @@ public class CommvaultBackupProvider extends AdapterBase implements BackupProvid "backup.plugin.commvault.task.poll.max.retry", "120", "The max number of retrying times when the management server polls for Commvault task status.", true, ConfigKey.Scope.Zone); - private final ConfigKey CommvaultClientVerboseLogs = new ConfigKey<>("Advanced", Boolean.class, + private ConfigKey CommvaultClientVerboseLogs = new ConfigKey<>("Advanced", Boolean.class, "backup.plugin.commvault.client.verbosity", "false", "Produce Verbose logs in Hypervisor", true, ConfigKey.Scope.Zone); - private static final String RSYNC_COMMAND = "rsync -az %s %s"; - private static final String RM_COMMAND = "rm -rf %s"; - private static final String CURRRENT_DEVICE = "virsh domblklist --domain %s | tail -n 3 | head -n 1 | awk '{print $1}'"; - private static final String ATTACH_DISK_COMMAND = " virsh attach-disk %s %s %s --driver qemu --subdriver qcow2 --cache none"; - private static final int BASE_MAJOR = 11; - private static final int BASE_FR = 32; - private static final int BASE_MT = 89; - private static final Pattern VERSION_PATTERN = Pattern.compile("^(\\d+)\\s*SP\\s*(\\d+)(?:\\.(\\d+))?$", Pattern.CASE_INSENSITIVE); + private ConfigKey CommvaultBackupRestoreTimeout = new ConfigKey<>("Advanced", Integer.class, + "commvault.backup.restore.timeout", + "30", + "Timeout in seconds after which qemu-img execute when restoring", + true, + BackupFrameworkEnabled.key()); @Inject private BackupDao backupDao; @@ -187,16 +173,16 @@ public class CommvaultBackupProvider extends AdapterBase implements BackupProvid private VMInstanceDao vmInstanceDao; @Inject - private ManagementServerHostDao msHostDao; + private AccountService accountService; @Inject - private AccountService accountService; + DataStoreManager dataStoreMgr; @Inject - private UserVmJoinDao userVmJoinDao; + private AgentManager agentManager; @Inject - private SnapshotDao snapshotDao; + private VMSnapshotDao vmSnapshotDao; @Inject private PrimaryDataStoreDao primaryDataStoreDao; @@ -207,432 +193,406 @@ public class CommvaultBackupProvider extends AdapterBase implements BackupProvid @Inject private BackupManager backupManager; + @Inject + ResourceManager resourceManager; + @Inject private DiskOfferingDao diskOfferingDao; - private static String getUrlDomain(String url) throws URISyntaxException { - URI uri; - try { - uri = new URI(url); - } catch (URI.MalformedURIException e) { - throw new CloudRuntimeException("Failed to cast URI"); + private Long getClusterIdFromRootVolume(VirtualMachine vm) { + VolumeVO rootVolume = volumeDao.getInstanceRootVolume(vm.getId()); + StoragePoolVO rootDiskPool = primaryDataStoreDao.findById(rootVolume.getPoolId()); + if (rootDiskPool == null) { + return null; } - - return uri.getHost(); - } - - @Override - public ConfigKey[] getConfigKeys() { - return new ConfigKey[]{ - CommvaultUrl, - CommvaultUsername, - CommvaultPassword, - CommvaultValidateSSLSecurity, - CommvaultApiRequestTimeout, - CommvaultClientVerboseLogs - }; - } - - @Override - public String getName() { - return "commvault"; - } - - @Override - public String getDescription() { - return "Commvault Backup Plugin"; - } - - @Override - public String getConfigComponentName() { - return BackupService.class.getSimpleName(); + return rootDiskPool.getClusterId(); } - protected HostVO getLastVMHypervisorHost(VirtualMachine vm) { - HostVO host; + protected Host getVMHypervisorHost(VirtualMachine vm) { Long hostId = vm.getLastHostId(); + Long clusterId = null; - if (hostId == null) { - LOG.debug("Cannot find last host for vm. This should never happen, please check your database."); - return null; + if (hostId != null) { + Host host = hostDao.findById(hostId); + if (host.getStatus() == Status.Up) { + return host; + } + // Try to find any Up host in the same cluster + clusterId = host.getClusterId(); + } else { + // Try to find any Up host in the same cluster as the root volume + clusterId = getClusterIdFromRootVolume(vm); } - host = hostDao.findById(hostId); - if (host.getStatus() == Status.Up) { - return host; - } else { - // Try to find a host in the same cluster - List altClusterHosts = hostDao.findHypervisorHostInCluster(host.getClusterId()); - for (final HostVO candidateClusterHost : altClusterHosts) { - if ( candidateClusterHost.getStatus() == Status.Up ) { - LOG.debug(String.format("Found Host %s", candidateClusterHost)); - return candidateClusterHost; + if (clusterId != null) { + for (final Host hostInCluster : hostDao.findHypervisorHostInCluster(clusterId)) { + if (hostInCluster.getStatus() == Status.Up) { + LOG.debug("Found Host {} in cluster {}", hostInCluster, clusterId); + return hostInCluster; } } } - // Try to find a Host in the zone - List altZoneHosts = hostDao.findByDataCenterId(host.getDataCenterId()); - for (final HostVO candidateZoneHost : altZoneHosts) { - if ( candidateZoneHost.getStatus() == Status.Up && candidateZoneHost.getHypervisorType() == Hypervisor.HypervisorType.KVM ) { - LOG.debug("Found Host " + candidateZoneHost); - return candidateZoneHost; - } - } - return null; - } - protected HostVO getRunningVMHypervisorHost(VirtualMachine vm) { + // Try to find any Host in the zone + return resourceManager.findOneRandomRunningHostByHypervisor(Hypervisor.HypervisorType.KVM, vm.getDataCenterId()); + } - HostVO host; + protected Host getVMHypervisorHostForBackup(VirtualMachine vm) { Long hostId = vm.getHostId(); - + if (hostId == null && VirtualMachine.State.Running.equals(vm.getState())) { + throw new CloudRuntimeException(String.format("Unable to find the hypervisor host for %s. Make sure the virtual machine is running", vm.getName())); + } + if (VirtualMachine.State.Stopped.equals(vm.getState())) { + hostId = vm.getLastHostId(); + } if (hostId == null) { - throw new CloudRuntimeException("Unable to find the HYPERVISOR for " + vm.getName() + ". Make sure the virtual machine is running"); + throw new CloudRuntimeException(String.format("Unable to find the hypervisor host for stopped VM: %s", vm)); + } + final Host host = hostDao.findById(hostId); + if (host == null || !Status.Up.equals(host.getStatus()) || !Hypervisor.HypervisorType.KVM.equals(host.getHypervisorType())) { + throw new CloudRuntimeException("Unable to contact backend control plane to initiate backup"); } - - host = hostDao.findById(hostId); - return host; } - protected String getVMHypervisorCluster(HostVO host) { - - return clusterDao.findById(host.getClusterId()).getName(); - } + @Override + public Pair takeBackup(VirtualMachine vm, Boolean quiesceVM) { + final Host vmHost = getVMHypervisorHostForBackup(vm); + final HostVO vmHostVO = hostDao.findById(vmHost.getId()); + if (CollectionUtils.isNotEmpty(vmSnapshotDao.findByVmAndByType(vm.getId(), VMSnapshot.Type.DiskAndMemory))) { + LOG.debug("Commvault backup provider cannot take backups of a VM [{}] with disk-and-memory VM snapshots. Restoring the backup will corrupt any newer disk-and-memory " + + "VM snapshots.", vm); + throw new CloudRuntimeException(String.format("Cannot take backup of VM [%s] as it has disk-and-memory VM snapshots.", vm.getUuid())); + } - protected Ternary getKVMHyperisorCredentials(HostVO host) { + try { + String commvaultServer = getUrlDomain(CommvaultUrl.value()); + } catch (URISyntaxException e) { + throw new CloudRuntimeException(String.format("Failed to convert API to HOST : %s", e)); + } + // 백업 중인 작업 조회 + final CommvaultClient client = getClient(vm.getDataCenterId()); + boolean activeJob = client.getActiveJob(vm.getInstanceName()); + if (activeJob) { + throw new CloudRuntimeException("There are backup jobs running on the virtual machine. Please try again later."); + } - String username = null; - String password = null; + BackupOfferingVO vmBackupOffering = new BackupOfferingDaoImpl().findById(vm.getBackupOfferingId()); + String planId = vmBackupOffering.getExternalId(); - if (host != null && host.getHypervisorType() == Hypervisor.HypervisorType.KVM) { - hostDao.loadDetails(host); - password = host.getDetail("password"); - username = host.getDetail("username"); - } - if ( password == null || username == null) { - throw new CloudRuntimeException("Cannot find login credentials for HYPERVISOR " + Objects.requireNonNull(host).getUuid()); + // 클라이언트의 백업세트 조회하여 호스트 정의 + String checkVm = client.getVmBackupSetId(vmHost.getName(), vm.getInstanceName()); + if (checkVm == null) { + String clientId = client.getClientId(vmHost.getName()); + String applicationId = client.getApplicationId(clientId); + boolean result = client.createBackupSet(vm.getInstanceName(), applicationId, clientId, planId); + if (!result) { + throw new CloudRuntimeException("Execution of the API that creates a backup set of a virtual machine on the host failed."); + } } - return new Ternary<>(username, password, null); - } + final Date creationDate = new Date(); + final String backupPath = String.format("%s/%s/%s", COMMVAULT_DIRECTORY, vm.getInstanceName(), + new SimpleDateFormat("yyyy.MM.dd.HH.mm.ss").format(creationDate)); - private CommvaultClient getClient(final Long zoneId) { + BackupVO backupVO = createBackupObject(vm, backupPath); + CommvaultTakeBackupCommand command = new CommvaultTakeBackupCommand(vm.getInstanceName(), backupPath); + command.setQuiesce(quiesceVM); + + if (VirtualMachine.State.Stopped.equals(vm.getState())) { + List vmVolumes = volumeDao.findByInstance(vm.getId()); + vmVolumes.sort(Comparator.comparing(Volume::getDeviceId)); + Pair, List> volumePoolsAndPaths = getVolumePoolsAndPaths(vmVolumes); + command.setVolumePools(volumePoolsAndPaths.first()); + command.setVolumePaths(volumePoolsAndPaths.second()); + } + + BackupAnswer answer; try { - return new CommvaultClient(CommvaultUrl.valueIn(zoneId), CommvaultUsername.valueIn(zoneId), CommvaultPassword.valueIn(zoneId), - CommvaultValidateSSLSecurity.valueIn(zoneId), CommvaultApiRequestTimeout.valueIn(zoneId)); - } catch (URISyntaxException e) { - throw new CloudRuntimeException("Failed to parse Commvault API URL: " + e.getMessage()); - } catch (NoSuchAlgorithmException | KeyManagementException e) { - LOG.error("Failed to build Commvault API client due to: ", e); + answer = (BackupAnswer) agentManager.send(vmHost.getId(), command); + } catch (AgentUnavailableException e) { + LOG.error("Unable to contact backend control plane to initiate backup for VM {}", vm.getInstanceName()); + backupVO.setStatus(Backup.Status.Failed); + backupDao.remove(backupVO.getId()); + throw new CloudRuntimeException("Unable to contact backend control plane to initiate backup"); + } catch (OperationTimedoutException e) { + LOG.error("Operation to initiate backup timed out for VM {}", vm.getInstanceName()); + backupVO.setStatus(Backup.Status.Failed); + backupDao.remove(backupVO.getId()); + throw new CloudRuntimeException("Operation to initiate backup timed out, please try again"); } - throw new CloudRuntimeException("Failed to build Commvault API client"); - } - @Override - public boolean checkBackupAgent(final Long zoneId) { - Map checkResult = new HashMap<>(); - final CommvaultClient client = getClient(zoneId); - String csVersionInfo = client.getCvtVersion(); - boolean version = versionCheck(csVersionInfo); - if (version) { - List Hosts = hostDao.findByDataCenterId(zoneId); - for (final HostVO host : Hosts) { - if (host.getStatus() == Status.Up && host.getHypervisorType() == Hypervisor.HypervisorType.KVM) { - String checkHost = client.getClientId(host.getName()); - if (checkHost == null) { - return false; + if (answer != null && answer.getResult()) { + int sshPort = NumbersUtil.parseInt(configDao.getValue("kvm.ssh.port"), 22); + Ternary credentials = getKVMHyperisorCredentials(vmHostVO); + String cmd = String.format(RM_COMMAND, backupPath); + // 생성된 백업 폴더 경로로 해당 백업 세트의 백업 콘텐츠 경로 업데이트 + String clientId = client.getClientId(vmHost.getName()); + String subClientEntity = client.getSubclient(clientId, vm.getInstanceName()); + if (subClientEntity == null) { + LOG.error("Failed to take backup for VM " + vm.getInstanceName() + " to get subclient info commvault api"); + } else { + JSONObject jsonObject = new JSONObject(subClientEntity); + String subclientId = String.valueOf(jsonObject.get("subclientId")); + String applicationId = String.valueOf(jsonObject.get("applicationId")); + String backupsetId = String.valueOf(jsonObject.get("backupsetId")); + String instanceId = String.valueOf(jsonObject.get("instanceId")); + String backupsetName = String.valueOf(jsonObject.get("backupsetName")); + String displayName = String.valueOf(jsonObject.get("displayName")); + String commCellName = String.valueOf(jsonObject.get("commCellName")); + String companyId = String.valueOf(jsonObject.getJSONObject("entityInfo").get("companyId")); + String companyName = String.valueOf(jsonObject.getJSONObject("entityInfo").get("companyName")); + String instanceName = String.valueOf(jsonObject.get("instanceName")); + String appName = String.valueOf(jsonObject.get("appName")); + String clientName = String.valueOf(jsonObject.get("clientName")); + String subclientGUID = String.valueOf(jsonObject.get("subclientGUID")); + String subclientName = String.valueOf(jsonObject.get("subclientName")); + String csGUID = String.valueOf(jsonObject.get("csGUID")); + boolean upResult = client.updateBackupSet(backupPath, subclientId, clientId, planId, applicationId, backupsetId, instanceId, subclientName, backupsetName); + if (upResult) { + String planName = client.getPlanName(planId); + String storagePolicyId = client.getStoragePolicyId(planName); + if (planName == null || storagePolicyId == null) { + LOG.error("Failed to take backup for VM " + vm.getInstanceName() + " to get storage policy id commvault api"); } else { - boolean installJob = client.getInstallActiveJob(host.getPrivateIpAddress()); - boolean checkInstall = client.getClientProps(checkHost); - if (installJob || !checkInstall) { - if (!checkInstall) { - LOG.error("The host is registered with the client, but the readiness status is not normal and you must manually check the client status."); + // 백업 실행 + String jobId = client.createBackup(subclientId, storagePolicyId, displayName, commCellName, clientId, companyId, companyName, instanceName, appName, applicationId, clientName, backupsetId, instanceId, subclientGUID, subclientName, csGUID, backupsetName); + if (jobId != null) { + String jobStatus = client.getJobStatus(jobId); + String externalId = backupPath + "," + jobId; + if (jobStatus.equalsIgnoreCase("Completed")) { + String jobDetails = client.getJobDetails(jobId); + if (jobDetails != null) { + JSONObject jsonObject2 = new JSONObject(jobDetails); + String endTime = String.valueOf(jsonObject2.getJSONObject("job").getJSONObject("jobDetail").getJSONObject("detailInfo").get("endTime")); + long timestamp = Long.parseLong(endTime) * 1000L; + Date endDate = new Date(timestamp); + SimpleDateFormat formatterDateTime = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss"); + String formattedString = formatterDateTime.format(endDate); + String size = String.valueOf(jsonObject2.getJSONObject("job").getJSONObject("jobDetail").getJSONObject("detailInfo").get("sizeOfApplication")); + String type = String.valueOf(jsonObject2.getJSONObject("job").getJSONObject("jobDetail").getJSONObject("generalInfo").get("backupType")); + backupVO.setExternalId(externalId); + backupVO.setType(type.toUpperCase()); + try { + backupVO.setDate(formatterDateTime.parse(formattedString)); + } catch (ParseException e) { + String msg = String.format("Unable to parse date [%s].", endTime); + LOG.error(msg, e); + throw new CloudRuntimeException(msg, e); + } + backupVO.setSize(Long.parseLong(size)); + backupVO.setStatus(Backup.Status.BackedUp); + List vols = new ArrayList<>(volumeDao.findByInstance(vm.getId())); + backupVO.setBackedUpVolumes(backupManager.createVolumeInfoFromVolumes(vols)); + if (backupDao.update(backupVO.getId(), backupVO)) { + executeDeleteBackupPathCommand(vmHostVO, credentials.first(), credentials.second(), sshPort, cmd); + return new Pair<>(true, backupVO); + } else { + executeDeleteBackupPathCommand(vmHostVO, credentials.first(), credentials.second(), sshPort, cmd); + throw new CloudRuntimeException("Failed to update backup"); + } + } else { + backupVO.setExternalId(externalId); + LOG.error("Failed to take backup for VM " + vm.getInstanceName() + " to get details job commvault api"); + } + } else { + backupVO.setExternalId(externalId); + LOG.error("Failed to take backup for VM " + vm.getInstanceName() + " to create backup job status is " + jobStatus); } - return false; + } else { + LOG.error("Failed to take backup for VM " + vm.getInstanceName() + " to create backup job commvault api"); } } + } else { + LOG.error("Failed to take backup for VM " + vm.getInstanceName() + " to update backupset content path commvault api"); } } - return true; + backupVO.setStatus(Backup.Status.Failed); + backupDao.remove(backupVO.getId()); + executeDeleteBackupPathCommand(vmHostVO, credentials.first(), credentials.second(), sshPort, cmd); + return new Pair<>(false, null); + } else { + LOG.error("Failed to take backup for VM {}: {}", vm.getInstanceName(), answer != null ? answer.getDetails() : "No answer received"); + if (answer.getNeedsCleanup()) { + LOG.error("Backup cleanup failed for VM {}. Leaving the backup in Error state.", vm.getInstanceName()); + backupVO.setStatus(Backup.Status.Error); + backupDao.update(backupVO.getId(), backupVO); + } else { + backupVO.setStatus(Backup.Status.Failed); + backupDao.remove(backupVO.getId()); + } + return new Pair<>(false, null); } - return false; } + private BackupVO createBackupObject(VirtualMachine vm, String backupPath) { + BackupVO backup = new BackupVO(); + backup.setVmId(vm.getId()); + backup.setExternalId(backupPath); + backup.setType("FULL"); + backup.setDate(new Date()); + long virtualSize = 0L; + for (final Volume volume: volumeDao.findByInstance(vm.getId())) { + if (Volume.State.Ready.equals(volume.getState())) { + virtualSize += volume.getSize(); + } + } + backup.setProtectedSize(virtualSize); + backup.setStatus(Backup.Status.BackingUp); + backup.setBackupOfferingId(vm.getBackupOfferingId()); + backup.setAccountId(vm.getAccountId()); + backup.setDomainId(vm.getDomainId()); + backup.setZoneId(vm.getDataCenterId()); + backup.setName(backupManager.getBackupNameFromVM(vm)); + Map details = backupManager.getBackupDetailsFromVM(vm); + backup.setDetails(details); + + return backupDao.persist(backup); + } + + // 백업에서 새 인스턴스 생성 @Override - public boolean installBackupAgent(final Long zoneId) { - Map failResult = new HashMap<>(); - final CommvaultClient client = getClient(zoneId); - List Hosts = hostDao.findByDataCenterId(zoneId); - for (final HostVO host : Hosts) { - if (host.getStatus() == Status.Up && host.getHypervisorType() == Hypervisor.HypervisorType.KVM) { - String commCell = client.getCommcell(); - JSONObject jsonObject = new JSONObject(commCell); - String commCellId = String.valueOf(jsonObject.get("commCellId")); - String commServeHostName = String.valueOf(jsonObject.get("commCellName")); - Ternary credentials = getKVMHyperisorCredentials(host); - boolean installJob = true; - LOG.info("checking for install agent on the Commvault Backup Provider in host " + host.getPrivateIpAddress()); - // 설치가 진행중인 호스트가 있는지 확인 - while (installJob) { - installJob = client.getInstallActiveJob(host.getName()); - try { - Thread.sleep(30000); - } catch (InterruptedException e) { - LOG.error("checkBackupAgent get install active job result sleep interrupted error"); - } + public Pair restoreBackupToVM(VirtualMachine vm, Backup backup, String hostIp, String dataStoreUuid) { + return restoreVMBackup(vm, backup); + } + + // 가상머신 백업 복원 + @Override + public boolean restoreVMFromBackup(VirtualMachine vm, Backup backup) { + return restoreVMBackup(vm, backup).first(); + } + + private Pair restoreVMBackup(VirtualMachine vm, Backup backup) { + try { + String commvaultServer = getUrlDomain(CommvaultUrl.value()); + } catch (URISyntaxException e) { + throw new CloudRuntimeException(String.format("Failed to convert API to HOST : %s", e)); + } + final CommvaultClient client = getClient(vm.getDataCenterId()); + final String externalId = backup.getExternalId(); + String jobId = externalId.substring(externalId.lastIndexOf(',') + 1).trim(); + final String path = externalId.substring(0, externalId.lastIndexOf(',')); + String jobDetails = client.getJobDetails(jobId); + if (jobDetails == null) { + throw new CloudRuntimeException("Failed to get job details commvault api"); + } + JSONObject jsonObject = new JSONObject(jobDetails); + String endTime = String.valueOf(jsonObject.getJSONObject("job").getJSONObject("jobDetail").getJSONObject("detailInfo").get("endTime")); + String subclientId = String.valueOf(jsonObject.getJSONObject("job").getJSONObject("jobDetail").getJSONObject("generalInfo").getJSONObject("subclient").get("subclientId")); + String displayName = String.valueOf(jsonObject.getJSONObject("job").getJSONObject("jobDetail").getJSONObject("generalInfo").getJSONObject("subclient").get("displayName")); + String clientId = String.valueOf(jsonObject.getJSONObject("job").getJSONObject("jobDetail").getJSONObject("generalInfo").getJSONObject("subclient").get("clientId")); + String companyId = String.valueOf(jsonObject.getJSONObject("job").getJSONObject("jobDetail").getJSONObject("generalInfo").getJSONObject("company").get("companyId")); + String companyName = String.valueOf(jsonObject.getJSONObject("job").getJSONObject("jobDetail").getJSONObject("generalInfo").getJSONObject("company").get("companyName")); + String instanceName = String.valueOf(jsonObject.getJSONObject("job").getJSONObject("jobDetail").getJSONObject("generalInfo").getJSONObject("subclient").get("instanceName")); + String appName = String.valueOf(jsonObject.getJSONObject("job").getJSONObject("jobDetail").getJSONObject("generalInfo").getJSONObject("subclient").get("appName")); + String applicationId = String.valueOf(jsonObject.getJSONObject("job").getJSONObject("jobDetail").getJSONObject("generalInfo").getJSONObject("subclient").get("applicationId")); + String clientName = String.valueOf(jsonObject.getJSONObject("job").getJSONObject("jobDetail").getJSONObject("generalInfo").getJSONObject("subclient").get("clientName")); + String backupsetId = String.valueOf(jsonObject.getJSONObject("job").getJSONObject("jobDetail").getJSONObject("generalInfo").getJSONObject("subclient").get("backupsetId")); + String instanceId = String.valueOf(jsonObject.getJSONObject("job").getJSONObject("jobDetail").getJSONObject("generalInfo").getJSONObject("subclient").get("instanceId")); + String backupsetName = String.valueOf(jsonObject.getJSONObject("job").getJSONObject("jobDetail").getJSONObject("generalInfo").getJSONObject("subclient").get("backupsetName")); + String commCellId = String.valueOf(jsonObject.getJSONObject("job").getJSONObject("jobDetail").getJSONObject("generalInfo").getJSONObject("commcell").get("commCellId")); + String backupsetGUID = client.getVmBackupSetGuid(clientName, backupsetName); + if (backupsetGUID == null) { + throw new CloudRuntimeException("Failed to get vm backup set guid commvault api"); + } + // 복원된 호스트 정의 + final HostVO restoreHost = hostDao.findByName(clientName); + final HostVO restoreHostVO = hostDao.findById(restoreHost.getId()); + LOG.info(String.format("Restoring vm %s from backup %s on the Commvault Backup Provider", vm, backup)); + // 복원 실행 + String jobId2 = client.restoreFullVM(subclientId, displayName, backupsetGUID, clientId, companyId, companyName, instanceName, appName, applicationId, clientName, backupsetId, instanceId, backupsetName, commCellId, endTime, path); + if (jobId2 != null) { + String jobStatus = client.getJobStatus(jobId2); + if (jobStatus.equalsIgnoreCase("Completed")) { + List backedVolumesUUIDs = backup.getBackedUpVolumes().stream() + .sorted(Comparator.comparingLong(Backup.VolumeInfo::getDeviceId)) + .map(Backup.VolumeInfo::getUuid) + .collect(Collectors.toList()); + + List restoreVolumes = volumeDao.findByInstance(vm.getId()).stream() + .sorted(Comparator.comparingLong(VolumeVO::getDeviceId)) + .collect(Collectors.toList()); + + LOG.debug("Restoring vm {} from backup {} on the Commvault Backup Provider", vm, backup); + // 가상머신이 실행중인 호스트 정의 + final Host vmHost = getVMHypervisorHost(vm); + final HostVO vmHostVO = hostDao.findById(vmHost.getId()); + CommvaultRestoreBackupCommand restoreCommand = new CommvaultRestoreBackupCommand(); + LOG.info(path); + restoreCommand.setBackupPath(path); + restoreCommand.setVmName(vm.getName()); + restoreCommand.setBackupVolumesUUIDs(backedVolumesUUIDs); + Pair, List> volumePoolsAndPaths = getVolumePoolsAndPaths(restoreVolumes); + restoreCommand.setRestoreVolumePools(volumePoolsAndPaths.first()); + restoreCommand.setRestoreVolumePaths(volumePoolsAndPaths.second()); + restoreCommand.setVmExists(vm.getRemoved() == null); + restoreCommand.setVmState(vm.getState()); + restoreCommand.setTimeout(CommvaultBackupRestoreTimeout.value()); + // 복원된 호스트와 가상머신이 실행중인 호스트가 같은 경우 null, 다른 경우 추가 + restoreCommand.setHostName(restoreHost.getId() == vmHost.getId() ? null : restoreHost.getName()); + + BackupAnswer answer; + try { + answer = (BackupAnswer) agentManager.send(vmHost.getId(), restoreCommand); + } catch (AgentUnavailableException e) { + throw new CloudRuntimeException("Unable to contact backend control plane to initiate backup"); + } catch (OperationTimedoutException e) { + throw new CloudRuntimeException("Operation to restore backup timed out, please try again"); } - String checkHost = client.getClientId(host.getName()); - // 호스트가 클라이언트에 등록되지 않은 경우 - if (checkHost == null) { - String jobId = client.installAgent(host.getPrivateIpAddress(), commCellId, commServeHostName, credentials.first(), credentials.second()); - if (jobId != null) { - String jobStatus = client.getJobStatus(jobId); - if (!jobStatus.equalsIgnoreCase("Completed")) { - LOG.error("installing agent on the Commvault Backup Provider failed jogId : " + jobId + " , jobStatus : " + jobStatus); - ActionEventUtils.onActionEvent(User.UID_SYSTEM, Account.ACCOUNT_ID_SYSTEM, Domain.ROOT_DOMAIN, EventTypes.EVENT_HOST_AGENT_INSTALL, - "Failed install the commvault client agent on the host : " + host.getPrivateIpAddress(), User.UID_SYSTEM, ApiCommandResourceType.Host.toString()); - failResult.put(host.getPrivateIpAddress(), jobId); - } - } else { - return false; - } - } else { - // 호스트가 클라이언트에는 등록되었지만 구성이 정상적으로 되지 않은 경우 준비 상태 체크 - boolean checkInstall = client.getClientCheckReadiness(checkHost); - if (!checkInstall) { - LOG.error("The host is registered with the client, but the readiness status is not normal and you must manually check the client status."); - ActionEventUtils.onActionEvent(User.UID_SYSTEM, Account.ACCOUNT_ID_SYSTEM, Domain.ROOT_DOMAIN, EventTypes.EVENT_HOST_AGENT_INSTALL, - "Failed check readiness the commvault client agent on the host : " + host.getPrivateIpAddress(), User.UID_SYSTEM, ApiCommandResourceType.Host.toString()); - return false; + if (!answer.getResult()) { + int sshPort = NumbersUtil.parseInt(configDao.getValue("kvm.ssh.port"), 22); + Ternary credentials = getKVMHyperisorCredentials(vmHostVO); + String command = String.format(RM_COMMAND, path); + executeDeleteBackupPathCommand(vmHostVO, credentials.first(), credentials.second(), sshPort, command); + if (restoreHost.getId() != vmHost.getId()) { + credentials = getKVMHyperisorCredentials(restoreHostVO); + command = String.format(RM_COMMAND, path); + executeDeleteBackupPathCommand(restoreHostVO, credentials.first(), credentials.second(), sshPort, command); } } + return new Pair<>(answer.getResult(), answer.getDetails()); + } else { + throw new CloudRuntimeException("Failed to restore Full VM commvault api resulted in " + jobStatus); } + } else { + throw new CloudRuntimeException("Failed to restore Full VM commvault api"); } - if (!failResult.isEmpty()) { - return false; + } + + private Pair, List> getVolumePoolsAndPaths(List volumes) { + List volumePools = new ArrayList<>(); + List volumePaths = new ArrayList<>(); + for (VolumeVO volume : volumes) { + StoragePoolVO storagePool = primaryDataStoreDao.findById(volume.getPoolId()); + if (Objects.isNull(storagePool)) { + throw new CloudRuntimeException("Unable to find storage pool associated to the volume"); + } + + DataStore dataStore = dataStoreMgr.getDataStore(storagePool.getId(), DataStoreRole.Primary); + volumePools.add(dataStore != null ? (PrimaryDataStoreTO)dataStore.getTO() : null); + + String volumePathPrefix = getVolumePathPrefix(storagePool); + volumePaths.add(String.format("%s/%s", volumePathPrefix, volume.getPath())); } - return true; - } - - @Override - public boolean importBackupPlan(final Long zoneId, final String retentionPeriod, final String externalId) { - final CommvaultClient client = getClient(zoneId); - // 선택한 백업 정책의 RPO 편집 Commvault API 호출 - String type = "deleteRpo"; - String taskId = client.getScheduleTaskId(type, externalId); - if (taskId != null) { - String subTaskId = client.getSubTaskId(taskId); - if (subTaskId != null) { - boolean result = client.deleteSchedulePolicy(taskId, subTaskId); - if (!result) { - throw new CloudRuntimeException("Failed to delete schedule policy commvault api"); - } - } - } else { - throw new CloudRuntimeException("Failed to get plan details schedule task id commvault api"); - } - // 선택한 백업 정책의 보존 기간 변경 Commvault API 호출 - type = "updateRpo"; - String planEntity = client.getScheduleTaskId(type, externalId); - JSONObject jsonObject = new JSONObject(planEntity); - String planType = String.valueOf(jsonObject.get("planType")); - String planName = String.valueOf(jsonObject.get("planName")); - String planSubtype = String.valueOf(jsonObject.get("planSubtype")); - String planId = String.valueOf(jsonObject.get("planId")); - JSONObject entityInfo = jsonObject.getJSONObject("entityInfo"); - String companyId = String.valueOf(entityInfo.get("companyId")); - String storagePolicyId = client.getStoragePolicyId(planName); - if (storagePolicyId == null) { - throw new CloudRuntimeException("Failed to get plan storage policy id commvault api"); - } - boolean result = client.getStoragePolicyDetails(planId, storagePolicyId, retentionPeriod); - if (result) { - // 호스트에 선택한 백업 정책 설정 Commvault API 호출 - String path = "/"; - List Hosts = hostDao.findByDataCenterId(zoneId); - for (final HostVO host : Hosts) { - String backupSetId = client.getDefaultBackupSetId(host.getName()); - if (backupSetId != null) { - if (!client.setBackupSet(path, planType, planName, planSubtype, planId, companyId, backupSetId)) { - throw new CloudRuntimeException("Failed to setting backup plan for client commvault api"); - } - } - } - return true; - } else { - throw new CloudRuntimeException("Failed to edit plan schedule retention period commvault api"); - } - } - - @Override - public boolean updateBackupPlan(final Long zoneId, final String retentionPeriod, final String externalId) { - final CommvaultClient client = getClient(zoneId); - String type = "updateRpo"; - String planEntity = client.getScheduleTaskId(type, externalId); - JSONObject jsonObject = new JSONObject(planEntity); - String planType = String.valueOf(jsonObject.get("planType")); - String planName = String.valueOf(jsonObject.get("planName")); - String planSubtype = String.valueOf(jsonObject.get("planSubtype")); - String planId = String.valueOf(jsonObject.get("planId")); - JSONObject entityInfo = jsonObject.getJSONObject("entityInfo"); - String companyId = String.valueOf(entityInfo.get("companyId")); - String storagePolicyId = client.getStoragePolicyId(planName); - if (storagePolicyId == null) { - throw new CloudRuntimeException("Failed to get plan storage policy id commvault api"); - } - return client.getStoragePolicyDetails(planId, storagePolicyId, retentionPeriod); - } - - @Override - public List listBackupOfferings(Long zoneId) { - return getClient(zoneId).listPlans(); - } - - @Override - public boolean isValidProviderOffering(Long zoneId, String uuid) { - List policies = listBackupOfferings(zoneId); - if (CollectionUtils.isEmpty(policies)) { - return false; - } - for (final BackupOffering policy : policies) { - if (policy.getExternalId().equals(uuid)) { - return true; - } - } - return false; - } - - @Override - public boolean assignVMToBackupOffering(VirtualMachine vm, BackupOffering backupOffering) { - HostVO hostVO; - final CommvaultClient client = getClient(vm.getDataCenterId()); - if (vm.getState() == VirtualMachine.State.Running) { - hostVO = getRunningVMHypervisorHost(vm); - } else { - hostVO = getLastVMHypervisorHost(vm); - } - String clientId = client.getClientId(hostVO.getName()); - String applicationId = client.getApplicationId(clientId); - return client.createBackupSet(vm.getInstanceName(), applicationId, clientId, backupOffering.getExternalId()); - } - - @Override - public boolean removeVMFromBackupOffering(VirtualMachine vm) { - final CommvaultClient client = getClient(vm.getDataCenterId()); - List Hosts = hostDao.findByDataCenterId(vm.getDataCenterId()); - for (final HostVO host : Hosts) { - if (host.getHypervisorType() == Hypervisor.HypervisorType.KVM) { - String backupSetId = client.getVmBackupSetId(host.getName(), vm.getInstanceName()); - if (backupSetId != null) { - return client.deleteBackupSet(backupSetId); - } - } - } - return false; - } - - @Override - public boolean restoreVMFromBackup(VirtualMachine vm, Backup backup) { - List backedVolumes = backup.getBackedUpVolumes(); - List volumes = backedVolumes.stream() - .map(volume -> volumeDao.findByUuid(volume.getUuid())) - .sorted((v1, v2) -> Long.compare(v1.getDeviceId(), v2.getDeviceId())) - .collect(Collectors.toList()); - try { - String commvaultServer = getUrlDomain(CommvaultUrl.value()); - } catch (URISyntaxException e) { - throw new CloudRuntimeException(String.format("Failed to convert API to HOST : %s", e)); - } - final CommvaultClient client = getClient(vm.getDataCenterId()); - final String externalId = backup.getExternalId(); - String jobId = externalId.substring(externalId.lastIndexOf(',') + 1).trim(); - String path = externalId.substring(0, externalId.lastIndexOf(',')); - String jobDetails = client.getJobDetails(jobId); - if (jobDetails == null) { - throw new CloudRuntimeException("Failed to get job details commvault api"); - } - JSONObject jsonObject = new JSONObject(jobDetails); - String endTime = String.valueOf(jsonObject.getJSONObject("job").getJSONObject("jobDetail").getJSONObject("detailInfo").get("endTime")); - String subclientId = String.valueOf(jsonObject.getJSONObject("job").getJSONObject("jobDetail").getJSONObject("generalInfo").getJSONObject("subclient").get("subclientId")); - String displayName = String.valueOf(jsonObject.getJSONObject("job").getJSONObject("jobDetail").getJSONObject("generalInfo").getJSONObject("subclient").get("displayName")); - String clientId = String.valueOf(jsonObject.getJSONObject("job").getJSONObject("jobDetail").getJSONObject("generalInfo").getJSONObject("subclient").get("clientId")); - String companyId = String.valueOf(jsonObject.getJSONObject("job").getJSONObject("jobDetail").getJSONObject("generalInfo").getJSONObject("company").get("companyId")); - String companyName = String.valueOf(jsonObject.getJSONObject("job").getJSONObject("jobDetail").getJSONObject("generalInfo").getJSONObject("company").get("companyName")); - String instanceName = String.valueOf(jsonObject.getJSONObject("job").getJSONObject("jobDetail").getJSONObject("generalInfo").getJSONObject("subclient").get("instanceName")); - String appName = String.valueOf(jsonObject.getJSONObject("job").getJSONObject("jobDetail").getJSONObject("generalInfo").getJSONObject("subclient").get("appName")); - String applicationId = String.valueOf(jsonObject.getJSONObject("job").getJSONObject("jobDetail").getJSONObject("generalInfo").getJSONObject("subclient").get("applicationId")); - String clientName = String.valueOf(jsonObject.getJSONObject("job").getJSONObject("jobDetail").getJSONObject("generalInfo").getJSONObject("subclient").get("clientName")); - String backupsetId = String.valueOf(jsonObject.getJSONObject("job").getJSONObject("jobDetail").getJSONObject("generalInfo").getJSONObject("subclient").get("backupsetId")); - String instanceId = String.valueOf(jsonObject.getJSONObject("job").getJSONObject("jobDetail").getJSONObject("generalInfo").getJSONObject("subclient").get("instanceId")); - String backupsetName = String.valueOf(jsonObject.getJSONObject("job").getJSONObject("jobDetail").getJSONObject("generalInfo").getJSONObject("subclient").get("backupsetName")); - String commCellId = String.valueOf(jsonObject.getJSONObject("job").getJSONObject("jobDetail").getJSONObject("generalInfo").getJSONObject("commcell").get("commCellId")); - String backupsetGUID = client.getVmBackupSetGuid(clientName, backupsetName); - if (backupsetGUID == null) { - throw new CloudRuntimeException("Failed to get vm backup set guid commvault api"); - } - LOG.info(String.format("Restoring vm %s from backup %s on the Commvault Backup Provider", vm, backup)); - // 복원 실행 - int sshPort = NumbersUtil.parseInt(configDao.getValue("kvm.ssh.port"), 22); - HostVO hostVO = hostDao.findByName(clientName); - Ternary credentials = getKVMHyperisorCredentials(hostVO); - String jobId2 = client.restoreFullVM(subclientId, displayName, backupsetGUID, clientId, companyId, companyName, instanceName, appName, applicationId, clientName, backupsetId, instanceId, backupsetName, commCellId, endTime, path); - if (jobId2 != null) { - String jobStatus = client.getJobStatus(jobId2); - if (jobStatus.equalsIgnoreCase("Completed")) { - String snapshotId = backup.getSnapshotId(); - Map checkResult = new HashMap<>(); - if (snapshotId != null || !snapshotId.isEmpty()) { - String[] snapshots = snapshotId.split(","); - for (int i=0; i < snapshots.length; i++) { - SnapshotVO snapshot = snapshotDao.findByUuidIncludingRemoved(snapshots[i]); - VolumeVO volume = volumeDao.findById(snapshot.getVolumeId()); - StoragePoolVO storagePool = primaryDataStoreDao.findById(volume.getPoolId()); - String volumePath = String.format("%s/%s", storagePool.getPath(), volume.getPath()); - SnapshotDataStoreVO snapshotStore = snapshotStoreDao.findDestroyedReferenceBySnapshot(snapshot.getSnapshotId(), DataStoreRole.Primary); - String snapshotPath = snapshotStore.getInstallPath(); - String command = String.format(RSYNC_COMMAND, snapshotPath, volumePath); - if (executeRestoreCommand(hostVO, credentials.first(), credentials.second(), sshPort, command)) { - Date restoreJobEnd = new Date(); - if (snapshots.length > 1) { - String[] paths = path.split(","); - checkResult.put(snapshots[i], paths[i]); - } else { - checkResult.put(snapshots[i], path); - } - } else { - if (!checkResult.isEmpty()) { - for (String value : checkResult.values()) { - command = String.format(RM_COMMAND, value); - executeDeleteSnapshotCommand(hostVO, credentials.first(), credentials.second(), sshPort, command); - } - } - return false; - } - } - if (!checkResult.isEmpty()) { - for (String value : checkResult.values()) { - String command = String.format(RM_COMMAND, value); - executeDeleteSnapshotCommand(hostVO, credentials.first(), credentials.second(), sshPort, command); - } - } - return true; - } - } else { - // 복원 실패 - LOG.error("restoreBackup commvault api resulted in " + jobStatus); - return false; - } - } - return false; + return new Pair<>(volumePools, volumePaths); + } + + private String getVolumePathPrefix(StoragePoolVO storagePool) { + String volumePathPrefix; + if (ScopeType.HOST.equals(storagePool.getScope()) || + Storage.StoragePoolType.SharedMountPoint.equals(storagePool.getPoolType()) || + Storage.StoragePoolType.RBD.equals(storagePool.getPoolType())) { + volumePathPrefix = storagePool.getPath(); + } else { + // Should be Storage.StoragePoolType.NetworkFilesystem + volumePathPrefix = String.format("/mnt/%s", storagePool.getUuid()); + } + return volumePathPrefix; } + // 백업 볼륨 복원 및 연결 @Override public Pair restoreBackedUpVolume(Backup backup, Backup.VolumeInfo backupVolumeInfo, String hostIp, String dataStoreUuid, Pair vmNameAndState) { - final VolumeVO volume = volumeDao.findByUuid(backupVolumeInfo.getUuid()); - List backedVolumes = backup.getBackedUpVolumes(); - final DiskOffering diskOffering = diskOfferingDao.findByUuid(backupVolumeInfo.getDiskOfferingId()); - final StoragePoolVO pool = primaryDataStoreDao.findByUuid(dataStoreUuid); try { String commvaultServer = getUrlDomain(CommvaultUrl.value()); } catch (URISyntaxException e) { @@ -642,7 +602,7 @@ public Pair restoreBackedUpVolume(Backup backup, Backup.VolumeI final Long zoneId = backup.getZoneId(); final CommvaultClient client = getClient(zoneId); String jobId = externalId.substring(externalId.lastIndexOf(',') + 1).trim(); - String path = externalId.substring(0, externalId.lastIndexOf(',')); + final String path = externalId.substring(0, externalId.lastIndexOf(',')); String jobDetails = client.getJobDetails(jobId); if (jobDetails == null) { throw new CloudRuntimeException("Failed to get job details commvault api"); @@ -666,424 +626,115 @@ public Pair restoreBackedUpVolume(Backup backup, Backup.VolumeI if (backupsetGUID == null) { throw new CloudRuntimeException("Failed to get vm backup set guid commvault api"); } - LOG.info(String.format("Restoring volume %s from backup %s on the Commvault Backup Provider", volume.getUuid(), backup)); // 복원 실행 - int sshPort = NumbersUtil.parseInt(configDao.getValue("kvm.ssh.port"), 22); - HostVO hostVO = hostDao.findByName(clientName); - Ternary credentials = getKVMHyperisorCredentials(hostVO); String jobId2 = client.restoreFullVM(subclientId, displayName, backupsetGUID, clientId, companyId, companyName, instanceName, appName, applicationId, clientName, backupsetId, instanceId, backupsetName, commCellId, endTime, path); if (jobId2 != null) { String jobStatus = client.getJobStatus(jobId2); if (jobStatus.equalsIgnoreCase("Completed")) { - String snapshotId = backup.getSnapshotId(); - Map checkResult = new HashMap<>(); - String restoreVolume = null; - if (snapshotId != null || !snapshotId.isEmpty()) { - String[] snapshots = snapshotId.split(","); - for (int i=0; i < snapshots.length; i++) { - SnapshotVO snapshot = snapshotDao.findByUuidIncludingRemoved(snapshots[i]); - VolumeVO volumes = volumeDao.findById(snapshot.getVolumeId()); - StoragePoolVO storagePool = primaryDataStoreDao.findById(volumes.getPoolId()); - String volumePath = String.format("%s/%s", storagePool.getPath(), volume.getPath()); - SnapshotDataStoreVO snapshotStore = snapshotStoreDao.findDestroyedReferenceBySnapshot(snapshot.getSnapshotId(), DataStoreRole.Primary); - String snapshotPath = snapshotStore.getInstallPath(); - if (volumes.getPath().equalsIgnoreCase(volume.getPath())) { - VMInstanceVO backupSourceVm = vmInstanceDao.findById(backup.getVmId()); - VolumeVO restoredVolume = new VolumeVO(Volume.Type.DATADISK, null, backup.getZoneId(), - backup.getDomainId(), backup.getAccountId(), 0, null, - backup.getSize(), null, null, null); - String volumeName = volume != null ? volume.getName() : backupVolumeInfo.getUuid(); - restoredVolume.setName("RV-"+volumeName); - restoredVolume.setProvisioningType(diskOffering.getProvisioningType()); - restoredVolume.setUpdated(new Date()); - restoredVolume.setUuid(UUID.randomUUID().toString()); - restoredVolume.setRemoved(null); - restoredVolume.setDisplayVolume(true); - restoredVolume.setPoolId(pool.getId()); - restoredVolume.setPath(restoredVolume.getUuid()); - restoredVolume.setState(Volume.State.Copying); - restoredVolume.setSize(backupVolumeInfo.getSize()); - restoredVolume.setDiskOfferingId(diskOffering.getId()); - try { - volumeDao.persist(restoredVolume); - } catch (Exception e) { - throw new CloudRuntimeException("Unable to craft restored volume due to: "+e); - } - String reVolumePath = String.format("%s/%s", storagePool.getPath(), restoredVolume.getUuid()); - String command = String.format(RSYNC_COMMAND, snapshotPath, reVolumePath); - if (executeRestoreCommand(hostVO, credentials.first(), credentials.second(), sshPort, command)) { - Date restoreJobEnd = new Date(); - LOG.info("Restore Job for jobID " + jobId2 + " completed successfully at " + restoreJobEnd); - if (VirtualMachine.State.Running.equals(vmNameAndState.second())) { - final VMInstanceVO vm = vmInstanceDao.findVMByInstanceName(vmNameAndState.first()); - HostVO rvHostVO = hostDao.findById(vm.getHostId()); - Ternary rvCredentials = getKVMHyperisorCredentials(rvHostVO); - command = String.format(CURRRENT_DEVICE, vmNameAndState.first()); - String currentDevice = executeDeviceCommand(rvHostVO, rvCredentials.first(), rvCredentials.second(), sshPort, command); - if (currentDevice == null || currentDevice.contains("error")) { - volumeDao.expunge(restoredVolume.getId()); - command = String.format(RM_COMMAND, snapshotPath); - executeDeleteSnapshotCommand(hostVO, credentials.first(), credentials.second(), sshPort, command); - throw new CloudRuntimeException("Failed to get current device execute command VM to location " + volume.getPath()); - } else { - currentDevice = currentDevice.replaceAll("\\s", ""); - char lastChar = currentDevice.charAt(currentDevice.length() - 1); - char incrementedChar = (char) (lastChar + 1); - String rvDevice = currentDevice.substring(0, currentDevice.length() - 1) + incrementedChar; - command = String.format(ATTACH_DISK_COMMAND, vmNameAndState.first(), reVolumePath, rvDevice); - if (!executeAttachCommand(rvHostVO, rvCredentials.first(), rvCredentials.second(), sshPort, command)) { - volumeDao.expunge(restoredVolume.getId()); - command = String.format(RM_COMMAND, snapshotPath); - executeDeleteSnapshotCommand(hostVO, credentials.first(), credentials.second(), sshPort, command); - throw new CloudRuntimeException(String.format("Failed to attach volume to VM: %s", vmNameAndState.first())); - } - } - } - checkResult.put(snapshots[i], volumePath); - restoreVolume = restoredVolume.getUuid(); - } else { - volumeDao.expunge(restoredVolume.getId()); - LOG.info("Restore Job for jobID " + jobId2 + " completed failed."); - command = String.format(RM_COMMAND, snapshotPath); - executeDeleteSnapshotCommand(hostVO, credentials.first(), credentials.second(), sshPort, command); - } - } else { - String command = String.format(RM_COMMAND, snapshotPath); - executeDeleteSnapshotCommand(hostVO, credentials.first(), credentials.second(), sshPort, command); - } - } - if (!checkResult.isEmpty()) { - return new Pair<>(true,restoreVolume); - } else { - throw new CloudRuntimeException("Failed to restore VM to location " + volume.getPath()); + final int sshPort = NumbersUtil.parseInt(configDao.getValue("kvm.ssh.port"), 22); + final VolumeVO volume = volumeDao.findByUuid(backupVolumeInfo.getUuid()); + final DiskOffering diskOffering = diskOfferingDao.findByUuid(backupVolumeInfo.getDiskOfferingId()); + String cacheMode = null; + final VMInstanceVO vm = vmInstanceDao.findVMByInstanceName(vmNameAndState.first()); + List listVolumes = volumeDao.findByInstanceAndType(vm.getId(), Type.ROOT); + if(CollectionUtils.isNotEmpty(listVolumes)) { + VolumeVO rootDisk = listVolumes.get(0); + DiskOffering baseDiskOffering = diskOfferingDao.findById(rootDisk.getDiskOfferingId()); + if (baseDiskOffering.getCacheMode() != null) { + cacheMode = baseDiskOffering.getCacheMode().toString(); } } - } else { - // 복원 실패 - LOG.error("restoreBackup commvault api resulted in " + jobStatus); - throw new CloudRuntimeException("Failed to restore VM to location " + volume.getPath() + " commvault api resulted in " + jobStatus); - } - } - return new Pair<>(false,null); - } - - @Override - public Pair takeBackup(VirtualMachine vm, Boolean quiesceVM) { - String hostName = null; - try { - String commvaultServer = getUrlDomain(CommvaultUrl.value()); - } catch (URISyntaxException e) { - throw new CloudRuntimeException(String.format("Failed to convert API to HOST : %s", e)); - } - // 백업 중인 작업 조회 - final CommvaultClient client = getClient(vm.getDataCenterId()); - boolean activeJob = client.getActiveJob(vm.getInstanceName()); - if (activeJob) { - throw new CloudRuntimeException("There are backup jobs running on the virtual machine. Please try again later."); - } - // 클라이언트의 백업세트 조회하여 호스트 정의 - List Hosts = hostDao.findByDataCenterId(vm.getDataCenterId()); - for (final HostVO host : Hosts) { - if (host.getHypervisorType() == Hypervisor.HypervisorType.KVM) { - String checkVm = client.getVmBackupSetId(host.getName(), vm.getInstanceName()); - if (checkVm != null) { - hostName = host.getName(); - } - } - } - BackupOfferingVO vmBackupOffering = new BackupOfferingDaoImpl().findById(vm.getBackupOfferingId()); - String planId = vmBackupOffering.getExternalId(); - // 스냅샷 생성 mold-API 호출 - String[] properties = getServerProperties(); - ManagementServerHostVO msHost = msHostDao.findByMsid(ManagementServerNode.getManagementServerId()); - String moldUrl = properties[1] + "://" + msHost.getServiceIP() + ":" + properties[0] + "/client/api/"; - String moldMethod = "POST"; - String moldCommand = "createSnapshotBackup"; - UserAccount user = accountService.getActiveUserAccount("admin", 1L); - String apiKey = user.getApiKey(); - String secretKey = user.getSecretKey(); - if (apiKey == null || secretKey == null) { - throw new CloudRuntimeException("Failed because the API key and Secret key for the admin account do not exist."); - } - UserVmJoinVO userVM = userVmJoinDao.findById(vm.getId()); - List volumes = volumeDao.findByInstance(userVM.getId()); - volumes.sort(Comparator.comparing(Volume::getDeviceId)); - StringJoiner joiner = new StringJoiner(","); - Map checkResult = new HashMap<>(); - for (VolumeVO vol : volumes) { - Map snapParams = new HashMap<>(); - snapParams.put("volumeid", Long.toString(vol.getId())); - snapParams.put("backup", "true"); - // snapParams.put("quiescevm", String.valueOf(quiesceVM)); - String createSnapResult = moldCreateSnapshotBackupAPI(moldUrl, moldCommand, moldMethod, apiKey, secretKey, snapParams); - if (createSnapResult == null) { - if (!checkResult.isEmpty()) { - for (String value : checkResult.values()) { - Map snapshotParams = new HashMap<>(); - snapshotParams.put("id", value); - moldMethod = "GET"; - moldCommand = "deleteSnapshot"; - moldDeleteSnapshotAPI(moldUrl, moldCommand, moldMethod, apiKey, secretKey, snapshotParams); - } - } - LOG.error("Failed to request createSnapshot Mold-API."); - return new Pair<>(false, null); - } else { - JSONObject jsonObject = new JSONObject(createSnapResult); - String jobId = jsonObject.get("jobid").toString(); - String snapId = jsonObject.get("id").toString(); - int jobStatus = getAsyncJobResult(moldUrl, apiKey, secretKey, jobId); - if (jobStatus == 2) { - Map snapshotParams = new HashMap<>(); - snapshotParams.put("id", snapId); - moldMethod = "GET"; - moldCommand = "deleteSnapshot"; - moldDeleteSnapshotAPI(moldUrl, moldCommand, moldMethod, apiKey, secretKey, snapshotParams); - if (!checkResult.isEmpty()) { - for (String value : checkResult.values()) { - snapshotParams = new HashMap<>(); - snapshotParams.put("id", value); - moldMethod = "GET"; - moldCommand = "deleteSnapshot"; - moldDeleteSnapshotAPI(moldUrl, moldCommand, moldMethod, apiKey, secretKey, snapshotParams); - } - } - LOG.error("createSnapshot Mold-API async job resulted in failure."); - return new Pair<>(false, null); - } - checkResult.put(vol.getId(), snapId); - SnapshotDataStoreVO snapshotStore = snapshotStoreDao.findLatestSnapshotForVolume(vol.getId(), DataStoreRole.Primary); - joiner.add(snapshotStore.getInstallPath()); - } - } - String path = joiner.toString(); - String backupPath = path; - // 가상머신이 실행중인 경우 가상머신 xml 파일 함께 백업 - HostVO hostVO = null; - StoragePoolVO storagePool = null; - String storagePath = null; - String command = null; - Ternary credentials = null; - int sshPort = NumbersUtil.parseInt(configDao.getValue("kvm.ssh.port"), 22); - if (vm.getState() == VirtualMachine.State.Running) { - hostVO = getRunningVMHypervisorHost(vm); - credentials = getKVMHyperisorCredentials(hostVO); - List rootVolumesOfVm = volumeDao.findByInstanceAndType(userVM.getId(), Volume.Type.ROOT); - if (!rootVolumesOfVm.isEmpty()) { - storagePool = primaryDataStoreDao.findById(rootVolumesOfVm.get(0).getPoolId()); - storagePath = storagePool.getPath(); - command = String.format( - "mkdir -p %1$s && " + - "virsh -c qemu:///system dumpxml '%2$s' > %1$s/domain-config.xml && " + - "virsh -c qemu:///system dominfo '%2$s' > %1$s/dominfo.xml && " + - "virsh -c qemu:///system domiflist '%2$s' > %1$s/domiflist.xml && " + - "virsh -c qemu:///system domblklist '%2$s' > %1$s/domblklist.xml", - storagePath + "/" + vm.getInstanceName(), vm.getInstanceName() - ); - if (!executeTakeBackupCommand(hostVO, credentials.first(), credentials.second(), sshPort, command)) { - command = String.format(RM_COMMAND, storagePath + "/" + vm.getInstanceName()); - executeDeleteXmlCommand(hostVO, credentials.first(), credentials.second(), sshPort, command); + final StoragePoolVO pool = primaryDataStoreDao.findByUuid(dataStoreUuid); + // 백업 볼륨 복원 및 연결 시 연결할 가상머신이 실행중인 경우 해당 호스트, 정지중인 경우 랜덤 호스트 정의백업 + final HostVO vmHost = hostDao.findByIp(hostIp); + final HostVO vmHostVO = hostDao.findById(vmHost.getId()); + // 복원된 호스트 정의 + final HostVO restoreHost = hostDao.findByName(clientName); + final HostVO restoreHostVO = hostDao.findById(restoreHost.getId()); + LOG.info(String.format("Restoring volume %s from backup %s on the Commvault Backup Provider", volume.getUuid(), backup)); + LOG.debug("Restoring vm volume {} from backup {} on the Commvault Backup Provider", backupVolumeInfo, backup); + VolumeVO restoredVolume = new VolumeVO(Volume.Type.DATADISK, null, backup.getZoneId(), + backup.getDomainId(), backup.getAccountId(), 0, null, + backup.getSize(), null, null, null); + String volumeUUID = UUID.randomUUID().toString(); + String volumeName = volume != null ? volume.getName() : backupVolumeInfo.getUuid(); + restoredVolume.setName("RestoredVol-" + volumeName); + restoredVolume.setProvisioningType(diskOffering.getProvisioningType()); + restoredVolume.setUpdated(new Date()); + restoredVolume.setUuid(volumeUUID); + restoredVolume.setRemoved(null); + restoredVolume.setDisplayVolume(true); + restoredVolume.setPoolId(pool.getId()); + restoredVolume.setPoolType(pool.getPoolType()); + restoredVolume.setPath(restoredVolume.getUuid()); + restoredVolume.setState(Volume.State.Copying); + restoredVolume.setSize(backupVolumeInfo.getSize()); + restoredVolume.setDiskOfferingId(diskOffering.getId()); + if (pool.getPoolType() != Storage.StoragePoolType.RBD) { + restoredVolume.setFormat(Storage.ImageFormat.QCOW2); } else { - joiner.add(storagePath + "/" + vm.getInstanceName()); - path = joiner.toString(); - } - } - } - // 생성된 스냅샷의 경로로 해당 백업 세트의 백업 콘텐츠 경로 업데이트 - String clientId = client.getClientId(hostName); - String subClientEntity = client.getSubclient(clientId, vm.getInstanceName()); - if (subClientEntity == null) { - if (!checkResult.isEmpty()) { - for (String value : checkResult.values()) { - Map snapshotParams = new HashMap<>(); - snapshotParams.put("id", value); - moldMethod = "GET"; - moldCommand = "deleteSnapshot"; - moldDeleteSnapshotAPI(moldUrl, moldCommand, moldMethod, apiKey, secretKey, snapshotParams); + restoredVolume.setFormat(Storage.ImageFormat.RAW); } - if (vm.getState() == VirtualMachine.State.Running) { - command = String.format(RM_COMMAND, storagePath + "/" + vm.getInstanceName()); - executeDeleteXmlCommand(hostVO, credentials.first(), credentials.second(), sshPort, command); + + CommvaultRestoreBackupCommand restoreCommand = new CommvaultRestoreBackupCommand(); + restoreCommand.setBackupPath(path); + restoreCommand.setVmName(vmNameAndState.first()); + restoreCommand.setRestoreVolumePaths(Collections.singletonList(String.format("%s/%s", getVolumePathPrefix(pool), volumeUUID))); + DataStore dataStore = dataStoreMgr.getDataStore(pool.getId(), DataStoreRole.Primary); + restoreCommand.setRestoreVolumePools(Collections.singletonList(dataStore != null ? (PrimaryDataStoreTO)dataStore.getTO() : null)); + restoreCommand.setDiskType(backupVolumeInfo.getType().name().toLowerCase(Locale.ROOT)); + restoreCommand.setVmExists(null); + restoreCommand.setVmState(vmNameAndState.second()); + restoreCommand.setRestoreVolumeUUID(backupVolumeInfo.getUuid()); + restoreCommand.setTimeout(CommvaultBackupRestoreTimeout.value()); + restoreCommand.setCacheMode(cacheMode); + // 복원된 호스트와 가상머신이 실행중인 호스트가 같은 경우 null, 다른 경우 추가 + restoreCommand.setHostName(restoreHost.getId() == vmHost.getId() ? null : restoreHost.getName()); + + BackupAnswer answer; + try { + answer = (BackupAnswer) agentManager.send(vmHost.getId(), restoreCommand); + } catch (AgentUnavailableException e) { + throw new CloudRuntimeException("Unable to contact backend control plane to initiate backup"); + } catch (OperationTimedoutException e) { + throw new CloudRuntimeException("Operation to restore backed up volume timed out, please try again"); } - } - throw new CloudRuntimeException("Failed to get subclient info commvault api"); - } - JSONObject jsonObject = new JSONObject(subClientEntity); - String subclientId = String.valueOf(jsonObject.get("subclientId")); - String applicationId = String.valueOf(jsonObject.get("applicationId")); - String backupsetId = String.valueOf(jsonObject.get("backupsetId")); - String instanceId = String.valueOf(jsonObject.get("instanceId")); - String backupsetName = String.valueOf(jsonObject.get("backupsetName")); - String displayName = String.valueOf(jsonObject.get("displayName")); - String commCellName = String.valueOf(jsonObject.get("commCellName")); - String companyId = String.valueOf(jsonObject.getJSONObject("entityInfo").get("companyId")); - String companyName = String.valueOf(jsonObject.getJSONObject("entityInfo").get("companyName")); - String instanceName = String.valueOf(jsonObject.get("instanceName")); - String appName = String.valueOf(jsonObject.get("appName")); - String clientName = String.valueOf(jsonObject.get("clientName")); - String subclientGUID = String.valueOf(jsonObject.get("subclientGUID")); - String subclientName = String.valueOf(jsonObject.get("subclientName")); - String csGUID = String.valueOf(jsonObject.get("csGUID")); - boolean upResult = client.updateBackupSet(path, subclientId, clientId, planId, applicationId, backupsetId, instanceId, subclientName, backupsetName); - if (upResult) { - String planName = client.getPlanName(planId); - String storagePolicyId = client.getStoragePolicyId(planName); - if (planName == null || storagePolicyId == null) { - if (!checkResult.isEmpty()) { - for (String value : checkResult.values()) { - Map snapshotParams = new HashMap<>(); - snapshotParams.put("id", value); - moldMethod = "GET"; - moldCommand = "deleteSnapshot"; - moldDeleteSnapshotAPI(moldUrl, moldCommand, moldMethod, apiKey, secretKey, snapshotParams); + + if (answer.getResult()) { + try { + volumeDao.persist(restoredVolume); + } catch (Exception e) { + throw new CloudRuntimeException("Unable to create restored volume due to: " + e); } - if (vm.getState() == VirtualMachine.State.Running) { - command = String.format(RM_COMMAND, storagePath + "/" + vm.getInstanceName()); - executeDeleteXmlCommand(hostVO, credentials.first(), credentials.second(), sshPort, command); - } - } - throw new CloudRuntimeException("Failed to get storage Policy id commvault api"); - } - // 백업 실행 - String jobId = client.createBackup(subclientId, storagePolicyId, displayName, commCellName, clientId, companyId, companyName, instanceName, appName, applicationId, clientName, backupsetId, instanceId, subclientGUID, subclientName, csGUID, backupsetName); - if (jobId != null) { - String jobStatus = client.getJobStatus(jobId); - if (jobStatus.equalsIgnoreCase("Completed")) { - String jobDetails = client.getJobDetails(jobId); - if (jobDetails != null) { - JSONObject jsonObject2 = new JSONObject(jobDetails); - String endTime = String.valueOf(jsonObject2.getJSONObject("job").getJSONObject("jobDetail").getJSONObject("detailInfo").get("endTime")); - long timestamp = Long.parseLong(endTime) * 1000L; - Date endDate = new Date(timestamp); - SimpleDateFormat formatterDateTime = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss"); - String formattedString = formatterDateTime.format(endDate); - String size = String.valueOf(jsonObject2.getJSONObject("job").getJSONObject("jobDetail").getJSONObject("detailInfo").get("sizeOfApplication")); - String type = String.valueOf(jsonObject2.getJSONObject("job").getJSONObject("jobDetail").getJSONObject("generalInfo").get("backupType")); - String externalId = backupPath + "," + jobId; - BackupVO backup = new BackupVO(); - backup.setVmId(vm.getId()); - backup.setExternalId(externalId); - backup.setType(type.toUpperCase()); - try { - backup.setDate(formatterDateTime.parse(formattedString)); - } catch (ParseException e) { - String msg = String.format("Unable to parse date [%s].", endTime); - LOG.error(msg, e); - throw new CloudRuntimeException(msg, e); - } - backup.setSize(Long.parseLong(size)); - long virtualSize = 0L; - for (final Volume volume: volumeDao.findByInstance(vm.getId())) { - if (Volume.State.Ready.equals(volume.getState())) { - virtualSize += volume.getSize(); - } - } - backup.setProtectedSize(Long.valueOf(virtualSize)); - backup.setStatus(org.apache.cloudstack.backup.Backup.Status.BackedUp); - backup.setBackupOfferingId(vm.getBackupOfferingId()); - backup.setAccountId(vm.getAccountId()); - backup.setDomainId(vm.getDomainId()); - backup.setZoneId(vm.getDataCenterId()); - backup.setName(backupManager.getBackupNameFromVM(vm)); - List vols = new ArrayList<>(volumeDao.findByInstance(vm.getId())); - backup.setBackedUpVolumes(backupManager.createVolumeInfoFromVolumes(vols)); - Map details = backupManager.getBackupDetailsFromVM(vm); - backup.setDetails(details); - StringJoiner snapshots = new StringJoiner(","); - for (String value : checkResult.values()) { - snapshots.add(value); - } - backup.setSnapshotId(snapshots.toString()); - backupDao.persist(backup); - // 백업 오퍼링 할당 시의 볼륨 정보만 담겨있어서 이후 추가된 볼륨에 대해 복원 시 오류로 백업 시 볼륨 정보 업데이트 - VMInstanceVO vmInstance = vmInstanceDao.findByIdIncludingRemoved(vm.getId()); - vmInstance.setBackupVolumes(backupManager.createVolumeInfoFromVolumes(vols)); - vmInstanceDao.update(vm.getId(), vmInstance); - // 백업 성공 후 스냅샷 삭제 - for (String value : checkResult.values()) { - Map snapshotParams = new HashMap<>(); - snapshotParams.put("id", value); - moldMethod = "GET"; - moldCommand = "deleteSnapshot"; - moldDeleteSnapshotAPI(moldUrl, moldCommand, moldMethod, apiKey, secretKey, snapshotParams); - } - if (vm.getState() == VirtualMachine.State.Running) { - command = String.format(RM_COMMAND, storagePath + "/" + vm.getInstanceName()); - executeDeleteXmlCommand(hostVO, credentials.first(), credentials.second(), sshPort, command); - } - return new Pair<>(true, backup); - } else { - // 백업 실패 - if (!checkResult.isEmpty()) { - for (String value : checkResult.values()) { - Map snapshotParams = new HashMap<>(); - snapshotParams.put("id", value); - moldMethod = "GET"; - moldCommand = "deleteSnapshot"; - moldDeleteSnapshotAPI(moldUrl, moldCommand, moldMethod, apiKey, secretKey, snapshotParams); - } - } - if (vm.getState() == VirtualMachine.State.Running) { - command = String.format(RM_COMMAND, storagePath + "/" + vm.getInstanceName()); - executeDeleteXmlCommand(hostVO, credentials.first(), credentials.second(), sshPort, command); - } - LOG.error("createBackup commvault api resulted in " + jobStatus); - return new Pair<>(false, null); + if (restoreHost.getId() != vmHost.getId()) { + Ternary credentials = getKVMHyperisorCredentials(restoreHostVO); + String command = String.format(RM_COMMAND, path); + executeDeleteBackupPathCommand(restoreHostVO, credentials.first(), credentials.second(), sshPort, command); } + return new Pair<>(answer.getResult(), answer.getDetails()); } else { - // 백업 실패 - if (!checkResult.isEmpty()) { - for (String value : checkResult.values()) { - Map snapshotParams = new HashMap<>(); - snapshotParams.put("id", value); - moldMethod = "GET"; - moldCommand = "deleteSnapshot"; - moldDeleteSnapshotAPI(moldUrl, moldCommand, moldMethod, apiKey, secretKey, snapshotParams); - } - } - if (vm.getState() == VirtualMachine.State.Running) { - command = String.format(RM_COMMAND, storagePath + "/" + vm.getInstanceName()); - executeDeleteXmlCommand(hostVO, credentials.first(), credentials.second(), sshPort, command); + Ternary credentials = getKVMHyperisorCredentials(vmHostVO); + String command = String.format(RM_COMMAND, path); + executeDeleteBackupPathCommand(vmHostVO, credentials.first(), credentials.second(), sshPort, command); + if (restoreHost.getId() != vmHost.getId()) { + credentials = getKVMHyperisorCredentials(restoreHostVO); + command = String.format(RM_COMMAND, path); + executeDeleteBackupPathCommand(restoreHostVO, credentials.first(), credentials.second(), sshPort, command); } - LOG.error("createBackup commvault api resulted in " + jobStatus); - return new Pair<>(false, null); } } else { - // 백업 실패 - if (!checkResult.isEmpty()) { - for (String value : checkResult.values()) { - Map snapshotParams = new HashMap<>(); - snapshotParams.put("id", value); - moldMethod = "GET"; - moldCommand = "deleteSnapshot"; - moldDeleteSnapshotAPI(moldUrl, moldCommand, moldMethod, apiKey, secretKey, snapshotParams); - } - } - if (vm.getState() == VirtualMachine.State.Running) { - command = String.format(RM_COMMAND, storagePath + "/" + vm.getInstanceName()); - executeDeleteXmlCommand(hostVO, credentials.first(), credentials.second(), sshPort, command); - } - LOG.error("failed request createBackup commvault api"); - return new Pair<>(false, null); + LOG.error("Failed to restore backup for VM " + vmNameAndState.first() + " to restore backup job status is " + jobStatus); } } else { - // 백업 경로 업데이트 실패 - if (!checkResult.isEmpty()) { - for (String value : checkResult.values()) { - Map snapshotParams = new HashMap<>(); - snapshotParams.put("id", value); - moldMethod = "GET"; - moldCommand = "deleteSnapshot"; - moldDeleteSnapshotAPI(moldUrl, moldCommand, moldMethod, apiKey, secretKey, snapshotParams); - } - } - if (vm.getState() == VirtualMachine.State.Running) { - command = String.format(RM_COMMAND, storagePath + "/" + vm.getInstanceName()); - executeDeleteXmlCommand(hostVO, credentials.first(), credentials.second(), sshPort, command); - } - LOG.error("updateBackupSet commvault api resulted in failure."); - return new Pair<>(false, null); + LOG.error("Failed to restore backup for VM " + vmNameAndState.first() + " to restore backup job commvault api"); } + return new Pair<>(false, null); + } + + private Optional getBackedUpVolumeInfo(List backedUpVolumes, String volumeUuid) { + return backedUpVolumes.stream() + .filter(v -> v.getUuid().equals(volumeUuid)) + .findFirst(); } @Override @@ -1108,6 +759,125 @@ public boolean deleteBackup(Backup backup, boolean forced) { } } + public void syncBackupMetrics(Long zoneId) { + } + + @Override + public List listRestorePoints(VirtualMachine vm) { + return null; + } + + @Override + public Backup createNewBackupEntryForRestorePoint(Backup.RestorePoint restorePoint, VirtualMachine vm) { + return null; + } + + @Override + public boolean assignVMToBackupOffering(VirtualMachine vm, BackupOffering backupOffering) { + final CommvaultClient client = getClient(vm.getDataCenterId()); + final Host host = getVMHypervisorHostForBackup(vm); + String clientId = client.getClientId(host.getName()); + String applicationId = client.getApplicationId(clientId); + return client.createBackupSet(vm.getInstanceName(), applicationId, clientId, backupOffering.getExternalId()); + } + + @Override + public boolean removeVMFromBackupOffering(VirtualMachine vm) { + final CommvaultClient client = getClient(vm.getDataCenterId()); + List Hosts = hostDao.findByDataCenterId(vm.getDataCenterId()); + boolean allDeleted = true; + for (final HostVO host : Hosts) { + if (host.getHypervisorType() == Hypervisor.HypervisorType.KVM) { + String backupSetId = client.getVmBackupSetId(host.getName(), vm.getInstanceName()); + if (backupSetId != null) { + boolean deleted = client.deleteBackupSet(backupSetId); + if (!deleted) { + allDeleted = false; + LOG.error("Failed to delete backupSetId: " + backupSetId +" for VM: " + vm.getInstanceName()); + } + } + } + } + return allDeleted; + } + + // 하위 클라이언트 삭제 시 백업본 데이터는 그대로 남아있지만, 해당 하위 클라이언트가 삭제되었기 때문에 스케줄도 삭제시켜야하며 + // 남아있는 백업본 데이터는 mold에서 관리하지 않고, commvault 의 plan 보존기간에 따라 데이터 에이징 됨. + @Override + public boolean willDeleteBackupsOnOfferingRemoval() { + return true; + } + + @Override + public boolean supportsInstanceFromBackup() { + return true; + } + + @Override + public boolean supportsMemoryVmSnapshot() { + return false; + } + + @Override + public Pair getBackupStorageStats(Long zoneId) { + return new Pair<>(0L, 0L); + } + + @Override + public void syncBackupStorageStats(Long zoneId) { + } + + @Override + public List listBackupOfferings(Long zoneId) { + return getClient(zoneId).listPlans(); + } + + @Override + public boolean isValidProviderOffering(Long zoneId, String uuid) { + List policies = listBackupOfferings(zoneId); + if (CollectionUtils.isEmpty(policies)) { + return false; + } + for (final BackupOffering policy : policies) { + if (policy.getExternalId().equals(uuid)) { + return true; + } + } + return false; + } + + @Override + public Boolean crossZoneInstanceCreationEnabled(BackupOffering backupOffering) { + return false; + } + + @Override + public ConfigKey[] getConfigKeys() { + return new ConfigKey[]{ + CommvaultUrl, + CommvaultUsername, + CommvaultPassword, + CommvaultValidateSSLSecurity, + CommvaultApiRequestTimeout, + CommvaultClientVerboseLogs + }; + } + + @Override + public String getName() { + return "commvault"; + } + + @Override + public String getDescription() { + return "Commvault Backup Plugin"; + } + + @Override + public String getConfigComponentName() { + return BackupService.class.getSimpleName(); + } + @Override public void syncBackups(VirtualMachine vm) { try { @@ -1125,7 +895,6 @@ public void syncBackups(VirtualMachine vm) { JSONObject jsonObject = new JSONObject(jobDetails); String retainedUntil = String.valueOf(jsonObject.getJSONObject("job").getJSONObject("jobDetail").getJSONObject("generalInfo").get("retainedUntil")); String storagePolicyId = String.valueOf(jsonObject.getJSONObject("job").getJSONObject("jobDetail").getJSONObject("generalInfo").getJSONObject("storagePolicy").get("storagePolicyId")); - // 보존 기간 sync BackupOfferingVO vmBackupOffering = new BackupOfferingDaoImpl().findById(vm.getBackupOfferingId()); BackupOfferingVO offering = backupOfferingDao.createForUpdate(vmBackupOffering.getId()); String retentionDay = client.getRetentionPeriod(storagePolicyId); @@ -1150,352 +919,213 @@ public void syncBackups(VirtualMachine vm) { return; } - protected static String moldCreateSnapshotBackupAPI(String region, String command, String method, String apiKey, String secretKey, Map params) { - try { - String readLine = null; - StringBuffer sb = null; - String apiParams = buildParamsMold(command, params); - String urlFinal = buildUrl(apiParams, region, apiKey, secretKey); - URL url = new URL(urlFinal); - HttpURLConnection connection = null; - if (region.contains("https")) { - final SSLContext sslContext = SSLUtils.getSSLContext(); - sslContext.init(null, new TrustManager[]{new TrustAllManager()}, new SecureRandom()); - HttpsURLConnection httpsConnection = (HttpsURLConnection) url.openConnection(); - httpsConnection.setSSLSocketFactory(sslContext.getSocketFactory()); - connection = httpsConnection; - } else { - connection = (HttpURLConnection) url.openConnection(); - } - connection.setDoOutput(true); - connection.setRequestMethod(method); - connection.setConnectTimeout(10000); - connection.setReadTimeout(180000); - connection.setRequestProperty("Accept", "application/json"); - connection.setRequestProperty("Content-type", "application/x-www-form-urlencoded"); - if (connection.getResponseCode() == 200) { - BufferedReader br = new BufferedReader(new InputStreamReader(connection.getInputStream(), "UTF-8")); - sb = new StringBuffer(); - while ((readLine = br.readLine()) != null) { - sb.append(readLine); - } - } else { - String msg = "Failed to request mold API. response code : " + connection.getResponseCode(); - LOG.error(msg); - return null; - } - JSONObject jObject = XML.toJSONObject(sb.toString()); - JSONObject response = (JSONObject) jObject.get("createsnapshotbackupresponse"); - return response.toString(); - } catch (Exception e) { - LOG.error(String.format("Mold API endpoint not available"), e); - return null; - } - } - - protected static String moldDeleteSnapshotAPI(String region, String command, String method, String apiKey, String secretKey, Map params) { - try { - String readLine = null; - StringBuffer sb = null; - String apiParams = buildParamsMold(command, params); - String urlFinal = buildUrl(apiParams, region, apiKey, secretKey); - URL url = new URL(urlFinal); - HttpURLConnection connection = null; - if (region.contains("https")) { - final SSLContext sslContext = SSLUtils.getSSLContext(); - sslContext.init(null, new TrustManager[]{new TrustAllManager()}, new SecureRandom()); - HttpsURLConnection httpsConnection = (HttpsURLConnection) url.openConnection(); - httpsConnection.setSSLSocketFactory(sslContext.getSocketFactory()); - connection = httpsConnection; - } else { - connection = (HttpURLConnection) url.openConnection(); - } - connection.setDoOutput(true); - connection.setRequestMethod(method); - connection.setConnectTimeout(10000); - connection.setReadTimeout(180000); - connection.setRequestProperty("Accept", "application/json"); - connection.setRequestProperty("Content-type", "application/x-www-form-urlencoded"); - if (connection.getResponseCode() == 200) { - BufferedReader br = new BufferedReader(new InputStreamReader(connection.getInputStream(), "UTF-8")); - sb = new StringBuffer(); - while ((readLine = br.readLine()) != null) { - sb.append(readLine); - } - } else { - String msg = "Failed to request mold API. response code : " + connection.getResponseCode(); - LOG.error(msg); - return null; - } - JSONObject jObject = XML.toJSONObject(sb.toString()); - JSONObject response = (JSONObject) jObject.get("deletesnapshotresponse"); - return response.toString(); - } catch (Exception e) { - LOG.error(String.format("Mold API endpoint not available"), e); - return null; - } - } - - protected static String moldQueryAsyncJobResultAPI(String region, String command, String method, String apiKey, String secretKey, Map params) { - try { - String readLine = null; - StringBuffer sb = null; - String apiParams = buildParamsMold(command, params); - String urlFinal = buildUrl(apiParams, region, apiKey, secretKey); - URL url = new URL(urlFinal); - HttpURLConnection connection = null; - if (region.contains("https")) { - final SSLContext sslContext = SSLUtils.getSSLContext(); - sslContext.init(null, new TrustManager[]{new TrustAllManager()}, new SecureRandom()); - HttpsURLConnection httpsConnection = (HttpsURLConnection) url.openConnection(); - httpsConnection.setSSLSocketFactory(sslContext.getSocketFactory()); - connection = httpsConnection; - } else { - connection = (HttpURLConnection) url.openConnection(); - } - connection.setDoOutput(true); - connection.setRequestMethod(method); - connection.setConnectTimeout(10000); - connection.setReadTimeout(180000); - connection.setRequestProperty("Accept", "application/json"); - connection.setRequestProperty("Content-type", "application/x-www-form-urlencoded"); - if (connection.getResponseCode() == 200) { - BufferedReader br = new BufferedReader(new InputStreamReader(connection.getInputStream(), "UTF-8")); - sb = new StringBuffer(); - while ((readLine = br.readLine()) != null) { - sb.append(readLine); - } - } else { - String msg = "Failed to request mold API. response code : " + connection.getResponseCode(); - LOG.error(msg); - return null; - } - JSONObject jObject = XML.toJSONObject(sb.toString()); - JSONObject response = (JSONObject) jObject.get("queryasyncjobresultresponse"); - return response.get("jobstatus").toString(); - } catch (Exception e) { - LOG.error(String.format("Mold API endpoint not available"), e); - return null; - } - } - - protected static String buildParamsMold(String command, Map params) { - StringBuffer paramString = new StringBuffer("command=" + command); - if (params != null) { - try { - for(Map.Entry param : params.entrySet() ){ - String key = param.getKey(); - String value = param.getValue(); - paramString.append("&" + param.getKey() + "=" + URLEncoder.encode(param.getValue(), "UTF-8")); + @Override + public boolean checkBackupAgent(final Long zoneId) { + Map checkResult = new HashMap<>(); + final CommvaultClient client = getClient(zoneId); + String csVersionInfo = client.getCvtVersion(); + boolean version = versionCheck(csVersionInfo); + if (version) { + List Hosts = hostDao.findByDataCenterId(zoneId); + for (final HostVO host : Hosts) { + if (host.getStatus() == Status.Up && host.getHypervisorType() == Hypervisor.HypervisorType.KVM) { + String checkHost = client.getClientId(host.getName()); + if (checkHost == null) { + return false; + } else { + boolean installJob = client.getInstallActiveJob(host.getPrivateIpAddress()); + boolean checkInstall = client.getClientProps(checkHost); + if (installJob || !checkInstall) { + if (!checkInstall) { + LOG.error("The host is registered with the client, but the readiness status is not normal and you must manually check the client status."); + } + return false; + } + } } - } catch (UnsupportedEncodingException e) { - LOG.error(e.getMessage()); - return null; } + return true; } - return paramString.toString(); + return false; } - private static String buildUrl(String apiParams, String region, String apiKey, String secretKey) { - String encodedApiKey; - try { - encodedApiKey = URLEncoder.encode(apiKey, "UTF-8"); - List sortedParams = new ArrayList(); - sortedParams.add("apikey=" + encodedApiKey.toLowerCase()); - StringTokenizer st = new StringTokenizer(apiParams, "&"); - String url = null; - boolean first = true; - while (st.hasMoreTokens()) { - String paramValue = st.nextToken(); - String param = paramValue.substring(0, paramValue.indexOf("=")); - String value = paramValue.substring(paramValue.indexOf("=") + 1, paramValue.length()); - if (first) { - url = param + "=" + value; - first = false; - } else { - url = url + "&" + param + "=" + value; + @Override + public boolean installBackupAgent(final Long zoneId) { + Map failResult = new HashMap<>(); + final CommvaultClient client = getClient(zoneId); + List Hosts = hostDao.findByDataCenterId(zoneId); + for (final HostVO host : Hosts) { + if (host.getStatus() == Status.Up && host.getHypervisorType() == Hypervisor.HypervisorType.KVM) { + String commCell = client.getCommcell(); + JSONObject jsonObject = new JSONObject(commCell); + String commCellId = String.valueOf(jsonObject.get("commCellId")); + String commServeHostName = String.valueOf(jsonObject.get("commCellName")); + Ternary credentials = getKVMHyperisorCredentials(host); + boolean installJob = true; + LOG.info("checking for install agent on the Commvault Backup Provider in host " + host.getPrivateIpAddress()); + // 설치가 진행중인 호스트가 있는지 확인 + while (installJob) { + installJob = client.getInstallActiveJob(host.getName()); + try { + Thread.sleep(30000); + } catch (InterruptedException e) { + LOG.error("checkBackupAgent get install active job result sleep interrupted error"); + } } - sortedParams.add(param.toLowerCase() + "=" + value.toLowerCase()); - } - Collections.sort(sortedParams); - String sortedUrl = null; - first = true; - for (String param : sortedParams) { - if (first) { - sortedUrl = param; - first = false; + String checkHost = client.getClientId(host.getName()); + // 호스트가 클라이언트에 등록되지 않은 경우 + if (checkHost == null) { + String jobId = client.installAgent(host.getPrivateIpAddress(), commCellId, commServeHostName, credentials.first(), credentials.second()); + if (jobId != null) { + String jobStatus = client.getJobStatus(jobId); + if (!jobStatus.equalsIgnoreCase("Completed")) { + LOG.error("installing agent on the Commvault Backup Provider failed jogId : " + jobId + " , jobStatus : " + jobStatus); + ActionEventUtils.onActionEvent(User.UID_SYSTEM, Account.ACCOUNT_ID_SYSTEM, Domain.ROOT_DOMAIN, EventTypes.EVENT_HOST_AGENT_INSTALL, + "Failed install the commvault client agent on the host : " + host.getPrivateIpAddress(), User.UID_SYSTEM, ApiCommandResourceType.Host.toString()); + failResult.put(host.getPrivateIpAddress(), jobId); + } + } else { + return false; + } } else { - sortedUrl = sortedUrl + "&" + param; + // 호스트가 클라이언트에는 등록되었지만 구성이 정상적으로 되지 않은 경우 준비 상태 체크 + boolean checkInstall = client.getClientCheckReadiness(checkHost); + if (!checkInstall) { + LOG.error("The host is registered with the client, but the readiness status is not normal and you must manually check the client status."); + ActionEventUtils.onActionEvent(User.UID_SYSTEM, Account.ACCOUNT_ID_SYSTEM, Domain.ROOT_DOMAIN, EventTypes.EVENT_HOST_AGENT_INSTALL, + "Failed check readiness the commvault client agent on the host : " + host.getPrivateIpAddress(), User.UID_SYSTEM, ApiCommandResourceType.Host.toString()); + return false; + } } } - String encodedSignature = signRequest(sortedUrl, secretKey); - String finalUrl = region + "?" + apiParams + "&apiKey=" + apiKey + "&signature=" + encodedSignature; - return finalUrl; - } catch (UnsupportedEncodingException e) { - LOG.error(e.getMessage()); - return null; } - } - - private static String signRequest(String request, String key) { - try { - Mac mac = Mac.getInstance("HmacSHA256"); - SecretKeySpec keySpec = new SecretKeySpec(key.getBytes(), "HmacSHA256"); - mac.init(keySpec); - mac.update(request.getBytes()); - byte[] encryptedBytes = mac.doFinal(); - return URLEncoder.encode(Base64.encodeBase64String(encryptedBytes), "UTF-8"); - } catch (Exception ex) { - LOG.error(ex.getMessage()); - return null; + if (!failResult.isEmpty()) { + return false; } + return true; } - private int getAsyncJobResult(String moldUrl, String apiKey, String secretKey, String jobId) throws CloudRuntimeException { - int jobStatus = 0; - String moldCommand = "queryAsyncJobResult"; - String moldMethod = "GET"; - Map params = new HashMap<>(); - params.put("jobid", jobId); - while (jobStatus == 0) { - String result = moldQueryAsyncJobResultAPI(moldUrl, moldCommand, moldMethod, apiKey, secretKey, params); - if (result != null) { - jobStatus = Integer.parseInt(result); - try { - Thread.sleep(10000); - } catch (InterruptedException e) { - LOG.error("create snapshot get asyncjob result sleep interrupted error"); + @Override + public boolean importBackupPlan(final Long zoneId, final String retentionPeriod, final String externalId) { + final CommvaultClient client = getClient(zoneId); + // 선택한 백업 정책의 RPO 편집 Commvault API 호출 + String type = "deleteRpo"; + String taskId = client.getScheduleTaskId(type, externalId); + if (taskId != null) { + String subTaskId = client.getSubTaskId(taskId); + if (subTaskId != null) { + boolean result = client.deleteSchedulePolicy(taskId, subTaskId); + if (!result) { + throw new CloudRuntimeException("Failed to delete schedule policy commvault api"); } - } else { - throw new CloudRuntimeException("Failed to request queryAsyncJobResult Mold-API."); } + } else { + throw new CloudRuntimeException("Failed to get plan details schedule task id commvault api"); } - return jobStatus; - } - - private Optional getBackedUpVolumeInfo(List backedUpVolumes, String volumeUuid) { - return backedUpVolumes.stream() - .filter(v -> v.getUuid().equals(volumeUuid)) - .findFirst(); - } - - private String[] getServerProperties() { - String[] serverInfo = null; - final String HTTP_PORT = "http.port"; - final String HTTPS_ENABLE = "https.enable"; - final String HTTPS_PORT = "https.port"; - final File confFile = PropertiesUtil.findConfigFile("server.properties"); - try { - InputStream is = new FileInputStream(confFile); - String port = null; - String protocol = null; - final Properties properties = ServerProperties.getServerProperties(is); - if (properties.getProperty(HTTPS_ENABLE).equals("true")){ - port = properties.getProperty(HTTPS_PORT); - protocol = "https"; - } else { - port = properties.getProperty(HTTP_PORT); - protocol = "http"; - } - serverInfo = new String[]{port, protocol}; - } catch (final IOException e) { - LOG.debug("Failed to read configuration from server.properties file", e); + // 선택한 백업 정책의 보존 기간 변경 Commvault API 호출 + type = "updateRpo"; + String planEntity = client.getScheduleTaskId(type, externalId); + JSONObject jsonObject = new JSONObject(planEntity); + String planType = String.valueOf(jsonObject.get("planType")); + String planName = String.valueOf(jsonObject.get("planName")); + String planSubtype = String.valueOf(jsonObject.get("planSubtype")); + String planId = String.valueOf(jsonObject.get("planId")); + JSONObject entityInfo = jsonObject.getJSONObject("entityInfo"); + String companyId = String.valueOf(entityInfo.get("companyId")); + String storagePolicyId = client.getStoragePolicyId(planName); + if (storagePolicyId == null) { + throw new CloudRuntimeException("Failed to get plan storage policy id commvault api"); } - return serverInfo; - } - - private boolean executeTakeBackupCommand(HostVO host, String username, String password, int port, String command) { - try { - Pair response = SshHelper.sshExecute(host.getPrivateIpAddress(), port, - username, null, password, command, 120000, 120000, 3600000); - - if (!response.first()) { - LOG.error(String.format("take backup vm xml file failed on HYPERVISOR %s due to: %s", host, response.second())); - } else { - return true; + boolean result = client.getStoragePolicyDetails(planId, storagePolicyId, retentionPeriod); + if (result) { + // 호스트에 선택한 백업 정책 설정 Commvault API 호출 + String path = "/"; + List Hosts = hostDao.findByDataCenterId(zoneId); + for (final HostVO host : Hosts) { + String backupSetId = client.getDefaultBackupSetId(host.getName()); + if (backupSetId != null) { + if (!client.setBackupSet(path, planType, planName, planSubtype, planId, companyId, backupSetId)) { + throw new CloudRuntimeException("Failed to setting backup plan for client commvault api"); + } + } } - } catch (final Exception e) { - throw new CloudRuntimeException(String.format("Failed to take backup vm xml file on host %s due to: %s", host.getName(), e.getMessage())); + return true; + } else { + throw new CloudRuntimeException("Failed to edit plan schedule retention period commvault api"); } - return false; } - private boolean executeDeleteXmlCommand(HostVO host, String username, String password, int port, String command) { - try { - Pair response = SshHelper.sshExecute(host.getPrivateIpAddress(), port, - username, null, password, command, 120000, 120000, 3600000); - - if (!response.first()) { - LOG.error(String.format("Delete xml file failed on HYPERVISOR %s due to: %s", host, response.second())); - } else { - return true; - } - } catch (final Exception e) { - throw new CloudRuntimeException(String.format("Failed to delete xml file on host %s due to: %s", host.getName(), e.getMessage())); + @Override + public boolean updateBackupPlan(final Long zoneId, final String retentionPeriod, final String externalId) { + final CommvaultClient client = getClient(zoneId); + String type = "updateRpo"; + String planEntity = client.getScheduleTaskId(type, externalId); + JSONObject jsonObject = new JSONObject(planEntity); + String planType = String.valueOf(jsonObject.get("planType")); + String planName = String.valueOf(jsonObject.get("planName")); + String planSubtype = String.valueOf(jsonObject.get("planSubtype")); + String planId = String.valueOf(jsonObject.get("planId")); + JSONObject entityInfo = jsonObject.getJSONObject("entityInfo"); + String companyId = String.valueOf(entityInfo.get("companyId")); + String storagePolicyId = client.getStoragePolicyId(planName); + if (storagePolicyId == null) { + throw new CloudRuntimeException("Failed to get plan storage policy id commvault api"); } - return false; + return client.getStoragePolicyDetails(planId, storagePolicyId, retentionPeriod); } - private String executeDeviceCommand(HostVO host, String username, String password, int port, String command) { + private static String getUrlDomain(String url) throws URISyntaxException { + URI uri; try { - Pair response = SshHelper.sshExecute(host.getPrivateIpAddress(), port, - username, null, password, command, 120000, 120000, 3600000); - - if (!response.first()) { - LOG.error(String.format("get current device failed on HYPERVISOR %s due to: %s", host, response.second())); - } else { - return response.second(); - } - } catch (final Exception e) { - throw new CloudRuntimeException(String.format("Failed to get current device backup on host %s due to: %s", host.getName(), e.getMessage())); + uri = new URI(url); + } catch (URI.MalformedURIException e) { + throw new CloudRuntimeException("Failed to cast URI"); } - return null; + + return uri.getHost(); } - private boolean executeAttachCommand(HostVO host, String username, String password, int port, String command) { + private CommvaultClient getClient(final Long zoneId) { try { - Pair response = SshHelper.sshExecute(host.getPrivateIpAddress(), port, - username, null, password, command, 120000, 120000, 3600000); - - if (!response.first()) { - LOG.error(String.format("Attach voulme failed on HYPERVISOR %s due to: %s", host, response.second())); - } else { - return true; - } - } catch (final Exception e) { - throw new CloudRuntimeException(String.format("Failed to attach volume backup on host %s due to: %s", host.getName(), e.getMessage())); + return new CommvaultClient(CommvaultUrl.valueIn(zoneId), CommvaultUsername.valueIn(zoneId), CommvaultPassword.valueIn(zoneId), + CommvaultValidateSSLSecurity.valueIn(zoneId), CommvaultApiRequestTimeout.valueIn(zoneId)); + } catch (URISyntaxException e) { + throw new CloudRuntimeException("Failed to parse Commvault API URL: " + e.getMessage()); + } catch (NoSuchAlgorithmException | KeyManagementException e) { + LOG.error("Failed to build Commvault API client due to: ", e); } - return false; + throw new CloudRuntimeException("Failed to build Commvault API client"); } - private boolean executeRestoreCommand(HostVO host, String username, String password, int port, String command) { - try { - Pair response = SshHelper.sshExecute(host.getPrivateIpAddress(), port, - username, null, password, command, 120000, 120000, 3600000); + protected Ternary getKVMHyperisorCredentials(HostVO host) { - if (!response.first()) { - LOG.error(String.format("Restore failed on HYPERVISOR %s due to: %s", host, response.second())); - } else { - return true; - } - } catch (final Exception e) { - throw new CloudRuntimeException(String.format("Failed to restore backup on host %s due to: %s", host.getName(), e.getMessage())); + String username = null; + String password = null; + + if (host != null && host.getHypervisorType() == Hypervisor.HypervisorType.KVM) { + hostDao.loadDetails(host); + password = host.getDetail("password"); + username = host.getDetail("username"); } - return false; + if ( password == null || username == null) { + throw new CloudRuntimeException("Cannot find login credentials for HYPERVISOR " + Objects.requireNonNull(host).getUuid()); + } + + return new Ternary<>(username, password, null); } - private boolean executeDeleteSnapshotCommand(HostVO host, String username, String password, int port, String command) { + private boolean executeDeleteBackupPathCommand(HostVO host, String username, String password, int port, String command) { try { Pair response = SshHelper.sshExecute(host.getPrivateIpAddress(), port, username, null, password, command, 120000, 120000, 3600000); if (!response.first()) { - LOG.error(String.format("Restore failed on HYPERVISOR %s due to: %s", host, response.second())); + LOG.error(String.format("failed on HYPERVISOR %s due to: %s", host, response.second())); } else { return true; } } catch (final Exception e) { - throw new CloudRuntimeException(String.format("Failed to restore backup on host %s due to: %s", host.getName(), e.getMessage())); + throw new CloudRuntimeException(String.format("Failed to delete backup path on host %s due to: %s", host.getName(), e.getMessage())); } return false; } @@ -1541,48 +1171,4 @@ public static boolean versionCheck(String csVersionInfo) { return true; } - @Override - public boolean supportsInstanceFromBackup() { - return false; - } - - @Override - public Pair getBackupStorageStats(Long zoneId) { - return new Pair<>(0L, 0L); - } - - @Override - public void syncBackupStorageStats(Long zoneId) { - } - - // 하위 클라이언트 삭제 시 백업본 데이터는 그대로 남아있지만, 해당 하위 클라이언트가 삭제되었기 때문에 스케줄도 삭제시켜야하며 - // 남아있는 백업본 데이터는 mold에서 관리하지 않고, commvault 의 plan 보존기간에 따라 데이터 에이징 됨. - @Override - public boolean willDeleteBackupsOnOfferingRemoval() { - return true; - } - - @Override - public Pair restoreBackupToVM(VirtualMachine vm, Backup backup, String hostIp, String dataStoreUuid) { - return new Pair<>(true, null); - } - - @Override - public List listRestorePoints(VirtualMachine vm) { - return null; - } - - @Override - public Backup createNewBackupEntryForRestorePoint(Backup.RestorePoint restorePoint, VirtualMachine vm) { - return null; - } - - public void syncBackupMetrics(Long zoneId) { - } - - @Override - public Boolean crossZoneInstanceCreationEnabled(BackupOffering backupOffering) { - return false; - } - } \ No newline at end of file diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java index d8313d7c4ad1..857446c57187 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java @@ -426,6 +426,7 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv private String vmActivityCheckPathRbd; private String vmActivityCheckPathClvm; private String nasBackupPath; + private String cvtBackupPath; private String securityGroupPath; private String ovsPvlanDhcpHostPath; private String ovsPvlanVmPath; @@ -856,6 +857,10 @@ public String getNasBackupPath() { return nasBackupPath; } + public String getCvtBackupPath() { + return cvtBackupPath; + } + public String getOvsPvlanDhcpHostPath() { return ovsPvlanDhcpHostPath; } @@ -1193,6 +1198,11 @@ public boolean configure(final String name, final Map params) th throw new ConfigurationException("Unable to find nasbackup.sh"); } + cvtBackupPath = Script.findScript(kvmScriptsDir, "cvtbackup.sh"); + if (cvtBackupPath == null) { + throw new ConfigurationException("Unable to find cvtbackup.sh"); + } + createTmplPath = Script.findScript(storageScriptsDir, "createtmplt.sh"); if (createTmplPath == null) { throw new ConfigurationException("Unable to find the createtmplt.sh"); diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCommvaultRestoreBackupCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCommvaultRestoreBackupCommandWrapper.java new file mode 100644 index 000000000000..5ffce02fd138 --- /dev/null +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCommvaultRestoreBackupCommandWrapper.java @@ -0,0 +1,319 @@ +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// + +package com.cloud.hypervisor.kvm.resource.wrapper; + +import com.cloud.agent.api.Answer; +import com.cloud.hypervisor.kvm.resource.LibvirtComputingResource; +import com.cloud.hypervisor.kvm.storage.KVMPhysicalDisk; +import com.cloud.hypervisor.kvm.storage.KVMStoragePool; +import com.cloud.hypervisor.kvm.storage.KVMStoragePoolManager; +import com.cloud.resource.CommandWrapper; +import com.cloud.resource.ResourceWrapper; +import com.cloud.storage.Storage; +import com.cloud.utils.Pair; +import com.cloud.utils.exception.CloudRuntimeException; +import com.cloud.utils.script.Script; +import com.cloud.vm.VirtualMachine; +import org.apache.cloudstack.backup.BackupAnswer; +import org.apache.cloudstack.backup.CommvaultRestoreBackupCommand; +import org.apache.cloudstack.storage.to.PrimaryDataStoreTO; +import org.apache.cloudstack.utils.qemu.QemuImg; +import org.apache.cloudstack.utils.qemu.QemuImgException; +import org.apache.cloudstack.utils.qemu.QemuImgFile; +import org.apache.commons.io.FileUtils; +import org.apache.commons.lang3.StringUtils; +import org.libvirt.LibvirtException; + +import java.io.File; +import java.io.IOException; +import java.util.List; +import java.util.Locale; +import java.util.Objects; + +@ResourceWrapper(handles = CommvaultRestoreBackupCommand.class) +public class LibvirtCommvaultRestoreBackupCommandWrapper extends CommandWrapper { + private static final String FILE_PATH_PLACEHOLDER = "%s/%s"; + private static final String ATTACH_QCOW2_DISK_COMMAND = " virsh attach-disk %s %s %s --driver qemu --subdriver qcow2 --cache none"; + private static final String ATTACH_RBD_DISK_XML_COMMAND = " virsh attach-device %s /dev/stdin < backedVolumeUUIDs = command.getBackupVolumesUUIDs(); + List restoreVolumePools = command.getRestoreVolumePools(); + List restoreVolumePaths = command.getRestoreVolumePaths(); + String restoreVolumeUuid = command.getRestoreVolumeUUID(); + int timeout = command.getWait(); + String cacheMode = command.getCacheMode(); + String hostName = command.getHostName(); + KVMStoragePoolManager storagePoolMgr = serverResource.getStoragePoolMgr(); + + String newVolumeId = null; + try { + if (hostName != null) { + fetchBackupFile(hostName, backupPath); + } + if (Objects.isNull(vmExists)) { + PrimaryDataStoreTO volumePool = restoreVolumePools.get(0); + String volumePath = restoreVolumePaths.get(0); + int lastIndex = volumePath.lastIndexOf("/"); + newVolumeId = volumePath.substring(lastIndex + 1); + restoreVolume(storagePoolMgr, backupPath, volumePool, volumePath, diskType, restoreVolumeUuid, + new Pair<>(vmName, command.getVmState()), timeout, cacheMode); + } else if (Boolean.TRUE.equals(vmExists)) { + restoreVolumesOfExistingVM(storagePoolMgr, restoreVolumePools, restoreVolumePaths, backedVolumeUUIDs, backupPath, timeout); + } else { + restoreVolumesOfDestroyedVMs(storagePoolMgr, restoreVolumePools, restoreVolumePaths, vmName, backupPath, timeout); + } + } catch (CloudRuntimeException e) { + String errorMessage = e.getMessage() != null ? e.getMessage() : ""; + return new BackupAnswer(command, false, errorMessage); + } + + return new BackupAnswer(command, true, newVolumeId); + } + + private void verifyBackupFile(String backupPath, String volUuid) { + if (!checkBackupPathExists(backupPath)) { + throw new CloudRuntimeException(String.format("Backup file for the volume [%s] does not exist.", volUuid)); + } + if (!checkBackupFileImage(backupPath)) { + throw new CloudRuntimeException(String.format("Backup qcow2 file for the volume [%s] is corrupt.", volUuid)); + } + } + + private void restoreVolumesOfExistingVM(KVMStoragePoolManager storagePoolMgr, List restoreVolumePools, List restoreVolumePaths, List backedVolumesUUIDs, + String backupPath, int timeout) { + String diskType = "root"; + try { + for (int idx = 0; idx < restoreVolumePaths.size(); idx++) { + PrimaryDataStoreTO restoreVolumePool = restoreVolumePools.get(idx); + String restoreVolumePath = restoreVolumePaths.get(idx); + String backupVolumeUuid = backedVolumesUUIDs.get(idx); + Pair bkpPathAndVolUuid = getBackupPath(null, backupPath, diskType, backupVolumeUuid); + diskType = "datadisk"; + verifyBackupFile(bkpPathAndVolUuid.first(), bkpPathAndVolUuid.second()); + if (!replaceVolumeWithBackup(storagePoolMgr, restoreVolumePool, restoreVolumePath, bkpPathAndVolUuid.first(), timeout)) { + throw new CloudRuntimeException(String.format("Unable to restore contents from the backup volume [%s].", bkpPathAndVolUuid.second())); + } + } + } finally { + deleteBackupDirectory(backupPath); + } + } + + private void restoreVolumesOfDestroyedVMs(KVMStoragePoolManager storagePoolMgr, List volumePools, List volumePaths, String vmName, String backupPath, int timeout) { + String diskType = "root"; + try { + for (int i = 0; i < volumePaths.size(); i++) { + PrimaryDataStoreTO volumePool = volumePools.get(i); + String volumePath = volumePaths.get(i); + Pair bkpPathAndVolUuid = getBackupPath(volumePath, backupPath, diskType, null); + diskType = "datadisk"; + verifyBackupFile(bkpPathAndVolUuid.first(), bkpPathAndVolUuid.second()); + if (!replaceVolumeWithBackup(storagePoolMgr, volumePool, volumePath, bkpPathAndVolUuid.first(), timeout)) { + throw new CloudRuntimeException(String.format("Unable to restore contents from the backup volume [%s].", bkpPathAndVolUuid.second())); + } + } + } finally { + deleteBackupDirectory(backupPath); + } + } + + private void restoreVolume(KVMStoragePoolManager storagePoolMgr, String backupPath, PrimaryDataStoreTO volumePool, String volumePath, String diskType, String volumeUUID, + Pair vmNameAndState, int timeout, String cacheMode) { + Pair bkpPathAndVolUuid; + try { + bkpPathAndVolUuid = getBackupPath(volumePath, backupPath, diskType, volumeUUID); + verifyBackupFile(bkpPathAndVolUuid.first(), bkpPathAndVolUuid.second()); + if (!replaceVolumeWithBackup(storagePoolMgr, volumePool, volumePath, bkpPathAndVolUuid.first(), timeout, true)) { + throw new CloudRuntimeException(String.format("Unable to restore contents from the backup volume [%s].", bkpPathAndVolUuid.second())); + } + if (VirtualMachine.State.Running.equals(vmNameAndState.second())) { + if (!attachVolumeToVm(storagePoolMgr, vmNameAndState.first(), volumePool, volumePath, cacheMode)) { + throw new CloudRuntimeException(String.format("Failed to attach volume to VM: %s", vmNameAndState.first())); + } + } + } finally { + deleteBackupDirectory(backupPath); + } + } + + private void deleteBackupDirectory(String backupDirectory) { + try { + FileUtils.deleteDirectory(new File(backupDirectory)); + } catch (IOException e) { + logger.error(String.format("Failed to delete backup directory: %s", backupDirectory), e); + throw new CloudRuntimeException("Failed to delete the backup directory"); + } + } + + private Pair getBackupPath(String volumePath, String backupPath, String diskType, String volumeUuid) { + String volUuid = Objects.isNull(volumeUuid) ? volumePath.substring(volumePath.lastIndexOf(File.separator) + 1) : volumeUuid; + String backupFileName = String.format("%s.%s.qcow2", diskType.toLowerCase(Locale.ROOT), volUuid); + backupPath = String.format(FILE_PATH_PLACEHOLDER, backupPath, backupFileName); + return new Pair<>(backupPath, volUuid); + } + + private boolean checkBackupFileImage(String backupPath) { + int exitValue = Script.runSimpleBashScriptForExitValue(String.format("qemu-img check %s", backupPath)); + return exitValue == 0; + } + + private boolean checkBackupPathExists(String backupPath) { + int exitValue = Script.runSimpleBashScriptForExitValue(String.format("ls %s", backupPath)); + return exitValue == 0; + } + + private boolean replaceVolumeWithBackup(KVMStoragePoolManager storagePoolMgr, PrimaryDataStoreTO volumePool, String volumePath, String backupPath, int timeout) { + return replaceVolumeWithBackup(storagePoolMgr, volumePool, volumePath, backupPath, timeout, false); + } + + private boolean replaceVolumeWithBackup(KVMStoragePoolManager storagePoolMgr, PrimaryDataStoreTO volumePool, String volumePath, String backupPath, int timeout, boolean createTargetVolume) { + if (volumePool.getPoolType() != Storage.StoragePoolType.RBD) { + int exitValue = Script.runSimpleBashScriptForExitValue(String.format(RSYNC_COMMAND, backupPath, volumePath)); + return exitValue == 0; + } + + return replaceRbdVolumeWithBackup(storagePoolMgr, volumePool, volumePath, backupPath, timeout, createTargetVolume); + } + + private boolean replaceRbdVolumeWithBackup(KVMStoragePoolManager storagePoolMgr, PrimaryDataStoreTO volumePool, String volumePath, String backupPath, int timeout, boolean createTargetVolume) { + KVMStoragePool volumeStoragePool = storagePoolMgr.getStoragePool(volumePool.getPoolType(), volumePool.getUuid()); + QemuImg qemu; + try { + qemu = new QemuImg(timeout * 1000, true, false); + if (!createTargetVolume) { + KVMPhysicalDisk rdbDisk = volumeStoragePool.getPhysicalDisk(volumePath); + logger.debug("Restoring RBD volume: {}", rdbDisk.toString()); + qemu.setSkipTargetVolumeCreation(true); + } + } catch (LibvirtException ex) { + throw new CloudRuntimeException("Failed to create qemu-img command to restore RBD volume with backup", ex); + } + + QemuImgFile srcBackupFile = null; + QemuImgFile destVolumeFile = null; + try { + srcBackupFile = new QemuImgFile(backupPath, QemuImg.PhysicalDiskFormat.QCOW2); + String rbdDestVolumeFile = KVMPhysicalDisk.RBDStringBuilder(volumeStoragePool, volumePath); + destVolumeFile = new QemuImgFile(rbdDestVolumeFile, QemuImg.PhysicalDiskFormat.RAW); + + logger.debug("Starting convert backup {} to RBD volume {}", backupPath, volumePath); + qemu.convert(srcBackupFile, destVolumeFile); + logger.debug("Successfully converted backup {} to RBD volume {}", backupPath, volumePath); + } catch (QemuImgException | LibvirtException e) { + String srcFilename = srcBackupFile != null ? srcBackupFile.getFileName() : null; + String destFilename = destVolumeFile != null ? destVolumeFile.getFileName() : null; + logger.error("Failed to convert backup {} to volume {}, the error was: {}", srcFilename, destFilename, e.getMessage()); + return false; + } + + return true; + } + + private boolean attachVolumeToVm(KVMStoragePoolManager storagePoolMgr, String vmName, PrimaryDataStoreTO volumePool, String volumePath, String cacheMode) { + String deviceToAttachDiskTo = getDeviceToAttachDisk(vmName); + int exitValue; + if (volumePool.getPoolType() != Storage.StoragePoolType.RBD) { + exitValue = Script.runSimpleBashScriptForExitValue(String.format(ATTACH_QCOW2_DISK_COMMAND, vmName, volumePath, deviceToAttachDiskTo)); + } else { + String xmlForRbdDisk = getXmlForRbdDisk(storagePoolMgr, volumePool, volumePath, deviceToAttachDiskTo, cacheMode); + logger.debug("RBD disk xml to attach: {}", xmlForRbdDisk); + exitValue = Script.runSimpleBashScriptForExitValue(String.format(ATTACH_RBD_DISK_XML_COMMAND, vmName, xmlForRbdDisk)); + } + return exitValue == 0; + } + + private String getDeviceToAttachDisk(String vmName) { + String currentDevice = Script.runSimpleBashScript(String.format(CURRRENT_DEVICE, vmName)); + char lastChar = currentDevice.charAt(currentDevice.length() - 1); + char incrementedChar = (char) (lastChar + 1); + return currentDevice.substring(0, currentDevice.length() - 1) + incrementedChar; + } + + private String getXmlForRbdDisk(KVMStoragePoolManager storagePoolMgr, PrimaryDataStoreTO volumePool, String volumePath, String deviceToAttachDiskTo, String cacheMode) { + StringBuilder diskBuilder = new StringBuilder(); + diskBuilder.append("\n\n"); + + diskBuilder.append(" \n"); + + diskBuilder.append("\n"); + for (String sourceHost : volumePool.getHost().split(",")) { + diskBuilder.append("\n"); + } + diskBuilder.append("\n"); + String authUserName = null; + final KVMStoragePool primaryPool = storagePoolMgr.getStoragePool(volumePool.getPoolType(), volumePool.getUuid()); + if (primaryPool != null) { + authUserName = primaryPool.getAuthUserName(); + } + if (StringUtils.isNotBlank(authUserName)) { + diskBuilder.append("\n"); + diskBuilder.append("\n"); + diskBuilder.append("\n"); + } + diskBuilder.append("\n"); + diskBuilder.append("\n"); + return diskBuilder.toString(); + } + + private void fetchBackupFile(String hostName, String backupPath) { + int mkdirExit = Script.runSimpleBashScriptForExitValue(String.format(MKDIR_P, backupPath)); + if (mkdirExit != 0) { + throw new CloudRuntimeException(String.format("Failed to create local backup directory: %s", backupPath)); + } + + String cmd = String.format(RSYNC_DIR_FROM_REMOTE, hostName, backupPath, backupPath); + logger.debug("Fetching commvault backup directory from remote host. cmd={}", cmd); + + int exit = Script.runSimpleBashScriptForExitValue(cmd); + if (exit != 0) { + throw new CloudRuntimeException(String.format( + "Failed to fetch backup directory from remote host [%s]. remotePath=[%s], localPath=[%s]", + hostName, backupPath, backupPath)); + } + } +} \ No newline at end of file diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCommvaultTakeBackupCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCommvaultTakeBackupCommandWrapper.java new file mode 100644 index 000000000000..277d38e8573d --- /dev/null +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCommvaultTakeBackupCommandWrapper.java @@ -0,0 +1,91 @@ +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// + +package com.cloud.hypervisor.kvm.resource.wrapper; + +import com.cloud.agent.api.Answer; +import com.cloud.hypervisor.kvm.resource.LibvirtComputingResource; +import com.cloud.hypervisor.kvm.storage.KVMPhysicalDisk; +import com.cloud.hypervisor.kvm.storage.KVMStoragePool; +import com.cloud.hypervisor.kvm.storage.KVMStoragePoolManager; +import com.cloud.resource.CommandWrapper; +import com.cloud.resource.ResourceWrapper; +import com.cloud.storage.Storage; +import com.cloud.utils.Pair; +import com.cloud.utils.script.Script; +import org.apache.cloudstack.backup.BackupAnswer; +import org.apache.cloudstack.backup.CommvaultTakeBackupCommand; +import org.apache.cloudstack.storage.to.PrimaryDataStoreTO; + +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; + +@ResourceWrapper(handles = CommvaultTakeBackupCommand.class) +public class LibvirtCommvaultTakeBackupCommandWrapper extends CommandWrapper { + private static final Integer EXIT_CLEANUP_FAILED = 20; + @Override + public Answer execute(CommvaultTakeBackupCommand command, LibvirtComputingResource libvirtComputingResource) { + final String vmName = command.getVmName(); + final String backupPath = command.getBackupPath(); + List volumePools = command.getVolumePools(); + final List volumePaths = command.getVolumePaths(); + KVMStoragePoolManager storagePoolMgr = libvirtComputingResource.getStoragePoolMgr(); + + List diskPaths = new ArrayList<>(); + if (Objects.nonNull(volumePaths)) { + for (int idx = 0; idx < volumePaths.size(); idx++) { + PrimaryDataStoreTO volumePool = volumePools.get(idx); + String volumePath = volumePaths.get(idx); + if (volumePool.getPoolType() != Storage.StoragePoolType.RBD) { + diskPaths.add(volumePath); + } else { + KVMStoragePool volumeStoragePool = storagePoolMgr.getStoragePool(volumePool.getPoolType(), volumePool.getUuid()); + String rbdDestVolumeFile = KVMPhysicalDisk.RBDStringBuilder(volumeStoragePool, volumePath); + diskPaths.add(rbdDestVolumeFile); + } + } + } + + List commands = new ArrayList<>(); + commands.add(new String[]{ + libvirtComputingResource.getCvtBackupPath(), + "-o", "backup", + "-v", vmName, + "-p", backupPath, + "-q", command.getQuiesce() != null && command.getQuiesce() ? "true" : "false", + "-d", diskPaths.isEmpty() ? "" : String.join(",", diskPaths) + }); + + Pair result = Script.executePipedCommands(commands, libvirtComputingResource.getCmdsTimeout()); + + if (result.first() != 0) { + logger.debug("Failed to take VM backup"); + BackupAnswer answer = new BackupAnswer(command, false, null); + if (result.first() == EXIT_CLEANUP_FAILED) { + logger.debug("Backup cleanup failed"); + answer.setNeedsCleanup(true); + } + return answer; + } + + BackupAnswer answer = new BackupAnswer(command, true, "success"); + return answer; + } +} diff --git a/scripts/vm/hypervisor/kvm/cvtbackup.sh b/scripts/vm/hypervisor/kvm/cvtbackup.sh new file mode 100644 index 000000000000..0493654fce02 --- /dev/null +++ b/scripts/vm/hypervisor/kvm/cvtbackup.sh @@ -0,0 +1,255 @@ +#!/usr/bin/bash + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +set -eo pipefail + +# CloudStack B&R Commvault Backup and Recovery Tool for KVM + +# TODO: do libvirt/logging etc checks + +### Declare variables ### + +OP="" +VM="" +BACKUP_DIR="" +DISK_PATHS="" +QUIESCE="" +logFile="/var/log/cloudstack/agent/agent.log" + +EXIT_CLEANUP_FAILED=20 + +log() { + [[ "$verb" -eq 1 ]] && builtin echo "$@" + if [[ "$1" == "-ne" || "$1" == "-e" || "$1" == "-n" ]]; then + builtin echo -e "$(date '+%Y-%m-%d %H-%M-%S>')" "${@: 2}" >> "$logFile" + else + builtin echo "$(date '+%Y-%m-%d %H-%M-%S>')" "$@" >> "$logFile" + fi +} + +vercomp() { + local IFS=. + local i ver1=($1) ver2=($3) + + # Compare each segment of the version numbers + for ((i=0; i<${#ver1[@]}; i++)); do + if [[ -z ${ver2[i]} ]]; then + ver2[i]=0 + fi + + if ((10#${ver1[i]} > 10#${ver2[i]})); then + return 0 # Version 1 is greater + elif ((10#${ver1[i]} < 10#${ver2[i]})); then + return 2 # Version 2 is greater + fi + done + return 0 # Versions are equal +} + +sanity_checks() { + hvVersion=$(virsh version | grep hypervisor | awk '{print $(NF)}') + libvVersion=$(virsh version | grep libvirt | awk '{print $(NF)}' | tail -n 1) + apiVersion=$(virsh version | grep API | awk '{print $(NF)}') + + # Compare qemu version (hvVersion >= 4.2.0) + vercomp "$hvVersion" ">=" "4.2.0" + hvStatus=$? + + # Compare libvirt version (libvVersion >= 7.2.0) + vercomp "$libvVersion" ">=" "7.2.0" + libvStatus=$? + + if [[ $hvStatus -eq 0 && $libvStatus -eq 0 ]]; then + log -ne "Success... [ QEMU: $hvVersion Libvirt: $libvVersion apiVersion: $apiVersion ]" + else + echo "Failure... Your QEMU version $hvVersion or libvirt version $libvVersion is unsupported. Consider upgrading to the required minimum version of QEMU: 4.2.0 and Libvirt: 7.2.0" + exit 1 + fi + + log -ne "Environment Sanity Checks successfully passed" +} + +### Operation methods ### + +backup_running_vm() { + mkdir -p "$dest" || { echo "Failed to create backup directory $dest"; exit 1; } + + name="root" + echo "" > $dest/backup.xml + for disk in $(virsh -c qemu:///system domblklist $VM --details 2>/dev/null | awk '/disk/{print$3}'); do + volpath=$(virsh -c qemu:///system domblklist $VM --details | awk "/$disk/{print $4}" | sed 's/.*\///') + echo "" >> $dest/backup.xml + name="datadisk" + done + echo "" >> $dest/backup.xml + + local thaw=0 + if [[ ${QUIESCE} == "true" ]]; then + log -ne "Pause option is enabled on a running virtual machine" + if virsh -c qemu:///system qemu-agent-command "$VM" '{"execute":"guest-fsfreeze-freeze"}' > /dev/null 2>/dev/null; then + thaw=1 + fi + fi + + # Start push backup + local backup_begin=0 + if virsh -c qemu:///system backup-begin --domain $VM --backupxml $dest/backup.xml 2>&1 > /dev/null; then + backup_begin=1; + fi + + if [[ $thaw -eq 1 ]]; then + if ! response=$(virsh -c qemu:///system qemu-agent-command "$VM" '{"execute":"guest-fsfreeze-thaw"}' 2>&1 > /dev/null); then + echo "Failed to thaw the filesystem for vm $VM: $response" + cleanup + exit 1 + fi + fi + + if [[ $backup_begin -ne 1 ]]; then + cleanup + exit 1 + fi + + # Backup domain information + virsh -c qemu:///system dumpxml $VM > $dest/domain-config.xml 2>/dev/null + virsh -c qemu:///system dominfo $VM > $dest/dominfo.xml 2>/dev/null + virsh -c qemu:///system domiflist $VM > $dest/domiflist.xml 2>/dev/null + virsh -c qemu:///system domblklist $VM > $dest/domblklist.xml 2>/dev/null + + while true; do + status=$(virsh -c qemu:///system domjobinfo $VM --completed --keep-completed | awk '/Job type:/ {print $3}') + case "$status" in + Completed) + break ;; + Failed) + echo "Virsh backup job failed" + cleanup ;; + esac + sleep 5 + done + sync + +} + +backup_stopped_vm() { + mkdir -p "$dest" || { echo "Failed to create backup directory $dest"; exit 1; } + + IFS="," + + name="root" + for disk in $DISK_PATHS; do + if [[ "$disk" == rbd:* ]]; then + # disk for rbd => rbd:/:mon_host=... + # sample: rbd:cloudstack/53d5c355-d726-4d3e-9422-046a503a0b12:mon_host=10.0.1.2... + beforeUuid="${disk#*/}" # Remove up to first slash after rbd: + volUuid="${beforeUuid%%:*}" # Remove everything after colon to get the uuid + else + volUuid="${disk##*/}" + fi + output="$dest/$name.$volUuid.qcow2" + if ! qemu-img convert -O qcow2 "$disk" "$output" > "$logFile" 2> >(cat >&2); then + echo "qemu-img convert failed for $disk $output" + cleanup + fi + name="datadisk" + done + sync + +} + +cleanup() { + local status=0 + + rm -rf "$dest" || { echo "Failed to delete $dest"; status=1; } + + if [[ $status -ne 0 ]]; then + echo "Backup cleanup failed" + exit $EXIT_CLEANUP_FAILED + fi +} + +function usage { + echo "" + echo "Usage: $0 -o -v|--vm -p -d -q|--quiesce " + echo "" + exit 1 +} + +while [[ $# -gt 0 ]]; do + case $1 in + -o|--operation) + OP="$2" + shift + shift + ;; + -v|--vm) + VM="$2" + shift + shift + ;; + -p|--path) + BACKUP_DIR="$2" + shift + shift + ;; + -q|--quiesce) + QUIESCE="$2" + shift + shift + ;; + -d|--diskpaths) + DISK_PATHS="$2" + shift + shift + ;; + -h|--help) + usage + shift + ;; + *) + echo "Invalid option: $1" + usage + ;; + esac +done + +if [[ -z "$BACKUP_DIR" ]]; then + echo "Backup path (-p|--path) is required" + exit 1 +fi + +dest="$BACKUP_DIR" + +# Perform Initial sanity checks +sanity_checks + +if [[ "$OP" != "backup" ]]; then + echo "Unsupported operation: $OP" + exit 1 +fi + +STATE=$(virsh -c qemu:///system list | awk -v vm="$VM" '$2 == vm {print $3}') + +if [[ -n "$STATE" && "$STATE" == "running" ]]; then + backup_running_vm +else + backup_stopped_vm +fi + +exit 0 diff --git a/server/src/main/java/org/apache/cloudstack/backup/BackupManagerImpl.java b/server/src/main/java/org/apache/cloudstack/backup/BackupManagerImpl.java index 80809c1d64c5..5de15017be78 100644 --- a/server/src/main/java/org/apache/cloudstack/backup/BackupManagerImpl.java +++ b/server/src/main/java/org/apache/cloudstack/backup/BackupManagerImpl.java @@ -128,8 +128,6 @@ import com.cloud.storage.GuestOSVO; import com.cloud.storage.ScopeType; import com.cloud.storage.Storage; -import com.cloud.storage.StoragePoolStatus; -import com.cloud.storage.Storage.StoragePoolType; import com.cloud.storage.Volume; import com.cloud.storage.VolumeApiService; import com.cloud.storage.VolumeVO; @@ -331,20 +329,6 @@ public BackupOffering importBackupOffering(final ImportBackupOfferingCmd cmd) { } final BackupProvider provider = getBackupProvider(providerName); - if ("commvault".equals(providerName)) { - List pools = primaryDataStoreDao.listByDataCenterId(cmd.getZoneId()); - boolean validPool = false; - for (StoragePoolVO pool : pools) { - if (pool.getStatus() == StoragePoolStatus.Up && pool.getPoolType() == StoragePoolType.SharedMountPoint) { - validPool = true; - break; - } - } - if (!validPool) { - throw new CloudRuntimeException("The backup offering cannot be imported because storage of type SharedMountPoint with storage status Up does not exist."); - } - } - if (!provider.isValidProviderOffering(cmd.getZoneId(), cmd.getExternalId())) { throw new CloudRuntimeException("Backup offering '" + cmd.getExternalId() + "' does not exist on provider " + provider.getName() + " on zone " + cmd.getZoneId()); } @@ -711,8 +695,8 @@ public BackupSchedule configureBackupSchedule(CreateBackupScheduleCmd cmd) { final int maxBackups = validateAndGetDefaultBackupRetentionIfRequired(cmd.getMaxBackups(), offering, vm); - if (!"nas".equals(offering.getProvider()) && cmd.getQuiesceVM() != null) { - throw new InvalidParameterValueException("Quiesce VM option is supported only for NAS backup provider"); + if ((!"nas".equals(offering.getProvider()) && !"commvault".equals(offering.getProvider())) && cmd.getQuiesceVM() != null) { + throw new InvalidParameterValueException("Quiesce VM option is supported only for NAS, Commvault backup provider"); } final String timezoneId = timeZone.getID(); @@ -913,8 +897,8 @@ public boolean createBackup(CreateBackupCmd cmd, Object job) throws ResourceAllo throw new CloudRuntimeException("The assigned backup offering does not allow ad-hoc user backup"); } - if (!"nas".equals(offering.getProvider()) && cmd.getQuiesceVM() != null) { - throw new InvalidParameterValueException("Quiesce VM option is supported only for NAS backup provider"); + if ((!"nas".equals(offering.getProvider()) && !"commvault".equals(offering.getProvider())) && cmd.getQuiesceVM() != null) { + throw new InvalidParameterValueException("Quiesce VM option is supported only for NAS, Commvault backup provider"); } Long backupScheduleId = getBackupScheduleId(job); @@ -1513,7 +1497,7 @@ public boolean restoreBackupToVM(final Long backupId, final Long vmId) throws Cl String host = null; String dataStore = null; - if (!"nas".equals(offering.getProvider())) { + if (!"nas".equals(offering.getProvider()) && !"commvault".equals(offering.getProvider())) { Pair restoreInfo = getRestoreVolumeHostAndDatastore(vm); host = restoreInfo.first().getPrivateIpAddress(); dataStore = restoreInfo.second().getUuid(); @@ -1591,7 +1575,7 @@ public boolean restoreBackupVolumeAndAttachToVM(final String backedUpVolumeUuid, BackupProvider backupProvider = getBackupProvider(offering.getProvider()); VolumeVO backedUpVolume = volumeDao.findByUuid(backedUpVolumeUuid); Pair restoreInfo; - if (!"nas".equals(offering.getProvider()) || (backedUpVolume == null)) { + if ((!"nas".equals(offering.getProvider()) && !"commvault".equals(offering.getProvider())) || backedUpVolume == null) { restoreInfo = getRestoreVolumeHostAndDatastore(vm); } else { restoreInfo = getRestoreVolumeHostAndDatastoreForNas(vm, backedUpVolume); diff --git a/ui/src/utils/plugins.js b/ui/src/utils/plugins.js index 1a1b7aefe6e2..2c7c79b954e2 100644 --- a/ui/src/utils/plugins.js +++ b/ui/src/utils/plugins.js @@ -666,7 +666,7 @@ export const backupUtilPlugin = { if (!provider && typeof provider !== 'string') { return false } - return ['nas'].includes(provider.toLowerCase()) + return ['nas', 'commvault'].includes(provider.toLowerCase()) } } } diff --git a/ui/src/views/compute/DeployVM.vue b/ui/src/views/compute/DeployVM.vue index b3e2dd922b9b..3395605d0f51 100644 --- a/ui/src/views/compute/DeployVM.vue +++ b/ui/src/views/compute/DeployVM.vue @@ -2667,6 +2667,7 @@ export default { duration: 0 }) } + this.performPostDeployBackupActions(vm) if (!values.stayonpage) { // eventBus.emit('vm-refresh-data') } diff --git a/ui/src/views/compute/StartBackup.vue b/ui/src/views/compute/StartBackup.vue index de2680d14476..96c337ab0bd0 100644 --- a/ui/src/views/compute/StartBackup.vue +++ b/ui/src/views/compute/StartBackup.vue @@ -40,7 +40,7 @@ - +