CLOUDSTACK-10146: Bypass Secondary Storage for KVM templates (#2379)

This feature allows using templates and ISOs avoiding secondary storage as intermediate cache on KVM. The virtual machine deployment process is enhanced to supported bypassed registered templates and ISOs, delegating the work of downloading them to primary storage to the KVM agent instead of the SSVM agent.

Template and ISO registration:
- When hypervisor is KVM, a checkbox is displayed with 'Direct Download' label.
- API methods registerTemplate and registerISO are both extended with this new parameter directdownload.
- On template or ISO registration, no download job is sent to SSVM agent, CloudStack would only persist an entry on template_store_ref indicating that template or ISO has been marked as 'Direct Download' (bypassing Secondary Storage). These entries are persisted as:
template_id = Template or ISO id on vm_template table
store_id NULL
download_state = BYPASSED
state = Ready
(Note: these entries allow users to deploy virtual machine from registered templates or ISOs)
- An URL validation command is sent to a random KVM host to check if template/ISO location can be reached. Metalink are also supported by this feature. In case of a metalink, it is fetched and URL check is performed on each of its URLs.
- Checksum should be provided as indicated on #2246: {ALGORITHM}CHKSUMHASH
- After template or ISO is registered, it would be displayed in the UI

Virtual machine deployment:
When a 'Direct Download' template is selected for deployment, CloudStack would delegate template downloading to destination storage pool via destination host by a new pluggable download manager.
Download manager would handle template downloading depending on URL protocol. In case of HTTP, request headers can be set by the user via vm_template_details. Those details should be persisted as:
Key: HTTP_HEADER
Value: HEADERNAME:HEADERVALUE

In case of HTTPS, a new API method is added uploadTemplateDirectDownloadCertificate to allow user importing a client certificate into all KVM hosts' keystore before deployment.
After template or ISO is downloaded to primary storage, usual entry would be persisted on template_spool_ref indicating the mapping between template/ISO and storage pool.

Author: nvazquez

agent/src/com/cloud/agent/Agent.java

@@ -37,6 +37,7 @@
 
 import javax.naming.ConfigurationException;
 
+import org.apache.cloudstack.agent.directdownload.SetupDirectDownloadCertificate;
 import org.apache.cloudstack.ca.SetupCertificateAnswer;
 import org.apache.cloudstack.ca.SetupCertificateCommand;
 import org.apache.cloudstack.ca.SetupKeyStoreCommand;
@@ -551,6 +552,8 @@ protected void processRequest(final Request request, final Link link) {
                         answer = setupAgentKeystore((SetupKeyStoreCommand) cmd);
                     } else if (cmd instanceof SetupCertificateCommand && ((SetupCertificateCommand) cmd).isHandleByAgent()) {
                         answer = setupAgentCertificate((SetupCertificateCommand) cmd);
+                    } else if (cmd instanceof SetupDirectDownloadCertificate) {
+                        answer = setupDirectDownloadCertificate((SetupDirectDownloadCertificate) cmd);
                     } else {
                         if (cmd instanceof ReadyCommand) {
                             processReadyCommand(cmd);
@@ -600,6 +603,31 @@ protected void processRequest(final Request request, final Link link) {
         }
     }
 
+    private Answer setupDirectDownloadCertificate(SetupDirectDownloadCertificate cmd) {
+        String certificate = cmd.getCertificate();
+        String certificateName = cmd.getCertificateName();
+        s_logger.info("Importing certificate " + certificateName + " into keystore");
+
+        final File agentFile = PropertiesUtil.findConfigFile("agent.properties");
+        if (agentFile == null) {
+            return new Answer(cmd, false, "Failed to find agent.properties file");
+        }
+
+        final String keyStoreFile = agentFile.getParent() + "/" + KeyStoreUtils.defaultKeystoreFile;
+
+        String cerFile = agentFile.getParent() + "/" + certificateName + ".cer";
+        Script.runSimpleBashScript(String.format("echo '%s' > %s", certificate, cerFile));
+
+        String privatePasswordFormat = "sed -n '/keystore.passphrase/p' '%s' 2>/dev/null  | sed 's/keystore.passphrase=//g' 2>/dev/null";
+        String privatePasswordCmd = String.format(privatePasswordFormat, agentFile.getAbsolutePath());
+        String privatePassword = Script.runSimpleBashScript(privatePasswordCmd);
+
+        String importCommandFormat = "keytool -importcert -file %s -keystore %s -alias '%s' -storepass '%s' -noprompt";
+        String importCmd = String.format(importCommandFormat, cerFile, keyStoreFile, certificateName, privatePassword);
+        Script.runSimpleBashScript(importCmd);
+        return new Answer(cmd, true, "Certificate " + certificateName + " imported");
+    }
+
     public Answer setupAgentKeystore(final SetupKeyStoreCommand cmd) {
         final String keyStorePassword = cmd.getKeystorePassword();
         final long validityDays = cmd.getValidityDays();

agent/src/com/cloud/agent/direct/download/DirectTemplateDownloader.java

@@ -0,0 +1,71 @@
+//
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+//
+
+package com.cloud.agent.direct.download;
+
+public interface DirectTemplateDownloader {
+
+    class DirectTemplateInformation {
+        private String installPath;
+        private Long size;
+        private String checksum;
+
+        public DirectTemplateInformation(String installPath, Long size, String checksum) {
+            this.installPath = installPath;
+            this.size = size;
+            this.checksum = checksum;
+        }
+
+        public String getInstallPath() {
+            return installPath;
+        }
+
+        public Long getSize() {
+            return size;
+        }
+
+        public String getChecksum() {
+            return checksum;
+        }
+    }
+
+    /**
+     * Perform template download to pool specified on downloader creation
+     * @return true if successful, false if not
+     */
+    boolean downloadTemplate();
+
+    /**
+     * Perform extraction (if necessary) and installation of previously downloaded template
+     * @return true if successful, false if not
+     */
+    boolean extractAndInstallDownloadedTemplate();
+
+    /**
+     * Get template information after it is properly installed on pool
+     * @return template information
+     */
+    DirectTemplateInformation getTemplateInformation();
+
+    /**
+     * Perform checksum validation of previously downloadeed template
+     * @return true if successful, false if not
+     */
+    boolean validateChecksum();
+}

agent/src/com/cloud/agent/direct/download/DirectTemplateDownloaderImpl.java

@@ -0,0 +1,185 @@
+//
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+//
+package com.cloud.agent.direct.download;
+
+import com.cloud.utils.exception.CloudRuntimeException;
+import com.cloud.utils.script.Script;
+import org.apache.cloudstack.utils.security.ChecksumValue;
+import org.apache.commons.lang.StringUtils;
+
+import java.io.File;
+import java.util.UUID;
+
+public abstract class DirectTemplateDownloaderImpl implements DirectTemplateDownloader {
+
+    private String url;
+    private String destPoolPath;
+    private Long templateId;
+    private String downloadedFilePath;
+    private String installPath;
+    private String checksum;
+
+    protected DirectTemplateDownloaderImpl(final String url, final String destPoolPath, final Long templateId, final String checksum) {
+        this.url = url;
+        this.destPoolPath = destPoolPath;
+        this.templateId = templateId;
+        this.checksum = checksum;
+    }
+
+    private static String directDownloadDir = "template";
+
+    /**
+     * Return direct download temporary path to download template
+     */
+    protected static String getDirectDownloadTempPath(Long templateId) {
+        String templateIdAsString = String.valueOf(templateId);
+        return directDownloadDir + File.separator + templateIdAsString.substring(0,1) +
+                File.separator + templateIdAsString;
+    }
+
+    /**
+     * Create folder on path if it does not exist
+     */
+    protected void createFolder(String path) {
+        File dir = new File(path);
+        if (!dir.exists()) {
+            dir.mkdirs();
+        }
+    }
+
+    public String getUrl() {
+        return url;
+    }
+
+    public String getDestPoolPath() {
+        return destPoolPath;
+    }
+
+    public Long getTemplateId() {
+        return templateId;
+    }
+
+    public String getDownloadedFilePath() {
+        return downloadedFilePath;
+    }
+
+    public void setDownloadedFilePath(String filePath) {
+        this.downloadedFilePath = filePath;
+    }
+
+    /**
+     * Return filename from url
+     */
+    public String getFileNameFromUrl() {
+        String[] urlParts = url.split("/");
+        return urlParts[urlParts.length - 1];
+    }
+
+    /**
+     * Checks if downloaded template is extractable
+     * @return true if it should be extracted, false if not
+     */
+    private boolean isTemplateExtractable() {
+        String type = Script.runSimpleBashScript("file " + downloadedFilePath + " | awk -F' ' '{print $2}'");
+        return type.equalsIgnoreCase("bzip2") || type.equalsIgnoreCase("gzip") || type.equalsIgnoreCase("zip");
+    }
+
+    @Override
+    public boolean extractAndInstallDownloadedTemplate() {
+        installPath = UUID.randomUUID().toString();
+        if (isTemplateExtractable()) {
+            extractDownloadedTemplate();
+        } else {
+            Script.runSimpleBashScript("mv " + downloadedFilePath + " " + getInstallFullPath());
+        }
+        return true;
+    }
+
+    /**
+     * Return install full path
+     */
+    private String getInstallFullPath() {
+        return destPoolPath + File.separator + installPath;
+    }
+
+    /**
+     * Return extract command to execute given downloaded file
+     */
+    private String getExtractCommandForDownloadedFile() {
+        if (downloadedFilePath.endsWith(".zip")) {
+            return "unzip -p " + downloadedFilePath + " | cat > " + getInstallFullPath();
+        } else if (downloadedFilePath.endsWith(".bz2")) {
+            return "bunzip2 -c " + downloadedFilePath + " > " + getInstallFullPath();
+        } else if (downloadedFilePath.endsWith(".gz")) {
+            return "gunzip -c " + downloadedFilePath + " > " + getInstallFullPath();
+        } else {
+            throw new CloudRuntimeException("Unable to extract template " + templateId + " on " + downloadedFilePath);
+        }
+    }
+
+    /**
+     * Extract downloaded template into installPath, remove compressed file
+     */
+    private void extractDownloadedTemplate() {
+        String extractCommand = getExtractCommandForDownloadedFile();
+        Script.runSimpleBashScript(extractCommand);
+        Script.runSimpleBashScript("rm -f " + downloadedFilePath);
+    }
+
+    @Override
+    public DirectTemplateInformation getTemplateInformation() {
+        String sizeResult = Script.runSimpleBashScript("ls -als " + getInstallFullPath() + " | awk '{print $1}'");
+        long size = Long.parseLong(sizeResult);
+        return new DirectTemplateInformation(installPath, size, checksum);
+    }
+
+    /**
+     * Return checksum command from algorithm
+     */
+    private String getChecksumCommandFromAlgorithm(String algorithm) {
+        if (algorithm.equalsIgnoreCase("MD5")) {
+            return "md5sum";
+        } else if (algorithm.equalsIgnoreCase("SHA-1")) {
+            return "sha1sum";
+        } else if (algorithm.equalsIgnoreCase("SHA-224")) {
+            return "sha224sum";
+        } else if (algorithm.equalsIgnoreCase("SHA-256")) {
+            return "sha256sum";
+        } else if (algorithm.equalsIgnoreCase("SHA-384")) {
+            return "sha384sum";
+        } else if (algorithm.equalsIgnoreCase("SHA-512")) {
+            return "sha512sum";
+        } else {
+            throw new CloudRuntimeException("Unknown checksum algorithm: " + algorithm);
+        }
+    }
+
+    @Override
+    public boolean validateChecksum() {
+        if (StringUtils.isNotBlank(checksum)) {
+            ChecksumValue providedChecksum = new ChecksumValue(checksum);
+            String algorithm = providedChecksum.getAlgorithm();
+            String checksumCommand = "echo '%s %s' | %s -c --quiet";
+            String cmd = String.format(checksumCommand, providedChecksum.getChecksum(), downloadedFilePath, getChecksumCommandFromAlgorithm(algorithm));
+            int result = Script.runSimpleBashScriptForExitValue(cmd);
+            return result == 0;
+        }
+        return true;
+    }
+}