Skip to content

Commit 4f00c51

Browse files
davidjumanidavidjumani
committed
Allowing for upgrades
1 parent bbf1789 commit 4f00c51

15 files changed

Lines changed: 251 additions & 439 deletions

‎plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java‎

Lines changed: 15 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1140,7 +1140,8 @@ public boolean startKubernetesCluster(long kubernetesClusterId, boolean onCreate
11401140
}
11411141
if (onCreate) {
11421142
// Start for Kubernetes cluster in 'Created' state
1143-
String[] keys = getServiceUserKeys();
1143+
Account owner = accountService.getActiveAccountById(kubernetesCluster.getAccountId());
1144+
String[] keys = getServiceUserKeys(owner);
11441145
KubernetesClusterStartWorker startWorker =
11451146
new KubernetesClusterStartWorker(kubernetesCluster, this, keys);
11461147
startWorker = ComponentContext.inject(startWorker);
@@ -1276,14 +1277,16 @@ public KubernetesClusterConfigResponse getKubernetesClusterConfig(GetKubernetesC
12761277
return response;
12771278
}
12781279

1279-
private String[] getServiceUserKeys() {
1280-
Account caller = CallContext.current().getCallingAccount();
1281-
String username = caller.getAccountName() + "-" + KUBEADMIN_ACCOUNT_NAME;
1282-
UserAccount kubeadmin = accountService.getActiveUserAccount(username, caller.getDomainId());
1280+
private String[] getServiceUserKeys(Account owner) {
1281+
if (owner == null) {
1282+
owner = CallContext.current().getCallingAccount();
1283+
}
1284+
String username = owner.getAccountName() + "-" + KUBEADMIN_ACCOUNT_NAME;
1285+
UserAccount kubeadmin = accountService.getActiveUserAccount(username, owner.getDomainId());
12831286
String[] keys = null;
12841287
if (kubeadmin == null) {
1285-
User kube = userDao.persist(new UserVO(caller.getAccountId(), username, UUID.randomUUID().toString(), caller.getAccountName(), KUBEADMIN_ACCOUNT_NAME, "kubeadmin",
1286-
null, UUID.randomUUID().toString(), User.Source.UNKNOWN));
1288+
User kube = userDao.persist(new UserVO(owner.getAccountId(), username, UUID.randomUUID().toString(), owner.getAccountName(),
1289+
KUBEADMIN_ACCOUNT_NAME, "kubeadmin", null, UUID.randomUUID().toString(), User.Source.UNKNOWN));
12871290
keys = accountService.createApiKeyAndSecretKey(kube.getId());
12881291
} else {
12891292
String apiKey = kubeadmin.getApiKey();
@@ -1323,9 +1326,12 @@ public boolean upgradeKubernetesCluster(UpgradeKubernetesClusterCmd cmd) throws
13231326
logAndThrow(Level.ERROR, "Kubernetes Service plugin is disabled");
13241327
}
13251328
validateKubernetesClusterUpgradeParameters(cmd);
1329+
KubernetesClusterVO kubernetesCluster = kubernetesClusterDao.findById(cmd.getId());
1330+
Account owner = accountService.getActiveAccountById(kubernetesCluster.getAccountId());
1331+
String[] keys = getServiceUserKeys(owner);
13261332
KubernetesClusterUpgradeWorker upgradeWorker =
1327-
new KubernetesClusterUpgradeWorker(kubernetesClusterDao.findById(cmd.getId()),
1328-
kubernetesSupportedVersionDao.findById(cmd.getKubernetesVersionId()), this);
1333+
new KubernetesClusterUpgradeWorker(kubernetesClusterDao.findById(cmd.getId()),
1334+
kubernetesSupportedVersionDao.findById(cmd.getKubernetesVersionId()), this, keys);
13291335
upgradeWorker = ComponentContext.inject(upgradeWorker);
13301336
return upgradeWorker.upgradeCluster();
13311337
}

‎plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterActionWorker.java‎

Lines changed: 71 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,9 @@
1717

1818
package com.cloud.kubernetes.cluster.actionworkers;
1919

20+
import java.io.BufferedWriter;
2021
import java.io.File;
22+
import java.io.FileWriter;
2123
import java.io.IOException;
2224
import java.util.ArrayList;
2325
import java.util.Collections;
@@ -28,6 +30,7 @@
2830

2931
import org.apache.cloudstack.api.ApiConstants;
3032
import org.apache.cloudstack.ca.CAManager;
33+
import org.apache.cloudstack.config.ApiServiceConfiguration;
3134
import org.apache.cloudstack.engine.orchestration.service.NetworkOrchestrationService;
3235
import org.apache.cloudstack.framework.config.dao.ConfigurationDao;
3336
import org.apache.commons.collections.CollectionUtils;
@@ -70,6 +73,7 @@
7073
import com.cloud.utils.exception.CloudRuntimeException;
7174
import com.cloud.utils.fsm.NoTransitionException;
7275
import com.cloud.utils.fsm.StateMachine2;
76+
import com.cloud.utils.ssh.SshHelper;
7377
import com.cloud.vm.UserVmService;
7478
import com.cloud.vm.dao.UserVmDao;
7579
import com.google.common.base.Strings;
@@ -127,6 +131,11 @@ public class KubernetesClusterActionWorker {
127131
protected String publicIpAddress;
128132
protected int sshPort;
129133

134+
protected final String autoscaleScriptFilename = "autoscale-kube-cluster";
135+
protected final String deploySecretsScriptFilename = "deploy-cloudstack-secret";
136+
protected File autoscaleScriptFile;
137+
protected File deploySecretsScriptFile;
138+
130139
protected KubernetesClusterActionWorker(final KubernetesCluster kubernetesCluster, final KubernetesClusterManagerImpl clusterManager) {
131140
this.kubernetesCluster = kubernetesCluster;
132141
this.kubernetesClusterDao = clusterManager.kubernetesClusterDao;
@@ -178,7 +187,7 @@ protected void logMessage(final Level logLevel, final String message, final Exce
178187
}
179188

180189
protected void logTransitStateDetachIsoAndThrow(final Level logLevel, final String message, final KubernetesCluster kubernetesCluster,
181-
final List<UserVm> clusterVMs, final KubernetesCluster.Event event, final Exception e) throws CloudRuntimeException {
190+
final List<UserVm> clusterVMs, final KubernetesCluster.Event event, final Exception e) throws CloudRuntimeException {
182191
logMessage(logLevel, message, e);
183192
stateTransitTo(kubernetesCluster.getId(), event);
184193
detachIsoKubernetesVMs(clusterVMs);
@@ -383,4 +392,65 @@ protected boolean stateTransitTo(long kubernetesClusterId, KubernetesCluster.Eve
383392
return false;
384393
}
385394
}
395+
396+
protected boolean createCloudStackSecret(String[] keys) {
397+
File pkFile = getManagementServerSshPublicKeyFile();
398+
Pair<String, Integer> publicIpSshPort = getKubernetesClusterServerIpSshPort(null);
399+
publicIpAddress = publicIpSshPort.first();
400+
sshPort = publicIpSshPort.second();
401+
402+
List<KubernetesClusterVmMapVO> clusterVMs = getKubernetesClusterVMMaps();
403+
if (CollectionUtils.isEmpty(clusterVMs)) {
404+
return false;
405+
}
406+
407+
final UserVm userVm = userVmDao.findById(clusterVMs.get(0).getVmId());
408+
409+
String hostName = userVm.getHostName();
410+
if (!Strings.isNullOrEmpty(hostName)) {
411+
hostName = hostName.toLowerCase();
412+
}
413+
414+
try {
415+
Pair<Boolean, String> result = SshHelper.sshExecute(publicIpAddress, sshPort, CLUSTER_NODE_VM_USER,
416+
pkFile, null, String.format("sudo /opt/bin/deploy-cloudstack-secret -u '%s' -k '%s' -s '%s'",
417+
ApiServiceConfiguration.ApiServletPath.value(), keys[0], keys[1]),
418+
10000, 10000, 60000);
419+
return result.first();
420+
} catch (Exception e) {
421+
String msg = String.format("Failed to add cloudstack-secret to Kubernetes cluster: %s", kubernetesCluster.getName());
422+
LOGGER.warn(msg, e);
423+
}
424+
return true;
425+
}
426+
427+
protected File retrieveScriptFile(String filename) {
428+
File file = null;
429+
try {
430+
String data = readResourceFile("/script/" + filename);
431+
file = File.createTempFile(filename, ".sh");
432+
BufferedWriter writer = new BufferedWriter(new FileWriter(file));
433+
writer.write(data);
434+
writer.close();
435+
} catch (IOException e) {
436+
logAndThrow(Level.ERROR, String.format("Failed to upgrade Kubernetes cluster ID: %s, unable to prepare upgrade script %s", kubernetesCluster.getUuid(), filename), e);
437+
}
438+
return file;
439+
}
440+
441+
protected void copyAutoscalerScripts(final UserVm vm, final int index) throws Exception {
442+
// TODO: This might be a bad way to do it. Better to fetch the pf rules and try
443+
int nodeSshPort = sshPort == 22 ? sshPort : sshPort + index;
444+
String nodeAddress = (index > 0 && sshPort == 22) ? vm.getPrivateIpAddress() : publicIpAddress;
445+
SshHelper.scpTo(nodeAddress, nodeSshPort, CLUSTER_NODE_VM_USER, sshKeyFile, null,
446+
"~/", autoscaleScriptFile.getAbsolutePath(), "0755");
447+
SshHelper.scpTo(nodeAddress, nodeSshPort, CLUSTER_NODE_VM_USER, sshKeyFile, null,
448+
"~/", deploySecretsScriptFile.getAbsolutePath(), "0755");
449+
String cmdStr = String.format("sudo mv ~/%s /opt/bin/%s", autoscaleScriptFile.getName(), autoscaleScriptFilename);
450+
SshHelper.sshExecute(publicIpAddress, nodeSshPort, CLUSTER_NODE_VM_USER, sshKeyFile, null,
451+
cmdStr, 10000, 10000, 10 * 60 * 1000);
452+
cmdStr = String.format("sudo mv ~/%s /opt/bin/%s", deploySecretsScriptFile.getName(), deploySecretsScriptFilename);
453+
SshHelper.sshExecute(publicIpAddress, nodeSshPort, CLUSTER_NODE_VM_USER, sshKeyFile, null,
454+
cmdStr, 10000, 10000, 10 * 60 * 1000);
455+
}
386456
}

‎plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterResourceModifierActionWorker.java‎

Lines changed: 4 additions & 35 deletions
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,6 @@
3232
import org.apache.cloudstack.api.BaseCmd;
3333
import org.apache.cloudstack.api.command.user.firewall.CreateFirewallRuleCmd;
3434
import org.apache.cloudstack.api.command.user.vm.StartVMCmd;
35-
import org.apache.cloudstack.config.ApiServiceConfiguration;
3635
import org.apache.commons.codec.binary.Base64;
3736
import org.apache.commons.collections.CollectionUtils;
3837
import org.apache.log4j.Level;
@@ -520,37 +519,6 @@ protected String getKubernetesClusterNodeNamePrefix() {
520519
return prefix;
521520
}
522521

523-
protected boolean createSecret(String[] keys) {
524-
File pkFile = getManagementServerSshPublicKeyFile();
525-
Pair<String, Integer> publicIpSshPort = getKubernetesClusterServerIpSshPort(null);
526-
publicIpAddress = publicIpSshPort.first();
527-
sshPort = publicIpSshPort.second();
528-
529-
List<KubernetesClusterVmMapVO> clusterVMs = getKubernetesClusterVMMaps();
530-
if (CollectionUtils.isEmpty(clusterVMs)) {
531-
return false;
532-
}
533-
534-
final UserVm userVm = userVmDao.findById(clusterVMs.get(0).getVmId());
535-
536-
String hostName = userVm.getHostName();
537-
if (!Strings.isNullOrEmpty(hostName)) {
538-
hostName = hostName.toLowerCase();
539-
}
540-
541-
try {
542-
Pair<Boolean, String> result = SshHelper.sshExecute(publicIpAddress, sshPort, CLUSTER_NODE_VM_USER,
543-
pkFile, null, String.format("sudo /opt/bin/deploy-cloudstack-secret -u '%s' -k '%s' -s '%s'",
544-
ApiServiceConfiguration.ApiServletPath.value(), keys[0], keys[1]),
545-
10000, 10000, 60000);
546-
return result.first();
547-
} catch (Exception e) {
548-
String msg = String.format("Failed to add cloudstack-secret to Kubernetes cluster: %s", kubernetesCluster.getName());
549-
LOGGER.warn(msg, e);
550-
}
551-
return true;
552-
}
553-
554522
protected KubernetesClusterVO updateKubernetesClusterEntry(final Long cores, final Long memory,
555523
final Long size, final Long serviceOfferingId, final Boolean autoscaleEnabled, final Long minSize, final Long maxSize) {
556524
return Transaction.execute(new TransactionCallback<KubernetesClusterVO>() {
@@ -612,19 +580,20 @@ protected boolean autoscaleCluster(boolean enable, Long minSize, Long maxSize) {
612580

613581
try {
614582
if (enable) {
583+
String data = readResourceFile("/script/try-autoscaling");
615584
Pair<Boolean, String> result = SshHelper.sshExecute(publicIpAddress, sshPort, CLUSTER_NODE_VM_USER,
616-
pkFile, null, String.format("sudo /opt/bin/autoscale-kube-cluster -i %s -e -M %d -m %d", kubernetesCluster.getUuid(), maxSize, minSize),
585+
pkFile, null, String.format(data, kubernetesCluster.getUuid(), maxSize, minSize),
617586
10000, 10000, 60000);
618587
if (!result.first()) {
619-
return false;
588+
throw new CloudRuntimeException(result.second());
620589
}
621590
updateKubernetesClusterEntry(true, minSize, maxSize);
622591
} else {
623592
Pair<Boolean, String> result = SshHelper.sshExecute(publicIpAddress, sshPort, CLUSTER_NODE_VM_USER,
624593
pkFile, null, String.format("sudo /opt/bin/autoscale-kube-cluster -d"),
625594
10000, 10000, 60000);
626595
if (!result.first()) {
627-
return false;
596+
throw new CloudRuntimeException(result.second());
628597
}
629598
updateKubernetesClusterEntry(false, null, null);
630599
}

‎plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterScaleWorker.java‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -134,13 +134,15 @@ private void scaleKubernetesClusterNetworkRules(final List<Long> clusterVMIds, f
134134
throw new ManagementServerException(String.format("No source NAT IP addresses found for network ID: %s, Kubernetes cluster ID: %s", network.getUuid(), kubernetesCluster.getUuid()));
135135
}
136136

137+
// TODO : Remove indiv rules per vm
137138
// Remove existing SSH firewall rules
138139
FirewallRule firewallRule = removeSshFirewallRule(publicIp);
139140
if (firewallRule == null) {
140141
throw new ManagementServerException("Firewall rule for node SSH access can't be provisioned");
141142
}
142143
int existingFirewallRuleSourcePortEnd = firewallRule.getSourcePortEnd();
143144
final int scaledTotalNodeCount = clusterSize == null ? (int)kubernetesCluster.getTotalNodeCount() : (int)(clusterSize + kubernetesCluster.getMasterNodeCount());
145+
// TODO : Provision indiv rules per vm
144146
// Provision new SSH firewall rules
145147
try {
146148
provisionFirewallRules(publicIp, owner, CLUSTER_NODES_DEFAULT_START_SSH_PORT, CLUSTER_NODES_DEFAULT_START_SSH_PORT + scaledTotalNodeCount - 1);
@@ -159,6 +161,7 @@ private void scaleKubernetesClusterNetworkRules(final List<Long> clusterVMIds, f
159161
}
160162

161163
try {
164+
// TODO : Provision indiv rules per vm
162165
provisionSshPortForwardingRules(publicIp, network, owner, clusterVMIds, existingFirewallRuleSourcePortEnd + 1);
163166
} catch (ResourceUnavailableException | NetworkRuleConflictException e) {
164167
throw new ManagementServerException(String.format("Failed to activate SSH port forwarding rules for the Kubernetes cluster ID: %s", kubernetesCluster.getUuid()), e);

‎plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterStartWorker.java‎

Lines changed: 16 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -394,6 +394,7 @@ private void setupKubernetesClusterNetworkRules(Network network, List<UserVm> cl
394394
}
395395

396396
try {
397+
// TODO : Create indiv fw rules for each vm
397398
int endPort = CLUSTER_NODES_DEFAULT_START_SSH_PORT + clusterVMs.size() - 1;
398399
provisionFirewallRules(publicIp, owner, CLUSTER_NODES_DEFAULT_START_SSH_PORT, endPort);
399400
if (LOGGER.isInfoEnabled()) {
@@ -569,13 +570,27 @@ public boolean startKubernetesClusterOnCreate() {
569570
if (!isKubernetesClusterDashboardServiceRunning(true, startTimeoutTime)) {
570571
logTransitStateAndThrow(Level.ERROR, String.format("Failed to setup Kubernetes cluster ID: %s in usable state as unable to get Dashboard service running for the cluster", kubernetesCluster.getUuid()), kubernetesCluster.getId(),KubernetesCluster.Event.OperationFailed);
571572
}
572-
if (!createSecret(keys)) {
573+
retrieveScriptFiles();
574+
for (int i = 0; i < clusterVMs.size(); ++i) {
575+
UserVm vm = clusterVMs.get(i);
576+
try {
577+
copyAutoscalerScripts(vm, i);
578+
} catch (Exception e) {
579+
throw new CloudRuntimeException(e);
580+
}
581+
}
582+
if (!createCloudStackSecret(keys)) {
573583
logTransitStateAndThrow(Level.ERROR, String.format("Failed to setup keys for Kubernetes cluster ID: %s", kubernetesCluster.getUuid()), kubernetesCluster.getId(),KubernetesCluster.Event.OperationFailed);
574584
}
575585
stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.OperationSucceeded);
576586
return true;
577587
}
578588

589+
private void retrieveScriptFiles() {
590+
autoscaleScriptFile = retrieveScriptFile(autoscaleScriptFilename);
591+
deploySecretsScriptFile = retrieveScriptFile(deploySecretsScriptFilename);
592+
}
593+
579594
public boolean startStoppedKubernetesCluster() throws CloudRuntimeException {
580595
init();
581596
if (LOGGER.isInfoEnabled()) {

‎plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterUpgradeWorker.java‎

Lines changed: 12 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -17,10 +17,7 @@
1717

1818
package com.cloud.kubernetes.cluster.actionworkers;
1919

20-
import java.io.BufferedWriter;
2120
import java.io.File;
22-
import java.io.FileWriter;
23-
import java.io.IOException;
2421
import java.util.ArrayList;
2522
import java.util.List;
2623

@@ -45,26 +42,24 @@ public class KubernetesClusterUpgradeWorker extends KubernetesClusterActionWorke
4542

4643
private List<UserVm> clusterVMs = new ArrayList<>();
4744
private KubernetesSupportedVersion upgradeVersion;
45+
private final String upgradeScriptFilename = "upgrade-kubernetes.sh";
4846
private File upgradeScriptFile;
4947
private long upgradeTimeoutTime;
48+
private String[] keys;
5049

5150
public KubernetesClusterUpgradeWorker(final KubernetesCluster kubernetesCluster,
5251
final KubernetesSupportedVersion upgradeVersion,
53-
final KubernetesClusterManagerImpl clusterManager) {
52+
final KubernetesClusterManagerImpl clusterManager,
53+
final String[] keys) {
5454
super(kubernetesCluster, clusterManager);
5555
this.upgradeVersion = upgradeVersion;
56+
this.keys = keys;
5657
}
5758

58-
private void retrieveUpgradeScriptFile() {
59-
try {
60-
String upgradeScriptData = readResourceFile("/script/upgrade-kubernetes.sh");
61-
upgradeScriptFile = File.createTempFile("upgrade-kuberntes", ".sh");
62-
BufferedWriter upgradeScriptFileWriter = new BufferedWriter(new FileWriter(upgradeScriptFile));
63-
upgradeScriptFileWriter.write(upgradeScriptData);
64-
upgradeScriptFileWriter.close();
65-
} catch (IOException e) {
66-
logAndThrow(Level.ERROR, String.format("Failed to upgrade Kubernetes cluster ID: %s, unable to prepare upgrade script", kubernetesCluster.getUuid()), e);
67-
}
59+
private void retrieveScriptFiles() {
60+
upgradeScriptFile = retrieveScriptFile(upgradeScriptFilename);
61+
autoscaleScriptFile = retrieveScriptFile(autoscaleScriptFilename);
62+
deploySecretsScriptFile = retrieveScriptFile(deploySecretsScriptFilename);
6863
}
6964

7065
private Pair<Boolean, String> runInstallScriptOnVM(final UserVm vm, final int index) throws Exception {
@@ -110,6 +105,8 @@ private void upgradeKubernetesClusterNodes() {
110105
logTransitStateDetachIsoAndThrow(Level.ERROR, String.format("Failed to upgrade Kubernetes cluster ID: %s, upgrade action timed out", kubernetesCluster.getUuid()), kubernetesCluster, clusterVMs, KubernetesCluster.Event.OperationFailed, null);
111106
}
112107
try {
108+
copyAutoscalerScripts(vm, i);
109+
createCloudStackSecret(keys);
113110
result = runInstallScriptOnVM(vm, i);
114111
} catch (Exception e) {
115112
logTransitStateDetachIsoAndThrow(Level.ERROR, String.format("Failed to upgrade Kubernetes cluster ID: %s, unable to upgrade Kubernetes node on VM ID: %s", kubernetesCluster.getUuid(), vm.getUuid()), kubernetesCluster, clusterVMs, KubernetesCluster.Event.OperationFailed, e);
@@ -151,7 +148,7 @@ public boolean upgradeCluster() throws CloudRuntimeException {
151148
if (CollectionUtils.isEmpty(clusterVMs)) {
152149
logAndThrow(Level.ERROR, String.format("Upgrade failed for Kubernetes cluster ID: %s, unable to retrieve VMs for cluster", kubernetesCluster.getUuid()));
153150
}
154-
retrieveUpgradeScriptFile();
151+
retrieveScriptFiles();
155152
stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.UpgradeRequested);
156153
attachIsoKubernetesVMs(clusterVMs, upgradeVersion);
157154
upgradeKubernetesClusterNodes();

0 commit comments

Comments
 (0)