Skip to content

Commit 626f4a6

Browse files
farthinderfarthinder
authored
2.3.12 release - Remove groovy yaml (#32)
* HarborManagerContainer.groovy * Switched to using ObjectMapper for yaml manipulation. * Added extra startup/shutdown attempts, seems to be needed now potentially due to these being run on arm arch * The log container is now configured to be run as root per harbor github issue HarborDeployment.groovy * Added extra startup/shutdown attempts, seems to be needed now potentially due to these being run on arm arch JsmH2Deployment.groovy * Now waits for jira to become responsive after snapshotting * pom.xml * Bumped to 2.3.12
1 parent e2559f6 commit 626f4a6

File tree

5 files changed

+93
-61
lines changed

5 files changed

+93
-61
lines changed

‎pom.xml‎

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@
66

77
<groupId>com.eficode</groupId>
88
<artifactId>devstack</artifactId>
9-
<version>2.3.11-SNAPSHOT</version>
9+
<version>2.3.12-SNAPSHOT</version>
1010
<packaging>jar</packaging>
1111

1212
<name>DevStack</name>
@@ -35,11 +35,13 @@
3535
<scope>provided</scope>
3636
<type>pom</type>
3737
</dependency>
38+
<!-- https://mvnrepository.com/artifact/com.fasterxml.jackson.dataformat/jackson-dataformat-yaml -->
3839
<dependency>
39-
<groupId>org.codehaus.groovy</groupId>
40-
<artifactId>groovy-yaml</artifactId>
41-
<version>${groovy.version}</version>
40+
<groupId>com.fasterxml.jackson.dataformat</groupId>
41+
<artifactId>jackson-dataformat-yaml</artifactId>
42+
<version>2.15.3</version>
4243
</dependency>
44+
4345
<dependency>
4446
<groupId>org.codehaus.groovy</groupId>
4547
<artifactId>groovy-json</artifactId>

‎src/main/groovy/com/eficode/devstack/container/impl/HarborManagerContainer.groovy‎

Lines changed: 58 additions & 45 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,8 @@
11
package com.eficode.devstack.container.impl
22

3-
import groovy.yaml.YamlBuilder
4-
import groovy.yaml.YamlSlurper
3+
4+
import com.fasterxml.jackson.databind.ObjectMapper
5+
import com.fasterxml.jackson.dataformat.yaml.YAMLFactory
56
import org.apache.groovy.json.internal.LazyMap
67

78
import java.nio.file.Files
@@ -14,6 +15,8 @@ class HarborManagerContainer extends DoodContainer {
1415
String harborBaseUrl
1516
String basePath
1617

18+
ObjectMapper objectMapper = new ObjectMapper(new YAMLFactory())
19+
1720
/**
1821
* HarborManagerContainer setups a container that runs the Harbor installation scripts, which in turn creates all the "real"
1922
* harbor containers
@@ -30,14 +33,14 @@ class HarborManagerContainer extends DoodContainer {
3033
this.harborBaseUrl = baseUrl
3134
this.harborVersion = harborVersion
3235
this.containerName = host + "-manager"
33-
this.basePath = baseDir[-1] == "/" ? baseDir + host : baseDir + "/" + host
36+
this.basePath = baseDir[-1] == "/" ? baseDir + host : baseDir + "/" + host
3437

3538
if (dockerHost && dockerCertPath) {
3639
assert setupSecureRemoteConnection(dockerHost, dockerCertPath): "Error setting up secure remote docker connection"
3740
}
3841

3942
prepareBindMount("/var/run/docker.sock", "/var/run/docker.sock") // Mount docker socket
40-
prepareBindMount(baseDir , baseDir, false) //Mount data dir, data in this dir needs to be accessible by both engine and manager-container using the same path
43+
prepareBindMount(baseDir, baseDir, false) //Mount data dir, data in this dir needs to be accessible by both engine and manager-container using the same path
4144

4245
}
4346

@@ -55,22 +58,19 @@ class HarborManagerContainer extends DoodContainer {
5558
extractDomainFromUrl(harborBaseUrl)
5659
}
5760

58-
String getPort(){
59-
extractPortFromUrl( harborBaseUrl)
61+
String getPort() {
62+
extractPortFromUrl(harborBaseUrl)
6063
}
6164

6265

63-
64-
6566
@Override
6667
boolean runAfterDockerSetup() {
6768

6869
log.info("Setting up Harbor")
6970

7071
//Make sure basePath is empty or does not exist
7172
ArrayList<String> cmdOutput = runBashCommandInContainer("""ls "$basePath" | wc -l""", 5)
72-
assert cmdOutput == ["0"] || cmdOutput.any{it.startsWith("ls: cannot access")} : "Harbor base path is not empty: $basePath"
73-
73+
assert cmdOutput == ["0"] || cmdOutput.any { it.startsWith("ls: cannot access") }: "Harbor base path is not empty: $basePath"
7474

7575

7676
cmdOutput = runBashCommandInContainer("apt install -y wget; echo status: \$?", 100)
@@ -91,16 +91,27 @@ class HarborManagerContainer extends DoodContainer {
9191
log.info("\tFinished downloading and extracting Harbor")
9292

9393

94-
assert modifyInstallYml() : "Error updating Harbor install config file: harbor.yml"
94+
assert modifyInstallYml(): "Error updating Harbor install config file: harbor.yml"
9595

9696
log.info("\tStarting installation")
97-
cmdOutput = runBashCommandInContainer(installPath + "/harbor/install.sh ; echo status: \$?", 400 )
98-
assert cmdOutput.last().contains("status: 0"): "Error installing harbor:" + cmdOutput.join("\n")
97+
cmdOutput = runBashCommandInContainer(installPath + "/harbor/install.sh ; echo status: \$?", 400)
98+
if (!cmdOutput.last().contains("status: 0")) {
99+
log.warn("\tThere where problems during setup of harbor, potentially because dockerCompose has yet to mo modified, will modify and try again")
100+
}
99101

102+
cmdOutput = runBashCommandInContainer("cd " + installPath + "/harbor && docker-compose stop && echo status: \$?", 80)
103+
assert cmdOutput.last().contains("status: 0"): "Error stopping harbor before modifying docker-compose file:" + cmdOutput.join("\n")
100104

101-
assert modifyDockerCompose() : "Error modifying Harbors docker-compose file"
105+
assert modifyDockerCompose(): "Error modifying Harbors docker-compose file"
102106

107+
108+
sleep(5000)
103109
cmdOutput = runBashCommandInContainer("cd " + installPath + "/harbor && docker-compose up -d && echo status: \$?", 80)
110+
if (cmdOutput.last() != "status: 0" || cmdOutput.toString().contains("error")) {
111+
log.warn("\tThere was an error starting harbor after docker compose modification, this is common and a second attempt will be made")
112+
sleep(5000)
113+
cmdOutput = runBashCommandInContainer("cd " + installPath + "/harbor && docker-compose up -d && echo status: \$?", 120)
114+
}
104115
assert cmdOutput.last().contains("status: 0"): "Error applying the modified docker-compose file:" + cmdOutput.join("\n")
105116

106117
return true
@@ -114,61 +125,60 @@ class HarborManagerContainer extends DoodContainer {
114125
*/
115126
boolean modifyDockerCompose() {
116127

117-
118-
119-
120128
log.info("\tCustomizing Harbor docker compose")
121-
Path tmpDir= Files.createTempDirectory("harbor-compose")
129+
Path tmpDir = Files.createTempDirectory("harbor-compose")
122130
String tmpDirPath = tmpDir.toFile().absolutePath
123131

124132

125-
126133
ArrayList<File> files = copyFilesFromContainer("${installPath}/harbor/docker-compose.yml", tmpDirPath + "/")
127134

128-
assert files.size() == 1 && files.first().name == "docker-compose.yml" : "Error, could not find docker-compose.yml file"
135+
assert files.size() == 1 && files.first().name == "docker-compose.yml": "Error, could not find docker-compose.yml file"
129136
File yamlFile = files.first()
130137
log.debug("\t\tRetried docker compose file from container:" + yamlFile.absolutePath)
131138

132139

133-
LazyMap originalYml = new YamlSlurper().parse(yamlFile) as LazyMap
134-
LazyMap modifiedYml = new YamlSlurper().parse(yamlFile) as LazyMap
140+
//LazyMap originalYml = new YamlSlurper().parse(yamlFile) as LazyMap
141+
//LazyMap modifiedYml = new YamlSlurper().parse(yamlFile) as LazyMap
142+
LazyMap originalYml = objectMapper.readValue(yamlFile, LazyMap.class)
143+
LazyMap modifiedYml = objectMapper.readValue(yamlFile, LazyMap.class)
135144

136145

137-
138-
modifiedYml.services.each {Entry<String, LazyMap> service ->
139-
log.debug("\t"*3 + "Customising Docker Service:" + service.key)
146+
modifiedYml.services.each { Entry<String, LazyMap> service ->
147+
log.debug("\t" * 3 + "Customising Docker Service:" + service.key)
140148

141149
service.value.networks = containerDefaultNetworks
142-
log.trace("\t"*4 + "Set networks to:" + service.value.networks)
143-
log.trace("\t"*4 + "Used to be:" + originalYml.services.get(service.key).networks)
150+
log.trace("\t" * 4 + "Set networks to:" + service.value.networks)
151+
log.trace("\t" * 4 + "Used to be:" + originalYml.services.get(service.key).networks)
144152

145153
}
146154

147-
log.debug("\t"*3 + "Customising Docker Network")
155+
log.debug("\t" * 3 + "Customising Docker Network")
148156
modifiedYml.remove("networks")
149157

150158
Map<String, LazyMap> networks = [:]
151-
containerDefaultNetworks.each {networkName ->
152-
networks.put(networkName as String,["external": true, "name":networkName] as LazyMap )
159+
containerDefaultNetworks.each { networkName ->
160+
networks.put(networkName as String, ["external": true, "name": networkName] as LazyMap)
153161
}
154162
modifiedYml.put("networks", networks)
155163
//modifiedYml.put("networks", ["default": [external: [name: networkName]]])
156164

157-
log.trace("\t"*4 + "Set networks to:" + modifiedYml.networks)
158-
log.trace("\t"*4 + "Used to be:" + originalYml.networks)
159-
165+
log.trace("\t" * 4 + "Set networks to:" + modifiedYml.networks)
166+
log.trace("\t" * 4 + "Used to be:" + originalYml.networks)
160167

161168

162-
YamlBuilder yamlBuilder = new YamlBuilder()
163-
yamlBuilder(modifiedYml)
169+
//Change the user of log container to root https://github.com/goharbor/harbor/issues/16669
170+
(modifiedYml.services.find { Entry<String, LazyMap> service -> service.key == "log" } as Entry<String, Map>).value.put("user", "root")
164171

172+
//YamlBuilder yamlBuilder = new YamlBuilder()
173+
//yamlBuilder(modifiedYml)
165174

166-
yamlFile.write(yamlBuilder.toString())
175+
//yamlFile.write(yamlBuilder.toString())
176+
yamlFile.write( objectMapper.writeValueAsString(modifiedYml))
167177

168-
assert copyFileToContainer(yamlFile.absolutePath, installPath + "/harbor/") : "Error copying updated YAML file to container"
178+
assert copyFileToContainer(yamlFile.absolutePath, installPath + "/harbor/"): "Error copying updated YAML file to container"
169179
tmpDir.deleteDir()
170180

171-
log.info("\tFinished customizing installation configuration")
181+
log.info("\tFinished customizing docker-compose file")
172182

173183
return true
174184

@@ -180,17 +190,19 @@ class HarborManagerContainer extends DoodContainer {
180190
boolean modifyInstallYml() {
181191

182192

183-
Path tmpDir= Files.createTempDirectory("harbor-conf")
193+
Path tmpDir = Files.createTempDirectory("harbor-conf")
184194
String tmpDirPath = tmpDir.toFile().absolutePath
185195

186196

187197
ArrayList<File> files = copyFilesFromContainer("${installPath}/harbor/harbor.yml.tmpl", tmpDirPath + "/")
188198

189-
assert files.size() == 1 && files.first().name == "harbor.yml.tmpl" : "Error, could not find template config file"
199+
assert files.size() == 1 && files.first().name == "harbor.yml.tmpl": "Error, could not find template config file"
190200
File yamlFile = files.first()
191201

192202

193-
LazyMap originalYml = new YamlSlurper().parse(yamlFile) as LazyMap
203+
//LazyMap originalYml = new YamlSlurper().parse(yamlFile) as LazyMap
204+
LazyMap originalYml = objectMapper.readValue(yamlFile, LazyMap.class)
205+
194206

195207

196208
LazyMap modifiedYml = originalYml
@@ -201,14 +213,15 @@ class HarborManagerContainer extends DoodContainer {
201213

202214

203215

204-
YamlBuilder yamlBuilder = new YamlBuilder()
205-
yamlBuilder(modifiedYml)
216+
//YamlBuilder yamlBuilder = new YamlBuilder()
217+
//yamlBuilder(modifiedYml)
206218

207219
File modifiedYamlFile = new File(tmpDirPath + "/harbor.yml")
208220
modifiedYamlFile.createNewFile()
209-
modifiedYamlFile.write(yamlBuilder.toString().replaceAll("\"", ""))
221+
//modifiedYamlFile.write(yamlBuilder.toString().replaceAll("\"", ""))
222+
modifiedYamlFile.write(objectMapper.writeValueAsString(originalYml).replaceAll("\"", ""))
210223

211-
assert copyFileToContainer(modifiedYamlFile.absolutePath, installPath + "/harbor/") : "Error copying updated YAML file to container"
224+
assert copyFileToContainer(modifiedYamlFile.absolutePath, installPath + "/harbor/"): "Error copying updated YAML file to container"
212225
tmpDir.deleteDir()
213226

214227
log.info("\tFinished customizing installation configuration")

‎src/main/groovy/com/eficode/devstack/deployment/impl/HarborDeployment.groovy‎

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -74,9 +74,14 @@ class HarborDeployment implements Deployment {
7474

7575
assert managerContainer.startContainer()
7676
sleep(5000)
77-
ArrayList<String> cmdOutput = managerContainer.runBashCommandInContainer("cd ${managerContainer.installPath}/harbor ; docker-compose start ; echo status: \$?", 120)
77+
ArrayList<String> cmdOutput = managerContainer.runBashCommandInContainer("cd ${managerContainer.installPath}/harbor && docker-compose start && echo status: \$?", 120)
78+
if (cmdOutput.last() != "status: 0" || cmdOutput.toString().contains("error")) {
79+
log.warn("\tThere was an error starting harbor deployment, this is common and a second attempt will be made")
80+
sleep(5000)
81+
cmdOutput = managerContainer.runBashCommandInContainer("cd ${managerContainer.installPath}/harbor && docker-compose start && echo status: \$?", 120)
82+
}
7883
assert cmdOutput.last() == "status: 0": "Error starting harbor:" + cmdOutput.join("\n")
79-
84+
sleep(5000)
8085
return true
8186
}
8287

@@ -86,7 +91,7 @@ class HarborDeployment implements Deployment {
8691

8792

8893
assert managerContainer.startContainer()
89-
ArrayList<String> cmdOutput = managerContainer.runBashCommandInContainer("cd ${managerContainer.installPath}/harbor ; docker-compose stop ; echo status: \$?", 120)
94+
ArrayList<String> cmdOutput = managerContainer.runBashCommandInContainer("cd ${managerContainer.installPath}/harbor && docker-compose stop && echo status: \$?", 120)
9095
assert cmdOutput.last() == "status: 0": "Error stopping harbor:" + cmdOutput.join("\n")
9196

9297
assert managerContainer.stopContainer()

‎src/main/groovy/com/eficode/devstack/deployment/impl/JsmH2Deployment.groovy‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -118,7 +118,10 @@ class JsmH2Deployment implements Deployment{
118118
if (setupDeployment()) {
119119
if (snapshotAfterCreation) {
120120
log.info("\tSnapshotting the newly created container")
121-
return jsmContainer.snapshotJiraHome() != null
121+
assert jsmContainer.snapshotJiraHome() != null : "Error snapshotting container:" + jsmContainer.shortId
122+
log.info("\tWaiting for JIRA to start back up")
123+
assert jiraRest.waitForJiraToBeResponsive() : "Error waiting for JIRA to start up after creating snapshot"
124+
return true
122125
}else {
123126
return true
124127
}

‎src/test/groovy/com/eficode/devstack/deployment/impl/HarborDeploymentTest.groovy‎

Lines changed: 17 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -37,13 +37,17 @@ class HarborDeploymentTest extends DevStackSpec {
3737
def "Test the basics"(String dockerHost, String certPath, String harborBaseUrl, String harborVersion, String harborBaseDir) {
3838

3939
when:
40+
41+
new File(harborBaseDir).exists() && new File(harborBaseDir).deleteDir()
42+
new File(harborBaseDir).mkdirs()
43+
4044
HarborDeployment hd = new HarborDeployment(harborBaseUrl, harborVersion, harborBaseDir, dockerHost, certPath)
4145

4246
if (dockerHost && certPath) {
4347
assert hd.managerContainer.dockerClient.dockerClientConfig.host == hd.managerContainer.extractDomainFromUrl(dockerHost): "Connection to remote Docker host was not setup"
4448

45-
}else {
46-
assert hd.managerContainer.dockerClient.dockerClientConfig.host == "/var/run/docker.sock": "Connection to local Docker host was not setup"
49+
} else {
50+
assert hd.managerContainer.dockerClient.dockerClientConfig.host.endsWith("run/docker.sock"): "Connection to local Docker host was not setup as expected"
4751

4852
}
4953

@@ -56,10 +60,14 @@ class HarborDeploymentTest extends DevStackSpec {
5660
Unirest.get(harborBaseUrl).basicAuth("admin", "Harbor12345").asEmpty().status == 200
5761
hd.harborContainers.id.contains(hd.managerContainer.inspectContainer().id) //Make sure harborContainers returns manager container as well
5862
hd.harborContainers.every { it.state in [ContainerState.Status.Running.value, ContainerState.Status.Restarting.value] }
59-
hd.harborContainers.every {it.networkSettings.networks.keySet().toList() == [hd.deploymentNetworkName]}
60-
hd.harborContainers.collect {it.names.first()}.every {containerName ->
63+
hd.harborContainers.every {
64+
assert it.networkSettings.networks.keySet().toList() == [hd.deploymentNetworkName] : it.names.join(",") + " container has the wrong network: " + it.networkSettings.networks.keySet().toList().join(", ")
65+
return true
66+
}
67+
hd.harborContainers.collect { it.names.first() }.every { containerName ->
6168
String hostname = containerName[1..-1]
62-
hd.managerContainer.runBashCommandInContainer("ping ${hostname} -c 1 && echo Status: \$?", 5).last().contains("Status: 0")
69+
assert hd.managerContainer.runBashCommandInContainer("ping ${hostname} -c 1 && echo Status: \$?", 5).last().contains("Status: 0") : "Management container can not ping $hostname"
70+
return true
6371
}
6472

6573
when: "Stopping the deployment"
@@ -85,9 +93,10 @@ class HarborDeploymentTest extends DevStackSpec {
8593
hd.getHarborContainers().size() == 0
8694

8795
where:
88-
dockerHost | certPath | harborBaseUrl | harborVersion | harborBaseDir
89-
"" | "" | "http://localhost" | "v2.6.0" | "/tmp"
90-
dockerRemoteHost | dockerCertPath | "http://harbor.domain.se" | "v2.6.0" | "/tmp"
96+
dockerHost | certPath | harborBaseUrl | harborVersion | harborBaseDir
97+
"" | "" | "http://localhost" | "v2.7.3" | "/tmp/harbor"
98+
"" | "" | "http://localhost" | "v2.6.0" | "/tmp/harbor"
99+
//dockerRemoteHost | dockerCertPath | "http://harbor.domain.se" | "v2.6.0" | "/tmp"
91100

92101
}
93102
}

0 commit comments

Comments
 (0)