Skip to content

Commit 0f2cd31

Browse files
bafflesbaffles
committed
Add a SBT task to fetch the dependencies for the packaging step.
1 parent e76c051 commit 0f2cd31

File tree

5 files changed

+355
-4
lines changed

5 files changed

+355
-4
lines changed

‎project/Build.scala‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@ import com.earldouglas.xsbtwebplugin._
2828
import WebPlugin._
2929
import WebappPlugin._
3030
import Distributor.{ Keys => DistribKeys, distribSettings }
31+
import DependencyFetcher.dependencyFetcherSettings
3132
import sbtassembly.Plugin._
3233
import AssemblyKeys._
3334

@@ -66,6 +67,7 @@ object BuildDef extends Build with VersionSystem {
6667
.settings(webappProjectSettings: _*)
6768
.settings(EclipseKeys.withSource := true)
6869
.settings(distribSettings: _*)
70+
.settings(dependencyFetcherSettings: _*)
6971
.settings(assemblySettings: _*)
7072
.settings(
7173
resolvers ++= dependencyResolvers,

‎project/BuildKeys.scala‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,4 +27,6 @@ object BuildKeys {
2727
val packageEmbeddedWin32 = TaskKey[File]("package-embedded-win32", "Creates a ZIP distribution of the node-webkit embedded version of the current project for Windows (32-bit)")
2828
val packageEmbeddedOsx = TaskKey[File]("package-embedded-osx", "Creates a ZIP distribution of the node-webkit embedded version of the current project for OS X (32/64-bit)")
2929
val packageEmbeddedLinuxX86 = TaskKey[File]("package-embedded-linux-x86", "Creates a ZIP distribution of the node-webkit embedded version of the current project for Linux (x86)")
30+
31+
val fetchDependencies = TaskKey[Unit]("fetch-package-dependencies")
3032
}

‎project/DependencyFetcher.scala‎

Lines changed: 344 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,344 @@
1+
/*
2+
* Code Pulse: A real-time code coverage testing tool. For more information
3+
* see http://code-pulse.com
4+
*
5+
* Copyright (C) 2014 Applied Visions - http://securedecisions.avi.com
6+
*
7+
* Licensed under the Apache License, Version 2.0 (the "License");
8+
* you may not use this file except in compliance with the License.
9+
* You may obtain a copy of the License at
10+
*
11+
* http://www.apache.org/licenses/LICENSE-2.0
12+
*
13+
* Unless required by applicable law or agreed to in writing, software
14+
* distributed under the License is distributed on an "AS IS" BASIS,
15+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16+
* See the License for the specific language governing permissions and
17+
* limitations under the License.
18+
*/
19+
20+
import sbt._
21+
import Keys._
22+
import BuildKeys._
23+
import Project.Initialize
24+
25+
import scala.collection.JavaConversions._
26+
27+
import java.io.{ BufferedInputStream, InputStream, File, FileInputStream, FileOutputStream }
28+
import java.net.{ HttpURLConnection, URL, URLConnection }
29+
30+
import org.apache.commons.compress.archivers.tar.TarArchiveInputStream
31+
import org.apache.commons.compress.archivers.zip.ZipArchiveInputStream
32+
import org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream
33+
import org.apache.commons.io.IOUtils
34+
35+
/** Helper for fetching the Code Pulse dependencies from the web.
36+
*
37+
* @author robertf
38+
*/
39+
object DependencyFetcher extends BuildExtra {
40+
41+
import Distributor.Keys.{ Distribution, distDeps }
42+
43+
sealed trait Platform
44+
object Platform {
45+
case object Unspecified extends Platform
46+
case object Windows extends Platform
47+
case object Linux extends Platform
48+
case object OSX extends Platform
49+
}
50+
51+
sealed trait FileFormat
52+
object FileFormat {
53+
case object Raw extends FileFormat
54+
case object Zip extends FileFormat
55+
case object TarGz extends FileFormat
56+
}
57+
58+
case class DependencyFile(platform: Platform, url: String, format: FileFormat)
59+
60+
sealed trait Dependency { def name: String; def destPath: String }
61+
sealed trait PackageHelper { def trimPath(platform: Platform)(path: String): String }
62+
case class JreDependency(name: String, rawPath: String, destPath: String, files: List[DependencyFile]) extends Dependency with PackageHelper {
63+
private val trimPathRegex = ("^\\Q" + rawPath + "\\E(?:\\.jre)?/").r
64+
def trimPath(platform: Platform)(path: String) = trimPathRegex.replaceFirstIn(path, "")
65+
}
66+
case class PlatformDependency(name: String, destPath: String, files: List[DependencyFile]) extends Dependency
67+
case class CommonDependency(name: String, destPath: String, file: DependencyFile) extends Dependency
68+
case class ToolDependency(name: String, destPath: String, file: DependencyFile) extends Dependency
69+
70+
val BufferLen = 4096
71+
72+
object Keys {
73+
val PackageDependencies = config("package-dependencies")
74+
75+
val jreWindows = SettingKey[String]("jre-windows")
76+
val jreLinux = SettingKey[String]("jre-linux")
77+
val jreOsx = SettingKey[String]("jre-osx")
78+
79+
val nwkWindows = SettingKey[String]("nwk-windows")
80+
val nwkLinux = SettingKey[String]("nwk-linux")
81+
val nwkOsx = SettingKey[String]("nwk-osx")
82+
83+
val jetty = SettingKey[String]("jetty")
84+
val resourcer = SettingKey[String]("resourcer")
85+
86+
val dependencyList = TaskKey[Seq[Dependency]]("dependency-list")
87+
}
88+
89+
import Keys._
90+
91+
private class ProgressInputStream(underlying: InputStream, message: String, size: Long) extends InputStream {
92+
private var bytesRead = 0L
93+
private def progress(chunk: Int) {
94+
bytesRead += chunk
95+
val pct = 100 * bytesRead / size
96+
print(message + "... " + pct + "% complete\r")
97+
}
98+
99+
override def read() = {
100+
val b = underlying.read
101+
if (b != -1) progress(1)
102+
b
103+
}
104+
105+
override def read(b: Array[Byte]) = {
106+
val chunk = underlying.read(b)
107+
if (chunk >= 0) progress(chunk)
108+
chunk
109+
}
110+
111+
override def read(b: Array[Byte], off: Int, len: Int) = {
112+
val chunk = underlying.read(b, off, len)
113+
if (chunk >= 0) progress(chunk)
114+
chunk
115+
}
116+
117+
override def available() = underlying.available
118+
override def close() = underlying.close
119+
override def mark(readlimit: Int) = underlying.mark(readlimit)
120+
override def markSupported() = underlying.markSupported
121+
override def reset() = underlying.reset
122+
override def skip(n: Long) = underlying.skip(n)
123+
}
124+
125+
private def doDownload(label: String, log: Logger, dep: Dependency, file: DependencyFile, dest: File, preConnect: URLConnection => Unit = _ => ()) {
126+
val conn = {
127+
def connect(url: URL): HttpURLConnection = {
128+
val conn = url.openConnection.asInstanceOf[HttpURLConnection]
129+
preConnect(conn)
130+
131+
conn setInstanceFollowRedirects true
132+
conn setDoInput true
133+
134+
conn.connect
135+
136+
val status = conn.getResponseCode
137+
status match {
138+
case HttpURLConnection.HTTP_MOVED_TEMP | HttpURLConnection.HTTP_MOVED_PERM | HttpURLConnection.HTTP_SEE_OTHER =>
139+
val redirect = conn getHeaderField "Location"
140+
connect(new URL(redirect))
141+
142+
case _ => conn
143+
}
144+
}
145+
146+
connect(new URL(file.url))
147+
}
148+
149+
val stream = new BufferedInputStream(
150+
new ProgressInputStream(conn.getInputStream, "Downloading " + label, conn.getContentLengthLong)
151+
)
152+
153+
try {
154+
lazy val pathTrimmer: String => String = dep match {
155+
case ph: PackageHelper => ph.trimPath(file.platform)
156+
case _ => identity
157+
}
158+
159+
file.format match {
160+
case FileFormat.Raw => processRaw(stream, dest)
161+
case FileFormat.Zip => processZip(stream, dest, pathTrimmer)
162+
case FileFormat.TarGz => processTarGz(stream, dest, pathTrimmer)
163+
}
164+
} finally {
165+
IOUtils closeQuietly stream
166+
}
167+
168+
// "Downloading <label>... xxx% complete" = 18 extra characters to clear
169+
log.info("Downloaded " + label + " ")
170+
}
171+
172+
private def processRaw(stream: InputStream, destFile: File) {
173+
val fos = new FileOutputStream(destFile)
174+
try {
175+
IOUtils.copyLarge(stream, fos)
176+
} finally {
177+
IOUtils closeQuietly fos
178+
}
179+
}
180+
181+
private def processZip(stream: InputStream, destFolder: File, pathTrim: String => String) {
182+
destFolder.mkdirs
183+
184+
val zin = new ZipArchiveInputStream(stream)
185+
186+
try {
187+
val entries = Iterator.continually { zin.getNextZipEntry }.takeWhile { _ != null }
188+
val buffer = new Array[Byte](BufferLen)
189+
190+
for (entry <- entries if !entry.isDirectory) {
191+
val out = destFolder / pathTrim(entry.getName)
192+
out.getParentFile.mkdirs
193+
194+
val fos = new FileOutputStream(out)
195+
try {
196+
val reads = Iterator.continually { zin.read(buffer, 0, BufferLen) }.takeWhile { _ > 0 }
197+
for (read <- reads) fos.write(buffer, 0, read)
198+
} finally {
199+
IOUtils closeQuietly fos
200+
}
201+
202+
out setLastModified entry.getLastModifiedDate.getTime
203+
}
204+
} finally {
205+
IOUtils closeQuietly zin
206+
}
207+
}
208+
209+
private def processTarGz(stream: InputStream, destFolder: File, pathTrim: String => String) {
210+
destFolder.mkdirs
211+
212+
val gzin = new GzipCompressorInputStream(stream)
213+
val tarin = new TarArchiveInputStream(gzin)
214+
215+
try {
216+
val entries = Iterator.continually { tarin.getNextTarEntry }.takeWhile { _ != null }
217+
val buffer = new Array[Byte](BufferLen)
218+
219+
for (entry <- entries if !entry.isDirectory) {
220+
val out = destFolder / pathTrim(entry.getName)
221+
out.getParentFile.mkdirs
222+
223+
val fos = new FileOutputStream(out)
224+
try {
225+
val reads = Iterator.continually { tarin.read(buffer, 0, BufferLen) }.takeWhile { _ > 0 }
226+
for (read <- reads) fos.write(buffer, 0, read)
227+
} finally {
228+
IOUtils closeQuietly fos
229+
}
230+
231+
out setLastModified entry.getLastModifiedDate.getTime
232+
}
233+
} finally {
234+
IOUtils closeQuietly tarin
235+
IOUtils closeQuietly gzin
236+
}
237+
}
238+
239+
object Settings {
240+
val fetchDependenciesTask: Initialize[Task[Unit]] = (distDeps in Distribution, dependencyList in PackageDependencies, streams) map { (distDepsFolder, deps, streams) =>
241+
val log = streams.log
242+
243+
val commonFolder = distDepsFolder / "common"
244+
val toolsFolder = distDepsFolder / "tools"
245+
def getPlatformFolder(platform: Platform) = platform match {
246+
case Platform.Windows => distDepsFolder / "win32"
247+
case Platform.Linux => distDepsFolder / "linux-x86"
248+
case Platform.OSX => distDepsFolder / "osx"
249+
case Platform.Unspecified => commonFolder
250+
}
251+
252+
deps foreach {
253+
case dep @ JreDependency(name, _, destPath, files) =>
254+
for (file @ DependencyFile(platform, url, _) <- files)
255+
doDownload(
256+
name + " [" + platform + "]", log,
257+
dep, file,
258+
getPlatformFolder(platform) / destPath,
259+
{ _.setRequestProperty("Cookie", "oraclelicense=accept-securebackup-cookie") }
260+
)
261+
262+
case dep @ PlatformDependency(name, destPath, files) =>
263+
for (file @ DependencyFile(platform, url, _) <- files)
264+
doDownload(
265+
name + " [" + platform + "]", log,
266+
dep, file,
267+
getPlatformFolder(platform) / destPath
268+
)
269+
270+
case dep @ CommonDependency(name, destPath, file) =>
271+
doDownload(
272+
name + " [common]", log,
273+
dep, file,
274+
commonFolder / destPath
275+
)
276+
277+
case dep @ ToolDependency(name, destPath, file) =>
278+
doDownload(
279+
name + " [tool]", log,
280+
dep, file,
281+
toolsFolder / destPath
282+
)
283+
}
284+
}
285+
}
286+
287+
import Settings._
288+
289+
lazy val dependencyFetcherSettings: Seq[Setting[_]] = Seq(
290+
dependencyList in PackageDependencies := Nil,
291+
292+
jreWindows in PackageDependencies := "http://download.oracle.com/otn-pub/java/jdk/7u55-b13/jre-7u55-windows-i586.tar.gz",
293+
jreLinux in PackageDependencies := "http://download.oracle.com/otn-pub/java/jdk/7u55-b13/jre-7u55-linux-i586.tar.gz",
294+
jreOsx in PackageDependencies := "http://download.oracle.com/otn-pub/java/jdk/7u55-b13/jre-7u55-macosx-x64.tar.gz",
295+
dependencyList in PackageDependencies <+= (jreWindows in PackageDependencies, jreLinux in PackageDependencies, jreOsx in PackageDependencies) map { (jreWin, jreLin, jreOsx) =>
296+
JreDependency(
297+
name = "jre 7u55", rawPath = "jre1.7.0_55", destPath = "jre",
298+
files = List(
299+
DependencyFile(Platform.Windows, jreWin, FileFormat.TarGz),
300+
DependencyFile(Platform.Linux, jreLin, FileFormat.TarGz),
301+
DependencyFile(Platform.OSX, jreOsx, FileFormat.TarGz)
302+
)
303+
)
304+
},
305+
306+
nwkWindows in PackageDependencies := "http://dl.node-webkit.org/v0.9.2/node-webkit-v0.9.2-win-ia32.zip",
307+
nwkLinux in PackageDependencies := "http://dl.node-webkit.org/v0.9.2/node-webkit-v0.9.2-linux-ia32.tar.gz",
308+
nwkOsx in PackageDependencies := "http://dl.node-webkit.org/v0.9.2/node-webkit-v0.9.2-osx-ia32.zip",
309+
dependencyList in PackageDependencies <+= (nwkWindows in PackageDependencies, nwkLinux in PackageDependencies, nwkOsx in PackageDependencies) map { (nwkWin, nwkLin, nwkOsx) =>
310+
new PlatformDependency(
311+
name = "node-webkit v0.9.2", destPath = "node-webkit",
312+
files = List(
313+
DependencyFile(Platform.Windows, nwkWin, FileFormat.Zip),
314+
DependencyFile(Platform.Linux, nwkLin, FileFormat.TarGz),
315+
DependencyFile(Platform.OSX, nwkOsx, FileFormat.Zip)
316+
)
317+
) with PackageHelper {
318+
private val trimPathRegex = ("^\\Qnode-webkit-v0.9.2-linux-ia32\\E/").r
319+
def trimPath(platform: Platform)(path: String) = platform match {
320+
case Platform.Linux => trimPathRegex.replaceFirstIn(path, "")
321+
case _ => path
322+
}
323+
}
324+
},
325+
326+
jetty in PackageDependencies := "http://mirrors.xmission.com/eclipse/jetty/9.1.4.v20140401/dist/jetty-distribution-9.1.4.v20140401.zip",
327+
//resourcer in PackageDependencies := "http://anolis.codeplex.com/downloads/get/81545",
328+
resourcer in PackageDependencies := "https://dl.dropboxusercontent.com/s/zifogi9efgtsq1s/Anolis.Resourcer-0.9.zip?dl=1",
329+
dependencyList in PackageDependencies <++= (jetty in PackageDependencies, resourcer in PackageDependencies) map { (jetty, resourcer) =>
330+
val jettyDep = new CommonDependency("Jetty 9.1.4 v20140401", "jetty", DependencyFile(Platform.Unspecified, jetty, FileFormat.Zip)) with PackageHelper {
331+
private val trimPathRegex = ("^\\Qjetty-distribution-9.1.4.v20140401\\E/").r
332+
def trimPath(platform: Platform)(path: String) = {
333+
trimPathRegex.replaceFirstIn(path, "")
334+
}
335+
}
336+
337+
val resourcerDep = ToolDependency("Resourcer", "resourcer", DependencyFile(Platform.Unspecified, resourcer, FileFormat.Zip))
338+
339+
jettyDep :: resourcerDep :: Nil
340+
},
341+
342+
fetchDependencies <<= fetchDependenciesTask
343+
)
344+
}

‎project/Distributor.scala‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -103,7 +103,7 @@ object Distributor extends BuildExtra {
103103
val jetty = depsDir / "common" / "jetty"
104104

105105
if (!jetty.exists)
106-
sys.error("Missing jetty. Please download and place in " + jetty + ".")
106+
sys.error("Missing jetty. Please run `fetch-package-dependencies` or download and place in " + jetty + ".")
107107

108108
val jettyExclusions = List(
109109
"demo-base/", "etc/", "start.d/", "start.ini"
@@ -130,7 +130,7 @@ object Distributor extends BuildExtra {
130130
val nwk = deps / platform / "node-webkit"
131131

132132
if (!nwk.exists)
133-
sys.error("Missing node-webkit for " + platform + ". Please download and place in " + nwk + ".")
133+
sys.error("Missing node-webkit for " + platform + ". Please run `fetch-package-dependencies` or download and place in " + nwk + ".")
134134

135135
val nwkFiles: Seq[ZipEntry] = nwk.*** x rebase(nwk, rootZipFolder) map {
136136
// replace \ in paths with /, for easier matching below
@@ -243,7 +243,7 @@ object Distributor extends BuildExtra {
243243
val jreDest = rootZipFolder + "jre/"
244244

245245
if (!jre.exists)
246-
sys.error("Missing JRE for " + platform + ". Please download and place in " + jre + ".")
246+
sys.error("Missing JRE for " + platform + ". Please run `fetch-package-dependencies` or download and place in " + jre + ".")
247247

248248
val jreFiles: Seq[ZipEntry] = appResource(platform, rootZipFolder, jre.*** x rebase(jre, jreDest)) map {
249249
// replace \ in paths with /, for easier matching below

0 commit comments

Comments
 (0)