Analyze JSP files to estimate their instruction count.

Author: baffles

codepulse/src/main/scala/com/secdec/codepulse/data/jsp/JasperJspAdapter.scala

@@ -29,32 +29,33 @@ import com.secdec.codepulse.data.bytecode.CodeTreeNode
   * @author robertf
   */
 class JasperJspAdapter extends JspAdapter {
-	private val jsps = List.newBuilder[String]
+	private val jsps = List.newBuilder[(String, Int)]
 	private val webinfs = List.newBuilder[String]
 
-	def addJsp(path: String) = jsps += path
+	def addJsp(path: String, size: Int) = jsps += path -> size
 	def addWebinf(path: String) = webinfs += path
 
 	def build(codeForestBuilder: CodeForestBuilder): List[(String, Int)] = {
 		val jsps = this.jsps.result
 		val webinfs = this.webinfs.result
 
-		val jspClasses = Set.newBuilder[String]
+		val jspClasses = Set.newBuilder[(String, Int)]
 
 		for (webinf <- webinfs) {
 			val parent = Option(new File(webinf).getParent) getOrElse ""
 			val parentLen = parent.length
 			jspClasses ++= jsps
-				.filter(_.startsWith(parent))
-				.map(_.substring(parentLen))
+				.filter(_._1.startsWith(parent))
+				.map { case (clazz, size) => clazz.substring(parentLen) -> size }
 		}
 
 		// build up
-		jspClasses.result.toList map { clazz =>
-			val jspClassName = JasperUtils.makeJavaClass(clazz)
-			val displayName = clazz.split('/').filter(!_.isEmpty).toList
-			val node = codeForestBuilder.getOrAddJsp(displayName, 100)
-			jspClassName -> node.id
+		jspClasses.result.toList map {
+			case (clazz, size) =>
+				val jspClassName = JasperUtils.makeJavaClass(clazz)
+				val displayName = clazz.split('/').filter(!_.isEmpty).toList
+				val node = codeForestBuilder.getOrAddJsp(displayName, size)
+				jspClassName -> node.id
 		}
 	}
 }

codepulse/src/main/scala/com/secdec/codepulse/data/jsp/JspAdapter.scala

@@ -26,7 +26,7 @@ import com.secdec.codepulse.data.bytecode.CodeForestBuilder
   * @author robertf
   */
 trait JspAdapter {
-	def addJsp(path: String): Unit
+	def addJsp(path: String, size: Int): Unit
 	def addWebinf(path: String): Unit
 	def build(cfb: CodeForestBuilder): List[(String, Int)]
 }

codepulse/src/main/scala/com/secdec/codepulse/data/jsp/JspAnalyzer.scala

@@ -0,0 +1,61 @@
+/*
+ * Code Pulse: A real-time code coverage testing tool. For more information
+ * see http://code-pulse.com
+ *
+ * Copyright (C) 2014 Applied Visions - http://securedecisions.avi.com
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.secdec.codepulse.data.jsp
+
+/** Analyzer for JSP files. The purpose is to make a rough approximation of
+  * how many instructions are contained in a JSP file.
+  *
+  * We do this by scanning through the file and first counting how many
+  * statements are within. Then we apply a multiplier to the statement count
+  * and add a constant overhead value to approximate the number of instructions
+  * for the file.
+  *
+  * @author robertf
+  */
+object JspAnalyzer {
+	val ConstantOverhead = 96
+	val LineMultiplier = 3
+	val StatementMultiplier = 5
+
+	private val Block = (raw"(?s)<(%(?:=|!)?)(.*?)%>").r
+
+	private val ExpressionBlock = "%="
+	private val ScriptBlock = "%"
+	private val DeclarationBlock = "%!"
+
+	def analyze(contents: String) = {
+		val blocks = Block.findAllMatchIn(contents).toList
+		val statementCount = blocks.map { m =>
+			val statementCount = (m group 2).count(_ == ';')
+
+			m group 1 match {
+				case ExpressionBlock => 1 // count as 1 statement
+				case ScriptBlock => 1 + statementCount // (1 + number of ';') statements
+				case DeclarationBlock => statementCount // (number of ';') statements
+				case _ => 0
+			}
+		}.sum
+
+		val blockLineCount = blocks.map(_ group 2).map(_ count (_ == '\n')).sum
+		val lineCount = contents.count(_ == '\n') - blockLineCount
+
+		ConstantOverhead + StatementMultiplier * statementCount + LineMultiplier * lineCount
+	}
+}

codepulse/src/main/scala/com/secdec/codepulse/tracer/TraceUploadData.scala

@@ -25,11 +25,13 @@ import com.secdec.codepulse.data.bytecode.CodeForestBuilder
 import com.secdec.codepulse.data.trace.TraceId
 import com.secdec.codepulse.tracer.export.TraceImporter
 import com.secdec.codepulse.util.RichFile.RichFile
+import com.secdec.codepulse.util.SmartLoader
 import com.secdec.codepulse.util.ZipEntryChecker
 import net.liftweb.common.Box
 import net.liftweb.common.Failure
 import org.apache.commons.io.FilenameUtils
 import com.secdec.codepulse.data.jsp.JasperJspAdapter
+import com.secdec.codepulse.data.jsp.JspAnalyzer
 import com.secdec.codepulse.data.trace.TraceData
 import scala.concurrent.Future
 import scala.concurrent.ExecutionContext.Implicits.global
@@ -111,6 +113,8 @@ object TraceUploadData {
 		//TODO: make this configurable somehow
 		val jspAdapter = new JasperJspAdapter
 
+		val loader = new SmartLoader
+
 		ZipEntryChecker.forEachEntry(file) { (filename, entry, contents) =>
 			val groupName = if (filename == file.getName) RootGroupName else s"JARs/$filename"
 			if (!entry.isDirectory) {
@@ -123,7 +127,9 @@ object TraceUploadData {
 						} methodCorrelationsBuilder += (name -> treeNode.id)
 
 					case "jsp" =>
-						jspAdapter addJsp entry.getName
+						val jspContents = loader loadStream contents
+						val jspSize = JspAnalyzer analyze jspContents
+						jspAdapter.addJsp(entry.getName, jspSize)
 
 					case _ => // nothing
 				}

codepulse/src/main/scala/com/secdec/codepulse/util/SmartLoader.scala

@@ -0,0 +1,86 @@
+/*
+ * Code Pulse: A real-time code coverage testing tool. For more information
+ * see http://code-pulse.com
+ *
+ * Copyright (C) 2014 Applied Visions - http://securedecisions.avi.com
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.secdec.codepulse.util
+
+import java.io.ByteArrayInputStream
+import java.io.ByteArrayOutputStream
+import java.io.InputStream
+
+import scala.io.Codec
+import scala.io.Codec.string2codec
+import scala.io.Source
+
+import org.mozilla.universalchardet.UniversalDetector
+
+/** Loader that detects the proper character set when loading the contents of a
+  * stream. Uses juniversalchardet.
+  *
+  * @author robertf
+  */
+class SmartLoader {
+	private val BufferSize = 4096
+
+	val detector = new UniversalDetector(null)
+
+	/** Detect the character set used for `stream`.
+	  * *NOTE*: this consumes the stream.
+	  */
+	def detectCharset(stream: InputStream): String = {
+		detector.reset
+
+		val buffer = new Array[Byte](BufferSize)
+
+		val chunks = Iterator.continually {
+			stream.read(buffer, 0, buffer.length) -> buffer
+		} takeWhile { _._1 > 0 }
+
+		for ((len, buffer) <- chunks.takeWhile(_ => !detector.isDone)) {
+			detector.handleData(buffer, 0, len)
+		}
+
+		detector.dataEnd
+		detector.getDetectedCharset
+	}
+
+	/** Detect charset for `stream` and decode all contents of the stream,
+	  * returning a string. This buffers the entire contents of the stream in
+	  * memory.
+	  */
+	def loadStream(stream: InputStream): String = {
+		val bytes = {
+			val bos = new ByteArrayOutputStream
+			val buffer = new Array[Byte](BufferSize)
+
+			val chunks = Iterator.continually {
+				stream.read(buffer, 0, buffer.length) -> buffer
+			} takeWhile { _._1 > 0 }
+
+			for ((len, buffer) <- chunks) {
+				bos.write(buffer, 0, len)
+			}
+
+			bos.flush
+			bos.toByteArray
+		}
+
+		val charset = Option(detectCharset(new ByteArrayInputStream(bytes))).map[Codec](identity)
+		Source.fromInputStream(new ByteArrayInputStream(bytes))(charset getOrElse Codec.UTF8).mkString
+	}
+}

project/Build.scala

@@ -35,7 +35,7 @@ object BuildDef extends Build with VersionSystem {
 
 	lazy val liftDependencies = Seq(lift_webkit, servletApi, logback, slf4j)
 	lazy val testDependencies = Seq(junit, specs, scalatest)
-	lazy val libDependencies = Seq(akka, reactive, jna, commons.io, concLinkedHashMap) ++ asm ++ jackson
+	lazy val libDependencies = Seq(akka, reactive, jna, commons.io, concLinkedHashMap, juniversalchardet) ++ asm ++ jackson
 	lazy val dbDependencies = Seq(slick, h2)
 
 	val baseCompilerSettings = Seq(

project/Dependencies.scala

@@ -46,6 +46,7 @@ object Dependencies {
 		"com.fasterxml.jackson.core" % "jackson-core" % "2.3.2",
 		"com.fasterxml.jackson.module" %% "jackson-module-scala" % "2.3.2"
 	)
+	lazy val juniversalchardet = "com.googlecode.juniversalchardet" % "juniversalchardet" % "1.0.3"
 
 	// database related
 	lazy val slick = "com.typesafe.slick" %% "slick" % "2.0.1"