Merge branch 'dev/java-parse' into dev/llc

Author: Lenny Halseth

build.sbt

@@ -189,6 +189,7 @@ lazy val CodePulse = Project("CodePulse", file("codepulse"))
 			Dependencies.akka, Dependencies.reactive,
 			Dependencies.commons.io, Dependencies.commons.lang,
 			Dependencies.concLinkedHashMap, Dependencies.juniversalchardet, Dependencies.dependencyCheckCore,
-			Dependencies.slick, Dependencies.h2
+			Dependencies.slick, Dependencies.h2,
+			Dependencies.javaparser
 		) ++ Dependencies.asm ++ Dependencies.jackson ++ Dependencies.jna ++ Dependencies.logging
 	)

codepulse/src/main/resources/toserve/pages/projects/projects.js

@@ -248,7 +248,7 @@ $(document).ready(function(){
 					selection.refresh = true
 
 					sourceView.setSourceView(mode, source)
-					sourceView.scrollToLine(selection.metadata.methodStartLine - 1)
+					sourceView.scrollToLine(selection.metadata.methodStartLine)
 
 					coverageUpdateSubscription = Bacon.onValues(Trace.coverageRecords, Trace.activeRecordingsProp, function(coverage, activeRecordings) {
 						if (selection.refresh) {

codepulse/src/main/scala/com/secdec/codepulse/data/bytecode/MethodContentVisitor.scala

@@ -25,15 +25,15 @@ import java.io.InputStream
 
 object AsmVisitors {
 	// counterCallback(instructionCount)
-	type CounterCallback = (Int, Int, Int) => Unit
+	type CounterCallback = (Int, Int) => Unit
 
 	// methodCallback(methodSignature, instructionCount)
-	type MethodCallback = (String, String, Int, Int, Int) => Unit
+	type MethodCallback = (String, String, Int, Int) => Unit
 
 	def parseMethodsFromClass(classBytes: InputStream) = {
 		val reader = new ClassReader(classBytes)
-		val builder = List.newBuilder[(String, String, Int, Int, Int)]
-		val visitor = new ClassStructureVisitor2({ (file, sig, size, lineCount, methodStartLine) => builder += ((file, sig, size, lineCount, methodStartLine)) })
+		val builder = List.newBuilder[(String, String, Int, Int)]
+		val visitor = new ClassStructureVisitor2({ (file, sig, size, lineCount) => builder += ((file, sig, size, lineCount)) })
 		reader.accept(visitor, ClassReader.SKIP_FRAMES)
 		builder.result()
 	}
@@ -42,12 +42,10 @@ object AsmVisitors {
 class MethodContentVisitor(counterCallback: AsmVisitors.CounterCallback) extends MethodVisitor(Opcodes.ASM5) {
 	private var instructionCounter = 0
 	private var lineCounter = 0
-	private var methodStart = false
-	private var methodStartLine = -1
 
 	def inc() = instructionCounter += 1
-	override def visitCode = { instructionCounter = 0; lineCounter = 0; methodStart = true; methodStartLine = -1 }
-	override def visitEnd = { counterCallback(instructionCounter, lineCounter, methodStartLine) }
+	override def visitCode = { instructionCounter = 0; lineCounter = 0 }
+	override def visitEnd = { counterCallback(instructionCounter, lineCounter) }
 
 	override def visitFieldInsn(opcode: Int, owner: String, name: String, desc: String): Unit = inc()
 	override def visitIincInsn(v: Int, amt: Int): Unit = inc()
@@ -62,15 +60,7 @@ class MethodContentVisitor(counterCallback: AsmVisitors.CounterCallback) extends
 	override def visitTableSwitchInsn(min: Int, max: Int, dflt: Label, labels: Label*): Unit = inc()
 	override def visitTypeInsn(opcode: Int, tp: String): Unit = inc()
 	override def visitVarInsn(opcode: Int, v: Int): Unit = inc()
-	override def visitLineNumber(line: Int, start: Label): Unit = {
-		lineCounter += 1
-
-		if(methodStart) {
-			methodStartLine = line
-			methodStart = false
-		}
-
-	}
+	override def visitLineNumber(line: Int, start: Label): Unit = lineCounter += 1
 }
 
 class ClassStructureVisitor2(methodCallback: AsmVisitors.MethodCallback) extends ClassVisitor(Opcodes.ASM5) {
@@ -87,8 +77,8 @@ class ClassStructureVisitor2(methodCallback: AsmVisitors.MethodCallback) extends
 
 	override def visitMethod(access: Int, name: String, desc: String, sig: String, exceptions: Array[String]): MethodVisitor = {
 		val methodSignature = classSignature + "." + name + ";" + access + ";" + desc
-		val counterCallback: AsmVisitors.CounterCallback = (insnCount, lineCount, methodStartLine) => {
-			methodCallback(classFile, methodSignature, insnCount, lineCount, methodStartLine)
+		val counterCallback: AsmVisitors.CounterCallback = (insnCount, lineCount) => {
+			methodCallback(classFile, methodSignature, insnCount, lineCount)
 		}
 		new MethodContentVisitor(counterCallback)
 	}

codepulse/src/main/scala/com/secdec/codepulse/data/bytecode/parse/AccessFlags.scala

@@ -0,0 +1,99 @@
+/*
+ * Copyright 2018 Secure Decisions, a division of Applied Visions, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * This material is based on research sponsored by the Department of Homeland
+ * Security (DHS) Science and Technology Directorate, Cyber Security Division
+ * (DHS S&T/CSD) via contract number HHSP233201600058C.
+ */
+package com.secdec.codepulse.data.bytecode.parse
+
+import org.objectweb.asm.{Opcodes => O}
+
+/** Typesafe wrapper around an integer value that represents
+  * the combination of any number of `Opcodes.ACC_*` values.
+  *
+  * Note that not all of the flags provided by this class are
+  * necessarily used by Code Dx. They are simply mapped 1:1
+  * from what is found in the javadocs at
+  * http://asm.ow2.org/asm50/javadoc/user/org/objectweb/asm/Opcodes.html
+  *
+  * The `toStringFor...` methods try to choose a "sane" set of
+  * flags to present for stringification.
+  */
+case class AccessFlags(bits: Int) extends AnyVal {
+
+	@inline def is(thoseBits: Int) = (bits & thoseBits) != 0
+
+	def isAbstract = is(O.ACC_ABSTRACT)
+	def isAnnotation = is(O.ACC_ANNOTATION)
+	def isBridge = is(O.ACC_BRIDGE)
+	def isDeprecated = is(O.ACC_DEPRECATED)
+	def isEnum = is(O.ACC_ENUM)
+	def isFinal = is(O.ACC_FINAL)
+	def isInterface = is(O.ACC_INTERFACE)
+	def isMandated = is(O.ACC_MANDATED)
+	def isNative = is(O.ACC_NATIVE)
+	def isPrivate = is(O.ACC_PRIVATE)
+	def isProtected = is(O.ACC_PROTECTED)
+	def isPublic = is(O.ACC_PUBLIC)
+	def isStatic = is(O.ACC_STATIC)
+	def isStrict = is(O.ACC_STRICT)
+	def isSuper = is(O.ACC_SUPER)
+	def isSynchronized = is(O.ACC_SYNCHRONIZED)
+	def isSynthetic = is(O.ACC_SYNTHETIC)
+	def isTransient = is(O.ACC_TRANSIENT)
+	def isVarargs = is(O.ACC_VARARGS)
+	def isVolatile = is(O.ACC_VOLATILE)
+
+	private def stringifyWith(f: (String => Unit) => Unit) = {
+		val sb = new StringBuilder
+		var addedFirst = false
+		def add(s: String): Unit = {
+			if(addedFirst) sb += ' '
+			else addedFirst = true
+			sb append s
+		}
+		f(add _)
+		sb.result()
+	}
+
+	def toStringForClass = stringifyWith { add =>
+		if(isPublic) add("public")
+		if(isPrivate) add("private")
+		if(isProtected) add("protected")
+
+		if(isFinal) add("final")
+		if(isStatic) add("static")
+		if(isAbstract) add("abstract")
+
+		if(isAnnotation) add("@interface")
+		else if(isEnum) add("enum")
+		else if(isInterface) add("interface")
+		else add("class")
+	}
+
+	def toStringForMethod = stringifyWith { add =>
+		if(isPublic) add("public")
+		if(isPrivate) add("private")
+		if(isProtected) add("protected")
+
+		if(isFinal) add("final")
+		if(isStatic) add("static")
+		if(isAbstract) add("abstract")
+		if(isNative) add("native")
+		if(isSynchronized) add("synchronized")
+	}
+
+}

codepulse/src/main/scala/com/secdec/codepulse/data/bytecode/parse/BinaryClassSignature.scala

@@ -0,0 +1,99 @@
+/*
+ * Copyright 2018 Secure Decisions, a division of Applied Visions, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * This material is based on research sponsored by the Department of Homeland
+ * Security (DHS) Science and Technology Directorate, Cyber Security Division
+ * (DHS S&T/CSD) via contract number HHSP233201600058C.
+ */
+package com.secdec.codepulse.data.bytecode.parse
+
+/** Base trait describing "Binary Classes" in a codebase.
+  * A class signature should typically include a name, and type signature where applicable,
+  * but the actual details are up to the implementing classes.
+  * The main focus of this base trait is converting class signatures to and from their "memoized" forms.
+  * Classes stored in the database will have their "memo" in a dedicated column.
+  */
+sealed trait BinaryClassSignature {
+	def codebaseType: CodebaseType
+	def toMemo: BinaryClassSignatureMemo
+}
+
+object BinaryClassSignature {
+	def fromMemo(memo: BinaryClassSignatureMemo): BinaryClassSignature = {
+		CodebaseMemoParts.fromRaw(memo.raw) getOrElse { unrecognized(memo) } match {
+			case CodebaseMemoParts(CodebaseType.Java, 1, body) => JavaBinaryClassSignature.parseMemoV1(body)
+			case _ => unrecognized(memo)
+		}
+	}
+
+	@inline private def unrecognized(memo: BinaryClassSignatureMemo) = {
+		throw new IllegalArgumentException(s"Unrecognized binary class signature memo: ${memo.raw}")
+	}
+}
+
+/** "MEMO" representation of a `BinaryClassDeclaration`.
+  *
+  * Technically this is a value class wrapper for String, and it exists
+  * simply for the purpose of providing a type-level hint about what the
+  * String contains. Clients should not attempt to interact directly with
+  * the `raw` data. Instead they should call `BinaryClassDeclaration.fromMemo` to
+  * convert the memo to a nicer in-memory representation.
+  */
+final case class BinaryClassSignatureMemo(val raw: String) extends AnyVal
+
+// -------------------------------------------------------------
+//                           JAVA
+// -------------------------------------------------------------
+
+case class JavaBinaryClassSignature(
+	access: AccessFlags,
+	name: ClassName,
+	signature: JVMClassSignature
+) extends BinaryClassSignature {
+
+	def codebaseType: CodebaseType = CodebaseType.Java
+
+	def toMemo = new BinaryClassSignatureMemo(CodebaseMemoParts.buildRaw(codebaseType, 1, sb => {
+		sb append access.bits
+		sb append ';'
+		sb append name.slashedName
+		sb append ';'
+		sb append JVMSignatureConverter.toString(signature)
+	}))
+
+	override def toString = {
+		val sb = new StringBuilder
+		sb append access.toStringForClass
+		if(sb.nonEmpty) sb += ' '
+		sb append name.dottedName
+		if(signature.typeParams.nonEmpty) sb append signature.typeParams.mkString("<", ", ", ">")
+		sb append " extends " append signature.superClass
+		if(signature.interfaces.nonEmpty) sb append signature.interfaces.mkString(" implements ", ", ", "")
+		sb.result()
+	}
+}
+
+object JavaBinaryClassSignature {
+	def parseMemoV1(body: String) = {
+		// format is "<access>;<slashedName>;<signature>"
+		// note that signature will usually have ";" in it, so we limit the split to 3 parts
+		val Array(accessStr, slashedName, rawSignature) = body.split(";", 3)
+		JavaBinaryClassSignature(
+			AccessFlags(accessStr.toInt),
+			ClassName.fromSlashed(slashedName),
+			JVMSignatureConverter.fromString[JVMClassSignature](rawSignature)
+		)
+	}
+}

codepulse/src/main/scala/com/secdec/codepulse/data/bytecode/parse/BinaryMethodSignature.scala

@@ -0,0 +1,105 @@
+/*
+ * Copyright 2018 Secure Decisions, a division of Applied Visions, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * This material is based on research sponsored by the Department of Homeland
+ * Security (DHS) Science and Technology Directorate, Cyber Security Division
+ * (DHS S&T/CSD) via contract number HHSP233201600058C.
+ */
+package com.secdec.codepulse.data.bytecode.parse
+
+sealed trait BinaryMethodSignature extends MethodSignature {
+	def toMemo: BinaryMethodSignatureMemo
+}
+object BinaryMethodSignature {
+	def fromMemo(memo: BinaryMethodSignatureMemo): BinaryMethodSignature = {
+		CodebaseMemoParts.fromRaw(memo.raw) getOrElse { unrecognized(memo) } match {
+			case CodebaseMemoParts(CodebaseType.Java, 1, body) => JavaBinaryMethodSignature.parseMemoV1(body)
+			case _ => unrecognized(memo)
+		}
+	}
+
+	@inline private def unrecognized(memo: BinaryMethodSignatureMemo): Nothing = {
+		throw new IllegalArgumentException(s"Unrecognized binary method signature memo: ${memo.raw}")
+	}
+}
+
+/** "MEMO" representation of a `BinaryMethodSignature`.
+  *
+  * Technically this is a value class wrapper for String, and it exists
+  * simply for the purpose of providing a type-level hint about what the
+  * String contains. Clients should not attempt to interact directly with
+  * the `raw` data. Instead, they should call `MethodSignature.fromMemo`
+  * to convert the memo to a nicer in-memory representation.
+  *
+  * @param raw
+  */
+final case class BinaryMethodSignatureMemo(raw: String) extends AnyVal
+
+// -------------------------------------------------------------
+//                           JAVA
+// -------------------------------------------------------------
+
+/** Represents a java method signature found via bytecode inspection.
+  * Method signatures in this form can be converted to and from "memos"
+  * in order to be stored in the database.
+  */
+case class JavaBinaryMethodSignature(
+	access: AccessFlags,
+	name: String,
+	typeSignature: JVMMethodTypeSignature
+) extends BinaryMethodSignature {
+
+	def codebaseType: CodebaseType = CodebaseType.Java
+
+	def simplifiedReturnType = SimplifiedType of typeSignature.returnType
+	def simplifiedArgumentTypes = typeSignature.paramTypes.map(SimplifiedType.of)
+
+	def toMemo = new BinaryMethodSignatureMemo(CodebaseMemoParts.buildRaw(codebaseType, 1, sb => {
+		sb append access.bits
+		sb append ';'
+		sb append name
+		sb append ';'
+		sb append JVMSignatureConverter.toString(typeSignature)
+	}))
+
+	override def toString = {
+		val sb = new StringBuilder
+		sb append access.toStringForMethod
+		sb += ' '
+		if(typeSignature.typeParams.nonEmpty) sb append typeSignature.typeParams.mkString("<", ", ", "> ")
+		sb append typeSignature.returnType
+		sb += ' '
+		sb append name
+		sb append typeSignature.paramTypes.mkString("(", ", ", ")")
+		if(typeSignature.exceptionTypes.nonEmpty) sb append typeSignature.exceptionTypes.mkString(" throws ", ", ", "")
+		sb.result()
+	}
+
+}
+
+object JavaBinaryMethodSignature {
+
+	def parseMemoV1(body: String) = {
+		// format is "<access>;<name>;<signature>"
+		// note that signature will usually have ";" in it, so we limit the split to 3 parts
+		val Array(accessStr, name, rawSignature) = body.split(";", 3)
+		JavaBinaryMethodSignature(
+			AccessFlags(accessStr.toInt),
+			name,
+			JVMSignatureConverter.fromString[JVMMethodTypeSignature](rawSignature)
+		)
+	}
+
+}