Use Java Parser to get start lines for Java methods and enable scrolling to method in source viewer

Author: Lenny Halseth

build.sbt

@@ -190,6 +190,7 @@ lazy val CodePulse = Project("CodePulse", file("codepulse"))
 			Dependencies.commons.io, Dependencies.commons.lang,
 			Dependencies.concLinkedHashMap, Dependencies.juniversalchardet, Dependencies.dependencyCheckCore,
 			Dependencies.slick, Dependencies.h2,
-			Dependencies.antlr
+			Dependencies.antlr,
+			Dependencies.javaparser
 		) ++ Dependencies.asm ++ Dependencies.jackson ++ Dependencies.jna ++ Dependencies.logging
 	)

codepulse/src/main/resources/toserve/pages/projects/projects.js

@@ -248,7 +248,7 @@ $(document).ready(function(){
 					selection.refresh = true
 
 					sourceView.setSourceView(mode, source)
-					sourceView.scrollToLine(selection.metadata.methodStartLine - 1)
+					sourceView.scrollToLine(selection.metadata.methodStartLine)
 
 					coverageUpdateSubscription = Bacon.onValues(Trace.coverageRecords, Trace.activeRecordingsProp, function(coverage, activeRecordings) {
 						if (selection.refresh) {

…de/parse/MethodDeclarationListener.scala …/data/bytecode/parse/ParseListener.scalacodepulse/src/main/scala/com/secdec/codepulse/data/bytecode/parse/MethodDeclarationListener.scala renamed to codepulse/src/main/scala/com/secdec/codepulse/data/bytecode/parse/ParseListener.scala codepulse/src/main/scala/com/secdec/codepulse/data/bytecode/parse/MethodDeclarationListener.scala renamed to codepulse/src/main/scala/com/secdec/codepulse/data/bytecode/parse/ParseListener.scala

@@ -21,12 +21,41 @@ package com.secdec.codepulse.data.bytecode.parse
 
 import com.secdec.codepulse.parsers.java9
 import com.secdec.codepulse.parsers.java9.{ Java9BaseListener, Java9Parser }
+import scala.collection.JavaConverters._
+
+import org.antlr.v4.runtime.CharStreams
+import org.antlr.v4.runtime.misc.Interval
+
+case class MethodInfo(name: String, startLine: Int)
+case class ClassInfo(name: String, memberClasses: List[ClassInfo], memberMethods: List[MethodInfo])
+
+class ParseListener(callback: (String, Int) => Unit) extends Java9BaseListener {
+	var currentClass: Option[ClassInfo] = None
+	var currentMethods = List.empty[MethodInfo]
+
+	override def enterClassDeclaration(ctx: Java9Parser.ClassDeclarationContext) = {
+
+	}
+
+	override def exitClassDeclaration(ctx: Java9Parser.ClassDeclarationContext) = {
+
+	}
 
-class MethodDeclarationListener(callback: (String, Int) => Unit) extends Java9BaseListener {
 	override def enterMethodDeclaration(ctx: Java9Parser.MethodDeclarationContext) = {
-		val name = ctx.getText()
 		val start = ctx.start.getLine()
 
+		val modifiers = ctx.methodModifier().asScala.map(_.getText())
+		val header = ctx.methodHeader().getText()
+
+		val name = modifiers.mkString("") + header
+
 		callback(name, start)
+
+//		val input = ctx.methodHeader().getInputStream()
+//		val interval = new Interval(ctx.methodHeader().start.getStartIndex(), ctx.methodHeader().stop.getStopIndex())
+//		val t = input.getText(interval)
+
+		val method = MethodInfo(name, start)
+		currentMethods = method :: currentMethods
 	}
 }

codepulse/src/main/scala/com/secdec/codepulse/data/storage/ZippedStorage.scala

@@ -132,8 +132,8 @@ class ZippedStorage(zipFile: ZipFile) extends Storage {
 	protected def readEntries[T](filter: ZipEntry => Boolean, entryPath: Option[NestedPath], filename: String, stream: InputStream, recursive: Boolean = true)(read: (String, Option[NestedPath], ZipEntry, InputStream) => Unit): Unit = {
 		val zipStream = new ZipInputStream(stream)
 		try {
-			val entryStream = Stream.continually(Try { zipStream.getNextEntry })
-				.map(_.toOption.flatMap { Option(_) })
+			val entryStream = Stream.continually(Try {zipStream.getNextEntry})
+				.map(_.toOption.flatMap {Option(_)})
 				.takeWhile(_.isDefined)
 				.flatten
 
@@ -155,14 +155,16 @@ class ZippedStorage(zipFile: ZipFile) extends Storage {
 				}
 
 				if (entry.isDirectory) {
-					read(filename, nestedPath, entry, zipStream) // caller may need to process directories, too
-					readEntries(filter, nestedPath, s"$filename/${entry.getName}", new CloseShieldInputStream(zipStream), true)(read)
+					if (filter(entry)) read(filename, nestedPath, entry, zipStream) // caller may need to process directories, too
+					if (recursive) readEntries(filter, nestedPath, s"$filename/${entry.getName}", new CloseShieldInputStream(zipStream), recursive)(read)
 				}
 				else if (Storage.isZip(entry.getName) && recursive)
-					readEntries(filter, nestedPath, s"$filename/${entry.getName}", new CloseShieldInputStream(zipStream), true)(read)
-				else if(filter(entry))
+					readEntries(filter, nestedPath, s"$filename/${entry.getName}", new CloseShieldInputStream(zipStream), recursive)(read)
+				else if (filter(entry))
 					read(filename, entryPath, entry, zipStream)
 			}
+		} catch {
+			case ex: Exception => System.out.println(ex)
 		} finally {
 			zipStream.close
 		}

codepulse/src/main/scala/com/secdec/codepulse/input/LanguageProcessor.scala

@@ -36,6 +36,11 @@ trait LanguageProcessor {
 		sourceExtensions.contains(extension)
 	}
 
+	final def sourceType(typeExtension: String)(entry: ZipEntry): Boolean = {
+		val extension = FilenameUtils.getExtension(entry.getName)
+		typeExtension == extension
+	}
+
 	def canProcess(storage: Storage): Boolean
 	def process(storage: Storage, treeNodeData: TreeNodeDataAccess, sourceData: SourceDataAccess): Unit
 }

codepulse/src/main/scala/com/secdec/codepulse/input/bytecode/ByteCodeProcessor.scala

@@ -23,24 +23,35 @@ import java.io.InputStream
 import scala.collection.mutable.{ HashMap, MultiMap, Set }
 
 import akka.actor.{ Actor, Stash }
-import com.secdec.codepulse.data.bytecode.parse.MethodDeclarationListener
+import com.google.common.io.CharStreams
+import com.secdec.codepulse.data.bytecode.parse.ParseListener
 import com.secdec.codepulse.data.bytecode.{ AsmVisitors, CodeForestBuilder, CodeTreeNodeKind }
 import com.secdec.codepulse.data.jsp.{ JasperJspAdapter, JspAnalyzer }
 import com.secdec.codepulse.data.model.{ MethodSignatureNode, SourceDataAccess, TreeNodeDataAccess, TreeNodeImporter }
 import com.secdec.codepulse.data.storage.Storage
 import com.secdec.codepulse.events.GeneralEventBus
 import com.secdec.codepulse.input.pathnormalization.{ FilePath, NestedPath, PathNormalization }
 import com.secdec.codepulse.input.{ CanProcessFile, LanguageProcessor }
+import com.secdec.codepulse.parsers.java9.Java9Parser.CompilationUnitContext
 import com.secdec.codepulse.parsers.java9.{ Java9Lexer, Java9Parser }
 import com.secdec.codepulse.processing.{ ProcessEnvelope, ProcessStatus }
 import com.secdec.codepulse.processing.ProcessStatus.{ DataInputAvailable, ProcessDataAvailable }
 import com.secdec.codepulse.util.SmartLoader.Success
 import com.secdec.codepulse.util.SmartLoader
 import org.apache.commons.io.FilenameUtils
 import net.liftweb.common.Loggable
-import org.antlr.v4.runtime.CharStreams
-import org.antlr.v4.runtime.CommonTokenStream
-import org.antlr.v4.runtime.tree.ParseTreeWalker
+//import org.antlr.v4.runtime.CharStreams
+//import org.antlr.v4.runtime.CommonTokenStream
+//import org.antlr.v4.runtime.tree.ParseTreeWalker
+import org.apache.commons.io.input.CloseShieldInputStream
+import com.github.javaparser.JavaParser
+import com.github.javaparser.ParseException
+import com.github.javaparser.ast.body.MethodDeclaration
+import com.github.javaparser.ast.visitor.VoidVisitorAdapter
+import com.github.javaparser.ast.Node
+import com.github.javaparser.ast.body.ClassOrInterfaceDeclaration
+import com.github.javaparser.ast.CompilationUnit
+import com.github.javaparser.ast.PackageDeclaration
 
 class ByteCodeProcessor(eventBus: GeneralEventBus) extends Actor with Stash with LanguageProcessor with Loggable {
 	val group = "Classes"
@@ -97,6 +108,15 @@ class ByteCodeProcessor(eventBus: GeneralEventBus) extends Actor with Stash with
 			})
 		}
 
+		val sourceMethods = new HashMap[String, List[(String, Int)]]
+		storage.readEntries(sourceType("java") _) { (filename, entryPath, entry, contents) =>
+			val methodsAndStarts = parseJava9(contents)
+			sourceMethods.get(entry.getName()) match {
+				case None => sourceMethods += (entry.getName() -> methodsAndStarts)
+				case Some(entries) => sourceMethods += (entry.getName() -> (entries ::: methodsAndStarts))
+			}
+		}
+
 		def getPackageFromSig(sig: String): String = {
 			val packageContainer = sig.split(";").take(1)
 			val packagePart = packageContainer(0).split("/").dropRight(1)
@@ -118,8 +138,7 @@ class ByteCodeProcessor(eventBus: GeneralEventBus) extends Actor with Stash with
 			if (!entry.isDirectory) {
 				FilenameUtils.getExtension(entry.getName) match {
 					case "class" =>
-						val methods = AsmVisitors.parseMethodsFromClass(contents)
-						val methodsAndStartLines = parseJava9(contents)
+						val methods = AsmVisitors.parseMethodsFromClass(new CloseShieldInputStream(contents))
 						for {
 							(file, name, size, lineCount, methodStartLine) <- methods
 							pkg = getPackageFromSig(name)
@@ -132,7 +151,15 @@ class ByteCodeProcessor(eventBus: GeneralEventBus) extends Actor with Stash with
 								case Some(np) => authoritativePath(groupName, np).map (_.toString)
 								case None => None
 							}
-							treeNode <- builder.getOrAddMethod(groupName, name, size, authority, Option(lineCount), Option(methodStartLine))
+							methodsAndStartLines = authority.flatMap(sourceMethods.get)
+							startLine = methodsAndStartLines.flatMap { l =>
+								val aName = name.split(";").take(1)(0)
+								val potentials = l.filter { case (qualifiedName, line) => qualifiedName == name.split(";").take(1)(0) }
+								val selection = potentials.headOption.getOrElse((("", 0)))
+
+								Some(selection._2)
+							}
+							treeNode <- builder.getOrAddMethod(groupName, name, size, authority, Option(lineCount), startLine)
 						} {
 							methodCorrelationsBuilder += (name -> treeNode.id)
 						}
@@ -193,24 +220,60 @@ class ByteCodeProcessor(eventBus: GeneralEventBus) extends Actor with Stash with
 	}
 
 	private def parseJava9(contents: InputStream): List[(String, Int)] = {
-		// contents to stream
-		val stream = CharStreams.fromStream(contents)
+		def mkString(s1: String, s2: String, sep: String): String = {
+			if(s1.isEmpty) s2
+			else if(s2.isEmpty) s1
+			else s1 + sep + s2
+		}
+
+		def getQualifiedClass(n: Node, s: String): String = {
+			n match {
+				case m: MethodDeclaration => if(m.getParentNode().isPresent()) {
+					getQualifiedClass(m.getParentNode().get(), s)
+				} else {
+					s
+				}
+				case c: ClassOrInterfaceDeclaration => if(c.getParentNode().isPresent()) {
+					getQualifiedClass(c.getParentNode().get(), mkString(c.getName().toString, s, "$"))//c.getName + "." + s)
+				} else {
+					s
+				}
+				case cu: CompilationUnit => if(cu.getPackageDeclaration().isPresent()) {
+					getQualifiedClass(cu.getPackageDeclaration().get(), s)
+				} else {
+					s
+				}
+				case p: PackageDeclaration => mkString(p.getName().toString.replace(".", "/"), s, "/")//p.getName + "." + s
+			}
+		}
 
-		// create lexer, token, and parser
-		val lexer = new Java9Lexer(stream)
-		val tokens = new CommonTokenStream(lexer)
-		val parser = new Java9Parser(tokens)
+		var methodStarts = List[(String, Int)]()
 
-		// read input
-		val ctx = parser.methodDeclaration()
+		try {
+			val sourceContent = new CloseShieldInputStream(contents)
+			val cu = JavaParser.parse(sourceContent)
 
-		// walk the tree, listening for what we need
-		val walker = new ParseTreeWalker()
+			val methodVisitor = new VoidVisitorAdapter[Void] {
+				override def visit(n: MethodDeclaration, arg: Void): Unit = {
+					super.visit(n, arg)
+					System.out.println(getQualifiedClass(n, "") + "\t" + n.getName() + "\t" + n.getBegin())
 
-		var methodStarts = List[(String, Int)]()
-		val listener = new MethodDeclarationListener((method, start) => methodStarts = (method, start) :: methodStarts)
+					val qualifiedName = mkString(getQualifiedClass(n, ""), n.getName().toString, ".")
+					val startLine = if(n.getBegin().isPresent()) {
+						val location = n.getBegin().get()
+						location.line
+					} else {
+						0
+					}
 
-		walker.walk(listener, ctx)
+					methodStarts = (qualifiedName, startLine) :: methodStarts
+				}
+			}
+
+			cu.accept(methodVisitor, null)
+		} catch {
+			case ex: Exception => System.out.println(ex)
+		}
 
 		methodStarts
 	}

project/Dependencies.scala

@@ -80,4 +80,5 @@ object Dependencies {
 	lazy val dispatch = "net.databinder.dispatch" %% "dispatch-core" % "0.11.4"
 
 	lazy val antlr = "org.antlr" % "antlr4-maven-plugin" % "4.7"
+	lazy val javaparser = "com.github.javaparser" % "javaparser-core" % "3.6.13"
 }