Skip to content

Commit 569253e

Browse files
Lenny HalsethLenny Halseth
Lenny Halseth
authored and
Lenny Halseth
committed
Merge branch 'dev/llc' into dev/java-parse
2 parents 4e09ae5 + cf551e4 commit 569253e

File tree

12 files changed

+287
-220
lines changed

12 files changed

+287
-220
lines changed

‎agent/src/main/java/com/codedx/codepulse/agent/trace/InstrumentationClassVisitor.java‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,23 @@
1+
/*
2+
* Copyright 2018 Secure Decisions, a division of Applied Visions, Inc.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*
16+
* This material is based on research sponsored by the Department of Homeland
17+
* Security (DHS) Science and Technology Directorate, Cyber Security Division
18+
* (DHS S&T/CSD) via contract number HHSP233201600058C.
19+
*/
20+
121
package com.codedx.codepulse.agent.trace;
222

323
import org.objectweb.asm.ClassVisitor;

‎agent/src/main/java/com/codedx/codepulse/agent/trace/InstrumentationMethodVisitor.java‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,23 @@
1+
/*
2+
* Copyright 2018 Secure Decisions, a division of Applied Visions, Inc.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*
16+
* This material is based on research sponsored by the Department of Homeland
17+
* Security (DHS) Science and Technology Directorate, Cyber Security Division
18+
* (DHS S&T/CSD) via contract number HHSP233201600058C.
19+
*/
20+
121
package com.codedx.codepulse.agent.trace;
222

323
import org.objectweb.asm.Label;

‎codepulse/src/main/scala/com/secdec/codepulse/data/storage/ZippedStorage.scala‎

Lines changed: 2 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -200,11 +200,8 @@ class ZippedStorage(zipFile: ZipFile) extends Storage {
200200
}
201201
}
202202

203-
lazy val recurse = entry.isDirectory || (recursive &&
204-
Storage.isZip(entry.getName) &&
205-
find(s"$filename/${entry.getName}", nestedPath, new CloseShieldInputStream(zipStream), true)(predicate))
206-
207-
predicate(filename, entryPath, entry, zipStream) || recurse
203+
predicate(filename, entryPath, entry, zipStream) ||
204+
((entry.isDirectory || (recursive && Storage.isZip(entry.getName))) && (find(s"$filename/${entry.getName}", nestedPath, new CloseShieldInputStream(zipStream), true)(predicate)))
208205
}
209206
} finally {
210207
zipStream.close

‎codepulse/src/main/scala/com/secdec/codepulse/input/InputFileProcessor.scala‎

Lines changed: 68 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,68 @@
1+
/*
2+
* Copyright 2018 Secure Decisions, a division of Applied Visions, Inc.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*
16+
* This material is based on research sponsored by the Department of Homeland
17+
* Security (DHS) Science and Technology Directorate, Cyber Security Division
18+
* (DHS S&T/CSD) via contract number HHSP233201600058C.
19+
*/
20+
21+
package com.secdec.codepulse.input
22+
23+
import akka.actor.{Actor, Stash}
24+
import com.codedx.codepulse.utility.Loggable
25+
import com.secdec.codepulse.data.storage.Storage
26+
import com.secdec.codepulse.events.GeneralEventBus
27+
import com.secdec.codepulse.input.bytecode.ByteCodeProcessor
28+
import com.secdec.codepulse.input.dotnet.DotNETProcessor
29+
import com.secdec.codepulse.processing.{ProcessEnvelope, ProcessStatus}
30+
import com.secdec.codepulse.processing.ProcessStatus.{DataInputAvailable, ProcessDataAvailable}
31+
32+
class InputFileProcessor(eventBus: GeneralEventBus) extends Actor with Stash with Loggable {
33+
34+
import com.secdec.codepulse.util.Actor._
35+
36+
val languageProcessors: List[LanguageProcessor] = new ByteCodeProcessor :: new DotNETProcessor :: Nil
37+
38+
override def receive = {
39+
case ProcessEnvelope(_, DataInputAvailable(identifier, storage, treeNodeData, sourceData, post)) => {
40+
try {
41+
val languageProcessor = languageProcessors.find(x => x.canProcess(storage))
42+
if (languageProcessor.isDefined) {
43+
languageProcessor.get.process(storage, treeNodeData, sourceData)
44+
post()
45+
eventBus.publish(ProcessDataAvailable(identifier, storage, treeNodeData, sourceData))
46+
}
47+
} catch {
48+
case exception: Exception => eventBus.publish(ProcessStatus.asEnvelope(ProcessStatus.Failed(identifier, "InputFileProcessor", Some(exception))))
49+
}
50+
}
51+
52+
case CanProcessFile(file) => {
53+
try {
54+
Storage(file) match {
55+
case Some(storage) => {
56+
val languageProcessor = languageProcessors.find(x => x.canProcess(storage))
57+
sender ! languageProcessor.isDefined
58+
}
59+
case _ => sender ! false
60+
}
61+
} catch {
62+
case ex: Exception => {
63+
logAndSendFailure(logger, "Unable to complete can-process test for input file", sender, ex)
64+
}
65+
}
66+
}
67+
}
68+
}

‎codepulse/src/main/scala/com/secdec/codepulse/input/bytecode/ByteCodeProcessor.scala‎

Lines changed: 3 additions & 35 deletions
Original file line numberDiff line numberDiff line change
@@ -22,27 +22,16 @@ package com.secdec.codepulse.input.bytecode
2222
import java.io.InputStream
2323
import scala.collection.mutable.{ HashMap, MultiMap, Set }
2424

25-
import akka.actor.{ Actor, Stash }
26-
import com.google.common.io.CharStreams
27-
import com.secdec.codepulse.data.bytecode.parse.ParseListener
2825
import com.secdec.codepulse.data.bytecode.{ AsmVisitors, CodeForestBuilder, CodeTreeNodeKind }
2926
import com.secdec.codepulse.data.jsp.{ JasperJspAdapter, JspAnalyzer }
3027
import com.secdec.codepulse.data.model.{ MethodSignatureNode, SourceDataAccess, TreeNodeDataAccess, TreeNodeImporter }
3128
import com.secdec.codepulse.data.storage.Storage
32-
import com.secdec.codepulse.events.GeneralEventBus
3329
import com.secdec.codepulse.input.pathnormalization.{ FilePath, NestedPath, PathNormalization }
34-
import com.secdec.codepulse.input.{ CanProcessFile, LanguageProcessor }
35-
import com.secdec.codepulse.parsers.java9.Java9Parser.CompilationUnitContext
36-
import com.secdec.codepulse.parsers.java9.{ Java9Lexer, Java9Parser }
37-
import com.secdec.codepulse.processing.{ ProcessEnvelope, ProcessStatus }
38-
import com.secdec.codepulse.processing.ProcessStatus.{ DataInputAvailable, ProcessDataAvailable }
30+
import com.secdec.codepulse.input.LanguageProcessor
3931
import com.secdec.codepulse.util.SmartLoader.Success
4032
import com.secdec.codepulse.util.SmartLoader
4133
import org.apache.commons.io.FilenameUtils
4234
import net.liftweb.common.Loggable
43-
//import org.antlr.v4.runtime.CharStreams
44-
//import org.antlr.v4.runtime.CommonTokenStream
45-
//import org.antlr.v4.runtime.tree.ParseTreeWalker
4635
import org.apache.commons.io.input.CloseShieldInputStream
4736
import com.github.javaparser.JavaParser
4837
import com.github.javaparser.ParseException
@@ -53,35 +42,14 @@ import com.github.javaparser.ast.body.ClassOrInterfaceDeclaration
5342
import com.github.javaparser.ast.CompilationUnit
5443
import com.github.javaparser.ast.PackageDeclaration
5544

56-
class ByteCodeProcessor(eventBus: GeneralEventBus) extends Actor with Stash with LanguageProcessor with Loggable {
45+
class ByteCodeProcessor() extends LanguageProcessor with Loggable {
5746
val group = "Classes"
5847
val traceGroups = (group :: CodeForestBuilder.JSPGroupName :: Nil).toSet
5948
val sourceExtensions = List("java", "jsp")
6049

61-
def receive = {
62-
case ProcessEnvelope(_, DataInputAvailable(identifier, storage, treeNodeData, sourceData, post)) => {
63-
try {
64-
if(canProcess(storage)) {
65-
process(storage, treeNodeData, sourceData)
66-
post()
67-
eventBus.publish(ProcessDataAvailable(identifier, storage, treeNodeData, sourceData))
68-
}
69-
} catch {
70-
case exception: Exception => eventBus.publish(ProcessStatus.asEnvelope(ProcessStatus.Failed(identifier, "Java ByteCode Processor", Some(exception))))
71-
}
72-
}
73-
74-
case CanProcessFile(file) => {
75-
Storage(file) match {
76-
case Some(storage) => sender ! canProcess(storage)
77-
case _ => sender ! false
78-
}
79-
}
80-
}
81-
8250
def canProcess(storage: Storage): Boolean = {
8351
storage.find() { (filename, entryPath, entry, contents) =>
84-
!entry.isDirectory && FilenameUtils.getExtension(entry.getName) == "class"
52+
!entry.isDirectory && (FilenameUtils.getExtension(entry.getName) == "class" || FilenameUtils.getExtension(entry.getName) == "jsp")
8553
}
8654
}
8755

‎codepulse/src/main/scala/com/secdec/codepulse/input/dotnet/DotNETProcessor.scala‎

Lines changed: 2 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -22,48 +22,23 @@ package com.secdec.codepulse.input.dotnet
2222
import java.io.File
2323
import scala.collection.mutable.{ HashMap, MultiMap, Set }
2424

25-
import akka.actor.{ Actor, Stash }
2625
import com.secdec.codepulse.data.bytecode.{ CodeForestBuilder, CodeTreeNodeKind }
2726
import com.secdec.codepulse.data.dotnet.{ DotNet, SymbolReaderHTTPServiceConnector, SymbolService }
2827
import com.secdec.codepulse.data.model.{ MethodSignatureNode, SourceDataAccess, TreeNodeDataAccess, TreeNodeImporter }
2928
import com.secdec.codepulse.data.storage.Storage
30-
import com.secdec.codepulse.events.GeneralEventBus
31-
import com.secdec.codepulse.input.{ CanProcessFile, LanguageProcessor }
32-
import com.secdec.codepulse.processing.{ ProcessEnvelope, ProcessStatus }
33-
import com.secdec.codepulse.processing.ProcessStatus.{ DataInputAvailable, ProcessDataAvailable }
29+
import com.secdec.codepulse.input.LanguageProcessor
3430
import com.secdec.codepulse.input.pathnormalization.{ FilePath, NestedPath, PathNormalization }
3531
import scala.language.postfixOps
3632
import org.apache.commons.io.FilenameUtils
3733

38-
class DotNETProcessor(eventBus: GeneralEventBus) extends Actor with Stash with LanguageProcessor {
34+
class DotNETProcessor() extends LanguageProcessor {
3935
val group = "Classes"
4036
val traceGroups = (group :: Nil).toSet
4137
val sourceExtensions = List("cs", "vb", "fs", "fsi", "fsx", "fsscript", "cpp")
4238

4339
val symbolService = new SymbolService
4440
symbolService.create
4541

46-
def receive = {
47-
case ProcessEnvelope(_, DataInputAvailable(identifier, storage, treeNodeData, sourceData, post)) => {
48-
try {
49-
if(canProcess(storage)) {
50-
process(storage, treeNodeData, sourceData)
51-
post()
52-
eventBus.publish(ProcessDataAvailable(identifier, storage, treeNodeData, sourceData))
53-
}
54-
} catch {
55-
case exception: Exception => eventBus.publish(ProcessStatus.Failed(identifier, "dotNET Processor", Some(exception)))
56-
}
57-
}
58-
59-
case CanProcessFile(file) => {
60-
Storage(file) match {
61-
case Some(storage) => sender ! canProcess(storage)
62-
case _ => sender ! false
63-
}
64-
}
65-
}
66-
6742
def canProcess(storage: Storage): Boolean = {
6843
storage.find() { (_, _, entry, _) =>
6944
val extension = FilenameUtils.getExtension(entry.getName)

‎codepulse/src/main/scala/com/secdec/codepulse/input/project/ProjectInput.scala‎

Lines changed: 26 additions & 69 deletions
Original file line numberDiff line numberDiff line change
@@ -20,71 +20,57 @@
2020
package com.secdec.codepulse.input.project
2121

2222
import java.io.File
23-
import scala.concurrent.ExecutionContext.Implicits.global
2423

25-
import akka.actor.{ Actor, ActorRef, Stash }
26-
import scala.collection.mutable.{ Map => MutableMap }
24+
import scala.concurrent.ExecutionContext.Implicits.global
25+
import akka.actor.{Actor, ActorRef, Stash}
26+
import com.codedx.codepulse.utility.Loggable
2727

28-
import com.secdec.codepulse.data.model.{ ProjectData, ProjectId }
29-
import com.secdec.codepulse.data.storage.{ Storage, StorageManager }
28+
import scala.collection.mutable.{Map => MutableMap}
29+
import com.secdec.codepulse.data.model.{ProjectData, ProjectId}
30+
import com.secdec.codepulse.data.storage.{Storage, StorageManager}
3031
import com.secdec.codepulse.events.GeneralEventBus
31-
import com.secdec.codepulse.input.LanguageProcessor
3232
import com.secdec.codepulse.processing.ProcessEnvelope
3333
import com.secdec.codepulse.processing.ProcessStatus._
34-
import com.secdec.codepulse.tracer.{ BootVar, generalEventBus, projectDataProvider, projectManager }
34+
import com.secdec.codepulse.tracer.{BootVar, generalEventBus, projectDataProvider, projectManager}
3535

3636
trait ProjectLoader {
3737
protected def createProject: ProjectData
3838
}
3939

40-
case class CreateProject(processors: List[BootVar[ActorRef]], inputFile: File, load: (ProjectData, Storage, GeneralEventBus) => Unit)
40+
case class CreateProject(inputFile: File, load: (ProjectData, Storage, GeneralEventBus) => Unit)
41+
42+
class ProjectInputActor extends Actor with Stash with ProjectLoader with Loggable {
4143

42-
class ProjectInputActor extends Actor with Stash with ProjectLoader {
44+
import com.secdec.codepulse.util.Actor._
4345

4446
val projectCreationProcessors = MutableMap.empty[String, List[BootVar[ActorRef]]]
4547

4648
val projectProcessingSuccesses = MutableMap.empty[String, Integer]
4749

4850
val projectProcessingFailures = MutableMap.empty[String, Integer]
4951

50-
// TODO: handle data input by creating a project and broadcasting with 'DataInputAvailable' with project payload
51-
// TODO: capture failed state to cause a failed message and redirect (as necessary) for the user
52-
5352
def receive = {
54-
case CreateProject(processors, inputFile, load) => {
55-
val projectData = createProject
56-
addProjectCreators(projectData, processors)
57-
58-
StorageManager.storeInput(projectData.id, inputFile) match {
59-
case Some(storage) =>
60-
load(projectData, storage, generalEventBus)
61-
sender ! projectData
62-
case _ => throw new IllegalStateException(s"Unable to create storage for project ${projectData.id.num}")
53+
case CreateProject(inputFile, load) => {
54+
try {
55+
val projectData = createProject
56+
StorageManager.storeInput(projectData.id, inputFile) match {
57+
case Some(storage) =>
58+
load(projectData, storage, generalEventBus)
59+
sender ! projectData
60+
case _ => sender ! akka.actor.Status.Failure(new IllegalStateException(s"Unable to create storage for project ${projectData.id.num}"))
61+
}
62+
}
63+
catch {
64+
case ex: Exception => {
65+
logAndSendFailure(logger, "Unable to complete create-project operation for input file", sender, ex)
66+
}
6367
}
6468
}
65-
case ProcessEnvelope(_, ProcessDataAvailable(identifier, file, treeNodeData, sourceData)) => {
66-
val numberOfProcessors = projectCreationProcessors.get(identifier).get.length
67-
val succeeded = projectProcessorSucceeded(identifier)
68-
val failed = projectProcessingFailures.get(identifier).get
69-
70-
val remaining = numberOfProcessors - (succeeded + failed)
71-
72-
if(remaining == 0 && succeeded >= 1) {
69+
case ProcessEnvelope(_, ProcessDataAvailable(identifier, _, _, _)) => {
7370
projectManager getProject ProjectId(identifier.toInt) foreach(_.notifyLoadingFinished())
74-
clearProjectCreators(identifier)
75-
}
7671
}
7772
case ProcessEnvelope(_, Failed(identifier, action, Some(exception))) if action != "Dependency Check" => {
78-
val numberOfProcessors = projectCreationProcessors.get(identifier).get.length
79-
val succeeded = projectProcessingSuccesses.get(identifier).get
80-
val failed = projectProcessorFailed(identifier)
81-
82-
val remaining = numberOfProcessors - (succeeded + failed)
83-
84-
if(remaining == 0 && failed == numberOfProcessors) {
8573
projectManager.removeUnloadedProject(ProjectId(identifier.toInt), exception.getMessage)
86-
clearProjectCreators(identifier)
87-
}
8874
}
8975
}
9076

@@ -94,33 +80,4 @@ class ProjectInputActor extends Actor with Stash with ProjectLoader {
9480

9581
projectData
9682
}
97-
98-
protected def addProjectCreators(projectData: ProjectData, processors: List[BootVar[ActorRef]]): Integer = {
99-
val id = projectData.id.num.toString
100-
projectCreationProcessors.put(id, processors)
101-
projectProcessingSuccesses.put(id, 0)
102-
projectProcessingFailures.put(id, 0)
103-
processors.length
104-
}
105-
106-
protected def clearProjectCreators(projectId: String) = {
107-
projectCreationProcessors.remove(projectId)
108-
projectProcessingSuccesses.remove(projectId)
109-
projectProcessingFailures.remove(projectId)
110-
}
111-
112-
protected def projectProcessorSucceeded(projectId: String): Integer = {
113-
incrementProcessor(projectId, projectProcessingSuccesses)
114-
}
115-
116-
protected def projectProcessorFailed(projectId: String) = {
117-
incrementProcessor(projectId, projectProcessingFailures)
118-
}
119-
120-
private def incrementProcessor(key: String, map: MutableMap[String, Integer]): Integer = {
121-
val count = map.get(key).get + 1
122-
map.update(key, count)
123-
124-
count
125-
}
12683
}

0 commit comments

Comments
 (0)