diff --git a/ .git-blame-ignore-revs b/ .git-blame-ignore-revs new file mode 100644 index 0000000000..8c9f5418d9 --- /dev/null +++ b/ .git-blame-ignore-revs @@ -0,0 +1,2 @@ +# Scalafmt apply +7fb89c767649f28c0405d5f4ba8afa181614504f diff --git a/README.md b/README.md index ecc1b637b7..17035a11f5 100644 --- a/README.md +++ b/README.md @@ -113,9 +113,7 @@ import org.opencypher.morpheus.api.MorpheusSession import org.opencypher.morpheus.api.io.{MorpheusNodeTable, MorpheusRelationshipTable} import org.opencypher.morpheus.util.App -/** - * Demonstrates basic usage of the Morpheus API by loading an example graph from [[DataFrame]]s. - */ +/** Demonstrates basic usage of the Morpheus API by loading an example graph from [[DataFrame]]s. */ object DataFrameInputExample extends App { // 1) Create Morpheus session and retrieve Spark session implicit val morpheus: MorpheusSession = MorpheusSession.local() @@ -124,15 +122,23 @@ object DataFrameInputExample extends App { import spark.sqlContext.implicits._ // 2) Generate some DataFrames that we'd like to interpret as a property graph. - val nodesDF = spark.createDataset(Seq( - (0L, "Alice", 42L), - (1L, "Bob", 23L), - (2L, "Eve", 84L) - )).toDF("id", "name", "age") - val relsDF = spark.createDataset(Seq( - (0L, 0L, 1L, "23/01/1987"), - (1L, 1L, 2L, "12/12/2009") - )).toDF("id", "source", "target", "since") + val nodesDF = spark + .createDataset( + Seq( + (0L, "Alice", 42L), + (1L, "Bob", 23L), + (2L, "Eve", 84L) + ) + ) + .toDF("id", "name", "age") + val relsDF = spark + .createDataset( + Seq( + (0L, 0L, 1L, "23/01/1987"), + (1L, 1L, 2L, "12/12/2009") + ) + ) + .toDF("id", "source", "target", "since") // 3) Generate node- and relationship tables that wrap the DataFrames. The mapping between graph elements and columns // is derived using naming conventions for identifier columns. @@ -147,7 +153,8 @@ object DataFrameInputExample extends App { // 6) Collect results into string by selecting a specific column. // This operation may be very expensive as it materializes results locally. - val names: Set[String] = result.records.table.df.collect().map(_.getAs[String]("n_name")).toSet + val names: Set[String] = + result.records.table.df.collect().map(_.getAs[String]("n_name")).toSet println(names) } diff --git a/build.gradle b/build.gradle index f298117089..f7c4f5b78d 100644 --- a/build.gradle +++ b/build.gradle @@ -3,8 +3,8 @@ plugins { alias(libs.plugins.champeau.jmh).apply(false) alias(libs.plugins.license).apply(false) - alias(libs.plugins.scalastyle).apply(false) alias(libs.plugins.shadowjar).apply(false) + alias(libs.plugins.spotless).apply(false) alias(libs.plugins.versionCatalogUpdate) } diff --git a/build.style.gradle b/build.style.gradle index 4b43a5c160..24f608f8cf 100644 --- a/build.style.gradle +++ b/build.style.gradle @@ -1,8 +1,9 @@ subprojects { - apply plugin: 'com.github.alisiikh.scalastyle' + apply plugin: 'com.diffplug.spotless' - scalastyle { - scalaVersion = libs.versions.scala.major.get() - config = rootProject.file("etc/scalastyle_config.xml") + spotless { + scala { + scalafmt().configFile(rootProject.file('etc/.scalafmt.conf')) + } } } diff --git a/etc/.scalafmt.conf b/etc/.scalafmt.conf new file mode 100644 index 0000000000..511b7259d7 --- /dev/null +++ b/etc/.scalafmt.conf @@ -0,0 +1,9 @@ +version = 3.8.1 +runner.dialect = scala212 + +maxColumn = 100 +indent.defnSite = 2 + +docstrings.style = SpaceAsterisk +docstrings.oneline = fold +docstrings.blankFirstLine = yes \ No newline at end of file diff --git a/etc/scalastyle_config.xml b/etc/scalastyle_config.xml deleted file mode 100644 index 5d0e634fc1..0000000000 --- a/etc/scalastyle_config.xml +++ /dev/null @@ -1,16 +0,0 @@ - - Configuration - - - - - - - - - - - scala.Some,scala.Equals,scala.Int,scala.Long,scala.Double,org.junit.Test - - - diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index b11bb2a6e0..5bf4860a12 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -19,7 +19,7 @@ mockito = "5.18.0" neo4j-driver = "1.7.2" netty = "4.2.7.Final" # @pin - let's prevent automatic updates for the moment -scala-full = "2.12.20" +scala-full = "2.12.20" # Note, needs to stay aligned with scalafmt.conf # @pin - let's prevent automatic updates for the moment scala-major = "2.12" scalacheck = "1.19.0" @@ -83,6 +83,6 @@ testing-scalatest-junit = { module = "org.scalatestplus:junit-5-13_2.12", versio champeau-jmh = "me.champeau.jmh:0.7.3" # @pin - version is self-hosted license = "com.github.hierynomus.license:0.16.3-63da64d" -scalastyle = "com.github.alisiikh.scalastyle:3.5.0" +spotless = "com.diffplug.spotless:8.1.0" shadowjar = "com.gradleup.shadow:9.2.2" versionCatalogUpdate = "nl.littlerobots.version-catalog-update:1.0.1" diff --git a/graph-ddl/src/main/scala/org/opencypher/graphddl/GraphDdl.scala b/graph-ddl/src/main/scala/org/opencypher/graphddl/GraphDdl.scala index bedc3e842c..c9b0a9d3d4 100644 --- a/graph-ddl/src/main/scala/org/opencypher/graphddl/GraphDdl.scala +++ b/graph-ddl/src/main/scala/org/opencypher/graphddl/GraphDdl.scala @@ -29,7 +29,11 @@ package org.opencypher.graphddl import cats.instances.map._ import cats.syntax.semigroup._ import org.opencypher.graphddl.GraphDdl._ -import org.opencypher.graphddl.GraphDdlAst.{ColumnIdentifier, KeyDefinition, PropertyToColumnMappingDefinition} +import org.opencypher.graphddl.GraphDdlAst.{ + ColumnIdentifier, + KeyDefinition, + PropertyToColumnMappingDefinition +} import org.opencypher.graphddl.GraphDdlException._ import org.opencypher.okapi.api.graph.GraphName import org.opencypher.okapi.api.schema.PropertyKeys @@ -41,9 +45,7 @@ import scala.language.higherKinds object GraphDdl { - /** - * Type definition for a mapping from property name to source column name - */ + /** Type definition for a mapping from property name to source column name */ type PropertyMappings = Map[String, String] def apply(ddl: String): GraphDdl = @@ -53,7 +55,10 @@ object GraphDdl { val ddlParts = DdlParts(ddl.statements) - val global = PartialGraphType.empty.push(name = "global", statements = ddlParts.elementTypes) + val global = PartialGraphType.empty.push( + name = "global", + statements = ddlParts.elementTypes + ) val graphTypes = ddlParts.graphTypes .keyBy(_.name) @@ -62,7 +67,8 @@ object GraphDdl { global.push(graphType.name, graphType.statements) } } - .view.force + .view + .force val graphs = ddlParts.graphs .map { graph => @@ -86,10 +92,14 @@ object GraphDdl { def apply(statements: List[DdlStatement]): DdlParts = { val result = statements.foldLeft(DdlParts.empty) { - case (parts, s: SetSchemaDefinition) => parts.copy(maybeSetSchema = Some(s)) - case (parts, s: ElementTypeDefinition) => parts.copy(elementTypes = parts.elementTypes :+ s) - case (parts, s: GraphTypeDefinition) => parts.copy(graphTypes = parts.graphTypes :+ s) - case (parts, s: GraphDefinition) => parts.copy(graphs = parts.graphs :+ GraphDefinitionWithContext(s, parts.maybeSetSchema)) + case (parts, s: SetSchemaDefinition) => + parts.copy(maybeSetSchema = Some(s)) + case (parts, s: ElementTypeDefinition) => + parts.copy(elementTypes = parts.elementTypes :+ s) + case (parts, s: GraphTypeDefinition) => + parts.copy(graphTypes = parts.graphTypes :+ s) + case (parts, s: GraphDefinition) => + parts.copy(graphs = parts.graphs :+ GraphDefinitionWithContext(s, parts.maybeSetSchema)) } result.elementTypes.validateDistinctBy(_.name, "Duplicate element type") result.graphTypes.validateDistinctBy(_.name, "Duplicate graph type") @@ -111,9 +121,12 @@ object GraphDdl { def apply(statements: List[GraphTypeStatement]): GraphTypeParts = { val result = statements.foldLeft(GraphTypeParts.empty) { - case (parts, s: ElementTypeDefinition) => parts.copy(elementTypes = parts.elementTypes :+ s) - case (parts, s: NodeTypeDefinition) => parts.copy(nodeTypes = parts.nodeTypes :+ s) - case (parts, s: RelationshipTypeDefinition) => parts.copy(relTypes = parts.relTypes :+ s) + case (parts, s: ElementTypeDefinition) => + parts.copy(elementTypes = parts.elementTypes :+ s) + case (parts, s: NodeTypeDefinition) => + parts.copy(nodeTypes = parts.nodeTypes :+ s) + case (parts, s: RelationshipTypeDefinition) => + parts.copy(relTypes = parts.relTypes :+ s) } result.elementTypes.validateDistinctBy(_.name, "Duplicate element type") result @@ -132,9 +145,12 @@ object GraphDdl { def apply(mappings: List[GraphStatement]): GraphParts = mappings.foldLeft(GraphParts.empty) { - case (parts, s: GraphTypeStatement) => parts.copy(graphTypeStatements = parts.graphTypeStatements :+ s) - case (parts, s: NodeMappingDefinition) => parts.copy(nodeMappings = parts.nodeMappings :+ s) - case (parts, s: RelationshipMappingDefinition) => parts.copy(relMappings = parts.relMappings :+ s) + case (parts, s: GraphTypeStatement) => + parts.copy(graphTypeStatements = parts.graphTypeStatements :+ s) + case (parts, s: NodeMappingDefinition) => + parts.copy(nodeMappings = parts.nodeMappings :+ s) + case (parts, s: RelationshipMappingDefinition) => + parts.copy(relMappings = parts.relMappings :+ s) } } @@ -170,10 +186,14 @@ object GraphDdl { parent.map(_.allEdgeTypes).getOrElse(Map.empty) ++ edgeTypes /** Validates, resolves and pushes the statements to form a child GraphType */ - def push(name: String, statements: List[GraphTypeStatement]): PartialGraphType = { + def push( + name: String, + statements: List[GraphTypeStatement] + ): PartialGraphType = { val parts = GraphTypeParts(statements) - val local = PartialGraphType(Some(this), name, parts.elementTypes.keyBy(_.name)) + val local = + PartialGraphType(Some(this), name, parts.elementTypes.keyBy(_.name)) PartialGraphType( parent = Some(this), @@ -184,24 +204,43 @@ object GraphDdl { ) } - def toGraphType: GraphType = GraphType(name, - allElementTypes.values.toSet.map { elementTypeDef: ElementTypeDefinition => toElementType(elementTypeDef) }, - allNodeTypes.map { case (nodeTypeDef, _) => toNodeType(nodeTypeDef) }.toSet, - allEdgeTypes.map { case (relTypeDef, _) => toRelType(relTypeDef) }.toSet) + def toGraphType: GraphType = GraphType( + name, + allElementTypes.values.toSet.map { elementTypeDef: ElementTypeDefinition => + toElementType(elementTypeDef) + }, + allNodeTypes.map { case (nodeTypeDef, _) => + toNodeType(nodeTypeDef) + }.toSet, + allEdgeTypes.map { case (relTypeDef, _) => toRelType(relTypeDef) }.toSet + ) - def toElementType(elementTypeDefinition: ElementTypeDefinition): ElementType = { - val parentTypes = elementTypeDefinition.parents.map(resolveElementType).map(toElementType).map(_.name) - ElementType(elementTypeDefinition.name, parentTypes, elementTypeDefinition.properties, elementTypeDefinition.maybeKey) + def toElementType( + elementTypeDefinition: ElementTypeDefinition + ): ElementType = { + val parentTypes = elementTypeDefinition.parents + .map(resolveElementType) + .map(toElementType) + .map(_.name) + ElementType( + elementTypeDefinition.name, + parentTypes, + elementTypeDefinition.properties, + elementTypeDefinition.maybeKey + ) } def toNodeType(nodeTypeDefinition: NodeTypeDefinition): NodeType = NodeType(resolveNodeLabels(nodeTypeDefinition)) - def toRelType(relationshipTypeDefinition: RelationshipTypeDefinition): RelationshipType = + def toRelType( + relationshipTypeDefinition: RelationshipTypeDefinition + ): RelationshipType = RelationshipType( startNodeType = toNodeType(relationshipTypeDefinition.startNodeType), labels = resolveRelationshipLabel(relationshipTypeDefinition), - endNodeType = toNodeType(relationshipTypeDefinition.endNodeType)) + endNodeType = toNodeType(relationshipTypeDefinition.endNodeType) + ) private def resolveNodeTypes( parts: GraphTypeParts, @@ -210,7 +249,13 @@ object GraphDdl { parts.nodeTypes .map(nodeType => NodeTypeDefinition(local.resolveNodeLabels(nodeType))) .validateDistinctBy(identity, "Duplicate node type") - .map(nodeType => nodeType -> tryWithNode(nodeType)(mergeProperties(nodeType.elementTypes.flatMap(local.resolveElementTypes)))) + .map(nodeType => + nodeType -> tryWithNode(nodeType)( + mergeProperties( + nodeType.elementTypes.flatMap(local.resolveElementTypes) + ) + ) + ) .toMap } @@ -219,50 +264,87 @@ object GraphDdl { local: PartialGraphType ): Map[RelationshipTypeDefinition, PropertyKeys] = { parts.relTypes - .map(relType => RelationshipTypeDefinition( - startNodeType = NodeTypeDefinition(elementTypes = local.resolveNodeLabels(relType.startNodeType)), - elementTypes = local.resolveRelationshipLabel(relType), - endNodeType = NodeTypeDefinition(elementTypes = local.resolveNodeLabels(relType.endNodeType)))) + .map(relType => + RelationshipTypeDefinition( + startNodeType = + NodeTypeDefinition(elementTypes = local.resolveNodeLabels(relType.startNodeType)), + elementTypes = local.resolveRelationshipLabel(relType), + endNodeType = + NodeTypeDefinition(elementTypes = local.resolveNodeLabels(relType.endNodeType)) + ) + ) .validateDistinctBy(identity, "Duplicate relationship type") - .map(relType => relType -> tryWithRel(relType)(mergeProperties(relType.elementTypes.flatMap(local.resolveElementTypes)))) + .map(relType => + relType -> tryWithRel(relType)( + mergeProperties( + relType.elementTypes.flatMap(local.resolveElementTypes) + ) + ) + ) .toMap } - private def resolveNodeLabels(nodeTypeDefinition: NodeTypeDefinition): Set[String] = - tryWithNode(nodeTypeDefinition)(nodeTypeDefinition.elementTypes.flatMap(resolveElementTypes).map(_.name)) + private def resolveNodeLabels( + nodeTypeDefinition: NodeTypeDefinition + ): Set[String] = + tryWithNode(nodeTypeDefinition)( + nodeTypeDefinition.elementTypes.flatMap(resolveElementTypes).map(_.name) + ) - private def resolveRelationshipLabel(relTypeDefinition: RelationshipTypeDefinition): Set[String] = - tryWithRel(relTypeDefinition)(relTypeDefinition.elementTypes.flatMap(resolveElementTypes).map(_.name)) + private def resolveRelationshipLabel( + relTypeDefinition: RelationshipTypeDefinition + ): Set[String] = + tryWithRel(relTypeDefinition)( + relTypeDefinition.elementTypes.flatMap(resolveElementTypes).map(_.name) + ) - private def mergeProperties(elementTypes: Set[ElementTypeDefinition]): PropertyKeys = { + private def mergeProperties( + elementTypes: Set[ElementTypeDefinition] + ): PropertyKeys = { elementTypes .flatMap(_.properties) .foldLeft(PropertyKeys.empty) { case (props, (name, cypherType)) => props.get(name).filter(_ != cypherType) match { case Some(t) => incompatibleTypes(name, cypherType, t) - case None => props.updated(name, cypherType) + case None => props.updated(name, cypherType) } } } - private def resolveElementTypes(name: String): Set[ElementTypeDefinition] = { + private def resolveElementTypes( + name: String + ): Set[ElementTypeDefinition] = { val elementType = resolveElementType(name) detectCircularDependency(elementType) resolveParents(elementType) } private def resolveElementType(name: String): ElementTypeDefinition = - allElementTypes.getOrElse(name, unresolved(s"Unresolved element type", name)) + allElementTypes.getOrElse( + name, + unresolved(s"Unresolved element type", name) + ) - private def resolveParents(node: ElementTypeDefinition): Set[ElementTypeDefinition] = + private def resolveParents( + node: ElementTypeDefinition + ): Set[ElementTypeDefinition] = node.parents.map(resolveElementType).flatMap(resolveParents) + node private def detectCircularDependency(node: ElementTypeDefinition): Unit = { - def traverse(node: ElementTypeDefinition, path: List[ElementTypeDefinition]): Unit = { + def traverse( + node: ElementTypeDefinition, + path: List[ElementTypeDefinition] + ): Unit = { node.parents.foreach { p => - val parentElementType = allElementTypes.getOrElse(p, unresolved(s"Unresolved element type", p)) + val parentElementType = allElementTypes.getOrElse( + p, + unresolved(s"Unresolved element type", p) + ) if (path.contains(parentElementType)) { - illegalInheritance("Circular dependency detected", (path.map(_.name) :+ p).mkString(" -> ")) + illegalInheritance( + "Circular dependency detected", + (path.map(_.name) :+ p).mkString(" -> ") + ) } traverse(parentElementType, path :+ parentElementType) } @@ -277,29 +359,55 @@ object GraphDdl { graph: GraphDefinitionWithContext ): Graph = { val parts = GraphParts(graph.definition.statements) - val graphTypeName = graph.definition.maybeGraphTypeName.getOrElse(graph.definition.name) + val graphTypeName = + graph.definition.maybeGraphTypeName.getOrElse(graph.definition.name) val partialGraphType = parent .push(graphTypeName, parts.graphTypeStatements) - .push(graphTypeName, parts.nodeMappings.map(_.nodeType) ++ parts.relMappings.map(_.relType)) + .push( + graphTypeName, + parts.nodeMappings.map(_.nodeType) ++ parts.relMappings.map(_.relType) + ) val graphType = partialGraphType.toGraphType val g = Graph( name = GraphName(graph.definition.name), graphType = graphType, nodeToViewMappings = parts.nodeMappings - .flatMap(nmd => toNodeToViewMappings(partialGraphType.toNodeType(nmd.nodeType), graphType, graph.maybeSetSchema, nmd)) + .flatMap(nmd => + toNodeToViewMappings( + partialGraphType.toNodeType(nmd.nodeType), + graphType, + graph.maybeSetSchema, + nmd + ) + ) .validateDistinctBy(_.key, "Duplicate node mapping") .keyBy(_.key), edgeToViewMappings = parts.relMappings - .flatMap(rmd => toEdgeToViewMappings(partialGraphType.toRelType(rmd.relType), graphType, graph.maybeSetSchema, rmd)) + .flatMap(rmd => + toEdgeToViewMappings( + partialGraphType.toRelType(rmd.relType), + graphType, + graph.maybeSetSchema, + rmd + ) + ) .validateDistinctBy(_.key, "Duplicate relationship mapping") ) - g.edgeToViewMappings.flatMap(evm => Seq( - evm.startNode.nodeViewKey -> evm.startNodeJoinColumns.toSet, - evm.endNode.nodeViewKey -> evm.endNodeJoinColumns.toSet - )).distinct - .validateDistinctBy({ case (nvk, _) => nvk }, msg = "Inconsistent join column definition", illegalConstraint) + g.edgeToViewMappings + .flatMap(evm => + Seq( + evm.startNode.nodeViewKey -> evm.startNodeJoinColumns.toSet, + evm.endNode.nodeViewKey -> evm.endNodeJoinColumns.toSet + ) + ) + .distinct + .validateDistinctBy( + { case (nvk, _) => nvk }, + msg = "Inconsistent join column definition", + illegalConstraint + ) g } @@ -310,9 +418,12 @@ object GraphDdl { nmd: NodeMappingDefinition ): Seq[NodeToViewMapping] = { nmd.nodeToView.map { nvd => - tryWithContext(s"Error in node mapping for: ${nmd.nodeType.elementTypes.mkString(",")}") { + tryWithContext( + s"Error in node mapping for: ${nmd.nodeType.elementTypes.mkString(",")}" + ) { - val nodeKey = NodeViewKey(nodeType, toViewId(maybeSetSchema, nvd.viewId)) + val nodeKey = + NodeViewKey(nodeType, toViewId(maybeSetSchema, nvd.viewId)) tryWithContext(s"Error in node mapping for: $nodeKey") { NodeToViewMapping( @@ -338,7 +449,8 @@ object GraphDdl { rmd.relTypeToView.map { rvd => tryWithContext(s"Error in relationship mapping for: ${rmd.relType}") { - val edgeKey = EdgeViewKey(relType, toViewId(maybeSetSchema, rvd.viewDef.viewId)) + val edgeKey = + EdgeViewKey(relType, toViewId(maybeSetSchema, rvd.viewDef.viewId)) tryWithContext(s"Error in relationship mapping for: $edgeKey") { EdgeToViewMapping( @@ -347,22 +459,29 @@ object GraphDdl { startNode = StartNode( nodeViewKey = NodeViewKey( nodeType = edgeKey.relType.startNodeType, - viewId = toViewId(maybeSetSchema, rvd.startNodeTypeToView.viewDef.viewId) + viewId = toViewId( + maybeSetSchema, + rvd.startNodeTypeToView.viewDef.viewId + ) ), - joinPredicates = rvd.startNodeTypeToView.joinOn.joinPredicates.map(toJoin( - nodeAlias = rvd.startNodeTypeToView.viewDef.alias, - edgeAlias = rvd.viewDef.alias - )) + joinPredicates = rvd.startNodeTypeToView.joinOn.joinPredicates.map( + toJoin( + nodeAlias = rvd.startNodeTypeToView.viewDef.alias, + edgeAlias = rvd.viewDef.alias + ) + ) ), endNode = EndNode( nodeViewKey = NodeViewKey( nodeType = edgeKey.relType.endNodeType, viewId = toViewId(maybeSetSchema, rvd.endNodeTypeToView.viewDef.viewId) ), - joinPredicates = rvd.endNodeTypeToView.joinOn.joinPredicates.map(toJoin( - nodeAlias = rvd.endNodeTypeToView.viewDef.alias, - edgeAlias = rvd.viewDef.alias - )) + joinPredicates = rvd.endNodeTypeToView.joinOn.joinPredicates.map( + toJoin( + nodeAlias = rvd.endNodeTypeToView.viewDef.alias, + edgeAlias = rvd.viewDef.alias + ) + ) ), propertyMappings = toPropertyMappings( elementTypes = edgeKey.relType.labels, @@ -380,18 +499,29 @@ object GraphDdl { viewId: List[String] ): ViewId = ViewId(maybeSetSchema, viewId) - private def toJoin(nodeAlias: String, edgeAlias: String)(join: (ColumnIdentifier, ColumnIdentifier)): Join = { + private def toJoin(nodeAlias: String, edgeAlias: String)( + join: (ColumnIdentifier, ColumnIdentifier) + ): Join = { val (left, right) = join val (leftAlias, rightAlias) = (left.head, right.head) - val (leftColumn, rightColumn) = (left.tail.mkString("."), right.tail.mkString(".")) + val (leftColumn, rightColumn) = + (left.tail.mkString("."), right.tail.mkString(".")) (leftAlias, rightAlias) match { - case (`nodeAlias`, `edgeAlias`) => Join(nodeColumn = leftColumn, edgeColumn = rightColumn) - case (`edgeAlias`, `nodeAlias`) => Join(nodeColumn = rightColumn, edgeColumn = leftColumn) + case (`nodeAlias`, `edgeAlias`) => + Join(nodeColumn = leftColumn, edgeColumn = rightColumn) + case (`edgeAlias`, `nodeAlias`) => + Join(nodeColumn = rightColumn, edgeColumn = leftColumn) case _ => val aliases = Set(nodeAlias, edgeAlias) - if (!aliases.contains(leftAlias)) unresolved("Unresolved alias", leftAlias, aliases) - if (!aliases.contains(rightAlias)) unresolved("Unresolved alias", rightAlias, aliases) - unresolved(s"Unable to resolve aliases", s"$leftAlias, $rightAlias", aliases) + if (!aliases.contains(leftAlias)) + unresolved("Unresolved alias", leftAlias, aliases) + if (!aliases.contains(rightAlias)) + unresolved("Unresolved alias", rightAlias, aliases) + unresolved( + s"Unable to resolve aliases", + s"$leftAlias, $rightAlias", + aliases + ) } } @@ -418,27 +548,40 @@ object GraphDdl { private def tryWithGraph[T](name: String)(block: => T): T = tryWithContext(s"Error in graph: $name")(block) - private def tryWithNode[T](nodeTypeDefinition: NodeTypeDefinition)(block: => T): T = + private def tryWithNode[T](nodeTypeDefinition: NodeTypeDefinition)( + block: => T + ): T = tryWithContext(s"Error in node type: $nodeTypeDefinition")(block) - private def tryWithRel[T](relationshipTypeDefinition: RelationshipTypeDefinition)(block: => T): T = - tryWithContext(s"Error in relationship type: $relationshipTypeDefinition")(block) + private def tryWithRel[T]( + relationshipTypeDefinition: RelationshipTypeDefinition + )(block: => T): T = + tryWithContext(s"Error in relationship type: $relationshipTypeDefinition")( + block + ) - private implicit class TraversableOps[T, C[X] <: Traversable[X]](elems: C[T]) { + private implicit class TraversableOps[T, C[X] <: Traversable[X]]( + elems: C[T] + ) { def keyBy[K](key: T => K): Map[K, T] = elems.map(t => key(t) -> t).toMap - def validateDistinctBy[K](key: T => K, msg: String, error: (String, Any) => Nothing = duplicate): C[T] = { + def validateDistinctBy[K]( + key: T => K, + msg: String, + error: (String, Any) => Nothing = duplicate + ): C[T] = { elems.groupBy(key).foreach { case (k, values) if values.size > 1 => error(msg, k) - case _ => + case _ => } elems } } private implicit class MapOps[K, V](map: Map[K, V]) { - def getOrFail(key: K, msg: String): V = map.getOrElse(key, unresolved(msg, key, map.keySet)) + def getOrFail(key: K, msg: String): V = + map.getOrElse(key, unresolved(msg, key, map.keySet)) } } @@ -455,10 +598,13 @@ case class Graph( edgeToViewMappings: List[EdgeToViewMapping] ) { - def nodeIdColumnsFor(nodeViewKey: NodeViewKey): Option[List[String]] = edgeToViewMappings.collectFirst { - case evm: EdgeToViewMapping if evm.startNode.nodeViewKey == nodeViewKey => evm.startNodeJoinColumns - case evm: EdgeToViewMapping if evm.endNode.nodeViewKey == nodeViewKey => evm.endNodeJoinColumns - } + def nodeIdColumnsFor(nodeViewKey: NodeViewKey): Option[List[String]] = + edgeToViewMappings.collectFirst { + case evm: EdgeToViewMapping if evm.startNode.nodeViewKey == nodeViewKey => + evm.startNodeJoinColumns + case evm: EdgeToViewMapping if evm.endNode.nodeViewKey == nodeViewKey => + evm.endNodeJoinColumns + } } object GraphType { @@ -472,11 +618,14 @@ case class GraphType( relTypes: Set[RelationshipType] = Set.empty ) { - private lazy val elementTypesByName = elementTypes.map(et => et.name -> et).toMap + private lazy val elementTypesByName = + elementTypes.map(et => et.name -> et).toMap - def nodeElementTypes: Set[ElementType] = nodeTypes.flatMap(_.labels).map(elementTypesByName) + def nodeElementTypes: Set[ElementType] = + nodeTypes.flatMap(_.labels).map(elementTypesByName) - def relElementTypes: Set[ElementType] = relTypes.flatMap(_.labels).map(elementTypesByName) + def relElementTypes: Set[ElementType] = + relTypes.flatMap(_.labels).map(elementTypesByName) def nodePropertyKeys(labels: String*): PropertyKeys = nodePropertyKeys(NodeType(labels.toSet)) @@ -484,57 +633,99 @@ case class GraphType( def nodePropertyKeys(nodeType: NodeType): PropertyKeys = getPropertyKeys(nodeType.labels) - def relationshipPropertyKeys(startLabel: String, label: String, endLabel: String): PropertyKeys = + def relationshipPropertyKeys( + startLabel: String, + label: String, + endLabel: String + ): PropertyKeys = relationshipPropertyKeys(RelationshipType(startLabel, label, endLabel)) - def relationshipPropertyKeys(startLabels: Set[String], labels: Set[String], endLabels: Set[String]): PropertyKeys = - relationshipPropertyKeys(RelationshipType(NodeType(startLabels), labels, NodeType(endLabels))) + def relationshipPropertyKeys( + startLabels: Set[String], + labels: Set[String], + endLabels: Set[String] + ): PropertyKeys = + relationshipPropertyKeys( + RelationshipType(NodeType(startLabels), labels, NodeType(endLabels)) + ) - def relationshipPropertyKeys(relationshipType: RelationshipType): PropertyKeys = + def relationshipPropertyKeys( + relationshipType: RelationshipType + ): PropertyKeys = getPropertyKeys(relationshipType.labels) def withName(name: String): GraphType = copy(name = name) - def withElementType(label: String, propertyKeys: (String, CypherType)*): GraphType = + def withElementType( + label: String, + propertyKeys: (String, CypherType)* + ): GraphType = withElementType(ElementType(name = label, properties = propertyKeys.toMap)) - def withElementType(label: String, parents: Set[String], propertyKeys: (String, CypherType)*): GraphType = - withElementType(ElementType(name = label, parents = parents, properties = propertyKeys.toMap)) + def withElementType( + label: String, + parents: Set[String], + propertyKeys: (String, CypherType)* + ): GraphType = + withElementType( + ElementType( + name = label, + parents = parents, + properties = propertyKeys.toMap + ) + ) def withElementType(elementType: ElementType): GraphType = { validateElementType(elementType) copy(elementTypes = elementTypes + elementType) } - def withNodeType(labels: String*): GraphType = withNodeType(NodeType(labels: _*)) + def withNodeType(labels: String*): GraphType = withNodeType( + NodeType(labels: _*) + ) def withNodeType(nodeType: NodeType): GraphType = { validateNodeType(nodeType) copy(nodeTypes = nodeTypes + expandNodeType(nodeType)) } - def withRelationshipType(startLabel: String, label: String, endLabel: String): GraphType = + def withRelationshipType( + startLabel: String, + label: String, + endLabel: String + ): GraphType = withRelationshipType(Set(startLabel), Set(label), Set(endLabel)) - def withRelationshipType(startLabels: Set[String], labels: Set[String], endLabels: Set[String]): GraphType = - withRelationshipType(RelationshipType(NodeType(startLabels), labels, NodeType(endLabels))) + def withRelationshipType( + startLabels: Set[String], + labels: Set[String], + endLabels: Set[String] + ): GraphType = + withRelationshipType( + RelationshipType(NodeType(startLabels), labels, NodeType(endLabels)) + ) def withRelationshipType(relationshipType: RelationshipType): GraphType = { validateRelType(relationshipType) copy(relTypes = relTypes + expandRelType(relationshipType)) } - private def validateElementType(elementType: ElementType): Unit = tryWithContext(elementType.toString) { - if (elementTypes.contains(elementType)) { - duplicate("Element type already exists", elementType) + private def validateElementType(elementType: ElementType): Unit = + tryWithContext(elementType.toString) { + if (elementTypes.contains(elementType)) { + duplicate("Element type already exists", elementType) + } + elementType.parents.foreach(validateElementTypeHierarchy) } - elementType.parents.foreach(validateElementTypeHierarchy) - } private def validateElementTypeHierarchy(parent: String): Unit = { if (elementTypes.exists(_.name == parent)) { - elementTypes.collectFirst { case et if et.name == parent => et }.get.parents.foreach(validateElementTypeHierarchy) + elementTypes + .collectFirst { case et if et.name == parent => et } + .get + .parents + .foreach(validateElementTypeHierarchy) } else { illegalInheritance("Element type not found", parent) } @@ -547,17 +738,20 @@ case class GraphType( RelationshipType( startNodeType = expandNodeType(relType.startNodeType), labels = relType.labels.flatMap(getElementTypes).map(_.name), - endNodeType = expandNodeType(relType.endNodeType)) + endNodeType = expandNodeType(relType.endNodeType) + ) - private def validateNodeType(nodeType: NodeType): Unit = tryWithContext(nodeType.toString) { - nodeType.labels.foreach(validateElementTypeHierarchy) - } + private def validateNodeType(nodeType: NodeType): Unit = + tryWithContext(nodeType.toString) { + nodeType.labels.foreach(validateElementTypeHierarchy) + } - private def validateRelType(relType: RelationshipType): Unit = tryWithContext(relType.toString) { - validateNodeType(relType.startNodeType) - validateNodeType(relType.endNodeType) - relType.labels.foreach(validateElementTypeHierarchy) - } + private def validateRelType(relType: RelationshipType): Unit = + tryWithContext(relType.toString) { + validateNodeType(relType.startNodeType) + validateNodeType(relType.endNodeType) + relType.labels.foreach(validateElementTypeHierarchy) + } private def getElementTypes(label: String): Set[ElementType] = { val children = elementTypes.filter(_.name == label) @@ -568,23 +762,36 @@ case class GraphType( labels.flatMap(getElementTypes).map(_.properties).reduce(_ |+| _) } -case class ViewId(maybeSetSchema: Option[SetSchemaDefinition], parts: List[String]) { +case class ViewId( + maybeSetSchema: Option[SetSchemaDefinition], + parts: List[String] +) { lazy val dataSource: String = { (maybeSetSchema, parts) match { - case (_, ds :: _ :: _ :: Nil) => ds + case (_, ds :: _ :: _ :: Nil) => ds case (None, ds :: tail) if tail.nonEmpty => ds - case (Some(setSchema), _) => setSchema.dataSource - case _ => malformed("Relative view identifier requires a preceding SET SCHEMA statement", parts.mkString(".")) + case (Some(setSchema), _) => setSchema.dataSource + case _ => + malformed( + "Relative view identifier requires a preceding SET SCHEMA statement", + parts.mkString(".") + ) } } lazy val tableName: String = (maybeSetSchema, parts) match { - case (_, _ :: schema :: view :: Nil) => s"$schema.$view" + case (_, _ :: schema :: view :: Nil) => s"$schema.$view" case (Some(SetSchemaDefinition(_, schema)), view :: Nil) => s"$schema.$view" case (None, view) if view.size < 3 => - malformed("Relative view identifier requires a preceding SET SCHEMA statement", view.mkString(".")) + malformed( + "Relative view identifier requires a preceding SET SCHEMA statement", + view.mkString(".") + ) case (Some(_), view) if view.size > 1 => - malformed("Relative view identifier must have exactly one segment", view.mkString(".")) + malformed( + "Relative view identifier must have exactly one segment", + view.mkString(".") + ) } } @@ -610,9 +817,11 @@ case class EdgeToViewMapping( ) extends ElementToViewMapping { override def key: EdgeViewKey = EdgeViewKey(relType, view) - def startNodeJoinColumns: List[String] = startNode.joinPredicates.map(_.nodeColumn) + def startNodeJoinColumns: List[String] = + startNode.joinPredicates.map(_.nodeColumn) - def endNodeJoinColumns: List[String] = endNode.joinPredicates.map(_.nodeColumn) + def endNodeJoinColumns: List[String] = + endNode.joinPredicates.map(_.nodeColumn) } case class StartNode( @@ -646,12 +855,25 @@ case class NodeType(labels: Set[String]) { } object RelationshipType { - def apply(startNodeElementType: String, label: String, endNodeElementType: String): RelationshipType = - RelationshipType(NodeType(startNodeElementType), Set(label), NodeType(endNodeElementType)) + def apply( + startNodeElementType: String, + label: String, + endNodeElementType: String + ): RelationshipType = + RelationshipType( + NodeType(startNodeElementType), + Set(label), + NodeType(endNodeElementType) + ) } -case class RelationshipType(startNodeType: NodeType, labels: Set[String], endNodeType: NodeType) { - override def toString: String = s"$startNodeType-[${labels.mkString(",")}]->$endNodeType" +case class RelationshipType( + startNodeType: NodeType, + labels: Set[String], + endNodeType: NodeType +) { + override def toString: String = + s"$startNodeType-[${labels.mkString(",")}]->$endNodeType" } trait ElementViewKey { diff --git a/graph-ddl/src/main/scala/org/opencypher/graphddl/GraphDdlAst.scala b/graph-ddl/src/main/scala/org/opencypher/graphddl/GraphDdlAst.scala index 36e45aaf8b..af76a5f5ab 100644 --- a/graph-ddl/src/main/scala/org/opencypher/graphddl/GraphDdlAst.scala +++ b/graph-ddl/src/main/scala/org/opencypher/graphddl/GraphDdlAst.scala @@ -53,72 +53,102 @@ sealed trait GraphTypeStatement extends GraphStatement case class SetSchemaDefinition( dataSource: String, schema: String -) extends GraphDdlAst with DdlStatement +) extends GraphDdlAst + with DdlStatement case class ElementTypeDefinition( name: String, parents: Set[String] = Set.empty, properties: Map[String, CypherType] = Map.empty, maybeKey: Option[KeyDefinition] = None -) extends GraphDdlAst with DdlStatement with GraphTypeStatement +) extends GraphDdlAst + with DdlStatement + with GraphTypeStatement case class GraphTypeDefinition( name: String, statements: List[GraphTypeStatement] = List.empty -) extends GraphDdlAst with DdlStatement +) extends GraphDdlAst + with DdlStatement case class GraphDefinition( name: String, maybeGraphTypeName: Option[String] = None, statements: List[GraphStatement] = List.empty -) extends GraphDdlAst with DdlStatement +) extends GraphDdlAst + with DdlStatement object NodeTypeDefinition { - def apply(elementTypes: String*): NodeTypeDefinition = NodeTypeDefinition(elementTypes.toSet) + def apply(elementTypes: String*): NodeTypeDefinition = NodeTypeDefinition( + elementTypes.toSet + ) } case class NodeTypeDefinition( elementTypes: Set[String] -) extends GraphDdlAst with GraphTypeStatement { +) extends GraphDdlAst + with GraphTypeStatement { override def toString: String = s"(${elementTypes.mkString(",")})" } object RelationshipTypeDefinition { - def apply(startNodeElementType: String, elementType: String, endNodeElementType: String): RelationshipTypeDefinition = - RelationshipTypeDefinition(NodeTypeDefinition(startNodeElementType), Set(elementType), NodeTypeDefinition(endNodeElementType)) - - def apply(startNodeElementTypes: String*)(elementTypes: String*)(endNodeElementTypes: String*): RelationshipTypeDefinition = - RelationshipTypeDefinition(NodeTypeDefinition(startNodeElementTypes.toSet), elementTypes.toSet, NodeTypeDefinition(endNodeElementTypes.toSet)) + def apply( + startNodeElementType: String, + elementType: String, + endNodeElementType: String + ): RelationshipTypeDefinition = + RelationshipTypeDefinition( + NodeTypeDefinition(startNodeElementType), + Set(elementType), + NodeTypeDefinition(endNodeElementType) + ) + + def apply(startNodeElementTypes: String*)( + elementTypes: String* + )(endNodeElementTypes: String*): RelationshipTypeDefinition = + RelationshipTypeDefinition( + NodeTypeDefinition(startNodeElementTypes.toSet), + elementTypes.toSet, + NodeTypeDefinition(endNodeElementTypes.toSet) + ) } case class RelationshipTypeDefinition( startNodeType: NodeTypeDefinition, elementTypes: Set[String], endNodeType: NodeTypeDefinition -) extends GraphDdlAst with GraphTypeStatement { - override def toString: String = s"$startNodeType-[${elementTypes.mkString(",")}]->$endNodeType" +) extends GraphDdlAst + with GraphTypeStatement { + override def toString: String = + s"$startNodeType-[${elementTypes.mkString(",")}]->$endNodeType" } trait ElementToViewDefinition { def maybePropertyMapping: Option[PropertyToColumnMappingDefinition] } -case class NodeToViewDefinition ( +case class NodeToViewDefinition( viewId: List[String], - override val maybePropertyMapping: Option[PropertyToColumnMappingDefinition] = None -) extends GraphDdlAst with ElementToViewDefinition + override val maybePropertyMapping: Option[ + PropertyToColumnMappingDefinition + ] = None +) extends GraphDdlAst + with ElementToViewDefinition case class NodeMappingDefinition( nodeType: NodeTypeDefinition, nodeToView: List[NodeToViewDefinition] = List.empty -) extends GraphDdlAst with GraphStatement +) extends GraphDdlAst + with GraphStatement case class ViewDefinition( viewId: List[String], alias: String ) extends GraphDdlAst -case class JoinOnDefinition(joinPredicates: List[(ColumnIdentifier, ColumnIdentifier)]) extends GraphDdlAst +case class JoinOnDefinition( + joinPredicates: List[(ColumnIdentifier, ColumnIdentifier)] +) extends GraphDdlAst case class NodeTypeToViewDefinition( nodeType: NodeTypeDefinition, @@ -128,12 +158,16 @@ case class NodeTypeToViewDefinition( case class RelationshipTypeToViewDefinition( viewDef: ViewDefinition, - override val maybePropertyMapping: Option[PropertyToColumnMappingDefinition] = None, + override val maybePropertyMapping: Option[ + PropertyToColumnMappingDefinition + ] = None, startNodeTypeToView: NodeTypeToViewDefinition, endNodeTypeToView: NodeTypeToViewDefinition -) extends GraphDdlAst with ElementToViewDefinition +) extends GraphDdlAst + with ElementToViewDefinition case class RelationshipMappingDefinition( relType: RelationshipTypeDefinition, relTypeToView: List[RelationshipTypeToViewDefinition] -) extends GraphDdlAst with GraphStatement +) extends GraphDdlAst + with GraphStatement diff --git a/graph-ddl/src/main/scala/org/opencypher/graphddl/GraphDdlException.scala b/graph-ddl/src/main/scala/org/opencypher/graphddl/GraphDdlException.scala index cf7a2ca59a..fb0211f8f3 100644 --- a/graph-ddl/src/main/scala/org/opencypher/graphddl/GraphDdlException.scala +++ b/graph-ddl/src/main/scala/org/opencypher/graphddl/GraphDdlException.scala @@ -28,65 +28,95 @@ package org.opencypher.graphddl import org.opencypher.okapi.api.types.CypherType -abstract class GraphDdlException(msg: String, cause: Option[Exception] = None) extends RuntimeException(msg, cause.orNull) with Serializable { +abstract class GraphDdlException(msg: String, cause: Option[Exception] = None) + extends RuntimeException(msg, cause.orNull) + with Serializable { import GraphDdlException._ def getFullMessage: String = causeChain(this).map(_.getMessage).mkString("\n") } private[graphddl] object GraphDdlException { - def unresolved(desc: String, reference: Any): Nothing = throw UnresolvedReferenceException( - s"$desc: $reference" - ) + def unresolved(desc: String, reference: Any): Nothing = + throw UnresolvedReferenceException( + s"$desc: $reference" + ) - def unresolved(desc: String, reference: Any, available: Traversable[Any]): Nothing = throw UnresolvedReferenceException( + def unresolved( + desc: String, + reference: Any, + available: Traversable[Any] + ): Nothing = throw UnresolvedReferenceException( s"""$desc: $reference |Expected one of: ${available.mkString(", ")}""".stripMargin ) - def duplicate(desc: String, definition: Any): Nothing = throw DuplicateDefinitionException( - s"$desc: $definition" - ) + def duplicate(desc: String, definition: Any): Nothing = + throw DuplicateDefinitionException( + s"$desc: $definition" + ) - def illegalInheritance(desc: String, reference: Any): Nothing = throw IllegalInheritanceException( - s"$desc: $reference" - ) + def illegalInheritance(desc: String, reference: Any): Nothing = + throw IllegalInheritanceException( + s"$desc: $reference" + ) - def illegalConstraint(desc: String, reference: Any): Nothing = throw IllegalConstraintException( - s"$desc: $reference" - ) + def illegalConstraint(desc: String, reference: Any): Nothing = + throw IllegalConstraintException( + s"$desc: $reference" + ) def incompatibleTypes(msg: String): Nothing = throw TypeException(msg) - def incompatibleTypes(key: String, t1: CypherType, t2: CypherType): Nothing = throw TypeException( - s"""|Incompatible property types for property key: $key + def incompatibleTypes(key: String, t1: CypherType, t2: CypherType): Nothing = + throw TypeException( + s"""|Incompatible property types for property key: $key |Conflicting types: $t1 and $t2""".stripMargin - ) + ) def malformed(desc: String, identifier: String): Nothing = throw MalformedIdentifier(s"$desc: $identifier") def tryWithContext[T](msg: String)(block: => T): T = - try { block } catch { case e: Exception => throw ContextualizedException(msg, Some(e)) } + try { block } + catch { case e: Exception => throw ContextualizedException(msg, Some(e)) } def causeChain(e: Throwable): List[Throwable] = causeChain(e, Set.empty) def causeChain(e: Throwable, seen: Set[Throwable]): List[Throwable] = { val newSeen = seen + e - e :: Option(e.getCause).filterNot(newSeen).toList.flatMap(causeChain(_, newSeen)) + e :: Option(e.getCause) + .filterNot(newSeen) + .toList + .flatMap(causeChain(_, newSeen)) } } -case class UnresolvedReferenceException(msg: String, cause: Option[Exception] = None) extends GraphDdlException(msg, cause) +case class UnresolvedReferenceException( + msg: String, + cause: Option[Exception] = None +) extends GraphDdlException(msg, cause) -case class DuplicateDefinitionException(msg: String, cause: Option[Exception] = None) extends GraphDdlException(msg, cause) +case class DuplicateDefinitionException( + msg: String, + cause: Option[Exception] = None +) extends GraphDdlException(msg, cause) -case class IllegalInheritanceException(msg: String, cause: Option[Exception] = None) extends GraphDdlException(msg, cause) +case class IllegalInheritanceException( + msg: String, + cause: Option[Exception] = None +) extends GraphDdlException(msg, cause) -case class IllegalConstraintException(msg: String, cause: Option[Exception] = None) extends GraphDdlException(msg, cause) +case class IllegalConstraintException( + msg: String, + cause: Option[Exception] = None +) extends GraphDdlException(msg, cause) -case class TypeException(msg: String, cause: Option[Exception] = None) extends GraphDdlException(msg, cause) +case class TypeException(msg: String, cause: Option[Exception] = None) + extends GraphDdlException(msg, cause) -case class MalformedIdentifier(msg: String, cause: Option[Exception] = None) extends GraphDdlException(msg, cause) +case class MalformedIdentifier(msg: String, cause: Option[Exception] = None) + extends GraphDdlException(msg, cause) -case class ContextualizedException(msg: String, cause: Option[Exception] = None) extends GraphDdlException(msg, cause) \ No newline at end of file +case class ContextualizedException(msg: String, cause: Option[Exception] = None) + extends GraphDdlException(msg, cause) diff --git a/graph-ddl/src/main/scala/org/opencypher/graphddl/GraphDdlParser.scala b/graph-ddl/src/main/scala/org/opencypher/graphddl/GraphDdlParser.scala index 19afea972b..4a83b62f38 100644 --- a/graph-ddl/src/main/scala/org/opencypher/graphddl/GraphDdlParser.scala +++ b/graph-ddl/src/main/scala/org/opencypher/graphddl/GraphDdlParser.scala @@ -36,14 +36,14 @@ case class DdlParsingException( locationPointer: String, expected: String, tracedFailure: TracedFailure -) extends RuntimeException( - s"""|Failed at index $index: +) extends RuntimeException(s"""|Failed at index $index: | |Expected:\t$expected | |$locationPointer | - |${tracedFailure.msg}""".stripMargin) with Serializable + |${tracedFailure.msg}""".stripMargin) + with Serializable object GraphDdlParser { @@ -54,10 +54,17 @@ object GraphDdlParser { val before = index - math.max(index - 20, 0) val after = math.min(index + 20, extra.input.length) - index val locationPointer = - s"""|\t${extra.input.slice(index - before, index + after).replace('\n', ' ')} + s"""|\t${extra.input + .slice(index - before, index + after) + .replace('\n', ' ')} |\t${"~" * before + "^" + "~" * after} """.stripMargin - throw DdlParsingException(index, locationPointer, expected, extra.trace()) + throw DdlParsingException( + index, + locationPointer, + expected, + extra.trace() + ) } } @@ -81,7 +88,6 @@ object GraphDdlParser { private def SET[_: P]: P[Unit] = keyword("SET") private def SCHEMA[_: P]: P[Unit] = keyword("SCHEMA") - // ==== Element types ==== private def property[_: P]: P[(String, CypherType)] = @@ -91,16 +97,24 @@ object GraphDdlParser { P("(" ~/ property.rep(min = 0, sep = ",").map(_.toMap) ~/ ")") private def keyDefinition[_: P]: P[(String, Set[String])] = - P(KEY ~/ identifier.! ~/ "(" ~/ identifier.!.rep(min = 1, sep = ",").map(_.toSet) ~/ ")") + P( + KEY ~/ identifier.! ~/ "(" ~/ identifier.!.rep(min = 1, sep = ",") + .map(_.toSet) ~/ ")" + ) private def extendsDefinition[_: P]: P[Set[String]] = P(EXTENDS ~/ identifier.!.rep(min = 1, sep = ",").map(_.toSet)) def elementTypeDefinition[_: P]: P[ElementTypeDefinition] = - P(identifier.! ~/ extendsDefinition.? ~/ properties.? ~/ keyDefinition.?).map { - case (id, maybeParents, maybeProps, maybeKey) => - ElementTypeDefinition(id, maybeParents.getOrElse(Set.empty), maybeProps.getOrElse(Map.empty), maybeKey) - } + P(identifier.! ~/ extendsDefinition.? ~/ properties.? ~/ keyDefinition.?) + .map { case (id, maybeParents, maybeProps, maybeKey) => + ElementTypeDefinition( + id, + maybeParents.getOrElse(Set.empty), + maybeProps.getOrElse(Map.empty), + maybeKey + ) + } def globalElementTypeDefinition[_: P]: P[ElementTypeDefinition] = P(CREATE ~ ELEMENT ~/ TYPE ~/ elementTypeDefinition) @@ -117,17 +131,23 @@ object GraphDdlParser { P("(" ~ elementTypes ~ ")").map(NodeTypeDefinition(_)) def relTypeDefinition[_: P]: P[RelationshipTypeDefinition] = - P(nodeTypeDefinition ~ "-" ~ "[" ~ elementTypes ~ "]" ~ "->" ~ nodeTypeDefinition).map { - case (startNodeType, eType, endNodeType) => RelationshipTypeDefinition(startNodeType, eType, endNodeType) + P( + nodeTypeDefinition ~ "-" ~ "[" ~ elementTypes ~ "]" ~ "->" ~ nodeTypeDefinition + ).map { case (startNodeType, eType, endNodeType) => + RelationshipTypeDefinition(startNodeType, eType, endNodeType) } def graphTypeStatements[_: P]: P[List[GraphDdlAst with GraphTypeStatement]] = // Note: Order matters here. relTypeDefinition must appear before nodeTypeDefinition since they parse the same prefix - P("(" ~/ (elementTypeDefinition | relTypeDefinition | nodeTypeDefinition ).rep(sep = "," ~/ Pass).map(_.toList) ~/ ")") + P( + "(" ~/ (elementTypeDefinition | relTypeDefinition | nodeTypeDefinition) + .rep(sep = "," ~/ Pass) + .map(_.toList) ~/ ")" + ) def graphTypeDefinition[_: P]: P[GraphTypeDefinition] = - P(CREATE ~ GRAPH ~ TYPE ~/ identifier.! ~/ graphTypeStatements).map(GraphTypeDefinition.tupled) - + P(CREATE ~ GRAPH ~ TYPE ~/ identifier.! ~/ graphTypeStatements) + .map(GraphTypeDefinition.tupled) // ==== Graph ==== @@ -135,7 +155,9 @@ object GraphDdlParser { P(escapedIdentifier.repX(min = 1, max = 3, sep = ".")).map(_.toList) private def propertyToColumn[_: P]: P[(String, String)] = - P(identifier.! ~ AS ~/ identifier.!).map { case (column, propertyKey) => propertyKey -> column } + P(identifier.! ~ AS ~/ identifier.!).map { case (column, propertyKey) => + propertyKey -> column + } // TODO: avoid toMap to not accidentally swallow duplicate property keys def propertyMappingDefinition[_: P]: P[Map[String, String]] = { @@ -143,10 +165,15 @@ object GraphDdlParser { } def nodeToViewDefinition[_: P]: P[NodeToViewDefinition] = - P(FROM ~/ viewId ~/ propertyMappingDefinition.?).map(NodeToViewDefinition.tupled) + P(FROM ~/ viewId ~/ propertyMappingDefinition.?) + .map(NodeToViewDefinition.tupled) def nodeMappingDefinition[_: P]: P[NodeMappingDefinition] = { - P(nodeTypeDefinition ~ nodeToViewDefinition.rep(min = 1, sep = ",".?).map(_.toList)).map(NodeMappingDefinition.tupled) + P( + nodeTypeDefinition ~ nodeToViewDefinition + .rep(min = 1, sep = ",".?) + .map(_.toList) + ).map(NodeMappingDefinition.tupled) } def nodeMappings[_: P]: P[List[NodeMappingDefinition]] = @@ -159,40 +186,60 @@ object GraphDdlParser { P(columnIdentifier ~/ "=" ~/ columnIdentifier) private def joinOnDefinition[_: P]: P[JoinOnDefinition] = - P(JOIN ~/ ON ~/ joinTuple.rep(min = 1, sep = AND)).map(_.toList).map(JoinOnDefinition) + P(JOIN ~/ ON ~/ joinTuple.rep(min = 1, sep = AND)) + .map(_.toList) + .map(JoinOnDefinition) private def viewDefinition[_: P]: P[ViewDefinition] = P(viewId ~/ identifier.!).map(ViewDefinition.tupled) private def nodeTypeToViewDefinition[_: P]: P[NodeTypeToViewDefinition] = - P(nodeTypeDefinition ~/ FROM ~/ viewDefinition ~/ joinOnDefinition).map(NodeTypeToViewDefinition.tupled) + P(nodeTypeDefinition ~/ FROM ~/ viewDefinition ~/ joinOnDefinition) + .map(NodeTypeToViewDefinition.tupled) private def relTypeToViewDefinition[_: P]: P[RelationshipTypeToViewDefinition] = - P(FROM ~/ viewDefinition ~/ propertyMappingDefinition.? ~/ START ~/ NODES ~/ nodeTypeToViewDefinition ~/ END ~/ NODES ~/ nodeTypeToViewDefinition).map(RelationshipTypeToViewDefinition.tupled) + P( + FROM ~/ viewDefinition ~/ propertyMappingDefinition.? ~/ START ~/ NODES ~/ nodeTypeToViewDefinition ~/ END ~/ NODES ~/ nodeTypeToViewDefinition + ).map(RelationshipTypeToViewDefinition.tupled) def relationshipMappingDefinition[_: P]: P[RelationshipMappingDefinition] = { - P(relTypeDefinition ~ relTypeToViewDefinition.rep(min = 1, sep = ",".?).map(_.toList)).map(RelationshipMappingDefinition.tupled) + P( + relTypeDefinition ~ relTypeToViewDefinition + .rep(min = 1, sep = ",".?) + .map(_.toList) + ).map(RelationshipMappingDefinition.tupled) } def relationshipMappings[_: P]: P[List[RelationshipMappingDefinition]] = P(relationshipMappingDefinition.rep(min = 1, sep = ",").map(_.toList)) private def graphStatements[_: P]: P[List[GraphDdlAst with GraphStatement]] = - // Note: Order matters here - P("(" ~/ (relationshipMappingDefinition | nodeMappingDefinition | elementTypeDefinition | relTypeDefinition | nodeTypeDefinition ).rep(sep = "," ~/ Pass).map(_.toList) ~/ ")") + // Note: Order matters here + P( + "(" ~/ (relationshipMappingDefinition | nodeMappingDefinition | elementTypeDefinition | relTypeDefinition | nodeTypeDefinition) + .rep(sep = "," ~/ Pass) + .map(_.toList) ~/ ")" + ) def graphDefinition[_: P]: P[GraphDefinition] = { - P(CREATE ~ GRAPH ~ identifier.! ~/ (OF ~/ identifier.!).? ~/ graphStatements) - .map { case (gName, graphTypeRef, statements) => GraphDefinition(gName, graphTypeRef, statements) } + P( + CREATE ~ GRAPH ~ identifier.! ~/ (OF ~/ identifier.!).? ~/ graphStatements + ) + .map { case (gName, graphTypeRef, statements) => + GraphDefinition(gName, graphTypeRef, statements) + } } // ==== DDL ==== def setSchemaDefinition[_: P]: P[SetSchemaDefinition] = - P(SET ~/ SCHEMA ~ identifier.! ~/ "." ~/ identifier.! ~ ";".?).map(SetSchemaDefinition.tupled) + P(SET ~/ SCHEMA ~ identifier.! ~/ "." ~/ identifier.! ~ ";".?) + .map(SetSchemaDefinition.tupled) def ddlStatement[_: P]: P[GraphDdlAst with DdlStatement] = - P(setSchemaDefinition | globalElementTypeDefinition | graphTypeDefinition | graphDefinition) + P( + setSchemaDefinition | globalElementTypeDefinition | graphTypeDefinition | graphDefinition + ) def ddlDefinitions[_: P]: P[DdlDefinition] = // allow for whitespace/comments at the start diff --git a/graph-ddl/src/test/scala/org/opencypher/graphddl/GraphDdlParserTest.scala b/graph-ddl/src/test/scala/org/opencypher/graphddl/GraphDdlParserTest.scala index 5155d4d2d8..3b289582e2 100644 --- a/graph-ddl/src/test/scala/org/opencypher/graphddl/GraphDdlParserTest.scala +++ b/graph-ddl/src/test/scala/org/opencypher/graphddl/GraphDdlParserTest.scala @@ -52,14 +52,15 @@ class GraphDdlParserTest extends BaseTestSuite with MockitoSugar with TestNameFi case _ => } - parsed should matchPattern { - case Success(`expectation`, _) => + parsed should matchPattern { case Success(`expectation`, _) => } } - private def failure[T, Elem](parser: P[_] => P[T], input: String = testName): Unit = { - parse(input, parser) should matchPattern { - case Failure(_, _, _) => + private def failure[T, Elem]( + parser: P[_] => P[T], + input: String = testName + ): Unit = { + parse(input, parser) should matchPattern { case Failure(_, _, _) => } } @@ -71,10 +72,17 @@ class GraphDdlParserTest extends BaseTestSuite with MockitoSugar with TestNameFi val before = index - math.max(index - 20, 0) val after = math.min(index + 20, extra.input.length) - index val locationPointer = - s"""|\t${extra.input.slice(index - before, index + after).replace('\n', ' ')} + s"""|\t${extra.input + .slice(index - before, index + after) + .replace('\n', ' ')} |\t${"~" * before + "^" + "~" * after} """.stripMargin - throw DdlParsingException(index, locationPointer, expected, extra.trace()) + throw DdlParsingException( + index, + locationPointer, + expected, + extra.trace() + ) } } @@ -103,15 +111,24 @@ class GraphDdlParserTest extends BaseTestSuite with MockitoSugar with TestNameFi } it("parses A ( foo string? )") { - success(elementTypeDefinition(_), ElementTypeDefinition("A", properties = Map("foo" -> CTString.nullable))) + success( + elementTypeDefinition(_), + ElementTypeDefinition("A", properties = Map("foo" -> CTString.nullable)) + ) } it("parses A ( key FLOAT )") { - success(elementTypeDefinition(_), ElementTypeDefinition("A", properties = Map("key" -> CTFloat))) + success( + elementTypeDefinition(_), + ElementTypeDefinition("A", properties = Map("key" -> CTFloat)) + ) } it("parses A ( key FLOAT? )") { - success(elementTypeDefinition(_), ElementTypeDefinition("A", properties = Map("key" -> CTFloat.nullable))) + success( + elementTypeDefinition(_), + ElementTypeDefinition("A", properties = Map("key" -> CTFloat.nullable)) + ) } it("!parses A ( key _ STRING )") { @@ -119,23 +136,44 @@ class GraphDdlParserTest extends BaseTestSuite with MockitoSugar with TestNameFi } it("parses A ( key1 FLOAT, key2 STRING)") { - success(elementTypeDefinition(_), ElementTypeDefinition("A", properties = Map("key1" -> CTFloat, "key2" -> CTString))) + success( + elementTypeDefinition(_), + ElementTypeDefinition( + "A", + properties = Map("key1" -> CTFloat, "key2" -> CTString) + ) + ) } it("parses A ( key DATE )") { - success(elementTypeDefinition(_), ElementTypeDefinition("A", properties = Map("key" -> CTDate))) + success( + elementTypeDefinition(_), + ElementTypeDefinition("A", properties = Map("key" -> CTDate)) + ) } it("parses A ( key DATE? )") { - success(elementTypeDefinition(_), ElementTypeDefinition("A", properties = Map("key" -> CTDate.nullable))) + success( + elementTypeDefinition(_), + ElementTypeDefinition("A", properties = Map("key" -> CTDate.nullable)) + ) } it("parses A ( key LOCALDATETIME )") { - success(elementTypeDefinition(_), ElementTypeDefinition("A", properties = Map("key" -> CTLocalDateTime))) + success( + elementTypeDefinition(_), + ElementTypeDefinition("A", properties = Map("key" -> CTLocalDateTime)) + ) } it("parses A ( key LOCALDATETIME? )") { - success(elementTypeDefinition(_), ElementTypeDefinition("A", properties = Map("key" -> CTLocalDateTime.nullable))) + success( + elementTypeDefinition(_), + ElementTypeDefinition( + "A", + properties = Map("key" -> CTLocalDateTime.nullable) + ) + ) } it("parses A ()") { @@ -143,27 +181,53 @@ class GraphDdlParserTest extends BaseTestSuite with MockitoSugar with TestNameFi } it("parses A EXTENDS B ()") { - success(elementTypeDefinition(_), ElementTypeDefinition("A", parents = Set("B"))) + success( + elementTypeDefinition(_), + ElementTypeDefinition("A", parents = Set("B")) + ) } it("parses A <: B ()") { - success(elementTypeDefinition(_), ElementTypeDefinition("A", parents = Set("B"))) + success( + elementTypeDefinition(_), + ElementTypeDefinition("A", parents = Set("B")) + ) } it("parses A EXTENDS B, C ()") { - success(elementTypeDefinition(_), ElementTypeDefinition("A", parents = Set("B", "C"))) + success( + elementTypeDefinition(_), + ElementTypeDefinition("A", parents = Set("B", "C")) + ) } it("parses A <: B, C ()") { - success(elementTypeDefinition(_), ElementTypeDefinition("A", parents = Set("B", "C"))) + success( + elementTypeDefinition(_), + ElementTypeDefinition("A", parents = Set("B", "C")) + ) } it("parses A EXTENDS B, C ( key STRING )") { - success(elementTypeDefinition(_), ElementTypeDefinition("A", parents = Set("B", "C"), properties = Map("key" -> CTString))) + success( + elementTypeDefinition(_), + ElementTypeDefinition( + "A", + parents = Set("B", "C"), + properties = Map("key" -> CTString) + ) + ) } it("parses A <: B, C ( key STRING )") { - success(elementTypeDefinition(_), ElementTypeDefinition("A", parents = Set("B", "C"), properties = Map("key" -> CTString))) + success( + elementTypeDefinition(_), + ElementTypeDefinition( + "A", + parents = Set("B", "C"), + properties = Map("key" -> CTString) + ) + ) } } @@ -173,15 +237,32 @@ class GraphDdlParserTest extends BaseTestSuite with MockitoSugar with TestNameFi } it("parses CREATE ELEMENT TYPE A ( foo STRING ) ") { - success(globalElementTypeDefinition(_), ElementTypeDefinition("A", properties = Map("foo" -> CTString))) + success( + globalElementTypeDefinition(_), + ElementTypeDefinition("A", properties = Map("foo" -> CTString)) + ) } it("parses CREATE ELEMENT TYPE A KEY A_NK (foo, bar)") { - success(globalElementTypeDefinition(_), ElementTypeDefinition("A", properties = Map.empty, maybeKey = Some("A_NK" -> Set("foo", "bar")))) + success( + globalElementTypeDefinition(_), + ElementTypeDefinition( + "A", + properties = Map.empty, + maybeKey = Some("A_NK" -> Set("foo", "bar")) + ) + ) } it("parses CREATE ELEMENT TYPE A ( foo STRING ) KEY A_NK (foo, bar)") { - success(globalElementTypeDefinition(_), ElementTypeDefinition("A", properties = Map("foo" -> CTString), maybeKey = Some("A_NK" -> Set("foo", "bar")))) + success( + globalElementTypeDefinition(_), + ElementTypeDefinition( + "A", + properties = Map("foo" -> CTString), + maybeKey = Some("A_NK" -> Set("foo", "bar")) + ) + ) } it("!parses CREATE ELEMENT TYPE A ( foo STRING ) KEY A ()") { @@ -203,7 +284,10 @@ class GraphDdlParserTest extends BaseTestSuite with MockitoSugar with TestNameFi } it("parses (A)-[R,S]->(B)") { - success(relTypeDefinition(_), RelationshipTypeDefinition("A")("R", "S")("B")) + success( + relTypeDefinition(_), + RelationshipTypeDefinition("A")("R", "S")("B") + ) } } @@ -220,14 +304,27 @@ class GraphDdlParserTest extends BaseTestSuite with MockitoSugar with TestNameFi | |CREATE ELEMENT TYPE TYPE_1 | - |CREATE ELEMENT TYPE TYPE_2 ( prop BOOLEAN? ) """.stripMargin) shouldEqual - DdlDefinition(List( - SetSchemaDefinition("foo", "bar"), - ElementTypeDefinition("A", properties = Map("name" -> CTString)), - ElementTypeDefinition("B", properties = Map("sequence" -> CTInteger, "nationality" -> CTString.nullable, "age" -> CTInteger.nullable)), - ElementTypeDefinition("TYPE_1"), - ElementTypeDefinition("TYPE_2", properties = Map("prop" -> CTBoolean.nullable)) - )) + |CREATE ELEMENT TYPE TYPE_2 ( prop BOOLEAN? ) """.stripMargin + ) shouldEqual + DdlDefinition( + List( + SetSchemaDefinition("foo", "bar"), + ElementTypeDefinition("A", properties = Map("name" -> CTString)), + ElementTypeDefinition( + "B", + properties = Map( + "sequence" -> CTInteger, + "nationality" -> CTString.nullable, + "age" -> CTInteger.nullable + ) + ), + ElementTypeDefinition("TYPE_1"), + ElementTypeDefinition( + "TYPE_2", + properties = Map("prop" -> CTBoolean.nullable) + ) + ) + ) } it("parses a schema with node type, and rel type definitions") { @@ -245,24 +342,32 @@ class GraphDdlParserTest extends BaseTestSuite with MockitoSugar with TestNameFi | (A, B)-[TYPE_2]->(A) |) """.stripMargin - success(graphTypeDefinition(_), GraphTypeDefinition( - name = "mySchema", - statements = List( - NodeTypeDefinition("A"), - NodeTypeDefinition("B"), - NodeTypeDefinition("A", "B"), - RelationshipTypeDefinition("A", "TYPE_1", "B"), - RelationshipTypeDefinition("A", "B")("TYPE_2")("A") - )), input) + success( + graphTypeDefinition(_), + GraphTypeDefinition( + name = "mySchema", + statements = List( + NodeTypeDefinition("A"), + NodeTypeDefinition("B"), + NodeTypeDefinition("A", "B"), + RelationshipTypeDefinition("A", "TYPE_1", "B"), + RelationshipTypeDefinition("A", "B")("TYPE_2")("A") + ) + ), + input + ) } it("parses CREATE GRAPH TYPE mySchema ( (A)-[TYPE]->(B) )") { - success(graphTypeDefinition(_), + success( + graphTypeDefinition(_), GraphTypeDefinition( name = "mySchema", statements = List( RelationshipTypeDefinition("A", "TYPE", "B") - ))) + ) + ) + ) } it("parses a schema with node and rel definitions in any order") { @@ -277,16 +382,21 @@ class GraphDdlParserTest extends BaseTestSuite with MockitoSugar with TestNameFi | (B)-[TYPE_2]->(A, B) |) """.stripMargin - success(graphTypeDefinition(_), GraphTypeDefinition( - name = "mySchema", - statements = List( - RelationshipTypeDefinition("A", "B")("TYPE_1")("B"), - NodeTypeDefinition("A"), - NodeTypeDefinition("A", "B"), - RelationshipTypeDefinition("A", "TYPE_1", "A"), - NodeTypeDefinition("B"), - RelationshipTypeDefinition("B")("TYPE_2")("A", "B") - )), input) + success( + graphTypeDefinition(_), + GraphTypeDefinition( + name = "mySchema", + statements = List( + RelationshipTypeDefinition("A", "B")("TYPE_1")("B"), + NodeTypeDefinition("A"), + NodeTypeDefinition("A", "B"), + RelationshipTypeDefinition("A", "TYPE_1", "A"), + NodeTypeDefinition("B"), + RelationshipTypeDefinition("B")("TYPE_2")("A", "B") + ) + ), + input + ) } } @@ -305,23 +415,37 @@ class GraphDdlParserTest extends BaseTestSuite with MockitoSugar with TestNameFi ElementTypeDefinition("B"), NodeTypeDefinition("A", "B"), RelationshipTypeDefinition("A", "B")("B")("C"), - NodeMappingDefinition(NodeTypeDefinition("A", "B"), List(NodeToViewDefinition(List("view_a_b")))), + NodeMappingDefinition( + NodeTypeDefinition("A", "B"), + List(NodeToViewDefinition(List("view_a_b"))) + ), RelationshipMappingDefinition( relType = RelationshipTypeDefinition("A", "B")("B")("C"), - relTypeToView = List(RelationshipTypeToViewDefinition( - viewDef = ViewDefinition(List("baz"), "alias_baz"), - startNodeTypeToView = NodeTypeToViewDefinition( - NodeTypeDefinition("A", "B"), - ViewDefinition(List("foo"), "alias_foo"), - JoinOnDefinition(List( - (List("alias_foo", "COLUMN_A"), List("edge", "COLUMN_A")), - (List("alias_foo", "COLUMN_C"), List("edge", "COLUMN_D"))))), - endNodeTypeToView = NodeTypeToViewDefinition( - NodeTypeDefinition("C"), - ViewDefinition(List("bar"), "alias_bar"), - JoinOnDefinition(List( - (List("alias_bar", "COLUMN_A"), List("edge", "COLUMN_A"))))) - ))) + relTypeToView = List( + RelationshipTypeToViewDefinition( + viewDef = ViewDefinition(List("baz"), "alias_baz"), + startNodeTypeToView = NodeTypeToViewDefinition( + NodeTypeDefinition("A", "B"), + ViewDefinition(List("foo"), "alias_foo"), + JoinOnDefinition( + List( + (List("alias_foo", "COLUMN_A"), List("edge", "COLUMN_A")), + (List("alias_foo", "COLUMN_C"), List("edge", "COLUMN_D")) + ) + ) + ), + endNodeTypeToView = NodeTypeToViewDefinition( + NodeTypeDefinition("C"), + ViewDefinition(List("bar"), "alias_bar"), + JoinOnDefinition( + List( + (List("alias_bar", "COLUMN_A"), List("edge", "COLUMN_A")) + ) + ) + ) + ) + ) + ) ) parse( """|CREATE GRAPH myGraph ( @@ -337,13 +461,19 @@ class GraphDdlParserTest extends BaseTestSuite with MockitoSugar with TestNameFi | END NODES (C) FROM bar alias_bar | JOIN ON alias_bar.COLUMN_A = edge.COLUMN_A |) - """.stripMargin, graphDefinition(_)) should matchPattern { - case Success(GraphDefinition("myGraph", None, `expectedGraphStatements`), _) => + """.stripMargin, + graphDefinition(_) + ) should matchPattern { + case Success( + GraphDefinition("myGraph", None, `expectedGraphStatements`), + _ + ) => } } it("fails if node / edge type definitions are present") { - failure(graphDefinition(_), + failure( + graphDefinition(_), """|CREATE GRAPH myGraph ( | A ( foo STRING ) , | B, @@ -352,44 +482,137 @@ class GraphDdlParserTest extends BaseTestSuite with MockitoSugar with TestNameFi | [B] | |) - """.stripMargin) + """.stripMargin + ) } } describe("Node mappings and relationship mappings") { it("parses (A) FROM view") { - success(nodeMappingDefinition(_), NodeMappingDefinition(NodeTypeDefinition("A"), List(NodeToViewDefinition(List("view"))))) + success( + nodeMappingDefinition(_), + NodeMappingDefinition( + NodeTypeDefinition("A"), + List(NodeToViewDefinition(List("view"))) + ) + ) } - it("parses (A) FROM view (column1 AS propertyKey1, column2 AS propertyKey2)") { - success(nodeMappingDefinition(_), NodeMappingDefinition(NodeTypeDefinition("A"), List(NodeToViewDefinition(List("view"), Some(Map("propertyKey1" -> "column1", "propertyKey2" -> "column2")))))) + it( + "parses (A) FROM view (column1 AS propertyKey1, column2 AS propertyKey2)" + ) { + success( + nodeMappingDefinition(_), + NodeMappingDefinition( + NodeTypeDefinition("A"), + List( + NodeToViewDefinition( + List("view"), + Some( + Map("propertyKey1" -> "column1", "propertyKey2" -> "column2") + ) + ) + ) + ) + ) } it("parses (A) FROM viewA FROM viewB") { - success(nodeMappingDefinition(_), NodeMappingDefinition(NodeTypeDefinition("A"), List(NodeToViewDefinition(List("viewA")), NodeToViewDefinition(List("viewB"))))) + success( + nodeMappingDefinition(_), + NodeMappingDefinition( + NodeTypeDefinition("A"), + List( + NodeToViewDefinition(List("viewA")), + NodeToViewDefinition(List("viewB")) + ) + ) + ) } it("parses (A) FROM viewA, (B) FROM viewB") { - success(nodeMappings(_), List(NodeMappingDefinition(NodeTypeDefinition("A"), List(NodeToViewDefinition(List("viewA")))), NodeMappingDefinition(NodeTypeDefinition("B"), List(NodeToViewDefinition(List("viewB")))))) + success( + nodeMappings(_), + List( + NodeMappingDefinition( + NodeTypeDefinition("A"), + List(NodeToViewDefinition(List("viewA"))) + ), + NodeMappingDefinition( + NodeTypeDefinition("B"), + List(NodeToViewDefinition(List("viewB"))) + ) + ) + ) } - it("parses (A) FROM viewA (column1 AS propertyKey1, column2 AS propertyKey2) FROM viewB (column1 AS propertyKey1, column2 AS propertyKey2)") { - success(nodeMappings(_), List( - NodeMappingDefinition(NodeTypeDefinition("A"), List( - NodeToViewDefinition(List("viewA"), Some(Map("propertyKey1" -> "column1", "propertyKey2" -> "column2"))), - NodeToViewDefinition(List("viewB"), Some(Map("propertyKey1" -> "column1", "propertyKey2" -> "column2"))))) - )) + it( + "parses (A) FROM viewA (column1 AS propertyKey1, column2 AS propertyKey2) FROM viewB (column1 AS propertyKey1, column2 AS propertyKey2)" + ) { + success( + nodeMappings(_), + List( + NodeMappingDefinition( + NodeTypeDefinition("A"), + List( + NodeToViewDefinition( + List("viewA"), + Some( + Map("propertyKey1" -> "column1", "propertyKey2" -> "column2") + ) + ), + NodeToViewDefinition( + List("viewB"), + Some( + Map("propertyKey1" -> "column1", "propertyKey2" -> "column2") + ) + ) + ) + ) + ) + ) } it("parses (A) FROM `foo.json`") { - success(nodeMappingDefinition(_), NodeMappingDefinition(NodeTypeDefinition("A"), List(NodeToViewDefinition(List("foo.json"))))) + success( + nodeMappingDefinition(_), + NodeMappingDefinition( + NodeTypeDefinition("A"), + List(NodeToViewDefinition(List("foo.json"))) + ) + ) } - it("parses (A) FROM viewA (column1 AS propertyKey1, column2 AS propertyKey2), (B) FROM viewB (column1 AS propertyKey1, column2 AS propertyKey2)") { - success(nodeMappings(_), List( - NodeMappingDefinition(NodeTypeDefinition("A"), List(NodeToViewDefinition(List("viewA"), Some(Map("propertyKey1" -> "column1", "propertyKey2" -> "column2"))))), - NodeMappingDefinition(NodeTypeDefinition("B"), List(NodeToViewDefinition(List("viewB"), Some(Map("propertyKey1" -> "column1", "propertyKey2" -> "column2"))))) - )) + it( + "parses (A) FROM viewA (column1 AS propertyKey1, column2 AS propertyKey2), (B) FROM viewB (column1 AS propertyKey1, column2 AS propertyKey2)" + ) { + success( + nodeMappings(_), + List( + NodeMappingDefinition( + NodeTypeDefinition("A"), + List( + NodeToViewDefinition( + List("viewA"), + Some( + Map("propertyKey1" -> "column1", "propertyKey2" -> "column2") + ) + ) + ) + ), + NodeMappingDefinition( + NodeTypeDefinition("B"), + List( + NodeToViewDefinition( + List("viewB"), + Some( + Map("propertyKey1" -> "column1", "propertyKey2" -> "column2") + ) + ) + ) + ) + ) + ) } it("parses a relationship mapping definition") { @@ -402,45 +625,79 @@ class GraphDdlParserTest extends BaseTestSuite with MockitoSugar with TestNameFi | JOIN ON alias_bar.COLUMN_A = edge.COLUMN_A """.stripMargin - success(relationshipMappingDefinition(_), RelationshipMappingDefinition( - relType = RelationshipTypeDefinition("X", "Y", "Z"), - relTypeToView = List(RelationshipTypeToViewDefinition( - viewDef = ViewDefinition(List("baz"), "alias_baz"), - startNodeTypeToView = NodeTypeToViewDefinition( - NodeTypeDefinition("A", "B"), - ViewDefinition(List("foo"), "alias_foo"), - JoinOnDefinition(List( - (List("alias_foo", "COLUMN_A"), List("edge", "COLUMN_A")), - (List("alias_foo", "COLUMN_C"), List("edge", "COLUMN_D"))))), - endNodeTypeToView = NodeTypeToViewDefinition( - NodeTypeDefinition("C"), - ViewDefinition(List("bar"), "alias_bar"), - JoinOnDefinition(List( - (List("alias_bar", "COLUMN_A"), List("edge", "COLUMN_A"))))) - ))), input) - } - - it("parses a relationship mapping definition with custom property to column mapping") { + success( + relationshipMappingDefinition(_), + RelationshipMappingDefinition( + relType = RelationshipTypeDefinition("X", "Y", "Z"), + relTypeToView = List( + RelationshipTypeToViewDefinition( + viewDef = ViewDefinition(List("baz"), "alias_baz"), + startNodeTypeToView = NodeTypeToViewDefinition( + NodeTypeDefinition("A", "B"), + ViewDefinition(List("foo"), "alias_foo"), + JoinOnDefinition( + List( + (List("alias_foo", "COLUMN_A"), List("edge", "COLUMN_A")), + (List("alias_foo", "COLUMN_C"), List("edge", "COLUMN_D")) + ) + ) + ), + endNodeTypeToView = NodeTypeToViewDefinition( + NodeTypeDefinition("C"), + ViewDefinition(List("bar"), "alias_bar"), + JoinOnDefinition( + List( + (List("alias_bar", "COLUMN_A"), List("edge", "COLUMN_A")) + ) + ) + ) + ) + ) + ), + input + ) + } + + it( + "parses a relationship mapping definition with custom property to column mapping" + ) { val input = """|(a)-[a]->(a) FROM baz alias_baz ( colA AS foo, colB AS bar ) | START NODES (A, B) FROM foo alias_foo JOIN ON alias_foo.COLUMN_A = edge.COLUMN_A | END NODES (C) FROM bar alias_bar JOIN ON alias_bar.COLUMN_A = edge.COLUMN_A """.stripMargin - success(relationshipMappingDefinition(_), RelationshipMappingDefinition( - relType = RelationshipTypeDefinition("a", "a", "a"), - relTypeToView = List(RelationshipTypeToViewDefinition( - viewDef = ViewDefinition(List("baz"), "alias_baz"), - maybePropertyMapping = Some(Map("foo" -> "colA", "bar" -> "colB")), - startNodeTypeToView = NodeTypeToViewDefinition( - NodeTypeDefinition("A", "B"), - ViewDefinition(List("foo"), "alias_foo"), - JoinOnDefinition(List((List("alias_foo", "COLUMN_A"), List("edge", "COLUMN_A"))))), - endNodeTypeToView = NodeTypeToViewDefinition( - NodeTypeDefinition("C"), - ViewDefinition(List("bar"), "alias_bar"), - JoinOnDefinition(List((List("alias_bar", "COLUMN_A"), List("edge", "COLUMN_A"))))) - ))), input) + success( + relationshipMappingDefinition(_), + RelationshipMappingDefinition( + relType = RelationshipTypeDefinition("a", "a", "a"), + relTypeToView = List( + RelationshipTypeToViewDefinition( + viewDef = ViewDefinition(List("baz"), "alias_baz"), + maybePropertyMapping = Some(Map("foo" -> "colA", "bar" -> "colB")), + startNodeTypeToView = NodeTypeToViewDefinition( + NodeTypeDefinition("A", "B"), + ViewDefinition(List("foo"), "alias_foo"), + JoinOnDefinition( + List( + (List("alias_foo", "COLUMN_A"), List("edge", "COLUMN_A")) + ) + ) + ), + endNodeTypeToView = NodeTypeToViewDefinition( + NodeTypeDefinition("C"), + ViewDefinition(List("bar"), "alias_bar"), + JoinOnDefinition( + List( + (List("alias_bar", "COLUMN_A"), List("edge", "COLUMN_A")) + ) + ) + ) + ) + ) + ), + input + ) } it("parses a relationship label set definition") { @@ -459,19 +716,27 @@ class GraphDdlParserTest extends BaseTestSuite with MockitoSugar with TestNameFi startNodeTypeToView = NodeTypeToViewDefinition( NodeTypeDefinition("A"), ViewDefinition(List("foo"), "alias_foo"), - JoinOnDefinition(List((List("alias_foo", "COLUMN_A"), List("edge", "COLUMN_A"))))), + JoinOnDefinition( + List((List("alias_foo", "COLUMN_A"), List("edge", "COLUMN_A"))) + ) + ), endNodeTypeToView = NodeTypeToViewDefinition( NodeTypeDefinition("B"), ViewDefinition(List("bar"), "alias_bar"), - JoinOnDefinition(List((List("alias_bar", "COLUMN_A"), List("edge", "COLUMN_A"))))) + JoinOnDefinition( + List((List("alias_bar", "COLUMN_A"), List("edge", "COLUMN_A"))) + ) + ) ) success( relationshipMappingDefinition(_), RelationshipMappingDefinition( RelationshipTypeDefinition("A", "TYPE_1", "B"), - List(relMappingDef, relMappingDef)), - input) + List(relMappingDef, relMappingDef) + ), + input + ) } it("parses relationship label sets") { @@ -498,22 +763,33 @@ class GraphDdlParserTest extends BaseTestSuite with MockitoSugar with TestNameFi startNodeTypeToView = NodeTypeToViewDefinition( NodeTypeDefinition("A"), ViewDefinition(List("foo"), "alias_foo"), - JoinOnDefinition(List((List("alias_foo", "COLUMN_A"), List("edge", "COLUMN_A"))))), + JoinOnDefinition( + List((List("alias_foo", "COLUMN_A"), List("edge", "COLUMN_A"))) + ) + ), endNodeTypeToView = NodeTypeToViewDefinition( NodeTypeDefinition("B"), ViewDefinition(List("bar"), "alias_bar"), - JoinOnDefinition(List((List("alias_bar", "COLUMN_A"), List("edge", "COLUMN_A"))))) + JoinOnDefinition( + List((List("alias_bar", "COLUMN_A"), List("edge", "COLUMN_A"))) + ) + ) ) - success(relationshipMappings(_), + success( + relationshipMappings(_), List( RelationshipMappingDefinition( RelationshipTypeDefinition("A", "TYPE_1", "B"), - List(relMappingDef, relMappingDef)), + List(relMappingDef, relMappingDef) + ), RelationshipMappingDefinition( RelationshipTypeDefinition("A", "TYPE_2", "B"), - List(relMappingDef, relMappingDef)) - ), input) + List(relMappingDef, relMappingDef) + ) + ), + input + ) } } diff --git a/graph-ddl/src/test/scala/org/opencypher/graphddl/GraphDdlTest.scala b/graph-ddl/src/test/scala/org/opencypher/graphddl/GraphDdlTest.scala index dc52530594..e618ee46a5 100644 --- a/graph-ddl/src/test/scala/org/opencypher/graphddl/GraphDdlTest.scala +++ b/graph-ddl/src/test/scala/org/opencypher/graphddl/GraphDdlTest.scala @@ -67,38 +67,68 @@ class GraphDdlTest extends AnyFunSpec with Matchers { it("converts to GraphDDL IR") { val graphDdl = GraphDdl(ddlString) - val maybeSetSchema = Some(SetSchemaDefinition("dataSourceName", "fooDatabaseName")) + val maybeSetSchema = + Some(SetSchemaDefinition("dataSourceName", "fooDatabaseName")) - val personKey1 = NodeViewKey(NodeType("Person"), ViewId(maybeSetSchema, List("personView1"))) - val personKey2 = NodeViewKey(NodeType("Person"), ViewId(maybeSetSchema, List("personView2"))) - val bookKey = NodeViewKey(NodeType("Book"), ViewId(maybeSetSchema, List("bookView"))) - val readsKey1 = EdgeViewKey(RelationshipType("Person", "READS", "Book"), ViewId(maybeSetSchema, List("readsView1"))) - val readsKey2 = EdgeViewKey(RelationshipType("Person", "READS", "Book"), ViewId(maybeSetSchema, List("readsView2"))) + val personKey1 = NodeViewKey( + NodeType("Person"), + ViewId(maybeSetSchema, List("personView1")) + ) + val personKey2 = NodeViewKey( + NodeType("Person"), + ViewId(maybeSetSchema, List("personView2")) + ) + val bookKey = + NodeViewKey(NodeType("Book"), ViewId(maybeSetSchema, List("bookView"))) + val readsKey1 = EdgeViewKey( + RelationshipType("Person", "READS", "Book"), + ViewId(maybeSetSchema, List("readsView1")) + ) + val readsKey2 = EdgeViewKey( + RelationshipType("Person", "READS", "Book"), + ViewId(maybeSetSchema, List("readsView2")) + ) val expected = GraphDdl( Map( - GraphName("fooGraph") -> Graph(GraphName("fooGraph"), + GraphName("fooGraph") -> Graph( + GraphName("fooGraph"), GraphType.empty .withName("fooSchema") - .withElementType(ElementType("Person", Set.empty, Map("name" -> CTString, "age" -> CTInteger))) - .withElementType(ElementType("Book", Set.empty, Map("title" -> CTString))) - .withElementType(ElementType("READS", Set.empty, Map("rating" -> CTFloat))) + .withElementType( + ElementType( + "Person", + Set.empty, + Map("name" -> CTString, "age" -> CTInteger) + ) + ) + .withElementType( + ElementType("Book", Set.empty, Map("title" -> CTString)) + ) + .withElementType( + ElementType("READS", Set.empty, Map("rating" -> CTFloat)) + ) .withNodeType(NodeType("Person")) .withNodeType(NodeType("Book")) - .withRelationshipType(RelationshipType("Person", "READS", "Book")), + .withRelationshipType( + RelationshipType("Person", "READS", "Book") + ), Map( personKey1 -> NodeToViewMapping( nodeType = NodeType("Person"), view = personKey1.viewId, - propertyMappings = Map("name" -> "person_name1", "age" -> "age")), + propertyMappings = Map("name" -> "person_name1", "age" -> "age") + ), personKey2 -> NodeToViewMapping( nodeType = NodeType("Person"), view = personKey2.viewId, - propertyMappings = Map("name" -> "person_name2", "age" -> "age")), + propertyMappings = Map("name" -> "person_name2", "age" -> "age") + ), bookKey -> NodeToViewMapping( nodeType = NodeType("Book"), view = bookKey.viewId, - propertyMappings = Map("title" -> "book_title")) + propertyMappings = Map("title" -> "book_title") + ) ), List( EdgeToViewMapping( @@ -106,13 +136,15 @@ class GraphDdlTest extends AnyFunSpec with Matchers { view = readsKey1.viewId, startNode = StartNode(personKey1, List(Join("person_id1", "person"))), endNode = EndNode(bookKey, List(Join("book_id", "book"))), - propertyMappings = Map("rating" -> "value1")), + propertyMappings = Map("rating" -> "value1") + ), EdgeToViewMapping( relType = RelationshipType("Person", "READS", "Book"), view = readsKey2.viewId, startNode = StartNode(personKey2, List(Join("person_id2", "person"))), endNode = EndNode(bookKey, List(Join("book_id", "book"))), - propertyMappings = Map("rating" -> "value2")) + propertyMappings = Map("rating" -> "value2") + ) ) ) ) @@ -122,8 +154,7 @@ class GraphDdlTest extends AnyFunSpec with Matchers { } it("allows compact inline graph definition") { - val ddl = GraphDdl( - """SET SCHEMA ds1.db1 + val ddl = GraphDdl("""SET SCHEMA ds1.db1 |CREATE GRAPH myGraph ( | A (x STRING), B (y STRING), | (A) FROM a, @@ -133,7 +164,10 @@ class GraphDdlTest extends AnyFunSpec with Matchers { |) """.stripMargin) - val A_a = NodeViewKey(NodeType("A"), ViewId(Some(SetSchemaDefinition("ds1", "db1")), List("a"))) + val A_a = NodeViewKey( + NodeType("A"), + ViewId(Some(SetSchemaDefinition("ds1", "db1")), List("a")) + ) ddl.graphs(GraphName("myGraph")) shouldEqual Graph( name = GraphName("myGraph"), @@ -144,10 +178,16 @@ class GraphDdlTest extends AnyFunSpec with Matchers { .withNodeType(NodeType("A")) .withRelationshipType(RelationshipType("A", "B", "A")), nodeToViewMappings = Map( - A_a -> NodeToViewMapping(NodeType("A"), ViewId(Some(SetSchemaDefinition("ds1", "db1")), List("a")), Map("x" -> "x")) + A_a -> NodeToViewMapping( + NodeType("A"), + ViewId(Some(SetSchemaDefinition("ds1", "db1")), List("a")), + Map("x" -> "x") + ) ), edgeToViewMappings = List( - EdgeToViewMapping(RelationshipType("A", "B", "A"), ViewId(Some(SetSchemaDefinition("ds1", "db1")), List("b")), + EdgeToViewMapping( + RelationshipType("A", "B", "A"), + ViewId(Some(SetSchemaDefinition("ds1", "db1")), List("b")), StartNode(A_a, List(Join("id", "id"))), EndNode(A_a, List(Join("id", "id"))), Map("y" -> "y") @@ -159,8 +199,7 @@ class GraphDdlTest extends AnyFunSpec with Matchers { it("allows these equivalent graph definitions") { val ddls = List( // most compact form - GraphDdl( - """SET SCHEMA ds1.db1 + GraphDdl("""SET SCHEMA ds1.db1 |CREATE GRAPH myGraph ( | A (x STRING), B (y STRING), | (A) FROM a, @@ -170,8 +209,7 @@ class GraphDdlTest extends AnyFunSpec with Matchers { |) """.stripMargin), // mixed order - GraphDdl( - """SET SCHEMA ds1.db1 + GraphDdl("""SET SCHEMA ds1.db1 |CREATE GRAPH myGraph ( | (A)-[B]->(A) FROM b e | START NODES (A) FROM a n JOIN ON e.id = n.id @@ -181,8 +219,7 @@ class GraphDdlTest extends AnyFunSpec with Matchers { |) """.stripMargin), // explicit node and rel type definition - GraphDdl( - """SET SCHEMA ds1.db1 + GraphDdl("""SET SCHEMA ds1.db1 |CREATE GRAPH myGraph ( | A (x STRING), B (y STRING), | (A), (A)-[B]->(A), @@ -193,8 +230,7 @@ class GraphDdlTest extends AnyFunSpec with Matchers { |) """.stripMargin), // pure type definitions extracted to graph type - GraphDdl( - """SET SCHEMA ds1.db1 + GraphDdl("""SET SCHEMA ds1.db1 |CREATE GRAPH TYPE myType ( | A (x STRING), B (y STRING), | (A), (A)-[B]->(A) @@ -207,8 +243,7 @@ class GraphDdlTest extends AnyFunSpec with Matchers { |) """.stripMargin), // shadowing - GraphDdl( - """SET SCHEMA ds1.db1 + GraphDdl("""SET SCHEMA ds1.db1 |CREATE GRAPH TYPE myType ( | A (x STRING), B (foo STRING), | (A), (A)-[B]->(A) @@ -222,8 +257,7 @@ class GraphDdlTest extends AnyFunSpec with Matchers { |) """.stripMargin), // only label types in graph type - GraphDdl( - """SET SCHEMA ds1.db1 + GraphDdl("""SET SCHEMA ds1.db1 |CREATE GRAPH TYPE myType ( | A (x STRING), B (foo STRING) |) @@ -248,8 +282,7 @@ class GraphDdlTest extends AnyFunSpec with Matchers { it("allows these equivalent graph type definitions") { val ddls = List( // explicit node and rel type definitions - GraphDdl( - """CREATE GRAPH TYPE myType ( + GraphDdl("""CREATE GRAPH TYPE myType ( | A (x STRING), B (y STRING), C (z STRING), | (A), (C), | (A)-[B]->(C) @@ -257,8 +290,7 @@ class GraphDdlTest extends AnyFunSpec with Matchers { |CREATE GRAPH myGraph OF myType () """.stripMargin), // shadowing - GraphDdl( - """CREATE ELEMENT TYPE A (foo STRING) + GraphDdl("""CREATE ELEMENT TYPE A (foo STRING) |CREATE GRAPH TYPE myType ( | A (x STRING), B (y STRING), C (z STRING), | (A), (C), @@ -315,7 +347,9 @@ class GraphDdlTest extends AnyFunSpec with Matchers { GraphDdl(ddl).graphs(graphName).graphType shouldEqual expected expected.nodePropertyKeys("A") shouldEqual Map("name" -> CTString) - expected.relationshipPropertyKeys("A", "A", "A") shouldEqual Map("name" -> CTString) + expected.relationshipPropertyKeys("A", "A", "A") shouldEqual Map( + "name" -> CTString + ) } it("can construct schema with node and edge labels") { @@ -342,7 +376,11 @@ class GraphDdlTest extends AnyFunSpec with Matchers { GraphDdl(ddl).graphs(graphName).graphType shouldEqual expected expected.nodePropertyKeys("Node1") shouldEqual Map("val" -> CTString) expected.nodePropertyKeys("Node2") shouldEqual Map("val" -> CTString) - expected.relationshipPropertyKeys("Node1", "REL", "Node2") shouldEqual Map("name" -> CTString) + expected.relationshipPropertyKeys( + "Node1", + "REL", + "Node2" + ) shouldEqual Map("name" -> CTString) } it("can construct schema with inherited node and edge labels") { @@ -380,11 +418,34 @@ class GraphDdlTest extends AnyFunSpec with Matchers { .withRelationshipType("Node2", "REL3", "Node2") GraphDdl(ddl).graphs(graphName).graphType shouldEqual expected expected.nodePropertyKeys("Node1") shouldEqual Map("foo" -> CTString) - expected.nodePropertyKeys("Node1", "Node2") shouldEqual Map("foo" -> CTString, "bar" -> CTInteger) - expected.nodePropertyKeys("Node1", "Node2", "Node3") shouldEqual Map("foo" -> CTString, "bar" -> CTInteger, "baz" -> CTBoolean) - expected.relationshipPropertyKeys(Set("Node1"), Set("REL1"), Set("Node1")) shouldEqual Map("name" -> CTString) - expected.relationshipPropertyKeys(Set("Node1"), Set("REL1", "REL2"), Set("Node1", "Node2")) shouldEqual Map("name" -> CTString, "since" -> CTInteger) - expected.relationshipPropertyKeys(Set("Node1", "Node2"), Set("REL1", "REL2", "REL3"), Set("Node1", "Node2")) shouldEqual Map("name" -> CTString, "since" -> CTInteger, "age" -> CTBoolean) + expected.nodePropertyKeys("Node1", "Node2") shouldEqual Map( + "foo" -> CTString, + "bar" -> CTInteger + ) + expected.nodePropertyKeys("Node1", "Node2", "Node3") shouldEqual Map( + "foo" -> CTString, + "bar" -> CTInteger, + "baz" -> CTBoolean + ) + expected.relationshipPropertyKeys( + Set("Node1"), + Set("REL1"), + Set("Node1") + ) shouldEqual Map("name" -> CTString) + expected.relationshipPropertyKeys( + Set("Node1"), + Set("REL1", "REL2"), + Set("Node1", "Node2") + ) shouldEqual Map("name" -> CTString, "since" -> CTInteger) + expected.relationshipPropertyKeys( + Set("Node1", "Node2"), + Set("REL1", "REL2", "REL3"), + Set("Node1", "Node2") + ) shouldEqual Map( + "name" -> CTString, + "since" -> CTInteger, + "age" -> CTBoolean + ) } it("prefers local label over global label") { @@ -418,9 +479,9 @@ class GraphDdlTest extends AnyFunSpec with Matchers { |""".stripMargin val expected = GraphType(typeName) - .withElementType(ElementType( - name = "Node", - maybeKey = Some("akey" -> Set("val")))) + .withElementType( + ElementType(name = "Node", maybeKey = Some("akey" -> Set("val"))) + ) .withNodeType("Node") GraphDdl(ddl).graphs(graphName).graphType shouldEqual expected } @@ -450,7 +511,11 @@ class GraphDdlTest extends AnyFunSpec with Matchers { |""".stripMargin val expected = GraphType(typeName) - .withElementType("MyLabel", "property" -> CTString, "data" -> CTInteger.nullable) + .withElementType( + "MyLabel", + "property" -> CTString, + "data" -> CTInteger.nullable + ) .withElementType("LocalLabel1", "property" -> CTString) .withElementType("LocalLabel2") .withElementType("REL_TYPE1", "property" -> CTBoolean) @@ -458,14 +523,33 @@ class GraphDdlTest extends AnyFunSpec with Matchers { .withNodeType("MyLabel") .withNodeType("LocalLabel1") .withNodeType("LocalLabel1", "LocalLabel2") - .withRelationshipType(Set("MyLabel"), Set("REL_TYPE1"), Set("LocalLabel1")) - .withRelationshipType(Set("LocalLabel1", "LocalLabel2"), Set("REL_TYPE2"), Set("MyLabel")) + .withRelationshipType( + Set("MyLabel"), + Set("REL_TYPE1"), + Set("LocalLabel1") + ) + .withRelationshipType( + Set("LocalLabel1", "LocalLabel2"), + Set("REL_TYPE2"), + Set("MyLabel") + ) GraphDdl(ddl).graphs(graphName).graphType shouldEqual expected - expected.nodePropertyKeys("MyLabel") shouldEqual Map("property" -> CTString, "data" -> CTInteger.nullable) - expected.nodePropertyKeys("LocalLabel1") shouldEqual Map("property" -> CTString) + expected.nodePropertyKeys("MyLabel") shouldEqual Map( + "property" -> CTString, + "data" -> CTInteger.nullable + ) + expected.nodePropertyKeys("LocalLabel1") shouldEqual Map( + "property" -> CTString + ) expected.nodePropertyKeys("LocalLabel2") shouldEqual Map.empty - expected.nodePropertyKeys("LocalLabel1", "LocalLabel2") shouldEqual Map("property" -> CTString) - expected.relationshipPropertyKeys(Set("MyLabel"), Set("REL_TYPE1"), Set("LocalLabel1")) shouldEqual Map("property" -> CTBoolean) + expected.nodePropertyKeys("LocalLabel1", "LocalLabel2") shouldEqual Map( + "property" -> CTString + ) + expected.relationshipPropertyKeys( + Set("MyLabel"), + Set("REL_TYPE1"), + Set("LocalLabel1") + ) shouldEqual Map("property" -> CTBoolean) } it("merges property keys for label combination") { @@ -488,10 +572,15 @@ class GraphDdlTest extends AnyFunSpec with Matchers { .withNodeType("A", "B") GraphDdl(ddl).graphs(graphName).graphType shouldEqual expected expected.nodePropertyKeys("A") shouldEqual Map("foo" -> CTString) - expected.nodePropertyKeys("A", "B") shouldEqual Map("foo" -> CTString, "bar" -> CTString) + expected.nodePropertyKeys("A", "B") shouldEqual Map( + "foo" -> CTString, + "bar" -> CTString + ) } - it("merges property keys for label combination based on element type hierarchy") { + it( + "merges property keys for label combination based on element type hierarchy" + ) { // Given val ddl = s"""|CREATE ELEMENT TYPE A ( foo STRING ) @@ -506,15 +595,26 @@ class GraphDdlTest extends AnyFunSpec with Matchers { val expected = GraphType(typeName) .withElementType("A", "foo" -> CTString) - .withElementType(ElementType(name = "B", parents = Set("A"), properties = Map("bar" -> CTString))) + .withElementType( + ElementType( + name = "B", + parents = Set("A"), + properties = Map("bar" -> CTString) + ) + ) .withNodeType("A") .withNodeType("A", "B") GraphDdl(ddl).graphs(graphName).graphType shouldEqual expected expected.nodePropertyKeys("A") shouldEqual Map("foo" -> CTString) - expected.nodePropertyKeys("A", "B") shouldEqual Map("foo" -> CTString, "bar" -> CTString) + expected.nodePropertyKeys("A", "B") shouldEqual Map( + "foo" -> CTString, + "bar" -> CTString + ) } - it("merges property keys for label combination based on element type with multi-inheritance") { + it( + "merges property keys for label combination based on element type with multi-inheritance" + ) { // Given val ddl = s"""|CREATE ELEMENT TYPE A ( a STRING ) @@ -536,9 +636,27 @@ class GraphDdlTest extends AnyFunSpec with Matchers { val expected = GraphType(typeName) .withElementType("A", "a" -> CTString) - .withElementType(ElementType(name = "B", parents = Set("A"), properties = Map("b" -> CTString))) - .withElementType(ElementType(name = "C", parents = Set("A"), properties = Map("c" -> CTString))) - .withElementType(ElementType(name = "D", parents = Set("B", "C"), properties = Map("d" -> CTInteger))) + .withElementType( + ElementType( + name = "B", + parents = Set("A"), + properties = Map("b" -> CTString) + ) + ) + .withElementType( + ElementType( + name = "C", + parents = Set("A"), + properties = Map("c" -> CTString) + ) + ) + .withElementType( + ElementType( + name = "D", + parents = Set("B", "C"), + properties = Map("d" -> CTInteger) + ) + ) .withElementType("E", "e" -> CTFloat) .withNodeType("A") .withNodeType("A", "B") @@ -548,11 +666,31 @@ class GraphDdlTest extends AnyFunSpec with Matchers { .withNodeType("A", "B", "C", "D", "E") GraphDdl(ddl).graphs(graphName).graphType shouldEqual expected expected.nodePropertyKeys("A") shouldEqual Map("a" -> CTString) - expected.nodePropertyKeys("A", "B") shouldEqual Map("a" -> CTString, "b" -> CTString) - expected.nodePropertyKeys("A", "C") shouldEqual Map("a" -> CTString, "c" -> CTString) - expected.nodePropertyKeys("A", "B", "C", "D") shouldEqual Map("a" -> CTString, "b" -> CTString, "c" -> CTString, "d" -> CTInteger) - expected.nodePropertyKeys("A", "E") shouldEqual Map("a" -> CTString, "e" -> CTFloat) - expected.nodePropertyKeys("A", "B", "C", "D", "E") shouldEqual Map("a" -> CTString, "b" -> CTString, "c" -> CTString, "d" -> CTInteger, "e" -> CTFloat) + expected.nodePropertyKeys("A", "B") shouldEqual Map( + "a" -> CTString, + "b" -> CTString + ) + expected.nodePropertyKeys("A", "C") shouldEqual Map( + "a" -> CTString, + "c" -> CTString + ) + expected.nodePropertyKeys("A", "B", "C", "D") shouldEqual Map( + "a" -> CTString, + "b" -> CTString, + "c" -> CTString, + "d" -> CTInteger + ) + expected.nodePropertyKeys("A", "E") shouldEqual Map( + "a" -> CTString, + "e" -> CTFloat + ) + expected.nodePropertyKeys("A", "B", "C", "D", "E") shouldEqual Map( + "a" -> CTString, + "b" -> CTString, + "c" -> CTString, + "d" -> CTInteger, + "e" -> CTFloat + ) } it("merges identical property keys with same type") { @@ -579,8 +717,7 @@ class GraphDdlTest extends AnyFunSpec with Matchers { } it("parses correct schema") { - val ddlDefinition: DdlDefinition = parseDdl( - s"""|SET SCHEMA foo.bar; + val ddlDefinition: DdlDefinition = parseDdl(s"""|SET SCHEMA foo.bar; | |CREATE ELEMENT TYPE A ( name STRING ) | @@ -615,47 +752,92 @@ class GraphDdlTest extends AnyFunSpec with Matchers { |) |""".stripMargin) ddlDefinition should equalWithTracing( - DdlDefinition(List( - SetSchemaDefinition("foo", "bar"), - ElementTypeDefinition("A", properties = Map("name" -> CTString)), - ElementTypeDefinition("B", properties = Map("sequence" -> CTInteger, "nationality" -> CTString.nullable, "age" -> CTInteger.nullable)), - ElementTypeDefinition("TYPE_1"), - ElementTypeDefinition("TYPE_2", properties = Map("prop" -> CTBoolean.nullable)), - GraphTypeDefinition( - name = typeName, - statements = List( - ElementTypeDefinition("A", properties = Map("foo" -> CTInteger)), - ElementTypeDefinition("C"), - NodeTypeDefinition("A"), - NodeTypeDefinition("B"), - NodeTypeDefinition("A", "B"), - NodeTypeDefinition("C"), - RelationshipTypeDefinition("A", "TYPE_1", "B"), - RelationshipTypeDefinition("A", "B")("TYPE_2")("C") - )), - GraphDefinition( - name = graphName.value, - maybeGraphTypeName = Some(typeName), - statements = List( - NodeMappingDefinition(NodeTypeDefinition("A"), List(NodeToViewDefinition(List("foo")))), - RelationshipMappingDefinition( + DdlDefinition( + List( + SetSchemaDefinition("foo", "bar"), + ElementTypeDefinition("A", properties = Map("name" -> CTString)), + ElementTypeDefinition( + "B", + properties = Map( + "sequence" -> CTInteger, + "nationality" -> CTString.nullable, + "age" -> CTInteger.nullable + ) + ), + ElementTypeDefinition("TYPE_1"), + ElementTypeDefinition( + "TYPE_2", + properties = Map("prop" -> CTBoolean.nullable) + ), + GraphTypeDefinition( + name = typeName, + statements = List( + ElementTypeDefinition( + "A", + properties = Map("foo" -> CTInteger) + ), + ElementTypeDefinition("C"), + NodeTypeDefinition("A"), + NodeTypeDefinition("B"), + NodeTypeDefinition("A", "B"), + NodeTypeDefinition("C"), RelationshipTypeDefinition("A", "TYPE_1", "B"), - List(RelationshipTypeToViewDefinition( - viewDef = ViewDefinition(List("baz"), "edge"), - startNodeTypeToView = NodeTypeToViewDefinition( - NodeTypeDefinition("A"), - ViewDefinition(List("foo"), "alias_foo"), - JoinOnDefinition(List((List("alias_foo", "COLUMN_A"), List("edge", "COLUMN_A"))))), - endNodeTypeToView = NodeTypeToViewDefinition( - NodeTypeDefinition("B"), - ViewDefinition(List("bar"), "alias_bar"), - JoinOnDefinition(List((List("alias_bar", "COLUMN_A"), List("edge", "COLUMN_A"))))) - ))))) - )) + RelationshipTypeDefinition("A", "B")("TYPE_2")("C") + ) + ), + GraphDefinition( + name = graphName.value, + maybeGraphTypeName = Some(typeName), + statements = List( + NodeMappingDefinition( + NodeTypeDefinition("A"), + List(NodeToViewDefinition(List("foo"))) + ), + RelationshipMappingDefinition( + RelationshipTypeDefinition("A", "TYPE_1", "B"), + List( + RelationshipTypeToViewDefinition( + viewDef = ViewDefinition(List("baz"), "edge"), + startNodeTypeToView = NodeTypeToViewDefinition( + NodeTypeDefinition("A"), + ViewDefinition(List("foo"), "alias_foo"), + JoinOnDefinition( + List( + ( + List("alias_foo", "COLUMN_A"), + List("edge", "COLUMN_A") + ) + ) + ) + ), + endNodeTypeToView = NodeTypeToViewDefinition( + NodeTypeDefinition("B"), + ViewDefinition(List("bar"), "alias_bar"), + JoinOnDefinition( + List( + ( + List("alias_bar", "COLUMN_A"), + List("edge", "COLUMN_A") + ) + ) + ) + ) + ) + ) + ) + ) + ) + ) + ) ) val expected = GraphType(typeName) .withElementType("A", "foo" -> CTInteger) - .withElementType("B", "sequence" -> CTInteger, "nationality" -> CTString.nullable, "age" -> CTInteger.nullable) + .withElementType( + "B", + "sequence" -> CTInteger, + "nationality" -> CTString.nullable, + "age" -> CTInteger.nullable + ) .withElementType("C") .withElementType("TYPE_1") .withElementType("TYPE_2", "prop" -> CTBoolean.nullable) @@ -667,11 +849,28 @@ class GraphDdlTest extends AnyFunSpec with Matchers { .withRelationshipType(Set("A", "B"), Set("TYPE_2"), Set("C")) GraphDdl(ddlDefinition).graphs(graphName).graphType shouldEqual expected expected.nodePropertyKeys("A") shouldEqual Map("foo" -> CTInteger) - expected.nodePropertyKeys("B") shouldEqual Map("sequence" -> CTInteger, "nationality" -> CTString.nullable, "age" -> CTInteger.nullable) - expected.nodePropertyKeys("A", "B") shouldEqual Map("foo" -> CTInteger, "sequence" -> CTInteger, "nationality" -> CTString.nullable, "age" -> CTInteger.nullable) + expected.nodePropertyKeys("B") shouldEqual Map( + "sequence" -> CTInteger, + "nationality" -> CTString.nullable, + "age" -> CTInteger.nullable + ) + expected.nodePropertyKeys("A", "B") shouldEqual Map( + "foo" -> CTInteger, + "sequence" -> CTInteger, + "nationality" -> CTString.nullable, + "age" -> CTInteger.nullable + ) expected.nodePropertyKeys("C") shouldEqual Map.empty - expected.relationshipPropertyKeys("A", "TYPE_1", "B") shouldEqual Map.empty - expected.relationshipPropertyKeys(Set("A", "B"), Set("TYPE_2"), Set("C")) shouldEqual Map("prop" -> CTBoolean.nullable) + expected.relationshipPropertyKeys( + "A", + "TYPE_1", + "B" + ) shouldEqual Map.empty + expected.relationshipPropertyKeys( + Set("A", "B"), + Set("TYPE_2"), + Set("C") + ) shouldEqual Map("prop" -> CTBoolean.nullable) } it("creates implicit node/edge types from mappings") { @@ -708,7 +907,11 @@ class GraphDdlTest extends AnyFunSpec with Matchers { expected.nodePropertyKeys("A") shouldEqual Map("foo" -> CTInteger) expected.nodePropertyKeys("B") shouldEqual Map.empty expected.nodePropertyKeys("A", "B") shouldEqual Map("foo" -> CTInteger) - expected.relationshipPropertyKeys("A", "TYPE_1", "B") shouldEqual Map.empty + expected.relationshipPropertyKeys( + "A", + "TYPE_1", + "B" + ) shouldEqual Map.empty } it("resolves element types from parent graph type") { @@ -747,7 +950,11 @@ class GraphDdlTest extends AnyFunSpec with Matchers { expected.nodePropertyKeys("A") shouldEqual Map("foo" -> CTInteger) expected.nodePropertyKeys("B") shouldEqual Map.empty expected.nodePropertyKeys("A", "B") shouldEqual Map("foo" -> CTInteger) - expected.relationshipPropertyKeys("A", "TYPE_1", "B") shouldEqual Map.empty + expected.relationshipPropertyKeys( + "A", + "TYPE_1", + "B" + ) shouldEqual Map.empty } it("resolves shadowed element types") { @@ -788,7 +995,11 @@ class GraphDdlTest extends AnyFunSpec with Matchers { expected.nodePropertyKeys("A") shouldEqual Map("bar" -> CTString) expected.nodePropertyKeys("B") shouldEqual Map.empty expected.nodePropertyKeys("A", "B") shouldEqual Map("bar" -> CTString) - expected.relationshipPropertyKeys("A", "TYPE_1", "B") shouldEqual Map.empty + expected.relationshipPropertyKeys( + "A", + "TYPE_1", + "B" + ) shouldEqual Map.empty } it("resolves most local element type") { @@ -819,26 +1030,46 @@ class GraphDdlTest extends AnyFunSpec with Matchers { describe("Join key extraction") { it("extracts join keys for a given node view key in start node position") { - val maybeJoinColumns = GraphDdl(ddlString).graphs(GraphName("fooGraph")) - .nodeIdColumnsFor(NodeViewKey( - NodeType("Person"), - ViewId(Some(SetSchemaDefinition("dataSourceName", "fooDatabaseName")), List("personView1")))) + val maybeJoinColumns = GraphDdl(ddlString) + .graphs(GraphName("fooGraph")) + .nodeIdColumnsFor( + NodeViewKey( + NodeType("Person"), + ViewId( + Some(SetSchemaDefinition("dataSourceName", "fooDatabaseName")), + List("personView1") + ) + ) + ) maybeJoinColumns shouldEqual Some(List("person_id1")) } it("extracts join keys for a given node view key in end node position") { - val maybeJoinColumns = GraphDdl(ddlString).graphs(GraphName("fooGraph")) - .nodeIdColumnsFor(NodeViewKey( - NodeType("Book"), - ViewId(Some(SetSchemaDefinition("dataSourceName", "fooDatabaseName")), List("bookView")))) + val maybeJoinColumns = GraphDdl(ddlString) + .graphs(GraphName("fooGraph")) + .nodeIdColumnsFor( + NodeViewKey( + NodeType("Book"), + ViewId( + Some(SetSchemaDefinition("dataSourceName", "fooDatabaseName")), + List("bookView") + ) + ) + ) maybeJoinColumns shouldEqual Some(List("book_id")) } it("does not extract join keys for an invalid node view key") { - val maybeJoinColumns = GraphDdl(ddlString).graphs(GraphName("fooGraph")) - .nodeIdColumnsFor(NodeViewKey(NodeType("A"), ViewId(None, List("dataSourceName", "fooDatabaseName", "A")))) + val maybeJoinColumns = GraphDdl(ddlString) + .graphs(GraphName("fooGraph")) + .nodeIdColumnsFor( + NodeViewKey( + NodeType("A"), + ViewId(None, List("dataSourceName", "fooDatabaseName", "A")) + ) + ) maybeJoinColumns shouldEqual None } @@ -847,8 +1078,7 @@ class GraphDdlTest extends AnyFunSpec with Matchers { describe("SET SCHEMA") { it("allows SET SCHEMA and fully qualified names") { - val ddl = GraphDdl( - """SET SCHEMA ds1.db1 + val ddl = GraphDdl("""SET SCHEMA ds1.db1 | |CREATE GRAPH TYPE fooSchema ( | Person, @@ -863,20 +1093,34 @@ class GraphDdlTest extends AnyFunSpec with Matchers { """.stripMargin) ddl.graphs(GraphName("fooGraph")).nodeToViewMappings.keys shouldEqual Set( - NodeViewKey(NodeType("Person"), ViewId(Some(SetSchemaDefinition("ds1", "db1")), List("personView"))), - NodeViewKey(NodeType("Account"), ViewId(Some(SetSchemaDefinition("ds1", "db1")), List("ds2", "db2", "accountView"))) + NodeViewKey( + NodeType("Person"), + ViewId(Some(SetSchemaDefinition("ds1", "db1")), List("personView")) + ), + NodeViewKey( + NodeType("Account"), + ViewId( + Some(SetSchemaDefinition("ds1", "db1")), + List("ds2", "db2", "accountView") + ) + ) ) } } describe("validate EXTENDS syntax for mappings") { - val A_a = NodeViewKey(NodeType("A"), ViewId(Some(SetSchemaDefinition("ds1", "db1")), List("a"))) - val A_ab = NodeViewKey(NodeType("A", "B"), ViewId(Some(SetSchemaDefinition("ds1", "db1")), List("a_b"))) + val A_a = NodeViewKey( + NodeType("A"), + ViewId(Some(SetSchemaDefinition("ds1", "db1")), List("a")) + ) + val A_ab = NodeViewKey( + NodeType("A", "B"), + ViewId(Some(SetSchemaDefinition("ds1", "db1")), List("a_b")) + ) it("allows compact inline graph definition with complex node type") { - val ddl = GraphDdl( - """SET SCHEMA ds1.db1 + val ddl = GraphDdl("""SET SCHEMA ds1.db1 |CREATE GRAPH myGraph ( | A (x STRING), | B (y STRING), @@ -907,18 +1151,32 @@ class GraphDdlTest extends AnyFunSpec with Matchers { .withNodeType(NodeType("A")) .withNodeType(NodeType("A", "B")) .withRelationshipType(RelationshipType("A", "R", "A")) - .withRelationshipType(RelationshipType(NodeType("A", "B"), Set("R"), NodeType("A"))), + .withRelationshipType( + RelationshipType(NodeType("A", "B"), Set("R"), NodeType("A")) + ), nodeToViewMappings = Map( - A_a -> NodeToViewMapping(NodeType("A"), ViewId(Some(SetSchemaDefinition("ds1", "db1")), List("a")), Map("x" -> "x")), - A_ab -> NodeToViewMapping(NodeType("A", "B"), ViewId(Some(SetSchemaDefinition("ds1", "db1")), List("a_b")), Map("x" -> "x", "y" -> "y")) + A_a -> NodeToViewMapping( + NodeType("A"), + ViewId(Some(SetSchemaDefinition("ds1", "db1")), List("a")), + Map("x" -> "x") + ), + A_ab -> NodeToViewMapping( + NodeType("A", "B"), + ViewId(Some(SetSchemaDefinition("ds1", "db1")), List("a_b")), + Map("x" -> "x", "y" -> "y") + ) ), edgeToViewMappings = List( - EdgeToViewMapping(RelationshipType("A", "R", "A"), ViewId(Some(SetSchemaDefinition("ds1", "db1")), List("r")), + EdgeToViewMapping( + RelationshipType("A", "R", "A"), + ViewId(Some(SetSchemaDefinition("ds1", "db1")), List("r")), StartNode(A_a, List(Join("id", "id"))), EndNode(A_a, List(Join("id", "id"))), Map("y" -> "y") ), - EdgeToViewMapping(RelationshipType(NodeType("A", "B"), Set("R"), NodeType("A")), ViewId(Some(SetSchemaDefinition("ds1", "db1")), List("r")), + EdgeToViewMapping( + RelationshipType(NodeType("A", "B"), Set("R"), NodeType("A")), + ViewId(Some(SetSchemaDefinition("ds1", "db1")), List("r")), StartNode(A_ab, List(Join("id", "id"))), EndNode(A_a, List(Join("id", "id"))), Map("y" -> "y") @@ -927,9 +1185,10 @@ class GraphDdlTest extends AnyFunSpec with Matchers { ) } - it("allows compact inline graph definition with complex node type based on inheritance") { - val ddl = GraphDdl( - """SET SCHEMA ds1.db1 + it( + "allows compact inline graph definition with complex node type based on inheritance" + ) { + val ddl = GraphDdl("""SET SCHEMA ds1.db1 |CREATE GRAPH myGraph ( | A (x STRING), | B EXTENDS A (y STRING), @@ -959,18 +1218,32 @@ class GraphDdlTest extends AnyFunSpec with Matchers { .withNodeType(NodeType("A")) .withNodeType(NodeType("A", "B")) .withRelationshipType(RelationshipType("A", "R", "A")) - .withRelationshipType(RelationshipType(NodeType("A", "B"), Set("R"), NodeType("A"))), + .withRelationshipType( + RelationshipType(NodeType("A", "B"), Set("R"), NodeType("A")) + ), nodeToViewMappings = Map( - A_a -> NodeToViewMapping(NodeType("A"), ViewId(Some(SetSchemaDefinition("ds1", "db1")), List("a")), Map("x" -> "x")), - A_ab -> NodeToViewMapping(NodeType("A", "B"), ViewId(Some(SetSchemaDefinition("ds1", "db1")), List("a_b")), Map("x" -> "x", "y" -> "y")) + A_a -> NodeToViewMapping( + NodeType("A"), + ViewId(Some(SetSchemaDefinition("ds1", "db1")), List("a")), + Map("x" -> "x") + ), + A_ab -> NodeToViewMapping( + NodeType("A", "B"), + ViewId(Some(SetSchemaDefinition("ds1", "db1")), List("a_b")), + Map("x" -> "x", "y" -> "y") + ) ), edgeToViewMappings = List( - EdgeToViewMapping(RelationshipType("A", "R", "A"), ViewId(Some(SetSchemaDefinition("ds1", "db1")), List("r")), + EdgeToViewMapping( + RelationshipType("A", "R", "A"), + ViewId(Some(SetSchemaDefinition("ds1", "db1")), List("r")), StartNode(A_a, List(Join("id", "id"))), EndNode(A_a, List(Join("id", "id"))), Map("y" -> "y") ), - EdgeToViewMapping(RelationshipType(NodeType("A", "B"), Set("R"), NodeType("A")), ViewId(Some(SetSchemaDefinition("ds1", "db1")), List("r")), + EdgeToViewMapping( + RelationshipType(NodeType("A", "B"), Set("R"), NodeType("A")), + ViewId(Some(SetSchemaDefinition("ds1", "db1")), List("r")), StartNode(A_ab, List(Join("id", "id"))), EndNode(A_a, List(Join("id", "id"))), Map("y" -> "y") @@ -993,7 +1266,9 @@ class GraphDdlTest extends AnyFunSpec with Matchers { |) """.stripMargin ) - e.getFullMessage should (include("fooSchema") and include("node type") and include("(A,B)")) + e.getFullMessage should (include("fooSchema") and include( + "node type" + ) and include("(A,B)")) } it("fails on duplicate anonymous node types") { @@ -1009,7 +1284,9 @@ class GraphDdlTest extends AnyFunSpec with Matchers { |) """.stripMargin ) - e.getFullMessage should (include("fooSchema") and include("node type") and include("(A,B,X)")) + e.getFullMessage should (include("fooSchema") and include( + "node type" + ) and include("(A,B,X)")) } it("fails on duplicate relationship types") { @@ -1023,7 +1300,9 @@ class GraphDdlTest extends AnyFunSpec with Matchers { |) """.stripMargin ) - e.getFullMessage should (include("fooSchema") and include("relationship type") and include("(A)-[B]->(A)")) + e.getFullMessage should (include("fooSchema") and include( + "relationship type" + ) and include("(A)-[B]->(A)")) } it("fails on duplicate relationship types using anonymous node types") { @@ -1039,12 +1318,13 @@ class GraphDdlTest extends AnyFunSpec with Matchers { |) """.stripMargin ) - e.getFullMessage should (include("fooSchema") and include("relationship type") and include("(A,B)-[FOO]->(X)")) + e.getFullMessage should (include("fooSchema") and include( + "relationship type" + ) and include("(A,B)-[FOO]->(X)")) } it("fails on duplicate node mappings") { - val e = the[GraphDdlException] thrownBy GraphDdl( - """ + val e = the[GraphDdlException] thrownBy GraphDdl(""" |SET SCHEMA db.schema | |CREATE GRAPH TYPE fooSchema ( @@ -1056,12 +1336,13 @@ class GraphDdlTest extends AnyFunSpec with Matchers { | FROM personView |) """.stripMargin) - e.getFullMessage should (include("fooGraph") and include("(Person)") and include("personView")) + e.getFullMessage should (include("fooGraph") and include( + "(Person)" + ) and include("personView")) } it("fails on duplicate relationship mappings") { - val e = the[GraphDdlException] thrownBy GraphDdl( - """ + val e = the[GraphDdlException] thrownBy GraphDdl(""" |SET SCHEMA db.schema | |CREATE GRAPH TYPE fooSchema ( @@ -1080,12 +1361,13 @@ class GraphDdlTest extends AnyFunSpec with Matchers { | END NODES (Person) FROM a n JOIN ON e.id = n.id |) """.stripMargin) - e.getFullMessage should (include("fooGraph") and include("(Person)-[KNOWS]->(Person)") and include("pkpView")) + e.getFullMessage should (include("fooGraph") and include( + "(Person)-[KNOWS]->(Person)" + ) and include("pkpView")) } it("fails on duplicate global labels") { - val e = the[GraphDdlException] thrownBy GraphDdl( - """ + val e = the[GraphDdlException] thrownBy GraphDdl(""" |CREATE ELEMENT TYPE Person |CREATE ELEMENT TYPE Person """.stripMargin) @@ -1093,8 +1375,7 @@ class GraphDdlTest extends AnyFunSpec with Matchers { } it("fails on duplicate local labels") { - val e = the[GraphDdlException] thrownBy GraphDdl( - """ + val e = the[GraphDdlException] thrownBy GraphDdl(""" |CREATE GRAPH TYPE fooSchema ( | Person, | Person @@ -1104,8 +1385,7 @@ class GraphDdlTest extends AnyFunSpec with Matchers { } it("fails on duplicate graph types") { - val e = the[GraphDdlException] thrownBy GraphDdl( - """ + val e = the[GraphDdlException] thrownBy GraphDdl(""" |CREATE GRAPH TYPE fooSchema () |CREATE GRAPH TYPE fooSchema () """.stripMargin) @@ -1113,8 +1393,7 @@ class GraphDdlTest extends AnyFunSpec with Matchers { } it("fails on duplicate graphs") { - val e = the[GraphDdlException] thrownBy GraphDdl( - """ + val e = the[GraphDdlException] thrownBy GraphDdl(""" |CREATE GRAPH TYPE fooSchema () |CREATE GRAPH fooGraph OF fooSchema () |CREATE GRAPH fooGraph OF fooSchema () @@ -1123,53 +1402,58 @@ class GraphDdlTest extends AnyFunSpec with Matchers { } it("fails on unresolved graph type") { - val e = the[GraphDdlException] thrownBy GraphDdl( - """ + val e = the[GraphDdlException] thrownBy GraphDdl(""" |CREATE GRAPH TYPE fooSchema () |CREATE GRAPH fooGraph OF barSchema () """.stripMargin) - e.getFullMessage should (include("fooGraph") and include("fooSchema") and include("barSchema")) + e.getFullMessage should (include("fooGraph") and include( + "fooSchema" + ) and include("barSchema")) } it("fails on unresolved labels") { - val e = the[GraphDdlException] thrownBy GraphDdl( - """ + val e = the[GraphDdlException] thrownBy GraphDdl(""" |CREATE GRAPH TYPE fooSchema ( | Person1, | Person2, | (Person3, Person4) |) """.stripMargin) - e.getFullMessage should (include("fooSchema") and include("Person3") and include("Person4")) + e.getFullMessage should (include("fooSchema") and include( + "Person3" + ) and include("Person4")) } it("fails on unresolved labels in mapping") { - val e = the[GraphDdlException] thrownBy GraphDdl( - """ + val e = the[GraphDdlException] thrownBy GraphDdl(""" |CREATE GRAPH fooGraph ( | Person1, | Person2, | (Person3, Person4) FROM x |) """.stripMargin) - e.getFullMessage should (include("fooGraph") and include("Person3") and include("Person4")) + e.getFullMessage should (include("fooGraph") and include( + "Person3" + ) and include("Person4")) } it("fails on incompatible property types") { - val e = the[GraphDdlException] thrownBy GraphDdl( - """ + val e = the[GraphDdlException] thrownBy GraphDdl(""" |CREATE GRAPH TYPE fooSchema ( | Person1 ( age STRING ) , | Person2 ( age INTEGER ) , | (Person1, Person2) |) """.stripMargin) - e.getFullMessage should (include("fooSchema") and include("Person1") and include("Person2") and include("age") and include("STRING") and include("INTEGER")) + e.getFullMessage should (include("fooSchema") and include( + "Person1" + ) and include("Person2") and include("age") and include( + "STRING" + ) and include("INTEGER")) } it("fails on unresolved property names") { - val e = the[GraphDdlException] thrownBy GraphDdl( - """ + val e = the[GraphDdlException] thrownBy GraphDdl(""" |SET SCHEMA a.b |CREATE GRAPH TYPE fooSchema ( | Person ( age1 STRING ) , @@ -1179,12 +1463,13 @@ class GraphDdlTest extends AnyFunSpec with Matchers { | (Person) FROM personView ( person_name AS age2 ) |) """.stripMargin) - e.getFullMessage should (include("fooGraph") and include("Person") and include("personView") and include("age1") and include("age2")) + e.getFullMessage should (include("fooGraph") and include( + "Person" + ) and include("personView") and include("age1") and include("age2")) } it("fails on unresolved inherited element types") { - val e = the[GraphDdlException] thrownBy GraphDdl( - """ + val e = the[GraphDdlException] thrownBy GraphDdl(""" |SET SCHEMA a.b |CREATE GRAPH TYPE fooSchema ( | Person ( name STRING ) , @@ -1195,12 +1480,15 @@ class GraphDdlTest extends AnyFunSpec with Matchers { | (Employee) FROM employeeView ( person_name AS name, emp_dept AS dept ) |) """.stripMargin) - e.getFullMessage should (include("fooSchema") and include("Employee") and include("MissingPerson")) + e.getFullMessage should (include("fooSchema") and include( + "Employee" + ) and include("MissingPerson")) } - it("fails on unresolved inherited element types within inlined graph type") { - val e = the[GraphDdlException] thrownBy GraphDdl( - """ + it( + "fails on unresolved inherited element types within inlined graph type" + ) { + val e = the[GraphDdlException] thrownBy GraphDdl(""" |SET SCHEMA a.b |CREATE GRAPH fooGraph ( | Person ( name STRING ), @@ -1209,12 +1497,13 @@ class GraphDdlTest extends AnyFunSpec with Matchers { | (Employee) FROM employeeView ( person_name AS name, emp_dept AS dept ) |) """.stripMargin) - e.getFullMessage should (include("fooGraph") and include("Employee") and include("MissingPerson")) + e.getFullMessage should (include("fooGraph") and include( + "Employee" + ) and include("MissingPerson")) } it("fails on cyclic element type inheritance") { - val e = the[GraphDdlException] thrownBy GraphDdl( - """ + val e = the[GraphDdlException] thrownBy GraphDdl(""" |SET SCHEMA a.b |CREATE GRAPH fooGraph ( | A EXTENDS B ( a STRING ), @@ -1224,12 +1513,13 @@ class GraphDdlTest extends AnyFunSpec with Matchers { | (B) FROM b ( A_a AS a, B_b AS b ) |) """.stripMargin) - e.getFullMessage should (include("Circular dependency") and include("A -> B -> A")) + e.getFullMessage should (include("Circular dependency") and include( + "A -> B -> A" + )) } it("fails on conflicting property types in inheritance hierarchy") { - val e = the[GraphDdlException] thrownBy GraphDdl( - """ + val e = the[GraphDdlException] thrownBy GraphDdl(""" |SET SCHEMA a.b |CREATE GRAPH fooGraph ( | A ( x STRING ), @@ -1239,7 +1529,9 @@ class GraphDdlTest extends AnyFunSpec with Matchers { | (C) |) """.stripMargin) - e.getFullMessage should (include("(A,B,C)") and include("x") and include("INTEGER") and include("STRING")) + e.getFullMessage should (include("(A,B,C)") and include("x") and include( + "INTEGER" + ) and include("STRING")) } it("fails if an unknown property key is mapped to a column") { @@ -1284,7 +1576,9 @@ class GraphDdlTest extends AnyFunSpec with Matchers { GraphDdl(parseDdl(ddlString)).graphs(GraphName("myGraph")) } - e.getFullMessage should (include("Inconsistent join column definition") and include("(A)") and include("view_A")) + e.getFullMessage should (include( + "Inconsistent join column definition" + ) and include("(A)") and include("view_A")) } } } diff --git a/morpheus-examples/src/main/scala/org/opencypher/morpheus/examples/CaseClassExample.scala b/morpheus-examples/src/main/scala/org/opencypher/morpheus/examples/CaseClassExample.scala index 0feba319f5..f059e78b88 100644 --- a/morpheus-examples/src/main/scala/org/opencypher/morpheus/examples/CaseClassExample.scala +++ b/morpheus-examples/src/main/scala/org/opencypher/morpheus/examples/CaseClassExample.scala @@ -32,8 +32,8 @@ import org.opencypher.morpheus.api.io.{Node, Relationship, RelationshipType} import org.opencypher.morpheus.util.App /** - * Demonstrates basic usage of the Morpheus API by loading an example network via Scala case classes and running a Cypher - * query on it. + * Demonstrates basic usage of the Morpheus API by loading an example network via Scala case + * classes and running a Cypher query on it. */ object CaseClassExample extends App { @@ -41,7 +41,8 @@ object CaseClassExample extends App { implicit val morpheus: MorpheusSession = MorpheusSession.local() // 2) Load social network data via case class instances - val socialNetwork = morpheus.readFrom(SocialNetworkData.persons, SocialNetworkData.friendships) + val socialNetwork = + morpheus.readFrom(SocialNetworkData.persons, SocialNetworkData.friendships) // 3) Query graph with Cypher val results = socialNetwork.cypher( @@ -54,9 +55,7 @@ object CaseClassExample extends App { results.show } -/** - * Specify schema and data with case classes. - */ +/** Specify schema and data with case classes. */ object SocialNetworkData { case class Person(id: Long, name: String, age: Int) extends Node @@ -69,6 +68,9 @@ object SocialNetworkData { val carol = Person(2, "Carol", 15) val persons = List(alice, bob, carol) - val friendships = List(Friend(0, alice.id, bob.id, "23/01/1987"), Friend(1, bob.id, carol.id, "12/12/2009")) + val friendships = List( + Friend(0, alice.id, bob.id, "23/01/1987"), + Friend(1, bob.id, carol.id, "12/12/2009") + ) } // end::full-example[] diff --git a/morpheus-examples/src/main/scala/org/opencypher/morpheus/examples/CatalogExample.scala b/morpheus-examples/src/main/scala/org/opencypher/morpheus/examples/CatalogExample.scala index a5f26e37b6..bb92b8c087 100644 --- a/morpheus-examples/src/main/scala/org/opencypher/morpheus/examples/CatalogExample.scala +++ b/morpheus-examples/src/main/scala/org/opencypher/morpheus/examples/CatalogExample.scala @@ -60,6 +60,9 @@ object CatalogExample extends App { morpheus.cypher(s"FROM GRAPH $namespace.$graphName MATCH (n) RETURN n").show // Access graph via API - morpheus.catalog.graph(QualifiedGraphName(namespace, graphName)).cypher("MATCH (n) RETURN n").show + morpheus.catalog + .graph(QualifiedGraphName(namespace, graphName)) + .cypher("MATCH (n) RETURN n") + .show } // end::full-example[] diff --git a/morpheus-examples/src/main/scala/org/opencypher/morpheus/examples/CensusHiveExample.scala b/morpheus-examples/src/main/scala/org/opencypher/morpheus/examples/CensusHiveExample.scala index 3e957406a3..f13ae43631 100644 --- a/morpheus-examples/src/main/scala/org/opencypher/morpheus/examples/CensusHiveExample.scala +++ b/morpheus-examples/src/main/scala/org/opencypher/morpheus/examples/CensusHiveExample.scala @@ -43,13 +43,12 @@ object CensusHiveExample extends App { implicit val sparkSession: SparkSession = morpheus.sparkSession // end::create-session[] - // tag::register-sql-source-in-session[] // Register a SQL source (for Hive) in the Cypher session val graphName = "Census_1901" val sqlGraphSource = GraphSources - .sql(resource("ddl/census.ddl").getFile) - .withSqlDataSourceConfigs("CENSUS" -> Hive) + .sql(resource("ddl/census.ddl").getFile) + .withSqlDataSourceConfigs("CENSUS" -> Hive) // tag::prepare-sql-database[] // Create the data in Hive @@ -66,8 +65,8 @@ object CensusHiveExample extends App { // tag::query-graph[] // Run a simple Cypher query - census.cypher( - s""" + census + .cypher(s""" |FROM GRAPH sql.$graphName |MATCH (n:Person)-[r]->(m) |WHERE n.age >= 30 diff --git a/morpheus-examples/src/main/scala/org/opencypher/morpheus/examples/CensusJdbcExample.scala b/morpheus-examples/src/main/scala/org/opencypher/morpheus/examples/CensusJdbcExample.scala index 21013a02f6..26032a1e8a 100644 --- a/morpheus-examples/src/main/scala/org/opencypher/morpheus/examples/CensusJdbcExample.scala +++ b/morpheus-examples/src/main/scala/org/opencypher/morpheus/examples/CensusJdbcExample.scala @@ -60,8 +60,8 @@ object CensusJdbcExample extends App { val census = morpheus.catalog.graph("sql." + graphName) // Run a simple Cypher query - census.cypher( - s""" + census + .cypher(s""" |FROM GRAPH sql.$graphName |MATCH (n:Person)-[r]->(m) |WHERE n.age >= 30 diff --git a/morpheus-examples/src/main/scala/org/opencypher/morpheus/examples/CustomDataFrameInputExample.scala b/morpheus-examples/src/main/scala/org/opencypher/morpheus/examples/CustomDataFrameInputExample.scala index 230f40d9cf..d55f4cc177 100644 --- a/morpheus-examples/src/main/scala/org/opencypher/morpheus/examples/CustomDataFrameInputExample.scala +++ b/morpheus-examples/src/main/scala/org/opencypher/morpheus/examples/CustomDataFrameInputExample.scala @@ -36,8 +36,8 @@ import org.opencypher.morpheus.util.App import org.opencypher.okapi.api.io.conversion.{NodeMappingBuilder, RelationshipMappingBuilder} /** - * Demonstrates basic usage of the Morpheus API by loading an example network from existing [[DataFrame]]s including - * custom element mappings and running a Cypher query on it. + * Demonstrates basic usage of the Morpheus API by loading an example network from existing + * [[DataFrame]]s including custom element mappings and running a Cypher query on it. */ object CustomDataFrameInputExample extends App { @@ -55,17 +55,25 @@ object CustomDataFrameInputExample extends App { // 2) Generate some DataFrames that we'd like to interpret as a property graph. // tag::prepare-dataframes[] - val nodeData: DataFrame = spark.createDataFrame(Seq( - ("Alice", 42L), - ("Bob", 23L), - ("Eve", 84L) - )).toDF("FIRST_NAME", "AGE") + val nodeData: DataFrame = spark + .createDataFrame( + Seq( + ("Alice", 42L), + ("Bob", 23L), + ("Eve", 84L) + ) + ) + .toDF("FIRST_NAME", "AGE") val nodesDF = nodeData.withColumn("ID", nodeData.col("FIRST_NAME")) - val relsDF: DataFrame = spark.createDataFrame(Seq( - (0L, "Alice", "Bob", Date.valueOf("1987-01-23")), - (1L, "Bob", "Eve", Date.valueOf("2009-12-12")) - )).toDF("REL_ID", "SOURCE_ID", "TARGET_ID", "CONNECTED_SINCE") + val relsDF: DataFrame = spark + .createDataFrame( + Seq( + (0L, "Alice", "Bob", Date.valueOf("1987-01-23")), + (1L, "Bob", "Eve", Date.valueOf("2009-12-12")) + ) + ) + .toDF("REL_ID", "SOURCE_ID", "TARGET_ID", "CONNECTED_SINCE") // end::prepare-dataframes[] // 3) Generate node- and relationship tables that wrap the DataFrames and describe their contained data. @@ -107,11 +115,13 @@ object CustomDataFrameInputExample extends App { // This operation may be very expensive as it materializes results locally. // 6a) type safe version, discards values with wrong type // tag::collect-results-typesafe[] - val safeNames: Set[String] = result.records.collect.flatMap(_ ("n.name").as[String]).toSet + val safeNames: Set[String] = + result.records.collect.flatMap(_("n.name").as[String]).toSet // end::collect-results-typesafe[] // 6b) unsafe version, throws an exception when value cannot be cast // tag::collect-results-nontypesafe[] - val unsafeNames: Set[String] = result.records.collect.map(_ ("n.name").cast[String]).toSet + val unsafeNames: Set[String] = + result.records.collect.map(_("n.name").cast[String]).toSet // end::collect-results-nontypesafe[] println(safeNames) diff --git a/morpheus-examples/src/main/scala/org/opencypher/morpheus/examples/Customer360Example.scala b/morpheus-examples/src/main/scala/org/opencypher/morpheus/examples/Customer360Example.scala index b277da07fd..e5241a9a2d 100644 --- a/morpheus-examples/src/main/scala/org/opencypher/morpheus/examples/Customer360Example.scala +++ b/morpheus-examples/src/main/scala/org/opencypher/morpheus/examples/Customer360Example.scala @@ -39,20 +39,21 @@ import org.opencypher.okapi.neo4j.io.testing.Neo4jTestUtils._ * Example: Customer360 * * A business has a record of interactions between its customers and its employed customer reps. - * This record is modelled as a stream of events that get logged in a simple CSV file. - * As time progresses, more data is added to the file. + * This record is modelled as a stream of events that get logged in a simple CSV file. As time + * progresses, more data is added to the file. * - * A graph perspective is applied to this dataset, where we can identify customers who interact with - * customer reps with policies and accounts. - * From a business analysis perspective, we are interested in finding out which customers and which - * customer reps are involved in the most problematic interactions, such as cancellations. + * A graph perspective is applied to this dataset, where we can identify customers who interact + * with customer reps with policies and accounts. From a business analysis perspective, we are + * interested in finding out which customers and which customer reps are involved in the most + * problematic interactions, such as cancellations. * - * Using Graph DDL we load a set of Hive views into a graph, which we seed into a Neo4j transactional database. - * As time progresses, we need to update our transactional database with the incoming deltas, which are taken - * at regular time intervals. - * In this demo we showcase one such time interval, and merge a delta graph into the Neo4j database. + * Using Graph DDL we load a set of Hive views into a graph, which we seed into a Neo4j + * transactional database. As time progresses, we need to update our transactional database with + * the incoming deltas, which are taken at regular time intervals. In this demo we showcase one + * such time interval, and merge a delta graph into the Neo4j database. * - * One may observe that the Cypher queries shown here may also be executed directly in Neo4j, with the same results. + * One may observe that the Cypher queries shown here may also be executed directly in Neo4j, with + * the same results. */ object Customer360Example extends App { val boltUrl = namedArg("--bolt-url").getOrElse("bolt://localhost:7687") @@ -80,7 +81,10 @@ object Customer360Example extends App { // Register a Neo4j PGDS in the session's catalog // This enables reading graphs from a Neo4j database into Morpheus - morpheus.registerSource(Namespace("transactional"), GraphSources.cypher.neo4j(neo4j.config)) + morpheus.registerSource( + Namespace("transactional"), + GraphSources.cypher.neo4j(neo4j.config) + ) // Register a file system PGDS in the session's catalog // This allows storing snapshots of graphs persistently for processing in later Morpheus sessions @@ -88,31 +92,34 @@ object Customer360Example extends App { // We also integrate this with Hive so that the dataframes stored on disk are also visible in Hive views // To make this work we ensure that there the Hive database exists morpheus.sql("CREATE DATABASE IF NOT EXISTS snapshots") - morpheus.registerSource(Namespace("snapshots"), + morpheus.registerSource( + Namespace("snapshots"), GraphSources.fs("snapshots-root", Some("snapshots")).parquet ) println("PGDSs registered") - val c360Seed = morpheus.cypher( - """ + val c360Seed = morpheus + .cypher(""" |FROM c360.interactions_seed |RETURN GRAPH - """.stripMargin).graph + """.stripMargin) + .graph /* * Find customers who have reported the most problematic interactions (complaints or cancellations) * List the top 6 customers and their interaction statistics */ - c360Seed.cypher( - """ + c360Seed + .cypher(""" |MATCH (c:Customer)--(i:Interaction)--(:CustomerRep) |WITH c, i.type AS type, count(*) AS cnt |WHERE type IN ['cancel', 'complaint'] |RETURN c, type, cnt |ORDER BY cnt DESC |LIMIT 42 - """.stripMargin).show + """.stripMargin) + .show // Speed up merge operation. Requires Neo4j Enterprise Edition // Neo4jGraphMerge.createIndexes(entireGraphName, neo4j.dataSourceConfig, c360Seed.schema.nodeKeys) @@ -125,8 +132,8 @@ object Customer360Example extends App { /* * We can also execute the same query based on the graph we merged into the Neo4j instance, seeing the same results. */ - morpheus.cypher( - s""" + morpheus + .cypher(s""" |FROM transactional.$entireGraphName |MATCH (c:Customer)--(i:Interaction)--(rep:CustomerRep) |WITH c, i.type AS type, count(*) AS cnt @@ -134,14 +141,15 @@ object Customer360Example extends App { |RETURN c, type, cnt |ORDER BY cnt DESC |LIMIT 42 - """.stripMargin).show + """.stripMargin) + .show /* * Find customer reps who have received the most problematic reports (complaints or cancellations) * List the top 9 customer reps and their interaction statistics */ - morpheus.cypher( - """ + morpheus + .cypher(""" |FROM c360.interactions_seed |MATCH (c:Customer)--(i:Interaction)--(rep:CustomerRep) |WITH rep, i.type AS type, count(*) AS cnt @@ -149,13 +157,14 @@ object Customer360Example extends App { |RETURN rep, type, cnt |ORDER BY cnt DESC |LIMIT 12 - """.stripMargin).show + """.stripMargin) + .show /* * We can also execute the same query based on the graph we merged into the Neo4j instance, seeing the same results. */ - morpheus.cypher( - s""" + morpheus + .cypher(s""" |FROM transactional.$entireGraphName |MATCH (c:Customer)--(i:Interaction)--(rep:CustomerRep) |WITH rep, i.type AS type, count(*) AS cnt @@ -163,7 +172,8 @@ object Customer360Example extends App { |RETURN rep, type, cnt |ORDER BY cnt DESC |LIMIT 12 - """.stripMargin).show + """.stripMargin) + .show // Time moves forward and more interactions happen // We want to synchronize our transactional database with the new data @@ -178,8 +188,8 @@ object Customer360Example extends App { // Find the updated statistics on customer rep interactions // Here we execute the query from Spark by importing the necessary data from Neo4j on the fly - morpheus.cypher( - s""" + morpheus + .cypher(s""" |FROM transactional.$entireGraphName |MATCH (c:Customer)--(i:Interaction)--(rep:CustomerRep) |WITH rep, i.type AS type, count(*) AS cnt @@ -187,7 +197,8 @@ object Customer360Example extends App { |RETURN rep, type, cnt |ORDER BY cnt DESC |LIMIT 19 - """.stripMargin).show + """.stripMargin) + .show // Reset Neo4j test instance and close the session and driver neo4j.close() diff --git a/morpheus-examples/src/main/scala/org/opencypher/morpheus/examples/CypherSQLRoundtripExample.scala b/morpheus-examples/src/main/scala/org/opencypher/morpheus/examples/CypherSQLRoundtripExample.scala index e667ea7b26..b9b05d65f3 100644 --- a/morpheus-examples/src/main/scala/org/opencypher/morpheus/examples/CypherSQLRoundtripExample.scala +++ b/morpheus-examples/src/main/scala/org/opencypher/morpheus/examples/CypherSQLRoundtripExample.scala @@ -33,10 +33,9 @@ import org.opencypher.morpheus.util.App import org.opencypher.okapi.api.graph.Namespace /** - * Demonstrates usage patterns where Cypher and SQL can be interleaved in the - * same processing chain, by using the tabular output of a Cypher query as a - * SQL table, and using the output of a SQL query as an input driving table - * for a Cypher query. + * Demonstrates usage patterns where Cypher and SQL can be interleaved in the same processing + * chain, by using the tabular output of a Cypher query as a SQL table, and using the output of a + * SQL query as an input driving table for a Cypher query. */ object CypherSQLRoundtripExample extends App { // 1) Create Morpheus session @@ -45,10 +44,14 @@ object CypherSQLRoundtripExample extends App { // 2) Register a file based data source at the session // It contains a purchase network graph called 'products' val graphDir = getClass.getResource("/fs-graphsource/csv").getFile - morpheus.registerSource(Namespace("myDataSource"), GraphSources.fs(rootPath = graphDir).csv) + morpheus.registerSource( + Namespace("myDataSource"), + GraphSources.fs(rootPath = graphDir).csv + ) // 3) Load social network data via case class instances - val socialNetwork = morpheus.readFrom(SocialNetworkData.persons, SocialNetworkData.friendships) + val socialNetwork = + morpheus.readFrom(SocialNetworkData.persons, SocialNetworkData.friendships) // 4) Query for a view of the people in the social network val result = socialNetwork.cypher( @@ -58,17 +61,23 @@ object CypherSQLRoundtripExample extends App { ) // 5) Register the result as a table called people - result.records.asMorpheus.df.toDF("age", "name").createOrReplaceTempView("people") + result.records.asMorpheus.df + .toDF("age", "name") + .createOrReplaceTempView("people") // 6) Query the registered table using SQL val sqlResults = morpheus.sql("SELECT age, name FROM people") // 7) Use the results from the SQL query as driving table for a Cypher query on a graph contained in the data source - val result2 = morpheus.catalog.graph("myDataSource.products").cypher( - s""" + val result2 = morpheus.catalog + .graph("myDataSource.products") + .cypher( + s""" |MATCH (c:Customer {name: name})-->(p:Product) |RETURN c.name, age, p.title - """.stripMargin, drivingTable = Some(sqlResults)) + """.stripMargin, + drivingTable = Some(sqlResults) + ) // 8) Print the results result2.show diff --git a/morpheus-examples/src/main/scala/org/opencypher/morpheus/examples/DataFrameInputExample.scala b/morpheus-examples/src/main/scala/org/opencypher/morpheus/examples/DataFrameInputExample.scala index feec7bacda..b2a9209208 100644 --- a/morpheus-examples/src/main/scala/org/opencypher/morpheus/examples/DataFrameInputExample.scala +++ b/morpheus-examples/src/main/scala/org/opencypher/morpheus/examples/DataFrameInputExample.scala @@ -32,9 +32,7 @@ import org.opencypher.morpheus.api.MorpheusSession import org.opencypher.morpheus.api.io.{MorpheusNodeTable, MorpheusRelationshipTable} import org.opencypher.morpheus.util.App -/** - * Demonstrates basic usage of the Morpheus API by loading an example graph from [[DataFrame]]s. - */ +/** Demonstrates basic usage of the Morpheus API by loading an example graph from [[DataFrame]]s. */ object DataFrameInputExample extends App { // 1) Create Morpheus session and retrieve Spark session implicit val morpheus: MorpheusSession = MorpheusSession.local() @@ -43,15 +41,23 @@ object DataFrameInputExample extends App { import spark.sqlContext.implicits._ // 2) Generate some DataFrames that we'd like to interpret as a property graph. - val nodesDF = spark.createDataset(Seq( - (0L, "Alice", 42L), - (1L, "Bob", 23L), - (2L, "Eve", 84L) - )).toDF("id", "name", "age") - val relsDF = spark.createDataset(Seq( - (0L, 0L, 1L, "23/01/1987"), - (1L, 1L, 2L, "12/12/2009") - )).toDF("id", "source", "target", "since") + val nodesDF = spark + .createDataset( + Seq( + (0L, "Alice", 42L), + (1L, "Bob", 23L), + (2L, "Eve", 84L) + ) + ) + .toDF("id", "name", "age") + val relsDF = spark + .createDataset( + Seq( + (0L, 0L, 1L, "23/01/1987"), + (1L, 1L, 2L, "12/12/2009") + ) + ) + .toDF("id", "source", "target", "since") // 3) Generate node- and relationship tables that wrap the DataFrames. The mapping between graph elements and columns // is derived using naming conventions for identifier columns. @@ -66,7 +72,8 @@ object DataFrameInputExample extends App { // 6) Collect results into string by selecting a specific column. // This operation may be very expensive as it materializes results locally. - val names: Set[String] = result.records.table.df.collect().map(_.getAs[String]("n_name")).toSet + val names: Set[String] = + result.records.table.df.collect().map(_.getAs[String]("n_name")).toSet println(names) } diff --git a/morpheus-examples/src/main/scala/org/opencypher/morpheus/examples/DataFrameOutputExample.scala b/morpheus-examples/src/main/scala/org/opencypher/morpheus/examples/DataFrameOutputExample.scala index 07c8d15525..db553847f8 100644 --- a/morpheus-examples/src/main/scala/org/opencypher/morpheus/examples/DataFrameOutputExample.scala +++ b/morpheus-examples/src/main/scala/org/opencypher/morpheus/examples/DataFrameOutputExample.scala @@ -33,20 +33,19 @@ import org.opencypher.morpheus.api.MorpheusSession._ import org.opencypher.morpheus.util.App import org.opencypher.okapi.api.graph.CypherResult -/** - * Shows how to access a Cypher query result as a [[DataFrame]]. - */ +/** Shows how to access a Cypher query result as a [[DataFrame]]. */ object DataFrameOutputExample extends App { // 1) Create Morpheus session and retrieve Spark session implicit val morpheus: MorpheusSession = MorpheusSession.local() // 2) Load social network data via case class instances - val socialNetwork = morpheus.readFrom(SocialNetworkData.persons, SocialNetworkData.friendships) + val socialNetwork = + morpheus.readFrom(SocialNetworkData.persons, SocialNetworkData.friendships) // 3) Query graph with Cypher - val results: CypherResult = socialNetwork.cypher( - """|MATCH (a:Person)-[r:FRIEND_OF]->(b) + val results: CypherResult = + socialNetwork.cypher("""|MATCH (a:Person)-[r:FRIEND_OF]->(b) |RETURN a.name, b.name, r.since""".stripMargin) // 4) Extract DataFrame representing the query result @@ -58,20 +57,20 @@ object DataFrameOutputExample extends App { projection.show() } -/** - * Alternative to accessing a Cypher query result as a [[DataFrame]]. - */ +/** Alternative to accessing a Cypher query result as a [[DataFrame]]. */ object DataFrameOutputUsingAliasExample extends App { // 1) Create Morpheus session and retrieve Spark session implicit val morpheus: MorpheusSession = MorpheusSession.local() // 2) Load social network data via case class instances - val socialNetwork = morpheus.readFrom(SocialNetworkData.persons, SocialNetworkData.friendships) + val socialNetwork = + morpheus.readFrom(SocialNetworkData.persons, SocialNetworkData.friendships) // 3) Query graph with Cypher val results = socialNetwork.cypher( """|MATCH (a:Person)-[r:FRIEND_OF]->(b) - |RETURN a.name AS person1, b.name AS person2, r.since AS friendsSince""".stripMargin) + |RETURN a.name AS person1, b.name AS person2, r.since AS friendsSince""".stripMargin + ) // 4) Extract DataFrame representing the query result val df: DataFrame = results.records.asDataFrame diff --git a/morpheus-examples/src/main/scala/org/opencypher/morpheus/examples/DataSourceExample.scala b/morpheus-examples/src/main/scala/org/opencypher/morpheus/examples/DataSourceExample.scala index 8639679611..8c38507056 100644 --- a/morpheus-examples/src/main/scala/org/opencypher/morpheus/examples/DataSourceExample.scala +++ b/morpheus-examples/src/main/scala/org/opencypher/morpheus/examples/DataSourceExample.scala @@ -35,7 +35,8 @@ object DataSourceExample extends App { implicit val session: MorpheusSession = MorpheusSession.local() // 2) Load social network data via case class instances - val socialNetwork = session.readFrom(SocialNetworkData.persons, SocialNetworkData.friendships) + val socialNetwork = + session.readFrom(SocialNetworkData.persons, SocialNetworkData.friendships) session.catalog.store("sn", socialNetwork) diff --git a/morpheus-examples/src/main/scala/org/opencypher/morpheus/examples/GraphXPageRankExample.scala b/morpheus-examples/src/main/scala/org/opencypher/morpheus/examples/GraphXPageRankExample.scala index a11c6f8e4b..712c3b134a 100644 --- a/morpheus-examples/src/main/scala/org/opencypher/morpheus/examples/GraphXPageRankExample.scala +++ b/morpheus-examples/src/main/scala/org/opencypher/morpheus/examples/GraphXPageRankExample.scala @@ -38,8 +38,9 @@ import org.opencypher.okapi.api.io.conversion.NodeMappingBuilder /** * Round trip Morpheus -> GraphX -> Morpheus * - * This example demonstrates how Morpheus results can be used to construct a GraphX graph and invoke a GraphX algorithm - * on it. The computed ranks are imported back into Morpheus and used in a Cypher query. + * This example demonstrates how Morpheus results can be used to construct a GraphX graph and + * invoke a GraphX algorithm on it. The computed ranks are imported back into Morpheus and used in + * a Cypher query. */ object GraphXPageRankExample extends App { @@ -47,11 +48,11 @@ object GraphXPageRankExample extends App { implicit val morpheus: MorpheusSession = MorpheusSession.local() // 2) Load social network data via case class instances - val socialNetwork = morpheus.readFrom(SocialNetworkData.persons, SocialNetworkData.friendships) + val socialNetwork = + morpheus.readFrom(SocialNetworkData.persons, SocialNetworkData.friendships) // 3) Query graph with Cypher - val nodes = socialNetwork.cypher( - """|MATCH (n:Person) + val nodes = socialNetwork.cypher("""|MATCH (n:Person) |RETURN id(n), n.name""".stripMargin) val rels = socialNetwork.cypher( @@ -61,29 +62,39 @@ object GraphXPageRankExample extends App { ) // 4) Create GraphX compatible RDDs from nodes and relationships - val graphXNodeRDD = nodes.records.asDataFrame.rdd.map(row => decodeLong(row.getAs[Array[Byte]](0)) -> row.getString(1)) - val graphXRelRDD = rels.records.asDataFrame.rdd.map(row => Edge(decodeLong(row.getAs[Array[Byte]](0)), decodeLong(row.getAs[Array[Byte]](1)), ())) + val graphXNodeRDD = nodes.records.asDataFrame.rdd.map(row => + decodeLong(row.getAs[Array[Byte]](0)) -> row.getString(1) + ) + val graphXRelRDD = rels.records.asDataFrame.rdd.map(row => + Edge( + decodeLong(row.getAs[Array[Byte]](0)), + decodeLong(row.getAs[Array[Byte]](1)), + () + ) + ) // 5) Compute Page Rank via GraphX val graph = Graph(graphXNodeRDD, graphXRelRDD) val ranks = graph.pageRank(0.0001).vertices // 6) Convert RDD to DataFrame - val rankTable = morpheus.sparkSession.createDataFrame(ranks) + val rankTable = morpheus.sparkSession + .createDataFrame(ranks) .withColumnRenamed("_1", "id") .withColumnRenamed("_2", "rank") // 7) Create property graph from rank data - val ranksNodeMapping = NodeMappingBuilder.on("id").withPropertyKey("rank").build - val rankNodes = morpheus.readFrom(MorpheusElementTable.create(ranksNodeMapping, rankTable)) + val ranksNodeMapping = + NodeMappingBuilder.on("id").withPropertyKey("rank").build + val rankNodes = + morpheus.readFrom(MorpheusElementTable.create(ranksNodeMapping, rankTable)) // 8) Mount both graphs in the session morpheus.catalog.store("ranks", rankNodes) morpheus.catalog.store("sn", socialNetwork) // 9) Query across both graphs to print names with corresponding ranks, sorted by rank - val result = morpheus.cypher( - """|FROM GRAPH ranks + val result = morpheus.cypher("""|FROM GRAPH ranks |MATCH (r) |WITH id(r) as id, r.rank as rank |FROM GRAPH sn @@ -93,13 +104,13 @@ object GraphXPageRankExample extends App { |ORDER BY rank DESC""".stripMargin) result.show - //+---------------------------------------------+ - //| name | rank | - //+---------------------------------------------+ - //| 'Carol' | 1.4232365145228216 | - //| 'Bob' | 1.0235131396957122 | - //| 'Alice' | 0.5532503457814661 | - //+---------------------------------------------+ + // +---------------------------------------------+ + // | name | rank | + // +---------------------------------------------+ + // | 'Carol' | 1.4232365145228216 | + // | 'Bob' | 1.0235131396957122 | + // | 'Alice' | 0.5532503457814661 | + // +---------------------------------------------+ } // end::full-example[] diff --git a/morpheus-examples/src/main/scala/org/opencypher/morpheus/examples/HiveSupportExample.scala b/morpheus-examples/src/main/scala/org/opencypher/morpheus/examples/HiveSupportExample.scala index 031551b8f5..8b724b5f8b 100644 --- a/morpheus-examples/src/main/scala/org/opencypher/morpheus/examples/HiveSupportExample.scala +++ b/morpheus-examples/src/main/scala/org/opencypher/morpheus/examples/HiveSupportExample.scala @@ -40,18 +40,21 @@ object HiveSupportExample extends App { session.sparkSession.sql(s"DROP DATABASE IF EXISTS $hiveDatabaseName CASCADE") session.sparkSession.sql(s"CREATE DATABASE IF NOT EXISTS $hiveDatabaseName") - - val socialNetwork = session.readFrom(SocialNetworkData.persons, SocialNetworkData.friendships) - val tmp = s"file:///${System.getProperty("java.io.tmpdir").replace("\\", "/")}/${System.currentTimeMillis()}" + val socialNetwork = + session.readFrom(SocialNetworkData.persons, SocialNetworkData.friendships) + val tmp = + s"file:///${System.getProperty("java.io.tmpdir").replace("\\", "/")}/${System.currentTimeMillis()}" val fs = GraphSources.fs(tmp, Some(hiveDatabaseName)).parquet val graphName = GraphName("sn") fs.store(graphName, socialNetwork) - val nodeTableName = HiveTableName(hiveDatabaseName, graphName, Node, Set("Person")) + val nodeTableName = + HiveTableName(hiveDatabaseName, graphName, Node, Set("Person")) - val result = session.sql(s"SELECT * FROM $nodeTableName WHERE property_age >= 15") + val result = + session.sql(s"SELECT * FROM $nodeTableName WHERE property_age >= 15") result.show diff --git a/morpheus-examples/src/main/scala/org/opencypher/morpheus/examples/LdbcHiveExample.scala b/morpheus-examples/src/main/scala/org/opencypher/morpheus/examples/LdbcHiveExample.scala index 735f8df309..50fccec618 100644 --- a/morpheus-examples/src/main/scala/org/opencypher/morpheus/examples/LdbcHiveExample.scala +++ b/morpheus-examples/src/main/scala/org/opencypher/morpheus/examples/LdbcHiveExample.scala @@ -40,13 +40,13 @@ import org.opencypher.okapi.api.graph.Namespace /** * This demo reads data generated by the LDBC SNB data generator and performs the following steps: * - * 1) Loads the raw CSV files into Hive tables - * 2) Normalizes tables according to the LDBC schema (i.e. place -> [City, Country, Continent] - * 3) Generates a Graph DDL script based on LDBC naming conventions (if not already existing) - * 4) Initializes a SQL PGDS based on the generated Graph DDL file - * 5) Runs a Cypher query over the LDBC graph in Spark + * 1) Loads the raw CSV files into Hive tables 2) Normalizes tables according to the LDBC schema + * (i.e. place -> [City, Country, Continent] 3) Generates a Graph DDL script based on LDBC naming + * conventions (if not already existing) 4) Initializes a SQL PGDS based on the generated Graph DDL + * file 5) Runs a Cypher query over the LDBC graph in Spark * - * More detail about the LDBC SNB data generator are available under https://github.com/ldbc/ldbc_snb_datagen + * More detail about the LDBC SNB data generator are available under + * https://github.com/ldbc/ldbc_snb_datagen */ object LdbcHiveExample extends App { @@ -92,12 +92,13 @@ object LdbcHiveExample extends App { session.registerSource(Namespace("sql"), sqlGraphSource) - session.cypher( - s""" + session + .cypher(s""" |FROM GRAPH sql.LDBC |MATCH (n:Person)-[:islocatedin]->(c:City) |RETURN n.firstName, c.name |ORDER BY n.firstName, c.name |LIMIT 20 - """.stripMargin).show + """.stripMargin) + .show } diff --git a/morpheus-examples/src/main/scala/org/opencypher/morpheus/examples/MultipleGraphExample.scala b/morpheus-examples/src/main/scala/org/opencypher/morpheus/examples/MultipleGraphExample.scala index 1cdaeb26ba..44cc889f58 100644 --- a/morpheus-examples/src/main/scala/org/opencypher/morpheus/examples/MultipleGraphExample.scala +++ b/morpheus-examples/src/main/scala/org/opencypher/morpheus/examples/MultipleGraphExample.scala @@ -32,26 +32,32 @@ import org.opencypher.morpheus.util.App import org.opencypher.okapi.api.graph.Namespace /** - * Demonstrates multiple graph capabilities by loading a social network from case class objects and a purchase network - * from CSV data and schema files. The example connects both networks via matching user and customer names. A Cypher - * query is then used to compute products that friends have bought. + * Demonstrates multiple graph capabilities by loading a social network from case class objects and + * a purchase network from CSV data and schema files. The example connects both networks via + * matching user and customer names. A Cypher query is then used to compute products that friends + * have bought. */ object MultipleGraphExample extends App { // 1) Create Morpheus session implicit val morpheus: MorpheusSession = MorpheusSession.local() // 2) Load social network data via case class instances - val socialNetwork = morpheus.readFrom(SocialNetworkData.persons, SocialNetworkData.friendships) + val socialNetwork = + morpheus.readFrom(SocialNetworkData.persons, SocialNetworkData.friendships) morpheus.catalog.store("socialNetwork", socialNetwork) // 3) Register a file system graph source to the catalog // Note: if files were stored in HDFS, the file path would indicate so by starting with hdfs:// val csvFolder = getClass.getResource("/fs-graphsource/csv").getFile - morpheus.registerSource(Namespace("purchases"), GraphSources.fs(rootPath = csvFolder).csv) + morpheus.registerSource( + Namespace("purchases"), + GraphSources.fs(rootPath = csvFolder).csv + ) // 5) Create new edges between users and customers with the same name - val recommendationGraph = morpheus.cypher( - """|FROM GRAPH socialNetwork + val recommendationGraph = morpheus + .cypher( + """|FROM GRAPH socialNetwork |MATCH (p:Person) |FROM GRAPH purchases.products |MATCH (c:Customer) @@ -60,7 +66,8 @@ object MultipleGraphExample extends App { | CREATE (p)-[:IS]->(c) |RETURN GRAPH """.stripMargin - ).graph + ) + .graph // 6) Query for product recommendations val recommendations = recommendationGraph.cypher( @@ -69,7 +76,8 @@ object MultipleGraphExample extends App { | (customer)-[:BOUGHT]->(product:Product) |RETURN DISTINCT product.title AS recommendation, person.name AS for |ORDER BY recommendation - """.stripMargin) + """.stripMargin + ) recommendations.show } diff --git a/morpheus-examples/src/main/scala/org/opencypher/morpheus/examples/Neo4jCustomSchemaExample.scala b/morpheus-examples/src/main/scala/org/opencypher/morpheus/examples/Neo4jCustomSchemaExample.scala index 7d4fe27d78..ee0fb1b725 100644 --- a/morpheus-examples/src/main/scala/org/opencypher/morpheus/examples/Neo4jCustomSchemaExample.scala +++ b/morpheus-examples/src/main/scala/org/opencypher/morpheus/examples/Neo4jCustomSchemaExample.scala @@ -66,14 +66,15 @@ object Neo4jCustomSchemaExample extends App { // Convert the schema to a serializable JSON format val jsonString = schema.toJson // Get a filesystem to store the file on - val fileSystem = FileSystem.get(session.sparkSession.sparkContext.hadoopConfiguration) + val fileSystem = + FileSystem.get(session.sparkSession.sparkContext.hadoopConfiguration) // Write the file for future reading // Commented-out; this file already exists in the beginning of this example // fileSystem.writeFile(getClass.getResource("/").getPath + "schema.json", jsonString) // run queries! - session.cypher( - """ + session + .cypher(""" |FROM GRAPH socialNetwork.graph |MATCH (a:Person) |WHERE a.age > 15 diff --git a/morpheus-examples/src/main/scala/org/opencypher/morpheus/examples/Neo4jMergeExample.scala b/morpheus-examples/src/main/scala/org/opencypher/morpheus/examples/Neo4jMergeExample.scala index 4e7cd9bc98..b2d19f93ff 100644 --- a/morpheus-examples/src/main/scala/org/opencypher/morpheus/examples/Neo4jMergeExample.scala +++ b/morpheus-examples/src/main/scala/org/opencypher/morpheus/examples/Neo4jMergeExample.scala @@ -37,7 +37,8 @@ import org.opencypher.okapi.neo4j.io.testing.Neo4jTestUtils._ /** * Demonstrates merging a graph into an existing Neo4j database. * - * This merge requires node and relationship keys to identify same elements in the merge graph and the Neo4j database. + * This merge requires node and relationship keys to identify same elements in the merge graph and + * the Neo4j database. */ object Neo4jMergeExample extends App { val boltUrl = namedArg("--bolt-url").getOrElse("bolt://localhost:7687") @@ -65,8 +66,8 @@ object Neo4jMergeExample extends App { val relKeys = Map("FRIEND_OF" -> Set("id"), "MARRIED_TO" -> Set("id")) // Create a merge graph with updated data - val mergeGraph = morpheus.cypher( - """ + val mergeGraph = morpheus + .cypher(""" |CONSTRUCT | CREATE (a:Person { name: 'Alice', age: 11 }) | CREATE (b:Person { name: 'Bob', age: 21}) @@ -76,37 +77,52 @@ object Neo4jMergeExample extends App { | CREATE (b)-[:MARRIED_TO { id: 1 }]->(t) | CREATE (c)-[:FRIEND_OF { id: 2, since: '23/01/2019' }]->(t) |RETURN GRAPH - """.stripMargin).graph + """.stripMargin) + .graph // Speed up merge operation. Requires Neo4j Enterprise Edition // Neo4jGraphMerge.createIndexes(entireGraphName, neo4j.dataSourceConfig, nodeKeys) // Merge graph into existing Neo4j database - Neo4jGraphMerge.merge(entireGraphName, mergeGraph, neo4j.config, Some(nodeKeys), Some(relKeys)) + Neo4jGraphMerge.merge( + entireGraphName, + mergeGraph, + neo4j.config, + Some(nodeKeys), + Some(relKeys) + ) // Register Property Graph Data Source (PGDS) to read the updated graph from Neo4j - morpheus.registerSource(Namespace("updatedSocialNetwork"), GraphSources.cypher.neo4j(neo4j.config)) + morpheus.registerSource( + Namespace("updatedSocialNetwork"), + GraphSources.cypher.neo4j(neo4j.config) + ) // Access the graphs via their qualified graph names - val updatedSocialNetwork = morpheus.catalog.graph("updatedSocialNetwork.graph") + val updatedSocialNetwork = + morpheus.catalog.graph("updatedSocialNetwork.graph") - updatedSocialNetwork.cypher( - """ + updatedSocialNetwork + .cypher(""" |MATCH (p:Person) |RETURN p |ORDER BY p.name - """.stripMargin).records.show + """.stripMargin) + .records + .show - updatedSocialNetwork.cypher( - """ + updatedSocialNetwork + .cypher(""" |MATCH (p:Person) |OPTIONAL MATCH (p)-[r:FRIEND_OF|MARRIED_TO]->(o:Person) |RETURN p, r, o |ORDER BY p.name, r.since - """.stripMargin).records.show + """.stripMargin) + .records + .show // Reset Neo4j test instance and close the session and driver neo4j.close() - } +} // end::full-example[] diff --git a/morpheus-examples/src/main/scala/org/opencypher/morpheus/examples/Neo4jReadWriteExample.scala b/morpheus-examples/src/main/scala/org/opencypher/morpheus/examples/Neo4jReadWriteExample.scala index 27e13cd832..18e348349d 100644 --- a/morpheus-examples/src/main/scala/org/opencypher/morpheus/examples/Neo4jReadWriteExample.scala +++ b/morpheus-examples/src/main/scala/org/opencypher/morpheus/examples/Neo4jReadWriteExample.scala @@ -44,28 +44,31 @@ object Neo4jReadWriteExample extends App { val neo4j = connectNeo4j("", boltUrl) // Register Property Graph Data Sources (PGDS) - private val neo4jPgds: Neo4jPropertyGraphDataSource = GraphSources.cypher.neo4j(neo4j.config) - private val filePgds: FSGraphSource = GraphSources.fs(rootPath = getClass.getResource("/fs-graphsource/csv").getFile).csv + private val neo4jPgds: Neo4jPropertyGraphDataSource = + GraphSources.cypher.neo4j(neo4j.config) + private val filePgds: FSGraphSource = GraphSources + .fs(rootPath = getClass.getResource("/fs-graphsource/csv").getFile) + .csv morpheus.registerSource(Namespace("Neo4j"), neo4jPgds) morpheus.registerSource(Namespace("CSV"), filePgds) // Copy products graph from File-based PGDS to Neo4j PGDS - morpheus.cypher( - s""" + morpheus.cypher(s""" |CATALOG CREATE GRAPH Neo4j.products { | FROM GRAPH CSV.products RETURN GRAPH |} """.stripMargin) // Read graph from Neo4j and run a Cypher query - morpheus.cypher( - s""" + morpheus + .cypher(s""" |FROM Neo4j.products |MATCH (n:Customer)-[r:BOUGHT]->(m:Product) |RETURN m.title AS product, avg(r.rating) AS avg_rating, count(n) AS purchases |ORDER BY avg_rating DESC, purchases DESC, product ASC - """.stripMargin).show + """.stripMargin) + .show // Clear Neo4j test instance and close session / driver neo4j.close() diff --git a/morpheus-examples/src/main/scala/org/opencypher/morpheus/examples/Neo4jWorkflowExample.scala b/morpheus-examples/src/main/scala/org/opencypher/morpheus/examples/Neo4jWorkflowExample.scala index ee706d3000..fa673c92a2 100644 --- a/morpheus-examples/src/main/scala/org/opencypher/morpheus/examples/Neo4jWorkflowExample.scala +++ b/morpheus-examples/src/main/scala/org/opencypher/morpheus/examples/Neo4jWorkflowExample.scala @@ -49,8 +49,16 @@ object Neo4jWorkflowExample extends App { val neo4j = connectNeo4j(personNetwork, boltUrl) // Register Property Graph Data Sources (PGDS) - morpheus.registerSource(Namespace("socialNetwork"), GraphSources.cypher.neo4j(neo4j.config)) - morpheus.registerSource(Namespace("purchases"), GraphSources.fs(rootPath = getClass.getResource("/fs-graphsource/csv").getFile).csv) + morpheus.registerSource( + Namespace("socialNetwork"), + GraphSources.cypher.neo4j(neo4j.config) + ) + morpheus.registerSource( + Namespace("purchases"), + GraphSources + .fs(rootPath = getClass.getResource("/fs-graphsource/csv").getFile) + .csv + ) // Access the graphs via their qualified graph names val socialNetwork = morpheus.catalog.graph("socialNetwork.graph") @@ -58,8 +66,9 @@ object Neo4jWorkflowExample extends App { // Build new integrated graph that connects the social and product graphs and // create new edges between users and customers with the same name - val integratedGraph = morpheus.cypher( - """|FROM GRAPH socialNetwork.graph + val integratedGraph = morpheus + .cypher( + """|FROM GRAPH socialNetwork.graph |MATCH (p:Person) |FROM GRAPH purchases.products |MATCH (c:Customer) @@ -69,16 +78,19 @@ object Neo4jWorkflowExample extends App { | CREATE (p)-[:IS]->(c) |RETURN GRAPH """.stripMargin - ).getGraph.get + ) + .getGraph + .get // Query for product recommendations and create new edges between users and the product they should buy - val recommendationGraph = integratedGraph.cypher( - """|MATCH (person:Person)-[:FRIEND_OF]-(friend:Person), + val recommendationGraph = integratedGraph + .cypher("""|MATCH (person:Person)-[:FRIEND_OF]-(friend:Person), | (friend)-[:IS]->(customer:Customer), | (customer)-[:BOUGHT]->(product:Product) |CONSTRUCT | CREATE (person)-[:SHOULD_BUY]->(product) - |RETURN GRAPH""".stripMargin).graph + |RETURN GRAPH""".stripMargin) + .graph // Use the Neo4jGraphMerge utility to write the products and the recommendations back to Neo4j @@ -87,16 +99,24 @@ object Neo4jWorkflowExample extends App { val nodeKeys = Map("Person" -> Set("name"), "Product" -> Set("title")) // Write the recommendations back to Neo4j - Neo4jGraphMerge.merge(entireGraphName, recommendationGraph, neo4j.config, Some(nodeKeys)) + Neo4jGraphMerge.merge( + entireGraphName, + recommendationGraph, + neo4j.config, + Some(nodeKeys) + ) // Proof that the write-back to Neo4j worked, retrieve and print updated Neo4j results val updatedNeo4jSource = GraphSources.cypher.neo4j(neo4j.config) morpheus.registerSource(Namespace("updated-neo4j"), updatedNeo4jSource) - val socialNetworkWithRanks = morpheus.catalog.graph(QualifiedGraphName(Namespace("updated-neo4j"), entireGraphName)) - socialNetworkWithRanks.cypher( - """MATCH (person:Person)-[:SHOULD_BUY]->(product:Product) + val socialNetworkWithRanks = morpheus.catalog.graph( + QualifiedGraphName(Namespace("updated-neo4j"), entireGraphName) + ) + socialNetworkWithRanks + .cypher("""MATCH (person:Person)-[:SHOULD_BUY]->(product:Product) |RETURN person.name AS person, product.title AS should_buy - |ORDER BY person, should_buy""".stripMargin).show + |ORDER BY person, should_buy""".stripMargin) + .show // Clear Neo4j test instance and close session / driver neo4j.close() diff --git a/morpheus-examples/src/main/scala/org/opencypher/morpheus/examples/NorthwindJdbcExample.scala b/morpheus-examples/src/main/scala/org/opencypher/morpheus/examples/NorthwindJdbcExample.scala index 975a1d0f58..8b472e9032 100644 --- a/morpheus-examples/src/main/scala/org/opencypher/morpheus/examples/NorthwindJdbcExample.scala +++ b/morpheus-examples/src/main/scala/org/opencypher/morpheus/examples/NorthwindJdbcExample.scala @@ -58,25 +58,28 @@ object NorthwindJdbcExample extends App { morpheus.registerSource(Namespace("sql"), sqlGraphSource) // print the number of nodes in the graph - morpheus.cypher( - """ + morpheus + .cypher(""" |FROM GRAPH sql.Northwind |MATCH (n) |RETURN count(n) - """.stripMargin).records.show + """.stripMargin) + .records + .show // print the schema of the graph println(morpheus.catalog.graph("sql.Northwind").schema.pretty) // run a simple query - morpheus.cypher( - """ + morpheus + .cypher(""" |FROM GRAPH sql.Northwind |MATCH (e:Employee)-[:REPORTS_TO]->(:Employee)<-[:HAS_EMPLOYEE]-(o:Order) |RETURN o.customerID AS customer, o.orderDate AS orderedAt, e.lastName AS handledBy, e.title AS employee | ORDER BY orderedAt, handledBy, customer | LIMIT 50 - |""".stripMargin).show + |""".stripMargin) + .show } // end::full-example[] diff --git a/morpheus-examples/src/main/scala/org/opencypher/morpheus/examples/RecommendationExample.scala b/morpheus-examples/src/main/scala/org/opencypher/morpheus/examples/RecommendationExample.scala index a05a6f9e53..08a39e8e4f 100644 --- a/morpheus-examples/src/main/scala/org/opencypher/morpheus/examples/RecommendationExample.scala +++ b/morpheus-examples/src/main/scala/org/opencypher/morpheus/examples/RecommendationExample.scala @@ -34,9 +34,9 @@ import org.opencypher.okapi.neo4j.io.MetaLabelSupport._ import org.opencypher.okapi.neo4j.io.testing.Neo4jTestUtils._ /** - * This application demonstrates the integration of three data sources into a single graph which is used for computing - * recommendations. Two graphs are loaded from separate Neo4j databases, one graph is loaded from csv files stored in - * the local file system. + * This application demonstrates the integration of three data sources into a single graph which is + * used for computing recommendations. Two graphs are loaded from separate Neo4j databases, one + * graph is loaded from csv files stored in the local file system. */ object RecommendationExample extends App { val boltUrl1 = namedArg("--bolt-url-1").getOrElse("bolt://localhost:7687") @@ -53,18 +53,29 @@ object RecommendationExample extends App { // The graph within Neo4j is partitioned into regions using a property key. Within the data source, we map each // partition to a separate graph name (i.e. US and EU) - morpheus.registerSource(Namespace("usSocialNetwork"), GraphSources.cypher.neo4j(neo4jServerUS.config)) - morpheus.registerSource(Namespace("euSocialNetwork"), GraphSources.cypher.neo4j(neo4jServerEU.config)) + morpheus.registerSource( + Namespace("usSocialNetwork"), + GraphSources.cypher.neo4j(neo4jServerUS.config) + ) + morpheus.registerSource( + Namespace("euSocialNetwork"), + GraphSources.cypher.neo4j(neo4jServerEU.config) + ) // File-based CSV GDS - morpheus.registerSource(Namespace("purchases"), GraphSources.fs(rootPath = s"${getClass.getResource("/fs-graphsource/csv").getFile}").csv) + morpheus.registerSource( + Namespace("purchases"), + GraphSources + .fs(rootPath = s"${getClass.getResource("/fs-graphsource/csv").getFile}") + .csv + ) // Start analytical workload /** - * Returns a query that creates a graph containing persons that live in the same city and - * know each other via 1 to 2 hops. The created graph contains a CLOSE_TO relationship between - * each such pair of persons and is stored in the session catalog using the given graph name. + * Returns a query that creates a graph containing persons that live in the same city and know + * each other via 1 to 2 hops. The created graph contains a CLOSE_TO relationship between each + * such pair of persons and is stored in the session catalog using the given graph name. */ def cityFriendsQuery(fromGraph: String): String = s"""FROM GRAPH $fromGraph @@ -77,16 +88,18 @@ object RecommendationExample extends App { """.stripMargin // Find persons that are close to each other in the US social network - val usFriends = morpheus.cypher(cityFriendsQuery(s"usSocialNetwork.$entireGraphName")).graph + val usFriends = + morpheus.cypher(cityFriendsQuery(s"usSocialNetwork.$entireGraphName")).graph // Find persons that are close to each other in the EU social network - val euFriends = morpheus.cypher(cityFriendsQuery(s"euSocialNetwork.$entireGraphName")).graph + val euFriends = + morpheus.cypher(cityFriendsQuery(s"euSocialNetwork.$entireGraphName")).graph // Union the US and EU graphs into a single graph 'allFriends' and store it in the session morpheus.catalog.store("allFriends", usFriends.unionAll(euFriends)) // Connect the social network with the products network using equal person and customer emails - val connectedCustomers = morpheus.cypher( - s"""FROM GRAPH allFriends + val connectedCustomers = morpheus + .cypher(s"""FROM GRAPH allFriends |MATCH (p:Person) |FROM GRAPH purchases.products |MATCH (c:Customer) @@ -94,19 +107,21 @@ object RecommendationExample extends App { |CONSTRUCT ON purchases.products, allFriends | CREATE (c)-[:IS]->(p) |RETURN GRAPH - """.stripMargin).graph + """.stripMargin) + .graph // Compute recommendations for 'target' based on their interests and what persons close to the // 'target' have already bought and given a helpful and positive rating - val recommendationTable = connectedCustomers.cypher( - s"""|MATCH (target:Person)<-[:CLOSE_TO]-(person:Person), + val recommendationTable = connectedCustomers + .cypher(s"""|MATCH (target:Person)<-[:CLOSE_TO]-(person:Person), | (target)-[:HAS_INTEREST]->(i:Interest), | (person)<-[:IS]-(x:Customer)-[b:BOUGHT]->(product:Product {category: i.name}) |WHERE b.rating >= 4 AND (b.helpful * 1.0) / b.votes > 0.6 |WITH * ORDER BY product.rank |RETURN DISTINCT product.title AS product, target.name AS name |LIMIT 3 - """.stripMargin).records + """.stripMargin) + .records // Print the results recommendationTable.show diff --git a/morpheus-examples/src/main/scala/org/opencypher/morpheus/examples/UpdateExample.scala b/morpheus-examples/src/main/scala/org/opencypher/morpheus/examples/UpdateExample.scala index cf432e554d..6d0e0996af 100644 --- a/morpheus-examples/src/main/scala/org/opencypher/morpheus/examples/UpdateExample.scala +++ b/morpheus-examples/src/main/scala/org/opencypher/morpheus/examples/UpdateExample.scala @@ -37,19 +37,17 @@ import org.opencypher.okapi.api.value.CypherValue.CypherMap import scala.collection.JavaConverters._ -/** - * Demonstrates how to retrieve Property Graph elements as a Dataset and update them. - */ +/** Demonstrates how to retrieve Property Graph elements as a Dataset and update them. */ object UpdateExample extends App { // 1) Create Morpheus session and retrieve Spark session implicit val session: MorpheusSession = MorpheusSession.local() // 2) Load social network data via case class instances - val socialNetwork = session.readFrom(SocialNetworkData.persons, SocialNetworkData.friendships) + val socialNetwork = + session.readFrom(SocialNetworkData.persons, SocialNetworkData.friendships) // 3) Query graph with Cypher - val results = socialNetwork.cypher( - """|MATCH (p:Person) + val results = socialNetwork.cypher("""|MATCH (p:Person) |WHERE p.age >= 18 |RETURN p""".stripMargin) @@ -58,7 +56,10 @@ object UpdateExample extends App { // 5) Add a new label and property to the nodes val adults: Dataset[MorpheusNode] = ds.map { record: CypherMap => - record("p").cast[MorpheusNode].withLabel("Adult").withProperty("canVote", true) + record("p") + .cast[MorpheusNode] + .withLabel("Adult") + .withProperty("canVote", true) } // 6) Print updated nodes diff --git a/morpheus-examples/src/main/scala/org/opencypher/morpheus/examples/ViewsExample.scala b/morpheus-examples/src/main/scala/org/opencypher/morpheus/examples/ViewsExample.scala index dd2f5ee7a2..638abcbbbc 100644 --- a/morpheus-examples/src/main/scala/org/opencypher/morpheus/examples/ViewsExample.scala +++ b/morpheus-examples/src/main/scala/org/opencypher/morpheus/examples/ViewsExample.scala @@ -30,16 +30,13 @@ package org.opencypher.morpheus.examples import org.opencypher.morpheus.api.MorpheusSession import org.opencypher.morpheus.util.App -/** - * Demonstrates how to use views to filter graphs. - */ +/** Demonstrates how to use views to filter graphs. */ object ViewsExample extends App { // 1) Create Morpheus session and retrieve Spark session implicit val morpheus: MorpheusSession = MorpheusSession.local() // 2) Create a graph - morpheus.cypher( - """|CATALOG CREATE GRAPH sn { + morpheus.cypher("""|CATALOG CREATE GRAPH sn { | CONSTRUCT | CREATE (a: Person {age: 18}) | CREATE (b: Person {age: 20}) @@ -60,8 +57,7 @@ object ViewsExample extends App { """.stripMargin) // 3) Define a view that filters young friends - morpheus.cypher( - """ + morpheus.cypher(""" |CATALOG CREATE VIEW youngFriends($people) { | FROM $people | MATCH (p1: Person)-[r]->(p2: Person) @@ -73,8 +69,7 @@ object ViewsExample extends App { """.stripMargin) // 3) Apply `youngFriends` view to graph `sn` - val results = morpheus.cypher( - """|FROM youngFriends(sn) + val results = morpheus.cypher("""|FROM youngFriends(sn) |MATCH (p: Person)-[r]->(e) |RETURN p, r, e |ORDER BY p.age""".stripMargin) diff --git a/morpheus-examples/src/main/scala/org/opencypher/morpheus/integration/Customer360IntegrationDemo.scala b/morpheus-examples/src/main/scala/org/opencypher/morpheus/integration/Customer360IntegrationDemo.scala index 23260e7a29..7667874f20 100644 --- a/morpheus-examples/src/main/scala/org/opencypher/morpheus/integration/Customer360IntegrationDemo.scala +++ b/morpheus-examples/src/main/scala/org/opencypher/morpheus/integration/Customer360IntegrationDemo.scala @@ -38,16 +38,20 @@ import org.opencypher.okapi.neo4j.io.MetaLabelSupport._ import org.opencypher.okapi.neo4j.io.Neo4jConfig /** - * This demo shows the transformation from a single CSV file into a PropertyGraph that is eventually stored in Neo4j. + * This demo shows the transformation from a single CSV file into a PropertyGraph that is + * eventually stored in Neo4j. * - * The input is a de-normalized wide table containing interactions between customers and employees. Those interactions - * are normalized by splitting them into several views and registering them as Hive tables. + * The input is a de-normalized wide table containing interactions between customers and employees. + * Those interactions are normalized by splitting them into several views and registering them as + * Hive tables. * - * A Graph DDL file describes the PropertyGraph schema and the mapping between Hive views and PropertyGraph elements - * (i.e. nodes and relationships). After loading the graph via Spark, Cypher 10 Multiple Graph support is used to - * extend the graph with new relationships. The resulting graph is merged to Neo4j where it can be further analyzed. + * A Graph DDL file describes the PropertyGraph schema and the mapping between Hive views and + * PropertyGraph elements (i.e. nodes and relationships). After loading the graph via Spark, Cypher + * 10 Multiple Graph support is used to extend the graph with new relationships. The resulting + * graph is merged to Neo4j where it can be further analyzed. * - * This integration demo requires a running Neo4j Enterprise installation listening on bolt://localhost:7687. + * This integration demo requires a running Neo4j Enterprise installation listening on + * bolt://localhost:7687. */ object Customer360IntegrationDemo extends App { @@ -57,19 +61,21 @@ object Customer360IntegrationDemo extends App { implicit val spark: SparkSession = session.sparkSession - val inputSchema: StructType = StructType(Seq( - StructField("interactionId", LongType, nullable = false), - StructField("date", StringType, nullable = false), - StructField("customerIdx", LongType, nullable = false), - StructField("empNo", LongType, nullable = false), - StructField("empName", StringType, nullable = false), - StructField("type", StringType, nullable = false), - StructField("outcomeScore", StringType, nullable = false), - StructField("accountHolderId", StringType, nullable = false), - StructField("policyAccountNumber", StringType, nullable = false), - StructField("customerId", StringType, nullable = false), - StructField("customerName", StringType, nullable = false) - )) + val inputSchema: StructType = StructType( + Seq( + StructField("interactionId", LongType, nullable = false), + StructField("date", StringType, nullable = false), + StructField("customerIdx", LongType, nullable = false), + StructField("empNo", LongType, nullable = false), + StructField("empName", StringType, nullable = false), + StructField("type", StringType, nullable = false), + StructField("outcomeScore", StringType, nullable = false), + StructField("accountHolderId", StringType, nullable = false), + StructField("policyAccountNumber", StringType, nullable = false), + StructField("customerId", StringType, nullable = false), + StructField("customerName", StringType, nullable = false) + ) + ) // Import wide table from CSV file val importCsv = spark.read @@ -89,17 +95,56 @@ object Customer360IntegrationDemo extends App { importCsv.write.saveAsTable(s"$inputTableName") // Create views for nodes - createView(inputTableName, "interactions", true, "interactionId", "date", "type", "outcomeScore") - createView(inputTableName, "customers", true, "customerIdx", "customerId", "customerName") + createView( + inputTableName, + "interactions", + true, + "interactionId", + "date", + "type", + "outcomeScore" + ) + createView( + inputTableName, + "customers", + true, + "customerIdx", + "customerId", + "customerName" + ) createView(inputTableName, "account_holders", true, "accountHolderId") createView(inputTableName, "policies", true, "policyAccountNumber") createView(inputTableName, "customer_reps", true, "empNo", "empName") // Create views for relationships - createView(inputTableName, "has_customer_reps", false, "interactionId", "empNo") - createView(inputTableName, "has_customers", false, "interactionId", "customerId") - createView(inputTableName, "has_policies", false, "interactionId", "policyAccountNumber") - createView(inputTableName, "has_account_holders", false, "interactionId", "accountHolderId") + createView( + inputTableName, + "has_customer_reps", + false, + "interactionId", + "empNo" + ) + createView( + inputTableName, + "has_customers", + false, + "interactionId", + "customerId" + ) + createView( + inputTableName, + "has_policies", + false, + "interactionId", + "policyAccountNumber" + ) + createView( + inputTableName, + "has_account_holders", + false, + "interactionId", + "accountHolderId" + ) // Create SQL PropertyGraph DataSource val sqlGraphSource = GraphSources @@ -111,16 +156,18 @@ object Customer360IntegrationDemo extends App { val g = session.catalog.graph("c360.interactions") session.catalog.store("foo", g) - val c360Interactions = session.cypher( - """ + val c360Interactions = session + .cypher(""" |FROM GRAPH foo |MATCH (i:Interaction)-[rel]->(other) |CONSTRUCT ON foo | CREATE (other)-[:HAS_INTERACTION]->(i) |RETURN GRAPH - """.stripMargin).graph + """.stripMargin) + .graph - val neo4jConfig = Neo4jConfig(new URI("bolt://localhost:7687"), "neo4j", Some("admin")) + val neo4jConfig = + Neo4jConfig(new URI("bolt://localhost:7687"), "neo4j", Some("admin")) val nodeKeys = Map( "Interaction" -> Set("interactionId"), @@ -134,13 +181,22 @@ object Customer360IntegrationDemo extends App { Neo4jGraphMerge.createIndexes(entireGraphName, neo4jConfig, nodeKeys) // Merge interaction graph into Neo4j database with "Customer 360" data - Neo4jGraphMerge.merge(entireGraphName, c360Interactions, neo4jConfig, Some(nodeKeys)) + Neo4jGraphMerge.merge( + entireGraphName, + c360Interactions, + neo4jConfig, + Some(nodeKeys) + ) - def createView(fromTable: String, viewName: String, distinct: Boolean, columns: String*): Unit = { + def createView( + fromTable: String, + viewName: String, + distinct: Boolean, + columns: String* + ): Unit = { val distinctString = if (distinct) "DISTINCT" else "" - spark.sql( - s""" + spark.sql(s""" |CREATE VIEW $databaseName.$viewName AS | SELECT $distinctString ${columns.mkString(", ")} FROM $fromTable """.stripMargin) diff --git a/morpheus-examples/src/main/scala/org/opencypher/morpheus/integration/yelp/Part1_YelpImport.scala b/morpheus-examples/src/main/scala/org/opencypher/morpheus/integration/yelp/Part1_YelpImport.scala index 3c129ea6e0..49e8a336ad 100644 --- a/morpheus-examples/src/main/scala/org/opencypher/morpheus/integration/yelp/Part1_YelpImport.scala +++ b/morpheus-examples/src/main/scala/org/opencypher/morpheus/integration/yelp/Part1_YelpImport.scala @@ -75,38 +75,47 @@ object Part1_YelpImport extends App { def createPropertyGraph(yelpTables: YelpTables): PropertyGraph = { // Define node tables // (:User) - val userNodeTable = MorpheusElementTable.create(NodeMappingBuilder.on(sourceIdKey) - .withImpliedLabel(userLabel) - .withPropertyKey("name") - .withPropertyKey("yelping_since") - .withPropertyKey("elite") - .build, - yelpTables.userDf.prependIdColumn(sourceIdKey, userLabel)) + val userNodeTable = MorpheusElementTable.create( + NodeMappingBuilder + .on(sourceIdKey) + .withImpliedLabel(userLabel) + .withPropertyKey("name") + .withPropertyKey("yelping_since") + .withPropertyKey("elite") + .build, + yelpTables.userDf.prependIdColumn(sourceIdKey, userLabel) + ) // (:Business) - val businessNodeTable = MorpheusElementTable.create(NodeMappingBuilder.on(sourceIdKey) - .withImpliedLabel(businessLabel) - .withPropertyKey("businessId", "business_id") - .withPropertyKey("name") - .withPropertyKey("address") - .withPropertyKey("city") - .withPropertyKey("state") - .build, - yelpTables.businessDf.prependIdColumn(sourceIdKey, businessLabel)) + val businessNodeTable = MorpheusElementTable.create( + NodeMappingBuilder + .on(sourceIdKey) + .withImpliedLabel(businessLabel) + .withPropertyKey("businessId", "business_id") + .withPropertyKey("name") + .withPropertyKey("address") + .withPropertyKey("city") + .withPropertyKey("state") + .build, + yelpTables.businessDf.prependIdColumn(sourceIdKey, businessLabel) + ) // Define relationship tables // (:User)-[:REVIEWS]->(:Business) - val reviewRelTable = MorpheusElementTable.create(RelationshipMappingBuilder.on(sourceIdKey) - .withSourceStartNodeKey(sourceStartNodeKey) - .withSourceEndNodeKey(sourceEndNodeKey) - .withRelType(reviewRelType) - .withPropertyKey("stars") - .withPropertyKey("date") - .build, + val reviewRelTable = MorpheusElementTable.create( + RelationshipMappingBuilder + .on(sourceIdKey) + .withSourceStartNodeKey(sourceStartNodeKey) + .withSourceEndNodeKey(sourceEndNodeKey) + .withRelType(reviewRelType) + .withPropertyKey("stars") + .withPropertyKey("date") + .build, yelpTables.reviewDf .prependIdColumn(sourceIdKey, reviewRelType) .prependIdColumn(sourceStartNodeKey, userLabel) - .prependIdColumn(sourceEndNodeKey, businessLabel)) + .prependIdColumn(sourceEndNodeKey, businessLabel) + ) // Create property graph morpheus.graphs.create(businessNodeTable, userNodeTable, reviewRelTable) diff --git a/morpheus-examples/src/main/scala/org/opencypher/morpheus/integration/yelp/Part2_YelpGraphLibrary.scala b/morpheus-examples/src/main/scala/org/opencypher/morpheus/integration/yelp/Part2_YelpGraphLibrary.scala index c0fa050ddc..14d69a2e6c 100644 --- a/morpheus-examples/src/main/scala/org/opencypher/morpheus/integration/yelp/Part2_YelpGraphLibrary.scala +++ b/morpheus-examples/src/main/scala/org/opencypher/morpheus/integration/yelp/Part2_YelpGraphLibrary.scala @@ -29,7 +29,11 @@ package org.opencypher.morpheus.integration.yelp import org.apache.log4j.{Level, Logger} import org.opencypher.morpheus.api.{GraphSources, MorpheusSession} import org.opencypher.morpheus.impl.MorpheusConverters._ -import org.opencypher.morpheus.integration.yelp.YelpConstants.{defaultYelpGraphFolder, yelpGraphName, _} +import org.opencypher.morpheus.integration.yelp.YelpConstants.{ + defaultYelpGraphFolder, + yelpGraphName, + _ +} object Part2_YelpGraphLibrary extends App { Logger.getRootLogger.setLevel(Level.ERROR) @@ -45,8 +49,7 @@ object Part2_YelpGraphLibrary extends App { // Construct City sub graph log("Construct city sub graph", 1) - cypher( - s""" + cypher(s""" |CATALOG CREATE GRAPH $cityGraphName { | FROM GRAPH $fsNamespace.$yelpGraphName | MATCH (business:Business)<-[r:REVIEWS]-(user1:User) @@ -58,15 +61,18 @@ object Part2_YelpGraphLibrary extends App { """.stripMargin) // Cache graph before performing multiple projections - catalog.source(catalog.sessionNamespace).graph(cityGraphName).asMorpheus.cache() + catalog + .source(catalog.sessionNamespace) + .graph(cityGraphName) + .asMorpheus + .cache() // Create multiple projections of the City graph and store them in yearly buckets log(s"Create graph projections for '$city'", 1) (2015 to 2018) foreach { year => log(s"For year $year", 2) // Compute (:User)-[:REVIEWS]->(:Business) graph - cypher( - s""" + cypher(s""" |CATALOG CREATE GRAPH $fsNamespace.${reviewGraphName(year)} { | FROM GRAPH $cityGraphName | MATCH (business:Business)<-[r:REVIEWS]-(user:User) @@ -78,8 +84,7 @@ object Part2_YelpGraphLibrary extends App { """.stripMargin) // Compute (:Business)-[:CO_REVIEWED]->(:Business) graph - cypher( - s""" + cypher(s""" |CATALOG CREATE GRAPH $fsNamespace.${coReviewedGraphName(year)} { | FROM GRAPH $fsNamespace.${reviewGraphName(year)} | MATCH (business1:Business)<-[r1:REVIEWS]-(user:User)-[r2:REVIEWS]->(business2:Business) @@ -94,8 +99,7 @@ object Part2_YelpGraphLibrary extends App { """.stripMargin) // Compute (:User)-[:CO_REVIEWS]->(:User) graph - cypher( - s""" + cypher(s""" |CATALOG CREATE GRAPH $fsNamespace.${coReviewsGraphName(year)} { | FROM GRAPH $fsNamespace.${reviewGraphName(year)} | MATCH (business:Business)<-[r1:REVIEWS]-(user1:User), @@ -111,9 +115,10 @@ object Part2_YelpGraphLibrary extends App { """.stripMargin) // Compute (:User)-[:CO_REVIEWS]->(:User), (:User)-[:REVIEWS]->(:Business) graph - cypher( - s""" - |CATALOG CREATE GRAPH $fsNamespace.${coReviewAndBusinessGraphName(year)} { + cypher(s""" + |CATALOG CREATE GRAPH $fsNamespace.${coReviewAndBusinessGraphName( + year + )} { | FROM GRAPH $fsNamespace.${reviewGraphName(year)} | MATCH (business:Business)<-[r1:REVIEWS]-(user1:User), | (business)<-[r2:REVIEWS]-(user2:User) diff --git a/morpheus-examples/src/main/scala/org/opencypher/morpheus/integration/yelp/Part3_YelpHiveIntegration.scala b/morpheus-examples/src/main/scala/org/opencypher/morpheus/integration/yelp/Part3_YelpHiveIntegration.scala index b303dc6a83..1504423827 100644 --- a/morpheus-examples/src/main/scala/org/opencypher/morpheus/integration/yelp/Part3_YelpHiveIntegration.scala +++ b/morpheus-examples/src/main/scala/org/opencypher/morpheus/integration/yelp/Part3_YelpHiveIntegration.scala @@ -53,7 +53,8 @@ object Part3_YelpHiveIntegration extends App { prepareDemoData() val h2Config = SqlDataSourceConfig.Jdbc( - url = s"jdbc:h2:mem:$yelpBookDB.db;INIT=CREATE SCHEMA IF NOT EXISTS $yelpBookDB;DB_CLOSE_DELAY=30;", + url = + s"jdbc:h2:mem:$yelpBookDB.db;INIT=CREATE SCHEMA IF NOT EXISTS $yelpBookDB;DB_CLOSE_DELAY=30;", driver = "org.h2.Driver" ) @@ -88,12 +89,14 @@ object Part3_YelpHiveIntegration extends App { // Load integrated graph using SQL Property Graph Data Source using above DDL script and two data sources val sqlPgds = GraphSources .sql(GraphDdl(graphDdl)) - .withSqlDataSourceConfigs("HIVE" -> SqlDataSourceConfig.Hive, "H2" -> h2Config) + .withSqlDataSourceConfigs( + "HIVE" -> SqlDataSourceConfig.Hive, + "H2" -> h2Config + ) registerSource(Namespace("federation"), sqlPgds) - cypher( - s""" + cypher(s""" |FROM GRAPH federation.$integratedGraphName |MATCH (user1:User)-[:REVIEWS]->(b:Business)<-[:REVIEWS]-(user2:User) |RETURN EXISTS((user1)-[:FRIEND]-(user2)), count(b) AS coReviews @@ -119,9 +122,18 @@ object Part3_YelpHiveIntegration extends App { sql(s"CREATE DATABASE $yelpDB") sql(s"USE $yelpDB") - read.json(s"$inputPath/$cityGraphName/$yelpDB/business.json").write.saveAsTable(s"$yelpDB.business") - read.json(s"$inputPath/$cityGraphName/$yelpDB/user.json").write.saveAsTable(s"$yelpDB.user") - read.json(s"$inputPath/$cityGraphName/$yelpDB/review.json").write.saveAsTable(s"$yelpDB.review") + read + .json(s"$inputPath/$cityGraphName/$yelpDB/business.json") + .write + .saveAsTable(s"$yelpDB.business") + read + .json(s"$inputPath/$cityGraphName/$yelpDB/user.json") + .write + .saveAsTable(s"$yelpDB.user") + read + .json(s"$inputPath/$cityGraphName/$yelpDB/review.json") + .write + .saveAsTable(s"$yelpDB.review") } def prepareDemoData(): Unit = if (!Paths.get(inputPath).toFile.exists()) { diff --git a/morpheus-examples/src/main/scala/org/opencypher/morpheus/integration/yelp/Part4_BusinessTrends.scala b/morpheus-examples/src/main/scala/org/opencypher/morpheus/integration/yelp/Part4_BusinessTrends.scala index 5c8852a8d0..186da82f5e 100644 --- a/morpheus-examples/src/main/scala/org/opencypher/morpheus/integration/yelp/Part4_BusinessTrends.scala +++ b/morpheus-examples/src/main/scala/org/opencypher/morpheus/integration/yelp/Part4_BusinessTrends.scala @@ -49,8 +49,7 @@ object Part4_BusinessTrends extends App { log("Write to Neo4j and compute pageRank", 1) (2017 to 2018) foreach { year => log(s"For year $year", 2) - cypher( - s""" + cypher(s""" |CATALOG CREATE GRAPH $neo4jNamespace.${coReviewedGraphName(year)} { | FROM $fsNamespace.${coReviewedGraphName(year)} | RETURN GRAPH @@ -70,8 +69,12 @@ object Part4_BusinessTrends extends App { | weightProperty: "reviewCount" |}) |YIELD nodes, loadMillis, computeMillis, writeMillis - |RETURN nodes, loadMillis + computeMillis + writeMillis AS total""".stripMargin).head - log(s"Computing page rank on ${pageRankStats("nodes")} nodes took ${pageRankStats("total")} ms", 2) + |RETURN nodes, loadMillis + computeMillis + writeMillis AS total""".stripMargin + ).head + log( + s"Computing page rank on ${pageRankStats("nodes")} nodes took ${pageRankStats("total")} ms", + 2 + ) } } @@ -80,15 +83,18 @@ object Part4_BusinessTrends extends App { // Load graphs from Neo4j into Spark and compute trend rank for each business based on their page ranks. log("Load graphs back to Spark and compute trend rank", 1) - cypher( - s""" + cypher(s""" |CATALOG CREATE GRAPH $businessTrendsGraphName { | FROM GRAPH $neo4jNamespace.${coReviewedGraphName(2017)} | MATCH (b1:Business) | FROM GRAPH $neo4jNamespace.${coReviewedGraphName(2018)} | MATCH (b2:Business) | WHERE b1.businessId = b2.businessId - | WITH b1 AS b, (b2.${pageRankProp(2018)} / ${normalizationFactor(2018)}) - (b1.${pageRankProp(2017)} / ${normalizationFactor(2017)}) AS trendRank + | WITH b1 AS b, (b2.${pageRankProp(2018)} / ${normalizationFactor( + 2018 + )}) - (b1.${pageRankProp(2017)} / ${normalizationFactor( + 2017 + )}) AS trendRank | CONSTRUCT | CREATE (newB COPY OF b) | SET newB.trendRank = trendRank @@ -97,8 +103,7 @@ object Part4_BusinessTrends extends App { """.stripMargin) // Top 10 Increasing popularity - cypher( - s""" + cypher(s""" |FROM GRAPH $businessTrendsGraphName |MATCH (b:Business) |RETURN b.name AS name, b.address AS address, b.trendRank AS trendRank @@ -106,9 +111,12 @@ object Part4_BusinessTrends extends App { |LIMIT 10 """.stripMargin).show - def normalizationFactor(year: Int): Double = neo4jConfig.cypherWithNewSession( - s""" + def normalizationFactor(year: Int): Double = neo4jConfig + .cypherWithNewSession(s""" |MATCH (b:Business) |RETURN sum(b.${pageRankProp(year)}) AS nf - """.stripMargin).head("nf").cast[CypherFloat].value + """.stripMargin) + .head("nf") + .cast[CypherFloat] + .value } diff --git a/morpheus-examples/src/main/scala/org/opencypher/morpheus/integration/yelp/Part5_BusinessRecommendations.scala b/morpheus-examples/src/main/scala/org/opencypher/morpheus/integration/yelp/Part5_BusinessRecommendations.scala index 30eb823fb9..ab51a22b73 100644 --- a/morpheus-examples/src/main/scala/org/opencypher/morpheus/integration/yelp/Part5_BusinessRecommendations.scala +++ b/morpheus-examples/src/main/scala/org/opencypher/morpheus/integration/yelp/Part5_BusinessRecommendations.scala @@ -48,10 +48,14 @@ object Part5_BusinessRecommendations extends App { val year = 2017 - log("Write to Neo4j, detect communities and find similar users within communities", 1) - cypher( - s""" - |CATALOG CREATE GRAPH $neo4jNamespace.${coReviewAndBusinessGraphName(year)} { + log( + "Write to Neo4j, detect communities and find similar users within communities", + 1 + ) + cypher(s""" + |CATALOG CREATE GRAPH $neo4jNamespace.${coReviewAndBusinessGraphName( + year + )} { | FROM $fsNamespace.${coReviewAndBusinessGraphName(year)} | RETURN GRAPH |} @@ -59,20 +63,26 @@ object Part5_BusinessRecommendations extends App { // Use Neo4j Graph Algorithms to compute Louvain clusters and Jaccard similarity within clusters neo4jConfig.withSession { implicit session => - log("Find communities via Louvain", 1) val louvainStats = neo4jCypher( s""" - |CALL algo.louvain('${coReviewAndBusinessGraphName(year).metaLabel}', 'CO_REVIEWS', { + |CALL algo.louvain('${coReviewAndBusinessGraphName( + year + ).metaLabel}', 'CO_REVIEWS', { | write: true, | weightProperty: 'reviewCount', | writeProperty: '${communityProp(year)}' |}) |YIELD communityCount, nodes, loadMillis, computeMillis, writeMillis - |RETURN communityCount, nodes, loadMillis + computeMillis + writeMillis AS total""".stripMargin).head - log(s"Computing Louvain modularity on ${louvainStats("nodes")} nodes took ${louvainStats("total")} ms", 1) + |RETURN communityCount, nodes, loadMillis + computeMillis + writeMillis AS total""".stripMargin + ).head + log( + s"Computing Louvain modularity on ${louvainStats("nodes")} nodes took ${louvainStats("total")} ms", + 1 + ) - val communityNumber = louvainStats("communityCount").cast[CypherInteger].value.toInt + val communityNumber = + louvainStats("communityCount").cast[CypherInteger].value.toInt log(s"Find similar users within $communityNumber communities", 1) // We use Jaccard similarity because it doesn't require equal length vectors @@ -114,5 +124,3 @@ object Part5_BusinessRecommendations extends App { recommendations.show } - - diff --git a/morpheus-examples/src/main/scala/org/opencypher/morpheus/integration/yelp/YelpConstants.scala b/morpheus-examples/src/main/scala/org/opencypher/morpheus/integration/yelp/YelpConstants.scala index 694fb4803c..184af0618e 100644 --- a/morpheus-examples/src/main/scala/org/opencypher/morpheus/integration/yelp/YelpConstants.scala +++ b/morpheus-examples/src/main/scala/org/opencypher/morpheus/integration/yelp/YelpConstants.scala @@ -33,7 +33,8 @@ import org.opencypher.okapi.neo4j.io.Neo4jConfig object YelpConstants { - val neo4jConfig = Neo4jConfig(new URI("bolt://localhost:7687"), "neo4j", Some("yelp")) + val neo4jConfig = + Neo4jConfig(new URI("bolt://localhost:7687"), "neo4j", Some("yelp")) val yelpGraphName = GraphName("yelp") @@ -58,9 +59,15 @@ object YelpConstants { val businessTrendsGraphName = GraphName("businessTrends") def reviewGraphName(year: Int) = GraphName(s"$cityGraphName.review.y$year") - def coReviewsGraphName(year: Int) = GraphName(s"$cityGraphName.coReviews.y$year") - def coReviewedGraphName(year: Int) = GraphName(s"$cityGraphName.coReviewed.y$year") - def coReviewAndBusinessGraphName(year: Int) = GraphName(s"$cityGraphName.coReviewsAndBusiness.y$year") + def coReviewsGraphName(year: Int) = GraphName( + s"$cityGraphName.coReviews.y$year" + ) + def coReviewedGraphName(year: Int) = GraphName( + s"$cityGraphName.coReviewed.y$year" + ) + def coReviewAndBusinessGraphName(year: Int) = GraphName( + s"$cityGraphName.coReviewsAndBusiness.y$year" + ) def pageRankProp(year: Int) = s"pageRank$year" def pageRankCoReviewProp(year: Int) = s"pageRankCoReview$year" diff --git a/morpheus-examples/src/main/scala/org/opencypher/morpheus/integration/yelp/YelpHelpers.scala b/morpheus-examples/src/main/scala/org/opencypher/morpheus/integration/yelp/YelpHelpers.scala index 703f190433..e9b1facf28 100644 --- a/morpheus-examples/src/main/scala/org/opencypher/morpheus/integration/yelp/YelpHelpers.scala +++ b/morpheus-examples/src/main/scala/org/opencypher/morpheus/integration/yelp/YelpHelpers.scala @@ -41,7 +41,9 @@ object YelpHelpers { reviewDf: DataFrame ) - def loadYelpTables(inputPath: String)(implicit spark: SparkSession): YelpTables = { + def loadYelpTables( + inputPath: String + )(implicit spark: SparkSession): YelpTables = { import spark.implicits._ log("read business.json", 2) @@ -51,13 +53,27 @@ object YelpHelpers { log("read user.json", 2) val rawUserDf = spark.read.json(s"$inputPath/user.json") - val businessDf = rawBusinessDf.select($"business_id".as(sourceIdKey), $"business_id", $"name", $"address", $"city", $"state") - val reviewDf = rawReviewDf.select($"review_id".as(sourceIdKey), $"user_id".as(sourceStartNodeKey), $"business_id".as(sourceEndNodeKey), $"stars", $"date".cast(DateType)) + val businessDf = rawBusinessDf.select( + $"business_id".as(sourceIdKey), + $"business_id", + $"name", + $"address", + $"city", + $"state" + ) + val reviewDf = rawReviewDf.select( + $"review_id".as(sourceIdKey), + $"user_id".as(sourceStartNodeKey), + $"business_id".as(sourceEndNodeKey), + $"stars", + $"date".cast(DateType) + ) val userDf = rawUserDf.select( $"user_id".as(sourceIdKey), $"name", $"yelping_since".cast(DateType), - functions.split($"elite", ",").cast(ArrayType(LongType)).as("elite")) + functions.split($"elite", ",").cast(ArrayType(LongType)).as("elite") + ) YelpTables(userDf, businessDf, reviewDf) } @@ -69,18 +85,25 @@ object YelpHelpers { import spark.implicits._ rawBusinessDf.select($"city", $"state").distinct().show() - rawBusinessDf.withColumnRenamed("business_id", "id") + rawBusinessDf + .withColumnRenamed("business_id", "id") .join(rawReviewDf, $"id" === $"business_id") .groupBy($"city", $"state") - .count().as("count") + .count() + .as("count") .orderBy($"count".desc, $"state".asc) .show(100) } - def extractYelpCitySubset(inputPath: String, outputPath: String, city: String)(implicit spark: SparkSession): Unit = { + def extractYelpCitySubset( + inputPath: String, + outputPath: String, + city: String + )(implicit spark: SparkSession): Unit = { import spark.implicits._ - def emailColumn(userId: String): Column = functions.concat($"$userId", functions.lit("@yelp.com")) + def emailColumn(userId: String): Column = + functions.concat($"$userId", functions.lit("@yelp.com")) val rawUserDf = spark.read.json(s"$inputPath/user.json") val rawReviewDf = spark.read.json(s"$inputPath/review.json") @@ -97,7 +120,10 @@ object YelpHelpers { .join(reviewDf, Seq("user_id"), "left_semi") .withColumn("email", emailColumn("user_id")) val friendDf = userDf - .select($"email".as("user1_email"), functions.explode(functions.split($"friends", ", ")).as("user2_id")) + .select( + $"email".as("user1_email"), + functions.explode(functions.split($"friends", ", ")).as("user2_id") + ) .withColumn("user2_email", emailColumn("user2_id")) .select(s"user1_email", s"user2_email") @@ -109,6 +135,8 @@ object YelpHelpers { implicit class DataFrameOps(df: DataFrame) { def prependIdColumn(idColumn: String, prefix: String): DataFrame = - df.transformColumns(idColumn)(column => functions.concat(functions.lit(prefix), column).as(idColumn)) + df.transformColumns(idColumn)(column => + functions.concat(functions.lit(prefix), column).as(idColumn) + ) } } diff --git a/morpheus-examples/src/main/scala/org/opencypher/morpheus/snippets/SqlPGDS.scala b/morpheus-examples/src/main/scala/org/opencypher/morpheus/snippets/SqlPGDS.scala index 33c8abc24c..df111dfd89 100644 --- a/morpheus-examples/src/main/scala/org/opencypher/morpheus/snippets/SqlPGDS.scala +++ b/morpheus-examples/src/main/scala/org/opencypher/morpheus/snippets/SqlPGDS.scala @@ -1,29 +1,26 @@ /** - * Copyright (c) 2016-2019 "Neo4j Sweden, AB" [https://neo4j.com] - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * Attribution Notice under the terms of the Apache License 2.0 - * - * This work was created by the collective efforts of the openCypher community. - * Without limiting the terms of Section 6, any Derivative Work that is not - * approved by the public consensus process of the openCypher Implementers Group - * should not be described as “Cypher” (and Cypher® is a registered trademark of - * Neo4j Inc.) or as "openCypher". Extensions by implementers or prototypes or - * proposals for change that have been documented or implemented should only be - * described as "implementation extensions to Cypher" or as "proposed changes to - * Cypher that are not yet approved by the openCypher community". - */ + * Copyright (c) 2016-2019 "Neo4j Sweden, AB" [https://neo4j.com] + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the + * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing permissions and + * limitations under the License. + * + * Attribution Notice under the terms of the Apache License 2.0 + * + * This work was created by the collective efforts of the openCypher community. Without limiting + * the terms of Section 6, any Derivative Work that is not approved by the public consensus process + * of the openCypher Implementers Group should not be described as “Cypher” (and Cypher® is a + * registered trademark of Neo4j Inc.) or as "openCypher". Extensions by implementers or prototypes + * or proposals for change that have been documented or implemented should only be described as + * "implementation extensions to Cypher" or as "proposed changes to Cypher that are not yet + * approved by the openCypher community". + */ package org.opencypher.morpheus.snippets import org.opencypher.morpheus.api.io.sql.SqlDataSourceConfig diff --git a/morpheus-examples/src/main/scala/org/opencypher/morpheus/util/CensusDB.scala b/morpheus-examples/src/main/scala/org/opencypher/morpheus/util/CensusDB.scala index 0e58dda9f6..9b167f083c 100644 --- a/morpheus-examples/src/main/scala/org/opencypher/morpheus/util/CensusDB.scala +++ b/morpheus-examples/src/main/scala/org/opencypher/morpheus/util/CensusDB.scala @@ -47,68 +47,80 @@ object CensusDB { val townInput = Input( table = "TOWN", csvPath = "/census/csv/town.csv", - dfSchema = toStructType(Seq( - "CITY_NAME" -> StringType, - "REGION" -> StringType)) + dfSchema = toStructType(Seq("CITY_NAME" -> StringType, "REGION" -> StringType)) ) val residentsInput = Input( table = "RESIDENTS", csvPath = "/census/csv/residents.csv", - dfSchema = toStructType(Seq( - "CITY_NAME" -> StringType, - "FIRST_NAME" -> StringType, - "LAST_NAME" -> StringType, - "PERSON_NUMBER" -> StringType, - "REGION" -> StringType)) + dfSchema = toStructType( + Seq( + "CITY_NAME" -> StringType, + "FIRST_NAME" -> StringType, + "LAST_NAME" -> StringType, + "PERSON_NUMBER" -> StringType, + "REGION" -> StringType + ) + ) ) val visitorsInput = Input( table = "VISITORS", csvPath = "/census/csv/visitors.csv", - dfSchema = toStructType(Seq( - "AGE" -> LongType, - "CITY_NAME" -> StringType, - "COUNTRY" -> StringType, - "DATE_OF_ENTRY" -> StringType, - "ENTRY_SEQUENCE" -> LongType, - "FIRST_NAME" -> StringType, - "ISO3166" -> StringType, - "LAST_NAME" -> StringType, - "PASSPORT_NUMBER" -> LongType, - "REGION" -> StringType)) + dfSchema = toStructType( + Seq( + "AGE" -> LongType, + "CITY_NAME" -> StringType, + "COUNTRY" -> StringType, + "DATE_OF_ENTRY" -> StringType, + "ENTRY_SEQUENCE" -> LongType, + "FIRST_NAME" -> StringType, + "ISO3166" -> StringType, + "LAST_NAME" -> StringType, + "PASSPORT_NUMBER" -> LongType, + "REGION" -> StringType + ) + ) ) val licensedDogsInput = Input( table = "LICENSED_DOGS", csvPath = "/census/csv/licensed_dogs.csv", - dfSchema = toStructType(Seq( - "CITY_NAME" -> StringType, - "LICENCE_DATE" -> StringType, - "LICENCE_NUMBER" -> LongType, - "NAME" -> StringType, - "PERSON_NUMBER" -> StringType, - "REGION" -> StringType)) + dfSchema = toStructType( + Seq( + "CITY_NAME" -> StringType, + "LICENCE_DATE" -> StringType, + "LICENCE_NUMBER" -> LongType, + "NAME" -> StringType, + "PERSON_NUMBER" -> StringType, + "REGION" -> StringType + ) + ) ) def toStructType(columns: Seq[(String, DataType)]): StructType = { - val structFields = columns.map { - case (columnName, dataType: DataType) => StructField(columnName, dataType, nullable = false) + val structFields = columns.map { case (columnName, dataType: DataType) => + StructField(columnName, dataType, nullable = false) } StructType(structFields.toList) } private def readResourceAsString(name: String): String = - Source.fromFile(getClass.getResource(name).toURI) + Source + .fromFile(getClass.getResource(name).toURI) .getLines() .filterNot(line => line.startsWith("#") || line.startsWith("CREATE INDEX")) .mkString(Properties.lineSeparator) val databaseName: String = "CENSUS" - val createViewsSql: String = readResourceAsString("/census/sql/census_views.sql") + val createViewsSql: String = readResourceAsString( + "/census/sql/census_views.sql" + ) - def createJdbcData(sqlDataSourceConfig: SqlDataSourceConfig.Jdbc)(implicit sparkSession: SparkSession): Unit = { + def createJdbcData( + sqlDataSourceConfig: SqlDataSourceConfig.Jdbc + )(implicit sparkSession: SparkSession): Unit = { // Populate the data populateData(townInput, sqlDataSourceConfig) @@ -123,7 +135,9 @@ object CensusDB { } } - def createHiveData(sqlDataSourceConfig: SqlDataSourceConfig)(implicit sparkSession: SparkSession): Unit = { + def createHiveData( + sqlDataSourceConfig: SqlDataSourceConfig + )(implicit sparkSession: SparkSession): Unit = { // Create the database sparkSession.sql(s"DROP DATABASE IF EXISTS CENSUS CASCADE").count @@ -133,17 +147,23 @@ object CensusDB { populateData(townInput, sqlDataSourceConfig, Some(FileFormat.parquet)) populateData(residentsInput, sqlDataSourceConfig, Some(FileFormat.parquet)) populateData(visitorsInput, sqlDataSourceConfig, Some(FileFormat.parquet)) - populateData(licensedDogsInput, sqlDataSourceConfig, Some(FileFormat.parquet)) + populateData( + licensedDogsInput, + sqlDataSourceConfig, + Some(FileFormat.parquet) + ) // Create the views createViewsSql.split(";").foreach(sparkSession.sql) } // TODO: Can we use our data sources to populate data? - private def populateData(input: Input, cfg: SqlDataSourceConfig, format: Option[StorageFormat] = None) - (implicit sparkSession: SparkSession): Unit = { - val writer = sparkSession - .read + private def populateData( + input: Input, + cfg: SqlDataSourceConfig, + format: Option[StorageFormat] = None + )(implicit sparkSession: SparkSession): Unit = { + val writer = sparkSession.read .option("header", "true") .schema(input.dfSchema) .csv(getClass.getResource(s"${input.csvPath}").getPath) diff --git a/morpheus-examples/src/main/scala/org/opencypher/morpheus/util/LdbcUtil.scala b/morpheus-examples/src/main/scala/org/opencypher/morpheus/util/LdbcUtil.scala index 58f8580376..563d3b35bf 100644 --- a/morpheus-examples/src/main/scala/org/opencypher/morpheus/util/LdbcUtil.scala +++ b/morpheus-examples/src/main/scala/org/opencypher/morpheus/util/LdbcUtil.scala @@ -50,30 +50,45 @@ object LdbcUtil { "person_workat_organisation", // list properties "person_email_emailaddress", - "person_speaks_language") + "person_speaks_language" + ) case class NodeType(nodeTable: String) { override def toString: String = s"(${nodeTable.toNodeLabel})" } - case class EdgeType(startNodeType: NodeType, edgeType: String, endNodeType: NodeType) { - def edgeTable: String = s"${startNodeType.nodeTable}_${edgeType}_${endNodeType.nodeTable}" + case class EdgeType( + startNodeType: NodeType, + edgeType: String, + endNodeType: NodeType + ) { + def edgeTable: String = + s"${startNodeType.nodeTable}_${edgeType}_${endNodeType.nodeTable}" override def toString: String = s"$startNodeType-[$edgeType]->$endNodeType" } /** - * Generates a Graph Ddl script for the LDBC graph. The extraction of nodes and relationship types and their - * corresponding table/view mappings is based on the naming conventions used by the LDBC data generator. + * Generates a Graph Ddl script for the LDBC graph. The extraction of nodes and relationship + * types and their corresponding table/view mappings is based on the naming conventions used by + * the LDBC data generator. * - * @param datasource the data source used in the SET SCHEMA definition - * @param database the Hive database to get the table information from - * @param spark a Spark instance - * @return a Graph Ddl string + * @param datasource + * the data source used in the SET SCHEMA definition + * @param database + * the Hive database to get the table information from + * @param spark + * a Spark instance + * @return + * a Graph Ddl string */ - def toGraphDDL(datasource: String, database: String)(implicit spark: SparkSession): String = { + def toGraphDDL(datasource: String, database: String)(implicit + spark: SparkSession + ): String = { // select all tables (including views) - val tableNames = spark.sql(s"SHOW TABLES FROM $database").collect() - .filterNot(_.getBoolean(2)) //ignore temp tables + val tableNames = spark + .sql(s"SHOW TABLES FROM $database") + .collect() + .filterNot(_.getBoolean(2)) // ignore temp tables .map(_.getString(1)) .filterNot(excludeTables.contains) .toSet @@ -81,15 +96,20 @@ object LdbcUtil { val nodeTables = tableNames.filterNot(_.contains("_")) val edgeTables = tableNames -- nodeTables - val elementTypes = toElementType(database, nodeTables ++ edgeTables).toSeq.sorted + val elementTypes = + toElementType(database, nodeTables ++ edgeTables).toSeq.sorted - val nodeMappings = nodeTables.map(nodeTable => s"(${nodeTable.toNodeLabel}) FROM $nodeTable") + val nodeMappings = + nodeTables.map(nodeTable => s"(${nodeTable.toNodeLabel}) FROM $nodeTable") val edgeMappings = edgeTables .map(_.split("_")) .map(tokens => EdgeType(NodeType(tokens(0)), tokens(1), NodeType(tokens(2)))) .map { - case et@EdgeType(snt@NodeType(startNodeTable), _, ent@NodeType(endNodeTable)) => - + case et @ EdgeType( + snt @ NodeType(startNodeTable), + _, + ent @ NodeType(endNodeTable) + ) => val startJoinExpr = if (startNodeTable == endNodeTable) { s"edge.$startNodeTable.id0 = node.id" } else { @@ -102,7 +122,8 @@ object LdbcUtil { s"edge.$endNodeTable.id = node.id" } - val startNodes = s"$snt FROM $startNodeTable node JOIN ON $startJoinExpr" + val startNodes = + s"$snt FROM $startNodeTable node JOIN ON $startJoinExpr" val endNodes = s"$ent FROM $endNodeTable node JOIN ON $endJoinExpr" et -> s"FROM ${et.edgeTable} edge START NODES $startNodes END NODES $endNodes" @@ -111,10 +132,15 @@ object LdbcUtil { .map { case (relType, mappings) => val fromClauses = mappings.map(_._2) s"""$relType - |${fromClauses.mkString("\t\t", Properties.lineSeparator + "\t\t", "")}""".stripMargin + |${fromClauses.mkString( + "\t\t", + Properties.lineSeparator + "\t\t", + "" + )}""".stripMargin } - s"""-- generated by ${getClass.getSimpleName.dropRight(1)} on ${Calendar.getInstance().getTime} + s"""-- generated by ${getClass.getSimpleName + .dropRight(1)} on ${Calendar.getInstance().getTime} | |SET SCHEMA $datasource.$database | @@ -122,7 +148,11 @@ object LdbcUtil { | |CREATE GRAPH $database ( | -- Node types including mappings - | ${nodeMappings.mkString("", "," + Properties.lineSeparator + "\t", ",")} + | ${nodeMappings.mkString( + "", + "," + Properties.lineSeparator + "\t", + "," + )} | | -- Edge types including mappings | ${edgeMappings.mkString("," + Properties.lineSeparator + "\t")} @@ -130,29 +160,31 @@ object LdbcUtil { """.stripMargin } - private def toElementType(database: String, tableNames: Set[String])(implicit spark: SparkSession): Set[String] = - tableNames.map { - tableName => - val tableInfos = spark.sql(s"DESCRIBE TABLE $database.$tableName").collect() - - val label = if (tableName.contains("_")) { - tableName.split("_")(1) - } else { - tableName.toNodeLabel + private def toElementType(database: String, tableNames: Set[String])(implicit + spark: SparkSession + ): Set[String] = + tableNames.map { tableName => + val tableInfos = + spark.sql(s"DESCRIBE TABLE $database.$tableName").collect() + + val label = if (tableName.contains("_")) { + tableName.split("_")(1) + } else { + tableName.toNodeLabel + } + val properties = tableInfos + .filterNot(row => row.getString(0) == "id" || row.getString(0).contains(".id")) + .map { row => + val propertyKey = row.getString(0) + val propertyType = row.getString(1).toCypherType + s"$propertyKey $propertyType" } - val properties = tableInfos - .filterNot(row => row.getString(0) == "id" || row.getString(0).contains(".id")) - .map { row => - val propertyKey = row.getString(0) - val propertyType = row.getString(1).toCypherType - s"$propertyKey $propertyType" - } - if (properties.nonEmpty) { - s"CREATE ELEMENT TYPE $label ( ${properties.mkString(", ")} )" - } else { - s"CREATE ELEMENT TYPE $label" - } + if (properties.nonEmpty) { + s"CREATE ELEMENT TYPE $label ( ${properties.mkString(", ")} )" + } else { + s"CREATE ELEMENT TYPE $label" + } } implicit class StringHelpers(val s: String) extends AnyVal { @@ -160,13 +192,13 @@ object LdbcUtil { def toNodeLabel: String = s.capitalize def toCypherType: String = s.toUpperCase match { - case "STRING" => "STRING" - case "BIGINT" => "INTEGER" - case "INT" => "INTEGER" + case "STRING" => "STRING" + case "BIGINT" => "INTEGER" + case "INT" => "INTEGER" case "BOOLEAN" => "BOOLEAN" - case "FLOAT" => "FLOAT" - case "DOUBLE" => "FLOAT" - case "DATE" => "DATE" + case "FLOAT" => "FLOAT" + case "DOUBLE" => "FLOAT" + case "DATE" => "DATE" // TODO: map correctly as soon as we support timestamp case "TIMESTAMP" => "STRING" } diff --git a/morpheus-examples/src/main/scala/org/opencypher/morpheus/util/LoadInteractionsInHive.scala b/morpheus-examples/src/main/scala/org/opencypher/morpheus/util/LoadInteractionsInHive.scala index 171161bb14..1852436518 100644 --- a/morpheus-examples/src/main/scala/org/opencypher/morpheus/util/LoadInteractionsInHive.scala +++ b/morpheus-examples/src/main/scala/org/opencypher/morpheus/util/LoadInteractionsInHive.scala @@ -35,22 +35,29 @@ object LoadInteractionsInHive { val databaseName = "customers" val baseTableName = s"$databaseName.csv_input" - def load(show: Boolean = false)(implicit session: MorpheusSession): DataFrame = { - - val datafile = getClass.getResource("/customer-interactions/csv/customer-interactions.csv").toURI.getPath - val structType = StructType(Seq( - StructField("interactionId", LongType, nullable = false), - StructField("date", StringType, nullable = false), - StructField("customerIdx", LongType, nullable = false), - StructField("empNo", LongType, nullable = false), - StructField("empName", StringType, nullable = false), - StructField("type", StringType, nullable = false), - StructField("outcomeScore", StringType, nullable = false), - StructField("accountHolderId", StringType, nullable = false), - StructField("policyAccountNumber", StringType, nullable = false), - StructField("customerId", StringType, nullable = false), - StructField("customerName", StringType, nullable = false) - )) + def load( + show: Boolean = false + )(implicit session: MorpheusSession): DataFrame = { + + val datafile = getClass + .getResource("/customer-interactions/csv/customer-interactions.csv") + .toURI + .getPath + val structType = StructType( + Seq( + StructField("interactionId", LongType, nullable = false), + StructField("date", StringType, nullable = false), + StructField("customerIdx", LongType, nullable = false), + StructField("empNo", LongType, nullable = false), + StructField("empName", StringType, nullable = false), + StructField("type", StringType, nullable = false), + StructField("outcomeScore", StringType, nullable = false), + StructField("accountHolderId", StringType, nullable = false), + StructField("policyAccountNumber", StringType, nullable = false), + StructField("customerId", StringType, nullable = false), + StructField("customerName", StringType, nullable = false) + ) + ) val baseTable: DataFrame = session.sparkSession.read .format("csv") @@ -66,35 +73,76 @@ object LoadInteractionsInHive { baseTable.write.saveAsTable(s"$baseTableName") // Create views for nodes - createView(baseTableName, "interactions", true, "interactionId", "date", "type", "outcomeScore") - createView(baseTableName, "customers", true, "customerIdx", "customerId", "customerName") + createView( + baseTableName, + "interactions", + true, + "interactionId", + "date", + "type", + "outcomeScore" + ) + createView( + baseTableName, + "customers", + true, + "customerIdx", + "customerId", + "customerName" + ) createView(baseTableName, "account_holders", true, "accountHolderId") createView(baseTableName, "policies", true, "policyAccountNumber") createView(baseTableName, "customer_reps", true, "empNo", "empName") // Create views for relationships - createView(baseTableName, "has_customer_reps", false, "interactionId", "empNo") - createView(baseTableName, "has_customers", false, "interactionId", "customerIdx") - createView(baseTableName, "has_policies", false, "interactionId", "policyAccountNumber") - createView(baseTableName, "has_account_holders", false, "interactionId", "accountHolderId") + createView( + baseTableName, + "has_customer_reps", + false, + "interactionId", + "empNo" + ) + createView( + baseTableName, + "has_customers", + false, + "interactionId", + "customerIdx" + ) + createView( + baseTableName, + "has_policies", + false, + "interactionId", + "policyAccountNumber" + ) + createView( + baseTableName, + "has_account_holders", + false, + "interactionId", + "accountHolderId" + ) baseTable } - def createView(fromTable: String, viewName: String, distinct: Boolean, columns: String*) - (implicit session: MorpheusSession): Unit = { + def createView( + fromTable: String, + viewName: String, + distinct: Boolean, + columns: String* + )(implicit session: MorpheusSession): Unit = { val distinctString = if (distinct) "DISTINCT" else "" - session.sql( - s""" + session.sql(s""" |CREATE VIEW $databaseName.${viewName}_SEED AS | SELECT $distinctString ${columns.mkString(", ")} | FROM $fromTable | WHERE date < '2017-01-01' """.stripMargin) - session.sql( - s""" + session.sql(s""" |CREATE VIEW $databaseName.${viewName}_DELTA AS | SELECT $distinctString ${columns.mkString(", ")} | FROM $fromTable @@ -102,4 +150,4 @@ object LoadInteractionsInHive { """.stripMargin) } -} \ No newline at end of file +} diff --git a/morpheus-examples/src/main/scala/org/opencypher/morpheus/util/NorthwindDB.scala b/morpheus-examples/src/main/scala/org/opencypher/morpheus/util/NorthwindDB.scala index 3ac91ea70b..89034f37c9 100644 --- a/morpheus-examples/src/main/scala/org/opencypher/morpheus/util/NorthwindDB.scala +++ b/morpheus-examples/src/main/scala/org/opencypher/morpheus/util/NorthwindDB.scala @@ -43,19 +43,26 @@ object NorthwindDB { connection.setSchema("NORTHWIND") // create the SQL db schema - connection.execute(readResourceAsString("/northwind/sql/northwind_schema.sql")) + connection.execute( + readResourceAsString("/northwind/sql/northwind_schema.sql") + ) // populate tables with data - connection.execute(readResourceAsString("/northwind/sql/northwind_data.sql")) + connection.execute( + readResourceAsString("/northwind/sql/northwind_data.sql") + ) // create views that hide problematic columns - connection.execute(readResourceAsString("/northwind/sql/northwind_views.sql")) + connection.execute( + readResourceAsString("/northwind/sql/northwind_views.sql") + ) } } private def readResourceAsString(name: String): String = - using(Source.fromFile(getClass.getResource(name).toURI))(_ - .getLines() - .filterNot(line => line.startsWith("#") || line.startsWith("CREATE INDEX")) - .mkString(Properties.lineSeparator)) + using(Source.fromFile(getClass.getResource(name).toURI))( + _.getLines() + .filterNot(line => line.startsWith("#") || line.startsWith("CREATE INDEX")) + .mkString(Properties.lineSeparator) + ) } diff --git a/morpheus-examples/src/test/scala/org/opencypher/morpheus/examples/CustomDataFrameInputExampleTest.scala b/morpheus-examples/src/test/scala/org/opencypher/morpheus/examples/CustomDataFrameInputExampleTest.scala index 4993b6e460..4b7f134a66 100644 --- a/morpheus-examples/src/test/scala/org/opencypher/morpheus/examples/CustomDataFrameInputExampleTest.scala +++ b/morpheus-examples/src/test/scala/org/opencypher/morpheus/examples/CustomDataFrameInputExampleTest.scala @@ -27,10 +27,12 @@ package org.opencypher.morpheus.examples class CustomDataFrameInputExampleTest extends ExampleTestBase { - it("should produce the correct output") { - validate( - CustomDataFrameInputExample.main(Array.empty), - getClass.getResource("/example_outputs/CustomDataFrameInputExample.out").toURI - ) - } + it("should produce the correct output") { + validate( + CustomDataFrameInputExample.main(Array.empty), + getClass + .getResource("/example_outputs/CustomDataFrameInputExample.out") + .toURI + ) + } } diff --git a/morpheus-examples/src/test/scala/org/opencypher/morpheus/examples/CypherSQLRoundtripExampleTest.scala b/morpheus-examples/src/test/scala/org/opencypher/morpheus/examples/CypherSQLRoundtripExampleTest.scala index adeb46e1a9..8bbaf22894 100644 --- a/morpheus-examples/src/test/scala/org/opencypher/morpheus/examples/CypherSQLRoundtripExampleTest.scala +++ b/morpheus-examples/src/test/scala/org/opencypher/morpheus/examples/CypherSQLRoundtripExampleTest.scala @@ -30,7 +30,9 @@ class CypherSQLRoundtripExampleTest extends ExampleTestBase { it("should produce the correct output") { validate( CypherSQLRoundtripExample.main(Array.empty), - getClass.getResource("/example_outputs/CypherSQLRoundtripExample.out").toURI + getClass + .getResource("/example_outputs/CypherSQLRoundtripExample.out") + .toURI ) } } diff --git a/morpheus-examples/src/test/scala/org/opencypher/morpheus/examples/DataFrameInputExampleTest.scala b/morpheus-examples/src/test/scala/org/opencypher/morpheus/examples/DataFrameInputExampleTest.scala index 1d7931468c..24b6561919 100644 --- a/morpheus-examples/src/test/scala/org/opencypher/morpheus/examples/DataFrameInputExampleTest.scala +++ b/morpheus-examples/src/test/scala/org/opencypher/morpheus/examples/DataFrameInputExampleTest.scala @@ -27,8 +27,10 @@ package org.opencypher.morpheus.examples class DataFrameInputExampleTest extends ExampleTestBase { - it("should produce the correct output") { - validate(DataFrameInputExample.main(Array.empty), - getClass.getResource("/example_outputs/DataFrameInputExample.out").toURI) - } + it("should produce the correct output") { + validate( + DataFrameInputExample.main(Array.empty), + getClass.getResource("/example_outputs/DataFrameInputExample.out").toURI + ) + } } diff --git a/morpheus-examples/src/test/scala/org/opencypher/morpheus/examples/DataFrameOutputExampleTest.scala b/morpheus-examples/src/test/scala/org/opencypher/morpheus/examples/DataFrameOutputExampleTest.scala index 35f6348dc6..29cd77c4c6 100644 --- a/morpheus-examples/src/test/scala/org/opencypher/morpheus/examples/DataFrameOutputExampleTest.scala +++ b/morpheus-examples/src/test/scala/org/opencypher/morpheus/examples/DataFrameOutputExampleTest.scala @@ -28,7 +28,9 @@ package org.opencypher.morpheus.examples class DataFrameOutputExampleTest extends ExampleTestBase { it("should produce the correct output") { - validate(DataFrameOutputExample.main(Array.empty), - getClass.getResource("/example_outputs/DataFrameOutputExample.out").toURI) + validate( + DataFrameOutputExample.main(Array.empty), + getClass.getResource("/example_outputs/DataFrameOutputExample.out").toURI + ) } } diff --git a/morpheus-examples/src/test/scala/org/opencypher/morpheus/examples/DataSourceExampleTest.scala b/morpheus-examples/src/test/scala/org/opencypher/morpheus/examples/DataSourceExampleTest.scala index e71b2f789f..f25d42626d 100644 --- a/morpheus-examples/src/test/scala/org/opencypher/morpheus/examples/DataSourceExampleTest.scala +++ b/morpheus-examples/src/test/scala/org/opencypher/morpheus/examples/DataSourceExampleTest.scala @@ -28,7 +28,9 @@ package org.opencypher.morpheus.examples class DataSourceExampleTest extends ExampleTestBase { it("should produce the correct output") { - validate(DataSourceExample.main(Array.empty), - getClass.getResource("/example_outputs/DataSourceExample.out").toURI) + validate( + DataSourceExample.main(Array.empty), + getClass.getResource("/example_outputs/DataSourceExample.out").toURI + ) } } diff --git a/morpheus-examples/src/test/scala/org/opencypher/morpheus/examples/ExampleTestBase.scala b/morpheus-examples/src/test/scala/org/opencypher/morpheus/examples/ExampleTestBase.scala index b5766354bd..d8bfd25185 100644 --- a/morpheus-examples/src/test/scala/org/opencypher/morpheus/examples/ExampleTestBase.scala +++ b/morpheus-examples/src/test/scala/org/opencypher/morpheus/examples/ExampleTestBase.scala @@ -49,7 +49,9 @@ abstract class ExampleTestBase extends AnyFunSpec with Matchers with BeforeAndAf val source = Source.fromFile(expectedOut) val expectedLines = source.getLines().toList val appLines = capture(app).split(System.lineSeparator()) - withClue(s"${appLines.mkString("\n")} not equal to ${expectedLines.mkString("\n")}") { + withClue( + s"${appLines.mkString("\n")} not equal to ${expectedLines.mkString("\n")}" + ) { appLines.toBag shouldEqual expectedLines.toBag } } diff --git a/morpheus-examples/src/test/scala/org/opencypher/morpheus/examples/GraphXPageRankExampleTest.scala b/morpheus-examples/src/test/scala/org/opencypher/morpheus/examples/GraphXPageRankExampleTest.scala index fe0e9d0490..ac145509a7 100644 --- a/morpheus-examples/src/test/scala/org/opencypher/morpheus/examples/GraphXPageRankExampleTest.scala +++ b/morpheus-examples/src/test/scala/org/opencypher/morpheus/examples/GraphXPageRankExampleTest.scala @@ -28,8 +28,10 @@ package org.opencypher.morpheus.examples class GraphXPageRankExampleTest extends ExampleTestBase { it("should produce the correct output") { - validate(GraphXPageRankExample.main(Array.empty), - getClass.getResource("/example_outputs/GraphXPageRankExample.out").toURI) + validate( + GraphXPageRankExample.main(Array.empty), + getClass.getResource("/example_outputs/GraphXPageRankExample.out").toURI + ) } } diff --git a/morpheus-examples/src/test/scala/org/opencypher/morpheus/examples/HiveSupportExampleTest.scala b/morpheus-examples/src/test/scala/org/opencypher/morpheus/examples/HiveSupportExampleTest.scala index 0eaf2cfebc..74caa3694e 100644 --- a/morpheus-examples/src/test/scala/org/opencypher/morpheus/examples/HiveSupportExampleTest.scala +++ b/morpheus-examples/src/test/scala/org/opencypher/morpheus/examples/HiveSupportExampleTest.scala @@ -29,8 +29,10 @@ package org.opencypher.morpheus.examples class HiveSupportExampleTest extends ExampleTestBase { it("should produce the correct output") { - validate(HiveSupportExample.main(Array.empty), - getClass.getResource("/example_outputs/HiveSupportExample.out").toURI) + validate( + HiveSupportExample.main(Array.empty), + getClass.getResource("/example_outputs/HiveSupportExample.out").toURI + ) } } diff --git a/morpheus-examples/src/test/scala/org/opencypher/morpheus/examples/MetaTest.scala b/morpheus-examples/src/test/scala/org/opencypher/morpheus/examples/MetaTest.scala index 6f7c40d910..f40d8f6367 100644 --- a/morpheus-examples/src/test/scala/org/opencypher/morpheus/examples/MetaTest.scala +++ b/morpheus-examples/src/test/scala/org/opencypher/morpheus/examples/MetaTest.scala @@ -41,39 +41,60 @@ class MetaTest extends BaseTestSuite { val readmeName = "README.md" val moduleName = "morpheus-examples" - private val rootFolderPath = findRootFolderPath(Paths.get(".").toAbsolutePath.normalize.toString) - private val examplePath = DataFrameInputExample.getClass.getName.dropRight(1).replace(".", File.separator) - private val exampleClassName = examplePath.substring(examplePath.lastIndexOf(File.separator) + 1) + ".scala" - private val examplePackagePath = examplePath.substring(0, examplePath.lastIndexOf(File.separator)) + private val rootFolderPath = findRootFolderPath( + Paths.get(".").toAbsolutePath.normalize.toString + ) + private val examplePath = DataFrameInputExample.getClass.getName + .dropRight(1) + .replace(".", File.separator) + private val exampleClassName = examplePath.substring( + examplePath.lastIndexOf(File.separator) + 1 + ) + ".scala" + private val examplePackagePath = + examplePath.substring(0, examplePath.lastIndexOf(File.separator)) private val readmePath = Paths.get(rootFolderPath, readmeName).toString private def absolutePackagePath(scope: String) = - Paths.get(rootFolderPath, moduleName, Paths.get("src", scope, "scala").toString, examplePackagePath).toString + Paths + .get( + rootFolderPath, + moduleName, + Paths.get("src", scope, "scala").toString, + examplePackagePath + ) + .toString private val dataFrameInputExamplePath = Paths.get(absolutePackagePath("main"), exampleClassName).toString it("should exist a test for each Morpheus example") { - val exampleClassPrefixes = new File(absolutePackagePath("main")).listFiles() + val exampleClassPrefixes = new File(absolutePackagePath("main")) + .listFiles() .map(_.getName) .filter(_.contains("Example")) .map(example => example.substring(0, example.indexOf("Example"))) - val exampleTestClassPrefixes = new File(absolutePackagePath("test")).listFiles() + val exampleTestClassPrefixes = new File(absolutePackagePath("test")) + .listFiles() .map(_.getName) .filter(_.contains("Example")) - exampleClassPrefixes.forall(prefix => exampleTestClassPrefixes.exists(_.startsWith(prefix))) shouldBe true + exampleClassPrefixes.forall(prefix => + exampleTestClassPrefixes.exists(_.startsWith(prefix)) + ) shouldBe true } /** - * Tests whether the README example is aligned with the code contained in [[DataFrameInputExample]]. + * Tests whether the README example is aligned with the code contained in + * [[DataFrameInputExample]]. */ it("the code in the readme matches the example") { val readmeLines = Source.fromFile(readmePath).getLines.toVector - val readmeSourceCodeBlocks = extractMarkdownScalaSourceBlocks(readmeLines).map(_.canonical).toSet + val readmeSourceCodeBlocks = + extractMarkdownScalaSourceBlocks(readmeLines).map(_.canonical).toSet - val exampleSourceCodeLines = Source.fromFile(dataFrameInputExamplePath).getLines.toVector + val exampleSourceCodeLines = + Source.fromFile(dataFrameInputExamplePath).getLines.toVector val exampleSourceCode = ScalaSourceCode(exampleSourceCodeLines).canonical readmeSourceCodeBlocks should contain(exampleSourceCode) @@ -81,17 +102,18 @@ class MetaTest extends BaseTestSuite { case class ScalaSourceCode(lines: Vector[String]) { def canonical: Vector[String] = lines - .dropWhile(line => !line.startsWith("import")) // Drop license and everything else before the first import - .filterNot(_.contains("// tag::")).filterNot(_.contains("// end::")) // Filter documentation tags + .dropWhile(line => + !line.startsWith("import") + ) // Drop license and everything else before the first import + .filterNot(_.contains("// tag::")) + .filterNot(_.contains("// end::")) // Filter documentation tags .filterNot(_.contains("import App")) // Filter custom App import .filterNot(_ == "") // Filter empty lines override def toString: String = lines.mkString("\n") } - /** - * Find the root folder path even if the tests are executed in a child path. - */ + /** Find the root folder path even if the tests are executed in a child path. */ def findRootFolderPath(potentialChildFolderPath: String): String = { @tailrec def recFindRootFolderPath(folder: String): String = { if (isRootFolderPath(folder)) { @@ -103,18 +125,25 @@ class MetaTest extends BaseTestSuite { Try(recFindRootFolderPath(potentialChildFolderPath)).getOrElse( throw new PathNotFoundException( - s"Directory $potentialChildFolderPath is not a sub-folder of the project root directory.")) + s"Directory $potentialChildFolderPath is not a sub-folder of the project root directory." + ) + ) } /** - * Check by testing if the CONTRIBUTING.adoc file can be found. This works even if the root folder has a different name. + * Check by testing if the CONTRIBUTING.adoc file can be found. This works even if the root + * folder has a different name. */ - def isRootFolderPath(path: String): Boolean = Paths.get(path, "CONTRIBUTING.adoc").toFile.exists - - def extractMarkdownScalaSourceBlocks(lines: Vector[String]): Seq[ScalaSourceCode] = { - val currentParsingState: (Vector[ScalaSourceCode], Option[Vector[String]]) = (Vector.empty, None) - val sourceCodeSnippets = lines.foldLeft(currentParsingState) { - case ((sourceBlocks, currentBlock), currentLine) => + def isRootFolderPath(path: String): Boolean = + Paths.get(path, "CONTRIBUTING.adoc").toFile.exists + + def extractMarkdownScalaSourceBlocks( + lines: Vector[String] + ): Seq[ScalaSourceCode] = { + val currentParsingState: (Vector[ScalaSourceCode], Option[Vector[String]]) = + (Vector.empty, None) + val sourceCodeSnippets = lines + .foldLeft(currentParsingState) { case ((sourceBlocks, currentBlock), currentLine) => currentBlock match { case Some(block) => if (currentLine == "```") { @@ -129,7 +158,8 @@ class MetaTest extends BaseTestSuite { (sourceBlocks, None) } } - }._1 + } + ._1 sourceCodeSnippets } } diff --git a/morpheus-examples/src/test/scala/org/opencypher/morpheus/examples/MultipleGraphExampleTest.scala b/morpheus-examples/src/test/scala/org/opencypher/morpheus/examples/MultipleGraphExampleTest.scala index b54624b57b..fd2d946538 100644 --- a/morpheus-examples/src/test/scala/org/opencypher/morpheus/examples/MultipleGraphExampleTest.scala +++ b/morpheus-examples/src/test/scala/org/opencypher/morpheus/examples/MultipleGraphExampleTest.scala @@ -29,7 +29,9 @@ package org.opencypher.morpheus.examples class MultipleGraphExampleTest extends ExampleTestBase { it("should produce the correct output") { - validate(MultipleGraphExample.main(Array.empty), - getClass.getResource("/example_outputs/MultipleGraphExample.out").toURI) + validate( + MultipleGraphExample.main(Array.empty), + getClass.getResource("/example_outputs/MultipleGraphExample.out").toURI + ) } } diff --git a/morpheus-examples/src/test/scala/org/opencypher/morpheus/examples/Neo4jCustomSchemaExampleTest.scala b/morpheus-examples/src/test/scala/org/opencypher/morpheus/examples/Neo4jCustomSchemaExampleTest.scala index 268b434ec4..ab6f62929f 100644 --- a/morpheus-examples/src/test/scala/org/opencypher/morpheus/examples/Neo4jCustomSchemaExampleTest.scala +++ b/morpheus-examples/src/test/scala/org/opencypher/morpheus/examples/Neo4jCustomSchemaExampleTest.scala @@ -33,7 +33,11 @@ class Neo4jCustomSchemaExampleTest extends ExampleTestBase with Neo4jServerFixtu override def dataFixture: String = "" it("should produce the correct output") { - validate(Neo4jCustomSchemaExample.main(Array("--bolt-url", boltUrl)), - getClass.getResource("/example_outputs/Neo4jCustomSchemaExample.out").toURI) + validate( + Neo4jCustomSchemaExample.main(Array("--bolt-url", boltUrl)), + getClass + .getResource("/example_outputs/Neo4jCustomSchemaExample.out") + .toURI + ) } } diff --git a/morpheus-examples/src/test/scala/org/opencypher/morpheus/examples/Neo4jMergeExampleTest.scala b/morpheus-examples/src/test/scala/org/opencypher/morpheus/examples/Neo4jMergeExampleTest.scala index ea343bf819..387f60ccff 100644 --- a/morpheus-examples/src/test/scala/org/opencypher/morpheus/examples/Neo4jMergeExampleTest.scala +++ b/morpheus-examples/src/test/scala/org/opencypher/morpheus/examples/Neo4jMergeExampleTest.scala @@ -33,7 +33,9 @@ class Neo4jMergeExampleTest extends ExampleTestBase with Neo4jServerFixture { override def dataFixture: String = "" it("should produce the correct output") { - validate(Neo4jMergeExample.main(Array("--bolt-url", boltUrl)), - getClass.getResource("/example_outputs/Neo4jMergeExample.out").toURI) + validate( + Neo4jMergeExample.main(Array("--bolt-url", boltUrl)), + getClass.getResource("/example_outputs/Neo4jMergeExample.out").toURI + ) } } diff --git a/morpheus-examples/src/test/scala/org/opencypher/morpheus/examples/Neo4jReadWriteExampleTest.scala b/morpheus-examples/src/test/scala/org/opencypher/morpheus/examples/Neo4jReadWriteExampleTest.scala index df40dbaf9b..050181967d 100644 --- a/morpheus-examples/src/test/scala/org/opencypher/morpheus/examples/Neo4jReadWriteExampleTest.scala +++ b/morpheus-examples/src/test/scala/org/opencypher/morpheus/examples/Neo4jReadWriteExampleTest.scala @@ -33,8 +33,10 @@ class Neo4jReadWriteExampleTest extends ExampleTestBase with Neo4jServerFixture override def dataFixture: String = "" it("should produce the correct output") { - validate(Neo4jReadWriteExample.main(Array("--bolt-url", boltUrl)), - getClass.getResource("/example_outputs/Neo4jReadWriteExample.out").toURI) + validate( + Neo4jReadWriteExample.main(Array("--bolt-url", boltUrl)), + getClass.getResource("/example_outputs/Neo4jReadWriteExample.out").toURI + ) } } diff --git a/morpheus-examples/src/test/scala/org/opencypher/morpheus/examples/Neo4jWorkflowExampleTest.scala b/morpheus-examples/src/test/scala/org/opencypher/morpheus/examples/Neo4jWorkflowExampleTest.scala index bc1df782cf..e387eee52c 100644 --- a/morpheus-examples/src/test/scala/org/opencypher/morpheus/examples/Neo4jWorkflowExampleTest.scala +++ b/morpheus-examples/src/test/scala/org/opencypher/morpheus/examples/Neo4jWorkflowExampleTest.scala @@ -33,7 +33,9 @@ class Neo4jWorkflowExampleTest extends ExampleTestBase with Neo4jServerFixture { override def dataFixture: String = "" it("should produce the correct output") { - validate(Neo4jWorkflowExample.main(Array("--bolt-url", boltUrl)), - getClass.getResource("/example_outputs/Neo4jWorkflowExample.out").toURI) + validate( + Neo4jWorkflowExample.main(Array("--bolt-url", boltUrl)), + getClass.getResource("/example_outputs/Neo4jWorkflowExample.out").toURI + ) } } diff --git a/morpheus-examples/src/test/scala/org/opencypher/morpheus/examples/NorthwindJdbcExampleTest.scala b/morpheus-examples/src/test/scala/org/opencypher/morpheus/examples/NorthwindJdbcExampleTest.scala index 60c682521d..1bf38fefbd 100644 --- a/morpheus-examples/src/test/scala/org/opencypher/morpheus/examples/NorthwindJdbcExampleTest.scala +++ b/morpheus-examples/src/test/scala/org/opencypher/morpheus/examples/NorthwindJdbcExampleTest.scala @@ -55,8 +55,7 @@ class NorthwindJdbcExampleTest extends ExampleTestBase { val sqlResult = H2Utils.withConnection(dataSourceConfig) { conn => val stmt = conn.createStatement() var resultTuples = List.empty[Seq[CypherValue]] - val result = stmt.executeQuery( - """ + val result = stmt.executeQuery(""" |SELECT | ord.CustomerID AS customer, | ord.OrderDate AS orderedAt, @@ -85,12 +84,17 @@ class NorthwindJdbcExampleTest extends ExampleTestBase { ) resultTuples = resultTuples :+ tuple } - toTable(Seq("customer", "orderedAt", "handledBy", "employee"), resultTuples.map(row => row.map(_.toCypherString()))).dropRight(1) + toTable( + Seq("customer", "orderedAt", "handledBy", "employee"), + resultTuples.map(row => row.map(_.toCypherString())) + ).dropRight(1) } // We only need the last query result from the expected example output val cypherResult = Source - .fromFile(getClass.getResource("/example_outputs/NorthwindJdbcExample.out").toURI) + .fromFile( + getClass.getResource("/example_outputs/NorthwindJdbcExample.out").toURI + ) .getLines() .toList .takeRight(55) diff --git a/morpheus-examples/src/test/scala/org/opencypher/morpheus/examples/RecommendationExampleTest.scala b/morpheus-examples/src/test/scala/org/opencypher/morpheus/examples/RecommendationExampleTest.scala index e53e3af085..078a28e1f9 100644 --- a/morpheus-examples/src/test/scala/org/opencypher/morpheus/examples/RecommendationExampleTest.scala +++ b/morpheus-examples/src/test/scala/org/opencypher/morpheus/examples/RecommendationExampleTest.scala @@ -29,8 +29,10 @@ package org.opencypher.morpheus.examples class RecommendationExampleTest extends ExampleTestBase { // TODO: enable when spark planning bug is fixed ignore("should produce the correct output") { - validate(RecommendationExample.main(Array.empty), - getClass.getResource("/example_outputs/RecommendationExample.out").toURI) + validate( + RecommendationExample.main(Array.empty), + getClass.getResource("/example_outputs/RecommendationExample.out").toURI + ) } } diff --git a/morpheus-examples/src/test/scala/org/opencypher/morpheus/examples/UpdateExampleTest.scala b/morpheus-examples/src/test/scala/org/opencypher/morpheus/examples/UpdateExampleTest.scala index 3ae43260b0..d23ff6aae1 100644 --- a/morpheus-examples/src/test/scala/org/opencypher/morpheus/examples/UpdateExampleTest.scala +++ b/morpheus-examples/src/test/scala/org/opencypher/morpheus/examples/UpdateExampleTest.scala @@ -28,7 +28,9 @@ package org.opencypher.morpheus.examples class UpdateExampleTest extends ExampleTestBase { it("should produce the correct output") { - validate(UpdateExample.main(Array.empty), - getClass.getResource("/example_outputs/UpdateExample.out").toURI) + validate( + UpdateExample.main(Array.empty), + getClass.getResource("/example_outputs/UpdateExample.out").toURI + ) } } diff --git a/morpheus-examples/src/test/scala/org/opencypher/morpheus/examples/ViewsExampleTest.scala b/morpheus-examples/src/test/scala/org/opencypher/morpheus/examples/ViewsExampleTest.scala index 42fd2df5d0..00c55e6e1c 100644 --- a/morpheus-examples/src/test/scala/org/opencypher/morpheus/examples/ViewsExampleTest.scala +++ b/morpheus-examples/src/test/scala/org/opencypher/morpheus/examples/ViewsExampleTest.scala @@ -28,7 +28,9 @@ package org.opencypher.morpheus.examples class ViewsExampleTest extends ExampleTestBase { it("should produce the correct output") { - validate(ViewsExample.main(Array.empty), - getClass.getResource("/example_outputs/ViewsExample.out").toURI) + validate( + ViewsExample.main(Array.empty), + getClass.getResource("/example_outputs/ViewsExample.out").toURI + ) } } diff --git a/morpheus-examples/src/test/scala/org/opencypher/morpheus/snippets/SqlPGDSTest.scala b/morpheus-examples/src/test/scala/org/opencypher/morpheus/snippets/SqlPGDSTest.scala index 158cf35c16..a5f4b77fdc 100644 --- a/morpheus-examples/src/test/scala/org/opencypher/morpheus/snippets/SqlPGDSTest.scala +++ b/morpheus-examples/src/test/scala/org/opencypher/morpheus/snippets/SqlPGDSTest.scala @@ -1,34 +1,31 @@ /** - * Copyright (c) 2016-2019 "Neo4j Sweden, AB" [https://neo4j.com] - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * Attribution Notice under the terms of the Apache License 2.0 - * - * This work was created by the collective efforts of the openCypher community. - * Without limiting the terms of Section 6, any Derivative Work that is not - * approved by the public consensus process of the openCypher Implementers Group - * should not be described as “Cypher” (and Cypher® is a registered trademark of - * Neo4j Inc.) or as "openCypher". Extensions by implementers or prototypes or - * proposals for change that have been documented or implemented should only be - * described as "implementation extensions to Cypher" or as "proposed changes to - * Cypher that are not yet approved by the openCypher community". - */ + * Copyright (c) 2016-2019 "Neo4j Sweden, AB" [https://neo4j.com] + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the + * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing permissions and + * limitations under the License. + * + * Attribution Notice under the terms of the Apache License 2.0 + * + * This work was created by the collective efforts of the openCypher community. Without limiting + * the terms of Section 6, any Derivative Work that is not approved by the public consensus process + * of the openCypher Implementers Group should not be described as “Cypher” (and Cypher® is a + * registered trademark of Neo4j Inc.) or as "openCypher". Extensions by implementers or prototypes + * or proposals for change that have been documented or implemented should only be described as + * "implementation extensions to Cypher" or as "proposed changes to Cypher that are not yet + * approved by the openCypher community". + */ package org.opencypher.morpheus.snippets import org.opencypher.morpheus.examples.ExampleTestBase -class SqlPGDSTest extends ExampleTestBase { +class SqlPGDSTest extends ExampleTestBase { it("should produce the correct output") { validate( diff --git a/morpheus-jmh/src/jmh/scala/org/opencypher/morpheus/jmh/ConcatColumnBenchmark.scala b/morpheus-jmh/src/jmh/scala/org/opencypher/morpheus/jmh/ConcatColumnBenchmark.scala index 1333097054..968155fae6 100644 --- a/morpheus-jmh/src/jmh/scala/org/opencypher/morpheus/jmh/ConcatColumnBenchmark.scala +++ b/morpheus-jmh/src/jmh/scala/org/opencypher/morpheus/jmh/ConcatColumnBenchmark.scala @@ -56,13 +56,19 @@ class ConcatColumnBenchmark { @Benchmark def concatWs(): Int = { - val result = df.withColumn("c", functions.concat_ws("|", df.col("i"), df.col("s"), df.col("b"))) + val result = df.withColumn( + "c", + functions.concat_ws("|", df.col("i"), df.col("s"), df.col("b")) + ) result.select("c").collect().length } @Benchmark def serialize(): Int = { - val result = df.withColumn("c", MorpheusFunctions.serialize(df.col("i"), df.col("s"), df.col("b"))) + val result = df.withColumn( + "c", + MorpheusFunctions.serialize(df.col("i"), df.col("s"), df.col("b")) + ) result.select("c").collect().length } diff --git a/morpheus-jmh/src/jmh/scala/org/opencypher/morpheus/jmh/JoinBenchmark.scala b/morpheus-jmh/src/jmh/scala/org/opencypher/morpheus/jmh/JoinBenchmark.scala index 1d5787ad7b..7bc4be738a 100644 --- a/morpheus-jmh/src/jmh/scala/org/opencypher/morpheus/jmh/JoinBenchmark.scala +++ b/morpheus-jmh/src/jmh/scala/org/opencypher/morpheus/jmh/JoinBenchmark.scala @@ -73,7 +73,7 @@ class JoinBenchmark { leftData = List.fill(leftRandomCount)(Random.nextLong()) ++ joinRange rightData = List.fill(rightRandomCount)(Random.nextLong()) ++ joinRange - def prepareDf[A : TypeTag](data: List[A]): DataFrame = { + def prepareDf[A: TypeTag](data: List[A]): DataFrame = { sparkSession.createDataFrame(data.map(Tuple1(_))).toDF(idColumn) } @@ -89,47 +89,55 @@ class JoinBenchmark { leftByteArray = prepareDf(leftData.map(longToByteArray)).partitionAndCache rightByteArray = prepareDf(rightData.map(longToByteArray)).partitionAndCache - leftEfficientString = prepareDf(leftData.map(longToByteArray)).castIdToString.partitionAndCache - rightEfficientString = prepareDf(rightData.map(longToByteArray)).castIdToString.partitionAndCache + leftEfficientString = prepareDf( + leftData.map(longToByteArray) + ).castIdToString.partitionAndCache + rightEfficientString = prepareDf( + rightData.map(longToByteArray) + ).castIdToString.partitionAndCache } @Benchmark def joinLongIds(): Long = leftLong.join(rightLong, idColumn).count() @Benchmark - def joinArrayLongIds(): Long = leftArrayLong.join(rightArrayLong, idColumn).count() + def joinArrayLongIds(): Long = + leftArrayLong.join(rightArrayLong, idColumn).count() @Benchmark - def joinNaiveStringIds(): Long = leftNaiveString.join(rightNaiveString, idColumn).count() + def joinNaiveStringIds(): Long = + leftNaiveString.join(rightNaiveString, idColumn).count() @Benchmark - def joinByteArrayIds(): Long = leftByteArray.join(rightByteArray, idColumn).count() + def joinByteArrayIds(): Long = + leftByteArray.join(rightByteArray, idColumn).count() @Benchmark - def joinEfficientStringIds(): Long = leftEfficientString.join(rightEfficientString, idColumn).count() - + def joinEfficientStringIds(): Long = + leftEfficientString.join(rightEfficientString, idColumn).count() implicit class DataFrameSetup(df: DataFrame) { - def partitionAndCache : DataFrame = { + def partitionAndCache: DataFrame = { val cached = df.repartition(10).persist(StorageLevel.MEMORY_ONLY) cached.count() cached } - def castIdToString: DataFrame = df.select(df.col(idColumn).cast(StringType).as(idColumn)) + def castIdToString: DataFrame = + df.select(df.col(idColumn).cast(StringType).as(idColumn)) } private def longToByteArray(l: Long): Array[Byte] = { val a = new Array[Byte](8) - a(0) = (l & 0xFF).asInstanceOf[Byte] - a(1) = ((l >> 8) & 0xFF).asInstanceOf[Byte] - a(2) = ((l >> 16) & 0xFF).asInstanceOf[Byte] - a(3) = ((l >> 24) & 0xFF).asInstanceOf[Byte] - a(4) = ((l >> 32) & 0xFF).asInstanceOf[Byte] - a(5) = ((l >> 40) & 0xFF).asInstanceOf[Byte] - a(6) = ((l >> 48) & 0xFF).asInstanceOf[Byte] - a(7) = ((l >> 56) & 0xFF).asInstanceOf[Byte] + a(0) = (l & 0xff).asInstanceOf[Byte] + a(1) = ((l >> 8) & 0xff).asInstanceOf[Byte] + a(2) = ((l >> 16) & 0xff).asInstanceOf[Byte] + a(3) = ((l >> 24) & 0xff).asInstanceOf[Byte] + a(4) = ((l >> 32) & 0xff).asInstanceOf[Byte] + a(5) = ((l >> 40) & 0xff).asInstanceOf[Byte] + a(6) = ((l >> 48) & 0xff).asInstanceOf[Byte] + a(7) = ((l >> 56) & 0xff).asInstanceOf[Byte] a } } diff --git a/morpheus-spark-cypher/src/main/scala/org/opencypher/morpheus/api/GraphSources.scala b/morpheus-spark-cypher/src/main/scala/org/opencypher/morpheus/api/GraphSources.scala index 341c051044..21dfcdea4b 100644 --- a/morpheus-spark-cypher/src/main/scala/org/opencypher/morpheus/api/GraphSources.scala +++ b/morpheus-spark-cypher/src/main/scala/org/opencypher/morpheus/api/GraphSources.scala @@ -46,13 +46,16 @@ object GraphSources { rootPath: String, hiveDatabaseName: Option[String] = None, filesPerTable: Option[Int] = Some(1) - )(implicit session: MorpheusSession) = FSGraphSources(rootPath, hiveDatabaseName, filesPerTable) + )(implicit session: MorpheusSession) = + FSGraphSources(rootPath, hiveDatabaseName, filesPerTable) def cypher: CypherGraphSources.type = CypherGraphSources - def sql(graphDdlPath: String)(implicit session: MorpheusSession) = SqlGraphSources(graphDdlPath) + def sql(graphDdlPath: String)(implicit session: MorpheusSession) = + SqlGraphSources(graphDdlPath) - def sql(graphDdl: GraphDdl)(implicit session: MorpheusSession) = SqlGraphSources(graphDdl) + def sql(graphDdl: GraphDdl)(implicit session: MorpheusSession) = + SqlGraphSources(graphDdl) } object FSGraphSources { @@ -60,7 +63,8 @@ object FSGraphSources { rootPath: String, hiveDatabaseName: Option[String] = None, filesPerTable: Option[Int] = Some(1) - )(implicit session: MorpheusSession): FSGraphSourceFactory = FSGraphSourceFactory(rootPath, hiveDatabaseName, filesPerTable) + )(implicit session: MorpheusSession): FSGraphSourceFactory = + FSGraphSourceFactory(rootPath, hiveDatabaseName, filesPerTable) case class FSGraphSourceFactory( rootPath: String, @@ -70,46 +74,84 @@ object FSGraphSources { def csv: FSGraphSource = new CsvGraphSource(rootPath, filesPerTable) - def parquet: FSGraphSource = new FSGraphSource(rootPath, FileFormat.parquet, hiveDatabaseName, filesPerTable) - - def orc: FSGraphSource = new FSGraphSource(rootPath, FileFormat.orc, hiveDatabaseName, filesPerTable) with EscapeAtSymbol + def parquet: FSGraphSource = new FSGraphSource( + rootPath, + FileFormat.parquet, + hiveDatabaseName, + filesPerTable + ) + + def orc: FSGraphSource = new FSGraphSource( + rootPath, + FileFormat.orc, + hiveDatabaseName, + filesPerTable + ) with EscapeAtSymbol } /** - * Creates a data sink that is capable of writing a property graph into the Neo4j bulk import CSV format - * (see [[https://neo4j.com/docs/operations-manual/current/tools/import/]]). The data sink generates a shell script - * within the graph output folder that simplifies the import process. + * Creates a data sink that is capable of writing a property graph into the Neo4j bulk import CSV + * format (see [[https://neo4j.com/docs/operations-manual/current/tools/import/]]). The data sink + * generates a shell script within the graph output folder that simplifies the import process. * - * @param rootPath Directory where the graph is being stored in - * @param arrayDelimiter Delimiter for array properties - * @param morpheus Morpheus session - * @return Neo4j Bulk CSV data sink + * @param rootPath + * Directory where the graph is being stored in + * @param arrayDelimiter + * Delimiter for array properties + * @param morpheus + * Morpheus session + * @return + * Neo4j Bulk CSV data sink */ - def neo4jBulk(rootPath: String, arrayDelimiter: String = "|")(implicit morpheus: MorpheusSession): Neo4jBulkCSVDataSink = { + def neo4jBulk(rootPath: String, arrayDelimiter: String = "|")(implicit + morpheus: MorpheusSession + ): Neo4jBulkCSVDataSink = { new Neo4jBulkCSVDataSink(rootPath, arrayDelimiter) } } object CypherGraphSources { + /** * Creates a Neo4j Property Graph Data Source * - * @param config Neo4j connection configuration - * @param maybeSchema Optional Neo4j schema to avoid computation on Neo4j server - * @param omitIncompatibleProperties If set to true, import failures do not throw runtime exceptions but omit the unsupported - * properties instead and log warnings - * @param morpheus Morpheus session - * @return Neo4j Property Graph Data Source + * @param config + * Neo4j connection configuration + * @param maybeSchema + * Optional Neo4j schema to avoid computation on Neo4j server + * @param omitIncompatibleProperties + * If set to true, import failures do not throw runtime exceptions but omit the unsupported + * properties instead and log warnings + * @param morpheus + * Morpheus session + * @return + * Neo4j Property Graph Data Source */ - def neo4j(config: Neo4jConfig, maybeSchema: Option[PropertyGraphSchema] = None, omitIncompatibleProperties: Boolean = false) - (implicit morpheus: MorpheusSession): Neo4jPropertyGraphDataSource = - Neo4jPropertyGraphDataSource(config, maybeSchema = maybeSchema, omitIncompatibleProperties = omitIncompatibleProperties) + def neo4j( + config: Neo4jConfig, + maybeSchema: Option[PropertyGraphSchema] = None, + omitIncompatibleProperties: Boolean = false + )(implicit morpheus: MorpheusSession): Neo4jPropertyGraphDataSource = + Neo4jPropertyGraphDataSource( + config, + maybeSchema = maybeSchema, + omitIncompatibleProperties = omitIncompatibleProperties + ) // TODO: document - def neo4j(config: Neo4jConfig, schemaFile: String, omitIncompatibleProperties: Boolean) - (implicit morpheus: MorpheusSession): Neo4jPropertyGraphDataSource = { - val schemaString = using(Source.fromFile(Paths.get(schemaFile).toUri))(_.getLines().mkString(Properties.lineSeparator)) - Neo4jPropertyGraphDataSource(config, maybeSchema = Some(PropertyGraphSchema.fromJson(schemaString)), omitIncompatibleProperties = omitIncompatibleProperties) + def neo4j( + config: Neo4jConfig, + schemaFile: String, + omitIncompatibleProperties: Boolean + )(implicit morpheus: MorpheusSession): Neo4jPropertyGraphDataSource = { + val schemaString = using(Source.fromFile(Paths.get(schemaFile).toUri))( + _.getLines().mkString(Properties.lineSeparator) + ) + Neo4jPropertyGraphDataSource( + config, + maybeSchema = Some(PropertyGraphSchema.fromJson(schemaString)), + omitIncompatibleProperties = omitIncompatibleProperties + ) } } @@ -117,30 +159,54 @@ import org.opencypher.morpheus.api.io.sql.IdGenerationStrategy._ object SqlGraphSources { - case class SqlGraphSourceFactory(graphDdl: GraphDdl, idGenerationStrategy: IdGenerationStrategy) - (implicit morpheus: MorpheusSession) { + case class SqlGraphSourceFactory( + graphDdl: GraphDdl, + idGenerationStrategy: IdGenerationStrategy + )(implicit morpheus: MorpheusSession) { - def withIdGenerationStrategy(idGenerationStrategy: IdGenerationStrategy): SqlGraphSourceFactory = + def withIdGenerationStrategy( + idGenerationStrategy: IdGenerationStrategy + ): SqlGraphSourceFactory = copy(idGenerationStrategy = idGenerationStrategy) - def withSqlDataSourceConfigs(sqlDataSourceConfigsPath: String): SqlPropertyGraphDataSource = { - val jsonString = using(Source.fromFile(sqlDataSourceConfigsPath, "UTF-8"))(_.getLines().mkString(Properties.lineSeparator)) - val sqlDataSourceConfigs = SqlDataSourceConfig.dataSourcesFromString(jsonString) + def withSqlDataSourceConfigs( + sqlDataSourceConfigsPath: String + ): SqlPropertyGraphDataSource = { + val jsonString = using( + Source.fromFile(sqlDataSourceConfigsPath, "UTF-8") + )(_.getLines().mkString(Properties.lineSeparator)) + val sqlDataSourceConfigs = + SqlDataSourceConfig.dataSourcesFromString(jsonString) withSqlDataSourceConfigs(sqlDataSourceConfigs) } - def withSqlDataSourceConfigs(sqlDataSourceConfigs: (String, SqlDataSourceConfig)*): SqlPropertyGraphDataSource = + def withSqlDataSourceConfigs( + sqlDataSourceConfigs: (String, SqlDataSourceConfig)* + ): SqlPropertyGraphDataSource = withSqlDataSourceConfigs(sqlDataSourceConfigs.toMap) - def withSqlDataSourceConfigs(sqlDataSourceConfigs: Map[String, SqlDataSourceConfig]): SqlPropertyGraphDataSource = - SqlPropertyGraphDataSource(graphDdl, sqlDataSourceConfigs, idGenerationStrategy) + def withSqlDataSourceConfigs( + sqlDataSourceConfigs: Map[String, SqlDataSourceConfig] + ): SqlPropertyGraphDataSource = + SqlPropertyGraphDataSource( + graphDdl, + sqlDataSourceConfigs, + idGenerationStrategy + ) } - def apply(graphDdlPath: String)(implicit morpheus: MorpheusSession): SqlGraphSourceFactory = { + def apply( + graphDdlPath: String + )(implicit morpheus: MorpheusSession): SqlGraphSourceFactory = { val content = using(Source.fromFile(graphDdlPath, "UTF-8"))(_.mkString) SqlGraphSources(GraphDdl(content)) } - def apply(graphDdl: GraphDdl)(implicit morpheus: MorpheusSession): SqlGraphSourceFactory = - SqlGraphSourceFactory(graphDdl = graphDdl, idGenerationStrategy = SerializedId) + def apply( + graphDdl: GraphDdl + )(implicit morpheus: MorpheusSession): SqlGraphSourceFactory = + SqlGraphSourceFactory( + graphDdl = graphDdl, + idGenerationStrategy = SerializedId + ) } diff --git a/morpheus-spark-cypher/src/main/scala/org/opencypher/morpheus/api/MorpheusSession.scala b/morpheus-spark-cypher/src/main/scala/org/opencypher/morpheus/api/MorpheusSession.scala index 5c27b3c885..56d55ca93d 100644 --- a/morpheus-spark-cypher/src/main/scala/org/opencypher/morpheus/api/MorpheusSession.scala +++ b/morpheus-spark-cypher/src/main/scala/org/opencypher/morpheus/api/MorpheusSession.scala @@ -47,12 +47,15 @@ import scala.reflect.runtime.universe._ /** * Spark-specific [[CypherSession]] implementation. * - * This class is the main entry point for working with Morpheus. It wraps a [[SparkSession]] and allows - * running Cypher queries over a set of distributed Spark [[DataFrame]]s. + * This class is the main entry point for working with Morpheus. It wraps a [[SparkSession]] and + * allows running Cypher queries over a set of distributed Spark [[DataFrame]]s. * - * @param sparkSession The Spark session representing the cluster to execute on + * @param sparkSession + * The Spark session representing the cluster to execute on */ -sealed class MorpheusSession(val sparkSession: SparkSession) extends RelationalCypherSession[DataFrameTable] with Serializable { +sealed class MorpheusSession(val sparkSession: SparkSession) + extends RelationalCypherSession[DataFrameTable] + with Serializable { override type Result = RelationalCypherResult[DataFrameTable] @@ -64,21 +67,31 @@ sealed class MorpheusSession(val sparkSession: SparkSession) extends RelationalC override val graphs: MorpheusGraphFactory = MorpheusGraphFactory() - override val elementTables: MorpheusElementTableFactory.type = MorpheusElementTableFactory + override val elementTables: MorpheusElementTableFactory.type = + MorpheusElementTableFactory + /** * Reads a graph from sequences of nodes and relationships. * - * @param nodes sequence of nodes - * @param relationships sequence of relationships - * @tparam N node type implementing [[io.Node]] - * @tparam R relationship type implementing [[io.Relationship]] - * @return graph defined by the sequences + * @param nodes + * sequence of nodes + * @param relationships + * sequence of relationships + * @tparam N + * node type implementing [[io.Node]] + * @tparam R + * relationship type implementing [[io.Relationship]] + * @return + * graph defined by the sequences */ - def readFrom[N <: Node : TypeTag, R <: Relationship : TypeTag]( + def readFrom[N <: Node: TypeTag, R <: Relationship: TypeTag]( nodes: Seq[N], relationships: Seq[R] = Seq.empty ): PropertyGraph = { - graphs.create(MorpheusNodeTable(nodes), MorpheusRelationshipTable(relationships)) + graphs.create( + MorpheusNodeTable(nodes), + MorpheusRelationshipTable(relationships) + ) } def sql(query: String): MorpheusRecords = @@ -90,31 +103,49 @@ object MorpheusSession extends Serializable { /** * Creates a new [[MorpheusSession]] based on the given [[org.apache.spark.sql.SparkSession]]. * - * @return Morpheus session + * @return + * Morpheus session */ - def create(implicit sparkSession: SparkSession): MorpheusSession = new MorpheusSession(sparkSession) + def create(implicit sparkSession: SparkSession): MorpheusSession = + new MorpheusSession(sparkSession) def localSparkConf: SparkConf = { val conf = new SparkConf(true) conf.set("spark.sql.codegen.wholeStage", "true") conf.set("spark.sql.shuffle.partitions", "12") - conf.set("spark.default.parallelism", Runtime.getRuntime.availableProcessors().toString) + conf.set( + "spark.default.parallelism", + Runtime.getRuntime.availableProcessors().toString + ) // Required for left outer join without join expressions in OPTIONAL MATCH (leads to cartesian product) conf.set("spark.sql.crossJoin.enabled", "true") // We should probably remove this setting. Hide errors like: You're using untyped Scala UDF, which does not have the input type information. Spark may blindly pass null to the Scala closure with primitive-type argument, and the closure will see the default value of the Java type for the null argument, e.g. `udf((x: Int) => x, IntegerType)`, the result is 0 for null input. conf.set("spark.sql.legacy.allowUntypedScalaUDF", "true") // Store Hive tables in local temp folder - conf.set("spark.sql.warehouse.dir", Files.createTempDirectory("spark-warehouse").toString) - conf.set("spark.hadoop.hive.metastore.warehouse.dir", Files.createTempDirectory("hive-warehouse").toString) + conf.set( + "spark.sql.warehouse.dir", + Files.createTempDirectory("spark-warehouse").toString + ) + conf.set( + "spark.hadoop.hive.metastore.warehouse.dir", + Files.createTempDirectory("hive-warehouse").toString + ) // Configure Hive to run with in-memory Derby (skips writing metastore_db) - conf.set("javax.jdo.option.ConnectionURL", "jdbc:derby:memory:metastore_db;create=true") - conf.set("javax.jdo.option.ConnectionDriverName", "org.apache.derby.jdbc.EmbeddedDriver") + conf.set( + "javax.jdo.option.ConnectionURL", + "jdbc:derby:memory:metastore_db;create=true" + ) + conf.set( + "javax.jdo.option.ConnectionDriverName", + "org.apache.derby.jdbc.EmbeddedDriver" + ) conf } /** - * Creates a new [[MorpheusSession]] that wraps a local Spark session with Morpheus default parameters. + * Creates a new [[MorpheusSession]] that wraps a local Spark session with Morpheus default + * parameters. */ def local(): MorpheusSession = { val session = SparkSession @@ -140,31 +171,42 @@ object MorpheusSession extends Serializable { */ // TODO is this still used? implicit class RecordsAsDF(val records: CypherRecords) extends AnyVal { + /** * Extracts the underlying [[org.apache.spark.sql#DataFrame]] from the given [[records]]. * - * Note that the column names in the returned DF do not necessarily correspond to the names of the Cypher RETURN - * items, e.g. "RETURN n.name" does not mean that the column for that item is named "n.name". + * Note that the column names in the returned DF do not necessarily correspond to the names of + * the Cypher RETURN items, e.g. "RETURN n.name" does not mean that the column for that item is + * named "n.name". * - * @return [[org.apache.spark.sql#DataFrame]] representing the records + * @return + * [[org.apache.spark.sql#DataFrame]] representing the records */ def asDataFrame: DataFrame = records match { case morpheus: MorpheusRecords => morpheus.table.df - case _ => throw UnsupportedOperationException(s"can only handle Morpheus records, got $records") + case _ => + throw UnsupportedOperationException( + s"can only handle Morpheus records, got $records" + ) } /** - * Converts all values stored in this table to instances of the corresponding CypherValue class. - * In particular, this de-flattens, or collects, flattened elements (nodes and relationships) into - * compact CypherNode/CypherRelationship objects. + * Converts all values stored in this table to instances of the corresponding CypherValue + * class. In particular, this de-flattens, or collects, flattened elements (nodes and + * relationships) into compact CypherNode/CypherRelationship objects. * - * All values on each row are inserted into a CypherMap object mapped to the corresponding field name. + * All values on each row are inserted into a CypherMap object mapped to the corresponding + * field name. * - * @return [[org.apache.spark.sql.Dataset]] of CypherMaps + * @return + * [[org.apache.spark.sql.Dataset]] of CypherMaps */ def asDataset: Dataset[CypherMap] = records match { case morpheus: MorpheusRecords => morpheus.toCypherMaps - case _ => throw UnsupportedOperationException(s"can only handle Morpheus records, got $records") + case _ => + throw UnsupportedOperationException( + s"can only handle Morpheus records, got $records" + ) } } } diff --git a/morpheus-spark-cypher/src/main/scala/org/opencypher/morpheus/api/io/AbstractPropertyGraphDataSource.scala b/morpheus-spark-cypher/src/main/scala/org/opencypher/morpheus/api/io/AbstractPropertyGraphDataSource.scala index 9d0bcc15bf..dba54c0858 100644 --- a/morpheus-spark-cypher/src/main/scala/org/opencypher/morpheus/api/io/AbstractPropertyGraphDataSource.scala +++ b/morpheus-spark-cypher/src/main/scala/org/opencypher/morpheus/api/io/AbstractPropertyGraphDataSource.scala @@ -49,26 +49,36 @@ import scala.util.{Failure, Success} object AbstractPropertyGraphDataSource { - def nodeColsWithCypherType(schema: PropertyGraphSchema, labelCombination: Set[String]): Map[String, CypherType] = { - val propertyColsWithCypherType = schema.nodePropertyKeysForCombinations(Set(labelCombination)).map { - case (key, cypherType) => key.toPropertyColumnName -> cypherType - } + def nodeColsWithCypherType( + schema: PropertyGraphSchema, + labelCombination: Set[String] + ): Map[String, CypherType] = { + val propertyColsWithCypherType = + schema.nodePropertyKeysForCombinations(Set(labelCombination)).map { case (key, cypherType) => + key.toPropertyColumnName -> cypherType + } propertyColsWithCypherType + (GraphElement.sourceIdKey -> CTIdentity) } - def relColsWithCypherType(schema: PropertyGraphSchema, relType: String): Map[String, CypherType] = { - val propertyColsWithCypherType = schema.relationshipPropertyKeys(relType).map { - case (key, cypherType) => key.toPropertyColumnName -> cypherType - } - propertyColsWithCypherType ++ Relationship.nonPropertyAttributes.map(_ -> CTIdentity) + def relColsWithCypherType( + schema: PropertyGraphSchema, + relType: String + ): Map[String, CypherType] = { + val propertyColsWithCypherType = + schema.relationshipPropertyKeys(relType).map { case (key, cypherType) => + key.toPropertyColumnName -> cypherType + } + propertyColsWithCypherType ++ Relationship.nonPropertyAttributes.map( + _ -> CTIdentity + ) } } /** * Abstract data source implementation that takes care of caching graph names and schemas. * - * It automatically creates initializes a ScanGraphs an only requires the implementor to provider simpler methods for - * reading/writing files and tables. + * It automatically creates initializes a ScanGraphs an only requires the implementor to provider + * simpler methods for reading/writing files and tables. */ abstract class AbstractPropertyGraphDataSource extends MorpheusPropertyGraphDataSource { @@ -78,7 +88,8 @@ abstract class AbstractPropertyGraphDataSource extends MorpheusPropertyGraphData protected var schemaCache: Map[GraphName, MorpheusSchema] = Map.empty - protected var graphNameCache: Set[GraphName] = listGraphNames.map(GraphName).toSet + protected var graphNameCache: Set[GraphName] = + listGraphNames.map(GraphName).toSet protected def listGraphNames: List[String] @@ -88,21 +99,43 @@ abstract class AbstractPropertyGraphDataSource extends MorpheusPropertyGraphData protected def writeSchema(graphName: GraphName, schema: MorpheusSchema): Unit - protected def readMorpheusGraphMetaData(graphName: GraphName): MorpheusGraphMetaData - - protected def writeMorpheusGraphMetaData(graphName: GraphName, morpheusGraphMetaData: MorpheusGraphMetaData): Unit - - protected def readNodeTable(graphName: GraphName, labelCombination: Set[String], sparkSchema: StructType): DataFrame - - protected def writeNodeTable(graphName: GraphName, labelCombination: Set[String], table: DataFrame): Unit - - protected def readRelationshipTable(graphName: GraphName, relKey: String, sparkSchema: StructType): DataFrame - - protected def writeRelationshipTable(graphName: GraphName, relKey: String, table: DataFrame): Unit + protected def readMorpheusGraphMetaData( + graphName: GraphName + ): MorpheusGraphMetaData + + protected def writeMorpheusGraphMetaData( + graphName: GraphName, + morpheusGraphMetaData: MorpheusGraphMetaData + ): Unit + + protected def readNodeTable( + graphName: GraphName, + labelCombination: Set[String], + sparkSchema: StructType + ): DataFrame + + protected def writeNodeTable( + graphName: GraphName, + labelCombination: Set[String], + table: DataFrame + ): Unit + + protected def readRelationshipTable( + graphName: GraphName, + relKey: String, + sparkSchema: StructType + ): DataFrame + + protected def writeRelationshipTable( + graphName: GraphName, + relKey: String, + table: DataFrame + ): Unit override def graphNames: Set[GraphName] = graphNameCache - override def hasGraph(graphName: GraphName): Boolean = graphNameCache.contains(graphName) + override def hasGraph(graphName: GraphName): Boolean = + graphNameCache.contains(graphName) override def delete(graphName: GraphName): Unit = { deleteGraph(graphName) @@ -116,17 +149,28 @@ abstract class AbstractPropertyGraphDataSource extends MorpheusPropertyGraphData } else { val morpheusSchema: MorpheusSchema = schema(name).get val nodeTables = morpheusSchema.allCombinations.map { combo => - val df = readNodeTable(name, combo, morpheusSchema.canonicalNodeStructType(combo)) + val df = readNodeTable( + name, + combo, + morpheusSchema.canonicalNodeStructType(combo) + ) MorpheusNodeTable(combo, df) } val relTables = morpheusSchema.relationshipTypes.map { relType => - val df = readRelationshipTable(name, relType, morpheusSchema.canonicalRelStructType(relType)) + val df = readRelationshipTable( + name, + relType, + morpheusSchema.canonicalRelStructType(relType) + ) MorpheusRelationshipTable(relType, df) } if (nodeTables.isEmpty) { morpheus.graphs.empty } else { - morpheus.graphs.create(Some(morpheusSchema), (nodeTables ++ relTables).toSeq: _*) + morpheus.graphs.create( + Some(morpheusSchema), + (nodeTables ++ relTables).toSeq: _* + ) } } } @@ -141,14 +185,16 @@ abstract class AbstractPropertyGraphDataSource extends MorpheusPropertyGraphData } } - override def store(graphName: GraphName, graph: PropertyGraph): Unit = { checkStorable(graphName) - val poolSize = morpheus.sparkSession.sparkContext.statusTracker.getExecutorInfos.length + val poolSize = + morpheus.sparkSession.sparkContext.statusTracker.getExecutorInfos.length implicit val executionContext: ExecutionContextExecutorService = - ExecutionContext.fromExecutorService(Executors.newFixedThreadPool(poolSize)) + ExecutionContext.fromExecutorService( + Executors.newFixedThreadPool(poolSize) + ) try { val relationalGraph = graph.asMorpheus @@ -156,18 +202,29 @@ abstract class AbstractPropertyGraphDataSource extends MorpheusPropertyGraphData val schema = relationalGraph.schema.asMorpheus schemaCache += graphName -> schema graphNameCache += graphName - writeMorpheusGraphMetaData(graphName, MorpheusGraphMetaData(tableStorageFormat.name)) + writeMorpheusGraphMetaData( + graphName, + MorpheusGraphMetaData(tableStorageFormat.name) + ) writeSchema(graphName, schema) val nodeWrites = schema.labelCombinations.combos.map { combo => Future { - writeNodeTable(graphName, combo, relationalGraph.canonicalNodeTable(combo)) + writeNodeTable( + graphName, + combo, + relationalGraph.canonicalNodeTable(combo) + ) } } val relWrites = schema.relationshipTypes.map { relType => Future { - writeRelationshipTable(graphName, relType, relationalGraph.canonicalRelationshipTable(relType)) + writeRelationshipTable( + graphName, + relType, + relationalGraph.canonicalRelationshipTable(relType) + ) } } @@ -183,7 +240,9 @@ abstract class AbstractPropertyGraphDataSource extends MorpheusPropertyGraphData graphNameCache = listGraphNames.map(GraphName).toSet } - protected def waitForWriteCompletion(writeFutures: Set[Future[Unit]])(implicit ec: ExecutionContext): Unit = { + protected def waitForWriteCompletion( + writeFutures: Set[Future[Unit]] + )(implicit ec: ExecutionContext): Unit = { writeFutures.foreach { writeFuture => Await.ready(writeFuture, Duration.Inf) writeFuture.onComplete { diff --git a/morpheus-spark-cypher/src/main/scala/org/opencypher/morpheus/api/io/GraphElement.scala b/morpheus-spark-cypher/src/main/scala/org/opencypher/morpheus/api/io/GraphElement.scala index 59fd895535..f5d5cc7305 100644 --- a/morpheus-spark-cypher/src/main/scala/org/opencypher/morpheus/api/io/GraphElement.scala +++ b/morpheus-spark-cypher/src/main/scala/org/opencypher/morpheus/api/io/GraphElement.scala @@ -41,9 +41,9 @@ object GraphElement { } /** - * If a node has no label annotation, then the class name is used as its label. - * If a `Labels` annotation, for example `@Labels("Person", "Mammal")`, is present, - * then the labels from that annotation are used instead. + * If a node has no label annotation, then the class name is used as its label. If a `Labels` + * annotation, for example `@Labels("Person", "Mammal")`, is present, then the labels from that + * annotation are used instead. */ trait Node extends GraphElement @@ -52,13 +52,14 @@ object Relationship { val sourceEndNodeKey: String = SourceEndNodeKey.name - val nonPropertyAttributes: Set[String] = Set(GraphElement.sourceIdKey, sourceStartNodeKey, sourceEndNodeKey) + val nonPropertyAttributes: Set[String] = + Set(GraphElement.sourceIdKey, sourceStartNodeKey, sourceEndNodeKey) } /** * If a relationship has no type annotation, then the class name in upper case is used as its type. - * If a `Type` annotation, for example `@RelationshipType("FRIEND_OF")` is present, - * then the type from that annotation is used instead. + * If a `Type` annotation, for example `@RelationshipType("FRIEND_OF")` is present, then the type + * from that annotation is used instead. */ trait Relationship extends GraphElement { def source: Long @@ -67,25 +68,29 @@ trait Relationship extends GraphElement { } /** - * Annotation to use when mapping a case class to a node with more than one label, or a label different to the class name. + * Annotation to use when mapping a case class to a node with more than one label, or a label + * different to the class name. * * {{{ * @ Labels("Person", "Employee") * case class Employee(id: Long, name: String, salary: Double) * }}} * - * @param labels the labels that the node has. + * @param labels + * the labels that the node has. */ case class Labels(labels: String*) extends StaticAnnotation /** - * Annotation to use when mapping a case class to a relationship with a different relationship type to the class name. + * Annotation to use when mapping a case class to a relationship with a different relationship type + * to the class name. * * {{{ * @ RelationshipType("FRIEND_OF") * case class Friend(id: Long, src: Long, dst: Long, since: Int) * }}} * - * @param relType the relationship type that the relationship has. + * @param relType + * the relationship type that the relationship has. */ case class RelationshipType(relType: String) extends StaticAnnotation diff --git a/morpheus-spark-cypher/src/main/scala/org/opencypher/morpheus/api/io/MorpheusTable.scala b/morpheus-spark-cypher/src/main/scala/org/opencypher/morpheus/api/io/MorpheusTable.scala index dbefdcb1f2..1626181460 100644 --- a/morpheus-spark-cypher/src/main/scala/org/opencypher/morpheus/api/io/MorpheusTable.scala +++ b/morpheus-spark-cypher/src/main/scala/org/opencypher/morpheus/api/io/MorpheusTable.scala @@ -31,7 +31,11 @@ import org.opencypher.morpheus.api.MorpheusSession import org.opencypher.morpheus.impl.table.SparkTable.{DataFrameTable, _} import org.opencypher.morpheus.impl.util.Annotation import org.opencypher.morpheus.impl.{MorpheusRecords, RecordBehaviour} -import org.opencypher.okapi.api.io.conversion.{ElementMapping, NodeMappingBuilder, RelationshipMappingBuilder} +import org.opencypher.okapi.api.io.conversion.{ + ElementMapping, + NodeMappingBuilder, + RelationshipMappingBuilder +} import org.opencypher.okapi.impl.util.StringEncodingUtilities._ import org.opencypher.okapi.relational.api.io.ElementTable import org.opencypher.okapi.relational.api.table.RelationalElementTableFactory @@ -47,14 +51,17 @@ case object MorpheusElementTableFactory extends RelationalElementTableFactory[Da } } -case class MorpheusElementTable private[morpheus]( +case class MorpheusElementTable private[morpheus] ( override val mapping: ElementMapping, override val table: DataFrameTable -) extends ElementTable[DataFrameTable] with RecordBehaviour { +) extends ElementTable[DataFrameTable] + with RecordBehaviour { override type Records = MorpheusElementTable - private[morpheus] def records(implicit morpheus: MorpheusSession): MorpheusRecords = morpheus.records.fromElementTable(elementTable = this) + private[morpheus] def records(implicit + morpheus: MorpheusSession + ): MorpheusRecords = morpheus.records.fromElementTable(elementTable = this) override def cache(): MorpheusElementTable = { table.cache() @@ -63,7 +70,10 @@ case class MorpheusElementTable private[morpheus]( } object MorpheusElementTable { - def create(mapping: ElementMapping, table: DataFrameTable): MorpheusElementTable = { + def create( + mapping: ElementMapping, + table: DataFrameTable + ): MorpheusElementTable = { val sourceIdColumns = mapping.allSourceIdKeys val idCols = table.df.encodeIdColumns(sourceIdColumns: _*) val remainingCols = mapping.allSourcePropertyKeys.map(table.df.col) @@ -75,25 +85,39 @@ object MorpheusElementTable { object MorpheusNodeTable { - def apply[E <: Node : TypeTag](nodes: Seq[E])(implicit morpheus: MorpheusSession): MorpheusElementTable = { + def apply[E <: Node: TypeTag]( + nodes: Seq[E] + )(implicit morpheus: MorpheusSession): MorpheusElementTable = { val nodeLabels = Annotation.labels[E] val nodeDF = morpheus.sparkSession.createDataFrame(nodes) - val nodeProperties = nodeDF.columns.filter(_ != GraphElement.sourceIdKey).toSet - val nodeMapping = NodeMappingBuilder.create(nodeIdKey = GraphElement.sourceIdKey, impliedLabels = nodeLabels, propertyKeys = nodeProperties) + val nodeProperties = + nodeDF.columns.filter(_ != GraphElement.sourceIdKey).toSet + val nodeMapping = NodeMappingBuilder.create( + nodeIdKey = GraphElement.sourceIdKey, + impliedLabels = nodeLabels, + propertyKeys = nodeProperties + ) MorpheusElementTable.create(nodeMapping, nodeDF) } /** - * Creates a node table from the given [[DataFrame]]. By convention, there needs to be one column storing node - * identifiers and named after [[GraphElement.sourceIdKey]]. All remaining columns are interpreted as node property columns, the column name is used as property - * key. + * Creates a node table from the given [[DataFrame]]. By convention, there needs to be one column + * storing node identifiers and named after [[GraphElement.sourceIdKey]]. All remaining columns + * are interpreted as node property columns, the column name is used as property key. * - * @param impliedLabels implied node labels - * @param nodeDF node data - * @return a node table with inferred node mapping + * @param impliedLabels + * implied node labels + * @param nodeDF + * node data + * @return + * a node table with inferred node mapping */ - def apply(impliedLabels: Set[String], nodeDF: DataFrame): MorpheusElementTable = { - val propertyColumnNames = nodeDF.columns.filter(_ != GraphElement.sourceIdKey).toSet + def apply( + impliedLabels: Set[String], + nodeDF: DataFrame + ): MorpheusElementTable = { + val propertyColumnNames = + nodeDF.columns.filter(_ != GraphElement.sourceIdKey).toSet val propertyKeyMapping = propertyColumnNames.map(p => p.toProperty -> p) val mapping = NodeMappingBuilder @@ -108,36 +132,52 @@ object MorpheusNodeTable { object MorpheusRelationshipTable { - def apply[E <: Relationship : TypeTag](relationships: Seq[E])(implicit morpheus: MorpheusSession): MorpheusElementTable = { + def apply[E <: Relationship: TypeTag]( + relationships: Seq[E] + )(implicit morpheus: MorpheusSession): MorpheusElementTable = { val relationshipType: String = Annotation.relType[E] val relationshipDF = morpheus.sparkSession.createDataFrame(relationships) - val relationshipProperties = relationshipDF.columns.filter(!Relationship.nonPropertyAttributes.contains(_)).toSet + val relationshipProperties = relationshipDF.columns + .filter(!Relationship.nonPropertyAttributes.contains(_)) + .toSet - val relationshipMapping = RelationshipMappingBuilder.create(GraphElement.sourceIdKey, + val relationshipMapping = RelationshipMappingBuilder.create( + GraphElement.sourceIdKey, Relationship.sourceStartNodeKey, Relationship.sourceEndNodeKey, relationshipType, - relationshipProperties) + relationshipProperties + ) MorpheusElementTable.create(relationshipMapping, relationshipDF) } /** - * Creates a relationship table from the given [[DataFrame]]. By convention, there needs to be one column storing - * relationship identifiers and named after [[GraphElement.sourceIdKey]], one column storing source node identifiers - * and named after [[Relationship.sourceStartNodeKey]] and one column storing target node identifiers and named after - * [[Relationship.sourceEndNodeKey]]. All remaining columns are interpreted as relationship property columns, the - * column name is used as property key. + * Creates a relationship table from the given [[DataFrame]]. By convention, there needs to be + * one column storing relationship identifiers and named after [[GraphElement.sourceIdKey]], one + * column storing source node identifiers and named after [[Relationship.sourceStartNodeKey]] and + * one column storing target node identifiers and named after [[Relationship.sourceEndNodeKey]]. + * All remaining columns are interpreted as relationship property columns, the column name is + * used as property key. * - * Column names prefixed with `property#` are decoded by [[org.opencypher.okapi.impl.util.StringEncodingUtilities]] to - * recover the original property name. + * Column names prefixed with `property#` are decoded by + * [[org.opencypher.okapi.impl.util.StringEncodingUtilities]] to recover the original property + * name. * - * @param relationshipType relationship type - * @param relationshipDF relationship data - * @return a relationship table with inferred relationship mapping + * @param relationshipType + * relationship type + * @param relationshipDF + * relationship data + * @return + * a relationship table with inferred relationship mapping */ - def apply(relationshipType: String, relationshipDF: DataFrame): MorpheusElementTable = { - val propertyColumnNames = relationshipDF.columns.filter(!Relationship.nonPropertyAttributes.contains(_)).toSet + def apply( + relationshipType: String, + relationshipDF: DataFrame + ): MorpheusElementTable = { + val propertyColumnNames = relationshipDF.columns + .filter(!Relationship.nonPropertyAttributes.contains(_)) + .toSet val propertyKeyMapping = propertyColumnNames.map(p => p.toProperty -> p) val mapping = RelationshipMappingBuilder @@ -151,5 +191,3 @@ object MorpheusRelationshipTable { MorpheusElementTable.create(mapping, relationshipDF) } } - - diff --git a/morpheus-spark-cypher/src/main/scala/org/opencypher/morpheus/api/io/StorageFormat.scala b/morpheus-spark-cypher/src/main/scala/org/opencypher/morpheus/api/io/StorageFormat.scala index 0b76c3f542..8190017ed9 100644 --- a/morpheus-spark-cypher/src/main/scala/org/opencypher/morpheus/api/io/StorageFormat.scala +++ b/morpheus-spark-cypher/src/main/scala/org/opencypher/morpheus/api/io/StorageFormat.scala @@ -32,7 +32,8 @@ import org.opencypher.okapi.impl.util.JsonUtils.FlatOption._ import ujson._ trait StorageFormat { - def name: String = getClass.getSimpleName.dropRight("Format$".length).toLowerCase + def name: String = + getClass.getSimpleName.dropRight("Format$".length).toLowerCase } object StorageFormat { @@ -47,20 +48,25 @@ object StorageFormat { val nonFileFormatNames: Set[String] = nonFileFormats.keySet private def unexpected(name: String, available: Iterable[String]) = - throw IllegalArgumentException(s"Supported storage format (one of ${available.mkString("[", ", ", "]")})", name) + throw IllegalArgumentException( + s"Supported storage format (one of ${available.mkString("[", ", ", "]")})", + name + ) - implicit def rwStorageFormat: ReadWriter[StorageFormat] = readwriter[Value].bimap[StorageFormat]( - (storageFormat: StorageFormat) => storageFormat.name, - (storageFormatName: Value) => { - val formatString = storageFormatName.str - nonFileFormats.getOrElse(formatString, FileFormat(formatString)) - } - ) + implicit def rwStorageFormat: ReadWriter[StorageFormat] = + readwriter[Value].bimap[StorageFormat]( + (storageFormat: StorageFormat) => storageFormat.name, + (storageFormatName: Value) => { + val formatString = storageFormatName.str + nonFileFormats.getOrElse(formatString, FileFormat(formatString)) + } + ) - implicit def rwFileFormat: ReadWriter[FileFormat] = readwriter[Value].bimap[FileFormat]( - (fileFormat: FileFormat) => fileFormat.name, - (fileFormatName: Value) => FileFormat(fileFormatName.str) - ) + implicit def rwFileFormat: ReadWriter[FileFormat] = + readwriter[Value].bimap[FileFormat]( + (fileFormat: FileFormat) => fileFormat.name, + (fileFormatName: Value) => FileFormat(fileFormatName.str) + ) } @@ -79,6 +85,9 @@ object FileFormat { } case class FileFormat(override val name: String) extends StorageFormat { - assert(!nonFileFormatNames.contains(name), - s"Cannot create a file format with a name in ${nonFileFormatNames.mkString("[", ", ", "]")} ") + assert( + !nonFileFormatNames.contains(name), + s"Cannot create a file format with a name in ${nonFileFormatNames + .mkString("[", ", ", "]")} " + ) } diff --git a/morpheus-spark-cypher/src/main/scala/org/opencypher/morpheus/api/io/edgelist/EdgeListDataSource.scala b/morpheus-spark-cypher/src/main/scala/org/opencypher/morpheus/api/io/edgelist/EdgeListDataSource.scala index 5919c3689c..8022c4c533 100644 --- a/morpheus-spark-cypher/src/main/scala/org/opencypher/morpheus/api/io/edgelist/EdgeListDataSource.scala +++ b/morpheus-spark-cypher/src/main/scala/org/opencypher/morpheus/api/io/edgelist/EdgeListDataSource.scala @@ -53,34 +53,44 @@ object EdgeListDataSource { } /** - * A read-only data source that is able to read graphs from edge list files. Input files are expected to contain one - * edge per row, e.g. + * A read-only data source that is able to read graphs from edge list files. Input files are + * expected to contain one edge per row, e.g. * - * 0 1 - * 1 2 + * 0 1 1 2 * * describes a graph with two edges (one from vertex 0 to 1 and one from vertex 1 to 2). * * The data source can be parameterized with options used by the underlying Spark Csv reader. * - * @param path path to the edge list file - * @param options Spark Csv reader options - * @param morpheus Morpheus session + * @param path + * path to the edge list file + * @param options + * Spark Csv reader options + * @param morpheus + * Morpheus session */ -case class EdgeListDataSource(path: String, options: Map[String, String] = Map.empty)(implicit morpheus: MorpheusSession) - extends PropertyGraphDataSource { +case class EdgeListDataSource( + path: String, + options: Map[String, String] = Map.empty +)(implicit morpheus: MorpheusSession) + extends PropertyGraphDataSource { override def hasGraph(name: GraphName): Boolean = name == GRAPH_NAME override def graph(name: GraphName): PropertyGraph = { - val reader = options.foldLeft(morpheus.sparkSession.read) { - case (current, (key, value)) => current.option(key, value) + val reader = options.foldLeft(morpheus.sparkSession.read) { case (current, (key, value)) => + current.option(key, value) } val rawRels = reader - .schema(StructType(Seq( - StructField(sourceStartNodeKey, LongType), - StructField(sourceEndNodeKey, LongType)))) + .schema( + StructType( + Seq( + StructField(sourceStartNodeKey, LongType), + StructField(sourceEndNodeKey, LongType) + ) + ) + ) .csv(path) .withColumn(sourceIdKey, functions.monotonically_increasing_id()) .select(sourceIdKey, sourceStartNodeKey, sourceEndNodeKey) @@ -90,16 +100,23 @@ case class EdgeListDataSource(path: String, options: Map[String, String] = Map.e .union(rawRels.select(rawRels.col(sourceEndNodeKey).as(sourceIdKey))) .distinct() - morpheus.graphs.create(MorpheusNodeTable(Set(NODE_LABEL), rawNodes), MorpheusRelationshipTable(REL_TYPE, rawRels)) + morpheus.graphs.create( + MorpheusNodeTable(Set(NODE_LABEL), rawNodes), + MorpheusRelationshipTable(REL_TYPE, rawRels) + ) } - override def schema(name: GraphName): Option[PropertyGraphSchema] = Some(SCHEMA) + override def schema(name: GraphName): Option[PropertyGraphSchema] = Some( + SCHEMA + ) override def store(name: GraphName, graph: PropertyGraph): Unit = throw UnsupportedOperationException("Storing an edge list is not supported") override def delete(name: GraphName): Unit = - throw UnsupportedOperationException("Deleting an edge list is not supported") + throw UnsupportedOperationException( + "Deleting an edge list is not supported" + ) override val graphNames: Set[GraphName] = Set(GRAPH_NAME) } diff --git a/morpheus-spark-cypher/src/main/scala/org/opencypher/morpheus/api/io/fs/EscapeAtSymbol.scala b/morpheus-spark-cypher/src/main/scala/org/opencypher/morpheus/api/io/fs/EscapeAtSymbol.scala index 2b0a414a0a..d5ae103b9e 100644 --- a/morpheus-spark-cypher/src/main/scala/org/opencypher/morpheus/api/io/fs/EscapeAtSymbol.scala +++ b/morpheus-spark-cypher/src/main/scala/org/opencypher/morpheus/api/io/fs/EscapeAtSymbol.scala @@ -42,7 +42,10 @@ trait EscapeAtSymbol extends FSGraphSource { super.writeTable(path, newTable) } - abstract override def readTable(path: String, schema: StructType): DataFrame = { + abstract override def readTable( + path: String, + schema: StructType + ): DataFrame = { val readMapping = encodedColumnNames(schema).toMap val readSchema = StructType(schema.fields.map { f => f.copy(name = readMapping.getOrElse(f.name, f.name)) @@ -55,12 +58,16 @@ trait EscapeAtSymbol extends FSGraphSource { private def encodedColumnNames(schema: StructType): Seq[(String, String)] = { schema.fields .map(f => f.name -> f.name.replaceAll(atSymbol, unicodeEscaping)) - .filterNot { case (from, to) => from == to} + .filterNot { case (from, to) => from == to } } private def schemaCheck(schema: StructType): Unit = { - val invalidFields = schema.fields.filter(f => f.name.contains(unicodeEscaping)).map(_.name) - if (invalidFields.nonEmpty) sys.error(s"Orc fields: $invalidFields cannot contain special encoding string: '$unicodeEscaping'") + val invalidFields = + schema.fields.filter(f => f.name.contains(unicodeEscaping)).map(_.name) + if (invalidFields.nonEmpty) + sys.error( + s"Orc fields: $invalidFields cannot contain special encoding string: '$unicodeEscaping'" + ) } } diff --git a/morpheus-spark-cypher/src/main/scala/org/opencypher/morpheus/api/io/fs/FSGraphSource.scala b/morpheus-spark-cypher/src/main/scala/org/opencypher/morpheus/api/io/fs/FSGraphSource.scala index 3a1bf95b2c..0a512e0cb1 100644 --- a/morpheus-spark-cypher/src/main/scala/org/opencypher/morpheus/api/io/fs/FSGraphSource.scala +++ b/morpheus-spark-cypher/src/main/scala/org/opencypher/morpheus/api/io/fs/FSGraphSource.scala @@ -44,13 +44,18 @@ import org.opencypher.okapi.api.graph.{GraphName, Node, Relationship} /** * Data source implementation that handles the writing of files and tables to a filesystem. * - * By default Spark is used to write tables and the Hadoop filesystem configured in Spark is used to write files. - * The file/folder/table structure into which the graphs are stored is defined in [[DefaultGraphDirectoryStructure]]. + * By default Spark is used to write tables and the Hadoop filesystem configured in Spark is used + * to write files. The file/folder/table structure into which the graphs are stored is defined in + * [[DefaultGraphDirectoryStructure]]. * - * @param rootPath path where the graphs are stored - * @param tableStorageFormat Spark configuration parameter for the table format - * @param hiveDatabaseName optional Hive database to write tables to - * @param filesPerTable optional parameter that specifies how many files a table is coalesced into, by default 1 + * @param rootPath + * path where the graphs are stored + * @param tableStorageFormat + * Spark configuration parameter for the table format + * @param hiveDatabaseName + * optional Hive database to write tables to + * @param filesPerTable + * optional parameter that specifies how many files a table is coalesced into, by default 1 */ class FSGraphSource( val rootPath: String, @@ -58,31 +63,41 @@ class FSGraphSource( val hiveDatabaseName: Option[String] = None, val filesPerTable: Option[Int] = None )(override implicit val morpheus: MorpheusSession) - extends AbstractPropertyGraphDataSource with JsonSerialization { + extends AbstractPropertyGraphDataSource + with JsonSerialization { protected val directoryStructure = DefaultGraphDirectoryStructure(rootPath) import directoryStructure._ protected lazy val fileSystem: FileSystem = { - FileSystem.get(new URI(rootPath), morpheus.sparkSession.sparkContext.hadoopConfiguration) + FileSystem.get( + new URI(rootPath), + morpheus.sparkSession.sparkContext.hadoopConfiguration + ) } - protected def listDirectories(path: String): List[String] = fileSystem.listDirectories(path) + protected def listDirectories(path: String): List[String] = + fileSystem.listDirectories(path) - protected def deleteDirectory(path: String): Unit = fileSystem.deleteDirectory(path) + protected def deleteDirectory(path: String): Unit = + fileSystem.deleteDirectory(path) protected def readFile(path: String): String = fileSystem.readFile(path) - protected def writeFile(path: String, content: String): Unit = fileSystem.writeFile(path, content) + protected def writeFile(path: String, content: String): Unit = + fileSystem.writeFile(path, content) protected def readTable(path: String, schema: StructType): DataFrame = { - morpheus.sparkSession.read.format(tableStorageFormat.name).schema(schema).load(path) + morpheus.sparkSession.read + .format(tableStorageFormat.name) + .schema(schema) + .load(path) } protected def writeTable(path: String, table: DataFrame): Unit = { val coalescedTable = filesPerTable match { - case None => table + case None => table case Some(numFiles) => table.coalesce(numFiles) } // TODO: Consider changing computation of canonical tables so `null` typed columns don't make it here @@ -90,13 +105,21 @@ class FSGraphSource( val h :: t = coalescedTable.columns.collect { case c if coalescedTable.schema(c).dataType != NullType => c }.toList - coalescedTable.select(h, t: _*).write.format(tableStorageFormat.name).save(path) + coalescedTable + .select(h, t: _*) + .write + .format(tableStorageFormat.name) + .save(path) } override protected def listGraphNames: List[String] = { def traverse(parent: String): List[String] = { val children = listDirectories(parent) - if (children.contains(nodeTablesDirectoryName) || children.contains(relationshipTablesDirectoryName)) { + if ( + children.contains(nodeTablesDirectoryName) || children.contains( + relationshipTablesDirectoryName + ) + ) { List(parent) } else { children.flatMap(child => traverse(parent / child)) @@ -124,11 +147,20 @@ class FSGraphSource( readTable(pathToNodeTable(graphName, labels), sparkSchema) } - override protected def writeNodeTable(graphName: GraphName, labels: Set[String], table: DataFrame): Unit = { + override protected def writeNodeTable( + graphName: GraphName, + labels: Set[String], + table: DataFrame + ): Unit = { writeTable(pathToNodeTable(graphName, labels), table) if (hiveDatabaseName.isDefined) { - val hiveNodeTableName = HiveTableName(hiveDatabaseName.get, graphName, Node, labels) - writeHiveTable(pathToNodeTable(graphName, labels), hiveNodeTableName, table.schema) + val hiveNodeTableName = + HiveTableName(hiveDatabaseName.get, graphName, Node, labels) + writeHiveTable( + pathToNodeTable(graphName, labels), + hiveNodeTableName, + table.schema + ) } } @@ -140,16 +172,38 @@ class FSGraphSource( readTable(pathToRelationshipTable(graphName, relKey), sparkSchema) } - override protected def writeRelationshipTable(graphName: GraphName, relKey: String, table: DataFrame): Unit = { + override protected def writeRelationshipTable( + graphName: GraphName, + relKey: String, + table: DataFrame + ): Unit = { writeTable(pathToRelationshipTable(graphName, relKey), table) if (hiveDatabaseName.isDefined) { - val hiveRelationshipTableName = HiveTableName(hiveDatabaseName.get, graphName, Relationship, Set(relKey)) - writeHiveTable(pathToRelationshipTable(graphName, relKey), hiveRelationshipTableName, table.schema) + val hiveRelationshipTableName = HiveTableName( + hiveDatabaseName.get, + graphName, + Relationship, + Set(relKey) + ) + writeHiveTable( + pathToRelationshipTable(graphName, relKey), + hiveRelationshipTableName, + table.schema + ) } } - private def writeHiveTable(pathToTable: String, hiveTableName: String, schema: StructType): Unit = { - morpheus.sparkSession.catalog.createTable(hiveTableName, tableStorageFormat.name, schema, Map("path" -> pathToTable)) + private def writeHiveTable( + pathToTable: String, + hiveTableName: String, + schema: StructType + ): Unit = { + morpheus.sparkSession.catalog.createTable( + hiveTableName, + tableStorageFormat.name, + schema, + Map("path" -> pathToTable) + ) morpheus.sparkSession.catalog.refreshTable(hiveTableName) } @@ -159,12 +213,18 @@ class FSGraphSource( val relTypes = graphSchema.relationshipTypes labelCombinations.foreach { combo => - val tableName = HiveTableName(hiveDatabaseName.get, graphName, Node, combo) + val tableName = + HiveTableName(hiveDatabaseName.get, graphName, Node, combo) morpheus.sparkSession.sql(s"DROP TABLE IF EXISTS $tableName") } relTypes.foreach { relType => - val tableName = HiveTableName(hiveDatabaseName.get, graphName, Relationship, Set(relType)) + val tableName = HiveTableName( + hiveDatabaseName.get, + graphName, + Relationship, + Set(relType) + ) morpheus.sparkSession.sql(s"DROP TABLE IF EXISTS $tableName") } } @@ -173,32 +233,45 @@ class FSGraphSource( readFile(pathToGraphSchema(graphName)) } - override protected def writeJsonSchema(graphName: GraphName, schema: String): Unit = { + override protected def writeJsonSchema( + graphName: GraphName, + schema: String + ): Unit = { writeFile(pathToGraphSchema(graphName), schema) } - override protected def readJsonMorpheusGraphMetaData(graphName: GraphName): String = { + override protected def readJsonMorpheusGraphMetaData( + graphName: GraphName + ): String = { readFile(pathToMorpheusMetaData(graphName)) } - override protected def writeJsonMorpheusGraphMetaData(graphName: GraphName, morpheusGraphMetaData: String): Unit = { + override protected def writeJsonMorpheusGraphMetaData( + graphName: GraphName, + morpheusGraphMetaData: String + ): Unit = { writeFile(pathToMorpheusMetaData(graphName), morpheusGraphMetaData) } } /** - * Spark CSV does not support storing BinaryType columns by default. This data source implementation encodes BinaryType - * columns to Hex-encoded strings and decodes such columns back to BinaryType. This feature is required because ids - * within Morpheus are stored as BinaryType. + * Spark CSV does not support storing BinaryType columns by default. This data source + * implementation encodes BinaryType columns to Hex-encoded strings and decodes such columns back + * to BinaryType. This feature is required because ids within Morpheus are stored as BinaryType. */ -class CsvGraphSource(rootPath: String, filesPerTable: Option[Int] = None)(override implicit val morpheus: MorpheusSession) - extends FSGraphSource(rootPath, FileFormat.csv, None, filesPerTable) { +class CsvGraphSource(rootPath: String, filesPerTable: Option[Int] = None)( + override implicit val morpheus: MorpheusSession +) extends FSGraphSource(rootPath, FileFormat.csv, None, filesPerTable) { override protected def writeTable(path: String, table: DataFrame): Unit = super.writeTable(path, table.encodeBinaryToHexString) - protected override def readNodeTable(graphName: GraphName, labels: Set[String], sparkSchema: StructType): DataFrame = + protected override def readNodeTable( + graphName: GraphName, + labels: Set[String], + sparkSchema: StructType + ): DataFrame = readElementTable(graphName, Left(labels), sparkSchema) protected override def readRelationshipTable( @@ -216,7 +289,8 @@ class CsvGraphSource(rootPath: String, filesPerTable: Option[Int] = None)(overri val tableWithEncodedStrings = labelsOrRelKey match { case Left(labels) => super.readNodeTable(graphName, labels, readSchema) - case Right(relKey) => super.readRelationshipTable(graphName, relKey, readSchema) + case Right(relKey) => + super.readRelationshipTable(graphName, relKey, readSchema) } tableWithEncodedStrings.decodeHexStringToBinary(sparkSchema.binaryColumns) diff --git a/morpheus-spark-cypher/src/main/scala/org/opencypher/morpheus/api/io/fs/GraphDirectoryStructure.scala b/morpheus-spark-cypher/src/main/scala/org/opencypher/morpheus/api/io/fs/GraphDirectoryStructure.scala index 149f703efc..c0c2b917bd 100644 --- a/morpheus-spark-cypher/src/main/scala/org/opencypher/morpheus/api/io/fs/GraphDirectoryStructure.scala +++ b/morpheus-spark-cypher/src/main/scala/org/opencypher/morpheus/api/io/fs/GraphDirectoryStructure.scala @@ -69,9 +69,11 @@ object DefaultGraphDirectoryStructure { // Because an empty path does not work, we need a special directory name for nodes without labels. val noLabelNodeDirectoryName: String = "__NO_LABEL__" - def nodeTableDirectoryName(labels: Set[String]): String = concatDirectoryNames(labels.toSeq.sorted) + def nodeTableDirectoryName(labels: Set[String]): String = + concatDirectoryNames(labels.toSeq.sorted) - def relKeyTableDirectoryName(relKey: String): String = relKey.encodeSpecialCharacters + def relKeyTableDirectoryName(relKey: String): String = + relKey.encodeSpecialCharacters def concatDirectoryNames(seq: Seq[String]): String = { if (seq.isEmpty) { @@ -82,7 +84,8 @@ object DefaultGraphDirectoryStructure { } } -case class DefaultGraphDirectoryStructure(dataSourceRootPath: String) extends GraphDirectoryStructure { +case class DefaultGraphDirectoryStructure(dataSourceRootPath: String) + extends GraphDirectoryStructure { import DefaultGraphDirectoryStructure._ @@ -98,12 +101,22 @@ case class DefaultGraphDirectoryStructure(dataSourceRootPath: String) extends Gr pathToGraphDirectory(graphName) / morpheusMetaDataFileName } - override def pathToNodeTable(graphName: GraphName, labels: Set[String]): String = { - pathToGraphDirectory(graphName) / nodeTablesDirectoryName / nodeTableDirectoryName(labels) + override def pathToNodeTable( + graphName: GraphName, + labels: Set[String] + ): String = { + pathToGraphDirectory( + graphName + ) / nodeTablesDirectoryName / nodeTableDirectoryName(labels) } - override def pathToRelationshipTable(graphName: GraphName, relKey: String): String = { - pathToGraphDirectory(graphName) / relationshipTablesDirectoryName / relKeyTableDirectoryName(relKey) + override def pathToRelationshipTable( + graphName: GraphName, + relKey: String + ): String = { + pathToGraphDirectory( + graphName + ) / relationshipTablesDirectoryName / relKeyTableDirectoryName(relKey) } } diff --git a/morpheus-spark-cypher/src/main/scala/org/opencypher/morpheus/api/io/fs/HadoopFSHelpers.scala b/morpheus-spark-cypher/src/main/scala/org/opencypher/morpheus/api/io/fs/HadoopFSHelpers.scala index 5ff7b79565..bd9ef849d1 100644 --- a/morpheus-spark-cypher/src/main/scala/org/opencypher/morpheus/api/io/fs/HadoopFSHelpers.scala +++ b/morpheus-spark-cypher/src/main/scala/org/opencypher/morpheus/api/io/fs/HadoopFSHelpers.scala @@ -44,7 +44,8 @@ object HadoopFSHelpers { def listDirectories(path: String): List[String] = { val p = new Path(path) createDirectoryIfNotExists(p) - fileSystem.listStatus(p) + fileSystem + .listStatus(p) .filter(_.isDirectory) .map(_.getPath.getName) .toList @@ -55,8 +56,13 @@ object HadoopFSHelpers { } def readFile(path: String): String = { - using(new BufferedReader(new InputStreamReader(fileSystem.open(new Path(path)), "UTF-8"))) { reader => - def readLines = Stream.cons(reader.readLine(), Stream.continually(reader.readLine)) + using( + new BufferedReader( + new InputStreamReader(fileSystem.open(new Path(path)), "UTF-8") + ) + ) { reader => + def readLines = + Stream.cons(reader.readLine(), Stream.continually(reader.readLine)) readLines.takeWhile(_ != null).mkString } } @@ -66,7 +72,9 @@ object HadoopFSHelpers { val parentDirectory = p.getParent createDirectoryIfNotExists(parentDirectory) using(fileSystem.create(p)) { outputStream => - using(new BufferedWriter(new OutputStreamWriter(outputStream, "UTF-8"))) { bufferedWriter => + using( + new BufferedWriter(new OutputStreamWriter(outputStream, "UTF-8")) + ) { bufferedWriter => bufferedWriter.write(content) } } diff --git a/morpheus-spark-cypher/src/main/scala/org/opencypher/morpheus/api/io/json/JsonSerialization.scala b/morpheus-spark-cypher/src/main/scala/org/opencypher/morpheus/api/io/json/JsonSerialization.scala index 07c7c9f915..e27ab32de4 100644 --- a/morpheus-spark-cypher/src/main/scala/org/opencypher/morpheus/api/io/json/JsonSerialization.scala +++ b/morpheus-spark-cypher/src/main/scala/org/opencypher/morpheus/api/io/json/JsonSerialization.scala @@ -43,21 +43,34 @@ trait JsonSerialization { protected def readJsonMorpheusGraphMetaData(graphName: GraphName): String - protected def writeJsonMorpheusGraphMetaData(graphName: GraphName, morpheusGraphMetaData: String): Unit + protected def writeJsonMorpheusGraphMetaData( + graphName: GraphName, + morpheusGraphMetaData: String + ): Unit - override protected[io] def readSchema(graphName: GraphName): MorpheusSchema = { + override protected[io] def readSchema( + graphName: GraphName + ): MorpheusSchema = { PropertyGraphSchema.fromJson(readJsonSchema(graphName)).asMorpheus } - override protected def writeSchema(graphName: GraphName, schema: MorpheusSchema): Unit = { + override protected def writeSchema( + graphName: GraphName, + schema: MorpheusSchema + ): Unit = { writeJsonSchema(graphName, schema.schema.toJson) } - override protected def readMorpheusGraphMetaData(graphName: GraphName): MorpheusGraphMetaData = { + override protected def readMorpheusGraphMetaData( + graphName: GraphName + ): MorpheusGraphMetaData = { MorpheusGraphMetaData.fromJson(readJsonMorpheusGraphMetaData(graphName)) } - override protected def writeMorpheusGraphMetaData(graphName: GraphName, morpheusGraphMetaData: MorpheusGraphMetaData): Unit = { + override protected def writeMorpheusGraphMetaData( + graphName: GraphName, + morpheusGraphMetaData: MorpheusGraphMetaData + ): Unit = { writeJsonMorpheusGraphMetaData(graphName, morpheusGraphMetaData.toJson) } diff --git a/morpheus-spark-cypher/src/main/scala/org/opencypher/morpheus/api/io/neo4j/AbstractNeo4jDataSource.scala b/morpheus-spark-cypher/src/main/scala/org/opencypher/morpheus/api/io/neo4j/AbstractNeo4jDataSource.scala index 969fa9c6fb..66cc31e08f 100644 --- a/morpheus-spark-cypher/src/main/scala/org/opencypher/morpheus/api/io/neo4j/AbstractNeo4jDataSource.scala +++ b/morpheus-spark-cypher/src/main/scala/org/opencypher/morpheus/api/io/neo4j/AbstractNeo4jDataSource.scala @@ -41,11 +41,21 @@ abstract class AbstractNeo4jDataSource extends AbstractPropertyGraphDataSource { override def tableStorageFormat: StorageFormat = Neo4jFormat - override protected[io] def readSchema(graphName: GraphName): MorpheusSchema = { + override protected[io] def readSchema( + graphName: GraphName + ): MorpheusSchema = { SchemaFromProcedure(config, omitIncompatibleProperties).asMorpheus } - override protected def writeSchema(graphName: GraphName, schema: MorpheusSchema): Unit = () - override protected def readMorpheusGraphMetaData(graphName: GraphName): MorpheusGraphMetaData = MorpheusGraphMetaData(tableStorageFormat.name) - override protected def writeMorpheusGraphMetaData(graphName: GraphName, morpheusGraphMetaData: MorpheusGraphMetaData): Unit = () + override protected def writeSchema( + graphName: GraphName, + schema: MorpheusSchema + ): Unit = () + override protected def readMorpheusGraphMetaData( + graphName: GraphName + ): MorpheusGraphMetaData = MorpheusGraphMetaData(tableStorageFormat.name) + override protected def writeMorpheusGraphMetaData( + graphName: GraphName, + morpheusGraphMetaData: MorpheusGraphMetaData + ): Unit = () } diff --git a/morpheus-spark-cypher/src/main/scala/org/opencypher/morpheus/api/io/neo4j/Neo4jBulkCSVDataSink.scala b/morpheus-spark-cypher/src/main/scala/org/opencypher/morpheus/api/io/neo4j/Neo4jBulkCSVDataSink.scala index 45b26df8f5..471175be7f 100644 --- a/morpheus-spark-cypher/src/main/scala/org/opencypher/morpheus/api/io/neo4j/Neo4jBulkCSVDataSink.scala +++ b/morpheus-spark-cypher/src/main/scala/org/opencypher/morpheus/api/io/neo4j/Neo4jBulkCSVDataSink.scala @@ -65,13 +65,17 @@ object Neo4jBulkCSVDataSink { implicit class DataTypeOps(val dt: DataType) extends AnyVal { def toNeo4jBulkImportType: String = { dt match { - case StringType => "string" - case LongType => "int" - case BooleanType => "boolean" - case DoubleType => "double" + case StringType => "string" + case LongType => "int" + case BooleanType => "boolean" + case DoubleType => "double" case ArrayType(inner, _) => s"${inner.toNeo4jBulkImportType}[]" - case NullType => "string" - case other => throw IllegalArgumentException("supported Neo4j bulk import type", other) + case NullType => "string" + case other => + throw IllegalArgumentException( + "supported Neo4j bulk import type", + other + ) } } } @@ -79,28 +83,40 @@ object Neo4jBulkCSVDataSink { } /** - * This data sink writes a [[PropertyGraph]] into the Neo4j Bulk Import Format - * (see [[https://neo4j.com/docs/operations-manual/current/tools/import/]]). + * This data sink writes a [[PropertyGraph]] into the Neo4j Bulk Import Format (see + * [[https://neo4j.com/docs/operations-manual/current/tools/import/]]). * - * In addition, it generates an import shell script that is parameterized with the path to the Neo4j installation and - * runs the import. + * In addition, it generates an import shell script that is parameterized with the path to the + * Neo4j installation and runs the import. * - * @param rootPath Directory where the graph is being stored in - * @param arrayDelimiter Delimiter for array properties - * @param morpheus Morpheus session + * @param rootPath + * Directory where the graph is being stored in + * @param arrayDelimiter + * Delimiter for array properties + * @param morpheus + * Morpheus session */ -class Neo4jBulkCSVDataSink(override val rootPath: String, arrayDelimiter: String = "|")(implicit morpheus: MorpheusSession) - extends CsvGraphSource(rootPath) { +class Neo4jBulkCSVDataSink( + override val rootPath: String, + arrayDelimiter: String = "|" +)(implicit morpheus: MorpheusSession) + extends CsvGraphSource(rootPath) { override protected def writeSchema( graphName: GraphName, schema: MorpheusSchema ): Unit = { val nodeArguments = schema.labelCombinations.combos.toSeq.map { labels => - s"""--nodes:${labels.mkString(":")} "${schemaFileForNodes(graphName, labels)},${dataFileForNodes(graphName, labels)}"""" + s"""--nodes:${labels.mkString(":")} "${schemaFileForNodes( + graphName, + labels + )},${dataFileForNodes(graphName, labels)}"""" } val relArguments = schema.relationshipTypes.toSeq.map { relType => - s"""--relationships:$relType "${schemaFileForRelationships(graphName, relType)},${dataFileForRelationships(graphName, relType)}"""" + s"""--relationships:$relType "${schemaFileForRelationships( + graphName, + relType + )},${dataFileForRelationships(graphName, relType)}"""" } val importScript = String.format( @@ -108,30 +124,42 @@ class Neo4jBulkCSVDataSink(override val rootPath: String, arrayDelimiter: String graphName.value, arrayDelimiter, nodeArguments.mkString(" ", " \\\n ", ""), - relArguments.mkString(" ", " \\\n ", "")) + relArguments.mkString(" ", " \\\n ", "") + ) - fileSystem.writeFile(directoryStructure.pathToGraphDirectory(graphName) / SCRIPT_NAME, importScript) + fileSystem.writeFile( + directoryStructure.pathToGraphDirectory(graphName) / SCRIPT_NAME, + importScript + ) } def schemaFileForNodes( graphName: GraphName, labels: Set[String] - ): String = directoryStructure.pathToNodeTable(graphName, labels).replaceFirst(SCHEME_REGEX, "") / "schema.csv" + ): String = directoryStructure + .pathToNodeTable(graphName, labels) + .replaceFirst(SCHEME_REGEX, "") / "schema.csv" def dataFileForNodes( graphName: GraphName, labels: Set[String] - ): String = directoryStructure.pathToNodeTable(graphName, labels).replaceFirst(SCHEME_REGEX, "") / "part(.*)\\.csv" + ): String = directoryStructure + .pathToNodeTable(graphName, labels) + .replaceFirst(SCHEME_REGEX, "") / "part(.*)\\.csv" def schemaFileForRelationships( graphName: GraphName, relType: String - ): String = directoryStructure.pathToRelationshipTable(graphName, relType).replaceFirst(SCHEME_REGEX, "") / "schema.csv" + ): String = directoryStructure + .pathToRelationshipTable(graphName, relType) + .replaceFirst(SCHEME_REGEX, "") / "schema.csv" def dataFileForRelationships( graphName: GraphName, relType: String - ): String = directoryStructure.pathToRelationshipTable(graphName, relType).replaceFirst(SCHEME_REGEX, "") / "part(.*)\\.csv" + ): String = directoryStructure + .pathToRelationshipTable(graphName, relType) + .replaceFirst(SCHEME_REGEX, "") / "part(.*)\\.csv" override protected def writeNodeTable( graphName: GraphName, @@ -148,42 +176,69 @@ class Neo4jBulkCSVDataSink(override val rootPath: String, arrayDelimiter: String relKey: String, table: DataFrame ): Unit = { - val tableWithoutId = table.drop(table.schema.fieldNames.find(_ == GraphElement.sourceIdKey).get) - - super.writeRelationshipTable(graphName, relKey, stringifyArrayColumns(tableWithoutId)) - - writeHeaderFile(schemaFileForRelationships(graphName, relKey), tableWithoutId.schema.fields) + val tableWithoutId = table.drop( + table.schema.fieldNames.find(_ == GraphElement.sourceIdKey).get + ) + + super.writeRelationshipTable( + graphName, + relKey, + stringifyArrayColumns(tableWithoutId) + ) + + writeHeaderFile( + schemaFileForRelationships(graphName, relKey), + tableWithoutId.schema.fields + ) } private def stringifyArrayColumns(table: DataFrame): DataFrame = { - val arrayColumns = table.schema.fields.collect { - case StructField(name, _: ArrayType, _, _) => name + val arrayColumns = table.schema.fields.collect { case StructField(name, _: ArrayType, _, _) => + name } - arrayColumns.foldLeft(table) { - case (acc, arrayColumn) => acc.withColumn(arrayColumn, functions.concat_ws(arrayDelimiter, acc.col(arrayColumn))) - }.select(table.columns.head, table.columns.tail: _*) + arrayColumns + .foldLeft(table) { case (acc, arrayColumn) => + acc.withColumn( + arrayColumn, + functions.concat_ws(arrayDelimiter, acc.col(arrayColumn)) + ) + } + .select(table.columns.head, table.columns.tail: _*) } - private def writeHeaderFile(path: String, fields: Array[StructField]): Unit = { - val neoSchema = fields.map { - //TODO: use Neo4jDefaults here - case field if field.name == GraphElement.sourceIdKey => s"___morpheusID:ID" - case field if field.name == Relationship.sourceStartNodeKey => ":START_ID" - case field if field.name == Relationship.sourceEndNodeKey => ":END_ID" - case field if field.name.isPropertyColumnName => s"${field.name.toProperty}:${field.dataType.toNeo4jBulkImportType}" - }.mkString(",") + private def writeHeaderFile( + path: String, + fields: Array[StructField] + ): Unit = { + val neoSchema = fields + .map { + // TODO: use Neo4jDefaults here + case field if field.name == GraphElement.sourceIdKey => + s"___morpheusID:ID" + case field if field.name == Relationship.sourceStartNodeKey => + ":START_ID" + case field if field.name == Relationship.sourceEndNodeKey => ":END_ID" + case field if field.name.isPropertyColumnName => + s"${field.name.toProperty}:${field.dataType.toNeo4jBulkImportType}" + } + .mkString(",") fileSystem.writeFile(path, neoSchema) } override def hasGraph(graphName: GraphName): Boolean = false - override def graphNames: Set[GraphName] = throw UnsupportedOperationException("Write-only PGDS") + override def graphNames: Set[GraphName] = throw UnsupportedOperationException( + "Write-only PGDS" + ) - override def delete(graphName: GraphName): Unit = throw UnsupportedOperationException("Write-only PGDS") + override def delete(graphName: GraphName): Unit = + throw UnsupportedOperationException("Write-only PGDS") - override def graph(name: GraphName): PropertyGraph = throw UnsupportedOperationException("Write-only PGDS") + override def graph(name: GraphName): PropertyGraph = + throw UnsupportedOperationException("Write-only PGDS") - override def schema(graphName: GraphName): Option[MorpheusSchema] = throw UnsupportedOperationException("Write-only PGDS") + override def schema(graphName: GraphName): Option[MorpheusSchema] = + throw UnsupportedOperationException("Write-only PGDS") } diff --git a/morpheus-spark-cypher/src/main/scala/org/opencypher/morpheus/api/io/neo4j/Neo4jPropertyGraphDataSource.scala b/morpheus-spark-cypher/src/main/scala/org/opencypher/morpheus/api/io/neo4j/Neo4jPropertyGraphDataSource.scala index 5fabbadc2f..66855b26c4 100644 --- a/morpheus-spark-cypher/src/main/scala/org/opencypher/morpheus/api/io/neo4j/Neo4jPropertyGraphDataSource.scala +++ b/morpheus-spark-cypher/src/main/scala/org/opencypher/morpheus/api/io/neo4j/Neo4jPropertyGraphDataSource.scala @@ -63,22 +63,24 @@ case class Neo4jPropertyGraphDataSource( override val config: Neo4jConfig, maybeSchema: Option[PropertyGraphSchema] = None, override val omitIncompatibleProperties: Boolean = false -)(implicit val morpheus: MorpheusSession) extends AbstractNeo4jDataSource with Logging { +)(implicit val morpheus: MorpheusSession) + extends AbstractNeo4jDataSource + with Logging { graphNameCache += entireGraphName override def hasGraph(graphName: GraphName): Boolean = graphName match { case `entireGraphName` => true - case _ => super.hasGraph(graphName) + case _ => super.hasGraph(graphName) } override protected def listGraphNames: List[String] = { - val labelResult = config.cypherWithNewSession( - """|CALL db.labels() + val labelResult = config.cypherWithNewSession("""|CALL db.labels() |YIELD label |RETURN collect(label) AS labels """.stripMargin) - val allLabels = labelResult.head("labels").cast[CypherList].value.map(_.toString) + val allLabels = + labelResult.head("labels").cast[CypherList].value.map(_.toString) val metaLabelGraphNames = allLabels .filter(_.startsWith(metaPrefix)) @@ -92,14 +94,19 @@ case class Neo4jPropertyGraphDataSource( maybeSchema.getOrElse(super.readSchema(entireGraphName)) } - override protected[io] def readSchema(graphName: GraphName): MorpheusSchema = { + override protected[io] def readSchema( + graphName: GraphName + ): MorpheusSchema = { val filteredSchema = graphName.getMetaLabel match { case None => entireGraphSchema case Some(metaLabel) => - val containsMetaLabel = entireGraphSchema.labelPropertyMap.filterForLabels(metaLabel) - val cleanLabelPropertyMap = containsMetaLabel.withoutMetaLabel(metaLabel).withoutMetaProperty - val cleanRelTypePropertyMap = entireGraphSchema.relTypePropertyMap.withoutMetaProperty + val containsMetaLabel = + entireGraphSchema.labelPropertyMap.filterForLabels(metaLabel) + val cleanLabelPropertyMap = + containsMetaLabel.withoutMetaLabel(metaLabel).withoutMetaProperty + val cleanRelTypePropertyMap = + entireGraphSchema.relTypePropertyMap.withoutMetaProperty PropertyGraphSchemaImpl(cleanLabelPropertyMap, cleanRelTypePropertyMap) } filteredSchema.asMorpheus @@ -111,7 +118,11 @@ case class Neo4jPropertyGraphDataSource( sparkSchema: StructType ): DataFrame = { val graphSchema = schema(graphName).get - val flatQuery = ElementReader.flatExactLabelQuery(labels, graphSchema, graphName.getMetaLabel) + val flatQuery = ElementReader.flatExactLabelQuery( + labels, + graphSchema, + graphName.getMetaLabel + ) val neo4jConnection = Neo4j(config, morpheus.sparkSession) val rdd = neo4jConnection.cypher(flatQuery).loadRowRdd @@ -128,7 +139,11 @@ case class Neo4jPropertyGraphDataSource( sparkSchema: StructType ): DataFrame = { val graphSchema = schema(graphName).get - val flatQuery = ElementReader.flatRelTypeQuery(relKey, graphSchema, graphName.getMetaLabel) + val flatQuery = ElementReader.flatRelTypeQuery( + relKey, + graphSchema, + graphName.getMetaLabel + ) val neo4jConnection = Neo4j(config, morpheus.sparkSession) val rdd = neo4jConnection.cypher(flatQuery).loadRowRdd @@ -136,19 +151,25 @@ case class Neo4jPropertyGraphDataSource( // encode Neo4j identifiers to BinaryType morpheus.sparkSession .createDataFrame(rdd, sparkSchema.convertTypes(BinaryType, LongType)) - .transformColumns(idPropertyKey, startIdPropertyKey, endIdPropertyKey)(_.encodeLongAsMorpheusId) + .transformColumns(idPropertyKey, startIdPropertyKey, endIdPropertyKey)( + _.encodeLongAsMorpheusId + ) } override protected def deleteGraph(graphName: GraphName): Unit = { graphName.getMetaLabel match { case Some(metaLabel) => config.withSession { session => - session.run( - s"""|MATCH (n:$metaLabel) + session + .run(s"""|MATCH (n:$metaLabel) |DETACH DELETE n - """.stripMargin).consume() + """.stripMargin) + .consume() } - case None => throw UnsupportedOperationException("Deleting the entire Neo4j graph is not supported") + case None => + throw UnsupportedOperationException( + "Deleting the entire Neo4j graph is not supported" + ) } } @@ -159,12 +180,21 @@ case class Neo4jPropertyGraphDataSource( val metaLabel = graphName.getMetaLabel match { case Some(meta) => meta - case None => throw UnsupportedOperationException("Writing to the global Neo4j graph is not supported") + case None => + throw UnsupportedOperationException( + "Writing to the global Neo4j graph is not supported" + ) } config.withSession { session => - logger.debug(s"Creating database uniqueness constraint on ${metaLabel.cypherLabelPredicate}.$metaPropertyKey") - session.run(s"CREATE CONSTRAINT ON (n${metaLabel.cypherLabelPredicate}) ASSERT n.$metaPropertyKey IS UNIQUE").consume() + logger.debug( + s"Creating database uniqueness constraint on ${metaLabel.cypherLabelPredicate}.$metaPropertyKey" + ) + session + .run( + s"CREATE CONSTRAINT ON (n${metaLabel.cypherLabelPredicate}) ASSERT n.$metaPropertyKey IS UNIQUE" + ) + .consume() } val writesCompleted = for { @@ -178,46 +208,58 @@ case class Neo4jPropertyGraphDataSource( } // No need to implement these as we overwrite {{{org.opencypher.morppheus.api.io.neo4j.Neo4jPropertyGraphDataSource.store}}} - override protected def writeNodeTable(graphName: GraphName, labels: Set[String], table: DataFrame): Unit = () - override protected def writeRelationshipTable(graphName: GraphName, relKey: String, table: DataFrame): Unit = () + override protected def writeNodeTable( + graphName: GraphName, + labels: Set[String], + table: DataFrame + ): Unit = () + override protected def writeRelationshipTable( + graphName: GraphName, + relKey: String, + table: DataFrame + ): Unit = () } case object Writers { - def writeNodes(graph: PropertyGraph, metaLabel: String, config: Neo4jConfig) - (implicit morpheus: MorpheusSession): Set[Future[Unit]] = { + def writeNodes(graph: PropertyGraph, metaLabel: String, config: Neo4jConfig)(implicit + morpheus: MorpheusSession + ): Set[Future[Unit]] = { val result: Set[Future[Unit]] = graph.schema.labelCombinations.combos.map { combo => - val nodeScan = graph.nodes("n", CTNode(combo), exactLabelMatch = true).asMorpheus + val nodeScan = + graph.nodes("n", CTNode(combo), exactLabelMatch = true).asMorpheus val mapping = computeMapping(nodeScan) - nodeScan - .df - .encodeBinaryToHexString - .rdd + nodeScan.df.encodeBinaryToHexString.rdd .foreachPartitionAsync { i => - if (i.nonEmpty) ElementWriter.createNodes(i, mapping, config, combo + metaLabel)(rowToListValue) + if (i.nonEmpty) + ElementWriter.createNodes(i, mapping, config, combo + metaLabel)( + rowToListValue + ) } } result } - def writeRelationships(graph: PropertyGraph, metaLabel: String, config: Neo4jConfig) - (implicit morpheus: MorpheusSession): Set[Future[Unit]] = { + def writeRelationships( + graph: PropertyGraph, + metaLabel: String, + config: Neo4jConfig + )(implicit morpheus: MorpheusSession): Set[Future[Unit]] = { graph.schema.relationshipTypes.map { relType => val relScan = graph.relationships("r", CTRelationship(relType)).asMorpheus val mapping = computeMapping(relScan) val header = relScan.header val relVar = header.elementVars.head - val startExpr = header.expressionsFor(relVar).collect { case s: StartNode => s }.head - val endExpr = header.expressionsFor(relVar).collect { case e: EndNode => e }.head + val startExpr = + header.expressionsFor(relVar).collect { case s: StartNode => s }.head + val endExpr = + header.expressionsFor(relVar).collect { case e: EndNode => e }.head val startColumn = relScan.header.column(startExpr) val endColumn = relScan.header.column(endExpr) val startIndex = relScan.df.columns.indexOf(startColumn) val endIndex = relScan.df.columns.indexOf(endColumn) - relScan - .df - .encodeBinaryToHexString - .rdd + relScan.df.encodeBinaryToHexString.rdd .foreachPartitionAsync { i => if (i.nonEmpty) { ElementWriter.createRelationships( @@ -238,10 +280,10 @@ case object Writers { def castValue(v: Any): Any = v match { case a: mutable.WrappedArray[_] if a.size == 1 && a.head == null => null case a: mutable.WrappedArray[_] => a.array.map(o => castValue(o)) - case d: java.sql.Date => d.toLocalDate - case ts: java.sql.Timestamp => ts.toLocalDateTime - case ci: CalendarInterval => ci.toJavaDuration - case other => other + case d: java.sql.Date => d.toLocalDate + case ts: java.sql.Timestamp => ts.toLocalDateTime + case ci: CalendarInterval => ci.toJavaDuration + case other => other } val array = new Array[Value](row.size) @@ -256,8 +298,8 @@ case object Writers { private def computeMapping(nodeScan: MorpheusRecords): Array[String] = { val header = nodeScan.header val nodeVar = header.elementVars.head - val properties: Set[Property] = header.expressionsFor(nodeVar).collect { - case p: Property => p + val properties: Set[Property] = header.expressionsFor(nodeVar).collect { case p: Property => + p } val columns = nodeScan.df.columns diff --git a/morpheus-spark-cypher/src/main/scala/org/opencypher/morpheus/api/io/neo4j/sync/Neo4jGraphMerge.scala b/morpheus-spark-cypher/src/main/scala/org/opencypher/morpheus/api/io/neo4j/sync/Neo4jGraphMerge.scala index d34109c00f..9eb799f4fa 100644 --- a/morpheus-spark-cypher/src/main/scala/org/opencypher/morpheus/api/io/neo4j/sync/Neo4jGraphMerge.scala +++ b/morpheus-spark-cypher/src/main/scala/org/opencypher/morpheus/api/io/neo4j/sync/Neo4jGraphMerge.scala @@ -47,32 +47,37 @@ import scala.concurrent.ExecutionContext.Implicits.global import scala.concurrent.duration._ import scala.concurrent.{Await, Future} -/** - * Utility class that allows to merge a graph into an existing Neo4j database. - */ +/** Utility class that allows to merge a graph into an existing Neo4j database. */ object Neo4jGraphMerge extends Logging { /** * Defines a set of properties which uniquely identify a node with a given label. * - * @see [[org.opencypher.okapi.api.schema.PropertyGraphSchema#nodeKeys]] + * @see + * [[org.opencypher.okapi.api.schema.PropertyGraphSchema#nodeKeys]] */ type NodeKeys = Map[String, Set[String]] + /** * Defines a set of properties which uniquely identify a relationship with a given type. * - * @see [[org.opencypher.okapi.api.schema.PropertyGraphSchema#relationshipKeys]] + * @see + * [[org.opencypher.okapi.api.schema.PropertyGraphSchema#relationshipKeys]] */ type RelationshipKeys = Map[String, Set[String]] /** - * Creates node indexes for the sub-graph specified by `graphName` in the specified Neo4j database. - * This speeds up the Neo4j merge feature. + * Creates node indexes for the sub-graph specified by `graphName` in the specified Neo4j + * database. This speeds up the Neo4j merge feature. * - * @note This feature requires the Neo4j Enterprise Edition. - * @param graphName which sub-graph to create the indexes for - * @param config access config for the Neo4j database on which the indexes are created - * @param nodeKeys node keys that identify a node uniquely + * @note + * This feature requires the Neo4j Enterprise Edition. + * @param graphName + * which sub-graph to create the indexes for + * @param config + * access config for the Neo4j database on which the indexes are created + * @param nodeKeys + * node keys that identify a node uniquely */ def createIndexes( graphName: GraphName, @@ -84,50 +89,63 @@ object Neo4jGraphMerge extends Logging { config.withSession { session => maybeMetaLabel match { case None => - nodeKeys.foreach { - case (label, keys) => - val nodeVar = "n" - val propertyString = keys.map(k => s"$nodeVar.`$k`").mkString("(", ", ", ")") - val query = s"CREATE CONSTRAINT ON ($nodeVar${label.cypherLabelPredicate}) ASSERT $propertyString IS NODE KEY" - logger.debug(s"Creating node key constraints: $query") - session.run(query).consume + nodeKeys.foreach { case (label, keys) => + val nodeVar = "n" + val propertyString = + keys.map(k => s"$nodeVar.`$k`").mkString("(", ", ", ")") + val query = + s"CREATE CONSTRAINT ON ($nodeVar${label.cypherLabelPredicate}) ASSERT $propertyString IS NODE KEY" + logger.debug(s"Creating node key constraints: $query") + session.run(query).consume } nodeKeys.keySet.foreach { label => - val cmd = s"CREATE INDEX ON ${label.cypherLabelPredicate}(`$metaPropertyKey`)" + val cmd = + s"CREATE INDEX ON ${label.cypherLabelPredicate}(`$metaPropertyKey`)" logger.debug(s"Creating index for meta property key: $cmd") session.run(cmd).consume } case Some(ml) => - nodeKeys.foreach { - case (label, properties) => - val propertyString = properties.map(p => s"`$p`").mkString("(", ", ", ")") - val cmd = s"CREATE INDEX ON ${label.cypherLabelPredicate}$propertyString" - logger.debug(s"Creating index for node keys: $cmd") - session.run(cmd).consume + nodeKeys.foreach { case (label, properties) => + val propertyString = + properties.map(p => s"`$p`").mkString("(", ", ", ")") + val cmd = + s"CREATE INDEX ON ${label.cypherLabelPredicate}$propertyString" + logger.debug(s"Creating index for node keys: $cmd") + session.run(cmd).consume } - val command = s"CREATE INDEX ON ${ml.cypherLabelPredicate}(`$metaPropertyKey`)" - logger.debug(s"Creating sub-graph index for meta label and meta property key: $command") + val command = + s"CREATE INDEX ON ${ml.cypherLabelPredicate}(`$metaPropertyKey`)" + logger.debug( + s"Creating sub-graph index for meta label and meta property key: $command" + ) session.run(command).consume } } } /** - * Merges the given graph into the sub-graph specified by `graphName` within an existing Neo4j database. - * Properties in the Neo4j graph will be overwritten by values in the merge graph, missing ones are added. + * Merges the given graph into the sub-graph specified by `graphName` within an existing Neo4j + * database. Properties in the Neo4j graph will be overwritten by values in the merge graph, + * missing ones are added. * - * Nodes and relationships are identified by their element keys defined by the graph's schema. They can be overridden - * by optional node and relationship keys + * Nodes and relationships are identified by their element keys defined by the graph's schema. + * They can be overridden by optional node and relationship keys * - * @param graphName which sub-graph in the Neo4j graph to merge the delta to - * @param graph graph that is merged into the existing Neo4j database - * @param config access config for the Neo4j database into which the graph is merged - * @param nodeKeys additional node keys that override node keys defined by the schema - * @param relationshipKeys additional relationship keys that override relationship keys defined by the schema - * @param morpheus Morpheus session + * @param graphName + * which sub-graph in the Neo4j graph to merge the delta to + * @param graph + * graph that is merged into the existing Neo4j database + * @param config + * access config for the Neo4j database into which the graph is merged + * @param nodeKeys + * additional node keys that override node keys defined by the schema + * @param relationshipKeys + * additional relationship keys that override relationship keys defined by the schema + * @param morpheus + * Morpheus session */ def merge( graphName: GraphName, @@ -136,17 +154,34 @@ object Neo4jGraphMerge extends Logging { nodeKeys: Option[NodeKeys] = None, relationshipKeys: Option[RelationshipKeys] = None )(implicit morpheus: MorpheusSession): Unit = { - val updatedSchema = combineElementKeys(graph.schema, nodeKeys, relationshipKeys) + val updatedSchema = + combineElementKeys(graph.schema, nodeKeys, relationshipKeys) val maybeMetaLabel = graphName.getMetaLabel val maybeMetaLabelString = maybeMetaLabel.toSet[String].cypherLabelPredicate val writesCompleted = for { - _ <- Future.sequence(MergeWriters.writeNodes(maybeMetaLabel, graph, config, updatedSchema.nodeKeys)) - _ <- Future.sequence(MergeWriters.writeRelationships(maybeMetaLabel, graph, config, updatedSchema.relationshipKeys)) + _ <- Future.sequence( + MergeWriters.writeNodes( + maybeMetaLabel, + graph, + config, + updatedSchema.nodeKeys + ) + ) + _ <- Future.sequence( + MergeWriters.writeRelationships( + maybeMetaLabel, + graph, + config, + updatedSchema.relationshipKeys + ) + ) _ <- Future { config.withSession { session => - session.run(s"MATCH (n$maybeMetaLabelString) REMOVE n.$metaPropertyKey").consume() + session + .run(s"MATCH (n$maybeMetaLabelString) REMOVE n.$metaPropertyKey") + .consume() } } } yield Future {} @@ -160,11 +195,11 @@ object Neo4jGraphMerge extends Logging { nodeKeys: Option[NodeKeys], relationshipKeys: Option[RelationshipKeys] ): PropertyGraphSchema = { - val withNodeKeys = nodeKeys.getOrElse(Map.empty).foldLeft(schema) { - case (acc, (label, keys)) => acc.withNodeKey(label, keys) + val withNodeKeys = nodeKeys.getOrElse(Map.empty).foldLeft(schema) { case (acc, (label, keys)) => + acc.withNodeKey(label, keys) } - relationshipKeys.getOrElse(Map.empty).foldLeft(withNodeKeys) { - case (acc, (typ, keys)) => acc.withRelationshipKey(typ, keys) + relationshipKeys.getOrElse(Map.empty).foldLeft(withNodeKeys) { case (acc, (typ, keys)) => + acc.withRelationshipKey(typ, keys) } } } @@ -178,17 +213,28 @@ case object MergeWriters { ): Set[Future[Unit]] = { val result: Set[Future[Unit]] = graph.schema.labelCombinations.combos.map { combo => val comboWithMetaLabel = combo ++ maybeMetaLabel - val nodeScan = graph.nodes("n", CTNode(combo), exactLabelMatch = true).asMorpheus + val nodeScan = + graph.nodes("n", CTNode(combo), exactLabelMatch = true).asMorpheus val mapping = computeMapping(nodeScan, includeId = true) - val keys = combo.find(nodeKeys.contains).map(nodeKeys).getOrElse( - throw SchemaException(s"Could not find a node key for label combination $combo") - ) + val keys = combo + .find(nodeKeys.contains) + .map(nodeKeys) + .getOrElse( + throw SchemaException( + s"Could not find a node key for label combination $combo" + ) + ) - nodeScan - .df - .rdd + nodeScan.df.rdd .foreachPartitionAsync { i => - if (i.nonEmpty) ElementWriter.mergeNodes(i, mapping, config, comboWithMetaLabel, keys)(rowToListValue) + if (i.nonEmpty) + ElementWriter.mergeNodes( + i, + mapping, + config, + comboWithMetaLabel, + keys + )(rowToListValue) } } result @@ -206,16 +252,16 @@ case object MergeWriters { val header = relScan.header val relVar = header.elementVars.head - val startExpr = header.expressionsFor(relVar).collect { case s: StartNode => s }.head - val endExpr = header.expressionsFor(relVar).collect { case e: EndNode => e }.head + val startExpr = + header.expressionsFor(relVar).collect { case s: StartNode => s }.head + val endExpr = + header.expressionsFor(relVar).collect { case e: EndNode => e }.head val startColumn = relScan.header.column(startExpr) val endColumn = relScan.header.column(endExpr) val startIndex = relScan.df.columns.indexOf(startColumn) val endIndex = relScan.df.columns.indexOf(endColumn) - relScan - .df - .rdd + relScan.df.rdd .foreachPartitionAsync { i => if (i.nonEmpty) { ElementWriter.mergeRelationships( @@ -243,11 +289,14 @@ case object MergeWriters { new ListValue(array: _*) } - private def computeMapping(elementScan: MorpheusRecords, includeId: Boolean): Array[String] = { + private def computeMapping( + elementScan: MorpheusRecords, + includeId: Boolean + ): Array[String] = { val header = elementScan.header val nodeVar = header.elementVars.head - val properties: Set[Property] = header.expressionsFor(nodeVar).collect { - case p: Property => p + val properties: Set[Property] = header.expressionsFor(nodeVar).collect { case p: Property => + p } val columns = elementScan.df.columns diff --git a/morpheus-spark-cypher/src/main/scala/org/opencypher/morpheus/api/io/sql/GraphDdlConversions.scala b/morpheus-spark-cypher/src/main/scala/org/opencypher/morpheus/api/io/sql/GraphDdlConversions.scala index a0babb93bb..66086873a4 100644 --- a/morpheus-spark-cypher/src/main/scala/org/opencypher/morpheus/api/io/sql/GraphDdlConversions.scala +++ b/morpheus-spark-cypher/src/main/scala/org/opencypher/morpheus/api/io/sql/GraphDdlConversions.scala @@ -79,7 +79,7 @@ object GraphDdlConversions { .map { case (label, (parents, propertyKeys)) => val maybeKey = allKeys.get(label) match { case Some(s) => Some(label -> s) - case None => None + case None => None } ElementType(label, parents, propertyKeys, maybeKey) } @@ -89,11 +89,16 @@ object GraphDdlConversions { graphType.withElementType(elementType) } .foldLeftOver(schema.labelCombinations.combos) { case (graphType, labelCombo) => - val nodeLabels = if (labelCombo.isEmpty) Set(NO_LABEL) else labelCombo + val nodeLabels = + if (labelCombo.isEmpty) Set(NO_LABEL) else labelCombo graphType.withNodeType(nodeLabels.toSeq: _*) } .foldLeftOver(schema.schemaPatterns) { case (graphType, pattern) => - graphType.withRelationshipType(pattern.sourceLabelCombination, Set(pattern.relType), pattern.targetLabelCombination) + graphType.withRelationshipType( + pattern.sourceLabelCombination, + Set(pattern.relType), + pattern.targetLabelCombination + ) } } @@ -105,41 +110,55 @@ object GraphDdlConversions { output } else { - val sortedElementTypes = elementTypes.sortBy { case (name, keys) => (name, keys.size) } + val sortedElementTypes = elementTypes.sortBy { case (name, keys) => + (name, keys.size) + } val (label, propertyKeys) = sortedElementTypes.head val propertyKeysSet = propertyKeys.toSet.filterNot(_._2.isNullable) val updatedOutput = sortedElementTypes.foldLeft(output) { - case (currentOutput, (currentLabel, _)) if currentLabel == label && currentOutput.contains(currentLabel) => + case (currentOutput, (currentLabel, _)) + if currentLabel == label && currentOutput.contains( + currentLabel + ) => currentOutput case (currentOutput, (currentLabel, currentPropertyKeys)) if currentLabel == label => - currentOutput.updated(currentLabel, Set.empty[String] -> currentPropertyKeys) + currentOutput.updated( + currentLabel, + Set.empty[String] -> currentPropertyKeys + ) case (currentOutput, (currentLabel, currentPropertyKeys)) => val currentPropertyKeysSet = currentPropertyKeys.toSet - val updatedPropertyKeys = if (propertyKeysSet.subsetOf(currentPropertyKeysSet)) { - (currentPropertyKeysSet -- propertyKeysSet).toMap - } else { - currentPropertyKeys - } + val updatedPropertyKeys = + if (propertyKeysSet.subsetOf(currentPropertyKeysSet)) { + (currentPropertyKeysSet -- propertyKeysSet).toMap + } else { + currentPropertyKeys + } val isSubType = updatedPropertyKeys.size < currentPropertyKeys.size val currentParents = currentOutput.get(currentLabel) match { case Some((parents, _)) => parents - case None => Set.empty[String] + case None => Set.empty[String] } val updatedParents = if (isSubType) { currentParents + label } else { currentParents } - currentOutput.updated(currentLabel, updatedParents -> updatedPropertyKeys) + currentOutput.updated( + currentLabel, + updatedParents -> updatedPropertyKeys + ) } - val remainingElementTypes = sortedElementTypes.tail.map { case (name, _) => name -> updatedOutput(name)._2 } + val remainingElementTypes = sortedElementTypes.tail.map { case (name, _) => + name -> updatedOutput(name)._2 + } extractInheritance(remainingElementTypes, updatedOutput) } @@ -160,22 +179,29 @@ object GraphDdlConversions { } lazy val allPatterns: Set[SchemaPattern] = - graphType.relTypes.map(relType => SchemaPattern( - sourceLabelCombination = relType.startNodeType.labels, - relType = getSingleLabel(relType), - targetLabelCombination = relType.endNodeType.labels - )) + graphType.relTypes.map(relType => + SchemaPattern( + sourceLabelCombination = relType.startNodeType.labels, + relType = getSingleLabel(relType), + targetLabelCombination = relType.endNodeType.labels + ) + ) def asOkapiSchema: PropertyGraphSchema = PropertyGraphSchema.empty .foldLeftOver(graphType.nodeTypes) { case (schema, nodeType) => - val combo = if (nodeType.labels.contains(NO_LABEL)) Set.empty[String] else nodeType.labels + val combo = + if (nodeType.labels.contains(NO_LABEL)) Set.empty[String] + else nodeType.labels schema.withNodePropertyKeys(combo, graphType.nodePropertyKeys(nodeType)) } .foldLeftOver(graphType.nodeElementTypes) { case (schema, eType) => eType.maybeKey.fold(schema)(key => schema.withNodeKey(eType.name, key._2)) } .foldLeftOver(graphType.relTypes) { case (schema, relType) => - schema.withRelationshipPropertyKeys(getSingleLabel(relType), graphType.relationshipPropertyKeys(relType)) + schema.withRelationshipPropertyKeys( + getSingleLabel(relType), + graphType.relationshipPropertyKeys(relType) + ) } .foldLeftOver(graphType.relElementTypes) { case (schema, eType) => eType.maybeKey.fold(schema)(key => schema.withNodeKey(eType.name, key._2)) diff --git a/morpheus-spark-cypher/src/main/scala/org/opencypher/morpheus/api/io/sql/SqlDataSourceConfig.scala b/morpheus-spark-cypher/src/main/scala/org/opencypher/morpheus/api/io/sql/SqlDataSourceConfig.scala index b387bb52cd..9177f0d3ba 100644 --- a/morpheus-spark-cypher/src/main/scala/org/opencypher/morpheus/api/io/sql/SqlDataSourceConfig.scala +++ b/morpheus-spark-cypher/src/main/scala/org/opencypher/morpheus/api/io/sql/SqlDataSourceConfig.scala @@ -32,8 +32,8 @@ import ujson.Value import scala.util.{Failure, Success, Try} -case class SqlDataSourceConfigException(msg: String, cause: Throwable = null) extends Throwable(msg, cause) - +case class SqlDataSourceConfigException(msg: String, cause: Throwable = null) + extends Throwable(msg, cause) import org.opencypher.okapi.impl.util.JsonUtils.FlatOption._ @@ -51,24 +51,29 @@ object SqlDataSourceConfig { private final val UJSON_TYPE_KEY = "$type" private final val MORPHEUS_TYPE_KEY = "type" - implicit val rw: ReadWriter[SqlDataSourceConfig] = readwriter[Value].bimap[SqlDataSourceConfig]( - // Rename discriminator key from ujson default, to a more friendly version - cfg => writeJs(cfg)(defaultMacroRW) match { - case Obj(obj) => obj.collect { - case (UJSON_TYPE_KEY, value) => MORPHEUS_TYPE_KEY -> value - case other => other - } - case other => other // Note, case objects are serialised as strings - }, - // Revert name change so we can use the ujson reader - js => read[SqlDataSourceConfig](js match { - case Obj(obj) => obj.map { - case (MORPHEUS_TYPE_KEY, value) => UJSON_TYPE_KEY -> value - case other => other - } - case _ => js // Note, case objects are serialised as strings - })(defaultMacroRW) - ) + implicit val rw: ReadWriter[SqlDataSourceConfig] = + readwriter[Value].bimap[SqlDataSourceConfig]( + // Rename discriminator key from ujson default, to a more friendly version + cfg => + writeJs(cfg)(defaultMacroRW) match { + case Obj(obj) => + obj.collect { + case (UJSON_TYPE_KEY, value) => MORPHEUS_TYPE_KEY -> value + case other => other + } + case other => other // Note, case objects are serialised as strings + }, + // Revert name change so we can use the ujson reader + js => + read[SqlDataSourceConfig](js match { + case Obj(obj) => + obj.map { + case (MORPHEUS_TYPE_KEY, value) => UJSON_TYPE_KEY -> value + case other => other + } + case _ => js // Note, case objects are serialised as strings + })(defaultMacroRW) + ) def toJson(dataSource: SqlDataSourceConfig, indent: Int = 4): String = write[SqlDataSourceConfig](dataSource, indent) @@ -80,14 +85,21 @@ object SqlDataSourceConfig { Try(read[Map[String, SqlDataSourceConfig]](jsonStr)) match { case Success(result) => result case Failure(ex) => - throw SqlDataSourceConfigException(s"Malformed SQL configuration file: ${ex.getMessage}", ex) + throw SqlDataSourceConfigException( + s"Malformed SQL configuration file: ${ex.getMessage}", + ex + ) } - /** Configures a data source that reads tables via JDBC + /** + * Configures a data source that reads tables via JDBC * - * @param url the JDBC URI to use when connecting to the JDBC server - * @param driver class name of the JDBC driver to use for the JDBC connection - * @param options extra options passed to Spark when configuring the reader + * @param url + * the JDBC URI to use when connecting to the JDBC server + * @param driver + * class name of the JDBC driver to use for the JDBC connection + * @param options + * extra options passed to Spark when configuring the reader */ @upickle.implicits.key("jdbc") case class Jdbc( @@ -98,8 +110,10 @@ object SqlDataSourceConfig { override def format: StorageFormat = JdbcFormat } - /** Configures a data source that reads tables from Hive - * @note The Spark session needs to be configured with `.enableHiveSupport()` + /** + * Configures a data source that reads tables from Hive + * @note + * The Spark session needs to be configured with `.enableHiveSupport()` */ @upickle.implicits.key("hive") case object Hive extends SqlDataSourceConfig { @@ -107,11 +121,15 @@ object SqlDataSourceConfig { override def options: Map[String, String] = Map.empty } - /** Configures a data source that reads tables from files + /** + * Configures a data source that reads tables from files * - * @param format the file format passed to Spark when configuring the reader - * @param basePath the root folder used for file based formats - * @param options extra options passed to Spark when configuring the reader + * @param format + * the file format passed to Spark when configuring the reader + * @param basePath + * the root folder used for file based formats + * @param options + * extra options passed to Spark when configuring the reader */ @upickle.implicits.key("file") case class File( diff --git a/morpheus-spark-cypher/src/main/scala/org/opencypher/morpheus/api/io/sql/SqlPropertyGraphDataSource.scala b/morpheus-spark-cypher/src/main/scala/org/opencypher/morpheus/api/io/sql/SqlPropertyGraphDataSource.scala index eb4ab95775..0c440b6858 100644 --- a/morpheus-spark-cypher/src/main/scala/org/opencypher/morpheus/api/io/sql/SqlPropertyGraphDataSource.scala +++ b/morpheus-spark-cypher/src/main/scala/org/opencypher/morpheus/api/io/sql/SqlPropertyGraphDataSource.scala @@ -47,10 +47,18 @@ import org.opencypher.morpheus.impl.table.SparkTable._ import org.opencypher.morpheus.schema.MorpheusSchema import org.opencypher.morpheus.schema.MorpheusSchema._ import org.opencypher.okapi.api.graph._ -import org.opencypher.okapi.api.io.conversion.{ElementMapping, NodeMappingBuilder, RelationshipMappingBuilder} +import org.opencypher.okapi.api.io.conversion.{ + ElementMapping, + NodeMappingBuilder, + RelationshipMappingBuilder +} import org.opencypher.okapi.api.schema.PropertyGraphSchema import org.opencypher.okapi.api.types.{CTNode, CTRelationship, CTVoid} -import org.opencypher.okapi.impl.exception.{GraphNotFoundException, IllegalArgumentException, UnsupportedOperationException} +import org.opencypher.okapi.impl.exception.{ + GraphNotFoundException, + IllegalArgumentException, + UnsupportedOperationException +} import org.opencypher.okapi.impl.util.StringEncodingUtilities._ import scala.reflect.io.Path @@ -63,12 +71,22 @@ object SqlPropertyGraphDataSource { idGenerationStrategy: IdGenerationStrategy = SerializedId )(implicit morpheus: MorpheusSession): SqlPropertyGraphDataSource = { - val unsupportedDataSources = sqlDataSourceConfigs.filter { case (_, config) => config.format == FileFormat.csv } - if (unsupportedDataSources.nonEmpty) throw IllegalArgumentException( - expected = "Supported FileFormat for SQL Property Graph Data Source", - actual = s"${FileFormat.csv} used in the following data source configs: ${unsupportedDataSources.keys.mkString("[", ", ", "]")}") - - new SqlPropertyGraphDataSource(graphDdl, sqlDataSourceConfigs, idGenerationStrategy) + val unsupportedDataSources = sqlDataSourceConfigs.filter { case (_, config) => + config.format == FileFormat.csv + } + if (unsupportedDataSources.nonEmpty) + throw IllegalArgumentException( + expected = "Supported FileFormat for SQL Property Graph Data Source", + actual = + s"${FileFormat.csv} used in the following data source configs: ${unsupportedDataSources.keys + .mkString("[", ", ", "]")}" + ) + + new SqlPropertyGraphDataSource( + graphDdl, + sqlDataSourceConfigs, + idGenerationStrategy + ) } } @@ -76,28 +94,41 @@ case class SqlPropertyGraphDataSource( graphDdl: GraphDdl, sqlDataSourceConfigs: Map[String, SqlDataSourceConfig], idGenerationStrategy: IdGenerationStrategy -)(implicit val morpheus: MorpheusSession) extends MorpheusPropertyGraphDataSource { +)(implicit val morpheus: MorpheusSession) + extends MorpheusPropertyGraphDataSource { val relSourceIdKey: String = "rel_" + sourceIdKey private val className = getClass.getSimpleName - override def hasGraph(graphName: GraphName): Boolean = graphDdl.graphs.contains(graphName) + override def hasGraph(graphName: GraphName): Boolean = + graphDdl.graphs.contains(graphName) override def graph(graphName: GraphName): PropertyGraph = { - val ddlGraph = graphDdl.graphs.getOrElse(graphName, throw GraphNotFoundException(s"Graph $graphName not found")) + val ddlGraph = graphDdl.graphs.getOrElse( + graphName, + throw GraphNotFoundException(s"Graph $graphName not found") + ) val schema = ddlGraph.graphType.asOkapiSchema val nodeTables = extractNodeTables(ddlGraph, schema) val relationshipTables = extractRelationshipTables(ddlGraph, schema) - val patternTables = extractNodeRelTables(ddlGraph, schema, idGenerationStrategy) + val patternTables = + extractNodeRelTables(ddlGraph, schema, idGenerationStrategy) - morpheus.graphs.create(Some(schema), nodeTables.head, nodeTables.tail ++ relationshipTables ++ patternTables: _*) + morpheus.graphs.create( + Some(schema), + nodeTables.head, + nodeTables.tail ++ relationshipTables ++ patternTables: _* + ) } - override def schema(name: GraphName): Option[MorpheusSchema] = graphDdl.graphs.get(name).map(_.graphType.asOkapiSchema.asMorpheus) + override def schema(name: GraphName): Option[MorpheusSchema] = + graphDdl.graphs.get(name).map(_.graphType.asOkapiSchema.asMorpheus) - override def store(name: GraphName, graph: PropertyGraph): Unit = unsupported("storing a graph") + override def store(name: GraphName, graph: PropertyGraph): Unit = unsupported( + "storing a graph" + ) private def unsupported(operation: String): Nothing = throw UnsupportedOperationException(s"$className does not allow $operation") @@ -113,43 +144,53 @@ case class SqlPropertyGraphDataSource( ddlGraph: Graph, schema: PropertyGraphSchema ): Seq[MorpheusElementTable] = { - ddlGraph.nodeToViewMappings.mapValues(nvm => readTable(nvm.view)).map { - case (nodeViewKey, df) => + ddlGraph.nodeToViewMappings + .mapValues(nvm => readTable(nvm.view)) + .map { case (nodeViewKey, df) => val nodeViewMapping = ddlGraph.nodeToViewMappings(nodeViewKey) - val (propertyMapping, nodeColumns) = extractNode(ddlGraph, schema, nodeViewMapping, df) + val (propertyMapping, nodeColumns) = + extractNode(ddlGraph, schema, nodeViewMapping, df) val nodeDf = df.select(nodeColumns: _*) - val mapping = NodeMappingBuilder.on(sourceIdKey) + val mapping = NodeMappingBuilder + .on(sourceIdKey) .withImpliedLabels(nodeViewMapping.nodeType.labels.toSeq: _*) .withPropertyKeyMappings(propertyMapping.toSeq: _*) .build MorpheusElementTable.create(mapping, nodeDf) - }.toSeq + } + .toSeq } private def extractRelationshipTables( ddlGraph: Graph, schema: PropertyGraphSchema ): Seq[MorpheusElementTable] = { - ddlGraph.edgeToViewMappings.map(evm => evm -> readTable(evm.view)).map { - case (evm, df) => - val (propertyMapping, relColumns) = extractRelationship(ddlGraph, schema, evm, df) - val relDf = df.select(relColumns: _*) - - val relElementType = evm.key.relType.labels.toList match { - case relType :: Nil => relType - case other => throw IllegalArgumentException(expected = "Single relationship type", actual = s"${other.mkString(",")}") - } + ddlGraph.edgeToViewMappings.map(evm => evm -> readTable(evm.view)).map { case (evm, df) => + val (propertyMapping, relColumns) = + extractRelationship(ddlGraph, schema, evm, df) + val relDf = df.select(relColumns: _*) + + val relElementType = evm.key.relType.labels.toList match { + case relType :: Nil => relType + case other => + throw IllegalArgumentException( + expected = "Single relationship type", + actual = s"${other.mkString(",")}" + ) + } - val mapping = RelationshipMappingBuilder - .on(relSourceIdKey).from(sourceStartNodeKey).to(sourceEndNodeKey) - .relType(relElementType) - .withPropertyKeyMappings(propertyMapping.toSeq: _*) - .build + val mapping = RelationshipMappingBuilder + .on(relSourceIdKey) + .from(sourceStartNodeKey) + .to(sourceEndNodeKey) + .relType(relElementType) + .withPropertyKeyMappings(propertyMapping.toSeq: _*) + .build - MorpheusElementTable.create(mapping, relDf) + MorpheusElementTable.create(mapping, relDf) } } @@ -161,31 +202,39 @@ case class SqlPropertyGraphDataSource( ddlGraph.edgeToViewMappings .filter(evm => evm.view == evm.startNode.nodeViewKey.viewId) .map(evm => evm -> readTable(evm.view)) - .map { - case (evm, df) => - val nodeViewKey = evm.startNode.nodeViewKey - val nodeViewMapping = ddlGraph.nodeToViewMappings(nodeViewKey) - - val (nodePropertyMapping, nodeColumns) = extractNode(ddlGraph, schema, nodeViewMapping, df) - val (relPropertyMapping, relColumns) = extractRelationship(ddlGraph, schema, evm, df) - - val patternColumns = nodeColumns ++ relColumns - val patternDf = df.select(patternColumns: _*) - - val pattern = NodeRelPattern(CTNode(nodeViewMapping.nodeType.labels), CTRelationship(evm.relType.labels)) - val patternMapping = ElementMapping( - pattern, - Map( - pattern.nodeElement -> nodePropertyMapping, - pattern.relElement -> relPropertyMapping - ), - Map( - pattern.nodeElement -> Map(SourceIdKey -> sourceIdKey), - pattern.relElement -> Map(SourceIdKey -> relSourceIdKey, SourceStartNodeKey -> sourceStartNodeKey, SourceEndNodeKey -> sourceEndNodeKey) + .map { case (evm, df) => + val nodeViewKey = evm.startNode.nodeViewKey + val nodeViewMapping = ddlGraph.nodeToViewMappings(nodeViewKey) + + val (nodePropertyMapping, nodeColumns) = + extractNode(ddlGraph, schema, nodeViewMapping, df) + val (relPropertyMapping, relColumns) = + extractRelationship(ddlGraph, schema, evm, df) + + val patternColumns = nodeColumns ++ relColumns + val patternDf = df.select(patternColumns: _*) + + val pattern = NodeRelPattern( + CTNode(nodeViewMapping.nodeType.labels), + CTRelationship(evm.relType.labels) + ) + val patternMapping = ElementMapping( + pattern, + Map( + pattern.nodeElement -> nodePropertyMapping, + pattern.relElement -> relPropertyMapping + ), + Map( + pattern.nodeElement -> Map(SourceIdKey -> sourceIdKey), + pattern.relElement -> Map( + SourceIdKey -> relSourceIdKey, + SourceStartNodeKey -> sourceStartNodeKey, + SourceEndNodeKey -> sourceEndNodeKey ) ) + ) - MorpheusElementTable.create(patternMapping, patternDf) + MorpheusElementTable.create(patternMapping, patternDf) } } @@ -197,19 +246,31 @@ case class SqlPropertyGraphDataSource( ): (Map[String, String], Seq[Column]) = { val nodeIdColumn = { - val inputNodeIdColumns = ddlGraph.nodeIdColumnsFor(nodeViewMapping.key) match { - case Some(columnNames) => columnNames - case None => df.columns.map(_.decodeSpecialCharacters).toList - } + val inputNodeIdColumns = + ddlGraph.nodeIdColumnsFor(nodeViewMapping.key) match { + case Some(columnNames) => columnNames + case None => df.columns.map(_.decodeSpecialCharacters).toList + } - generateIdColumn(df, nodeViewMapping.key, inputNodeIdColumns, sourceIdKey, schema) + generateIdColumn( + df, + nodeViewMapping.key, + inputNodeIdColumns, + sourceIdKey, + schema + ) } - val nodeProperties = generatePropertyColumns(nodeViewMapping, df, ddlGraph, schema) - val nodePropertyColumns = nodeProperties.map { case (_, _, col) => col }.toSeq + val nodeProperties = + generatePropertyColumns(nodeViewMapping, df, ddlGraph, schema) + val nodePropertyColumns = nodeProperties.map { case (_, _, col) => + col + }.toSeq val nodeColumns = nodeIdColumn +: nodePropertyColumns - val nodePropertyMapping = nodeProperties.map { case (property, columnName, _) => property -> columnName } + val nodePropertyMapping = nodeProperties.map { case (property, columnName, _) => + property -> columnName + } nodePropertyMapping.toMap -> nodeColumns } @@ -221,37 +282,71 @@ case class SqlPropertyGraphDataSource( df: DataFrame ): (Map[String, String], Seq[Column]) = { - val relIdColumn = generateIdColumn(df, evm.key, df.columns.find(_ == SourceIdKey.name).toList, relSourceIdKey, schema) - val relSourceIdColumn = generateIdColumn(df, evm.startNode.nodeViewKey, evm.startNode.joinPredicates.map(_.edgeColumn), sourceStartNodeKey, schema) - val relTargetIdColumn = generateIdColumn(df, evm.endNode.nodeViewKey, evm.endNode.joinPredicates.map(_.edgeColumn), sourceEndNodeKey, schema) - val relProperties = generatePropertyColumns(evm, df, ddlGraph, schema, Some("relationship")) + val relIdColumn = generateIdColumn( + df, + evm.key, + df.columns.find(_ == SourceIdKey.name).toList, + relSourceIdKey, + schema + ) + val relSourceIdColumn = generateIdColumn( + df, + evm.startNode.nodeViewKey, + evm.startNode.joinPredicates.map(_.edgeColumn), + sourceStartNodeKey, + schema + ) + val relTargetIdColumn = generateIdColumn( + df, + evm.endNode.nodeViewKey, + evm.endNode.joinPredicates.map(_.edgeColumn), + sourceEndNodeKey, + schema + ) + val relProperties = + generatePropertyColumns(evm, df, ddlGraph, schema, Some("relationship")) val relPropertyColumns = relProperties.map { case (_, _, col) => col } - val relColumns = Seq(relIdColumn, relSourceIdColumn, relTargetIdColumn) ++ relPropertyColumns - val relPropertyMapping = relProperties.map { case (property, columnName, _) => property -> columnName } + val relColumns = Seq( + relIdColumn, + relSourceIdColumn, + relTargetIdColumn + ) ++ relPropertyColumns + val relPropertyMapping = relProperties.map { case (property, columnName, _) => + property -> columnName + } relPropertyMapping.toMap -> relColumns } private def readTable(viewId: ViewId): DataFrame = { - val sqlDataSourceConfig = sqlDataSourceConfigs.get(viewId.dataSource) match { - case None => - val knownDataSources = sqlDataSourceConfigs.keys.mkString("'", "';'", "'") - throw SqlDataSourceConfigException(s"Data source '${viewId.dataSource}' not configured; see data sources configuration. Known data sources: $knownDataSources") - case Some(config) => - config - } + val sqlDataSourceConfig = + sqlDataSourceConfigs.get(viewId.dataSource) match { + case None => + val knownDataSources = + sqlDataSourceConfigs.keys.mkString("'", "';'", "'") + throw SqlDataSourceConfigException( + s"Data source '${viewId.dataSource}' not configured; see data sources configuration. Known data sources: $knownDataSources" + ) + case Some(config) => + config + } val inputTable = sqlDataSourceConfig match { - case Hive => readSqlTable(viewId, Hive) + case Hive => readSqlTable(viewId, Hive) case jdbc: Jdbc => readSqlTable(viewId, jdbc) case file: File => readFile(viewId, file) } - inputTable.toDF(inputTable.columns.map(_.toLowerCase.encodeSpecialCharacters).toSeq: _*) + inputTable.toDF( + inputTable.columns.map(_.toLowerCase.encodeSpecialCharacters).toSeq: _* + ) } - private def readSqlTable(viewId: ViewId, sqlDataSourceConfig: SqlDataSourceConfig) = { + private def readSqlTable( + viewId: ViewId, + sqlDataSourceConfig: SqlDataSourceConfig + ) = { val spark = morpheus.sparkSession implicit class DataFrameReaderOps(read: DataFrameReader) { @@ -280,14 +375,19 @@ case class SqlPropertyGraphDataSource( val spark = morpheus.sparkSession val viewPath = viewId.parts.lastOption.getOrElse( - malformed("File names must be defined with the data source", viewId.parts.mkString("."))) + malformed( + "File names must be defined with the data source", + viewId.parts.mkString(".") + ) + ) val filePath = if (new URI(viewPath).isAbsolute) { viewPath } else { dataSourceConfig.basePath match { case Some(rootPath) => (Path(rootPath) / Path(viewPath)).toString() - case None => unsupported("Relative view file names require basePath to be set") + case None => + unsupported("Relative view file names require basePath to be set") } } @@ -313,8 +413,10 @@ case class SqlPropertyGraphDataSource( def getTargetType(elementTypes: Set[String], property: String): DataType = { val maybeCT = viewKey match { - case _: NodeViewKey => schema.nodePropertyKeyType(elementTypes, property) - case _: EdgeViewKey => schema.relationshipPropertyKeyType(elementTypes, property) + case _: NodeViewKey => + schema.nodePropertyKeyType(elementTypes, property) + case _: EdgeViewKey => + schema.relationshipPropertyKeyType(elementTypes, property) } maybeCT.getOrElse(CTVoid).getSparkType @@ -322,26 +424,29 @@ case class SqlPropertyGraphDataSource( val propertyMappings = mapping.propertyMappings - propertyMappings.map { - case (property, colName) => - val normalizedColName = colName.toLowerCase().encodeSpecialCharacters - val sourceColumn = df.col(normalizedColName) - val sourceType = df.schema.apply(normalizedColName).dataType - val targetType = getTargetType(elementTypes, property) - - val withCorrectType = (sourceType, targetType) match { - case _ if sourceType == targetType => sourceColumn - case (IntegerType, LongType) => sourceColumn.cast(targetType) - case (d1: DecimalType, d2: DecimalType) if d2.getCypherType().superTypeOf(d1.getCypherType()) => sourceColumn.cast(targetType) - case _ => throw IllegalArgumentException( + propertyMappings.map { case (property, colName) => + val normalizedColName = colName.toLowerCase().encodeSpecialCharacters + val sourceColumn = df.col(normalizedColName) + val sourceType = df.schema.apply(normalizedColName).dataType + val targetType = getTargetType(elementTypes, property) + + val withCorrectType = (sourceType, targetType) match { + case _ if sourceType == targetType => sourceColumn + case (IntegerType, LongType) => sourceColumn.cast(targetType) + case (d1: DecimalType, d2: DecimalType) + if d2.getCypherType().superTypeOf(d1.getCypherType()) => + sourceColumn.cast(targetType) + case _ => + throw IllegalArgumentException( s"Property `$property` to be a subtype of $targetType", s"Property $sourceColumn with type $sourceType" ) - } + } - val targetColumnName = maybePrefix.getOrElse("") + property.toPropertyColumnName + val targetColumnName = + maybePrefix.getOrElse("") + property.toPropertyColumnName - (property, targetColumnName, withCorrectType.as(targetColumnName)) + (property, targetColumnName, withCorrectType.as(targetColumnName)) } } @@ -352,31 +457,44 @@ case class SqlPropertyGraphDataSource( newIdColumn: String, schema: PropertyGraphSchema ): Column = { - val idColumns = if(idColumnNames.nonEmpty) { - idColumnNames.map(_.toLowerCase.encodeSpecialCharacters).map(dataFrame.col) - } - else { + val idColumns = if (idColumnNames.nonEmpty) { + idColumnNames + .map(_.toLowerCase.encodeSpecialCharacters) + .map(dataFrame.col) + } else { List(monotonically_increasing_id(), lit(dataFrame.hashCode())) } idGenerationStrategy match { case HashedId => - val viewLiteral = functions.lit(elementViewKey.viewId.parts.mkString(".")) - val elementTypeLiterals = elementViewKey.elementType.toSeq.sorted.map(functions.lit) + val viewLiteral = + functions.lit(elementViewKey.viewId.parts.mkString(".")) + val elementTypeLiterals = + elementViewKey.elementType.toSeq.sorted.map(functions.lit) val columnsToHash = Seq(viewLiteral) ++ elementTypeLiterals ++ idColumns - MorpheusFunctions.hash64(columnsToHash: _*).encodeLongAsMorpheusId(newIdColumn) + MorpheusFunctions + .hash64(columnsToHash: _*) + .encodeLongAsMorpheusId(newIdColumn) case SerializedId => val typeToId: Map[List[String], Int] = - (schema.labelCombinations.combos.map(_.toList.sorted) ++ schema.relationshipTypes.map(List(_))) - .toList + (schema.labelCombinations.combos.map( + _.toList.sorted + ) ++ schema.relationshipTypes.map(List(_))).toList .sortBy(s => s.mkString) - .zipWithIndex.toMap - val elementTypeToIntegerId = typeToId(elementViewKey.elementType.toList.sorted) - val columnsToSerialize = functions.lit(elementTypeToIntegerId) :: idColumns + .zipWithIndex + .toMap + val elementTypeToIntegerId = typeToId( + elementViewKey.elementType.toList.sorted + ) + val columnsToSerialize = + functions.lit(elementTypeToIntegerId) :: idColumns MorpheusFunctions.serialize(columnsToSerialize: _*).as(newIdColumn) } } - private def notFound(needle: Any, haystack: Traversable[Any] = Traversable.empty): Nothing = + private def notFound( + needle: Any, + haystack: Traversable[Any] = Traversable.empty + ): Nothing = throw IllegalArgumentException( expected = if (haystack.nonEmpty) s"one of ${stringList(haystack)}" else "", actual = needle diff --git a/morpheus-spark-cypher/src/main/scala/org/opencypher/morpheus/api/io/util/CachedDataSource.scala b/morpheus-spark-cypher/src/main/scala/org/opencypher/morpheus/api/io/util/CachedDataSource.scala index e8088c45f6..89762c158b 100644 --- a/morpheus-spark-cypher/src/main/scala/org/opencypher/morpheus/api/io/util/CachedDataSource.scala +++ b/morpheus-spark-cypher/src/main/scala/org/opencypher/morpheus/api/io/util/CachedDataSource.scala @@ -35,25 +35,31 @@ import org.opencypher.okapi.api.schema.PropertyGraphSchema import scala.collection.mutable /** - * Wraps a [[PropertyGraphDataSource]] and introduces a caching mechanism. First time a graph is read, it is cached in - * Spark according to the given [[StorageLevel]] and put in the datasource internal cache. When a graph is removed from - * the data source, it is uncached in Spark and removed from the datasource internal cache. + * Wraps a [[PropertyGraphDataSource]] and introduces a caching mechanism. First time a graph is + * read, it is cached in Spark according to the given [[StorageLevel]] and put in the datasource + * internal cache. When a graph is removed from the data source, it is uncached in Spark and + * removed from the datasource internal cache. * - * @param dataSource property graph data source - * @param storageLevel storage level for caching the graph + * @param dataSource + * property graph data source + * @param storageLevel + * storage level for caching the graph */ case class CachedDataSource( dataSource: PropertyGraphDataSource, - storageLevel: StorageLevel) extends PropertyGraphDataSource { + storageLevel: StorageLevel +) extends PropertyGraphDataSource { private val cache: mutable.Map[GraphName, PropertyGraph] = mutable.Map.empty - override def graph(name: GraphName): PropertyGraph = cache.getOrElse(name, { - val g = dataSource.graph(name) - g.asMorpheus.tables.foreach(_.persist(storageLevel)) - cache.put(name, g) - g - }) + override def graph(name: GraphName): PropertyGraph = cache.getOrElse( + name, { + val g = dataSource.graph(name) + g.asMorpheus.tables.foreach(_.persist(storageLevel)) + cache.put(name, g) + g + } + ) override def delete(name: GraphName): Unit = cache.get(name) match { case Some(g) => @@ -65,11 +71,14 @@ case class CachedDataSource( dataSource.delete(name) } - override def hasGraph(name: GraphName): Boolean = cache.contains(name) || dataSource.hasGraph(name) + override def hasGraph(name: GraphName): Boolean = + cache.contains(name) || dataSource.hasGraph(name) - override def schema(name: GraphName): Option[PropertyGraphSchema] = dataSource.schema(name) + override def schema(name: GraphName): Option[PropertyGraphSchema] = + dataSource.schema(name) - override def store(name: GraphName, graph: PropertyGraph): Unit = dataSource.store(name, graph) + override def store(name: GraphName, graph: PropertyGraph): Unit = + dataSource.store(name, graph) override def graphNames: Set[GraphName] = dataSource.graphNames } diff --git a/morpheus-spark-cypher/src/main/scala/org/opencypher/morpheus/api/io/util/HiveTableName.scala b/morpheus-spark-cypher/src/main/scala/org/opencypher/morpheus/api/io/util/HiveTableName.scala index 213f0c377b..b0cd9bbd4e 100644 --- a/morpheus-spark-cypher/src/main/scala/org/opencypher/morpheus/api/io/util/HiveTableName.scala +++ b/morpheus-spark-cypher/src/main/scala/org/opencypher/morpheus/api/io/util/HiveTableName.scala @@ -32,14 +32,18 @@ import org.opencypher.okapi.impl.util.StringEncodingUtilities._ case object HiveTableName { - def apply(databaseName: String, + def apply( + databaseName: String, graphName: GraphName, elementType: GraphElementType, - elementIdentifiers: Set[String]): String = { + elementIdentifiers: Set[String] + ): String = { val elementString = elementType.name.toLowerCase - val tableName = s"${graphName.path.replace('/', '_')}_${elementString}_${elementIdentifiers.toSeq.sorted.mkString("_")}".encodeSpecialCharacters + val tableName = + s"${graphName.path.replace('/', '_')}_${elementString}_${elementIdentifiers.toSeq.sorted + .mkString("_")}".encodeSpecialCharacters s"$databaseName.$tableName" } diff --git a/morpheus-spark-cypher/src/main/scala/org/opencypher/morpheus/api/io/util/MorpheusGraphExport.scala b/morpheus-spark-cypher/src/main/scala/org/opencypher/morpheus/api/io/util/MorpheusGraphExport.scala index 6408f261c0..ef71e9870c 100644 --- a/morpheus-spark-cypher/src/main/scala/org/opencypher/morpheus/api/io/util/MorpheusGraphExport.scala +++ b/morpheus-spark-cypher/src/main/scala/org/opencypher/morpheus/api/io/util/MorpheusGraphExport.scala @@ -43,28 +43,51 @@ object MorpheusGraphExport { implicit class CanonicalTableSparkSchema(val schema: PropertyGraphSchema) extends AnyVal { def canonicalNodeStructType(labels: Set[String]): StructType = { - val id = StructField(GraphElement.sourceIdKey, BinaryType, nullable = false) - val properties = schema.nodePropertyKeys(labels).toSeq - .map { case (propertyName, cypherType) => propertyName.toPropertyColumnName -> cypherType } + val id = + StructField(GraphElement.sourceIdKey, BinaryType, nullable = false) + val properties = schema + .nodePropertyKeys(labels) + .toSeq + .map { case (propertyName, cypherType) => + propertyName.toPropertyColumnName -> cypherType + } .sortBy { case (propertyColumnName, _) => propertyColumnName } .map { case (propertyColumnName, cypherType) => - StructField(propertyColumnName, cypherType.getSparkType, cypherType.isNullable) + StructField( + propertyColumnName, + cypherType.getSparkType, + cypherType.isNullable + ) } StructType(id +: properties) } def canonicalRelStructType(relType: String): StructType = { - val id = StructField(GraphElement.sourceIdKey, BinaryType, nullable = false) - val sourceId = StructField(Relationship.sourceStartNodeKey, BinaryType, nullable = false) - val targetId = StructField(Relationship.sourceEndNodeKey, BinaryType, nullable = false) - val properties = schema.relationshipPropertyKeys(relType).toSeq.sortBy(_._1).map { case (propertyName, cypherType) => - StructField(propertyName.toPropertyColumnName, cypherType.getSparkType, cypherType.isNullable) - } + val id = + StructField(GraphElement.sourceIdKey, BinaryType, nullable = false) + val sourceId = StructField( + Relationship.sourceStartNodeKey, + BinaryType, + nullable = false + ) + val targetId = + StructField(Relationship.sourceEndNodeKey, BinaryType, nullable = false) + val properties = + schema.relationshipPropertyKeys(relType).toSeq.sortBy(_._1).map { + case (propertyName, cypherType) => + StructField( + propertyName.toPropertyColumnName, + cypherType.getSparkType, + cypherType.isNullable + ) + } StructType(id +: sourceId +: targetId +: properties) } } - implicit class CanonicalTableExport(graph: RelationalCypherGraph[DataFrameTable]) { + implicit class CanonicalTableExport( + graph: RelationalCypherGraph[DataFrameTable] + ) { def canonicalNodeTable(labels: Set[String]): DataFrame = { val ct = CTNode(labels) @@ -74,12 +97,15 @@ object MorpheusGraphExport { val idRename = header.column(v) -> GraphElement.sourceIdKey val properties: Set[Property] = header.propertiesFor(v) - val propertyRenames = properties.map { p => header.column(p) -> p.key.name.toPropertyColumnName } - - val selectColumns = (idRename :: propertyRenames.toList.sortBy(_._2)).map { - case (oldName, newName) => nodeRecords.table.df.col(oldName).as(newName) + val propertyRenames = properties.map { p => + header.column(p) -> p.key.name.toPropertyColumnName } + val selectColumns = + (idRename :: propertyRenames.toList.sortBy(_._2)).map { case (oldName, newName) => + nodeRecords.table.df.col(oldName).as(newName) + } + nodeRecords.table.df.select(selectColumns: _*) } @@ -90,15 +116,21 @@ object MorpheusGraphExport { val header = relRecords.header val idRename = header.column(v) -> GraphElement.sourceIdKey - val sourceIdRename = header.column(header.startNodeFor(v)) -> Relationship.sourceStartNodeKey - val targetIdRename = header.column(header.endNodeFor(v)) -> Relationship.sourceEndNodeKey + val sourceIdRename = + header.column(header.startNodeFor(v)) -> Relationship.sourceStartNodeKey + val targetIdRename = + header.column(header.endNodeFor(v)) -> Relationship.sourceEndNodeKey val properties: Set[Property] = relRecords.header.propertiesFor(v) - val propertyRenames = properties.map { p => relRecords.header.column(p) -> p.key.name.toPropertyColumnName } - - val selectColumns = (idRename :: sourceIdRename :: targetIdRename :: propertyRenames.toList.sorted).map { - case (oldName, newName) => relRecords.table.df.col(oldName).as(newName) + val propertyRenames = properties.map { p => + relRecords.header.column(p) -> p.key.name.toPropertyColumnName } + val selectColumns = + (idRename :: sourceIdRename :: targetIdRename :: propertyRenames.toList.sorted) + .map { case (oldName, newName) => + relRecords.table.df.col(oldName).as(newName) + } + relRecords.table.df.select(selectColumns: _*) } diff --git a/morpheus-spark-cypher/src/main/scala/org/opencypher/morpheus/api/value/MorpheusElement.scala b/morpheus-spark-cypher/src/main/scala/org/opencypher/morpheus/api/value/MorpheusElement.scala index bb25dc28c8..f57b8726ec 100644 --- a/morpheus-spark-cypher/src/main/scala/org/opencypher/morpheus/api/value/MorpheusElement.scala +++ b/morpheus-spark-cypher/src/main/scala/org/opencypher/morpheus/api/value/MorpheusElement.scala @@ -41,7 +41,8 @@ object MorpheusElement { implicit class LongIdEncoding(val l: Long) extends AnyVal { - def withPrefix(prefix: Int): Array[Byte] = l.encodeAsMorpheusId.withPrefix(prefix.toByte) + def withPrefix(prefix: Int): Array[Byte] = + l.encodeAsMorpheusId.withPrefix(prefix.toByte) def encodeAsMorpheusId: Array[Byte] = encodeLong(l) @@ -71,11 +72,15 @@ object MorpheusNode { } /** - * Representation of a [[Node]] in Morpheus. A node contains an id of type [[Long]], a set of string labels and a map of properties. + * Representation of a [[Node]] in Morpheus. A node contains an id of type [[Long]], a set of + * string labels and a map of properties. * - * @param id the id of the node, unique within the containing graph. - * @param labels the labels of the node. - * @param properties the properties of the node. + * @param id + * the id of the node, unique within the containing graph. + * @param labels + * the labels of the node. + * @param properties + * the properties of the node. */ case class MorpheusNode( override val id: Seq[Byte], @@ -85,10 +90,15 @@ case class MorpheusNode( override type I = MorpheusNode - override def copy(id: Seq[Byte] = id, labels: Set[String] = labels, properties: CypherMap = properties): MorpheusNode = { + override def copy( + id: Seq[Byte] = id, + labels: Set[String] = labels, + properties: CypherMap = properties + ): MorpheusNode = { MorpheusNode(id, labels, properties) } - override def toString: String = s"${getClass.getSimpleName}(id=${id.toHex}, labels=$labels, properties=$properties)" + override def toString: String = + s"${getClass.getSimpleName}(id=${id.toHex}, labels=$labels, properties=$properties)" } object MorpheusRelationship { @@ -98,7 +108,12 @@ object MorpheusRelationship { startId: Long, endId: Long, relType: String - ): MorpheusRelationship = MorpheusRelationship(id.encodeAsMorpheusId, startId.encodeAsMorpheusId, endId.encodeAsMorpheusId, relType) + ): MorpheusRelationship = MorpheusRelationship( + id.encodeAsMorpheusId, + startId.encodeAsMorpheusId, + endId.encodeAsMorpheusId, + relType + ) def apply( id: Long, @@ -106,17 +121,29 @@ object MorpheusRelationship { endId: Long, relType: String, properties: CypherMap - ): MorpheusRelationship = MorpheusRelationship(id.encodeAsMorpheusId, startId.encodeAsMorpheusId, endId.encodeAsMorpheusId, relType, properties) + ): MorpheusRelationship = MorpheusRelationship( + id.encodeAsMorpheusId, + startId.encodeAsMorpheusId, + endId.encodeAsMorpheusId, + relType, + properties + ) } /** - * Representation of a [[Relationship]] in Morpheus. A relationship contains an id of type [[Long]], ids of its adjacent nodes, a relationship type and a map of properties. + * Representation of a [[Relationship]] in Morpheus. A relationship contains an id of type + * [[Long]], ids of its adjacent nodes, a relationship type and a map of properties. * - * @param id the id of the relationship, unique within the containing graph. - * @param startId the id of the source node. - * @param endId the id of the target node. - * @param relType the relationship type. - * @param properties the properties of the node. + * @param id + * the id of the relationship, unique within the containing graph. + * @param startId + * the id of the source node. + * @param endId + * the id of the target node. + * @param relType + * the relationship type. + * @param properties + * the properties of the node. */ case class MorpheusRelationship( override val id: Seq[Byte], @@ -134,8 +161,10 @@ case class MorpheusRelationship( endId: Seq[Byte] = endId, relType: String = relType, properties: CypherMap = properties - ): MorpheusRelationship = MorpheusRelationship(id, startId, endId, relType, properties) + ): MorpheusRelationship = + MorpheusRelationship(id, startId, endId, relType, properties) - override def toString: String = s"${getClass.getSimpleName}(id=${id.toHex}, startId=${startId.toHex}, endId=${endId.toHex}, relType=$relType, properties=$properties)" + override def toString: String = + s"${getClass.getSimpleName}(id=${id.toHex}, startId=${startId.toHex}, endId=${endId.toHex}, relType=$relType, properties=$properties)" } diff --git a/morpheus-spark-cypher/src/main/scala/org/opencypher/morpheus/impl/MorpheusConverters.scala b/morpheus-spark-cypher/src/main/scala/org/opencypher/morpheus/impl/MorpheusConverters.scala index ff628012b8..7006f5bab9 100644 --- a/morpheus-spark-cypher/src/main/scala/org/opencypher/morpheus/impl/MorpheusConverters.scala +++ b/morpheus-spark-cypher/src/main/scala/org/opencypher/morpheus/impl/MorpheusConverters.scala @@ -44,40 +44,42 @@ object MorpheusConverters { implicit class RichSession(val session: CypherSession) extends AnyVal { def asMorpheus: MorpheusSession = session match { case asMorpheus: MorpheusSession => asMorpheus - case other => unsupported("Morpheus session", other) + case other => unsupported("Morpheus session", other) } } implicit class RichPropertyGraph(val graph: PropertyGraph) extends AnyVal { - - def asMorpheus: RelationalCypherGraph[DataFrameTable] = graph.asInstanceOf[RelationalCypherGraph[_]] match { - case asMorpheus: RelationalCypherGraph[_] => - Try { - asMorpheus.asInstanceOf[RelationalCypherGraph[DataFrameTable]] - } match { - case Success(value) => value - case Failure(_) => unsupported("Morpheus graphs", asMorpheus) - } - case other => unsupported("Morpheus graphs", other) - } + def asMorpheus: RelationalCypherGraph[DataFrameTable] = + graph.asInstanceOf[RelationalCypherGraph[_]] match { + case asMorpheus: RelationalCypherGraph[_] => + Try { + asMorpheus.asInstanceOf[RelationalCypherGraph[DataFrameTable]] + } match { + case Success(value) => value + case Failure(_) => unsupported("Morpheus graphs", asMorpheus) + } + case other => unsupported("Morpheus graphs", other) + } } implicit class RichCypherRecords(val records: CypherRecords) extends AnyVal { def asMorpheus: MorpheusRecords = records match { case asMorpheus: MorpheusRecords => asMorpheus - case other => unsupported("Morpheus records", other) + case other => unsupported("Morpheus records", other) } } implicit class RichCypherResult(val records: CypherResult) extends AnyVal { - def asMorpheus(implicit morpheus: MorpheusSession): RelationalCypherResult[DataFrameTable] = records match { + def asMorpheus(implicit + morpheus: MorpheusSession + ): RelationalCypherResult[DataFrameTable] = records match { case relational: RelationalCypherResult[_] => Try { relational.asInstanceOf[RelationalCypherResult[DataFrameTable]] } match { case Success(value) => value - case Failure(_) => unsupported("Morpheus results", morpheus) + case Failure(_) => unsupported("Morpheus results", morpheus) } case other => unsupported("Morpheus results", other) } diff --git a/morpheus-spark-cypher/src/main/scala/org/opencypher/morpheus/impl/MorpheusFunctions.scala b/morpheus-spark-cypher/src/main/scala/org/opencypher/morpheus/impl/MorpheusFunctions.scala index 6de3be1f5c..acb78136cc 100644 --- a/morpheus-spark-cypher/src/main/scala/org/opencypher/morpheus/impl/MorpheusFunctions.scala +++ b/morpheus-spark-cypher/src/main/scala/org/opencypher/morpheus/impl/MorpheusFunctions.scala @@ -52,14 +52,17 @@ object MorpheusFunctions { val PI_LIT: Column = lit(Math.PI) // See: https://issues.apache.org/jira/browse/SPARK-20193 @nowarn - val EMPTY_STRUCT: Column = udf(() => new GenericRowWithSchema(Array(), StructType(Nil)), StructType(Nil))() + val EMPTY_STRUCT: Column = udf( + () => new GenericRowWithSchema(Array(), StructType(Nil)), + StructType(Nil) + )() implicit class RichColumn(column: Column) { /** - * This is a copy of {{{org.apache.spark.sql.Column#getItem}}}. The original method only allows fixed - * values (Int, or String) as index although the underlying implementation seem capable of processing arbitrary - * expressions. This method exposes these features + * This is a copy of {{{org.apache.spark.sql.Column#getItem}}}. The original method only allows + * fixed values (Int, or String) as index although the underlying implementation seem capable + * of processing arbitrary expressions. This method exposes these features */ def get(idx: Column): Column = new Column(UnresolvedExtractValue(column.expr, idx.expr)) @@ -70,23 +73,30 @@ object MorpheusFunctions { /** * Configurable wrapper around `monotonically_increasing_id` * - * @param partitionStartDelta Conceptually this number is added to the `partitionIndex` from which the Spark function - * `monotonically_increasing_id` starts assigning IDs. + * @param partitionStartDelta + * Conceptually this number is added to the `partitionIndex` from which the Spark function + * `monotonically_increasing_id` starts assigning IDs. */ // TODO: Document inherited limitations with regard to the maximum number of rows per data frame // TODO: Document the maximum number of partitions (before entering tag space) def partitioned_id_assignment(partitionStartDelta: Int): Column = monotonically_increasing_id() + (partitionStartDelta.toLong << rowIdSpaceBitsUsedByMonotonicallyIncreasingId) - def list_slice(list: Column, maybeFrom: Option[Column], maybeTo: Option[Column]): Column = { + def list_slice( + list: Column, + maybeFrom: Option[Column], + maybeTo: Option[Column] + ): Column = { val start = maybeFrom.map(_ + ONE_LIT).getOrElse(ONE_LIT) - val length = If((size(list) === ZERO_LIT).expr, NULL_LIT.expr, ((maybeTo.getOrElse(size(list)) - start) + ONE_LIT).expr) + val length = If( + (size(list) === ZERO_LIT).expr, + NULL_LIT.expr, + ((maybeTo.getOrElse(size(list)) - start) + ONE_LIT).expr + ) new Column(Slice(list.expr, start.expr, length)) } - /** - * Alternative version of `array_contains` that takes a column as the value. - */ + /** Alternative version of `array_contains` that takes a column as the value. */ def array_contains(column: Column, value: Column): Column = new Column(ArrayContains(column.expr, value.expr)) @@ -97,36 +107,71 @@ object MorpheusFunctions { new Column(Serialize(columns.map(_.expr))) } - def regex_match(text: Column, pattern: Column): Column = new Column(RLike(text.expr, pattern.expr)) + def regex_match(text: Column, pattern: Column): Column = new Column( + RLike(text.expr, pattern.expr) + ) def get_array_item(array: Column, index: Int): Column = { new Column(GetArrayItem(array.expr, functions.lit(index).expr)) } - private val x: NamedLambdaVariable = NamedLambdaVariable("x", StructType(Seq(StructField("item", StringType), StructField("flag", BooleanType))), nullable = false) + private val x: NamedLambdaVariable = NamedLambdaVariable( + "x", + StructType( + Seq(StructField("item", StringType), StructField("flag", BooleanType)) + ), + nullable = false + ) private val TRUE_EXPR: Expression = functions.lit(true).expr def filter_true[T: TypeTag](items: Seq[T], mask: Seq[Column]): Column = { - filter_with_mask(items, mask, LambdaFunction(EqualTo(GetStructField(x, 1), TRUE_EXPR), Seq(x), hidden = false)) + filter_with_mask( + items, + mask, + LambdaFunction( + EqualTo(GetStructField(x, 1), TRUE_EXPR), + Seq(x), + hidden = false + ) + ) } def filter_not_null[T: TypeTag](items: Seq[T], mask: Seq[Column]): Column = { - filter_with_mask(items, mask, LambdaFunction(IsNotNull(GetStructField(x, 1)), Seq(x), hidden = false)) + filter_with_mask( + items, + mask, + LambdaFunction(IsNotNull(GetStructField(x, 1)), Seq(x), hidden = false) + ) } - def make_big_decimal(unscaledVal: Column, precision: Int, scale: Int): Column = { + def make_big_decimal( + unscaledVal: Column, + precision: Int, + scale: Int + ): Column = { new Column(MakeDecimal(unscaledVal.expr, precision, scale)) } - private def filter_with_mask[T: TypeTag](items: Seq[T], mask: Seq[Column], predicate: LambdaFunction): Column = { - require(items.size == mask.size, s"Array filtering requires for the items and the mask to have the same length.") + private def filter_with_mask[T: TypeTag]( + items: Seq[T], + mask: Seq[Column], + predicate: LambdaFunction + ): Column = { + require( + items.size == mask.size, + s"Array filtering requires for the items and the mask to have the same length." + ) if (items.isEmpty) { functions.array() } else { val itemLiterals = functions.array(items.map(functions.typedLit): _*) - val zippedArray = functions.arrays_zip(itemLiterals, functions.array(mask: _*)) + val zippedArray = + functions.arrays_zip(itemLiterals, functions.array(mask: _*)) val filtered = ArrayFilter(zippedArray.expr, predicate) - val transform = ArrayTransform(filtered, LambdaFunction(GetStructField(x, 0), Seq(x), hidden = false)) + val transform = ArrayTransform( + filtered, + LambdaFunction(GetStructField(x, 0), Seq(x), hidden = false) + ) new Column(transform) } } @@ -137,22 +182,43 @@ object MorpheusFunctions { else struct(structColumns: _*) } - def switch(branches: Seq[(Column, Column)], maybeDefault: Option[Column]): Column = { - new Column(CaseWhen(branches.map { case (c, v) => c.expr -> v.expr } , maybeDefault.map(_.expr))) + def switch( + branches: Seq[(Column, Column)], + maybeDefault: Option[Column] + ): Column = { + new Column( + CaseWhen( + branches.map { case (c, v) => c.expr -> v.expr }, + maybeDefault.map(_.expr) + ) + ) } /** - * Alternative version of {{{org.apache.spark.sql.functions.translate}}} that takes {{{org.apache.spark.sql.Column}}}s for search and replace strings. + * Alternative version of {{{org.apache.spark.sql.functions.translate}}} that takes + * {{{org.apache.spark.sql.Column}}} s for search and replace strings. */ - def translate(src: Column, matchingString: Column, replaceString: Column): Column = { - new Column(StringTranslate(src.expr, matchingString.expr, replaceString.expr)) + def translate( + src: Column, + matchingString: Column, + replaceString: Column + ): Column = { + new Column( + StringTranslate(src.expr, matchingString.expr, replaceString.expr) + ) } - def column_for(expr: Expr)(implicit header: RecordHeader, df: DataFrame): Column = { - val columnName = header.getColumn(expr).getOrElse(throw IllegalArgumentException( - expected = s"Expression in ${header.expressions.mkString("[", ", ", "]")}", - actual = expr) - ) + def column_for( + expr: Expr + )(implicit header: RecordHeader, df: DataFrame): Column = { + val columnName = header + .getColumn(expr) + .getOrElse( + throw IllegalArgumentException( + expected = s"Expression in ${header.expressions.mkString("[", ", ", "]")}", + actual = expr + ) + ) if (df.columns.contains(columnName)) { df.col(columnName) } else { @@ -166,11 +232,12 @@ object MorpheusFunctions { v.cypherType.ensureSparkCompatible() v match { case list: CypherList => array(list.value.map(_.toSparkLiteral): _*) - case map: CypherMap => create_struct( - map.value.map { case (key, value) => - value.toSparkLiteral.as(key.toString) - }.toSeq - ) + case map: CypherMap => + create_struct( + map.value.map { case (key, value) => + value.toSparkLiteral.as(key.toString) + }.toSeq + ) case _ => lit(v.unwrap) } } diff --git a/morpheus-spark-cypher/src/main/scala/org/opencypher/morpheus/impl/MorpheusRecords.scala b/morpheus-spark-cypher/src/main/scala/org/opencypher/morpheus/impl/MorpheusRecords.scala index c193336e2b..5738bae8b6 100644 --- a/morpheus-spark-cypher/src/main/scala/org/opencypher/morpheus/impl/MorpheusRecords.scala +++ b/morpheus-spark-cypher/src/main/scala/org/opencypher/morpheus/impl/MorpheusRecords.scala @@ -36,28 +36,41 @@ import org.opencypher.morpheus.impl.table.SparkTable.{DataFrameTable, _} import org.opencypher.okapi.api.types._ import org.opencypher.okapi.api.value.CypherValue.{CypherMap, CypherValue} import org.opencypher.okapi.relational.api.io.ElementTable -import org.opencypher.okapi.relational.api.table.{RelationalCypherRecords, RelationalCypherRecordsFactory} +import org.opencypher.okapi.relational.api.table.{ + RelationalCypherRecords, + RelationalCypherRecordsFactory +} import org.opencypher.okapi.relational.impl.table._ import scala.collection.JavaConverters._ -case class MorpheusRecordsFactory()(implicit morpheus: MorpheusSession) extends RelationalCypherRecordsFactory[DataFrameTable] { +case class MorpheusRecordsFactory()(implicit morpheus: MorpheusSession) + extends RelationalCypherRecordsFactory[DataFrameTable] { override type Records = MorpheusRecords override def unit(): MorpheusRecords = { - val initialDataFrame = morpheus.sparkSession.createDataFrame(Seq(EmptyRow())) + val initialDataFrame = + morpheus.sparkSession.createDataFrame(Seq(EmptyRow())) MorpheusRecords(RecordHeader.empty, initialDataFrame) } - override def empty(initialHeader: RecordHeader = RecordHeader.empty): MorpheusRecords = { + override def empty( + initialHeader: RecordHeader = RecordHeader.empty + ): MorpheusRecords = { val initialSparkStructType = initialHeader.toStructType - val initialDataFrame = morpheus.sparkSession.createDataFrame(Collections.emptyList[Row](), initialSparkStructType) + val initialDataFrame = morpheus.sparkSession.createDataFrame( + Collections.emptyList[Row](), + initialSparkStructType + ) MorpheusRecords(initialHeader, initialDataFrame) } - override def fromElementTable(elementTable: ElementTable[DataFrameTable]): MorpheusRecords = { - val withCypherCompatibleTypes = elementTable.table.df.withCypherCompatibleTypes + override def fromElementTable( + elementTable: ElementTable[DataFrameTable] + ): MorpheusRecords = { + val withCypherCompatibleTypes = + elementTable.table.df.withCypherCompatibleTypes MorpheusRecords(elementTable.header, withCypherCompatibleTypes) } @@ -67,8 +80,8 @@ case class MorpheusRecordsFactory()(implicit morpheus: MorpheusSession) extends maybeDisplayNames: Option[Seq[String]] ): MorpheusRecords = { val displayNames = maybeDisplayNames match { - case s@Some(_) => s - case None => Some(header.vars.map(_.withoutType).toSeq) + case s @ Some(_) => s + case None => Some(header.vars.map(_.withoutType).toSeq) } MorpheusRecords(header, table, displayNames) } @@ -76,11 +89,16 @@ case class MorpheusRecordsFactory()(implicit morpheus: MorpheusSession) extends /** * Wraps a Spark SQL table (DataFrame) in a MorpheusRecords, making it understandable by Cypher. * - * @param df table to wrap. - * @param morpheus session to which the resulting MorpheusRecords is tied. - * @return a Cypher table. + * @param df + * table to wrap. + * @param morpheus + * session to which the resulting MorpheusRecords is tied. + * @return + * a Cypher table. */ - private[morpheus] def wrap(df: DataFrame)(implicit morpheus: MorpheusSession): MorpheusRecords = { + private[morpheus] def wrap( + df: DataFrame + )(implicit morpheus: MorpheusSession): MorpheusRecords = { val compatibleDf = df.withCypherCompatibleTypes MorpheusRecords(compatibleDf.schema.toRecordHeader, compatibleDf) } @@ -92,7 +110,9 @@ case class MorpheusRecords( header: RecordHeader, table: DataFrameTable, override val logicalColumns: Option[Seq[String]] = None -)(implicit morpheus: MorpheusSession) extends RelationalCypherRecords[DataFrameTable] with RecordBehaviour { +)(implicit morpheus: MorpheusSession) + extends RelationalCypherRecords[DataFrameTable] + with RecordBehaviour { override type Records = MorpheusRecords def df: DataFrame = table.df @@ -134,7 +154,6 @@ trait RecordBehaviour extends RelationalCypherRecords[DataFrameTable] { override def collect: Array[CypherMap] = toCypherMaps.collect() - def toCypherMaps: Dataset[CypherMap] = { import encoders._ table.df.map(rowToCypherMap(header.exprToColumn.toSeq)) diff --git a/morpheus-spark-cypher/src/main/scala/org/opencypher/morpheus/impl/SparkSQLExprMapper.scala b/morpheus-spark-cypher/src/main/scala/org/opencypher/morpheus/impl/SparkSQLExprMapper.scala index b590af0654..229776f79a 100644 --- a/morpheus-spark-cypher/src/main/scala/org/opencypher/morpheus/impl/SparkSQLExprMapper.scala +++ b/morpheus-spark-cypher/src/main/scala/org/opencypher/morpheus/impl/SparkSQLExprMapper.scala @@ -26,7 +26,18 @@ */ package org.opencypher.morpheus.impl -import org.apache.spark.sql.catalyst.expressions.{ArrayAggregate, ArrayExists, ArrayFilter, ArrayTransform, CaseWhen, ExprId, LambdaFunction, Literal, NamedLambdaVariable, StringSplit} +import org.apache.spark.sql.catalyst.expressions.{ + ArrayAggregate, + ArrayExists, + ArrayFilter, + ArrayTransform, + CaseWhen, + ExprId, + LambdaFunction, + Literal, + NamedLambdaVariable, + StringSplit +} import org.apache.spark.sql.functions.{array_contains => _, translate => _, _} import org.apache.spark.sql.types._ import org.apache.spark.sql.{Column, DataFrame} @@ -50,17 +61,22 @@ object SparkSQLExprMapper { implicit class RichExpression(expr: Expr) { /** - * Converts `expr` with the `withConvertedChildren` function, which is passed the converted child expressions as its - * argument. + * Converts `expr` with the `withConvertedChildren` function, which is passed the converted + * child expressions as its argument. * - * Iff the expression has `expr.nullInNullOut == true`, then any child being mapped to `null` will also result in - * the parent expression being mapped to null. + * Iff the expression has `expr.nullInNullOut == true`, then any child being mapped to `null` + * will also result in the parent expression being mapped to null. * - * For these expressions the `withConvertedChildren` function is guaranteed to not receive any `null` - * values from the evaluated children. + * For these expressions the `withConvertedChildren` function is guaranteed to not receive any + * `null` values from the evaluated children. */ - def nullSafeConversion(expr: Expr)(withConvertedChildren: Seq[Column] => Column) - (implicit header: RecordHeader, df: DataFrame, parameters: CypherMap): Column = { + def nullSafeConversion( + expr: Expr + )(withConvertedChildren: Seq[Column] => Column)(implicit + header: RecordHeader, + df: DataFrame, + parameters: CypherMap + ): Column = { if (expr.cypherType == CTNull) { NULL_LIT } else if (expr.cypherType == CTTrue) { @@ -69,10 +85,16 @@ object SparkSQLExprMapper { FALSE_LIT } else { val evaluatedArgs = expr.children.map(_.asSparkSQLExpr) - val withConvertedChildrenResult = withConvertedChildren(evaluatedArgs).expr + val withConvertedChildrenResult = withConvertedChildren( + evaluatedArgs + ).expr if (expr.children.nonEmpty && expr.nullInNullOut && expr.cypherType.isNullable) { - val nullPropagationCases = evaluatedArgs.map(_.isNull.expr).zip(Seq.fill(evaluatedArgs.length)(NULL_LIT.expr)) - new Column(CaseWhen(nullPropagationCases, withConvertedChildrenResult)) + val nullPropagationCases = evaluatedArgs + .map(_.isNull.expr) + .zip(Seq.fill(evaluatedArgs.length)(NULL_LIT.expr)) + new Column( + CaseWhen(nullPropagationCases, withConvertedChildrenResult) + ) } else { new Column(withConvertedChildrenResult) } @@ -82,29 +104,50 @@ object SparkSQLExprMapper { /** * Attempts to create a Spark SQL expression from the Morpheus expression. * - * @param header the header of the [[MorpheusRecords]] in which the expression should be evaluated. - * @param df the dataframe containing the data over which the expression should be evaluated. - * @param parameters query parameters - * @return Some Spark SQL expression if the input was mappable, otherwise None. + * @param header + * the header of the [[MorpheusRecords]] in which the expression should be evaluated. + * @param df + * the dataframe containing the data over which the expression should be evaluated. + * @param parameters + * query parameters + * @return + * Some Spark SQL expression if the input was mappable, otherwise None. */ - def asSparkSQLExpr(implicit header: RecordHeader, df: DataFrame, parameters: CypherMap): Column = { + def asSparkSQLExpr(implicit + header: RecordHeader, + df: DataFrame, + parameters: CypherMap + ): Column = { val outCol = expr match { case v: LambdaVar => - val sparkType = v.cypherType.toSparkType.getOrElse(throw IllegalStateException(s"No valid dataType for LambdaVar $v")) - new Column(NamedLambdaVariable(v.name, sparkType, nullable = v.cypherType.isNullable, ExprId(v.hashCode.toLong))) + val sparkType = v.cypherType.toSparkType.getOrElse( + throw IllegalStateException(s"No valid dataType for LambdaVar $v") + ) + new Column( + NamedLambdaVariable( + v.name, + sparkType, + nullable = v.cypherType.isNullable, + ExprId(v.hashCode.toLong) + ) + ) // Evaluate based on already present data; no recursion - case _: Var | _: HasLabel | _: HasType | _: StartNode | _: EndNode => column_for(expr) + case _: Var | _: HasLabel | _: HasType | _: StartNode | _: EndNode => + column_for(expr) // Evaluate bottom-up case _ => nullSafeConversion(expr)(convert) } header.getColumn(expr) match { - case None => outCol + case None => outCol case Some(colName) => outCol.as(colName) } } - private def convert(convertedChildren: Seq[Column]) - (implicit header: RecordHeader, df: DataFrame, parameters: CypherMap): Column = { + private def convert(convertedChildren: Seq[Column])(implicit + header: RecordHeader, + df: DataFrame, + parameters: CypherMap + ): Column = { def child0: Column = convertedChildren.head @@ -113,57 +156,73 @@ object SparkSQLExprMapper { def child2: Column = convertedChildren(2) expr match { - case _: ListLit => array(convertedChildren: _*) - case l: Lit[_] => lit(l.v) + case _: ListLit => array(convertedChildren: _*) + case l: Lit[_] => lit(l.v) case _: AliasExpr => child0 - case Param(name) => parameters(name).toSparkLiteral + case Param(name) => parameters(name).toSparkLiteral // Predicates case _: Equals => child0 === child1 - case _: Not => !child0 - case Size(e) => { - e.cypherType match { - case CTString => length(child0) - case _ => size(child0) // it's a list - } - }.cast(LongType) - case _: Ands => convertedChildren.foldLeft(TRUE_LIT)(_ && _) - case _: Ors => convertedChildren.foldLeft(FALSE_LIT)(_ || _) - case _: IsNull => child0.isNull - case _: IsNotNull => child0.isNotNull - case _: Exists => child0.isNotNull - case _: LessThan => child0 < child1 - case _: LessThanOrEqual => child0 <= child1 + case _: Not => !child0 + case Size(e) => + { + e.cypherType match { + case CTString => length(child0) + case _ => size(child0) // it's a list + } + }.cast(LongType) + case _: Ands => convertedChildren.foldLeft(TRUE_LIT)(_ && _) + case _: Ors => convertedChildren.foldLeft(FALSE_LIT)(_ || _) + case _: IsNull => child0.isNull + case _: IsNotNull => child0.isNotNull + case _: Exists => child0.isNotNull + case _: LessThan => child0 < child1 + case _: LessThanOrEqual => child0 <= child1 case _: GreaterThanOrEqual => child0 >= child1 - case _: GreaterThan => child0 > child1 + case _: GreaterThan => child0 > child1 case _: StartsWith => child0.startsWith(child1) - case _: EndsWith => child0.endsWith(child1) - case _: Contains => child0.contains(child1) + case _: EndsWith => child0.endsWith(child1) + case _: Contains => child0.contains(child1) case _: RegexMatch => regex_match(child0, child1) // Other - case Explode(list) => list.cypherType match { - case CTNull => explode(NULL_LIT.cast(ArrayType(NullType))) - case _ => explode(child0) - } - - case _: ElementProperty => if (!header.contains(expr)) NULL_LIT else column_for(expr) - case MapProperty(_, key) => if (expr.cypherType.material == CTVoid) NULL_LIT else child0.getField(key.name) - case DateProperty(_, key) => temporalAccessor[java.sql.Date](child0, key.name) - case LocalDateTimeProperty(_, key) => temporalAccessor[java.sql.Timestamp](child0, key.name) - case DurationProperty(_, key) => TemporalUdfs.durationAccessor(key.name.toLowerCase).apply(child0) + case Explode(list) => + list.cypherType match { + case CTNull => explode(NULL_LIT.cast(ArrayType(NullType))) + case _ => explode(child0) + } - case LocalDateTime(maybeDateExpr) => maybeDateExpr.map(e => lit(e.resolveTimestamp).cast(DataTypes.TimestampType)).getOrElse(current_timestamp()) - case Date(maybeDateExpr) => maybeDateExpr.map(e => lit(e.resolveDate).cast(DataTypes.DateType)).getOrElse(current_timestamp()) + case _: ElementProperty => + if (!header.contains(expr)) NULL_LIT else column_for(expr) + case MapProperty(_, key) => + if (expr.cypherType.material == CTVoid) NULL_LIT + else child0.getField(key.name) + case DateProperty(_, key) => + temporalAccessor[java.sql.Date](child0, key.name) + case LocalDateTimeProperty(_, key) => + temporalAccessor[java.sql.Timestamp](child0, key.name) + case DurationProperty(_, key) => + TemporalUdfs.durationAccessor(key.name.toLowerCase).apply(child0) + + case LocalDateTime(maybeDateExpr) => + maybeDateExpr + .map(e => lit(e.resolveTimestamp).cast(DataTypes.TimestampType)) + .getOrElse(current_timestamp()) + case Date(maybeDateExpr) => + maybeDateExpr + .map(e => lit(e.resolveDate).cast(DataTypes.DateType)) + .getOrElse(current_timestamp()) case Duration(durationExpr) => lit(durationExpr.resolveInterval) - case In(lhs, rhs) => rhs.cypherType.material match { - case CTList(inner) if inner.couldBeSameTypeAs(lhs.cypherType) => array_contains(child1, child0) - case _ => NULL_LIT - } + case In(lhs, rhs) => + rhs.cypherType.material match { + case CTList(inner) if inner.couldBeSameTypeAs(lhs.cypherType) => + array_contains(child1, child0) + case _ => NULL_LIT + } - //list-access + // list-access case _: Head => element_at(child0, 1) case _: Last => element_at(child0, -1) @@ -176,137 +235,184 @@ object SparkSQLExprMapper { if ((lhInner | rhInner).isSparkCompatible) { concat(child0, child1) } else { - throw SparkSQLMappingException(s"Lists of different inner types are not supported (${lhInner.material}, ${rhInner.material})") + throw SparkSQLMappingException( + s"Lists of different inner types are not supported (${lhInner.material}, ${rhInner.material})" + ) } - case (CTList(inner), nonListType) if (inner | nonListType).isSparkCompatible => concat(child0, array(child1)) - case (nonListType, CTList(inner)) if (inner | nonListType).isSparkCompatible => concat(array(child0), child1) - case (CTString, _) if rhsCT.subTypeOf(CTNumber) => concat(child0, child1.cast(StringType)) - case (_, CTString) if lhsCT.subTypeOf(CTNumber) => concat(child0.cast(StringType), child1) + case (CTList(inner), nonListType) if (inner | nonListType).isSparkCompatible => + concat(child0, array(child1)) + case (nonListType, CTList(inner)) if (inner | nonListType).isSparkCompatible => + concat(array(child0), child1) + case (CTString, _) if rhsCT.subTypeOf(CTNumber) => + concat(child0, child1.cast(StringType)) + case (_, CTString) if lhsCT.subTypeOf(CTNumber) => + concat(child0.cast(StringType), child1) case (CTString, CTString) => concat(child0, child1) case (CTDate, CTDuration) => TemporalUdfs.dateAdd(child0, child1) - case _ => child0 + child1 + case _ => child0 + child1 } - case Subtract(lhs, rhs) if lhs.cypherType.material.subTypeOf(CTDate) && rhs.cypherType.material.subTypeOf(CTDuration) => + case Subtract(lhs, rhs) + if lhs.cypherType.material.subTypeOf( + CTDate + ) && rhs.cypherType.material.subTypeOf(CTDuration) => TemporalUdfs.dateSubtract(child0, child1) case _: Subtract => child0 - child1 case _: Multiply => child0 * child1 case div: Divide => (child0 / child1).cast(div.cypherType.getSparkType) - case _: Modulo => child0 % child1 + case _: Modulo => child0 % child1 // Id functions - case _: Id => child0 + case _: Id => child0 case PrefixId(_, prefix) => child0.addPrefix(lit(prefix)) case ToId(e) => e.cypherType.material match { - case CTInteger => child0.encodeLongAsMorpheusId + case CTInteger => child0.encodeLongAsMorpheusId case ct if ct.toSparkType.contains(BinaryType) => child0 - case other => throw IllegalArgumentException("a type that may be converted to an ID", other) + case other => + throw IllegalArgumentException( + "a type that may be converted to an ID", + other + ) } // Functions case _: MonotonicallyIncreasingId => monotonically_increasing_id() case Labels(e) => - val possibleLabels = header.labelsFor(e.owner.get).toSeq.sortBy(_.label.name) + val possibleLabels = + header.labelsFor(e.owner.get).toSeq.sortBy(_.label.name) val labelBooleanFlagsCol = possibleLabels.map(_.asSparkSQLExpr) - val nodeLabels = filter_true(possibleLabels.map(_.label.name), labelBooleanFlagsCol) + val nodeLabels = + filter_true(possibleLabels.map(_.label.name), labelBooleanFlagsCol) nodeLabels case Type(e) => - val possibleRelTypes = header.typesFor(e.owner.get).toSeq.sortBy(_.relType.name) + val possibleRelTypes = + header.typesFor(e.owner.get).toSeq.sortBy(_.relType.name) val relTypeBooleanFlagsCol = possibleRelTypes.map(_.asSparkSQLExpr) - val relTypes = filter_true(possibleRelTypes.map(_.relType.name), relTypeBooleanFlagsCol) + val relTypes = filter_true( + possibleRelTypes.map(_.relType.name), + relTypeBooleanFlagsCol + ) val relType = get_array_item(relTypes, index = 0) relType case Keys(e) => e.cypherType.material match { case element if element.subTypeOf(CTElement) => - val possibleProperties = header.propertiesFor(e.owner.get).toSeq.sortBy(_.key.name) + val possibleProperties = + header.propertiesFor(e.owner.get).toSeq.sortBy(_.key.name) val propertyNames = possibleProperties.map(_.key.name) val propertyValues = possibleProperties.map(_.asSparkSQLExpr) filter_not_null(propertyNames, propertyValues) case CTMap(inner) => val mapColumn = child0 - val (propertyKeys, propertyValues) = inner.keys.map { e => - // Whe have to make sure that every column has the same type (true or null) - e -> when(mapColumn.getField(e).isNotNull, TRUE_LIT).otherwise(NULL_LIT) - }.toSeq.unzip + val (propertyKeys, propertyValues) = inner.keys + .map { e => + // Whe have to make sure that every column has the same type (true or null) + e -> when(mapColumn.getField(e).isNotNull, TRUE_LIT) + .otherwise(NULL_LIT) + } + .toSeq + .unzip filter_not_null(propertyKeys, propertyValues) - case other => throw IllegalArgumentException("an Expression with type CTNode, CTRelationship or CTMap", other) + case other => + throw IllegalArgumentException( + "an Expression with type CTNode, CTRelationship or CTMap", + other + ) } case Properties(e) => e.cypherType.material match { case element if element.subTypeOf(CTElement) => - val propertyExpressions = header.propertiesFor(e.owner.get).toSeq.sortBy(_.key.name) + val propertyExpressions = + header.propertiesFor(e.owner.get).toSeq.sortBy(_.key.name) val propertyColumns = propertyExpressions - .map(propertyExpression => propertyExpression.asSparkSQLExpr.as(propertyExpression.key.name)) + .map(propertyExpression => + propertyExpression.asSparkSQLExpr.as( + propertyExpression.key.name + ) + ) create_struct(propertyColumns) case _: CTMap => child0 case other => - throw IllegalArgumentException("a node, relationship or map", other, "Invalid input to properties function") + throw IllegalArgumentException( + "a node, relationship or map", + other, + "Invalid input to properties function" + ) } - case StartNodeFunction(e) => header.startNodeFor(e.owner.get).asSparkSQLExpr + case StartNodeFunction(e) => + header.startNodeFor(e.owner.get).asSparkSQLExpr case EndNodeFunction(e) => header.endNodeFor(e.owner.get).asSparkSQLExpr - case _: ToFloat => child0.cast(DoubleType) + case _: ToFloat => child0.cast(DoubleType) case _: ToInteger => child0.cast(IntegerType) - case _: ToString => child0.cast(StringType) + case _: ToString => child0.cast(StringType) case _: ToBoolean => child0.cast(BooleanType) - case _: Trim => trim(child0) - case _: LTrim => ltrim(child0) - case _: RTrim => rtrim(child0) + case _: Trim => trim(child0) + case _: LTrim => ltrim(child0) + case _: RTrim => rtrim(child0) case _: Reverse => reverse(child0) case _: ToUpper => upper(child0) case _: ToLower => lower(child0) - case _: Range => sequence(child0, child1, convertedChildren.lift(2).getOrElse(ONE_LIT)) + case _: Range => + sequence(child0, child1, convertedChildren.lift(2).getOrElse(ONE_LIT)) case _: Replace => translate(child0, child1, child2) - case _: Substring => child0.substr(child1 + ONE_LIT, convertedChildren.lift(2).getOrElse(length(child0) - child1)) - case _: Split => new Column(StringSplit(child0.expr, child1.expr, lit(-1).expr)) + case _: Substring => + child0.substr( + child1 + ONE_LIT, + convertedChildren.lift(2).getOrElse(length(child0) - child1) + ) + case _: Split => + new Column(StringSplit(child0.expr, child1.expr, lit(-1).expr)) // Mathematical functions - case E => E_LIT + case E => E_LIT case Pi => PI_LIT - case _: Sqrt => sqrt(child0) - case _: Log => log(child0) + case _: Sqrt => sqrt(child0) + case _: Log => log(child0) case _: Log10 => log(10.0, child0) - case _: Exp => exp(child0) - case _: Abs => abs(child0) - case _: Ceil => ceil(child0).cast(DoubleType) + case _: Exp => exp(child0) + case _: Abs => abs(child0) + case _: Ceil => ceil(child0).cast(DoubleType) case _: Floor => floor(child0).cast(DoubleType) - case Rand => rand() + case Rand => rand() case _: Round => round(child0).cast(DoubleType) - case _: Sign => signum(child0).cast(IntegerType) - - case _: Acos => acos(child0) - case _: Asin => asin(child0) - case _: Atan => atan(child0) - case _: Atan2 => atan2(child0, child1) - case _: Cos => cos(child0) - case Cot(e) => Divide(IntegerLit(1), Tan(e)).asSparkSQLExpr + case _: Sign => signum(child0).cast(IntegerType) + + case _: Acos => acos(child0) + case _: Asin => asin(child0) + case _: Atan => atan(child0) + case _: Atan2 => atan2(child0, child1) + case _: Cos => cos(child0) + case Cot(e) => Divide(IntegerLit(1), Tan(e)).asSparkSQLExpr case _: Degrees => degrees(child0) - case Haversin(e) => Divide(Subtract(IntegerLit(1), Cos(e)), IntegerLit(2)).asSparkSQLExpr + case Haversin(e) => + Divide(Subtract(IntegerLit(1), Cos(e)), IntegerLit(2)).asSparkSQLExpr case _: Radians => radians(child0) - case _: Sin => sin(child0) - case _: Tan => tan(child0) + case _: Sin => sin(child0) + case _: Tan => tan(child0) // Time functions case Timestamp => current_timestamp().cast(LongType) // Bit operations case _: BitwiseAnd => child0.bitwiseAND(child1) - case _: BitwiseOr => child0.bitwiseOR(child1) - case ShiftLeft(_, IntegerLit(shiftBits)) => shiftleft(child0, shiftBits.toInt) - case ShiftRightUnsigned(_, IntegerLit(shiftBits)) => shiftrightunsigned(child0, shiftBits.toInt) + case _: BitwiseOr => child0.bitwiseOR(child1) + case ShiftLeft(_, IntegerLit(shiftBits)) => + shiftleft(child0, shiftBits.toInt) + case ShiftRightUnsigned(_, IntegerLit(shiftBits)) => + shiftrightunsigned(child0, shiftBits.toInt) // Pattern Predicate case ep: ExistsPatternExpr => ep.targetField.asSparkSQLExpr @@ -316,11 +422,12 @@ object SparkSQLExprMapper { coalesce(columns: _*) case CaseExpr(_, maybeDefault) => - val (maybeConvertedDefault, convertedAlternatives) = if (maybeDefault.isDefined) { - Some(convertedChildren.head) -> convertedChildren.tail - } else { - None -> convertedChildren - } + val (maybeConvertedDefault, convertedAlternatives) = + if (maybeDefault.isDefined) { + Some(convertedChildren.head) -> convertedChildren.tail + } else { + None -> convertedChildren + } val indexed = convertedAlternatives.zipWithIndex val conditions = indexed.collect { case (c, i) if i % 2 == 0 => c } val values = indexed.collect { case (c, i) if i % 2 == 1 => c } @@ -330,28 +437,43 @@ object SparkSQLExprMapper { case ContainerIndex(container, index) => val containerCol = container.asSparkSQLExpr container.cypherType.material match { - case c if c.subTypeOf(CTContainer) => containerCol.get(index.asSparkSQLExpr) - case other => throw NotImplementedException(s"Accessing $other by index is not supported") + case c if c.subTypeOf(CTContainer) => + containerCol.get(index.asSparkSQLExpr) + case other => + throw NotImplementedException( + s"Accessing $other by index is not supported" + ) } - case _: ListSliceFromTo => list_slice(child0, Some(child1), Some(child2)) + case _: ListSliceFromTo => + list_slice(child0, Some(child1), Some(child2)) case _: ListSliceFrom => list_slice(child0, Some(child1), None) - case _: ListSliceTo => list_slice(child0, None, Some(child1)) - - case ListComprehension(variable, innerPredicate, extractExpression, listExpr) => + case _: ListSliceTo => list_slice(child0, None, Some(child1)) + + case ListComprehension( + variable, + innerPredicate, + extractExpression, + listExpr + ) => val lambdaVar = variable.asSparkSQLExpr.expr match { case v: NamedLambdaVariable => v - case err => throw IllegalStateException(s"$variable should be converted into a NamedLambdaVariable instead of $err") + case err => + throw IllegalStateException( + s"$variable should be converted into a NamedLambdaVariable instead of $err" + ) } val filteredExpr = innerPredicate match { case Some(filterExpr) => - val filterFunc = LambdaFunction(filterExpr.asSparkSQLExpr.expr, Seq(lambdaVar)) + val filterFunc = + LambdaFunction(filterExpr.asSparkSQLExpr.expr, Seq(lambdaVar)) ArrayFilter(listExpr.asSparkSQLExpr.expr, filterFunc) case None => listExpr.asSparkSQLExpr.expr } - val result = extractExpression match{ + val result = extractExpression match { case Some(extractExpr) => - val extractFunc = LambdaFunction(extractExpr.asSparkSQLExpr.expr, Seq(lambdaVar)) + val extractFunc = + LambdaFunction(extractExpr.asSparkSQLExpr.expr, Seq(lambdaVar)) ArrayTransform(filteredExpr, extractFunc) case None => filteredExpr } @@ -361,21 +483,34 @@ object SparkSQLExprMapper { val convertedChildrenExpr = convertedChildren.map(_.expr) val (initVar, accVar) = convertedChildrenExpr.slice(0, 2) match { case Seq(i: NamedLambdaVariable, a: NamedLambdaVariable) => i -> a - case err => throw IllegalStateException(s"($v,$acc) should be converted into a NamedLambdaVariables instead of $err") + case err => + throw IllegalStateException( + s"($v,$acc) should be converted into a NamedLambdaVariables instead of $err" + ) } - val reduceFunc = LambdaFunction(convertedChildrenExpr(2), Seq(initVar, accVar)) + val reduceFunc = + LambdaFunction(convertedChildrenExpr(2), Seq(initVar, accVar)) val finishFunc = LambdaFunction(accVar, Seq(accVar)) - val reduceExpr = ArrayAggregate(convertedChildrenExpr(4), convertedChildrenExpr(3), reduceFunc, finishFunc) + val reduceExpr = ArrayAggregate( + convertedChildrenExpr(4), + convertedChildrenExpr(3), + reduceFunc, + finishFunc + ) new Column(reduceExpr) case predExpr: IterablePredicateExpr => val convertedChildrenExpr = convertedChildren.map(_.expr) val lambdaVar = child0.expr match { case v: NamedLambdaVariable => v - case err => throw IllegalStateException(s"${predExpr.exprs.head} should be converted into a NamedLambdaVariable instead of $err") + case err => + throw IllegalStateException( + s"${predExpr.exprs.head} should be converted into a NamedLambdaVariable instead of $err" + ) } - val filterFunc = LambdaFunction(convertedChildrenExpr(1), Seq(lambdaVar)) + val filterFunc = + LambdaFunction(convertedChildrenExpr(1), Seq(lambdaVar)) predExpr match { case _: ListAll => val lengthBefore = size(convertedChildren(2)) @@ -395,31 +530,42 @@ object SparkSQLExprMapper { ONE_LIT === lengthAfter } - case MapExpression(items) => expr.cypherType.material match { - case CTMap(_) => - val innerColumns = items.map { - case (key, innerExpr) => innerExpr.asSparkSQLExpr.as(key) - }.toSeq - create_struct(innerColumns) - case other => throw IllegalArgumentException("an expression of type CTMap", other) - } + case MapExpression(items) => + expr.cypherType.material match { + case CTMap(_) => + val innerColumns = items.map { case (key, innerExpr) => + innerExpr.asSparkSQLExpr.as(key) + }.toSeq + create_struct(innerColumns) + case other => + throw IllegalArgumentException( + "an expression of type CTMap", + other + ) + } case MapProjection(mapOwner, items, includeAllProps) => - val convertedItems = items.map { case (key, value) => value.asSparkSQLExpr.as(key) } + val convertedItems = items.map { case (key, value) => + value.asSparkSQLExpr.as(key) + } val itemKeys = items.map { case (propKey, _) => propKey } val intersectedMapItems = if (includeAllProps) { mapOwner.cypherType.material match { case x if x.subTypeOf(CTElement) => - val uniqueEntityProps = header.propertiesFor(mapOwner).filterNot(p => itemKeys.contains(p.key.name)) - val propertyColumns = uniqueEntityProps.map(p => p.asSparkSQLExpr.as(p.key.name)) + val uniqueEntityProps = header + .propertiesFor(mapOwner) + .filterNot(p => itemKeys.contains(p.key.name)) + val propertyColumns = + uniqueEntityProps.map(p => p.asSparkSQLExpr.as(p.key.name)) convertedItems ++ propertyColumns case CTMap(inner) => - val uniqueMapKeys = inner.keys.filterNot(key => itemKeys.contains(key)) - val uniqueMapColumns = uniqueMapKeys.map(key => child0.getField(key).as(key)) + val uniqueMapKeys = + inner.keys.filterNot(key => itemKeys.contains(key)) + val uniqueMapColumns = + uniqueMapKeys.map(key => child0.getField(key).as(key)) convertedItems ++ uniqueMapColumns } - } - else { + } else { convertedItems } create_struct(intersectedMapItems) @@ -437,44 +583,55 @@ object SparkSQLExprMapper { case _: Avg => expr.cypherType match { case CTDuration => udaf(TemporalUdafs.DurationAvg).apply(child0) - case _ => avg(child0) + case _ => avg(child0) } case _: Max => expr.cypherType match { case CTDuration => udaf(TemporalUdafs.DurationMax).apply(child0) - case _ => max(child0) + case _ => max(child0) } case _: Min => expr.cypherType match { case CTDuration => udaf(TemporalUdafs.DurationMin).apply(child0) - case _ => min(child0) + case _ => min(child0) } case _: PercentileCont => val percentile = child1.expr match { case Literal(v, DoubleType) => v.asInstanceOf[Double] - case _ => throw IllegalArgumentException("Literal as percentage for percentileCont()", child1.expr) + case _ => + throw IllegalArgumentException( + "Literal as percentage for percentileCont()", + child1.expr + ) } PercentileUdafs.percentileCont(percentile)(child0) - case _:PercentileDisc => + case _: PercentileDisc => val percentile = child1.expr match { case Literal(v, DoubleType) => v.asInstanceOf[Double] - case _ => throw IllegalArgumentException("Literal as percentage for percentileDisc()", child1.expr) + case _ => + throw IllegalArgumentException( + "Literal as percentage for percentileDisc()", + child1.expr + ) } - PercentileUdafs.percentileDisc(percentile, child0.expr.dataType)(child0) - case _: StDev => stddev(child0) + PercentileUdafs.percentileDisc(percentile, child0.expr.dataType)( + child0 + ) + case _: StDev => stddev(child0) case _: StDevP => stddev_pop(child0) case _: Sum => expr.cypherType match { case CTDuration => udaf(TemporalUdafs.DurationSum).apply(child0) - case _ => sum(child0) + case _ => sum(child0) } - case BigDecimal(_, precision, scale) => make_big_decimal(child0, precision.toInt, scale.toInt) case _ => - throw NotImplementedException(s"No support for converting Cypher expression $expr to a Spark SQL expression") + throw NotImplementedException( + s"No support for converting Cypher expression $expr to a Spark SQL expression" + ) } } diff --git a/morpheus-spark-cypher/src/main/scala/org/opencypher/morpheus/impl/convert/SparkConversions.scala b/morpheus-spark-cypher/src/main/scala/org/opencypher/morpheus/impl/convert/SparkConversions.scala index 4709a9b6ba..292bf3792d 100644 --- a/morpheus-spark-cypher/src/main/scala/org/opencypher/morpheus/impl/convert/SparkConversions.scala +++ b/morpheus-spark-cypher/src/main/scala/org/opencypher/morpheus/impl/convert/SparkConversions.scala @@ -47,7 +47,8 @@ object SparkConversions { def toStructField(column: String): StructField = { ct.toSparkType match { case Some(st) => StructField(column, st, ct.isNullable) - case None => throw IllegalArgumentException("CypherType supported by Morpheus", ct) + case None => + throw IllegalArgumentException("CypherType supported by Morpheus", ct) } } @@ -55,30 +56,40 @@ object SparkConversions { case CTNull => Some(NullType) case _ => ct.material match { - case CTString => Some(StringType) - case CTInteger => Some(LongType) - case CTBigDecimal(p, s) => Some(DataTypes.createDecimalType(p, s)) - case CTFloat => Some(DoubleType) - case CTLocalDateTime => Some(TimestampType) - case CTDate => Some(DateType) - case CTDuration => Some(CalendarIntervalType) - case CTIdentity => Some(BinaryType) - case b if b.subTypeOf(CTBoolean) => Some(BooleanType) + case CTString => Some(StringType) + case CTInteger => Some(LongType) + case CTBigDecimal(p, s) => Some(DataTypes.createDecimalType(p, s)) + case CTFloat => Some(DoubleType) + case CTLocalDateTime => Some(TimestampType) + case CTDate => Some(DateType) + case CTDuration => Some(CalendarIntervalType) + case CTIdentity => Some(BinaryType) + case b if b.subTypeOf(CTBoolean) => Some(BooleanType) case n if n.subTypeOf(CTElement.nullable) => Some(BinaryType) // Spark uses String as the default array inner type - case CTMap(inner) => Some(StructType(inner.map { case (key, vType) => vType.toStructField(key) }.toSeq)) + case CTMap(inner) => + Some(StructType(inner.map { case (key, vType) => + vType.toStructField(key) + }.toSeq)) case CTEmptyList => Some(ArrayType(StringType, containsNull = false)) - case CTList(CTNull) => Some(ArrayType(StringType, containsNull = true)) - case CTList(inner) if inner.subTypeOf(CTBoolean.nullable) => Some(ArrayType(BooleanType, containsNull = inner.isNullable)) - case CTList(elemType) if elemType.toSparkType.isDefined => elemType.toSparkType.map(ArrayType(_, elemType.isNullable)) - case l if l.subTypeOf(CTList(CTNumber.nullable)) => Some(ArrayType(DoubleType, containsNull = l.isNullable)) + case CTList(CTNull) => + Some(ArrayType(StringType, containsNull = true)) + case CTList(inner) if inner.subTypeOf(CTBoolean.nullable) => + Some(ArrayType(BooleanType, containsNull = inner.isNullable)) + case CTList(elemType) if elemType.toSparkType.isDefined => + elemType.toSparkType.map(ArrayType(_, elemType.isNullable)) + case l if l.subTypeOf(CTList(CTNumber.nullable)) => + Some(ArrayType(DoubleType, containsNull = l.isNullable)) case _ => None } } def getSparkType: DataType = toSparkType match { case Some(t) => t - case None => throw SparkSQLMappingException(s"Mapping of CypherType $ct to Spark type is unsupported") + case None => + throw SparkSQLMappingException( + s"Mapping of CypherType $ct to Spark type is unsupported" + ) } def isSparkCompatible: Boolean = toSparkType.isDefined @@ -93,7 +104,11 @@ object SparkConversions { val exprToColumn = structType.fields.map { field => val cypherType = field.toCypherType match { case Some(ct) => ct - case None => throw IllegalArgumentException("a supported Spark type", field.dataType) + case None => + throw IllegalArgumentException( + "a supported Spark type", + field.dataType + ) } Var(field.name)(cypherType) -> field.name } @@ -101,42 +116,49 @@ object SparkConversions { RecordHeader(exprToColumn.toMap) } - def binaryColumns: Set[String] = structType.fields.filter(_.dataType == BinaryType).map(_.name).toSet + def binaryColumns: Set[String] = + structType.fields.filter(_.dataType == BinaryType).map(_.name).toSet - def convertTypes(from: DataType, to: DataType): StructType = StructType(structType.map { - case sf: StructField if sf.dataType == from => sf.copy(dataType = to) - case sf: StructField => sf - }) + def convertTypes(from: DataType, to: DataType): StructType = StructType( + structType.map { + case sf: StructField if sf.dataType == from => sf.copy(dataType = to) + case sf: StructField => sf + } + ) } implicit class StructFieldOps(val field: StructField) extends AnyVal { - def toCypherType: Option[CypherType] = field.dataType.toCypherType(field.nullable) + def toCypherType: Option[CypherType] = + field.dataType.toCypherType(field.nullable) } implicit class DataTypeOps(val dt: DataType) extends AnyVal { def toCypherType(nullable: Boolean = false): Option[CypherType] = { val result = dt match { - case StringType => Some(CTString) - case IntegerType => Some(CTInteger) - case LongType => Some(CTInteger) - case BooleanType => Some(CTBoolean) - case DoubleType => Some(CTFloat) - case dt: DecimalType => Some(CTBigDecimal(dt.precision, dt.scale)) - case TimestampType => Some(CTLocalDateTime) - case DateType => Some(CTDate) - case CalendarIntervalType => Some(CTDuration) + case StringType => Some(CTString) + case IntegerType => Some(CTInteger) + case LongType => Some(CTInteger) + case BooleanType => Some(CTBoolean) + case DoubleType => Some(CTFloat) + case dt: DecimalType => Some(CTBigDecimal(dt.precision, dt.scale)) + case TimestampType => Some(CTLocalDateTime) + case DateType => Some(CTDate) + case CalendarIntervalType => Some(CTDuration) case ArrayType(NullType, _) => Some(CTEmptyList) - case BinaryType => Some(CTIdentity) + case BinaryType => Some(CTIdentity) case ArrayType(elemType, containsNull) => elemType.toCypherType(containsNull).map(CTList(_)) case NullType => Some(CTNull) case StructType(fields) => - val convertedFields = fields.map { field => field.name -> field.dataType.toCypherType(field.nullable) }.toMap + val convertedFields = fields.map { field => + field.name -> field.dataType.toCypherType(field.nullable) + }.toMap val containsNone = convertedFields.exists { case (_, None) => true - case _ => false + case _ => false } - if (containsNone) None else Some(CTMap(convertedFields.mapValues(_.get))) + if (containsNone) None + else Some(CTMap(convertedFields.mapValues(_.get))) case _ => None } @@ -146,26 +168,31 @@ object SparkConversions { def getCypherType(nullable: Boolean = false): CypherType = toCypherType(nullable) match { case Some(ct) => ct - case None => throw SparkSQLMappingException(s"Mapping of Spark type $dt to Cypher type is unsupported") + case None => + throw SparkSQLMappingException( + s"Mapping of Spark type $dt to Cypher type is unsupported" + ) } /** * Checks if the given data type is supported within the Cypher type system. * - * @return true, iff the data type is supported + * @return + * true, iff the data type is supported */ def isCypherCompatible: Boolean = cypherCompatibleDataType.isDefined /** * Converts the given Spark data type into a Cypher type system compatible Spark data type. * - * @return some Cypher-compatible Spark data type or none if not compatible + * @return + * some Cypher-compatible Spark data type or none if not compatible */ def cypherCompatibleDataType: Option[DataType] = dt match { - case ByteType | ShortType | IntegerType => Some(LongType) - case FloatType => Some(DoubleType) + case ByteType | ShortType | IntegerType => Some(LongType) + case FloatType => Some(DoubleType) case compatible if dt.toCypherType().isDefined => Some(compatible) - case _ => None + case _ => None } } @@ -175,10 +202,12 @@ object SparkConversions { val structFields = header.columns.toSeq.sorted.map { column => val expressions = header.expressionsFor(column) val commonType = expressions.map(_.cypherType).reduce(_ join _) - assert(commonType.isSparkCompatible, + assert( + commonType.isSparkCompatible, s""" |Expressions $expressions with common super type $commonType mapped to column $column have no compatible data type. - """.stripMargin) + """.stripMargin + ) commonType.toStructField(column) } StructType(structFields) @@ -192,10 +221,10 @@ object SparkConversions { def allNull: Boolean = allNull(row.size) - def allNull(rowSize: Int): Boolean = (for (i <- 0 until rowSize) yield row.isNullAt(i)).reduce(_ && _) + def allNull(rowSize: Int): Boolean = + (for (i <- 0 until rowSize) yield row.isNullAt(i)).reduce(_ && _) } - object SparkCypherValueConverter extends CypherValueConverter { override def convert(v: Any): Option[CypherValue] = v match { case interval: CalendarInterval => Some(interval.toDuration) @@ -210,5 +239,6 @@ object SparkConversions { } } - implicit val sparkCypherValueConverter: CypherValueConverter = SparkCypherValueConverter + implicit val sparkCypherValueConverter: CypherValueConverter = + SparkCypherValueConverter } diff --git a/morpheus-spark-cypher/src/main/scala/org/opencypher/morpheus/impl/convert/rowToCypherMap.scala b/morpheus-spark-cypher/src/main/scala/org/opencypher/morpheus/impl/convert/rowToCypherMap.scala index 0e3096e8af..53b3358fbc 100644 --- a/morpheus-spark-cypher/src/main/scala/org/opencypher/morpheus/impl/convert/rowToCypherMap.scala +++ b/morpheus-spark-cypher/src/main/scala/org/opencypher/morpheus/impl/convert/rowToCypherMap.scala @@ -42,16 +42,18 @@ final case class rowToCypherMap(exprToColumn: Seq[(Expr, String)]) extends (Row private val header = RecordHeader(exprToColumn.toMap) override def apply(row: Row): CypherMap = { - val values = header.returnItems.map(r => r.name -> constructValue(row, r)).toSeq + val values = + header.returnItems.map(r => r.name -> constructValue(row, r)).toSeq CypherMap(values: _*) } // TODO: Validate all column types. At the moment null values are cast to the expected type... private def constructValue(row: Row, v: Var): CypherValue = { v.cypherType.material match { - case n if n.subTypeOf(CTNode.nullable) => collectNode(row, v) + case n if n.subTypeOf(CTNode.nullable) => collectNode(row, v) case r if r.subTypeOf(CTRelationship.nullable) => collectRel(row, v) - case l if l.subTypeOf(CTList.nullable) && !header.exprToColumn.contains(v) => collectComplexList(row, v) + case l if l.subTypeOf(CTList.nullable) && !header.exprToColumn.contains(v) => + collectComplexList(row, v) case _ => constructFromExpression(row, v) } } @@ -66,7 +68,6 @@ final case class rowToCypherMap(exprToColumn: Seq[(Expr, String)]) extends (Row idValue match { case null => CypherNull case id: Array[_] => - val labels = header .labelsFor(v) .map { l => l.label.name -> row.getAs[Boolean](header.column(l)) } @@ -79,7 +80,10 @@ final case class rowToCypherMap(exprToColumn: Seq[(Expr, String)]) extends (Row .toMap MorpheusNode(id.asInstanceOf[Array[Byte]], labels, properties) - case invalidID => throw UnsupportedOperationException(s"MorpheusNode ID has to be an Array[Byte] instead of ${invalidID.getClass}") + case invalidID => + throw UnsupportedOperationException( + s"MorpheusNode ID has to be an Array[Byte] instead of ${invalidID.getClass}" + ) } } @@ -108,21 +112,29 @@ final case class rowToCypherMap(exprToColumn: Seq[(Expr, String)]) extends (Row source.asInstanceOf[Array[Byte]], target.asInstanceOf[Array[Byte]], relType, - properties) - case invalidID => throw UnsupportedOperationException(s"MorpheusRelationship ID has to be an Array[Byte] instead of ${invalidID.getClass}") + properties + ) + case invalidID => + throw UnsupportedOperationException( + s"MorpheusRelationship ID has to be an Array[Byte] instead of ${invalidID.getClass}" + ) } } private def collectComplexList(row: Row, expr: Var): CypherList = { - val elements = header.ownedBy(expr).collect { - case p: ListSegment => p - }.toSeq.sortBy(_.index) + val elements = header + .ownedBy(expr) + .collect { case p: ListSegment => + p + } + .toSeq + .sortBy(_.index) val values = elements .map(constructValue(row, _)) .filter { case CypherNull => false - case _ => true + case _ => true } CypherList(values) diff --git a/morpheus-spark-cypher/src/main/scala/org/opencypher/morpheus/impl/encoders/CypherValueEncoders.scala b/morpheus-spark-cypher/src/main/scala/org/opencypher/morpheus/impl/encoders/CypherValueEncoders.scala index 442464775e..e097946596 100644 --- a/morpheus-spark-cypher/src/main/scala/org/opencypher/morpheus/impl/encoders/CypherValueEncoders.scala +++ b/morpheus-spark-cypher/src/main/scala/org/opencypher/morpheus/impl/encoders/CypherValueEncoders.scala @@ -32,9 +32,11 @@ import org.opencypher.morpheus.api.value.{MorpheusNode, MorpheusRelationship} import org.opencypher.okapi.api.value.CypherValue._ trait CypherValueEncoders extends LowPriorityCypherValueEncoders { - implicit def cypherNodeEncoder: ExpressionEncoder[MorpheusNode] = kryo[MorpheusNode] + implicit def cypherNodeEncoder: ExpressionEncoder[MorpheusNode] = + kryo[MorpheusNode] - implicit def cypherRelationshipEncoder: ExpressionEncoder[MorpheusRelationship] = kryo[MorpheusRelationship] + implicit def cypherRelationshipEncoder: ExpressionEncoder[MorpheusRelationship] = + kryo[MorpheusRelationship] implicit def cypherMapEncoder: ExpressionEncoder[CypherMap] = kryo[CypherMap] } diff --git a/morpheus-spark-cypher/src/main/scala/org/opencypher/morpheus/impl/encoders/LowPriorityCypherValueEncoders.scala b/morpheus-spark-cypher/src/main/scala/org/opencypher/morpheus/impl/encoders/LowPriorityCypherValueEncoders.scala index e433c2c758..8cd5dcb8b2 100644 --- a/morpheus-spark-cypher/src/main/scala/org/opencypher/morpheus/impl/encoders/LowPriorityCypherValueEncoders.scala +++ b/morpheus-spark-cypher/src/main/scala/org/opencypher/morpheus/impl/encoders/LowPriorityCypherValueEncoders.scala @@ -34,10 +34,12 @@ import org.opencypher.okapi.api.value.CypherValue.CypherValue import scala.language.implicitConversions trait LowPriorityCypherValueEncoders { - implicit def asExpressionEncoder[T](v: Encoder[T]): ExpressionEncoder[T] = v.asInstanceOf[ExpressionEncoder[T]] + implicit def asExpressionEncoder[T](v: Encoder[T]): ExpressionEncoder[T] = + v.asInstanceOf[ExpressionEncoder[T]] - implicit def cypherValueEncoder: ExpressionEncoder[CypherValue] = kryo[CypherValue] + implicit def cypherValueEncoder: ExpressionEncoder[CypherValue] = + kryo[CypherValue] - implicit def cypherRecordEncoder: ExpressionEncoder[Map[String, CypherValue]] = kryo[Map[String, CypherValue]] + implicit def cypherRecordEncoder: ExpressionEncoder[Map[String, CypherValue]] = + kryo[Map[String, CypherValue]] } - diff --git a/morpheus-spark-cypher/src/main/scala/org/opencypher/morpheus/impl/expressions/AddPrefix.scala b/morpheus-spark-cypher/src/main/scala/org/opencypher/morpheus/impl/expressions/AddPrefix.scala index dbb373280e..ad64baf2f9 100644 --- a/morpheus-spark-cypher/src/main/scala/org/opencypher/morpheus/impl/expressions/AddPrefix.scala +++ b/morpheus-spark-cypher/src/main/scala/org/opencypher/morpheus/impl/expressions/AddPrefix.scala @@ -1,46 +1,52 @@ /** - * Copyright (c) 2016-2019 "Neo4j Sweden, AB" [https://neo4j.com] - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * Attribution Notice under the terms of the Apache License 2.0 - * - * This work was created by the collective efforts of the openCypher community. - * Without limiting the terms of Section 6, any Derivative Work that is not - * approved by the public consensus process of the openCypher Implementers Group - * should not be described as “Cypher” (and Cypher® is a registered trademark of - * Neo4j Inc.) or as "openCypher". Extensions by implementers or prototypes or - * proposals for change that have been documented or implemented should only be - * described as "implementation extensions to Cypher" or as "proposed changes to - * Cypher that are not yet approved by the openCypher community". - */ + * Copyright (c) 2016-2019 "Neo4j Sweden, AB" [https://neo4j.com] + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the + * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing permissions and + * limitations under the License. + * + * Attribution Notice under the terms of the Apache License 2.0 + * + * This work was created by the collective efforts of the openCypher community. Without limiting + * the terms of Section 6, any Derivative Work that is not approved by the public consensus process + * of the openCypher Implementers Group should not be described as “Cypher” (and Cypher® is a + * registered trademark of Neo4j Inc.) or as "openCypher". Extensions by implementers or prototypes + * or proposals for change that have been documented or implemented should only be described as + * "implementation extensions to Cypher" or as "proposed changes to Cypher that are not yet + * approved by the openCypher community". + */ package org.opencypher.morpheus.impl.expressions import org.apache.spark.sql.Column import org.apache.spark.sql.catalyst.expressions.codegen.{CodegenContext, ExprCode} -import org.apache.spark.sql.catalyst.expressions.{BinaryExpression, ExpectsInputTypes, Expression, NullIntolerant} +import org.apache.spark.sql.catalyst.expressions.{ + BinaryExpression, + ExpectsInputTypes, + Expression, + NullIntolerant +} import org.apache.spark.sql.types.{BinaryType, ByteType, DataType} /** * Spark expression that adds a byte prefix to an array of bytes. * - * @param left expression of ByteType that is added as a prefix - * @param right expression of BinaryType to which a prefix is added + * @param left + * expression of ByteType that is added as a prefix + * @param right + * expression of BinaryType to which a prefix is added */ case class AddPrefix( left: Expression, // byte prefix right: Expression // binary array -) extends BinaryExpression with NullIntolerant with ExpectsInputTypes { +) extends BinaryExpression + with NullIntolerant + with ExpectsInputTypes { override val dataType: DataType = BinaryType @@ -51,9 +57,16 @@ case class AddPrefix( } override protected def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = - defineCodeGen(ctx, ev, (a, p) => s"(byte[])(${AddPrefix.getClass.getName.dropRight(1)}.addPrefix($a, $p))") + defineCodeGen( + ctx, + ev, + (a, p) => s"(byte[])(${AddPrefix.getClass.getName.dropRight(1)}.addPrefix($a, $p))" + ) - override protected def withNewChildrenInternal(newLeft: Expression, newRight: Expression): Expression = copy(newLeft, newRight) + override protected def withNewChildrenInternal( + newLeft: Expression, + newRight: Expression + ): Expression = copy(newLeft, newRight) } object AddPrefix { diff --git a/morpheus-spark-cypher/src/main/scala/org/opencypher/morpheus/impl/expressions/EncodeLong.scala b/morpheus-spark-cypher/src/main/scala/org/opencypher/morpheus/impl/expressions/EncodeLong.scala index 9af32ba8ee..1150958d4e 100644 --- a/morpheus-spark-cypher/src/main/scala/org/opencypher/morpheus/impl/expressions/EncodeLong.scala +++ b/morpheus-spark-cypher/src/main/scala/org/opencypher/morpheus/impl/expressions/EncodeLong.scala @@ -1,43 +1,49 @@ /** * Copyright (c) 2016-2019 "Neo4j Sweden, AB" [https://neo4j.com] * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and + * Unless required by applicable law or agreed to in writing, software distributed under the + * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing permissions and * limitations under the License. * * Attribution Notice under the terms of the Apache License 2.0 * - * This work was created by the collective efforts of the openCypher community. - * Without limiting the terms of Section 6, any Derivative Work that is not - * approved by the public consensus process of the openCypher Implementers Group - * should not be described as “Cypher” (and Cypher® is a registered trademark of - * Neo4j Inc.) or as "openCypher". Extensions by implementers or prototypes or - * proposals for change that have been documented or implemented should only be - * described as "implementation extensions to Cypher" or as "proposed changes to - * Cypher that are not yet approved by the openCypher community". + * This work was created by the collective efforts of the openCypher community. Without limiting + * the terms of Section 6, any Derivative Work that is not approved by the public consensus process + * of the openCypher Implementers Group should not be described as “Cypher” (and Cypher® is a + * registered trademark of Neo4j Inc.) or as "openCypher". Extensions by implementers or prototypes + * or proposals for change that have been documented or implemented should only be described as + * "implementation extensions to Cypher" or as "proposed changes to Cypher that are not yet + * approved by the openCypher community". */ package org.opencypher.morpheus.impl.expressions import org.apache.spark.sql.Column import org.apache.spark.sql.catalyst.expressions.codegen.{CodegenContext, ExprCode} -import org.apache.spark.sql.catalyst.expressions.{ExpectsInputTypes, Expression, NullIntolerant, UnaryExpression} +import org.apache.spark.sql.catalyst.expressions.{ + ExpectsInputTypes, + Expression, + NullIntolerant, + UnaryExpression +} import org.apache.spark.sql.types.{BinaryType, DataType, LongType} import org.opencypher.morpheus.api.value.MorpheusElement._ /** * Spark expression that encodes a long into a byte array using variable-length encoding. * - * @param child expression of type LongType that is encoded by this expression + * @param child + * expression of type LongType that is encoded by this expression */ -case class EncodeLong(child: Expression) extends UnaryExpression with NullIntolerant with ExpectsInputTypes { +case class EncodeLong(child: Expression) + extends UnaryExpression + with NullIntolerant + with ExpectsInputTypes { override val dataType: DataType = BinaryType @@ -46,10 +52,19 @@ case class EncodeLong(child: Expression) extends UnaryExpression with NullIntole override protected def nullSafeEval(input: Any): Any = EncodeLong.encodeLong(input.asInstanceOf[Long]) - override protected def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = - defineCodeGen(ctx, ev, c => s"(byte[])(${EncodeLong.getClass.getName.dropRight(1)}.encodeLong($c))") - - override protected def withNewChildInternal(newChild: Expression): Expression = copy(newChild) + override protected def doGenCode( + ctx: CodegenContext, + ev: ExprCode + ): ExprCode = + defineCodeGen( + ctx, + ev, + c => s"(byte[])(${EncodeLong.getClass.getName.dropRight(1)}.encodeLong($c))" + ) + + override protected def withNewChildInternal( + newChild: Expression + ): Expression = copy(newChild) } object EncodeLong { @@ -82,7 +97,10 @@ object EncodeLong { // Same encoding as as Base 128 Varints @ https://developers.google.com/protocol-buffers/docs/encoding @inline final def decodeLong(input: Array[Byte]): Long = { - assert(input.nonEmpty, "`decodeLong` requires a non-empty array as its input") + assert( + input.nonEmpty, + "`decodeLong` requires a non-empty array as its input" + ) var index = 0 var currentByte = input(index) var decoded = currentByte & varLength7BitMask @@ -94,14 +112,17 @@ object EncodeLong { decoded |= (currentByte & varLength7BitMask) << nextLeftShift nextLeftShift += 7 } - assert(index == input.length - 1, - s"`decodeLong` received an input array ${input.toSeq.toHex} with extra bytes that could not be decoded.") + assert( + index == input.length - 1, + s"`decodeLong` received an input array ${input.toSeq.toHex} with extra bytes that could not be decoded." + ) decoded } implicit class ColumnLongOps(val c: Column) extends AnyVal { - def encodeLongAsMorpheusId(name: String): Column = encodeLongAsMorpheusId.as(name) + def encodeLongAsMorpheusId(name: String): Column = + encodeLongAsMorpheusId.as(name) def encodeLongAsMorpheusId: Column = new Column(EncodeLong(c.expr)) diff --git a/morpheus-spark-cypher/src/main/scala/org/opencypher/morpheus/impl/expressions/PercentileUdafs.scala b/morpheus-spark-cypher/src/main/scala/org/opencypher/morpheus/impl/expressions/PercentileUdafs.scala index 80c6fe12ea..72910a9ab4 100644 --- a/morpheus-spark-cypher/src/main/scala/org/opencypher/morpheus/impl/expressions/PercentileUdafs.scala +++ b/morpheus-spark-cypher/src/main/scala/org/opencypher/morpheus/impl/expressions/PercentileUdafs.scala @@ -35,14 +35,19 @@ import org.opencypher.okapi.impl.exception.IllegalArgumentException import scala.annotation.nowarn import scala.collection.mutable - // As abs(percentile_rank() - given_percentage) inside min() is not allowed object PercentileUdafs extends Logging { @nowarn abstract class PercentileAggregation(percentile: Double) extends UserDefinedAggregateFunction { - def inputSchema: StructType = StructType(Array(StructField("value", DoubleType))) - def bufferSchema: StructType = StructType(Array(StructField("array_buffer", ArrayType(DoubleType, containsNull = false)))) + def inputSchema: StructType = StructType( + Array(StructField("value", DoubleType)) + ) + def bufferSchema: StructType = StructType( + Array( + StructField("array_buffer", ArrayType(DoubleType, containsNull = false)) + ) + ) def deterministic: Boolean = true def initialize(buffer: MutableAggregationBuffer): Unit = { buffer(0) = Array[DoubleType]() @@ -53,23 +58,27 @@ object PercentileUdafs extends Logging { } } def merge(buffer1: MutableAggregationBuffer, buffer2: Row): Unit = { - buffer1(0) = buffer1(0).asInstanceOf[mutable.WrappedArray[Double]] ++ buffer2(0).asInstanceOf[mutable.WrappedArray[Double]] + buffer1(0) = buffer1(0).asInstanceOf[mutable.WrappedArray[Double]] ++ buffer2(0) + .asInstanceOf[mutable.WrappedArray[Double]] } } - class PercentileDisc(percentile: Double, numberType: DataType) extends PercentileAggregation(percentile) { + class PercentileDisc(percentile: Double, numberType: DataType) + extends PercentileAggregation(percentile) { def dataType: DataType = numberType def evaluate(buffer: Row): Any = { - val sortedValues = buffer(0).asInstanceOf[mutable.WrappedArray[Double]].sortWith(_ < _) + val sortedValues = + buffer(0).asInstanceOf[mutable.WrappedArray[Double]].sortWith(_ < _) if (sortedValues.isEmpty) return null val position = (sortedValues.length * percentile).round.toInt - val result = if (position == 0) sortedValues(0) else sortedValues(position - 1) + val result = + if (position == 0) sortedValues(0) else sortedValues(position - 1) dataType match { - case LongType => result.toLong + case LongType => result.toLong case DoubleType => result - case e => throw IllegalArgumentException("a Integer or a Float", e) + case e => throw IllegalArgumentException("a Integer or a Float", e) } } @@ -79,7 +88,8 @@ object PercentileUdafs extends Logging { def dataType: DataType = DoubleType def evaluate(buffer: Row): Any = { - val sortedValues = buffer(0).asInstanceOf[mutable.WrappedArray[Double]].sortWith(_ < _) + val sortedValues = + buffer(0).asInstanceOf[mutable.WrappedArray[Double]].sortWith(_ < _) if (sortedValues.isEmpty) return null val exact_position = 1 + ((sortedValues.length - 1) * percentile) @@ -87,9 +97,13 @@ object PercentileUdafs extends Logging { val succ = exact_position.ceil.toInt val weight = succ - exact_position exact_position match { - case pos if pos < 1 => (1 - weight) * sortedValues(succ) + weight * sortedValues(prec) + case pos if pos < 1 => + (1 - weight) * sortedValues(succ) + weight * sortedValues(prec) case pos if pos == succ => sortedValues(prec - 1) - case _ => (1 - weight) * sortedValues(succ - 1) + weight * sortedValues(prec - 1) + case _ => + (1 - weight) * sortedValues(succ - 1) + weight * sortedValues( + prec - 1 + ) } } } diff --git a/morpheus-spark-cypher/src/main/scala/org/opencypher/morpheus/impl/expressions/Serialize.scala b/morpheus-spark-cypher/src/main/scala/org/opencypher/morpheus/impl/expressions/Serialize.scala index 73986703b8..76f78dabc2 100644 --- a/morpheus-spark-cypher/src/main/scala/org/opencypher/morpheus/impl/expressions/Serialize.scala +++ b/morpheus-spark-cypher/src/main/scala/org/opencypher/morpheus/impl/expressions/Serialize.scala @@ -41,7 +41,8 @@ import org.opencypher.okapi.impl.exception /** * Encodes all children to a byte array (BinaryType). * - * For each child, writes the length of the serialized child followed by the actual serialized child. + * For each child, writes the length of the serialized child followed by the actual serialized + * child. */ case class Serialize(children: Seq[Expression]) extends Expression { @@ -55,11 +56,16 @@ case class Serialize(children: Seq[Expression]) extends Expression { val out = new ByteArrayOutputStream() children.foreach { child => child.dataType match { - case BinaryType => write(child.eval(input).asInstanceOf[Array[Byte]], out) - case StringType => write(child.eval(input).asInstanceOf[UTF8String], out) + case BinaryType => + write(child.eval(input).asInstanceOf[Array[Byte]], out) + case StringType => + write(child.eval(input).asInstanceOf[UTF8String], out) case IntegerType => write(child.eval(input).asInstanceOf[Int], out) - case LongType => write(child.eval(input).asInstanceOf[Long], out) - case other => throw exception.UnsupportedOperationException(s"Cannot serialize Spark data type $other.") + case LongType => write(child.eval(input).asInstanceOf[Long], out) + case other => + throw exception.UnsupportedOperationException( + s"Cannot serialize Spark data type $other." + ) } } out.toByteArray @@ -71,28 +77,37 @@ case class Serialize(children: Seq[Expression]) extends Expression { ): ExprCode = { ev.isNull = FalseLiteral val out = ctx.freshName("out") - val serializeChildren = children.map { child => - val childEval = child.genCode(ctx) - s"""|${childEval.code} + val serializeChildren = children + .map { child => + val childEval = child.genCode(ctx) + s"""|${childEval.code} |if (!${childEval.isNull}) { - | ${Serialize.getClass.getName.dropRight(1)}.write(${childEval.value}, $out); + | ${Serialize.getClass.getName.dropRight( + 1 + )}.write(${childEval.value}, $out); |}""".stripMargin - }.mkString("\n") + } + .mkString("\n") val baos = classOf[ByteArrayOutputStream].getName - ev.copy( - code = code"""|$baos $out = new $baos(); + ev.copy(code = code"""|$baos $out = new $baos(); |$serializeChildren |byte[] ${ev.value} = $out.toByteArray();""".stripMargin) } - override protected def withNewChildrenInternal(newChildren: scala.IndexedSeq[Expression]): Expression = copy(newChildren.toIndexedSeq) + override protected def withNewChildrenInternal( + newChildren: scala.IndexedSeq[Expression] + ): Expression = copy(newChildren.toIndexedSeq) } object Serialize { - val supportedTypes: Set[DataType] = Set(BinaryType, StringType, IntegerType, LongType) + val supportedTypes: Set[DataType] = + Set(BinaryType, StringType, IntegerType, LongType) - @inline final def write(value: Array[Byte], out: ByteArrayOutputStream): Unit = { + @inline final def write( + value: Array[Byte], + out: ByteArrayOutputStream + ): Unit = { out.write(encodeLong(value.length)) out.write(value) } @@ -102,14 +117,19 @@ object Serialize { out: ByteArrayOutputStream ): Unit = write(if (value) 1.toLong else 0.toLong, out) - @inline final def write(value: Byte, out: ByteArrayOutputStream): Unit = write(value.toLong, out) + @inline final def write(value: Byte, out: ByteArrayOutputStream): Unit = + write(value.toLong, out) - @inline final def write(value: Int, out: ByteArrayOutputStream): Unit = write(value.toLong, out) + @inline final def write(value: Int, out: ByteArrayOutputStream): Unit = + write(value.toLong, out) - @inline final def write(value: Long, out: ByteArrayOutputStream): Unit = write(encodeLong(value), out) + @inline final def write(value: Long, out: ByteArrayOutputStream): Unit = + write(encodeLong(value), out) - @inline final def write(value: UTF8String, out: ByteArrayOutputStream): Unit = write(value.getBytes, out) + @inline final def write(value: UTF8String, out: ByteArrayOutputStream): Unit = + write(value.getBytes, out) - @inline final def write(value: String, out: ByteArrayOutputStream): Unit = write(value.getBytes, out) + @inline final def write(value: String, out: ByteArrayOutputStream): Unit = + write(value.getBytes, out) } diff --git a/morpheus-spark-cypher/src/main/scala/org/opencypher/morpheus/impl/graph/MorpheusGraphFactory.scala b/morpheus-spark-cypher/src/main/scala/org/opencypher/morpheus/impl/graph/MorpheusGraphFactory.scala index 972fb5cb08..3bf603e15d 100644 --- a/morpheus-spark-cypher/src/main/scala/org/opencypher/morpheus/impl/graph/MorpheusGraphFactory.scala +++ b/morpheus-spark-cypher/src/main/scala/org/opencypher/morpheus/impl/graph/MorpheusGraphFactory.scala @@ -29,14 +29,20 @@ package org.opencypher.morpheus.impl.graph import org.opencypher.morpheus.api.MorpheusSession import org.opencypher.morpheus.impl.table.SparkTable.DataFrameTable import org.opencypher.morpheus.schema.MorpheusSchema._ -import org.opencypher.okapi.relational.api.graph.{RelationalCypherGraph, RelationalCypherGraphFactory} +import org.opencypher.okapi.relational.api.graph.{ + RelationalCypherGraph, + RelationalCypherGraphFactory +} import org.opencypher.okapi.relational.api.planning.RelationalRuntimeContext -case class MorpheusGraphFactory()(implicit val session: MorpheusSession) extends RelationalCypherGraphFactory[DataFrameTable] { +case class MorpheusGraphFactory()(implicit val session: MorpheusSession) + extends RelationalCypherGraphFactory[DataFrameTable] { override type Graph = RelationalCypherGraph[DataFrameTable] - override def unionGraph(graphs: RelationalCypherGraph[DataFrameTable]*)(implicit context: RelationalRuntimeContext[DataFrameTable]): Graph = { - val unionGraph = super.unionGraph(graphs:_*) - unionGraph.schema.asMorpheus //to check for schema compatibility + override def unionGraph( + graphs: RelationalCypherGraph[DataFrameTable]* + )(implicit context: RelationalRuntimeContext[DataFrameTable]): Graph = { + val unionGraph = super.unionGraph(graphs: _*) + unionGraph.schema.asMorpheus // to check for schema compatibility unionGraph } } diff --git a/morpheus-spark-cypher/src/main/scala/org/opencypher/morpheus/impl/io/MorpheusPropertyGraphDataSource.scala b/morpheus-spark-cypher/src/main/scala/org/opencypher/morpheus/impl/io/MorpheusPropertyGraphDataSource.scala index b6e2ddd4ec..7c33a7ee6e 100644 --- a/morpheus-spark-cypher/src/main/scala/org/opencypher/morpheus/impl/io/MorpheusPropertyGraphDataSource.scala +++ b/morpheus-spark-cypher/src/main/scala/org/opencypher/morpheus/impl/io/MorpheusPropertyGraphDataSource.scala @@ -34,6 +34,8 @@ trait MorpheusPropertyGraphDataSource extends PropertyGraphDataSource { protected def checkStorable(name: GraphName): Unit = { if (hasGraph(name)) - throw GraphAlreadyExistsException(s"A graph with name $name is already stored in this graph data source.") + throw GraphAlreadyExistsException( + s"A graph with name $name is already stored in this graph data source." + ) } } diff --git a/morpheus-spark-cypher/src/main/scala/org/opencypher/morpheus/impl/io/neo4j/external/Executor.scala b/morpheus-spark-cypher/src/main/scala/org/opencypher/morpheus/impl/io/neo4j/external/Executor.scala index f4e7913ab7..880ad73ccf 100644 --- a/morpheus-spark-cypher/src/main/scala/org/opencypher/morpheus/impl/io/neo4j/external/Executor.scala +++ b/morpheus-spark-cypher/src/main/scala/org/opencypher/morpheus/impl/io/neo4j/external/Executor.scala @@ -48,13 +48,17 @@ private object Executor { private def toJava(x: Any): AnyRef = x match { case y: Seq[_] => y.asJava - case _ => x.asInstanceOf[AnyRef] + case _ => x.asInstanceOf[AnyRef] } val EMPTY = Array.empty[Any] - class Neo4jQueryResult(val schema: StructType, val rows: Iterator[Array[Any]]) { - def sparkRows: Iterator[Row] = rows.map(row => new GenericRowWithSchema(row, schema)) + class Neo4jQueryResult( + val schema: StructType, + val rows: Iterator[Array[Any]] + ) { + def sparkRows: Iterator[Row] = + rows.map(row => new GenericRowWithSchema(row, schema)) def fields: Array[String] = schema.fieldNames } @@ -65,7 +69,11 @@ private object Executor { i } - def execute(config: Neo4jConfig, query: String, parameters: Map[String, Any]): Neo4jQueryResult = { + def execute( + config: Neo4jConfig, + query: String, + parameters: Map[String, Any] + ): Neo4jQueryResult = { config.withSession { session => val result: StatementResult = session.run(query, toJava(parameters)) if (!result.hasNext) { @@ -75,12 +83,17 @@ private object Executor { val keyCount = peek.size() if (keyCount == 0) { val res: Neo4jQueryResult = - new Neo4jQueryResult(new StructType(), Array.fill[Array[Any]](rows(result))(EMPTY).toIterator) + new Neo4jQueryResult( + new StructType(), + Array.fill[Array[Any]](rows(result))(EMPTY).toIterator + ) result.consume() return res } val keys = peek.keys().asScala - val fields = keys.map(k => (k, peek.get(k).`type`())).map(keyType => CypherTypes.field(keyType)) + val fields = keys + .map(k => (k, peek.get(k).`type`())) + .map(keyType => CypherTypes.field(keyType)) val schema = StructType(fields) val it = result.asScala.map { record => @@ -103,11 +116,16 @@ private object Executor { v match { case l: ListValue => l.asList(mapFunction).toArray - case d: LocalDateTimeValue => java.sql.Timestamp.valueOf(d.asLocalDateTime()) + case d: LocalDateTimeValue => + java.sql.Timestamp.valueOf(d.asLocalDateTime()) case d: DateValue => java.sql.Date.valueOf(d.asLocalDate()) case d: DurationValue => val iso = d.asIsoDuration() - new CalendarInterval(iso.months().toInt, iso.days().toInt, iso.nanoseconds() / 1000) + new CalendarInterval( + iso.months().toInt, + iso.days().toInt, + iso.nanoseconds() / 1000 + ) case other => other.asObject() } } @@ -121,20 +139,23 @@ private object CypherTypes { val NULL: NullType.type = types.NullType def apply(typ: String): DataType = typ.toUpperCase match { - case "LONG" => INTEGER - case "INT" => INTEGER + case "LONG" => INTEGER + case "INT" => INTEGER case "INTEGER" => INTEGER - case "FLOAT" => FlOAT - case "DOUBLE" => FlOAT + case "FLOAT" => FlOAT + case "DOUBLE" => FlOAT case "NUMERIC" => FlOAT - case "STRING" => STRING + case "STRING" => STRING case "BOOLEAN" => BOOLEAN - case "BOOL" => BOOLEAN - case "NULL" => NULL - case _ => STRING + case "BOOL" => BOOLEAN + case "NULL" => NULL + case _ => STRING } - def toSparkType(typeSystem: TypeSystem, typ: Type): org.apache.spark.sql.types.DataType = + def toSparkType( + typeSystem: TypeSystem, + typ: Type + ): org.apache.spark.sql.types.DataType = if (typ == typeSystem.BOOLEAN()) CypherTypes.BOOLEAN else if (typ == typeSystem.STRING()) CypherTypes.STRING else if (typ == typeSystem.INTEGER()) CypherTypes.INTEGER @@ -143,16 +164,23 @@ private object CypherTypes { else CypherTypes.STRING def field(keyType: (String, Type)): StructField = { - StructField(keyType._1, CypherTypes.toSparkType(InternalTypeSystem.TYPE_SYSTEM, keyType._2)) + StructField( + keyType._1, + CypherTypes.toSparkType(InternalTypeSystem.TYPE_SYSTEM, keyType._2) + ) } def schemaFromNamedType(schemaInfo: Seq[(String, String)]): StructType = { - val fields = schemaInfo.map(field => StructField(field._1, CypherTypes(field._2), nullable = true)) + val fields = + schemaInfo.map(field => StructField(field._1, CypherTypes(field._2), nullable = true)) StructType(fields) } - def schemaFromDataType(schemaInfo: Seq[(String, types.DataType)]): StructType = { - val fields = schemaInfo.map(field => StructField(field._1, field._2, nullable = true)) + def schemaFromDataType( + schemaInfo: Seq[(String, types.DataType)] + ): StructType = { + val fields = + schemaInfo.map(field => StructField(field._1, field._2, nullable = true)) StructType(fields) } } diff --git a/morpheus-spark-cypher/src/main/scala/org/opencypher/morpheus/impl/io/neo4j/external/Neo4j.scala b/morpheus-spark-cypher/src/main/scala/org/opencypher/morpheus/impl/io/neo4j/external/Neo4j.scala index 5493788eb4..4fc0ac3def 100644 --- a/morpheus-spark-cypher/src/main/scala/org/opencypher/morpheus/impl/io/neo4j/external/Neo4j.scala +++ b/morpheus-spark-cypher/src/main/scala/org/opencypher/morpheus/impl/io/neo4j/external/Neo4j.scala @@ -76,18 +76,24 @@ object Neo4j { } case class Partitions( - partitions: Long = 1, - batchSize: Long = Neo4j.UNDEFINED, - rows: Long = Neo4j.UNDEFINED, - rowSource: Option[() => Long] = None) { + partitions: Long = 1, + batchSize: Long = Neo4j.UNDEFINED, + rows: Long = Neo4j.UNDEFINED, + rowSource: Option[() => Long] = None +) { - def upper(v1: Long, v2: Long): Long = v1 / v2 + Math.signum(v1 % v2).asInstanceOf[Long] + def upper(v1: Long, v2: Long): Long = + v1 / v2 + Math.signum(v1 % v2).asInstanceOf[Long] def effective(): Partitions = { - if (this.batchSize == Neo4j.UNDEFINED && this.rows == Neo4j.UNDEFINED) return Partitions() - if (this.batchSize == Neo4j.UNDEFINED) return this.copy(batchSize = upper(rows, partitions)) - if (this.rows == Neo4j.UNDEFINED) return this.copy(rows = this.batchSize * this.partitions) - if (this.partitions == 1) return this.copy(partitions = upper(rows, batchSize)) + if (this.batchSize == Neo4j.UNDEFINED && this.rows == Neo4j.UNDEFINED) + return Partitions() + if (this.batchSize == Neo4j.UNDEFINED) + return this.copy(batchSize = upper(rows, partitions)) + if (this.rows == Neo4j.UNDEFINED) + return this.copy(rows = this.batchSize * this.partitions) + if (this.partitions == 1) + return this.copy(partitions = upper(rows, batchSize)) this } @@ -100,7 +106,10 @@ case class Partitions( } } -case class Neo4j(config: Neo4jConfig, session: SparkSession) extends QueriesDsl with PartitionsDsl with LoadDsl { +case class Neo4j(config: Neo4jConfig, session: SparkSession) + extends QueriesDsl + with PartitionsDsl + with LoadDsl { var nodes: Query = Query(null) var rels: Query = Query(null) @@ -109,7 +118,10 @@ case class Neo4j(config: Neo4jConfig, session: SparkSession) extends QueriesDsl // configure plain query - override def cypher(cypher: String, params: Map[String, Any] = Map.empty): Neo4j = { + override def cypher( + cypher: String, + params: Map[String, Any] = Map.empty + ): Neo4j = { this.nodes = Query(cypher, this.nodes.params ++ params) this } @@ -127,7 +139,10 @@ case class Neo4j(config: Neo4jConfig, session: SparkSession) extends QueriesDsl override def nodes(cypher: String, params: Map[String, Any]): Neo4j = this.cypher(cypher, params) - override def rels(cypher: String, params: Map[String, Any] = Map.empty): Neo4j = { + override def rels( + cypher: String, + params: Map[String, Any] = Map.empty + ): Neo4j = { this.rels = Query(cypher, params) this } @@ -158,14 +173,26 @@ case class Neo4j(config: Neo4jConfig, session: SparkSession) extends QueriesDsl override def loadNodeRdds: RDD[Row] = if (!nodes.isEmpty) { - new Neo4jRDD(session.sparkContext, nodes.query, config, nodes.params, partitions) + new Neo4jRDD( + session.sparkContext, + nodes.query, + config, + nodes.params, + partitions + ) } else { throw IllegalArgumentException("node query") } override def loadRelRdd: RDD[Row] = if (!rels.isEmpty) { - new Neo4jRDD(session.sparkContext, rels.query, config, rels.params, partitions) + new Neo4jRDD( + session.sparkContext, + rels.query, + config, + rels.params, + partitions + ) } else { throw IllegalArgumentException("relationship query") } diff --git a/morpheus-spark-cypher/src/main/scala/org/opencypher/morpheus/impl/io/neo4j/external/Neo4jPartition.scala b/morpheus-spark-cypher/src/main/scala/org/opencypher/morpheus/impl/io/neo4j/external/Neo4jPartition.scala index 5f5eb5faee..35961ea713 100644 --- a/morpheus-spark-cypher/src/main/scala/org/opencypher/morpheus/impl/io/neo4j/external/Neo4jPartition.scala +++ b/morpheus-spark-cypher/src/main/scala/org/opencypher/morpheus/impl/io/neo4j/external/Neo4jPartition.scala @@ -28,10 +28,15 @@ package org.opencypher.morpheus.impl.io.neo4j.external import org.apache.spark.Partition -private class Neo4jPartition(idx: Long = 0, skip: Long = 0, limit: Long = Long.MaxValue) extends Partition { +private class Neo4jPartition( + idx: Long = 0, + skip: Long = 0, + limit: Long = Long.MaxValue +) extends Partition { override def index: Int = idx.toInt val window: Map[String, Any] = Map("_limit" -> limit, "_skip" -> skip) - override def toString: String = s"Neo4jRDD index $index skip $skip limit: $limit" + override def toString: String = + s"Neo4jRDD index $index skip $skip limit: $limit" } diff --git a/morpheus-spark-cypher/src/main/scala/org/opencypher/morpheus/impl/io/neo4j/external/Neo4jRDD.scala b/morpheus-spark-cypher/src/main/scala/org/opencypher/morpheus/impl/io/neo4j/external/Neo4jRDD.scala index 92759a32f5..18234ac5e2 100644 --- a/morpheus-spark-cypher/src/main/scala/org/opencypher/morpheus/impl/io/neo4j/external/Neo4jRDD.scala +++ b/morpheus-spark-cypher/src/main/scala/org/opencypher/morpheus/impl/io/neo4j/external/Neo4jRDD.scala @@ -32,24 +32,32 @@ import org.apache.spark.{Partition, SparkContext, TaskContext} import org.opencypher.okapi.neo4j.io.Neo4jConfig private class Neo4jRDD( - sc: SparkContext, - val query: String, - val neo4jConfig: Neo4jConfig, - val parameters: Map[String, Any] = Map.empty, - partitions: Partitions = Partitions()) - extends RDD[Row](sc, Nil) { + sc: SparkContext, + val query: String, + val neo4jConfig: Neo4jConfig, + val parameters: Map[String, Any] = Map.empty, + partitions: Partitions = Partitions() +) extends RDD[Row](sc, Nil) { - override def compute(partition: Partition, context: TaskContext): Iterator[Row] = { + override def compute( + partition: Partition, + context: TaskContext + ): Iterator[Row] = { val neo4jPartition: Neo4jPartition = partition.asInstanceOf[Neo4jPartition] - Executor.execute(neo4jConfig, query, parameters ++ neo4jPartition.window).sparkRows + Executor + .execute(neo4jConfig, query, parameters ++ neo4jPartition.window) + .sparkRows } override protected def getPartitions: Array[Partition] = { val p = partitions.effective() - Range(0, p.partitions.toInt).map(idx => new Neo4jPartition(idx, p.skip(idx), p.limit(idx))).toArray + Range(0, p.partitions.toInt) + .map(idx => new Neo4jPartition(idx, p.skip(idx), p.limit(idx))) + .toArray } - override def toString(): String = s"Neo4jRDD partitions $partitions $query using $parameters" + override def toString(): String = + s"Neo4jRDD partitions $partitions $query using $parameters" } diff --git a/morpheus-spark-cypher/src/main/scala/org/opencypher/morpheus/impl/table/SparkTable.scala b/morpheus-spark-cypher/src/main/scala/org/opencypher/morpheus/impl/table/SparkTable.scala index 26e2270892..c42181900e 100644 --- a/morpheus-spark-cypher/src/main/scala/org/opencypher/morpheus/impl/table/SparkTable.scala +++ b/morpheus-spark-cypher/src/main/scala/org/opencypher/morpheus/impl/table/SparkTable.scala @@ -56,34 +56,49 @@ object SparkTable { override def physicalColumns: Seq[String] = df.columns - override def columnType: Map[String, CypherType] = physicalColumns.map(c => c -> df.cypherTypeForColumn(c)).toMap - - override def rows: Iterator[String => CypherValue] = df.toLocalIterator.asScala.map { row => - physicalColumns.map(c => c -> CypherValue(row.get(row.fieldIndex(c)))).toMap - } + override def columnType: Map[String, CypherType] = + physicalColumns.map(c => c -> df.cypherTypeForColumn(c)).toMap + + override def rows: Iterator[String => CypherValue] = + df.toLocalIterator.asScala.map { row => + physicalColumns + .map(c => c -> CypherValue(row.get(row.fieldIndex(c)))) + .toMap + } override def size: Long = df.count() - override def select(col: (String, String), cols: (String, String)*): DataFrameTable = { + override def select( + col: (String, String), + cols: (String, String)* + ): DataFrameTable = { val columns = col +: cols if (df.columns.toSeq == columns.map { case (_, alias) => alias }) { df } else { // Spark interprets dots in column names as struct accessors. Hence, we need to escape column names by default. - df.select(columns.map { case (colName, alias) => df.col(s"`$colName`").as(alias) }: _*) + df.select(columns.map { case (colName, alias) => + df.col(s"`$colName`").as(alias) + }: _*) } } - override def filter(expr: Expr)(implicit header: RecordHeader, parameters: CypherMap): DataFrameTable = { + override def filter( + expr: Expr + )(implicit header: RecordHeader, parameters: CypherMap): DataFrameTable = { df.where(expr.asSparkSQLExpr(header, df, parameters)) } - override def withColumns(columns: (Expr, String)*) - (implicit header: RecordHeader, parameters: CypherMap): DataFrameTable = { - val initialColumnNameToColumn: Map[String, Column] = df.columns.map(c => c -> df.col(c)).toMap - val updatedColumns = columns.foldLeft(initialColumnNameToColumn) { case (columnMap, (expr, columnName)) => - val column = expr.asSparkSQLExpr(header, df, parameters).as(columnName) - columnMap + (columnName -> column) + override def withColumns( + columns: (Expr, String)* + )(implicit header: RecordHeader, parameters: CypherMap): DataFrameTable = { + val initialColumnNameToColumn: Map[String, Column] = + df.columns.map(c => c -> df.col(c)).toMap + val updatedColumns = columns.foldLeft(initialColumnNameToColumn) { + case (columnMap, (expr, columnName)) => + val column = + expr.asSparkSQLExpr(header, df, parameters).as(columnName) + columnMap + (columnName -> column) } // TODO: Re-enable this check as soon as types (and their nullability) are correctly inferred in typing phase // if (!expr.cypherType.isNullable) { @@ -103,12 +118,13 @@ object SparkTable { df.drop(cols: _*) } - override def orderBy(sortItems: (Expr, Order)*) - (implicit header: RecordHeader, parameters: CypherMap): DataFrameTable = { + override def orderBy( + sortItems: (Expr, Order)* + )(implicit header: RecordHeader, parameters: CypherMap): DataFrameTable = { val mappedSortItems = sortItems.map { case (expr, order) => val mappedExpr = expr.asSparkSQLExpr(header, df, parameters) order match { - case Ascending => mappedExpr.asc + case Ascending => mappedExpr.asc case Descending => mappedExpr.desc } } @@ -127,12 +143,15 @@ object SparkTable { } override def limit(items: Long): DataFrameTable = { - if (items > Int.MaxValue) throw IllegalArgumentException("an integer", items) + if (items > Int.MaxValue) + throw IllegalArgumentException("an integer", items) df.limit(items.toInt) } - override def group(by: Set[Var], aggregations: Map[String, Aggregator]) - (implicit header: RecordHeader, parameters: CypherMap): DataFrameTable = { + override def group( + by: Set[Var], + aggregations: Map[String, Aggregator] + )(implicit header: RecordHeader, parameters: CypherMap): DataFrameTable = { def withInnerExpr(expr: Expr)(f: Column => Column) = f(expr.asSparkSQLExpr(header, df, parameters)) @@ -148,8 +167,8 @@ object SparkTable { Right(df) } - val sparkAggFunctions = aggregations.map { - case (columnName, aggFunc) => aggFunc.asSparkSQLExpr(header, df, parameters).as(columnName) + val sparkAggFunctions = aggregations.map { case (columnName, aggFunc) => + aggFunc.asSparkSQLExpr(header, df, parameters).as(columnName) } data.fold( @@ -166,20 +185,26 @@ object SparkTable { case (leftType, rightType) if !leftType.nullable.couldBeSameTypeAs(rightType.nullable) => throw IllegalArgumentException( "Equal column data types for union all (differing nullability is OK)", - s"Left fields: ${df.schema.fields.mkString(", ")}\n\tRight fields: ${other.df.schema.fields.mkString(", ")}") + s"Left fields: ${df.schema.fields + .mkString(", ")}\n\tRight fields: ${other.df.schema.fields.mkString(", ")}" + ) case _ => } df.union(other.df) } - override def join(other: DataFrameTable, joinType: JoinType, joinCols: (String, String)*): DataFrameTable = { + override def join( + other: DataFrameTable, + joinType: JoinType, + joinCols: (String, String)* + ): DataFrameTable = { val joinTypeString = joinType match { - case InnerJoin => "inner" - case LeftOuterJoin => "left_outer" + case InnerJoin => "inner" + case LeftOuterJoin => "left_outer" case RightOuterJoin => "right_outer" - case FullOuterJoin => "full_outer" - case CrossJoin => "cross" + case FullOuterJoin => "full_outer" + case CrossJoin => "cross" } joinType match { @@ -187,8 +212,13 @@ object SparkTable { df.crossJoin(other.df) case LeftOuterJoin - if joinCols.isEmpty && df.sparkSession.conf.get("spark.sql.crossJoin.enabled", "false") == "false" => - throw UnsupportedOperationException("OPTIONAL MATCH support requires spark.sql.crossJoin.enabled=true") + if joinCols.isEmpty && df.sparkSession.conf.get( + "spark.sql.crossJoin.enabled", + "false" + ) == "false" => + throw UnsupportedOperationException( + "OPTIONAL MATCH support requires spark.sql.crossJoin.enabled=true" + ) case _ => df.safeJoin(other.df, joinCols, joinTypeString) @@ -197,12 +227,18 @@ object SparkTable { override def distinct: DataFrameTable = df.dropDuplicates() - override def distinct(cols: String*): DataFrameTable = df.dropDuplicates(cols.toSeq) + override def distinct(cols: String*): DataFrameTable = + df.dropDuplicates(cols.toSeq) override def cache(): DataFrameTable = { val planToCache = df.queryExecution.analyzed - if (df.sparkSession.sharedState.cacheManager.lookupCachedData(planToCache).nonEmpty) { - df.sparkSession.sharedState.cacheManager.cacheQuery(df, None, StorageLevel.MEMORY_ONLY) + if ( + df.sparkSession.sharedState.cacheManager + .lookupCachedData(planToCache) + .nonEmpty + ) { + df.sparkSession.sharedState.cacheManager + .cacheQuery(df, None, StorageLevel.MEMORY_ONLY) } this } @@ -220,28 +256,41 @@ object SparkTable { } implicit class DataFrameMeta(val df: DataFrame) extends AnyVal { + /** * Returns the corresponding Cypher type for the given column name in the data frame. * - * @param columnName column name - * @return Cypher type for column + * @param columnName + * column name + * @return + * Cypher type for column */ def cypherTypeForColumn(columnName: String): CypherType = { val structField = structFieldForColumn(columnName) - val compatibleCypherType = structField.dataType.cypherCompatibleDataType.flatMap(_.toCypherType(structField.nullable)) + val compatibleCypherType = structField.dataType.cypherCompatibleDataType + .flatMap(_.toCypherType(structField.nullable)) compatibleCypherType.getOrElse( - throw IllegalArgumentException("a supported Spark DataType that can be converted to CypherType", structField.dataType)) + throw IllegalArgumentException( + "a supported Spark DataType that can be converted to CypherType", + structField.dataType + ) + ) } /** * Returns the struct field for the given column. * - * @param columnName column name - * @return struct field + * @param columnName + * column name + * @return + * struct field */ def structFieldForColumn(columnName: String): StructField = { if (df.schema.fieldIndex(columnName) < 0) { - throw IllegalArgumentException(s"column with name $columnName", s"columns with names ${df.columns.mkString("[", ", ", "]")}") + throw IllegalArgumentException( + s"column with name $columnName", + s"columns with names ${df.columns.mkString("[", ", ", "]")}" + ) } df.schema.fields(df.schema.fieldIndex(columnName)) } @@ -249,8 +298,11 @@ object SparkTable { implicit class DataFrameValidation(val df: DataFrame) extends AnyVal { - def validateColumnTypes(expectedColsWithType: Map[String, CypherType]): Unit = { - val missingColumns = expectedColsWithType.keySet -- df.schema.fieldNames.toSet + def validateColumnTypes( + expectedColsWithType: Map[String, CypherType] + ): Unit = { + val missingColumns = + expectedColsWithType.keySet -- df.schema.fieldNames.toSet if (missingColumns.nonEmpty) { throw IllegalArgumentException( @@ -262,24 +314,27 @@ object SparkTable { ) } - val structFields = df.schema.fields.map(field => field.name -> field).toMap + val structFields = + df.schema.fields.map(field => field.name -> field).toMap - expectedColsWithType.foreach { - case (column, cypherType) => - val structField = structFields(column) + expectedColsWithType.foreach { case (column, cypherType) => + val structField = structFields(column) - val structFieldType = structField.toCypherType match { - case Some(cType) => cType - case None => throw IllegalArgumentException( + val structFieldType = structField.toCypherType match { + case Some(cType) => cType + case None => + throw IllegalArgumentException( expected = s"Cypher-compatible DataType for column $column", - actual = structField.dataType) - } + actual = structField.dataType + ) + } - if (!structFieldType.material.subTypeOf(cypherType.material)) { - throw IllegalArgumentException( - expected = s"Sub-type of $cypherType for column: $column", - actual = structFieldType) - } + if (!structFieldType.material.subTypeOf(cypherType.material)) { + throw IllegalArgumentException( + expected = s"Sub-type of $cypherType for column: $column", + actual = structFieldType + ) + } } } } @@ -287,9 +342,11 @@ object SparkTable { implicit class DataFrameTransformation(val df: DataFrame) extends AnyVal { def safeAddColumn(name: String, col: Column): DataFrame = { - require(!df.columns.contains(name), + require( + !df.columns.contains(name), s"Cannot add column `$name`. A column with that name exists already. " + - s"Use `safeReplaceColumn` if you intend to replace that column.") + s"Use `safeReplaceColumn` if you intend to replace that column." + ) df.withColumn(name, col) } @@ -300,8 +357,11 @@ object SparkTable { } def safeReplaceColumn(name: String, newColumn: Column): DataFrame = { - require(df.columns.contains(name), s"Cannot replace column `$name`. No column with that name exists. " + - s"Use `safeAddColumn` if you intend to add that column.") + require( + df.columns.contains(name), + s"Cannot replace column `$name`. No column with that name exists. " + + s"Use `safeAddColumn` if you intend to add that column." + ) df.safeAddColumn(name, newColumn) } @@ -310,15 +370,23 @@ object SparkTable { } def safeRenameColumns(renames: Map[String, String]): DataFrame = { - if (renames.isEmpty || renames.forall { case (oldColumn, newColumn) => oldColumn == newColumn }) { + if ( + renames.isEmpty || renames.forall { case (oldColumn, newColumn) => + oldColumn == newColumn + } + ) { df } else { - renames.foreach { case (oldName, newName) => if (oldName != newName) require(!df.columns.contains(newName), - s"Cannot rename column `$oldName` to `$newName`. A column with name `$newName` exists already.") + renames.foreach { case (oldName, newName) => + if (oldName != newName) + require( + !df.columns.contains(newName), + s"Cannot rename column `$oldName` to `$newName`. A column with name `$newName` exists already." + ) } val newColumns = df.columns.map { case col if renames.contains(col) => renames(col) - case col => col + case col => col } df.toDF(newColumns: _*) } @@ -326,19 +394,27 @@ object SparkTable { def safeDropColumns(names: String*): DataFrame = { val nonExistentColumns = names.toSet -- df.columns - require(nonExistentColumns.isEmpty, - s"Cannot drop column(s) ${nonExistentColumns.map(c => s"`$c`").mkString(", ")}. They do not exist.") + require( + nonExistentColumns.isEmpty, + s"Cannot drop column(s) ${nonExistentColumns.map(c => s"`$c`").mkString(", ")}. They do not exist." + ) df.drop(names: _*) } - def safeJoin(other: DataFrame, joinCols: Seq[(String, String)], joinType: String): DataFrame = { + def safeJoin( + other: DataFrame, + joinCols: Seq[(String, String)], + joinType: String + ): DataFrame = { require(joinCols.map(_._1).forall(col => !other.columns.contains(col))) require(joinCols.map(_._2).forall(col => !df.columns.contains(col))) val joinExpr = if (joinCols.nonEmpty) { - joinCols.map { - case (l, r) => df.col(l) === other.col(r) - }.reduce((acc, expr) => acc && expr) + joinCols + .map { case (l, r) => + df.col(l) === other.col(r) + } + .reduce((acc, expr) => acc && expr) } else { functions.lit(true) } @@ -346,18 +422,22 @@ object SparkTable { } def prefixColumns(prefix: String): DataFrame = - df.safeRenameColumns(df.columns.map(column => column -> s"$prefix$column").toMap) + df.safeRenameColumns( + df.columns.map(column => column -> s"$prefix$column").toMap + ) def removePrefix(prefix: String): DataFrame = { val columnRenames = df.columns.collect { - case column if column.startsWith(prefix) => column -> column.substring(prefix.length) + case column if column.startsWith(prefix) => + column -> column.substring(prefix.length) } df.safeRenameColumns(columnRenames.toMap) } def encodeBinaryToHexString: DataFrame = { val columnsToSelect = df.schema.map { - case sf: StructField if sf.dataType == BinaryType => functions.hex(df.col(sf.name)).as(sf.name) + case sf: StructField if sf.dataType == BinaryType => + functions.hex(df.col(sf.name)).as(sf.name) case sf: StructField => df.col(sf.name) } df.select(columnsToSelect: _*) @@ -366,7 +446,7 @@ object SparkTable { def transformColumns(cols: String*)(f: Column => Column): DataFrame = { val columnsToSelect = df.columns.map { case c if cols.contains(c) => f(df.col(c)) - case c => df.col(c) + case c => df.col(c) } df.select(columnsToSelect: _*) } @@ -374,7 +454,10 @@ object SparkTable { def decodeHexStringToBinary(hexColumns: Set[String]): DataFrame = { val columnsToSelect = df.schema.map { case sf: StructField if hexColumns.contains(sf.name) => - assert(sf.dataType == StringType, "Can only decode hex columns of StringType to BinaryType") + assert( + sf.dataType == StringType, + "Can only decode hex columns of StringType to BinaryType" + ) functions.unhex(df.col(sf.name)).as(sf.name) case sf: StructField => df.col(sf.name) } @@ -385,13 +468,16 @@ object SparkTable { idColumns.map { key => df.structFieldForColumn(key).dataType match { case LongType => df.col(key).encodeLongAsMorpheusId(key) - case IntegerType => df.col(key).cast(LongType).encodeLongAsMorpheusId(key) + case IntegerType => + df.col(key).cast(LongType).encodeLongAsMorpheusId(key) case StringType => df.col(key).cast(BinaryType) case BinaryType => df.col(key) - case unsupportedType => throw IllegalArgumentException( - expected = s"Column `$key` should have a valid identifier data type, such as [`$BinaryType`, `$StringType`, `$LongType`, `$IntegerType`]", - actual = s"Unsupported column type `$unsupportedType`" - ) + case unsupportedType => + throw IllegalArgumentException( + expected = + s"Column `$key` should have a valid identifier data type, such as [`$BinaryType`, `$StringType`, `$LongType`, `$IntegerType`]", + actual = s"Unsupported column type `$unsupportedType`" + ) } } } @@ -399,115 +485,146 @@ object SparkTable { /** * Cast all integer columns in a DataFrame to long. * - * @return a DataFrame with all integer values cast to long + * @return + * a DataFrame with all integer values cast to long */ def castToLong: DataFrame = { def convertColumns(field: StructField, col: Column): Column = { val convertedCol = field.dataType match { case StructType(inner) => - val columns = inner.map(i => convertColumns(i, col.getField(i.name)).as(i.name)) + val columns = + inner.map(i => convertColumns(i, col.getField(i.name)).as(i.name)) functions.struct(columns: _*) - case ArrayType(IntegerType, nullable) => col.cast(ArrayType(LongType, nullable)) + case ArrayType(IntegerType, nullable) => + col.cast(ArrayType(LongType, nullable)) case IntegerType => col.cast(LongType) - case _ => col + case _ => col } if (col == convertedCol) col else convertedCol.as(field.name) } - val convertedColumns = df.schema.fields.map { field => convertColumns(field, df.col(field.name)) } - if (df.columns.map(df.col).sameElements(convertedColumns)) df else df.select(convertedColumns: _*) + val convertedColumns = df.schema.fields.map { field => + convertColumns(field, df.col(field.name)) + } + if (df.columns.map(df.col).sameElements(convertedColumns)) df + else df.select(convertedColumns: _*) } /** * Adds a new column `hashColumn` containing the hash value of the given input columns. * - * The hash is generated using [[org.apache.spark.sql.catalyst.expressions.Murmur3Hash]] based on the given column - * sequence. To decrease collision probability, we: + * The hash is generated using [[org.apache.spark.sql.catalyst.expressions.Murmur3Hash]] based + * on the given column sequence. To decrease collision probability, we: * - * 1) generate a first hash for the given column sequence - * 2) shift the hash into the upper bits of a 64 bit long - * 3) generate a second hash using the reversed input column sequence - * 4) store the hash in the lower 32 bits of the final id + * 1) generate a first hash for the given column sequence 2) shift the hash into the upper bits + * of a 64 bit long 3) generate a second hash using the reversed input column sequence 4) store + * the hash in the lower 32 bits of the final id * - * @param columns input columns for the hash function - * @param hashColumn column storing the result of the hash function - * @return DataFrame with an additional column that contains the hash ID + * @param columns + * input columns for the hash function + * @param hashColumn + * column storing the result of the hash function + * @return + * DataFrame with an additional column that contains the hash ID */ def withHashColumn(columns: Seq[Column], hashColumn: String): DataFrame = { - require(columns.nonEmpty, "Hash function requires a non-empty sequence of columns as input.") - df.safeAddColumn(hashColumn, MorpheusFunctions.hash64(columns: _*).encodeLongAsMorpheusId) + require( + columns.nonEmpty, + "Hash function requires a non-empty sequence of columns as input." + ) + df.safeAddColumn( + hashColumn, + MorpheusFunctions.hash64(columns: _*).encodeLongAsMorpheusId + ) } /** - * Adds a new column `serializedColumn` containing the serialized values of the given input columns. + * Adds a new column `serializedColumn` containing the serialized values of the given input + * columns. * - * @param columns input columns for the serialization function - * @param serializedColumn column storing the result of the serialization function - * @return DataFrame with an additional column that contains the serialized ID + * @param columns + * input columns for the serialization function + * @param serializedColumn + * column storing the result of the serialization function + * @return + * DataFrame with an additional column that contains the serialized ID */ - def withSerializedIdColumn(columns: Seq[Column], serializedColumn: String): DataFrame = { - require(columns.nonEmpty, "Serialized ID function requires a non-empty sequence of columns as input.") + def withSerializedIdColumn( + columns: Seq[Column], + serializedColumn: String + ): DataFrame = { + require( + columns.nonEmpty, + "Serialized ID function requires a non-empty sequence of columns as input." + ) df.safeAddColumn(serializedColumn, serialize(columns: _*)) } - /** - * Normalises the DataFrame by lifting numeric fields to Long and similar ops. - */ + /** Normalises the DataFrame by lifting numeric fields to Long and similar ops. */ def withCypherCompatibleTypes: DataFrame = { val toCast = df.schema.fields.filter(f => f.toCypherType.isEmpty) - val dfWithCompatibleTypes: DataFrame = toCast.foldLeft(df) { - case (currentDf, field) => - val castType = field.dataType.cypherCompatibleDataType.getOrElse( - throw IllegalArgumentException( - s"a Spark type supported by Cypher.", - s"type ${field.dataType} of field $field")) - currentDf.withColumn(field.name, currentDf.col(field.name).cast(castType)) + val dfWithCompatibleTypes: DataFrame = toCast.foldLeft(df) { case (currentDf, field) => + val castType = field.dataType.cypherCompatibleDataType.getOrElse( + throw IllegalArgumentException( + s"a Spark type supported by Cypher.", + s"type ${field.dataType} of field $field" + ) + ) + currentDf.withColumn( + field.name, + currentDf.col(field.name).cast(castType) + ) } dfWithCompatibleTypes } } implicit class DataFrameSequence(val dataFrames: Seq[DataFrame]) extends AnyVal { + /** - * Takes a sequence of DataFrames and adds long identifiers to all of them. Identifiers are guaranteed to be unique - * across all given DataFrames. The DataFrames are returned in the same order as the input. + * Takes a sequence of DataFrames and adds long identifiers to all of them. Identifiers are + * guaranteed to be unique across all given DataFrames. The DataFrames are returned in the same + * order as the input. * - * @param idColumnName column name for the generated id - * @return a sequence of DataFrames with unique long identifiers + * @param idColumnName + * column name for the generated id + * @return + * a sequence of DataFrames with unique long identifiers */ def addUniqueIds(idColumnName: String): Seq[DataFrame] = { // We need to know how many partitions a DF has in order to avoid writing into the id space of another DF. // This is why require a running sum of number of partitions because we add the DF-specific sum to the offset that // Sparks monotonically_increasing_id adds. val dfPartitionCounts = dataFrames.map(_.rdd.getNumPartitions) - val dfPartitionStartDeltas = dfPartitionCounts.scan(0)(_ + _).dropRight(1) // drop last delta, as we don't need it - - dataFrames.zip(dfPartitionStartDeltas).map { - case (df, partitionStartDelta) => - df.safeAddColumn(idColumnName, partitioned_id_assignment(partitionStartDelta)) + val dfPartitionStartDeltas = dfPartitionCounts + .scan(0)(_ + _) + .dropRight(1) // drop last delta, as we don't need it + + dataFrames.zip(dfPartitionStartDeltas).map { case (df, partitionStartDelta) => + df.safeAddColumn( + idColumnName, + partitioned_id_assignment(partitionStartDelta) + ) } } } implicit class DataFrameDebug(val df: DataFrame) extends AnyVal { - /** - * Prints timing of Spark computation for DF. - */ + + /** Prints timing of Spark computation for DF. */ def printExecutionTiming(description: String): Unit = { printTiming(s"$description") { df.count() // Force computation of DF } } - /** - * Prints Spark physical plan. - */ + /** Prints Spark physical plan. */ def printPhysicalPlan(): Unit = { println("Spark plan:") implicit val sc: SparkContext = df.sparkSession.sparkContext val sparkPlan: SparkPlan = df.queryExecution.executedPlan val planString = sparkPlan.treeString(verbose = false).flatMap { - case '\n' => Seq('\n', '\t') + case '\n' => Seq('\n', '\t') case other => Seq(other) } println(s"\t$planString") diff --git a/morpheus-spark-cypher/src/main/scala/org/opencypher/morpheus/impl/temporal/TemporalConversions.scala b/morpheus-spark-cypher/src/main/scala/org/opencypher/morpheus/impl/temporal/TemporalConversions.scala index 7cea7a4f4b..7f12b34600 100644 --- a/morpheus-spark-cypher/src/main/scala/org/opencypher/morpheus/impl/temporal/TemporalConversions.scala +++ b/morpheus-spark-cypher/src/main/scala/org/opencypher/morpheus/impl/temporal/TemporalConversions.scala @@ -1,28 +1,25 @@ /** * Copyright (c) 2016-2019 "Neo4j Sweden, AB" [https://neo4j.com] * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and + * Unless required by applicable law or agreed to in writing, software distributed under the + * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing permissions and * limitations under the License. * * Attribution Notice under the terms of the Apache License 2.0 * - * This work was created by the collective efforts of the openCypher community. - * Without limiting the terms of Section 6, any Derivative Work that is not - * approved by the public consensus process of the openCypher Implementers Group - * should not be described as “Cypher” (and Cypher® is a registered trademark of - * Neo4j Inc.) or as "openCypher". Extensions by implementers or prototypes or - * proposals for change that have been documented or implemented should only be - * described as "implementation extensions to Cypher" or as "proposed changes to - * Cypher that are not yet approved by the openCypher community". + * This work was created by the collective efforts of the openCypher community. Without limiting + * the terms of Section 6, any Derivative Work that is not approved by the public consensus process + * of the openCypher Implementers Group should not be described as “Cypher” (and Cypher® is a + * registered trademark of Neo4j Inc.) or as "openCypher". Extensions by implementers or prototypes + * or proposals for change that have been documented or implemented should only be described as + * "implementation extensions to Cypher" or as "proposed changes to Cypher that are not yet + * approved by the openCypher community". */ package org.opencypher.morpheus.impl.temporal @@ -34,7 +31,12 @@ import org.opencypher.okapi.impl.temporal.TemporalTypesHelper._ import org.apache.spark.sql.{Column, functions} import org.apache.spark.unsafe.types.CalendarInterval import org.opencypher.okapi.api.value.CypherValue.{CypherInteger, CypherMap, CypherString} -import org.opencypher.okapi.impl.exception.{IllegalArgumentException, IllegalStateException, NotImplementedException, UnsupportedOperationException} +import org.opencypher.okapi.impl.exception.{ + IllegalArgumentException, + IllegalStateException, + NotImplementedException, + UnsupportedOperationException +} import org.opencypher.okapi.impl.temporal.{Duration, TemporalConstants} import org.opencypher.okapi.ir.api.expr.{Expr, MapExpression, NullLit, Param} import org.opencypher.okapi.impl.temporal.{Duration => DurationValue} @@ -48,12 +50,15 @@ object TemporalConversions extends Logging { /** * Converts the Okapi representation of a duration into the spark representation. * - * @note This conversion is lossy, as the Sparks [[CalendarInterval]] only has a resolution down to microseconds. - * Additionally it uses an approximate representation of days. + * @note + * This conversion is lossy, as the Sparks [[CalendarInterval]] only has a resolution down to + * microseconds. Additionally it uses an approximate representation of days. */ def toCalendarInterval: CalendarInterval = { if (duration.nanos % 1000 != 0) { - logger.warn("Spark does not support durations with nanosecond resolution, truncating!") + logger.warn( + "Spark does not support durations with nanosecond resolution, truncating!" + ) } val microseconds = duration.nanos / 1000 + @@ -70,15 +75,22 @@ object TemporalConversions extends Logging { /** * Converts the Spark representation of a duration into the Okapi representation. * - * @note To ensure compatibility with the reverse operation we estimate the number of days from the given seconds. + * @note + * To ensure compatibility with the reverse operation we estimate the number of days from the + * given seconds. */ implicit class RichCalendarInterval(calendarInterval: CalendarInterval) { def toDuration: Duration = { - val daysInSeconds = calendarInterval.days * DateTimeConstants.SECONDS_PER_DAY - val seconds = daysInSeconds + (calendarInterval.microseconds / DateTimeConstants.MICROS_PER_SECOND) - val normalizedDays = seconds / (DateTimeConstants.MICROS_PER_DAY / DateTimeConstants.MICROS_PER_SECOND) - val normalizedSeconds = seconds % (DateTimeConstants.MICROS_PER_DAY / DateTimeConstants.MICROS_PER_SECOND) - val normalizedNanos = calendarInterval.microseconds % DateTimeConstants.MICROS_PER_SECOND * 1000 + val daysInSeconds = + calendarInterval.days * DateTimeConstants.SECONDS_PER_DAY + val seconds = + daysInSeconds + (calendarInterval.microseconds / DateTimeConstants.MICROS_PER_SECOND) + val normalizedDays = + seconds / (DateTimeConstants.MICROS_PER_DAY / DateTimeConstants.MICROS_PER_SECOND) + val normalizedSeconds = + seconds % (DateTimeConstants.MICROS_PER_DAY / DateTimeConstants.MICROS_PER_SECOND) + val normalizedNanos = + calendarInterval.microseconds % DateTimeConstants.MICROS_PER_SECOND * 1000 Duration( months = calendarInterval.months, @@ -104,7 +116,10 @@ object TemporalConversions extends Logging { .map(java.sql.Timestamp.valueOf) .map { case ts if ts.getNanos % 1000 == 0 => ts - case _ => throw IllegalStateException("Spark does not support nanosecond resolution in 'localdatetime'") + case _ => + throw IllegalStateException( + "Spark does not support nanosecond resolution in 'localdatetime'" + ) } .orNull } @@ -118,22 +133,31 @@ object TemporalConversions extends Logging { def resolveInterval(implicit parameters: CypherMap): CalendarInterval = { expr.resolveTemporalArgument.map { - case Left(m) => DurationValue(m.mapValues(_.toLong)).toCalendarInterval + case Left(m) => DurationValue(m.mapValues(_.toLong)).toCalendarInterval case Right(s) => DurationValue.parse(s).toCalendarInterval }.orNull } - def resolveTemporalArgument(implicit parameters: CypherMap): Option[Either[Map[String, Int], String]] = { + def resolveTemporalArgument(implicit + parameters: CypherMap + ): Option[Either[Map[String, Int], String]] = { expr match { case MapExpression(inner) => val map = inner.map { - case (key, Param(name)) => key -> (parameters(name) match { - case CypherString(s) => s.toInt - case CypherInteger(i) => i.toInt - case other => throw IllegalArgumentException("A map value of type CypherString or CypherInteger", other) - }) + case (key, Param(name)) => + key -> (parameters(name) match { + case CypherString(s) => s.toInt + case CypherInteger(i) => i.toInt + case other => + throw IllegalArgumentException( + "A map value of type CypherString or CypherInteger", + other + ) + }) case (key, e) => - throw NotImplementedException(s"Parsing temporal values is currently only supported for Literal-Maps, got $key -> $e") + throw NotImplementedException( + s"Parsing temporal values is currently only supported for Literal-Maps, got $key -> $e" + ) } Some(Left(map)) @@ -141,7 +165,11 @@ object TemporalConversions extends Logging { case Param(name) => val s = parameters(name) match { case CypherString(str) => str - case other => throw IllegalArgumentException(s"Parameter `$name` to be a CypherString", other) + case other => + throw IllegalArgumentException( + s"Parameter `$name` to be a CypherString", + other + ) } Some(Right(s)) @@ -149,30 +177,39 @@ object TemporalConversions extends Logging { case NullLit => None case other => - throw NotImplementedException(s"Parsing temporal values is currently only supported for Literal-Maps and String literals, got $other") + throw NotImplementedException( + s"Parsing temporal values is currently only supported for Literal-Maps and String literals, got $other" + ) } } } - def temporalAccessor[I: TypeTag](temporalColumn: Column, accessor: String): Column = { + def temporalAccessor[I: TypeTag]( + temporalColumn: Column, + accessor: String + ): Column = { accessor.toLowerCase match { - case "year" => functions.year(temporalColumn) - case "quarter" => functions.quarter(temporalColumn) - case "month" => functions.month(temporalColumn) - case "week" => functions.weekofyear(temporalColumn) - case "day" => functions.dayofmonth(temporalColumn) - case "ordinalday" => functions.dayofyear(temporalColumn) - case "weekyear" => TemporalUdfs.weekYear[I].apply(temporalColumn) + case "year" => functions.year(temporalColumn) + case "quarter" => functions.quarter(temporalColumn) + case "month" => functions.month(temporalColumn) + case "week" => functions.weekofyear(temporalColumn) + case "day" => functions.dayofmonth(temporalColumn) + case "ordinalday" => functions.dayofyear(temporalColumn) + case "weekyear" => TemporalUdfs.weekYear[I].apply(temporalColumn) case "dayofquarter" => TemporalUdfs.dayOfQuarter[I].apply(temporalColumn) - case "dayofweek" | "weekday" => TemporalUdfs.dayOfWeek[I].apply(temporalColumn) + case "dayofweek" | "weekday" => + TemporalUdfs.dayOfWeek[I].apply(temporalColumn) - case "hour" => functions.hour(temporalColumn) - case "minute" => functions.minute(temporalColumn) - case "second" => functions.second(temporalColumn) + case "hour" => functions.hour(temporalColumn) + case "minute" => functions.minute(temporalColumn) + case "second" => functions.second(temporalColumn) case "millisecond" => TemporalUdfs.milliseconds[I].apply(temporalColumn) case "microsecond" => TemporalUdfs.microseconds[I].apply(temporalColumn) - case other => throw UnsupportedOperationException(s"Unknown Temporal Accessor: $other") + case other => + throw UnsupportedOperationException( + s"Unknown Temporal Accessor: $other" + ) } } } diff --git a/morpheus-spark-cypher/src/main/scala/org/opencypher/morpheus/impl/temporal/TemporalUdafs.scala b/morpheus-spark-cypher/src/main/scala/org/opencypher/morpheus/impl/temporal/TemporalUdafs.scala index 5caa4551f8..a069f6b14a 100644 --- a/morpheus-spark-cypher/src/main/scala/org/opencypher/morpheus/impl/temporal/TemporalUdafs.scala +++ b/morpheus-spark-cypher/src/main/scala/org/opencypher/morpheus/impl/temporal/TemporalUdafs.scala @@ -1,28 +1,25 @@ /** * Copyright (c) 2016-2019 "Neo4j Sweden, AB" [https://neo4j.com] * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and + * Unless required by applicable law or agreed to in writing, software distributed under the + * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing permissions and * limitations under the License. * * Attribution Notice under the terms of the Apache License 2.0 * - * This work was created by the collective efforts of the openCypher community. - * Without limiting the terms of Section 6, any Derivative Work that is not - * approved by the public consensus process of the openCypher Implementers Group - * should not be described as “Cypher” (and Cypher® is a registered trademark of - * Neo4j Inc.) or as "openCypher". Extensions by implementers or prototypes or - * proposals for change that have been documented or implemented should only be - * described as "implementation extensions to Cypher" or as "proposed changes to - * Cypher that are not yet approved by the openCypher community". + * This work was created by the collective efforts of the openCypher community. Without limiting + * the terms of Section 6, any Derivative Work that is not approved by the public consensus process + * of the openCypher Implementers Group should not be described as “Cypher” (and Cypher® is a + * registered trademark of Neo4j Inc.) or as "openCypher". Extensions by implementers or prototypes + * or proposals for change that have been documented or implemented should only be described as + * "implementation extensions to Cypher" or as "proposed changes to Cypher that are not yet + * approved by the openCypher community". */ package org.opencypher.morpheus.impl.temporal @@ -39,51 +36,90 @@ object TemporalUdafs extends Logging { private val intervalEncoder = ExpressionEncoder(CalendarIntervalEncoder) - trait SimpleDurationAggregation extends Aggregator[CalendarInterval, CalendarInterval, CalendarInterval] { - final override def finish(reduction: CalendarInterval): CalendarInterval = reduction - final override def bufferEncoder: Encoder[CalendarInterval] = intervalEncoder - final override def outputEncoder: Encoder[CalendarInterval] = intervalEncoder + trait SimpleDurationAggregation + extends Aggregator[CalendarInterval, CalendarInterval, CalendarInterval] { + final override def finish(reduction: CalendarInterval): CalendarInterval = + reduction + final override def bufferEncoder: Encoder[CalendarInterval] = + intervalEncoder + final override def outputEncoder: Encoder[CalendarInterval] = + intervalEncoder } object DurationSum extends SimpleDurationAggregation { override def zero: CalendarInterval = new CalendarInterval(0, 0, 0L) - override def reduce(b: CalendarInterval, a: CalendarInterval): CalendarInterval = IntervalUtils.add(b, a) - override def merge(b1: CalendarInterval, b2: CalendarInterval): CalendarInterval = IntervalUtils.add(b1, b2) + override def reduce( + b: CalendarInterval, + a: CalendarInterval + ): CalendarInterval = IntervalUtils.add(b, a) + override def merge( + b1: CalendarInterval, + b2: CalendarInterval + ): CalendarInterval = IntervalUtils.add(b1, b2) } object DurationMax extends SimpleDurationAggregation { override def zero: CalendarInterval = new CalendarInterval(0, 0, 0L) - override def reduce(b: CalendarInterval, a: CalendarInterval): CalendarInterval = { + override def reduce( + b: CalendarInterval, + a: CalendarInterval + ): CalendarInterval = { if (b.toDuration.compare(a.toDuration) >= 0) b else a } - override def merge(b1: CalendarInterval, b2: CalendarInterval): CalendarInterval = reduce(b1, b2) + override def merge( + b1: CalendarInterval, + b2: CalendarInterval + ): CalendarInterval = reduce(b1, b2) } object DurationMin extends SimpleDurationAggregation { - final override def zero: CalendarInterval = new CalendarInterval(Int.MaxValue, Int.MaxValue, Long.MaxValue) + final override def zero: CalendarInterval = + new CalendarInterval(Int.MaxValue, Int.MaxValue, Long.MaxValue) - override def reduce(b: CalendarInterval, a: CalendarInterval): CalendarInterval = { + override def reduce( + b: CalendarInterval, + a: CalendarInterval + ): CalendarInterval = { if (b.toDuration.compare(a.toDuration) >= 0) a else b } - override def merge(b1: CalendarInterval, b2: CalendarInterval): CalendarInterval = reduce(b1, b2) + override def merge( + b1: CalendarInterval, + b2: CalendarInterval + ): CalendarInterval = reduce(b1, b2) } - case class DurationAvgRunningSum(months: Int, days: Int, micros: Long, count: Long) - - object DurationAvg extends Aggregator[CalendarInterval, DurationAvgRunningSum, CalendarInterval] { + case class DurationAvgRunningSum( + months: Int, + days: Int, + micros: Long, + count: Long + ) + + object DurationAvg + extends Aggregator[ + CalendarInterval, + DurationAvgRunningSum, + CalendarInterval + ] { override def zero: DurationAvgRunningSum = DurationAvgRunningSum(0, 0, 0, 0) - override def reduce(b: DurationAvgRunningSum, a: CalendarInterval): DurationAvgRunningSum = DurationAvgRunningSum( + override def reduce( + b: DurationAvgRunningSum, + a: CalendarInterval + ): DurationAvgRunningSum = DurationAvgRunningSum( months = b.months + a.months, days = b.days + a.days, micros = b.micros + a.microseconds, count = b.count + 1 ) - override def merge(b1: DurationAvgRunningSum, b2: DurationAvgRunningSum): DurationAvgRunningSum = { + override def merge( + b1: DurationAvgRunningSum, + b2: DurationAvgRunningSum + ): DurationAvgRunningSum = { DurationAvgRunningSum( months = b1.months + b2.months, days = b1.days + b2.days, @@ -93,9 +129,17 @@ object TemporalUdafs extends Logging { } override def finish(reduction: DurationAvgRunningSum): CalendarInterval = - IntervalUtils.divideExact(new CalendarInterval(reduction.months, reduction.days, reduction.micros), reduction.count) + IntervalUtils.divideExact( + new CalendarInterval( + reduction.months, + reduction.days, + reduction.micros + ), + reduction.count + ) - override def bufferEncoder: Encoder[DurationAvgRunningSum] = Encoders.product[DurationAvgRunningSum] + override def bufferEncoder: Encoder[DurationAvgRunningSum] = + Encoders.product[DurationAvgRunningSum] override def outputEncoder: Encoder[CalendarInterval] = intervalEncoder } } diff --git a/morpheus-spark-cypher/src/main/scala/org/opencypher/morpheus/impl/temporal/TemporalUdfs.scala b/morpheus-spark-cypher/src/main/scala/org/opencypher/morpheus/impl/temporal/TemporalUdfs.scala index 26495feae1..d6fe188787 100644 --- a/morpheus-spark-cypher/src/main/scala/org/opencypher/morpheus/impl/temporal/TemporalUdfs.scala +++ b/morpheus-spark-cypher/src/main/scala/org/opencypher/morpheus/impl/temporal/TemporalUdfs.scala @@ -1,29 +1,26 @@ /** - * Copyright (c) 2016-2019 "Neo4j Sweden, AB" [https://neo4j.com] - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * Attribution Notice under the terms of the Apache License 2.0 - * - * This work was created by the collective efforts of the openCypher community. - * Without limiting the terms of Section 6, any Derivative Work that is not - * approved by the public consensus process of the openCypher Implementers Group - * should not be described as “Cypher” (and Cypher® is a registered trademark of - * Neo4j Inc.) or as "openCypher". Extensions by implementers or prototypes or - * proposals for change that have been documented or implemented should only be - * described as "implementation extensions to Cypher" or as "proposed changes to - * Cypher that are not yet approved by the openCypher community". - */ + * Copyright (c) 2016-2019 "Neo4j Sweden, AB" [https://neo4j.com] + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the + * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing permissions and + * limitations under the License. + * + * Attribution Notice under the terms of the Apache License 2.0 + * + * This work was created by the collective efforts of the openCypher community. Without limiting + * the terms of Section 6, any Derivative Work that is not approved by the public consensus process + * of the openCypher Implementers Group should not be described as “Cypher” (and Cypher® is a + * registered trademark of Neo4j Inc.) or as "openCypher". Extensions by implementers or prototypes + * or proposals for change that have been documented or implemented should only be described as + * "implementation extensions to Cypher" or as "proposed changes to Cypher that are not yet + * approved by the openCypher community". + */ package org.opencypher.morpheus.impl.temporal import java.sql.{Date, Timestamp} @@ -39,23 +36,22 @@ import scala.reflect.runtime.universe._ object TemporalUdfs extends Logging { - /** - * Adds a duration to a date. - * Duration components on a sub-day level are ignored - */ + /** Adds a duration to a date. Duration components on a sub-day level are ignored */ val dateAdd: UserDefinedFunction = udf[Date, Date, CalendarInterval]((date: Date, interval: CalendarInterval) => { if (date == null || interval == null) { null } else { - val days = interval.days + interval.microseconds / DateTimeConstants.MICROS_PER_DAY + val days = + interval.days + interval.microseconds / DateTimeConstants.MICROS_PER_DAY if (interval.microseconds % DateTimeConstants.MICROS_PER_DAY != 0) { - logger.warn("Arithmetic with Date and Duration can lead to incorrect results when sub-day values are present.") + logger.warn( + "Arithmetic with Date and Duration can lead to incorrect results when sub-day values are present." + ) } - val reducedLocalDate = date - .toLocalDate + val reducedLocalDate = date.toLocalDate .plusMonths(interval.months) .plusDays(days) @@ -63,23 +59,22 @@ object TemporalUdfs extends Logging { } }) - /** - * Subtracts a duration from a date. - * Duration components on a sub-day level are ignored - */ + /** Subtracts a duration from a date. Duration components on a sub-day level are ignored */ val dateSubtract: UserDefinedFunction = udf[Date, Date, CalendarInterval]((date: Date, interval: CalendarInterval) => { if (date == null || interval == null) { null } else { - val days = interval.days + interval.microseconds / DateTimeConstants.MICROS_PER_DAY + val days = + interval.days + interval.microseconds / DateTimeConstants.MICROS_PER_DAY if (interval.microseconds % DateTimeConstants.MICROS_PER_DAY != 0) { - logger.warn("Arithmetic with Date and Duration can lead to incorrect results when sub-day values are present.") + logger.warn( + "Arithmetic with Date and Duration can lead to incorrect results when sub-day values are present." + ) } - val reducedLocalDate = date - .toLocalDate + val reducedLocalDate = date.toLocalDate .minusMonths(interval.months) .minusDays(days) @@ -87,33 +82,28 @@ object TemporalUdfs extends Logging { } }) - /** - * Returns the week based year of a given temporal type. - */ - def weekYear[I: TypeTag]: UserDefinedFunction = dateAccessor[I](IsoFields.WEEK_BASED_YEAR) - - /** - * Returns the day of the quarter of a given temporal type. - */ - def dayOfQuarter[I: TypeTag]: UserDefinedFunction = dateAccessor[I](IsoFields.DAY_OF_QUARTER) - - /** - * Returns the day of the week of a given temporal type. - */ - def dayOfWeek[I: TypeTag]: UserDefinedFunction = dateAccessor[I](ChronoField.DAY_OF_WEEK) - - /** - * Returns the milliseconds. - */ - def milliseconds[I: TypeTag]: UserDefinedFunction = timeAccessor[I](ChronoField.MILLI_OF_SECOND) - - /** - * Returns the microseconds. - */ - def microseconds[I: TypeTag]: UserDefinedFunction = timeAccessor[I](ChronoField.MICRO_OF_SECOND) - - def durationAccessor(accessor: String): UserDefinedFunction = udf[java.lang.Long, CalendarInterval]( - (duration: CalendarInterval) => { + /** Returns the week based year of a given temporal type. */ + def weekYear[I: TypeTag]: UserDefinedFunction = + dateAccessor[I](IsoFields.WEEK_BASED_YEAR) + + /** Returns the day of the quarter of a given temporal type. */ + def dayOfQuarter[I: TypeTag]: UserDefinedFunction = + dateAccessor[I](IsoFields.DAY_OF_QUARTER) + + /** Returns the day of the week of a given temporal type. */ + def dayOfWeek[I: TypeTag]: UserDefinedFunction = + dateAccessor[I](ChronoField.DAY_OF_WEEK) + + /** Returns the milliseconds. */ + def milliseconds[I: TypeTag]: UserDefinedFunction = + timeAccessor[I](ChronoField.MILLI_OF_SECOND) + + /** Returns the microseconds. */ + def microseconds[I: TypeTag]: UserDefinedFunction = + timeAccessor[I](ChronoField.MICRO_OF_SECOND) + + def durationAccessor(accessor: String): UserDefinedFunction = + udf[java.lang.Long, CalendarInterval]((duration: CalendarInterval) => { if (duration == null) { null } else { @@ -122,41 +112,64 @@ object TemporalUdfs extends Logging { val daysInMicros = days * DateTimeConstants.MICROS_PER_DAY val l: Long = accessor match { - case "years" => duration.months / 12 + case "years" => duration.months / 12 case "quarters" => duration.months / 3 - case "months" => duration.months - case "weeks" => duration.days / DateTimeConstants.DAYS_PER_WEEK + duration.microseconds / DateTimeConstants.MICROS_PER_DAY / 7 - case "days" => duration.days + duration.microseconds / DateTimeConstants.MICROS_PER_DAY - case "hours" => (duration.microseconds - daysInMicros) / DateTimeConstants.MICROS_PER_HOUR - case "minutes" => (duration.microseconds - daysInMicros) / DateTimeConstants.MICROS_PER_MINUTE - case "seconds" => (duration.microseconds - daysInMicros) / DateTimeConstants.MICROS_PER_SECOND - case "milliseconds" => (duration.microseconds - daysInMicros) / DateTimeConstants.MICROS_PER_MILLIS + case "months" => duration.months + case "weeks" => + duration.days / DateTimeConstants.DAYS_PER_WEEK + duration.microseconds / DateTimeConstants.MICROS_PER_DAY / 7 + case "days" => + duration.days + duration.microseconds / DateTimeConstants.MICROS_PER_DAY + case "hours" => + (duration.microseconds - daysInMicros) / DateTimeConstants.MICROS_PER_HOUR + case "minutes" => + (duration.microseconds - daysInMicros) / DateTimeConstants.MICROS_PER_MINUTE + case "seconds" => + (duration.microseconds - daysInMicros) / DateTimeConstants.MICROS_PER_SECOND + case "milliseconds" => + (duration.microseconds - daysInMicros) / DateTimeConstants.MICROS_PER_MILLIS case "microseconds" => duration.microseconds - daysInMicros - case "quartersofyear" => (duration.months / 3) % 4 + case "quartersofyear" => (duration.months / 3) % 4 case "monthsofquarter" => duration.months % 3 - case "monthsofyear" => duration.months % 12 - case "daysofweek" => (duration.microseconds / DateTimeConstants.MICROS_PER_DAY) % 7 - case "minutesofhour" => ((duration.microseconds - daysInMicros) / DateTimeConstants.MICROS_PER_MINUTE) % 60 - case "secondsofminute" => ((duration.microseconds - daysInMicros) / DateTimeConstants.MICROS_PER_SECOND) % 60 - case "millisecondsofsecond" => ((duration.microseconds - daysInMicros) / DateTimeConstants.MICROS_PER_MILLIS) % 1000 - case "microsecondsofsecond" => (duration.microseconds - daysInMicros) % 1000000 - - case other => throw UnsupportedOperationException(s"Unknown Duration accessor: $other") + case "monthsofyear" => duration.months % 12 + case "daysofweek" => + (duration.microseconds / DateTimeConstants.MICROS_PER_DAY) % 7 + case "minutesofhour" => + ((duration.microseconds - daysInMicros) / DateTimeConstants.MICROS_PER_MINUTE) % 60 + case "secondsofminute" => + ((duration.microseconds - daysInMicros) / DateTimeConstants.MICROS_PER_SECOND) % 60 + case "millisecondsofsecond" => + ((duration.microseconds - daysInMicros) / DateTimeConstants.MICROS_PER_MILLIS) % 1000 + case "microsecondsofsecond" => + (duration.microseconds - daysInMicros) % 1000000 + + case other => + throw UnsupportedOperationException( + s"Unknown Duration accessor: $other" + ) } java.lang.Long.valueOf(l) } - } - ) + }) - private def dateAccessor[I: TypeTag](accessor: TemporalField): UserDefinedFunction = udf[Long, I] { - case d: Date => d.toLocalDate.get(accessor) + private def dateAccessor[I: TypeTag]( + accessor: TemporalField + ): UserDefinedFunction = udf[Long, I] { + case d: Date => d.toLocalDate.get(accessor) case l: Timestamp => l.toLocalDateTime.get(accessor) - case other => throw UnsupportedOperationException(s"Date Accessor '$accessor' is not supported for '$other'.") + case other => + throw UnsupportedOperationException( + s"Date Accessor '$accessor' is not supported for '$other'." + ) } - private def timeAccessor[I: TypeTag](accessor: TemporalField): UserDefinedFunction = udf[Long, I] { + private def timeAccessor[I: TypeTag]( + accessor: TemporalField + ): UserDefinedFunction = udf[Long, I] { case l: Timestamp => l.toLocalDateTime.get(accessor) - case other => throw UnsupportedOperationException(s"Time Accessor '$accessor' is not supported for '$other'.") + case other => + throw UnsupportedOperationException( + s"Time Accessor '$accessor' is not supported for '$other'." + ) } } diff --git a/morpheus-spark-cypher/src/main/scala/org/opencypher/morpheus/impl/util/Annotation.scala b/morpheus-spark-cypher/src/main/scala/org/opencypher/morpheus/impl/util/Annotation.scala index 9911ddbb4d..744b61e309 100644 --- a/morpheus-spark-cypher/src/main/scala/org/opencypher/morpheus/impl/util/Annotation.scala +++ b/morpheus-spark-cypher/src/main/scala/org/opencypher/morpheus/impl/util/Annotation.scala @@ -47,14 +47,16 @@ private[morpheus] object Annotation { } } - def get[A <: StaticAnnotation: TypeTag, E: TypeTag]: Option[A] = synchronized { - val maybeAnnotation = staticClass[E].annotations.find(_.tree.tpe =:= typeOf[A]) - maybeAnnotation.map { annotation => - val tb = typeTag[E].mirror.mkToolBox() - val instance = tb.eval(tb.untypecheck(annotation.tree)).asInstanceOf[A] - instance + def get[A <: StaticAnnotation: TypeTag, E: TypeTag]: Option[A] = + synchronized { + val maybeAnnotation = + staticClass[E].annotations.find(_.tree.tpe =:= typeOf[A]) + maybeAnnotation.map { annotation => + val tb = typeTag[E].mirror.mkToolBox() + val instance = tb.eval(tb.untypecheck(annotation.tree)).asInstanceOf[A] + instance + } } - } private def runtimeClass[E: TypeTag]: Class[E] = synchronized { val tag = typeTag[E] diff --git a/morpheus-spark-cypher/src/main/scala/org/opencypher/morpheus/schema/MorpheusSchema.scala b/morpheus-spark-cypher/src/main/scala/org/opencypher/morpheus/schema/MorpheusSchema.scala index 9589b2b7b5..9d1a9e9141 100644 --- a/morpheus-spark-cypher/src/main/scala/org/opencypher/morpheus/schema/MorpheusSchema.scala +++ b/morpheus-spark-cypher/src/main/scala/org/opencypher/morpheus/schema/MorpheusSchema.scala @@ -41,41 +41,51 @@ object MorpheusSchema { implicit class MorpheusSchemaConverter(schema: PropertyGraphSchema) { /** - * Converts a given schema into a Morpheus specific schema. The conversion fails if the schema specifies property types - * that cannot be represented in Morpheus and throws a [[org.opencypher.okapi.impl.exception.SchemaException]]. + * Converts a given schema into a Morpheus specific schema. The conversion fails if the schema + * specifies property types that cannot be represented in Morpheus and throws a + * [[org.opencypher.okapi.impl.exception.SchemaException]]. */ def asMorpheus: MorpheusSchema = { schema match { case s: MorpheusSchema => s case s: PropertyGraphSchema => - val combosByLabel = schema.labels.map(label => label -> s.labelCombinations.combos.filter(_.contains(label))) - - combosByLabel.foreach { - case (_, combos) => - val keysForAllCombosOfLabel = combos.map(combo => combo -> schema.nodePropertyKeys(combo)) - for { - (combo1, keys1) <- keysForAllCombosOfLabel - (combo2, keys2) <- keysForAllCombosOfLabel - } yield { - (keys1.keySet intersect keys2.keySet).foreach { k => - val t1 = keys1(k) - val t2 = keys2(k) - val join = t1.join(t2) - if (!join.isSparkCompatible) { - val explanation = if (combo1 == combo2) { - s"The unsupported type is specified on label combination ${combo1.mkString("[", ", ", "]")}." - } else { - s"The conflict appears between label combinations ${combo1.mkString("[", ", ", "]")} and ${combo2.mkString("[", ", ", "]")}." - } - throw SchemaException(s"The property type '$join' for property '$k' can not be stored in a Spark column. " + explanation) + val combosByLabel = schema.labels.map(label => + label -> s.labelCombinations.combos.filter(_.contains(label)) + ) + + combosByLabel.foreach { case (_, combos) => + val keysForAllCombosOfLabel = + combos.map(combo => combo -> schema.nodePropertyKeys(combo)) + for { + (combo1, keys1) <- keysForAllCombosOfLabel + (combo2, keys2) <- keysForAllCombosOfLabel + } yield { + (keys1.keySet intersect keys2.keySet).foreach { k => + val t1 = keys1(k) + val t2 = keys2(k) + val join = t1.join(t2) + if (!join.isSparkCompatible) { + val explanation = if (combo1 == combo2) { + s"The unsupported type is specified on label combination ${combo1 + .mkString("[", ", ", "]")}." + } else { + s"The conflict appears between label combinations ${combo1 + .mkString("[", ", ", "]")} and ${combo2.mkString("[", ", ", "]")}." } + throw SchemaException( + s"The property type '$join' for property '$k' can not be stored in a Spark column. " + explanation + ) } } + } } new MorpheusSchema(s) - case other => throw UnsupportedOperationException(s"${other.getClass.getSimpleName} does not have Tag support") + case other => + throw UnsupportedOperationException( + s"${other.getClass.getSimpleName} does not have Tag support" + ) } } @@ -83,7 +93,8 @@ object MorpheusSchema { } -case class MorpheusSchema private[schema](schema: PropertyGraphSchema) extends PropertyGraphSchema { +case class MorpheusSchema private[schema] (schema: PropertyGraphSchema) + extends PropertyGraphSchema { override def labels: Set[String] = schema.labels @@ -91,65 +102,117 @@ case class MorpheusSchema private[schema](schema: PropertyGraphSchema) extends P override def relationshipTypes: Set[String] = schema.relationshipTypes - override def relationshipKeys: Map[String, Set[String]] = schema.relationshipKeys + override def relationshipKeys: Map[String, Set[String]] = + schema.relationshipKeys override def labelPropertyMap: LabelPropertyMap = schema.labelPropertyMap - override def relTypePropertyMap: RelTypePropertyMap = schema.relTypePropertyMap + override def relTypePropertyMap: RelTypePropertyMap = + schema.relTypePropertyMap override def schemaPatterns: Set[SchemaPattern] = schema.schemaPatterns - override def withSchemaPatterns(patterns: SchemaPattern*): PropertyGraphSchema = schema.withSchemaPatterns(patterns: _*) + override def withSchemaPatterns( + patterns: SchemaPattern* + ): PropertyGraphSchema = schema.withSchemaPatterns(patterns: _*) override def impliedLabels: ImpliedLabels = schema.impliedLabels override def labelCombinations: LabelCombinations = schema.labelCombinations - override def impliedLabels(knownLabels: Set[String]): Set[String] = schema.impliedLabels(knownLabels) + override def impliedLabels(knownLabels: Set[String]): Set[String] = + schema.impliedLabels(knownLabels) - override def nodePropertyKeys(labels: Set[String]): PropertyKeys = schema.nodePropertyKeys(labels) + override def nodePropertyKeys(labels: Set[String]): PropertyKeys = + schema.nodePropertyKeys(labels) override def allCombinations: Set[Set[String]] = schema.allCombinations - override def combinationsFor(knownLabels: Set[String]): Set[Set[String]] = schema.combinationsFor(knownLabels) + override def combinationsFor(knownLabels: Set[String]): Set[Set[String]] = + schema.combinationsFor(knownLabels) - override def nodePropertyKeyType(labels: Set[String], key: String): Option[CypherType] = schema.nodePropertyKeyType(labels, key) + override def nodePropertyKeyType( + labels: Set[String], + key: String + ): Option[CypherType] = schema.nodePropertyKeyType(labels, key) - override def nodePropertyKeysForCombinations(labelCombinations: Set[Set[String]]): PropertyKeys = schema.nodePropertyKeysForCombinations(labelCombinations) + override def nodePropertyKeysForCombinations( + labelCombinations: Set[Set[String]] + ): PropertyKeys = schema.nodePropertyKeysForCombinations(labelCombinations) - override def relationshipPropertyKeyType(types: Set[String], key: String): Option[CypherType] = schema.relationshipPropertyKeyType(types, key) + override def relationshipPropertyKeyType( + types: Set[String], + key: String + ): Option[CypherType] = schema.relationshipPropertyKeyType(types, key) - override def relationshipPropertyKeys(typ: String): PropertyKeys = schema.relationshipPropertyKeys(typ) + override def relationshipPropertyKeys(typ: String): PropertyKeys = + schema.relationshipPropertyKeys(typ) - override def relationshipPropertyKeysForTypes(knownTypes: Set[String]): PropertyKeys = schema.relationshipPropertyKeysForTypes(knownTypes) + override def relationshipPropertyKeysForTypes( + knownTypes: Set[String] + ): PropertyKeys = schema.relationshipPropertyKeysForTypes(knownTypes) - override def withNodePropertyKeys(nodeLabels: Set[String], keys: PropertyKeys): PropertyGraphSchema = schema.withNodePropertyKeys(nodeLabels, keys) + override def withNodePropertyKeys( + nodeLabels: Set[String], + keys: PropertyKeys + ): PropertyGraphSchema = schema.withNodePropertyKeys(nodeLabels, keys) - override def withRelationshipPropertyKeys(typ: String, keys: PropertyKeys): PropertyGraphSchema = schema.withRelationshipPropertyKeys(typ, keys) + override def withRelationshipPropertyKeys( + typ: String, + keys: PropertyKeys + ): PropertyGraphSchema = schema.withRelationshipPropertyKeys(typ, keys) - override def ++(other: PropertyGraphSchema): PropertyGraphSchema = schema ++ other + override def ++(other: PropertyGraphSchema): PropertyGraphSchema = + schema ++ other override def pretty: String = schema.pretty override def isEmpty: Boolean = schema.isEmpty - override def forNode(labelConstraints: Set[String]): PropertyGraphSchema = schema.forNode(labelConstraints) + override def forNode(labelConstraints: Set[String]): PropertyGraphSchema = + schema.forNode(labelConstraints) - override def forRelationship(relType: CTRelationship): PropertyGraphSchema = schema.forRelationship(relType) + override def forRelationship(relType: CTRelationship): PropertyGraphSchema = + schema.forRelationship(relType) - override def dropPropertiesFor(combo: Set[String]): PropertyGraphSchema = schema.dropPropertiesFor(combo) + override def dropPropertiesFor(combo: Set[String]): PropertyGraphSchema = + schema.dropPropertiesFor(combo) - override def withOverwrittenNodePropertyKeys(nodeLabels: Set[String], propertyKeys: PropertyKeys): PropertyGraphSchema = schema.withOverwrittenNodePropertyKeys(nodeLabels, propertyKeys) + override def withOverwrittenNodePropertyKeys( + nodeLabels: Set[String], + propertyKeys: PropertyKeys + ): PropertyGraphSchema = + schema.withOverwrittenNodePropertyKeys(nodeLabels, propertyKeys) - override def withOverwrittenRelationshipPropertyKeys(relType: String, propertyKeys: PropertyKeys): PropertyGraphSchema = schema.withOverwrittenRelationshipPropertyKeys(relType, propertyKeys) + override def withOverwrittenRelationshipPropertyKeys( + relType: String, + propertyKeys: PropertyKeys + ): PropertyGraphSchema = + schema.withOverwrittenRelationshipPropertyKeys(relType, propertyKeys) override def toJson: String = schema.toJson - override def explicitSchemaPatterns: Set[SchemaPattern] = schema.explicitSchemaPatterns - - override def schemaPatternsFor(knownSourceLabels: Set[String], knownRelTypes: Set[String], knownTargetLabels: Set[String]): Set[SchemaPattern] = schema.schemaPatternsFor(knownSourceLabels, knownRelTypes, knownTargetLabels) - - override def withNodeKey(label: String, nodeKey: Set[String]): PropertyGraphSchema = schema.withNodeKey(label, nodeKey) - - override def withRelationshipKey(relationshipType: String, relationshipKey: Set[String]): PropertyGraphSchema = schema.withRelationshipKey(relationshipType, relationshipKey) + override def explicitSchemaPatterns: Set[SchemaPattern] = + schema.explicitSchemaPatterns + + override def schemaPatternsFor( + knownSourceLabels: Set[String], + knownRelTypes: Set[String], + knownTargetLabels: Set[String] + ): Set[SchemaPattern] = schema.schemaPatternsFor( + knownSourceLabels, + knownRelTypes, + knownTargetLabels + ) + + override def withNodeKey( + label: String, + nodeKey: Set[String] + ): PropertyGraphSchema = schema.withNodeKey(label, nodeKey) + + override def withRelationshipKey( + relationshipType: String, + relationshipKey: Set[String] + ): PropertyGraphSchema = + schema.withRelationshipKey(relationshipType, relationshipKey) } diff --git a/morpheus-tck/src/generator/scala/org/opencypher/morpheus/testing/MorpheusTestGenerator.scala b/morpheus-tck/src/generator/scala/org/opencypher/morpheus/testing/MorpheusTestGenerator.scala index 8b80cb8d58..3af710517b 100644 --- a/morpheus-tck/src/generator/scala/org/opencypher/morpheus/testing/MorpheusTestGenerator.scala +++ b/morpheus-tck/src/generator/scala/org/opencypher/morpheus/testing/MorpheusTestGenerator.scala @@ -32,31 +32,42 @@ import org.opencypher.okapi.impl.exception.IllegalArgumentException import org.opencypher.okapi.tck.test.AcceptanceTestGenerator object MorpheusTestGenerator extends App { - val imports = List("import org.opencypher.morpheus.testing.MorpheusTestSuite", - "import org.opencypher.morpheus.testing.support.creation.graphs.ScanGraphFactory") - val generator = AcceptanceTestGenerator(imports, + val imports = List( + "import org.opencypher.morpheus.testing.MorpheusTestSuite", + "import org.opencypher.morpheus.testing.support.creation.graphs.ScanGraphFactory" + ) + val generator = AcceptanceTestGenerator( + imports, graphFactoryName = "ScanGraphFactory", createGraphMethodName = "initGraph", emptyGraphMethodName = "apply(InMemoryTestGraph.empty)", testSuiteName = "MorpheusTestSuite", targetPackageName = "org.opencypher.morpheus.testing", addGitIgnore = true, - checkSideEffects = false) + checkSideEffects = false + ) if (args.isEmpty) { - val defaultOutDir = new File("morpheus-tck/src/test/scala/org/opencypher/morpheus/testing/") - val defaultResFiles = new File("morpheus-tck/src/test/resources/").listFiles() + val defaultOutDir = new File( + "morpheus-tck/src/test/scala/org/opencypher/morpheus/testing/" + ) + val defaultResFiles = + new File("morpheus-tck/src/test/resources/").listFiles() generator.generateAllScenarios(defaultOutDir, defaultResFiles) - } - else { - //parameter names specified in gradle task - val (outDir, resFiles) = (new File(args(0)), Option(new File(args(1)).listFiles())) + } else { + // parameter names specified in gradle task + val (outDir, resFiles) = + (new File(args(0)), Option(new File(args(1)).listFiles())) resFiles match { case Some(files) => val scenarioNames = args(2) if (scenarioNames.nonEmpty) - generator.generateGivenScenarios(outDir, files, scenarioNames.split('|')) + generator.generateGivenScenarios( + outDir, + files, + scenarioNames.split('|') + ) else generator.generateAllScenarios(outDir, files) case None => throw IllegalArgumentException("resource Dir does not exist") diff --git a/morpheus-tck/src/test/scala/org/opencypher/morpheus/testing/TckSparkCypherTest.scala b/morpheus-tck/src/test/scala/org/opencypher/morpheus/testing/TckSparkCypherTest.scala index f03cba26aa..31439ccc7d 100644 --- a/morpheus-tck/src/test/scala/org/opencypher/morpheus/testing/TckSparkCypherTest.scala +++ b/morpheus-tck/src/test/scala/org/opencypher/morpheus/testing/TckSparkCypherTest.scala @@ -42,23 +42,37 @@ class TckSparkCypherTest extends MorpheusTestSuite { // Defines the graphs to run on private val factories = Table( - ("factory","additional_blacklist"), + ("factory", "additional_blacklist"), (ScanGraphFactory, Set.empty[String]) ) private val defaultFactory: TestGraphFactory = ScanGraphFactory - private val failingBlacklist = getClass.getResource("/failing_blacklist").getFile - private val temporalBlacklist = getClass.getResource("/temporal_blacklist").getFile - private val wontFixBlacklistFile = getClass.getResource("/wont_fix_blacklist").getFile - private val failureReportingBlacklistFile = getClass.getResource("/failure_reporting_blacklist").getFile - private val scenarios = ScenariosFor(failingBlacklist, temporalBlacklist, wontFixBlacklistFile, failureReportingBlacklistFile) + private val failingBlacklist = + getClass.getResource("/failing_blacklist").getFile + private val temporalBlacklist = + getClass.getResource("/temporal_blacklist").getFile + private val wontFixBlacklistFile = + getClass.getResource("/wont_fix_blacklist").getFile + private val failureReportingBlacklistFile = + getClass.getResource("/failure_reporting_blacklist").getFile + private val scenarios = ScenariosFor( + failingBlacklist, + temporalBlacklist, + wontFixBlacklistFile, + failureReportingBlacklistFile + ) // white list tests are run on all factories forAll(factories) { (factory, additional_blacklist) => forAll(scenarios.whiteList) { scenario => if (!additional_blacklist.contains(scenario.toString)) { - test(s"[${factory.name}, ${WhiteList.name}] $scenario", WhiteList, tckMorpheusTag, Tag(factory.name)) { + test( + s"[${factory.name}, ${WhiteList.name}] $scenario", + WhiteList, + tckMorpheusTag, + Tag(factory.name) + ) { scenario(TCKGraph(factory, morpheus.graphs.empty)).execute() } } @@ -67,12 +81,18 @@ class TckSparkCypherTest extends MorpheusTestSuite { // black list tests are run on default factory forAll(scenarios.blackList) { scenario => - test(s"[${defaultFactory.name}, ${BlackList.name}] $scenario", BlackList, tckMorpheusTag) { + test( + s"[${defaultFactory.name}, ${BlackList.name}] $scenario", + BlackList, + tckMorpheusTag + ) { val tckGraph = TCKGraph(defaultFactory, morpheus.graphs.empty) Try(scenario(tckGraph).execute()) match { case Success(_) => - throw new RuntimeException(s"A blacklisted scenario actually worked: $scenario") + throw new RuntimeException( + s"A blacklisted scenario actually worked: $scenario" + ) case Failure(_) => () } @@ -83,31 +103,37 @@ class TckSparkCypherTest extends MorpheusTestSuite { val white = scenarios.whiteList.groupBy(_.featureName).mapValues(_.size) val black = scenarios.blackList.groupBy(_.featureName).mapValues(_.size) - val failingScenarios = using(Source.fromFile(failingBlacklist))(_.getLines().size) - val failingTemporalScenarios = using(Source.fromFile(temporalBlacklist))(_.getLines().size) - val failureReportingScenarios = using(Source.fromFile(failureReportingBlacklistFile))(_.getLines().size) + val failingScenarios = + using(Source.fromFile(failingBlacklist))(_.getLines().size) + val failingTemporalScenarios = + using(Source.fromFile(temporalBlacklist))(_.getLines().size) + val failureReportingScenarios = + using(Source.fromFile(failureReportingBlacklistFile))(_.getLines().size) val allFeatures = white.keySet ++ black.keySet - val perFeatureCoverage = allFeatures.foldLeft(Map.empty[String, Float]) { - case (acc, feature) => - val w = white.getOrElse(feature, 0).toFloat - val b = black.getOrElse(feature, 0).toFloat - val percentage = (w / (w + b)) * 100 - acc.updated(feature, percentage) + val perFeatureCoverage = allFeatures.foldLeft(Map.empty[String, Float]) { case (acc, feature) => + val w = white.getOrElse(feature, 0).toFloat + val b = black.getOrElse(feature, 0).toFloat + val percentage = (w / (w + b)) * 100 + acc.updated(feature, percentage) } - val allScenarios = scenarios.blacklist.size + scenarios.whiteList.size.toFloat - val readOnlyScenarios = scenarios.whiteList.size + failingScenarios + failureReportingScenarios.toFloat + failingTemporalScenarios - val smallReadOnlyScenarios = scenarios.whiteList.size + failingScenarios.toFloat + val allScenarios = + scenarios.blacklist.size + scenarios.whiteList.size.toFloat + val readOnlyScenarios = + scenarios.whiteList.size + failingScenarios + failureReportingScenarios.toFloat + failingTemporalScenarios + val smallReadOnlyScenarios = + scenarios.whiteList.size + failingScenarios.toFloat val overallCoverage = scenarios.whiteList.size / allScenarios val readOnlyCoverage = scenarios.whiteList.size / readOnlyScenarios - val smallReadOnlyCoverage = scenarios.whiteList.size / smallReadOnlyScenarios + val smallReadOnlyCoverage = + scenarios.whiteList.size / smallReadOnlyScenarios val featureCoverageReport = perFeatureCoverage.toSeq .sortBy { case (_, coverage) => coverage } .reverse - .map{ case (feature, coverage) => s" $feature: $coverage%" } + .map { case (feature, coverage) => s" $feature: $coverage%" } .mkString("\n") val report = s""" @@ -129,7 +155,8 @@ class TckSparkCypherTest extends MorpheusTestSuite { } ignore("run single scenario") { - scenarios.get("Should add or subtract duration to or from date") + scenarios + .get("Should add or subtract duration to or from date") .foreach(scenario => scenario(TCKGraph(defaultFactory, morpheus.graphs.empty)).execute()) } } diff --git a/morpheus-testing/src/main/scala/org/opencypher/morpheus/testing/MorpheusTestSuite.scala b/morpheus-testing/src/main/scala/org/opencypher/morpheus/testing/MorpheusTestSuite.scala index 5706819025..a0761e3f9a 100644 --- a/morpheus-testing/src/main/scala/org/opencypher/morpheus/testing/MorpheusTestSuite.scala +++ b/morpheus-testing/src/main/scala/org/opencypher/morpheus/testing/MorpheusTestSuite.scala @@ -41,8 +41,11 @@ abstract class MorpheusTestSuite with GraphMatchingTestSupport with RecordMatchingTestSupport { - def catalog(qgn: QualifiedGraphName): Option[RelationalCypherGraph[DataFrameTable]] = None + def catalog( + qgn: QualifiedGraphName + ): Option[RelationalCypherGraph[DataFrameTable]] = None - implicit val context: RelationalRuntimeContext[DataFrameTable] = RelationalRuntimeContext(catalog) + implicit val context: RelationalRuntimeContext[DataFrameTable] = + RelationalRuntimeContext(catalog) } diff --git a/morpheus-testing/src/main/scala/org/opencypher/morpheus/testing/TestSparkSession.scala b/morpheus-testing/src/main/scala/org/opencypher/morpheus/testing/TestSparkSession.scala index f3c15ee5dd..2004bbe874 100644 --- a/morpheus-testing/src/main/scala/org/opencypher/morpheus/testing/TestSparkSession.scala +++ b/morpheus-testing/src/main/scala/org/opencypher/morpheus/testing/TestSparkSession.scala @@ -27,16 +27,14 @@ /** * Copyright (c) 2016-2017 "Neo4j, Inc." [https://neo4j.com] * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and + * Unless required by applicable law or agreed to in writing, software distributed under the + * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing permissions and * limitations under the License. */ package org.opencypher.morpheus.testing diff --git a/morpheus-testing/src/main/scala/org/opencypher/morpheus/testing/api/io/MorpheusPGDSAcceptanceTest.scala b/morpheus-testing/src/main/scala/org/opencypher/morpheus/testing/api/io/MorpheusPGDSAcceptanceTest.scala index 0760570959..0a53ca37be 100644 --- a/morpheus-testing/src/main/scala/org/opencypher/morpheus/testing/api/io/MorpheusPGDSAcceptanceTest.scala +++ b/morpheus-testing/src/main/scala/org/opencypher/morpheus/testing/api/io/MorpheusPGDSAcceptanceTest.scala @@ -34,14 +34,17 @@ import org.opencypher.morpheus.impl.table.SparkTable.DataFrameTable import org.opencypher.morpheus.testing.MorpheusTestSuite import org.opencypher.morpheus.testing.support.creation.graphs.ScanGraphFactory -trait MorpheusPGDSAcceptanceTest extends PGDSAcceptanceTest[MorpheusSession, ScanGraph[DataFrameTable]] { +trait MorpheusPGDSAcceptanceTest + extends PGDSAcceptanceTest[MorpheusSession, ScanGraph[DataFrameTable]] { self: MorpheusTestSuite => trait MorpheusTestContextFactory extends TestContextFactory { override def initSession: MorpheusSession = morpheus } - override def initGraph(createStatements: String): ScanGraph[DataFrameTable] = { + override def initGraph( + createStatements: String + ): ScanGraph[DataFrameTable] = { ScanGraphFactory(CreateGraphFactory(createStatements)) } diff --git a/morpheus-testing/src/main/scala/org/opencypher/morpheus/testing/api/value/MorpheusTestValues.scala b/morpheus-testing/src/main/scala/org/opencypher/morpheus/testing/api/value/MorpheusTestValues.scala index 2236c8145d..c5a44c99e6 100644 --- a/morpheus-testing/src/main/scala/org/opencypher/morpheus/testing/api/value/MorpheusTestValues.scala +++ b/morpheus-testing/src/main/scala/org/opencypher/morpheus/testing/api/value/MorpheusTestValues.scala @@ -44,17 +44,40 @@ object MorpheusTestValues { implicit lazy val RELATIONSHIP_valueGroups: ValueGroups[CypherValue] = Seq( Seq[CypherValue]( - MorpheusRelationship(1, 1, 1, "KNOWS", CypherMap("a" -> 1, "b" -> CypherNull)), - MorpheusRelationship(1, 2, 4, "FORGETS", CypherMap("a" -> 1.0, "b" -> CypherNull)) + MorpheusRelationship( + 1, + 1, + 1, + "KNOWS", + CypherMap("a" -> 1, "b" -> CypherNull) + ), + MorpheusRelationship( + 1, + 2, + 4, + "FORGETS", + CypherMap("a" -> 1.0, "b" -> CypherNull) + ) ), - Seq[CypherValue](MorpheusRelationship(10, 1, 1, "KNOWS", CypherMap("a" -> 1))), Seq[CypherValue]( - MorpheusRelationship(20, 1, 1, "KNOWS", CypherMap("a" -> 1, "b" -> 1))), + MorpheusRelationship(10, 1, 1, "KNOWS", CypherMap("a" -> 1)) + ), + Seq[CypherValue]( + MorpheusRelationship(20, 1, 1, "KNOWS", CypherMap("a" -> 1, "b" -> 1)) + ), Seq[CypherValue]( - MorpheusRelationship(21, 0, -1, "KNOWS", CypherMap("b" -> CypherNull))), + MorpheusRelationship(21, 0, -1, "KNOWS", CypherMap("b" -> CypherNull)) + ), Seq[CypherValue](MorpheusRelationship(30, 1, 1, "_-&", CypherMap.empty)), Seq[CypherValue]( - MorpheusRelationship(40, 1, 1, "", CypherMap("c" -> 10, "b" -> CypherNull))), + MorpheusRelationship( + 40, + 1, + 1, + "", + CypherMap("c" -> 10, "b" -> CypherNull) + ) + ), Seq[CypherValue](CypherNull) ) @@ -65,11 +88,23 @@ object MorpheusTestValues { ), Seq[CypherValue](MorpheusNode(10L, Set.empty[String], CypherMap("a" -> 1))), Seq[CypherValue]( - MorpheusNode(20L, Set("MathGuy"), CypherMap("a" -> 1, "b" -> 1))), + MorpheusNode(20L, Set("MathGuy"), CypherMap("a" -> 1, "b" -> 1)) + ), Seq[CypherValue]( - MorpheusNode(21L, Set("MathGuy", "FanOfNulls"), CypherMap("b" -> CypherNull))), + MorpheusNode( + 21L, + Set("MathGuy", "FanOfNulls"), + CypherMap("b" -> CypherNull) + ) + ), Seq[CypherValue](MorpheusNode(30L, Set("NoOne"), CypherMap.empty)), - Seq[CypherValue](MorpheusNode(40L, Set.empty[String], CypherMap("c" -> 10, "b" -> CypherNull))), + Seq[CypherValue]( + MorpheusNode( + 40L, + Set.empty[String], + CypherMap("c" -> 10, "b" -> CypherNull) + ) + ), Seq[CypherValue](CypherNull) ) @@ -196,13 +231,15 @@ object MorpheusTestValues { val materials: ValueGroups[CypherValue] = allGroups.flatMap(_.materialValueGroups) val nulls: ValueGroups[CypherValue] = Seq( - allGroups.flatMap(_.CypherNullableValueGroups).flatten) + allGroups.flatMap(_.CypherNullableValueGroups).flatten + ) materials ++ nulls } implicit final class CypherValueGroups[V <: CypherValue]( - elements: ValueGroups[V]) { + elements: ValueGroups[V] + ) { def materialValueGroups: ValueGroups[V] = elements.map(_.filterNot(_.isNull)).filter(_.nonEmpty) diff --git a/morpheus-testing/src/main/scala/org/opencypher/morpheus/testing/fixture/GraphConstructionFixture.scala b/morpheus-testing/src/main/scala/org/opencypher/morpheus/testing/fixture/GraphConstructionFixture.scala index 167f8d66d5..b121bcf917 100644 --- a/morpheus-testing/src/main/scala/org/opencypher/morpheus/testing/fixture/GraphConstructionFixture.scala +++ b/morpheus-testing/src/main/scala/org/opencypher/morpheus/testing/fixture/GraphConstructionFixture.scala @@ -39,6 +39,9 @@ trait GraphConstructionFixture { def graphFactory: TestGraphFactory = ScanGraphFactory - def initGraph(query: String, additionalPatterns: Seq[Pattern] = Seq.empty): RelationalCypherGraph[DataFrameTable] = + def initGraph( + query: String, + additionalPatterns: Seq[Pattern] = Seq.empty + ): RelationalCypherGraph[DataFrameTable] = ScanGraphFactory(CreateGraphFactory(query), additionalPatterns).asMorpheus } diff --git a/morpheus-testing/src/main/scala/org/opencypher/morpheus/testing/fixture/H2Fixture.scala b/morpheus-testing/src/main/scala/org/opencypher/morpheus/testing/fixture/H2Fixture.scala index 2f23446fef..c262f4884e 100644 --- a/morpheus-testing/src/main/scala/org/opencypher/morpheus/testing/fixture/H2Fixture.scala +++ b/morpheus-testing/src/main/scala/org/opencypher/morpheus/testing/fixture/H2Fixture.scala @@ -30,16 +30,19 @@ import org.opencypher.morpheus.api.io.sql.SqlDataSourceConfig import org.opencypher.morpheus.testing.utils.H2Utils._ import org.opencypher.okapi.testing.BaseTestSuite - trait H2Fixture extends SparkSessionFixture { self: BaseTestSuite => def createH2Database(cfg: SqlDataSourceConfig.Jdbc, name: String): Unit = { - withConnection(cfg) { conn => conn.execute(s"CREATE SCHEMA IF NOT EXISTS $name")} + withConnection(cfg) { conn => + conn.execute(s"CREATE SCHEMA IF NOT EXISTS $name") + } } def dropH2Database(cfg: SqlDataSourceConfig.Jdbc, name: String): Unit = { - withConnection(cfg) { conn => conn.execute(s"DROP SCHEMA IF EXISTS $name CASCADE")} + withConnection(cfg) { conn => + conn.execute(s"DROP SCHEMA IF EXISTS $name CASCADE") + } } def freshH2Database(cfg: SqlDataSourceConfig.Jdbc, name: String): Unit = { diff --git a/morpheus-testing/src/main/scala/org/opencypher/morpheus/testing/fixture/MiniDFSClusterFixture.scala b/morpheus-testing/src/main/scala/org/opencypher/morpheus/testing/fixture/MiniDFSClusterFixture.scala index 450be5f1ae..771e759ab5 100644 --- a/morpheus-testing/src/main/scala/org/opencypher/morpheus/testing/fixture/MiniDFSClusterFixture.scala +++ b/morpheus-testing/src/main/scala/org/opencypher/morpheus/testing/fixture/MiniDFSClusterFixture.scala @@ -50,16 +50,20 @@ trait MiniDFSClusterFixture extends BaseTestFixture { protected def fsTestGraphPath: Option[String] = None protected lazy val cluster: MiniDFSCluster = { - val cluster = new MiniDFSCluster.Builder(sparkSession.sparkContext.hadoopConfiguration).build() + val cluster = new MiniDFSCluster.Builder( + sparkSession.sparkContext.hadoopConfiguration + ).build() cluster.waitClusterUp() // copy from local FS to HDFS if necessary if (dfsTestGraphPath.isDefined) { val dfsPathString = dfsTestGraphPath.get - val fsPathString = fsTestGraphPath.getOrElse(getClass.getResource(dfsPathString).toString) + val fsPathString = + fsTestGraphPath.getOrElse(getClass.getResource(dfsPathString).toString) cluster.getFileSystem.copyFromLocalFile( new Path(fsPathString), - new Path(dfsPathString)) + new Path(dfsPathString) + ) } cluster } @@ -72,11 +76,13 @@ trait MiniDFSClusterFixture extends BaseTestFixture { protected def clusterConfig: Configuration = { sparkSession.sparkContext.hadoopConfiguration - .set("fs.default.name", new URIBuilder() - .setScheme(HDFS_URI_SCHEME) - .setHost(cluster.getNameNode.getHostAndPort) - .build() - .toString + .set( + "fs.default.name", + new URIBuilder() + .setScheme(HDFS_URI_SCHEME) + .setHost(cluster.getNameNode.getHostAndPort) + .build() + .toString ) sparkSession.sparkContext.hadoopConfiguration } diff --git a/morpheus-testing/src/main/scala/org/opencypher/morpheus/testing/fixture/MorpheusSessionFixture.scala b/morpheus-testing/src/main/scala/org/opencypher/morpheus/testing/fixture/MorpheusSessionFixture.scala index 4f2c20e1db..7f036da335 100644 --- a/morpheus-testing/src/main/scala/org/opencypher/morpheus/testing/fixture/MorpheusSessionFixture.scala +++ b/morpheus-testing/src/main/scala/org/opencypher/morpheus/testing/fixture/MorpheusSessionFixture.scala @@ -36,7 +36,11 @@ trait MorpheusSessionFixture extends BaseTestFixture { abstract override protected def afterEach(): Unit = { // delete all session graphs via their qualified graph name - morpheus.catalog.source(morpheus.catalog.sessionNamespace).graphNames.map(_.value).foreach(morpheus.catalog.dropGraph) + morpheus.catalog + .source(morpheus.catalog.sessionNamespace) + .graphNames + .map(_.value) + .foreach(morpheus.catalog.dropGraph) morpheus.catalog.store(morpheus.emptyGraphQgn, morpheus.graphs.empty) super.afterEach() } diff --git a/morpheus-testing/src/main/scala/org/opencypher/morpheus/testing/fixture/OpenCypherDataFixture.scala b/morpheus-testing/src/main/scala/org/opencypher/morpheus/testing/fixture/OpenCypherDataFixture.scala index 13448218bc..1040e8a2c7 100644 --- a/morpheus-testing/src/main/scala/org/opencypher/morpheus/testing/fixture/OpenCypherDataFixture.scala +++ b/morpheus-testing/src/main/scala/org/opencypher/morpheus/testing/fixture/OpenCypherDataFixture.scala @@ -27,16 +27,14 @@ /** * Copyright (c) 2016-2017 "Neo4j, Inc." [https://neo4j.com] * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and + * Unless required by applicable law or agreed to in writing, software distributed under the + * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing permissions and * limitations under the License. */ package org.opencypher.morpheus.testing.fixture @@ -119,8 +117,14 @@ trait OpenCypherDataFixture extends TestDataFixture { val nbrRels = 28 val schema: MorpheusSchema = PropertyGraphSchema.empty - .withNodePropertyKeys("Person")("name" -> CTString, "birthyear" -> CTInteger) - .withNodePropertyKeys("Person", "Actor")("name" -> CTString, "birthyear" -> CTInteger) + .withNodePropertyKeys("Person")( + "name" -> CTString, + "birthyear" -> CTInteger + ) + .withNodePropertyKeys("Person", "Actor")( + "name" -> CTString, + "birthyear" -> CTInteger + ) .withNodePropertyKeys("City")("name" -> CTString) .withNodePropertyKeys("Film")("title" -> CTString) // .withNodePropertyKeys("Movie")("title" -> CTInteger) @@ -132,4 +136,3 @@ trait OpenCypherDataFixture extends TestDataFixture { .withRelationshipPropertyKeys("ACTED_IN")("charactername" -> CTString) .asMorpheus } - diff --git a/morpheus-testing/src/main/scala/org/opencypher/morpheus/testing/fixture/RecordsVerificationFixture.scala b/morpheus-testing/src/main/scala/org/opencypher/morpheus/testing/fixture/RecordsVerificationFixture.scala index c966d26b1e..390c434a34 100644 --- a/morpheus-testing/src/main/scala/org/opencypher/morpheus/testing/fixture/RecordsVerificationFixture.scala +++ b/morpheus-testing/src/main/scala/org/opencypher/morpheus/testing/fixture/RecordsVerificationFixture.scala @@ -35,9 +35,13 @@ import org.opencypher.okapi.testing.Bag._ trait RecordsVerificationFixture { - self: MorpheusTestSuite => + self: MorpheusTestSuite => - protected def verify(records: RelationalCypherRecords[DataFrameTable], expectedExprs: Seq[Expr], expectedData: Bag[Row]): Unit = { + protected def verify( + records: RelationalCypherRecords[DataFrameTable], + expectedExprs: Seq[Expr], + expectedData: Bag[Row] + ): Unit = { val df = records.table.df val header = records.header val expectedColumns = expectedExprs.map(header.column) @@ -45,12 +49,16 @@ trait RecordsVerificationFixture { df.columns.toSet should equal(expectedColumns.toSet) // Array equality is based on reference, not structure. Hence, we need to convert to lists. - val actual = df.select(expectedColumns.head, expectedColumns.tail: _*).collect().map { r => - Row(r.toSeq.map { - case c: Array[_] => c.toList - case other => other - }: _*) - }.toBag + val actual = df + .select(expectedColumns.head, expectedColumns.tail: _*) + .collect() + .map { r => + Row(r.toSeq.map { + case c: Array[_] => c.toList + case other => other + }: _*) + } + .toBag actual should equal(expectedData) } diff --git a/morpheus-testing/src/main/scala/org/opencypher/morpheus/testing/fixture/TeamDataFixture.scala b/morpheus-testing/src/main/scala/org/opencypher/morpheus/testing/fixture/TeamDataFixture.scala index 5436eb6da6..f584b6c6d4 100644 --- a/morpheus-testing/src/main/scala/org/opencypher/morpheus/testing/fixture/TeamDataFixture.scala +++ b/morpheus-testing/src/main/scala/org/opencypher/morpheus/testing/fixture/TeamDataFixture.scala @@ -29,7 +29,11 @@ package org.opencypher.morpheus.testing.fixture import org.apache.spark.sql.{DataFrame, Row} import org.opencypher.morpheus.api.io.MorpheusElementTable import org.opencypher.morpheus.api.value.{MorpheusNode, MorpheusRelationship} -import org.opencypher.okapi.api.io.conversion.{ElementMapping, NodeMappingBuilder, RelationshipMappingBuilder} +import org.opencypher.okapi.api.io.conversion.{ + ElementMapping, + NodeMappingBuilder, + RelationshipMappingBuilder +} import org.opencypher.okapi.api.schema.PropertyGraphSchema import org.opencypher.okapi.api.types._ import org.opencypher.okapi.api.value.CypherValue.{CypherList, CypherMap} @@ -50,10 +54,14 @@ trait TeamDataFixture extends TestDataFixture { val nHasLabelPerson: Expr = HasLabel(n, Label("Person")) val nHasLabelProgrammer: Expr = HasLabel(n, Label("Programmer")) val nHasLabelBrogrammer: Expr = HasLabel(n, Label("Brogrammer")) - val nHasPropertyLanguage: Expr = ElementProperty(n, PropertyKey("language"))(CTString) - val nHasPropertyLuckyNumber: Expr = ElementProperty(n, PropertyKey("luckyNumber"))(CTInteger) - val nHasPropertyTitle: Expr = ElementProperty(n, PropertyKey("title"))(CTString) - val nHasPropertyYear: Expr = ElementProperty(n, PropertyKey("year"))(CTInteger) + val nHasPropertyLanguage: Expr = + ElementProperty(n, PropertyKey("language"))(CTString) + val nHasPropertyLuckyNumber: Expr = + ElementProperty(n, PropertyKey("luckyNumber"))(CTInteger) + val nHasPropertyTitle: Expr = + ElementProperty(n, PropertyKey("title"))(CTString) + val nHasPropertyYear: Expr = + ElementProperty(n, PropertyKey("year"))(CTInteger) val nHasPropertyName: Expr = ElementProperty(n, PropertyKey("name"))(CTString) val r: Var = Var("r")(CTRelationship) @@ -62,8 +70,10 @@ trait TeamDataFixture extends TestDataFixture { val rHasTypeKnows: Expr = HasType(r, RelType("KNOWS")) val rHasTypeReads: Expr = HasType(r, RelType("READS")) val rHasTypeInfluences: Expr = HasType(r, RelType("INFLUENCES")) - val rHasPropertyRecommends: Expr = ElementProperty(r, PropertyKey("recommends"))(CTBoolean) - val rHasPropertySince: Expr = ElementProperty(r, PropertyKey("since"))(CTInteger) + val rHasPropertyRecommends: Expr = + ElementProperty(r, PropertyKey("recommends"))(CTBoolean) + val rHasPropertySince: Expr = + ElementProperty(r, PropertyKey("since"))(CTInteger) override def dataFixture = """ @@ -78,9 +88,20 @@ trait TeamDataFixture extends TestDataFixture { """ lazy val dataFixtureSchema: PropertyGraphSchema = PropertyGraphSchema.empty - .withNodePropertyKeys("Person", "German")("name" -> CTString, "luckyNumber" -> CTInteger, "languages" -> CTList(CTString).nullable) - .withNodePropertyKeys("Person", "Swede")("name" -> CTString, "luckyNumber" -> CTInteger) - .withNodePropertyKeys("Person")("name" -> CTString, "luckyNumber" -> CTInteger, "languages" -> CTEmptyList) + .withNodePropertyKeys("Person", "German")( + "name" -> CTString, + "luckyNumber" -> CTInteger, + "languages" -> CTList(CTString).nullable + ) + .withNodePropertyKeys("Person", "Swede")( + "name" -> CTString, + "luckyNumber" -> CTInteger + ) + .withNodePropertyKeys("Person")( + "name" -> CTString, + "luckyNumber" -> CTInteger, + "languages" -> CTEmptyList + ) .withRelationshipPropertyKeys("KNOWS")("since" -> CTInteger) override lazy val nbrNodes = 4 @@ -88,44 +109,128 @@ trait TeamDataFixture extends TestDataFixture { override def nbrRels = 3 lazy val teamDataGraphNodes: Bag[CypherMap] = Bag( - CypherMap("n" -> MorpheusNode(0L, Set("Person", "German"), CypherMap("name" -> "Stefan", "luckyNumber" -> 42L, "languages" -> CypherList("German", "English", "Klingon")))), - CypherMap("n" -> MorpheusNode(1L, Set("Person", "Swede"), CypherMap("name" -> "Mats", "luckyNumber" -> 23L))), - CypherMap("n" -> MorpheusNode(2L, Set("Person", "German"), CypherMap("name" -> "Martin", "luckyNumber" -> 1337L))), - CypherMap("n" -> MorpheusNode(3L, Set("Person", "German"), CypherMap("name" -> "Max", "luckyNumber" -> 8L))), - CypherMap("n" -> MorpheusNode(4L, Set("Person"), CypherMap("name" -> "Donald", "luckyNumber" -> 8L, "languages" -> CypherList()))) + CypherMap( + "n" -> MorpheusNode( + 0L, + Set("Person", "German"), + CypherMap( + "name" -> "Stefan", + "luckyNumber" -> 42L, + "languages" -> CypherList("German", "English", "Klingon") + ) + ) + ), + CypherMap( + "n" -> MorpheusNode( + 1L, + Set("Person", "Swede"), + CypherMap("name" -> "Mats", "luckyNumber" -> 23L) + ) + ), + CypherMap( + "n" -> MorpheusNode( + 2L, + Set("Person", "German"), + CypherMap("name" -> "Martin", "luckyNumber" -> 1337L) + ) + ), + CypherMap( + "n" -> MorpheusNode( + 3L, + Set("Person", "German"), + CypherMap("name" -> "Max", "luckyNumber" -> 8L) + ) + ), + CypherMap( + "n" -> MorpheusNode( + 4L, + Set("Person"), + CypherMap( + "name" -> "Donald", + "luckyNumber" -> 8L, + "languages" -> CypherList() + ) + ) + ) ) lazy val teamDataGraphRels: Bag[CypherMap] = Bag( - CypherMap("r" -> MorpheusRelationship(0, 0, 1, "KNOWS", CypherMap("since" -> 2016))), - CypherMap("r" -> MorpheusRelationship(1, 1, 2, "KNOWS", CypherMap("since" -> 2016))), - CypherMap("r" -> MorpheusRelationship(2, 2, 3, "KNOWS", CypherMap("since" -> 2016))) + CypherMap( + "r" -> MorpheusRelationship(0, 0, 1, "KNOWS", CypherMap("since" -> 2016)) + ), + CypherMap( + "r" -> MorpheusRelationship(1, 1, 2, "KNOWS", CypherMap("since" -> 2016)) + ), + CypherMap( + "r" -> MorpheusRelationship(2, 2, 3, "KNOWS", CypherMap("since" -> 2016)) + ) ) /** * Returns the expected graph tags for the test graph in /resources/csv/sn * - * @return expected graph tags + * @return + * expected graph tags */ lazy val csvTestGraphTags: Set[Int] = Set(0, 1) /** * Returns the expected nodes for the test graph in /resources/csv/sn * - * @return expected nodes + * @return + * expected nodes */ lazy val csvTestGraphNodes: Bag[Row] = Bag( - Row(1L, true, true, true, false, wrap(Array("german", "english")), 42L, "Stefan"), - Row(2L, true, false, true, true, wrap(Array("swedish", "english", "german")), 23L, "Mats"), - Row(3L, true, true, true, false, wrap(Array("german", "english")), 1337L, "Martin"), - Row(4L, true, true, true, false, wrap(Array("german", "swedish", "english")), 8L, "Max") + Row( + 1L, + true, + true, + true, + false, + wrap(Array("german", "english")), + 42L, + "Stefan" + ), + Row( + 2L, + true, + false, + true, + true, + wrap(Array("swedish", "english", "german")), + 23L, + "Mats" + ), + Row( + 3L, + true, + true, + true, + false, + wrap(Array("german", "english")), + 1337L, + "Martin" + ), + Row( + 4L, + true, + true, + true, + false, + wrap(Array("german", "swedish", "english")), + 8L, + "Max" + ) ) // TODO: figure out why the column order is different for the calls in this and the next method /** * Returns the rels for the test graph in /resources/csv/sn as expected by a - * [[org.opencypher.okapi.relational.api.graph.RelationalCypherGraph[DataFrameTable]#relationships]] call. + * [[org.opencypher.okapi.relational.api.graph.RelationalCypherGraph[DataFrameTable]#relationships]] + * call. * - * @return expected rels + * @return + * expected rels */ lazy val csvTestGraphRels: Bag[Row] = Bag( Row(1L, 10L, "KNOWS", 2L, 2016L), @@ -135,9 +240,11 @@ trait TeamDataFixture extends TestDataFixture { /** * Returns the rels for the test graph in /resources/csv/sn as expected by a - * [[[org.opencypher.okapi.relational.api.graph.RelationalCypherGraph[DataFrameTable]#records]] call. + * [[[org.opencypher.okapi.relational.api.graph.RelationalCypherGraph[DataFrameTable]#records]] + * call. * - * @return expected rels + * @return + * expected rels */ lazy val csvTestGraphRelsFromRecords: Bag[Row] = Bag( Row(10L, 1L, "KNOWS", 2L, 2016L), @@ -183,35 +290,43 @@ trait TeamDataFixture extends TestDataFixture { .withPropertyKey("luckyNumber" -> "NUM") .build - protected lazy val personDF: DataFrame = morpheus.sparkSession.createDataFrame( - Seq( - (1L, "Mats", 23L), - (2L, "Martin", 42L), - (3L, "Max", 1337L), - (4L, "Stefan", 9L)) - ).toDF("ID", "NAME", "NUM") + protected lazy val personDF: DataFrame = morpheus.sparkSession + .createDataFrame( + Seq( + (1L, "Mats", 23L), + (2L, "Martin", 42L), + (3L, "Max", 1337L), + (4L, "Stefan", 9L) + ) + ) + .toDF("ID", "NAME", "NUM") - lazy val personTable: MorpheusElementTable = MorpheusElementTable.create(personMapping, personDF) + lazy val personTable: MorpheusElementTable = + MorpheusElementTable.create(personMapping, personDF) protected lazy val knowsMapping: ElementMapping = RelationshipMappingBuilder - .on("ID").from("SRC") + .on("ID") + .from("SRC") .to("DST") .relType("KNOWS") .withPropertyKey("since" -> "SINCE") .build - - protected lazy val knowsDF: DataFrame = morpheus.sparkSession.createDataFrame( - Seq( - (1L, 1L, 2L, 2017L), - (1L, 2L, 3L, 2016L), - (1L, 3L, 4L, 2015L), - (2L, 4L, 3L, 2016L), - (2L, 5L, 4L, 2013L), - (3L, 6L, 4L, 2016L)) - ).toDF("SRC", "ID", "DST", "SINCE") - - lazy val knowsTable: MorpheusElementTable = MorpheusElementTable.create(knowsMapping, knowsDF) + protected lazy val knowsDF: DataFrame = morpheus.sparkSession + .createDataFrame( + Seq( + (1L, 1L, 2L, 2017L), + (1L, 2L, 3L, 2016L), + (1L, 3L, 4L, 2015L), + (2L, 4L, 3L, 2016L), + (2L, 5L, 4L, 2013L), + (3L, 6L, 4L, 2016L) + ) + ) + .toDF("SRC", "ID", "DST", "SINCE") + + lazy val knowsTable: MorpheusElementTable = + MorpheusElementTable.create(knowsMapping, knowsDF) private lazy val programmerMapping: ElementMapping = NodeMappingBuilder .on("ID") @@ -222,15 +337,19 @@ trait TeamDataFixture extends TestDataFixture { .withPropertyKey("language" -> "LANG") .build - private lazy val programmerDF: DataFrame = morpheus.sparkSession.createDataFrame( - Seq( - (100L, "Alice", 42L, "C"), - (200L, "Bob", 23L, "D"), - (300L, "Eve", 84L, "F"), - (400L, "Carl", 49L, "R") - )).toDF("ID", "NAME", "NUM", "LANG") + private lazy val programmerDF: DataFrame = morpheus.sparkSession + .createDataFrame( + Seq( + (100L, "Alice", 42L, "C"), + (200L, "Bob", 23L, "D"), + (300L, "Eve", 84L, "F"), + (400L, "Carl", 49L, "R") + ) + ) + .toDF("ID", "NAME", "NUM", "LANG") - lazy val programmerTable: MorpheusElementTable= MorpheusElementTable.create(programmerMapping, programmerDF) + lazy val programmerTable: MorpheusElementTable = + MorpheusElementTable.create(programmerMapping, programmerDF) private lazy val brogrammerMapping: ElementMapping = NodeMappingBuilder .on("ID") @@ -239,16 +358,20 @@ trait TeamDataFixture extends TestDataFixture { .withPropertyKey("language" -> "LANG") .build - private lazy val brogrammerDF = morpheus.sparkSession.createDataFrame( - Seq( - (100L, "Node"), - (200L, "Coffeescript"), - (300L, "Javascript"), - (400L, "Typescript") - )).toDF("ID", "LANG") + private lazy val brogrammerDF = morpheus.sparkSession + .createDataFrame( + Seq( + (100L, "Node"), + (200L, "Coffeescript"), + (300L, "Javascript"), + (400L, "Typescript") + ) + ) + .toDF("ID", "LANG") // required to test conflicting input data - lazy val brogrammerTable: MorpheusElementTable = MorpheusElementTable.create(brogrammerMapping, brogrammerDF) + lazy val brogrammerTable: MorpheusElementTable = + MorpheusElementTable.create(brogrammerMapping, brogrammerDF) private lazy val bookMapping: ElementMapping = NodeMappingBuilder .on("ID") @@ -257,34 +380,54 @@ trait TeamDataFixture extends TestDataFixture { .withPropertyKey("year" -> "YEAR") .build - private lazy val bookDF: DataFrame = morpheus.sparkSession.createDataFrame( - Seq( - (10L, "1984", 1949L), - (20L, "Cryptonomicon", 1999L), - (30L, "The Eye of the World", 1990L), - (40L, "The Circle", 2013L) - )).toDF("ID", "NAME", "YEAR") + private lazy val bookDF: DataFrame = morpheus.sparkSession + .createDataFrame( + Seq( + (10L, "1984", 1949L), + (20L, "Cryptonomicon", 1999L), + (30L, "The Eye of the World", 1990L), + (40L, "The Circle", 2013L) + ) + ) + .toDF("ID", "NAME", "YEAR") - lazy val bookTable: MorpheusElementTable = MorpheusElementTable.create(bookMapping, bookDF) + lazy val bookTable: MorpheusElementTable = + MorpheusElementTable.create(bookMapping, bookDF) private lazy val readsMapping: ElementMapping = RelationshipMappingBuilder - .on("ID").from("SRC").to("DST").relType("READS").withPropertyKey("recommends" -> "RECOMMENDS").build - - private lazy val readsDF = morpheus.sparkSession.createDataFrame( - Seq( - (100L, 100L, 10L, true), - (200L, 200L, 40L, true), - (300L, 300L, 30L, true), - (400L, 400L, 20L, false) - )).toDF("SRC", "ID", "DST", "RECOMMENDS") - - lazy val readsTable: MorpheusElementTable = MorpheusElementTable.create(readsMapping, readsDF) - - private lazy val influencesMapping: ElementMapping = RelationshipMappingBuilder - .on("ID").from("SRC").to("DST").relType("INFLUENCES").build - - private lazy val influencesDF: DataFrame = morpheus.sparkSession.createDataFrame( - Seq((10L, 1000L, 20L))).toDF("SRC", "ID", "DST") + .on("ID") + .from("SRC") + .to("DST") + .relType("READS") + .withPropertyKey("recommends" -> "RECOMMENDS") + .build - lazy val influencesTable: MorpheusElementTable = MorpheusElementTable.create(influencesMapping, influencesDF) + private lazy val readsDF = morpheus.sparkSession + .createDataFrame( + Seq( + (100L, 100L, 10L, true), + (200L, 200L, 40L, true), + (300L, 300L, 30L, true), + (400L, 400L, 20L, false) + ) + ) + .toDF("SRC", "ID", "DST", "RECOMMENDS") + + lazy val readsTable: MorpheusElementTable = + MorpheusElementTable.create(readsMapping, readsDF) + + private lazy val influencesMapping: ElementMapping = + RelationshipMappingBuilder + .on("ID") + .from("SRC") + .to("DST") + .relType("INFLUENCES") + .build + + private lazy val influencesDF: DataFrame = morpheus.sparkSession + .createDataFrame(Seq((10L, 1000L, 20L))) + .toDF("SRC", "ID", "DST") + + lazy val influencesTable: MorpheusElementTable = + MorpheusElementTable.create(influencesMapping, influencesDF) } diff --git a/morpheus-testing/src/main/scala/org/opencypher/morpheus/testing/support/ElementTableCreationSupport.scala b/morpheus-testing/src/main/scala/org/opencypher/morpheus/testing/support/ElementTableCreationSupport.scala index 4047b2590e..8a3b9b6178 100644 --- a/morpheus-testing/src/main/scala/org/opencypher/morpheus/testing/support/ElementTableCreationSupport.scala +++ b/morpheus-testing/src/main/scala/org/opencypher/morpheus/testing/support/ElementTableCreationSupport.scala @@ -1,29 +1,26 @@ /** - * Copyright (c) 2016-2019 "Neo4j Sweden, AB" [https://neo4j.com] - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * Attribution Notice under the terms of the Apache License 2.0 - * - * This work was created by the collective efforts of the openCypher community. - * Without limiting the terms of Section 6, any Derivative Work that is not - * approved by the public consensus process of the openCypher Implementers Group - * should not be described as “Cypher” (and Cypher® is a registered trademark of - * Neo4j Inc.) or as "openCypher". Extensions by implementers or prototypes or - * proposals for change that have been documented or implemented should only be - * described as "implementation extensions to Cypher" or as "proposed changes to - * Cypher that are not yet approved by the openCypher community". - */ + * Copyright (c) 2016-2019 "Neo4j Sweden, AB" [https://neo4j.com] + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the + * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing permissions and + * limitations under the License. + * + * Attribution Notice under the terms of the Apache License 2.0 + * + * This work was created by the collective efforts of the openCypher community. Without limiting + * the terms of Section 6, any Derivative Work that is not approved by the public consensus process + * of the openCypher Implementers Group should not be described as “Cypher” (and Cypher® is a + * registered trademark of Neo4j Inc.) or as "openCypher". Extensions by implementers or prototypes + * or proposals for change that have been documented or implemented should only be described as + * "implementation extensions to Cypher" or as "proposed changes to Cypher that are not yet + * approved by the openCypher community". + */ package org.opencypher.morpheus.testing.support import org.apache.spark.sql.DataFrame @@ -35,33 +32,39 @@ import org.opencypher.okapi.impl.util.StringEncodingUtilities._ trait ElementTableCreationSupport { /** - * This helper creates an ElementTable based on column name conventions. - * For every pattern element with name NAME, the following column names are recognized: + * This helper creates an ElementTable based on column name conventions. For every pattern + * element with name NAME, the following column names are recognized: * - * - NAME_id / NAME_source / NAME_target -> recognized as id, source or target columns, eg. *node_source* - * - NAME_PROPERTY_property -> as a property column with property name PROPERTY, eg. *node_name_property* + * - NAME_id / NAME_source / NAME_target -> recognized as id, source or target columns, eg. + * *node_source* + * - NAME_PROPERTY_property -> as a property column with property name PROPERTY, eg. + * *node_name_property* * * Implicit types are retrieved from the pattern elements cypher types */ - def constructElementTable(pattern: Pattern, df: DataFrame): MorpheusElementTable = { + def constructElementTable( + pattern: Pattern, + df: DataFrame + ): MorpheusElementTable = { val mapping = pattern.elements.foldLeft(ElementMapping.empty(pattern)) { case (acc, patternElement) => - - val patternElementColumns = df - .columns + val patternElementColumns = df.columns .filter(_.startsWith(s"${patternElement.name}_")) val idMapping: Map[IdKey, String] = patternElementColumns.collect { - case id if id.endsWith("_id") => SourceIdKey -> id + case id if id.endsWith("_id") => SourceIdKey -> id case src if src.endsWith("_source") => SourceStartNodeKey -> src case tgt if tgt.endsWith("_target") => SourceEndNodeKey -> tgt }.toMap - val propertyMapping: Map[String, String] = patternElementColumns.collect { - case prop if prop.endsWith("_property") => - val encodedKey = prop.replaceFirst(s"${patternElement.name}_", "").replaceFirst("_property", "") - encodedKey.decodeSpecialCharacters -> prop - }.toMap + val propertyMapping: Map[String, String] = + patternElementColumns.collect { + case prop if prop.endsWith("_property") => + val encodedKey = prop + .replaceFirst(s"${patternElement.name}_", "") + .replaceFirst("_property", "") + encodedKey.decodeSpecialCharacters -> prop + }.toMap acc.copy( properties = acc.properties.updated(patternElement, propertyMapping), diff --git a/morpheus-testing/src/main/scala/org/opencypher/morpheus/testing/support/GraphMatchingTestSupport.scala b/morpheus-testing/src/main/scala/org/opencypher/morpheus/testing/support/GraphMatchingTestSupport.scala index 2fd871fcb1..e42a01855f 100644 --- a/morpheus-testing/src/main/scala/org/opencypher/morpheus/testing/support/GraphMatchingTestSupport.scala +++ b/morpheus-testing/src/main/scala/org/opencypher/morpheus/testing/support/GraphMatchingTestSupport.scala @@ -39,16 +39,28 @@ trait GraphMatchingTestSupport { self: BaseTestSuite with SparkSessionFixture with MorpheusSessionFixture => - private def getElementIds(records: RelationalCypherRecords[DataFrameTable]): Set[List[Byte]] = { + private def getElementIds( + records: RelationalCypherRecords[DataFrameTable] + ): Set[List[Byte]] = { val elementVar = records.header.vars.toSeq match { case Seq(v) => v - case other => throw new UnsupportedOperationException(s"Expected records with 1 element, got $other") + case other => + throw new UnsupportedOperationException( + s"Expected records with 1 element, got $other" + ) } - records.table.df.select(records.header.column(elementVar)).collect().map(_.getAs[Array[Byte]](0).toList).toSet + records.table.df + .select(records.header.column(elementVar)) + .collect() + .map(_.getAs[Array[Byte]](0).toList) + .toSet } - private def verify(actual: RelationalCypherGraph[DataFrameTable], expected: RelationalCypherGraph[DataFrameTable]): Assertion = { + private def verify( + actual: RelationalCypherGraph[DataFrameTable], + expected: RelationalCypherGraph[DataFrameTable] + ): Assertion = { val expectedNodeIds = getElementIds(expected.nodes("n")) val expectedRelIds = getElementIds(expected.relationships("r")) @@ -59,19 +71,25 @@ trait GraphMatchingTestSupport { expectedRelIds should equal(actualRelIds) } - implicit class GraphsMatcher(graphs: Map[String, RelationalCypherGraph[DataFrameTable]]) { - def shouldMatch(expectedGraphs: RelationalCypherGraph[DataFrameTable]*): Unit = { + implicit class GraphsMatcher( + graphs: Map[String, RelationalCypherGraph[DataFrameTable]] + ) { + def shouldMatch( + expectedGraphs: RelationalCypherGraph[DataFrameTable]* + ): Unit = { withClue("expected and actual must have same size") { graphs.size should equal(expectedGraphs.size) } - graphs.values.zip(expectedGraphs).foreach { - case (actual, expected) => verify(actual, expected) + graphs.values.zip(expectedGraphs).foreach { case (actual, expected) => + verify(actual, expected) } } } implicit class GraphMatcher(graph: RelationalCypherGraph[DataFrameTable]) { - def shouldMatch(expectedGraph: RelationalCypherGraph[DataFrameTable]): Unit = verify(graph, expectedGraph) + def shouldMatch( + expectedGraph: RelationalCypherGraph[DataFrameTable] + ): Unit = verify(graph, expectedGraph) } } diff --git a/morpheus-testing/src/main/scala/org/opencypher/morpheus/testing/support/RecordMatchingTestSupport.scala b/morpheus-testing/src/main/scala/org/opencypher/morpheus/testing/support/RecordMatchingTestSupport.scala index 45a9bb44ef..c566e94cfd 100644 --- a/morpheus-testing/src/main/scala/org/opencypher/morpheus/testing/support/RecordMatchingTestSupport.scala +++ b/morpheus-testing/src/main/scala/org/opencypher/morpheus/testing/support/RecordMatchingTestSupport.scala @@ -50,15 +50,18 @@ trait RecordMatchingTestSupport { implicit class RichRow(r: Row) { - def getCypherValue(expr: Expr, header: RecordHeader) - (implicit context: RelationalRuntimeContext[DataFrameTable]): CypherValue = { + def getCypherValue(expr: Expr, header: RecordHeader)(implicit + context: RelationalRuntimeContext[DataFrameTable] + ): CypherValue = { expr match { case Param(name) => context.parameters(name) case _ => header.getColumn(expr) match { - case None => throw IllegalArgumentException( - expected = s"column for $expr", - actual = header.pretty) + case None => + throw IllegalArgumentException( + expected = s"column for $expr", + actual = header.pretty + ) case Some(column) => CypherValue(r.get(r.schema.fieldIndex(column))) } } diff --git a/morpheus-testing/src/main/scala/org/opencypher/morpheus/testing/support/creation/graphs/GraphFactoryTest.scala b/morpheus-testing/src/main/scala/org/opencypher/morpheus/testing/support/creation/graphs/GraphFactoryTest.scala index 282bf5d6a7..f90d00a6ac 100644 --- a/morpheus-testing/src/main/scala/org/opencypher/morpheus/testing/support/creation/graphs/GraphFactoryTest.scala +++ b/morpheus-testing/src/main/scala/org/opencypher/morpheus/testing/support/creation/graphs/GraphFactoryTest.scala @@ -47,7 +47,10 @@ import org.opencypher.okapi.ir.impl.util.VarConverters._ import org.opencypher.okapi.testing.Bag import org.opencypher.okapi.testing.propertygraph.CreateGraphFactory -abstract class GraphFactoryTest extends MorpheusTestSuite with GraphMatchingTestSupport with RecordsVerificationFixture { +abstract class GraphFactoryTest + extends MorpheusTestSuite + with GraphMatchingTestSupport + with RecordsVerificationFixture { def factory: TestGraphFactory val createQuery: String = @@ -63,74 +66,98 @@ abstract class GraphFactoryTest extends MorpheusTestSuite with GraphMatchingTest |CREATE (martin)-[:SPEAKS]->(orbital) """.stripMargin - val personAstronautTable: MorpheusElementTable = MorpheusElementTable.create(NodeMappingBuilder - .on("ID") - .withImpliedLabels("Person", "Astronaut") - .withPropertyKey("name" -> "NAME") - .withPropertyKey("birthday" -> "BIRTHDAY") - .build, morpheus.sparkSession.createDataFrame( - Seq((0L, "Max", Date.valueOf("1991-07-10")))).toDF("ID", "NAME", "BIRTHDAY")) - - val personMartianTable: MorpheusElementTable = MorpheusElementTable.create(NodeMappingBuilder - .on("ID") - .withImpliedLabels("Person", "Martian") - .withPropertyKey("name" -> "NAME") - .build, morpheus.sparkSession.createDataFrame( - Seq((1L, "Martin"))).toDF("ID", "NAME")) - - val languageTable: MorpheusElementTable = MorpheusElementTable.create(NodeMappingBuilder - .on("ID") - .withImpliedLabel("Language") - .withPropertyKey("title" -> "TITLE") - .build, morpheus.sparkSession.createDataFrame( - Seq( - (2L, "Swedish"), - (3L, "German"), - (4L, "Orbital")) - ).toDF("ID", "TITLE")) - - val knowsScan: MorpheusElementTable = MorpheusElementTable.create(RelationshipMappingBuilder - .on("ID") - .from("SRC").to("DST").relType("KNOWS").build, morpheus.sparkSession.createDataFrame( - Seq( - (0L, 5L, 2L), - (0L, 6L, 3L), - (1L, 7L, 3L), - (1L, 8L, 4L)) - ).toDF("SRC", "ID", "DST")) + val personAstronautTable: MorpheusElementTable = MorpheusElementTable.create( + NodeMappingBuilder + .on("ID") + .withImpliedLabels("Person", "Astronaut") + .withPropertyKey("name" -> "NAME") + .withPropertyKey("birthday" -> "BIRTHDAY") + .build, + morpheus.sparkSession + .createDataFrame(Seq((0L, "Max", Date.valueOf("1991-07-10")))) + .toDF("ID", "NAME", "BIRTHDAY") + ) + + val personMartianTable: MorpheusElementTable = MorpheusElementTable.create( + NodeMappingBuilder + .on("ID") + .withImpliedLabels("Person", "Martian") + .withPropertyKey("name" -> "NAME") + .build, + morpheus.sparkSession + .createDataFrame(Seq((1L, "Martin"))) + .toDF("ID", "NAME") + ) + + val languageTable: MorpheusElementTable = MorpheusElementTable.create( + NodeMappingBuilder + .on("ID") + .withImpliedLabel("Language") + .withPropertyKey("title" -> "TITLE") + .build, + morpheus.sparkSession + .createDataFrame( + Seq((2L, "Swedish"), (3L, "German"), (4L, "Orbital")) + ) + .toDF("ID", "TITLE") + ) + + val knowsScan: MorpheusElementTable = MorpheusElementTable.create( + RelationshipMappingBuilder + .on("ID") + .from("SRC") + .to("DST") + .relType("KNOWS") + .build, + morpheus.sparkSession + .createDataFrame( + Seq((0L, 5L, 2L), (0L, 6L, 3L), (1L, 7L, 3L), (1L, 8L, 4L)) + ) + .toDF("SRC", "ID", "DST") + ) test("testSchema") { val propertyGraph = CreateGraphFactory(createQuery) - factory(propertyGraph).schema should equal(PropertyGraphSchema.empty - .withNodePropertyKeys("Person", "Astronaut")("name" -> CTString, "birthday" -> CTDate) - .withNodePropertyKeys("Person", "Martian")("name" -> CTString) - .withNodePropertyKeys("Language")("title" -> CTString) - .withRelationshipType("SPEAKS") - .asMorpheus) + factory(propertyGraph).schema should equal( + PropertyGraphSchema.empty + .withNodePropertyKeys("Person", "Astronaut")( + "name" -> CTString, + "birthday" -> CTDate + ) + .withNodePropertyKeys("Person", "Martian")("name" -> CTString) + .withNodePropertyKeys("Language")("title" -> CTString) + .withRelationshipType("SPEAKS") + .asMorpheus + ) } test("testAsScanGraph") { val propertyGraph = CreateGraphFactory(createQuery) val g = factory(propertyGraph).asMorpheus - g shouldMatch morpheus.graphs.create(personAstronautTable, personMartianTable, languageTable, knowsScan) + g shouldMatch morpheus.graphs.create( + personAstronautTable, + personMartianTable, + languageTable, + knowsScan + ) } it("can create graphs containing list properties") { - val propertyGraph = CreateGraphFactory( - """ + val propertyGraph = CreateGraphFactory(""" |CREATE ( {l: [1,2,3]} ) """.stripMargin) val g = factory(propertyGraph).asMorpheus - g.cypher("MATCH (n) RETURN n.l as list").records.toMaps should equal(Bag( - CypherMap("list" -> List(1,2,3)) - )) + g.cypher("MATCH (n) RETURN n.l as list").records.toMaps should equal( + Bag( + CypherMap("list" -> List(1, 2, 3)) + ) + ) } it("can handle nodes with the same label but different properties") { - val propertyGraph = CreateGraphFactory( - """ + val propertyGraph = CreateGraphFactory(""" |CREATE ( { } ) |CREATE ( {val1: 1} ) |CREATE ( {val1: 1, val2: "foo"} ) @@ -138,19 +165,27 @@ abstract class GraphFactoryTest extends MorpheusTestSuite with GraphMatchingTest val g = factory(propertyGraph).asMorpheus - g.cypher("MATCH (n) RETURN n.val1, n.val2").records.toMaps should equal(Bag( - CypherMap("n.val1" -> 1, "n.val2" -> "foo"), - CypherMap("n.val1" -> 1, "n.val2" -> null), - CypherMap("n.val1" -> null, "n.val2" -> null) - )) + g.cypher("MATCH (n) RETURN n.val1, n.val2").records.toMaps should equal( + Bag( + CypherMap("n.val1" -> 1, "n.val2" -> "foo"), + CypherMap("n.val1" -> 1, "n.val2" -> null), + CypherMap("n.val1" -> null, "n.val2" -> null) + ) + ) } - it("extracts additional patterns"){ - val nodeRelPattern = NodeRelPattern(CTNode("Person", "Martian"), CTRelationship("SPEAKS")) - val tripletPattern = TripletPattern(CTNode("Person", "Martian"), CTRelationship("SPEAKS"), CTNode("Language")) + it("extracts additional patterns") { + val nodeRelPattern = + NodeRelPattern(CTNode("Person", "Martian"), CTRelationship("SPEAKS")) + val tripletPattern = TripletPattern( + CTNode("Person", "Martian"), + CTRelationship("SPEAKS"), + CTNode("Language") + ) val propertyGraph = CreateGraphFactory(createQuery) - val g = factory(propertyGraph, Seq(nodeRelPattern, tripletPattern)).asMorpheus + val g = + factory(propertyGraph, Seq(nodeRelPattern, tripletPattern)).asMorpheus g.patterns should contain(nodeRelPattern) g.patterns should contain(tripletPattern) @@ -171,8 +206,26 @@ abstract class GraphFactoryTest extends MorpheusTestSuite with GraphMatchingTest ) val data = Bag( - Row(1L.encodeAsMorpheusId.toList, true, true, "Martin", 7L.encodeAsMorpheusId.toList, true, 1L.encodeAsMorpheusId.toList, 3L.encodeAsMorpheusId.toList), - Row(1L.encodeAsMorpheusId.toList, true, true, "Martin", 8L.encodeAsMorpheusId.toList, true, 1L.encodeAsMorpheusId.toList, 4L.encodeAsMorpheusId.toList) + Row( + 1L.encodeAsMorpheusId.toList, + true, + true, + "Martin", + 7L.encodeAsMorpheusId.toList, + true, + 1L.encodeAsMorpheusId.toList, + 3L.encodeAsMorpheusId.toList + ), + Row( + 1L.encodeAsMorpheusId.toList, + true, + true, + "Martin", + 8L.encodeAsMorpheusId.toList, + true, + 1L.encodeAsMorpheusId.toList, + 4L.encodeAsMorpheusId.toList + ) ) val scan = g.scanOperator(nodeRelPattern) @@ -200,9 +253,32 @@ abstract class GraphFactoryTest extends MorpheusTestSuite with GraphMatchingTest ) val data = Bag( - - Row(1L.encodeAsMorpheusId.toList, true, true, "Martin", 8L.encodeAsMorpheusId.toList, true, 1L.encodeAsMorpheusId.toList, 4L.encodeAsMorpheusId.toList, 4L.encodeAsMorpheusId.toList, true, "Orbital"), - Row(1L.encodeAsMorpheusId.toList, true, true, "Martin", 7L.encodeAsMorpheusId.toList, true, 1L.encodeAsMorpheusId.toList, 3L.encodeAsMorpheusId.toList, 3L.encodeAsMorpheusId.toList, true, "German") + Row( + 1L.encodeAsMorpheusId.toList, + true, + true, + "Martin", + 8L.encodeAsMorpheusId.toList, + true, + 1L.encodeAsMorpheusId.toList, + 4L.encodeAsMorpheusId.toList, + 4L.encodeAsMorpheusId.toList, + true, + "Orbital" + ), + Row( + 1L.encodeAsMorpheusId.toList, + true, + true, + "Martin", + 7L.encodeAsMorpheusId.toList, + true, + 1L.encodeAsMorpheusId.toList, + 3L.encodeAsMorpheusId.toList, + 3L.encodeAsMorpheusId.toList, + true, + "German" + ) ) val scan = g.scanOperator(tripletPattern) diff --git a/morpheus-testing/src/main/scala/org/opencypher/morpheus/testing/support/creation/graphs/ScanGraphFactory.scala b/morpheus-testing/src/main/scala/org/opencypher/morpheus/testing/support/creation/graphs/ScanGraphFactory.scala index c67d5ce936..806fb0005c 100644 --- a/morpheus-testing/src/main/scala/org/opencypher/morpheus/testing/support/creation/graphs/ScanGraphFactory.scala +++ b/morpheus-testing/src/main/scala/org/opencypher/morpheus/testing/support/creation/graphs/ScanGraphFactory.scala @@ -46,18 +46,25 @@ import org.opencypher.okapi.impl.exception.{IllegalArgumentException, IllegalSta import org.opencypher.okapi.impl.temporal.Duration import org.opencypher.okapi.impl.util.StringEncodingUtilities._ import org.opencypher.okapi.relational.impl.graph.ScanGraph -import org.opencypher.okapi.testing.propertygraph.{InMemoryTestGraph, InMemoryTestNode, InMemoryTestRelationship} +import org.opencypher.okapi.testing.propertygraph.{ + InMemoryTestGraph, + InMemoryTestNode, + InMemoryTestRelationship +} import scala.collection.JavaConverters._ object ScanGraphFactory extends TestGraphFactory with ElementTableCreationSupport { - override def apply(propertyGraph: InMemoryTestGraph, additionalPatterns: Seq[Pattern]) - (implicit morpheus: MorpheusSession): ScanGraph[DataFrameTable] = { + override def apply( + propertyGraph: InMemoryTestGraph, + additionalPatterns: Seq[Pattern] + )(implicit morpheus: MorpheusSession): ScanGraph[DataFrameTable] = { val schema = computeSchema(propertyGraph).asMorpheus - val nodePatterns = schema.labelCombinations.combos.map(labels => NodePattern(CTNode(labels))) + val nodePatterns = + schema.labelCombinations.combos.map(labels => NodePattern(CTNode(labels))) val relPatterns = schema.relationshipTypes.map(typ => RelationshipPattern(CTRelationship(typ))) val scans = (nodePatterns ++ relPatterns ++ additionalPatterns).map { pattern => @@ -70,8 +77,13 @@ object ScanGraphFactory extends TestGraphFactory with ElementTableCreationSuppor override def name: String = "ScanGraphFactory" - private def extractEmbeddings(pattern: Pattern, graph: InMemoryTestGraph, schema: PropertyGraphSchema) - (implicit morpheus: MorpheusSession): Seq[Map[PatternElement, Element[Long]]] = { + private def extractEmbeddings( + pattern: Pattern, + graph: InMemoryTestGraph, + schema: PropertyGraphSchema + )(implicit + morpheus: MorpheusSession + ): Seq[Map[PatternElement, Element[Long]]] = { val candidates = pattern.elements.map { element => element.cypherType match { @@ -79,41 +91,52 @@ object ScanGraphFactory extends TestGraphFactory with ElementTableCreationSuppor element -> graph.nodes.filter(_.labels == labels) case CTRelationship(types, _) => element -> graph.relationships.filter(rel => types.contains(rel.relType)) - case other => throw IllegalArgumentException("Node or Relationship type", other) + case other => + throw IllegalArgumentException("Node or Relationship type", other) } }.toMap val unitEmbedding = Seq( Map.empty[PatternElement, Element[Long]] ) - val initialEmbeddings = pattern.elements.foldLeft(unitEmbedding) { - case (acc, patternElement) => - val elementCandidates = candidates(patternElement) - - for { - row <- acc - elementCandidate <- elementCandidates - } yield row.updated(patternElement, elementCandidate) + val initialEmbeddings = pattern.elements.foldLeft(unitEmbedding) { case (acc, patternElement) => + val elementCandidates = candidates(patternElement) + + for { + row <- acc + elementCandidate <- elementCandidates + } yield row.updated(patternElement, elementCandidate) } - pattern.topology.foldLeft(initialEmbeddings) { - case (acc, (relElement, connection)) => - connection match { - case Connection(Some(sourceNode), None, _) => acc.filter { row => - row(sourceNode).id == row(relElement).asInstanceOf[InMemoryTestRelationship].startId + pattern.topology.foldLeft(initialEmbeddings) { case (acc, (relElement, connection)) => + connection match { + case Connection(Some(sourceNode), None, _) => + acc.filter { row => + row(sourceNode).id == row(relElement) + .asInstanceOf[InMemoryTestRelationship] + .startId } - case Connection(None, Some(targetElement), _) => acc.filter { row => - row(targetElement).id == row(relElement).asInstanceOf[InMemoryTestRelationship].endId + case Connection(None, Some(targetElement), _) => + acc.filter { row => + row(targetElement).id == row(relElement) + .asInstanceOf[InMemoryTestRelationship] + .endId } - case Connection(Some(sourceNode), Some(targetElement), _) => acc.filter { row => + case Connection(Some(sourceNode), Some(targetElement), _) => + acc.filter { row => val rel = row(relElement).asInstanceOf[InMemoryTestRelationship] - row(sourceNode).id == rel.startId && row(targetElement).id == rel.endId + row(sourceNode).id == rel.startId && row( + targetElement + ).id == rel.endId } - case Connection(None, None, _) => throw IllegalStateException("Connection without source or target node") - } + case Connection(None, None, _) => + throw IllegalStateException( + "Connection without source or target node" + ) + } } } @@ -123,53 +146,75 @@ object ScanGraphFactory extends TestGraphFactory with ElementTableCreationSuppor schema: PropertyGraphSchema )(implicit morpheus: MorpheusSession): MorpheusElementTable = { - val unitData: Seq[Seq[Any]] = Seq(embeddings.indices.map(_ => Seq.empty[Any]): _*) - - val (columns, data) = pattern.elements.foldLeft(Seq.empty[StructField] -> unitData) { - case ((accColumns, accData), element) => - - element.cypherType match { - case CTNode(labels, _) => - val propertyKeys = schema.nodePropertyKeys(labels) - val propertyFields = getPropertyStructFields(element, propertyKeys) - - val nodeData = embeddings.map { embedding => - val node = embedding(element).asInstanceOf[InMemoryTestNode] - - val propertyValues = propertyKeys.keySet.toSeq.map(p => node.properties.get(p).map(toSparkValue).orNull) - Seq(node.id) ++ propertyValues - } - - val newData = accData.zip(nodeData).map { case (l, r) => l ++ r } - val newColumns = accColumns ++ Seq(StructField(s"${element.name.encodeSpecialCharacters}_id", LongType)) ++ propertyFields - - newColumns -> newData - - - case CTRelationship(types, _) => - val propertyKeys = schema.relationshipPropertyKeys(types.head) - val propertyFields = getPropertyStructFields(element, propertyKeys) - - val relData = embeddings.map { embedding => - val rel = embedding(element).asInstanceOf[InMemoryTestRelationship] - val propertyValues = propertyKeys.keySet.toSeq.map(p => rel.properties.get(p).map(toSparkValue).orNull) - Seq(rel.id, rel.startId, rel.endId) ++ propertyValues - } - - val newData = accData.zip(relData).map { case (l, r) => l ++ r } - val newColumns = accColumns ++ - Seq( - StructField(s"${element.name.encodeSpecialCharacters}_id", LongType), - StructField(s"${element.name.encodeSpecialCharacters}_source", LongType), - StructField(s"${element.name.encodeSpecialCharacters}_target", LongType) - ) ++ - propertyFields - - newColumns -> newData + val unitData: Seq[Seq[Any]] = Seq( + embeddings.indices.map(_ => Seq.empty[Any]): _* + ) - case other => throw IllegalArgumentException("Node or Relationship type", other) - } - } + val (columns, data) = + pattern.elements.foldLeft(Seq.empty[StructField] -> unitData) { + case ((accColumns, accData), element) => + element.cypherType match { + case CTNode(labels, _) => + val propertyKeys = schema.nodePropertyKeys(labels) + val propertyFields = + getPropertyStructFields(element, propertyKeys) + + val nodeData = embeddings.map { embedding => + val node = embedding(element).asInstanceOf[InMemoryTestNode] + + val propertyValues = propertyKeys.keySet.toSeq.map(p => + node.properties.get(p).map(toSparkValue).orNull + ) + Seq(node.id) ++ propertyValues + } + + val newData = accData.zip(nodeData).map { case (l, r) => l ++ r } + val newColumns = accColumns ++ Seq( + StructField( + s"${element.name.encodeSpecialCharacters}_id", + LongType + ) + ) ++ propertyFields + + newColumns -> newData + + case CTRelationship(types, _) => + val propertyKeys = schema.relationshipPropertyKeys(types.head) + val propertyFields = + getPropertyStructFields(element, propertyKeys) + + val relData = embeddings.map { embedding => + val rel = + embedding(element).asInstanceOf[InMemoryTestRelationship] + val propertyValues = + propertyKeys.keySet.toSeq.map(p => rel.properties.get(p).map(toSparkValue).orNull) + Seq(rel.id, rel.startId, rel.endId) ++ propertyValues + } + + val newData = accData.zip(relData).map { case (l, r) => l ++ r } + val newColumns = accColumns ++ + Seq( + StructField( + s"${element.name.encodeSpecialCharacters}_id", + LongType + ), + StructField( + s"${element.name.encodeSpecialCharacters}_source", + LongType + ), + StructField( + s"${element.name.encodeSpecialCharacters}_target", + LongType + ) + ) ++ + propertyFields + + newColumns -> newData + + case other => + throw IllegalArgumentException("Node or Relationship type", other) + } + } val df = morpheus.sparkSession.createDataFrame( data.map { r => Row(r: _*) }.asJava, @@ -179,20 +224,29 @@ object ScanGraphFactory extends TestGraphFactory with ElementTableCreationSuppor constructElementTable(pattern, df) } - protected def getPropertyStructFields(patternElement: PatternElement, propKeys: PropertyKeys): Seq[StructField] = { + protected def getPropertyStructFields( + patternElement: PatternElement, + propKeys: PropertyKeys + ): Seq[StructField] = { propKeys.foldLeft(Seq.empty[StructField]) { case (fields, key) => - fields :+ StructField(s"${patternElement.name}_${key._1.encodeSpecialCharacters}_property", key._2.getSparkType, key._2.isNullable) + fields :+ StructField( + s"${patternElement.name}_${key._1.encodeSpecialCharacters}_property", + key._2.getSparkType, + key._2.isNullable + ) } } private def toSparkValue(v: CypherValue): Any = { v.getValue match { case Some(date: LocalDate) => java.sql.Date.valueOf(date) - case Some(localDateTime: LocalDateTime) => java.sql.Timestamp.valueOf(localDateTime) + case Some(localDateTime: LocalDateTime) => + java.sql.Timestamp.valueOf(localDateTime) case Some(dur: Duration) => dur.toCalendarInterval - case Some(l: List[_]) => l.collect { case c: CypherValue => toSparkValue(c) } + case Some(l: List[_]) => + l.collect { case c: CypherValue => toSparkValue(c) } case Some(other) => other - case None => null + case None => null } } } diff --git a/morpheus-testing/src/main/scala/org/opencypher/morpheus/testing/support/creation/graphs/TestGraphFactory.scala b/morpheus-testing/src/main/scala/org/opencypher/morpheus/testing/support/creation/graphs/TestGraphFactory.scala index e267de4761..4cbe80bdcb 100644 --- a/morpheus-testing/src/main/scala/org/opencypher/morpheus/testing/support/creation/graphs/TestGraphFactory.scala +++ b/morpheus-testing/src/main/scala/org/opencypher/morpheus/testing/support/creation/graphs/TestGraphFactory.scala @@ -34,8 +34,12 @@ import org.opencypher.morpheus.impl.MorpheusConverters._ import org.opencypher.morpheus.impl.table.SparkTable.DataFrameTable trait TestGraphFactory extends CypherTestGraphFactory[MorpheusSession] { - def initGraph(createQuery: String, additionalPatterns: Seq[Pattern] = Seq.empty) - (implicit morpheus: MorpheusSession): RelationalCypherGraph[DataFrameTable] = { + def initGraph( + createQuery: String, + additionalPatterns: Seq[Pattern] = Seq.empty + )(implicit + morpheus: MorpheusSession + ): RelationalCypherGraph[DataFrameTable] = { apply(CreateGraphFactory(createQuery), additionalPatterns).asMorpheus } } diff --git a/morpheus-testing/src/main/scala/org/opencypher/morpheus/testing/utils/FileSystemUtils.scala b/morpheus-testing/src/main/scala/org/opencypher/morpheus/testing/utils/FileSystemUtils.scala index fe62bdd743..0efc53c294 100644 --- a/morpheus-testing/src/main/scala/org/opencypher/morpheus/testing/utils/FileSystemUtils.scala +++ b/morpheus-testing/src/main/scala/org/opencypher/morpheus/testing/utils/FileSystemUtils.scala @@ -35,16 +35,21 @@ import scala.util.Properties object FileSystemUtils { - def deleteDirectory(path: Path): Unit = Files.walk(path).iterator().asScala.toList + def deleteDirectory(path: Path): Unit = Files + .walk(path) + .iterator() + .asScala + .toList .reverse .map(_.toFile) .foreach(_.delete()) - def readFile(fileName: String): String = Source.fromFile(fileName) + def readFile(fileName: String): String = Source + .fromFile(fileName) .getLines() .mkString(Properties.lineSeparator) - def writeFile(fileName: String, content: String): Unit = + def writeFile(fileName: String, content: String): Unit = Files.write(Paths.get(fileName), content.getBytes(StandardCharsets.UTF_8)) } diff --git a/morpheus-testing/src/main/scala/org/opencypher/morpheus/testing/utils/H2Utils.scala b/morpheus-testing/src/main/scala/org/opencypher/morpheus/testing/utils/H2Utils.scala index 108a44c22a..286ea638c5 100644 --- a/morpheus-testing/src/main/scala/org/opencypher/morpheus/testing/utils/H2Utils.scala +++ b/morpheus-testing/src/main/scala/org/opencypher/morpheus/testing/utils/H2Utils.scala @@ -36,14 +36,17 @@ object H2Utils { implicit class ConnOps(conn: Connection) { def run[T](code: Statement => T): T = { val stmt = conn.createStatement() - try { code(stmt) } finally { stmt.close() } + try { code(stmt) } + finally { stmt.close() } } def execute(sql: String): Boolean = conn.run(_.execute(sql)) def query(sql: String): ResultSet = conn.run(_.executeQuery(sql)) def update(sql: String): Int = conn.run(_.executeUpdate(sql)) } - def withConnection[T](cfg: SqlDataSourceConfig.Jdbc)(code: Connection => T): T = { + def withConnection[T]( + cfg: SqlDataSourceConfig.Jdbc + )(code: Connection => T): T = { Class.forName(cfg.driver) val conn = (cfg.options.get("user"), cfg.options.get("password")) match { case (Some(user), Some(pass)) => diff --git a/morpheus-testing/src/test/scala/org/opencypher/morpheus/SparkTests.scala b/morpheus-testing/src/test/scala/org/opencypher/morpheus/SparkTests.scala index 1a929a4fef..06e323ea70 100644 --- a/morpheus-testing/src/test/scala/org/opencypher/morpheus/SparkTests.scala +++ b/morpheus-testing/src/test/scala/org/opencypher/morpheus/SparkTests.scala @@ -32,13 +32,16 @@ class SparkTests extends MorpheusTestSuite { // Reproduces https://issues.apache.org/jira/browse/SPARK-23855, which was relevant to Spark 2.2 it("should correctly perform a join after a cross") { - val df1 = sparkSession.createDataFrame(Seq(Tuple1(0L))) + val df1 = sparkSession + .createDataFrame(Seq(Tuple1(0L))) .toDF("a") - val df2 = sparkSession.createDataFrame(Seq(Tuple1(1L))) + val df2 = sparkSession + .createDataFrame(Seq(Tuple1(1L))) .toDF("b") - val df3 = sparkSession.createDataFrame(Seq(Tuple1(0L))) + val df3 = sparkSession + .createDataFrame(Seq(Tuple1(0L))) .toDF("c") val cross = df1.crossJoin(df2) diff --git a/morpheus-testing/src/test/scala/org/opencypher/morpheus/api/io/CachedDataSourceTest.scala b/morpheus-testing/src/test/scala/org/opencypher/morpheus/api/io/CachedDataSourceTest.scala index 2455a704ef..a44f02b2ab 100644 --- a/morpheus-testing/src/test/scala/org/opencypher/morpheus/api/io/CachedDataSourceTest.scala +++ b/morpheus-testing/src/test/scala/org/opencypher/morpheus/api/io/CachedDataSourceTest.scala @@ -37,7 +37,10 @@ import org.opencypher.okapi.relational.api.graph.RelationalCypherGraph import org.opencypher.okapi.relational.impl.graph.ScanGraph import org.scalatest.BeforeAndAfterEach -class CachedDataSourceTest extends MorpheusTestSuite with GraphConstructionFixture with BeforeAndAfterEach { +class CachedDataSourceTest + extends MorpheusTestSuite + with GraphConstructionFixture + with BeforeAndAfterEach { override val testNamespace: Namespace = morpheus.catalog.sessionNamespace private val testDataSource = morpheus.catalog.source(testNamespace) @@ -88,7 +91,8 @@ class CachedDataSourceTest extends MorpheusTestSuite with GraphConstructionFixtu } private def assert(g: PropertyGraph, storageLevel: StorageLevel): Unit = { - g.asInstanceOf[ScanGraph[DataFrameTable]].scans + g.asInstanceOf[ScanGraph[DataFrameTable]] + .scans .map(_.table.df) .foreach(_.storageLevel should equal(storageLevel)) } diff --git a/morpheus-testing/src/test/scala/org/opencypher/morpheus/api/io/FullPGDSAcceptanceTest.scala b/morpheus-testing/src/test/scala/org/opencypher/morpheus/api/io/FullPGDSAcceptanceTest.scala index ff6eeee873..f555449ca6 100644 --- a/morpheus-testing/src/test/scala/org/opencypher/morpheus/api/io/FullPGDSAcceptanceTest.scala +++ b/morpheus-testing/src/test/scala/org/opencypher/morpheus/api/io/FullPGDSAcceptanceTest.scala @@ -1,28 +1,25 @@ /** * Copyright (c) 2016-2019 "Neo4j Sweden, AB" [https://neo4j.com] * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and + * Unless required by applicable law or agreed to in writing, software distributed under the + * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing permissions and * limitations under the License. * * Attribution Notice under the terms of the Apache License 2.0 * - * This work was created by the collective efforts of the openCypher community. - * Without limiting the terms of Section 6, any Derivative Work that is not - * approved by the public consensus process of the openCypher Implementers Group - * should not be described as “Cypher” (and Cypher® is a registered trademark of - * Neo4j Inc.) or as "openCypher". Extensions by implementers or prototypes or - * proposals for change that have been documented or implemented should only be - * described as "implementation extensions to Cypher" or as "proposed changes to - * Cypher that are not yet approved by the openCypher community". + * This work was created by the collective efforts of the openCypher community. Without limiting + * the terms of Section 6, any Derivative Work that is not approved by the public consensus process + * of the openCypher Implementers Group should not be described as “Cypher” (and Cypher® is a + * registered trademark of Neo4j Inc.) or as "openCypher". Extensions by implementers or prototypes + * or proposals for change that have been documented or implemented should only be described as + * "implementation extensions to Cypher" or as "proposed changes to Cypher that are not yet + * approved by the openCypher community". */ package org.opencypher.morpheus.api.io @@ -54,8 +51,12 @@ import org.opencypher.okapi.impl.exception.IllegalArgumentException import org.opencypher.okapi.impl.io.SessionGraphDataSource import org.opencypher.okapi.impl.util.StringEncodingUtilities._ -class FullPGDSAcceptanceTest extends MorpheusTestSuite - with MorpheusPGDSAcceptanceTest with MiniDFSClusterFixture with H2Fixture with HiveFixture { +class FullPGDSAcceptanceTest + extends MorpheusTestSuite + with MorpheusPGDSAcceptanceTest + with MiniDFSClusterFixture + with H2Fixture + with HiveFixture { // === Run scenarios with all factories @@ -63,16 +64,20 @@ class FullPGDSAcceptanceTest extends MorpheusTestSuite private val sqlWhitelist = allScenarios .filterNot(_.name == "API: Correct schema for graph #1") - .filterNot(_.name == "API: PropertyGraphDataSource: correct node/rel count for graph #2") + .filterNot( + _.name == "API: PropertyGraphDataSource: correct node/rel count for graph #2" + ) allSqlContextFactories.foreach(executeScenariosWithContext(sqlWhitelist, _)) - allFileSystemContextFactories.foreach(executeScenariosWithContext(allScenarios, _)) + allFileSystemContextFactories.foreach( + executeScenariosWithContext(allScenarios, _) + ) // === Generate context factories for Neo4j, Session, FileSystem, and SQL property graph data sources lazy val fileFormatOptions = List(csv, parquet, orc) - lazy val filesPerTableOptions = List(1) //, 10 + lazy val filesPerTableOptions = List(1) // , 10 lazy val idGenerationOptions = List(SerializedId, HashedId) lazy val allFileSystemContextFactories: List[TestContextFactory] = { @@ -93,9 +98,11 @@ class FullPGDSAcceptanceTest extends MorpheusTestSuite } yield SQLWithLocalFSContextFactory(format, filesPerTable, idGeneration) } - lazy val sqlHiveContextFactories: List[TestContextFactory] = idGenerationOptions.map(SQLWithHiveContextFactory) + lazy val sqlHiveContextFactories: List[TestContextFactory] = + idGenerationOptions.map(SQLWithHiveContextFactory) - lazy val sqlH2ContextFactories: List[TestContextFactory] = idGenerationOptions.map(SQLWithH2ContextFactory) + lazy val sqlH2ContextFactories: List[TestContextFactory] = + idGenerationOptions.map(SQLWithH2ContextFactory) lazy val allSqlContextFactories: List[TestContextFactory] = { sqlFileSystemContextFactories ++ sqlHiveContextFactories ++ sqlH2ContextFactories @@ -107,7 +114,9 @@ class FullPGDSAcceptanceTest extends MorpheusTestSuite override def toString: String = s"SESSION-PGDS" - override def initPgds(graphNames: List[GraphName]): PropertyGraphDataSource = { + override def initPgds( + graphNames: List[GraphName] + ): PropertyGraphDataSource = { val pgds = new SessionGraphDataSource graphNames.foreach(gn => pgds.store(gn, graph(gn))) pgds @@ -118,7 +127,8 @@ class FullPGDSAcceptanceTest extends MorpheusTestSuite idGenerationStrategy: IdGenerationStrategy ) extends SQLContextFactory { - override def toString: String = s"SQL-PGDS-H2-${idGenerationStrategy.toString}" + override def toString: String = + s"SQL-PGDS-H2-${idGenerationStrategy.toString}" override def initializeContext(graphNames: List[GraphName]): TestContext = { createH2Database(sqlDataSourceConfig, databaseName) @@ -150,7 +160,8 @@ class FullPGDSAcceptanceTest extends MorpheusTestSuite idGenerationStrategy: IdGenerationStrategy ) extends SQLContextFactory { - override def toString: String = s"SQL-PGDS-HIVE-${idGenerationStrategy.toString}" + override def toString: String = + s"SQL-PGDS-HIVE-${idGenerationStrategy.toString}" override def initializeContext(graphNames: List[GraphName]): TestContext = { createHiveDatabase(databaseName) @@ -174,17 +185,23 @@ class FullPGDSAcceptanceTest extends MorpheusTestSuite override val fileFormat: FileFormat, override val filesPerTable: Int, idGenerationStrategy: IdGenerationStrategy - ) extends LocalFileSystemContextFactory(fileFormat, filesPerTable) with SQLContextFactory { + ) extends LocalFileSystemContextFactory(fileFormat, filesPerTable) + with SQLContextFactory { - override def toString: String = s"SQL-PGDS-${fileFormat.name.toUpperCase}-FORMAT-$filesPerTable-FILE(S)-PER-TABLE-${idGenerationStrategy.toString}" + override def toString: String = + s"SQL-PGDS-${fileFormat.name.toUpperCase}-FORMAT-$filesPerTable-FILE(S)-PER-TABLE-${idGenerationStrategy.toString}" override def writeTable(df: DataFrame, tableName: String): Unit = { val path = basePath + s"/${tableName.replace(s"$databaseName.", "")}" val encodedDf = fileFormat match { case FileFormat.csv => df.encodeBinaryToHexString - case _ => df + case _ => df } - encodedDf.write.mode(SaveMode.Overwrite).option("header", "true").format(fileFormat.name).save(path) + encodedDf.write + .mode(SaveMode.Overwrite) + .option("header", "true") + .format(fileFormat.name) + .save(path) } override def sqlDataSourceConfig: SqlDataSourceConfig = { @@ -196,10 +213,26 @@ class FullPGDSAcceptanceTest extends MorpheusTestSuite private val graphTypes = Map( g1 -> GraphType.empty - .withElementType("A", "name" -> CTString, "type" -> CTString.nullable, "size" -> CTInteger.nullable, "date" -> CTDate.nullable) - .withElementType("B", "name" -> CTString.nullable, "type" -> CTString, "size" -> CTInteger.nullable, "datetime" -> CTLocalDateTime.nullable) + .withElementType( + "A", + "name" -> CTString, + "type" -> CTString.nullable, + "size" -> CTInteger.nullable, + "date" -> CTDate.nullable + ) + .withElementType( + "B", + "name" -> CTString.nullable, + "type" -> CTString, + "size" -> CTInteger.nullable, + "datetime" -> CTLocalDateTime.nullable + ) .withElementType("C", "name" -> CTString) - .withElementType("R", "since" -> CTInteger, "before" -> CTBoolean.nullable) + .withElementType( + "R", + "since" -> CTInteger, + "before" -> CTBoolean.nullable + ) .withElementType("S", "since" -> CTInteger) .withElementType("T") .withNodeType("A") @@ -213,7 +246,8 @@ class FullPGDSAcceptanceTest extends MorpheusTestSuite .withRelationshipType(Set("A", "C"), Set("T"), Set("A", "B")), g3 -> GraphType.empty.withElementType("A").withNodeType("A"), g4 -> GraphType.empty.withElementType("A").withNodeType("A"), - g5 -> GraphType.empty.withElementType("USER", "name" -> CTString) + g5 -> GraphType.empty + .withElementType("USER", "name" -> CTString) .withElementType("BUSINESS") .withElementType("REVIEWS", "rating" -> CTFloat.nullable) .withNodeType("USER") @@ -231,58 +265,98 @@ class FullPGDSAcceptanceTest extends MorpheusTestSuite protected val databaseName = "SQLPGDS" - override def releasePgds(implicit ctx: TestContext): Unit = () // SQL PGDS does not support graph deletion + override def releasePgds(implicit ctx: TestContext): Unit = + () // SQL PGDS does not support graph deletion - override def initPgds(graphNames: List[GraphName]): SqlPropertyGraphDataSource = { + override def initPgds( + graphNames: List[GraphName] + ): SqlPropertyGraphDataSource = { val ddls = graphNames.map { gn => val g = graph(gn) val okapiSchema = g.schema - val graphType = graphTypes.getOrElse(gn, throw IllegalArgumentException(s"GraphType for $gn")) - val ddl = g.defaultDdl(gn, graphType, Some(dataSourceName), Some(databaseName)) - - ddl.graphs(gn).nodeToViewMappings.foreach { case (key: NodeViewKey, mapping: NodeToViewMapping) => - val nodeDf = g.canonicalNodeTable(key.nodeType.labels).removePrefix(propertyPrefix) - val allKeys = graphType.nodePropertyKeys(key.nodeType) - val missingPropertyKeys = allKeys.keySet -- okapiSchema.nodePropertyKeys(key.nodeType.labels).keySet - val addColumns = missingPropertyKeys.map(key => key -> functions.lit(null).cast(allKeys(key).getSparkType)) - val alignedNodeDf = nodeDf.safeAddColumns(addColumns.toSeq: _*) - writeTable(alignedNodeDf, mapping.view.tableName) + val graphType = graphTypes.getOrElse( + gn, + throw IllegalArgumentException(s"GraphType for $gn") + ) + val ddl = + g.defaultDdl(gn, graphType, Some(dataSourceName), Some(databaseName)) + + ddl.graphs(gn).nodeToViewMappings.foreach { + case (key: NodeViewKey, mapping: NodeToViewMapping) => + val nodeDf = g + .canonicalNodeTable(key.nodeType.labels) + .removePrefix(propertyPrefix) + val allKeys = graphType.nodePropertyKeys(key.nodeType) + val missingPropertyKeys = allKeys.keySet -- okapiSchema + .nodePropertyKeys(key.nodeType.labels) + .keySet + val addColumns = missingPropertyKeys + .map(key => key -> functions.lit(null).cast(allKeys(key).getSparkType)) + val alignedNodeDf = nodeDf.safeAddColumns(addColumns.toSeq: _*) + writeTable(alignedNodeDf, mapping.view.tableName) } ddl.graphs(gn).edgeToViewMappings.foreach { edgeToViewMapping => - val startNodeDf = g.canonicalNodeTable(edgeToViewMapping.relType.startNodeType.labels) - val endNodeDf = g.canonicalNodeTable(edgeToViewMapping.relType.endNodeType.labels) + val startNodeDf = + g.canonicalNodeTable(edgeToViewMapping.relType.startNodeType.labels) + val endNodeDf = + g.canonicalNodeTable(edgeToViewMapping.relType.endNodeType.labels) val relType = edgeToViewMapping.relType val relationshipType = relType.labels.toList match { case rType :: Nil => rType - case other => throw IllegalArgumentException(expected = "Single relationship type", actual = s"${other.mkString(",")}") + case other => + throw IllegalArgumentException( + expected = "Single relationship type", + actual = s"${other.mkString(",")}" + ) } - val allRelsDf = g.canonicalRelationshipTable(relationshipType).removePrefix(propertyPrefix) + val allRelsDf = g + .canonicalRelationshipTable(relationshipType) + .removePrefix(propertyPrefix) val relDfColumns = allRelsDf.columns.toSeq val tmpNodeId = s"node_${GraphElement.sourceIdKey}" - val tmpStartNodeDf = startNodeDf.withColumnRenamed(GraphElement.sourceIdKey, tmpNodeId) - val tmpEndNodeDf = endNodeDf.withColumnRenamed(GraphElement.sourceIdKey, tmpNodeId) + val tmpStartNodeDf = + startNodeDf.withColumnRenamed(GraphElement.sourceIdKey, tmpNodeId) + val tmpEndNodeDf = + endNodeDf.withColumnRenamed(GraphElement.sourceIdKey, tmpNodeId) val startNodesWithRelsDf = tmpStartNodeDf - .join(allRelsDf, tmpStartNodeDf.col(tmpNodeId) === allRelsDf.col(Relationship.sourceStartNodeKey)) + .join( + allRelsDf, + tmpStartNodeDf.col(tmpNodeId) === allRelsDf + .col(Relationship.sourceStartNodeKey) + ) .select(relDfColumns.head, relDfColumns.tail: _*) val relsDf = startNodesWithRelsDf - .join(tmpEndNodeDf, startNodesWithRelsDf.col(Relationship.sourceEndNodeKey) === tmpEndNodeDf.col(tmpNodeId)) + .join( + tmpEndNodeDf, + startNodesWithRelsDf.col( + Relationship.sourceEndNodeKey + ) === tmpEndNodeDf.col(tmpNodeId) + ) .select(relDfColumns.head, relDfColumns.tail: _*) val allKeys = graphType.relationshipPropertyKeys(relType) - val missingPropertyKeys = allKeys.keySet -- okapiSchema.relationshipPropertyKeys(relType.labels.head).keySet - val addColumns = missingPropertyKeys.map(key => key -> functions.lit(null).cast(allKeys(key).getSparkType)) + val missingPropertyKeys = allKeys.keySet -- okapiSchema + .relationshipPropertyKeys(relType.labels.head) + .keySet + val addColumns = missingPropertyKeys + .map(key => key -> functions.lit(null).cast(allKeys(key).getSparkType)) val alignedRelsDf = relsDf.safeAddColumns(addColumns.toSeq: _*) writeTable(alignedRelsDf, edgeToViewMapping.view.tableName) } ddl } - val ddl = ddls.foldLeft(graphddl.GraphDdl(Map.empty[GraphName, Graph]))(_ ++ _) - SqlPropertyGraphDataSource(ddl, Map(dataSourceName -> sqlDataSourceConfig), idGenerationStrategy) + val ddl = + ddls.foldLeft(graphddl.GraphDdl(Map.empty[GraphName, Graph]))(_ ++ _) + SqlPropertyGraphDataSource( + ddl, + Map(dataSourceName -> sqlDataSourceConfig), + idGenerationStrategy + ) } } @@ -291,7 +365,8 @@ class FullPGDSAcceptanceTest extends MorpheusTestSuite val filesPerTable: Int ) extends FileSystemContextFactory { - override def toString: String = s"HDFS-PGDS-${fileFormat.name.toUpperCase}-FORMAT-$filesPerTable-FILE(S)-PER-TABLE" + override def toString: String = + s"HDFS-PGDS-${fileFormat.name.toUpperCase}-FORMAT-$filesPerTable-FILE(S)-PER-TABLE" override def initializeContext(graphNames: List[GraphName]): TestContext = { super.initializeContext(graphNames) @@ -315,13 +390,15 @@ class FullPGDSAcceptanceTest extends MorpheusTestSuite val filesPerTable: Int ) extends FileSystemContextFactory { - override def toString: String = s"LocalFS-PGDS-${fileFormat.name.toUpperCase}-FORMAT-$filesPerTable-FILE(S)-PER-TABLE" + override def toString: String = + s"LocalFS-PGDS-${fileFormat.name.toUpperCase}-FORMAT-$filesPerTable-FILE(S)-PER-TABLE" protected var tempDir: java.nio.file.Path = _ def basePath: String = s"file://${tempDir.toAbsolutePath}" - def graphSourceFactory: FSGraphSourceFactory = GraphSources.fs(basePath, filesPerTable = Some(filesPerTable)) + def graphSourceFactory: FSGraphSourceFactory = + GraphSources.fs(basePath, filesPerTable = Some(filesPerTable)) override def initializeContext(graphNames: List[GraphName]): TestContext = { tempDir = Files.createTempDirectory(getClass.getSimpleName) @@ -340,12 +417,15 @@ class FullPGDSAcceptanceTest extends MorpheusTestSuite def graphSourceFactory: FSGraphSourceFactory - override def initPgds(graphNames: List[GraphName]): PropertyGraphDataSource = { + override def initPgds( + graphNames: List[GraphName] + ): PropertyGraphDataSource = { val pgds = fileFormat match { - case FileFormat.csv => graphSourceFactory.csv + case FileFormat.csv => graphSourceFactory.csv case FileFormat.parquet => graphSourceFactory.parquet - case FileFormat.orc => graphSourceFactory.orc - case other => throw IllegalArgumentException("A supported file format", other) + case FileFormat.orc => graphSourceFactory.orc + case other => + throw IllegalArgumentException("A supported file format", other) } graphNames.foreach(gn => pgds.store(gn, graph(gn))) pgds diff --git a/morpheus-testing/src/test/scala/org/opencypher/morpheus/api/io/edgelist/EdgeListDataSourceTest.scala b/morpheus-testing/src/test/scala/org/opencypher/morpheus/api/io/edgelist/EdgeListDataSourceTest.scala index 0e3f1b49fd..59f6408f0a 100644 --- a/morpheus-testing/src/test/scala/org/opencypher/morpheus/api/io/edgelist/EdgeListDataSourceTest.scala +++ b/morpheus-testing/src/test/scala/org/opencypher/morpheus/api/io/edgelist/EdgeListDataSourceTest.scala @@ -44,14 +44,18 @@ class EdgeListDataSourceTest extends MorpheusTestSuite with BeforeAndAfterAll wi |1 3 """.stripMargin - private val tempFile = File.createTempFile(s"morpheus_edgelist_${System.currentTimeMillis()}", "edgelist") + private val tempFile = File.createTempFile( + s"morpheus_edgelist_${System.currentTimeMillis()}", + "edgelist" + ) - private val dataSource = EdgeListDataSource( - tempFile.getAbsolutePath, - Map("delimiter" -> " ")) + private val dataSource = + EdgeListDataSource(tempFile.getAbsolutePath, Map("delimiter" -> " ")) it("should return a static schema") { - dataSource.schema(EdgeListDataSource.GRAPH_NAME) should equal(Some(EdgeListDataSource.SCHEMA)) + dataSource.schema(EdgeListDataSource.GRAPH_NAME) should equal( + Some(EdgeListDataSource.SCHEMA) + ) } it("should contain only one graph named 'graph'") { diff --git a/morpheus-testing/src/test/scala/org/opencypher/morpheus/api/io/fs/DefaultFileSystemTest.scala b/morpheus-testing/src/test/scala/org/opencypher/morpheus/api/io/fs/DefaultFileSystemTest.scala index 363d7a53c9..c253a59074 100644 --- a/morpheus-testing/src/test/scala/org/opencypher/morpheus/api/io/fs/DefaultFileSystemTest.scala +++ b/morpheus-testing/src/test/scala/org/opencypher/morpheus/api/io/fs/DefaultFileSystemTest.scala @@ -50,13 +50,16 @@ class DefaultFileSystemTest extends MorpheusTestSuite { } it("creates a data source root folder when it does not exist yet") { - val graph = morpheus.cypher( - """ + val graph = morpheus + .cypher(""" |CONSTRUCT | CREATE () |RETURN GRAPH - """.stripMargin).graph - val ds = FSGraphSources(tempDir.toAbsolutePath.resolve("someNewFolder1/someNewFolder2").toString).csv + """.stripMargin) + .graph + val ds = FSGraphSources( + tempDir.toAbsolutePath.resolve("someNewFolder1/someNewFolder2").toString + ).csv ds.store(GraphName("foo"), graph) } diff --git a/morpheus-testing/src/test/scala/org/opencypher/morpheus/api/io/fs/FSGraphSourceTest.scala b/morpheus-testing/src/test/scala/org/opencypher/morpheus/api/io/fs/FSGraphSourceTest.scala index da7e510be0..ed2b886d42 100644 --- a/morpheus-testing/src/test/scala/org/opencypher/morpheus/api/io/fs/FSGraphSourceTest.scala +++ b/morpheus-testing/src/test/scala/org/opencypher/morpheus/api/io/fs/FSGraphSourceTest.scala @@ -48,13 +48,17 @@ class FSGraphSourceTest extends MorpheusTestSuite with ScanGraphInit { private val testDatabaseName = "test" override protected def beforeEach(): Unit = { - morpheus.sparkSession.sql(s"CREATE DATABASE IF NOT EXISTS $testDatabaseName") + morpheus.sparkSession.sql( + s"CREATE DATABASE IF NOT EXISTS $testDatabaseName" + ) tempDir = Files.createTempDirectory(getClass.getSimpleName) super.beforeEach() } override protected def afterEach(): Unit = { - morpheus.sparkSession.sql(s"DROP DATABASE IF EXISTS $testDatabaseName CASCADE") + morpheus.sparkSession.sql( + s"DROP DATABASE IF EXISTS $testDatabaseName CASCADE" + ) FileUtils.deleteDirectory(tempDir.toFile) super.afterEach() } @@ -62,18 +66,26 @@ class FSGraphSourceTest extends MorpheusTestSuite with ScanGraphInit { describe("Hive support") { val graphName = GraphName("foo") - val nodeTableName = HiveTableName(testDatabaseName, graphName, Node, Set("L")) - val relTableName = HiveTableName(testDatabaseName, graphName, Relationship, Set("R")) - val testGraph = initGraph("CREATE (:L {prop: 'a'})-[:R {prop: 'b'}]->(:L {prop: 'c'})") + val nodeTableName = + HiveTableName(testDatabaseName, graphName, Node, Set("L")) + val relTableName = + HiveTableName(testDatabaseName, graphName, Relationship, Set("R")) + val testGraph = + initGraph("CREATE (:L {prop: 'a'})-[:R {prop: 'b'}]->(:L {prop: 'c'})") it("writes nodes and relationships to hive tables") { val given = testGraph - val fs = new FSGraphSource("file:///" + tempDir.toAbsolutePath.toString.replace("\\", "/"), - FileFormat.parquet, Some(testDatabaseName), None) + val fs = new FSGraphSource( + "file:///" + tempDir.toAbsolutePath.toString.replace("\\", "/"), + FileFormat.parquet, + Some(testDatabaseName), + None + ) fs.store(graphName, given) - val nodeResult = morpheus.sparkSession.sql(s"SELECT * FROM $nodeTableName") + val nodeResult = + morpheus.sparkSession.sql(s"SELECT * FROM $nodeTableName") nodeResult.collect().toSet should equal( Set( Row(1.encodeAsMorpheusId, "c"), @@ -84,7 +96,12 @@ class FSGraphSourceTest extends MorpheusTestSuite with ScanGraphInit { val relResult = morpheus.sparkSession.sql(s"SELECT * FROM $relTableName") relResult.collect().toSet should equal( Set( - Row(2.encodeAsMorpheusId, 0.encodeAsMorpheusId, 1.encodeAsMorpheusId, "b") + Row( + 2.encodeAsMorpheusId, + 0.encodeAsMorpheusId, + 1.encodeAsMorpheusId, + "b" + ) ) ) } @@ -92,18 +109,28 @@ class FSGraphSourceTest extends MorpheusTestSuite with ScanGraphInit { it("deletes the hive database if the graph is deleted") { val given = testGraph - val fs = new FSGraphSource("file:///" + tempDir.toAbsolutePath.toString.replace("\\", "/"), - FileFormat.parquet, Some(testDatabaseName), None) + val fs = new FSGraphSource( + "file:///" + tempDir.toAbsolutePath.toString.replace("\\", "/"), + FileFormat.parquet, + Some(testDatabaseName), + None + ) fs.store(graphName, given) - morpheus.sparkSession.sql(s"SELECT * FROM $nodeTableName").collect().toSet should not be empty - morpheus.sparkSession.sql(s"SELECT * FROM $relTableName").collect().toSet should not be empty + morpheus.sparkSession + .sql(s"SELECT * FROM $nodeTableName") + .collect() + .toSet should not be empty + morpheus.sparkSession + .sql(s"SELECT * FROM $relTableName") + .collect() + .toSet should not be empty fs.delete(graphName) - an [AnalysisException] shouldBe thrownBy { + an[AnalysisException] shouldBe thrownBy { morpheus.sparkSession.sql(s"SELECT * FROM $nodeTableName") } - an [AnalysisException] shouldBe thrownBy { + an[AnalysisException] shouldBe thrownBy { morpheus.sparkSession.sql(s"SELECT * FROM $relTableName") } } @@ -114,19 +141,24 @@ class FSGraphSourceTest extends MorpheusTestSuite with ScanGraphInit { it("encodes unsupported charaters") { val graphName = GraphName("orcGraph") - val given = initGraph( - """ + val given = initGraph(""" |CREATE (:A {`foo@bar`: 42}) """.stripMargin) - val fs = GraphSources.fs("file:///" + tempDir.toAbsolutePath.toString.replace("\\", "/")).orc + val fs = GraphSources + .fs("file:///" + tempDir.toAbsolutePath.toString.replace("\\", "/")) + .orc fs.store(graphName, given) val graph = fs.graph(graphName) - graph.nodes("n").toMaps should equal(Bag( - CypherMap("n" -> MorpheusNode(0, Set("A"), CypherMap("foo@bar" -> 42))) - )) + graph.nodes("n").toMaps should equal( + Bag( + CypherMap( + "n" -> MorpheusNode(0, Set("A"), CypherMap("foo@bar" -> 42)) + ) + ) + ) } } diff --git a/morpheus-testing/src/test/scala/org/opencypher/morpheus/api/io/neo4j/Neo4jBulkCSVDataSinkTest.scala b/morpheus-testing/src/test/scala/org/opencypher/morpheus/api/io/neo4j/Neo4jBulkCSVDataSinkTest.scala index 656b6cd532..c29f9d62e6 100644 --- a/morpheus-testing/src/test/scala/org/opencypher/morpheus/api/io/neo4j/Neo4jBulkCSVDataSinkTest.scala +++ b/morpheus-testing/src/test/scala/org/opencypher/morpheus/api/io/neo4j/Neo4jBulkCSVDataSinkTest.scala @@ -39,7 +39,11 @@ import org.scalatest.BeforeAndAfterAll import java.nio.file.{Files, Path} import scala.io.Source -class Neo4jBulkCSVDataSinkTest extends MorpheusTestSuite with TeamDataFixture with ScanGraphInit with BeforeAndAfterAll { +class Neo4jBulkCSVDataSinkTest + extends MorpheusTestSuite + with TeamDataFixture + with ScanGraphInit + with BeforeAndAfterAll { protected var tempDir: Path = _ private val graphName = GraphName("teamdata") @@ -48,7 +52,9 @@ class Neo4jBulkCSVDataSinkTest extends MorpheusTestSuite with TeamDataFixture wi override protected def beforeAll(): Unit = { super.beforeAll() tempDir = Files.createTempDirectory(getClass.getSimpleName) - val graph: RelationalCypherGraph[SparkTable.DataFrameTable] = initGraph(dataFixture) + val graph: RelationalCypherGraph[SparkTable.DataFrameTable] = initGraph( + dataFixture + ) val dataSource = new Neo4jBulkCSVDataSink(tempDir.toAbsolutePath.toString) dataSource.store(graphName, graph) morpheus.catalog.register(namespace, dataSource) @@ -59,7 +65,8 @@ class Neo4jBulkCSVDataSinkTest extends MorpheusTestSuite with TeamDataFixture wi super.afterAll() } - private def ds: Neo4jBulkCSVDataSink = morpheus.catalog.source(namespace).asInstanceOf[Neo4jBulkCSVDataSink] + private def ds: Neo4jBulkCSVDataSink = + morpheus.catalog.source(namespace).asInstanceOf[Neo4jBulkCSVDataSink] it("writes the correct script file") { val root = ds.rootPath @@ -87,19 +94,27 @@ class Neo4jBulkCSVDataSinkTest extends MorpheusTestSuite with TeamDataFixture wi } it("writes the correct schema files") { - Source.fromFile(ds.schemaFileForNodes(graphName, Set("Person", "German"))).mkString should equal( + Source + .fromFile(ds.schemaFileForNodes(graphName, Set("Person", "German"))) + .mkString should equal( "___morpheusID:ID,languages:string[],luckyNumber:int,name:string" ) - Source.fromFile(ds.schemaFileForNodes(graphName, Set("Person"))).mkString should equal( + Source + .fromFile(ds.schemaFileForNodes(graphName, Set("Person"))) + .mkString should equal( "___morpheusID:ID,languages:string[],luckyNumber:int,name:string" ) - Source.fromFile(ds.schemaFileForNodes(graphName, Set("Person", "Swede"))).mkString should equal( + Source + .fromFile(ds.schemaFileForNodes(graphName, Set("Person", "Swede"))) + .mkString should equal( "___morpheusID:ID,luckyNumber:int,name:string" ) - Source.fromFile(ds.schemaFileForRelationships(graphName, "KNOWS")).mkString should equal( + Source + .fromFile(ds.schemaFileForRelationships(graphName, "KNOWS")) + .mkString should equal( ":START_ID,:END_ID,since:int" ) } diff --git a/morpheus-testing/src/test/scala/org/opencypher/morpheus/api/io/neo4j/Neo4jPropertyGraphDataSourceEmptyGraphTest.scala b/morpheus-testing/src/test/scala/org/opencypher/morpheus/api/io/neo4j/Neo4jPropertyGraphDataSourceEmptyGraphTest.scala index 3dab396e60..bc3aed2642 100644 --- a/morpheus-testing/src/test/scala/org/opencypher/morpheus/api/io/neo4j/Neo4jPropertyGraphDataSourceEmptyGraphTest.scala +++ b/morpheus-testing/src/test/scala/org/opencypher/morpheus/api/io/neo4j/Neo4jPropertyGraphDataSourceEmptyGraphTest.scala @@ -39,14 +39,17 @@ class Neo4jPropertyGraphDataSourceEmptyGraphTest extends MorpheusTestSuite with it("can read an empty graph") { val graph = Neo4jPropertyGraphDataSource(neo4jConfig).graph(entireGraphName) - val result = graph.cypher( - """ + val result = graph + .cypher(""" |MATCH (n) |RETURN count(n) as count - """.stripMargin).records + """.stripMargin) + .records - result.toMaps should equal(Bag( - CypherMap("count" -> 0) - )) + result.toMaps should equal( + Bag( + CypherMap("count" -> 0) + ) + ) } } diff --git a/morpheus-testing/src/test/scala/org/opencypher/morpheus/api/io/neo4j/Neo4jPropertyGraphDataSourceTest.scala b/morpheus-testing/src/test/scala/org/opencypher/morpheus/api/io/neo4j/Neo4jPropertyGraphDataSourceTest.scala index 18c1751568..98e8ab86b7 100644 --- a/morpheus-testing/src/test/scala/org/opencypher/morpheus/api/io/neo4j/Neo4jPropertyGraphDataSourceTest.scala +++ b/morpheus-testing/src/test/scala/org/opencypher/morpheus/api/io/neo4j/Neo4jPropertyGraphDataSourceTest.scala @@ -50,81 +50,128 @@ import org.opencypher.okapi.testing.Bag import org.opencypher.okapi.testing.Bag._ class Neo4jPropertyGraphDataSourceTest - extends MorpheusTestSuite + extends MorpheusTestSuite with Neo4jServerFixture with TeamDataFixture { it("should cache the schema during and between queries") { - val spiedPGDS = spy[Neo4jPropertyGraphDataSource](CypherGraphSources.neo4j(neo4jConfig)) + val spiedPGDS = + spy[Neo4jPropertyGraphDataSource](CypherGraphSources.neo4j(neo4jConfig)) morpheus.registerSource(Namespace("pgds"), spiedPGDS) - morpheus.cypher( - s""" + morpheus + .cypher( + s""" |FROM pgds.$entireGraphName |MATCH (n) |RETURN 1 """.stripMargin - ).records.size + ) + .records + .size - morpheus.cypher( - s""" + morpheus + .cypher( + s""" |FROM pgds.$entireGraphName |MATCH (n) |RETURN 1 """.stripMargin - ).records.size + ) + .records + .size // we will request schema many times but should only compute it once verify(spiedPGDS, Mockito.atLeast(2)).schema(entireGraphName) - verify(spiedPGDS.asInstanceOf[AbstractPropertyGraphDataSource], times(1)).readSchema(entireGraphName) + verify(spiedPGDS.asInstanceOf[AbstractPropertyGraphDataSource], times(1)) + .readSchema(entireGraphName) } it("can read lists from Neo4j") { - val graph = CypherGraphSources.neo4j(neo4jConfig).graph(entireGraphName).asMorpheus - graph.cypher("MATCH (n) RETURN n.languages").records.iterator.toBag should equal(Bag( - CypherMap("n.languages" -> Seq("German", "English", "Klingon")), - CypherMap("n.languages" -> Seq()), - CypherMap("n.languages" -> CypherNull), - CypherMap("n.languages" -> CypherNull), - CypherMap("n.languages" -> CypherNull) - )) + val graph = + CypherGraphSources.neo4j(neo4jConfig).graph(entireGraphName).asMorpheus + graph + .cypher("MATCH (n) RETURN n.languages") + .records + .iterator + .toBag should equal( + Bag( + CypherMap("n.languages" -> Seq("German", "English", "Klingon")), + CypherMap("n.languages" -> Seq()), + CypherMap("n.languages" -> CypherNull), + CypherMap("n.languages" -> CypherNull), + CypherMap("n.languages" -> CypherNull) + ) + ) } it("should load a graph from Neo4j via DataSource") { - val graph = CypherGraphSources.neo4j(neo4jConfig).graph(entireGraphName).asMorpheus - graph.nodes("n").asMorpheus.toCypherMaps.collect.toBag.nodeValuesWithoutIds shouldEqual teamDataGraphNodes.nodeValuesWithoutIds - graph.relationships("r").asMorpheus.toCypherMaps.collect.toBag.relValuesWithoutIds shouldEqual teamDataGraphRels.relValuesWithoutIds + val graph = + CypherGraphSources.neo4j(neo4jConfig).graph(entireGraphName).asMorpheus + graph + .nodes("n") + .asMorpheus + .toCypherMaps + .collect + .toBag + .nodeValuesWithoutIds shouldEqual teamDataGraphNodes.nodeValuesWithoutIds + graph + .relationships("r") + .asMorpheus + .toCypherMaps + .collect + .toBag + .relValuesWithoutIds shouldEqual teamDataGraphRels.relValuesWithoutIds } it("should load a graph from Neo4j via catalog") { val testNamespace = Namespace("myNeo4j") - morpheus.registerSource(testNamespace, CypherGraphSources.neo4j(neo4jConfig)) + morpheus.registerSource( + testNamespace, + CypherGraphSources.neo4j(neo4jConfig) + ) - val nodes: CypherResult = morpheus.cypher(s"FROM GRAPH $testNamespace.$entireGraphName MATCH (n) RETURN n") + val nodes: CypherResult = morpheus.cypher( + s"FROM GRAPH $testNamespace.$entireGraphName MATCH (n) RETURN n" + ) nodes.records.collect.toBag.nodeValuesWithoutIds shouldEqual teamDataGraphNodes.nodeValuesWithoutIds - val edges = morpheus.cypher(s"FROM GRAPH $testNamespace.$entireGraphName MATCH ()-[r]->() RETURN r") + val edges = morpheus.cypher( + s"FROM GRAPH $testNamespace.$entireGraphName MATCH ()-[r]->() RETURN r" + ) edges.records.collect.toBag.relValuesWithoutIds shouldEqual teamDataGraphRels.relValuesWithoutIds } - it("should omit properties with unsupported types if corresponding flag is set") { - neo4jConfig.cypherWithNewSession(s"""CREATE (n:Unsupported:${metaPrefix}test { foo: duration('P2.5W'), bar: 42 })""") + it( + "should omit properties with unsupported types if corresponding flag is set" + ) { + neo4jConfig.cypherWithNewSession( + s"""CREATE (n:Unsupported:${metaPrefix}test { foo: duration('P2.5W'), bar: 42 })""" + ) - val dataSource = CypherGraphSources.neo4j(neo4jConfig, omitIncompatibleProperties = true) + val dataSource = + CypherGraphSources.neo4j(neo4jConfig, omitIncompatibleProperties = true) val graph = dataSource.graph(GraphName("test")).asMorpheus val nodes = graph.nodes("n").asMorpheus.toCypherMaps.collect.toList nodes.size shouldBe 1 nodes.head.value match { - case n: MorpheusNode => n should equal(MorpheusNode(n.id, Set("Unsupported"), CypherMap("bar" -> 42L))) + case n: MorpheusNode => + n should equal( + MorpheusNode(n.id, Set("Unsupported"), CypherMap("bar" -> 42L)) + ) case other => IllegalArgumentException("a MorpheusNode", other) } } - it("should throw exception if properties with unsupported types are being imported") { + it( + "should throw exception if properties with unsupported types are being imported" + ) { a[SchemaException] should be thrownBy { - neo4jConfig.cypherWithNewSession(s"""CREATE (n:Unsupported:${metaPrefix}test { foo: time(), bar: 42 })""") + neo4jConfig.cypherWithNewSession( + s"""CREATE (n:Unsupported:${metaPrefix}test { foo: time(), bar: 42 })""" + ) val dataSource = CypherGraphSources.neo4j(neo4jConfig) val graph = dataSource.graph(GraphName("test")).asMorpheus diff --git a/morpheus-testing/src/test/scala/org/opencypher/morpheus/api/io/neo4j/Neo4jPropertyGraphDataSourceWriteTest.scala b/morpheus-testing/src/test/scala/org/opencypher/morpheus/api/io/neo4j/Neo4jPropertyGraphDataSourceWriteTest.scala index fd980e58b2..8f09f12bc9 100644 --- a/morpheus-testing/src/test/scala/org/opencypher/morpheus/api/io/neo4j/Neo4jPropertyGraphDataSourceWriteTest.scala +++ b/morpheus-testing/src/test/scala/org/opencypher/morpheus/api/io/neo4j/Neo4jPropertyGraphDataSourceWriteTest.scala @@ -37,13 +37,12 @@ import org.opencypher.okapi.testing.Bag import org.opencypher.okapi.testing.Bag._ class Neo4jPropertyGraphDataSourceWriteTest - extends MorpheusTestSuite + extends MorpheusTestSuite with Neo4jServerFixture - with ScanGraphInit{ + with ScanGraphInit { it("can write a graph to Neo4j") { - val g = initGraph( - """ + val g = initGraph(""" |CREATE (a:A {val: 1})-[:REL {val: 3}]->(b:B {val: 2}) |CREATE (b)-[:REL {val: 4}]->(a) """.stripMargin) @@ -52,10 +51,15 @@ class Neo4jPropertyGraphDataSourceWriteTest dataSource.store(GraphName("g1"), g) - neo4jConfig.cypherWithNewSession("MATCH (n)-[r]->(m) RETURN n.val, r.val, m.val").map(x => CypherMap(x.toSeq:_*)).toBag should equal(Bag( - CypherMap("n.val" -> 1, "r.val" -> 3, "m.val" -> 2), - CypherMap("n.val" -> 2, "r.val" -> 4, "m.val" -> 1) - )) + neo4jConfig + .cypherWithNewSession("MATCH (n)-[r]->(m) RETURN n.val, r.val, m.val") + .map(x => CypherMap(x.toSeq: _*)) + .toBag should equal( + Bag( + CypherMap("n.val" -> 1, "r.val" -> 3, "m.val" -> 2), + CypherMap("n.val" -> 2, "r.val" -> 4, "m.val" -> 1) + ) + ) } override def dataFixture: String = "" diff --git a/morpheus-testing/src/test/scala/org/opencypher/morpheus/api/io/neo4j/sync/Neo4JGraphMergeTest.scala b/morpheus-testing/src/test/scala/org/opencypher/morpheus/api/io/neo4j/sync/Neo4JGraphMergeTest.scala index 26a6c9dc48..5500fcad7d 100644 --- a/morpheus-testing/src/test/scala/org/opencypher/morpheus/api/io/neo4j/sync/Neo4JGraphMergeTest.scala +++ b/morpheus-testing/src/test/scala/org/opencypher/morpheus/api/io/neo4j/sync/Neo4JGraphMergeTest.scala @@ -53,8 +53,8 @@ class Neo4JGraphMergeTest extends MorpheusTestSuite with Neo4jServerFixture with val nodeKeys = Map("Person" -> Set("id")) val relKeys = Map("R" -> Set("id")) - val initialGraph: RelationalCypherGraph[SparkTable.DataFrameTable] = initGraph( - """ + val initialGraph: RelationalCypherGraph[SparkTable.DataFrameTable] = + initGraph(""" |CREATE (s:Person {id: 1, name: "bar"}) |CREATE (e:Person:Employee {id: 2}) |CREATE (s)-[r:R {id: 1}]->(e) @@ -68,55 +68,133 @@ class Neo4JGraphMergeTest extends MorpheusTestSuite with Neo4jServerFixture with describe("merging into the entire graph") { it("can do basic Neo4j merge") { Neo4jGraphMerge.createIndexes(entireGraphName, neo4jConfig, nodeKeys) - Neo4jGraphMerge.merge(entireGraphName, initialGraph, neo4jConfig, Some(nodeKeys), Some(relKeys)) - - val readGraph = Neo4jPropertyGraphDataSource(neo4jConfig).graph(entireGraphName) - - readGraph.cypher("MATCH (n) RETURN n.id as id, n.name as name, labels(n) as labels").records.toMaps should equal(Bag( - CypherMap("id" -> 1, "name" -> "bar", "labels" -> Seq("Person")), - CypherMap("id" -> 2, "name" -> null, "labels" -> Seq("Employee", "Person")) - )) - - readGraph.cypher("MATCH (n)-[r]->(m) RETURN n.id as nid, r.id as id, m.id as mid").records.toMaps should equal(Bag( - CypherMap("nid" -> 1, "id" -> 1, "mid" -> 2) - )) + Neo4jGraphMerge.merge( + entireGraphName, + initialGraph, + neo4jConfig, + Some(nodeKeys), + Some(relKeys) + ) + + val readGraph = + Neo4jPropertyGraphDataSource(neo4jConfig).graph(entireGraphName) + + readGraph + .cypher( + "MATCH (n) RETURN n.id as id, n.name as name, labels(n) as labels" + ) + .records + .toMaps should equal( + Bag( + CypherMap("id" -> 1, "name" -> "bar", "labels" -> Seq("Person")), + CypherMap( + "id" -> 2, + "name" -> null, + "labels" -> Seq("Employee", "Person") + ) + ) + ) + + readGraph + .cypher( + "MATCH (n)-[r]->(m) RETURN n.id as nid, r.id as id, m.id as mid" + ) + .records + .toMaps should equal( + Bag( + CypherMap("nid" -> 1, "id" -> 1, "mid" -> 2) + ) + ) // Do not change a graph when the same graph is merged as a delta - Neo4jGraphMerge.merge(entireGraphName, initialGraph, neo4jConfig, Some(nodeKeys), Some(relKeys)) + Neo4jGraphMerge.merge( + entireGraphName, + initialGraph, + neo4jConfig, + Some(nodeKeys), + Some(relKeys) + ) val graphAfterSameMerge = Neo4jPropertyGraphDataSource(neo4jConfig) .graph(entireGraphName) - graphAfterSameMerge.cypher("MATCH (n) RETURN n.id as id, n.name as name, labels(n) as labels").records.toMaps should equal(Bag( - CypherMap("id" -> 1, "name" -> "bar", "labels" -> Seq("Person")), - CypherMap("id" -> 2, "name" -> null, "labels" -> Seq("Employee", "Person")) - )) - - graphAfterSameMerge.cypher("MATCH (n)-[r]->(m) RETURN n.id as nid, r.id as id, m.id as mid").records.toMaps should equal(Bag( - CypherMap("nid" -> 1, "id" -> 1, "mid" -> 2) - )) + graphAfterSameMerge + .cypher( + "MATCH (n) RETURN n.id as id, n.name as name, labels(n) as labels" + ) + .records + .toMaps should equal( + Bag( + CypherMap("id" -> 1, "name" -> "bar", "labels" -> Seq("Person")), + CypherMap( + "id" -> 2, + "name" -> null, + "labels" -> Seq("Employee", "Person") + ) + ) + ) + + graphAfterSameMerge + .cypher( + "MATCH (n)-[r]->(m) RETURN n.id as nid, r.id as id, m.id as mid" + ) + .records + .toMaps should equal( + Bag( + CypherMap("nid" -> 1, "id" -> 1, "mid" -> 2) + ) + ) // merge a delta - val delta = initGraph( - """ + val delta = initGraph(""" |CREATE (s:Person {id: 1, name: "baz", bar: 1}) |CREATE (e:Person {id: 2}) |CREATE (s)-[r:R {id: 1, name: 1}]->(e) |CREATE (s)-[r:R {id: 2}]->(e) """.stripMargin) - Neo4jGraphMerge.merge(entireGraphName, delta, neo4jConfig, Some(nodeKeys), Some(relKeys)) + Neo4jGraphMerge.merge( + entireGraphName, + delta, + neo4jConfig, + Some(nodeKeys), + Some(relKeys) + ) val graphAfterDeltaSync = Neo4jPropertyGraphDataSource(neo4jConfig) .graph(entireGraphName) - graphAfterDeltaSync.cypher("MATCH (n) RETURN n.id as id, n.name as name, n.bar as bar, labels(n) as labels").records.toMaps should equal(Bag( - CypherMap("id" -> 1, "name" -> "baz", "bar" -> 1, "labels" -> Seq("Person")), - CypherMap("id" -> 2, "name" -> null, "bar" -> null, "labels" -> Seq("Employee", "Person")) - )) - - graphAfterDeltaSync.cypher("MATCH (n)-[r]->(m) RETURN n.id as nid, r.id as id, r.name as name, m.id as mid").records.toMaps should equal(Bag( - CypherMap("nid" -> 1, "id" -> 1, "name" -> 1, "mid" -> 2), - CypherMap("nid" -> 1, "id" -> 2, "name" -> null, "mid" -> 2) - )) + graphAfterDeltaSync + .cypher( + "MATCH (n) RETURN n.id as id, n.name as name, n.bar as bar, labels(n) as labels" + ) + .records + .toMaps should equal( + Bag( + CypherMap( + "id" -> 1, + "name" -> "baz", + "bar" -> 1, + "labels" -> Seq("Person") + ), + CypherMap( + "id" -> 2, + "name" -> null, + "bar" -> null, + "labels" -> Seq("Employee", "Person") + ) + ) + ) + + graphAfterDeltaSync + .cypher( + "MATCH (n)-[r]->(m) RETURN n.id as nid, r.id as id, r.name as name, m.id as mid" + ) + .records + .toMaps should equal( + Bag( + CypherMap("nid" -> 1, "id" -> 1, "name" -> 1, "mid" -> 2), + CypherMap("nid" -> 1, "id" -> 2, "name" -> null, "mid" -> 2) + ) + ) } it("merges when using the same element key for all labels") { @@ -126,23 +204,39 @@ class Neo4JGraphMergeTest extends MorpheusTestSuite with Neo4jServerFixture with Neo4jGraphMerge.createIndexes(entireGraphName, neo4jConfig, nodeKeys) val graphName = GraphName("graph") - val graph = initGraph( - """ + val graph = initGraph(""" |CREATE (s:Person {id: 1, name: "bar"}) |CREATE (e:Person:Employee {id: 2 }) |CREATE (f:Employee {id: 3}) |CREATE (s)-[r:R {id: 1}]->(e) """.stripMargin) - Neo4jGraphMerge.merge(entireGraphName, graph, neo4jConfig, Some(nodeKeys), Some(relKeys)) + Neo4jGraphMerge.merge( + entireGraphName, + graph, + neo4jConfig, + Some(nodeKeys), + Some(relKeys) + ) val readGraph = Neo4jPropertyGraphDataSource(neo4jConfig).graph(graphName) - readGraph.cypher("MATCH (n) RETURN n.id as id, n.name as name, labels(n) as labels").records.toMaps should equal(Bag( - CypherMap("id" -> 1, "name" -> "bar", "labels" -> Seq("Person")), - CypherMap("id" -> 2, "name" -> null, "labels" -> Seq("Employee", "Person")), - CypherMap("id" -> 3, "name" -> null, "labels" -> Seq("Employee")) - )) + readGraph + .cypher( + "MATCH (n) RETURN n.id as id, n.name as name, labels(n) as labels" + ) + .records + .toMaps should equal( + Bag( + CypherMap("id" -> 1, "name" -> "bar", "labels" -> Seq("Person")), + CypherMap( + "id" -> 2, + "name" -> null, + "labels" -> Seq("Employee", "Person") + ), + CypherMap("id" -> 3, "name" -> null, "labels" -> Seq("Employee")) + ) + ) } it("merges when using multiple element keys with different names") { @@ -152,23 +246,50 @@ class Neo4JGraphMergeTest extends MorpheusTestSuite with Neo4jServerFixture with Neo4jGraphMerge.createIndexes(entireGraphName, neo4jConfig, nodeKeys) val graphName = GraphName("graph") - val graph = initGraph( - """ + val graph = initGraph(""" |CREATE (s:Person {nId: 1, name: "bar"}) |CREATE (e:Person:Employee {nId: 2, mId: 3 }) |CREATE (f:Employee {mId: 2}) |CREATE (s)-[r:R {id: 1}]->(e) """.stripMargin) - Neo4jGraphMerge.merge(entireGraphName, graph, neo4jConfig, Some(nodeKeys), Some(relKeys)) + Neo4jGraphMerge.merge( + entireGraphName, + graph, + neo4jConfig, + Some(nodeKeys), + Some(relKeys) + ) val readGraph = Neo4jPropertyGraphDataSource(neo4jConfig).graph(graphName) - readGraph.cypher("MATCH (n) RETURN n.nId as nId, n.mId as mId, n.name as name, labels(n) as labels").records.toMaps should equal(Bag( - CypherMap("nId" -> 1, "mId" -> null, "name" -> "bar", "labels" -> Seq("Person")), - CypherMap("nId" -> 2, "mId" -> 3, "name" -> null, "labels" -> Seq("Employee", "Person")), - CypherMap("nId" -> null, "mId" -> 2, "name" -> null, "labels" -> Seq("Employee")) - )) + readGraph + .cypher( + "MATCH (n) RETURN n.nId as nId, n.mId as mId, n.name as name, labels(n) as labels" + ) + .records + .toMaps should equal( + Bag( + CypherMap( + "nId" -> 1, + "mId" -> null, + "name" -> "bar", + "labels" -> Seq("Person") + ), + CypherMap( + "nId" -> 2, + "mId" -> 3, + "name" -> null, + "labels" -> Seq("Employee", "Person") + ), + CypherMap( + "nId" -> null, + "mId" -> 2, + "name" -> null, + "labels" -> Seq("Employee") + ) + ) + ) } it("merges with element keys set in the schema") { @@ -176,7 +297,12 @@ class Neo4JGraphMergeTest extends MorpheusTestSuite with Neo4jServerFixture with Row(1L, "baz") ).asJava - val df = sparkSession.createDataFrame(data, StructType(Seq(StructField("id", LongType), StructField("name", StringType)))) + val df = sparkSession.createDataFrame( + data, + StructType( + Seq(StructField("id", LongType), StructField("name", StringType)) + ) + ) morpheus.sql("CREATE DATABASE IF NOT EXISTS db") df.write.saveAsTable("db.persons") @@ -193,18 +319,33 @@ class Neo4JGraphMergeTest extends MorpheusTestSuite with Neo4jServerFixture with |) """.stripMargin val hiveDataSourceConfig = SqlDataSourceConfig.Hive - val pgds = SqlPropertyGraphDataSource(GraphDdl(ddlString), Map("ds1" -> hiveDataSourceConfig)) + val pgds = SqlPropertyGraphDataSource( + GraphDdl(ddlString), + Map("ds1" -> hiveDataSourceConfig) + ) val graph = pgds.graph(GraphName("personGraph")) - Neo4jGraphMerge.createIndexes(entireGraphName, neo4jConfig, graph.schema.nodeKeys) + Neo4jGraphMerge.createIndexes( + entireGraphName, + neo4jConfig, + graph.schema.nodeKeys + ) Neo4jGraphMerge.merge(entireGraphName, graph, neo4jConfig) - val readGraph = Neo4jPropertyGraphDataSource(neo4jConfig).graph(entireGraphName) - - readGraph.cypher("MATCH (n:Person) RETURN n.id as id, n.name as name, labels(n) as labels").records.toMaps should equal(Bag( - CypherMap("id" -> 1, "name" -> "baz", "labels" -> Seq("Person")) - )) + val readGraph = + Neo4jPropertyGraphDataSource(neo4jConfig).graph(entireGraphName) + + readGraph + .cypher( + "MATCH (n:Person) RETURN n.id as id, n.name as name, labels(n) as labels" + ) + .records + .toMaps should equal( + Bag( + CypherMap("id" -> 1, "name" -> "baz", "labels" -> Seq("Person")) + ) + ) } it("checks if element keys are present in the merge graph") { @@ -212,38 +353,74 @@ class Neo4JGraphMergeTest extends MorpheusTestSuite with Neo4jServerFixture with val relKeys = Map("R" -> Set("id")) Neo4jGraphMerge.createIndexes(entireGraphName, neo4jConfig, nodeKeys) - val graph = initGraph( - """ + val graph = initGraph(""" |CREATE (s:Person {nId: 1, name: "bar"}) |CREATE (e:Person:Employee {nId: 2, mId: 3 }) |CREATE (f:Employee {mId: 2}) |CREATE (s)-[r:R {id: 1}]->(e) """.stripMargin) - Try(Neo4jGraphMerge.merge(entireGraphName, graph, neo4jConfig, Some(nodeKeys), Some(relKeys))) match { + Try( + Neo4jGraphMerge.merge( + entireGraphName, + graph, + neo4jConfig, + Some(nodeKeys), + Some(relKeys) + ) + ) match { case Success(_) => fail case Failure(exception) => - exception.getMessage should include("Properties [name] have nullable types") + exception.getMessage should include( + "Properties [name] have nullable types" + ) } } it("creates indexes correctly") { - val nodeKeys = Map("Person" -> Set("name", "bar"), "Employee" -> Set("baz")) + val nodeKeys = + Map("Person" -> Set("name", "bar"), "Employee" -> Set("baz")) val relKeys = Map("REL" -> Set("a")) Neo4jGraphMerge.createIndexes(entireGraphName, neo4jConfig, nodeKeys) - neo4jConfig.cypherWithNewSession("CALL db.constraints YIELD description").toSet should equal(Set( - Map("description" -> new CypherString("CONSTRAINT ON ( person:Person ) ASSERT (person.name, person.bar) IS NODE KEY")), - Map("description" -> new CypherString("CONSTRAINT ON ( employee:Employee ) ASSERT employee.baz IS NODE KEY")) - )) - - neo4jConfig.cypherWithNewSession("CALL db.indexes YIELD description").toSet should equal(Set( - Map("description" -> new CypherString(s"INDEX ON :Person($metaPropertyKey)")), - Map("description" -> new CypherString(s"INDEX ON :Person(name, bar)")), - Map("description" -> new CypherString(s"INDEX ON :Employee($metaPropertyKey)")), - Map("description" -> new CypherString(s"INDEX ON :Employee(baz)")) - )) + neo4jConfig + .cypherWithNewSession("CALL db.constraints YIELD description") + .toSet should equal( + Set( + Map( + "description" -> new CypherString( + "CONSTRAINT ON ( person:Person ) ASSERT (person.name, person.bar) IS NODE KEY" + ) + ), + Map( + "description" -> new CypherString( + "CONSTRAINT ON ( employee:Employee ) ASSERT employee.baz IS NODE KEY" + ) + ) + ) + ) + + neo4jConfig + .cypherWithNewSession("CALL db.indexes YIELD description") + .toSet should equal( + Set( + Map( + "description" -> new CypherString( + s"INDEX ON :Person($metaPropertyKey)" + ) + ), + Map( + "description" -> new CypherString(s"INDEX ON :Person(name, bar)") + ), + Map( + "description" -> new CypherString( + s"INDEX ON :Employee($metaPropertyKey)" + ) + ), + Map("description" -> new CypherString(s"INDEX ON :Employee(baz)")) + ) + ) } } @@ -252,89 +429,192 @@ class Neo4JGraphMergeTest extends MorpheusTestSuite with Neo4jServerFixture with val subGraphName = GraphName("name") Neo4jGraphMerge.createIndexes(subGraphName, neo4jConfig, nodeKeys) - Neo4jGraphMerge.merge(subGraphName, initialGraph, neo4jConfig, Some(nodeKeys), Some(relKeys)) - - val readGraph = Neo4jPropertyGraphDataSource(neo4jConfig).graph(subGraphName) - - readGraph.cypher("MATCH (n) RETURN n.id as id, n.name as name, labels(n) as labels").records.toMaps should equal(Bag( - CypherMap("id" -> 1, "name" -> "bar", "labels" -> Seq("Person")), - CypherMap("id" -> 2, "name" -> null, "labels" -> Seq("Employee", "Person")) - )) - - readGraph.cypher("MATCH (n)-[r]->(m) RETURN n.id as nid, r.id as id, m.id as mid").records.toMaps should equal(Bag( - CypherMap("nid" -> 1, "id" -> 1, "mid" -> 2) - )) + Neo4jGraphMerge.merge( + subGraphName, + initialGraph, + neo4jConfig, + Some(nodeKeys), + Some(relKeys) + ) + + val readGraph = + Neo4jPropertyGraphDataSource(neo4jConfig).graph(subGraphName) + + readGraph + .cypher( + "MATCH (n) RETURN n.id as id, n.name as name, labels(n) as labels" + ) + .records + .toMaps should equal( + Bag( + CypherMap("id" -> 1, "name" -> "bar", "labels" -> Seq("Person")), + CypherMap( + "id" -> 2, + "name" -> null, + "labels" -> Seq("Employee", "Person") + ) + ) + ) + + readGraph + .cypher( + "MATCH (n)-[r]->(m) RETURN n.id as nid, r.id as id, m.id as mid" + ) + .records + .toMaps should equal( + Bag( + CypherMap("nid" -> 1, "id" -> 1, "mid" -> 2) + ) + ) // Do not change a graph when the same graph is synced as a delta - Neo4jGraphMerge.merge(entireGraphName, initialGraph, neo4jConfig, Some(nodeKeys), Some(relKeys)) - val graphAfterSameSync = Neo4jPropertyGraphDataSource(neo4jConfig).graph(subGraphName) - - graphAfterSameSync.cypher("MATCH (n) RETURN n.id as id, n.name as name, labels(n) as labels").records.toMaps should equal(Bag( - CypherMap("id" -> 1, "name" -> "bar", "labels" -> Seq("Person")), - CypherMap("id" -> 2, "name" -> null, "labels" -> Seq("Employee", "Person")) - )) - - graphAfterSameSync.cypher("MATCH (n)-[r]->(m) RETURN n.id as nid, r.id as id, m.id as mid").records.toMaps should equal(Bag( - CypherMap("nid" -> 1, "id" -> 1, "mid" -> 2) - )) + Neo4jGraphMerge.merge( + entireGraphName, + initialGraph, + neo4jConfig, + Some(nodeKeys), + Some(relKeys) + ) + val graphAfterSameSync = + Neo4jPropertyGraphDataSource(neo4jConfig).graph(subGraphName) + + graphAfterSameSync + .cypher( + "MATCH (n) RETURN n.id as id, n.name as name, labels(n) as labels" + ) + .records + .toMaps should equal( + Bag( + CypherMap("id" -> 1, "name" -> "bar", "labels" -> Seq("Person")), + CypherMap( + "id" -> 2, + "name" -> null, + "labels" -> Seq("Employee", "Person") + ) + ) + ) + + graphAfterSameSync + .cypher( + "MATCH (n)-[r]->(m) RETURN n.id as nid, r.id as id, m.id as mid" + ) + .records + .toMaps should equal( + Bag( + CypherMap("nid" -> 1, "id" -> 1, "mid" -> 2) + ) + ) // Sync a delta - val delta = initGraph( - """ + val delta = initGraph(""" |CREATE (s:Person {id: 1, name: "baz", bar: 1}) |CREATE (e:Person {id: 2}) |CREATE (s)-[r:R {id: 1, name: 1}]->(e) |CREATE (s)-[r:R {id: 2}]->(e) """.stripMargin) - Neo4jGraphMerge.merge(subGraphName, delta, neo4jConfig, Some(nodeKeys), Some(relKeys)) - val graphAfterDeltaSync = Neo4jPropertyGraphDataSource(neo4jConfig).graph(subGraphName) - - graphAfterDeltaSync.cypher("MATCH (n) RETURN n.id as id, n.name as name, n.bar as bar, labels(n) as labels").records.toMaps should equal(Bag( - CypherMap("id" -> 1, "name" -> "baz", "bar" -> 1, "labels" -> Seq("Person")), - CypherMap("id" -> 2, "name" -> null, "bar" -> null, "labels" -> Seq("Employee", "Person")) - )) - - graphAfterDeltaSync.cypher("MATCH (n)-[r]->(m) RETURN n.id as nid, r.id as id, r.name as name, m.id as mid").records.toMaps should equal(Bag( - CypherMap("nid" -> 1, "id" -> 1, "name" -> 1, "mid" -> 2), - CypherMap("nid" -> 1, "id" -> 2, "name" -> null, "mid" -> 2) - )) + Neo4jGraphMerge.merge( + subGraphName, + delta, + neo4jConfig, + Some(nodeKeys), + Some(relKeys) + ) + val graphAfterDeltaSync = + Neo4jPropertyGraphDataSource(neo4jConfig).graph(subGraphName) + + graphAfterDeltaSync + .cypher( + "MATCH (n) RETURN n.id as id, n.name as name, n.bar as bar, labels(n) as labels" + ) + .records + .toMaps should equal( + Bag( + CypherMap( + "id" -> 1, + "name" -> "baz", + "bar" -> 1, + "labels" -> Seq("Person") + ), + CypherMap( + "id" -> 2, + "name" -> null, + "bar" -> null, + "labels" -> Seq("Employee", "Person") + ) + ) + ) + + graphAfterDeltaSync + .cypher( + "MATCH (n)-[r]->(m) RETURN n.id as nid, r.id as id, r.name as name, m.id as mid" + ) + .records + .toMaps should equal( + Bag( + CypherMap("nid" -> 1, "id" -> 1, "name" -> 1, "mid" -> 2), + CypherMap("nid" -> 1, "id" -> 2, "name" -> null, "mid" -> 2) + ) + ) } it("creates indexes correctly") { - val nodeKeys = Map("Person" -> Set("name", "bar"), "Employee" -> Set("baz")) + val nodeKeys = + Map("Person" -> Set("name", "bar"), "Employee" -> Set("baz")) val relKeys = Map("REL" -> Set("a")) val subGraphName = GraphName("myGraph") Neo4jGraphMerge.createIndexes(subGraphName, neo4jConfig, nodeKeys) - neo4jConfig.cypherWithNewSession("CALL db.constraints YIELD description").toSet shouldBe empty - - neo4jConfig.cypherWithNewSession("CALL db.indexes YIELD description").toSet should equal(Set( - Map("description" -> new CypherString(s"INDEX ON :${subGraphName.metaLabelForSubgraph}($metaPropertyKey)")), - Map("description" -> new CypherString(s"INDEX ON :Person(name, bar)")), - Map("description" -> new CypherString(s"INDEX ON :Employee(baz)")) - )) + neo4jConfig + .cypherWithNewSession("CALL db.constraints YIELD description") + .toSet shouldBe empty + + neo4jConfig + .cypherWithNewSession("CALL db.indexes YIELD description") + .toSet should equal( + Set( + Map( + "description" -> new CypherString( + s"INDEX ON :${subGraphName.metaLabelForSubgraph}($metaPropertyKey)" + ) + ), + Map( + "description" -> new CypherString(s"INDEX ON :Person(name, bar)") + ), + Map("description" -> new CypherString(s"INDEX ON :Employee(baz)")) + ) + ) } } describe("error handling") { it("should throw when a node key is missing") { - a[SchemaException] should be thrownBy Neo4jGraphMerge.merge(entireGraphName, initialGraph, neo4jConfig) + a[SchemaException] should be thrownBy Neo4jGraphMerge.merge( + entireGraphName, + initialGraph, + neo4jConfig + ) } - it("should throw when a missing element key is not only appearing with an implied label that has an element key") { + it( + "should throw when a missing element key is not only appearing with an implied label that has an element key" + ) { val nodeKeys = Map("Person" -> Set("id")) Neo4jGraphMerge.createIndexes(entireGraphName, neo4jConfig, nodeKeys) - val graph = initGraph( - """ + val graph = initGraph(""" |CREATE (s:Person {id: 1, name: "bar"}) |CREATE (e:Person:Employee {id: 2}) |CREATE (f:Employee {id: 3}) |CREATE (s)-[r:R {id: 1}]->(e) """.stripMargin) - a[SchemaException] should be thrownBy Neo4jGraphMerge.merge(entireGraphName, graph, neo4jConfig, Some(nodeKeys)) + a[SchemaException] should be thrownBy Neo4jGraphMerge.merge( + entireGraphName, + graph, + neo4jConfig, + Some(nodeKeys) + ) } } diff --git a/morpheus-testing/src/test/scala/org/opencypher/morpheus/api/io/sql/GraphDdlConversionsTest.scala b/morpheus-testing/src/test/scala/org/opencypher/morpheus/api/io/sql/GraphDdlConversionsTest.scala index 847b7417ea..ee265ab1b4 100644 --- a/morpheus-testing/src/test/scala/org/opencypher/morpheus/api/io/sql/GraphDdlConversionsTest.scala +++ b/morpheus-testing/src/test/scala/org/opencypher/morpheus/api/io/sql/GraphDdlConversionsTest.scala @@ -43,8 +43,7 @@ class GraphDdlConversionsTest extends BaseTestSuite { describe("GraphType to OKAPI schema") { it("converts a graph type with single element type references") { - GraphDdl( - """ + GraphDdl(""" |CREATE GRAPH myGraph ( | Person ( name STRING, age INTEGER ), | Book ( title STRING ) , @@ -53,16 +52,23 @@ class GraphDdlConversionsTest extends BaseTestSuite { | (Book), | (Person)-[READS]->(Book) |) - """.stripMargin).graphs(GraphName("myGraph")).graphType.asOkapiSchema should equal(PropertyGraphSchema.empty - .withNodePropertyKeys("Person")("name" -> CTString, "age" -> CTInteger) - .withNodePropertyKeys("Book")("title" -> CTString) - .withRelationshipPropertyKeys("READS")("rating" -> CTFloat) - .withSchemaPatterns(SchemaPattern("Person", "READS", "Book"))) + """.stripMargin) + .graphs(GraphName("myGraph")) + .graphType + .asOkapiSchema should equal( + PropertyGraphSchema.empty + .withNodePropertyKeys("Person")( + "name" -> CTString, + "age" -> CTInteger + ) + .withNodePropertyKeys("Book")("title" -> CTString) + .withRelationshipPropertyKeys("READS")("rating" -> CTFloat) + .withSchemaPatterns(SchemaPattern("Person", "READS", "Book")) + ) } it("converts a graph type with multiple element type references") { - GraphDdl( - """ + GraphDdl(""" |CREATE GRAPH myGraph ( | A (x STRING), | B (y STRING), @@ -72,17 +78,21 @@ class GraphDdlConversionsTest extends BaseTestSuite { | (A)-[R]->(A), | (A, B)-[R]->(A) |) - """.stripMargin).graphs(GraphName("myGraph")).graphType.asOkapiSchema should equal(PropertyGraphSchema.empty - .withNodePropertyKeys("A")("x" -> CTString) - .withNodePropertyKeys("A", "B")("x" -> CTString, "y" -> CTString) - .withRelationshipPropertyKeys("R")("y" -> CTString) - .withSchemaPatterns(SchemaPattern("A", "R", "A")) - .withSchemaPatterns(SchemaPattern(Set("A", "B"), "R", Set("A")))) + """.stripMargin) + .graphs(GraphName("myGraph")) + .graphType + .asOkapiSchema should equal( + PropertyGraphSchema.empty + .withNodePropertyKeys("A")("x" -> CTString) + .withNodePropertyKeys("A", "B")("x" -> CTString, "y" -> CTString) + .withRelationshipPropertyKeys("R")("y" -> CTString) + .withSchemaPatterns(SchemaPattern("A", "R", "A")) + .withSchemaPatterns(SchemaPattern(Set("A", "B"), "R", Set("A"))) + ) } it("converts a graph type with element type inheritance") { - GraphDdl( - """ + GraphDdl(""" |CREATE GRAPH myGraph ( | A (x STRING), | B EXTENDS A (y STRING), @@ -92,12 +102,17 @@ class GraphDdlConversionsTest extends BaseTestSuite { | (A)-[R]->(A), | (B)-[R]->(A) |) - """.stripMargin).graphs(GraphName("myGraph")).graphType.asOkapiSchema should equal(PropertyGraphSchema.empty - .withNodePropertyKeys("A")("x" -> CTString) - .withNodePropertyKeys("A", "B")("x" -> CTString, "y" -> CTString) - .withRelationshipPropertyKeys("R")("y" -> CTString) - .withSchemaPatterns(SchemaPattern("A", "R", "A")) - .withSchemaPatterns(SchemaPattern(Set("A", "B"), "R", Set("A")))) + """.stripMargin) + .graphs(GraphName("myGraph")) + .graphType + .asOkapiSchema should equal( + PropertyGraphSchema.empty + .withNodePropertyKeys("A")("x" -> CTString) + .withNodePropertyKeys("A", "B")("x" -> CTString, "y" -> CTString) + .withRelationshipPropertyKeys("R")("y" -> CTString) + .withSchemaPatterns(SchemaPattern("A", "R", "A")) + .withSchemaPatterns(SchemaPattern(Set("A", "B"), "R", Set("A"))) + ) } it("can construct schema with node label") { @@ -151,7 +166,6 @@ class GraphDdlConversionsTest extends BaseTestSuite { |CREATE GRAPH $graphName OF $typeName () |""".stripMargin - GraphDdl(ddl).graphs(graphName).graphType.asOkapiSchema shouldEqual PropertyGraphSchema.empty .withNodePropertyKeys("Node1")("val" -> CTString) @@ -173,9 +187,10 @@ class GraphDdlConversionsTest extends BaseTestSuite { |CREATE GRAPH $graphName OF $typeName () |""".stripMargin - GraphDdl(ddl).graphs(graphName).graphType.asOkapiSchema should equal( - PropertyGraphSchema.empty.withNodePropertyKeys("Node")("foo" -> CTInteger) + PropertyGraphSchema.empty.withNodePropertyKeys("Node")( + "foo" -> CTInteger + ) ) } @@ -191,7 +206,10 @@ class GraphDdlConversionsTest extends BaseTestSuite { GraphDdl(ddl).graphs(graphName).graphType.asOkapiSchema should equal( PropertyGraphSchema.empty - .withNodePropertyKeys("Node")("val" -> CTString, "another" -> CTString) + .withNodePropertyKeys("Node")( + "val" -> CTString, + "another" -> CTString + ) .withNodeKey("Node", Set("val")) ) } @@ -242,13 +260,26 @@ class GraphDdlConversionsTest extends BaseTestSuite { GraphDdl(ddl).graphs(graphName).graphType.asOkapiSchema should equal( PropertyGraphSchema.empty - .withNodePropertyKeys("MyLabel")("property" -> CTString, "data" -> CTInteger.nullable) + .withNodePropertyKeys("MyLabel")( + "property" -> CTString, + "data" -> CTInteger.nullable + ) .withNodePropertyKeys("LocalLabel1")("property" -> CTString) - .withNodePropertyKeys("LocalLabel1", "LocalLabel2")("property" -> CTString) + .withNodePropertyKeys("LocalLabel1", "LocalLabel2")( + "property" -> CTString + ) .withRelationshipPropertyKeys("REL_TYPE1")("property" -> CTBoolean) .withRelationshipPropertyKeys("REL_TYPE2")() - .withSchemaPatterns(SchemaPattern(Set("MyLabel"), "REL_TYPE1", Set("LocalLabel1"))) - .withSchemaPatterns(SchemaPattern(Set("LocalLabel1", "LocalLabel2"), "REL_TYPE2", Set("MyLabel"))) + .withSchemaPatterns( + SchemaPattern(Set("MyLabel"), "REL_TYPE1", Set("LocalLabel1")) + ) + .withSchemaPatterns( + SchemaPattern( + Set("LocalLabel1", "LocalLabel2"), + "REL_TYPE2", + Set("MyLabel") + ) + ) ) } @@ -272,7 +303,9 @@ class GraphDdlConversionsTest extends BaseTestSuite { ) } - it("merges property keys for label combination based on element type hierarchy") { + it( + "merges property keys for label combination based on element type hierarchy" + ) { // Given val ddl = s"""|CREATE ELEMENT TYPE A ( foo STRING ) @@ -292,7 +325,9 @@ class GraphDdlConversionsTest extends BaseTestSuite { ) } - it("merges property keys for label combination based on element type with multi-inheritance") { + it( + "merges property keys for label combination based on element type with multi-inheritance" + ) { // Given val ddl = s"""|CREATE ELEMENT TYPE A ( a STRING ) @@ -317,9 +352,20 @@ class GraphDdlConversionsTest extends BaseTestSuite { .withNodePropertyKeys("A")("a" -> CTString) .withNodePropertyKeys("A", "B")("a" -> CTString, "b" -> CTString) .withNodePropertyKeys("A", "C")("a" -> CTString, "c" -> CTString) - .withNodePropertyKeys("A", "B", "C", "D")("a" -> CTString, "b" -> CTString, "c" -> CTString, "d" -> CTInteger) + .withNodePropertyKeys("A", "B", "C", "D")( + "a" -> CTString, + "b" -> CTString, + "c" -> CTString, + "d" -> CTInteger + ) .withNodePropertyKeys("A", "E")("a" -> CTString, "e" -> CTFloat) - .withNodePropertyKeys("A", "B", "C", "D", "E")("a" -> CTString, "b" -> CTString, "c" -> CTString, "d" -> CTInteger, "e" -> CTFloat) + .withNodePropertyKeys("A", "B", "C", "D", "E")( + "a" -> CTString, + "b" -> CTString, + "c" -> CTString, + "d" -> CTInteger, + "e" -> CTFloat + ) ) } @@ -344,8 +390,7 @@ class GraphDdlConversionsTest extends BaseTestSuite { } it("parses correct schema") { - val ddlDefinition: DdlDefinition = parseDdl( - s"""|SET SCHEMA foo.bar; + val ddlDefinition: DdlDefinition = parseDdl(s"""|SET SCHEMA foo.bar; | |CREATE ELEMENT TYPE A ( name STRING ) | @@ -380,48 +425,100 @@ class GraphDdlConversionsTest extends BaseTestSuite { |) |""".stripMargin) ddlDefinition should equalWithTracing( - DdlDefinition(List( - SetSchemaDefinition("foo", "bar"), - ElementTypeDefinition("A", properties = Map("name" -> CTString)), - ElementTypeDefinition("B", properties = Map("sequence" -> CTInteger, "nationality" -> CTString.nullable, "age" -> CTInteger.nullable)), - ElementTypeDefinition("TYPE_1"), - ElementTypeDefinition("TYPE_2", properties = Map("prop" -> CTBoolean.nullable)), - GraphTypeDefinition( - name = typeName, - statements = List( - ElementTypeDefinition("A", properties = Map("foo" -> CTInteger)), - ElementTypeDefinition("C"), - NodeTypeDefinition("A"), - NodeTypeDefinition("B"), - NodeTypeDefinition("A", "B"), - NodeTypeDefinition("C"), - RelationshipTypeDefinition("A", "TYPE_1", "B"), - RelationshipTypeDefinition("A", "B")("TYPE_2")("C") - )), - GraphDefinition( - name = graphName.value, - maybeGraphTypeName = Some(typeName), - statements = List( - NodeMappingDefinition(NodeTypeDefinition("A"), List(NodeToViewDefinition(List("foo")))), - RelationshipMappingDefinition( + DdlDefinition( + List( + SetSchemaDefinition("foo", "bar"), + ElementTypeDefinition("A", properties = Map("name" -> CTString)), + ElementTypeDefinition( + "B", + properties = Map( + "sequence" -> CTInteger, + "nationality" -> CTString.nullable, + "age" -> CTInteger.nullable + ) + ), + ElementTypeDefinition("TYPE_1"), + ElementTypeDefinition( + "TYPE_2", + properties = Map("prop" -> CTBoolean.nullable) + ), + GraphTypeDefinition( + name = typeName, + statements = List( + ElementTypeDefinition( + "A", + properties = Map("foo" -> CTInteger) + ), + ElementTypeDefinition("C"), + NodeTypeDefinition("A"), + NodeTypeDefinition("B"), + NodeTypeDefinition("A", "B"), + NodeTypeDefinition("C"), RelationshipTypeDefinition("A", "TYPE_1", "B"), - List(RelationshipTypeToViewDefinition( - viewDef = ViewDefinition(List("baz"), "edge"), - startNodeTypeToView = NodeTypeToViewDefinition( - NodeTypeDefinition("A"), - ViewDefinition(List("foo"), "alias_foo"), - JoinOnDefinition(List((List("alias_foo", "COLUMN_A"), List("edge", "COLUMN_A"))))), - endNodeTypeToView = NodeTypeToViewDefinition( - NodeTypeDefinition("B"), - ViewDefinition(List("bar"), "alias_bar"), - JoinOnDefinition(List((List("alias_bar", "COLUMN_A"), List("edge", "COLUMN_A"))))) - ))))) - )) + RelationshipTypeDefinition("A", "B")("TYPE_2")("C") + ) + ), + GraphDefinition( + name = graphName.value, + maybeGraphTypeName = Some(typeName), + statements = List( + NodeMappingDefinition( + NodeTypeDefinition("A"), + List(NodeToViewDefinition(List("foo"))) + ), + RelationshipMappingDefinition( + RelationshipTypeDefinition("A", "TYPE_1", "B"), + List( + RelationshipTypeToViewDefinition( + viewDef = ViewDefinition(List("baz"), "edge"), + startNodeTypeToView = NodeTypeToViewDefinition( + NodeTypeDefinition("A"), + ViewDefinition(List("foo"), "alias_foo"), + JoinOnDefinition( + List( + ( + List("alias_foo", "COLUMN_A"), + List("edge", "COLUMN_A") + ) + ) + ) + ), + endNodeTypeToView = NodeTypeToViewDefinition( + NodeTypeDefinition("B"), + ViewDefinition(List("bar"), "alias_bar"), + JoinOnDefinition( + List( + ( + List("alias_bar", "COLUMN_A"), + List("edge", "COLUMN_A") + ) + ) + ) + ) + ) + ) + ) + ) + ) + ) + ) ) - GraphDdl(ddlDefinition).graphs(graphName).graphType.asOkapiSchema shouldEqual PropertyGraphSchema.empty + GraphDdl(ddlDefinition) + .graphs(graphName) + .graphType + .asOkapiSchema shouldEqual PropertyGraphSchema.empty .withNodePropertyKeys("A")("foo" -> CTInteger) - .withNodePropertyKeys("B")("sequence" -> CTInteger, "nationality" -> CTString.nullable, "age" -> CTInteger.nullable) - .withNodePropertyKeys("A", "B")("foo" -> CTInteger, "sequence" -> CTInteger, "nationality" -> CTString.nullable, "age" -> CTInteger.nullable) + .withNodePropertyKeys("B")( + "sequence" -> CTInteger, + "nationality" -> CTString.nullable, + "age" -> CTInteger.nullable + ) + .withNodePropertyKeys("A", "B")( + "foo" -> CTInteger, + "sequence" -> CTInteger, + "nationality" -> CTString.nullable, + "age" -> CTInteger.nullable + ) .withNodePropertyKeys(Set("C")) .withRelationshipType("TYPE_1") .withRelationshipPropertyKeys("TYPE_2")("prop" -> CTBoolean.nullable) @@ -451,7 +548,10 @@ class GraphDdlConversionsTest extends BaseTestSuite { |) |""".stripMargin ) - GraphDdl(ddlDefinition).graphs(graphName).graphType.asOkapiSchema shouldEqual PropertyGraphSchema.empty + GraphDdl(ddlDefinition) + .graphs(graphName) + .graphType + .asOkapiSchema shouldEqual PropertyGraphSchema.empty .withNodePropertyKeys("A")("foo" -> CTInteger) .withNodePropertyKeys("B")() .withNodePropertyKeys("A", "B")("foo" -> CTInteger) @@ -483,7 +583,10 @@ class GraphDdlConversionsTest extends BaseTestSuite { |) |""".stripMargin ) - GraphDdl(ddlDefinition).graphs(graphName).graphType.asOkapiSchema shouldEqual PropertyGraphSchema.empty + GraphDdl(ddlDefinition) + .graphs(graphName) + .graphType + .asOkapiSchema shouldEqual PropertyGraphSchema.empty .withNodePropertyKeys("A")("foo" -> CTInteger) .withNodePropertyKeys("B")() .withNodePropertyKeys("A", "B")("foo" -> CTInteger) @@ -517,7 +620,10 @@ class GraphDdlConversionsTest extends BaseTestSuite { |) |""".stripMargin ) - GraphDdl(ddlDefinition).graphs(graphName).graphType.asOkapiSchema shouldEqual PropertyGraphSchema.empty + GraphDdl(ddlDefinition) + .graphs(graphName) + .graphType + .asOkapiSchema shouldEqual PropertyGraphSchema.empty .withNodePropertyKeys("A")("bar" -> CTString) .withNodePropertyKeys("B")() .withNodePropertyKeys("A", "B")("bar" -> CTString) @@ -542,7 +648,10 @@ class GraphDdlConversionsTest extends BaseTestSuite { |) |""".stripMargin ) - GraphDdl(ddlDefinition).graphs(graphName).graphType.asOkapiSchema shouldEqual PropertyGraphSchema.empty + GraphDdl(ddlDefinition) + .graphs(graphName) + .graphType + .asOkapiSchema shouldEqual PropertyGraphSchema.empty .withNodePropertyKeys("X")("c" -> CTString) } @@ -581,15 +690,55 @@ class GraphDdlConversionsTest extends BaseTestSuite { val schema = PropertyGraphSchema.empty .withNodePropertyKeys("A")("a" -> CTString) .withNodePropertyKeys("A", "B")("a" -> CTString, "b" -> CTInteger) - .withNodePropertyKeys("A", "B", "C")("a" -> CTString, "b" -> CTInteger, "c" -> CTFloat) - .withNodePropertyKeys("A", "B", "D")("a" -> CTString, "b" -> CTInteger, "d" -> CTFloat) - .withNodePropertyKeys("A", "B", "C", "D", "E")("a" -> CTString, "b" -> CTInteger, "c" -> CTFloat, "d" -> CTFloat, "e" -> CTBoolean) + .withNodePropertyKeys("A", "B", "C")( + "a" -> CTString, + "b" -> CTInteger, + "c" -> CTFloat + ) + .withNodePropertyKeys("A", "B", "D")( + "a" -> CTString, + "b" -> CTInteger, + "d" -> CTFloat + ) + .withNodePropertyKeys("A", "B", "C", "D", "E")( + "a" -> CTString, + "b" -> CTInteger, + "c" -> CTFloat, + "d" -> CTFloat, + "e" -> CTBoolean + ) val expected = GraphType.empty - .withElementType("A", "a" -> CTString, "b" -> CTInteger.nullable, "c" -> CTFloat.nullable, "d" -> CTFloat.nullable, "e" -> CTBoolean.nullable) - .withElementType("B", Set("A"), "b" -> CTInteger, "c" -> CTFloat.nullable, "d" -> CTFloat.nullable, "e" -> CTBoolean.nullable) - .withElementType("C", Set("B"), "c" -> CTFloat, "d" -> CTFloat.nullable, "e" -> CTBoolean.nullable) - .withElementType("D", Set("B"), "d" -> CTFloat, "c" -> CTFloat.nullable, "e" -> CTBoolean.nullable) + .withElementType( + "A", + "a" -> CTString, + "b" -> CTInteger.nullable, + "c" -> CTFloat.nullable, + "d" -> CTFloat.nullable, + "e" -> CTBoolean.nullable + ) + .withElementType( + "B", + Set("A"), + "b" -> CTInteger, + "c" -> CTFloat.nullable, + "d" -> CTFloat.nullable, + "e" -> CTBoolean.nullable + ) + .withElementType( + "C", + Set("B"), + "c" -> CTFloat, + "d" -> CTFloat.nullable, + "e" -> CTBoolean.nullable + ) + .withElementType( + "D", + Set("B"), + "d" -> CTFloat, + "c" -> CTFloat.nullable, + "e" -> CTBoolean.nullable + ) .withElementType("E", Set("C", "D"), "e" -> CTBoolean) .withNodeType("A") .withNodeType("A", "B") @@ -602,14 +751,24 @@ class GraphDdlConversionsTest extends BaseTestSuite { actual shouldEqual expected } - it("converts multiple node types with overlapping labels but non-overlapping properties") { + it( + "converts multiple node types with overlapping labels but non-overlapping properties" + ) { PropertyGraphSchema.empty .withNodePropertyKeys("A")("a" -> CTString) .withNodePropertyKeys("B")("b" -> CTString) .withNodePropertyKeys("A", "B")("c" -> CTString) .asGraphType shouldEqual GraphType.empty - .withElementType("A", "a" -> CTString.nullable, "c" -> CTString.nullable) - .withElementType("B", "b" -> CTString.nullable, "c" -> CTString.nullable) + .withElementType( + "A", + "a" -> CTString.nullable, + "c" -> CTString.nullable + ) + .withElementType( + "B", + "b" -> CTString.nullable, + "c" -> CTString.nullable + ) .withNodeType("A") .withNodeType("B") .withNodeType("A", "B") diff --git a/morpheus-testing/src/test/scala/org/opencypher/morpheus/api/io/sql/SqlDataSourceConfigTest.scala b/morpheus-testing/src/test/scala/org/opencypher/morpheus/api/io/sql/SqlDataSourceConfigTest.scala index 1874f26e94..574e80600e 100644 --- a/morpheus-testing/src/test/scala/org/opencypher/morpheus/api/io/sql/SqlDataSourceConfigTest.scala +++ b/morpheus-testing/src/test/scala/org/opencypher/morpheus/api/io/sql/SqlDataSourceConfigTest.scala @@ -53,7 +53,9 @@ class SqlDataSourceConfigTest extends AnyFunSpec with Matchers { } it("parses multiple SQL data sources") { - val jsonString = Source.fromURL(getClass.getResource("/sql/sql-data-sources.json")).mkString + val jsonString = Source + .fromURL(getClass.getResource("/sql/sql-data-sources.json")) + .mkString val config = Map( "CENSUS_ORACLE" -> Jdbc( url = "jdbc:h2:mem:CENSUS.db;INIT=CREATE SCHEMA IF NOT EXISTS CENSUS;DB_CLOSE_DELAY=30;", diff --git a/morpheus-testing/src/test/scala/org/opencypher/morpheus/api/io/sql/SqlPropertyGraphDataSourceTest.scala b/morpheus-testing/src/test/scala/org/opencypher/morpheus/api/io/sql/SqlPropertyGraphDataSourceTest.scala index ebd2ebb321..f614b94885 100644 --- a/morpheus-testing/src/test/scala/org/opencypher/morpheus/api/io/sql/SqlPropertyGraphDataSourceTest.scala +++ b/morpheus-testing/src/test/scala/org/opencypher/morpheus/api/io/sql/SqlPropertyGraphDataSourceTest.scala @@ -43,7 +43,6 @@ import org.opencypher.okapi.testing.Bag import scala.math.BigDecimal.RoundingMode - class SqlPropertyGraphDataSourceTest extends MorpheusTestSuite with HiveFixture with H2Fixture { private val dataSourceName = "fooDataSource" @@ -79,16 +78,23 @@ class SqlPropertyGraphDataSourceTest extends MorpheusTestSuite with HiveFixture sparkSession .createDataFrame(Seq(Tuple1("Alice"))) .toDF("foo") - .write.mode(SaveMode.Overwrite).saveAsTable(s"$databaseName.$fooView") + .write + .mode(SaveMode.Overwrite) + .saveAsTable(s"$databaseName.$fooView") - val ds = SqlPropertyGraphDataSource(GraphDdl(ddlString), Map(dataSourceName -> Hive)) + val ds = SqlPropertyGraphDataSource( + GraphDdl(ddlString), + Map(dataSourceName -> Hive) + ) ds.graph(fooGraphName) .cypher("MATCH (n) RETURN labels(n) AS labels, n.foo AS foo") - .records.toMaps should equal( + .records + .toMaps should equal( Bag( CypherMap("labels" -> List("Foo"), "foo" -> "Alice") - )) + ) + ) } describe("bigdecimal specifics") { @@ -108,40 +114,57 @@ class SqlPropertyGraphDataSourceTest extends MorpheusTestSuite with HiveFixture """.stripMargin ) - def testBigDecimalConversion(sourcePrecision: Int, sourceScale: Int, targetPrecision: Int, targetScale: Int): Unit = { - val sourceValue = BigDecimal(1*Math.pow(10, sourcePrecision).toInt, sourceScale) + def testBigDecimalConversion( + sourcePrecision: Int, + sourceScale: Int, + targetPrecision: Int, + targetScale: Int + ): Unit = { + val sourceValue = + BigDecimal(1 * Math.pow(10, sourcePrecision).toInt, sourceScale) val sourceType = DecimalType(sourcePrecision, sourceScale) val targetValue = sourceValue.setScale(targetScale, RoundingMode.HALF_UP) val targetType = DecimalType(targetPrecision, targetScale) - sparkSession.createDataFrame(List(Row(sourceValue)).asJava, StructType(Seq(StructField("num", sourceType)))) - .write.mode(SaveMode.Overwrite).saveAsTable(s"$databaseName.$fooView") + sparkSession + .createDataFrame( + List(Row(sourceValue)).asJava, + StructType(Seq(StructField("num", sourceType))) + ) + .write + .mode(SaveMode.Overwrite) + .saveAsTable(s"$databaseName.$fooView") - val ds = SqlPropertyGraphDataSource(graphDdl(targetPrecision, targetScale), Map(dataSourceName -> Hive)) + val ds = SqlPropertyGraphDataSource( + graphDdl(targetPrecision, targetScale), + Map(dataSourceName -> Hive) + ) - val records = ds.graph(fooGraphName) + val records = ds + .graph(fooGraphName) .cypher("MATCH (n) RETURN labels(n) AS labels, n.num AS num") .records records.toMaps should equal( Bag( CypherMap("labels" -> List("Foo"), "num" -> targetValue) - )) + ) + ) records.asMorpheus.table.df.schema.fields(1) should equal( StructField("num", targetType) ) } it("reads bigdecimals as standard properties") { - testBigDecimalConversion(10,5, 10,5) + testBigDecimalConversion(10, 5, 10, 5) } it("lifts bigdecimals with lower precision, same scale") { - testBigDecimalConversion(14,4, 15,4) + testBigDecimalConversion(14, 4, 15, 4) } it("lifts bigdecimals with lower precision, lower scale") { - testBigDecimalConversion(14,3, 15,4) + testBigDecimalConversion(14, 3, 15, 4) } it("prevents bigdecimals with greater precision") { @@ -149,26 +172,44 @@ class SqlPropertyGraphDataSourceTest extends MorpheusTestSuite with HiveFixture val value2 = BigDecimal(123456, 2) val typ = DecimalType(6, 2) - sparkSession.createDataFrame(List(Row(value1), Row(value2)).asJava, StructType(Seq(StructField("num", typ)))) - .write.mode(SaveMode.Overwrite).saveAsTable(s"$databaseName.$fooView") + sparkSession + .createDataFrame( + List(Row(value1), Row(value2)).asJava, + StructType(Seq(StructField("num", typ))) + ) + .write + .mode(SaveMode.Overwrite) + .saveAsTable(s"$databaseName.$fooView") - val ds = SqlPropertyGraphDataSource(graphDdl(5, 2), Map(dataSourceName -> Hive)) + val ds = + SqlPropertyGraphDataSource(graphDdl(5, 2), Map(dataSourceName -> Hive)) - val e = the [IllegalArgumentException] thrownBy ds.graph(fooGraphName) - e.getMessage should (include("subtype of DecimalType(5,2)") and include("DecimalType(6,2)")) + val e = the[IllegalArgumentException] thrownBy ds.graph(fooGraphName) + e.getMessage should (include("subtype of DecimalType(5,2)") and include( + "DecimalType(6,2)" + )) } it("prevents bigdecimals with greater scale") { val value = BigDecimal(12345, 3) val typ = DecimalType(5, 3) - sparkSession.createDataFrame(List(Row(value)).asJava, StructType(Seq(StructField("num", typ)))) - .write.mode(SaveMode.Overwrite).saveAsTable(s"$databaseName.$fooView") + sparkSession + .createDataFrame( + List(Row(value)).asJava, + StructType(Seq(StructField("num", typ))) + ) + .write + .mode(SaveMode.Overwrite) + .saveAsTable(s"$databaseName.$fooView") - val ds = SqlPropertyGraphDataSource(graphDdl(5, 2), Map(dataSourceName -> Hive)) + val ds = + SqlPropertyGraphDataSource(graphDdl(5, 2), Map(dataSourceName -> Hive)) - val e = the [IllegalArgumentException] thrownBy ds.graph(fooGraphName) - e.getMessage should (include("subtype of DecimalType(5,2)") and include("DecimalType(5,3)")) + val e = the[IllegalArgumentException] thrownBy ds.graph(fooGraphName) + e.getMessage should (include("subtype of DecimalType(5,2)") and include( + "DecimalType(5,3)" + )) } } @@ -192,14 +233,24 @@ class SqlPropertyGraphDataSourceTest extends MorpheusTestSuite with HiveFixture sparkSession .createDataFrame(Seq(Tuple2("Alice", 42L))) .toDF("col1", "col2") - .write.mode(SaveMode.Overwrite).mode(SaveMode.Overwrite).saveAsTable(s"$databaseName.$fooView") + .write + .mode(SaveMode.Overwrite) + .mode(SaveMode.Overwrite) + .saveAsTable(s"$databaseName.$fooView") - val ds = SqlPropertyGraphDataSource(GraphDdl(ddlString), Map(dataSourceName -> Hive)) + val ds = SqlPropertyGraphDataSource( + GraphDdl(ddlString), + Map(dataSourceName -> Hive) + ) ds.graph(fooGraphName) - .cypher("MATCH (n) RETURN labels(n) AS labels, n.key1 AS key1, n.key2 as key2") - .records.toMaps should equal( - Bag(CypherMap("labels" -> List("Foo"), "key1" -> 42L, "key2" -> "Alice"))) + .cypher( + "MATCH (n) RETURN labels(n) AS labels, n.key1 AS key1, n.key2 as key2" + ) + .records + .toMaps should equal( + Bag(CypherMap("labels" -> List("Foo"), "key1" -> 42L, "key2" -> "Alice")) + ) } it("is case insensitive on the column names") { @@ -222,14 +273,24 @@ class SqlPropertyGraphDataSourceTest extends MorpheusTestSuite with HiveFixture sparkSession .createDataFrame(Seq(Tuple2("Alice", 42L))) .toDF("COL1", "COL2") - .write.mode(SaveMode.Overwrite).mode(SaveMode.Overwrite).saveAsTable(s"$databaseName.$fooView") + .write + .mode(SaveMode.Overwrite) + .mode(SaveMode.Overwrite) + .saveAsTable(s"$databaseName.$fooView") - val ds = SqlPropertyGraphDataSource(GraphDdl(ddlString), Map(dataSourceName -> Hive)) + val ds = SqlPropertyGraphDataSource( + GraphDdl(ddlString), + Map(dataSourceName -> Hive) + ) ds.graph(fooGraphName) - .cypher("MATCH (n) RETURN labels(n) AS labels, n.key1 AS key1, n.key2 as key2") - .records.toMaps should equal( - Bag(CypherMap("labels" -> List("Foo"), "key1" -> 42L, "key2" -> "Alice"))) + .cypher( + "MATCH (n) RETURN labels(n) AS labels, n.key1 AS key1, n.key2 as key2" + ) + .records + .toMaps should equal( + Bag(CypherMap("labels" -> List("Foo"), "key1" -> 42L, "key2" -> "Alice")) + ) } it("reads nodes from multiple tables") { @@ -256,21 +317,32 @@ class SqlPropertyGraphDataSourceTest extends MorpheusTestSuite with HiveFixture sparkSession .createDataFrame(Seq(Tuple1("Alice"))) .toDF("foo") - .write.mode(SaveMode.Overwrite).saveAsTable(s"$databaseName.$fooView") + .write + .mode(SaveMode.Overwrite) + .saveAsTable(s"$databaseName.$fooView") sparkSession .createDataFrame(Seq(Tuple1(0L))) .toDF("bar") - .write.mode(SaveMode.Overwrite).saveAsTable(s"$databaseName.$barView") + .write + .mode(SaveMode.Overwrite) + .saveAsTable(s"$databaseName.$barView") - val ds = SqlPropertyGraphDataSource(GraphDdl(ddlString), Map(dataSourceName -> Hive)) + val ds = SqlPropertyGraphDataSource( + GraphDdl(ddlString), + Map(dataSourceName -> Hive) + ) ds.graph(fooGraphName) - .cypher("MATCH (n) RETURN labels(n) AS labels, n.foo AS foo, n.bar as bar") - .records.toMaps should equal( + .cypher( + "MATCH (n) RETURN labels(n) AS labels, n.foo AS foo, n.bar as bar" + ) + .records + .toMaps should equal( Bag( CypherMap("labels" -> List("Foo"), "foo" -> "Alice", "bar" -> null), CypherMap("labels" -> List("Bar"), "foo" -> null, "bar" -> 0L) - )) + ) + ) } it("reads relationships from a table") { @@ -304,30 +376,58 @@ class SqlPropertyGraphDataSourceTest extends MorpheusTestSuite with HiveFixture sparkSession .createDataFrame(Seq((0L, "Alice"))) .toDF("person_id", "person_name") - .write.mode(SaveMode.Overwrite).saveAsTable(s"$databaseName.$personView") + .write + .mode(SaveMode.Overwrite) + .saveAsTable(s"$databaseName.$personView") sparkSession .createDataFrame(Seq((1L, "1984"))) .toDF("book_id", "book_title") - .write.mode(SaveMode.Overwrite).saveAsTable(s"$databaseName.$bookView") + .write + .mode(SaveMode.Overwrite) + .saveAsTable(s"$databaseName.$bookView") sparkSession .createDataFrame(Seq((0L, 1L, 42.23))) .toDF("person", "book", "rating") - .write.mode(SaveMode.Overwrite).saveAsTable(s"$databaseName.$readsView") + .write + .mode(SaveMode.Overwrite) + .saveAsTable(s"$databaseName.$readsView") - val ds = SqlPropertyGraphDataSource(GraphDdl(ddlString), Map(dataSourceName -> Hive)) + val ds = SqlPropertyGraphDataSource( + GraphDdl(ddlString), + Map(dataSourceName -> Hive) + ) ds.graph(fooGraphName) - .cypher("MATCH (n) RETURN labels(n) AS labels, n.name AS name, n.title as title") - .records.toMaps should equal( + .cypher( + "MATCH (n) RETURN labels(n) AS labels, n.name AS name, n.title as title" + ) + .records + .toMaps should equal( Bag( - CypherMap("labels" -> List("Person"), "name" -> "Alice", "title" -> null), + CypherMap( + "labels" -> List("Person"), + "name" -> "Alice", + "title" -> null + ), CypherMap("labels" -> List("Book"), "name" -> null, "title" -> "1984") - )) + ) + ) ds.graph(fooGraphName) - .cypher("MATCH (a)-[r]->(b) RETURN type(r) AS type, a.name as name, b.title as title, r.rating as rating") - .records.toMaps should equal( - Bag(CypherMap("type" -> "READS", "name" -> "Alice", "title" -> "1984", "rating" -> 42.23))) + .cypher( + "MATCH (a)-[r]->(b) RETURN type(r) AS type, a.name as name, b.title as title, r.rating as rating" + ) + .records + .toMaps should equal( + Bag( + CypherMap( + "type" -> "READS", + "name" -> "Alice", + "title" -> "1984", + "rating" -> 42.23 + ) + ) + ) } it("reads relationships from a table with colliding column names") { @@ -354,31 +454,66 @@ class SqlPropertyGraphDataSourceTest extends MorpheusTestSuite with HiveFixture """.stripMargin sparkSession - .createDataFrame(Seq( - (0L, 23L, "startValue", "endValue"), - (1L, 42L, "startValue", "endValue") - )).repartition(1) // to keep id generation predictable + .createDataFrame( + Seq( + (0L, 23L, "startValue", "endValue"), + (1L, 42L, "startValue", "endValue") + ) + ) + .repartition(1) // to keep id generation predictable .toDF("node_id", "id", "start", "end") - .write.mode(SaveMode.Overwrite).saveAsTable(s"$databaseName.$nodesView") + .write + .mode(SaveMode.Overwrite) + .saveAsTable(s"$databaseName.$nodesView") sparkSession .createDataFrame(Seq((0L, 1L, 1984L, "startValue", "endValue"))) .toDF("source_id", "target_id", "id", "start", "end") - .write.mode(SaveMode.Overwrite).saveAsTable(s"$databaseName.$relsView") + .write + .mode(SaveMode.Overwrite) + .saveAsTable(s"$databaseName.$relsView") - val ds = SqlPropertyGraphDataSource(GraphDdl(ddlString), Map(dataSourceName -> Hive)) + val ds = SqlPropertyGraphDataSource( + GraphDdl(ddlString), + Map(dataSourceName -> Hive) + ) ds.graph(fooGraphName) - .cypher("MATCH (n) RETURN labels(n) AS labels, n.id AS id, n.start as start, n.end as end") - .records.toMaps should equal( + .cypher( + "MATCH (n) RETURN labels(n) AS labels, n.id AS id, n.start as start, n.end as end" + ) + .records + .toMaps should equal( Bag( - CypherMap("labels" -> List("Node"), "id" -> 23, "start" -> "startValue", "end" -> "endValue"), - CypherMap("labels" -> List("Node"), "id" -> 42, "start" -> "startValue", "end" -> "endValue") - )) + CypherMap( + "labels" -> List("Node"), + "id" -> 23, + "start" -> "startValue", + "end" -> "endValue" + ), + CypherMap( + "labels" -> List("Node"), + "id" -> 42, + "start" -> "startValue", + "end" -> "endValue" + ) + ) + ) ds.graph(fooGraphName) - .cypher("MATCH (a)-[r]->(b) RETURN type(r) AS type, r.id as id, r.start as start, r.end as end") - .records.toMaps should equal( - Bag(CypherMap("type" -> "REL", "id" -> 1984L, "start" -> "startValue", "end" -> "endValue"))) + .cypher( + "MATCH (a)-[r]->(b) RETURN type(r) AS type, r.id as id, r.start as start, r.end as end" + ) + .records + .toMaps should equal( + Bag( + CypherMap( + "type" -> "REL", + "id" -> 1984L, + "start" -> "startValue", + "end" -> "endValue" + ) + ) + ) } it("reads relationships from multiple tables") { @@ -415,45 +550,75 @@ class SqlPropertyGraphDataSourceTest extends MorpheusTestSuite with HiveFixture sparkSession .createDataFrame(Seq((0L, "Alice"))) .toDF("person_id", "person_name") - .write.mode(SaveMode.Overwrite).saveAsTable(s"$databaseName.$personView") + .write + .mode(SaveMode.Overwrite) + .saveAsTable(s"$databaseName.$personView") sparkSession - .createDataFrame(Seq((1L, "1984"), (2L, "Scala with Cats"))).repartition(1) // to keep id generation predictable + .createDataFrame(Seq((1L, "1984"), (2L, "Scala with Cats"))) + .repartition(1) // to keep id generation predictable .toDF("book_id", "book_title") - .write.mode(SaveMode.Overwrite).saveAsTable(s"$databaseName.$bookView") + .write + .mode(SaveMode.Overwrite) + .saveAsTable(s"$databaseName.$bookView") sparkSession .createDataFrame(Seq((0L, 1L, 13.37))) .toDF("person", "book", "rating") - .write.mode(SaveMode.Overwrite).saveAsTable(s"$databaseName.$readsView1") + .write + .mode(SaveMode.Overwrite) + .saveAsTable(s"$databaseName.$readsView1") sparkSession .createDataFrame(Seq((0L, 2L, 13.37))) .toDF("p_id", "b_id", "rates") - .write.mode(SaveMode.Overwrite).saveAsTable(s"$databaseName.$readsView2") + .write + .mode(SaveMode.Overwrite) + .saveAsTable(s"$databaseName.$readsView2") - val ds = SqlPropertyGraphDataSource(GraphDdl(ddlString), Map(dataSourceName -> Hive)) + val ds = SqlPropertyGraphDataSource( + GraphDdl(ddlString), + Map(dataSourceName -> Hive) + ) ds.graph(fooGraphName) - .cypher("MATCH (n) RETURN labels(n) AS labels, n.name AS name, n.title as title") - .records.toMaps should equal( + .cypher( + "MATCH (n) RETURN labels(n) AS labels, n.name AS name, n.title as title" + ) + .records + .toMaps should equal( Bag( - CypherMap("labels" -> List("Person"), "name" -> "Alice", "title" -> null), + CypherMap( + "labels" -> List("Person"), + "name" -> "Alice", + "title" -> null + ), CypherMap("labels" -> List("Book"), "name" -> null, "title" -> "1984"), - CypherMap("labels" -> List("Book"), "name" -> null, "title" -> "Scala with Cats") - )) + CypherMap( + "labels" -> List("Book"), + "name" -> null, + "title" -> "Scala with Cats" + ) + ) + ) ds.graph(fooGraphName) .cypher("MATCH ()-[r]->() RETURN type(r) AS type, r.rating as rating") - .records.toMaps should equal( + .records + .toMaps should equal( Bag( CypherMap("type" -> "READS", "rating" -> 13.37), CypherMap("type" -> "READS", "rating" -> 13.37) - )) + ) + ) ds.graph(fooGraphName) - .cypher("MATCH ()-[r]->() WITH type(r) AS type, id(r) AS id RETURN count(distinct id) AS distinct_ids") - .records.toMaps should equal( + .cypher( + "MATCH ()-[r]->() WITH type(r) AS type, id(r) AS id RETURN count(distinct id) AS distinct_ids" + ) + .records + .toMaps should equal( Bag( CypherMap("distinct_ids" -> 2) - )) + ) + ) } it("reads nodes from multiple data sources") { @@ -479,11 +644,15 @@ class SqlPropertyGraphDataSourceTest extends MorpheusTestSuite with HiveFixture sparkSession .createDataFrame(Seq(Tuple1("Alice"))) .toDF("foo") - .write.mode(SaveMode.Overwrite).saveAsTable(s"db1.$fooView") + .write + .mode(SaveMode.Overwrite) + .saveAsTable(s"db1.$fooView") sparkSession .createDataFrame(Seq(Tuple1(0L))) .toDF("bar") - .write.mode(SaveMode.Overwrite).saveAsTable(s"db2.$barView") + .write + .mode(SaveMode.Overwrite) + .saveAsTable(s"db2.$barView") val configs = Map( "ds1" -> Hive, @@ -492,12 +661,16 @@ class SqlPropertyGraphDataSourceTest extends MorpheusTestSuite with HiveFixture val ds = SqlPropertyGraphDataSource(GraphDdl(ddlString), configs) ds.graph(fooGraphName) - .cypher("MATCH (n) RETURN labels(n) AS labels, n.foo AS foo, n.bar AS bar") - .records.toMaps should equal( + .cypher( + "MATCH (n) RETURN labels(n) AS labels, n.foo AS foo, n.bar AS bar" + ) + .records + .toMaps should equal( Bag( CypherMap("labels" -> List("Foo"), "foo" -> "Alice", "bar" -> null), CypherMap("labels" -> List("Bar"), "foo" -> null, "bar" -> 0L) - )) + ) + ) } it("reads nodes from hive and h2 data sources") { @@ -528,7 +701,9 @@ class SqlPropertyGraphDataSourceTest extends MorpheusTestSuite with HiveFixture sparkSession .createDataFrame(Seq(Tuple1("Alice"))) .toDF("foo") - .write.mode(SaveMode.Overwrite).saveAsTable(s"schema1.$fooView") + .write + .mode(SaveMode.Overwrite) + .saveAsTable(s"schema1.$fooView") // -- Add h2 data @@ -540,15 +715,22 @@ class SqlPropertyGraphDataSourceTest extends MorpheusTestSuite with HiveFixture // -- Read graph and validate - val ds = SqlPropertyGraphDataSource(GraphDdl(ddlString), Map("ds1" -> hiveDataSourceConfig, "ds2" -> h2DataSourceConfig)) + val ds = SqlPropertyGraphDataSource( + GraphDdl(ddlString), + Map("ds1" -> hiveDataSourceConfig, "ds2" -> h2DataSourceConfig) + ) ds.graph(fooGraphName) - .cypher("MATCH (n) RETURN labels(n) AS labels, n.foo AS foo, n.bar as bar") - .records.toMaps should equal( + .cypher( + "MATCH (n) RETURN labels(n) AS labels, n.foo AS foo, n.bar as bar" + ) + .records + .toMaps should equal( Bag( CypherMap("labels" -> List("Foo"), "foo" -> "Alice", "bar" -> null), CypherMap("labels" -> List("Bar"), "foo" -> null, "bar" -> 123L) - )) + ) + ) } it("should not auto-cast IntegerType columns to LongType") { @@ -556,7 +738,12 @@ class SqlPropertyGraphDataSourceTest extends MorpheusTestSuite with HiveFixture Row(1, 10L), Row(15, 800L) ).asJava - val df = sparkSession.createDataFrame(data, StructType(Seq(StructField("int", IntegerType), StructField("long", LongType)))) + val df = sparkSession.createDataFrame( + data, + StructType( + Seq(StructField("int", IntegerType), StructField("long", LongType)) + ) + ) morpheus.sql("CREATE DATABASE IF NOT EXISTS db") df.write.saveAsTable("db.int_long") @@ -573,12 +760,19 @@ class SqlPropertyGraphDataSourceTest extends MorpheusTestSuite with HiveFixture |) """.stripMargin - val pgds = SqlPropertyGraphDataSource(GraphDdl(ddlString), Map("ds1" -> Hive)) + val pgds = + SqlPropertyGraphDataSource(GraphDdl(ddlString), Map("ds1" -> Hive)) - pgds.graph(GraphName("fooGraph")).cypher("MATCH (n) RETURN n.int, n.long").records.toMaps should equal(Bag( - CypherMap("n.int" -> 1, "n.long" -> 10), - CypherMap("n.int" -> 15, "n.long" -> 800) - )) + pgds + .graph(GraphName("fooGraph")) + .cypher("MATCH (n) RETURN n.int, n.long") + .records + .toMaps should equal( + Bag( + CypherMap("n.int" -> 1, "n.long" -> 10), + CypherMap("n.int" -> 15, "n.long" -> 800) + ) + ) } it("should give good error message on bad SqlDataSource config") { @@ -590,10 +784,16 @@ class SqlPropertyGraphDataSourceTest extends MorpheusTestSuite with HiveFixture |) """.stripMargin - val pgds = SqlPropertyGraphDataSource(GraphDdl(ddlString), Map("known1" -> Hive, "known2" -> Hive)) + val pgds = SqlPropertyGraphDataSource( + GraphDdl(ddlString), + Map("known1" -> Hive, "known2" -> Hive) + ) - val e = the[SqlDataSourceConfigException] thrownBy pgds.graph(GraphName("g")) - e.getMessage should (include("unknown") and include("known1") and include("known2")) + val e = + the[SqlDataSourceConfigException] thrownBy pgds.graph(GraphName("g")) + e.getMessage should (include("unknown") and include("known1") and include( + "known2" + )) } it("reads nodes and rels from file-based sources") { @@ -615,28 +815,38 @@ class SqlPropertyGraphDataSourceTest extends MorpheusTestSuite with HiveFixture // -- Read graph and validate val ds = SqlPropertyGraphDataSource( GraphDdl(ddlString), - Map("parquet" -> File( - format = FileFormat.parquet, - basePath = Some("file://" + getClass.getResource("/parquet").getPath) - )) + Map( + "parquet" -> File( + format = FileFormat.parquet, + basePath = Some("file://" + getClass.getResource("/parquet").getPath) + ) + ) ) ds.graph(fooGraphName) - .cypher("MATCH (n) RETURN n.id AS id, labels(n) AS labels, n.name AS name") - .records.toMaps should equal( + .cypher( + "MATCH (n) RETURN n.id AS id, labels(n) AS labels, n.name AS name" + ) + .records + .toMaps should equal( Bag( CypherMap("id" -> 1, "labels" -> List("Person"), "name" -> "Alice"), CypherMap("id" -> 2, "labels" -> List("Person"), "name" -> "Bob"), CypherMap("id" -> 3, "labels" -> List("Person"), "name" -> "Eve") - )) + ) + ) ds.graph(fooGraphName) - .cypher("MATCH (a)-[r]->(b) RETURN type(r) AS type, a.id as startId, b.id as endId") - .records.toMaps should equal( + .cypher( + "MATCH (a)-[r]->(b) RETURN type(r) AS type, a.id as startId, b.id as endId" + ) + .records + .toMaps should equal( Bag( CypherMap("type" -> "KNOWS", "startId" -> 1, "endId" -> 2), CypherMap("type" -> "KNOWS", "startId" -> 2, "endId" -> 3) - )) + ) + ) } it("reads nodes and rels from file-based sources with absolute paths") { @@ -659,42 +869,55 @@ class SqlPropertyGraphDataSourceTest extends MorpheusTestSuite with HiveFixture // -- Read graph and validate val ds = SqlPropertyGraphDataSource( GraphDdl(ddlString), - Map("parquet" -> File( - format = FileFormat.parquet - )) + Map( + "parquet" -> File( + format = FileFormat.parquet + ) + ) ) ds.graph(fooGraphName) - .cypher("MATCH (n) RETURN n.id AS id, labels(n) AS labels, n.name AS name") - .records.toMaps should equal( + .cypher( + "MATCH (n) RETURN n.id AS id, labels(n) AS labels, n.name AS name" + ) + .records + .toMaps should equal( Bag( CypherMap("id" -> 1, "labels" -> List("Person"), "name" -> "Alice"), CypherMap("id" -> 2, "labels" -> List("Person"), "name" -> "Bob"), CypherMap("id" -> 3, "labels" -> List("Person"), "name" -> "Eve") - )) + ) + ) ds.graph(fooGraphName) - .cypher("MATCH (a)-[r]->(b) RETURN type(r) AS type, a.id as startId, b.id as endId") - .records.toMaps should equal( + .cypher( + "MATCH (a)-[r]->(b) RETURN type(r) AS type, a.id as startId, b.id as endId" + ) + .records + .toMaps should equal( Bag( CypherMap("type" -> "KNOWS", "startId" -> 1, "endId" -> 2), CypherMap("type" -> "KNOWS", "startId" -> 2, "endId" -> 3) - )) + ) + ) } - describe("Failure handling") { it("does not support reading from csv files") { val e = the[IllegalArgumentException] thrownBy { SqlPropertyGraphDataSource( GraphDdl.apply(""), - Map("IllegalDataSource" -> File( - format = FileFormat.csv - )) + Map( + "IllegalDataSource" -> File( + format = FileFormat.csv + ) + ) ) } - e.getMessage should (include(FileFormat.csv.toString) and include("IllegalDataSource")) + e.getMessage should (include(FileFormat.csv.toString) and include( + "IllegalDataSource" + )) } it("does not support relationship types with more than one label") { @@ -709,12 +932,16 @@ class SqlPropertyGraphDataSourceTest extends MorpheusTestSuite with HiveFixture |) |CREATE GRAPH fooGraph OF fooSchema ()""".stripMargin - val e = the[IllegalArgumentException] thrownBy SqlPropertyGraphDataSource(GraphDdl(ddlString), Map(dataSourceName -> Hive)).graph(GraphName("fooGraph")) - e.getMessage should (include("(A)-[A,B]->(A)") and include("single label")) + val e = the[IllegalArgumentException] thrownBy SqlPropertyGraphDataSource( + GraphDdl(ddlString), + Map(dataSourceName -> Hive) + ).graph(GraphName("fooGraph")) + e.getMessage should (include("(A)-[A,B]->(A)") and include( + "single label" + )) } } - describe("NodeRelPattern support") { val personView = "person_view" val cityView = "city_view" @@ -723,11 +950,15 @@ class SqlPropertyGraphDataSourceTest extends MorpheusTestSuite with HiveFixture sparkSession .createDataFrame(Seq((0L, "Alice", 1L, 2010))) .toDF("person_id", "person_name", "city_id", "since") - .write.mode(SaveMode.Overwrite).saveAsTable(s"$databaseName.$personView") + .write + .mode(SaveMode.Overwrite) + .saveAsTable(s"$databaseName.$personView") sparkSession .createDataFrame(Seq((1L, "Leipzig"))) .toDF("city_id", "city_name") - .write.mode(SaveMode.Overwrite).saveAsTable(s"$databaseName.$cityView") + .write + .mode(SaveMode.Overwrite) + .saveAsTable(s"$databaseName.$cityView") } it("reads NodeRelPatterns") { @@ -756,7 +987,10 @@ class SqlPropertyGraphDataSourceTest extends MorpheusTestSuite with HiveFixture prepareData() - val ds = SqlPropertyGraphDataSource(GraphDdl(ddlString), Map(dataSourceName -> Hive)) + val ds = SqlPropertyGraphDataSource( + GraphDdl(ddlString), + Map(dataSourceName -> Hive) + ) val graph = ds.graph(fooGraphName) @@ -766,15 +1000,20 @@ class SqlPropertyGraphDataSourceTest extends MorpheusTestSuite with HiveFixture graph.asMorpheus.scanOperator(pattern, true).table.df.count() should be(1) - val result = graph.cypher( - """ + val result = graph.cypher(""" |MATCH (p:Person)-[l:LIVES_IN]->(c:City) |RETURN p.name, l.since, c.name """.stripMargin) - result.records.toMaps should equal(Bag( - CypherMap("p.name" -> "Alice", "l.since" -> 2010, "c.name" -> "Leipzig") - )) + result.records.toMaps should equal( + Bag( + CypherMap( + "p.name" -> "Alice", + "l.since" -> 2010, + "c.name" -> "Leipzig" + ) + ) + ) } it("handles naming conflicts") { @@ -803,19 +1042,23 @@ class SqlPropertyGraphDataSourceTest extends MorpheusTestSuite with HiveFixture prepareData() - val ds = SqlPropertyGraphDataSource(GraphDdl(ddlString), Map(dataSourceName -> Hive)) + val ds = SqlPropertyGraphDataSource( + GraphDdl(ddlString), + Map(dataSourceName -> Hive) + ) val graph = ds.graph(fooGraphName) - val result = graph.cypher( - """ + val result = graph.cypher(""" |MATCH (p:Person)-[l:LIVES_IN]->(c:City) |RETURN p.since, l.since """.stripMargin) - result.records.toMaps should equal(Bag( - CypherMap("p.since" -> 2010, "l.since" -> 2010) - )) + result.records.toMaps should equal( + Bag( + CypherMap("p.since" -> 2010, "l.since" -> 2010) + ) + ) } } } diff --git a/morpheus-testing/src/test/scala/org/opencypher/morpheus/api/io/sql/util/DdlUtils.scala b/morpheus-testing/src/test/scala/org/opencypher/morpheus/api/io/sql/util/DdlUtils.scala index 2698c13803..9cb00c236c 100644 --- a/morpheus-testing/src/test/scala/org/opencypher/morpheus/api/io/sql/util/DdlUtils.scala +++ b/morpheus-testing/src/test/scala/org/opencypher/morpheus/api/io/sql/util/DdlUtils.scala @@ -1,34 +1,35 @@ /** * Copyright (c) 2016-2019 "Neo4j Sweden, AB" [https://neo4j.com] * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and + * Unless required by applicable law or agreed to in writing, software distributed under the + * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing permissions and * limitations under the License. * * Attribution Notice under the terms of the Apache License 2.0 * - * This work was created by the collective efforts of the openCypher community. - * Without limiting the terms of Section 6, any Derivative Work that is not - * approved by the public consensus process of the openCypher Implementers Group - * should not be described as “Cypher” (and Cypher® is a registered trademark of - * Neo4j Inc.) or as "openCypher". Extensions by implementers or prototypes or - * proposals for change that have been documented or implemented should only be - * described as "implementation extensions to Cypher" or as "proposed changes to - * Cypher that are not yet approved by the openCypher community". + * This work was created by the collective efforts of the openCypher community. Without limiting + * the terms of Section 6, any Derivative Work that is not approved by the public consensus process + * of the openCypher Implementers Group should not be described as “Cypher” (and Cypher® is a + * registered trademark of Neo4j Inc.) or as "openCypher". Extensions by implementers or prototypes + * or proposals for change that have been documented or implemented should only be described as + * "implementation extensions to Cypher" or as "proposed changes to Cypher that are not yet + * approved by the openCypher community". */ package org.opencypher.morpheus.api.io.sql.util import org.opencypher.graphddl._ import org.opencypher.morpheus.api.MorpheusSession -import org.opencypher.morpheus.api.io.fs.DefaultGraphDirectoryStructure.{concatDirectoryNames, nodeTableDirectoryName, relKeyTableDirectoryName} +import org.opencypher.morpheus.api.io.fs.DefaultGraphDirectoryStructure.{ + concatDirectoryNames, + nodeTableDirectoryName, + relKeyTableDirectoryName +} import org.opencypher.morpheus.api.io.{GraphElement, Relationship} import org.opencypher.okapi.api.graph.{GraphName, PropertyGraph} @@ -43,20 +44,37 @@ object DdlUtils { maybeSchemaDefinition: Option[SetSchemaDefinition] = None )(implicit morpheus: MorpheusSession): GraphDdl = { val schema = graphType - val pathPrefixParts: List[String] = List(maybeDataSourceName, maybeDatabaseName).flatten - val joinFromStartNode: List[Join] = List(Join(GraphElement.sourceIdKey, Relationship.sourceStartNodeKey)) - val joinFromEndNode: List[Join] = List(Join(GraphElement.sourceIdKey, Relationship.sourceEndNodeKey)) + val pathPrefixParts: List[String] = + List(maybeDataSourceName, maybeDatabaseName).flatten + val joinFromStartNode: List[Join] = List( + Join(GraphElement.sourceIdKey, Relationship.sourceStartNodeKey) + ) + val joinFromEndNode: List[Join] = List( + Join(GraphElement.sourceIdKey, Relationship.sourceEndNodeKey) + ) val tablePrefix = graphName.value.replaceAll("\\.", "_") def nodeViewId(labelCombination: Set[String]): ViewId = { - ViewId(maybeSchemaDefinition, pathPrefixParts :+ nodeTableDirectoryName(labelCombination + tablePrefix)) + ViewId( + maybeSchemaDefinition, + pathPrefixParts :+ nodeTableDirectoryName( + labelCombination + tablePrefix + ) + ) } - def relViewId(sourceLabelCombination: Set[String], relType: String, targetLabelCombination: Set[String]): ViewId = { - val relTypeDir = concatDirectoryNames(Seq( - nodeTableDirectoryName(sourceLabelCombination), - relKeyTableDirectoryName(tablePrefix + relType), - nodeTableDirectoryName(targetLabelCombination))) + def relViewId( + sourceLabelCombination: Set[String], + relType: String, + targetLabelCombination: Set[String] + ): ViewId = { + val relTypeDir = concatDirectoryNames( + Seq( + nodeTableDirectoryName(sourceLabelCombination), + relKeyTableDirectoryName(tablePrefix + relType), + nodeTableDirectoryName(targetLabelCombination) + ) + ) ViewId(maybeSchemaDefinition, pathPrefixParts :+ relTypeDir) } @@ -65,30 +83,50 @@ object DdlUtils { } val nodeToViewMappings: Map[NodeViewKey, NodeToViewMapping] = { - schema.nodeTypes.map { nodeType => - NodeToViewMapping( - nodeType, - nodeViewId(nodeType.labels), - schema.nodePropertyKeys(nodeType).keySet.map(k => k -> k).toMap) - }.map(mapping => mapping.key -> mapping).toMap + schema.nodeTypes + .map { nodeType => + NodeToViewMapping( + nodeType, + nodeViewId(nodeType.labels), + schema.nodePropertyKeys(nodeType).keySet.map(k => k -> k).toMap + ) + } + .map(mapping => mapping.key -> mapping) + .toMap } val edgeToViewMappings: List[EdgeToViewMapping] = schema.relTypes.map { relType => - val relKeyMapping = schema.relationshipPropertyKeys(relType).keySet.map(k => k -> k).toMap + val relKeyMapping = schema + .relationshipPropertyKeys(relType) + .keySet + .map(k => k -> k) + .toMap EdgeToViewMapping( relType, - relViewId(relType.startNodeType.labels, relType.labels.head, relType.endNodeType.labels), - StartNode(nodeViewKey(relType.startNodeType.labels), joinFromStartNode), + relViewId( + relType.startNodeType.labels, + relType.labels.head, + relType.endNodeType.labels + ), + StartNode( + nodeViewKey(relType.startNodeType.labels), + joinFromStartNode + ), EndNode(nodeViewKey(relType.endNodeType.labels), joinFromEndNode), - relKeyMapping) + relKeyMapping + ) }.toList - GraphDdl(Map(graphName -> Graph( - graphName, - graphType, - nodeToViewMappings, - edgeToViewMappings - ))) + GraphDdl( + Map( + graphName -> Graph( + graphName, + graphType, + nodeToViewMappings, + edgeToViewMappings + ) + ) + ) } } } diff --git a/morpheus-testing/src/test/scala/org/opencypher/morpheus/api/io/util/StringEncodingUtilitiesTest.scala b/morpheus-testing/src/test/scala/org/opencypher/morpheus/api/io/util/StringEncodingUtilitiesTest.scala index ab01a963a4..182cb0f710 100644 --- a/morpheus-testing/src/test/scala/org/opencypher/morpheus/api/io/util/StringEncodingUtilitiesTest.scala +++ b/morpheus-testing/src/test/scala/org/opencypher/morpheus/api/io/util/StringEncodingUtilitiesTest.scala @@ -32,7 +32,9 @@ import org.scalatestplus.scalacheck.ScalaCheckDrivenPropertyChecks class StringEncodingUtilitiesTest extends BaseTestSuite with ScalaCheckDrivenPropertyChecks { - it("encodes arbitrary strings with only letters, digits, underscores, hashes, and 'at' symbols") { + it( + "encodes arbitrary strings with only letters, digits, underscores, hashes, and 'at' symbols" + ) { forAll { s: String => val encoded = s.encodeSpecialCharacters val decoded = encoded.decodeSpecialCharacters diff --git a/morpheus-testing/src/test/scala/org/opencypher/morpheus/api/util/ZeppelinSupportTest.scala b/morpheus-testing/src/test/scala/org/opencypher/morpheus/api/util/ZeppelinSupportTest.scala index ada6a73cbf..4e6e5fc5c8 100644 --- a/morpheus-testing/src/test/scala/org/opencypher/morpheus/api/util/ZeppelinSupportTest.scala +++ b/morpheus-testing/src/test/scala/org/opencypher/morpheus/api/util/ZeppelinSupportTest.scala @@ -51,8 +51,7 @@ class ZeppelinSupportTest extends MorpheusTestSuite with TeamDataFixture with Sc // scalastyle:on line.contains.tab it("can render a graph from records") { - val graph = initGraph( - """ + val graph = initGraph(""" |CREATE (a:Person {val1: 1, val2: "foo"}) |CREATE (b:Person:Swedish {val1: 2, val2: "bar"}) |CREATE (c:Person {val1: 3, val2: "baz"}) @@ -60,7 +59,9 @@ class ZeppelinSupportTest extends MorpheusTestSuite with TeamDataFixture with Sc |CREATE (a)-[:KNOWS {since: 2016}]->(c) |CREATE (b)-[:KNOWS {since: 2017}]->(c) """.stripMargin) - val result = graph.cypher("MATCH (p:Person)-[k:KNOWS]->(p2:Person) RETURN p, k, p2 ORDER BY p.val1, k.since") + val result = graph.cypher( + "MATCH (p:Person)-[k:KNOWS]->(p2:Person) RETURN p, k, p2 ORDER BY p.val1, k.since" + ) val asGraph = result.records.toZeppelinGraph @@ -149,7 +150,10 @@ class ZeppelinSupportTest extends MorpheusTestSuite with TeamDataFixture with Sc def sorted(v: ujson.Value): ujson.Value = v match { case ujson.Obj(x) => val res = ujson.Obj() - x.mapValues(sorted(_)).toSeq.sortBy(_._1).foreach(e => res.value.put(e._1, e._2)) + x.mapValues(sorted(_)) + .toSeq + .sortBy(_._1) + .foreach(e => res.value.put(e._1, e._2)) res case ujson.Arr(x) => val res = ujson.Arr() @@ -159,10 +163,16 @@ class ZeppelinSupportTest extends MorpheusTestSuite with TeamDataFixture with Sc } it("supports Zeppelin network representation") { - val graph = morpheus.graphs.create(personTable, bookTable, readsTable, knowsTable, influencesTable) - val asJson = graph.toZeppelinJson()(CypherValue.Format.defaultValueFormatter) - val expected = ujson.read( - s""" + val graph = morpheus.graphs.create( + personTable, + bookTable, + readsTable, + knowsTable, + influencesTable + ) + val asJson = + graph.toZeppelinJson()(CypherValue.Format.defaultValueFormatter) + val expected = ujson.read(s""" |{ | "nodes": [ | { diff --git a/morpheus-testing/src/test/scala/org/opencypher/morpheus/api/value/MorpheusValueConversionTest.scala b/morpheus-testing/src/test/scala/org/opencypher/morpheus/api/value/MorpheusValueConversionTest.scala index ad39e328f3..24db738e82 100644 --- a/morpheus-testing/src/test/scala/org/opencypher/morpheus/api/value/MorpheusValueConversionTest.scala +++ b/morpheus-testing/src/test/scala/org/opencypher/morpheus/api/value/MorpheusValueConversionTest.scala @@ -58,7 +58,14 @@ class MorpheusValueConversionTest extends MorpheusValueTestSuite { val newValues = scalaValues.map(CypherValue(_)) assert(newValues == originalValues) originalValues.foreach { v => - assert(v.isOrContainsNull == (v == CypherNull || v.as[CypherMap].get.unwrap.valuesIterator.contains(null))) + assert( + v.isOrContainsNull == (v == CypherNull || v + .as[CypherMap] + .get + .unwrap + .valuesIterator + .contains(null)) + ) } } @@ -66,12 +73,18 @@ class MorpheusValueConversionTest extends MorpheusValueTestSuite { val originalValues = LIST_valueGroups.flatten val scalaValues = originalValues.map(_.unwrap) val newValues = scalaValues.map { - case null => CypherNull + case null => CypherNull case l: List[_] => CypherList(l: _*) } assert(newValues == originalValues) originalValues.foreach { v => - assert(v.isOrContainsNull == v.isNull || v.as[CypherList].get.unwrap.contains(null)) + assert( + v.isOrContainsNull == v.isNull || v + .as[CypherList] + .get + .unwrap + .contains(null) + ) } } @@ -79,7 +92,7 @@ class MorpheusValueConversionTest extends MorpheusValueTestSuite { val originalValues = STRING_valueGroups.flatten val scalaValues = originalValues.map(_.unwrap) val newValues = scalaValues.map { - case null => CypherNull + case null => CypherNull case s: java.lang.String => CypherString(s) } assert(newValues == originalValues) @@ -92,7 +105,7 @@ class MorpheusValueConversionTest extends MorpheusValueTestSuite { val originalValues = BOOLEAN_valueGroups.flatten val scalaValues = originalValues.map(_.value) val newValues = scalaValues.map { - case null => CypherNull + case null => CypherNull case b: Boolean => CypherBoolean(b) } assert(newValues == originalValues) @@ -105,7 +118,7 @@ class MorpheusValueConversionTest extends MorpheusValueTestSuite { val originalValues = INTEGER_valueGroups.flatten val scalaValues = originalValues.map(_.value) val newValues = scalaValues.map { - case null => CypherNull + case null => CypherNull case l: java.lang.Long => CypherInteger(l) } assert(newValues == originalValues) @@ -118,7 +131,7 @@ class MorpheusValueConversionTest extends MorpheusValueTestSuite { val originalValues = FLOAT_valueGroups.flatten val scalaValues = originalValues.map(_.value) val newValues = scalaValues.map { - case null => CypherNull + case null => CypherNull case d: java.lang.Double => CypherFloat(d) } assert(newValues.withoutNaNs == originalValues.withoutNaNs) @@ -131,7 +144,7 @@ class MorpheusValueConversionTest extends MorpheusValueTestSuite { val originalValues = BIGDECIMAL_valueGroups.flatten val scalaValues = originalValues.map(_.value) val newValues = scalaValues.map { - case null => CypherNull + case null => CypherNull case b: BigDecimal => CypherBigDecimal(b) } assert(newValues.withoutNaNs == originalValues.withoutNaNs) @@ -144,10 +157,10 @@ class MorpheusValueConversionTest extends MorpheusValueTestSuite { val originalValues = NUMBER_valueGroups.flatten val scalaValues = originalValues.map(_.value) val newValues = scalaValues.map { - case null => CypherNull - case l: java.lang.Long => CypherInteger(l) + case null => CypherNull + case l: java.lang.Long => CypherInteger(l) case d: java.lang.Double => CypherFloat(d) - case b: BigDecimal => CypherBigDecimal(b) + case b: BigDecimal => CypherBigDecimal(b) } assert(newValues.withoutNaNs == originalValues.withoutNaNs) originalValues.foreach { v => diff --git a/morpheus-testing/src/test/scala/org/opencypher/morpheus/api/value/MorpheusValueStructureTest.scala b/morpheus-testing/src/test/scala/org/opencypher/morpheus/api/value/MorpheusValueStructureTest.scala index 807ebf63f3..930e5dfa92 100644 --- a/morpheus-testing/src/test/scala/org/opencypher/morpheus/api/value/MorpheusValueStructureTest.scala +++ b/morpheus-testing/src/test/scala/org/opencypher/morpheus/api/value/MorpheusValueStructureTest.scala @@ -69,7 +69,8 @@ class MorpheusValueStructureTest extends MorpheusValueTestSuite { val reconstructedValueGroups = originalValueGroups.map { values => values.map { case CypherNull => CypherNull - case MorpheusNode(id, labels, properties) => MorpheusNode(id, labels, properties) + case MorpheusNode(id, labels, properties) => + MorpheusNode(id, labels, properties) case other => fail(s"Unexpected value $other") } } @@ -82,9 +83,9 @@ class MorpheusValueStructureTest extends MorpheusValueTestSuite { val originalValueGroups = MAP_valueGroups val reconstructedValueGroups = originalValueGroups.map { values => values.map { - case CypherNull => CypherNull + case CypherNull => CypherNull case CypherMap(m) => CypherValue(m) - case other => fail(s"Unexpected value $other") + case other => fail(s"Unexpected value $other") } } assert(reconstructedValueGroups == originalValueGroups) @@ -95,14 +96,13 @@ class MorpheusValueStructureTest extends MorpheusValueTestSuite { val actual = cypherValueGroups.map { values => values.map { case CypherMap(m) => CypherMap(m) - case other => fail(s"Unexpected value $other") + case other => fail(s"Unexpected value $other") } } assert(actual == cypherValueGroups) } } - describe("LIST") { it("constructs values") { val originalValueGroups = LIST_valueGroups @@ -110,8 +110,8 @@ class MorpheusValueStructureTest extends MorpheusValueTestSuite { val reconstructedValueGroups = originalValueGroups.map { values => values.map { case CypherList(l) => CypherList(l) - case CypherNull => CypherNull - case other => fail(s"Unexpected value $other") + case CypherNull => CypherNull + case other => fail(s"Unexpected value $other") } } reconstructedValueGroups should equal(originalValueGroups) @@ -122,7 +122,7 @@ class MorpheusValueStructureTest extends MorpheusValueTestSuite { val actual = cypherValueGroups.map { values => values.map { case CypherList(v) => CypherList(v) - case other => fail(s"Unexpected value $other") + case other => fail(s"Unexpected value $other") } } actual should equal(cypherValueGroups) @@ -135,8 +135,8 @@ class MorpheusValueStructureTest extends MorpheusValueTestSuite { val reconstructedValueGroups = originalValueGroups.map { values => values.map { case CypherString(s) => CypherString(s) - case CypherNull => CypherNull - case other => fail(s"Unexpected value $other") + case CypherNull => CypherNull + case other => fail(s"Unexpected value $other") } } reconstructedValueGroups should equal(originalValueGroups) @@ -144,7 +144,9 @@ class MorpheusValueStructureTest extends MorpheusValueTestSuite { it("Deconstruct STRING values") { val cypherValueGroups = STRING_valueGroups.materialValueGroups - val actual = cypherValueGroups.map { values => values.map { case CypherString(v) => CypherString(v) } } + val actual = cypherValueGroups.map { values => + values.map { case CypherString(v) => CypherString(v) } + } actual should equal(cypherValueGroups) CypherString.unapply(null.asInstanceOf[CypherString]) should equal(None) } @@ -156,8 +158,8 @@ class MorpheusValueStructureTest extends MorpheusValueTestSuite { val reconstructedValueGroups = originalValueGroups.map { values => values.map { case CypherBoolean(b) => CypherBoolean(b) - case CypherNull => CypherNull - case other => fail(s"Unexpected value $other") + case CypherNull => CypherNull + case other => fail(s"Unexpected value $other") } } reconstructedValueGroups should equal(originalValueGroups) @@ -165,7 +167,9 @@ class MorpheusValueStructureTest extends MorpheusValueTestSuite { it("Deconstruct BOOLEAN values") { val cypherValueGroups = BOOLEAN_valueGroups.materialValueGroups - val actual = cypherValueGroups.map { values => values.map { case CypherBoolean(v) => CypherBoolean(v) } } + val actual = cypherValueGroups.map { values => + values.map { case CypherBoolean(v) => CypherBoolean(v) } + } actual should equal(cypherValueGroups) } } @@ -176,8 +180,8 @@ class MorpheusValueStructureTest extends MorpheusValueTestSuite { val reconstructedValueGroups = originalValueGroups.map { values => values.map { case CypherInteger(l) => CypherInteger(l) - case CypherNull => CypherNull - case other => fail(s"Unexpected value $other") + case CypherNull => CypherNull + case other => fail(s"Unexpected value $other") } } reconstructedValueGroups should equal(originalValueGroups) @@ -185,7 +189,9 @@ class MorpheusValueStructureTest extends MorpheusValueTestSuite { it("Deconstruct INTEGER values") { val cypherValueGroups = INTEGER_valueGroups.materialValueGroups - val actual = cypherValueGroups.map { values => values.map { case CypherInteger(v) => CypherInteger(v) } } + val actual = cypherValueGroups.map { values => + values.map { case CypherInteger(v) => CypherInteger(v) } + } actual should equal(cypherValueGroups) } } @@ -196,16 +202,20 @@ class MorpheusValueStructureTest extends MorpheusValueTestSuite { val reconstructedValueGroups = originalValueGroups.map { values => values.map { case CypherFloat(d) => CypherFloat(d) - case CypherNull => CypherNull - case other => fail(s"Unexpected value $other") + case CypherNull => CypherNull + case other => fail(s"Unexpected value $other") } } - assert(reconstructedValueGroups.withoutNaNs == originalValueGroups.withoutNaNs) + assert( + reconstructedValueGroups.withoutNaNs == originalValueGroups.withoutNaNs + ) } it("Deconstruct FLOAT values") { val cypherValueGroups = FLOAT_valueGroups.materialValueGroups - val actual = cypherValueGroups.map { values => values.map { case CypherFloat(v) => CypherFloat(v) } } + val actual = cypherValueGroups.map { values => + values.map { case CypherFloat(v) => CypherFloat(v) } + } assert(actual.withoutNaNs == cypherValueGroups.withoutNaNs) } } @@ -216,16 +226,20 @@ class MorpheusValueStructureTest extends MorpheusValueTestSuite { val reconstructedValueGroups = originalValueGroups.map { values => values.map { case CypherBigDecimal(d) => CypherBigDecimal(d) - case CypherNull => CypherNull - case other => fail(s"Unexpected value $other") + case CypherNull => CypherNull + case other => fail(s"Unexpected value $other") } } - assert(reconstructedValueGroups.withoutNaNs == originalValueGroups.withoutNaNs) + assert( + reconstructedValueGroups.withoutNaNs == originalValueGroups.withoutNaNs + ) } it("deconstructs values") { val cypherValueGroups = BIGDECIMAL_valueGroups.materialValueGroups - val actual = cypherValueGroups.map { values => values.map { case CypherBigDecimal(v) => CypherBigDecimal(v) } } + val actual = cypherValueGroups.map { values => + values.map { case CypherBigDecimal(v) => CypherBigDecimal(v) } + } assert(actual.withoutNaNs == cypherValueGroups.withoutNaNs) } } @@ -233,17 +247,18 @@ class MorpheusValueStructureTest extends MorpheusValueTestSuite { describe("NUMBER") { it("constructs values") { val originalValueGroups = NUMBER_valueGroups - val reconstructedValueGroups = originalValueGroups.map { - values => - values.map { - case CypherNull => CypherNull - case CypherInteger(l) => CypherInteger(l) - case CypherFloat(d) => CypherFloat(d) - case CypherBigDecimal(d) => CypherBigDecimal(d) - case other => fail(s"Unexpected value $other") - } + val reconstructedValueGroups = originalValueGroups.map { values => + values.map { + case CypherNull => CypherNull + case CypherInteger(l) => CypherInteger(l) + case CypherFloat(d) => CypherFloat(d) + case CypherBigDecimal(d) => CypherBigDecimal(d) + case other => fail(s"Unexpected value $other") + } } - assert(reconstructedValueGroups.withoutNaNs == originalValueGroups.withoutNaNs) + assert( + reconstructedValueGroups.withoutNaNs == originalValueGroups.withoutNaNs + ) } it("deconstructs values") { @@ -253,29 +268,33 @@ class MorpheusValueStructureTest extends MorpheusValueTestSuite { case MorpheusNode(id, labels, properties) => CypherValue(MorpheusNode(id, labels, properties)) case MorpheusRelationship(id, source, target, relType, properties) => - CypherValue(MorpheusRelationship(id, source, target, relType, properties)) - case CypherMap(map) => CypherValue(map) - case CypherList(l) => CypherValue(l) - case CypherBoolean(b) => CypherValue(b) - case CypherString(s) => CypherString(s) - case CypherInteger(l) => CypherValue(l) - case CypherFloat(d) => CypherValue(d) + CypherValue( + MorpheusRelationship(id, source, target, relType, properties) + ) + case CypherMap(map) => CypherValue(map) + case CypherList(l) => CypherValue(l) + case CypherBoolean(b) => CypherValue(b) + case CypherString(s) => CypherString(s) + case CypherInteger(l) => CypherValue(l) + case CypherFloat(d) => CypherValue(d) case CypherBigDecimal(d) => CypherValue(d) - case CypherNull => CypherValue(null) - case other => fail(s"Unexpected value $other") + case CypherNull => CypherValue(null) + case other => fail(s"Unexpected value $other") } } - assert(reconstructedValueGroups.withoutNaNs == originalValueGroups.withoutNaNs) + assert( + reconstructedValueGroups.withoutNaNs == originalValueGroups.withoutNaNs + ) } } - describe("ANY"){ + describe("ANY") { it("deconstructs values") { val cypherValueGroups = ANY_valueGroups.materialValueGroups val actual = cypherValueGroups.map { values => values.map { case CypherValue(v) => CypherValue(v) - case other => fail(s"Unexpected value $other") + case other => fail(s"Unexpected value $other") } } assert(actual.withoutNaNs == cypherValueGroups.withoutNaNs) diff --git a/morpheus-testing/src/test/scala/org/opencypher/morpheus/api/value/MorpheusValueTestSuite.scala b/morpheus-testing/src/test/scala/org/opencypher/morpheus/api/value/MorpheusValueTestSuite.scala index d758f217a4..edf6f2b3c4 100644 --- a/morpheus-testing/src/test/scala/org/opencypher/morpheus/api/value/MorpheusValueTestSuite.scala +++ b/morpheus-testing/src/test/scala/org/opencypher/morpheus/api/value/MorpheusValueTestSuite.scala @@ -39,7 +39,7 @@ class MorpheusValueTestSuite extends BaseTestSuite with CypherValueEncoders { implicit class FilterValues(values: Seq[Any]) { def withoutNaNs: Seq[Any] = values.filter { case CypherFloat(d) => !d.isNaN - case _ => true + case _ => true } } diff --git a/morpheus-testing/src/test/scala/org/opencypher/morpheus/api/value/MorpheusValueToStringTest.scala b/morpheus-testing/src/test/scala/org/opencypher/morpheus/api/value/MorpheusValueToStringTest.scala index 0f68f944a0..7b412100fa 100644 --- a/morpheus-testing/src/test/scala/org/opencypher/morpheus/api/value/MorpheusValueToStringTest.scala +++ b/morpheus-testing/src/test/scala/org/opencypher/morpheus/api/value/MorpheusValueToStringTest.scala @@ -33,17 +33,57 @@ import org.opencypher.okapi.testing.BaseTestSuite class MorpheusValueToStringTest extends BaseTestSuite { test("node") { - MorpheusNode(1L, Set.empty[String], CypherMap.empty).toCypherString should equal("()") - MorpheusNode(1L, Set("A"), CypherMap.empty).toCypherString should equal("(:`A`)") - MorpheusNode(1L, Set("A", "B"), CypherMap.empty).toCypherString should equal("(:`A`:`B`)") - MorpheusNode(1L, Set("A", "B"), CypherMap("a" -> "b")).toCypherString should equal("(:`A`:`B` {`a`: 'b'})") - MorpheusNode(1L, Set("A", "B"), CypherMap("a" -> "b", "b" -> 1)).toCypherString should equal("(:`A`:`B` {`a`: 'b', `b`: 1})") - MorpheusNode(1L, Set.empty[String], CypherMap("a" -> "b", "b" -> 1)).toCypherString should equal("({`a`: 'b', `b`: 1})") + MorpheusNode( + 1L, + Set.empty[String], + CypherMap.empty + ).toCypherString should equal("()") + MorpheusNode(1L, Set("A"), CypherMap.empty).toCypherString should equal( + "(:`A`)" + ) + MorpheusNode( + 1L, + Set("A", "B"), + CypherMap.empty + ).toCypherString should equal("(:`A`:`B`)") + MorpheusNode( + 1L, + Set("A", "B"), + CypherMap("a" -> "b") + ).toCypherString should equal("(:`A`:`B` {`a`: 'b'})") + MorpheusNode( + 1L, + Set("A", "B"), + CypherMap("a" -> "b", "b" -> 1) + ).toCypherString should equal("(:`A`:`B` {`a`: 'b', `b`: 1})") + MorpheusNode( + 1L, + Set.empty[String], + CypherMap("a" -> "b", "b" -> 1) + ).toCypherString should equal("({`a`: 'b', `b`: 1})") } test("relationship") { - MorpheusRelationship(1L, 1L, 1L, "A", CypherMap.empty).toCypherString should equal("[:`A`]") - MorpheusRelationship(1L, 1L, 1L, "A", CypherMap("a" -> "b")).toCypherString should equal("[:`A` {`a`: 'b'}]") - MorpheusRelationship(1L, 1L, 1L, "A", CypherMap("a" -> "b", "b" -> 1)).toCypherString should equal("[:`A` {`a`: 'b', `b`: 1}]") + MorpheusRelationship( + 1L, + 1L, + 1L, + "A", + CypherMap.empty + ).toCypherString should equal("[:`A`]") + MorpheusRelationship( + 1L, + 1L, + 1L, + "A", + CypherMap("a" -> "b") + ).toCypherString should equal("[:`A` {`a`: 'b'}]") + MorpheusRelationship( + 1L, + 1L, + 1L, + "A", + CypherMap("a" -> "b", "b" -> 1) + ).toCypherString should equal("[:`A` {`a`: 'b', `b`: 1}]") } } diff --git a/morpheus-testing/src/test/scala/org/opencypher/morpheus/impl/MorpheusGraphOperationsTest.scala b/morpheus-testing/src/test/scala/org/opencypher/morpheus/impl/MorpheusGraphOperationsTest.scala index 7c34ec0611..51b0989dd1 100644 --- a/morpheus-testing/src/test/scala/org/opencypher/morpheus/impl/MorpheusGraphOperationsTest.scala +++ b/morpheus-testing/src/test/scala/org/opencypher/morpheus/impl/MorpheusGraphOperationsTest.scala @@ -34,7 +34,10 @@ import org.opencypher.okapi.api.types._ import org.opencypher.okapi.ir.api.expr._ import org.opencypher.okapi.testing.Bag -class MorpheusGraphOperationsTest extends MorpheusTestSuite with TeamDataFixture with RecordsVerificationFixture { +class MorpheusGraphOperationsTest + extends MorpheusTestSuite + with TeamDataFixture + with RecordsVerificationFixture { test("union") { val graph1 = morpheus.graphs.create(personTable, knowsTable) @@ -53,45 +56,256 @@ class MorpheusGraphOperationsTest extends MorpheusTestSuite with TeamDataFixture nHasPropertyLuckyNumber, nHasPropertyName, nHasPropertyTitle, - nHasPropertyYear) + nHasPropertyYear + ) - verify(nodeRecords, nExprs, Bag( - Row(1L.withPrefix(0).toList, false, true, false, null, 23L, "Mats", null, null), - Row(2L.withPrefix(0).toList, false, true, false, null, 42L, "Martin", null, null), - Row(3L.withPrefix(0).toList, false, true, false, null, 1337L, "Max", null, null), - Row(4L.withPrefix(0).toList, false, true, false, null, 9L, "Stefan", null, null), - Row(10L.withPrefix(1).toList, true, false, false, null, null, null, "1984", 1949L), - Row(20L.withPrefix(1).toList, true, false, false, null, null, null, "Cryptonomicon", 1999L), - Row(30L.withPrefix(1).toList, true, false, false, null, null, null, "The Eye of the World", 1990L), - Row(40L.withPrefix(1).toList, true, false, false, null, null, null, "The Circle", 2013L), - Row(100L.withPrefix(1).toList, false, true, true, "C", 42L, "Alice", null, null), - Row(200L.withPrefix(1).toList, false, true, true, "D", 23L, "Bob", null, null), - Row(300L.withPrefix(1).toList, false, true, true, "F", 84L, "Eve", null, null), - Row(400L.withPrefix(1).toList, false, true, true, "R", 49L, "Carl", null, null) - )) + verify( + nodeRecords, + nExprs, + Bag( + Row( + 1L.withPrefix(0).toList, + false, + true, + false, + null, + 23L, + "Mats", + null, + null + ), + Row( + 2L.withPrefix(0).toList, + false, + true, + false, + null, + 42L, + "Martin", + null, + null + ), + Row( + 3L.withPrefix(0).toList, + false, + true, + false, + null, + 1337L, + "Max", + null, + null + ), + Row( + 4L.withPrefix(0).toList, + false, + true, + false, + null, + 9L, + "Stefan", + null, + null + ), + Row( + 10L.withPrefix(1).toList, + true, + false, + false, + null, + null, + null, + "1984", + 1949L + ), + Row( + 20L.withPrefix(1).toList, + true, + false, + false, + null, + null, + null, + "Cryptonomicon", + 1999L + ), + Row( + 30L.withPrefix(1).toList, + true, + false, + false, + null, + null, + null, + "The Eye of the World", + 1990L + ), + Row( + 40L.withPrefix(1).toList, + true, + false, + false, + null, + null, + null, + "The Circle", + 2013L + ), + Row( + 100L.withPrefix(1).toList, + false, + true, + true, + "C", + 42L, + "Alice", + null, + null + ), + Row( + 200L.withPrefix(1).toList, + false, + true, + true, + "D", + 23L, + "Bob", + null, + null + ), + Row( + 300L.withPrefix(1).toList, + false, + true, + true, + "F", + 84L, + "Eve", + null, + null + ), + Row( + 400L.withPrefix(1).toList, + false, + true, + true, + "R", + 49L, + "Carl", + null, + null + ) + ) + ) val relRecords = result.relationships("r") - val rExprs = Seq(rStart, - r, - rHasTypeKnows, - rHasTypeReads, - rEnd, - rHasPropertyRecommends, - rHasPropertySince) + val rExprs = Seq( + rStart, + r, + rHasTypeKnows, + rHasTypeReads, + rEnd, + rHasPropertyRecommends, + rHasPropertySince + ) - verify(relRecords, rExprs, Bag( - Row(1L.withPrefix(0).toList, 1L.withPrefix(0).toList, true, false, 2L.withPrefix(0).toList, null, 2017L), - Row(1L.withPrefix(0).toList, 2L.withPrefix(0).toList, true, false, 3L.withPrefix(0).toList, null, 2016L), - Row(1L.withPrefix(0).toList, 3L.withPrefix(0).toList, true, false, 4L.withPrefix(0).toList, null, 2015L), - Row(2L.withPrefix(0).toList, 4L.withPrefix(0).toList, true, false, 3L.withPrefix(0).toList, null, 2016L), - Row(2L.withPrefix(0).toList, 5L.withPrefix(0).toList, true, false, 4L.withPrefix(0).toList, null, 2013L), - Row(3L.withPrefix(0).toList, 6L.withPrefix(0).toList, true, false, 4L.withPrefix(0).toList, null, 2016L), - Row(100L.withPrefix(1).toList, 100L.withPrefix(1).toList, false, true, 10L.withPrefix(1).toList, true, null), - Row(200L.withPrefix(1).toList, 200L.withPrefix(1).toList, false, true, 40L.withPrefix(1).toList, true, null), - Row(300L.withPrefix(1).toList, 300L.withPrefix(1).toList, false, true, 30L.withPrefix(1).toList, true, null), - Row(400L.withPrefix(1).toList, 400L.withPrefix(1).toList, false, true, 20L.withPrefix(1).toList, false, null) - )) + verify( + relRecords, + rExprs, + Bag( + Row( + 1L.withPrefix(0).toList, + 1L.withPrefix(0).toList, + true, + false, + 2L.withPrefix(0).toList, + null, + 2017L + ), + Row( + 1L.withPrefix(0).toList, + 2L.withPrefix(0).toList, + true, + false, + 3L.withPrefix(0).toList, + null, + 2016L + ), + Row( + 1L.withPrefix(0).toList, + 3L.withPrefix(0).toList, + true, + false, + 4L.withPrefix(0).toList, + null, + 2015L + ), + Row( + 2L.withPrefix(0).toList, + 4L.withPrefix(0).toList, + true, + false, + 3L.withPrefix(0).toList, + null, + 2016L + ), + Row( + 2L.withPrefix(0).toList, + 5L.withPrefix(0).toList, + true, + false, + 4L.withPrefix(0).toList, + null, + 2013L + ), + Row( + 3L.withPrefix(0).toList, + 6L.withPrefix(0).toList, + true, + false, + 4L.withPrefix(0).toList, + null, + 2016L + ), + Row( + 100L.withPrefix(1).toList, + 100L.withPrefix(1).toList, + false, + true, + 10L.withPrefix(1).toList, + true, + null + ), + Row( + 200L.withPrefix(1).toList, + 200L.withPrefix(1).toList, + false, + true, + 40L.withPrefix(1).toList, + true, + null + ), + Row( + 300L.withPrefix(1).toList, + 300L.withPrefix(1).toList, + false, + true, + 30L.withPrefix(1).toList, + true, + null + ), + Row( + 400L.withPrefix(1).toList, + 400L.withPrefix(1).toList, + false, + true, + 20L.withPrefix(1).toList, + false, + null + ) + ) + ) } } diff --git a/morpheus-testing/src/test/scala/org/opencypher/morpheus/impl/MorpheusGraphTest.scala b/morpheus-testing/src/test/scala/org/opencypher/morpheus/impl/MorpheusGraphTest.scala index 76e04a18c1..9be68da4dc 100644 --- a/morpheus-testing/src/test/scala/org/opencypher/morpheus/impl/MorpheusGraphTest.scala +++ b/morpheus-testing/src/test/scala/org/opencypher/morpheus/impl/MorpheusGraphTest.scala @@ -31,7 +31,11 @@ import org.opencypher.morpheus.api.io.MorpheusElementTable import org.opencypher.morpheus.api.value.MorpheusElement._ import org.opencypher.morpheus.impl.table.SparkTable.DataFrameTable import org.opencypher.morpheus.testing.MorpheusTestSuite -import org.opencypher.morpheus.testing.fixture.{GraphConstructionFixture, RecordsVerificationFixture, TeamDataFixture} +import org.opencypher.morpheus.testing.fixture.{ + GraphConstructionFixture, + RecordsVerificationFixture, + TeamDataFixture +} import org.opencypher.okapi.api.types._ import org.opencypher.okapi.relational.api.planning.RelationalRuntimeContext import org.opencypher.okapi.relational.api.table.RelationalCypherRecords @@ -40,16 +44,19 @@ import org.opencypher.okapi.testing.Bag import scala.reflect.runtime.universe -abstract class MorpheusGraphTest extends MorpheusTestSuite - with GraphConstructionFixture - with RecordsVerificationFixture - with TeamDataFixture { +abstract class MorpheusGraphTest + extends MorpheusTestSuite + with GraphConstructionFixture + with RecordsVerificationFixture + with TeamDataFixture { object MorpheusGraphTest { implicit class RecordOps(records: RelationalCypherRecords[DataFrameTable]) { def planStart: Start[DataFrameTable] = { - implicit val tableTypeTag: universe.TypeTag[DataFrameTable] = morpheus.tableTypeTag - implicit val context: RelationalRuntimeContext[DataFrameTable] = morpheus.basicRuntimeContext() + implicit val tableTypeTag: universe.TypeTag[DataFrameTable] = + morpheus.tableTypeTag + implicit val context: RelationalRuntimeContext[DataFrameTable] = + morpheus.basicRuntimeContext() Start.fromEmptyGraph(records) } } @@ -64,12 +71,17 @@ abstract class MorpheusGraphTest extends MorpheusTestSuite nHasPropertyLuckyNumber, nHasPropertyName ) - verify(nodes, cols, Bag(Row(4L.encodeAsMorpheusId.toList, true, 8L, "Donald"))) + verify( + nodes, + cols, + Bag(Row(4L.encodeAsMorpheusId.toList, true, 8L, "Donald")) + ) } it("should return only nodes with that exact label (multiple labels)") { val graph = initGraph(dataFixtureWithoutArrays) - val nodes = graph.nodes("n", CTNode("Person", "German"), exactLabelMatch = true) + val nodes = + graph.nodes("n", CTNode("Person", "German"), exactLabelMatch = true) val cols = Seq( n, nHasLabelGerman, @@ -87,27 +99,31 @@ abstract class MorpheusGraphTest extends MorpheusTestSuite it("should support the same node label from multiple node tables") { // this creates additional :Person nodes - val personsPart2 = morpheus.sparkSession.createDataFrame( - Seq( - (5L, false, "Soeren", 23L), - (6L, false, "Hannes", 42L)) - ).toDF("ID", "IS_SWEDE", "NAME", "NUM") + val personsPart2 = morpheus.sparkSession + .createDataFrame( + Seq((5L, false, "Soeren", 23L), (6L, false, "Hannes", 42L)) + ) + .toDF("ID", "IS_SWEDE", "NAME", "NUM") - val personTable2 = MorpheusElementTable.create(personTable.mapping, personsPart2) + val personTable2 = + MorpheusElementTable.create(personTable.mapping, personsPart2) val graph = morpheus.graphs.create(personTable, personTable2) graph.nodes("n").size shouldBe 6 } - it("should support the same relationship type from multiple relationship tables") { + it( + "should support the same relationship type from multiple relationship tables" + ) { // this creates additional :KNOWS relationships - val knowsParts2 = morpheus.sparkSession.createDataFrame( - Seq( - (1L, 7L, 2L, 2017L), - (1L, 8L, 3L, 2016L)) - ).toDF("SRC", "ID", "DST", "SINCE") + val knowsParts2 = morpheus.sparkSession + .createDataFrame( + Seq((1L, 7L, 2L, 2017L), (1L, 8L, 3L, 2016L)) + ) + .toDF("SRC", "ID", "DST", "SINCE") - val knowsTable2 = MorpheusElementTable.create(knowsTable.mapping, knowsParts2) + val knowsTable2 = + MorpheusElementTable.create(knowsTable.mapping, knowsParts2) val graph = morpheus.graphs.create(personTable, knowsTable, knowsTable2) graph.relationships("r").size shouldBe 8 diff --git a/morpheus-testing/src/test/scala/org/opencypher/morpheus/impl/MorpheusRecordsAcceptanceTest.scala b/morpheus-testing/src/test/scala/org/opencypher/morpheus/impl/MorpheusRecordsAcceptanceTest.scala index d755a7f152..f6e0856c56 100644 --- a/morpheus-testing/src/test/scala/org/opencypher/morpheus/impl/MorpheusRecordsAcceptanceTest.scala +++ b/morpheus-testing/src/test/scala/org/opencypher/morpheus/impl/MorpheusRecordsAcceptanceTest.scala @@ -40,32 +40,89 @@ import org.opencypher.okapi.testing.Bag._ import scala.language.reflectiveCalls -class MorpheusRecordsAcceptanceTest extends MorpheusTestSuite with ScanGraphInit with OpenCypherDataFixture { +class MorpheusRecordsAcceptanceTest + extends MorpheusTestSuite + with ScanGraphInit + with OpenCypherDataFixture { - private lazy val graph: RelationalCypherGraph[DataFrameTable] = initGraph(dataFixture) + private lazy val graph: RelationalCypherGraph[DataFrameTable] = initGraph( + dataFixture + ) it("convert nodes to CypherMaps") { // When - val result = graph.cypher("MATCH (a:Person) WHERE a.birthyear < 1930 RETURN a, a.name") + val result = + graph.cypher("MATCH (a:Person) WHERE a.birthyear < 1930 RETURN a, a.name") // Then - result.records.collect.toBag should equal(Bag( - CypherMap("a" -> MorpheusNode(0, Set("Actor", "Person"), CypherMap("birthyear" -> 1910, "name" -> "Rachel Kempson")), "a.name" -> "Rachel Kempson"), - CypherMap("a" -> MorpheusNode(1, Set("Actor", "Person"), CypherMap("birthyear" -> 1908, "name" -> "Michael Redgrave")), "a.name" -> "Michael Redgrave"), - CypherMap("a" -> MorpheusNode(10, Set("Actor", "Person"), CypherMap("birthyear" -> 1873, "name" -> "Roy Redgrave")), "a.name" -> "Roy Redgrave") - )) + result.records.collect.toBag should equal( + Bag( + CypherMap( + "a" -> MorpheusNode( + 0, + Set("Actor", "Person"), + CypherMap("birthyear" -> 1910, "name" -> "Rachel Kempson") + ), + "a.name" -> "Rachel Kempson" + ), + CypherMap( + "a" -> MorpheusNode( + 1, + Set("Actor", "Person"), + CypherMap("birthyear" -> 1908, "name" -> "Michael Redgrave") + ), + "a.name" -> "Michael Redgrave" + ), + CypherMap( + "a" -> MorpheusNode( + 10, + Set("Actor", "Person"), + CypherMap("birthyear" -> 1873, "name" -> "Roy Redgrave") + ), + "a.name" -> "Roy Redgrave" + ) + ) + ) } it("convert rels to CypherMaps") { // When - val result = graph.cypher("MATCH ()-[r:ACTED_IN]->() WHERE r.charactername ENDS WITH 'e' RETURN r") + val result = graph.cypher( + "MATCH ()-[r:ACTED_IN]->() WHERE r.charactername ENDS WITH 'e' RETURN r" + ) // Then - result.records.collect.toBag should equal(Bag( - CypherMap("r" -> MorpheusRelationship(46, 6, 18, "ACTED_IN", CypherMap("charactername" -> "Albus Dumbledore"))), - CypherMap("r" -> MorpheusRelationship(44, 2, 20, "ACTED_IN", CypherMap("charactername" -> "Guenevere"))), - CypherMap("r" -> MorpheusRelationship(49, 8, 19, "ACTED_IN", CypherMap("charactername" -> "Halle/Annie"))) - )) + result.records.collect.toBag should equal( + Bag( + CypherMap( + "r" -> MorpheusRelationship( + 46, + 6, + 18, + "ACTED_IN", + CypherMap("charactername" -> "Albus Dumbledore") + ) + ), + CypherMap( + "r" -> MorpheusRelationship( + 44, + 2, + 20, + "ACTED_IN", + CypherMap("charactername" -> "Guenevere") + ) + ), + CypherMap( + "r" -> MorpheusRelationship( + 49, + 8, + 19, + "ACTED_IN", + CypherMap("charactername" -> "Halle/Annie") + ) + ) + ) + ) } it("label scan and project") { @@ -74,70 +131,101 @@ class MorpheusRecordsAcceptanceTest extends MorpheusTestSuite with ScanGraphInit // Then result.records.size shouldBe 15 - result.records.collect should contain(CypherMap("a.name" -> "Rachel Kempson")) + result.records.collect should contain( + CypherMap("a.name" -> "Rachel Kempson") + ) } it("expand and project") { // When - val result = graph.cypher("MATCH (a:Actor)-[r]->(m:Film) RETURN a.birthyear, m.title") + val result = + graph.cypher("MATCH (a:Actor)-[r]->(m:Film) RETURN a.birthyear, m.title") // Then result.records.size shouldBe 8 - result.records.collect should contain(CypherMap("a.birthyear" -> 1952, "m.title" -> "Batman Begins")) + result.records.collect should contain( + CypherMap("a.birthyear" -> 1952, "m.title" -> "Batman Begins") + ) } it("filter rels on property") { // Given - val query = "MATCH (a:Actor)-[r:ACTED_IN]->() WHERE r.charactername = 'Guenevere' RETURN a, r" + val query = + "MATCH (a:Actor)-[r:ACTED_IN]->() WHERE r.charactername = 'Guenevere' RETURN a, r" // When val result = graph.cypher(query) // Then - result.records.collect.toBag should equal(Bag( - CypherMap( - "a" -> MorpheusNode(2, Set("Actor", "Person"), CypherMap("birthyear" -> 1937, "name" -> "Vanessa Redgrave")), - "r" -> MorpheusRelationship(44, 2, 20, "ACTED_IN", CypherMap("charactername" -> "Guenevere")) + result.records.collect.toBag should equal( + Bag( + CypherMap( + "a" -> MorpheusNode( + 2, + Set("Actor", "Person"), + CypherMap("birthyear" -> 1937, "name" -> "Vanessa Redgrave") + ), + "r" -> MorpheusRelationship( + 44, + 2, + 20, + "ACTED_IN", + CypherMap("charactername" -> "Guenevere") + ) + ) ) - )) + ) } it("filter nodes on property") { // When - val result = graph.cypher("MATCH (p:Person) WHERE p.birthyear = 1970 RETURN p.name") + val result = + graph.cypher("MATCH (p:Person) WHERE p.birthyear = 1970 RETURN p.name") // Then - result.records.collect.toBag should equal(Bag( - CypherMap("p.name" -> "Fake Bar"), - CypherMap("p.name" -> "Fake Foo"), - CypherMap("p.name" -> "Christopher Nolan") - )) + result.records.collect.toBag should equal( + Bag( + CypherMap("p.name" -> "Fake Bar"), + CypherMap("p.name" -> "Fake Foo"), + CypherMap("p.name" -> "Christopher Nolan") + ) + ) } it("expand and project, three properties") { // Given - val query = "MATCH (a:Actor)-[:ACTED_IN]->(f:Film) RETURN a.name, f.title, a.birthyear" + val query = + "MATCH (a:Actor)-[:ACTED_IN]->(f:Film) RETURN a.name, f.title, a.birthyear" // When val result = graph.cypher(query) // Then result.records.size shouldBe 8 - result.records.collect should contain(CypherMap("a.name" -> "Natasha Richardson", "f.title" -> "The Parent Trap", "a.birthyear" -> 1963)) + result.records.collect should contain( + CypherMap( + "a.name" -> "Natasha Richardson", + "f.title" -> "The Parent Trap", + "a.birthyear" -> 1963 + ) + ) } it("multiple hops of expand with different reltypes") { // Given - val query = "MATCH (c:City)<-[:BORN_IN]-(a:Actor)-[r:ACTED_IN]->(f:Film) RETURN a.name, c.name, f.title" + val query = + "MATCH (c:City)<-[:BORN_IN]-(a:Actor)-[r:ACTED_IN]->(f:Film) RETURN a.name, c.name, f.title" // When val records = graph.cypher(query).records - records.asMorpheus.df.collect().toBag should equal(Bag( + records.asMorpheus.df.collect().toBag should equal( + Bag( Row("Natasha Richardson", "London", "The Parent Trap"), Row("Dennis Quaid", "Houston", "The Parent Trap"), Row("Lindsay Lohan", "New York", "The Parent Trap"), Row("Vanessa Redgrave", "London", "Camelot") - )) + ) + ) } } diff --git a/morpheus-testing/src/test/scala/org/opencypher/morpheus/impl/MorpheusRecordsPrinterTest.scala b/morpheus-testing/src/test/scala/org/opencypher/morpheus/impl/MorpheusRecordsPrinterTest.scala index 55840d9089..a4baa5bb93 100644 --- a/morpheus-testing/src/test/scala/org/opencypher/morpheus/impl/MorpheusRecordsPrinterTest.scala +++ b/morpheus-testing/src/test/scala/org/opencypher/morpheus/impl/MorpheusRecordsPrinterTest.scala @@ -69,7 +69,8 @@ class MorpheusRecordsPrinterTest extends MorpheusTestSuite with GraphConstructio val header = RecordHeader.from(NodeVar("foo")(CTNode)) val emptyDf = morpheus.sparkSession.createDataFrame( Collections.emptyList[Row](), - StructType(Seq(StructField(header.column(NodeVar("foo")()), LongType)))) + StructType(Seq(StructField(header.column(NodeVar("foo")()), LongType))) + ) val records = MorpheusRecords(header, emptyDf, Some(Seq("foo"))) // When @@ -87,7 +88,9 @@ class MorpheusRecordsPrinterTest extends MorpheusTestSuite with GraphConstructio it("prints a single column with three rows") { // Given - val df = sparkSession.createDataFrame(Seq(Row1("myString"), Row1("foo"), Row1(null))).toDF("foo") + val df = sparkSession + .createDataFrame(Seq(Row1("myString"), Row1("foo"), Row1(null))) + .toDF("foo") val records = morpheus.records.wrap(df) // When @@ -109,11 +112,15 @@ class MorpheusRecordsPrinterTest extends MorpheusTestSuite with GraphConstructio it("prints three columns with three rows") { // Given - val df = sparkSession.createDataFrame(Seq( - Row3("myString", 4L, false), - Row3("foo", 99999999L, true), - Row3(null, -1L, true) - )).toDF("foo", "v", "veryLongColumnNameWithBoolean") + val df = sparkSession + .createDataFrame( + Seq( + Row3("myString", 4L, false), + Row3("foo", 99999999L, true), + Row3(null, -1L, true) + ) + ) + .toDF("foo", "v", "veryLongColumnNameWithBoolean") val records = morpheus.records.wrap(df) // When @@ -136,8 +143,7 @@ class MorpheusRecordsPrinterTest extends MorpheusTestSuite with GraphConstructio it("prints return property values without alias") { val given = - initGraph( - """ + initGraph(""" |CREATE (a:Person {name: "Alice"})-[:LIVES_IN]->(city:City)<-[:LIVES_IN]-(b:Person {name: "Bob"}) """.stripMargin) @@ -145,12 +151,12 @@ class MorpheusRecordsPrinterTest extends MorpheusTestSuite with GraphConstructio """MATCH (a:Person)-[:LIVES_IN]->(city:City)<-[:LIVES_IN]-(b:Person) |RETURN a.name, b.name |ORDER BY a.name - """.stripMargin) + """.stripMargin + ) print(when.records) - getString should equal( - """|╔═════════╤═════════╗ + getString should equal("""|╔═════════╤═════════╗ |║ a.name │ b.name ║ |╠═════════╪═════════╣ |║ 'Alice' │ 'Bob' ║ @@ -169,11 +175,17 @@ class MorpheusRecordsPrinterTest extends MorpheusTestSuite with GraphConstructio private case class Row1(foo: String) - private case class Row3(foo: String, v: Long, veryLongColumnNameWithBoolean: Boolean) + private case class Row3( + foo: String, + v: Long, + veryLongColumnNameWithBoolean: Boolean + ) private def getString = new String(baos.toByteArray, UTF_8) private def print(r: CypherRecords)(implicit options: PrintOptions): Unit = - RecordsPrinter.print(r)(options.stream(new PrintStream(baos, true, UTF_8.name()))) + RecordsPrinter.print(r)( + options.stream(new PrintStream(baos, true, UTF_8.name())) + ) } diff --git a/morpheus-testing/src/test/scala/org/opencypher/morpheus/impl/MorpheusRecordsTest.scala b/morpheus-testing/src/test/scala/org/opencypher/morpheus/impl/MorpheusRecordsTest.scala index 435b42b9b5..733adad3b5 100644 --- a/morpheus-testing/src/test/scala/org/opencypher/morpheus/impl/MorpheusRecordsTest.scala +++ b/morpheus-testing/src/test/scala/org/opencypher/morpheus/impl/MorpheusRecordsTest.scala @@ -43,26 +43,76 @@ import org.opencypher.okapi.relational.impl.table.RecordHeader import org.opencypher.okapi.testing.Bag import org.opencypher.okapi.testing.Bag._ -class MorpheusRecordsTest extends MorpheusTestSuite with GraphConstructionFixture with TeamDataFixture { +class MorpheusRecordsTest + extends MorpheusTestSuite + with GraphConstructionFixture + with TeamDataFixture { describe("column naming") { it("creates column names for simple expressions") { - morpheus.cypher("RETURN 1").records.asMorpheus.df.columns should equal(Array("1")) - morpheus.cypher("RETURN '\u0099'").records.asMorpheus.df.columns should equal(Array("'\u0099'")) - morpheus.cypher("RETURN 1 AS foo").records.asMorpheus.df.columns should equal(Array("foo")) - morpheus.cypher("RETURN 1 AS foo, 2 AS bar").records.asMorpheus.df.columns.toSet should equal(Set("foo", "bar")) - morpheus.cypher("RETURN true AND false").records.asMorpheus.df.columns.toSet should equal(Set("true AND false")) - morpheus.cypher("RETURN true AND false AND false").records.asMorpheus.df.columns.toSet should equal(Set("true AND false AND false")) - morpheus.cypher("RETURN 'foo' STARTS WITH 'f'").records.asMorpheus.df.columns.toSet should equal(Set("'foo' STARTS WITH 'f'")) + morpheus.cypher("RETURN 1").records.asMorpheus.df.columns should equal( + Array("1") + ) + morpheus + .cypher("RETURN '\u0099'") + .records + .asMorpheus + .df + .columns should equal(Array("'\u0099'")) + morpheus + .cypher("RETURN 1 AS foo") + .records + .asMorpheus + .df + .columns should equal(Array("foo")) + morpheus + .cypher("RETURN 1 AS foo, 2 AS bar") + .records + .asMorpheus + .df + .columns + .toSet should equal(Set("foo", "bar")) + morpheus + .cypher("RETURN true AND false") + .records + .asMorpheus + .df + .columns + .toSet should equal(Set("true AND false")) + morpheus + .cypher("RETURN true AND false AND false") + .records + .asMorpheus + .df + .columns + .toSet should equal(Set("true AND false AND false")) + morpheus + .cypher("RETURN 'foo' STARTS WITH 'f'") + .records + .asMorpheus + .df + .columns + .toSet should equal(Set("'foo' STARTS WITH 'f'")) } it("escapes property accessors") { - morpheus.cypher("MATCH (n) RETURN n.foo").records.asMorpheus.df.columns.toSet should equal(Set("n_foo")) + morpheus + .cypher("MATCH (n) RETURN n.foo") + .records + .asMorpheus + .df + .columns + .toSet should equal(Set("n_foo")) } it("creates column names for params") { - morpheus.cypher("RETURN $x", parameters = CypherMap("x" -> 1)).records.asMorpheus.df.columns should equal(Array("$x")) + morpheus + .cypher("RETURN $x", parameters = CypherMap("x" -> 1)) + .records + .asMorpheus + .df + .columns should equal(Array("$x")) } it("creates column names for node expressions") { @@ -71,23 +121,30 @@ class MorpheusRecordsTest extends MorpheusTestSuite with GraphConstructionFixtur val result = given.cypher("FROM GRAPH foo MATCH (n) RETURN n") - result.records.asMorpheus.df.columns.toSet should equal(Set("n", "n:L", "n_val")) + result.records.asMorpheus.df.columns.toSet should equal( + Set("n", "n:L", "n_val") + ) } it("creates column names for relationship expressions") { - val given = initGraph("CREATE ({val: 'a'})-[:R {prop: 'b'}]->({val: 'c'})") + val given = + initGraph("CREATE ({val: 'a'})-[:R {prop: 'b'}]->({val: 'c'})") morpheus.catalog.store("foo", given) val result = given.cypher("FROM GRAPH foo MATCH (n)-[r]->(m) RETURN r") - result.records.asMorpheus.df.columns.toSet should equal(Set("r", "r:R", "source(r)", "target(r)", "r_prop")) + result.records.asMorpheus.df.columns.toSet should equal( + Set("r", "r:R", "source(r)", "target(r)", "r_prop") + ) } it("retains user-specified order of return items") { val given = initGraph("CREATE (:L {val: 'a'})") morpheus.catalog.store("foo", given) - val result = given.cypher("FROM GRAPH foo MATCH (n) RETURN n.val AS bar, n, n.val AS foo") + val result = given.cypher( + "FROM GRAPH foo MATCH (n) RETURN n.val AS bar, n, n.val AS foo" + ) val dfColumns = result.records.asMorpheus.df.columns dfColumns.head should equal("bar") @@ -102,7 +159,9 @@ class MorpheusRecordsTest extends MorpheusTestSuite with GraphConstructionFixtur val result = given.cypher("FROM GRAPH foo MATCH (n) RETURN n, n.val") val dfColumns = result.records.asMorpheus.df.columns - dfColumns.collect { case col if col == "n_val" => col }.length should equal(1) + dfColumns.collect { + case col if col == "n_val" => col + }.length should equal(1) dfColumns.toSet should equal(Set("n", "n:L", "n_val")) } } @@ -111,39 +170,50 @@ class MorpheusRecordsTest extends MorpheusTestSuite with GraphConstructionFixtur // Given (generally produced by a SQL query) val records = morpheus.records.wrap(personDF) - records.header.expressions.map(s => s -> s.cypherType) should equal(Set( - Var("ID")() -> CTInteger, - Var("NAME")() -> CTString.nullable, - Var("NUM")() -> CTInteger - )) + records.header.expressions.map(s => s -> s.cypherType) should equal( + Set( + Var("ID")() -> CTInteger, + Var("NAME")() -> CTString.nullable, + Var("NUM")() -> CTInteger + ) + ) } it("can be registered and queried from SQL") { // Given - morpheus.records.fromElementTable(personTable).df.createOrReplaceTempView("people") + morpheus.records + .fromElementTable(personTable) + .df + .createOrReplaceTempView("people") // When val df = sparkSession.sql("SELECT * FROM people") // Then - df.collect().toBag should equal(Bag( - Row(1L.encodeAsMorpheusId, "Mats", 23), - Row(2L.encodeAsMorpheusId, "Martin", 42), - Row(3L.encodeAsMorpheusId, "Max", 1337), - Row(4L.encodeAsMorpheusId, "Stefan", 9) - )) + df.collect().toBag should equal( + Bag( + Row(1L.encodeAsMorpheusId, "Mats", 23), + Row(2L.encodeAsMorpheusId, "Martin", 42), + Row(3L.encodeAsMorpheusId, "Max", 1337), + Row(4L.encodeAsMorpheusId, "Stefan", 9) + ) + ) } it("verify MorpheusRecords header") { - val givenDF = sparkSession.createDataFrame( - Seq( - (1L, "Mats"), - (2L, "Martin"), - (3L, "Max"), - (4L, "Stefan") - )).toDF("ID", "NAME") - - val givenMapping = NodeMappingBuilder.on("ID") + val givenDF = sparkSession + .createDataFrame( + Seq( + (1L, "Mats"), + (2L, "Martin"), + (3L, "Max"), + (4L, "Stefan") + ) + ) + .toDF("ID", "NAME") + + val givenMapping = NodeMappingBuilder + .on("ID") .withImpliedLabel("Person") .withPropertyKey("name" -> "NAME") .build @@ -157,20 +227,25 @@ class MorpheusRecordsTest extends MorpheusTestSuite with GraphConstructionFixtur Set( elementVar, ElementProperty(elementVar, PropertyKey("name"))(CTString.nullable) - )) + ) + ) } it("verify MorpheusRecords header for relationship with a fixed type") { - val givenDF = sparkSession.createDataFrame( - Seq( - (10L, 1L, 2L, "red"), - (11L, 2L, 3L, "blue"), - (12L, 3L, 4L, "green"), - (13L, 4L, 1L, "yellow") - )).toDF("ID", "FROM", "TO", "COLOR") + val givenDF = sparkSession + .createDataFrame( + Seq( + (10L, 1L, 2L, "red"), + (11L, 2L, 3L, "blue"), + (12L, 3L, 4L, "green"), + (13L, 4L, 1L, "yellow") + ) + ) + .toDF("ID", "FROM", "TO", "COLOR") - val givenMapping = RelationshipMappingBuilder.on("ID") + val givenMapping = RelationshipMappingBuilder + .on("ID") .from("FROM") .to("TO") .relType("NEXT") @@ -195,7 +270,9 @@ class MorpheusRecordsTest extends MorpheusTestSuite with GraphConstructionFixtur // Validation happens in operators instead ignore("can not construct records with data/header column name conflict") { - val data = sparkSession.createDataFrame(Seq((1, "foo"), (2, "bar"))).toDF("int", "string") + val data = sparkSession + .createDataFrame(Seq((1, "foo"), (2, "bar"))) + .toDF("int", "string") val header = RecordHeader.from(Var("int")(), Var("notString")()) a[InternalException] shouldBe thrownBy { @@ -204,7 +281,9 @@ class MorpheusRecordsTest extends MorpheusTestSuite with GraphConstructionFixtur } it("can construct records with matching data/header") { - val data = sparkSession.createDataFrame(Seq((1L, "foo"), (2L, "bar"))).toDF("int", "string") + val data = sparkSession + .createDataFrame(Seq((1L, "foo"), (2L, "bar"))) + .toDF("int", "string") val records = morpheus.records.wrap(data) val v = Var("int")(CTInteger) @@ -223,6 +302,7 @@ class MorpheusRecordsTest extends MorpheusTestSuite with GraphConstructionFixtur "n" -> MorpheusNode(0L, Set("Foo"), CypherMap("p" -> 1)), "b" -> 5 ) - )) + ) + ) } } diff --git a/morpheus-testing/src/test/scala/org/opencypher/morpheus/impl/MorpheusSessionImplTest.scala b/morpheus-testing/src/test/scala/org/opencypher/morpheus/impl/MorpheusSessionImplTest.scala index 156b57f872..4f25a0a371 100644 --- a/morpheus-testing/src/test/scala/org/opencypher/morpheus/impl/MorpheusSessionImplTest.scala +++ b/morpheus-testing/src/test/scala/org/opencypher/morpheus/impl/MorpheusSessionImplTest.scala @@ -35,7 +35,10 @@ import org.opencypher.okapi.impl.io.SessionGraphDataSource import org.opencypher.okapi.testing.Bag import org.opencypher.okapi.testing.Bag._ -class MorpheusSessionImplTest extends MorpheusTestSuite with TeamDataFixture with GraphConstructionFixture { +class MorpheusSessionImplTest + extends MorpheusTestSuite + with TeamDataFixture + with GraphConstructionFixture { it("can use multiple session graph data sources") { morpheus.registerSource(Namespace("working"), new SessionGraphDataSource()) @@ -47,7 +50,9 @@ class MorpheusSessionImplTest extends MorpheusTestSuite with TeamDataFixture wit morpheus.catalog.store(QualifiedGraphName("session.a"), g1) morpheus.catalog.store(QualifiedGraphName("working.a"), g2) - morpheus.cypher("CATALOG CREATE GRAPH working.b { FROM GRAPH working.a RETURN GRAPH }") + morpheus.cypher( + "CATALOG CREATE GRAPH working.b { FROM GRAPH working.a RETURN GRAPH }" + ) morpheus.catalog.store(QualifiedGraphName("foo.bar.baz.a"), g3) val r1 = morpheus.cypher("FROM GRAPH a MATCH (n) RETURN n") @@ -55,40 +60,49 @@ class MorpheusSessionImplTest extends MorpheusTestSuite with TeamDataFixture wit val r3 = morpheus.cypher("FROM GRAPH working.b MATCH (n) RETURN n") val r4 = morpheus.cypher("FROM GRAPH foo.bar.baz.a MATCH (n) RETURN n") - r1.records.collect.toBag should equal(Bag( - CypherMap("n" -> MorpheusNode(0L, Set("A"))) - )) - r2.records.collect.toBag should equal(Bag( - CypherMap("n" -> MorpheusNode(0L, Set("B"))) - )) - r3.records.collect.toBag should equal(Bag( - CypherMap("n" -> MorpheusNode(0L, Set("B"))) - )) - r4.records.collect.toBag should equal(Bag( - CypherMap("n" -> MorpheusNode(0L, Set("C"))) - )) + r1.records.collect.toBag should equal( + Bag( + CypherMap("n" -> MorpheusNode(0L, Set("A"))) + ) + ) + r2.records.collect.toBag should equal( + Bag( + CypherMap("n" -> MorpheusNode(0L, Set("B"))) + ) + ) + r3.records.collect.toBag should equal( + Bag( + CypherMap("n" -> MorpheusNode(0L, Set("B"))) + ) + ) + r4.records.collect.toBag should equal( + Bag( + CypherMap("n" -> MorpheusNode(0L, Set("C"))) + ) + ) } it("can execute sql on registered tables") { morpheus.records.wrap(personDF).df.createOrReplaceTempView("people") morpheus.records.wrap(knowsDF).df.createOrReplaceTempView("knows") - val sqlResult = morpheus.sql( - """ + val sqlResult = morpheus.sql(""" |SELECT people.name AS me, knows.since AS since, p2.name AS you |FROM people |INNER JOIN knows ON knows.src = people.id |INNER JOIN people p2 ON knows.dst = p2.id """.stripMargin) - sqlResult.collect.toBag should equal(Bag( - CypherMap("me" -> "Mats", "since" -> 2017, "you" -> "Martin"), - CypherMap("me" -> "Mats", "since" -> 2016, "you" -> "Max"), - CypherMap("me" -> "Mats", "since" -> 2015, "you" -> "Stefan"), - CypherMap("me" -> "Martin", "since" -> 2016, "you" -> "Max"), - CypherMap("me" -> "Martin", "since" -> 2013, "you" -> "Stefan"), - CypherMap("me" -> "Max", "since" -> 2016, "you" -> "Stefan") - )) + sqlResult.collect.toBag should equal( + Bag( + CypherMap("me" -> "Mats", "since" -> 2017, "you" -> "Martin"), + CypherMap("me" -> "Mats", "since" -> 2016, "you" -> "Max"), + CypherMap("me" -> "Mats", "since" -> 2015, "you" -> "Stefan"), + CypherMap("me" -> "Martin", "since" -> 2016, "you" -> "Max"), + CypherMap("me" -> "Martin", "since" -> 2013, "you" -> "Stefan"), + CypherMap("me" -> "Max", "since" -> 2016, "you" -> "Stefan") + ) + ) } } diff --git a/morpheus-testing/src/test/scala/org/opencypher/morpheus/impl/ScanGraphTest.scala b/morpheus-testing/src/test/scala/org/opencypher/morpheus/impl/ScanGraphTest.scala index 1a0d375e4b..fed4cb0646 100644 --- a/morpheus-testing/src/test/scala/org/opencypher/morpheus/impl/ScanGraphTest.scala +++ b/morpheus-testing/src/test/scala/org/opencypher/morpheus/impl/ScanGraphTest.scala @@ -34,7 +34,11 @@ import org.opencypher.morpheus.impl.MorpheusConverters._ import org.opencypher.morpheus.testing.support.ElementTableCreationSupport import org.opencypher.morpheus.testing.support.creation.graphs.{ScanGraphFactory, TestGraphFactory} import org.opencypher.okapi.api.graph._ -import org.opencypher.okapi.api.io.conversion.{ElementMapping, NodeMappingBuilder, RelationshipMappingBuilder} +import org.opencypher.okapi.api.io.conversion.{ + ElementMapping, + NodeMappingBuilder, + RelationshipMappingBuilder +} import org.opencypher.okapi.api.types._ import org.opencypher.okapi.api.value.CypherValue.CypherMap import org.opencypher.okapi.ir.api.expr._ @@ -76,31 +80,231 @@ class ScanGraphTest extends MorpheusGraphTest with ElementTableCreationSupport { ) val nodeData = Bag( - Row(1L.encodeAsMorpheusId.withPrefix(0).toList, false, true, false, null, 23L, "Mats", null, null), - Row(2L.encodeAsMorpheusId.withPrefix(0).toList, false, true, false, null, 42L, "Martin", null, null), - Row(3L.encodeAsMorpheusId.withPrefix(0).toList, false, true, false, null, 1337L, "Max", null, null), - Row(4L.encodeAsMorpheusId.withPrefix(0).toList, false, true, false, null, 9L, "Stefan", null, null), - Row(10L.encodeAsMorpheusId.withPrefix(1).toList, true, false, false, null, null, null, "1984", 1949L), - Row(20L.encodeAsMorpheusId.withPrefix(1).toList, true, false, false, null, null, null, "Cryptonomicon", 1999L), - Row(30L.encodeAsMorpheusId.withPrefix(1).toList, true, false, false, null, null, null, "The Eye of the World", 1990L), - Row(40L.encodeAsMorpheusId.withPrefix(1).toList, true, false, false, null, null, null, "The Circle", 2013L), - Row(100L.encodeAsMorpheusId.withPrefix(1).toList, false, true, true, "C", 42L, "Alice", null, null), - Row(200L.encodeAsMorpheusId.withPrefix(1).toList, false, true, true, "D", 23L, "Bob", null, null), - Row(300L.encodeAsMorpheusId.withPrefix(1).toList, false, true, true, "F", 84L, "Eve", null, null), - Row(400L.encodeAsMorpheusId.withPrefix(1).toList, false, true, true, "R", 49L, "Carl", null, null) + Row( + 1L.encodeAsMorpheusId.withPrefix(0).toList, + false, + true, + false, + null, + 23L, + "Mats", + null, + null + ), + Row( + 2L.encodeAsMorpheusId.withPrefix(0).toList, + false, + true, + false, + null, + 42L, + "Martin", + null, + null + ), + Row( + 3L.encodeAsMorpheusId.withPrefix(0).toList, + false, + true, + false, + null, + 1337L, + "Max", + null, + null + ), + Row( + 4L.encodeAsMorpheusId.withPrefix(0).toList, + false, + true, + false, + null, + 9L, + "Stefan", + null, + null + ), + Row( + 10L.encodeAsMorpheusId.withPrefix(1).toList, + true, + false, + false, + null, + null, + null, + "1984", + 1949L + ), + Row( + 20L.encodeAsMorpheusId.withPrefix(1).toList, + true, + false, + false, + null, + null, + null, + "Cryptonomicon", + 1999L + ), + Row( + 30L.encodeAsMorpheusId.withPrefix(1).toList, + true, + false, + false, + null, + null, + null, + "The Eye of the World", + 1990L + ), + Row( + 40L.encodeAsMorpheusId.withPrefix(1).toList, + true, + false, + false, + null, + null, + null, + "The Circle", + 2013L + ), + Row( + 100L.encodeAsMorpheusId.withPrefix(1).toList, + false, + true, + true, + "C", + 42L, + "Alice", + null, + null + ), + Row( + 200L.encodeAsMorpheusId.withPrefix(1).toList, + false, + true, + true, + "D", + 23L, + "Bob", + null, + null + ), + Row( + 300L.encodeAsMorpheusId.withPrefix(1).toList, + false, + true, + true, + "F", + 84L, + "Eve", + null, + null + ), + Row( + 400L.encodeAsMorpheusId.withPrefix(1).toList, + false, + true, + true, + "R", + 49L, + "Carl", + null, + null + ) ) val relData = Bag( - Row(1L.encodeAsMorpheusId.withPrefix(0).toList, 1L.encodeAsMorpheusId.withPrefix(0).toList, true, false, 2L.encodeAsMorpheusId.withPrefix(0).toList, null, 2017L), - Row(1L.encodeAsMorpheusId.withPrefix(0).toList, 2L.encodeAsMorpheusId.withPrefix(0).toList, true, false, 3L.encodeAsMorpheusId.withPrefix(0).toList, null, 2016L), - Row(1L.encodeAsMorpheusId.withPrefix(0).toList, 3L.encodeAsMorpheusId.withPrefix(0).toList, true, false, 4L.encodeAsMorpheusId.withPrefix(0).toList, null, 2015L), - Row(2L.encodeAsMorpheusId.withPrefix(0).toList, 4L.encodeAsMorpheusId.withPrefix(0).toList, true, false, 3L.encodeAsMorpheusId.withPrefix(0).toList, null, 2016L), - Row(2L.encodeAsMorpheusId.withPrefix(0).toList, 5L.encodeAsMorpheusId.withPrefix(0).toList, true, false, 4L.encodeAsMorpheusId.withPrefix(0).toList, null, 2013L), - Row(3L.encodeAsMorpheusId.withPrefix(0).toList, 6L.encodeAsMorpheusId.withPrefix(0).toList, true, false, 4L.encodeAsMorpheusId.withPrefix(0).toList, null, 2016L), - Row(100L.encodeAsMorpheusId.withPrefix(1).toList, 100L.encodeAsMorpheusId.withPrefix(1).toList, false, true, 10L.encodeAsMorpheusId.withPrefix(1).toList, true, null), - Row(200L.encodeAsMorpheusId.withPrefix(1).toList, 200L.encodeAsMorpheusId.withPrefix(1).toList, false, true, 40L.encodeAsMorpheusId.withPrefix(1).toList, true, null), - Row(300L.encodeAsMorpheusId.withPrefix(1).toList, 300L.encodeAsMorpheusId.withPrefix(1).toList, false, true, 30L.encodeAsMorpheusId.withPrefix(1).toList, true, null), - Row(400L.encodeAsMorpheusId.withPrefix(1).toList, 400L.encodeAsMorpheusId.withPrefix(1).toList, false, true, 20L.encodeAsMorpheusId.withPrefix(1).toList, false, null) + Row( + 1L.encodeAsMorpheusId.withPrefix(0).toList, + 1L.encodeAsMorpheusId.withPrefix(0).toList, + true, + false, + 2L.encodeAsMorpheusId.withPrefix(0).toList, + null, + 2017L + ), + Row( + 1L.encodeAsMorpheusId.withPrefix(0).toList, + 2L.encodeAsMorpheusId.withPrefix(0).toList, + true, + false, + 3L.encodeAsMorpheusId.withPrefix(0).toList, + null, + 2016L + ), + Row( + 1L.encodeAsMorpheusId.withPrefix(0).toList, + 3L.encodeAsMorpheusId.withPrefix(0).toList, + true, + false, + 4L.encodeAsMorpheusId.withPrefix(0).toList, + null, + 2015L + ), + Row( + 2L.encodeAsMorpheusId.withPrefix(0).toList, + 4L.encodeAsMorpheusId.withPrefix(0).toList, + true, + false, + 3L.encodeAsMorpheusId.withPrefix(0).toList, + null, + 2016L + ), + Row( + 2L.encodeAsMorpheusId.withPrefix(0).toList, + 5L.encodeAsMorpheusId.withPrefix(0).toList, + true, + false, + 4L.encodeAsMorpheusId.withPrefix(0).toList, + null, + 2013L + ), + Row( + 3L.encodeAsMorpheusId.withPrefix(0).toList, + 6L.encodeAsMorpheusId.withPrefix(0).toList, + true, + false, + 4L.encodeAsMorpheusId.withPrefix(0).toList, + null, + 2016L + ), + Row( + 100L.encodeAsMorpheusId.withPrefix(1).toList, + 100L.encodeAsMorpheusId.withPrefix(1).toList, + false, + true, + 10L.encodeAsMorpheusId.withPrefix(1).toList, + true, + null + ), + Row( + 200L.encodeAsMorpheusId.withPrefix(1).toList, + 200L.encodeAsMorpheusId.withPrefix(1).toList, + false, + true, + 40L.encodeAsMorpheusId.withPrefix(1).toList, + true, + null + ), + Row( + 300L.encodeAsMorpheusId.withPrefix(1).toList, + 300L.encodeAsMorpheusId.withPrefix(1).toList, + false, + true, + 30L.encodeAsMorpheusId.withPrefix(1).toList, + true, + null + ), + Row( + 400L.encodeAsMorpheusId.withPrefix(1).toList, + 400L.encodeAsMorpheusId.withPrefix(1).toList, + false, + true, + 20L.encodeAsMorpheusId.withPrefix(1).toList, + false, + null + ) ) verify(result.nodes("n"), nodeCols, nodeData) @@ -108,29 +312,41 @@ class ScanGraphTest extends MorpheusGraphTest with ElementTableCreationSupport { } it("dont lose schema information when mapping") { - val nodes = MorpheusElementTable.create(NodeMappingBuilder.on("id").build, - morpheus.sparkSession.createDataFrame( - Seq( - Tuple1(10L), - Tuple1(11L), - Tuple1(12L), - Tuple1(20L), - Tuple1(21L), - Tuple1(22L), - Tuple1(25L), - Tuple1(50L), - Tuple1(51L) + val nodes = MorpheusElementTable.create( + NodeMappingBuilder.on("id").build, + morpheus.sparkSession + .createDataFrame( + Seq( + Tuple1(10L), + Tuple1(11L), + Tuple1(12L), + Tuple1(20L), + Tuple1(21L), + Tuple1(22L), + Tuple1(25L), + Tuple1(50L), + Tuple1(51L) + ) ) - ).toDF("id")) + .toDF("id") + ) - val rs = MorpheusElementTable.create(RelationshipMappingBuilder.on("ID").from("SRC").to("DST").relType("FOO").build, - morpheus.sparkSession.createDataFrame( - Seq( - (10L, 1000L, 20L), - (50L, 500L, 25L) + val rs = MorpheusElementTable.create( + RelationshipMappingBuilder + .on("ID") + .from("SRC") + .to("DST") + .relType("FOO") + .build, + morpheus.sparkSession + .createDataFrame( + Seq( + (10L, 1000L, 20L), + (50L, 500L, 25L) + ) ) - ).toDF("SRC", "ID", "DST")) - + .toDF("SRC", "ID", "DST") + ) val graph = morpheus.graphs.create(nodes, rs) @@ -140,7 +356,8 @@ class ScanGraphTest extends MorpheusGraphTest with ElementTableCreationSupport { Set( CypherMap("r" -> MorpheusRelationship(1000L, 10L, 20L, "FOO")), CypherMap("r" -> MorpheusRelationship(500L, 50L, 25L, "FOO")) - )) + ) + ) } it("Construct graph from single node scan") { @@ -174,19 +391,50 @@ class ScanGraphTest extends MorpheusGraphTest with ElementTableCreationSupport { nHasPropertyYear ) val data = Bag( - Row(1L.encodeAsMorpheusId.toList, false, true, 23L, "Mats", null, null), - Row(2L.encodeAsMorpheusId.toList, false, true, 42L, "Martin", null, null), - Row(3L.encodeAsMorpheusId.toList, false, true, 1337L, "Max", null, null), - Row(4L.encodeAsMorpheusId.toList, false, true, 9L, "Stefan", null, null), - Row(10L.encodeAsMorpheusId.toList, true, false, null, null, "1984", 1949L), - Row(20L.encodeAsMorpheusId.toList, true, false, null, null, "Cryptonomicon", 1999L), - Row(30L.encodeAsMorpheusId.toList, true, false, null, null, "The Eye of the World", 1990L), - Row(40L.encodeAsMorpheusId.toList, true, false, null, null, "The Circle", 2013L) + Row(1L.encodeAsMorpheusId.toList, false, true, 23L, "Mats", null, null), + Row(2L.encodeAsMorpheusId.toList, false, true, 42L, "Martin", null, null), + Row(3L.encodeAsMorpheusId.toList, false, true, 1337L, "Max", null, null), + Row(4L.encodeAsMorpheusId.toList, false, true, 9L, "Stefan", null, null), + Row( + 10L.encodeAsMorpheusId.toList, + true, + false, + null, + null, + "1984", + 1949L + ), + Row( + 20L.encodeAsMorpheusId.toList, + true, + false, + null, + null, + "Cryptonomicon", + 1999L + ), + Row( + 30L.encodeAsMorpheusId.toList, + true, + false, + null, + null, + "The Eye of the World", + 1990L + ), + Row( + 40L.encodeAsMorpheusId.toList, + true, + false, + null, + null, + "The Circle", + 2013L + ) ) verify(nodes, cols, data) } - it("Align node scans") { val fixture = """ @@ -203,23 +451,54 @@ class ScanGraphTest extends MorpheusGraphTest with ElementTableCreationSupport { graph.cypher("MATCH (n) RETURN n").records.size should equal(3) } - it("Align node scans when individual tables have the same node id and properties") { - val aDf = morpheus.sparkSession.createDataFrame(Seq( - (0L, "A") - )).toDF("_node_id", "name").withColumn("size", functions.lit(null)) - val aMapping: ElementMapping = NodeMappingBuilder.on("_node_id").withPropertyKey("name").withPropertyKey("size").withImpliedLabel("A").build + it( + "Align node scans when individual tables have the same node id and properties" + ) { + val aDf = morpheus.sparkSession + .createDataFrame( + Seq( + (0L, "A") + ) + ) + .toDF("_node_id", "name") + .withColumn("size", functions.lit(null)) + val aMapping: ElementMapping = NodeMappingBuilder + .on("_node_id") + .withPropertyKey("name") + .withPropertyKey("size") + .withImpliedLabel("A") + .build val aTable = MorpheusElementTable.create(aMapping, aDf) - val bDf = morpheus.sparkSession.createDataFrame(Seq( - (1L, "B") - )).toDF("_node_id", "name").withColumn("size", functions.lit(null)) - val bMapping = NodeMappingBuilder.on("_node_id").withPropertyKey("name").withPropertyKey("size").withImpliedLabel("B").build + val bDf = morpheus.sparkSession + .createDataFrame( + Seq( + (1L, "B") + ) + ) + .toDF("_node_id", "name") + .withColumn("size", functions.lit(null)) + val bMapping = NodeMappingBuilder + .on("_node_id") + .withPropertyKey("name") + .withPropertyKey("size") + .withImpliedLabel("B") + .build val bTable = MorpheusElementTable.create(bMapping, bDf) - val comboDf = morpheus.sparkSession.createDataFrame(Seq( - (2L, "COMBO", 2) - )).toDF("_node_id", "name", "size") - val comboMapping = NodeMappingBuilder.on("_node_id").withPropertyKey("name").withPropertyKey("size").withImpliedLabels("A", "B").build + val comboDf = morpheus.sparkSession + .createDataFrame( + Seq( + (2L, "COMBO", 2) + ) + ) + .toDF("_node_id", "name", "size") + val comboMapping = NodeMappingBuilder + .on("_node_id") + .withPropertyKey("name") + .withPropertyKey("size") + .withImpliedLabels("A", "B") + .build val comboTable = MorpheusElementTable.create(comboMapping, comboDf) val graph = morpheus.graphs.create(aTable, bTable, comboTable) @@ -239,12 +518,48 @@ class ScanGraphTest extends MorpheusGraphTest with ElementTableCreationSupport { rHasPropertySince ) val data = Bag( - Row(1L.encodeAsMorpheusId.toList, 1L.encodeAsMorpheusId.toList, true, 2L.encodeAsMorpheusId.toList, 2017L), - Row(1L.encodeAsMorpheusId.toList, 2L.encodeAsMorpheusId.toList, true, 3L.encodeAsMorpheusId.toList, 2016L), - Row(1L.encodeAsMorpheusId.toList, 3L.encodeAsMorpheusId.toList, true, 4L.encodeAsMorpheusId.toList, 2015L), - Row(2L.encodeAsMorpheusId.toList, 4L.encodeAsMorpheusId.toList, true, 3L.encodeAsMorpheusId.toList, 2016L), - Row(2L.encodeAsMorpheusId.toList, 5L.encodeAsMorpheusId.toList, true, 4L.encodeAsMorpheusId.toList, 2013L), - Row(3L.encodeAsMorpheusId.toList, 6L.encodeAsMorpheusId.toList, true, 4L.encodeAsMorpheusId.toList, 2016L) + Row( + 1L.encodeAsMorpheusId.toList, + 1L.encodeAsMorpheusId.toList, + true, + 2L.encodeAsMorpheusId.toList, + 2017L + ), + Row( + 1L.encodeAsMorpheusId.toList, + 2L.encodeAsMorpheusId.toList, + true, + 3L.encodeAsMorpheusId.toList, + 2016L + ), + Row( + 1L.encodeAsMorpheusId.toList, + 3L.encodeAsMorpheusId.toList, + true, + 4L.encodeAsMorpheusId.toList, + 2015L + ), + Row( + 2L.encodeAsMorpheusId.toList, + 4L.encodeAsMorpheusId.toList, + true, + 3L.encodeAsMorpheusId.toList, + 2016L + ), + Row( + 2L.encodeAsMorpheusId.toList, + 5L.encodeAsMorpheusId.toList, + true, + 4L.encodeAsMorpheusId.toList, + 2013L + ), + Row( + 3L.encodeAsMorpheusId.toList, + 6L.encodeAsMorpheusId.toList, + true, + 4L.encodeAsMorpheusId.toList, + 2016L + ) ) verify(rels, cols, data) @@ -263,14 +578,46 @@ class ScanGraphTest extends MorpheusGraphTest with ElementTableCreationSupport { nHasPropertyYear ) val data = Bag( - Row(1L.encodeAsMorpheusId.toList, false, true, 23L, "Mats", null, null), - Row(2L.encodeAsMorpheusId.toList, false, true, 42L, "Martin", null, null), - Row(3L.encodeAsMorpheusId.toList, false, true, 1337L, "Max", null, null), - Row(4L.encodeAsMorpheusId.toList, false, true, 9L, "Stefan", null, null), - Row(10L.encodeAsMorpheusId.toList, true, false, null, null, "1984", 1949L), - Row(20L.encodeAsMorpheusId.toList, true, false, null, null, "Cryptonomicon", 1999L), - Row(30L.encodeAsMorpheusId.toList, true, false, null, null, "The Eye of the World", 1990L), - Row(40L.encodeAsMorpheusId.toList, true, false, null, null, "The Circle", 2013L) + Row(1L.encodeAsMorpheusId.toList, false, true, 23L, "Mats", null, null), + Row(2L.encodeAsMorpheusId.toList, false, true, 42L, "Martin", null, null), + Row(3L.encodeAsMorpheusId.toList, false, true, 1337L, "Max", null, null), + Row(4L.encodeAsMorpheusId.toList, false, true, 9L, "Stefan", null, null), + Row( + 10L.encodeAsMorpheusId.toList, + true, + false, + null, + null, + "1984", + 1949L + ), + Row( + 20L.encodeAsMorpheusId.toList, + true, + false, + null, + null, + "Cryptonomicon", + 1999L + ), + Row( + 30L.encodeAsMorpheusId.toList, + true, + false, + null, + null, + "The Eye of the World", + 1990L + ), + Row( + 40L.encodeAsMorpheusId.toList, + true, + false, + null, + null, + "The Circle", + 2013L + ) ) verify(nodes, cols, data) @@ -295,7 +642,8 @@ class ScanGraphTest extends MorpheusGraphTest with ElementTableCreationSupport { } it("Extract all relationship scans") { - val graph = morpheus.graphs.create(personTable, bookTable, knowsTable, readsTable) + val graph = + morpheus.graphs.create(personTable, bookTable, knowsTable, readsTable) val rels = graph.relationships("r") val cols = Seq( rStart, @@ -307,23 +655,104 @@ class ScanGraphTest extends MorpheusGraphTest with ElementTableCreationSupport { rHasPropertySince ) val data = Bag( - Row(1L.encodeAsMorpheusId.toList, 1L.encodeAsMorpheusId.toList, true, false, 2L.encodeAsMorpheusId.toList, null, 2017L), - Row(1L.encodeAsMorpheusId.toList, 2L.encodeAsMorpheusId.toList, true, false, 3L.encodeAsMorpheusId.toList, null, 2016L), - Row(1L.encodeAsMorpheusId.toList, 3L.encodeAsMorpheusId.toList, true, false, 4L.encodeAsMorpheusId.toList, null, 2015L), - Row(2L.encodeAsMorpheusId.toList, 4L.encodeAsMorpheusId.toList, true, false, 3L.encodeAsMorpheusId.toList, null, 2016L), - Row(2L.encodeAsMorpheusId.toList, 5L.encodeAsMorpheusId.toList, true, false, 4L.encodeAsMorpheusId.toList, null, 2013L), - Row(3L.encodeAsMorpheusId.toList, 6L.encodeAsMorpheusId.toList, true, false, 4L.encodeAsMorpheusId.toList, null, 2016L), - Row(100L.encodeAsMorpheusId.toList, 100L.encodeAsMorpheusId.toList, false, true, 10L.encodeAsMorpheusId.toList, true, null), - Row(200L.encodeAsMorpheusId.toList, 200L.encodeAsMorpheusId.toList, false, true, 40L.encodeAsMorpheusId.toList, true, null), - Row(300L.encodeAsMorpheusId.toList, 300L.encodeAsMorpheusId.toList, false, true, 30L.encodeAsMorpheusId.toList, true, null), - Row(400L.encodeAsMorpheusId.toList, 400L.encodeAsMorpheusId.toList, false, true, 20L.encodeAsMorpheusId.toList, false, null) + Row( + 1L.encodeAsMorpheusId.toList, + 1L.encodeAsMorpheusId.toList, + true, + false, + 2L.encodeAsMorpheusId.toList, + null, + 2017L + ), + Row( + 1L.encodeAsMorpheusId.toList, + 2L.encodeAsMorpheusId.toList, + true, + false, + 3L.encodeAsMorpheusId.toList, + null, + 2016L + ), + Row( + 1L.encodeAsMorpheusId.toList, + 3L.encodeAsMorpheusId.toList, + true, + false, + 4L.encodeAsMorpheusId.toList, + null, + 2015L + ), + Row( + 2L.encodeAsMorpheusId.toList, + 4L.encodeAsMorpheusId.toList, + true, + false, + 3L.encodeAsMorpheusId.toList, + null, + 2016L + ), + Row( + 2L.encodeAsMorpheusId.toList, + 5L.encodeAsMorpheusId.toList, + true, + false, + 4L.encodeAsMorpheusId.toList, + null, + 2013L + ), + Row( + 3L.encodeAsMorpheusId.toList, + 6L.encodeAsMorpheusId.toList, + true, + false, + 4L.encodeAsMorpheusId.toList, + null, + 2016L + ), + Row( + 100L.encodeAsMorpheusId.toList, + 100L.encodeAsMorpheusId.toList, + false, + true, + 10L.encodeAsMorpheusId.toList, + true, + null + ), + Row( + 200L.encodeAsMorpheusId.toList, + 200L.encodeAsMorpheusId.toList, + false, + true, + 40L.encodeAsMorpheusId.toList, + true, + null + ), + Row( + 300L.encodeAsMorpheusId.toList, + 300L.encodeAsMorpheusId.toList, + false, + true, + 30L.encodeAsMorpheusId.toList, + true, + null + ), + Row( + 400L.encodeAsMorpheusId.toList, + 400L.encodeAsMorpheusId.toList, + false, + true, + 20L.encodeAsMorpheusId.toList, + false, + null + ) ) verify(rels, cols, data) } it("Extract relationship scan subset") { - val graph = morpheus.graphs.create(personTable, bookTable, knowsTable, readsTable) + val graph = + morpheus.graphs.create(personTable, bookTable, knowsTable, readsTable) val rels = graph.relationships("r", CTRelationship("KNOWS")) val cols = Seq( rStart, @@ -333,19 +762,61 @@ class ScanGraphTest extends MorpheusGraphTest with ElementTableCreationSupport { rHasPropertySince ) val data = Bag( - Row(1L.encodeAsMorpheusId.toList, 1L.encodeAsMorpheusId.toList, true, 2L.encodeAsMorpheusId.toList, 2017L), - Row(1L.encodeAsMorpheusId.toList, 2L.encodeAsMorpheusId.toList, true, 3L.encodeAsMorpheusId.toList, 2016L), - Row(1L.encodeAsMorpheusId.toList, 3L.encodeAsMorpheusId.toList, true, 4L.encodeAsMorpheusId.toList, 2015L), - Row(2L.encodeAsMorpheusId.toList, 4L.encodeAsMorpheusId.toList, true, 3L.encodeAsMorpheusId.toList, 2016L), - Row(2L.encodeAsMorpheusId.toList, 5L.encodeAsMorpheusId.toList, true, 4L.encodeAsMorpheusId.toList, 2013L), - Row(3L.encodeAsMorpheusId.toList, 6L.encodeAsMorpheusId.toList, true, 4L.encodeAsMorpheusId.toList, 2016L) + Row( + 1L.encodeAsMorpheusId.toList, + 1L.encodeAsMorpheusId.toList, + true, + 2L.encodeAsMorpheusId.toList, + 2017L + ), + Row( + 1L.encodeAsMorpheusId.toList, + 2L.encodeAsMorpheusId.toList, + true, + 3L.encodeAsMorpheusId.toList, + 2016L + ), + Row( + 1L.encodeAsMorpheusId.toList, + 3L.encodeAsMorpheusId.toList, + true, + 4L.encodeAsMorpheusId.toList, + 2015L + ), + Row( + 2L.encodeAsMorpheusId.toList, + 4L.encodeAsMorpheusId.toList, + true, + 3L.encodeAsMorpheusId.toList, + 2016L + ), + Row( + 2L.encodeAsMorpheusId.toList, + 5L.encodeAsMorpheusId.toList, + true, + 4L.encodeAsMorpheusId.toList, + 2013L + ), + Row( + 3L.encodeAsMorpheusId.toList, + 6L.encodeAsMorpheusId.toList, + true, + 4L.encodeAsMorpheusId.toList, + 2016L + ) ) verify(rels, cols, data) } it("Extract relationship scan strict subset") { - val graph = morpheus.graphs.create(personTable, bookTable, knowsTable, readsTable, influencesTable) + val graph = morpheus.graphs.create( + personTable, + bookTable, + knowsTable, + readsTable, + influencesTable + ) val rels = graph.relationships("r", CTRelationship("KNOWS", "INFLUENCES")) val cols = Seq( rStart, @@ -357,14 +828,63 @@ class ScanGraphTest extends MorpheusGraphTest with ElementTableCreationSupport { ) val data = Bag( // :KNOWS - Row(1L.encodeAsMorpheusId.toList, 1L.encodeAsMorpheusId.toList, false, true, 2L.encodeAsMorpheusId.toList, 2017L), - Row(1L.encodeAsMorpheusId.toList, 2L.encodeAsMorpheusId.toList, false, true, 3L.encodeAsMorpheusId.toList, 2016L), - Row(1L.encodeAsMorpheusId.toList, 3L.encodeAsMorpheusId.toList, false, true, 4L.encodeAsMorpheusId.toList, 2015L), - Row(2L.encodeAsMorpheusId.toList, 4L.encodeAsMorpheusId.toList, false, true, 3L.encodeAsMorpheusId.toList, 2016L), - Row(2L.encodeAsMorpheusId.toList, 5L.encodeAsMorpheusId.toList, false, true, 4L.encodeAsMorpheusId.toList, 2013L), - Row(3L.encodeAsMorpheusId.toList, 6L.encodeAsMorpheusId.toList, false, true, 4L.encodeAsMorpheusId.toList, 2016L), + Row( + 1L.encodeAsMorpheusId.toList, + 1L.encodeAsMorpheusId.toList, + false, + true, + 2L.encodeAsMorpheusId.toList, + 2017L + ), + Row( + 1L.encodeAsMorpheusId.toList, + 2L.encodeAsMorpheusId.toList, + false, + true, + 3L.encodeAsMorpheusId.toList, + 2016L + ), + Row( + 1L.encodeAsMorpheusId.toList, + 3L.encodeAsMorpheusId.toList, + false, + true, + 4L.encodeAsMorpheusId.toList, + 2015L + ), + Row( + 2L.encodeAsMorpheusId.toList, + 4L.encodeAsMorpheusId.toList, + false, + true, + 3L.encodeAsMorpheusId.toList, + 2016L + ), + Row( + 2L.encodeAsMorpheusId.toList, + 5L.encodeAsMorpheusId.toList, + false, + true, + 4L.encodeAsMorpheusId.toList, + 2013L + ), + Row( + 3L.encodeAsMorpheusId.toList, + 6L.encodeAsMorpheusId.toList, + false, + true, + 4L.encodeAsMorpheusId.toList, + 2016L + ), // :INFLUENCES - Row(10L.encodeAsMorpheusId.toList, 1000L.encodeAsMorpheusId.toList, true, false, 20L.encodeAsMorpheusId.toList, null) + Row( + 10L.encodeAsMorpheusId.toList, + 1000L.encodeAsMorpheusId.toList, + true, + false, + 20L.encodeAsMorpheusId.toList, + null + ) ) verify(rels, cols, data) @@ -382,10 +902,10 @@ class ScanGraphTest extends MorpheusGraphTest with ElementTableCreationSupport { nHasPropertyName ) val data = Bag( - Row(1L.encodeAsMorpheusId.toList, true, false, null, 23L, "Mats"), - Row(2L.encodeAsMorpheusId.toList, true, false, null, 42L, "Martin"), - Row(3L.encodeAsMorpheusId.toList, true, false, null, 1337L, "Max"), - Row(4L.encodeAsMorpheusId.toList, true, false, null, 9L, "Stefan"), + Row(1L.encodeAsMorpheusId.toList, true, false, null, 23L, "Mats"), + Row(2L.encodeAsMorpheusId.toList, true, false, null, 42L, "Martin"), + Row(3L.encodeAsMorpheusId.toList, true, false, null, 1337L, "Max"), + Row(4L.encodeAsMorpheusId.toList, true, false, null, 9L, "Stefan"), Row(100L.encodeAsMorpheusId.toList, true, true, "C", 42L, "Alice"), Row(200L.encodeAsMorpheusId.toList, true, true, "D", 23L, "Bob"), Row(300L.encodeAsMorpheusId.toList, true, true, "F", 84L, "Eve"), @@ -407,12 +927,19 @@ class ScanGraphTest extends MorpheusGraphTest with ElementTableCreationSupport { nHasPropertyName ) val data = Bag( - Row(1L.encodeAsMorpheusId.toList, false, true, null, 23L, "Mats"), - Row(2L.encodeAsMorpheusId.toList, false, true, null, 42L, "Martin"), - Row(3L.encodeAsMorpheusId.toList, false, true, null, 1337L, "Max"), - Row(4L.encodeAsMorpheusId.toList, false, true, null, 9L, "Stefan"), + Row(1L.encodeAsMorpheusId.toList, false, true, null, 23L, "Mats"), + Row(2L.encodeAsMorpheusId.toList, false, true, null, 42L, "Martin"), + Row(3L.encodeAsMorpheusId.toList, false, true, null, 1337L, "Max"), + Row(4L.encodeAsMorpheusId.toList, false, true, null, 9L, "Stefan"), Row(100L.encodeAsMorpheusId.toList, true, true, "Node", null, null), - Row(200L.encodeAsMorpheusId.toList, true, true, "Coffeescript", null, null), + Row( + 200L.encodeAsMorpheusId.toList, + true, + true, + "Coffeescript", + null, + null + ), Row(300L.encodeAsMorpheusId.toList, true, true, "Javascript", null, null), Row(400L.encodeAsMorpheusId.toList, true, true, "Typescript", null, null) ) @@ -429,7 +956,9 @@ class ScanGraphTest extends MorpheusGraphTest with ElementTableCreationSupport { |CREATE (a:Person {name: "Alice"}) |CREATE (b:Person {name: "Bob"}) |CREATE (a)-[:KNOWS {since: 2017}]->(b) - """.stripMargin, Seq(pattern)) + """.stripMargin, + Seq(pattern) + ) val scan = graph.scanOperator(pattern) val renamedScan = scan.assignScanName( @@ -452,21 +981,36 @@ class ScanGraphTest extends MorpheusGraphTest with ElementTableCreationSupport { ) val data = Bag( - Row(0L.encodeAsMorpheusId.toList, true, "Alice", 0L.encodeAsMorpheusId.toList, 2L.encodeAsMorpheusId.toList, true, 1L.encodeAsMorpheusId.toList, 2017L) + Row( + 0L.encodeAsMorpheusId.toList, + true, + "Alice", + 0L.encodeAsMorpheusId.toList, + 2L.encodeAsMorpheusId.toList, + true, + 1L.encodeAsMorpheusId.toList, + 2017L + ) ) verify(result, cols, data) } it("can scan for TripletPatterns") { - val pattern = TripletPattern(CTNode("Person"), CTRelationship("KNOWS"), CTNode("Person")) + val pattern = TripletPattern( + CTNode("Person"), + CTRelationship("KNOWS"), + CTNode("Person") + ) val graph = initGraph( """ |CREATE (a:Person {name: "Alice"}) |CREATE (b:Person {name: "Bob"}) |CREATE (a)-[:KNOWS {since: 2017}]->(b) - """.stripMargin, Seq(pattern)) + """.stripMargin, + Seq(pattern) + ) val scan = graph.scanOperator(pattern) val result = morpheus.records.from(scan.header, scan.table) @@ -489,17 +1033,38 @@ class ScanGraphTest extends MorpheusGraphTest with ElementTableCreationSupport { ) val data = Bag( - Row(0L.encodeAsMorpheusId.toList, true, "Alice", 2L.encodeAsMorpheusId.toList, true, 0L.encodeAsMorpheusId.toList, 1L.encodeAsMorpheusId.toList, 2017L, 1L.encodeAsMorpheusId.toList, true, "Bob") + Row( + 0L.encodeAsMorpheusId.toList, + true, + "Alice", + 2L.encodeAsMorpheusId.toList, + true, + 0L.encodeAsMorpheusId.toList, + 1L.encodeAsMorpheusId.toList, + 2017L, + 1L.encodeAsMorpheusId.toList, + true, + "Bob" + ) ) verify(result, cols, data) } it("can align different complex pattern scans") { - val scanPattern= TripletPattern(CTNode("Person"), CTRelationship("KNOWS"), CTNode) + val scanPattern = + TripletPattern(CTNode("Person"), CTRelationship("KNOWS"), CTNode) - val personPattern = TripletPattern(CTNode("Person"), CTRelationship("KNOWS"), CTNode("Person")) - val animalPattern = TripletPattern(CTNode("Person"), CTRelationship("KNOWS"), CTNode("Animal")) + val personPattern = TripletPattern( + CTNode("Person"), + CTRelationship("KNOWS"), + CTNode("Person") + ) + val animalPattern = TripletPattern( + CTNode("Person"), + CTRelationship("KNOWS"), + CTNode("Animal") + ) val graph = initGraph( """ @@ -508,7 +1073,9 @@ class ScanGraphTest extends MorpheusGraphTest with ElementTableCreationSupport { |CREATE (c:Animal {name: "Garfield"}) |CREATE (a)-[:KNOWS {since: 2017}]->(b) |CREATE (a)-[:KNOWS {since: 2017}]->(c) - """.stripMargin, Seq(personPattern, animalPattern)) + """.stripMargin, + Seq(personPattern, animalPattern) + ) val scan = graph.scanOperator(scanPattern) @@ -533,8 +1100,34 @@ class ScanGraphTest extends MorpheusGraphTest with ElementTableCreationSupport { ) val data = Bag( - Row(0L.encodeAsMorpheusId.toList, true, "Alice", 3L.encodeAsMorpheusId.toList, true, 0L.encodeAsMorpheusId.toList, 1L.encodeAsMorpheusId.toList, 2017L, 1L.encodeAsMorpheusId.toList, true, false, "Bob"), - Row(0L.encodeAsMorpheusId.toList, true, "Alice", 4L.encodeAsMorpheusId.toList, true, 0L.encodeAsMorpheusId.toList, 2L.encodeAsMorpheusId.toList, 2017L, 2L.encodeAsMorpheusId.toList, false, true, "Garfield") + Row( + 0L.encodeAsMorpheusId.toList, + true, + "Alice", + 3L.encodeAsMorpheusId.toList, + true, + 0L.encodeAsMorpheusId.toList, + 1L.encodeAsMorpheusId.toList, + 2017L, + 1L.encodeAsMorpheusId.toList, + true, + false, + "Bob" + ), + Row( + 0L.encodeAsMorpheusId.toList, + true, + "Alice", + 4L.encodeAsMorpheusId.toList, + true, + 0L.encodeAsMorpheusId.toList, + 2L.encodeAsMorpheusId.toList, + 2017L, + 2L.encodeAsMorpheusId.toList, + false, + true, + "Garfield" + ) ) verify(result, cols, data) diff --git a/morpheus-testing/src/test/scala/org/opencypher/morpheus/impl/SparkSQLExprMapperTest.scala b/morpheus-testing/src/test/scala/org/opencypher/morpheus/impl/SparkSQLExprMapperTest.scala index 659678434e..14b0e55e23 100644 --- a/morpheus-testing/src/test/scala/org/opencypher/morpheus/impl/SparkSQLExprMapperTest.scala +++ b/morpheus-testing/src/test/scala/org/opencypher/morpheus/impl/SparkSQLExprMapperTest.scala @@ -53,20 +53,26 @@ class SparkSQLExprMapperTest extends BaseTestSuite with SparkSessionFixture { val id = 257L val prefix = 2.toByte val expr = PrefixId(ToId(IntegerLit(id)), prefix) - expr.eval.asInstanceOf[Array[_]].toList should equal(prefix :: id.encodeAsMorpheusId.toList) + expr.eval.asInstanceOf[Array[_]].toList should equal( + prefix :: id.encodeAsMorpheusId.toList + ) } it("converts a CypherInteger to an ID") { val id = 257L val expr = ToId(IntegerLit(id)) - expr.eval.asInstanceOf[Array[_]].toList should equal(id.encodeAsMorpheusId.toList) + expr.eval.asInstanceOf[Array[_]].toList should equal( + id.encodeAsMorpheusId.toList + ) } it("converts a CypherInteger to an ID and prefixes it") { val id = 257L val prefix = 2.toByte val expr = PrefixId(ToId(IntegerLit(id)), prefix) - expr.eval.asInstanceOf[Array[_]].toList should equal(prefix :: id.encodeAsMorpheusId.toList) + expr.eval.asInstanceOf[Array[_]].toList should equal( + prefix :: id.encodeAsMorpheusId.toList + ) } it("converts a CypherInteger literal") { @@ -80,9 +86,17 @@ class SparkSQLExprMapperTest extends BaseTestSuite with SparkSessionFixture { } val df: DataFrame = sparkSession.createDataFrame( Collections.emptyList[Row](), - StructType(Seq(StructField(header.column(vA), IntegerType), StructField(header.column(vB), IntegerType)))) + StructType( + Seq( + StructField(header.column(vA), IntegerType), + StructField(header.column(vB), IntegerType) + ) + ) + ) - implicit def extractRecordHeaderFromResult[T](tuple: (RecordHeader, T)): RecordHeader = tuple._1 + implicit def extractRecordHeaderFromResult[T]( + tuple: (RecordHeader, T) + ): RecordHeader = tuple._1 } object ExprEval { @@ -92,9 +106,13 @@ object ExprEval { def eval(implicit spark: SparkSession): Any = { val df = spark.createDataFrame( Collections.emptyList[Row](), - StructType(Seq.empty)) + StructType(Seq.empty) + ) - expr.asSparkSQLExpr(RecordHeader.empty, df, CypherMap.empty).expr.eval(InternalRow.empty) + expr + .asSparkSQLExpr(RecordHeader.empty, df, CypherMap.empty) + .expr + .eval(InternalRow.empty) } } } diff --git a/morpheus-testing/src/test/scala/org/opencypher/morpheus/impl/SparkTableTest.scala b/morpheus-testing/src/test/scala/org/opencypher/morpheus/impl/SparkTableTest.scala index dbbb7c3a3e..40510afb24 100644 --- a/morpheus-testing/src/test/scala/org/opencypher/morpheus/impl/SparkTableTest.scala +++ b/morpheus-testing/src/test/scala/org/opencypher/morpheus/impl/SparkTableTest.scala @@ -42,37 +42,91 @@ class SparkTableTest extends MorpheusTestSuite with Matchers with ScalaCheckDriv it("it should cast integer columns to long") { - val df = sparkSession.createDataFrame(List( - Row(1, 2L, Array(42), Array(42), Array(42L), Row(42, 42L)) - ).asJava, StructType(Seq( - StructField("a", IntegerType, nullable = true), - StructField("b", LongType, nullable = false), - StructField("c", ArrayType(IntegerType, containsNull = true), nullable = true), - StructField("d", ArrayType(IntegerType, containsNull = false), nullable = false), - StructField("e", ArrayType(LongType, containsNull = false), nullable = false), - StructField("f", StructType(Seq( - StructField("foo", IntegerType, true), - StructField("bar", LongType, false) - )), nullable = true) - ))) + val df = sparkSession.createDataFrame( + List( + Row(1, 2L, Array(42), Array(42), Array(42L), Row(42, 42L)) + ).asJava, + StructType( + Seq( + StructField("a", IntegerType, nullable = true), + StructField("b", LongType, nullable = false), + StructField( + "c", + ArrayType(IntegerType, containsNull = true), + nullable = true + ), + StructField( + "d", + ArrayType(IntegerType, containsNull = false), + nullable = false + ), + StructField( + "e", + ArrayType(LongType, containsNull = false), + nullable = false + ), + StructField( + "f", + StructType( + Seq( + StructField("foo", IntegerType, true), + StructField("bar", LongType, false) + ) + ), + nullable = true + ) + ) + ) + ) val updatedDf = df.castToLong - updatedDf.schema should equal(StructType(Seq( - StructField("a", LongType, nullable = true), - StructField("b", LongType, nullable = false), - StructField("c", ArrayType(LongType, containsNull = true), nullable = true), - StructField("d", ArrayType(LongType, containsNull = false), nullable = false), - StructField("e", ArrayType(LongType, containsNull = false), nullable = false), - StructField("f", StructType(Seq( - StructField("foo", LongType, true), - StructField("bar", LongType, true) - )), nullable = false) - ))) + updatedDf.schema should equal( + StructType( + Seq( + StructField("a", LongType, nullable = true), + StructField("b", LongType, nullable = false), + StructField( + "c", + ArrayType(LongType, containsNull = true), + nullable = true + ), + StructField( + "d", + ArrayType(LongType, containsNull = false), + nullable = false + ), + StructField( + "e", + ArrayType(LongType, containsNull = false), + nullable = false + ), + StructField( + "f", + StructType( + Seq( + StructField("foo", LongType, true), + StructField("bar", LongType, true) + ) + ), + nullable = false + ) + ) + ) + ) - updatedDf.collect().toBag should equal(Bag( - Row(1L, 2L, new ofLong(Array(42L)), new ofLong(Array(42L)), new ofLong(Array(42L)), Row(42L, 42L)) - )) + updatedDf.collect().toBag should equal( + Bag( + Row( + 1L, + 2L, + new ofLong(Array(42L)), + new ofLong(Array(42L)), + new ofLong(Array(42L)), + Row(42L, 42L) + ) + ) + ) } // These tests verifies that https://issues.apache.org/jira/browse/SPARK-26572 is still fixed @@ -81,13 +135,21 @@ class SparkTableTest extends MorpheusTestSuite with Matchers with ScalaCheckDriv val baseTable = Seq(1, 1).toDF("idx") // Uses Spark distinct - val distinctWithId = baseTable.distinct.withColumn("id", functions.monotonically_increasing_id()) + val distinctWithId = baseTable.distinct.withColumn( + "id", + functions.monotonically_increasing_id() + ) val monotonicallyOnLeft = distinctWithId.join(baseTable, "idx") // Bug in Spark: "monotonically_increasing_id" is pushed down when it shouldn't be. Push down only happens when the // DF containing the "monotonically_increasing_id" expression is on the left side of the join. - monotonicallyOnLeft.select("id").collect().map(_.get(0)).distinct.length shouldBe 1 + monotonicallyOnLeft + .select("id") + .collect() + .map(_.get(0)) + .distinct + .length shouldBe 1 } } diff --git a/morpheus-testing/src/test/scala/org/opencypher/morpheus/impl/UnionGraphTest.scala b/morpheus-testing/src/test/scala/org/opencypher/morpheus/impl/UnionGraphTest.scala index 3b3a10bc37..8f3de3c8b5 100644 --- a/morpheus-testing/src/test/scala/org/opencypher/morpheus/impl/UnionGraphTest.scala +++ b/morpheus-testing/src/test/scala/org/opencypher/morpheus/impl/UnionGraphTest.scala @@ -28,13 +28,18 @@ package org.opencypher.morpheus.impl import org.apache.spark.sql.Row import org.opencypher.morpheus.api.value.MorpheusElement._ -import org.opencypher.morpheus.testing.fixture.{GraphConstructionFixture, RecordsVerificationFixture, TeamDataFixture} +import org.opencypher.morpheus.testing.fixture.{ + GraphConstructionFixture, + RecordsVerificationFixture, + TeamDataFixture +} import org.opencypher.okapi.testing.Bag -class UnionGraphTest extends MorpheusGraphTest - with GraphConstructionFixture - with RecordsVerificationFixture - with TeamDataFixture { +class UnionGraphTest + extends MorpheusGraphTest + with GraphConstructionFixture + with RecordsVerificationFixture + with TeamDataFixture { import MorpheusGraphTestData._ @@ -42,7 +47,11 @@ class UnionGraphTest extends MorpheusGraphTest def testGraph2 = initGraph("CREATE (:Person {name: 'Phil'})") it("supports UNION ALL") { - testGraph1.unionAll(testGraph2).cypher("""MATCH (n) RETURN DISTINCT id(n)""").records.size should equal(2) + testGraph1 + .unionAll(testGraph2) + .cypher("""MATCH (n) RETURN DISTINCT id(n)""") + .records + .size should equal(2) } it("supports UNION ALL on identical graphs") { @@ -79,8 +88,24 @@ class UnionGraphTest extends MorpheusGraphTest Row(2L.withPrefix(0).toList, false, true, 1337L, "Max", null, null), Row(3L.withPrefix(0).toList, false, true, 9L, "Stefan", null, null), Row(0L.withPrefix(1).toList, true, false, null, null, "1984", 1949L), - Row(1L.withPrefix(1).toList, true, false, null, null, "Cryptonomicon", 1999L), - Row(2L.withPrefix(1).toList, true, false, null, null, "The Eye of the World", 1990L), + Row( + 1L.withPrefix(1).toList, + true, + false, + null, + null, + "Cryptonomicon", + 1999L + ), + Row( + 2L.withPrefix(1).toList, + true, + false, + null, + null, + "The Eye of the World", + 1990L + ), Row(3L.withPrefix(1).toList, true, false, null, null, "The Circle", 2013L) ) diff --git a/morpheus-testing/src/test/scala/org/opencypher/morpheus/impl/acceptance/AggregationTests.scala b/morpheus-testing/src/test/scala/org/opencypher/morpheus/impl/acceptance/AggregationTests.scala index 28aae86aac..c5a812dd12 100644 --- a/morpheus-testing/src/test/scala/org/opencypher/morpheus/impl/acceptance/AggregationTests.scala +++ b/morpheus-testing/src/test/scala/org/opencypher/morpheus/impl/acceptance/AggregationTests.scala @@ -41,9 +41,11 @@ class AggregationTests extends MorpheusTestSuite with ScanGraphInit { val result = graph.cypher("MATCH (n) WITH AVG(n.val) AS res RETURN res") - result.records.collect.toBag should equal(Bag( - CypherMap("res" -> 4.0) - )) + result.records.collect.toBag should equal( + Bag( + CypherMap("res" -> 4.0) + ) + ) } it("avg(prop) with integers in RETURN") { @@ -51,9 +53,11 @@ class AggregationTests extends MorpheusTestSuite with ScanGraphInit { val result = graph.cypher("MATCH (n) RETURN AVG(n.val) AS res") - result.records.collect.toBag should equal(Bag( - CypherMap("res" -> 4.0) - )) + result.records.collect.toBag should equal( + Bag( + CypherMap("res" -> 4.0) + ) + ) } it("avg(prop) with integers in RETURN without alias") { @@ -61,9 +65,11 @@ class AggregationTests extends MorpheusTestSuite with ScanGraphInit { val result = graph.cypher("MATCH (n) RETURN AVG(n.val)") - result.records.toMaps should equal(Bag( - CypherMap("AVG(n.val)" -> 4.0) - )) + result.records.toMaps should equal( + Bag( + CypherMap("AVG(n.val)" -> 4.0) + ) + ) } it("avg(prop) with floats in WITH") { @@ -71,9 +77,11 @@ class AggregationTests extends MorpheusTestSuite with ScanGraphInit { val result = graph.cypher("MATCH (n) WITH AVG(n.val) AS res RETURN res") - result.records.toMaps should equal(Bag( - CypherMap("res" -> 3.5) - )) + result.records.toMaps should equal( + Bag( + CypherMap("res" -> 3.5) + ) + ) } it("avg(prop) with floats in RETURN") { @@ -81,9 +89,11 @@ class AggregationTests extends MorpheusTestSuite with ScanGraphInit { val result = graph.cypher("MATCH (n) RETURN AVG(n.val) AS res") - result.records.toMaps should equal(Bag( - CypherMap("res" -> 3.5) - )) + result.records.toMaps should equal( + Bag( + CypherMap("res" -> 3.5) + ) + ) } it("avg(prop) with single null value in WITH") { @@ -91,9 +101,11 @@ class AggregationTests extends MorpheusTestSuite with ScanGraphInit { val result = graph.cypher("MATCH (n) WITH AVG(n.val) AS res RETURN res") - result.records.toMaps should equal(Bag( - CypherMap("res" -> 32.5) - )) + result.records.toMaps should equal( + Bag( + CypherMap("res" -> 32.5) + ) + ) } it("avg(prop) with single null value in RETURN") { @@ -101,9 +113,11 @@ class AggregationTests extends MorpheusTestSuite with ScanGraphInit { val result = graph.cypher("MATCH (n) RETURN AVG(n.val) AS res") - result.records.toMaps should equal(Bag( - CypherMap("res" -> 32.5) - )) + result.records.toMaps should equal( + Bag( + CypherMap("res" -> 32.5) + ) + ) } it("avg(prop) with only null values in WITH") { @@ -111,9 +125,11 @@ class AggregationTests extends MorpheusTestSuite with ScanGraphInit { val result = graph.cypher("MATCH (n) WITH AVG(n.val) AS res RETURN res") - result.records.toMaps should equal(Bag( - CypherMap("res" -> null) - )) + result.records.toMaps should equal( + Bag( + CypherMap("res" -> null) + ) + ) } it("avg(prop) with only null values in RETURN") { @@ -121,17 +137,23 @@ class AggregationTests extends MorpheusTestSuite with ScanGraphInit { val result = graph.cypher("MATCH (n) RETURN AVG(n.val) AS res") - result.records.toMaps should equal(Bag( - CypherMap("res" -> null) - )) + result.records.toMaps should equal( + Bag( + CypherMap("res" -> null) + ) + ) } it("avg on durations") { - val result = morpheus.graphs.empty.cypher("UNWIND [duration('P1DT12H'), duration('P1DT20H')] AS d RETURN AVG(d) AS res") + val result = morpheus.graphs.empty.cypher( + "UNWIND [duration('P1DT12H'), duration('P1DT20H')] AS d RETURN AVG(d) AS res" + ) - result.records.toMaps should equal(Bag( - CypherMap("res" -> Duration(days = 1, hours = 16)) - )) + result.records.toMaps should equal( + Bag( + CypherMap("res" -> Duration(days = 1, hours = 16)) + ) + ) } } @@ -142,9 +164,11 @@ class AggregationTests extends MorpheusTestSuite with ScanGraphInit { val result = graph.cypher("MATCH (n) WITH count(*) AS nbrRows RETURN nbrRows") - result.records.toMaps should equal(Bag( - CypherMap("nbrRows" -> 6) - )) + result.records.toMaps should equal( + Bag( + CypherMap("nbrRows" -> 6) + ) + ) } it("count(*) in RETURN") { @@ -152,9 +176,11 @@ class AggregationTests extends MorpheusTestSuite with ScanGraphInit { val result = graph.cypher("MATCH (n) RETURN count(*) AS nbrRows") - result.records.toMaps should equal(Bag( - CypherMap("nbrRows" -> 6) - )) + result.records.toMaps should equal( + Bag( + CypherMap("nbrRows" -> 6) + ) + ) } it("count(n) in RETURN") { @@ -162,9 +188,11 @@ class AggregationTests extends MorpheusTestSuite with ScanGraphInit { val result = graph.cypher("MATCH (n) RETURN count(n) AS nbrRows") - result.records.toMaps should equal(Bag( - CypherMap("nbrRows" -> 6) - )) + result.records.toMaps should equal( + Bag( + CypherMap("nbrRows" -> 6) + ) + ) } it("count(n) in RETURN without alias") { @@ -172,9 +200,11 @@ class AggregationTests extends MorpheusTestSuite with ScanGraphInit { val result = graph.cypher("MATCH (n) RETURN count(n)") - result.records.toMaps should equal(Bag( - CypherMap("count(n)" -> 6) - )) + result.records.toMaps should equal( + Bag( + CypherMap("count(n)" -> 6) + ) + ) } it("count(*) in return without alias") { @@ -182,9 +212,11 @@ class AggregationTests extends MorpheusTestSuite with ScanGraphInit { val result = graph.cypher("MATCH (n) RETURN count(*)") - result.records.toMaps should equal(Bag( - CypherMap("count(*)" -> 6) - )) + result.records.toMaps should equal( + Bag( + CypherMap("count(*)" -> 6) + ) + ) } it("simple count(prop)") { @@ -192,9 +224,11 @@ class AggregationTests extends MorpheusTestSuite with ScanGraphInit { val result = graph.cypher("MATCH (n) WITH count(n.name) AS nonNullNames RETURN nonNullNames") - result.records.toMaps should equal(Bag( - CypherMap("nonNullNames" -> 3) - )) + result.records.toMaps should equal( + Bag( + CypherMap("nonNullNames" -> 3) + ) + ) } it("simple count(node)") { @@ -202,19 +236,25 @@ class AggregationTests extends MorpheusTestSuite with ScanGraphInit { val result = graph.cypher("MATCH (n) WITH count(n) AS nodes RETURN nodes") - result.records.toMaps should equal(Bag( - CypherMap("nodes" -> 6) - )) + result.records.toMaps should equal( + Bag( + CypherMap("nodes" -> 6) + ) + ) } it("count after expand") { - val graph = initGraph("CREATE ({name: 'foo'})-[:A]->(:B), ({name: 'bar'}), (), ()-[:A]->(:B), (), ({name: 'baz'})") + val graph = initGraph( + "CREATE ({name: 'foo'})-[:A]->(:B), ({name: 'bar'}), (), ()-[:A]->(:B), (), ({name: 'baz'})" + ) val result = graph.cypher("MATCH (n)-->(b:B) WITH count(b) AS nodes RETURN nodes") - result.records.toMaps should equal(Bag( - CypherMap("nodes" -> 2) - )) + result.records.toMaps should equal( + Bag( + CypherMap("nodes" -> 2) + ) + ) } it("count() with grouping in RETURN clause") { @@ -222,58 +262,69 @@ class AggregationTests extends MorpheusTestSuite with ScanGraphInit { val result = graph.cypher("MATCH (n) RETURN n.name AS name, count(*) AS amount") - result.records.toMaps should equal(Bag( - CypherMap("name" -> "foo", "amount" -> 2), - CypherMap("name" -> null, "amount" -> 3), - CypherMap("name" -> "baz", "amount" -> 1) - )) + result.records.toMaps should equal( + Bag( + CypherMap("name" -> "foo", "amount" -> 2), + CypherMap("name" -> null, "amount" -> 3), + CypherMap("name" -> "baz", "amount" -> 1) + ) + ) } it("count() with grouping in WITH clause") { val graph = initGraph("CREATE ({name: 'foo'}), ({name: 'foo'}), (), (), (), ({name: 'baz'})") - val result = graph.cypher("MATCH (n) WITH n.name AS name, count(*) AS amount RETURN name, amount") + val result = + graph.cypher("MATCH (n) WITH n.name AS name, count(*) AS amount RETURN name, amount") - result.records.toMaps should equal(Bag( - CypherMap("name" -> "foo", "amount" -> 2), - CypherMap("name" -> null, "amount" -> 3), - CypherMap("name" -> "baz", "amount" -> 1) - )) + result.records.toMaps should equal( + Bag( + CypherMap("name" -> "foo", "amount" -> 2), + CypherMap("name" -> null, "amount" -> 3), + CypherMap("name" -> "baz", "amount" -> 1) + ) + ) } it("count() with grouping on multiple keys") { - val graph = initGraph("CREATE ({name: 'foo', age: 42}), ({name: 'foo', age: 42}), ({name: 'foo', age: 23}), (), (), ({name: 'baz', age: 23})") + val graph = initGraph( + "CREATE ({name: 'foo', age: 42}), ({name: 'foo', age: 42}), ({name: 'foo', age: 23}), (), (), ({name: 'baz', age: 23})" + ) val result = graph - .cypher("MATCH (n) WITH n.name AS name, n.age AS age, count(*) AS amount RETURN name, age, amount") + .cypher( + "MATCH (n) WITH n.name AS name, n.age AS age, count(*) AS amount RETURN name, age, amount" + ) - result.records.toMaps should equal(Bag( - CypherMap("name" -> "foo", "age" -> 23, "amount" -> 1), - CypherMap("name" -> "foo", "age" -> 42, "amount" -> 2), - CypherMap("name" -> "baz", "age" -> 23, "amount" -> 1), - CypherMap("name" -> null, "age" -> null, "amount" -> 2) - )) + result.records.toMaps should equal( + Bag( + CypherMap("name" -> "foo", "age" -> 23, "amount" -> 1), + CypherMap("name" -> "foo", "age" -> 42, "amount" -> 2), + CypherMap("name" -> "baz", "age" -> 23, "amount" -> 1), + CypherMap("name" -> null, "age" -> null, "amount" -> 2) + ) + ) } it("counts distinct with grouping") { - val graph = initGraph( - """ + val graph = initGraph(""" |CREATE (a:Start{id: 1}) |CREATE (a)-[:REL]->({val: "foo"}) |CREATE (a)-[:REL]->({val: "foo"}) """.stripMargin) - val result = graph.cypher( - """ + val result = graph.cypher(""" |MATCH (a:Start)-->(b) | |RETURN a.id, | count(distinct b.val) AS val """.stripMargin) - result.records.toMaps should equal(Bag( - CypherMap("a.id" -> 1, "val" -> 1) - )) + result.records.toMaps should equal( + Bag( + CypherMap("a.id" -> 1, "val" -> 1) + ) + ) } } @@ -284,9 +335,11 @@ class AggregationTests extends MorpheusTestSuite with ScanGraphInit { val result = graph.cypher("MATCH (n) WITH MIN(n.val) AS res RETURN res") - result.records.toMaps should equal(Bag( - CypherMap("res" -> 23L) - )) + result.records.toMaps should equal( + Bag( + CypherMap("res" -> 23L) + ) + ) } it("min(prop) in RETURN") { @@ -294,9 +347,11 @@ class AggregationTests extends MorpheusTestSuite with ScanGraphInit { val result = graph.cypher("MATCH (n) RETURN MIN(n.val) AS res") - result.records.toMaps should equal(Bag( - CypherMap("res" -> 23L) - )) + result.records.toMaps should equal( + Bag( + CypherMap("res" -> 23L) + ) + ) } it("min(prop) with single null value in WITH") { @@ -304,9 +359,11 @@ class AggregationTests extends MorpheusTestSuite with ScanGraphInit { val result = graph.cypher("MATCH (n) WITH MIN(n.val) AS res RETURN res") - result.records.toMaps should equal(Bag( - CypherMap("res" -> 23L) - )) + result.records.toMaps should equal( + Bag( + CypherMap("res" -> 23L) + ) + ) } it("min(prop) with single null value in RETURN") { @@ -314,9 +371,11 @@ class AggregationTests extends MorpheusTestSuite with ScanGraphInit { val result = graph.cypher("MATCH (n) RETURN MIN(n.val) AS res") - result.records.toMaps should equal(Bag( - CypherMap("res" -> 23L) - )) + result.records.toMaps should equal( + Bag( + CypherMap("res" -> 23L) + ) + ) } it("min(prop) with single null value in RETURN without alias") { @@ -324,9 +383,11 @@ class AggregationTests extends MorpheusTestSuite with ScanGraphInit { val result = graph.cypher("MATCH (n) RETURN MIN(n.val)") - result.records.toMaps should equal(Bag( - CypherMap("MIN(n.val)" -> 23L) - )) + result.records.toMaps should equal( + Bag( + CypherMap("MIN(n.val)" -> 23L) + ) + ) } it("min(prop) with only null values in WITH") { @@ -334,9 +395,11 @@ class AggregationTests extends MorpheusTestSuite with ScanGraphInit { val result = graph.cypher("MATCH (n) WITH MIN(n.val) AS res RETURN res") - result.records.toMaps should equal(Bag( - CypherMap("res" -> null) - )) + result.records.toMaps should equal( + Bag( + CypherMap("res" -> null) + ) + ) } it("min(prop) with only null values in RETURN") { @@ -344,41 +407,59 @@ class AggregationTests extends MorpheusTestSuite with ScanGraphInit { val result = graph.cypher("MATCH (n) RETURN MIN(n.val) AS res") - result.records.toMaps should equal(Bag( - CypherMap("res" -> null) - )) + result.records.toMaps should equal( + Bag( + CypherMap("res" -> null) + ) + ) } it("min on dates") { - val result = morpheus.graphs.empty.cypher("UNWIND [date('2018-01-01'), date('2019-01-01')] AS d RETURN MIN(d) AS res") + val result = morpheus.graphs.empty.cypher( + "UNWIND [date('2018-01-01'), date('2019-01-01')] AS d RETURN MIN(d) AS res" + ) - result.records.toMaps should equal(Bag( - CypherMap("res" -> java.time.LocalDate.parse("2018-01-01")) - )) + result.records.toMaps should equal( + Bag( + CypherMap("res" -> java.time.LocalDate.parse("2018-01-01")) + ) + ) } it("min on datetimes") { - val result = morpheus.graphs.empty.cypher("UNWIND [localdatetime('2010-10-10T12:00'), localdatetime('2010-10-10T12:01')] AS d RETURN MIN(d) AS res") + val result = morpheus.graphs.empty.cypher( + "UNWIND [localdatetime('2010-10-10T12:00'), localdatetime('2010-10-10T12:01')] AS d RETURN MIN(d) AS res" + ) - result.records.toMaps should equal(Bag( - CypherMap("res" -> java.time.LocalDateTime.parse("2010-10-10T12:00")) - )) + result.records.toMaps should equal( + Bag( + CypherMap("res" -> java.time.LocalDateTime.parse("2010-10-10T12:00")) + ) + ) } it("min on durations") { - val result = morpheus.graphs.empty.cypher("UNWIND [duration('P1DT12H'), duration('P1DT200H')] AS d RETURN MIN(d) AS res") + val result = morpheus.graphs.empty.cypher( + "UNWIND [duration('P1DT12H'), duration('P1DT200H')] AS d RETURN MIN(d) AS res" + ) - result.records.toMaps should equal(Bag( - CypherMap("res" -> Duration(days = 1, hours = 12)) - )) + result.records.toMaps should equal( + Bag( + CypherMap("res" -> Duration(days = 1, hours = 12)) + ) + ) } it("min on combination of temporal types") { - val result = morpheus.graphs.empty.cypher("UNWIND [date('2018-01-01'), localdatetime('2010-10-10T12:01')] AS d RETURN MIN(d) AS res") + val result = morpheus.graphs.empty.cypher( + "UNWIND [date('2018-01-01'), localdatetime('2010-10-10T12:01')] AS d RETURN MIN(d) AS res" + ) - result.records.toMaps should equal(Bag( - CypherMap("res" -> java.time.LocalDateTime.parse("2010-10-10T12:01")) - )) + result.records.toMaps should equal( + Bag( + CypherMap("res" -> java.time.LocalDateTime.parse("2010-10-10T12:01")) + ) + ) } } @@ -389,9 +470,11 @@ class AggregationTests extends MorpheusTestSuite with ScanGraphInit { val result = graph.cypher("MATCH (n) WITH MAX(n.val) AS res RETURN res") - result.records.toMaps should equal(Bag( - CypherMap("res" -> 84L) - )) + result.records.toMaps should equal( + Bag( + CypherMap("res" -> 84L) + ) + ) } it("max(prop) in RETURN") { @@ -399,9 +482,11 @@ class AggregationTests extends MorpheusTestSuite with ScanGraphInit { val result = graph.cypher("MATCH (n) RETURN MAX(n.val) AS res") - result.records.toMaps should equal(Bag( - CypherMap("res" -> 84L) - )) + result.records.toMaps should equal( + Bag( + CypherMap("res" -> 84L) + ) + ) } it("max(prop) with single null value in WITH") { @@ -409,9 +494,11 @@ class AggregationTests extends MorpheusTestSuite with ScanGraphInit { val result = graph.cypher("MATCH (n) WITH MAX(n.val) AS res RETURN res") - result.records.toMaps should equal(Bag( - CypherMap("res" -> 42L) - )) + result.records.toMaps should equal( + Bag( + CypherMap("res" -> 42L) + ) + ) } it("max(prop) with single null value in RETURN") { @@ -419,9 +506,11 @@ class AggregationTests extends MorpheusTestSuite with ScanGraphInit { val result = graph.cypher("MATCH (n) RETURN MAX(n.val) AS res") - result.records.toMaps should equal(Bag( - CypherMap("res" -> 42L) - )) + result.records.toMaps should equal( + Bag( + CypherMap("res" -> 42L) + ) + ) } it("max(prop) with single null value in RETURN without alias") { @@ -429,9 +518,11 @@ class AggregationTests extends MorpheusTestSuite with ScanGraphInit { val result = graph.cypher("MATCH (n) RETURN MAX(n.val)") - result.records.toMaps should equal(Bag( - CypherMap("MAX(n.val)" -> 42L) - )) + result.records.toMaps should equal( + Bag( + CypherMap("MAX(n.val)" -> 42L) + ) + ) } it("simple max(prop) with only null values in WITH") { @@ -439,9 +530,11 @@ class AggregationTests extends MorpheusTestSuite with ScanGraphInit { val result = graph.cypher("MATCH (n) WITH MAX(n.val) AS res RETURN res") - result.records.toMaps should equal(Bag( - CypherMap("res" -> null) - )) + result.records.toMaps should equal( + Bag( + CypherMap("res" -> null) + ) + ) } it("simple max(prop) with only null values in RETURN") { @@ -449,46 +542,63 @@ class AggregationTests extends MorpheusTestSuite with ScanGraphInit { val result = graph.cypher("MATCH (n) RETURN MAX(n.val) AS res") - result.records.toMaps should equal(Bag( - CypherMap("res" -> null) - )) + result.records.toMaps should equal( + Bag( + CypherMap("res" -> null) + ) + ) } it("max on dates") { - val result = morpheus.graphs.empty.cypher("UNWIND [date('2018-01-01'), date('2019-01-01')] AS d RETURN MAX(d) AS res") + val result = morpheus.graphs.empty.cypher( + "UNWIND [date('2018-01-01'), date('2019-01-01')] AS d RETURN MAX(d) AS res" + ) - result.records.toMaps should equal(Bag( - CypherMap("res" -> java.time.LocalDate.parse("2019-01-01")) - )) + result.records.toMaps should equal( + Bag( + CypherMap("res" -> java.time.LocalDate.parse("2019-01-01")) + ) + ) } it("max on datetimes") { - val result = morpheus.graphs.empty.cypher("UNWIND [localdatetime('2010-10-10T12:00'), localdatetime('2010-10-10T12:01')] AS d RETURN MAX(d) AS res") + val result = morpheus.graphs.empty.cypher( + "UNWIND [localdatetime('2010-10-10T12:00'), localdatetime('2010-10-10T12:01')] AS d RETURN MAX(d) AS res" + ) - result.records.toMaps should equal(Bag( - CypherMap("res" -> java.time.LocalDateTime.parse("2010-10-10T12:01")) - )) + result.records.toMaps should equal( + Bag( + CypherMap("res" -> java.time.LocalDateTime.parse("2010-10-10T12:01")) + ) + ) } it("max on durations") { - val result = morpheus.graphs.empty.cypher("UNWIND [duration('P10DT12H'), duration('P1DT24H')] AS d RETURN MAX(d) AS res") + val result = morpheus.graphs.empty.cypher( + "UNWIND [duration('P10DT12H'), duration('P1DT24H')] AS d RETURN MAX(d) AS res" + ) - result.records.toMaps should equal(Bag( - CypherMap("res" -> Duration(days = 10, hours = 12)) - )) + result.records.toMaps should equal( + Bag( + CypherMap("res" -> Duration(days = 10, hours = 12)) + ) + ) } it("max on combination of temporal types") { - val result = morpheus.graphs.empty.cypher("UNWIND [date('2018-01-01'), localdatetime('2010-10-10T12:01')] AS d RETURN MAX(d) AS res") + val result = morpheus.graphs.empty.cypher( + "UNWIND [date('2018-01-01'), localdatetime('2010-10-10T12:01')] AS d RETURN MAX(d) AS res" + ) - result.records.toMaps should equal(Bag( - CypherMap("res" -> java.time.LocalDateTime.parse("2018-01-01T00:00")) - )) + result.records.toMaps should equal( + Bag( + CypherMap("res" -> java.time.LocalDateTime.parse("2018-01-01T00:00")) + ) + ) } } - describe("SUM") { it("sum(prop) with integers in WITH") { @@ -496,9 +606,11 @@ class AggregationTests extends MorpheusTestSuite with ScanGraphInit { val result = graph.cypher("MATCH (n) WITH SUM(n.val) AS res RETURN res") - result.records.toMaps should equal(Bag( - CypherMap("res" -> 12) - )) + result.records.toMaps should equal( + Bag( + CypherMap("res" -> 12) + ) + ) } it("sum(prop) with integers in RETURN") { @@ -506,9 +618,11 @@ class AggregationTests extends MorpheusTestSuite with ScanGraphInit { val result = graph.cypher("MATCH (n) RETURN SUM(n.val) AS res") - result.records.toMaps should equal(Bag( - CypherMap("res" -> 12) - )) + result.records.toMaps should equal( + Bag( + CypherMap("res" -> 12) + ) + ) } it("sum(prop) with floats in WITH") { @@ -516,9 +630,11 @@ class AggregationTests extends MorpheusTestSuite with ScanGraphInit { val result = graph.cypher("MATCH (n) WITH SUM(n.val) AS res RETURN res") - result.records.toMaps should equal(Bag( - CypherMap("res" -> 10.5) - )) + result.records.toMaps should equal( + Bag( + CypherMap("res" -> 10.5) + ) + ) } it("sum(prop) with floats in RETURN") { @@ -526,9 +642,11 @@ class AggregationTests extends MorpheusTestSuite with ScanGraphInit { val result = graph.cypher("MATCH (n) RETURN SUM(n.val) AS res") - result.records.toMaps should equal(Bag( - CypherMap("res" -> 10.5) - )) + result.records.toMaps should equal( + Bag( + CypherMap("res" -> 10.5) + ) + ) } it("sum(prop) with floats in RETURN without alias") { @@ -536,9 +654,11 @@ class AggregationTests extends MorpheusTestSuite with ScanGraphInit { val result = graph.cypher("MATCH (n) RETURN SUM(n.val)") - result.records.toMaps should equal(Bag( - CypherMap("SUM(n.val)" -> 10.5) - )) + result.records.toMaps should equal( + Bag( + CypherMap("SUM(n.val)" -> 10.5) + ) + ) } it("simple sum(prop) with single null value in WITH") { @@ -546,9 +666,11 @@ class AggregationTests extends MorpheusTestSuite with ScanGraphInit { val result = graph.cypher("MATCH (n) WITH SUM(n.val) AS res RETURN res") - result.records.toMaps should equal(Bag( - CypherMap("res" -> 65.0) - )) + result.records.toMaps should equal( + Bag( + CypherMap("res" -> 65.0) + ) + ) } it("simple sum(prop) with single null value in RETURN") { @@ -556,9 +678,11 @@ class AggregationTests extends MorpheusTestSuite with ScanGraphInit { val result = graph.cypher("MATCH (n) RETURN SUM(n.val) AS res") - result.records.toMaps should equal(Bag( - CypherMap("res" -> 65.0) - )) + result.records.toMaps should equal( + Bag( + CypherMap("res" -> 65.0) + ) + ) } it("simple sum(prop) with only null values in WITH") { @@ -566,9 +690,11 @@ class AggregationTests extends MorpheusTestSuite with ScanGraphInit { val result = graph.cypher("MATCH (n) WITH SUM(n.val) AS res RETURN res") - result.records.toMaps should equal(Bag( - CypherMap("res" -> null) - )) + result.records.toMaps should equal( + Bag( + CypherMap("res" -> null) + ) + ) } it("simple sum(prop) with only null values in RETURN") { @@ -576,156 +702,220 @@ class AggregationTests extends MorpheusTestSuite with ScanGraphInit { val result = graph.cypher("MATCH (n) RETURN SUM(n.val) AS res") - result.records.toMaps should equal(Bag( - CypherMap("res" -> null) - )) + result.records.toMaps should equal( + Bag( + CypherMap("res" -> null) + ) + ) } it("sum on durations") { - val result = morpheus.graphs.empty.cypher("UNWIND [duration('P1DT12H'), duration('P1DT24H')] AS d RETURN SUM(d) AS res") + val result = morpheus.graphs.empty.cypher( + "UNWIND [duration('P1DT12H'), duration('P1DT24H')] AS d RETURN SUM(d) AS res" + ) - result.records.toMaps should equal(Bag( - CypherMap("res" -> Duration(days = 3, hours = 12)) - )) + result.records.toMaps should equal( + Bag( + CypherMap("res" -> Duration(days = 3, hours = 12)) + ) + ) } } describe("stDev") { it("stDev on floats") { - val result = morpheus.graphs.empty.cypher("UNWIND [98.17, 112.3, 102.6, 94.3, 108.1] AS numbers RETURN round(stDev(numbers)*1000)/1000.0 AS res") + val result = morpheus.graphs.empty.cypher( + "UNWIND [98.17, 112.3, 102.6, 94.3, 108.1] AS numbers RETURN round(stDev(numbers)*1000)/1000.0 AS res" + ) - result.records.toMaps should equal(Bag( - CypherMap("res" -> 7.274) - )) + result.records.toMaps should equal( + Bag( + CypherMap("res" -> 7.274) + ) + ) } it("stDev on nullable list of floats") { - val result = morpheus.graphs.empty.cypher("UNWIND [98.17, null, 102.6, 94.3, 108.1] AS numbers RETURN round(stDev(numbers)*1000)/1000.0 AS res") + val result = morpheus.graphs.empty.cypher( + "UNWIND [98.17, null, 102.6, 94.3, 108.1] AS numbers RETURN round(stDev(numbers)*1000)/1000.0 AS res" + ) - result.records.toMaps should equal(Bag( - CypherMap("res" -> 5.936) - )) + result.records.toMaps should equal( + Bag( + CypherMap("res" -> 5.936) + ) + ) } it("stDev on null") { val result = morpheus.graphs.empty.cypher("RETURN stDev(null) AS res") // TODO: Spark returns null for stDev(null) instead of 0 - result.records.toMaps should equal(Bag( - CypherMap("res" -> null) - )) + result.records.toMaps should equal( + Bag( + CypherMap("res" -> null) + ) + ) } } describe("stDevP") { it("stDevP on floats") { - val result = morpheus.graphs.empty.cypher("UNWIND [98.17, 112.3, 102.6, 94.3, 108.1] AS numbers RETURN round(stDevP(numbers)*1000)/1000.0 AS res") + val result = morpheus.graphs.empty.cypher( + "UNWIND [98.17, 112.3, 102.6, 94.3, 108.1] AS numbers RETURN round(stDevP(numbers)*1000)/1000.0 AS res" + ) - result.records.toMaps should equal(Bag( - CypherMap("res" -> 6.506) - )) + result.records.toMaps should equal( + Bag( + CypherMap("res" -> 6.506) + ) + ) } it("stDevP on nullable list of floats") { - val result = morpheus.graphs.empty.cypher("UNWIND [98.17, null, 102.6, 94.3, 108.1] AS numbers RETURN round(stDevP(numbers)*1000)/1000.0 AS res") + val result = morpheus.graphs.empty.cypher( + "UNWIND [98.17, null, 102.6, 94.3, 108.1] AS numbers RETURN round(stDevP(numbers)*1000)/1000.0 AS res" + ) - result.records.toMaps should equal(Bag( - CypherMap("res" -> 5.140) - )) + result.records.toMaps should equal( + Bag( + CypherMap("res" -> 5.140) + ) + ) } it("stDevP on null") { val result = morpheus.graphs.empty.cypher("RETURN stDevP(null) AS res") // TODO: Spark returns null for stDevP(null) instead of 0 - result.records.toMaps should equal(Bag( - CypherMap("res" -> null) - )) + result.records.toMaps should equal( + Bag( + CypherMap("res" -> null) + ) + ) } } describe("percentileCont") { - it("percentileContil on integers"){ - val result = morpheus.graphs.empty.cypher("UNWIND [1,2] AS values RETURN percentileCont(values, 0.5) AS res") + it("percentileContil on integers") { + val result = morpheus.graphs.empty.cypher( + "UNWIND [1,2] AS values RETURN percentileCont(values, 0.5) AS res" + ) - result.records.toMaps should equal(Bag( - CypherMap("res" -> 1.5) - )) + result.records.toMaps should equal( + Bag( + CypherMap("res" -> 1.5) + ) + ) } - it("percentileContil with 1.0 as percentile"){ - val result = morpheus.graphs.empty.cypher("UNWIND [2,10,5,6] AS values RETURN percentileCont(values, 1.0) AS res") + it("percentileContil with 1.0 as percentile") { + val result = morpheus.graphs.empty.cypher( + "UNWIND [2,10,5,6] AS values RETURN percentileCont(values, 1.0) AS res" + ) - result.records.toMaps should equal(Bag( - CypherMap("res" -> 10.0) - )) + result.records.toMaps should equal( + Bag( + CypherMap("res" -> 10.0) + ) + ) } - it("percentileContil with 0.0 as percentile"){ - val result = morpheus.graphs.empty.cypher("UNWIND [2,10,5,6] AS values RETURN percentileCont(values, 0.0) AS res") + it("percentileContil with 0.0 as percentile") { + val result = morpheus.graphs.empty.cypher( + "UNWIND [2,10,5,6] AS values RETURN percentileCont(values, 0.0) AS res" + ) - result.records.toMaps should equal(Bag( - CypherMap("res" -> 2.0) - )) + result.records.toMaps should equal( + Bag( + CypherMap("res" -> 2.0) + ) + ) } - it("percentileContil on floats with null"){ - val result = morpheus.graphs.empty.cypher("UNWIND [10.0,null,2.0,6.0] AS values RETURN round(percentileCont(values, 0.62) * 1000) / 1000.0 AS res") + it("percentileContil on floats with null") { + val result = morpheus.graphs.empty.cypher( + "UNWIND [10.0,null,2.0,6.0] AS values RETURN round(percentileCont(values, 0.62) * 1000) / 1000.0 AS res" + ) - result.records.toMaps should equal(Bag( - CypherMap("res" -> 6.96) - )) + result.records.toMaps should equal( + Bag( + CypherMap("res" -> 6.96) + ) + ) } - it("percentileContil on floats"){ - val result = morpheus.graphs.empty.cypher("UNWIND [10.0,5.0,2.0,6.0] AS values RETURN percentileCont(values, 0.6) AS res") + it("percentileContil on floats") { + val result = morpheus.graphs.empty.cypher( + "UNWIND [10.0,5.0,2.0,6.0] AS values RETURN percentileCont(values, 0.6) AS res" + ) - result.records.toMaps should equal(Bag( - CypherMap("res" -> 5.8) - )) + result.records.toMaps should equal( + Bag( + CypherMap("res" -> 5.8) + ) + ) } } describe("percentileDisc") { - it("percentileDisc on integers"){ - val result = morpheus.graphs.empty.cypher("UNWIND [10,5,2,6] AS values RETURN percentileDisc(values, 0.5) AS res") + it("percentileDisc on integers") { + val result = morpheus.graphs.empty.cypher( + "UNWIND [10,5,2,6] AS values RETURN percentileDisc(values, 0.5) AS res" + ) - result.records.toMaps should equal(Bag( - CypherMap("res" -> 5) - )) + result.records.toMaps should equal( + Bag( + CypherMap("res" -> 5) + ) + ) } - it("percentileDisc with 1.0 as percentile"){ - val result = morpheus.graphs.empty.cypher("UNWIND [10.0,5.0,2.0,6.0] AS values RETURN percentileDisc(values, 1.0) AS res") + it("percentileDisc with 1.0 as percentile") { + val result = morpheus.graphs.empty.cypher( + "UNWIND [10.0,5.0,2.0,6.0] AS values RETURN percentileDisc(values, 1.0) AS res" + ) - result.records.toMaps should equal(Bag( - CypherMap("res" -> 10.0) - )) + result.records.toMaps should equal( + Bag( + CypherMap("res" -> 10.0) + ) + ) } - it("percentileDisc with 0.0 as percentile"){ - val result = morpheus.graphs.empty.cypher("UNWIND [10.0,5.0,2.0,6.0] AS values RETURN percentileDisc(values, 0.0) AS res") + it("percentileDisc with 0.0 as percentile") { + val result = morpheus.graphs.empty.cypher( + "UNWIND [10.0,5.0,2.0,6.0] AS values RETURN percentileDisc(values, 0.0) AS res" + ) - result.records.toMaps should equal(Bag( - CypherMap("res" -> 2.0) - )) + result.records.toMaps should equal( + Bag( + CypherMap("res" -> 2.0) + ) + ) } - it("percentileDisc on floats with null"){ - val result = morpheus.graphs.empty.cypher("UNWIND [10.0,null,2.0,6.0] AS values RETURN percentileDisc(values, 0.6) AS res") + it("percentileDisc on floats with null") { + val result = morpheus.graphs.empty.cypher( + "UNWIND [10.0,null,2.0,6.0] AS values RETURN percentileDisc(values, 0.6) AS res" + ) - result.records.toMaps should equal(Bag( - CypherMap("res" -> 6.0) - )) + result.records.toMaps should equal( + Bag( + CypherMap("res" -> 6.0) + ) + ) } - it("percentileDisc on floats"){ + it("percentileDisc on floats") { val graph = initGraph("CREATE ({age: 10.0}), ({age: 2.0}), ({age: 5.0}), ({age: 6.0})") val result = graph.cypher("MATCH (n) RETURN percentileDisc(n.age, 0.5) AS res") - result.records.toMaps should equal(Bag( - CypherMap("res" -> 5.0) - )) + result.records.toMaps should equal( + Bag( + CypherMap("res" -> 5.0) + ) + ) } } @@ -756,9 +946,11 @@ class AggregationTests extends MorpheusTestSuite with ScanGraphInit { val result = graph.cypher("MATCH (n) WITH COLLECT(n.val) AS res RETURN res") - result.records.toMaps should equal(Bag( - CypherMap("res" -> Seq(23.0, 42.0)) - )) + result.records.toMaps should equal( + Bag( + CypherMap("res" -> Seq(23.0, 42.0)) + ) + ) } it("simple collect(prop) with single null value in RETURN") { @@ -766,9 +958,11 @@ class AggregationTests extends MorpheusTestSuite with ScanGraphInit { val result = graph.cypher("MATCH (n) RETURN COLLECT(n.val) AS res") - result.records.toMaps should equal(Bag( - CypherMap("res" -> Seq(23.0, 42.0)) - )) + result.records.toMaps should equal( + Bag( + CypherMap("res" -> Seq(23.0, 42.0)) + ) + ) } it("simple collect(prop) with only null values in WITH") { @@ -776,9 +970,11 @@ class AggregationTests extends MorpheusTestSuite with ScanGraphInit { val result = graph.cypher("MATCH (n) WITH Collect(n.val) AS res RETURN res") - result.records.toMaps should equal(Bag( - CypherMap("res" -> Seq.empty) - )) + result.records.toMaps should equal( + Bag( + CypherMap("res" -> Seq.empty) + ) + ) } it("simple collect(prop) with only null values in RETURN") { @@ -786,36 +982,37 @@ class AggregationTests extends MorpheusTestSuite with ScanGraphInit { val result = graph.cypher("MATCH (n) RETURN COLLECT(n.val) AS res") - result.records.toMaps should equal(Bag( - CypherMap("res" -> Seq.empty) - )) + result.records.toMaps should equal( + Bag( + CypherMap("res" -> Seq.empty) + ) + ) } it("collects distinct lists with grouping") { - val graph = initGraph( - """ + val graph = initGraph(""" |CREATE (a:Start{id: 1}) |CREATE (a)-[:REL]->({val: "foo"}) |CREATE (a)-[:REL]->({val: "foo"}) """.stripMargin) - val result = graph.cypher( - """ + val result = graph.cypher(""" |MATCH (a:Start)-->(b) | |RETURN a.id, | collect(distinct b.val) AS val """.stripMargin) - result.records.toMaps should equal(Bag( - CypherMap("a.id" -> 1, "val" -> CypherList("foo")) - )) + result.records.toMaps should equal( + Bag( + CypherMap("a.id" -> 1, "val" -> CypherList("foo")) + ) + ) } } it("collects non-nullable strings that are not present on every match") { - val graph = initGraph( - """ + val graph = initGraph(""" |CREATE (a:Person{id: 1, name:'Anna'}) |CREATE (b:Person{id: 2, name:'Bob'}) |CREATE (p1:Purchase{id: 3}) @@ -826,7 +1023,8 @@ class AggregationTests extends MorpheusTestSuite with ScanGraphInit { """|MATCH (person:Person)-[:FRIEND_OF]-(friend:Person), |(friend)-[:IS]->(customer:Customer), |(customer)-[:BOUGHT]->(product:Product) - |RETURN person.name AS for, collect(DISTINCT product.title) AS recommendations""".stripMargin) + |RETURN person.name AS for, collect(DISTINCT product.title) AS recommendations""".stripMargin + ) } describe("Combinations") { @@ -834,8 +1032,7 @@ class AggregationTests extends MorpheusTestSuite with ScanGraphInit { it("multiple aggregates in WITH") { val graph = initGraph("CREATE ({val: 42}),({val: 23}),({val: 84})") - val result = graph.cypher( - """MATCH (n) + val result = graph.cypher("""MATCH (n) |WITH | AVG(n.val) AS avg, | COUNT(*) AS cnt, @@ -858,8 +1055,7 @@ class AggregationTests extends MorpheusTestSuite with ScanGraphInit { it("multiple aggregates in RETURN") { val graph = initGraph("CREATE ({val: 42}),({val: 23}),({val: 84})") - val result = graph.cypher( - """MATCH (n) + val result = graph.cypher("""MATCH (n) |RETURN | AVG(n.val) AS avg, | COUNT(*) AS cnt, @@ -879,10 +1075,10 @@ class AggregationTests extends MorpheusTestSuite with ScanGraphInit { } it("computes multiple aggregates with grouping in RETURN clause") { - val graph = initGraph("CREATE ({key: 'a', val: 42}),({key: 'a',val: 23}),({key: 'b', val: 84})") + val graph = + initGraph("CREATE ({key: 'a', val: 42}),({key: 'a',val: 23}),({key: 'b', val: 84})") - val result = graph.cypher( - """MATCH (n) + val result = graph.cypher("""MATCH (n) |RETURN | n.key AS key, | AVG(n.val) AS avg, @@ -892,18 +1088,35 @@ class AggregationTests extends MorpheusTestSuite with ScanGraphInit { | SUM(n.val) AS sum, | COLLECT(n.val) AS col""".stripMargin) - result.records.toMaps should equal(Bag( - CypherMap( - "key" -> "a", "avg" -> 32.5, "cnt" -> 2, "min" -> 23L, "max" -> 42L, "sum" -> 65, "col" -> Seq(23, 42)), - CypherMap("key" -> "b", "avg" -> 84.0, "cnt" -> 1, "min" -> 84, "max" -> 84, "sum" -> 84, "col" -> Seq(84)) - )) + result.records.toMaps should equal( + Bag( + CypherMap( + "key" -> "a", + "avg" -> 32.5, + "cnt" -> 2, + "min" -> 23L, + "max" -> 42L, + "sum" -> 65, + "col" -> Seq(23, 42) + ), + CypherMap( + "key" -> "b", + "avg" -> 84.0, + "cnt" -> 1, + "min" -> 84, + "max" -> 84, + "sum" -> 84, + "col" -> Seq(84) + ) + ) + ) } it("computes multiple aggregates with grouping in WITH clause") { - val graph = initGraph("CREATE ({key: 'a', val: 42}),({key: 'a',val: 23}),({key: 'b', val: 84})") + val graph = + initGraph("CREATE ({key: 'a', val: 42}),({key: 'a',val: 23}),({key: 'b', val: 84})") - val result = graph.cypher( - """MATCH (n) + val result = graph.cypher("""MATCH (n) |WITH | n.key AS key, | AVG(n.val) AS avg, @@ -914,11 +1127,28 @@ class AggregationTests extends MorpheusTestSuite with ScanGraphInit { | COLLECT(n.val) AS col |RETURN key, avg, cnt, min, max, sum, col""".stripMargin) - result.records.toMaps should equal(Bag( - CypherMap( - "key" -> "a", "avg" -> 32.5, "cnt" -> 2, "min" -> 23L, "max" -> 42L, "sum" -> 65, "col" -> Seq(23, 42)), - CypherMap("key" -> "b", "avg" -> 84.0, "cnt" -> 1, "min" -> 84, "max" -> 84, "sum" -> 84, "col" -> Seq(84)) - )) + result.records.toMaps should equal( + Bag( + CypherMap( + "key" -> "a", + "avg" -> 32.5, + "cnt" -> 2, + "min" -> 23L, + "max" -> 42L, + "sum" -> 65, + "col" -> Seq(23, 42) + ), + CypherMap( + "key" -> "b", + "avg" -> 84.0, + "cnt" -> 1, + "min" -> 84, + "max" -> 84, + "sum" -> 84, + "col" -> Seq(84) + ) + ) + ) } } } diff --git a/morpheus-testing/src/test/scala/org/opencypher/morpheus/impl/acceptance/BigDecimalTests.scala b/morpheus-testing/src/test/scala/org/opencypher/morpheus/impl/acceptance/BigDecimalTests.scala index 3e77754c5c..4b3d273ea6 100644 --- a/morpheus-testing/src/test/scala/org/opencypher/morpheus/impl/acceptance/BigDecimalTests.scala +++ b/morpheus-testing/src/test/scala/org/opencypher/morpheus/impl/acceptance/BigDecimalTests.scala @@ -35,7 +35,10 @@ class BigDecimalTests extends MorpheusTestSuite with ScanGraphInit { describe("general") { it("returns a big decimal") { - morpheus.cypher("RETURN bigdecimal(1234, 4, 2) AS decimal").records.toMaps should equal( + morpheus + .cypher("RETURN bigdecimal(1234, 4, 2) AS decimal") + .records + .toMaps should equal( Bag( CypherMap("decimal" -> BigDecimal(1234, 2)) ) @@ -46,7 +49,12 @@ class BigDecimalTests extends MorpheusTestSuite with ScanGraphInit { describe("arithmetics") { it("adds two big decimals") { - morpheus.cypher("RETURN bigdecimal(1234, 4, 2) + bigdecimal(12, 2, 1) AS decimal").records.toMaps should equal( + morpheus + .cypher( + "RETURN bigdecimal(1234, 4, 2) + bigdecimal(12, 2, 1) AS decimal" + ) + .records + .toMaps should equal( Bag( CypherMap("decimal" -> BigDecimal(1354, 2)) ) @@ -54,13 +62,19 @@ class BigDecimalTests extends MorpheusTestSuite with ScanGraphInit { } it("adds a big decimal and an integer") { - morpheus.cypher("RETURN bigdecimal(1234, 4, 2) + 10 AS decimal").records.toMaps should equal( + morpheus + .cypher("RETURN bigdecimal(1234, 4, 2) + 10 AS decimal") + .records + .toMaps should equal( Bag( CypherMap("decimal" -> BigDecimal(2234, 2)) ) ) - morpheus.cypher("RETURN 10 + bigdecimal(1234, 4, 2) AS decimal").records.toMaps should equal( + morpheus + .cypher("RETURN 10 + bigdecimal(1234, 4, 2) AS decimal") + .records + .toMaps should equal( Bag( CypherMap("decimal" -> BigDecimal(2234, 2)) ) @@ -68,13 +82,19 @@ class BigDecimalTests extends MorpheusTestSuite with ScanGraphInit { } it("adds a big decimal and a float") { - morpheus.cypher("RETURN bigdecimal(1234, 4, 2) + 10.2 AS decimal").records.toMaps should equal( + morpheus + .cypher("RETURN bigdecimal(1234, 4, 2) + 10.2 AS decimal") + .records + .toMaps should equal( Bag( CypherMap("decimal" -> 22.54) ) ) - morpheus.cypher("RETURN 10.2 + bigdecimal(1234, 4, 2) AS decimal").records.toMaps should equal( + morpheus + .cypher("RETURN 10.2 + bigdecimal(1234, 4, 2) AS decimal") + .records + .toMaps should equal( Bag( CypherMap("decimal" -> 22.54) ) @@ -82,13 +102,19 @@ class BigDecimalTests extends MorpheusTestSuite with ScanGraphInit { } it("subtracts a big decimal and an integer") { - morpheus.cypher("RETURN bigdecimal(1234, 4, 2) - 10 AS decimal").records.toMaps should equal( + morpheus + .cypher("RETURN bigdecimal(1234, 4, 2) - 10 AS decimal") + .records + .toMaps should equal( Bag( CypherMap("decimal" -> BigDecimal(234, 2)) ) ) - morpheus.cypher("RETURN 10 - bigdecimal(1234, 4, 2) AS decimal").records.toMaps should equal( + morpheus + .cypher("RETURN 10 - bigdecimal(1234, 4, 2) AS decimal") + .records + .toMaps should equal( Bag( CypherMap("decimal" -> BigDecimal(-234, 2)) ) @@ -96,7 +122,10 @@ class BigDecimalTests extends MorpheusTestSuite with ScanGraphInit { } it("subtracts a big decimal and a float") { - morpheus.cypher("RETURN bigdecimal(44, 2, 1) - 0.2 AS decimal").records.toMaps should equal( + morpheus + .cypher("RETURN bigdecimal(44, 2, 1) - 0.2 AS decimal") + .records + .toMaps should equal( Bag( CypherMap("decimal" -> 4.2) ) diff --git a/morpheus-testing/src/test/scala/org/opencypher/morpheus/impl/acceptance/BoundedVarExpandTests.scala b/morpheus-testing/src/test/scala/org/opencypher/morpheus/impl/acceptance/BoundedVarExpandTests.scala index fc7d3a0655..53d4f7c707 100644 --- a/morpheus-testing/src/test/scala/org/opencypher/morpheus/impl/acceptance/BoundedVarExpandTests.scala +++ b/morpheus-testing/src/test/scala/org/opencypher/morpheus/impl/acceptance/BoundedVarExpandTests.scala @@ -43,73 +43,88 @@ class BoundedVarExpandTests extends MorpheusTestSuite with ScanGraphInit { """ ) - val result = given.cypher( - """ + val result = given.cypher(""" |MATCH (a:A) |MATCH (a)-[:LIKES*0]->(c) |RETURN c.name""".stripMargin) - result.records.toMaps should equal(Bag( - CypherMap("c.name" -> "n0") - )) + result.records.toMaps should equal( + Bag( + CypherMap("c.name" -> "n0") + ) + ) } it("bounded to single relationship") { // Given - val given = initGraph("CREATE (s:Node {val: 'source'})-[:REL]->(:Node {val: 'mid1'})-[:REL]->(:Node {val: 'end'})") + val given = initGraph( + "CREATE (s:Node {val: 'source'})-[:REL]->(:Node {val: 'mid1'})-[:REL]->(:Node {val: 'end'})" + ) // When val result = given.cypher("MATCH (n:Node)-[r*0..1]->(m:Node) RETURN m.val") // Then - result.records.toMaps should equal(Bag( - CypherMap("m.val" -> "source"), - CypherMap("m.val" -> "mid1"), - CypherMap("m.val" -> "mid1"), - CypherMap("m.val" -> "end"), - CypherMap("m.val" -> "end") - )) + result.records.toMaps should equal( + Bag( + CypherMap("m.val" -> "source"), + CypherMap("m.val" -> "mid1"), + CypherMap("m.val" -> "mid1"), + CypherMap("m.val" -> "end"), + CypherMap("m.val" -> "end") + ) + ) } it("bounded with lower bound") { // Given - val given = initGraph("CREATE (:Node {val: 'source'})-[:REL]->(:Node {val: 'mid1'})-[:REL]->(:Node {val: 'end'})") + val given = initGraph( + "CREATE (:Node {val: 'source'})-[:REL]->(:Node {val: 'mid1'})-[:REL]->(:Node {val: 'end'})" + ) // When val result = given.cypher("MATCH (t:Node)-[r*2..3]->(y:Node) RETURN y.val") // Then - result.records.toMaps should equal(Bag( - CypherMap("y.val" -> "end") - )) + result.records.toMaps should equal( + Bag( + CypherMap("y.val" -> "end") + ) + ) } it("var expand with default lower and loop") { // Given - val given = initGraph("CREATE (a:Node {v: 'a'})-[:REL]->(:Node {v: 'b'})-[:REL]->(:Node {v: 'c'})-[:REL]->(a)") + val given = initGraph( + "CREATE (a:Node {v: 'a'})-[:REL]->(:Node {v: 'b'})-[:REL]->(:Node {v: 'c'})-[:REL]->(a)" + ) // When val result = given.cypher("MATCH (a:Node)-[r*..6]->(b:Node) RETURN b.v") // Then - result.records.toMaps should equal(Bag( - CypherMap("b.v" -> "a"), - CypherMap("b.v" -> "a"), - CypherMap("b.v" -> "a"), - CypherMap("b.v" -> "b"), - CypherMap("b.v" -> "b"), - CypherMap("b.v" -> "b"), - CypherMap("b.v" -> "c"), - CypherMap("b.v" -> "c"), - CypherMap("b.v" -> "c") - )) + result.records.toMaps should equal( + Bag( + CypherMap("b.v" -> "a"), + CypherMap("b.v" -> "a"), + CypherMap("b.v" -> "a"), + CypherMap("b.v" -> "b"), + CypherMap("b.v" -> "b"), + CypherMap("b.v" -> "b"), + CypherMap("b.v" -> "c"), + CypherMap("b.v" -> "c"), + CypherMap("b.v" -> "c") + ) + ) } it("var expand return var length rel as list of relationships") { // Given - val given = initGraph("CREATE (a:Node {v: 'a'})-[:REL]->(:Node {v: 'b'})-[:REL]->(:Node {v: 'c'})-[:REL]->(a)") + val given = initGraph( + "CREATE (a:Node {v: 'a'})-[:REL]->(:Node {v: 'b'})-[:REL]->(:Node {v: 'c'})-[:REL]->(a)" + ) // When val result = given.cypher("MATCH (a:Node)-[r*..6]->(b:Node) RETURN r") @@ -119,70 +134,87 @@ class BoundedVarExpandTests extends MorpheusTestSuite with ScanGraphInit { val rel3 = MorpheusRelationship(5, 3, 0, "REL") val elements = result.records.toMaps - elements should equal(Bag( - CypherMap("r" -> CypherList(Seq(rel1))), - CypherMap("r" -> CypherList(Seq(rel1, rel2))), - CypherMap("r" -> CypherList(Seq(rel1, rel2, rel3))), - CypherMap("r" -> CypherList(Seq(rel2))), - CypherMap("r" -> CypherList(Seq(rel2, rel3))), - CypherMap("r" -> CypherList(Seq(rel2, rel3, rel1))), - CypherMap("r" -> CypherList(Seq(rel3))), - CypherMap("r" -> CypherList(Seq(rel3, rel1))), - CypherMap("r" -> CypherList(Seq(rel3, rel1, rel2))) - )) + elements should equal( + Bag( + CypherMap("r" -> CypherList(Seq(rel1))), + CypherMap("r" -> CypherList(Seq(rel1, rel2))), + CypherMap("r" -> CypherList(Seq(rel1, rel2, rel3))), + CypherMap("r" -> CypherList(Seq(rel2))), + CypherMap("r" -> CypherList(Seq(rel2, rel3))), + CypherMap("r" -> CypherList(Seq(rel2, rel3, rel1))), + CypherMap("r" -> CypherList(Seq(rel3))), + CypherMap("r" -> CypherList(Seq(rel3, rel1))), + CypherMap("r" -> CypherList(Seq(rel3, rel1, rel2))) + ) + ) } it("var expand with rel type") { // Given - val given = initGraph("CREATE (a:Node {v: 'a'})-[:LOVES]->(:Node {v: 'b'})-[:KNOWS]->(:Node {v: 'c'})-[:HATES]->(a)") + val given = initGraph( + "CREATE (a:Node {v: 'a'})-[:LOVES]->(:Node {v: 'b'})-[:KNOWS]->(:Node {v: 'c'})-[:HATES]->(a)" + ) // When - val result = given.cypher("MATCH (a:Node)-[r:LOVES|KNOWS*..6]->(b:Node) RETURN b.v") + val result = + given.cypher("MATCH (a:Node)-[r:LOVES|KNOWS*..6]->(b:Node) RETURN b.v") // Then - result.records.toMaps should equal(Bag( - CypherMap("b.v" -> "b"), - CypherMap("b.v" -> "c"), - CypherMap("b.v" -> "c") - )) + result.records.toMaps should equal( + Bag( + CypherMap("b.v" -> "b"), + CypherMap("b.v" -> "c"), + CypherMap("b.v" -> "c") + ) + ) } // Property predicates on var-length patterns get rewritten in AST to WHERE all(_foo IN r | _foo.prop = value) // We could do that better by pre-filtering candidate relationships prior to the iterate step ignore("var expand with property filter") { // Given - val given = initGraph("CREATE (a:Node {v: 'a'})-[:R {v: 1}]->(:Node {v: 'b'})-[:R {v: 2}]->(:Node {v: 'c'})-[:R {v: 2}]->(a)") + val given = initGraph( + "CREATE (a:Node {v: 'a'})-[:R {v: 1}]->(:Node {v: 'b'})-[:R {v: 2}]->(:Node {v: 'c'})-[:R {v: 2}]->(a)" + ) // When - val result = given.cypher("MATCH (a:Node)-[r*..6 {v: 2}]->(b:Node) RETURN b.v") + val result = + given.cypher("MATCH (a:Node)-[r*..6 {v: 2}]->(b:Node) RETURN b.v") // Then - result.records.toMaps should equal(Bag( - CypherMap("b.v" -> "c"), - CypherMap("b.v" -> "a"), - CypherMap("b.v" -> "a") - )) + result.records.toMaps should equal( + Bag( + CypherMap("b.v" -> "c"), + CypherMap("b.v" -> "a"), + CypherMap("b.v" -> "a") + ) + ) } it("var expand with additional hop") { // Given - val given = initGraph("CREATE (a:Node {v: 'a'})-[:KNOWS]->(:Node {v: 'b'})-[:KNOWS]->(:Node {v: 'c'})-[:HATES]->(d:Node {v: 'd'})") + val given = initGraph( + "CREATE (a:Node {v: 'a'})-[:KNOWS]->(:Node {v: 'b'})-[:KNOWS]->(:Node {v: 'c'})-[:HATES]->(d:Node {v: 'd'})" + ) // When - val result = given.cypher("MATCH (a:Node)-[r:KNOWS*..6]->(b:Node)-[:HATES]->(c:Node) RETURN c.v") + val result = given.cypher( + "MATCH (a:Node)-[r:KNOWS*..6]->(b:Node)-[:HATES]->(c:Node) RETURN c.v" + ) // Then - result.records.toMaps should equal(Bag( - CypherMap("c.v" -> "d"), - CypherMap("c.v" -> "d") - )) + result.records.toMaps should equal( + Bag( + CypherMap("c.v" -> "d"), + CypherMap("c.v" -> "d") + ) + ) } it("var expand with expand into") { // Given - val given = initGraph( - """ + val given = initGraph(""" |CREATE (a:Person {name: "Philip"}) |CREATE (b:Person {name: "Stefan"}) |CREATE (c:City {name: "Berlondon"}) @@ -199,8 +231,14 @@ class BoundedVarExpandTests extends MorpheusTestSuite with ScanGraphInit { ) // Then - result.records.toMaps should equal(Bag( - CypherMap("a.name" -> "Philip", "b.name" -> "Stefan", "c.name" -> "Berlondon") - )) + result.records.toMaps should equal( + Bag( + CypherMap( + "a.name" -> "Philip", + "b.name" -> "Stefan", + "c.name" -> "Berlondon" + ) + ) + ) } } diff --git a/morpheus-testing/src/test/scala/org/opencypher/morpheus/impl/acceptance/CacheTests.scala b/morpheus-testing/src/test/scala/org/opencypher/morpheus/impl/acceptance/CacheTests.scala index 654d8781e2..ddcbbcc968 100644 --- a/morpheus-testing/src/test/scala/org/opencypher/morpheus/impl/acceptance/CacheTests.scala +++ b/morpheus-testing/src/test/scala/org/opencypher/morpheus/impl/acceptance/CacheTests.scala @@ -36,21 +36,22 @@ class CacheTests extends MorpheusTestSuite with ScanGraphInit { describe("scan caching") { it("caches a reused scan") { - val g = initGraph("""CREATE (p:Person {firstName: "Alice", lastName: "Foo"})""") - val result: CypherResult = g.cypher( - """ + val g = + initGraph("""CREATE (p:Person {firstName: "Alice", lastName: "Foo"})""") + val result: CypherResult = g.cypher(""" |MATCH (n: Person) |MATCH (m: Person) |WHERE n.name = m.name |RETURN n.name """.stripMargin) - result.asMorpheus.plans.relationalPlan.get.collect { case c: Cache[_] => c } should have size 2 + result.asMorpheus.plans.relationalPlan.get.collect { case c: Cache[_] => + c + } should have size 2 } it("caches all-node/relationship scans") { - val g = initGraph( - """ + val g = initGraph(""" |CREATE (a:Person {firstName: "Alice"}) |CREATE (b:Person {firstName: "Bob"}) |CREATE (c:Person {firstName: "Carol"}) @@ -60,18 +61,18 @@ class CacheTests extends MorpheusTestSuite with ScanGraphInit { |CREATE (book)-[:PUBLISHED_BY]->(publisher) """.stripMargin) - val result: CypherResult = g.cypher( - """ + val result: CypherResult = g.cypher(""" |MATCH (a)-->(b)-->(c) |RETURN a, b """.stripMargin) - result.asMorpheus.plans.relationalPlan.get.collect { case c: Cache[_] => c } should have size 5 + result.asMorpheus.plans.relationalPlan.get.collect { case c: Cache[_] => + c + } should have size 5 } it("caches all-node/relationship scans across MATCH statements") { - val g = initGraph( - """ + val g = initGraph(""" |CREATE (a:Person {firstName: "Alice"}) |CREATE (b:Person {firstName: "Bob"}) |CREATE (c:Person {firstName: "Carol"}) @@ -81,14 +82,15 @@ class CacheTests extends MorpheusTestSuite with ScanGraphInit { |CREATE (book)-[:PUBLISHED_BY]->(publisher) """.stripMargin) - val result: CypherResult = g.cypher( - """ + val result: CypherResult = g.cypher(""" |MATCH (a)-->(b) |MATCH (b)-->(c) |RETURN a, b """.stripMargin) - result.asMorpheus.plans.relationalPlan.get.collect { case c: Cache[_] => c } should have size 5 + result.asMorpheus.plans.relationalPlan.get.collect { case c: Cache[_] => + c + } should have size 5 } } } diff --git a/morpheus-testing/src/test/scala/org/opencypher/morpheus/impl/acceptance/CatalogDDLTests.scala b/morpheus-testing/src/test/scala/org/opencypher/morpheus/impl/acceptance/CatalogDDLTests.scala index fd0faeff18..168de68974 100644 --- a/morpheus-testing/src/test/scala/org/opencypher/morpheus/impl/acceptance/CatalogDDLTests.scala +++ b/morpheus-testing/src/test/scala/org/opencypher/morpheus/impl/acceptance/CatalogDDLTests.scala @@ -39,28 +39,29 @@ class CatalogDDLTests extends MorpheusTestSuite with ScanGraphInit with BeforeAn override def afterEach(): Unit = { super.afterEach() - morpheus.catalog.graphNames.filterNot(_ == morpheus.emptyGraphQgn).foreach(morpheus.catalog.dropGraph) + morpheus.catalog.graphNames + .filterNot(_ == morpheus.emptyGraphQgn) + .foreach(morpheus.catalog.dropGraph) morpheus.catalog.viewNames.foreach(morpheus.catalog.dropView) } describe("CATALOG CREATE GRAPH") { it("supports CATALOG CREATE GRAPH on the session") { - val inputGraph = initGraph( - """ + val inputGraph = initGraph(""" |CREATE (:A) """.stripMargin) morpheus.catalog.store("foo", inputGraph) - val result = morpheus.cypher( - """ + val result = morpheus.cypher(""" |CATALOG CREATE GRAPH bar { | FROM GRAPH foo | RETURN GRAPH |} """.stripMargin) - val sessionSource = morpheus.catalog.source(morpheus.catalog.sessionNamespace) + val sessionSource = + morpheus.catalog.source(morpheus.catalog.sessionNamespace) sessionSource.hasGraph(GraphName("bar")) shouldBe true sessionSource.graph(GraphName("bar")) shouldEqual inputGraph result.getGraph shouldBe None @@ -71,8 +72,7 @@ class CatalogDDLTests extends MorpheusTestSuite with ScanGraphInit with BeforeAn describe("CATALOG CREATE VIEW") { it("supports storing a VIEW") { - morpheus.cypher( - """ + morpheus.cypher(""" |CATALOG CREATE VIEW bar { | FROM GRAPH foo | RETURN GRAPH @@ -85,8 +85,7 @@ class CatalogDDLTests extends MorpheusTestSuite with ScanGraphInit with BeforeAn } it("throws an error when a view QGN collides with an existing view QGN") { - morpheus.cypher( - """ + morpheus.cypher(""" |CATALOG CREATE VIEW foo { | FROM GRAPH whatever | RETURN GRAPH @@ -94,8 +93,7 @@ class CatalogDDLTests extends MorpheusTestSuite with ScanGraphInit with BeforeAn """.stripMargin) a[ViewAlreadyExistsException] should be thrownBy { - morpheus.cypher( - """ + morpheus.cypher(""" |CATALOG CREATE VIEW foo { | FROM GRAPH whatever | RETURN GRAPH @@ -107,8 +105,7 @@ class CatalogDDLTests extends MorpheusTestSuite with ScanGraphInit with BeforeAn it("can still resolve a graph when a view with the same name exists") { - morpheus.cypher( - """ + morpheus.cypher(""" |CATALOG CREATE GRAPH foo { | CONSTRUCT | CREATE () @@ -116,22 +113,23 @@ class CatalogDDLTests extends MorpheusTestSuite with ScanGraphInit with BeforeAn |} """.stripMargin) - morpheus.cypher( - """ + morpheus.cypher(""" |CATALOG CREATE VIEW foo { | FROM GRAPH whatever | RETURN GRAPH |} """.stripMargin) - morpheus.cypher("FROM GRAPH foo MATCH (n) RETURN n").records.size shouldBe 1 + morpheus + .cypher("FROM GRAPH foo MATCH (n) RETURN n") + .records + .size shouldBe 1 } it("can still resolve a view when a graph with the same name exists") { - morpheus.cypher( - """ + morpheus.cypher(""" |CATALOG CREATE GRAPH bar { | CONSTRUCT | CREATE () @@ -140,8 +138,7 @@ class CatalogDDLTests extends MorpheusTestSuite with ScanGraphInit with BeforeAn |} """.stripMargin) - morpheus.cypher( - """ + morpheus.cypher(""" |CATALOG CREATE GRAPH foo { | CONSTRUCT | CREATE () @@ -149,8 +146,7 @@ class CatalogDDLTests extends MorpheusTestSuite with ScanGraphInit with BeforeAn |} """.stripMargin) - morpheus.cypher( - """ + morpheus.cypher(""" |CATALOG CREATE VIEW foo { | FROM bar | RETURN GRAPH @@ -161,10 +157,11 @@ class CatalogDDLTests extends MorpheusTestSuite with ScanGraphInit with BeforeAn } - it("throws an illegal argument exception, when no view with the given name is stored") { + it( + "throws an illegal argument exception, when no view with the given name is stored" + ) { an[IllegalArgumentException] should be thrownBy { - morpheus.cypher( - """ + morpheus.cypher(""" |FROM GRAPH someView() |MATCH (n) |RETURN n @@ -173,15 +170,13 @@ class CatalogDDLTests extends MorpheusTestSuite with ScanGraphInit with BeforeAn } it("supports simple nested CATALOG CREATE VIEW in a query") { - val inputGraphA = initGraph( - """ + val inputGraphA = initGraph(""" |CREATE (:A {val: 0}) """.stripMargin) morpheus.catalog.store("a", inputGraphA) - morpheus.cypher( - """ + morpheus.cypher(""" |CATALOG CREATE VIEW inc($g1) { | FROM GRAPH $g1 | MATCH (a: A) @@ -195,28 +190,29 @@ class CatalogDDLTests extends MorpheusTestSuite with ScanGraphInit with BeforeAn morpheus.catalog.catalogNames should contain(inc) morpheus.catalog.viewNames should contain(inc) - val result = morpheus.cypher( - """ + val result = morpheus.cypher(""" |FROM GRAPH inc(inc(inc(inc(a)))) |MATCH (n) |RETURN n.val as val """.stripMargin) - result.records.toMaps should equal(Bag(CypherMap( - "val" -> 4 - ))) + result.records.toMaps should equal( + Bag( + CypherMap( + "val" -> 4 + ) + ) + ) } it("disallows graph parameters as view invocation parameters") { - val inputGraphA = initGraph( - """ + val inputGraphA = initGraph(""" |CREATE (:A {val: 0}) """.stripMargin) morpheus.catalog.store("a", inputGraphA) - morpheus.cypher( - """ + morpheus.cypher(""" |CATALOG CREATE VIEW inc($g1) { | FROM GRAPH $g1 | MATCH (a: A) @@ -236,17 +232,17 @@ class CatalogDDLTests extends MorpheusTestSuite with ScanGraphInit with BeforeAn |FROM GRAPH inc($param) |MATCH (n) |RETURN n.val as val - """.stripMargin, CypherMap("param" -> "a")) + """.stripMargin, + CypherMap("param" -> "a") + ) } } it("supports CATALOG CREATE VIEW with two parameters") { - val inputGraphA = initGraph( - """ + val inputGraphA = initGraph(""" |CREATE (:A) """.stripMargin) - val inputGraphB = initGraph( - """ + val inputGraphB = initGraph(""" |CREATE (:B) |CREATE (:B) """.stripMargin) @@ -254,8 +250,7 @@ class CatalogDDLTests extends MorpheusTestSuite with ScanGraphInit with BeforeAn morpheus.catalog.store("a", inputGraphA) morpheus.catalog.store("b", inputGraphB) - morpheus.cypher( - """ + morpheus.cypher(""" |CATALOG CREATE VIEW bar($g1, $g2) { | FROM GRAPH $g1 | MATCH (a: A) @@ -272,11 +267,12 @@ class CatalogDDLTests extends MorpheusTestSuite with ScanGraphInit with BeforeAn morpheus.catalog.catalogNames should contain(bar) morpheus.catalog.viewNames should contain(bar) - val resultGraph = morpheus.cypher( - """ + val resultGraph = morpheus + .cypher(""" |FROM GRAPH bar(a, b) |RETURN GRAPH - """.stripMargin).graph + """.stripMargin) + .graph resultGraph.nodes("n").size shouldBe 3 resultGraph.nodes("a", CTNode("A")).size shouldBe 1 @@ -284,20 +280,17 @@ class CatalogDDLTests extends MorpheusTestSuite with ScanGraphInit with BeforeAn } it("supports nested CREATE VIEW with two parameters") { - val inputGraphA1 = initGraph( - """ + val inputGraphA1 = initGraph(""" |CREATE ({val: 1}) """.stripMargin) - val inputGraphA2 = initGraph( - """ + val inputGraphA2 = initGraph(""" |CREATE ({val: 1000}) """.stripMargin) morpheus.catalog.store("a1", inputGraphA1) morpheus.catalog.store("a2", inputGraphA2) - morpheus.cypher( - """ + morpheus.cypher(""" |CATALOG CREATE VIEW bar($g1, $g2) { | FROM GRAPH $g1 | MATCH (n) @@ -309,26 +302,29 @@ class CatalogDDLTests extends MorpheusTestSuite with ScanGraphInit with BeforeAn |} """.stripMargin) - val resultGraph = morpheus.cypher( - """ + val resultGraph = morpheus + .cypher(""" |FROM GRAPH bar(bar(a2, a1), bar(a1, a2)) |RETURN GRAPH - """.stripMargin).graph + """.stripMargin) + .graph resultGraph.nodes("n").size shouldBe 1 - resultGraph.cypher("MATCH (n) RETURN n.val").records.toMaps should equal(Bag( - CypherMap("n.val" -> 2002) - )) + resultGraph.cypher("MATCH (n) RETURN n.val").records.toMaps should equal( + Bag( + CypherMap("n.val" -> 2002) + ) + ) } - it("supports nested CREATE VIEW with two parameters and multiple constructed nodes") { - val inputGraphA = initGraph( - """ + it( + "supports nested CREATE VIEW with two parameters and multiple constructed nodes" + ) { + val inputGraphA = initGraph(""" |CREATE ({name: 'A1'}) |CREATE ({name: 'A2'}) """.stripMargin) - val inputGraphB = initGraph( - """ + val inputGraphB = initGraph(""" |CREATE ({name: 'B1'}) |CREATE ({name: 'B2'}) """.stripMargin) @@ -336,8 +332,7 @@ class CatalogDDLTests extends MorpheusTestSuite with ScanGraphInit with BeforeAn morpheus.catalog.store("a", inputGraphA) morpheus.catalog.store("b", inputGraphB) - morpheus.cypher( - """ + morpheus.cypher(""" |CATALOG CREATE VIEW bar($g1, $g2) { | FROM GRAPH $g1 | MATCH (n) @@ -350,23 +345,22 @@ class CatalogDDLTests extends MorpheusTestSuite with ScanGraphInit with BeforeAn |} """.stripMargin) - val resultGraph = morpheus.cypher( - """ + val resultGraph = morpheus + .cypher(""" |FROM GRAPH bar(bar(b, a), bar(a, b)) |RETURN GRAPH - """.stripMargin).graph + """.stripMargin) + .graph resultGraph.nodes("n").size shouldBe 8 } it("supports nested CREATE VIEW with two parameters with cloning") { - val inputGraphA = initGraph( - """ + val inputGraphA = initGraph(""" |CREATE ({name: 'A1'}) |CREATE ({name: 'A2'}) """.stripMargin) - val inputGraphB = initGraph( - """ + val inputGraphB = initGraph(""" |CREATE ({name: 'B1'}) |CREATE ({name: 'B2'}) """.stripMargin) @@ -374,8 +368,7 @@ class CatalogDDLTests extends MorpheusTestSuite with ScanGraphInit with BeforeAn morpheus.catalog.store("a", inputGraphA) morpheus.catalog.store("b", inputGraphB) - morpheus.cypher( - """ + morpheus.cypher(""" |CATALOG CREATE VIEW bar($g1, $g2) { | FROM GRAPH $g1 | MATCH (n) @@ -388,23 +381,24 @@ class CatalogDDLTests extends MorpheusTestSuite with ScanGraphInit with BeforeAn |} """.stripMargin) - val resultGraph = morpheus.cypher( - """ + val resultGraph = morpheus + .cypher(""" |FROM GRAPH bar(bar(b, a), bar(a, b)) |RETURN GRAPH - """.stripMargin).graph + """.stripMargin) + .graph resultGraph.nodes("n").size shouldBe 8 } - it("supports nested CREATE VIEW with two parameters and empty constructed nodes") { - val inputGraphA = initGraph( - """ + it( + "supports nested CREATE VIEW with two parameters and empty constructed nodes" + ) { + val inputGraphA = initGraph(""" |CREATE ({name: 'A1'}) |CREATE ({name: 'A2'}) """.stripMargin) - val inputGraphB = initGraph( - """ + val inputGraphB = initGraph(""" |CREATE ({name: 'B1'}) |CREATE ({name: 'B2'}) """.stripMargin) @@ -412,8 +406,7 @@ class CatalogDDLTests extends MorpheusTestSuite with ScanGraphInit with BeforeAn morpheus.catalog.store("a", inputGraphA) morpheus.catalog.store("b", inputGraphB) - morpheus.cypher( - """ + morpheus.cypher(""" |CATALOG CREATE VIEW bar($g1, $g2) { | FROM GRAPH $g1 | MATCH (n) @@ -426,11 +419,12 @@ class CatalogDDLTests extends MorpheusTestSuite with ScanGraphInit with BeforeAn |} """.stripMargin) - val resultGraph = morpheus.cypher( - """ + val resultGraph = morpheus + .cypher(""" |FROM GRAPH bar(bar(b, a), bar(a, b)) |RETURN GRAPH - """.stripMargin).graph + """.stripMargin) + .graph resultGraph.nodes("n").size shouldBe 42 } @@ -440,8 +434,7 @@ class CatalogDDLTests extends MorpheusTestSuite with ScanGraphInit with BeforeAn describe("DROP GRAPH/VIEW") { it("can drop a view") { - morpheus.cypher( - """ + morpheus.cypher(""" |CATALOG CREATE VIEW bar { | FROM GRAPH foo | RETURN GRAPH @@ -477,7 +470,9 @@ class CatalogDDLTests extends MorpheusTestSuite with ScanGraphInit with BeforeAn """.stripMargin ) - morpheus.catalog.source(morpheus.catalog.sessionNamespace).hasGraph(GraphName("foo")) shouldBe false + morpheus.catalog + .source(morpheus.catalog.sessionNamespace) + .hasGraph(GraphName("foo")) shouldBe false result.getGraph shouldBe None result.getRecords shouldBe None } diff --git a/morpheus-testing/src/test/scala/org/opencypher/morpheus/impl/acceptance/DrivingTableTests.scala b/morpheus-testing/src/test/scala/org/opencypher/morpheus/impl/acceptance/DrivingTableTests.scala index 2c7e914b20..895b802949 100644 --- a/morpheus-testing/src/test/scala/org/opencypher/morpheus/impl/acceptance/DrivingTableTests.scala +++ b/morpheus-testing/src/test/scala/org/opencypher/morpheus/impl/acceptance/DrivingTableTests.scala @@ -43,97 +43,132 @@ class DrivingTableTests extends MorpheusTestSuite with ScanGraphInit { Row(15, "Carol") ).asJava - val schema = StructType(Seq( - StructField("age", IntegerType), - StructField("name", StringType) - )) + val schema = StructType( + Seq( + StructField("age", IntegerType), + StructField("name", StringType) + ) + ) - val drivingTable: MorpheusRecords = morpheus.records.wrap(morpheus.sparkSession.createDataFrame(data, schema)) + val drivingTable: MorpheusRecords = + morpheus.records.wrap(morpheus.sparkSession.createDataFrame(data, schema)) describe("simple usages") { it("return data from the driving table") { - morpheus.cypher( - """ + morpheus + .cypher( + """ |RETURN age, name - """.stripMargin, drivingTable = Some(drivingTable)).records.toMaps should equal(Bag( - CypherMap("age" -> 10, "name" -> "Alice"), - CypherMap("age" -> 20, "name" -> "Bob"), - CypherMap("age" -> 15, "name" -> "Carol") - )) + """.stripMargin, + drivingTable = Some(drivingTable) + ) + .records + .toMaps should equal( + Bag( + CypherMap("age" -> 10, "name" -> "Alice"), + CypherMap("age" -> 20, "name" -> "Bob"), + CypherMap("age" -> 15, "name" -> "Carol") + ) + ) } it("can combine driving table with unwind") { - morpheus.cypher( - """ + morpheus + .cypher( + """ |UNWIND [1,2] AS i |RETURN i, age, name - """.stripMargin, drivingTable = Some(drivingTable)).records.toMaps should equal(Bag( - CypherMap("i" -> 1, "age" -> 10, "name" -> "Alice"), - CypherMap("i" -> 1, "age" -> 20, "name" -> "Bob"), - CypherMap("i" -> 1, "age" -> 15, "name" -> "Carol"), - CypherMap("i" -> 2, "age" -> 10, "name" -> "Alice"), - CypherMap("i" -> 2, "age" -> 20, "name" -> "Bob"), - CypherMap("i" -> 2, "age" -> 15, "name" -> "Carol") - )) + """.stripMargin, + drivingTable = Some(drivingTable) + ) + .records + .toMaps should equal( + Bag( + CypherMap("i" -> 1, "age" -> 10, "name" -> "Alice"), + CypherMap("i" -> 1, "age" -> 20, "name" -> "Bob"), + CypherMap("i" -> 1, "age" -> 15, "name" -> "Carol"), + CypherMap("i" -> 2, "age" -> 10, "name" -> "Alice"), + CypherMap("i" -> 2, "age" -> 20, "name" -> "Bob"), + CypherMap("i" -> 2, "age" -> 15, "name" -> "Carol") + ) + ) } } describe("matching on driving table") { it("can use driving table data for filters") { - val graph = initGraph( - """ + val graph = initGraph(""" |CREATE (:Person {name: "George", age: 20}) |CREATE (:Person {name: "Frank", age: 50}) |CREATE (:Person {name: "Jon", age: 15}) """.stripMargin) - graph.cypher( - """ + graph + .cypher( + """ |MATCH (p:Person) |WHERE p.age = age |RETURN p.name, name - """.stripMargin, drivingTable = Some(drivingTable)).records.toMaps should equal(Bag( - CypherMap("p.name" -> "George", "name" -> "Bob"), - CypherMap("p.name" -> "Jon", "name" -> "Carol") - )) + """.stripMargin, + drivingTable = Some(drivingTable) + ) + .records + .toMaps should equal( + Bag( + CypherMap("p.name" -> "George", "name" -> "Bob"), + CypherMap("p.name" -> "Jon", "name" -> "Carol") + ) + ) } it("can use driving table data for filters inside the pattern") { - val graph = initGraph( - """ + val graph = initGraph(""" |CREATE (:Person {name: "George", age: 20}) |CREATE (:Person {name: "Frank", age: 50}) |CREATE (:Person {name: "Jon", age: 15}) """.stripMargin) - graph.cypher( - """ + graph + .cypher( + """ |MATCH (p:Person {age: age}) |RETURN p.name, name - """.stripMargin, drivingTable = Some(drivingTable)).records.toMaps should equal(Bag( - CypherMap("p.name" -> "George", "name" -> "Bob"), - CypherMap("p.name" -> "Jon", "name" -> "Carol") - )) + """.stripMargin, + drivingTable = Some(drivingTable) + ) + .records + .toMaps should equal( + Bag( + CypherMap("p.name" -> "George", "name" -> "Bob"), + CypherMap("p.name" -> "Jon", "name" -> "Carol") + ) + ) } it("can use driving table for more complex matches") { - val graph = initGraph( - """ + val graph = initGraph(""" |CREATE (b:B) |CREATE (:Person {name: "George", age: 20})-[:REL]->(b) |CREATE (:Person {name: "Frank", age: 50})-[:REL]->(b) |CREATE (:Person {name: "Jon", age: 15})-[:REL]->(b) """.stripMargin) - graph.cypher( - """ + graph + .cypher( + """ |MATCH (p:Person {age: age}) |MATCH (p)-[]->() |RETURN p.name, name - """.stripMargin, drivingTable = Some(drivingTable)).records.toMaps should equal(Bag( - CypherMap("p.name" -> "George", "name" -> "Bob"), - CypherMap("p.name" -> "Jon", "name" -> "Carol") - )) + """.stripMargin, + drivingTable = Some(drivingTable) + ) + .records + .toMaps should equal( + Bag( + CypherMap("p.name" -> "George", "name" -> "Bob"), + CypherMap("p.name" -> "Jon", "name" -> "Carol") + ) + ) } } } diff --git a/morpheus-testing/src/test/scala/org/opencypher/morpheus/impl/acceptance/ExpandIntoTests.scala b/morpheus-testing/src/test/scala/org/opencypher/morpheus/impl/acceptance/ExpandIntoTests.scala index 7c21be883e..863f104ab0 100644 --- a/morpheus-testing/src/test/scala/org/opencypher/morpheus/impl/acceptance/ExpandIntoTests.scala +++ b/morpheus-testing/src/test/scala/org/opencypher/morpheus/impl/acceptance/ExpandIntoTests.scala @@ -34,8 +34,7 @@ class ExpandIntoTests extends MorpheusTestSuite with ScanGraphInit { it("test expand into for dangling edge") { // Given - val given = initGraph( - """ + val given = initGraph(""" |CREATE (p1:Person {name: "Alice"}) |CREATE (p2:Person {name: "Bob"}) |CREATE (p3:Person {name: "Eve"}) @@ -49,8 +48,7 @@ class ExpandIntoTests extends MorpheusTestSuite with ScanGraphInit { """.stripMargin) // When - val result = given.cypher( - """ + val result = given.cypher(""" |MATCH (p1:Person)-[e1:KNOWS]->(p2:Person), |(p2)-[e2:KNOWS]->(p3:Person), |(p1)-[e3:KNOWS]->(p3), @@ -59,26 +57,27 @@ class ExpandIntoTests extends MorpheusTestSuite with ScanGraphInit { """.stripMargin) // Then - result.records.toMaps should equal(Bag( - CypherMap( - "p1.name" -> "Alice", - "p2.name" -> "Bob", - "p3.name" -> "Eve", - "p4.name" -> "Carl" - ), - CypherMap( - "p1.name" -> "Alice", - "p2.name" -> "Bob", - "p3.name" -> "Eve", - "p4.name" -> "Richard" + result.records.toMaps should equal( + Bag( + CypherMap( + "p1.name" -> "Alice", + "p2.name" -> "Bob", + "p3.name" -> "Eve", + "p4.name" -> "Carl" + ), + CypherMap( + "p1.name" -> "Alice", + "p2.name" -> "Bob", + "p3.name" -> "Eve", + "p4.name" -> "Richard" + ) ) - )) + ) } it("test expand into for triangle") { // Given - val given = initGraph( - """ + val given = initGraph(""" |CREATE (p1:Person {name: "Alice"}) |CREATE (p2:Person {name: "Bob"}) |CREATE (p3:Person {name: "Eve"}) @@ -88,8 +87,7 @@ class ExpandIntoTests extends MorpheusTestSuite with ScanGraphInit { """.stripMargin) // When - val result = given.cypher( - """ + val result = given.cypher(""" |MATCH (p1:Person)-[e1:KNOWS]->(p2:Person), |(p2)-[e2:KNOWS]->(p3:Person), |(p1)-[e3:KNOWS]->(p3) @@ -97,19 +95,20 @@ class ExpandIntoTests extends MorpheusTestSuite with ScanGraphInit { """.stripMargin) // Then - result.records.toMaps should equal(Bag( - CypherMap( - "p1.name" -> "Alice", - "p2.name" -> "Bob", - "p3.name" -> "Eve" + result.records.toMaps should equal( + Bag( + CypherMap( + "p1.name" -> "Alice", + "p2.name" -> "Bob", + "p3.name" -> "Eve" + ) ) - )) + ) } it("Expand into after var expand") { // Given - val given = initGraph( - """ + val given = initGraph(""" |CREATE (p1:Person {name: "Alice"}) |CREATE (p2:Person {name: "Bob"}) |CREATE (comment:Comment) @@ -134,12 +133,14 @@ class ExpandIntoTests extends MorpheusTestSuite with ScanGraphInit { ) // Then - result.records.toMaps should equal(Bag( - CypherMap( - "p1.name" -> "Alice", - "p2.name" -> "Bob", - "post.content" -> "foobar" + result.records.toMaps should equal( + Bag( + CypherMap( + "p1.name" -> "Alice", + "p2.name" -> "Bob", + "post.content" -> "foobar" + ) ) - )) + ) } } diff --git a/morpheus-testing/src/test/scala/org/opencypher/morpheus/impl/acceptance/ExpressionTests.scala b/morpheus-testing/src/test/scala/org/opencypher/morpheus/impl/acceptance/ExpressionTests.scala index ef696be662..0a89ca5fb7 100644 --- a/morpheus-testing/src/test/scala/org/opencypher/morpheus/impl/acceptance/ExpressionTests.scala +++ b/morpheus-testing/src/test/scala/org/opencypher/morpheus/impl/acceptance/ExpressionTests.scala @@ -31,7 +31,12 @@ import org.opencypher.morpheus.testing.MorpheusTestSuite import org.opencypher.morpheus.testing.support.creation.graphs.ScanGraphFactory import org.opencypher.okapi.api.value.CypherValue import org.opencypher.okapi.api.value.CypherValue.Format.defaultValueFormatter -import org.opencypher.okapi.api.value.CypherValue.{CypherFloat, CypherInteger, CypherList, CypherMap} +import org.opencypher.okapi.api.value.CypherValue.{ + CypherFloat, + CypherInteger, + CypherList, + CypherMap +} import org.opencypher.okapi.api.value.GenCypherValue._ import org.opencypher.okapi.impl.exception.IllegalArgumentException import org.opencypher.okapi.impl.temporal.Duration @@ -49,34 +54,41 @@ class ExpressionTests extends MorpheusTestSuite with ScanGraphInit with Checkers it("slice") { val result = morpheus.cypher("RETURN ['a', 'b', 'c', 'd'][0..3] as r") - result.records.toMaps should equal(Bag( - CypherMap("r" -> CypherList("a", "b", "c")) - )) + result.records.toMaps should equal( + Bag( + CypherMap("r" -> CypherList("a", "b", "c")) + ) + ) } it("slice without from") { val result = morpheus.cypher("RETURN ['a', 'b', 'c', 'd'][..3] as r") - result.records.toMaps should equal(Bag( - CypherMap("r" -> CypherList("a", "b", "c")) - )) + result.records.toMaps should equal( + Bag( + CypherMap("r" -> CypherList("a", "b", "c")) + ) + ) } it("slice without to") { val result = morpheus.cypher("RETURN ['a', 'b', 'c', 'd'][0..] as r") - result.records.toMaps should equal(Bag( - CypherMap("r" -> CypherList("a", "b", "c", "d")) - )) + result.records.toMaps should equal( + Bag( + CypherMap("r" -> CypherList("a", "b", "c", "d")) + ) + ) } it("slice on empty list") { - val result = morpheus.cypher( - """ + val result = morpheus.cypher(""" |WITH [] AS things |Return things[1..] as r """.stripMargin) - result.records.toMaps should equal(Bag( - CypherMap("r" -> null) - )) + result.records.toMaps should equal( + Bag( + CypherMap("r" -> null) + ) + ) } } @@ -84,16 +96,14 @@ class ExpressionTests extends MorpheusTestSuite with ScanGraphInit with Checkers it("should evaluate a generic CASE expression with default") { // Given val given = - initGraph( - """ + initGraph(""" |CREATE (:Person {val: "foo"}) |CREATE (:Person {val: "bar"}) |CREATE (:Person {val: "baz"}) """.stripMargin) // When - val result = given.cypher( - """MATCH (n) + val result = given.cypher("""MATCH (n) |RETURN | n.val, | CASE @@ -104,26 +114,26 @@ class ExpressionTests extends MorpheusTestSuite with ScanGraphInit with Checkers """.stripMargin) // Then - result.records.toMaps should equal(Bag( - CypherMap("n.val" -> "foo", "result" -> 1), - CypherMap("n.val" -> "bar", "result" -> 2), - CypherMap("n.val" -> "baz", "result" -> 3)) + result.records.toMaps should equal( + Bag( + CypherMap("n.val" -> "foo", "result" -> 1), + CypherMap("n.val" -> "bar", "result" -> 2), + CypherMap("n.val" -> "baz", "result" -> 3) + ) ) } it("should evaluate a simple equality CASE expression") { // Given val given = - initGraph( - """ + initGraph(""" |CREATE (:Person {val: "foo"}) |CREATE (:Person {val: "bar"}) |CREATE (:Person {val: "baz"}) """.stripMargin) // When - val result = given.cypher( - """MATCH (n) + val result = given.cypher("""MATCH (n) |RETURN | n.val, | CASE n.val @@ -134,26 +144,26 @@ class ExpressionTests extends MorpheusTestSuite with ScanGraphInit with Checkers """.stripMargin) // Then - result.records.toMaps should equal(Bag( - CypherMap("n.val" -> "foo", "result" -> 1), - CypherMap("n.val" -> "bar", "result" -> 2), - CypherMap("n.val" -> "baz", "result" -> 3)) + result.records.toMaps should equal( + Bag( + CypherMap("n.val" -> "foo", "result" -> 1), + CypherMap("n.val" -> "bar", "result" -> 2), + CypherMap("n.val" -> "baz", "result" -> 3) + ) ) } it("should evaluate an inner CASE expression with default") { // Given val given = - initGraph( - """ + initGraph(""" |CREATE (:Person {val: "foo", amount: 42 }) |CREATE (:Person {val: "bar", amount: 23 }) |CREATE (:Person {val: "baz", amount: 84 }) """.stripMargin) // When - val result = given.cypher( - """MATCH (n) + val result = given.cypher("""MATCH (n) |RETURN | n.val, | sum(CASE n.val @@ -164,10 +174,12 @@ class ExpressionTests extends MorpheusTestSuite with ScanGraphInit with Checkers """.stripMargin) // Then - result.records.toMaps should equal(Bag( - CypherMap("n.val" -> "foo", "result" -> 42), - CypherMap("n.val" -> "bar", "result" -> 1984), - CypherMap("n.val" -> "baz", "result" -> 0)) + result.records.toMaps should equal( + Bag( + CypherMap("n.val" -> "foo", "result" -> 42), + CypherMap("n.val" -> "bar", "result" -> 1984), + CypherMap("n.val" -> "baz", "result" -> 0) + ) ) } @@ -176,8 +188,7 @@ class ExpressionTests extends MorpheusTestSuite with ScanGraphInit with Checkers describe("properties") { it("handles property expression on unknown label") { // Given - val given = initGraph( - """ + val given = initGraph(""" |CREATE (p:Person {firstName: "Alice", lastName: "Foo"}) """.stripMargin) @@ -195,26 +206,25 @@ class ExpressionTests extends MorpheusTestSuite with ScanGraphInit with Checkers it("handles unknown properties") { // Given - val given = initGraph( - """ + val given = initGraph(""" |CREATE (p:Person {firstName: "Alice", lastName: "Foo"}) """.stripMargin) // When - val result = given.cypher( - """ + val result = given.cypher(""" |MATCH (a:Person) |RETURN a.firstName, a.age """.stripMargin) // Then - result.records.toMaps should equal(Bag(CypherMap("a.age" -> null, "a.firstName" -> "Alice"))) + result.records.toMaps should equal( + Bag(CypherMap("a.age" -> null, "a.firstName" -> "Alice")) + ) } it("equality between properties") { // Given - val given = initGraph( - """ + val given = initGraph(""" |CREATE (:A {val: 1})-[:REL]->(:B {p: 2}) |CREATE (:A {val: 2})-[:REL]->(:B {p: 1}) |CREATE (:A {val: 100})-[:REL]->(:B {p: 100}) @@ -227,20 +237,22 @@ class ExpressionTests extends MorpheusTestSuite with ScanGraphInit with Checkers val result = given.cypher("MATCH (a:A)-->(b:B) RETURN a.val = b.p AS eq") // Then - result.records.toMaps should equal(Bag( - CypherMap("eq" -> false), - CypherMap("eq" -> false), - CypherMap("eq" -> true), - CypherMap("eq" -> null), - CypherMap("eq" -> null), - CypherMap("eq" -> null) - )) + result.records.toMaps should equal( + Bag( + CypherMap("eq" -> false), + CypherMap("eq" -> false), + CypherMap("eq" -> true), + CypherMap("eq" -> null), + CypherMap("eq" -> null), + CypherMap("eq" -> null) + ) + ) } it("filter rels on property regular expression") { // Given - val given = initGraph( - """CREATE (rachel:Person:Actor {name: 'Rachel Kempson', birthyear: 1910}) + val given = + initGraph("""CREATE (rachel:Person:Actor {name: 'Rachel Kempson', birthyear: 1910}) |CREATE (michael:Person:Actor {name: 'Michael Redgrave', birthyear: 1908}) |CREATE (corin:Person:Actor {name: 'Corin Redgrave', birthyear: 1939}) |CREATE (liam:Person:Actor {name: 'Liam Neeson', birthyear: 1952}) @@ -270,14 +282,18 @@ class ExpressionTests extends MorpheusTestSuite with ScanGraphInit with Checkers // Then val records = result.records.collect - records.toBag should equal(Bag(CypherMap("r.charactername" -> "Henri Ducard"), - CypherMap("r.charactername" -> "Albus Dumbledore"))) + records.toBag should equal( + Bag( + CypherMap("r.charactername" -> "Henri Ducard"), + CypherMap("r.charactername" -> "Albus Dumbledore") + ) + ) } it("filter nodes on property regular expression") { // Given - val given = initGraph( - """CREATE (rachel:Person:Actor {name: 'Rachel Kempson', birthyear: 1910}) + val given = + initGraph("""CREATE (rachel:Person:Actor {name: 'Rachel Kempson', birthyear: 1910}) |CREATE (michael:Person:Actor {name: 'Michael Redgrave', birthyear: 1908}) |CREATE (corin:Person:Actor {name: 'Corin Redgrave', birthyear: 1939}) |CREATE (liam:Person:Actor {name: 'Liam Neeson', birthyear: 1952}) @@ -307,37 +323,50 @@ class ExpressionTests extends MorpheusTestSuite with ScanGraphInit with Checkers // Then val records = result.records.collect - records.toBag should equal(Bag(CypherMap("p.name" -> "Michael Redgrave"), - CypherMap("p.name" -> "Corin Redgrave"), - CypherMap("p.name" -> "Jemma Redgrave"))) + records.toBag should equal( + Bag( + CypherMap("p.name" -> "Michael Redgrave"), + CypherMap("p.name" -> "Corin Redgrave"), + CypherMap("p.name" -> "Jemma Redgrave") + ) + ) } it("supports simple property expression") { // Given - val given = initGraph("CREATE (:Person {name: 'Mats'})-[:REL]->(:Person {name: 'Martin'})") + val given = initGraph( + "CREATE (:Person {name: 'Mats'})-[:REL]->(:Person {name: 'Martin'})" + ) // When val result = given.cypher("MATCH (p:Person) RETURN p.name") // Then - result.records.toMaps should equal(Bag( - CypherMap("p.name" -> "Mats"), - CypherMap("p.name" -> "Martin") - )) + result.records.toMaps should equal( + Bag( + CypherMap("p.name" -> "Mats"), + CypherMap("p.name" -> "Martin") + ) + ) } it("supports simple property expression on relationship") { // Given - val given = initGraph("CREATE (:Person {name: 'Mats'})-[:KNOWS {since: 2017}]->(:Person {name: 'Martin'})") + val given = initGraph( + "CREATE (:Person {name: 'Mats'})-[:KNOWS {since: 2017}]->(:Person {name: 'Martin'})" + ) // When - val result = given.cypher("MATCH (a:Person)-[r:KNOWS]->(b:Person) RETURN r.since") + val result = + given.cypher("MATCH (a:Person)-[r:KNOWS]->(b:Person) RETURN r.since") // Then - result.records.toMaps should equal(Bag( - CypherMap("r.since" -> 2017) - )) + result.records.toMaps should equal( + Bag( + CypherMap("r.since" -> 2017) + ) + ) } it("reports error message when property has ANY type") { @@ -356,118 +385,155 @@ class ExpressionTests extends MorpheusTestSuite with ScanGraphInit with Checkers test("less than") { // Given - val given = initGraph("CREATE ({val: 4})-[:REL]->({val: 5})-[:REL]->({val: 5})-[:REL]->({val: 2})-[:REL]->()") + val given = initGraph( + "CREATE ({val: 4})-[:REL]->({val: 5})-[:REL]->({val: 5})-[:REL]->({val: 2})-[:REL]->()" + ) // When val result = given.cypher("MATCH (n)-->(m) RETURN n.val < m.val") // Then - result.records.toMaps should equal(Bag( - CypherMap("n.val < m.val" -> true), - CypherMap("n.val < m.val" -> false), - CypherMap("n.val < m.val" -> false), - CypherMap("n.val < m.val" -> null) - )) + result.records.toMaps should equal( + Bag( + CypherMap("n.val < m.val" -> true), + CypherMap("n.val < m.val" -> false), + CypherMap("n.val < m.val" -> false), + CypherMap("n.val < m.val" -> null) + ) + ) } test("less than or equal") { // Given - val given = initGraph("CREATE ({val: 4})-[:REL]->({val: 5})-[:REL]->({val: 5})-[:REL]->({val: 2})-[:REL]->()") + val given = initGraph( + "CREATE ({val: 4})-[:REL]->({val: 5})-[:REL]->({val: 5})-[:REL]->({val: 2})-[:REL]->()" + ) // When val result = given.cypher("MATCH (n)-->(m) RETURN n.val <= m.val") // Then - result.records.toMaps should equal(Bag( - CypherMap("n.val <= m.val" -> true), - CypherMap("n.val <= m.val" -> true), - CypherMap("n.val <= m.val" -> false), - CypherMap("n.val <= m.val" -> null) - )) + result.records.toMaps should equal( + Bag( + CypherMap("n.val <= m.val" -> true), + CypherMap("n.val <= m.val" -> true), + CypherMap("n.val <= m.val" -> false), + CypherMap("n.val <= m.val" -> null) + ) + ) } test("greater than") { // Given - val given = initGraph("CREATE ({val: 4})-[:REL]->({val: 5})-[:REL]->({val: 5})-[:REL]->({val: 2})-[:REL]->()") + val given = initGraph( + "CREATE ({val: 4})-[:REL]->({val: 5})-[:REL]->({val: 5})-[:REL]->({val: 2})-[:REL]->()" + ) // When val result = given.cypher("MATCH (n)-->(m) RETURN n.val > m.val AS gt") // Then - result.records.toMaps should equal(Bag( - CypherMap("gt" -> false), - CypherMap("gt" -> false), - CypherMap("gt" -> true), - CypherMap("gt" -> null) - )) + result.records.toMaps should equal( + Bag( + CypherMap("gt" -> false), + CypherMap("gt" -> false), + CypherMap("gt" -> true), + CypherMap("gt" -> null) + ) + ) } test("greater than or equal") { // Given - val given = initGraph("CREATE ({val: 4})-[:REL]->({val: 5})-[:REL]->({val: 5})-[:REL]->({val: 2})-[:REL]->()") + val given = initGraph( + "CREATE ({val: 4})-[:REL]->({val: 5})-[:REL]->({val: 5})-[:REL]->({val: 2})-[:REL]->()" + ) // When val result = given.cypher("MATCH (n)-->(m) RETURN n.val >= m.val") // Then - result.records.toMaps should equal(Bag( - CypherMap("n.val >= m.val" -> false), - CypherMap("n.val >= m.val" -> true), - CypherMap("n.val >= m.val" -> true), - CypherMap("n.val >= m.val" -> null) - )) + result.records.toMaps should equal( + Bag( + CypherMap("n.val >= m.val" -> false), + CypherMap("n.val >= m.val" -> true), + CypherMap("n.val >= m.val" -> true), + CypherMap("n.val >= m.val" -> null) + ) + ) } it("supports integer addition") { - check(Prop.forAll(integer, integer) { (i1: CypherInteger, i2: CypherInteger) => - val query = s"RETURN ${i1.toCypherString} + ${i2.toCypherString} AS result" - if (BigInt(i1.unwrap) + BigInt(i2.unwrap) != BigInt(i1.unwrap + i2.unwrap)) { - // Long over-/underflow - val e = the[ParsingException] thrownBy morpheus.cypher(query).records.toMaps - Claim(e.getMessage.contains("SemanticError") && e.getMessage.contains("cannot be represented as an integer")) - } else { - val result = morpheus.cypher(query).records.toMaps - val expected = Bag(CypherMap("result" -> (i1.unwrap + i2.unwrap))) - Claim(result == expected) - } - }, minSuccessful(100)) + check( + Prop.forAll(integer, integer) { (i1: CypherInteger, i2: CypherInteger) => + val query = + s"RETURN ${i1.toCypherString} + ${i2.toCypherString} AS result" + if (BigInt(i1.unwrap) + BigInt(i2.unwrap) != BigInt(i1.unwrap + i2.unwrap)) { + // Long over-/underflow + val e = + the[ParsingException] thrownBy morpheus.cypher(query).records.toMaps + Claim( + e.getMessage.contains("SemanticError") && e.getMessage.contains( + "cannot be represented as an integer" + ) + ) + } else { + val result = morpheus.cypher(query).records.toMaps + val expected = Bag(CypherMap("result" -> (i1.unwrap + i2.unwrap))) + Claim(result == expected) + } + }, + minSuccessful(100) + ) } it("supports float addition") { - check(Prop.forAll(float, float) { (f1: CypherFloat, f2: CypherFloat) => - val query = s"RETURN ${f1.toCypherString} + ${f2.toCypherString} AS result" - val result = morpheus.cypher(query).records.toMaps - val expected = Bag(CypherMap("result" -> (f1.unwrap + f2.unwrap))) - Claim(result == expected) - }, minSuccessful(100)) + check( + Prop.forAll(float, float) { (f1: CypherFloat, f2: CypherFloat) => + val query = + s"RETURN ${f1.toCypherString} + ${f2.toCypherString} AS result" + val result = morpheus.cypher(query).records.toMaps + val expected = Bag(CypherMap("result" -> (f1.unwrap + f2.unwrap))) + Claim(result == expected) + }, + minSuccessful(100) + ) } it("supports addition after matching a pattern") { // Given - val given = initGraph("CREATE ({val: 4})-[:REL]->({val: 5, other: 3})-[:REL]->()") + val given = + initGraph("CREATE ({val: 4})-[:REL]->({val: 5, other: 3})-[:REL]->()") // When - val result = given.cypher("MATCH (n)-->(m) RETURN m.other + m.val + n.val AS res") + val result = + given.cypher("MATCH (n)-->(m) RETURN m.other + m.val + n.val AS res") // Then - result.records.toMaps should equal(Bag( - CypherMap("res" -> 12), - CypherMap("res" -> null) - )) + result.records.toMaps should equal( + Bag( + CypherMap("res" -> 12), + CypherMap("res" -> null) + ) + ) } test("subtraction with name") { // Given - val given = initGraph("CREATE ({val: 4})-[:REL]->({val: 5, other: 3})-[:REL]->()") + val given = + initGraph("CREATE ({val: 4})-[:REL]->({val: 5, other: 3})-[:REL]->()") // When - val result = given.cypher("MATCH (n)-->(m) RETURN m.val - n.val - m.other AS res") + val result = + given.cypher("MATCH (n)-->(m) RETURN m.val - n.val - m.other AS res") // Then - result.records.toMaps should equal(Bag( - CypherMap("res" -> -2), - CypherMap("res" -> null) - )) + result.records.toMaps should equal( + Bag( + CypherMap("res" -> -2), + CypherMap("res" -> null) + ) + ) } test("subtraction without name") { @@ -478,81 +544,101 @@ class ExpressionTests extends MorpheusTestSuite with ScanGraphInit with Checkers val result = given.cypher("MATCH (n:Node)-->(m:Node) RETURN m.val - n.val") // Then - result.records.toMaps should equal(Bag( - CypherMap("m.val - n.val" -> 1) - )) + result.records.toMaps should equal( + Bag( + CypherMap("m.val - n.val" -> 1) + ) + ) } test("multiplication with integer") { // Given - val given = initGraph("CREATE (:Node {val: 9})-[:REL]->(:Node {val: 2})-[:REL]->(:Node {val: 3})") + val given = initGraph( + "CREATE (:Node {val: 9})-[:REL]->(:Node {val: 2})-[:REL]->(:Node {val: 3})" + ) // When val result = given.cypher("MATCH (n:Node)-->(m:Node) RETURN n.val * m.val") // Then - result.records.toMaps should equal(Bag( - CypherMap("n.val * m.val" -> 18), - CypherMap("n.val * m.val" -> 6) - )) + result.records.toMaps should equal( + Bag( + CypherMap("n.val * m.val" -> 18), + CypherMap("n.val * m.val" -> 6) + ) + ) } test("multiplication with float") { // Given - val given = initGraph("CREATE (:Node {val: 4.5D})-[:REL]->(:Node {val: 2.5D})") + val given = + initGraph("CREATE (:Node {val: 4.5D})-[:REL]->(:Node {val: 2.5D})") // When val result = given.cypher("MATCH (n:Node)-->(m:Node) RETURN n.val * m.val") // Then - result.records.toMaps should equal(Bag( - CypherMap("n.val * m.val" -> 11.25) - )) + result.records.toMaps should equal( + Bag( + CypherMap("n.val * m.val" -> 11.25) + ) + ) } test("multiplication with integer and float") { // Given - val given = initGraph("CREATE (:Node {val: 9})-[:REL]->(:Node {val2: 2.5D})") + val given = + initGraph("CREATE (:Node {val: 9})-[:REL]->(:Node {val2: 2.5D})") // When val result = given.cypher("MATCH (n:Node)-->(m:Node) RETURN n.val * m.val2") // Then - result.records.toMaps should equal(Bag( - CypherMap("n.val * m.val2" -> 22.5) - )) + result.records.toMaps should equal( + Bag( + CypherMap("n.val * m.val2" -> 22.5) + ) + ) } test("division with no remainder") { // Given - val given = initGraph("CREATE (:Node {val: 9})-[:REL]->(:Node {val: 3})-[:REL]->(:Node {val: 2})") + val given = initGraph( + "CREATE (:Node {val: 9})-[:REL]->(:Node {val: 3})-[:REL]->(:Node {val: 2})" + ) // When val result = given.cypher("MATCH (n:Node)-->(m:Node) RETURN n.val / m.val") // Then - result.records.toMaps should equal(Bag( - CypherMap("n.val / m.val" -> 3), - CypherMap("n.val / m.val" -> 1) - )) + result.records.toMaps should equal( + Bag( + CypherMap("n.val / m.val" -> 3), + CypherMap("n.val / m.val" -> 1) + ) + ) } test("division integer and float and null") { // Given - val given = initGraph("CREATE (:Node {val: 9})-[:REL]->(:Node {val2: 4.5D})-[:REL]->(:Node)") + val given = initGraph( + "CREATE (:Node {val: 9})-[:REL]->(:Node {val2: 4.5D})-[:REL]->(:Node)" + ) // When val result = given.cypher("MATCH (n:Node)-->(m:Node) RETURN n.val / m.val2") // Then - result.records.toMaps should equal(Bag( - CypherMap("n.val / m.val2" -> 2.0), - CypherMap("n.val / m.val2" -> null) - )) + result.records.toMaps should equal( + Bag( + CypherMap("n.val / m.val2" -> 2.0), + CypherMap("n.val / m.val2" -> null) + ) + ) } @@ -564,9 +650,11 @@ class ExpressionTests extends MorpheusTestSuite with ScanGraphInit with Checkers val result = given.cypher("MATCH (n:Node) RETURN n.val / 0.5 AS res") // Then - result.records.toMaps should equal(Bag( - CypherMap("res" -> 9.0) - )) + result.records.toMaps should equal( + Bag( + CypherMap("res" -> 9.0) + ) + ) } @@ -576,9 +664,11 @@ class ExpressionTests extends MorpheusTestSuite with ScanGraphInit with Checkers val result = morpheus.cypher("RETURN 3 % 2 as res") // Then - result.records.toMaps should equal(Bag( - CypherMap("res" -> 1) - )) + result.records.toMaps should equal( + Bag( + CypherMap("res" -> 1) + ) + ) } it("computes modulo on an integer property") { @@ -589,9 +679,11 @@ class ExpressionTests extends MorpheusTestSuite with ScanGraphInit with Checkers val result = given.cypher("MATCH (n:Node) RETURN n.val % 2 AS res") // Then - result.records.toMaps should equal(Bag( - CypherMap("res" -> 1) - )) + result.records.toMaps should equal( + Bag( + CypherMap("res" -> 1) + ) + ) } it("computes modulo on float properties") { @@ -612,155 +704,162 @@ class ExpressionTests extends MorpheusTestSuite with ScanGraphInit with Checkers it("equality") { // Given - val given = initGraph( - """ + val given = initGraph(""" |CREATE (:Node {val: 4})-[:REL]->(:Node {val: 5}) |CREATE (:Node {val: 4})-[:REL]->(:Node {val: 4}) |CREATE (:Node)-[:REL]->(:Node {val: 5}) """.stripMargin) // When - val result = given.cypher("MATCH (n:Node)-->(m:Node) RETURN m.val = n.val AS res") + val result = + given.cypher("MATCH (n:Node)-->(m:Node) RETURN m.val = n.val AS res") // Then - result.records.toMaps should equal(Bag( - CypherMap("res" -> false), - CypherMap("res" -> true), - CypherMap("res" -> null) - )) + result.records.toMaps should equal( + Bag( + CypherMap("res" -> false), + CypherMap("res" -> true), + CypherMap("res" -> null) + ) + ) } describe("EXISTS with pattern") { it("evaluates basic exists pattern") { // Given - val given = initGraph( - """ + val given = initGraph(""" |CREATE (v {id: 1})-[:REL]->({id: 2})-[:REL]->(w {id: 3}) |CREATE (v)-[:REL]->(w) |CREATE (w)-[:REL]->({id: 4}) """.stripMargin) // When - val result = given.cypher("MATCH (a)-->(b) WITH a, b, EXISTS((a)-->()-->(b)) as con RETURN a.id, b.id, con") + val result = given.cypher( + "MATCH (a)-->(b) WITH a, b, EXISTS((a)-->()-->(b)) as con RETURN a.id, b.id, con" + ) // Then - result.records.toMaps should equal(Bag( - CypherMap("a.id" -> 1L, "b.id" -> 3L, "con" -> true), - CypherMap("a.id" -> 1L, "b.id" -> 2L, "con" -> false), - CypherMap("a.id" -> 2L, "b.id" -> 3L, "con" -> false), - CypherMap("a.id" -> 3L, "b.id" -> 4L, "con" -> false) - )) + result.records.toMaps should equal( + Bag( + CypherMap("a.id" -> 1L, "b.id" -> 3L, "con" -> true), + CypherMap("a.id" -> 1L, "b.id" -> 2L, "con" -> false), + CypherMap("a.id" -> 2L, "b.id" -> 3L, "con" -> false), + CypherMap("a.id" -> 3L, "b.id" -> 4L, "con" -> false) + ) + ) } it("evaluates exists pattern with var-length-expand") { // Given - val given = initGraph("CREATE (v {id: 1})-[:REL]->({id: 2})-[:REL]->({id: 3})<-[:REL]-(v)") + val given = initGraph( + "CREATE (v {id: 1})-[:REL]->({id: 2})-[:REL]->({id: 3})<-[:REL]-(v)" + ) // When - val result = given.cypher( - """ + val result = given.cypher(""" |MATCH (a)-->(b) |WITH a, b, EXISTS((a)-[*1..3]->()-->(b)) as con |RETURN a.id, b.id, con""".stripMargin) // Then - result.records.toMaps should equal(Bag( - CypherMap("a.id" -> 1L, "b.id" -> 2L, "con" -> false), - CypherMap("a.id" -> 1L, "b.id" -> 3L, "con" -> true), - CypherMap("a.id" -> 2L, "b.id" -> 3L, "con" -> false) - )) + result.records.toMaps should equal( + Bag( + CypherMap("a.id" -> 1L, "b.id" -> 2L, "con" -> false), + CypherMap("a.id" -> 1L, "b.id" -> 3L, "con" -> true), + CypherMap("a.id" -> 2L, "b.id" -> 3L, "con" -> false) + ) + ) } it("can evaluate simple exists pattern with node predicate") { // Given - val given = initGraph( - """ + val given = initGraph(""" |CREATE ({id: 1})-[:REL]->({id: 2, name: 'foo'}) |CREATE ({id: 3})-[:REL]->({id: 4, name: 'bar'}) """.stripMargin) // When - val result = given.cypher( - """ + val result = given.cypher(""" |MATCH (a) |WITH a, EXISTS((a)-->({name: 'foo'})) AS con |RETURN a.id, con""".stripMargin) - // Then - result.records.toMaps should equal(Bag( - CypherMap("a.id" -> 1L, "con" -> true), - CypherMap("a.id" -> 2L, "con" -> false), - CypherMap("a.id" -> 3L, "con" -> false), - CypherMap("a.id" -> 4L, "con" -> false) - )) + result.records.toMaps should equal( + Bag( + CypherMap("a.id" -> 1L, "con" -> true), + CypherMap("a.id" -> 2L, "con" -> false), + CypherMap("a.id" -> 3L, "con" -> false), + CypherMap("a.id" -> 4L, "con" -> false) + ) + ) } it("can evaluate simple exists pattern with relationship predicate") { // Given - val given = initGraph( - """ + val given = initGraph(""" |CREATE (v {id: 1})-[:REL {val: 'foo'}]->({id: 2})<-[:REL]-(v) |CREATE (w {id: 3})-[:REL {val: 'bar'}]->({id: 4})<-[:REL]-(w) """.stripMargin) // When - val result = given.cypher( - """ + val result = given.cypher(""" |MATCH (a)-->(b) |WITH DISTINCT a, b |WITH a, b, EXISTS((a)-[{val: 'foo'}]->(b)) AS con |RETURN a.id, b.id, con""".stripMargin) // Then - result.records.toMaps should equal(Bag( - CypherMap("a.id" -> 1L, "b.id" -> 2L, "con" -> true), - CypherMap("a.id" -> 3L, "b.id" -> 4L, "con" -> false) - )) + result.records.toMaps should equal( + Bag( + CypherMap("a.id" -> 1L, "b.id" -> 2L, "con" -> true), + CypherMap("a.id" -> 3L, "b.id" -> 4L, "con" -> false) + ) + ) } it("can evaluate simple exists pattern with node label predicate") { // Given - val given = initGraph( - """ + val given = initGraph(""" |CREATE (v:SRC {id: 1})-[:REL]->(:A) |CREATE (w:SRC {id: 2})-[:REL]->(:B) """.stripMargin) // When - val result = given.cypher( - """ + val result = given.cypher(""" |MATCH (a:SRC) |WITH a, EXISTS((a)-->(:A)) AS con |RETURN a.id, con""".stripMargin) // Then - result.records.toMaps should equal(Bag( - CypherMap("a.id" -> 1L, "con" -> true), - CypherMap("a.id" -> 2L, "con" -> false) - )) + result.records.toMaps should equal( + Bag( + CypherMap("a.id" -> 1L, "con" -> true), + CypherMap("a.id" -> 2L, "con" -> false) + ) + ) } it("can evaluate simple exists pattern with relationship type predicate") { // Given - val given = initGraph( - """ + val given = initGraph(""" |CREATE (v {id: 1})-[:A]->({id: 2})<-[:REL]-(v) |CREATE (w {id: 3})-[:B]->({id: 4})<-[:REL]-(w) """.stripMargin) // When - val result = given.cypher( - """ + val result = given.cypher(""" |MATCH (a)-[:REL]->(b) |WITH a, b, EXISTS((a)-[:A]->(b)) AS con |RETURN a.id, b.id, con""".stripMargin) // Then - result.records.toMaps should equal(Bag( - CypherMap("a.id" -> 1L, "b.id" -> 2L, "con" -> true), - CypherMap("a.id" -> 3L, "b.id" -> 4L, "con" -> false) - )) + result.records.toMaps should equal( + Bag( + CypherMap("a.id" -> 1L, "b.id" -> 2L, "con" -> true), + CypherMap("a.id" -> 3L, "b.id" -> 4L, "con" -> false) + ) + ) } it("can evaluate inverse exist pattern") { @@ -768,25 +867,25 @@ class ExpressionTests extends MorpheusTestSuite with ScanGraphInit with Checkers val given = initGraph("CREATE (v {id: 1})-[:REL]->({id: 2})") // When - val result = given.cypher( - """ + val result = given.cypher(""" |MATCH (a), (b) |WITH a, b, NOT EXISTS((a)-->(b)) AS con |RETURN a.id, b.id, con""".stripMargin) // Then - result.records.toMaps should equal(Bag( - CypherMap("a.id" -> 1L, "b.id" -> 1L, "con" -> true), - CypherMap("a.id" -> 1L, "b.id" -> 2L, "con" -> false), - CypherMap("a.id" -> 2L, "b.id" -> 1L, "con" -> true), - CypherMap("a.id" -> 2L, "b.id" -> 2L, "con" -> true) - )) + result.records.toMaps should equal( + Bag( + CypherMap("a.id" -> 1L, "b.id" -> 1L, "con" -> true), + CypherMap("a.id" -> 1L, "b.id" -> 2L, "con" -> false), + CypherMap("a.id" -> 2L, "b.id" -> 1L, "con" -> true), + CypherMap("a.id" -> 2L, "b.id" -> 2L, "con" -> true) + ) + ) } it("can evaluate exist pattern with derived node predicate") { // Given - val given = initGraph( - """ + val given = initGraph(""" |CREATE ({id: 1, val: 0})-[:REL]->({id: 2, val: 2})<-[:REL]-({id: 3, val: 10}) """.stripMargin) @@ -794,14 +893,17 @@ class ExpressionTests extends MorpheusTestSuite with ScanGraphInit with Checkers val result = given.cypher( """ |MATCH (a) - |WITH a, EXISTS((a)-->({val: a.val + 2})) AS other RETURN a.id, other""".stripMargin) + |WITH a, EXISTS((a)-->({val: a.val + 2})) AS other RETURN a.id, other""".stripMargin + ) // Then - result.records.toMaps should equal(Bag( - CypherMap("a.id" -> 1L, "other" -> true), - CypherMap("a.id" -> 2L, "other" -> false), - CypherMap("a.id" -> 3L, "other" -> false) - )) + result.records.toMaps should equal( + Bag( + CypherMap("a.id" -> 1L, "other" -> true), + CypherMap("a.id" -> 2L, "other" -> false), + CypherMap("a.id" -> 3L, "other" -> false) + ) + ) } } @@ -812,60 +914,66 @@ class ExpressionTests extends MorpheusTestSuite with ScanGraphInit with Checkers val result = graph.cypher( """ |WITH [$a, $b] as strings - |RETURN strings""".stripMargin, Map("a" -> CypherValue("bar"), "b" -> CypherValue("foo"))) + |RETURN strings""".stripMargin, + Map("a" -> CypherValue("bar"), "b" -> CypherValue("foo")) + ) - result.records.toMaps should equal(Bag( - CypherMap("strings" -> Seq("bar", "foo")) - )) + result.records.toMaps should equal( + Bag( + CypherMap("strings" -> Seq("bar", "foo")) + ) + ) } it("can convert string ListLiterals") { val graph = initGraph("CREATE ()") - val result = graph.cypher( - """ + val result = graph.cypher(""" |WITH ["bar", "foo"] as strings |RETURN strings""".stripMargin) - result.records.toMaps should equal(Bag( - CypherMap("strings" -> Seq("bar", "foo")) - )) + result.records.toMaps should equal( + Bag( + CypherMap("strings" -> Seq("bar", "foo")) + ) + ) } it("can convert ListLiterals with nested non literal expressions") { val graph = initGraph("CREATE ({val: 1}), ({val: 2})") - val result = graph.cypher( - """ + val result = graph.cypher(""" |MATCH (n) |WITH [n.val*10, n.val*100] as vals |RETURN vals""".stripMargin) - result.records.toMaps should equal(Bag( - CypherMap("vals" -> Seq(10, 100)), - CypherMap("vals" -> Seq(20, 200)) - )) + result.records.toMaps should equal( + Bag( + CypherMap("vals" -> Seq(10, 100)), + CypherMap("vals" -> Seq(20, 200)) + ) + ) } it("can build lists that include nulls") { - val result = morpheus.cypher( - """ + val result = morpheus.cypher(""" |RETURN [ | 1, | null |] AS p """.stripMargin) - result.records.toMaps should equal(Bag( - CypherMap("p" -> List(1, null)) - )) + result.records.toMaps should equal( + Bag( + CypherMap("p" -> List(1, null)) + ) + ) } } describe("ANDs") { it("can project ands") { - val graph = initGraph( - """ + val graph = initGraph(""" |CREATE ({v1: true, v2: true, v3: true}), ({v1: false, v2: true, v3: true}) """.stripMargin) @@ -876,21 +984,21 @@ class ExpressionTests extends MorpheusTestSuite with ScanGraphInit with Checkers | RETURN n.v1 """.stripMargin('|') - graph.cypher(query).records.toMaps should equal(Bag( - CypherMap("n.v1" -> true) - )) + graph.cypher(query).records.toMaps should equal( + Bag( + CypherMap("n.v1" -> true) + ) + ) } } describe("XOR") { it("can project xors") { - val graph = initGraph( - """ + val graph = initGraph(""" | CREATE ({v1: true, v2: true, res: false}), ({v1: true, v2: false, res: true}), | ({v1: false, v2: true, res: true}), ({v1: false, v2: false, res: false}) """.stripMargin) - val result = graph.cypher( - """ + val result = graph.cypher(""" | MATCH (n) | WHERE n.v1 XOR n.v2 = n.res | RETURN n @@ -902,8 +1010,7 @@ class ExpressionTests extends MorpheusTestSuite with ScanGraphInit with Checkers describe("ContainerIndex") { it("Can extract the nth element from a list with literal index") { - val graph = initGraph( - """ + val graph = initGraph(""" |CREATE ({v1: [1, 2, 3]}) """.stripMargin) @@ -913,14 +1020,15 @@ class ExpressionTests extends MorpheusTestSuite with ScanGraphInit with Checkers | RETURN n.v1[1] as val """.stripMargin('|') - graph.cypher(query).records.toMaps should equal(Bag( - CypherMap("val" -> 2) - )) + graph.cypher(query).records.toMaps should equal( + Bag( + CypherMap("val" -> 2) + ) + ) } it("Can extract the nth element from a list with expression index") { - val graph = initGraph( - """ + val graph = initGraph(""" |CREATE ({v1: [1, 2, 3]}) """.stripMargin) @@ -931,16 +1039,17 @@ class ExpressionTests extends MorpheusTestSuite with ScanGraphInit with Checkers | RETURN n.v1[i] as val """.stripMargin('|') - graph.cypher(query).records.toMaps should equal(Bag( - CypherMap("val" -> 1), - CypherMap("val" -> 2), - CypherMap("val" -> 3) - )) + graph.cypher(query).records.toMaps should equal( + Bag( + CypherMap("val" -> 1), + CypherMap("val" -> 2), + CypherMap("val" -> 3) + ) + ) } it("returns null when the index is out of bounds") { - val graph = initGraph( - """ + val graph = initGraph(""" |CREATE ({v1: [1, 2, 3]}) """.stripMargin) @@ -951,232 +1060,348 @@ class ExpressionTests extends MorpheusTestSuite with ScanGraphInit with Checkers | RETURN n.v1[i] as val """.stripMargin('|') - graph.cypher(query).records.toMaps should equal(Bag( - CypherMap("val" -> null), - CypherMap("val" -> null), - CypherMap("val" -> null) - )) + graph.cypher(query).records.toMaps should equal( + Bag( + CypherMap("val" -> null), + CypherMap("val" -> null), + CypherMap("val" -> null) + ) + ) } } describe("string concatenation") { it("can concat two strings from literals") { - morpheus.cypher( - """ + morpheus + .cypher(""" |RETURN "Hello" + "World" as hello - """.stripMargin).records.toMaps should equal(Bag( - CypherMap("hello" -> "HelloWorld") - )) + """.stripMargin) + .records + .toMaps should equal( + Bag( + CypherMap("hello" -> "HelloWorld") + ) + ) } it("can concat two properties") { - val g = initGraph( - """ + val g = initGraph(""" |CREATE (:A {a: "Hello"}) |CREATE (:B {b: "World"}) """.stripMargin) - g.cypher( - """ + g.cypher(""" |MATCH (a:A), (b:B) |RETURN a.a + b.b AS hello - """.stripMargin).records.toMaps should equal(Bag( - CypherMap("hello" -> "HelloWorld") - )) + """.stripMargin) + .records + .toMaps should equal( + Bag( + CypherMap("hello" -> "HelloWorld") + ) + ) } it("can concat a string and an integer") { - val g = initGraph( - """ + val g = initGraph(""" |CREATE (:A {a1: "Hello", a2: 42}) |CREATE (:B {b1: 42, b2: "Hello"}) """.stripMargin) - g.cypher( - """ + g.cypher(""" |MATCH (a:A), (b:B) |RETURN a.a1 + b.b1 AS hello, a.a2 + b.b2 as world - """.stripMargin).records.toMaps should equal(Bag( - CypherMap("hello" -> "Hello42", "world" -> "42Hello") - )) + """.stripMargin) + .records + .toMaps should equal( + Bag( + CypherMap("hello" -> "Hello42", "world" -> "42Hello") + ) + ) } it("can concat a string and a float") { - val g = initGraph( - """ + val g = initGraph(""" |CREATE (:A {a1: "Hello", a2: 42.0}) |CREATE (:B {b1: 42.0, b2: "Hello"}) """.stripMargin) - g.cypher( - """ + g.cypher(""" |MATCH (a:A), (b:B) |RETURN a.a1 + b.b1 AS hello, a.a2 + b.b2 as world - """.stripMargin).records.toMaps should equal(Bag( - CypherMap("hello" -> "Hello42.0", "world" -> "42.0Hello") - )) + """.stripMargin) + .records + .toMaps should equal( + Bag( + CypherMap("hello" -> "Hello42.0", "world" -> "42.0Hello") + ) + ) } } describe("list concatenation") { it("can concat empty lists") { - morpheus.cypher("RETURN [] + [] AS res") - .records.toMaps should equal(Bag( - CypherMap("res" -> CypherList()) - )) + morpheus.cypher("RETURN [] + [] AS res").records.toMaps should equal( + Bag( + CypherMap("res" -> CypherList()) + ) + ) } it("can concat empty list with nonempty list") { - morpheus.cypher("RETURN [] + ['foo'] AS res") - .records.toMaps should equal(Bag( - CypherMap("res" -> CypherList("foo")) - )) + morpheus.cypher("RETURN [] + ['foo'] AS res").records.toMaps should equal( + Bag( + CypherMap("res" -> CypherList("foo")) + ) + ) } it("can concat list of null with nonnull scalar value") { - morpheus.cypher("RETURN [null] + 'foo' AS res") - .records.toMaps should equal(Bag( - CypherMap("res" -> CypherList(null, "foo")) - )) + morpheus + .cypher("RETURN [null] + 'foo' AS res") + .records + .toMaps should equal( + Bag( + CypherMap("res" -> CypherList(null, "foo")) + ) + ) } it("can concat empty list with scalar value") { - morpheus.cypher("RETURN [] + '' AS res") - .records.toMaps should equal(Bag( - CypherMap("res" -> CypherList("")) - )) + morpheus.cypher("RETURN [] + '' AS res").records.toMaps should equal( + Bag( + CypherMap("res" -> CypherList("")) + ) + ) } it("can concat empty list with null scalar value") { - morpheus.cypher("RETURN [] + null AS res") - .records.toMaps should equal(Bag( - CypherMap("res" -> null) - )) + morpheus.cypher("RETURN [] + null AS res").records.toMaps should equal( + Bag( + CypherMap("res" -> null) + ) + ) } it("can concat two literal lists of Cypher integers") { - morpheus.cypher("RETURN [1] + [2] AS res") - .records.toMaps should equal(Bag( - CypherMap("res" -> CypherList(1, 2)) - )) + morpheus.cypher("RETURN [1] + [2] AS res").records.toMaps should equal( + Bag( + CypherMap("res" -> CypherList(1, 2)) + ) + ) } it("can concat two literal lists of strings") { - morpheus.cypher("RETURN ['foo'] + ['bar'] AS res") - .records.toMaps should equal(Bag( - CypherMap("res" -> CypherList("foo", "bar")) - )) + morpheus + .cypher("RETURN ['foo'] + ['bar'] AS res") + .records + .toMaps should equal( + Bag( + CypherMap("res" -> CypherList("foo", "bar")) + ) + ) } it("can concat two literal lists of boolean type") { - morpheus.cypher("RETURN [true] + [false] AS res") - .records.toMaps should equal(Bag( - CypherMap("res" -> CypherList(true, false)) - )) + morpheus + .cypher("RETURN [true] + [false] AS res") + .records + .toMaps should equal( + Bag( + CypherMap("res" -> CypherList(true, false)) + ) + ) } it("can concat two literal lists of float type") { - morpheus.cypher("RETURN [0.5] + [1.5] AS res") - .records.toMaps should equal(Bag( - CypherMap("res" -> CypherList(0.5, 1.5)) - )) + morpheus + .cypher("RETURN [0.5] + [1.5] AS res") + .records + .toMaps should equal( + Bag( + CypherMap("res" -> CypherList(0.5, 1.5)) + ) + ) } it("can concat two literal lists of date type") { val date = "2016-02-17" - morpheus.cypher("RETURN [date($date)] + [date($date)] AS res", CypherMap("date" -> date)) - .records.toMaps should equal(Bag( - CypherMap("res" -> CypherList(java.sql.Date.valueOf(date), java.sql.Date.valueOf(date))) - )) + morpheus + .cypher( + "RETURN [date($date)] + [date($date)] AS res", + CypherMap("date" -> date) + ) + .records + .toMaps should equal( + Bag( + CypherMap( + "res" -> CypherList( + java.sql.Date.valueOf(date), + java.sql.Date.valueOf(date) + ) + ) + ) + ) } it("can concat two literal lists of localdatetime type") { val date = "2016-02-17T06:11:00" - morpheus.cypher("RETURN [localdatetime($date)] + [localdatetime($date)] AS res", CypherMap("date" -> date)) - .records.toMaps should equal(Bag( - CypherMap("res" -> CypherList(java.time.LocalDateTime.parse(date), java.time.LocalDateTime.parse(date))) - )) + morpheus + .cypher( + "RETURN [localdatetime($date)] + [localdatetime($date)] AS res", + CypherMap("date" -> date) + ) + .records + .toMaps should equal( + Bag( + CypherMap( + "res" -> CypherList( + java.time.LocalDateTime.parse(date), + java.time.LocalDateTime.parse(date) + ) + ) + ) + ) } it("can concat two literal lists of duration type") { val duration = "P1WT2H" - morpheus.cypher("RETURN [duration($duration)] + [duration($duration)] AS res", CypherMap("duration" -> duration)) - .records.toMaps should equal(Bag( - CypherMap("res" -> CypherList(Duration.parse(duration), Duration.parse(duration))) - )) + morpheus + .cypher( + "RETURN [duration($duration)] + [duration($duration)] AS res", + CypherMap("duration" -> duration) + ) + .records + .toMaps should equal( + Bag( + CypherMap( + "res" -> CypherList( + Duration.parse(duration), + Duration.parse(duration) + ) + ) + ) + ) } it("can concat two literal lists of null type") { - morpheus.cypher("RETURN [null] + [null] AS res") - .records.toMaps should equal(Bag( - CypherMap("res" -> CypherList(null, null)) - )) + morpheus + .cypher("RETURN [null] + [null] AS res") + .records + .toMaps should equal( + Bag( + CypherMap("res" -> CypherList(null, null)) + ) + ) } it("can concat two lists of nulls from expressions type") { - morpheus.cypher("RETURN [acos(null)] + [acos(null)] AS res") - .records.toMaps should equal(Bag( - CypherMap("res" -> CypherList(null, null)) - )) + morpheus + .cypher("RETURN [acos(null)] + [acos(null)] AS res") + .records + .toMaps should equal( + Bag( + CypherMap("res" -> CypherList(null, null)) + ) + ) } it("can add integer literal to list of integer literals") { - morpheus.cypher("RETURN [1] + 1 AS res") - .records.toMaps should equal(Bag( - CypherMap("res" -> CypherList(1, 1)) - )) + morpheus.cypher("RETURN [1] + 1 AS res").records.toMaps should equal( + Bag( + CypherMap("res" -> CypherList(1, 1)) + ) + ) } it("can add string literal to list of string literals") { - morpheus.cypher("RETURN ['hello'] + 'world' AS res") - .records.toMaps should equal(Bag( - CypherMap("res" -> CypherList("hello", "world")) - )) + morpheus + .cypher("RETURN ['hello'] + 'world' AS res") + .records + .toMaps should equal( + Bag( + CypherMap("res" -> CypherList("hello", "world")) + ) + ) } it("can add boolean literal to list of boolean literals") { - morpheus.cypher("RETURN [true] + false AS res") - .records.toMaps should equal(Bag( - CypherMap("res" -> CypherList(true, false)) - )) + morpheus + .cypher("RETURN [true] + false AS res") + .records + .toMaps should equal( + Bag( + CypherMap("res" -> CypherList(true, false)) + ) + ) } it("can add float literal to list of float literals") { - morpheus.cypher("RETURN [0.5] + 0.5 AS res") - .records.toMaps should equal(Bag( - CypherMap("res" -> CypherList(0.5, 0.5)) - )) + morpheus.cypher("RETURN [0.5] + 0.5 AS res").records.toMaps should equal( + Bag( + CypherMap("res" -> CypherList(0.5, 0.5)) + ) + ) } it("can add date to list of dates") { val date = "2016-02-17" - morpheus.cypher("RETURN [date($date)] + date($date) AS res", CypherMap("date" -> date)) - .records.toMaps should equal(Bag( - CypherMap("res" -> CypherList(java.sql.Date.valueOf(date), java.sql.Date.valueOf(date))) - )) + morpheus + .cypher( + "RETURN [date($date)] + date($date) AS res", + CypherMap("date" -> date) + ) + .records + .toMaps should equal( + Bag( + CypherMap( + "res" -> CypherList( + java.sql.Date.valueOf(date), + java.sql.Date.valueOf(date) + ) + ) + ) + ) } it("can add localdatetime to list of localdatetime") { val date = "2016-02-17T06:11:00" - morpheus.cypher("RETURN [localdatetime($date)] + localdatetime($date) AS res", CypherMap("date" -> date)) - .records.toMaps should equal(Bag( - CypherMap("res" -> CypherList(java.time.LocalDateTime.parse(date), java.time.LocalDateTime.parse(date))) - )) + morpheus + .cypher( + "RETURN [localdatetime($date)] + localdatetime($date) AS res", + CypherMap("date" -> date) + ) + .records + .toMaps should equal( + Bag( + CypherMap( + "res" -> CypherList( + java.time.LocalDateTime.parse(date), + java.time.LocalDateTime.parse(date) + ) + ) + ) + ) } it("can add null literal to list of null literals") { - morpheus.cypher("RETURN [null] + null AS res") - .records.toMaps should equal(Bag( - CypherMap("res" -> null) - )) + morpheus + .cypher("RETURN [null] + null AS res") + .records + .toMaps should equal( + Bag( + CypherMap("res" -> null) + ) + ) } it("can add empty list to string") { - morpheus.cypher("RETURN 'hello' + [] AS res") - .records.toMaps should equal(Bag( - CypherMap("res" -> CypherList("hello")) - )) + morpheus.cypher("RETURN 'hello' + [] AS res").records.toMaps should equal( + Bag( + CypherMap("res" -> CypherList("hello")) + ) + ) } } @@ -1184,52 +1409,82 @@ class ExpressionTests extends MorpheusTestSuite with ScanGraphInit with Checkers describe("parameters") { it("can do list parameters") { - morpheus.cypher("RETURN $listParam AS res", CypherMap("listParam" -> CypherList(1, 2))) - .records.toMaps should equal(Bag( - CypherMap("res" -> CypherList(1, 2)) - )) + morpheus + .cypher( + "RETURN $listParam AS res", + CypherMap("listParam" -> CypherList(1, 2)) + ) + .records + .toMaps should equal( + Bag( + CypherMap("res" -> CypherList(1, 2)) + ) + ) } it("throws exception on mixed types in list parameter") { val e = the[SparkSQLMappingException] thrownBy - morpheus.cypher("RETURN $listParam AS res", CypherMap("listParam" -> CypherList(1, "string"))) - .records.toMaps - e.getMessage should (include("LIST(UNION(INTEGER, STRING))") and include("unsupported")) + morpheus + .cypher( + "RETURN $listParam AS res", + CypherMap("listParam" -> CypherList(1, "string")) + ) + .records + .toMaps + e.getMessage should (include("LIST(UNION(INTEGER, STRING))") and include( + "unsupported" + )) } it("can support empty list parameter") { - morpheus.cypher("RETURN $listParam AS res", CypherMap("listParam" -> CypherList())) - .records.toMaps should equal(Bag( - CypherMap("res" -> CypherList()) - )) + morpheus + .cypher( + "RETURN $listParam AS res", + CypherMap("listParam" -> CypherList()) + ) + .records + .toMaps should equal( + Bag( + CypherMap("res" -> CypherList()) + ) + ) } } describe("STARTS WITH") { it("returns true for matching strings") { - morpheus.cypher( - """ + morpheus + .cypher( + """ |RETURN "foobar" STARTS WITH "foo" as x """.stripMargin - ).records.toMaps should equal(Bag( - CypherMap("x" -> true) - )) + ) + .records + .toMaps should equal( + Bag( + CypherMap("x" -> true) + ) + ) } it("returns false for not matching strings") { - morpheus.cypher( - """ + morpheus + .cypher( + """ |RETURN "foobar" STARTS WITH "bar" as x """.stripMargin - ).records.toMaps should equal(Bag( - CypherMap("x" -> false) - )) + ) + .records + .toMaps should equal( + Bag( + CypherMap("x" -> false) + ) + ) } it("can handle nulls") { - val g = initGraph( - """ + val g = initGraph(""" |CREATE ({s: "foobar", r: null}) |CREATE ({s: null, r: "foo"}) """.stripMargin) @@ -1239,37 +1494,49 @@ class ExpressionTests extends MorpheusTestSuite with ScanGraphInit with Checkers |MATCH (n) |RETURN n.s STARTS WITh n.r as x """.stripMargin - ).records.toMaps should equal(Bag( - CypherMap("x" -> null), - CypherMap("x" -> null) - )) + ).records + .toMaps should equal( + Bag( + CypherMap("x" -> null), + CypherMap("x" -> null) + ) + ) } } describe("ENDS WITH") { it("returns true for matching strings") { - morpheus.cypher( - """ + morpheus + .cypher( + """ |RETURN "foobar" ENDS WITH "bar" as x """.stripMargin - ).records.toMaps should equal(Bag( - CypherMap("x" -> true) - )) + ) + .records + .toMaps should equal( + Bag( + CypherMap("x" -> true) + ) + ) } it("returns false for not matching strings") { - morpheus.cypher( - """ + morpheus + .cypher( + """ |RETURN "foobar" ENDS WITH "foo" as x """.stripMargin - ).records.toMaps should equal(Bag( - CypherMap("x" -> false) - )) + ) + .records + .toMaps should equal( + Bag( + CypherMap("x" -> false) + ) + ) } it("can handle nulls") { - val g = initGraph( - """ + val g = initGraph(""" |CREATE ({s: "foobar", r: null}) |CREATE ({s: null, r: "bar"}) """.stripMargin) @@ -1279,37 +1546,49 @@ class ExpressionTests extends MorpheusTestSuite with ScanGraphInit with Checkers |MATCH (n) |RETURN n.s STARTS WITh n.r as x """.stripMargin - ).records.toMaps should equal(Bag( - CypherMap("x" -> null), - CypherMap("x" -> null) - )) + ).records + .toMaps should equal( + Bag( + CypherMap("x" -> null), + CypherMap("x" -> null) + ) + ) } } describe("CONTAINS") { it("returns true for matching strings") { - morpheus.cypher( - """ + morpheus + .cypher( + """ |RETURN "foobarbaz" CONTAINS "baz" as x """.stripMargin - ).records.toMaps should equal(Bag( - CypherMap("x" -> true) - )) + ) + .records + .toMaps should equal( + Bag( + CypherMap("x" -> true) + ) + ) } it("returns false for not matching strings") { - morpheus.cypher( - """ + morpheus + .cypher( + """ |RETURN "foobarbaz" CONTAINS "abc" as x """.stripMargin - ).records.toMaps should equal(Bag( - CypherMap("x" -> false) - )) + ) + .records + .toMaps should equal( + Bag( + CypherMap("x" -> false) + ) + ) } it("can handle nulls") { - val g = initGraph( - """ + val g = initGraph(""" |CREATE ({s: "foobar", r: null}) |CREATE ({s: null, r: "bar"}) """.stripMargin) @@ -1319,77 +1598,86 @@ class ExpressionTests extends MorpheusTestSuite with ScanGraphInit with Checkers |MATCH (n) |RETURN n.s STARTS WITh n.r as x """.stripMargin - ).records.toMaps should equal(Bag( - CypherMap("x" -> null), - CypherMap("x" -> null) - )) + ).records + .toMaps should equal( + Bag( + CypherMap("x" -> null), + CypherMap("x" -> null) + ) + ) } } describe("properties") { it("can extract properties from nodes") { - val g = initGraph( - """ + val g = initGraph(""" |CREATE (:A {val1: "foo", val2: 42}) |CREATE (:A {val1: "bar", val2: 21}) |CREATE (:A) """.stripMargin) - val result = g.cypher( - """ + val result = g + .cypher(""" |MATCH (a:A) |RETURN properties(a) AS props - """.stripMargin).records - - result.toMaps should equal(Bag( - CypherMap("props" -> CypherMap("val1" -> "foo", "val2" -> 42)), - CypherMap("props" -> CypherMap("val1" -> "bar", "val2" -> 21)), - CypherMap("props" -> CypherMap("val1" -> null, "val2" -> null)) - )) + """.stripMargin) + .records + + result.toMaps should equal( + Bag( + CypherMap("props" -> CypherMap("val1" -> "foo", "val2" -> 42)), + CypherMap("props" -> CypherMap("val1" -> "bar", "val2" -> 21)), + CypherMap("props" -> CypherMap("val1" -> null, "val2" -> null)) + ) + ) } it("can extract properties from relationships") { - val g = initGraph( - """ + val g = initGraph(""" |CREATE (a), (b) |CREATE (a)-[:REL {val1: "foo", val2: 42}]->(b) |CREATE (a)-[:REL {val1: "bar", val2: 21}]->(b) |CREATE (a)-[:REL]->(b) """.stripMargin) - val result = g.cypher( - """ + val result = g + .cypher(""" |MATCH ()-[rel:REL]->() |RETURN properties(rel) as props - """.stripMargin).records - - result.toMaps should equal(Bag( - CypherMap("props" -> CypherMap("val1" -> "foo", "val2" -> 42)), - CypherMap("props" -> CypherMap("val1" -> "bar", "val2" -> 21)), - CypherMap("props" -> CypherMap("val1" -> null, "val2" -> null)) - )) + """.stripMargin) + .records + + result.toMaps should equal( + Bag( + CypherMap("props" -> CypherMap("val1" -> "foo", "val2" -> 42)), + CypherMap("props" -> CypherMap("val1" -> "bar", "val2" -> 21)), + CypherMap("props" -> CypherMap("val1" -> null, "val2" -> null)) + ) + ) } it("can extract properties from maps") { - val g = initGraph( - """ + val g = initGraph(""" |CREATE (a), (b) |CREATE (a)-[:REL {val1: "foo", val2: 42}]->(b) |CREATE (a)-[:REL {val1: "bar", val2: 21}]->(b) """.stripMargin) - val result = g.cypher( - """UNWIND [ + val result = g + .cypher("""UNWIND [ | {val1: "foo", val2: 42}, | {val1: "bar", val2: 21} |] as map |RETURN properties(map) as props - """.stripMargin).records + """.stripMargin) + .records - result.toMaps should equal(Bag( - CypherMap("props" -> CypherMap("val1" -> "foo", "val2" -> 42)), - CypherMap("props" -> CypherMap("val1" -> "bar", "val2" -> 21)) - )) + result.toMaps should equal( + Bag( + CypherMap("props" -> CypherMap("val1" -> "foo", "val2" -> 42)), + CypherMap("props" -> CypherMap("val1" -> "bar", "val2" -> 21)) + ) + ) } } @@ -1397,69 +1685,72 @@ class ExpressionTests extends MorpheusTestSuite with ScanGraphInit with Checkers describe("map support") { describe("map construction") { it("can construct static maps") { - val result = morpheus.cypher( - """ + val result = morpheus.cypher(""" |RETURN { | foo: "bar", | baz: 42 |} AS myMap """.stripMargin) - result.records.toMaps should equal(Bag( - CypherMap("myMap" -> Map("foo" -> "bar", "baz" -> 42)) - )) + result.records.toMaps should equal( + Bag( + CypherMap("myMap" -> Map("foo" -> "bar", "baz" -> 42)) + ) + ) } it("can construct Maps with expression values") { - val result = morpheus.cypher( - """ + val result = morpheus.cypher(""" |UNWIND [21, 42] as value |RETURN {foo: value} as myMap """.stripMargin) - result.records.toMaps should equal(Bag( - CypherMap("myMap" -> Map("foo" -> 21)), - CypherMap("myMap" -> Map("foo" -> 42)) - )) + result.records.toMaps should equal( + Bag( + CypherMap("myMap" -> Map("foo" -> 21)), + CypherMap("myMap" -> Map("foo" -> 42)) + ) + ) } it("can construct nodes with map properties") { - val g = initGraph( - """ + val g = initGraph(""" |CREATE (:A {val: "foo"}) """.stripMargin) - val result = g.cypher( - """ + val result = g + .cypher(""" |MATCH (a:A) |CONSTRUCT | CREATE (b {map: {val: a.val}}) |MATCH (n) |RETURN n.map as map - """.stripMargin).records + """.stripMargin) + .records - result.toMaps should equal(Bag( - CypherMap("map" -> CypherMap("val" -> "foo")) - )) + result.toMaps should equal( + Bag( + CypherMap("map" -> CypherMap("val" -> "foo")) + ) + ) } it("can return empty maps") { - val result = morpheus.cypher( - """ + val result = morpheus.cypher(""" |RETURN {} as myMap """.stripMargin) - result.records.toMaps should equal(Bag( - CypherMap("myMap" -> CypherMap()) - )) + result.records.toMaps should equal( + Bag( + CypherMap("myMap" -> CypherMap()) + ) + ) } } - describe("index access") { it("returns the element with literal key") { - val result = morpheus.cypher( - """ + val result = morpheus.cypher(""" |WITH { | foo: "bar", | baz: 42 @@ -1467,14 +1758,15 @@ class ExpressionTests extends MorpheusTestSuite with ScanGraphInit with Checkers |RETURN myMap["foo"] as foo, myMap["baz"] as baz """.stripMargin) - result.records.toMaps should equal(Bag( - CypherMap("foo" -> "bar", "baz" -> 42) - )) + result.records.toMaps should equal( + Bag( + CypherMap("foo" -> "bar", "baz" -> 42) + ) + ) } it("returns null if the literal key does not exist") { - val result = morpheus.cypher( - """ + val result = morpheus.cypher(""" |WITH { | foo: "bar", | baz: 42 @@ -1482,9 +1774,11 @@ class ExpressionTests extends MorpheusTestSuite with ScanGraphInit with Checkers |RETURN myMap["barbaz"] as barbaz """.stripMargin) - result.records.toMaps should equal(Bag( - CypherMap("barbaz" -> null) - )) + result.records.toMaps should equal( + Bag( + CypherMap("barbaz" -> null) + ) + ) } it("returns the element with parameter key") { @@ -1495,11 +1789,15 @@ class ExpressionTests extends MorpheusTestSuite with ScanGraphInit with Checkers | baz: 42 |} as myMap |RETURN myMap[$fooKey] as foo, myMap[$bazKey] as baz - """.stripMargin, CypherMap("fooKey" -> "foo", "bazKey" -> "baz")) - - result.records.toMaps should equal(Bag( - CypherMap("foo" -> "bar", "baz" -> 42) - )) + """.stripMargin, + CypherMap("fooKey" -> "foo", "bazKey" -> "baz") + ) + + result.records.toMaps should equal( + Bag( + CypherMap("foo" -> "bar", "baz" -> 42) + ) + ) } // TODO: This throws a spark analysis error as it cannot find the column @@ -1511,17 +1809,20 @@ class ExpressionTests extends MorpheusTestSuite with ScanGraphInit with Checkers | baz: 42 |} as myMap |RETURN myMap[$barbazKey] as barbaz - """.stripMargin, CypherMap("barbazKey" -> "barbaz")) - - result.records.toMaps should equal(Bag( - CypherMap("barbaz" -> null) - )) + """.stripMargin, + CypherMap("barbazKey" -> "barbaz") + ) + + result.records.toMaps should equal( + Bag( + CypherMap("barbaz" -> null) + ) + ) } // TODO: needs planning outside of SparkSQLExpressionMapper ignore("supports expression keys if all values have compatible types") { - val result = morpheus.cypher( - """ + val result = morpheus.cypher(""" |WITH { | foo: 1, | bar: 2 @@ -1530,18 +1831,19 @@ class ExpressionTests extends MorpheusTestSuite with ScanGraphInit with Checkers |RETURN myMap[key] as value """.stripMargin) - result.records.toMaps should equal(Bag( - CypherMap("value" -> 1), - CypherMap("value" -> 2) - )) + result.records.toMaps should equal( + Bag( + CypherMap("value" -> 1), + CypherMap("value" -> 2) + ) + ) } } } describe("list comprehension") { it("supports list comprehension with static mapping") { - val result = morpheus.cypher( - """ + val result = morpheus.cypher(""" |WITH [1, 2, 3] AS things |RETURN [n IN things | 1] AS value """.stripMargin) @@ -1552,8 +1854,7 @@ class ExpressionTests extends MorpheusTestSuite with ScanGraphInit with Checkers } it("supports list comprehension with simple mapping") { - val result = morpheus.cypher( - """ + val result = morpheus.cypher(""" |WITH [1, 2, 3] AS things |RETURN [n IN things | n*3] AS value """.stripMargin) @@ -1564,8 +1865,7 @@ class ExpressionTests extends MorpheusTestSuite with ScanGraphInit with Checkers } it("supports list comprehension with more complex mapping") { - val result = morpheus.cypher( - """ + val result = morpheus.cypher(""" |WITH ['1', '2', '3'] AS things |RETURN [n IN things | toInteger(n)*3 + toInteger(n)] AS value """.stripMargin) @@ -1576,8 +1876,7 @@ class ExpressionTests extends MorpheusTestSuite with ScanGraphInit with Checkers } it("supports list comprehension with inner predicate") { - val result = morpheus.cypher( - """ + val result = morpheus.cypher(""" |WITH [1, 2, 3] AS things |RETURN [n IN things WHERE n > 2] AS value """.stripMargin) @@ -1587,9 +1886,10 @@ class ExpressionTests extends MorpheusTestSuite with ScanGraphInit with Checkers ) } - it("supports list comprehension with inner predicate and more complex mapping") { - val result = morpheus.cypher( - """ + it( + "supports list comprehension with inner predicate and more complex mapping" + ) { + val result = morpheus.cypher(""" |WITH ['1', '2', '3'] AS things |RETURN [n IN things WHERE toInteger(n) > 2 | toInteger(n)*3 + toInteger(n)] AS value """.stripMargin) @@ -1600,8 +1900,7 @@ class ExpressionTests extends MorpheusTestSuite with ScanGraphInit with Checkers } it("supports nested list comprehensions") { - val result = morpheus.cypher( - """ + val result = morpheus.cypher(""" |WITH [[1,2,3], [2,2,3], [3,4]] AS things |RETURN [n IN things | [n IN n WHERE n < 2]] AS value """.stripMargin) @@ -1614,8 +1913,7 @@ class ExpressionTests extends MorpheusTestSuite with ScanGraphInit with Checkers describe("list reduce") { it("simple reduction") { - val result = morpheus.cypher( - """ + val result = morpheus.cypher(""" |WITH [1,2,3] AS things |RETURN reduce(acc = 0, n in things | acc + n) AS value """.stripMargin) @@ -1627,8 +1925,7 @@ class ExpressionTests extends MorpheusTestSuite with ScanGraphInit with Checkers describe("iterable predicates") { it("any") { - val result = morpheus.cypher( - """ + val result = morpheus.cypher(""" |WITH [1,2,3] AS things |RETURN any(n in things WHERE n < 2) AS value """.stripMargin) @@ -1638,8 +1935,7 @@ class ExpressionTests extends MorpheusTestSuite with ScanGraphInit with Checkers } it("none") { - val result = morpheus.cypher( - """ + val result = morpheus.cypher(""" |WITH [1,2,3] AS things |RETURN none(n in things WHERE n < 2) AS value """.stripMargin) @@ -1649,8 +1945,7 @@ class ExpressionTests extends MorpheusTestSuite with ScanGraphInit with Checkers } it("all") { - val result = morpheus.cypher( - """ + val result = morpheus.cypher(""" |WITH [1,2,3] AS things |RETURN all(n in things WHERE n < 4) AS value """.stripMargin) @@ -1660,8 +1955,7 @@ class ExpressionTests extends MorpheusTestSuite with ScanGraphInit with Checkers } it("negative single") { - val result = morpheus.cypher( - """ + val result = morpheus.cypher(""" |WITH [1,2,3] AS things |RETURN single(n in things WHERE n < 3) AS value """.stripMargin) @@ -1671,8 +1965,7 @@ class ExpressionTests extends MorpheusTestSuite with ScanGraphInit with Checkers } it("positive single") { - val result = morpheus.cypher( - """ + val result = morpheus.cypher(""" |WITH [1,2,3] AS things |RETURN single(n in things WHERE n < 2) AS value """.stripMargin) @@ -1684,9 +1977,9 @@ class ExpressionTests extends MorpheusTestSuite with ScanGraphInit with Checkers describe("map projection") { it("simple map-projection") { - val graph = ScanGraphFactory.initGraph("""CREATE ({prop: 1, name: "Morpheus"})""") - val result = graph.cypher( - """ + val graph = + ScanGraphFactory.initGraph("""CREATE ({prop: 1, name: "Morpheus"})""") + val result = graph.cypher(""" |MATCH (n) |RETURN n {.prop} AS map """.stripMargin) @@ -1696,9 +1989,9 @@ class ExpressionTests extends MorpheusTestSuite with ScanGraphInit with Checkers } it("map-projection with additional properties") { - val graph = ScanGraphFactory.initGraph("""CREATE ({prop: 1, name: "Morpheus"})""") - val result = graph.cypher( - """ + val graph = + ScanGraphFactory.initGraph("""CREATE ({prop: 1, name: "Morpheus"})""") + val result = graph.cypher(""" |MATCH (n) |RETURN n {.prop, age: 21} AS map """.stripMargin) @@ -1708,21 +2001,23 @@ class ExpressionTests extends MorpheusTestSuite with ScanGraphInit with Checkers } it("map-projection copying all properties") { - val graph = ScanGraphFactory.initGraph("""CREATE (:Person{prop: [1,2], name: "Morpheus"})""") - val result = graph.cypher( - """ + val graph = ScanGraphFactory.initGraph( + """CREATE (:Person{prop: [1,2], name: "Morpheus"})""" + ) + val result = graph.cypher(""" |MATCH (n:Person) |RETURN n {.*} AS map """.stripMargin) result.records.toMaps shouldEqual Bag( - CypherMap("map" -> Map("prop" -> List(1,2), "name" -> "Morpheus")) + CypherMap("map" -> Map("prop" -> List(1, 2), "name" -> "Morpheus")) ) } it("map-projection copying all properties and overwriting properties") { - val graph = ScanGraphFactory.initGraph("""CREATE (:Person{age: 1, name: "Morpheus"})""") - val result = graph.cypher( - """ + val graph = ScanGraphFactory.initGraph( + """CREATE (:Person{age: 1, name: "Morpheus"})""" + ) + val result = graph.cypher(""" |MATCH (n:Person) |RETURN n {.*, age: n.age+1} AS map """.stripMargin) @@ -1732,8 +2027,7 @@ class ExpressionTests extends MorpheusTestSuite with ScanGraphInit with Checkers } it("map-projection based on a map") { - val result = morpheus.cypher( - """ + val result = morpheus.cypher(""" |WITH {age: 1, name: "Morpheus"} as n |RETURN n {.*, age: n.age+1} AS map """.stripMargin) @@ -1743,4 +2037,3 @@ class ExpressionTests extends MorpheusTestSuite with ScanGraphInit with Checkers } } } - diff --git a/morpheus-testing/src/test/scala/org/opencypher/morpheus/impl/acceptance/FunctionTests.scala b/morpheus-testing/src/test/scala/org/opencypher/morpheus/impl/acceptance/FunctionTests.scala index 038cfe1960..4f4a9c7f57 100644 --- a/morpheus-testing/src/test/scala/org/opencypher/morpheus/impl/acceptance/FunctionTests.scala +++ b/morpheus-testing/src/test/scala/org/opencypher/morpheus/impl/acceptance/FunctionTests.scala @@ -438,7 +438,8 @@ class FunctionTests extends MorpheusTestSuite with ScanGraphInit { ) } it("multiple characters") { - val result = morpheus.cypher("RETURN replace('hello', 'ell', 'ipp') AS res") + val result = + morpheus.cypher("RETURN replace('hello', 'ell', 'ipp') AS res") result.records.toMaps should equal( Bag( CypherMap("res" -> "hippo") @@ -458,7 +459,7 @@ class FunctionTests extends MorpheusTestSuite with ScanGraphInit { result.records.toMaps should equal( Bag( CypherMap("res" -> null) - ) + ) ) } it("on null to-be-replaced") { @@ -478,7 +479,9 @@ class FunctionTests extends MorpheusTestSuite with ScanGraphInit { ) } it("on complex string expression") { - val result = morpheus.cypher("RETURN replace('he' + 'llo', 'l' + 'l', 'w' + 'w') AS res") + val result = morpheus.cypher( + "RETURN replace('he' + 'llo', 'l' + 'l', 'w' + 'w') AS res" + ) result.records.toMaps should equal( Bag( CypherMap("res" -> "hewwo") @@ -486,7 +489,9 @@ class FunctionTests extends MorpheusTestSuite with ScanGraphInit { ) } it("on complex expression evaluating to null") { - val result = morpheus.cypher("WITH ['ll', 'ww'] AS stringList RETURN replace('hello', stringList[0], stringList[2]) AS res") + val result = morpheus.cypher( + "WITH ['ll', 'ww'] AS stringList RETURN replace('hello', stringList[0], stringList[2]) AS res" + ) result.records.toMaps should equal( Bag( CypherMap("res" -> null) @@ -549,8 +554,7 @@ class FunctionTests extends MorpheusTestSuite with ScanGraphInit { it("trims more complex structures") { val given = initGraph("CREATE ({name: ' foo '})") - val result = given.cypher( - """ + val result = given.cypher(""" |MATCH (n) |WITH rtrim(n.name) AS name |RETURN rtrim(ltrim(name + '_bar ')) AS trimmed @@ -570,14 +574,20 @@ class FunctionTests extends MorpheusTestSuite with ScanGraphInit { val t1 = morpheus.cypher("RETURN timestamp()") val t2 = morpheus.cypher("RETURN timestamp()") - t1.records.toMaps.keys.map(_.value.head._2.value.asInstanceOf[Long]) should be <= + t1.records.toMaps.keys + .map(_.value.head._2.value.asInstanceOf[Long]) should be <= t2.records.toMaps.keys.map(_.value.head._2.value.asInstanceOf[Long]) } - it("should return the same value when called multiple times inside the same query") { + it( + "should return the same value when called multiple times inside the same query" + ) { val given = initGraph("CREATE (), ()") - val result = given.cypher("WITH timestamp() AS t1 MATCH (n) RETURN t1, timestamp() AS t2").records.toMaps + val result = given + .cypher("WITH timestamp() AS t1 MATCH (n) RETURN t1, timestamp() AS t2") + .records + .toMaps val expected = result.head._1("t1") @@ -604,7 +614,8 @@ class FunctionTests extends MorpheusTestSuite with ScanGraphInit { CypherMap("exists" -> true), CypherMap("exists" -> false), CypherMap("exists" -> false) - )) + ) + ) } it("exists() on null property") { @@ -615,7 +626,8 @@ class FunctionTests extends MorpheusTestSuite with ScanGraphInit { result.records.toMaps should equal( Bag( CypherMap("exists" -> false) - )) + ) + ) } it("exists() on null map") { @@ -626,7 +638,8 @@ class FunctionTests extends MorpheusTestSuite with ScanGraphInit { result.records.toMaps should equal( Bag( CypherMap("exists" -> false) - )) + ) + ) } } @@ -642,7 +655,8 @@ class FunctionTests extends MorpheusTestSuite with ScanGraphInit { CypherMap("type(r)" -> "KNOWS"), CypherMap("type(r)" -> "HATES"), CypherMap("type(r)" -> "REL") - )) + ) + ) } } @@ -653,7 +667,12 @@ class FunctionTests extends MorpheusTestSuite with ScanGraphInit { val result = given.cypher("MATCH (n) RETURN id(n)") - result.records.toMaps should equal(Bag(CypherMap("id(n)" -> 0L.encodeAsMorpheusId), CypherMap("id(n)" -> 1L.encodeAsMorpheusId))) + result.records.toMaps should equal( + Bag( + CypherMap("id(n)" -> 0L.encodeAsMorpheusId), + CypherMap("id(n)" -> 1L.encodeAsMorpheusId) + ) + ) } it("id for rel") { @@ -661,7 +680,12 @@ class FunctionTests extends MorpheusTestSuite with ScanGraphInit { val result = given.cypher("MATCH ()-[e]->() RETURN id(e)") - result.records.toMaps should equal(Bag(CypherMap("id(e)" -> 2L.encodeAsMorpheusId), CypherMap("id(e)" -> 4L.encodeAsMorpheusId))) + result.records.toMaps should equal( + Bag( + CypherMap("id(e)" -> 2L.encodeAsMorpheusId), + CypherMap("id(e)" -> 4L.encodeAsMorpheusId) + ) + ) } } @@ -677,7 +701,8 @@ class FunctionTests extends MorpheusTestSuite with ScanGraphInit { Bag( CypherMap("labels(a)" -> List("A")), CypherMap("labels(a)" -> List("B")) - )) + ) + ) } it("get multiple labels") { @@ -689,7 +714,8 @@ class FunctionTests extends MorpheusTestSuite with ScanGraphInit { Bag( CypherMap("labels(a)" -> List("A", "B")), CypherMap("labels(a)" -> List("C", "D")) - )) + ) + ) } it("unlabeled nodes") { @@ -702,7 +728,8 @@ class FunctionTests extends MorpheusTestSuite with ScanGraphInit { CypherMap("labels(a)" -> List("A")), CypherMap("labels(a)" -> List("C", "D")), CypherMap("labels(a)" -> List.empty) - )) + ) + ) } it("handle null") { @@ -727,7 +754,8 @@ class FunctionTests extends MorpheusTestSuite with ScanGraphInit { result.records.toMaps should equal( Bag( CypherMap("s" -> 2) - )) + ) + ) } it("size() on literal string") { @@ -738,7 +766,8 @@ class FunctionTests extends MorpheusTestSuite with ScanGraphInit { result.records.toMaps should equal( Bag( CypherMap("s" -> 5) - )) + ) + ) } it("size() on retrieved string") { @@ -749,7 +778,8 @@ class FunctionTests extends MorpheusTestSuite with ScanGraphInit { result.records.toMaps should equal( Bag( CypherMap("s" -> 5) - )) + ) + ) } it("size() on constructed list") { @@ -763,7 +793,8 @@ class FunctionTests extends MorpheusTestSuite with ScanGraphInit { CypherMap("s" -> 2), CypherMap("s" -> 1), CypherMap("s" -> 0) - )) + ) + ) } it("size() on null") { @@ -794,34 +825,37 @@ class FunctionTests extends MorpheusTestSuite with ScanGraphInit { it("keys()") { val given = initGraph("CREATE ({name:'Alice', age: 64, eyes:'brown'})") - val result = given.cypher("MATCH (a) WHERE a.name = 'Alice' RETURN keys(a) AS k") + val result = + given.cypher("MATCH (a) WHERE a.name = 'Alice' RETURN keys(a) AS k") val keysAsMap = result.records.toMaps keysAsMap should equal( Bag( CypherMap("k" -> List("age", "eyes", "name")) - )) + ) + ) } it("keys() does not return keys of unset properties") { - val given = initGraph( - """ + val given = initGraph(""" |CREATE (:Person {name:'Alice', age: 64, eyes:'brown'}) |CREATE (:Person {name:'Bob', eyes:'blue'}) """.stripMargin) - val result = given.cypher("MATCH (a: Person) WHERE a.name = 'Bob' RETURN keys(a) AS k") + val result = given.cypher( + "MATCH (a: Person) WHERE a.name = 'Bob' RETURN keys(a) AS k" + ) result.records.toMaps should equal( Bag( CypherMap("k" -> List("eyes", "name")) - )) + ) + ) } it("works with literal maps") { - val result = morpheus.cypher( - """ + val result = morpheus.cypher(""" |WITH {person: {name: 'Anne', age: 25}} AS p |RETURN keys(p) AS k1, keys(p["person"]) AS k2 """.stripMargin) @@ -829,38 +863,46 @@ class FunctionTests extends MorpheusTestSuite with ScanGraphInit { result.records.toMaps should equal( Bag( CypherMap("k1" -> List("person"), "k2" -> List("name", "age")) - )) + ) + ) } it("works with literal maps2") { - val result = morpheus.cypher( - """ + val result = morpheus.cypher(""" |RETURN keys({name: 'Alice', age: 38, address: {city: 'London', residential: true}}) AS k """.stripMargin) result.records.toMaps should equal( Bag( CypherMap("k" -> List("name", "age", "address")) - )) + ) + ) } it("works with predicate maps") { val result = morpheus.cypher( """ |RETURN keys($map) AS k - """.stripMargin, CypherMap("map" -> CypherMap("name" -> "Alice", "age" -> 38, "address" -> CypherMap("city" -> "London", "residential" -> true)))) + """.stripMargin, + CypherMap( + "map" -> CypherMap( + "name" -> "Alice", + "age" -> 38, + "address" -> CypherMap("city" -> "London", "residential" -> true) + ) + ) + ) result.records.toMaps should equal( Bag( CypherMap("k" -> List("name", "age", "address")) - )) + ) + ) } - it("works with null keys in maps") { - val result = morpheus.cypher( - """ + val result = morpheus.cypher(""" |UNWIND [ | 1, | null @@ -875,37 +917,71 @@ class FunctionTests extends MorpheusTestSuite with ScanGraphInit { Bag( CypherMap("k" -> List("key")), CypherMap("k" -> List()) - )) + ) + ) } } describe("startNode") { it("startNode()") { - val given = initGraph("CREATE ()-[:FOO {val: 'a'}]->(),()-[:FOO {val: 'b'}]->()") - - val result = given.cypher("MATCH ()-[r:FOO]->() RETURN r.val, startNode(r)") - - result.records.toMaps should equal( - Bag( - CypherMap("r.val" -> "a", "startNode(r)" -> MorpheusNode(0L.encodeAsMorpheusId.toSeq, Set.empty[String], CypherMap())), - CypherMap("r.val" -> "b", "startNode(r)" -> MorpheusNode(3L.encodeAsMorpheusId.toSeq, Set.empty[String], CypherMap())) - )) + val given = + initGraph("CREATE ()-[:FOO {val: 'a'}]->(),()-[:FOO {val: 'b'}]->()") + + val result = + given.cypher("MATCH ()-[r:FOO]->() RETURN r.val, startNode(r)") + + result.records.toMaps should equal( + Bag( + CypherMap( + "r.val" -> "a", + "startNode(r)" -> MorpheusNode( + 0L.encodeAsMorpheusId.toSeq, + Set.empty[String], + CypherMap() + ) + ), + CypherMap( + "r.val" -> "b", + "startNode(r)" -> MorpheusNode( + 3L.encodeAsMorpheusId.toSeq, + Set.empty[String], + CypherMap() + ) + ) + ) + ) } } describe("endNode") { it("endNode()") { - val given = initGraph("CREATE ()-[:FOO {val: 'a'}]->(),()-[:FOO {val: 'b'}]->()") + val given = + initGraph("CREATE ()-[:FOO {val: 'a'}]->(),()-[:FOO {val: 'b'}]->()") val result = given.cypher("MATCH (a)-[r]->() RETURN r.val, endNode(r)") result.records.toMaps should equal( Bag( - CypherMap("r.val" -> "a", "endNode(r)" -> MorpheusNode(1L.encodeAsMorpheusId.toSeq, Set.empty[String], CypherMap())), - CypherMap("r.val" -> "b", "endNode(r)" -> MorpheusNode(4L.encodeAsMorpheusId.toSeq, Set.empty[String], CypherMap())) - )) + CypherMap( + "r.val" -> "a", + "endNode(r)" -> MorpheusNode( + 1L.encodeAsMorpheusId.toSeq, + Set.empty[String], + CypherMap() + ) + ), + CypherMap( + "r.val" -> "b", + "endNode(r)" -> MorpheusNode( + 4L.encodeAsMorpheusId.toSeq, + Set.empty[String], + CypherMap() + ) + ) + ) + ) } } @@ -919,7 +995,8 @@ class FunctionTests extends MorpheusTestSuite with ScanGraphInit { result.records.toMaps should equal( Bag( CypherMap("myFloat" -> 1.0) - )) + ) + ) } it("toFloat from float") { @@ -930,7 +1007,8 @@ class FunctionTests extends MorpheusTestSuite with ScanGraphInit { result.records.toMaps should equal( Bag( CypherMap("myFloat" -> 1.0) - )) + ) + ) } it("toFloat from string") { @@ -941,7 +1019,8 @@ class FunctionTests extends MorpheusTestSuite with ScanGraphInit { result.records.toMaps should equal( Bag( CypherMap("myFloat" -> 42.0) - )) + ) + ) } } @@ -1071,7 +1150,9 @@ class FunctionTests extends MorpheusTestSuite with ScanGraphInit { it("can evaluate coalesce") { val given = initGraph("CREATE ({valA: 1}), ({valB: 2}), ({valC: 3}), ()") - val result = given.cypher("MATCH (n) RETURN coalesce(n.valA, n.valB, n.valC) AS value") + val result = given.cypher( + "MATCH (n) RETURN coalesce(n.valA, n.valB, n.valC) AS value" + ) result.records.collect.toBag should equal( Bag( @@ -1079,20 +1160,23 @@ class FunctionTests extends MorpheusTestSuite with ScanGraphInit { CypherMap("value" -> 2), CypherMap("value" -> 3), CypherMap("value" -> null) - )) + ) + ) } it("can evaluate coalesce on non-existing expressions") { val given = initGraph("CREATE ({valA: 1}), ({valB: 2}), ()") - val result = given.cypher("MATCH (n) RETURN coalesce(n.valD, n.valE) AS value") + val result = + given.cypher("MATCH (n) RETURN coalesce(n.valD, n.valE) AS value") result.records.collect.toBag should equal( Bag( CypherMap("value" -> null), CypherMap("value" -> null), CypherMap("value" -> null) - )) + ) + ) } } @@ -1101,8 +1185,7 @@ class FunctionTests extends MorpheusTestSuite with ScanGraphInit { it("toInteger() on a graph") { val given = initGraph("CREATE (:Person {age: '42'})") - val result = given.cypher( - """ + val result = given.cypher(""" |MATCH (n) |RETURN toInteger(n.age) AS age """.stripMargin) @@ -1117,7 +1200,8 @@ class FunctionTests extends MorpheusTestSuite with ScanGraphInit { it("toInteger() on float") { val given = initGraph("CREATE (:Person {weight: '82.9'})") - val result = given.cypher("MATCH (n) RETURN toInteger(n.weight) AS nWeight") + val result = + given.cypher("MATCH (n) RETURN toInteger(n.weight) AS nWeight") result.records.toMaps should equal( Bag( @@ -1501,57 +1585,69 @@ class FunctionTests extends MorpheusTestSuite with ScanGraphInit { describe("range") { it("can compute a range from literals") { - morpheus.cypher( - """UNWIND range(1, 3) AS x - |RETURN x""".stripMargin).records.toMaps should equal(Bag( - CypherMap("x" -> 1), - CypherMap("x" -> 2), - CypherMap("x" -> 3) - )) + morpheus + .cypher("""UNWIND range(1, 3) AS x + |RETURN x""".stripMargin) + .records + .toMaps should equal( + Bag( + CypherMap("x" -> 1), + CypherMap("x" -> 2), + CypherMap("x" -> 3) + ) + ) } it("can compute a range from literals with custom steps") { - morpheus.cypher( - """UNWIND range(1, 7, 3) AS x - |RETURN x""".stripMargin).records.toMaps should equal(Bag( - CypherMap("x" -> 1), - CypherMap("x" -> 4), - CypherMap("x" -> 7) - )) + morpheus + .cypher("""UNWIND range(1, 7, 3) AS x + |RETURN x""".stripMargin) + .records + .toMaps should equal( + Bag( + CypherMap("x" -> 1), + CypherMap("x" -> 4), + CypherMap("x" -> 7) + ) + ) } it("can compute a range from column values") { - val g = initGraph( - """ + val g = initGraph(""" |CREATE (:A {from: 1, to: 2}) |CREATE (:A {from: 1, to: 3}) |CREATE (:A {from: 1, to: 4}) """.stripMargin) - g.cypher( - """ + g.cypher(""" |MATCH (n) - |RETURN range(n.from, n.to) AS x""".stripMargin).records.toMaps should equal(Bag( - CypherMap("x" -> List(1, 2)), - CypherMap("x" -> List(1, 2, 3)), - CypherMap("x" -> List(1, 2, 3, 4)) - )) + |RETURN range(n.from, n.to) AS x""".stripMargin) + .records + .toMaps should equal( + Bag( + CypherMap("x" -> List(1, 2)), + CypherMap("x" -> List(1, 2, 3)), + CypherMap("x" -> List(1, 2, 3, 4)) + ) + ) } it("can compute a range with varying step values") { - val g = initGraph( - """ + val g = initGraph(""" |CREATE (:A {step: 2}) |CREATE (:A {step: 3}) """.stripMargin) - g.cypher( - """ + g.cypher(""" |MATCH (n) - |RETURN range(1, 4, n.step) AS x""".stripMargin).records.toMaps should equal(Bag( - CypherMap("x" -> List(1, 3)), - CypherMap("x" -> List(1, 4)) - )) + |RETURN range(1, 4, n.step) AS x""".stripMargin) + .records + .toMaps should equal( + Bag( + CypherMap("x" -> List(1, 3)), + CypherMap("x" -> List(1, 4)) + ) + ) } } @@ -1562,9 +1658,11 @@ class FunctionTests extends MorpheusTestSuite with ScanGraphInit { val result = g.cypher("RETURN substring('foobar', 3) AS substring") - result.records.toMaps should equal(Bag( - CypherMap("substring" -> "bar") - )) + result.records.toMaps should equal( + Bag( + CypherMap("substring" -> "bar") + ) + ) } it("returns substring from literal with given length") { @@ -1572,9 +1670,11 @@ class FunctionTests extends MorpheusTestSuite with ScanGraphInit { val result = g.cypher("RETURN substring('foobar', 0, 3) AS substring") - result.records.toMaps should equal(Bag( - CypherMap("substring" -> "foo") - )) + result.records.toMaps should equal( + Bag( + CypherMap("substring" -> "foo") + ) + ) } it("returns substring from literal with exceeding given length") { @@ -1582,9 +1682,11 @@ class FunctionTests extends MorpheusTestSuite with ScanGraphInit { val result = g.cypher("RETURN substring('foobar', 3, 10) AS substring") - result.records.toMaps should equal(Bag( - CypherMap("substring" -> "bar") - )) + result.records.toMaps should equal( + Bag( + CypherMap("substring" -> "bar") + ) + ) } it("returns empty string for length 0") { @@ -1592,9 +1694,11 @@ class FunctionTests extends MorpheusTestSuite with ScanGraphInit { val result = g.cypher("RETURN substring('foobar', 0, 0) AS substring") - result.records.toMaps should equal(Bag( - CypherMap("substring" -> "") - )) + result.records.toMaps should equal( + Bag( + CypherMap("substring" -> "") + ) + ) } it("returns empty string for exceeding start") { @@ -1602,9 +1706,11 @@ class FunctionTests extends MorpheusTestSuite with ScanGraphInit { val result = g.cypher("RETURN substring('foobar', 10) AS substring") - result.records.toMaps should equal(Bag( - CypherMap("substring" -> "") - )) + result.records.toMaps should equal( + Bag( + CypherMap("substring" -> "") + ) + ) } it("returns null for null") { @@ -1612,9 +1718,11 @@ class FunctionTests extends MorpheusTestSuite with ScanGraphInit { val result = g.cypher("RETURN substring(null, 0, 0) AS substring") - result.records.toMaps should equal(Bag( - CypherMap("substring" -> CypherNull) - )) + result.records.toMaps should equal( + Bag( + CypherMap("substring" -> CypherNull) + ) + ) } it("throws for negative length") { @@ -1622,126 +1730,140 @@ class FunctionTests extends MorpheusTestSuite with ScanGraphInit { val result = g.cypher("RETURN substring(null, 0, 0) AS substring") - result.records.toMaps should equal(Bag( - CypherMap("substring" -> CypherNull) - )) + result.records.toMaps should equal( + Bag( + CypherMap("substring" -> CypherNull) + ) + ) } } describe("list-access") { it("head") { - val result = morpheus.cypher( - """ + val result = morpheus.cypher(""" |WITH [1, 2, 3] AS things |Return head(things) as head """.stripMargin) - result.records.toMaps should equal(Bag( - CypherMap("head" -> 1) - )) + result.records.toMaps should equal( + Bag( + CypherMap("head" -> 1) + ) + ) } it("head on empty list") { - val result = morpheus.cypher( - """ + val result = morpheus.cypher(""" |WITH [] AS things |Return head(things) as head """.stripMargin) - result.records.toMaps should equal(Bag( - CypherMap("head" -> null) - )) + result.records.toMaps should equal( + Bag( + CypherMap("head" -> null) + ) + ) } it("tail") { - val result = morpheus.cypher( - """ + val result = morpheus.cypher(""" |WITH [1, 2, 3] AS things |Return tail(things) as tail """.stripMargin) - result.records.toMaps should equal(Bag( - CypherMap("tail" -> List(2, 3)) - )) + result.records.toMaps should equal( + Bag( + CypherMap("tail" -> List(2, 3)) + ) + ) } it("tail on empty list") { - val result = morpheus.cypher( - """ + val result = morpheus.cypher(""" |WITH [] AS things |Return tail(things) as tail """.stripMargin) - result.records.toMaps should equal(Bag( - CypherMap("tail" -> null) - )) + result.records.toMaps should equal( + Bag( + CypherMap("tail" -> null) + ) + ) } it("last") { - val result = morpheus.cypher( - """ + val result = morpheus.cypher(""" |WITH [1, 2, 3] AS things |Return last(things) as last """.stripMargin) - result.records.toMaps should equal(Bag( - CypherMap("last" -> 3) - )) + result.records.toMaps should equal( + Bag( + CypherMap("last" -> 3) + ) + ) } it("last on empty list") { - val result = morpheus.cypher( - """ + val result = morpheus.cypher(""" |WITH [] AS things |Return last(things) as last """.stripMargin) - result.records.toMaps should equal(Bag( - CypherMap("last" -> null) - )) + result.records.toMaps should equal( + Bag( + CypherMap("last" -> null) + ) + ) } } describe("reverse") { it("reverse on string") { - val result = morpheus.cypher( - """ + val result = morpheus.cypher(""" |RETURN reverse("anagram") as rev """.stripMargin) - result.records.toMaps should equal(Bag( - CypherMap("rev" -> "margana") - )) + result.records.toMaps should equal( + Bag( + CypherMap("rev" -> "margana") + ) + ) } it("reverse on lists") { - val result = morpheus.cypher( - """ + val result = morpheus.cypher(""" |RETURN reverse([1, 2, 3]) as rev """.stripMargin) - result.records.toMaps should equal(Bag( - CypherMap("rev" -> List(3, 2, 1)) - )) + result.records.toMaps should equal( + Bag( + CypherMap("rev" -> List(3, 2, 1)) + ) + ) } } describe("split") { it("split with constant delimiter") { - val result = morpheus.cypher( - """ + val result = morpheus.cypher(""" |RETURN split("1,2,3",",2,") as split """.stripMargin) - result.records.toMaps should equal(Bag( - CypherMap("split" -> List("1", "3")) - )) + result.records.toMaps should equal( + Bag( + CypherMap("split" -> List("1", "3")) + ) + ) } it("split with variable delimiter") { - val graph = ScanGraphFactory.initGraph( - """ + val graph = ScanGraphFactory.initGraph(""" |CREATE ({friends: 'Bob,Eve', delimiter:","}), | ({friends: 'Eve;Bob', delimiter:";"}) """.stripMargin) - val result = graph.cypher("""MATCH (n) RETURN split(n.friends, n.delimiter) as split""") - result.records.toMaps should equal(Bag( - CypherMap("split" -> List("Eve", "Bob")), - CypherMap("split" -> List("Bob", "Eve")) - )) + val result = graph.cypher( + """MATCH (n) RETURN split(n.friends, n.delimiter) as split""" + ) + result.records.toMaps should equal( + Bag( + CypherMap("split" -> List("Eve", "Bob")), + CypherMap("split" -> List("Bob", "Eve")) + ) + ) } } diff --git a/morpheus-testing/src/test/scala/org/opencypher/morpheus/impl/acceptance/GraphInit.scala b/morpheus-testing/src/test/scala/org/opencypher/morpheus/impl/acceptance/GraphInit.scala index 275d41978e..691636d624 100644 --- a/morpheus-testing/src/test/scala/org/opencypher/morpheus/impl/acceptance/GraphInit.scala +++ b/morpheus-testing/src/test/scala/org/opencypher/morpheus/impl/acceptance/GraphInit.scala @@ -33,13 +33,19 @@ import org.opencypher.okapi.api.graph.Pattern import org.opencypher.okapi.relational.api.graph.RelationalCypherGraph trait GraphInit { - def initGraph(createQuery: String, additionalPatterns: Seq[Pattern] = Seq.empty) - (implicit morpheus: MorpheusSession): RelationalCypherGraph[DataFrameTable] + def initGraph( + createQuery: String, + additionalPatterns: Seq[Pattern] = Seq.empty + )(implicit morpheus: MorpheusSession): RelationalCypherGraph[DataFrameTable] } trait ScanGraphInit extends GraphInit { - def initGraph(createQuery: String, additionalPatterns: Seq[Pattern] = Seq.empty) - (implicit morpheus: MorpheusSession): RelationalCypherGraph[DataFrameTable] = { + def initGraph( + createQuery: String, + additionalPatterns: Seq[Pattern] = Seq.empty + )(implicit + morpheus: MorpheusSession + ): RelationalCypherGraph[DataFrameTable] = { ScanGraphFactory.initGraph(createQuery, additionalPatterns) } } diff --git a/morpheus-testing/src/test/scala/org/opencypher/morpheus/impl/acceptance/MatchTests.scala b/morpheus-testing/src/test/scala/org/opencypher/morpheus/impl/acceptance/MatchTests.scala index 899105dbf6..8e4b87800a 100644 --- a/morpheus-testing/src/test/scala/org/opencypher/morpheus/impl/acceptance/MatchTests.scala +++ b/morpheus-testing/src/test/scala/org/opencypher/morpheus/impl/acceptance/MatchTests.scala @@ -75,30 +75,29 @@ class MatchTests extends MorpheusTestSuite with ScanGraphInit { it("matches a label") { // Given - val given = initGraph( - """ + val given = initGraph(""" |CREATE (p:Person {firstName: "Alice", lastName: "Foo"}) """.stripMargin) // When - val result = given.cypher( - """ + val result = given.cypher(""" |MATCH (a:Person) |RETURN a.firstName """.stripMargin) // Then - result.records.toMaps should equal(Bag(CypherMap("a.firstName" -> "Alice") - )) + result.records.toMaps should equal( + Bag(CypherMap("a.firstName" -> "Alice")) + ) } it("matches an unknown label") { // Given - val given = initGraph("CREATE (p:Person {firstName: 'Alice', lastName: 'Foo'})") + val given = + initGraph("CREATE (p:Person {firstName: 'Alice', lastName: 'Foo'})") // When - val result = given.cypher( - """ + val result = given.cypher(""" |MATCH (a:Animal) |RETURN a """.stripMargin) @@ -111,8 +110,7 @@ class MatchTests extends MorpheusTestSuite with ScanGraphInit { describe("multiple match clauses") { it("can handle multiple match clauses") { // Given - val given = initGraph( - """CREATE (p1:Person {name: "Alice"}) + val given = initGraph("""CREATE (p1:Person {name: "Alice"}) |CREATE (p2:Person {name: "Bob"}) |CREATE (p3:Person {name: "Eve"}) |CREATE (p1)-[:KNOWS]->(p2) @@ -120,8 +118,7 @@ class MatchTests extends MorpheusTestSuite with ScanGraphInit { """.stripMargin) // When - val result = given.cypher( - """MATCH (p1:Person) + val result = given.cypher("""MATCH (p1:Person) |MATCH (p1:Person)-[e1]->(p2:Person) |MATCH (p2)-[e2]->(p3:Person) |RETURN p1.name, p2.name, p3.name @@ -136,13 +133,13 @@ class MatchTests extends MorpheusTestSuite with ScanGraphInit { "p3.name" -> "Eve" ) - )) + ) + ) } it("cyphermorphism and multiple match clauses") { // Given - val given = initGraph( - """ + val given = initGraph(""" |CREATE (p1:Person {name: "Alice"}) |CREATE (p2:Person {name: "Bob"}) |CREATE (p1)-[:KNOWS]->(p2) @@ -150,8 +147,7 @@ class MatchTests extends MorpheusTestSuite with ScanGraphInit { """.stripMargin) // When - val result = given.cypher( - """ + val result = given.cypher(""" |MATCH (p1:Person)-[e1:KNOWS]->(p2:Person)-[e2:KNOWS]->(p3:Person) |MATCH (p3)-[e3:KNOWS]->(p4:Person) |RETURN p1.name, p2.name, p3.name, p4.name @@ -172,7 +168,8 @@ class MatchTests extends MorpheusTestSuite with ScanGraphInit { "p3.name" -> "Alice", "p4.name" -> "Bob" ) - )) + ) + ) } } @@ -180,8 +177,7 @@ class MatchTests extends MorpheusTestSuite with ScanGraphInit { it("disconnected components") { // Given - val given = initGraph( - """ + val given = initGraph(""" |CREATE (p1:Narcissist {name: "Alice"}) |CREATE (p2:Narcissist {name: "Bob"}) |CREATE (p1)-[:LOVES]->(p1) @@ -189,8 +185,7 @@ class MatchTests extends MorpheusTestSuite with ScanGraphInit { """.stripMargin) // When - val result = given.cypher( - """ + val result = given.cypher(""" |MATCH (a:Narcissist), (b:Narcissist) |RETURN a.name AS one, b.name AS two """.stripMargin) @@ -202,13 +197,13 @@ class MatchTests extends MorpheusTestSuite with ScanGraphInit { CypherMap("one" -> "Alice", "two" -> "Bob"), CypherMap("one" -> "Bob", "two" -> "Bob"), CypherMap("one" -> "Bob", "two" -> "Alice") - )) + ) + ) } it("joined components") { // Given - val given = initGraph( - """ + val given = initGraph(""" |CREATE (p1:Narcissist {name: "Alice"}) |CREATE (p2:Narcissist {name: "Bob"}) |CREATE (p1)-[:LOVES]->(p1) @@ -216,8 +211,7 @@ class MatchTests extends MorpheusTestSuite with ScanGraphInit { """.stripMargin) // When - val result = given.cypher( - """ + val result = given.cypher(""" |MATCH (a:Narcissist), (b:Narcissist) WHERE a.name = b.name |RETURN a.name AS one, b.name AS two """.stripMargin) @@ -227,14 +221,16 @@ class MatchTests extends MorpheusTestSuite with ScanGraphInit { Bag( CypherMap("one" -> "Alice", "two" -> "Alice"), CypherMap("one" -> "Bob", "two" -> "Bob") - )) + ) + ) // TODO: Move to plan based testing result.plans.logical should include("ValueJoin") } it("can evaluate cross Product between multiple match clauses") { - val graph = initGraph("CREATE (:A {val: 0}), (:B {val: 1})-[:REL]->(:C {val: 2})") + val graph = + initGraph("CREATE (:A {val: 0}), (:B {val: 1})-[:REL]->(:C {val: 2})") val query = """ |MATCH (a:A) @@ -242,9 +238,11 @@ class MatchTests extends MorpheusTestSuite with ScanGraphInit { |RETURN a.val, c.val """.stripMargin - graph.cypher(query).records.collect.toBag should equal(Bag( - CypherMap("a.val" -> 0, "c.val" -> 2) - )) + graph.cypher(query).records.collect.toBag should equal( + Bag( + CypherMap("a.val" -> 0, "c.val" -> 2) + ) + ) } } @@ -262,12 +260,15 @@ class MatchTests extends MorpheusTestSuite with ScanGraphInit { """.stripMargin ) - val result = given.cypher("MATCH (a:A)--(other) RETURN a.prop, other.prop") + val result = + given.cypher("MATCH (a:A)--(other) RETURN a.prop, other.prop") - result.records.collect.toBag should equal(Bag( - CypherMap("a.prop" -> "isA", "other.prop" -> "fromA"), - CypherMap("a.prop" -> "isA", "other.prop" -> "toA") - )) + result.records.collect.toBag should equal( + Bag( + CypherMap("a.prop" -> "isA", "other.prop" -> "fromA"), + CypherMap("a.prop" -> "isA", "other.prop" -> "toA") + ) + ) } it("matches an undirected relationship with two hops") { @@ -284,13 +285,16 @@ class MatchTests extends MorpheusTestSuite with ScanGraphInit { """.stripMargin ) - val result = given.cypher("MATCH (a:A)--()--(other) RETURN a.prop, other.prop") + val result = + given.cypher("MATCH (a:A)--()--(other) RETURN a.prop, other.prop") - result.records.collect.toBag should equal(Bag( - CypherMap("a.prop" -> "a", "other.prop" -> "c"), - CypherMap("a.prop" -> "a", "other.prop" -> "b"), - CypherMap("a.prop" -> "a", "other.prop" -> "d") - )) + result.records.collect.toBag should equal( + Bag( + CypherMap("a.prop" -> "a", "other.prop" -> "c"), + CypherMap("a.prop" -> "a", "other.prop" -> "b"), + CypherMap("a.prop" -> "a", "other.prop" -> "d") + ) + ) } it("matches an undirected pattern with pre-bound nodes") { @@ -303,18 +307,19 @@ class MatchTests extends MorpheusTestSuite with ScanGraphInit { """.stripMargin ) - val result = given.cypher( - """ + val result = given.cypher(""" |MATCH (a:A) |MATCH (b:B) |MATCH (a)--(b) |RETURN a.prop, b.prop """.stripMargin) - result.records.collect.toBag should equal(Bag( - CypherMap("a.prop" -> "a", "b.prop" -> "b"), - CypherMap("a.prop" -> "a", "b.prop" -> "b") - )) + result.records.collect.toBag should equal( + Bag( + CypherMap("a.prop" -> "a", "b.prop" -> "b"), + CypherMap("a.prop" -> "a", "b.prop" -> "b") + ) + ) } it("matches a mixed directed/undirected pattern") { @@ -330,14 +335,17 @@ class MatchTests extends MorpheusTestSuite with ScanGraphInit { """.stripMargin ) - val result = given.cypher("MATCH (a:A)--(a)<--(other) RETURN a.prop, other.prop") + val result = + given.cypher("MATCH (a:A)--(a)<--(other) RETURN a.prop, other.prop") - result.records.collect.toBag should equal(Bag( - CypherMap("a.prop" -> "a", "other.prop" -> "a"), - CypherMap("a.prop" -> "a", "other.prop" -> "a"), - CypherMap("a.prop" -> "a", "other.prop" -> "b"), - CypherMap("a.prop" -> "a", "other.prop" -> "b") - )) + result.records.collect.toBag should equal( + Bag( + CypherMap("a.prop" -> "a", "other.prop" -> "a"), + CypherMap("a.prop" -> "a", "other.prop" -> "a"), + CypherMap("a.prop" -> "a", "other.prop" -> "b"), + CypherMap("a.prop" -> "a", "other.prop" -> "b") + ) + ) } it("matches an undirected cyclic relationship") { @@ -352,9 +360,11 @@ class MatchTests extends MorpheusTestSuite with ScanGraphInit { val result = given.cypher("MATCH (a:A)--(a) RETURN a.prop") - result.records.collect.toBag should equal(Bag( - CypherMap("a.prop" -> "isA") - )) + result.records.collect.toBag should equal( + Bag( + CypherMap("a.prop" -> "isA") + ) + ) } it("matches an undirected variable-length relationship") { @@ -368,11 +378,14 @@ class MatchTests extends MorpheusTestSuite with ScanGraphInit { """.stripMargin ) - val result = given.cypher("MATCH (a:A)-[*2..2]-(other) RETURN a.prop, other.prop") + val result = + given.cypher("MATCH (a:A)-[*2..2]-(other) RETURN a.prop, other.prop") - result.records.collect.toBag should equal(Bag( - CypherMap("a.prop" -> "a", "other.prop" -> "c") - )) + result.records.collect.toBag should equal( + Bag( + CypherMap("a.prop" -> "a", "other.prop" -> "c") + ) + ) } } @@ -419,13 +432,14 @@ class MatchTests extends MorpheusTestSuite with ScanGraphInit { it("reports error on mismatched scans on constructed graph") { an[IllegalArgumentException] shouldBe thrownBy { - morpheus.cypher( - """ + morpheus + .cypher(""" |CONSTRUCT | CREATE (:A {p: 1}) | CREATE (:B {p: 'hi'}) |MATCH (n) - |RETURN count(*)""".stripMargin).show + |RETURN count(*)""".stripMargin) + .show } } } @@ -448,26 +462,34 @@ class MatchTests extends MorpheusTestSuite with ScanGraphInit { "MATCH (a:Person)-[:LIVES_IN]->(c:City)<-[:LIVES_IN]-(b:Person), (a)-[:KNOWS*1..2]->(b) RETURN a.name, b.name, c.name" ) - result.records.toMaps should equal(Bag( - CypherMap("a.name" -> "Philip", "b.name" -> "Stefan", "c.name" -> "The Pan-European Sprawl") - )) + result.records.toMaps should equal( + Bag( + CypherMap( + "a.name" -> "Philip", + "b.name" -> "Stefan", + "c.name" -> "The Pan-European Sprawl" + ) + ) + ) } describe("match disjunctions of relationship types") { it("can match a disjunction of two types") { val given = initGraph(sprawlGraphInit) - val result = given.cypher("MATCH ()-[r:LIVES_IN|KNOWS]->() RETURN type(r)") - result.records.toMaps should equal(Bag( - CypherMap("type(r)" -> "LIVES_IN"), - CypherMap("type(r)" -> "LIVES_IN"), - CypherMap("type(r)" -> "KNOWS") - )) + val result = + given.cypher("MATCH ()-[r:LIVES_IN|KNOWS]->() RETURN type(r)") + result.records.toMaps should equal( + Bag( + CypherMap("type(r)" -> "LIVES_IN"), + CypherMap("type(r)" -> "LIVES_IN"), + CypherMap("type(r)" -> "KNOWS") + ) + ) } it("can match a disjunction of four types with var length expand") { - val given = initGraph( - """ + val given = initGraph(""" |CREATE (a { val: 'a' }) |CREATE (b { val: 'b' }) |CREATE (c { val: 'c' }) @@ -477,18 +499,22 @@ class MatchTests extends MorpheusTestSuite with ScanGraphInit { |CREATE (b)-[:C]->(c) |CREATE (c)-[:D]->(d) """.stripMargin) - val result = given.cypher("MATCH (from)-[:A|B|C|D*1..3]->(to) RETURN from.val as from, to.val as to") - result.records.toMaps should equal(Bag( - CypherMap("from" -> "a", "to" -> "a"), - CypherMap("from" -> "a", "to" -> "b"), - CypherMap("from" -> "a", "to" -> "b"), - CypherMap("from" -> "a", "to" -> "c"), - CypherMap("from" -> "a", "to" -> "c"), - CypherMap("from" -> "a", "to" -> "d"), - CypherMap("from" -> "b", "to" -> "c"), - CypherMap("from" -> "b", "to" -> "d"), - CypherMap("from" -> "c", "to" -> "d") - )) + val result = given.cypher( + "MATCH (from)-[:A|B|C|D*1..3]->(to) RETURN from.val as from, to.val as to" + ) + result.records.toMaps should equal( + Bag( + CypherMap("from" -> "a", "to" -> "a"), + CypherMap("from" -> "a", "to" -> "b"), + CypherMap("from" -> "a", "to" -> "b"), + CypherMap("from" -> "a", "to" -> "c"), + CypherMap("from" -> "a", "to" -> "c"), + CypherMap("from" -> "a", "to" -> "d"), + CypherMap("from" -> "b", "to" -> "c"), + CypherMap("from" -> "b", "to" -> "d"), + CypherMap("from" -> "c", "to" -> "d") + ) + ) } } diff --git a/morpheus-testing/src/test/scala/org/opencypher/morpheus/impl/acceptance/MultipleGraphTests.scala b/morpheus-testing/src/test/scala/org/opencypher/morpheus/impl/acceptance/MultipleGraphTests.scala index e22abd9223..055d9b8933 100644 --- a/morpheus-testing/src/test/scala/org/opencypher/morpheus/impl/acceptance/MultipleGraphTests.scala +++ b/morpheus-testing/src/test/scala/org/opencypher/morpheus/impl/acceptance/MultipleGraphTests.scala @@ -47,21 +47,26 @@ import scala.language.existentials class MultipleGraphTests extends MorpheusTestSuite with ScanGraphInit { - def testGraph1: RelationalCypherGraph[SparkTable.DataFrameTable] = initGraph("CREATE (:Person {name: 'Mats'})") + def testGraph1: RelationalCypherGraph[SparkTable.DataFrameTable] = initGraph( + "CREATE (:Person {name: 'Mats'})" + ) - def testGraph2: RelationalCypherGraph[SparkTable.DataFrameTable] = initGraph("CREATE (:Person {name: 'Phil'})") + def testGraph2: RelationalCypherGraph[SparkTable.DataFrameTable] = initGraph( + "CREATE (:Person {name: 'Phil'})" + ) - def testGraph3: RelationalCypherGraph[SparkTable.DataFrameTable] = initGraph("CREATE (:Car {type: 'Toyota'})") + def testGraph3: RelationalCypherGraph[SparkTable.DataFrameTable] = initGraph( + "CREATE (:Car {type: 'Toyota'})" + ) - def testGraphRels: RelationalCypherGraph[SparkTable.DataFrameTable] = initGraph( - """|CREATE (mats:Person {name: 'Mats'}) + def testGraphRels: RelationalCypherGraph[SparkTable.DataFrameTable] = + initGraph("""|CREATE (mats:Person {name: 'Mats'}) |CREATE (max:Person {name: 'Max'}) |CREATE (max)-[:HAS_SIMILAR_NAME]->(mats) """.stripMargin) it("can read graph via parameter") { - val graph = initGraph( - """|CREATE (:A {v: 1}) + val graph = initGraph("""|CREATE (:A {v: 1}) |CREATE (:B {v: 100})""".stripMargin) morpheus.catalog.store("g1", graph) @@ -75,27 +80,29 @@ class MultipleGraphTests extends MorpheusTestSuite with ScanGraphInit { } it("creates multiple copies of the same node") { - val g = morpheus.cypher( - """ + val g = morpheus + .cypher(""" |CONSTRUCT | CREATE () |RETURN GRAPH - """.stripMargin).graph - val results = g.cypher( - """ + """.stripMargin) + .graph + val results = g + .cypher(""" |MATCH (a) |CONSTRUCT | CREATE (f COPY OF a)-[:FOO]->(g COPY OF a) |MATCH (n) |RETURN n - """.stripMargin).records + """.stripMargin) + .records results.size shouldBe 2 } it("can match on constructed graph") { - val results = morpheus.cypher( - """ + val results = morpheus + .cypher(""" |CONSTRUCT | CREATE () |MATCH (a) @@ -103,12 +110,13 @@ class MultipleGraphTests extends MorpheusTestSuite with ScanGraphInit { | CREATE (f COPY OF a)-[:FOO]->(g COPY OF a) |MATCH (n) |RETURN n - """.stripMargin).records + """.stripMargin) + .records results.size shouldBe 2 } - //TODO: This test has no useful expectation + // TODO: This test has no useful expectation it("CLONEs with an alias") { val query = """ @@ -156,7 +164,8 @@ class MultipleGraphTests extends MorpheusTestSuite with ScanGraphInit { result.records.toMaps should equal( Bag( CypherMap("name" -> "Phil") - )) + ) + ) } it("matches from different graphs") { @@ -177,15 +186,14 @@ class MultipleGraphTests extends MorpheusTestSuite with ScanGraphInit { result.records.toMaps should equal( Bag( CypherMap("name" -> "Phil", "car" -> "Toyota") - )) + ) + ) } it("constructs from different graphs with multiple distinct nodes") { - val g1 = initGraph( - """|CREATE (:A {v: 1}) + val g1 = initGraph("""|CREATE (:A {v: 1}) |CREATE (:B {v: 100})""".stripMargin) - val g2 = initGraph( - """|CREATE (:A {v: 2}) + val g2 = initGraph("""|CREATE (:A {v: 2}) |CREATE (:B {v: 200})""".stripMargin) morpheus.catalog.store("g1", g1) morpheus.catalog.store("g2", g2) @@ -252,7 +260,9 @@ class MultipleGraphTests extends MorpheusTestSuite with ScanGraphInit { result.graph.relationships("r").size should equal(2) } - it("should CONSTRUCT a graph with multiple unconnected anonymous CREATE clauses") { + it( + "should CONSTRUCT a graph with multiple unconnected anonymous CREATE clauses" + ) { val query = """|CONSTRUCT | CREATE (:A) @@ -280,10 +290,18 @@ class MultipleGraphTests extends MorpheusTestSuite with ScanGraphInit { result.getRecords shouldBe None result.graph.schema.labels should equal(Set("A")) - result.graph.schema should equal(PropertyGraphSchema.empty.withNodePropertyKeys("A")("name" -> CTString)) - result.graph.cypher("MATCH (a:A) RETURN a.name").records.iterator.toBag should equal(Bag( - CypherMap("a.name" -> "Mats") - )) + result.graph.schema should equal( + PropertyGraphSchema.empty.withNodePropertyKeys("A")("name" -> CTString) + ) + result.graph + .cypher("MATCH (a:A) RETURN a.name") + .records + .iterator + .toBag should equal( + Bag( + CypherMap("a.name" -> "Mats") + ) + ) } it("should construct a node property from a literal") { @@ -295,10 +313,18 @@ class MultipleGraphTests extends MorpheusTestSuite with ScanGraphInit { val result = morpheus.cypher(query) result.getRecords shouldBe None - result.graph.schema should equal(PropertyGraphSchema.empty.withNodePropertyKeys()("name" -> CTString)) - result.graph.cypher("MATCH (a) RETURN a.name").records.iterator.toBag should equal(Bag( - CypherMap("a.name" -> "Donald") - )) + result.graph.schema should equal( + PropertyGraphSchema.empty.withNodePropertyKeys()("name" -> CTString) + ) + result.graph + .cypher("MATCH (a) RETURN a.name") + .records + .iterator + .toBag should equal( + Bag( + CypherMap("a.name" -> "Donald") + ) + ) } it("should construct a node label and a node property from a literal") { @@ -311,10 +337,18 @@ class MultipleGraphTests extends MorpheusTestSuite with ScanGraphInit { result.getRecords shouldBe None result.graph.schema.labels should equal(Set("A")) - result.graph.schema should equal(PropertyGraphSchema.empty.withNodePropertyKeys("A")("name" -> CTString)) - result.graph.cypher("MATCH (a:A) RETURN a.name").records.iterator.toBag should equal(Bag( - CypherMap("a.name" -> "Donald") - )) + result.graph.schema should equal( + PropertyGraphSchema.empty.withNodePropertyKeys("A")("name" -> CTString) + ) + result.graph + .cypher("MATCH (a:A) RETURN a.name") + .records + .iterator + .toBag should equal( + Bag( + CypherMap("a.name" -> "Donald") + ) + ) } it("should construct multiple properties") { @@ -329,11 +363,20 @@ class MultipleGraphTests extends MorpheusTestSuite with ScanGraphInit { result.graph.schema.labels should equal(Set("A", "B")) result.graph.schema should equal( PropertyGraphSchema.empty - .withNodePropertyKeys(Set("A", "B"), PropertyKeys("name" -> CTString, "age" -> CTInteger)) + .withNodePropertyKeys( + Set("A", "B"), + PropertyKeys("name" -> CTString, "age" -> CTInteger) + ) + ) + result.graph + .cypher("MATCH (a:A:B) RETURN a.name") + .records + .iterator + .toBag should equal( + Bag( + CypherMap("a.name" -> "Donald") + ) ) - result.graph.cypher("MATCH (a:A:B) RETURN a.name").records.iterator.toBag should equal(Bag( - CypherMap("a.name" -> "Donald") - )) } it("should pick up labels of the outer match") { @@ -351,9 +394,15 @@ class MultipleGraphTests extends MorpheusTestSuite with ScanGraphInit { PropertyGraphSchema.empty .withNodePropertyKeys(Set("Person"), PropertyKeys("name" -> CTString)) ) - result.graph.cypher("MATCH (a:Person) RETURN a.name").records.iterator.toBag should equal(Bag( - CypherMap("a.name" -> "Mats") - )) + result.graph + .cypher("MATCH (a:Person) RETURN a.name") + .records + .iterator + .toBag should equal( + Bag( + CypherMap("a.name" -> "Mats") + ) + ) } it("should construct a relationship") { @@ -366,12 +415,20 @@ class MultipleGraphTests extends MorpheusTestSuite with ScanGraphInit { result.getRecords shouldBe None result.graph.schema.relationshipTypes should equal(Set("FOO")) - result.graph.schema should equal(PropertyGraphSchema.empty - .withNodePropertyKeys()() - .withRelationshipPropertyKeys("FOO", PropertyKeys("val" -> CTInteger))) - result.graph.cypher("MATCH ()-[r]->() RETURN r.val").records.iterator.toBag should equal(Bag( - CypherMap("r.val" -> 42) - )) + result.graph.schema should equal( + PropertyGraphSchema.empty + .withNodePropertyKeys()() + .withRelationshipPropertyKeys("FOO", PropertyKeys("val" -> CTInteger)) + ) + result.graph + .cypher("MATCH ()-[r]->() RETURN r.val") + .records + .iterator + .toBag should equal( + Bag( + CypherMap("r.val" -> 42) + ) + ) } it("should copy a relationship") { @@ -386,14 +443,19 @@ class MultipleGraphTests extends MorpheusTestSuite with ScanGraphInit { val result = testGraph1.cypher(query) result.getRecords shouldBe None - result.graph.cypher("MATCH ()-[r]->() RETURN r.val, r.name, type(r) as type").records.iterator.toBag should equal(Bag( - CypherMap("r.val" -> 42, "r.name" -> "Donald", "type" -> "FOO") - )) + result.graph + .cypher("MATCH ()-[r]->() RETURN r.val, r.name, type(r) as type") + .records + .iterator + .toBag should equal( + Bag( + CypherMap("r.val" -> 42, "r.name" -> "Donald", "type" -> "FOO") + ) + ) } it("should copy a mean relationship") { - val graph = initGraph( - """ + val graph = initGraph(""" |CREATE ()-[:FOO {val: 1, val2: 2}]->() |CREATE ()-[:BAR {val: 1, val2: 3}]->() """.stripMargin) @@ -408,10 +470,16 @@ class MultipleGraphTests extends MorpheusTestSuite with ScanGraphInit { result.getRecords shouldBe None - result.graph.cypher("MATCH ()-[r]->() RETURN r.val, r.val2, type(r) as type").records.iterator.toBag should equal(Bag( - CypherMap("r.val" -> 1, "r.val2" -> "Donald", "type" -> "BAZ"), - CypherMap("r.val" -> 1, "r.val2" -> "Donald", "type" -> "BAZ") - )) + result.graph + .cypher("MATCH ()-[r]->() RETURN r.val, r.val2, type(r) as type") + .records + .iterator + .toBag should equal( + Bag( + CypherMap("r.val" -> 1, "r.val2" -> "Donald", "type" -> "BAZ"), + CypherMap("r.val" -> 1, "r.val2" -> "Donald", "type" -> "BAZ") + ) + ) } it("should copy a node") { @@ -427,18 +495,24 @@ class MultipleGraphTests extends MorpheusTestSuite with ScanGraphInit { result.getRecords shouldBe None result.graph.schema.labels should equal(Set("Foo")) - result.graph.schema should equal(PropertyGraphSchema.empty - .withNodePropertyKeys("Foo")("foo" -> CTString) + result.graph.schema should equal( + PropertyGraphSchema.empty + .withNodePropertyKeys("Foo")("foo" -> CTString) ) - result.graph.cypher("MATCH (a) RETURN a.foo, labels(a) as labels").records.iterator.toBag should equal(Bag( - CypherMap("a.foo" -> "bar", "labels" -> Seq("Foo")) - )) + result.graph + .cypher("MATCH (a) RETURN a.foo, labels(a) as labels") + .records + .iterator + .toBag should equal( + Bag( + CypherMap("a.foo" -> "bar", "labels" -> Seq("Foo")) + ) + ) } it("should copy a node with labels") { - val graph = initGraph( - """ + val graph = initGraph(""" |CREATE (:A {val: 1}) |CREATE (:B {val: 2}) |CREATE (:A:C {val: 3}) @@ -453,16 +527,21 @@ class MultipleGraphTests extends MorpheusTestSuite with ScanGraphInit { val result = graph.cypher(query) result.getRecords shouldBe None - result.graph.cypher("MATCH (a) RETURN a.val, labels(a) as labels").records.iterator.toBag should equal(Bag( - CypherMap("a.val" -> 1, "labels" -> Seq("A")), - CypherMap("a.val" -> 2, "labels" -> Seq("B")), - CypherMap("a.val" -> 3, "labels" -> Seq("A", "C")) - )) + result.graph + .cypher("MATCH (a) RETURN a.val, labels(a) as labels") + .records + .iterator + .toBag should equal( + Bag( + CypherMap("a.val" -> 1, "labels" -> Seq("A")), + CypherMap("a.val" -> 2, "labels" -> Seq("B")), + CypherMap("a.val" -> 3, "labels" -> Seq("A", "C")) + ) + ) } it("can override in SET") { - val graph = initGraph( - """ + val graph = initGraph(""" |CREATE ({val: 1}) """.stripMargin) @@ -475,14 +554,19 @@ class MultipleGraphTests extends MorpheusTestSuite with ScanGraphInit { val result = graph.cypher(query) result.getRecords shouldBe None - result.graph.cypher("MATCH (a) RETURN a.val").records.iterator.toBag should equal(Bag( - CypherMap("a.val" -> 2) - )) + result.graph + .cypher("MATCH (a) RETURN a.val") + .records + .iterator + .toBag should equal( + Bag( + CypherMap("a.val" -> 2) + ) + ) } it("can override heterogeneous types in SET") { - val graph = initGraph( - """ + val graph = initGraph(""" |CREATE ({val: 1}) """.stripMargin) @@ -495,14 +579,21 @@ class MultipleGraphTests extends MorpheusTestSuite with ScanGraphInit { val result = graph.cypher(query) result.getRecords shouldBe None - result.graph.cypher("MATCH (a) RETURN a.val").records.iterator.toBag should equal(Bag( - CypherMap("a.val" -> "foo") - )) + result.graph + .cypher("MATCH (a) RETURN a.val") + .records + .iterator + .toBag should equal( + Bag( + CypherMap("a.val" -> "foo") + ) + ) } it("supports CLONE in CONSTRUCT") { - val res = testGraph1.unionAll(testGraph2).cypher( - """ + val res = testGraph1 + .unionAll(testGraph2) + .cypher(""" |MATCH (n),(m) |WHERE n.name = 'Mats' AND m.name = 'Phil' |CONSTRUCT @@ -516,8 +607,9 @@ class MultipleGraphTests extends MorpheusTestSuite with ScanGraphInit { } it("implicitly CLONEs in CONSTRUCT") { - val res = testGraph1.unionAll(testGraph2).cypher( - """ + val res = testGraph1 + .unionAll(testGraph2) + .cypher(""" |MATCH (n),(m) |WHERE n.name = 'Mats' AND m.name = 'Phil' |CONSTRUCT @@ -530,14 +622,12 @@ class MultipleGraphTests extends MorpheusTestSuite with ScanGraphInit { } it("constructs multiple relationships") { - val inputGraph = initGraph( - """ + val inputGraph = initGraph(""" |CREATE (p0 {name: 'Mats'}) |CREATE (p1 {name: 'Phil'}) """.stripMargin) - val res = inputGraph.cypher( - """ + val res = inputGraph.cypher(""" |MATCH (n),(m) |WHERE n <> m |CONSTRUCT @@ -550,8 +640,7 @@ class MultipleGraphTests extends MorpheusTestSuite with ScanGraphInit { } it("implicitly clones when constructing multiple relationships") { - val inputGraph = initGraph( - """ + val inputGraph = initGraph(""" |CREATE (p0 {name: 'Mats'}) |CREATE (p1 {name: 'Phil'}) |CREATE (p0)-[:KNOWS]->(p1) @@ -559,8 +648,7 @@ class MultipleGraphTests extends MorpheusTestSuite with ScanGraphInit { |CREATE (p1)-[:KNOWS]->(p0) """.stripMargin) - val res = inputGraph.cypher( - """ + val res = inputGraph.cypher(""" |MATCH (n)-[:KNOWS]->(m) |WITH DISTINCT n, m |CONSTRUCT @@ -573,8 +661,7 @@ class MultipleGraphTests extends MorpheusTestSuite with ScanGraphInit { } it("constructs multiple relationships 2") { - val inputGraph = initGraph( - """ + val inputGraph = initGraph(""" |CREATE (p0 {name: 'Mats'}) |CREATE (p1 {name: 'Phil'}) |CREATE (p0)-[:KNOWS]->(p1) @@ -582,8 +669,7 @@ class MultipleGraphTests extends MorpheusTestSuite with ScanGraphInit { |CREATE (p1)-[:KNOWS]->(p0) """.stripMargin) - val res = inputGraph.cypher( - """ + val res = inputGraph.cypher(""" |MATCH (n)-[:KNOWS]->(m) |CONSTRUCT | CLONE n AS n, m AS m @@ -596,8 +682,7 @@ class MultipleGraphTests extends MorpheusTestSuite with ScanGraphInit { } it("implicitly clones when constructing multiple relationships 2") { - val inputGraph = initGraph( - """ + val inputGraph = initGraph(""" |CREATE (p0 {name: 'Mats'}) |CREATE (p1 {name: 'Phil'}) |CREATE (p0)-[:KNOWS]->(p1) @@ -605,8 +690,7 @@ class MultipleGraphTests extends MorpheusTestSuite with ScanGraphInit { |CREATE (p1)-[:KNOWS]->(p0) """.stripMargin) - val res = inputGraph.cypher( - """ + val res = inputGraph.cypher(""" |MATCH (n)-[:KNOWS]->(m) |CONSTRUCT | CREATE (n)-[r:KNOWS]->(m) @@ -628,7 +712,9 @@ class MultipleGraphTests extends MorpheusTestSuite with ScanGraphInit { result.schema.asMorpheus should equal(testGraph1.schema) result.nodes("n").toMaps should equal(testGraph1.nodes("n").toMaps) - result.relationships("r").toMaps should equal(testGraph1.relationships("r").toMaps) + result.relationships("r").toMaps should equal( + testGraph1.relationships("r").toMaps + ) } it("CONSTRUCT ON a single graph without GraphUnionAll") { @@ -642,11 +728,15 @@ class MultipleGraphTests extends MorpheusTestSuite with ScanGraphInit { result.asMorpheus.maybeRelational match { case Some(relPlan) => - val switchOp = relPlan.collectFirst { case op: SwitchContext[_] => op }.get - val containsUnionGraph = switchOp.context.queryLocalCatalog.head._2 match { - case g: UnionGraph[_] => g.graphs.collectFirst { case op: UnionGraph[_] => op }.isDefined - case _ => false - } + val switchOp = relPlan.collectFirst { case op: SwitchContext[_] => + op + }.get + val containsUnionGraph = + switchOp.context.queryLocalCatalog.head._2 match { + case g: UnionGraph[_] => + g.graphs.collectFirst { case op: UnionGraph[_] => op }.isDefined + case _ => false + } withClue("CONSTRUCT plans union on a single input graph") { containsUnionGraph shouldBe false } @@ -666,8 +756,12 @@ class MultipleGraphTests extends MorpheusTestSuite with ScanGraphInit { val result = testGraph2.cypher(query).graph result.schema should equal(testGraph1.schema ++ testGraph2.schema) - result.nodes("n").toMaps should equal(testGraph1.unionAll(testGraph2).nodes("n").toMaps) - result.relationships("r").toMaps should equal(testGraph1.unionAll(testGraph2).relationships("r").toMaps) + result.nodes("n").toMaps should equal( + testGraph1.unionAll(testGraph2).nodes("n").toMaps + ) + result.relationships("r").toMaps should equal( + testGraph1.unionAll(testGraph2).relationships("r").toMaps + ) } it("CONSTRUCTS ON two graphs and adds a relationship") { @@ -684,16 +778,29 @@ class MultipleGraphTests extends MorpheusTestSuite with ScanGraphInit { val result = morpheus.cypher(query).graph - result.schema should equal((testGraph1.schema ++ testGraph2.schema).withRelationshipPropertyKeys("KNOWS")()) - result.nodes("n").toMaps should equal(testGraph1.unionAll(testGraph2).nodes("n").toMaps) - val resultRelationship = result.relationships("r").toMaps.head._1("r").asInstanceOf[MorpheusRelationship] + result.schema should equal( + (testGraph1.schema ++ testGraph2.schema).withRelationshipPropertyKeys( + "KNOWS" + )() + ) + result.nodes("n").toMaps should equal( + testGraph1.unionAll(testGraph2).nodes("n").toMaps + ) + val resultRelationship = result + .relationships("r") + .toMaps + .head + ._1("r") + .asInstanceOf[MorpheusRelationship] resultRelationship.startId should equal(0L.withPrefix(0).toList) resultRelationship.endId should equal(0L.withPrefix(1).toList) resultRelationship.id should equal(0L.withPrefix(-1).toList) resultRelationship.relType should equal("KNOWS") } - it("implictly clones when CONSTRUCTing ON two graphs and adding a relationship") { + it( + "implictly clones when CONSTRUCTing ON two graphs and adding a relationship" + ) { morpheus.catalog.store("one", testGraph1) morpheus.catalog.store("two", testGraph2) val query = @@ -707,9 +814,20 @@ class MultipleGraphTests extends MorpheusTestSuite with ScanGraphInit { val result = morpheus.cypher(query).graph - result.schema should equal((testGraph1.schema ++ testGraph2.schema).withRelationshipPropertyKeys("KNOWS")()) - result.nodes("n").toMaps should equal(testGraph1.unionAll(testGraph2).nodes("n").toMaps) - val resultRelationship = result.relationships("r").toMaps.head._1("r").asInstanceOf[MorpheusRelationship] + result.schema should equal( + (testGraph1.schema ++ testGraph2.schema).withRelationshipPropertyKeys( + "KNOWS" + )() + ) + result.nodes("n").toMaps should equal( + testGraph1.unionAll(testGraph2).nodes("n").toMaps + ) + val resultRelationship = result + .relationships("r") + .toMaps + .head + ._1("r") + .asInstanceOf[MorpheusRelationship] resultRelationship.startId should equal(0L.withPrefix(0).toList) resultRelationship.endId should equal(0L.withPrefix(1).toList) resultRelationship.id should equal(0L.withPrefix(-1).toList) @@ -726,10 +844,14 @@ class MultipleGraphTests extends MorpheusTestSuite with ScanGraphInit { val graph = morpheus.cypher(query).graph - graph.schema should equal(PropertyGraphSchema.empty.withNodePropertyKeys(Set.empty[String])) - graph.nodes("n").collect.toBag should equal(Bag( - CypherMap("n" -> MorpheusNode(0, Set.empty[String])) - )) + graph.schema should equal( + PropertyGraphSchema.empty.withNodePropertyKeys(Set.empty[String]) + ) + graph.nodes("n").collect.toBag should equal( + Bag( + CypherMap("n" -> MorpheusNode(0, Set.empty[String])) + ) + ) } it("construct match construct") { @@ -750,15 +872,43 @@ class MultipleGraphTests extends MorpheusTestSuite with ScanGraphInit { val graph = morpheus.cypher(query).graph graph.schema.asMorpheus should equal(testGraphRels.schema) - graph.nodes("n").collect.toBag should equal(Bag( - CypherMap("n" -> MorpheusNode(0.withPrefix(1), Set("Person"), CypherMap("name" -> "Mats"))), - CypherMap("n" -> MorpheusNode(1.withPrefix(1), Set("Person"), CypherMap("name" -> "Max"))), - CypherMap("n" -> MorpheusNode(0L.withPrefix(0), Set("Person"), CypherMap("name" -> "Mats"))), - CypherMap("n" -> MorpheusNode(1L.withPrefix(0), Set("Person"), CypherMap("name" -> "Max"))) - )) + graph.nodes("n").collect.toBag should equal( + Bag( + CypherMap( + "n" -> MorpheusNode( + 0.withPrefix(1), + Set("Person"), + CypherMap("name" -> "Mats") + ) + ), + CypherMap( + "n" -> MorpheusNode( + 1.withPrefix(1), + Set("Person"), + CypherMap("name" -> "Max") + ) + ), + CypherMap( + "n" -> MorpheusNode( + 0L.withPrefix(0), + Set("Person"), + CypherMap("name" -> "Mats") + ) + ), + CypherMap( + "n" -> MorpheusNode( + 1L.withPrefix(0), + Set("Person"), + CypherMap("name" -> "Max") + ) + ) + ) + ) } - it("does not clone twice when a variable is both constructed on and matched") { + it( + "does not clone twice when a variable is both constructed on and matched" + ) { morpheus.catalog.store("g1", testGraph1) morpheus.catalog.store("g2", testGraph2) val query = @@ -776,10 +926,24 @@ class MultipleGraphTests extends MorpheusTestSuite with ScanGraphInit { val graph = morpheus.cypher(query).graph graph.schema.asMorpheus should equal(testGraph1.schema) - graph.nodes("n").collect.toBag should equal(Bag( - CypherMap("n" -> MorpheusNode(0L.withPrefix(1), Set("Person"), CypherMap("name" -> "Mats"))), - CypherMap("n" -> MorpheusNode(0L.withPrefix(0), Set("Person"), CypherMap("name" -> "Phil"))) - )) + graph.nodes("n").collect.toBag should equal( + Bag( + CypherMap( + "n" -> MorpheusNode( + 0L.withPrefix(1), + Set("Person"), + CypherMap("name" -> "Mats") + ) + ), + CypherMap( + "n" -> MorpheusNode( + 0L.withPrefix(0), + Set("Person"), + CypherMap("name" -> "Phil") + ) + ) + ) + ) } it("allows CONSTRUCT ON with relationships") { @@ -795,25 +959,70 @@ class MultipleGraphTests extends MorpheusTestSuite with ScanGraphInit { val result = morpheus.cypher(query).graph - result.schema should equal((testGraph1.schema ++ testGraph2.schema).withRelationshipPropertyKeys("HAS_SIMILAR_NAME")()) + result.schema should equal( + (testGraph1.schema ++ testGraph2.schema).withRelationshipPropertyKeys( + "HAS_SIMILAR_NAME" + )() + ) - result.nodes("n").toMaps should equal(Bag( - CypherMap("n" -> MorpheusNode(0L.withPrefix(0), Set("Person"), CypherMap("name" -> "Mats"))), - CypherMap("n" -> MorpheusNode(1L.withPrefix(0), Set("Person"), CypherMap("name" -> "Max"))), - CypherMap("n" -> MorpheusNode(0L.withPrefix(1), Set("Person"), CypherMap("name" -> "Mats"))), - CypherMap("n" -> MorpheusNode(1L.withPrefix(1), Set("Person"), CypherMap("name" -> "Max"))) - )) + result.nodes("n").toMaps should equal( + Bag( + CypherMap( + "n" -> MorpheusNode( + 0L.withPrefix(0), + Set("Person"), + CypherMap("name" -> "Mats") + ) + ), + CypherMap( + "n" -> MorpheusNode( + 1L.withPrefix(0), + Set("Person"), + CypherMap("name" -> "Max") + ) + ), + CypherMap( + "n" -> MorpheusNode( + 0L.withPrefix(1), + Set("Person"), + CypherMap("name" -> "Mats") + ) + ), + CypherMap( + "n" -> MorpheusNode( + 1L.withPrefix(1), + Set("Person"), + CypherMap("name" -> "Max") + ) + ) + ) + ) - result.relationships("r").toMaps should equal(Bag( - CypherMap("r" -> MorpheusRelationship(2L.withPrefix(0), 1L.withPrefix(0), 0L.withPrefix(0), "HAS_SIMILAR_NAME")), - CypherMap("r" -> MorpheusRelationship(2L.withPrefix(1), 1L.withPrefix(1), 0L.withPrefix(1), "HAS_SIMILAR_NAME")) - )) + result.relationships("r").toMaps should equal( + Bag( + CypherMap( + "r" -> MorpheusRelationship( + 2L.withPrefix(0), + 1L.withPrefix(0), + 0L.withPrefix(0), + "HAS_SIMILAR_NAME" + ) + ), + CypherMap( + "r" -> MorpheusRelationship( + 2L.withPrefix(1), + 1L.withPrefix(1), + 0L.withPrefix(1), + "HAS_SIMILAR_NAME" + ) + ) + ) + ) } it("allows cloning from different graphs with nodes and relationships") { - def testGraphRels = initGraph( - """|CREATE (mats:Person {name: 'Mats'}) + def testGraphRels = initGraph("""|CREATE (mats:Person {name: 'Mats'}) |CREATE (max:Person {name: 'Max'}) |CREATE (max)-[:HAS_SIMILAR_NAME]->(mats) """.stripMargin) @@ -835,17 +1044,59 @@ class MultipleGraphTests extends MorpheusTestSuite with ScanGraphInit { val result = morpheus.cypher(query).graph result.schema.asMorpheus shouldEqual testGraphRels.schema - result.nodes("n").toMaps should equal(Bag( - CypherMap("n" -> MorpheusNode(0L.withPrefix(0), Set("Person"), CypherMap("name" -> "Mats"))), - CypherMap("n" -> MorpheusNode(1L.withPrefix(0), Set("Person"), CypherMap("name" -> "Max"))), - CypherMap("n" -> MorpheusNode(0L.withPrefix(1), Set("Person"), CypherMap("name" -> "Mats"))), - CypherMap("n" -> MorpheusNode(1L.withPrefix(1), Set("Person"), CypherMap("name" -> "Max"))) - )) - - result.relationships("r").toMaps should equal(Bag( - CypherMap("r" -> MorpheusRelationship(2L.withPrefix(0), 1L.withPrefix(0), 0L.withPrefix(0), "HAS_SIMILAR_NAME")), - CypherMap("r" -> MorpheusRelationship(2L.withPrefix(1), 1L.withPrefix(1), 0L.withPrefix(1), "HAS_SIMILAR_NAME")) - )) + result.nodes("n").toMaps should equal( + Bag( + CypherMap( + "n" -> MorpheusNode( + 0L.withPrefix(0), + Set("Person"), + CypherMap("name" -> "Mats") + ) + ), + CypherMap( + "n" -> MorpheusNode( + 1L.withPrefix(0), + Set("Person"), + CypherMap("name" -> "Max") + ) + ), + CypherMap( + "n" -> MorpheusNode( + 0L.withPrefix(1), + Set("Person"), + CypherMap("name" -> "Mats") + ) + ), + CypherMap( + "n" -> MorpheusNode( + 1L.withPrefix(1), + Set("Person"), + CypherMap("name" -> "Max") + ) + ) + ) + ) + + result.relationships("r").toMaps should equal( + Bag( + CypherMap( + "r" -> MorpheusRelationship( + 2L.withPrefix(0), + 1L.withPrefix(0), + 0L.withPrefix(0), + "HAS_SIMILAR_NAME" + ) + ), + CypherMap( + "r" -> MorpheusRelationship( + 2L.withPrefix(1), + 1L.withPrefix(1), + 0L.withPrefix(1), + "HAS_SIMILAR_NAME" + ) + ) + ) + ) } it("allows consecutive construction") { @@ -863,14 +1114,21 @@ class MultipleGraphTests extends MorpheusTestSuite with ScanGraphInit { result.getRecords shouldBe None result.graph.schema.relationshipTypes should equal(Set("KNOWS")) result.graph.schema.labels should equal(Set("A", "B")) - result.graph.schema should equal(PropertyGraphSchema.empty - .withNodePropertyKeys("A")() - .withNodePropertyKeys("B")() - .withRelationshipPropertyKeys("KNOWS")() + result.graph.schema should equal( + PropertyGraphSchema.empty + .withNodePropertyKeys("A")() + .withNodePropertyKeys("B")() + .withRelationshipPropertyKeys("KNOWS")() + ) + result.graph + .cypher("MATCH ()-[r]->() RETURN type(r)") + .records + .iterator + .toBag should equal( + Bag( + CypherMap("type(r)" -> "KNOWS") + ) ) - result.graph.cypher("MATCH ()-[r]->() RETURN type(r)").records.iterator.toBag should equal(Bag( - CypherMap("type(r)" -> "KNOWS") - )) } it("implictly clones when doing consecutive construction") { @@ -887,57 +1145,79 @@ class MultipleGraphTests extends MorpheusTestSuite with ScanGraphInit { result.getRecords shouldBe None result.graph.schema.relationshipTypes should equal(Set("KNOWS")) result.graph.schema.labels should equal(Set("A", "B")) - result.graph.schema should equal(PropertyGraphSchema.empty - .withNodePropertyKeys("A")() - .withNodePropertyKeys("B")() - .withRelationshipPropertyKeys("KNOWS")()) - result.graph.cypher("MATCH ()-[r]->() RETURN type(r)").records.iterator.toBag should equal(Bag( - CypherMap("type(r)" -> "KNOWS") - )) + result.graph.schema should equal( + PropertyGraphSchema.empty + .withNodePropertyKeys("A")() + .withNodePropertyKeys("B")() + .withRelationshipPropertyKeys("KNOWS")() + ) + result.graph + .cypher("MATCH ()-[r]->() RETURN type(r)") + .records + .iterator + .toBag should equal( + Bag( + CypherMap("type(r)" -> "KNOWS") + ) + ) } it("can construct a copy of a node with matched label") { - morpheus.cypher("CATALOG CREATE GRAPH foo { CONSTRUCT CREATE (:A) RETURN GRAPH }") + morpheus.cypher( + "CATALOG CREATE GRAPH foo { CONSTRUCT CREATE (:A) RETURN GRAPH }" + ) val graph = morpheus.cypher("FROM GRAPH foo RETURN GRAPH").graph - graph.cypher( - """MATCH (a:A) + graph + .cypher("""MATCH (a:A) |CONSTRUCT | CREATE (COPY OF a) |MATCH (n) |RETURN labels(n) - """.stripMargin).records.iterator.toBag should equal(Bag( - CypherMap("labels(n)" -> Seq("A")) - )) + """.stripMargin) + .records + .iterator + .toBag should equal( + Bag( + CypherMap("labels(n)" -> Seq("A")) + ) + ) } it("can construct with an input table expanded by unwind") { - morpheus.cypher("CATALOG CREATE GRAPH foo { CONSTRUCT CREATE (:A) RETURN GRAPH }") + morpheus.cypher( + "CATALOG CREATE GRAPH foo { CONSTRUCT CREATE (:A) RETURN GRAPH }" + ) - val data = morpheus.cypher("FROM GRAPH foo RETURN GRAPH").graph.cypher( - """MATCH (a:A) + val data = morpheus + .cypher("FROM GRAPH foo RETURN GRAPH") + .graph + .cypher("""MATCH (a:A) |UNWIND [1, 2, 3] AS i |CONSTRUCT | CREATE (f COPY OF a)-[:FOO]->(g COPY OF a) | CREATE (:B {name: 'foo'}) |MATCH (n) |RETURN n.name - """.stripMargin).records + """.stripMargin) + .records val nullRow = CypherMap("n.name" -> null) val fooRow = CypherMap("n.name" -> "foo") - data.iterator.toBag should equal(Bag( - nullRow, - nullRow, - nullRow, - nullRow, - nullRow, - nullRow, - fooRow, - fooRow, - fooRow - )) + data.iterator.toBag should equal( + Bag( + nullRow, + nullRow, + nullRow, + nullRow, + nullRow, + nullRow, + fooRow, + fooRow, + fooRow + ) + ) } it("should set a node property from a matched node") { @@ -952,10 +1232,17 @@ class MultipleGraphTests extends MorpheusTestSuite with ScanGraphInit { result.getRecords shouldBe None result.graph.schema.labels should equal(Set("A")) - result.graph.schema should equal(PropertyGraphSchema.empty.withNodePropertyKeys("A")("name" -> CTString)) - result.graph.cypher("MATCH (a:A) RETURN a.name").records.toMaps should equal(Bag( - CypherMap("a.name" -> "Mats") - )) + result.graph.schema should equal( + PropertyGraphSchema.empty.withNodePropertyKeys("A")("name" -> CTString) + ) + result.graph + .cypher("MATCH (a:A) RETURN a.name") + .records + .toMaps should equal( + Bag( + CypherMap("a.name" -> "Mats") + ) + ) } it("should set a node property from a literal") { @@ -969,10 +1256,17 @@ class MultipleGraphTests extends MorpheusTestSuite with ScanGraphInit { result.getRecords shouldBe None result.graph.schema.labels should equal(Set("A")) - result.graph.schema should equal(PropertyGraphSchema.empty.withNodePropertyKeys("A")("name" -> CTString)) - result.graph.cypher("MATCH (a:A) RETURN a.name").records.toMaps should equal(Bag( - CypherMap("a.name" -> "Donald") - )) + result.graph.schema should equal( + PropertyGraphSchema.empty.withNodePropertyKeys("A")("name" -> CTString) + ) + result.graph + .cypher("MATCH (a:A) RETURN a.name") + .records + .toMaps should equal( + Bag( + CypherMap("a.name" -> "Donald") + ) + ) } it("should set a node label") { @@ -992,11 +1286,14 @@ class MultipleGraphTests extends MorpheusTestSuite with ScanGraphInit { it("fails early on incompatible graphs in CONSTRUCT") { val g1 = morpheus.cypher("CONSTRUCT CREATE (:A {p: 1}) RETURN GRAPH").graph - val g2 = morpheus.cypher("CONSTRUCT CREATE (:A {p: 'foo'}) RETURN GRAPH").graph + val g2 = + morpheus.cypher("CONSTRUCT CREATE (:A {p: 'foo'}) RETURN GRAPH").graph morpheus.catalog.store("g1", g1) morpheus.catalog.store("g2", g2) - an[SchemaException] should be thrownBy morpheus.cypher("CONSTRUCT ON g1, g2 RETURN GRAPH") + an[SchemaException] should be thrownBy morpheus.cypher( + "CONSTRUCT ON g1, g2 RETURN GRAPH" + ) } it("should implicit clone a relationship #1") { @@ -1006,9 +1303,15 @@ class MultipleGraphTests extends MorpheusTestSuite with ScanGraphInit { | CREATE (a)-[r]->(b) |RETURN GRAPH""".stripMargin val result = testGraphRels.cypher(query) - result.graph.cypher("MATCH ()-[r]->() RETURN type(r) as type").records.iterator.toBag should equal(Bag( - CypherMap("type" -> "HAS_SIMILAR_NAME") - )) + result.graph + .cypher("MATCH ()-[r]->() RETURN type(r) as type") + .records + .iterator + .toBag should equal( + Bag( + CypherMap("type" -> "HAS_SIMILAR_NAME") + ) + ) } it("should implicit clone a relationship #2") { @@ -1019,9 +1322,15 @@ class MultipleGraphTests extends MorpheusTestSuite with ScanGraphInit { | CREATE (a)-[r]->(b) |RETURN GRAPH""".stripMargin val result = testGraphRels.cypher(query) - result.graph.cypher("MATCH ()-[r]->() RETURN type(r) as type").records.iterator.toBag should equal(Bag( - CypherMap("type" -> "HAS_SIMILAR_NAME") - )) + result.graph + .cypher("MATCH ()-[r]->() RETURN type(r) as type") + .records + .iterator + .toBag should equal( + Bag( + CypherMap("type" -> "HAS_SIMILAR_NAME") + ) + ) } it("should implicit clone a relationship #3") { @@ -1033,8 +1342,14 @@ class MultipleGraphTests extends MorpheusTestSuite with ScanGraphInit { | CREATE (x)-[r]->(y) |RETURN GRAPH""".stripMargin val result = testGraphRels.cypher(query) - result.graph.cypher("MATCH ()-[r]->() RETURN type(r) as type").records.iterator.toBag should equal(Bag( - CypherMap("type" -> "HAS_SIMILAR_NAME") - )) + result.graph + .cypher("MATCH ()-[r]->() RETURN type(r) as type") + .records + .iterator + .toBag should equal( + Bag( + CypherMap("type" -> "HAS_SIMILAR_NAME") + ) + ) } } diff --git a/morpheus-testing/src/test/scala/org/opencypher/morpheus/impl/acceptance/NullTests.scala b/morpheus-testing/src/test/scala/org/opencypher/morpheus/impl/acceptance/NullTests.scala index 44d658e498..57b4707759 100644 --- a/morpheus-testing/src/test/scala/org/opencypher/morpheus/impl/acceptance/NullTests.scala +++ b/morpheus-testing/src/test/scala/org/opencypher/morpheus/impl/acceptance/NullTests.scala @@ -40,10 +40,10 @@ class NullTests extends MorpheusTestSuite with ScanGraphInit with TestNameFixtur private def returnsValue(exp: Any, call: String = testName) = morpheus .cypher(s"RETURN $call AS res") - .records.toMaps + .records + .toMaps .shouldEqual(Bag(CypherMap("res" -> exp))) - describe("null input produces null") { it("calling: id(null)")(returnsNull()) it("calling: labels(null)")(returnsNull()) diff --git a/morpheus-testing/src/test/scala/org/opencypher/morpheus/impl/acceptance/OptionalMatchTests.scala b/morpheus-testing/src/test/scala/org/opencypher/morpheus/impl/acceptance/OptionalMatchTests.scala index b534d0d6d3..8498f87216 100644 --- a/morpheus-testing/src/test/scala/org/opencypher/morpheus/impl/acceptance/OptionalMatchTests.scala +++ b/morpheus-testing/src/test/scala/org/opencypher/morpheus/impl/acceptance/OptionalMatchTests.scala @@ -71,58 +71,61 @@ class OptionalMatchTests extends MorpheusTestSuite with ScanGraphInit { } it("supports stacked optional matches") { - val g = initGraph( - """ + val g = initGraph(""" |CREATE (:DoesExist {property: 42}) |CREATE (:DoesExist {property: 43}) |CREATE (:DoesExist {property: 44}) """.stripMargin) - val res = g.cypher( - """ + val res = g.cypher(""" |OPTIONAL MATCH (f:DoesExist) |OPTIONAL MATCH (n:DoesNotExist) |RETURN collect(DISTINCT n.property) AS a, collect(DISTINCT f.property) AS b """.stripMargin) - res.records.collect.toBag should equal(Bag( - CypherMap("a" -> List.empty, "b" -> List(42, 43, 44)) - )) + res.records.collect.toBag should equal( + Bag( + CypherMap("a" -> List.empty, "b" -> List(42, 43, 44)) + ) + ) } it("throws if spark.sql.crossJoin.enabled=false") { morpheus.sparkSession.conf.set("spark.sql.crossJoin.enabled", "false") - val e = the[org.opencypher.okapi.impl.exception.UnsupportedOperationException] thrownBy { + val e = the[ + org.opencypher.okapi.impl.exception.UnsupportedOperationException + ] thrownBy { try { - val g = initGraph( - """ + val g = initGraph(""" |CREATE (:DoesExist {property: 42}) |CREATE (:DoesExist {property: 43}) |CREATE (:DoesExist {property: 44}) """.stripMargin) - val res = g.cypher( - """ + val res = g.cypher(""" |OPTIONAL MATCH (f:DoesExist) |OPTIONAL MATCH (n:DoesNotExist) |RETURN collect(DISTINCT n.property) AS a, collect(DISTINCT f.property) AS b """.stripMargin) - res.records.collect.toBag should equal(Bag( - CypherMap("a" -> List.empty, "b" -> List(42, 43, 44)) - )) + res.records.collect.toBag should equal( + Bag( + CypherMap("a" -> List.empty, "b" -> List(42, 43, 44)) + ) + ) } finally { morpheus.sparkSession.conf.set("spark.sql.crossJoin.enabled", "true") } } - e.getMessage should (include("OPTIONAL MATCH") and include("spark.sql.crossJoin.enabled")) + e.getMessage should (include("OPTIONAL MATCH") and include( + "spark.sql.crossJoin.enabled" + )) } } it("optionally match") { // Given - val given = initGraph( - """ + val given = initGraph(""" |CREATE (p1:Person {name: "Alice"}) |CREATE (p2:Person {name: "Bob"}) |CREATE (p3:Person {name: "Eve"}) @@ -131,67 +134,67 @@ class OptionalMatchTests extends MorpheusTestSuite with ScanGraphInit { """.stripMargin) // When - val result = given.cypher( - """ + val result = given.cypher(""" |MATCH (p1:Person) |OPTIONAL MATCH (p1)-[e1]->(p2)-[e2]->(p3) |RETURN p1.name, p2.name, p3.name """.stripMargin) // Then - result.records.toMaps should equal(Bag( - CypherMap( - "p1.name" -> "Eve", - "p2.name" -> null, - "p3.name" -> null - ), - CypherMap( - "p1.name" -> "Bob", - "p2.name" -> null, - "p3.name" -> null - ), - CypherMap( - "p1.name" -> "Alice", - "p2.name" -> "Bob", - "p3.name" -> "Eve" + result.records.toMaps should equal( + Bag( + CypherMap( + "p1.name" -> "Eve", + "p2.name" -> null, + "p3.name" -> null + ), + CypherMap( + "p1.name" -> "Bob", + "p2.name" -> null, + "p3.name" -> null + ), + CypherMap( + "p1.name" -> "Alice", + "p2.name" -> "Bob", + "p3.name" -> "Eve" + ) ) - )) + ) } it("can optionally match with predicates") { // Given - val given = initGraph( - """ + val given = initGraph(""" |CREATE (p1:Person {name: "Alice"}) |CREATE (p2:Person {name: "Bob"}) |CREATE (p1)-[:KNOWS]->(p2) """.stripMargin) // When - val result = given.cypher( - """ + val result = given.cypher(""" |MATCH (p1:Person) |OPTIONAL MATCH (p1)-[e1:KNOWS]->(p2:Person) |RETURN p1.name, p2.name """.stripMargin) // Then - result.records.toMaps should equal(Bag( - CypherMap( - "p1.name" -> "Bob", - "p2.name" -> null - ), - CypherMap( - "p1.name" -> "Alice", - "p2.name" -> "Bob" + result.records.toMaps should equal( + Bag( + CypherMap( + "p1.name" -> "Bob", + "p2.name" -> null + ), + CypherMap( + "p1.name" -> "Alice", + "p2.name" -> "Bob" + ) ) - )) + ) } it("can optionally match already matched relationships") { // Given - val given = initGraph( - """ + val given = initGraph(""" |CREATE (p1:Person {name: "Alice"}) |CREATE (p2:Person {name: "Bob"}) |CREATE (p3:Person {name: "Eve"}) @@ -201,47 +204,47 @@ class OptionalMatchTests extends MorpheusTestSuite with ScanGraphInit { """.stripMargin) // When - val result = given.cypher( - """ + val result = given.cypher(""" |MATCH (p1:Person)-[e1:KNOWS]->(p2:Person) |OPTIONAL MATCH (p1)-[e2:KNOWS]->(p3:Person) |RETURN p1.name, p2.name, p3.name """.stripMargin) // Then - result.records.toMaps should equal(Bag( - CypherMap( - "p1.name" -> "Alice", - "p2.name" -> "Bob", - "p3.name" -> "Eve" - ), - CypherMap( - "p1.name" -> "Alice", - "p2.name" -> "Eve", - "p3.name" -> "Bob" - ), - CypherMap( - "p1.name" -> "Alice", - "p2.name" -> "Bob", - "p3.name" -> "Bob" - ), - CypherMap( - "p1.name" -> "Alice", - "p2.name" -> "Eve", - "p3.name" -> "Eve" - ), - CypherMap( - "p1.name" -> "Bob", - "p2.name" -> "Eve", - "p3.name" -> "Eve" + result.records.toMaps should equal( + Bag( + CypherMap( + "p1.name" -> "Alice", + "p2.name" -> "Bob", + "p3.name" -> "Eve" + ), + CypherMap( + "p1.name" -> "Alice", + "p2.name" -> "Eve", + "p3.name" -> "Bob" + ), + CypherMap( + "p1.name" -> "Alice", + "p2.name" -> "Bob", + "p3.name" -> "Bob" + ), + CypherMap( + "p1.name" -> "Alice", + "p2.name" -> "Eve", + "p3.name" -> "Eve" + ), + CypherMap( + "p1.name" -> "Bob", + "p2.name" -> "Eve", + "p3.name" -> "Eve" + ) ) - )) + ) } it("can optionally match incoming relationships") { // Given - val given = initGraph( - """ + val given = initGraph(""" |CREATE (p1:Person {name: "Alice"}) |CREATE (p2:Person {name: "Bob"}) |CREATE (p3:Person {name: "Frank"}) @@ -251,32 +254,32 @@ class OptionalMatchTests extends MorpheusTestSuite with ScanGraphInit { """.stripMargin) // When - val result = given.cypher( - """ + val result = given.cypher(""" |MATCH (p1:Person)-[e1:KNOWS]->(p2:Person) |OPTIONAL MATCH (p1)<-[e2:LOVES]-(p3:Person) |RETURN p1.name, p2.name, p3.name """.stripMargin) // Then - result.records.toMaps should equal(Bag( - CypherMap( - "p1.name" -> "Alice", - "p2.name" -> "Bob", - "p3.name" -> "Frank" - ), - CypherMap( - "p1.name" -> "Bob", - "p2.name" -> "Frank", - "p3.name" -> null + result.records.toMaps should equal( + Bag( + CypherMap( + "p1.name" -> "Alice", + "p2.name" -> "Bob", + "p3.name" -> "Frank" + ), + CypherMap( + "p1.name" -> "Bob", + "p2.name" -> "Frank", + "p3.name" -> null + ) ) - )) + ) } it("can optionally match with partial matches") { // Given - val given = initGraph( - """ + val given = initGraph(""" |CREATE (p1:Person {name: "Alice"}) |CREATE (p2:Person {name: "Bob"}) |CREATE (p3:Person {name: "Eve"}) @@ -285,37 +288,37 @@ class OptionalMatchTests extends MorpheusTestSuite with ScanGraphInit { """.stripMargin) // When - val result = given.cypher( - """ + val result = given.cypher(""" |MATCH (p1:Person) |OPTIONAL MATCH (p1)-[e1:KNOWS]->(p2:Person)-[e2:KNOWS]->(p3:Person) |RETURN p1.name, p2.name, p3.name """.stripMargin) // Then - result.records.toMaps should equal(Bag( - CypherMap( - "p1.name" -> "Alice", - "p2.name" -> "Bob", - "p3.name" -> "Eve" - ), - CypherMap( - "p1.name" -> "Bob", - "p2.name" -> null, - "p3.name" -> null - ), - CypherMap( - "p1.name" -> "Eve", - "p2.name" -> null, - "p3.name" -> null + result.records.toMaps should equal( + Bag( + CypherMap( + "p1.name" -> "Alice", + "p2.name" -> "Bob", + "p3.name" -> "Eve" + ), + CypherMap( + "p1.name" -> "Bob", + "p2.name" -> null, + "p3.name" -> null + ), + CypherMap( + "p1.name" -> "Eve", + "p2.name" -> null, + "p3.name" -> null + ) ) - )) + ) } it("can optionally match with duplicates") { // Given - val given = initGraph( - """ + val given = initGraph(""" |CREATE (p1:Person {name: "Alice"}) |CREATE (p2:Person {name: "Bob"}) |CREATE (p3:Person {name: "Eve"}) @@ -326,34 +329,34 @@ class OptionalMatchTests extends MorpheusTestSuite with ScanGraphInit { """.stripMargin) // When - val result = given.cypher( - """ + val result = given.cypher(""" |MATCH (a:Person)-[e1:KNOWS]->(b:Person) |OPTIONAL MATCH (b)-[e2:KNOWS]->(c:Person) |RETURN b.name, c.name """.stripMargin) // Then - result.records.toMaps should equal(Bag( - CypherMap( - "b.name" -> "Eve", - "c.name" -> "Paul" - ), - CypherMap( - "b.name" -> "Eve", - "c.name" -> "Paul" - ), - CypherMap( - "b.name" -> "Paul", - "c.name" -> null + result.records.toMaps should equal( + Bag( + CypherMap( + "b.name" -> "Eve", + "c.name" -> "Paul" + ), + CypherMap( + "b.name" -> "Eve", + "c.name" -> "Paul" + ), + CypherMap( + "b.name" -> "Paul", + "c.name" -> null + ) ) - )) + ) } it("can optionally match with duplicates and cycle") { // Given - val given = initGraph( - """ + val given = initGraph(""" |CREATE (p1:Person {name: "Alice"}) |CREATE (p2:Person {name: "Bob"}) |CREATE (p3:Person {name: "Eve"}) @@ -365,71 +368,70 @@ class OptionalMatchTests extends MorpheusTestSuite with ScanGraphInit { """.stripMargin) // When - val result = given.cypher( - """ + val result = given.cypher(""" |MATCH (a:Person)-[e1:KNOWS]->(b:Person)-[e2:KNOWS]->(c:Person) |OPTIONAL MATCH (c)-[e3:KNOWS]->(a) |RETURN a.name, b.name, c.name, e3.foo """.stripMargin) // Then - result.records.toMaps should equal(Bag( - CypherMap( - "a.name" -> "Alice", - "b.name" -> "Eve", - "c.name" -> "Paul", - "e3.foo" -> 42 - ), - CypherMap( - "a.name" -> "Eve", - "b.name" -> "Paul", - "c.name" -> "Alice", - "e3.foo" -> null - ), - CypherMap( - "a.name" -> "Paul", - "b.name" -> "Alice", - "c.name" -> "Eve", - "e3.foo" -> null - ), - CypherMap( - "a.name" -> "Bob", - "b.name" -> "Eve", - "c.name" -> "Paul", - "e3.foo" -> null + result.records.toMaps should equal( + Bag( + CypherMap( + "a.name" -> "Alice", + "b.name" -> "Eve", + "c.name" -> "Paul", + "e3.foo" -> 42 + ), + CypherMap( + "a.name" -> "Eve", + "b.name" -> "Paul", + "c.name" -> "Alice", + "e3.foo" -> null + ), + CypherMap( + "a.name" -> "Paul", + "b.name" -> "Alice", + "c.name" -> "Eve", + "e3.foo" -> null + ), + CypherMap( + "a.name" -> "Bob", + "b.name" -> "Eve", + "c.name" -> "Paul", + "e3.foo" -> null + ) ) - )) + ) } it("can match multiple optional matches") { - val graph = initGraph( - """ + val graph = initGraph(""" |CREATE (s {val: 1}) """.stripMargin) - val result = graph.cypher( - """ + val result = graph.cypher(""" |MATCH (a) |OPTIONAL MATCH (a)-->(b:NonExistent) |OPTIONAL MATCH (a)-->(c:NonExistent) |RETURN b,c """.stripMargin) - result.records.collect.toBag should equal(Bag( - CypherMap("b" -> CypherNull, "c" -> CypherNull) - )) + result.records.collect.toBag should equal( + Bag( + CypherMap("b" -> CypherNull, "c" -> CypherNull) + ) + ) } it("can start with an optional match") { - val g = initGraph( - """ + val g = initGraph(""" |CREATE (p1:Person {name: "Alice"}) |CREATE (p2:Person {name: "Bob"}) """.stripMargin) // When - val result = g.cypher( - """ + val result = g.cypher(""" |OPTIONAL MATCH (a:Foo) |WITH a |MATCH (b:Person) @@ -437,34 +439,42 @@ class OptionalMatchTests extends MorpheusTestSuite with ScanGraphInit { """.stripMargin) // Then - result.records.collect.toBag should equal(Bag( - CypherMap("a" -> CypherNull, "b" -> MorpheusNode(0L, Set("Person"), CypherMap("name" -> "Alice"))), - CypherMap("a" -> CypherNull, "b" -> MorpheusNode(1L, Set("Person"), CypherMap("name" -> "Bob"))) - )) + result.records.collect.toBag should equal( + Bag( + CypherMap( + "a" -> CypherNull, + "b" -> MorpheusNode(0L, Set("Person"), CypherMap("name" -> "Alice")) + ), + CypherMap( + "a" -> CypherNull, + "b" -> MorpheusNode(1L, Set("Person"), CypherMap("name" -> "Bob")) + ) + ) + ) } it("returns null IDs") { // Given - val given = initGraph( - """ + val given = initGraph(""" |CREATE (p1:Person {name: "Alice"}) """.stripMargin) // When - val result = given.cypher( - """ + val result = given.cypher(""" |MATCH (p1:Person) |OPTIONAL MATCH (p1)-[e1]->(p2) |RETURN id(p1), id(p2) """.stripMargin) // Then - result.records.toMaps should equal(Bag( - CypherMap( - "id(p1)" -> List(0), - "id(p2)" -> null + result.records.toMaps should equal( + Bag( + CypherMap( + "id(p1)" -> List(0), + "id(p2)" -> null + ) ) - )) + ) } } diff --git a/morpheus-testing/src/test/scala/org/opencypher/morpheus/impl/acceptance/PatternScanTests.scala b/morpheus-testing/src/test/scala/org/opencypher/morpheus/impl/acceptance/PatternScanTests.scala index 932c186b92..d38b9b9d18 100644 --- a/morpheus-testing/src/test/scala/org/opencypher/morpheus/impl/acceptance/PatternScanTests.scala +++ b/morpheus-testing/src/test/scala/org/opencypher/morpheus/impl/acceptance/PatternScanTests.scala @@ -47,20 +47,25 @@ class PatternScanTests extends MorpheusTestSuite with ScanGraphInit { Seq(pattern) ) - val res = g.cypher( - """ + val res = g.cypher(""" |MATCH (a:Person)-[:KNOWS]->(b:Person) |RETURN a.name, b.name """.stripMargin) - res.records.toMaps should equal(Bag( - CypherMap("a.name" -> "Alice", "b.name" -> "Bob") - )) + res.records.toMaps should equal( + Bag( + CypherMap("a.name" -> "Alice", "b.name" -> "Bob") + ) + ) } it("can combine multiple pattern scans") { val pattern1 = NodeRelPattern(CTNode("Person"), CTRelationship("KNOWS")) - val pattern2 = TripletPattern(CTNode("Person"), CTRelationship("LOVES"), CTNode("Person")) + val pattern2 = TripletPattern( + CTNode("Person"), + CTRelationship("LOVES"), + CTNode("Person") + ) val g = initGraph( """ @@ -74,21 +79,23 @@ class PatternScanTests extends MorpheusTestSuite with ScanGraphInit { Seq(pattern1, pattern2) ) - val res = g.cypher( - """ + val res = g.cypher(""" |MATCH (a:Person)-[:KNOWS]->(b:Person), | (a)-[:LOVES]->(c:Person) |RETURN a.name, b.name, c.name """.stripMargin) - res.records.toMaps should equal(Bag( - CypherMap("a.name" -> "Alice", "b.name" -> "Bob", "c.name" -> "Carol") - )) + res.records.toMaps should equal( + Bag( + CypherMap("a.name" -> "Alice", "b.name" -> "Bob", "c.name" -> "Carol") + ) + ) } it("combines multiple pattern scans") { val pattern1 = NodeRelPattern(CTNode("Person"), CTRelationship("KNOWS")) - val pattern2 = NodeRelPattern(CTNode("Person", "Employee"), CTRelationship("KNOWS")) + val pattern2 = + NodeRelPattern(CTNode("Person", "Employee"), CTRelationship("KNOWS")) val g = initGraph( """ @@ -102,16 +109,17 @@ class PatternScanTests extends MorpheusTestSuite with ScanGraphInit { Seq(pattern1, pattern2) ) - val res = g.cypher( - """ + val res = g.cypher(""" |MATCH (a:Person)-[:KNOWS]->(b:Person) |RETURN a.name, b.name """.stripMargin) - res.records.toMaps should equal(Bag( - CypherMap("a.name" -> "Alice", "b.name" -> "Bob"), - CypherMap("a.name" -> "Garfield", "b.name" -> "Bob") - )) + res.records.toMaps should equal( + Bag( + CypherMap("a.name" -> "Alice", "b.name" -> "Bob"), + CypherMap("a.name" -> "Garfield", "b.name" -> "Bob") + ) + ) } it("works if node rel scans do not cover all node label combinations") { @@ -129,16 +137,17 @@ class PatternScanTests extends MorpheusTestSuite with ScanGraphInit { Seq(pattern1) ) - val res = g.cypher( - """ + val res = g.cypher(""" |MATCH (a:Person)-[:KNOWS]->(b:Person) |RETURN a.name, b.name """.stripMargin) - res.records.toMaps should equal(Bag( - CypherMap("a.name" -> "Alice", "b.name" -> "Bob"), - CypherMap("a.name" -> "Garfield", "b.name" -> "Bob") - )) + res.records.toMaps should equal( + Bag( + CypherMap("a.name" -> "Alice", "b.name" -> "Bob"), + CypherMap("a.name" -> "Garfield", "b.name" -> "Bob") + ) + ) } it("works if node rel scans do not cover all rel type combinations") { @@ -155,20 +164,25 @@ class PatternScanTests extends MorpheusTestSuite with ScanGraphInit { Seq(pattern1) ) - val res = g.cypher( - """ + val res = g.cypher(""" |MATCH (a:Person)-[:KNOWS|LOVES]->(b:Person) |RETURN a.name, b.name """.stripMargin) - res.records.toMaps should equal(Bag( - CypherMap("a.name" -> "Alice", "b.name" -> "Bob"), - CypherMap("a.name" -> "Alice", "b.name" -> "Bob") - )) + res.records.toMaps should equal( + Bag( + CypherMap("a.name" -> "Alice", "b.name" -> "Bob"), + CypherMap("a.name" -> "Alice", "b.name" -> "Bob") + ) + ) } it("works if triplet scans do not cover all source node labels") { - val pattern = TripletPattern(CTNode("Person"), CTRelationship("KNOWS"), CTNode("Person")) + val pattern = TripletPattern( + CTNode("Person"), + CTRelationship("KNOWS"), + CTNode("Person") + ) val g = initGraph( """ @@ -182,20 +196,25 @@ class PatternScanTests extends MorpheusTestSuite with ScanGraphInit { Seq(pattern) ) - val res = g.cypher( - """ + val res = g.cypher(""" |MATCH (a)-[:KNOWS]->(b:Person) |RETURN a.name, b.name """.stripMargin) - res.records.toMaps should equal(Bag( - CypherMap("a.name" -> "Alice", "b.name" -> "Bob"), - CypherMap("a.name" -> "Garfield", "b.name" -> "Bob") - )) + res.records.toMaps should equal( + Bag( + CypherMap("a.name" -> "Alice", "b.name" -> "Bob"), + CypherMap("a.name" -> "Garfield", "b.name" -> "Bob") + ) + ) } it("works if triplet scans do not cover all target node labels") { - val pattern = TripletPattern(CTNode("Person"), CTRelationship("KNOWS"), CTNode("Person")) + val pattern = TripletPattern( + CTNode("Person"), + CTRelationship("KNOWS"), + CTNode("Person") + ) val g = initGraph( """ @@ -209,20 +228,25 @@ class PatternScanTests extends MorpheusTestSuite with ScanGraphInit { Seq(pattern) ) - val res = g.cypher( - """ + val res = g.cypher(""" |MATCH (a)-[:KNOWS]->(b) |RETURN a.name, b.name """.stripMargin) - res.records.toMaps should equal(Bag( - CypherMap("a.name" -> "Alice", "b.name" -> "Bob"), - CypherMap("a.name" -> "Alice", "b.name" -> "Garfield") - )) + res.records.toMaps should equal( + Bag( + CypherMap("a.name" -> "Alice", "b.name" -> "Bob"), + CypherMap("a.name" -> "Alice", "b.name" -> "Garfield") + ) + ) } it("works if they do not cover all rel types") { - val pattern = TripletPattern(CTNode("Person"), CTRelationship("KNOWS"), CTNode("Person")) + val pattern = TripletPattern( + CTNode("Person"), + CTRelationship("KNOWS"), + CTNode("Person") + ) val g = initGraph( """ @@ -235,15 +259,16 @@ class PatternScanTests extends MorpheusTestSuite with ScanGraphInit { Seq(pattern) ) - val res = g.cypher( - """ + val res = g.cypher(""" |MATCH (a)-[:KNOWS|LOVES]->(b) |RETURN a.name, b.name """.stripMargin) - res.records.toMaps should equal(Bag( - CypherMap("a.name" -> "Alice", "b.name" -> "Bob"), - CypherMap("a.name" -> "Alice", "b.name" -> "Bob") - )) + res.records.toMaps should equal( + Bag( + CypherMap("a.name" -> "Alice", "b.name" -> "Bob"), + CypherMap("a.name" -> "Alice", "b.name" -> "Bob") + ) + ) } } diff --git a/morpheus-testing/src/test/scala/org/opencypher/morpheus/impl/acceptance/PredicateTests.scala b/morpheus-testing/src/test/scala/org/opencypher/morpheus/impl/acceptance/PredicateTests.scala index 85201b5e1b..d12907980d 100644 --- a/morpheus-testing/src/test/scala/org/opencypher/morpheus/impl/acceptance/PredicateTests.scala +++ b/morpheus-testing/src/test/scala/org/opencypher/morpheus/impl/acceptance/PredicateTests.scala @@ -47,85 +47,107 @@ class PredicateTests extends MorpheusTestSuite with ScanGraphInit { val result = given.cypher("MATCH (n) WHERE exists(n.id) RETURN n.id") - result.records.toMaps should equal(Bag( - CypherMap("n.id" -> 1), - CypherMap("n.id" -> 2) - )) + result.records.toMaps should equal( + Bag( + CypherMap("n.id" -> 1), + CypherMap("n.id" -> 2) + ) + ) } it("in") { // Given - val given = initGraph("""CREATE (:A {val: 1}), (:A {val: 2}), (:A {val: 3})""") + val given = + initGraph("""CREATE (:A {val: 1}), (:A {val: 2}), (:A {val: 3})""") // When - val result = given.cypher("MATCH (a:A) WHERE a.val IN [-1, 2, 5, 0] RETURN a.val") + val result = + given.cypher("MATCH (a:A) WHERE a.val IN [-1, 2, 5, 0] RETURN a.val") // Then - result.records.toMaps should equal(Bag( - CypherMap("a.val" -> 2) - )) + result.records.toMaps should equal( + Bag( + CypherMap("a.val" -> 2) + ) + ) } it("in with parameter") { // Given - val given = initGraph("""CREATE (:A {val: 1}), (:A {val: 2}), (:A {val: 3})""") + val given = + initGraph("""CREATE (:A {val: 1}), (:A {val: 2}), (:A {val: 3})""") // When - val result = given.cypher("MATCH (a:A) WHERE a.val IN $list RETURN a.val", Map("list" -> CypherList(-1, 2, 5, 0))) + val result = given.cypher( + "MATCH (a:A) WHERE a.val IN $list RETURN a.val", + Map("list" -> CypherList(-1, 2, 5, 0)) + ) // Then - result.records.toMaps should equal(Bag( - CypherMap("a.val" -> 2) - )) + result.records.toMaps should equal( + Bag( + CypherMap("a.val" -> 2) + ) + ) } it("evaluates or") { // Given - val given = initGraph("""CREATE (:A {val: 1}), (:A {val: 2}), (:A {val: 3})""") + val given = + initGraph("""CREATE (:A {val: 1}), (:A {val: 2}), (:A {val: 3})""") // When - val result = given.cypher("MATCH (a:A) WHERE a.val = 1 OR a.val = 2 RETURN a.val") + val result = + given.cypher("MATCH (a:A) WHERE a.val = 1 OR a.val = 2 RETURN a.val") // Then - result.records.toMaps should equal(Bag( - CypherMap("a.val" -> 1), - CypherMap("a.val" -> 2) - )) + result.records.toMaps should equal( + Bag( + CypherMap("a.val" -> 1), + CypherMap("a.val" -> 2) + ) + ) } it("or on labels") { // Given - val given = initGraph("""CREATE (:A {val: 1}), (:B {val: 2}), (:C {val: 3})""") + val given = + initGraph("""CREATE (:A {val: 1}), (:B {val: 2}), (:C {val: 3})""") // When val result = given.cypher("MATCH (a) WHERE a:A OR a:B RETURN a.val") // Then - result.records.toMaps should equal(Bag( - CypherMap("a.val" -> 1), - CypherMap("a.val" -> 2) - )) + result.records.toMaps should equal( + Bag( + CypherMap("a.val" -> 1), + CypherMap("a.val" -> 2) + ) + ) } it("or on labels and properties") { // Given - val given = initGraph("""CREATE (:A {val: 1}), (:B {val: 2}), (:A:B {val: 3})""") + val given = + initGraph("""CREATE (:A {val: 1}), (:B {val: 2}), (:A:B {val: 3})""") // When - val result = given.cypher("MATCH (a) WHERE (a:A AND a.val = 1) OR (a:B) RETURN a.val") + val result = + given.cypher("MATCH (a) WHERE (a:A AND a.val = 1) OR (a:B) RETURN a.val") // Then - result.records.toMaps should equal(Bag( - CypherMap("a.val" -> 1), - CypherMap("a.val" -> 2), - CypherMap("a.val" -> 3) - )) + result.records.toMaps should equal( + Bag( + CypherMap("a.val" -> 1), + CypherMap("a.val" -> 2), + CypherMap("a.val" -> 3) + ) + ) } it("or with and") { // Given - val given = initGraph( - """CREATE (:A {val: 1, name: 'a'}) + val given = initGraph("""CREATE (:A {val: 1, name: 'a'}) |CREATE (:A {val: 2, name: 'a'}) |CREATE (:A {val: 3, name: 'e'}) |CREATE (:A {val: 4}) @@ -133,19 +155,22 @@ class PredicateTests extends MorpheusTestSuite with ScanGraphInit { """.stripMargin) // When - val result = given.cypher("MATCH (a:A) WHERE a.val = 1 OR (a.val >= 4 AND a.name = 'e') RETURN a.val, a.name") + val result = given.cypher( + "MATCH (a:A) WHERE a.val = 1 OR (a.val >= 4 AND a.name = 'e') RETURN a.val, a.name" + ) // Then - result.records.toMaps should equal(Bag( - CypherMap("a.val" -> 1, "a.name" -> "a"), - CypherMap("a.val" -> 5, "a.name" -> "e") - )) + result.records.toMaps should equal( + Bag( + CypherMap("a.val" -> 1, "a.name" -> "a"), + CypherMap("a.val" -> 5, "a.name" -> "e") + ) + ) } it("equality between properties") { // Given - val given = initGraph( - """|CREATE (:A {val: 1})-[:REL]->(:B {p: 2}) + val given = initGraph("""|CREATE (:A {val: 1})-[:REL]->(:B {p: 2}) |CREATE (:A {val: 2})-[:REL]->(:B {p: 1}) |CREATE (:A {val: 100})-[:REL]->(:B {p: 100}) |CREATE (:A {val: 1})-[:REL]->(:B) @@ -154,53 +179,65 @@ class PredicateTests extends MorpheusTestSuite with ScanGraphInit { """.stripMargin) // When - val result = given.cypher("MATCH (a:A)-->(b:B) WHERE a.val = b.p RETURN b.p") + val result = + given.cypher("MATCH (a:A)-->(b:B) WHERE a.val = b.p RETURN b.p") // Then - result.records.toMaps should equal(Bag( - CypherMap("b.p" -> 100) - )) + result.records.toMaps should equal( + Bag( + CypherMap("b.p" -> 100) + ) + ) } describe("comparison operators") { it("less than") { // Given - val given = initGraph("""CREATE (:Node {val: 4})-[:REL]->(:Node {val: 5})""") + val given = + initGraph("""CREATE (:Node {val: 4})-[:REL]->(:Node {val: 5})""") // When - val result = given.cypher("MATCH (n:Node)-->(m:Node) WHERE n.val < m.val RETURN n.val") + val result = given.cypher( + "MATCH (n:Node)-->(m:Node) WHERE n.val < m.val RETURN n.val" + ) // Then - result.records.toMaps should equal(Bag( - CypherMap("n.val" -> 4) - )) + result.records.toMaps should equal( + Bag( + CypherMap("n.val" -> 4) + ) + ) } it("compares less than between different types") { // Given - val given = initGraph( - """CREATE (:A {val: 4})-[:REL]->(:B {val2: 1.0}), + val given = initGraph("""CREATE (:A {val: 4})-[:REL]->(:B {val2: 1.0}), | (:A {val: 1})-[:REL]->(:B {val2: 4.0}) """.stripMargin) // When - val result = given.cypher("MATCH (a:A)-->(b:B) WHERE a.val < b.val2 RETURN a.val") + val result = + given.cypher("MATCH (a:A)-->(b:B) WHERE a.val < b.val2 RETURN a.val") // Then - result.records.collect.toBag should equal(Bag( - CypherMap("a.val" -> 1) - )) + result.records.collect.toBag should equal( + Bag( + CypherMap("a.val" -> 1) + ) + ) } it("fails when comparing less than between incompatible types") { // Given - val given = initGraph("""CREATE (:A {val: 4})-[:REL]->(:B {val2: 'string'})""") + val given = + initGraph("""CREATE (:A {val: 4})-[:REL]->(:B {val2: 'string'})""") // Where - val result = given.cypher("MATCH (a:A)-->(b:B) WHERE a.val < b.val2 RETURN a.val") + val result = + given.cypher("MATCH (a:A)-->(b:B) WHERE a.val < b.val2 RETURN a.val") // Then result.records.collect.toBag shouldBe empty @@ -208,47 +245,53 @@ class PredicateTests extends MorpheusTestSuite with ScanGraphInit { it("less than or equal") { // Given - val given = initGraph( - """ + val given = initGraph(""" |CREATE (:Node {id: 1, val: 4})-[:REL]->(:Node {id: 2, val: 5})-[:REL]->(:Node {id: 3, val: 5}) |""".stripMargin) // When - val - result = given.cypher("MATCH (n:Node)-->(m:Node) WHERE n.val <= m.val RETURN n.id, n.val") + val result = given.cypher( + "MATCH (n:Node)-->(m:Node) WHERE n.val <= m.val RETURN n.id, n.val" + ) // Then - result.records.toMaps should equal(Bag( - CypherMap("n.id" -> 1, "n.val" -> 4), - CypherMap("n.id" -> 2, "n.val" -> 5) - )) + result.records.toMaps should equal( + Bag( + CypherMap("n.id" -> 1, "n.val" -> 4), + CypherMap("n.id" -> 2, "n.val" -> 5) + ) + ) } it("compares less than or equal between different types") { // Given - val given = initGraph( - """CREATE (:A {val: 4})-[:REL]->(:B {val2: 4.0}), + val given = initGraph("""CREATE (:A {val: 4})-[:REL]->(:B {val2: 4.0}), | (:A {val: 1})-[:REL]->(:B {val2: 4.0}) """.stripMargin) // When - val result = given.cypher("MATCH (a:A)-->(b:B) WHERE a.val <= b.val2 RETURN a.val") + val result = + given.cypher("MATCH (a:A)-->(b:B) WHERE a.val <= b.val2 RETURN a.val") // Then - result.records.collect.toBag should equal(Bag( - CypherMap("a.val" -> 4), - CypherMap("a.val" -> 1) - )) + result.records.collect.toBag should equal( + Bag( + CypherMap("a.val" -> 4), + CypherMap("a.val" -> 1) + ) + ) } it("fails when comparing less than or equal between incompatible types") { // Given - val given = initGraph("""CREATE (:A {val: 4})-[:REL]->(:B {val2: 'string'})""") + val given = + initGraph("""CREATE (:A {val: 4})-[:REL]->(:B {val2: 'string'})""") // Where - val result = given.cypher("MATCH (a:A)-->(b:B) WHERE a.val <= b.val2 RETURN a.val") + val result = + given.cypher("MATCH (a:A)-->(b:B) WHERE a.val <= b.val2 RETURN a.val") // Then result.records.collect.toBag shouldBe empty @@ -256,41 +299,50 @@ class PredicateTests extends MorpheusTestSuite with ScanGraphInit { it("greater than") { // Given - val given = initGraph("""CREATE (:Node {val: 4})-[:REL]->(:Node {val: 5})""") + val given = + initGraph("""CREATE (:Node {val: 4})-[:REL]->(:Node {val: 5})""") // When - val result = given.cypher("MATCH (n:Node)<--(m:Node) WHERE n.val > m.val RETURN n.val") + val result = given.cypher( + "MATCH (n:Node)<--(m:Node) WHERE n.val > m.val RETURN n.val" + ) // Then - result.records.toMaps should equal(Bag( - CypherMap("n.val" -> 5) - )) + result.records.toMaps should equal( + Bag( + CypherMap("n.val" -> 5) + ) + ) } it("compares greater than between different types") { // Given - val given = initGraph( - """CREATE (:A {val: 4})-[:REL]->(:B {val2: 1.0}), + val given = initGraph("""CREATE (:A {val: 4})-[:REL]->(:B {val2: 1.0}), | (:A {val: 1})-[:REL]->(:B {val2: 4.0}) """.stripMargin) // When - val result = given.cypher("MATCH (a:A)-->(b:B) WHERE a.val > b.val2 RETURN a.val") + val result = + given.cypher("MATCH (a:A)-->(b:B) WHERE a.val > b.val2 RETURN a.val") // Then - result.records.collect.toBag should equal(Bag( - CypherMap("a.val" -> 4) - )) + result.records.collect.toBag should equal( + Bag( + CypherMap("a.val" -> 4) + ) + ) } it("fails when comparing greater than between incompatible types") { // Given - val given = initGraph("""CREATE (:A {val: 4})-[:REL]->(:B {val2: 'string'})""") + val given = + initGraph("""CREATE (:A {val: 4})-[:REL]->(:B {val2: 'string'})""") // Where - val result = given.cypher("MATCH (a:A)-->(b:B) WHERE a.val > b.val2 RETURN a.val") + val result = + given.cypher("MATCH (a:A)-->(b:B) WHERE a.val > b.val2 RETURN a.val") // Then result.records.collect.toBag shouldBe empty @@ -298,43 +350,53 @@ class PredicateTests extends MorpheusTestSuite with ScanGraphInit { it("greater than or equal") { // Given - val given = initGraph("""CREATE (:Node {id: 1, val: 4})-[:REL]->(:Node {id: 2, val: 5})-[:REL]->(:Node {id: 3, val: 5})""") + val given = initGraph( + """CREATE (:Node {id: 1, val: 4})-[:REL]->(:Node {id: 2, val: 5})-[:REL]->(:Node {id: 3, val: 5})""" + ) // When - val result = given.cypher("MATCH (n:Node)<--(m:Node) WHERE n.val >= m.val RETURN n.id, n.val") + val result = given.cypher( + "MATCH (n:Node)<--(m:Node) WHERE n.val >= m.val RETURN n.id, n.val" + ) // Then - result.records.toMaps should equal(Bag( - CypherMap("n.id" -> 2, "n.val" -> 5), - CypherMap("n.id" -> 3, "n.val" -> 5) - )) + result.records.toMaps should equal( + Bag( + CypherMap("n.id" -> 2, "n.val" -> 5), + CypherMap("n.id" -> 3, "n.val" -> 5) + ) + ) } it("compares greater than or equal between different types") { // Given - val given = initGraph( - """CREATE (:A {val: 4})-[:REL]->(:B {val2: 1.0}), + val given = initGraph("""CREATE (:A {val: 4})-[:REL]->(:B {val2: 1.0}), | (:A {val: 4})-[:REL]->(:B {val2: 4.0}) """.stripMargin) // When - val result = given.cypher("MATCH (a:A)-->(b:B) WHERE a.val >= b.val2 RETURN a.val") + val result = + given.cypher("MATCH (a:A)-->(b:B) WHERE a.val >= b.val2 RETURN a.val") // Then - result.records.collect.toBag should equal(Bag( - CypherMap("a.val" -> 4), - CypherMap("a.val" -> 4) - )) + result.records.collect.toBag should equal( + Bag( + CypherMap("a.val" -> 4), + CypherMap("a.val" -> 4) + ) + ) } it("fails when comparing greater than or equal between different types") { // Given - val given = initGraph("""CREATE (:A {val: 4})-[:REL]->(:B {val2: 'string'})""") + val given = + initGraph("""CREATE (:A {val: 4})-[:REL]->(:B {val2: 'string'})""") // Where - val result = given.cypher("MATCH (a:A)-->(b:B) WHERE a.val >= b.val2 RETURN a.val") + val result = + given.cypher("MATCH (a:A)-->(b:B) WHERE a.val >= b.val2 RETURN a.val") // Then result.records.collect.toBag shouldBe empty @@ -351,175 +413,207 @@ class PredicateTests extends MorpheusTestSuite with ScanGraphInit { |RETURN a.val """.stripMargin - graph.cypher(query).records.collect.toBag should equal(Bag( - CypherMap("a.val" -> 10) - )) + graph.cypher(query).records.collect.toBag should equal( + Bag( + CypherMap("a.val" -> 10) + ) + ) } it("float conversion for integer division") { // Given - val given = initGraph("""CREATE (:Node {id: 1, val: 4}), (:Node {id: 2, val: 5}), (:Node {id: 3, val: 5})""") + val given = initGraph( + """CREATE (:Node {id: 1, val: 4}), (:Node {id: 2, val: 5}), (:Node {id: 3, val: 5})""" + ) // When - val result = given.cypher("MATCH (n:Node) WHERE (n.val * 1.0) / n.id >= 2.5 RETURN n.id") + val result = given.cypher( + "MATCH (n:Node) WHERE (n.val * 1.0) / n.id >= 2.5 RETURN n.id" + ) // Then - result.records.toMaps should equal(Bag( - CypherMap("n.id" -> 1), - CypherMap("n.id" -> 2) - )) + result.records.toMaps should equal( + Bag( + CypherMap("n.id" -> 1), + CypherMap("n.id" -> 2) + ) + ) } it("basic pattern predicate") { // Given - val given = initGraph( - """ + val given = initGraph(""" |CREATE (v {id: 1})-[:REL]->({id: 2})-[:REL]->(w {id: 3}) |CREATE (v)-[:REL]->(w) |CREATE (w)-[:REL]->({id: 4}) """.stripMargin) // When - val result = given.cypher("MATCH (a)-->(b) WHERE (a)-->()-->(b) RETURN a.id, b.id") + val result = + given.cypher("MATCH (a)-->(b) WHERE (a)-->()-->(b) RETURN a.id, b.id") // Then - result.records.toMaps should equal(Bag( - CypherMap("a.id" -> 1L, "b.id" -> 3L) - )) + result.records.toMaps should equal( + Bag( + CypherMap("a.id" -> 1L, "b.id" -> 3L) + ) + ) } it("pattern predicate with var-length-expand") { // Given - val given = initGraph("CREATE (v {id: 1})-[:REL]->({id: 2})-[:REL]->({id: 3})<-[:REL]-(v)") + val given = initGraph( + "CREATE (v {id: 1})-[:REL]->({id: 2})-[:REL]->({id: 3})<-[:REL]-(v)" + ) // When - val result = given.cypher("MATCH (a)-->(b) WHERE (a)-[*1..3]->()-->(b) RETURN a.id, b.id") + val result = given.cypher( + "MATCH (a)-->(b) WHERE (a)-[*1..3]->()-->(b) RETURN a.id, b.id" + ) // Then - result.records.toMaps should equal(Bag( - CypherMap("a.id" -> 1L, "b.id" -> 3L) - )) + result.records.toMaps should equal( + Bag( + CypherMap("a.id" -> 1L, "b.id" -> 3L) + ) + ) } it("simple pattern predicate with node predicate") { // Given - val given = initGraph( - """ + val given = initGraph(""" |CREATE ({id: 1})-[:REL]->({name: 'foo'}) |CREATE ({id: 3})-[:REL]->({name: 'bar'}) """.stripMargin) // When - val result = given.cypher("MATCH (a) WHERE (a)-->({name: 'foo'}) RETURN a.id") + val result = + given.cypher("MATCH (a) WHERE (a)-->({name: 'foo'}) RETURN a.id") // Then - result.records.toMaps should equal(Bag( - CypherMap("a.id" -> 1L) - )) + result.records.toMaps should equal( + Bag( + CypherMap("a.id" -> 1L) + ) + ) } it("simple pattern predicate with relationship predicate") { // Given - val given = initGraph( - """ + val given = initGraph(""" |CREATE (v {id: 1})-[:REL {val: 'foo'}]->()-[:REL]->({id: 2})<-[:REL]-(v) |CREATE (w {id: 3})-[:REL {val: 'bar'}]->()-[:REL]->({id: 4})<-[:REL]-(w) """.stripMargin) // When - val result = given.cypher("MATCH (a)-->(b) WHERE (a)-[{val: 'foo'}]->()-->(b) RETURN a.id, b.id") + val result = given.cypher( + "MATCH (a)-->(b) WHERE (a)-[{val: 'foo'}]->()-->(b) RETURN a.id, b.id" + ) // Then - result.records.toMaps should equal(Bag( - CypherMap("a.id" -> 1L, "b.id" -> 2L) - )) + result.records.toMaps should equal( + Bag( + CypherMap("a.id" -> 1L, "b.id" -> 2L) + ) + ) } it("simple pattern predicate with node label predicate") { // Given - val given = initGraph( - """ + val given = initGraph(""" |CREATE (v{id: 1})-[:REL {val: 'foo'}]->(:A)-[:REL]->({id: 2})<-[:REL]-(v) |CREATE (w{id: 3})-[:REL {val: 'bar'}]->(:B)-[:REL]->({id: 4})<-[:REL]-(w) """.stripMargin) // When - val result = given.cypher("MATCH (a)-->(b) WHERE (a)-->(:A)-->(b) RETURN a.id, b.id") + val result = + given.cypher("MATCH (a)-->(b) WHERE (a)-->(:A)-->(b) RETURN a.id, b.id") // Then - result.records.toMaps should equal(Bag( - CypherMap("a.id" -> 1L, "b.id" -> 2L) - )) + result.records.toMaps should equal( + Bag( + CypherMap("a.id" -> 1L, "b.id" -> 2L) + ) + ) } it("simple pattern predicate with relationship type predicate") { // Given - val given = initGraph( - """ + val given = initGraph(""" |CREATE (v {id: 1})-[:A]->()-[:REL]->({id: 2})<-[:REL]-(v) |CREATE (w {id: 3})-[:B]->()-[:REL]->({id: 4})<-[:REL]-(w) """.stripMargin) // When - val result = given.cypher("MATCH (a)-->(b) WHERE (a)-[:A]->()-->(b) RETURN a.id, b.id") + val result = + given.cypher("MATCH (a)-->(b) WHERE (a)-[:A]->()-->(b) RETURN a.id, b.id") // Then - result.records.toMaps should equal(Bag( - CypherMap("a.id" -> 1L, "b.id" -> 2L) - )) + result.records.toMaps should equal( + Bag( + CypherMap("a.id" -> 1L, "b.id" -> 2L) + ) + ) } it("inverse pattern predicate") { // Given - val given = initGraph("CREATE (v {id: 1})-[:REL]->({id: 2})-[:REL]->({id: 3})<-[:REL]-(v)") + val given = initGraph( + "CREATE (v {id: 1})-[:REL]->({id: 2})-[:REL]->({id: 3})<-[:REL]-(v)" + ) // When - val result = given.cypher("MATCH (a)-->(b) WHERE NOT (a)-->()-->(b) RETURN a.id, b.id") + val result = + given.cypher("MATCH (a)-->(b) WHERE NOT (a)-->()-->(b) RETURN a.id, b.id") // Then - result.records.toMaps should equal(Bag( - CypherMap("a.id" -> 1L, "b.id" -> 2L), - CypherMap("a.id" -> 2L, "b.id" -> 3L) - )) + result.records.toMaps should equal( + Bag( + CypherMap("a.id" -> 1L, "b.id" -> 2L), + CypherMap("a.id" -> 2L, "b.id" -> 3L) + ) + ) } it("nested pattern predicate") { - val given = initGraph( - """ + val given = initGraph(""" |CREATE ({id: 1, age: 21}) |CREATE ({id: 2, age: 18, foo: true}) |CREATE ({id: 3, age: 18, foo: true})-[:KNOWS]->(:Foo) |CREATE ({id: 4, age: 18, foo: false})-[:KNOWS]->(:Foo) """.stripMargin) - val result = given.cypher( - """ + val result = given.cypher(""" |MATCH (a) |WHERE a.age > 20 OR ( (a)-[:KNOWS]->(:Foo) AND a.foo = true ) |RETURN a.id """.stripMargin) - result.records.toMaps should equal(Bag( - CypherMap("a.id" -> 1), - CypherMap("a.id" -> 3) - )) + result.records.toMaps should equal( + Bag( + CypherMap("a.id" -> 1), + CypherMap("a.id" -> 3) + ) + ) } it("pattern predicate with derived node predicate") { // Given - val given = initGraph( - """ + val given = initGraph(""" |CREATE ({id: 1, val: 0})-[:REL]->({id: 3, val: 2}) |CREATE ({id: 2, val: 0})-[:REL]->({id: 3, val: 1}) """.stripMargin) // When - val result = given.cypher("MATCH (a) WHERE (a)-->({val: a.val + 2}) RETURN a.id") + val result = + given.cypher("MATCH (a) WHERE (a)-->({val: a.val + 2}) RETURN a.id") // Then - result.records.toMaps should equal(Bag( - CypherMap("a.id" -> 1L) - )) + result.records.toMaps should equal( + Bag( + CypherMap("a.id" -> 1L) + ) + ) } it("multiple predicate patterns") { @@ -527,165 +621,194 @@ class PredicateTests extends MorpheusTestSuite with ScanGraphInit { val given = initGraph("CREATE ({id: 1})-[:REL]->({id: 2, foo: true})") // When - val result = given.cypher("MATCH (a) WHERE (a)-->({id: 2, foo: true}) RETURN a.id") + val result = + given.cypher("MATCH (a) WHERE (a)-->({id: 2, foo: true}) RETURN a.id") // Then - result.records.toMaps should equal(Bag( - CypherMap("a.id" -> 1L) - )) + result.records.toMaps should equal( + Bag( + CypherMap("a.id" -> 1L) + ) + ) } describe("Inline pattern predicates") { it("basic pattern predicate") { // Given - val given = initGraph( - """ + val given = initGraph(""" |CREATE (v {id: 1})-[:REL]->({id: 2})-[:REL]->(w {id: 3}) |CREATE (v)-[:REL]->(w) |CREATE (w)-[:REL]->({id: 4}) """.stripMargin) // When - val result = given.cypher("MATCH (a)-->(b) WHERE (a)-->()-->(b) RETURN a.id, b.id") + val result = + given.cypher("MATCH (a)-->(b) WHERE (a)-->()-->(b) RETURN a.id, b.id") // Then - result.records.toMaps should equal(Bag( - CypherMap("a.id" -> 1L, "b.id" -> 3L) - )) + result.records.toMaps should equal( + Bag( + CypherMap("a.id" -> 1L, "b.id" -> 3L) + ) + ) } it("pattern predicate with var-length-expand") { // Given - val given = initGraph("CREATE (v {id: 1})-[:REL]->({id: 2})-[:REL]->({id: 3})<-[:REL]-(v)") + val given = initGraph( + "CREATE (v {id: 1})-[:REL]->({id: 2})-[:REL]->({id: 3})<-[:REL]-(v)" + ) // When - val result = given.cypher("MATCH (a)-->(b) WHERE (a)-[*1..3]->()-->(b) RETURN a.id, b.id") + val result = given.cypher( + "MATCH (a)-->(b) WHERE (a)-[*1..3]->()-->(b) RETURN a.id, b.id" + ) // Then - result.records.toMaps should equal(Bag( - CypherMap("a.id" -> 1L, "b.id" -> 3L) - )) + result.records.toMaps should equal( + Bag( + CypherMap("a.id" -> 1L, "b.id" -> 3L) + ) + ) } it("simple pattern predicate with node predicate") { // Given - val given = initGraph( - """ + val given = initGraph(""" |CREATE ({id: 1})-[:REL]->({name: 'foo'}) |CREATE ({id: 3})-[:REL]->({name: 'bar'}) """.stripMargin) // When - val result = given.cypher("MATCH (a) WHERE (a)-->({name: 'foo'}) RETURN a.id") + val result = + given.cypher("MATCH (a) WHERE (a)-->({name: 'foo'}) RETURN a.id") // Then - result.records.toMaps should equal(Bag( - CypherMap("a.id" -> 1L) - )) + result.records.toMaps should equal( + Bag( + CypherMap("a.id" -> 1L) + ) + ) } it("simple pattern predicate with relationship predicate") { // Given - val given = initGraph( - """ + val given = initGraph(""" |CREATE (v {id: 1})-[:REL {val: 'foo'}]->()-[:REL]->({id: 2})<-[:REL]-(v) |CREATE (w {id: 3})-[:REL {val: 'bar'}]->()-[:REL]->({id: 4})<-[:REL]-(w) """.stripMargin) // When - val result = given.cypher("MATCH (a)-->(b) WHERE (a)-[{val: 'foo'}]->()-->(b) RETURN a.id, b.id") + val result = given.cypher( + "MATCH (a)-->(b) WHERE (a)-[{val: 'foo'}]->()-->(b) RETURN a.id, b.id" + ) // Then - result.records.toMaps should equal(Bag( - CypherMap("a.id" -> 1L, "b.id" -> 2L) - )) + result.records.toMaps should equal( + Bag( + CypherMap("a.id" -> 1L, "b.id" -> 2L) + ) + ) } it("simple pattern predicate with node label predicate") { // Given - val given = initGraph( - """ + val given = initGraph(""" |CREATE (v{id: 1})-[:REL {val: 'foo'}]->(:A)-[:REL]->({id: 2})<-[:REL]-(v) |CREATE (w{id: 3})-[:REL {val: 'bar'}]->(:B)-[:REL]->({id: 4})<-[:REL]-(w) """.stripMargin) // When - val result = given.cypher("MATCH (a)-->(b) WHERE (a)-->(:A)-->(b) RETURN a.id, b.id") + val result = + given.cypher("MATCH (a)-->(b) WHERE (a)-->(:A)-->(b) RETURN a.id, b.id") // Then - result.records.toMaps should equal(Bag( - CypherMap("a.id" -> 1L, "b.id" -> 2L) - )) + result.records.toMaps should equal( + Bag( + CypherMap("a.id" -> 1L, "b.id" -> 2L) + ) + ) } it("simple pattern predicate with relationship type predicate") { // Given - val given = initGraph( - """ + val given = initGraph(""" |CREATE (v {id: 1})-[:A]->()-[:REL]->({id: 2})<-[:REL]-(v) |CREATE (w {id: 3})-[:B]->()-[:REL]->({id: 4})<-[:REL]-(w) """.stripMargin) // When - val result = given.cypher("MATCH (a)-->(b) WHERE (a)-[:A]->()-->(b) RETURN a.id, b.id") + val result = given.cypher( + "MATCH (a)-->(b) WHERE (a)-[:A]->()-->(b) RETURN a.id, b.id" + ) // Then - result.records.toMaps should equal(Bag( - CypherMap("a.id" -> 1L, "b.id" -> 2L) - )) + result.records.toMaps should equal( + Bag( + CypherMap("a.id" -> 1L, "b.id" -> 2L) + ) + ) } it("inverse pattern predicate") { // Given - val given = initGraph("CREATE (v {id: 1})-[:REL]->({id: 2})-[:REL]->({id: 3})<-[:REL]-(v)") + val given = initGraph( + "CREATE (v {id: 1})-[:REL]->({id: 2})-[:REL]->({id: 3})<-[:REL]-(v)" + ) // When - val result = given.cypher("MATCH (a)-->(b) WHERE NOT (a)-->()-->(b) RETURN a.id, b.id") + val result = given.cypher( + "MATCH (a)-->(b) WHERE NOT (a)-->()-->(b) RETURN a.id, b.id" + ) // Then - result.records.toMaps should equal(Bag( - CypherMap("a.id" -> 1L, "b.id" -> 2L), - CypherMap("a.id" -> 2L, "b.id" -> 3L) - )) + result.records.toMaps should equal( + Bag( + CypherMap("a.id" -> 1L, "b.id" -> 2L), + CypherMap("a.id" -> 2L, "b.id" -> 3L) + ) + ) } it("nested pattern predicate") { - val given = initGraph( - """ + val given = initGraph(""" |CREATE ({id: 1, age: 21}) |CREATE ({id: 2, age: 18, foo: true}) |CREATE ({id: 3, age: 18, foo: true})-[:KNOWS]->(:Foo) |CREATE ({id: 4, age: 18, foo: false})-[:KNOWS]->(:Foo) """.stripMargin) - val result = given.cypher( - """ + val result = given.cypher(""" |MATCH (a) |WHERE a.age > 20 OR ( (a)-[:KNOWS]->(:Foo) AND a.foo = true ) |RETURN a.id """.stripMargin) - result.records.toMaps should equal(Bag( - CypherMap("a.id" -> 1), - CypherMap("a.id" -> 3) - )) + result.records.toMaps should equal( + Bag( + CypherMap("a.id" -> 1), + CypherMap("a.id" -> 3) + ) + ) } it("pattern predicate with derived node predicate") { // Given - val given = initGraph( - """ + val given = initGraph(""" |CREATE ({id: 1, val: 0})-[:REL]->({id: 3, val: 2}) |CREATE ({id: 2, val: 0})-[:REL]->({id: 3, val: 1}) """.stripMargin) // When - val result = given.cypher("MATCH (a) WHERE (a)-->({val: a.val + 2}) RETURN a.id") + val result = + given.cypher("MATCH (a) WHERE (a)-->({val: a.val + 2}) RETURN a.id") // Then - result.records.toMaps should equal(Bag( - CypherMap("a.id" -> 1L) - )) + result.records.toMaps should equal( + Bag( + CypherMap("a.id" -> 1L) + ) + ) } it("multiple predicate patterns") { @@ -693,166 +816,199 @@ class PredicateTests extends MorpheusTestSuite with ScanGraphInit { val given = initGraph("CREATE ({id: 1})-[:REL]->({id: 2, foo: true})") // When - val result = given.cypher("MATCH (a) WHERE (a)-->({id: 2, foo: true}) RETURN a.id") + val result = + given.cypher("MATCH (a) WHERE (a)-->({id: 2, foo: true}) RETURN a.id") // Then - result.records.toMaps should equal(Bag( - CypherMap("a.id" -> 1L) - )) + result.records.toMaps should equal( + Bag( + CypherMap("a.id" -> 1L) + ) + ) } } describe("Pattern predicates via exists") { it("basic pattern predicate") { // Given - val given = initGraph( - """ + val given = initGraph(""" |CREATE (v {id: 1})-[:REL]->({id: 2})-[:REL]->(w {id: 3}) |CREATE (v)-[:REL]->(w) |CREATE (w)-[:REL]->({id: 4}) """.stripMargin) // When - val result = given.cypher("MATCH (a)-->(b) WHERE EXISTS((a)-->()-->(b)) RETURN a.id, b.id") + val result = given.cypher( + "MATCH (a)-->(b) WHERE EXISTS((a)-->()-->(b)) RETURN a.id, b.id" + ) // Then - result.records.toMaps should equal(Bag( - CypherMap("a.id" -> 1L, "b.id" -> 3L) - )) + result.records.toMaps should equal( + Bag( + CypherMap("a.id" -> 1L, "b.id" -> 3L) + ) + ) } it("pattern predicate with var-length-expand") { // Given - val given = initGraph("CREATE (v {id: 1})-[:REL]->({id: 2})-[:REL]->({id: 3})<-[:REL]-(v)") + val given = initGraph( + "CREATE (v {id: 1})-[:REL]->({id: 2})-[:REL]->({id: 3})<-[:REL]-(v)" + ) // When - val result = given.cypher("MATCH (a)-->(b) WHERE EXISTS((a)-[*1..3]->()-->(b)) RETURN a.id, b.id") + val result = given.cypher( + "MATCH (a)-->(b) WHERE EXISTS((a)-[*1..3]->()-->(b)) RETURN a.id, b.id" + ) // Then - result.records.toMaps should equal(Bag( - CypherMap("a.id" -> 1L, "b.id" -> 3L) - )) + result.records.toMaps should equal( + Bag( + CypherMap("a.id" -> 1L, "b.id" -> 3L) + ) + ) } it("simple pattern predicate with node predicate") { // Given - val given = initGraph( - """ + val given = initGraph(""" |CREATE ({id: 1})-[:REL]->({name: 'foo'}) |CREATE ({id: 3})-[:REL]->({name: 'bar'}) """.stripMargin) // When - val result = given.cypher("MATCH (a) WHERE EXISTS((a)-->({name: 'foo'})) RETURN a.id") + val result = given.cypher( + "MATCH (a) WHERE EXISTS((a)-->({name: 'foo'})) RETURN a.id" + ) // Then - result.records.toMaps should equal(Bag( - CypherMap("a.id" -> 1L) - )) + result.records.toMaps should equal( + Bag( + CypherMap("a.id" -> 1L) + ) + ) } it("simple pattern predicate with relationship predicate") { // Given - val given = initGraph( - """ + val given = initGraph(""" |CREATE (v {id: 1})-[:REL {val: 'foo'}]->()-[:REL]->({id: 2})<-[:REL]-(v) |CREATE (w {id: 3})-[:REL {val: 'bar'}]->()-[:REL]->({id: 4})<-[:REL]-(w) """.stripMargin) // When - val result = given.cypher("MATCH (a)-->(b) WHERE EXISTS((a)-[{val: 'foo'}]->()-->(b)) RETURN a.id, b.id") + val result = given.cypher( + "MATCH (a)-->(b) WHERE EXISTS((a)-[{val: 'foo'}]->()-->(b)) RETURN a.id, b.id" + ) // Then - result.records.toMaps should equal(Bag( - CypherMap("a.id" -> 1L, "b.id" -> 2L) - )) + result.records.toMaps should equal( + Bag( + CypherMap("a.id" -> 1L, "b.id" -> 2L) + ) + ) } it("simple pattern predicate with node label predicate") { // Given - val given = initGraph( - """ + val given = initGraph(""" |CREATE (v{id: 1})-[:REL {val: 'foo'}]->(:A)-[:REL]->({id: 2})<-[:REL]-(v) |CREATE (w{id: 3})-[:REL {val: 'bar'}]->(:B)-[:REL]->({id: 4})<-[:REL]-(w) """.stripMargin) // When - val result = given.cypher("MATCH (a)-->(b) WHERE EXISTS((a)-->(:A)-->(b)) RETURN a.id, b.id") + val result = given.cypher( + "MATCH (a)-->(b) WHERE EXISTS((a)-->(:A)-->(b)) RETURN a.id, b.id" + ) // Then - result.records.toMaps should equal(Bag( - CypherMap("a.id" -> 1L, "b.id" -> 2L) - )) + result.records.toMaps should equal( + Bag( + CypherMap("a.id" -> 1L, "b.id" -> 2L) + ) + ) } it("simple pattern predicate with relationship type predicate") { // Given - val given = initGraph( - """ + val given = initGraph(""" |CREATE (v {id: 1})-[:A]->()-[:REL]->({id: 2})<-[:REL]-(v) |CREATE (w {id: 3})-[:B]->()-[:REL]->({id: 4})<-[:REL]-(w) """.stripMargin) // When - val result = given.cypher("MATCH (a)-->(b) WHERE EXISTS((a)-[:A]->()-->(b)) RETURN a.id, b.id") + val result = given.cypher( + "MATCH (a)-->(b) WHERE EXISTS((a)-[:A]->()-->(b)) RETURN a.id, b.id" + ) // Then - result.records.toMaps should equal(Bag( - CypherMap("a.id" -> 1L, "b.id" -> 2L) - )) + result.records.toMaps should equal( + Bag( + CypherMap("a.id" -> 1L, "b.id" -> 2L) + ) + ) } it("inverse pattern predicate") { // Given - val given = initGraph("CREATE (v {id: 1})-[:REL]->({id: 2})-[:REL]->({id: 3})<-[:REL]-(v)") + val given = initGraph( + "CREATE (v {id: 1})-[:REL]->({id: 2})-[:REL]->({id: 3})<-[:REL]-(v)" + ) // When - val result = given.cypher("MATCH (a)-->(b) WHERE NOT EXISTS((a)-->()-->(b)) RETURN a.id, b.id") + val result = given.cypher( + "MATCH (a)-->(b) WHERE NOT EXISTS((a)-->()-->(b)) RETURN a.id, b.id" + ) // Then - result.records.toMaps should equal(Bag( - CypherMap("a.id" -> 1L, "b.id" -> 2L), - CypherMap("a.id" -> 2L, "b.id" -> 3L) - )) + result.records.toMaps should equal( + Bag( + CypherMap("a.id" -> 1L, "b.id" -> 2L), + CypherMap("a.id" -> 2L, "b.id" -> 3L) + ) + ) } it("nested pattern predicate") { - val given = initGraph( - """ + val given = initGraph(""" |CREATE ({id: 1, age: 21}) |CREATE ({id: 2, age: 18, foo: true}) |CREATE ({id: 3, age: 18, foo: true})-[:KNOWS]->(:Foo) |CREATE ({id: 4, age: 18, foo: false})-[:KNOWS]->(:Foo) """.stripMargin) - val result = given.cypher( - """ + val result = given.cypher(""" |MATCH (a) |WHERE a.age > 20 OR ( EXISTS((a)-[:KNOWS]->(:Foo)) AND a.foo = true ) |RETURN a.id """.stripMargin) - result.records.toMaps should equal(Bag( - CypherMap("a.id" -> 1), - CypherMap("a.id" -> 3) - )) + result.records.toMaps should equal( + Bag( + CypherMap("a.id" -> 1), + CypherMap("a.id" -> 3) + ) + ) } it("pattern predicate with derived node predicate") { // Given - val given = initGraph( - """ + val given = initGraph(""" |CREATE ({id: 1, val: 0})-[:REL]->({id: 3, val: 2}) |CREATE ({id: 2, val: 0})-[:REL]->({id: 3, val: 1}) """.stripMargin) // When - val result = given.cypher("MATCH (a) WHERE EXISTS((a)-->({val: a.val + 2})) RETURN a.id") + val result = given.cypher( + "MATCH (a) WHERE EXISTS((a)-->({val: a.val + 2})) RETURN a.id" + ) // Then - result.records.toMaps should equal(Bag( - CypherMap("a.id" -> 1L) - )) + result.records.toMaps should equal( + Bag( + CypherMap("a.id" -> 1L) + ) + ) } it("multiple predicate patterns") { @@ -860,12 +1016,16 @@ class PredicateTests extends MorpheusTestSuite with ScanGraphInit { val given = initGraph("CREATE ({id: 1})-[:REL]->({id: 2, foo: true})") // When - val result = given.cypher("MATCH (a) WHERE EXISTS((a)-->({id: 2, foo: true})) RETURN a.id") + val result = given.cypher( + "MATCH (a) WHERE EXISTS((a)-->({id: 2, foo: true})) RETURN a.id" + ) // Then - result.records.toMaps should equal(Bag( - CypherMap("a.id" -> 1L) - )) + result.records.toMaps should equal( + Bag( + CypherMap("a.id" -> 1L) + ) + ) } } } diff --git a/morpheus-testing/src/test/scala/org/opencypher/morpheus/impl/acceptance/QualifiedGraphNameAcceptance.scala b/morpheus-testing/src/test/scala/org/opencypher/morpheus/impl/acceptance/QualifiedGraphNameAcceptance.scala index 05f6d76e6b..8bd86ae398 100644 --- a/morpheus-testing/src/test/scala/org/opencypher/morpheus/impl/acceptance/QualifiedGraphNameAcceptance.scala +++ b/morpheus-testing/src/test/scala/org/opencypher/morpheus/impl/acceptance/QualifiedGraphNameAcceptance.scala @@ -37,7 +37,8 @@ import org.opencypher.okapi.testing.Bag._ class QualifiedGraphNameAcceptance extends MorpheusTestSuite with ScanGraphInit { - val defaultGraph: RelationalCypherGraph[SparkTable.DataFrameTable] = initGraph("CREATE (:A)-[:REL]->(:B)") + val defaultGraph: RelationalCypherGraph[SparkTable.DataFrameTable] = + initGraph("CREATE (:A)-[:REL]->(:B)") def defaultDS: SessionGraphDataSource = { val ds = new SessionGraphDataSource() @@ -53,15 +54,21 @@ class QualifiedGraphNameAcceptance extends MorpheusTestSuite with ScanGraphInit describe("FROM GRAPH") { def assertFromGraph(namespace: String, graphName: String) = { - morpheus.cypher( - s""" + morpheus + .cypher( + s""" |FROM GRAPH $namespace.$graphName |MATCH (n) |RETURN COUNT(n) as cnt """.stripMargin - ).records.iterator.toBag should equal(Bag( - CypherMap("cnt" -> 2) - )) + ) + .records + .iterator + .toBag should equal( + Bag( + CypherMap("cnt" -> 2) + ) + ) } it("can load from escaped namespaces") { @@ -83,29 +90,41 @@ class QualifiedGraphNameAcceptance extends MorpheusTestSuite with ScanGraphInit val sessionDS = morpheus.catalog.source(morpheus.catalog.sessionNamespace) sessionDS.store(GraphName("my best graph"), defaultGraph) - morpheus.cypher( - s""" + morpheus + .cypher( + s""" |FROM GRAPH `my best graph` |MATCH (n) |RETURN COUNT(n) as cnt """.stripMargin - ).records.iterator.toBag should equal(Bag( - CypherMap("cnt" -> 2) - )) + ) + .records + .iterator + .toBag should equal( + Bag( + CypherMap("cnt" -> 2) + ) + ) } } describe("CONSTRUCT ON") { def assertConstructOn(namespace: String, graphName: String) = { - morpheus.cypher( - s""" + morpheus + .cypher( + s""" |CONSTRUCT ON $namespace.$graphName |MATCH (n) |RETURN COUNT(n) as cnt """.stripMargin - ).records.iterator.toBag should equal(Bag( - CypherMap("cnt" -> 2) - )) + ) + .records + .iterator + .toBag should equal( + Bag( + CypherMap("cnt" -> 2) + ) + ) } it("can construct on escaped namespaces") { @@ -127,15 +146,21 @@ class QualifiedGraphNameAcceptance extends MorpheusTestSuite with ScanGraphInit val sessionDS = morpheus.catalog.source(morpheus.catalog.sessionNamespace) sessionDS.store(GraphName("my best graph"), defaultGraph) - morpheus.cypher( - s""" + morpheus + .cypher( + s""" |CONSTRUCT ON `my best graph` |MATCH (n) |RETURN COUNT(n) as cnt """.stripMargin - ).records.iterator.toBag should equal(Bag( - CypherMap("cnt" -> 2) - )) + ) + .records + .iterator + .toBag should equal( + Bag( + CypherMap("cnt" -> 2) + ) + ) } } @@ -180,8 +205,8 @@ class QualifiedGraphNameAcceptance extends MorpheusTestSuite with ScanGraphInit """.stripMargin ) - morpheus - .catalog.source(morpheus.catalog.sessionNamespace) + morpheus.catalog + .source(morpheus.catalog.sessionNamespace) .hasGraph(GraphName("my best constructed graph")) should be(true) } } diff --git a/morpheus-testing/src/test/scala/org/opencypher/morpheus/impl/acceptance/ReturnTests.scala b/morpheus-testing/src/test/scala/org/opencypher/morpheus/impl/acceptance/ReturnTests.scala index e06f6b026c..08fcf2bd61 100644 --- a/morpheus-testing/src/test/scala/org/opencypher/morpheus/impl/acceptance/ReturnTests.scala +++ b/morpheus-testing/src/test/scala/org/opencypher/morpheus/impl/acceptance/ReturnTests.scala @@ -44,10 +44,14 @@ class ReturnTests extends MorpheusTestSuite with ScanGraphInit { val result = g.cypher("MATCH (a:A) WITH a, a.name AS foo RETURN a") - result.records.collect.toBag should equal(Bag( - CypherMap("a" -> MorpheusNode(0L, Set("A"), CypherMap("name" -> "me"))), - CypherMap("a" -> MorpheusNode(1L, Set("A"), CypherMap.empty)) - )) + result.records.collect.toBag should equal( + Bag( + CypherMap( + "a" -> MorpheusNode(0L, Set("A"), CypherMap("name" -> "me")) + ), + CypherMap("a" -> MorpheusNode(1L, Set("A"), CypherMap.empty)) + ) + ) } it("returns only returned fields with tricky alias") { @@ -55,10 +59,14 @@ class ReturnTests extends MorpheusTestSuite with ScanGraphInit { val result = g.cypher("MATCH (a:A) WITH a, a AS foo RETURN a") - result.records.collect.toBag should equal(Bag( - CypherMap("a" -> MorpheusNode(0L, Set("A"), CypherMap("name" -> "me"))), - CypherMap("a" -> MorpheusNode(1L, Set("A"), CypherMap.empty)) - )) + result.records.collect.toBag should equal( + Bag( + CypherMap( + "a" -> MorpheusNode(0L, Set("A"), CypherMap("name" -> "me")) + ), + CypherMap("a" -> MorpheusNode(1L, Set("A"), CypherMap.empty)) + ) + ) } it("return only returned fields with trickier aliasing") { @@ -68,10 +76,14 @@ class ReturnTests extends MorpheusTestSuite with ScanGraphInit { // perhaps copy all child expressions in RecordHeader val result = g.cypher("MATCH (a:A) WITH a, a AS foo RETURN foo AS b") - result.records.collect.toBag should equal(Bag( - CypherMap("b" -> MorpheusNode(0L, Set("A"), CypherMap("name" -> "me"))), - CypherMap("b" -> MorpheusNode(1L, Set("A"), CypherMap.empty)) - )) + result.records.collect.toBag should equal( + Bag( + CypherMap( + "b" -> MorpheusNode(0L, Set("A"), CypherMap("name" -> "me")) + ), + CypherMap("b" -> MorpheusNode(1L, Set("A"), CypherMap.empty)) + ) + ) } it("returns only returned fields without dependencies") { @@ -79,9 +91,11 @@ class ReturnTests extends MorpheusTestSuite with ScanGraphInit { val result = g.cypher("MATCH (a:A), (b) RETURN a") - result.records.collect.toBag should equal(Bag( - CypherMap("a" -> MorpheusNode(0L, Set("A"), CypherMap.empty)) - )) + result.records.collect.toBag should equal( + Bag( + CypherMap("a" -> MorpheusNode(0L, Set("A"), CypherMap.empty)) + ) + ) } it("can run a single return query") { @@ -105,10 +119,24 @@ class ReturnTests extends MorpheusTestSuite with ScanGraphInit { val result = given.cypher("MATCH (n) RETURN n") - result.records.toMaps should equal(Bag( - CypherMap("n" -> MorpheusNode(0L.encodeAsMorpheusId.toSeq, Set.empty[String], CypherMap("foo" -> "bar"))), - CypherMap("n" -> MorpheusNode(1L.encodeAsMorpheusId.toSeq, Set.empty[String], CypherMap())) - )) + result.records.toMaps should equal( + Bag( + CypherMap( + "n" -> MorpheusNode( + 0L.encodeAsMorpheusId.toSeq, + Set.empty[String], + CypherMap("foo" -> "bar") + ) + ), + CypherMap( + "n" -> MorpheusNode( + 1L.encodeAsMorpheusId.toSeq, + Set.empty[String], + CypherMap() + ) + ) + ) + ) } it("returns full rel") { @@ -116,21 +144,35 @@ class ReturnTests extends MorpheusTestSuite with ScanGraphInit { val result = given.cypher("MATCH ()-[r]->() RETURN r") - result.records.collect.toBag should equal(Bag( - CypherMap("r" -> MorpheusRelationship(2, 0, 1, "Rel", CypherMap("foo" -> "bar"))), - CypherMap("r" -> MorpheusRelationship(4, 1, 3, "Rel")) - )) + result.records.collect.toBag should equal( + Bag( + CypherMap( + "r" -> MorpheusRelationship( + 2, + 0, + 1, + "Rel", + CypherMap("foo" -> "bar") + ) + ), + CypherMap("r" -> MorpheusRelationship(4, 1, 3, "Rel")) + ) + ) } - it("returns relationship property from relationship without specific type") { + it( + "returns relationship property from relationship without specific type" + ) { val given = initGraph("CREATE ()-[:Rel {foo:'bar'}]->()-[:Rel]->()") val result = given.cypher("MATCH ()-[r]->() RETURN r.foo") - result.records.toMaps should equal(Bag( - CypherMap("r.foo" -> "bar"), - CypherMap("r.foo" -> null) - )) + result.records.toMaps should equal( + Bag( + CypherMap("r.foo" -> "bar"), + CypherMap("r.foo" -> null) + ) + ) } it("should be able to project expression with multiple references") { @@ -144,17 +186,17 @@ class ReturnTests extends MorpheusTestSuite with ScanGraphInit { |RETURN a.val """.stripMargin - - graph.cypher(query).records.collect.toBag should equal(Bag( - CypherMap("a.val" -> 0) - )) + graph.cypher(query).records.collect.toBag should equal( + Bag( + CypherMap("a.val" -> 0) + ) + ) } } describe("DISTINCT") { it("can return distinct properties") { - val given = initGraph( - """CREATE ({name:'bar'}) + val given = initGraph("""CREATE ({name:'bar'}) |CREATE ({name:'bar'}) |CREATE ({name:'baz'}) |CREATE ({name:'baz'}) @@ -164,76 +206,96 @@ class ReturnTests extends MorpheusTestSuite with ScanGraphInit { val result = given.cypher("MATCH (n) RETURN DISTINCT n.name AS name") - result.records.toMaps should equal(Bag( - CypherMap("name" -> "bar"), - CypherMap("name" -> "foo"), - CypherMap("name" -> "baz") - )) + result.records.toMaps should equal( + Bag( + CypherMap("name" -> "bar"), + CypherMap("name" -> "foo"), + CypherMap("name" -> "baz") + ) + ) } it("can return distinct properties for combinations") { - val given = initGraph( - """CREATE ({p1:'a', p2: 'a', p3: '1'}) + val given = initGraph("""CREATE ({p1:'a', p2: 'a', p3: '1'}) |CREATE ({p1:'a', p2: 'a', p3: '2'}) |CREATE ({p1:'a', p2: 'b', p3: '3'}) |CREATE ({p1:'b', p2: 'a', p3: '4'}) |CREATE ({p1:'b', p2: 'b', p3: '5'}) """.stripMargin) - val result = given.cypher("MATCH (n) RETURN DISTINCT n.p1 as p1, n.p2 as p2") - - result.records.toMaps should equal(Bag( - CypherMap("p2" -> "a", "p1" -> "a"), - CypherMap("p2" -> "a", "p1" -> "b"), - CypherMap("p2" -> "b", "p1" -> "a"), - CypherMap("p2" -> "b", "p1" -> "b") - )) + val result = + given.cypher("MATCH (n) RETURN DISTINCT n.p1 as p1, n.p2 as p2") + + result.records.toMaps should equal( + Bag( + CypherMap("p2" -> "a", "p1" -> "a"), + CypherMap("p2" -> "a", "p1" -> "b"), + CypherMap("p2" -> "b", "p1" -> "a"), + CypherMap("p2" -> "b", "p1" -> "b") + ) + ) } } describe("ORDER BY") { it("can order with default direction") { - val given = initGraph("""CREATE (:Node {val: 4}), (:Node {val: 3}), (:Node {val: 42})""") + val given = initGraph( + """CREATE (:Node {val: 4}), (:Node {val: 3}), (:Node {val: 42})""" + ) val result = given.cypher("MATCH (a) RETURN a.val AS val ORDER BY val") // Then - result.records.toMaps should equal(Bag( - CypherMap("val" -> 3L), - CypherMap("val" -> 4L), - CypherMap("val" -> 42L) - )) + result.records.toMaps should equal( + Bag( + CypherMap("val" -> 3L), + CypherMap("val" -> 4L), + CypherMap("val" -> 42L) + ) + ) } it("can order ascending") { - val given = initGraph("""CREATE (:Node {val: 4}),(:Node {val: 3}),(:Node {val: 42})""") + val given = initGraph( + """CREATE (:Node {val: 4}),(:Node {val: 3}),(:Node {val: 42})""" + ) - val result = given.cypher("MATCH (a) RETURN a.val as val ORDER BY val ASC") + val result = + given.cypher("MATCH (a) RETURN a.val as val ORDER BY val ASC") // Then - result.records.toMaps should equal(Bag( - CypherMap("val" -> 3L), - CypherMap("val" -> 4L), - CypherMap("val" -> 42L) - )) + result.records.toMaps should equal( + Bag( + CypherMap("val" -> 3L), + CypherMap("val" -> 4L), + CypherMap("val" -> 42L) + ) + ) } it("can order descending") { - val given = initGraph("""CREATE (:Node {val: 4}),(:Node {val: 3}),(:Node {val: 42})""") + val given = initGraph( + """CREATE (:Node {val: 4}),(:Node {val: 3}),(:Node {val: 42})""" + ) - val result = given.cypher("MATCH (a) RETURN a.val as val ORDER BY val DESC") + val result = + given.cypher("MATCH (a) RETURN a.val as val ORDER BY val DESC") // Then - result.records.toMaps should equal(Bag( - CypherMap("val" -> 42L), - CypherMap("val" -> 4L), - CypherMap("val" -> 3L) - )) + result.records.toMaps should equal( + Bag( + CypherMap("val" -> 42L), + CypherMap("val" -> 4L), + CypherMap("val" -> 3L) + ) + ) } } describe("SKIP") { it("can skip") { - val given = initGraph("""CREATE (:Node {val: 4}),(:Node {val: 3}),(:Node {val: 42})""") + val given = initGraph( + """CREATE (:Node {val: 4}),(:Node {val: 3}),(:Node {val: 42})""" + ) val result = given.cypher("MATCH (a) RETURN a.val as val SKIP 2") @@ -242,32 +304,44 @@ class ReturnTests extends MorpheusTestSuite with ScanGraphInit { } it("can order with skip") { - val given = initGraph("""CREATE (:Node {val: 4}),(:Node {val: 3}),(:Node {val: 42})""") + val given = initGraph( + """CREATE (:Node {val: 4}),(:Node {val: 3}),(:Node {val: 42})""" + ) - val result = given.cypher("MATCH (a) RETURN a.val as val ORDER BY val SKIP 1") + val result = + given.cypher("MATCH (a) RETURN a.val as val ORDER BY val SKIP 1") // Then - result.records.toMaps should equal(Bag( - CypherMap("val" -> 4L), - CypherMap("val" -> 42L) - )) + result.records.toMaps should equal( + Bag( + CypherMap("val" -> 4L), + CypherMap("val" -> 42L) + ) + ) } it("can order with (arithmetic) skip") { - val given = initGraph("""CREATE (:Node {val: 4}),(:Node {val: 3}),(:Node {val: 42})""") + val given = initGraph( + """CREATE (:Node {val: 4}),(:Node {val: 3}),(:Node {val: 42})""" + ) - val result = given.cypher("MATCH (a) RETURN a.val as val ORDER BY val SKIP 1 + 1") + val result = + given.cypher("MATCH (a) RETURN a.val as val ORDER BY val SKIP 1 + 1") // Then - result.records.toMaps should equal(Bag( - CypherMap("val" -> 42L) - )) + result.records.toMaps should equal( + Bag( + CypherMap("val" -> 42L) + ) + ) } } describe("limit") { it("can evaluate limit") { - val given = initGraph("""CREATE (:Node {val: 4}),(:Node {val: 3}),(:Node {val: 42})""") + val given = initGraph( + """CREATE (:Node {val: 4}),(:Node {val: 3}),(:Node {val: 42})""" + ) val result = given.cypher("MATCH (a) RETURN a.val as val LIMIT 1") @@ -283,51 +357,70 @@ class ReturnTests extends MorpheusTestSuite with ScanGraphInit { |MATCH (a) |WITH a |LIMIT $limit - |RETURN a""".stripMargin, Map("limit" -> CypherValue(1))) + |RETURN a""".stripMargin, + Map("limit" -> CypherValue(1)) + ) res.records.size } - it("can order with limit") { - val given = initGraph("""CREATE (:Node {val: 4}),(:Node {val: 3}),(:Node {val: 42})""") + val given = initGraph( + """CREATE (:Node {val: 4}),(:Node {val: 3}),(:Node {val: 42})""" + ) - val result = given.cypher("MATCH (a) RETURN a.val as val ORDER BY val LIMIT 1") + val result = + given.cypher("MATCH (a) RETURN a.val as val ORDER BY val LIMIT 1") // Then - result.records.toMaps should equal(Bag( - CypherMap("val" -> 3L) - )) + result.records.toMaps should equal( + Bag( + CypherMap("val" -> 3L) + ) + ) } it("can order with (arithmetic) limit") { - val given = initGraph("""CREATE (:Node {val: 4}),(:Node {val: 3}),(:Node {val: 42})""") + val given = initGraph( + """CREATE (:Node {val: 4}),(:Node {val: 3}),(:Node {val: 42})""" + ) - val result = given.cypher("MATCH (a) RETURN a.val as val ORDER BY val LIMIT 1 + 1") + val result = + given.cypher("MATCH (a) RETURN a.val as val ORDER BY val LIMIT 1 + 1") // Then - result.records.toMaps should equal(Bag( - CypherMap("val" -> 3L), - CypherMap("val" -> 4L) - )) + result.records.toMaps should equal( + Bag( + CypherMap("val" -> 3L), + CypherMap("val" -> 4L) + ) + ) } it("can order with skip and limit") { - val given = initGraph("""CREATE (:Node {val: 4}),(:Node {val: 3}),(:Node {val: 42})""") + val given = initGraph( + """CREATE (:Node {val: 4}),(:Node {val: 3}),(:Node {val: 42})""" + ) - val result = given.cypher("MATCH (a) RETURN a.val as val ORDER BY val SKIP 1 LIMIT 1") + val result = given.cypher( + "MATCH (a) RETURN a.val as val ORDER BY val SKIP 1 LIMIT 1" + ) // Then - result.records.toMaps should equal(Bag( - CypherMap("val" -> 4L) - )) + result.records.toMaps should equal( + Bag( + CypherMap("val" -> 4L) + ) + ) } } describe("MAPS") { it("returns maps") { val res = morpheus.cypher("RETURN { foo : 123, bar : '456'} AS m") - res.records.collect.toBag should equal(Bag(CypherMap("m" -> CypherMap("foo" -> 123, "bar" -> "456")))) + res.records.collect.toBag should equal( + Bag(CypherMap("m" -> CypherMap("foo" -> 123, "bar" -> "456"))) + ) } it("returns maps and support df struct access") { @@ -337,21 +430,28 @@ class ReturnTests extends MorpheusTestSuite with ScanGraphInit { } it("returns map elements") { - val res = morpheus.cypher("WITH { foo : 123, bar : '456'} AS m RETURN m.foo AS foo, m.bar AS bar") - res.records.collect.toBag should equal(Bag(CypherMap("foo" -> 123, "bar" -> "456"))) + val res = morpheus.cypher( + "WITH { foo : 123, bar : '456'} AS m RETURN m.foo AS foo, m.bar AS bar" + ) + res.records.collect.toBag should equal( + Bag(CypherMap("foo" -> 123, "bar" -> "456")) + ) } it("returns lists of maps") { - val res = morpheus.cypher( - """ + val res = morpheus.cypher(""" |RETURN [ | {foo: "bar"}, | {foo: "baz"} |] as maps """.stripMargin) - res.records.collect.toBag should equal(Bag( - CypherMap("maps" -> CypherList(Map("foo" -> "bar"), Map("foo" -> "baz"))) - )) + res.records.collect.toBag should equal( + Bag( + CypherMap( + "maps" -> CypherList(Map("foo" -> "bar"), Map("foo" -> "baz")) + ) + ) + ) } } diff --git a/morpheus-testing/src/test/scala/org/opencypher/morpheus/impl/acceptance/TemporalTests.scala b/morpheus-testing/src/test/scala/org/opencypher/morpheus/impl/acceptance/TemporalTests.scala index 37be2ef29e..a5ef6915aa 100644 --- a/morpheus-testing/src/test/scala/org/opencypher/morpheus/impl/acceptance/TemporalTests.scala +++ b/morpheus-testing/src/test/scala/org/opencypher/morpheus/impl/acceptance/TemporalTests.scala @@ -31,74 +31,159 @@ import java.time.format.DateTimeFormatter import org.opencypher.morpheus.testing.MorpheusTestSuite import org.opencypher.okapi.api.value.CypherValue.CypherMap -import org.opencypher.okapi.impl.exception.{IllegalArgumentException, IllegalStateException, UnsupportedOperationException} +import org.opencypher.okapi.impl.exception.{ + IllegalArgumentException, + IllegalStateException, + UnsupportedOperationException +} import org.opencypher.okapi.impl.temporal.Duration import org.opencypher.okapi.testing.Bag class TemporalTests extends MorpheusTestSuite with ScanGraphInit { private def shouldParseDate(given: String, expected: String): Unit = { - morpheus.cypher(s"RETURN date('$given') AS time").records.toMaps should equal( + morpheus + .cypher(s"RETURN date('$given') AS time") + .records + .toMaps should equal( Bag(CypherMap("time" -> java.time.LocalDate.parse(expected))) ) } private def shouldParseDateTime(given: String, expected: String): Unit = { - morpheus.cypher(s"RETURN localdatetime('$given') AS time").records.toMaps should equal( + morpheus + .cypher(s"RETURN localdatetime('$given') AS time") + .records + .toMaps should equal( Bag(CypherMap("time" -> java.time.LocalDateTime.parse(expected))) ) } describe("duration") { it("parses cypher compatible duration strings") { - morpheus.cypher("RETURN duration('P1Y2M20D') AS duration").records.toMaps should equal( + morpheus + .cypher("RETURN duration('P1Y2M20D') AS duration") + .records + .toMaps should equal( Bag(CypherMap("duration" -> Duration(years = 1, months = 2, days = 20))) ) - morpheus.cypher("RETURN duration('PT1S') AS duration").records.toMaps should equal( + morpheus + .cypher("RETURN duration('PT1S') AS duration") + .records + .toMaps should equal( Bag(CypherMap("duration" -> Duration(seconds = 1))) ) - morpheus.cypher("RETURN duration('PT111.123456S') AS duration").records.toMaps should equal( - Bag(CypherMap("duration" -> Duration(seconds = 111, nanoseconds = 123456000))) + morpheus + .cypher("RETURN duration('PT111.123456S') AS duration") + .records + .toMaps should equal( + Bag( + CypherMap( + "duration" -> Duration(seconds = 111, nanoseconds = 123456000) + ) + ) ) - morpheus.cypher("RETURN duration('PT1M10S') AS duration").records.toMaps should equal( + morpheus + .cypher("RETURN duration('PT1M10S') AS duration") + .records + .toMaps should equal( Bag(CypherMap("duration" -> Duration(minutes = 1, seconds = 10))) ) - morpheus.cypher("RETURN duration('PT3H1M10S') AS duration").records.toMaps should equal( - Bag(CypherMap("duration" -> Duration(hours = 3, minutes = 1, seconds = 10))) + morpheus + .cypher("RETURN duration('PT3H1M10S') AS duration") + .records + .toMaps should equal( + Bag( + CypherMap( + "duration" -> Duration(hours = 3, minutes = 1, seconds = 10) + ) + ) ) - morpheus.cypher("RETURN duration('P5DT3H1M10S') AS duration").records.toMaps should equal( - Bag(CypherMap("duration" -> Duration(days = 5, hours = 3, minutes = 1, seconds = 10))) + morpheus + .cypher("RETURN duration('P5DT3H1M10S') AS duration") + .records + .toMaps should equal( + Bag( + CypherMap( + "duration" -> Duration( + days = 5, + hours = 3, + minutes = 1, + seconds = 10 + ) + ) + ) ) - morpheus.cypher("RETURN duration('P1W5DT3H1M10S') AS duration") - .records.toMaps should equal( - Bag(CypherMap("duration" -> Duration(weeks = 1, days = 5, hours = 3, minutes = 1, seconds = 10))) + morpheus + .cypher("RETURN duration('P1W5DT3H1M10S') AS duration") + .records + .toMaps should equal( + Bag( + CypherMap( + "duration" -> Duration( + weeks = 1, + days = 5, + hours = 3, + minutes = 1, + seconds = 10 + ) + ) + ) ) - morpheus.cypher("RETURN duration('P12M1W5DT3H1M10S') AS duration") - .records.toMaps should equal( - Bag(CypherMap("duration" -> Duration(months = 12, weeks = 1, days = 5, hours = 3, minutes = 1, - seconds = 10))) + morpheus + .cypher("RETURN duration('P12M1W5DT3H1M10S') AS duration") + .records + .toMaps should equal( + Bag( + CypherMap( + "duration" -> Duration( + months = 12, + weeks = 1, + days = 5, + hours = 3, + minutes = 1, + seconds = 10 + ) + ) + ) ) - morpheus.cypher("RETURN duration('P3Y12M1W5DT3H1M10S') AS duration") - .records.toMaps should equal( - Bag(CypherMap("duration" -> Duration(years = 3, months = 12, weeks = 1, days = 5, hours = 3, - minutes = 1, seconds = 10))) + morpheus + .cypher("RETURN duration('P3Y12M1W5DT3H1M10S') AS duration") + .records + .toMaps should equal( + Bag( + CypherMap( + "duration" -> Duration( + years = 3, + months = 12, + weeks = 1, + days = 5, + hours = 3, + minutes = 1, + seconds = 10 + ) + ) + ) ) } it("constructs duration from a map") { - morpheus.cypher("RETURN duration({ seconds: 1 }) AS duration").records.toMaps should equal( + morpheus + .cypher("RETURN duration({ seconds: 1 }) AS duration") + .records + .toMaps should equal( Bag(CypherMap("duration" -> Duration(seconds = 1))) ) - morpheus.cypher( - """RETURN duration({ + morpheus + .cypher("""RETURN duration({ | years: 3, | months: 12, | weeks: 1, @@ -107,11 +192,22 @@ class TemporalTests extends MorpheusTestSuite with ScanGraphInit { | minutes: 1, | seconds: 10, | milliseconds: 10, - | microseconds: 10 }) AS duration""".stripMargin).records.toMaps should equal( + | microseconds: 10 }) AS duration""".stripMargin) + .records + .toMaps should equal( Bag( - CypherMap("duration" -> Duration( - years = 3, months = 12, weeks = 1, days = 5, - hours = 3, minutes = 1, seconds = 10, milliseconds = 10, microseconds = 10) + CypherMap( + "duration" -> Duration( + years = 3, + months = 12, + weeks = 1, + days = 5, + hours = 3, + minutes = 1, + seconds = 10, + milliseconds = 10, + microseconds = 10 + ) ) ) ) @@ -119,78 +215,137 @@ class TemporalTests extends MorpheusTestSuite with ScanGraphInit { describe("addition") { it("supports addition to duration") { - morpheus.cypher("RETURN duration('P1D') + duration('P1D') AS time").records.toMaps should equal( + morpheus + .cypher("RETURN duration('P1D') + duration('P1D') AS time") + .records + .toMaps should equal( Bag(CypherMap("time" -> Duration(days = 2))) ) } it("supports addition to date") { - morpheus.cypher("RETURN date('2010-10-10') + duration('P1D') AS time").records.toMaps should equal( + morpheus + .cypher("RETURN date('2010-10-10') + duration('P1D') AS time") + .records + .toMaps should equal( Bag(CypherMap("time" -> java.time.LocalDate.parse("2010-10-11"))) ) } it("supports addition to date with time part present") { - morpheus.cypher("RETURN date('2010-10-10') + duration('P1DT12H') AS time").records.toMaps should equal( + morpheus + .cypher("RETURN date('2010-10-10') + duration('P1DT12H') AS time") + .records + .toMaps should equal( Bag(CypherMap("time" -> java.time.LocalDate.parse("2010-10-11"))) ) } it("supports addition to localdatetime") { - morpheus.cypher("RETURN localdatetime('2010-10-10T12:00') + duration('P1D') AS time").records.toMaps should equal( - Bag(CypherMap("time" -> java.time.LocalDateTime.parse("2010-10-11T12:00:00"))) + morpheus + .cypher( + "RETURN localdatetime('2010-10-10T12:00') + duration('P1D') AS time" + ) + .records + .toMaps should equal( + Bag( + CypherMap( + "time" -> java.time.LocalDateTime.parse("2010-10-11T12:00:00") + ) + ) ) } it("supports addition to localdatetime with time part present") { - morpheus.cypher("RETURN localdatetime('2010-10-10T12:00') + duration('P1DT12H') AS time").records.toMaps should equal( - Bag(CypherMap("time" -> java.time.LocalDateTime.parse("2010-10-12T00:00:00"))) + morpheus + .cypher( + "RETURN localdatetime('2010-10-10T12:00') + duration('P1DT12H') AS time" + ) + .records + .toMaps should equal( + Bag( + CypherMap( + "time" -> java.time.LocalDateTime.parse("2010-10-12T00:00:00") + ) + ) ) } } describe("subtraction") { it("supports subtraction to duration") { - morpheus.cypher("RETURN duration('P1D') - duration('P1D') AS time").records.toMaps should equal( + morpheus + .cypher("RETURN duration('P1D') - duration('P1D') AS time") + .records + .toMaps should equal( Bag(CypherMap("time" -> Duration())) ) - morpheus.cypher("RETURN duration('P1D') - duration('PT12H') AS time").records.toMaps should equal( + morpheus + .cypher("RETURN duration('P1D') - duration('PT12H') AS time") + .records + .toMaps should equal( Bag(CypherMap("time" -> Duration(hours = 12))) ) } it("supports subtraction to date") { - morpheus.cypher("RETURN date('2010-10-10') - duration('P1D') AS time").records.toMaps should equal( + morpheus + .cypher("RETURN date('2010-10-10') - duration('P1D') AS time") + .records + .toMaps should equal( Bag(CypherMap("time" -> java.time.LocalDate.parse("2010-10-09"))) ) - morpheus.cypher( - """ + morpheus + .cypher(""" |WITH | date({year: 1984, month: 10, day: 11}) AS date, | duration({months: 1, days: -14}) as duration |RETURN date - duration AS diff - """.stripMargin).records.toMaps should equal( + """.stripMargin) + .records + .toMaps should equal( Bag(CypherMap("diff" -> java.time.LocalDate.parse("1984-09-25"))) ) } it("supports subtraction to date with time part present") { - morpheus.cypher("RETURN date('2010-10-10') - duration('P1DT12H') AS time").records.toMaps should equal( + morpheus + .cypher("RETURN date('2010-10-10') - duration('P1DT12H') AS time") + .records + .toMaps should equal( Bag(CypherMap("time" -> java.time.LocalDate.parse("2010-10-09"))) ) } it("supports subtraction to localdatetime") { - morpheus.cypher("RETURN localdatetime('2010-10-10T12:00') - duration('P1D') AS time").records.toMaps should equal( - Bag(CypherMap("time" -> java.time.LocalDateTime.parse("2010-10-09T12:00:00"))) + morpheus + .cypher( + "RETURN localdatetime('2010-10-10T12:00') - duration('P1D') AS time" + ) + .records + .toMaps should equal( + Bag( + CypherMap( + "time" -> java.time.LocalDateTime.parse("2010-10-09T12:00:00") + ) + ) ) } it("supports subtraction to localdatetime with time part present") { - morpheus.cypher("RETURN localdatetime('2010-10-10T12:00') - duration('P1DT12H') AS time").records.toMaps should equal( - Bag(CypherMap("time" -> java.time.LocalDateTime.parse("2010-10-09T00:00:00"))) + morpheus + .cypher( + "RETURN localdatetime('2010-10-10T12:00') - duration('P1DT12H') AS time" + ) + .records + .toMaps should equal( + Bag( + CypherMap( + "time" -> java.time.LocalDateTime.parse("2010-10-09T00:00:00") + ) + ) ) } } @@ -199,7 +354,10 @@ class TemporalTests extends MorpheusTestSuite with ScanGraphInit { describe("date") { it("returns a valid date") { - morpheus.cypher("RETURN date('2010-10-10') AS time").records.toMaps should equal( + morpheus + .cypher("RETURN date('2010-10-10') AS time") + .records + .toMaps should equal( Bag(CypherMap("time" -> java.time.LocalDate.parse("2010-10-10"))) ) } @@ -220,21 +378,31 @@ class TemporalTests extends MorpheusTestSuite with ScanGraphInit { "2015Q2" -> "2015-04-01", "2015-202" -> "2015-07-21", "2015202" -> "2015-07-21", - "2010" -> "2010-01-01").foreach { - case (given, expected) => shouldParseDate(given, expected) + "2010" -> "2010-01-01" + ).foreach { case (given, expected) => + shouldParseDate(given, expected) } } it("returns a valid date when constructed from a map") { - morpheus.cypher("RETURN date({ year: 2010, month: 10, day: 10 }) AS time").records.toMaps should equal( + morpheus + .cypher("RETURN date({ year: 2010, month: 10, day: 10 }) AS time") + .records + .toMaps should equal( Bag(CypherMap("time" -> java.time.LocalDate.parse("2010-10-10"))) ) - morpheus.cypher("RETURN date({ year: 2010, month: 10 }) AS time").records.toMaps should equal( + morpheus + .cypher("RETURN date({ year: 2010, month: 10 }) AS time") + .records + .toMaps should equal( Bag(CypherMap("time" -> java.time.LocalDate.parse("2010-10-01"))) ) - morpheus.cypher("RETURN date({ year: '2010' }) AS time").records.toMaps should equal( + morpheus + .cypher("RETURN date({ year: '2010' }) AS time") + .records + .toMaps should equal( Bag(CypherMap("time" -> java.time.LocalDate.parse("2010-01-01"))) ) } @@ -242,35 +410,51 @@ class TemporalTests extends MorpheusTestSuite with ScanGraphInit { it("throws an error if values of higher significance are omitted") { val e1 = the[IllegalArgumentException] thrownBy morpheus.cypher("RETURN date({ year: 2018, day: 356 })").records.toMaps - e1.getMessage should (include("valid significance order") and include("year, day")) + e1.getMessage should (include("valid significance order") and include( + "year, day" + )) val e2 = the[IllegalArgumentException] thrownBy morpheus.cypher("RETURN date({ month: 11, day: 2 })").records.toMaps - e2.getMessage should (include("`year` needs to be set") and include("month, day")) + e2.getMessage should (include("`year` needs to be set") and include( + "month, day" + )) val e3 = the[IllegalArgumentException] thrownBy morpheus.cypher("RETURN date({ day: 2 })").records.toMaps - e3.getMessage should (include("`year` needs to be set") and include("day")) + e3.getMessage should (include("`year` needs to be set") and include( + "day" + )) } it("throws an error if the date argument is wrong") { val e1 = the[IllegalArgumentException] thrownBy morpheus.cypher("RETURN date('2018-10-10-10')").records.toMaps - e1.getMessage should (include("valid date construction string") and include("2018-10-10-10")) + e1.getMessage should (include( + "valid date construction string" + ) and include("2018-10-10-10")) val e2 = the[IllegalArgumentException] thrownBy morpheus.cypher("RETURN date('201810101')").records.toMaps - e2.getMessage should (include("valid date construction string") and include("201810101")) + e2.getMessage should (include( + "valid date construction string" + ) and include("201810101")) } it("compares two dates") { - morpheus.cypher("RETURN date('2015-10-10') < date('2015-10-12') AS time").records.toMaps should equal( + morpheus + .cypher("RETURN date('2015-10-10') < date('2015-10-12') AS time") + .records + .toMaps should equal( Bag( CypherMap("time" -> true) ) ) - morpheus.cypher("RETURN date('2015-10-10') > date('2015-10-12') AS time").records.toMaps should equal( + morpheus + .cypher("RETURN date('2015-10-10') > date('2015-10-12') AS time") + .records + .toMaps should equal( Bag( CypherMap("time" -> false) ) @@ -279,7 +463,10 @@ class TemporalTests extends MorpheusTestSuite with ScanGraphInit { it("returns current date if no parameters are given") { val currentDate = new java.sql.Date(System.currentTimeMillis()).toString - morpheus.cypher(s"RETURN date('$currentDate') <= date() AS time").records.toMaps should equal( + morpheus + .cypher(s"RETURN date('$currentDate') <= date() AS time") + .records + .toMaps should equal( Bag( CypherMap("time" -> true) ) @@ -321,83 +508,128 @@ class TemporalTests extends MorpheusTestSuite with ScanGraphInit { "2010-10-10T21:40" -> "2010-10-10T21:40:00", "2010-10-10T2140" -> "2010-10-10T21:40:00", "2010-10-10T21" -> "2010-10-10T21:00:00" - ).foreach { - case (given, expected) => shouldParseDateTime(given, expected) + ).foreach { case (given, expected) => + shouldParseDateTime(given, expected) } } it("returns a valid localdatetime when constructed from a map") { - morpheus.cypher( - """RETURN localdatetime({ + morpheus + .cypher("""RETURN localdatetime({ |year: 2015, |month: 10, |day: 12, |hour: 12, |minute: 50, |second: 35, - |millisecond: 556}) AS time""".stripMargin).records.toMaps should equal( + |millisecond: 556}) AS time""".stripMargin) + .records + .toMaps should equal( Bag( - CypherMap("time" -> java.time.LocalDateTime.parse("2015-10-12T12:50:35.556")) + CypherMap( + "time" -> java.time.LocalDateTime.parse("2015-10-12T12:50:35.556") + ) ) ) - morpheus.cypher( - """RETURN localdatetime({ + morpheus + .cypher("""RETURN localdatetime({ |year: 2015, |month: 10, |day: 1, |hour: 12, - |minute: 50}) AS time""".stripMargin).records.toMaps should equal( + |minute: 50}) AS time""".stripMargin) + .records + .toMaps should equal( Bag( - CypherMap("time" -> java.time.LocalDateTime.parse("2015-10-01T12:50:00")) + CypherMap( + "time" -> java.time.LocalDateTime.parse("2015-10-01T12:50:00") + ) ) ) } it("throws an error if nanoseconds are specified") { - val e = the[IllegalStateException] thrownBy morpheus.cypher( - """RETURN localdatetime({ + val e = the[IllegalStateException] thrownBy morpheus + .cypher("""RETURN localdatetime({ |year: 2015, |month: 10, |day: 1, |hour: 12, |minute: 50, |second: 1, - |nanosecond: 42}) AS time""".stripMargin).records.toMaps + |nanosecond: 42}) AS time""".stripMargin) + .records + .toMaps e.getMessage should include("nanosecond resolution") } it("throws an error if values of higher significance are omitted") { - val e1 = the[IllegalArgumentException] thrownBy morpheus.cypher("RETURN localdatetime({year: 2011, minute: 50 })").records.toMaps - e1.getMessage should (include("valid significance order") and include("minute")) - - val e2 = the[IllegalArgumentException] thrownBy morpheus.cypher("RETURN localdatetime({ year: 2018, hour: 12, second: 14 })").records.toMaps - e2.getMessage should (include("valid significance order") and include("year, hour, second")) + val e1 = the[IllegalArgumentException] thrownBy morpheus + .cypher("RETURN localdatetime({year: 2011, minute: 50 })") + .records + .toMaps + e1.getMessage should (include("valid significance order") and include( + "minute" + )) + + val e2 = the[IllegalArgumentException] thrownBy morpheus + .cypher("RETURN localdatetime({ year: 2018, hour: 12, second: 14 })") + .records + .toMaps + e2.getMessage should (include("valid significance order") and include( + "year, hour, second" + )) } it("throws an error if the localdatetime string is malformed") { val e1 = the[IllegalArgumentException] thrownBy - morpheus.cypher("RETURN localdatetime('2018-10-10T12:10:30:15')").records.toMaps - e1.getMessage.should(include("valid time construction string") and include("12:10:30:15")) + morpheus + .cypher("RETURN localdatetime('2018-10-10T12:10:30:15')") + .records + .toMaps + e1.getMessage.should( + include("valid time construction string") and include("12:10:30:15") + ) val e2 = the[IllegalArgumentException] thrownBy - morpheus.cypher("RETURN localdatetime('20181010T1210301')").records.toMaps - e2.getMessage should (include("valid time construction string") and include("1210301")) + morpheus + .cypher("RETURN localdatetime('20181010T1210301')") + .records + .toMaps + e2.getMessage should (include( + "valid time construction string" + ) and include("1210301")) val e3 = the[IllegalArgumentException] thrownBy - morpheus.cypher("RETURN localdatetime('20181010123123T12:00')").records.toMaps - e3.getMessage should (include("valid date construction string") and include("20181010123123")) + morpheus + .cypher("RETURN localdatetime('20181010123123T12:00')") + .records + .toMaps + e3.getMessage should (include( + "valid date construction string" + ) and include("20181010123123")) } it("compares two datetimes") { - morpheus.cypher("RETURN localdatetime('2015-10-10T00:00:00') < localdatetime('2015-10-12T00:00:00') AS time").records.toMaps should equal( + morpheus + .cypher( + "RETURN localdatetime('2015-10-10T00:00:00') < localdatetime('2015-10-12T00:00:00') AS time" + ) + .records + .toMaps should equal( Bag( CypherMap("time" -> true) ) ) - morpheus.cypher("RETURN localdatetime('2015-10-10T00:00:00') > localdatetime('2015-10-12T00:00:00') AS time").records.toMaps should equal( + morpheus + .cypher( + "RETURN localdatetime('2015-10-10T00:00:00') > localdatetime('2015-10-12T00:00:00') AS time" + ) + .records + .toMaps should equal( Bag( CypherMap("time" -> false) ) @@ -406,8 +638,14 @@ class TemporalTests extends MorpheusTestSuite with ScanGraphInit { // TODO This pass locally ignore("uses the current date and time if no parameters are given") { - val currentDateTime = LocalDateTime.now().format(DateTimeFormatter.ISO_LOCAL_DATE_TIME) - morpheus.cypher(s"RETURN localdatetime('$currentDateTime') <= localdatetime() AS time").records.toMaps equals equal( + val currentDateTime = + LocalDateTime.now().format(DateTimeFormatter.ISO_LOCAL_DATE_TIME) + morpheus + .cypher( + s"RETURN localdatetime('$currentDateTime') <= localdatetime() AS time" + ) + .records + .toMaps equals equal( Bag( CypherMap("time" -> true) ) @@ -415,7 +653,10 @@ class TemporalTests extends MorpheusTestSuite with ScanGraphInit { } it("should propagate null") { - morpheus.cypher("RETURN localdatetime(null) as time").records.toMaps should equal( + morpheus + .cypher("RETURN localdatetime(null) as time") + .records + .toMaps should equal( Bag( CypherMap("time" -> null) ) @@ -425,13 +666,19 @@ class TemporalTests extends MorpheusTestSuite with ScanGraphInit { describe("temporal accessors") { it("propagates null values") { - morpheus.cypher("""RETURN date(null).year AS year""").records.toMaps should equal( + morpheus + .cypher("""RETURN date(null).year AS year""") + .records + .toMaps should equal( Bag( CypherMap("year" -> null) ) ) - morpheus.cypher("""RETURN localdatetime(null).year AS year""").records.toMaps should equal( + morpheus + .cypher("""RETURN localdatetime(null).year AS year""") + .records + .toMaps should equal( Bag( CypherMap("year" -> null) ) @@ -439,14 +686,20 @@ class TemporalTests extends MorpheusTestSuite with ScanGraphInit { } it("throws an error for unknown accessors") { - val e = the[UnsupportedOperationException] thrownBy morpheus.cypher("""RETURN date().foo AS foo""").records.toMaps + val e = the[UnsupportedOperationException] thrownBy morpheus + .cypher("""RETURN date().foo AS foo""") + .records + .toMaps e.getMessage should include("foo") } describe("year") { it("works on date") { - morpheus.cypher("""RETURN date('2015-10-10').year AS year""").records.toMaps should equal( + morpheus + .cypher("""RETURN date('2015-10-10').year AS year""") + .records + .toMaps should equal( Bag( CypherMap("year" -> 2015) ) @@ -454,7 +707,10 @@ class TemporalTests extends MorpheusTestSuite with ScanGraphInit { } it("works on localdatetime") { - morpheus.cypher("""RETURN localdatetime('2015-10-10T10:10').year AS year""").records.toMaps should equal( + morpheus + .cypher("""RETURN localdatetime('2015-10-10T10:10').year AS year""") + .records + .toMaps should equal( Bag( CypherMap("year" -> 2015) ) @@ -464,7 +720,10 @@ class TemporalTests extends MorpheusTestSuite with ScanGraphInit { describe("quarter") { it("works on date") { - morpheus.cypher("""RETURN date('2015-10-10').quarter AS quarter""").records.toMaps should equal( + morpheus + .cypher("""RETURN date('2015-10-10').quarter AS quarter""") + .records + .toMaps should equal( Bag( CypherMap("quarter" -> 4) ) @@ -472,7 +731,12 @@ class TemporalTests extends MorpheusTestSuite with ScanGraphInit { } it("works on localdatetime") { - morpheus.cypher("""RETURN localdatetime('2015-10-10T10:10').quarter AS quarter""").records.toMaps should equal( + morpheus + .cypher( + """RETURN localdatetime('2015-10-10T10:10').quarter AS quarter""" + ) + .records + .toMaps should equal( Bag( CypherMap("quarter" -> 4) ) @@ -482,7 +746,10 @@ class TemporalTests extends MorpheusTestSuite with ScanGraphInit { describe("month") { it("works on date") { - morpheus.cypher("""RETURN date('2015-10-10').month AS month""").records.toMaps should equal( + morpheus + .cypher("""RETURN date('2015-10-10').month AS month""") + .records + .toMaps should equal( Bag( CypherMap("month" -> 10) ) @@ -490,7 +757,10 @@ class TemporalTests extends MorpheusTestSuite with ScanGraphInit { } it("works on localdatetime") { - morpheus.cypher("""RETURN localdatetime('2015-10-10T10:10').month AS month""").records.toMaps should equal( + morpheus + .cypher("""RETURN localdatetime('2015-10-10T10:10').month AS month""") + .records + .toMaps should equal( Bag( CypherMap("month" -> 10) ) @@ -500,7 +770,10 @@ class TemporalTests extends MorpheusTestSuite with ScanGraphInit { describe("week") { it("works on date") { - morpheus.cypher("""RETURN date('2019-01-01').week AS week""").records.toMaps should equal( + morpheus + .cypher("""RETURN date('2019-01-01').week AS week""") + .records + .toMaps should equal( Bag( CypherMap("week" -> 1) ) @@ -508,7 +781,10 @@ class TemporalTests extends MorpheusTestSuite with ScanGraphInit { } it("works on localdatetime") { - morpheus.cypher("""RETURN localdatetime('2019-01-01T10:10').week AS week""").records.toMaps should equal( + morpheus + .cypher("""RETURN localdatetime('2019-01-01T10:10').week AS week""") + .records + .toMaps should equal( Bag( CypherMap("week" -> 1) ) @@ -518,7 +794,10 @@ class TemporalTests extends MorpheusTestSuite with ScanGraphInit { describe("weekYear") { it("works on date") { - morpheus.cypher("""RETURN date('1813-01-01').weekYear AS weekYear""").records.toMaps should equal( + morpheus + .cypher("""RETURN date('1813-01-01').weekYear AS weekYear""") + .records + .toMaps should equal( Bag( CypherMap("weekYear" -> 1812) ) @@ -526,7 +805,12 @@ class TemporalTests extends MorpheusTestSuite with ScanGraphInit { } it("works on localdatetime") { - morpheus.cypher("""RETURN localdatetime('1813-01-01T10:10').weekYear AS weekYear""").records.toMaps should equal( + morpheus + .cypher( + """RETURN localdatetime('1813-01-01T10:10').weekYear AS weekYear""" + ) + .records + .toMaps should equal( Bag( CypherMap("weekYear" -> 1812) ) @@ -536,7 +820,10 @@ class TemporalTests extends MorpheusTestSuite with ScanGraphInit { describe("dayOfQuarter") { it("works on date") { - morpheus.cypher("""RETURN date('2019-01-01').dayOfQuarter AS dayOfQuarter""").records.toMaps should equal( + morpheus + .cypher("""RETURN date('2019-01-01').dayOfQuarter AS dayOfQuarter""") + .records + .toMaps should equal( Bag( CypherMap("dayOfQuarter" -> 1) ) @@ -544,7 +831,12 @@ class TemporalTests extends MorpheusTestSuite with ScanGraphInit { } it("works on localdatetime") { - morpheus.cypher("""RETURN localdatetime('2019-01-01T10:10').dayOfQuarter AS dayOfQuarter""").records.toMaps should equal( + morpheus + .cypher( + """RETURN localdatetime('2019-01-01T10:10').dayOfQuarter AS dayOfQuarter""" + ) + .records + .toMaps should equal( Bag( CypherMap("dayOfQuarter" -> 1) ) @@ -554,7 +846,10 @@ class TemporalTests extends MorpheusTestSuite with ScanGraphInit { describe("day") { it("works on date") { - morpheus.cypher("""RETURN date('2019-05-10').day AS day""").records.toMaps should equal( + morpheus + .cypher("""RETURN date('2019-05-10').day AS day""") + .records + .toMaps should equal( Bag( CypherMap("day" -> 10) ) @@ -562,7 +857,10 @@ class TemporalTests extends MorpheusTestSuite with ScanGraphInit { } it("works on localdatetime") { - morpheus.cypher("""RETURN localdatetime('2019-05-10T10:10').day AS day""").records.toMaps should equal( + morpheus + .cypher("""RETURN localdatetime('2019-05-10T10:10').day AS day""") + .records + .toMaps should equal( Bag( CypherMap("day" -> 10) ) @@ -572,7 +870,10 @@ class TemporalTests extends MorpheusTestSuite with ScanGraphInit { describe("ordinalDay") { it("works on date") { - morpheus.cypher("""RETURN date('2019-05-10').ordinalDay AS ordinalDay""").records.toMaps should equal( + morpheus + .cypher("""RETURN date('2019-05-10').ordinalDay AS ordinalDay""") + .records + .toMaps should equal( Bag( CypherMap("ordinalDay" -> 130) ) @@ -580,7 +881,12 @@ class TemporalTests extends MorpheusTestSuite with ScanGraphInit { } it("works on localdatetime") { - morpheus.cypher("""RETURN localdatetime('2019-05-10T10:10').ordinalDay AS ordinalDay""").records.toMaps should equal( + morpheus + .cypher( + """RETURN localdatetime('2019-05-10T10:10').ordinalDay AS ordinalDay""" + ) + .records + .toMaps should equal( Bag( CypherMap("ordinalDay" -> 130) ) @@ -590,7 +896,10 @@ class TemporalTests extends MorpheusTestSuite with ScanGraphInit { describe("dayOfWeek") { it("works on date") { - morpheus.cypher("""RETURN date('2019-05-10').dayOfWeek AS dayOfWeek""").records.toMaps should equal( + morpheus + .cypher("""RETURN date('2019-05-10').dayOfWeek AS dayOfWeek""") + .records + .toMaps should equal( Bag( CypherMap("dayOfWeek" -> 5) ) @@ -598,7 +907,12 @@ class TemporalTests extends MorpheusTestSuite with ScanGraphInit { } it("works on localdatetime") { - morpheus.cypher("""RETURN localdatetime('2019-05-10T10:10').dayOfWeek AS dayOfWeek""").records.toMaps should equal( + morpheus + .cypher( + """RETURN localdatetime('2019-05-10T10:10').dayOfWeek AS dayOfWeek""" + ) + .records + .toMaps should equal( Bag( CypherMap("dayOfWeek" -> 5) ) @@ -608,7 +922,10 @@ class TemporalTests extends MorpheusTestSuite with ScanGraphInit { describe("hour") { it("works on datetime") { - morpheus.cypher("""RETURN localdatetime('2019-05-10T10:10').hour AS hour""").records.toMaps should equal( + morpheus + .cypher("""RETURN localdatetime('2019-05-10T10:10').hour AS hour""") + .records + .toMaps should equal( Bag( CypherMap("hour" -> 10) ) @@ -618,7 +935,12 @@ class TemporalTests extends MorpheusTestSuite with ScanGraphInit { describe("minute") { it("works on datetime") { - morpheus.cypher("""RETURN localdatetime('2019-05-10T10:11').minute AS minute""").records.toMaps should equal( + morpheus + .cypher( + """RETURN localdatetime('2019-05-10T10:11').minute AS minute""" + ) + .records + .toMaps should equal( Bag( CypherMap("minute" -> 11) ) @@ -628,7 +950,12 @@ class TemporalTests extends MorpheusTestSuite with ScanGraphInit { describe("second") { it("works on datetime") { - morpheus.cypher("""RETURN localdatetime('2019-05-10T10:10:12').second AS second""").records.toMaps should equal( + morpheus + .cypher( + """RETURN localdatetime('2019-05-10T10:10:12').second AS second""" + ) + .records + .toMaps should equal( Bag( CypherMap("second" -> 12) ) @@ -638,7 +965,12 @@ class TemporalTests extends MorpheusTestSuite with ScanGraphInit { describe("millisecond") { it("works on datetime") { - morpheus.cypher("""RETURN localdatetime('2019-05-10T10:10:12.113').millisecond AS millisecond""").records.toMaps should equal( + morpheus + .cypher( + """RETURN localdatetime('2019-05-10T10:10:12.113').millisecond AS millisecond""" + ) + .records + .toMaps should equal( Bag( CypherMap("millisecond" -> 113) ) @@ -648,7 +980,12 @@ class TemporalTests extends MorpheusTestSuite with ScanGraphInit { describe("microsecond") { it("works on datetime") { - morpheus.cypher("""RETURN localdatetime('2019-05-10T10:10:12.113114').microsecond AS microsecond""").records.toMaps should equal( + morpheus + .cypher( + """RETURN localdatetime('2019-05-10T10:10:12.113114').microsecond AS microsecond""" + ) + .records + .toMaps should equal( Bag( CypherMap("microsecond" -> 113114) ) @@ -658,7 +995,10 @@ class TemporalTests extends MorpheusTestSuite with ScanGraphInit { describe("duration based accessors") { it("supports years") { - morpheus.cypher("""RETURN duration({years: 2, months: 14}).years AS years""").records.toMaps should equal( + morpheus + .cypher("""RETURN duration({years: 2, months: 14}).years AS years""") + .records + .toMaps should equal( Bag( CypherMap("years" -> 3) ) @@ -666,7 +1006,12 @@ class TemporalTests extends MorpheusTestSuite with ScanGraphInit { } it("supports months") { - morpheus.cypher("""RETURN duration({years: 2, months: 14}).months AS months""").records.toMaps should equal( + morpheus + .cypher( + """RETURN duration({years: 2, months: 14}).months AS months""" + ) + .records + .toMaps should equal( Bag( CypherMap("months" -> 38) ) @@ -674,7 +1019,10 @@ class TemporalTests extends MorpheusTestSuite with ScanGraphInit { } it("supports weeks") { - morpheus.cypher("""RETURN duration({years: 2, days: 15}).weeks AS weeks""").records.toMaps should equal( + morpheus + .cypher("""RETURN duration({years: 2, days: 15}).weeks AS weeks""") + .records + .toMaps should equal( Bag( CypherMap("weeks" -> 2) ) @@ -682,7 +1030,10 @@ class TemporalTests extends MorpheusTestSuite with ScanGraphInit { } it("supports days") { - morpheus.cypher("""RETURN duration({years: 2, days: 14}).days AS days""").records.toMaps should equal( + morpheus + .cypher("""RETURN duration({years: 2, days: 14}).days AS days""") + .records + .toMaps should equal( Bag( CypherMap("days" -> 14) ) @@ -690,7 +1041,12 @@ class TemporalTests extends MorpheusTestSuite with ScanGraphInit { } it("supports hours") { - morpheus.cypher("""RETURN duration({years: 2, hours: 5, minutes: 60}).hours AS hours""").records.toMaps should equal( + morpheus + .cypher( + """RETURN duration({years: 2, hours: 5, minutes: 60}).hours AS hours""" + ) + .records + .toMaps should equal( Bag( CypherMap("hours" -> 6) ) @@ -698,7 +1054,12 @@ class TemporalTests extends MorpheusTestSuite with ScanGraphInit { } it("supports minutes") { - morpheus.cypher("""RETURN duration({years: 2, hours: 2, minutes: 2}).minutes AS minutes""").records.toMaps should equal( + morpheus + .cypher( + """RETURN duration({years: 2, hours: 2, minutes: 2}).minutes AS minutes""" + ) + .records + .toMaps should equal( Bag( CypherMap("minutes" -> 122) ) @@ -706,7 +1067,12 @@ class TemporalTests extends MorpheusTestSuite with ScanGraphInit { } it("supports seconds") { - morpheus.cypher("""RETURN duration({years: 2, minutes: 1, seconds: 2}).seconds AS seconds""").records.toMaps should equal( + morpheus + .cypher( + """RETURN duration({years: 2, minutes: 1, seconds: 2}).seconds AS seconds""" + ) + .records + .toMaps should equal( Bag( CypherMap("seconds" -> 62) ) @@ -714,7 +1080,12 @@ class TemporalTests extends MorpheusTestSuite with ScanGraphInit { } it("supports milliseconds") { - morpheus.cypher("""RETURN duration({years: 2, milliseconds: 142}).milliseconds AS millis""").records.toMaps should equal( + morpheus + .cypher( + """RETURN duration({years: 2, milliseconds: 142}).milliseconds AS millis""" + ) + .records + .toMaps should equal( Bag( CypherMap("millis" -> 142) ) @@ -722,7 +1093,12 @@ class TemporalTests extends MorpheusTestSuite with ScanGraphInit { } it("supports microseconds") { - morpheus.cypher("""RETURN duration({years: 2, microseconds: 142}).microseconds AS micros""").records.toMaps should equal( + morpheus + .cypher( + """RETURN duration({years: 2, microseconds: 142}).microseconds AS micros""" + ) + .records + .toMaps should equal( Bag( CypherMap("micros" -> 142) ) @@ -731,7 +1107,12 @@ class TemporalTests extends MorpheusTestSuite with ScanGraphInit { // not supported as Spark CalendarInterval does not support nanoseconds ignore("supports nanoseconds") { - morpheus.cypher("""RETURN duration({years: 2, microseconds: 1}).nanoseconds AS nanos""").records.toMaps should equal( + morpheus + .cypher( + """RETURN duration({years: 2, microseconds: 1}).nanoseconds AS nanos""" + ) + .records + .toMaps should equal( Bag( CypherMap("nanos" -> 1000) ) @@ -739,7 +1120,10 @@ class TemporalTests extends MorpheusTestSuite with ScanGraphInit { } it("propagates null") { - morpheus.cypher("""RETURN duration(null).years AS years""").records.toMaps should equal( + morpheus + .cypher("""RETURN duration(null).years AS years""") + .records + .toMaps should equal( Bag( CypherMap("years" -> null) ) diff --git a/morpheus-testing/src/test/scala/org/opencypher/morpheus/impl/acceptance/UnionTests.scala b/morpheus-testing/src/test/scala/org/opencypher/morpheus/impl/acceptance/UnionTests.scala index 8b90b8909f..8f9f12a6b0 100644 --- a/morpheus-testing/src/test/scala/org/opencypher/morpheus/impl/acceptance/UnionTests.scala +++ b/morpheus-testing/src/test/scala/org/opencypher/morpheus/impl/acceptance/UnionTests.scala @@ -37,22 +37,25 @@ class UnionTests extends MorpheusTestSuite with ScanGraphInit { describe("tabular union all") { it("unions simple queries") { - val result = morpheus.cypher( - """ + val result = morpheus + .cypher(""" |RETURN 1 AS one |UNION ALL |RETURN 2 AS one - """.stripMargin).records - - result.toMaps should equal(Bag( - CypherMap("one" -> 1), - CypherMap("one" -> 2) - )) + """.stripMargin) + .records + + result.toMaps should equal( + Bag( + CypherMap("one" -> 1), + CypherMap("one" -> 2) + ) + ) } it("supports stacked union all") { - val result = morpheus.cypher( - """ + val result = morpheus + .cypher(""" |RETURN 1 AS one |UNION ALL |RETURN 2 AS one @@ -60,113 +63,151 @@ class UnionTests extends MorpheusTestSuite with ScanGraphInit { |RETURN 2 AS one |UNION ALL |RETURN 3 AS one - """.stripMargin).records - - result.toMaps should equal(Bag( - CypherMap("one" -> 1), - CypherMap("one" -> 2), - CypherMap("one" -> 2), - CypherMap("one" -> 3) - )) + """.stripMargin) + .records + + result.toMaps should equal( + Bag( + CypherMap("one" -> 1), + CypherMap("one" -> 2), + CypherMap("one" -> 2), + CypherMap("one" -> 3) + ) + ) } it("supports union all with UNWIND") { - val result = morpheus.cypher( - """ + val result = morpheus + .cypher(""" |UNWIND [1, 2] AS i |RETURN i |UNION ALL |UNWIND [1, 2, 6] AS i |RETURN i - """.stripMargin).records - - result.toMaps should equal(Bag( - CypherMap("i" -> 1), - CypherMap("i" -> 2), - CypherMap("i" -> 1), - CypherMap("i" -> 2), - CypherMap("i" -> 6) - )) + """.stripMargin) + .records + + result.toMaps should equal( + Bag( + CypherMap("i" -> 1), + CypherMap("i" -> 2), + CypherMap("i" -> 1), + CypherMap("i" -> 2), + CypherMap("i" -> 6) + ) + ) } it("supports union all with MATCH on nodes") { - val g = initGraph( - """ + val g = initGraph(""" |CREATE (a: A {val: "foo"}) |CREATE (b: B {bar: "baz"}) """.stripMargin) - val result = g.cypher( - """ + val result = g + .cypher(""" |MATCH (a:A) |RETURN a AS node |UNION ALL |MATCH (b:B) |RETURN b AS node - """.stripMargin).records - - result.toMaps should equal(Bag( - CypherMap("node" -> MorpheusNode(0, Set("A"), CypherMap("val" -> "foo"))), - CypherMap("node" -> MorpheusNode(1, Set("B"), CypherMap("bar" -> "baz"))) - )) + """.stripMargin) + .records + + result.toMaps should equal( + Bag( + CypherMap( + "node" -> MorpheusNode(0, Set("A"), CypherMap("val" -> "foo")) + ), + CypherMap( + "node" -> MorpheusNode(1, Set("B"), CypherMap("bar" -> "baz")) + ) + ) + ) } it("supports union all with MATCH on nodes and relationships") { - val g = initGraph( - """ + val g = initGraph(""" |CREATE (a: A {val: "foo"}) |CREATE (b: B {bar: "baz"}) |CREATE (a)-[:REL1 {foo: 42}]->(b) |CREATE (b)-[:REL2 {bar: true}]->(a) """.stripMargin) - val result = g.cypher( - """ + val result = g + .cypher(""" |MATCH (a:A)-[r]->() |RETURN a AS node, r AS rel |UNION ALL |MATCH (b:B)-[r]->() |RETURN b AS node, r AS rel - """.stripMargin).records - - result.toMaps should equal(Bag( - CypherMap("node" -> MorpheusNode(0, Set("A"), CypherMap("val" -> "foo")), "rel" -> MorpheusRelationship(2, 0, 1, "REL1", CypherMap("foo" -> 42))), - CypherMap("node" -> MorpheusNode(1, Set("B"), CypherMap("bar" -> "baz")), "rel" -> MorpheusRelationship(3, 1, 0, "REL2", CypherMap("bar" -> true))) - )) + """.stripMargin) + .records + + result.toMaps should equal( + Bag( + CypherMap( + "node" -> MorpheusNode(0, Set("A"), CypherMap("val" -> "foo")), + "rel" -> MorpheusRelationship( + 2, + 0, + 1, + "REL1", + CypherMap("foo" -> 42) + ) + ), + CypherMap( + "node" -> MorpheusNode(1, Set("B"), CypherMap("bar" -> "baz")), + "rel" -> MorpheusRelationship( + 3, + 1, + 0, + "REL2", + CypherMap("bar" -> true) + ) + ) + ) + ) } } describe("tabular union") { it("unions simple queries") { - val result = morpheus.cypher( - """ + val result = morpheus + .cypher(""" |RETURN 1 AS one |UNION |RETURN 2 AS one - """.stripMargin).records - - result.toMaps should equal(Bag( - CypherMap("one" -> 1), - CypherMap("one" -> 2) - )) + """.stripMargin) + .records + + result.toMaps should equal( + Bag( + CypherMap("one" -> 1), + CypherMap("one" -> 2) + ) + ) } it("unions simple queries with duplicates") { - val result = morpheus.cypher( - """ + val result = morpheus + .cypher(""" |RETURN 1 AS one |UNION |RETURN 1 AS one - """.stripMargin).records + """.stripMargin) + .records - result.toMaps should equal(Bag( - CypherMap("one" -> 1) - )) + result.toMaps should equal( + Bag( + CypherMap("one" -> 1) + ) + ) } it("supports stacked union") { - val result = morpheus.cypher( - """ + val result = morpheus + .cypher(""" |RETURN 1 AS one |UNION |RETURN 2 AS one @@ -174,103 +215,134 @@ class UnionTests extends MorpheusTestSuite with ScanGraphInit { |RETURN 2 AS one |UNION |RETURN 3 AS one - """.stripMargin).records - - result.toMaps should equal(Bag( - CypherMap("one" -> 1), - CypherMap("one" -> 2), - CypherMap("one" -> 3) - )) + """.stripMargin) + .records + + result.toMaps should equal( + Bag( + CypherMap("one" -> 1), + CypherMap("one" -> 2), + CypherMap("one" -> 3) + ) + ) } it("supports union with UNWIND") { - val result = morpheus.cypher( - """ + val result = morpheus + .cypher(""" |UNWIND [1, 2] AS i |RETURN i |UNION |UNWIND [1, 2, 6] AS i |RETURN i - """.stripMargin).records - - result.toMaps should equal(Bag( - CypherMap("i" -> 1), - CypherMap("i" -> 2), - CypherMap("i" -> 6) - )) + """.stripMargin) + .records + + result.toMaps should equal( + Bag( + CypherMap("i" -> 1), + CypherMap("i" -> 2), + CypherMap("i" -> 6) + ) + ) } it("supports union with MATCH on nodes") { - val g = initGraph( - """ + val g = initGraph(""" |CREATE (a: A {val: "foo"}) |CREATE (b: B {bar: "baz"}) """.stripMargin) - val result = g.cypher( - """ + val result = g + .cypher(""" |MATCH (a:A), (b:B) |RETURN a AS node1, b AS node2 |UNION |MATCH (b:B), (a:A) |RETURN b AS node1, a AS node2 - """.stripMargin).records - - result.toMaps should equal(Bag( - CypherMap("node1" -> MorpheusNode(0, Set("A"), CypherMap("val" -> "foo")), "node2" -> MorpheusNode(1, Set("B"), CypherMap("bar" -> "baz"))), - CypherMap("node1" -> MorpheusNode(1, Set("B"), CypherMap("bar" -> "baz")), "node2" -> MorpheusNode(0, Set("A"), CypherMap("val" -> "foo"))) - )) + """.stripMargin) + .records + + result.toMaps should equal( + Bag( + CypherMap( + "node1" -> MorpheusNode(0, Set("A"), CypherMap("val" -> "foo")), + "node2" -> MorpheusNode(1, Set("B"), CypherMap("bar" -> "baz")) + ), + CypherMap( + "node1" -> MorpheusNode(1, Set("B"), CypherMap("bar" -> "baz")), + "node2" -> MorpheusNode(0, Set("A"), CypherMap("val" -> "foo")) + ) + ) + ) } it("supports union on duplicate nodes") { - val g = initGraph( - """ + val g = initGraph(""" |CREATE (a: A {val: "foo"}) """.stripMargin) - val result = g.cypher( - """ + val result = g + .cypher(""" |MATCH (a:A) |RETURN a AS node |UNION |MATCH (a:A) |RETURN a AS node - """.stripMargin).records - - result.toMaps should equal(Bag( - CypherMap("node" -> MorpheusNode(0, Set("A"), CypherMap("val" -> "foo"))) - )) + """.stripMargin) + .records + + result.toMaps should equal( + Bag( + CypherMap( + "node" -> MorpheusNode(0, Set("A"), CypherMap("val" -> "foo")) + ) + ) + ) } it("supports union on duplicate relationships") { - val g = initGraph( - """ + val g = initGraph(""" |CREATE (a) |CREATE (a)-[:REL {val: 42}]->(a) """.stripMargin) - val result = g.cypher( - """ + val result = g + .cypher(""" |MATCH ()-[r]->() |RETURN r AS rel |UNION |MATCH ()-[r]->() |RETURN r AS rel - """.stripMargin).records - - result.toMaps should equal(Bag( - CypherMap("rel" -> MorpheusRelationship(1, 0, 0, "REL", CypherMap("val" -> 42))) - )) + """.stripMargin) + .records + + result.toMaps should equal( + Bag( + CypherMap( + "rel" -> MorpheusRelationship( + 1, + 0, + 0, + "REL", + CypherMap("val" -> 42) + ) + ) + ) + ) } } describe("Graph union all") { it("union all on graphs") { val a = initGraph("CREATE ()") - morpheus.catalog.source(morpheus.catalog.sessionNamespace).store(GraphName("a"), a) - morpheus.catalog.source(morpheus.catalog.sessionNamespace).store(GraphName("b"), a) - val result = morpheus.cypher( - """ + morpheus.catalog + .source(morpheus.catalog.sessionNamespace) + .store(GraphName("a"), a) + morpheus.catalog + .source(morpheus.catalog.sessionNamespace) + .store(GraphName("b"), a) + val result = morpheus.cypher(""" |FROM a |RETURN GRAPH |UNION ALL @@ -284,20 +356,27 @@ class UnionTests extends MorpheusTestSuite with ScanGraphInit { it("union all fails on graphs with common properties") { val a = initGraph("CREATE (:one{test:1})") val b = initGraph("CREATE (:one{test:'hello'})") - morpheus.catalog.source(morpheus.catalog.sessionNamespace).store(GraphName("a"), a) - morpheus.catalog.source(morpheus.catalog.sessionNamespace).store(GraphName("b"), b) + morpheus.catalog + .source(morpheus.catalog.sessionNamespace) + .store(GraphName("a"), a) + morpheus.catalog + .source(morpheus.catalog.sessionNamespace) + .store(GraphName("b"), b) val e: SchemaException = the[SchemaException] thrownBy { - morpheus.cypher( - """ + morpheus + .cypher(""" |FROM a |RETURN GRAPH |UNION ALL |FROM b |RETURN GRAPH - """.stripMargin).graph + """.stripMargin) + .graph } - e.getMessage should (include("one") and include("test") and include("STRING") and include("INTEGER")) + e.getMessage should (include("one") and include("test") and include( + "STRING" + ) and include("INTEGER")) } } } diff --git a/morpheus-testing/src/test/scala/org/opencypher/morpheus/impl/acceptance/UnwindTests.scala b/morpheus-testing/src/test/scala/org/opencypher/morpheus/impl/acceptance/UnwindTests.scala index a43a0957d4..bd250f80f3 100644 --- a/morpheus-testing/src/test/scala/org/opencypher/morpheus/impl/acceptance/UnwindTests.scala +++ b/morpheus-testing/src/test/scala/org/opencypher/morpheus/impl/acceptance/UnwindTests.scala @@ -43,7 +43,8 @@ class UnwindTests extends MorpheusTestSuite with ScanGraphInit { CypherMap("item" -> 1), CypherMap("item" -> 2), CypherMap("item" -> 3) - )) + ) + ) } it("standalone unwind from literal") { @@ -56,7 +57,8 @@ class UnwindTests extends MorpheusTestSuite with ScanGraphInit { CypherMap("item" -> 1), CypherMap("item" -> 2), CypherMap("item" -> 3) - )) + ) + ) } it("unwind after match") { @@ -68,19 +70,40 @@ class UnwindTests extends MorpheusTestSuite with ScanGraphInit { result.records.toMaps.map(_.toString) should equal( Bag( - CypherMap("a" -> MorpheusNode(0L, Set("A"), CypherMap.empty), "item" -> 1), - CypherMap("a" -> MorpheusNode(0L, Set("A"), CypherMap.empty), "item" -> 2), - CypherMap("a" -> MorpheusNode(0L, Set("A"), CypherMap.empty), "item" -> 3), - CypherMap("a" -> MorpheusNode(1L, Set("B"), CypherMap("item" -> "1")), "item" -> 1), - CypherMap("a" -> MorpheusNode(1L, Set("B"), CypherMap("item" -> "1")), "item" -> 2), - CypherMap("a" -> MorpheusNode(1L, Set("B"), CypherMap("item" -> "1")), "item" -> 3) - ).map(_.toString)) + CypherMap( + "a" -> MorpheusNode(0L, Set("A"), CypherMap.empty), + "item" -> 1 + ), + CypherMap( + "a" -> MorpheusNode(0L, Set("A"), CypherMap.empty), + "item" -> 2 + ), + CypherMap( + "a" -> MorpheusNode(0L, Set("A"), CypherMap.empty), + "item" -> 3 + ), + CypherMap( + "a" -> MorpheusNode(1L, Set("B"), CypherMap("item" -> "1")), + "item" -> 1 + ), + CypherMap( + "a" -> MorpheusNode(1L, Set("B"), CypherMap("item" -> "1")), + "item" -> 2 + ), + CypherMap( + "a" -> MorpheusNode(1L, Set("B"), CypherMap("item" -> "1")), + "item" -> 3 + ) + ).map(_.toString) + ) } it("unwind from expression, aggregation") { - val graph = initGraph("CREATE (:A {v: 1}), (:A:B {v: 15}), (:A:C {v: -32}), (:A)") + val graph = + initGraph("CREATE (:A {v: 1}), (:A:B {v: 15}), (:A:C {v: -32}), (:A)") - val query = "MATCH (a:A) WITH collect(a.v) AS list UNWIND list AS item RETURN item" + val query = + "MATCH (a:A) WITH collect(a.v) AS list UNWIND list AS item RETURN item" val result = graph.cypher(query, Map("param" -> CypherList(1, 2, 3))) @@ -89,7 +112,8 @@ class UnwindTests extends MorpheusTestSuite with ScanGraphInit { CypherMap("item" -> 1), CypherMap("item" -> 15), CypherMap("item" -> -32) - )) + ) + ) } it("unwind from expression") { @@ -104,7 +128,8 @@ class UnwindTests extends MorpheusTestSuite with ScanGraphInit { CypherMap("item" -> 1), CypherMap("item" -> 2), CypherMap("item" -> -4) - )) + ) + ) } // https://issues.apache.org/jira/browse/SPARK-23610 @@ -112,7 +137,9 @@ class UnwindTests extends MorpheusTestSuite with ScanGraphInit { // We lack a way of encoding empty lists in a way that expands to a specific list type in Spark // Like putting it in a column with Array(Integer) as in this test ignore("unwind from expression with empty and null lists") { - val graph = initGraph("CREATE (:A {v: [1, 2]}), (:A:B {v: [-4]}), (:A:C {v: []}), (:A)") + val graph = initGraph( + "CREATE (:A {v: [1, 2]}), (:A:B {v: [-4]}), (:A:C {v: []}), (:A)" + ) val query = "MATCH (a:A) WITH a.v AS list UNWIND list AS item RETURN item" @@ -123,7 +150,8 @@ class UnwindTests extends MorpheusTestSuite with ScanGraphInit { CypherMap("item" -> 1), CypherMap("item" -> 2), CypherMap("item" -> -4) - )) + ) + ) } it("unwind from literal null expression") { @@ -168,10 +196,22 @@ class UnwindTests extends MorpheusTestSuite with ScanGraphInit { result.records.toMaps should equal( Bag( - CypherMap("a" -> MorpheusNode(0L, Set("A"), CypherMap.empty), "item" -> 3), - CypherMap("a" -> MorpheusNode(1L, Set("B"), CypherMap("item" -> "1")), "item" -> 3), - CypherMap("a" -> MorpheusNode(0L, Set("A"), CypherMap.empty), "item" -> 2), - CypherMap("a" -> MorpheusNode(1L, Set("B"), CypherMap("item" -> "1")), "item" -> 2) + CypherMap( + "a" -> MorpheusNode(0L, Set("A"), CypherMap.empty), + "item" -> 3 + ), + CypherMap( + "a" -> MorpheusNode(1L, Set("B"), CypherMap("item" -> "1")), + "item" -> 3 + ), + CypherMap( + "a" -> MorpheusNode(0L, Set("A"), CypherMap.empty), + "item" -> 2 + ), + CypherMap( + "a" -> MorpheusNode(1L, Set("B"), CypherMap("item" -> "1")), + "item" -> 2 + ) ) ) } diff --git a/morpheus-testing/src/test/scala/org/opencypher/morpheus/impl/acceptance/WithTests.scala b/morpheus-testing/src/test/scala/org/opencypher/morpheus/impl/acceptance/WithTests.scala index 689397fed2..224d9c1598 100644 --- a/morpheus-testing/src/test/scala/org/opencypher/morpheus/impl/acceptance/WithTests.scala +++ b/morpheus-testing/src/test/scala/org/opencypher/morpheus/impl/acceptance/WithTests.scala @@ -39,8 +39,7 @@ class WithTests extends MorpheusTestSuite with ScanGraphInit { val given = initGraph("""CREATE (:Node {val: 1}), (:Node {val: 2})""") // When - val result = given.cypher( - """MATCH (n:Node) + val result = given.cypher("""MATCH (n:Node) |WITH n.val AS foo |WITH foo + 2 AS bar |WITH bar + 2 AS foo @@ -52,7 +51,8 @@ class WithTests extends MorpheusTestSuite with ScanGraphInit { Bag( CypherMap("foo" -> 5), CypherMap("foo" -> 6) - )) + ) + ) } test("projecting constants") { @@ -60,8 +60,7 @@ class WithTests extends MorpheusTestSuite with ScanGraphInit { val given = initGraph("""CREATE (), ()""") // When - val result = given.cypher( - """MATCH () + val result = given.cypher("""MATCH () |WITH 3 AS foo |WITH foo + 2 AS bar |RETURN bar @@ -72,125 +71,152 @@ class WithTests extends MorpheusTestSuite with ScanGraphInit { Bag( CypherMap("bar" -> 5), CypherMap("bar" -> 5) - )) + ) + ) } test("projecting variables in scope") { // Given - val given = initGraph("""CREATE (:Node {val: 4})-[:Rel]->(:Node {val: 5})""") + val given = + initGraph("""CREATE (:Node {val: 4})-[:Rel]->(:Node {val: 5})""") // When - val result = given.cypher("MATCH (n:Node)-->(m:Node) WITH n, m RETURN n.val") + val result = + given.cypher("MATCH (n:Node)-->(m:Node) WITH n, m RETURN n.val") // Then result.records.toMaps should equal( Bag( CypherMap("n.val" -> 4) - )) + ) + ) } test("projecting property expression") { // Given - val given = initGraph("""CREATE (:Node {val: 4})-[:Rel]->(:Node {val: 5})""") + val given = + initGraph("""CREATE (:Node {val: 4})-[:Rel]->(:Node {val: 5})""") // When - val result = given.cypher("MATCH (n:Node)-->(m:Node) WITH n.val AS n_val RETURN n_val") + val result = + given.cypher("MATCH (n:Node)-->(m:Node) WITH n.val AS n_val RETURN n_val") // Then result.records.toMaps should equal( Bag( CypherMap("n_val" -> 4) - )) + ) + ) } test("projecting property expression with filter") { // Given - val given = initGraph("""CREATE (:Node {val: 3}), (:Node {val: 4}), (:Node {val: 5})""") + val given = initGraph( + """CREATE (:Node {val: 3}), (:Node {val: 4}), (:Node {val: 5})""" + ) // When - val result = given.cypher("MATCH (n:Node) WITH n.val AS n_val WHERE n_val <= 4 RETURN n_val") + val result = given.cypher( + "MATCH (n:Node) WITH n.val AS n_val WHERE n_val <= 4 RETURN n_val" + ) // Then result.records.toMaps should equal( Bag( CypherMap("n_val" -> 3), CypherMap("n_val" -> 4) - )) + ) + ) } test("projecting addition expression") { // Given - val given = initGraph("""CREATE (:Node {val: 4})-[:Rel]->(:Node {val: 5})""") + val given = + initGraph("""CREATE (:Node {val: 4})-[:Rel]->(:Node {val: 5})""") // When - val result = given.cypher("MATCH (n:Node)-->(m:Node) WITH n.val + m.val AS sum_n_m_val RETURN sum_n_m_val") + val result = given.cypher( + "MATCH (n:Node)-->(m:Node) WITH n.val + m.val AS sum_n_m_val RETURN sum_n_m_val" + ) // Then result.records.toMaps should equal( Bag( CypherMap("sum_n_m_val" -> 9) - )) + ) + ) } test("aliasing variables") { // Given - val given = initGraph("""CREATE (:Node {val: 4})-[:Rel]->(:Node {val: 5})""") + val given = + initGraph("""CREATE (:Node {val: 4})-[:Rel]->(:Node {val: 5})""") // When - val result = given.cypher("MATCH (n:Node)-[r]->(m:Node) WITH n.val + m.val AS sum WITH sum AS sum2 RETURN sum2") + val result = given.cypher( + "MATCH (n:Node)-[r]->(m:Node) WITH n.val + m.val AS sum WITH sum AS sum2 RETURN sum2" + ) // Then result.records.toMaps should equal( Bag( CypherMap("sum2" -> 9) - )) + ) + ) } test("projecting mixed expression") { // Given - val given = initGraph("""CREATE (:Node {val: 4})-[:Rel]->(:Node {val: 5})-[:Rel]->(:Node)""") + val given = initGraph( + """CREATE (:Node {val: 4})-[:Rel]->(:Node {val: 5})-[:Rel]->(:Node)""" + ) // When val result = given.cypher( - "MATCH (n:Node)-[r]->(m:Node) WITH n.val AS n_val, n.val + m.val AS sum_n_m_val RETURN sum_n_m_val, n_val") + "MATCH (n:Node)-[r]->(m:Node) WITH n.val AS n_val, n.val + m.val AS sum_n_m_val RETURN sum_n_m_val, n_val" + ) // Then result.records.toMaps should equal( Bag( CypherMap("sum_n_m_val" -> 9, "n_val" -> 4), CypherMap("sum_n_m_val" -> null, "n_val" -> 5) - )) + ) + ) } it("can project and predicates") { - val graph = initGraph( - """ + val graph = initGraph(""" |CREATE ({val1: 1, val2: 3, val3: 10}), ({val1: 1, val2: 2, val3: 3}) """.stripMargin) - val result = graph.cypher( - """ + val result = graph.cypher(""" |MATCH (n) |WITH n.val1 AS val1, n.val2 AS val2, n.val3 AS val3 |WHERE val1 >= 1 AND val2 > 2 AND val3 > 5 |RETURN val1, val2, val3 """.stripMargin) - result.records.collect.toBag should equal(Bag( - CypherMap("val1" -> 1, "val2" -> 3, "val3" -> 10) - )) + result.records.collect.toBag should equal( + Bag( + CypherMap("val1" -> 1, "val2" -> 3, "val3" -> 10) + ) + ) } test("order by") { - val given = initGraph("""CREATE (:Node {val: 4}),(:Node {val: 3}),(:Node {val: 42})""") + val given = initGraph( + """CREATE (:Node {val: 4}),(:Node {val: 3}),(:Node {val: 42})""" + ) - val result = given.cypher("MATCH (a) WITH a.val as val ORDER BY val RETURN val") + val result = + given.cypher("MATCH (a) WITH a.val as val ORDER BY val RETURN val") // Then result.records.toMaps should equal( @@ -198,13 +224,17 @@ class WithTests extends MorpheusTestSuite with ScanGraphInit { CypherMap("val" -> 3L), CypherMap("val" -> 4L), CypherMap("val" -> 42L) - )) + ) + ) } test("order by asc") { - val given = initGraph("""CREATE (:Node {val: 4}),(:Node {val: 3}),(:Node {val: 42})""") + val given = initGraph( + """CREATE (:Node {val: 4}),(:Node {val: 3}),(:Node {val: 42})""" + ) - val result = given.cypher("MATCH (a) WITH a.val as val ORDER BY val ASC RETURN val") + val result = + given.cypher("MATCH (a) WITH a.val as val ORDER BY val ASC RETURN val") // Then result.records.toMaps should equal( @@ -212,13 +242,17 @@ class WithTests extends MorpheusTestSuite with ScanGraphInit { CypherMap("val" -> 3L), CypherMap("val" -> 4L), CypherMap("val" -> 42L) - )) + ) + ) } test("order by desc") { - val given = initGraph("""CREATE (:Node {val: 4}),(:Node {val: 3}),(:Node {val: 42})""") + val given = initGraph( + """CREATE (:Node {val: 4}),(:Node {val: 3}),(:Node {val: 42})""" + ) - val result = given.cypher("MATCH (a) WITH a.val as val ORDER BY val DESC RETURN val") + val result = + given.cypher("MATCH (a) WITH a.val as val ORDER BY val DESC RETURN val") // Then result.records.toMaps should equal( @@ -226,11 +260,14 @@ class WithTests extends MorpheusTestSuite with ScanGraphInit { CypherMap("val" -> 42L), CypherMap("val" -> 4L), CypherMap("val" -> 3L) - )) + ) + ) } test("skip") { - val given = initGraph("""CREATE (:Node {val: 4}),(:Node {val: 3}),(:Node {val: 42})""") + val given = initGraph( + """CREATE (:Node {val: 4}),(:Node {val: 3}),(:Node {val: 42})""" + ) val result = given.cypher("MATCH (a) WITH a.val as val SKIP 2 RETURN val") @@ -239,32 +276,43 @@ class WithTests extends MorpheusTestSuite with ScanGraphInit { } test("order by with skip") { - val given = initGraph("""CREATE (:Node {val: 4}),(:Node {val: 3}),(:Node {val: 42})""") + val given = initGraph( + """CREATE (:Node {val: 4}),(:Node {val: 3}),(:Node {val: 42})""" + ) - val result = given.cypher("MATCH (a) WITH a.val as val ORDER BY val SKIP 1 RETURN val") + val result = + given.cypher("MATCH (a) WITH a.val as val ORDER BY val SKIP 1 RETURN val") // Then result.records.toMaps should equal( Bag( CypherMap("val" -> 4L), CypherMap("val" -> 42L) - )) + ) + ) } test("order by with (arithmetic) skip") { - val given = initGraph("""CREATE (:Node {val: 4}),(:Node {val: 3}),(:Node {val: 42})""") + val given = initGraph( + """CREATE (:Node {val: 4}),(:Node {val: 3}),(:Node {val: 42})""" + ) - val result = given.cypher("MATCH (a) WITH a.val as val ORDER BY val SKIP 1 + 1 RETURN val") + val result = given.cypher( + "MATCH (a) WITH a.val as val ORDER BY val SKIP 1 + 1 RETURN val" + ) // Then result.records.toMaps should equal( Bag( CypherMap("val" -> 42L) - )) + ) + ) } test("limit") { - val given = initGraph("""CREATE (:Node {val: 4}),(:Node {val: 3}),(:Node {val: 42})""") + val given = initGraph( + """CREATE (:Node {val: 4}),(:Node {val: 3}),(:Node {val: 42})""" + ) val result = given.cypher("MATCH (a) WITH a.val as val LIMIT 1 RETURN val") @@ -273,83 +321,97 @@ class WithTests extends MorpheusTestSuite with ScanGraphInit { } test("order by with limit") { - val given = initGraph("""CREATE (:Node {val: 4}),(:Node {val: 3}),(:Node {val: 42})""") + val given = initGraph( + """CREATE (:Node {val: 4}),(:Node {val: 3}),(:Node {val: 42})""" + ) - val result = given.cypher("MATCH (a) WITH a.val as val ORDER BY val LIMIT 1 RETURN val") + val result = given.cypher( + "MATCH (a) WITH a.val as val ORDER BY val LIMIT 1 RETURN val" + ) // Then result.records.toMaps should equal( Bag( CypherMap("val" -> 3L) - )) + ) + ) } test("order by with (arithmetic) limit") { - val given = initGraph("""CREATE (:Node {val: 4}),(:Node {val: 3}),(:Node {val: 42})""") + val given = initGraph( + """CREATE (:Node {val: 4}),(:Node {val: 3}),(:Node {val: 42})""" + ) - val result = given.cypher("MATCH (a) WITH a.val as val ORDER BY val LIMIT 1 + 1 RETURN val") + val result = given.cypher( + "MATCH (a) WITH a.val as val ORDER BY val LIMIT 1 + 1 RETURN val" + ) // Then result.records.toMaps should equal( Bag( CypherMap("val" -> 3L), CypherMap("val" -> 4L) - )) + ) + ) } test("order by with skip and limit") { - val given = initGraph("""CREATE (:Node {val: 4}),(:Node {val: 3}),(:Node {val: 42})""") + val given = initGraph( + """CREATE (:Node {val: 4}),(:Node {val: 3}),(:Node {val: 42})""" + ) - val result = given.cypher("MATCH (a) WITH a.val as val ORDER BY val SKIP 1 LIMIT 1 RETURN val") + val result = given.cypher( + "MATCH (a) WITH a.val as val ORDER BY val SKIP 1 LIMIT 1 RETURN val" + ) // Then result.records.toMaps should equal( Bag( CypherMap("val" -> 4L) - )) + ) + ) } - describe("NOT") { it("can project not of literal") { // Given - val given = initGraph( - """ + val given = initGraph(""" |CREATE () """.stripMargin) // When - val result = given.cypher( - """ + val result = given.cypher(""" |WITH true AS t, false AS f |WITH NOT true AS nt, not false AS nf |RETURN nt, nf""".stripMargin) // Then - result.records.toMaps should equal(Bag( - CypherMap("nt" -> false, "nf" -> true) - )) + result.records.toMaps should equal( + Bag( + CypherMap("nt" -> false, "nf" -> true) + ) + ) } it("can project not of expression") { // Given - val given = initGraph( - """ + val given = initGraph(""" |CREATE ({id: 1, val: true}), ({id: 2, val: false}) """.stripMargin) // When - val result = given.cypher( - """ + val result = given.cypher(""" |MATCH (n) |WITH n.id AS id, NOT n.val AS val2 |RETURN id, val2""".stripMargin) // Then - result.records.toMaps should equal(Bag( - CypherMap("id" -> 1L, "val2" -> false), - CypherMap("id" -> 2L, "val2" -> true) - )) + result.records.toMaps should equal( + Bag( + CypherMap("id" -> 1L, "val2" -> false), + CypherMap("id" -> 2L, "val2" -> true) + ) + ) } } } diff --git a/morpheus-testing/src/test/scala/org/opencypher/morpheus/impl/convert/ConvertersTest.scala b/morpheus-testing/src/test/scala/org/opencypher/morpheus/impl/convert/ConvertersTest.scala index 341120c0c9..41f32bc128 100644 --- a/morpheus-testing/src/test/scala/org/opencypher/morpheus/impl/convert/ConvertersTest.scala +++ b/morpheus-testing/src/test/scala/org/opencypher/morpheus/impl/convert/ConvertersTest.scala @@ -45,10 +45,9 @@ class ConvertersTest extends BaseTestSuite { BinaryType -> CTIdentity ) - mappings.foreach { - case (spark, cypher) => - spark.toCypherType(false) should equal(Some(cypher)) - spark.toCypherType(true) should equal(Some(cypher.nullable)) + mappings.foreach { case (spark, cypher) => + spark.toCypherType(false) should equal(Some(cypher)) + spark.toCypherType(true) should equal(Some(cypher.nullable)) } } @@ -74,9 +73,8 @@ class ConvertersTest extends BaseTestSuite { CTRelationship("BAR") -> BinaryType ) - mappings.foreach { - case (cypher, spark) => - cypher.getSparkType should equal(spark) + mappings.foreach { case (cypher, spark) => + cypher.getSparkType should equal(spark) } } @@ -92,13 +90,14 @@ class ConvertersTest extends BaseTestSuite { java.lang.Boolean.TRUE -> CTTrue, Array(1) -> CTList(CTInteger), Array() -> CTEmptyList, - Array(Int.box(1), Double.box(3.14)) -> CTList(CTUnion(CTInteger, CTFloat)), + Array(Int.box(1), Double.box(3.14)) -> CTList( + CTUnion(CTInteger, CTFloat) + ), Array(null, "foo") -> CTList(CTString.nullable) ) - mappings.foreach { - case (spark, cypher) => - CypherValue(spark).cypherType should equal(cypher) + mappings.foreach { case (spark, cypher) => + CypherValue(spark).cypherType should equal(cypher) } } } diff --git a/morpheus-testing/src/test/scala/org/opencypher/morpheus/impl/convert/SparkConversionsTest.scala b/morpheus-testing/src/test/scala/org/opencypher/morpheus/impl/convert/SparkConversionsTest.scala index 872f2c681c..8578518e36 100644 --- a/morpheus-testing/src/test/scala/org/opencypher/morpheus/impl/convert/SparkConversionsTest.scala +++ b/morpheus-testing/src/test/scala/org/opencypher/morpheus/impl/convert/SparkConversionsTest.scala @@ -34,54 +34,151 @@ import org.opencypher.okapi.testing.BaseTestSuite class SparkConversionsTest extends BaseTestSuite { it("should produce the correct StructField for non-nested types") { - CTInteger.toStructField("foo") should equal(StructField("foo", LongType, nullable = false)) - CTInteger.nullable.toStructField("foo") should equal(StructField("foo", LongType, nullable = true)) - CTFloat.toStructField("foo") should equal(StructField("foo", DoubleType, nullable = false)) - CTFloat.nullable.toStructField("foo") should equal(StructField("foo", DoubleType, nullable = true)) - CTBigDecimal(38, 12).toStructField("foo") should equal(StructField("foo", DataTypes.createDecimalType(38,12), nullable = false)) - CTBigDecimal(10, 5).nullable.toStructField("foo") should equal(StructField("foo", DataTypes.createDecimalType(10,5), nullable = true)) - CTBoolean.toStructField("foo") should equal(StructField("foo", BooleanType, nullable = false)) - CTBoolean.nullable.toStructField("foo") should equal(StructField("foo", BooleanType, nullable = true)) - CTString.toStructField("foo") should equal(StructField("foo", StringType, nullable = false)) - CTString.nullable.toStructField("foo") should equal(StructField("foo", StringType, nullable = true)) - CTLocalDateTime.toStructField("foo") should equal(StructField("foo", TimestampType, nullable = false)) - CTLocalDateTime.nullable.toStructField("foo") should equal(StructField("foo", TimestampType, nullable = true)) - CTDate.toStructField("foo") should equal(StructField("foo", DateType, nullable = false)) - CTDate.nullable.toStructField("foo") should equal(StructField("foo", DateType, nullable = true)) + CTInteger.toStructField("foo") should equal( + StructField("foo", LongType, nullable = false) + ) + CTInteger.nullable.toStructField("foo") should equal( + StructField("foo", LongType, nullable = true) + ) + CTFloat.toStructField("foo") should equal( + StructField("foo", DoubleType, nullable = false) + ) + CTFloat.nullable.toStructField("foo") should equal( + StructField("foo", DoubleType, nullable = true) + ) + CTBigDecimal(38, 12).toStructField("foo") should equal( + StructField("foo", DataTypes.createDecimalType(38, 12), nullable = false) + ) + CTBigDecimal(10, 5).nullable.toStructField("foo") should equal( + StructField("foo", DataTypes.createDecimalType(10, 5), nullable = true) + ) + CTBoolean.toStructField("foo") should equal( + StructField("foo", BooleanType, nullable = false) + ) + CTBoolean.nullable.toStructField("foo") should equal( + StructField("foo", BooleanType, nullable = true) + ) + CTString.toStructField("foo") should equal( + StructField("foo", StringType, nullable = false) + ) + CTString.nullable.toStructField("foo") should equal( + StructField("foo", StringType, nullable = true) + ) + CTLocalDateTime.toStructField("foo") should equal( + StructField("foo", TimestampType, nullable = false) + ) + CTLocalDateTime.nullable.toStructField("foo") should equal( + StructField("foo", TimestampType, nullable = true) + ) + CTDate.toStructField("foo") should equal( + StructField("foo", DateType, nullable = false) + ) + CTDate.nullable.toStructField("foo") should equal( + StructField("foo", DateType, nullable = true) + ) - CTNode(Set("A")).toStructField("foo") should equal(StructField("foo", BinaryType, nullable = false)) - CTNode(Set("A")).nullable.toStructField("foo") should equal(StructField("foo", BinaryType, nullable = true)) - CTRelationship("A").toStructField("foo") should equal(StructField("foo", BinaryType, nullable = false)) - CTRelationship("A").nullable.toStructField("foo") should equal(StructField("foo", BinaryType, nullable = true)) + CTNode(Set("A")).toStructField("foo") should equal( + StructField("foo", BinaryType, nullable = false) + ) + CTNode(Set("A")).nullable.toStructField("foo") should equal( + StructField("foo", BinaryType, nullable = true) + ) + CTRelationship("A").toStructField("foo") should equal( + StructField("foo", BinaryType, nullable = false) + ) + CTRelationship("A").nullable.toStructField("foo") should equal( + StructField("foo", BinaryType, nullable = true) + ) } it("should produce the correct StructField for nested types") { - CTList(CTInteger).toStructField("foo") should equal(StructField("foo", ArrayType(LongType, containsNull = false), nullable = false)) - CTList(CTInteger.nullable).toStructField("foo") should equal(StructField("foo", ArrayType(LongType, containsNull = true), nullable = false)) - CTList(CTInteger).nullable.toStructField("foo") should equal(StructField("foo", ArrayType(LongType, containsNull = false), nullable = true)) - CTList(CTInteger.nullable).nullable.toStructField("foo") should equal(StructField("foo", ArrayType(LongType, containsNull = true), nullable = true)) + CTList(CTInteger).toStructField("foo") should equal( + StructField( + "foo", + ArrayType(LongType, containsNull = false), + nullable = false + ) + ) + CTList(CTInteger.nullable).toStructField("foo") should equal( + StructField( + "foo", + ArrayType(LongType, containsNull = true), + nullable = false + ) + ) + CTList(CTInteger).nullable.toStructField("foo") should equal( + StructField( + "foo", + ArrayType(LongType, containsNull = false), + nullable = true + ) + ) + CTList(CTInteger.nullable).nullable.toStructField("foo") should equal( + StructField( + "foo", + ArrayType(LongType, containsNull = true), + nullable = true + ) + ) CTList(CTList(CTInteger)).toStructField("foo") should equal( - StructField("foo", ArrayType(ArrayType(LongType, containsNull = false), containsNull = false), nullable = false) + StructField( + "foo", + ArrayType( + ArrayType(LongType, containsNull = false), + containsNull = false + ), + nullable = false + ) ) CTList(CTList(CTInteger.nullable)).toStructField("foo") should equal( - StructField("foo", ArrayType(ArrayType(LongType, containsNull = true), containsNull = false), nullable = false) + StructField( + "foo", + ArrayType( + ArrayType(LongType, containsNull = true), + containsNull = false + ), + nullable = false + ) ) - CTList(CTList(CTInteger.nullable).nullable).toStructField("foo") should equal( - StructField("foo", ArrayType(ArrayType(LongType, containsNull = true), containsNull = true), nullable = false) + CTList(CTList(CTInteger.nullable).nullable) + .toStructField("foo") should equal( + StructField( + "foo", + ArrayType( + ArrayType(LongType, containsNull = true), + containsNull = true + ), + nullable = false + ) ) - CTList(CTList(CTInteger.nullable).nullable).nullable.toStructField("foo") should equal( - StructField("foo", ArrayType(ArrayType(LongType, containsNull = true), containsNull = true), nullable = true) + CTList(CTList(CTInteger.nullable).nullable).nullable + .toStructField("foo") should equal( + StructField( + "foo", + ArrayType( + ArrayType(LongType, containsNull = true), + containsNull = true + ), + nullable = true + ) ) - CTMap(Map("foo" -> CTInteger, "bar" -> CTString.nullable)).toStructField("myMap") should equal( - StructField("myMap", StructType(Seq( - StructField("foo", LongType, nullable = false), - StructField("bar", StringType, nullable = true) - )), nullable = false) + CTMap(Map("foo" -> CTInteger, "bar" -> CTString.nullable)) + .toStructField("myMap") should equal( + StructField( + "myMap", + StructType( + Seq( + StructField("foo", LongType, nullable = false), + StructField("bar", StringType, nullable = true) + ) + ), + nullable = false + ) ) } } diff --git a/morpheus-testing/src/test/scala/org/opencypher/morpheus/impl/encoders/EncodeLongTest.scala b/morpheus-testing/src/test/scala/org/opencypher/morpheus/impl/encoders/EncodeLongTest.scala index c4388d6e7a..6fb8349d99 100644 --- a/morpheus-testing/src/test/scala/org/opencypher/morpheus/impl/encoders/EncodeLongTest.scala +++ b/morpheus-testing/src/test/scala/org/opencypher/morpheus/impl/encoders/EncodeLongTest.scala @@ -39,19 +39,28 @@ import org.scalatestplus.scalacheck.Checkers class EncodeLongTest extends MorpheusTestSuite with Checkers { it("encodes longs correctly") { - check((l: Long) => { - val scala = l.encodeAsMorpheusId.toList - val spark = typedLit[Long](l).encodeLongAsMorpheusId.expr.eval().asInstanceOf[Array[Byte]].toList - scala === spark - }, minSuccessful(1000)) + check( + (l: Long) => { + val scala = l.encodeAsMorpheusId.toList + val spark = typedLit[Long](l).encodeLongAsMorpheusId.expr + .eval() + .asInstanceOf[Array[Byte]] + .toList + scala === spark + }, + minSuccessful(1000) + ) } it("encoding/decoding is symmetric") { - check((l: Long) => { - val encoded = l.encodeAsMorpheusId - val decoded = decodeLong(encoded) - decoded === l - }, minSuccessful(1000)) + check( + (l: Long) => { + val encoded = l.encodeAsMorpheusId + val decoded = decodeLong(encoded) + decoded === l + }, + minSuccessful(1000) + ) } it("scala version encodes longs correctly") { @@ -59,34 +68,47 @@ class EncodeLongTest extends MorpheusTestSuite with Checkers { } it("spark version encodes longs correctly") { - typedLit[Long](0L).encodeLongAsMorpheusId.expr.eval().asInstanceOf[Array[Byte]].array.toList should equal(List(0.toByte)) + typedLit[Long](0L).encodeLongAsMorpheusId.expr + .eval() + .asInstanceOf[Array[Byte]] + .array + .toList should equal(List(0.toByte)) } describe("Spark expression") { it("converts longs into byte arrays using expression interpreter") { - check((l: Long) => { - val positive = l & Long.MaxValue - val inputRow = new GenericInternalRow(Array[Any](positive)) - val encodeLong = EncodeLong(functions.lit(positive).expr) - val interpreted = encodeLong.eval(inputRow).asInstanceOf[Array[Byte]] - val decoded = decodeLong(interpreted) + check( + (l: Long) => { + val positive = l & Long.MaxValue + val inputRow = new GenericInternalRow(Array[Any](positive)) + val encodeLong = EncodeLong(functions.lit(positive).expr) + val interpreted = encodeLong.eval(inputRow).asInstanceOf[Array[Byte]] + val decoded = decodeLong(interpreted) - decoded === positive - }, minSuccessful(1000)) + decoded === positive + }, + minSuccessful(1000) + ) } it("converts longs into byte arrays using expression code gen") { - check((l: Long) => { - val positive = l & Long.MaxValue - val inputRow = new GenericInternalRow(Array[Any](positive)) - val encodeLong = EncodeLong(functions.lit(positive).expr) - val plan = GenerateMutableProjection.generate(Alias(encodeLong, s"Optimized($encodeLong)")() :: Nil) - val codegen = plan(inputRow).get(0, encodeLong.dataType).asInstanceOf[Array[Byte]] - val decoded = decodeLong(codegen) + check( + (l: Long) => { + val positive = l & Long.MaxValue + val inputRow = new GenericInternalRow(Array[Any](positive)) + val encodeLong = EncodeLong(functions.lit(positive).expr) + val plan = GenerateMutableProjection.generate( + Alias(encodeLong, s"Optimized($encodeLong)")() :: Nil + ) + val codegen = + plan(inputRow).get(0, encodeLong.dataType).asInstanceOf[Array[Byte]] + val decoded = decodeLong(codegen) - decoded === positive - }, minSuccessful(1000)) + decoded === positive + }, + minSuccessful(1000) + ) } } diff --git a/morpheus-testing/src/test/scala/org/opencypher/morpheus/impl/io/neo4j/external/Neo4jTest.scala b/morpheus-testing/src/test/scala/org/opencypher/morpheus/impl/io/neo4j/external/Neo4jTest.scala index 3770cda28f..981342a90e 100644 --- a/morpheus-testing/src/test/scala/org/opencypher/morpheus/impl/io/neo4j/external/Neo4jTest.scala +++ b/morpheus-testing/src/test/scala/org/opencypher/morpheus/impl/io/neo4j/external/Neo4jTest.scala @@ -30,9 +30,7 @@ import org.opencypher.morpheus.testing.fixture.SparkSessionFixture import org.opencypher.okapi.neo4j.io.testing.Neo4jServerFixture import org.opencypher.okapi.testing.BaseTestSuite -class Neo4jTest extends BaseTestSuite - with SparkSessionFixture - with Neo4jServerFixture { +class Neo4jTest extends BaseTestSuite with SparkSessionFixture with Neo4jServerFixture { override def dataFixture: String = """ @@ -47,7 +45,9 @@ class Neo4jTest extends BaseTestSuite lazy private val neo4j = Neo4j(neo4jConfig, sparkSession) test("run Cypher Query With Params") { - val result = neo4j.cypher("MATCH (n:Person) WHERE n.id <= {maxId} RETURN id(n)").param("maxId", 10) + val result = neo4j + .cypher("MATCH (n:Person) WHERE n.id <= {maxId} RETURN id(n)") + .param("maxId", 10) assertResult(10)(result.loadRowRdd.count()) } @@ -62,12 +62,20 @@ class Neo4jTest extends BaseTestSuite } test("run Cypher Query With Partition") { - val result = neo4j.cypher("MATCH (n:Person) RETURN id(n) SKIP {_skip} LIMIT {_limit}").partitions(4).batch(25) + val result = neo4j + .cypher("MATCH (n:Person) RETURN id(n) SKIP {_skip} LIMIT {_limit}") + .partitions(4) + .batch(25) assertResult(100)(result.loadRowRdd.count()) } test("run Cypher Rel Query WithPartition") { - val result = neo4j.cypher("MATCH (n:Person)-[r:KNOWS]->(m:Person) RETURN id(n) as src,id(m) as dst,type(r) as value SKIP {_skip} LIMIT {_limit}").partitions(7).batch(200) + val result = neo4j + .cypher( + "MATCH (n:Person)-[r:KNOWS]->(m:Person) RETURN id(n) as src,id(m) as dst,type(r) as value SKIP {_skip} LIMIT {_limit}" + ) + .partitions(7) + .batch(200) assertResult(1000)(result.loadRowRdd.count()) } } diff --git a/morpheus-testing/src/test/scala/org/opencypher/morpheus/impl/physical/RecordHeaderMismatch.scala b/morpheus-testing/src/test/scala/org/opencypher/morpheus/impl/physical/RecordHeaderMismatch.scala index fa8f0bbcc2..ffb357eb8f 100644 --- a/morpheus-testing/src/test/scala/org/opencypher/morpheus/impl/physical/RecordHeaderMismatch.scala +++ b/morpheus-testing/src/test/scala/org/opencypher/morpheus/impl/physical/RecordHeaderMismatch.scala @@ -41,7 +41,9 @@ import org.opencypher.okapi.relational.impl.operators.{RelationalOperator, Start class RecordHeaderMismatch extends MorpheusTestSuite { - it("throws a schema exception when the physical record header does not match the one computed based on the schema") { + it( + "throws a schema exception when the physical record header does not match the one computed based on the schema" + ) { val buggyGraph: RelationalCypherGraph[DataFrameTable] { type Session = MorpheusSession @@ -64,7 +66,10 @@ class RecordHeaderMismatch extends MorpheusTestSuite { override def tables: Seq[DataFrameTable] = Seq.empty // Always return empty records, which does not match what the schema promises - def scanOperator(searchPattern: Pattern, exactLabelMatch: Boolean): RelationalOperator[DataFrameTable] = { + def scanOperator( + searchPattern: Pattern, + exactLabelMatch: Boolean + ): RelationalOperator[DataFrameTable] = { Start.fromEmptyGraph(morpheus.records.empty()) } } diff --git a/morpheus-testing/src/test/scala/org/opencypher/morpheus/impl/physical/RelationalOptimizerTest.scala b/morpheus-testing/src/test/scala/org/opencypher/morpheus/impl/physical/RelationalOptimizerTest.scala index 2f5e948d6d..09961c8878 100644 --- a/morpheus-testing/src/test/scala/org/opencypher/morpheus/impl/physical/RelationalOptimizerTest.scala +++ b/morpheus-testing/src/test/scala/org/opencypher/morpheus/impl/physical/RelationalOptimizerTest.scala @@ -36,7 +36,12 @@ import org.opencypher.okapi.ir.api.expr.Var import org.opencypher.okapi.logical.impl.LogicalCatalogGraph import org.opencypher.okapi.relational.api.planning.RelationalRuntimeContext import org.opencypher.okapi.relational.api.table.Table -import org.opencypher.okapi.relational.impl.operators.{Cache, Join, RelationalOperator, SwitchContext} +import org.opencypher.okapi.relational.impl.operators.{ + Cache, + Join, + RelationalOperator, + SwitchContext +} import org.opencypher.okapi.relational.impl.planning.RelationalPlanner._ import org.opencypher.okapi.relational.impl.planning.{CrossJoin, RelationalOptimizer} @@ -45,15 +50,15 @@ class RelationalOptimizerTest extends MorpheusTestSuite with GraphConstructionFi implicit class OpContainsCache[T <: Table[T]](op: RelationalOperator[T]) { def containsCache: Boolean = op.exists { case _: Cache[T] => true - case _ => false + case _ => false } } test("Test insert Cache operators") { - implicit val context: RelationalRuntimeContext[DataFrameTable] = morpheus.basicRuntimeContext() + implicit val context: RelationalRuntimeContext[DataFrameTable] = + morpheus.basicRuntimeContext() - val g = initGraph( - """ + val g = initGraph(""" |CREATE () """.stripMargin) @@ -68,34 +73,38 @@ class RelationalOptimizerTest extends MorpheusTestSuite with GraphConstructionFi val pattern = NodePattern(CTNode) - val aPlan = planScan(None, logicalGraph, pattern, Map(aVar -> pattern.nodeElement)) - val bPlan = planScan(None, logicalGraph, pattern, Map(bVar -> pattern.nodeElement)) + val aPlan = + planScan(None, logicalGraph, pattern, Map(aVar -> pattern.nodeElement)) + val bPlan = + planScan(None, logicalGraph, pattern, Map(bVar -> pattern.nodeElement)) - val cPlan = planScan(None, logicalGraph, pattern, Map(cVar -> pattern.nodeElement)) - val dPlan = planScan(None, logicalGraph, pattern, Map(dVar -> pattern.nodeElement)) + val cPlan = + planScan(None, logicalGraph, pattern, Map(cVar -> pattern.nodeElement)) + val dPlan = + planScan(None, logicalGraph, pattern, Map(dVar -> pattern.nodeElement)) val join1 = aPlan.join(bPlan, Seq.empty, CrossJoin) val join2 = cPlan.join(dPlan, Seq.empty, CrossJoin) val plan = join1.join(join2, Seq.empty, CrossJoin) - val rewrittenPlan = RelationalOptimizer.process(plan) val eachSideOfAllJoinsContainsCache = rewrittenPlan.transform[Boolean] { case (Join(l, r, _, _), _) => l.containsCache && r.containsCache - case (_, childValues) => childValues.contains(true) + case (_, childValues) => childValues.contains(true) } - withClue(s"Each side of each join should contain a Cache operator. Actual tree was:\n${rewrittenPlan.pretty}") { + withClue( + s"Each side of each join should contain a Cache operator. Actual tree was:\n${rewrittenPlan.pretty}" + ) { eachSideOfAllJoinsContainsCache should equal(true) } } it("caches expand into for triangle") { // Given - val given = initGraph( - """ + val given = initGraph(""" |CREATE (p1:Person {name: "Alice"}) |CREATE (p2:Person {name: "Bob"}) |CREATE (p3:Person {name: "Eve"}) @@ -105,8 +114,7 @@ class RelationalOptimizerTest extends MorpheusTestSuite with GraphConstructionFi """.stripMargin) // When - val result = given.cypher( - """ + val result = given.cypher(""" |MATCH (p1:Person)-[e1:KNOWS]->(p2:Person), |(p2)-[e2:KNOWS]->(p3:Person), |(p1)-[e3:KNOWS]->(p3) @@ -116,13 +124,13 @@ class RelationalOptimizerTest extends MorpheusTestSuite with GraphConstructionFi val optimizedRelationalPlan = result.asMorpheus.plans.relationalPlan.get val cachedScans = optimizedRelationalPlan.transform[Int] { - case (s: SwitchContext[DataFrameTable], _) => if (s.containsCache) 1 else 0 + case (s: SwitchContext[DataFrameTable], _) => + if (s.containsCache) 1 else 0 case (_, childValues) => childValues.sum } // Then - withClue( - s"""Each scan should contain a Cache operator. Actual tree was: + withClue(s"""Each scan should contain a Cache operator. Actual tree was: ${optimizedRelationalPlan.pretty}""") { cachedScans shouldBe 6 } @@ -131,8 +139,7 @@ class RelationalOptimizerTest extends MorpheusTestSuite with GraphConstructionFi // Takes a long time to run with little extra info test("test caching expand into after var expand") { // Given - val given = initGraph( - """ + val given = initGraph(""" |CREATE (p1:Person {name: "Alice"}) |CREATE (p2:Person {name: "Bob"}) |CREATE (comment:Comment) @@ -157,15 +164,16 @@ class RelationalOptimizerTest extends MorpheusTestSuite with GraphConstructionFi ) // Then - val cacheOps = result.asMorpheus.plans.relationalPlan.get.collect { case c: Cache[DataFrameTable] => c } + val cacheOps = result.asMorpheus.plans.relationalPlan.get.collect { + case c: Cache[DataFrameTable] => c + } cacheOps.size shouldBe 115 } // Adds little extra info test("test caching optional match with duplicates") { // Given - val given = initGraph( - """ + val given = initGraph(""" |CREATE (p1:Person {name: "Alice"}) |CREATE (p2:Person {name: "Bob"}) |CREATE (p3:Person {name: "Eve"}) @@ -176,15 +184,16 @@ class RelationalOptimizerTest extends MorpheusTestSuite with GraphConstructionFi """.stripMargin) // When - val result = given.cypher( - """ + val result = given.cypher(""" |MATCH (a:Person)-[e1:KNOWS]->(b:Person) |OPTIONAL MATCH (b)-[e2:KNOWS]->(c:Person) |RETURN b.name, c.name """.stripMargin) // Then - val cacheOps = result.asMorpheus.plans.relationalPlan.get.collect { case c: Cache[DataFrameTable] => c } + val cacheOps = result.asMorpheus.plans.relationalPlan.get.collect { + case c: Cache[DataFrameTable] => c + } cacheOps.size shouldBe 7 } } diff --git a/morpheus-testing/src/test/scala/org/opencypher/morpheus/impl/table/MorpheusRecordHeaderTest.scala b/morpheus-testing/src/test/scala/org/opencypher/morpheus/impl/table/MorpheusRecordHeaderTest.scala index 6ff5db3159..ed3bd5e7c8 100644 --- a/morpheus-testing/src/test/scala/org/opencypher/morpheus/impl/table/MorpheusRecordHeaderTest.scala +++ b/morpheus-testing/src/test/scala/org/opencypher/morpheus/impl/table/MorpheusRecordHeaderTest.scala @@ -41,11 +41,17 @@ class MorpheusRecordHeaderTest extends BaseTestSuite { .withExpr(Var("b")(CTString.nullable)) .withExpr(Var("c")(CTList(CTString.nullable))) - header.toStructType.fields.toSet should equal(Set( - StructField(header.column(Var("a")()), StringType, nullable = false), - StructField(header.column(Var("b")()), StringType, nullable = true), - StructField(header.column(Var("c")()), ArrayType(StringType, containsNull = true), nullable = false) - )) + header.toStructType.fields.toSet should equal( + Set( + StructField(header.column(Var("a")()), StringType, nullable = false), + StructField(header.column(Var("b")()), StringType, nullable = true), + StructField( + header.column(Var("c")()), + ArrayType(StringType, containsNull = true), + nullable = false + ) + ) + ) } } diff --git a/morpheus-testing/src/test/scala/org/opencypher/morpheus/impl/table/MorpheusStructTypeTest.scala b/morpheus-testing/src/test/scala/org/opencypher/morpheus/impl/table/MorpheusStructTypeTest.scala index 70562e25a7..4c2db05c8c 100644 --- a/morpheus-testing/src/test/scala/org/opencypher/morpheus/impl/table/MorpheusStructTypeTest.scala +++ b/morpheus-testing/src/test/scala/org/opencypher/morpheus/impl/table/MorpheusStructTypeTest.scala @@ -36,18 +36,27 @@ import org.opencypher.okapi.testing.BaseTestSuite class MorpheusStructTypeTest extends BaseTestSuite { it("computes a header from a given struct type") { - val structType = StructType(Seq( - StructField("a", LongType, nullable = true), - StructField("b", StringType, nullable = false), - StructField("c", ArrayType(StringType, containsNull = true), nullable = false) - )) + val structType = StructType( + Seq( + StructField("a", LongType, nullable = true), + StructField("b", StringType, nullable = false), + StructField( + "c", + ArrayType(StringType, containsNull = true), + nullable = false + ) + ) + ) - structType.toRecordHeader should equal(RecordHeader(Map( - Var("a")(CTInteger.nullable) -> "a", - Var("b")(CTString) -> "b", - Var("c")(CTList(CTString.nullable)) -> "c" + structType.toRecordHeader should equal( + RecordHeader( + Map( + Var("a")(CTInteger.nullable) -> "a", + Var("b")(CTString) -> "b", + Var("c")(CTList(CTString.nullable)) -> "c" + ) + ) ) - )) } } diff --git a/morpheus-testing/src/test/scala/org/opencypher/morpheus/impl/table/PatternElementTableTest.scala b/morpheus-testing/src/test/scala/org/opencypher/morpheus/impl/table/PatternElementTableTest.scala index d146454d87..17f0860165 100644 --- a/morpheus-testing/src/test/scala/org/opencypher/morpheus/impl/table/PatternElementTableTest.scala +++ b/morpheus-testing/src/test/scala/org/opencypher/morpheus/impl/table/PatternElementTableTest.scala @@ -34,7 +34,11 @@ import org.opencypher.morpheus.api.value.MorpheusElement._ import org.opencypher.morpheus.api.value.MorpheusNode import org.opencypher.morpheus.testing.MorpheusTestSuite import org.opencypher.okapi.api.graph.Pattern -import org.opencypher.okapi.api.io.conversion.{ElementMapping, NodeMappingBuilder, RelationshipMappingBuilder} +import org.opencypher.okapi.api.io.conversion.{ + ElementMapping, + NodeMappingBuilder, + RelationshipMappingBuilder +} import org.opencypher.okapi.api.schema.PropertyGraphSchema import org.opencypher.okapi.api.types._ import org.opencypher.okapi.api.value.CypherValue.CypherMap @@ -69,29 +73,37 @@ class PatternElementTableTest extends MorpheusTestSuite { it("mapping from scala classes") { val personTableScala = MorpheusNodeTable(List(Person(0, "Alice", 15))) - personTableScala.mapping should equal(NodeMappingBuilder - .withSourceIdKey("id") - .withImpliedLabel("Person") - .withPropertyKey("name") - .withPropertyKey("age") - .build) - + personTableScala.mapping should equal( + NodeMappingBuilder + .withSourceIdKey("id") + .withImpliedLabel("Person") + .withPropertyKey("name") + .withPropertyKey("age") + .build + ) - val friends = List(Friend(0, 0, 1, "23/01/1987"), Friend(1, 1, 2, "12/12/2009")) + val friends = + List(Friend(0, 0, 1, "23/01/1987"), Friend(1, 1, 2, "12/12/2009")) val friendTableScala = MorpheusRelationshipTable(friends) - friendTableScala.mapping should equal(RelationshipMappingBuilder - .withSourceIdKey("id") - .withSourceStartNodeKey("source") - .withSourceEndNodeKey("target") - .withRelType("FRIEND_OF") - .withPropertyKey("since") - .build) + friendTableScala.mapping should equal( + RelationshipMappingBuilder + .withSourceIdKey("id") + .withSourceStartNodeKey("source") + .withSourceEndNodeKey("target") + .withRelType("FRIEND_OF") + .withPropertyKey("since") + .build + ) } // TODO: What is the expected column ordering? - ignore("throws an IllegalArgumentException when a relationship table does not have the expected column ordering") { - val df = sparkSession.createDataFrame(Seq((1, 1, 1, true))).toDF("ID", "TARGET", "SOURCE", "TYPE") + ignore( + "throws an IllegalArgumentException when a relationship table does not have the expected column ordering" + ) { + val df = sparkSession + .createDataFrame(Seq((1, 1, 1, true))) + .toDF("ID", "TARGET", "SOURCE", "TYPE") val relMapping = RelationshipMappingBuilder .on("ID") .from("SOURCE") @@ -105,28 +117,39 @@ class PatternElementTableTest extends MorpheusTestSuite { } it("NodeTable should create correct schema from given mapping") { - val df = sparkSession.createDataFrame(Seq((1L, "Mats", 23L))).toDF("ID", "FOO", "BAR") + val df = sparkSession + .createDataFrame(Seq((1L, "Mats", 23L))) + .toDF("ID", "FOO", "BAR") val nodeTable = MorpheusElementTable.create(nodeMapping, df) nodeTable.schema should equal( PropertyGraphSchema.empty - .withNodePropertyKeys("A", "B")("foo" -> CTString.nullable, "bar" -> CTInteger)) + .withNodePropertyKeys("A", "B")( + "foo" -> CTString.nullable, + "bar" -> CTInteger + ) + ) } it("NodeTable should create correct header from given mapping") { - val df = sparkSession.createDataFrame(Seq((1L, "Mats", 23L))).toDF("ID", "FOO", "BAR") + val df = sparkSession + .createDataFrame(Seq((1L, "Mats", 23L))) + .toDF("ID", "FOO", "BAR") val nodeTable = MorpheusElementTable.create(nodeMapping, df) val v = Var(Pattern.DEFAULT_NODE_NAME)(CTNode("A", "B")) - nodeTable.header should equal(RecordHeader(Map( - v -> "ID", - ElementProperty(v, PropertyKey("foo"))(CTString) -> "FOO", - ElementProperty(v, PropertyKey("bar"))(CTInteger) -> "BAR" + nodeTable.header should equal( + RecordHeader( + Map( + v -> "ID", + ElementProperty(v, PropertyKey("foo"))(CTString) -> "FOO", + ElementProperty(v, PropertyKey("bar"))(CTInteger) -> "BAR" + ) + ) ) - )) } it("Relationship table should create correct schema from given mapping") { @@ -138,7 +161,11 @@ class PatternElementTableTest extends MorpheusTestSuite { relationshipTable.schema should equal( PropertyGraphSchema.empty - .withRelationshipPropertyKeys("A")("foo" -> CTString.nullable, "bar" -> CTInteger)) + .withRelationshipPropertyKeys("A")( + "foo" -> CTString.nullable, + "bar" -> CTInteger + ) + ) } it("Relationship table should create correct header from given mapping") { @@ -150,23 +177,33 @@ class PatternElementTableTest extends MorpheusTestSuite { val v = Var(Pattern.DEFAULT_REL_NAME)(CTRelationship("A")) - relationshipTable.header should equal(RecordHeader( - Map( - v -> "ID", - StartNode(v)(CTNode) -> "FROM", - EndNode(v)(CTNode) -> "TO", - ElementProperty(v, PropertyKey("foo"))(CTString) -> "FOO", - ElementProperty(v, PropertyKey("bar"))(CTInteger) -> "BAR" + relationshipTable.header should equal( + RecordHeader( + Map( + v -> "ID", + StartNode(v)(CTNode) -> "FROM", + EndNode(v)(CTNode) -> "TO", + ElementProperty(v, PropertyKey("foo"))(CTString) -> "FOO", + ElementProperty(v, PropertyKey("bar"))(CTInteger) -> "BAR" + ) ) - )) + ) } it("NodeTable should cast compatible types in input DataFrame") { // The ASCII code of character `1` is 49 - val dfBinary = sparkSession.createDataFrame(Seq((Array[Byte](49.toByte), true, 10.toShort, 23.1f))).toDF("ID", "IS_C", "FOO", "BAR") - val dfLong = sparkSession.createDataFrame(Seq((49L, true, 10.toShort, 23.1f))).toDF("ID", "IS_C", "FOO", "BAR") - val dfInt = sparkSession.createDataFrame(Seq((49, true, 10.toShort, 23.1f))).toDF("ID", "IS_C", "FOO", "BAR") - val dfString = sparkSession.createDataFrame(Seq(("1", 10.toShort, 23.1f))).toDF("ID", "FOO", "BAR") + val dfBinary = sparkSession + .createDataFrame(Seq((Array[Byte](49.toByte), true, 10.toShort, 23.1f))) + .toDF("ID", "IS_C", "FOO", "BAR") + val dfLong = sparkSession + .createDataFrame(Seq((49L, true, 10.toShort, 23.1f))) + .toDF("ID", "IS_C", "FOO", "BAR") + val dfInt = sparkSession + .createDataFrame(Seq((49, true, 10.toShort, 23.1f))) + .toDF("ID", "IS_C", "FOO", "BAR") + val dfString = sparkSession + .createDataFrame(Seq(("1", 10.toShort, 23.1f))) + .toDF("ID", "FOO", "BAR") val dfs = Seq(dfBinary, dfString, dfLong, dfInt) @@ -175,73 +212,121 @@ class PatternElementTableTest extends MorpheusTestSuite { nodeTable.schema should equal( PropertyGraphSchema.empty - .withNodePropertyKeys("A", "B")("foo" -> CTInteger, "bar" -> CTFloat)) + .withNodePropertyKeys("A", "B")("foo" -> CTInteger, "bar" -> CTFloat) + ) - nodeTable.records.df.collect().toSet should equal(Set(Row(49L.encodeAsMorpheusId, 23.1f.toDouble, 10))) + nodeTable.records.df.collect().toSet should equal( + Set(Row(49L.encodeAsMorpheusId, 23.1f.toDouble, 10)) + ) } } it("NodeTable can handle shuffled columns due to cast") { - val df = sparkSession.createDataFrame(Seq((1L, 10.toShort, 23.1f))).toDF("ID", "FOO", "BAR") + val df = sparkSession + .createDataFrame(Seq((1L, 10.toShort, 23.1f))) + .toDF("ID", "FOO", "BAR") val nodeTable = MorpheusElementTable.create(nodeMapping, df) val graph = morpheus.graphs.create(nodeTable) - graph.nodes("n").collect.toSet should equal(Set( - CypherMap("n" -> MorpheusNode(1, Set("A", "B"), CypherMap("bar" -> 23.1f, "foo" -> 10))) - )) + graph.nodes("n").collect.toSet should equal( + Set( + CypherMap( + "n" -> MorpheusNode( + 1, + Set("A", "B"), + CypherMap("bar" -> 23.1f, "foo" -> 10) + ) + ) + ) + ) } - it("NodeTable should not accept wrong source id key type (should be compatible to LongType)") { + it( + "NodeTable should not accept wrong source id key type (should be compatible to LongType)" + ) { val e = the[IllegalArgumentException] thrownBy { - val df = sparkSession.createDataFrame(Seq(Tuple1(Date.valueOf("1987-01-23")))).toDF("ID") + val df = sparkSession + .createDataFrame(Seq(Tuple1(Date.valueOf("1987-01-23")))) + .toDF("ID") val nodeMapping = NodeMappingBuilder.on("ID").withImpliedLabel("A").build MorpheusElementTable.create(nodeMapping, df) } - e.getMessage should (include("Column `ID` should have a valid identifier data type") and include("Unsupported column type `DateType`")) + e.getMessage should (include( + "Column `ID` should have a valid identifier data type" + ) and include("Unsupported column type `DateType`")) } - it("RelationshipTable should not accept wrong sourceId, -StartNode, -EndNode key type") { - val relMapping = RelationshipMappingBuilder.on("ID").from("SOURCE").to("TARGET").relType("A").build + it( + "RelationshipTable should not accept wrong sourceId, -StartNode, -EndNode key type" + ) { + val relMapping = RelationshipMappingBuilder + .on("ID") + .from("SOURCE") + .to("TARGET") + .relType("A") + .build an[IllegalArgumentException] should be thrownBy { - val df = sparkSession.createDataFrame(Seq((1.toByte, 1, 1))).toDF("ID", "SOURCE", "TARGET") + val df = sparkSession + .createDataFrame(Seq((1.toByte, 1, 1))) + .toDF("ID", "SOURCE", "TARGET") MorpheusElementTable.create(relMapping, df) } an[IllegalArgumentException] should be thrownBy { - val df = sparkSession.createDataFrame(Seq((1, 1.toByte, 1))).toDF("ID", "SOURCE", "TARGET") + val df = sparkSession + .createDataFrame(Seq((1, 1.toByte, 1))) + .toDF("ID", "SOURCE", "TARGET") MorpheusElementTable.create(relMapping, df) } an[IllegalArgumentException] should be thrownBy { - val df = sparkSession.createDataFrame(Seq((1, 1, 1.toByte))).toDF("ID", "SOURCE", "TARGET") + val df = sparkSession + .createDataFrame(Seq((1, 1, 1.toByte))) + .toDF("ID", "SOURCE", "TARGET") MorpheusElementTable.create(relMapping, df) } } it("NodeTable should infer the correct schema") { - val df = sparkSession.createDataFrame(Seq((1L, "Alice", 1984, true, 13.37))) + val df = sparkSession + .createDataFrame(Seq((1L, "Alice", 1984, true, 13.37))) .toDF("id", "name", "birthYear", "isGood", "luckyNumber") val nodeTable = MorpheusNodeTable(Set("Person"), df) - nodeTable.schema should equal(PropertyGraphSchema.empty - .withNodePropertyKeys("Person")( - "name" -> CTString.nullable, - "birthYear" -> CTInteger, - "isGood" -> CTBoolean, - "luckyNumber" -> CTFloat)) + nodeTable.schema should equal( + PropertyGraphSchema.empty + .withNodePropertyKeys("Person")( + "name" -> CTString.nullable, + "birthYear" -> CTInteger, + "isGood" -> CTBoolean, + "luckyNumber" -> CTFloat + ) + ) } it("RelationshipTable should infer the correct schema") { - val df = sparkSession.createDataFrame(Seq((1L, 1L, 1L, "Alice", 1984, true, 13.37))) - .toDF("id", "source", "target", "name", "birthYear", "isGood", "luckyNumber") + val df = sparkSession + .createDataFrame(Seq((1L, 1L, 1L, "Alice", 1984, true, 13.37))) + .toDF( + "id", + "source", + "target", + "name", + "birthYear", + "isGood", + "luckyNumber" + ) val relationshipTable = MorpheusRelationshipTable("KNOWS", df) - relationshipTable.schema should equal(PropertyGraphSchema.empty - .withRelationshipPropertyKeys("KNOWS")( - "name" -> CTString.nullable, - "birthYear" -> CTInteger, - "isGood" -> CTBoolean, - "luckyNumber" -> CTFloat)) + relationshipTable.schema should equal( + PropertyGraphSchema.empty + .withRelationshipPropertyKeys("KNOWS")( + "name" -> CTString.nullable, + "birthYear" -> CTInteger, + "isGood" -> CTBoolean, + "luckyNumber" -> CTFloat + ) + ) } } diff --git a/morpheus-testing/src/test/scala/org/opencypher/morpheus/impl/util/AnnotationTest.scala b/morpheus-testing/src/test/scala/org/opencypher/morpheus/impl/util/AnnotationTest.scala index fe462e97bb..b2ddf3d608 100644 --- a/morpheus-testing/src/test/scala/org/opencypher/morpheus/impl/util/AnnotationTest.scala +++ b/morpheus-testing/src/test/scala/org/opencypher/morpheus/impl/util/AnnotationTest.scala @@ -53,18 +53,26 @@ case class TestAnnotation(foo: String) extends StaticAnnotation class AnnotationTest extends BaseTestSuite { test("read node label annotation") { - Annotation.labels[NodeWithoutAnnotation] should equal(Set(classOf[NodeWithoutAnnotation].getSimpleName)) + Annotation.labels[NodeWithoutAnnotation] should equal( + Set(classOf[NodeWithoutAnnotation].getSimpleName) + ) Annotation.labels[NodeWithEmptyAnnotation] should equal(Set.empty) Annotation.labels[NodeWithSingleAnnotation] should equal(Set("One")) - Annotation.labels[NodeWithMultipleAnnotations] should equal(Set("One", "Two", "Three")) + Annotation.labels[NodeWithMultipleAnnotations] should equal( + Set("One", "Two", "Three") + ) } test("read relationship type annotation") { - Annotation.relType[RelWithoutAnnotation] should equal(classOf[RelWithoutAnnotation].getSimpleName.toUpperCase) + Annotation.relType[RelWithoutAnnotation] should equal( + classOf[RelWithoutAnnotation].getSimpleName.toUpperCase + ) Annotation.relType[RelWithAnnotation] should equal("One") } test("read more general static annotation") { - Annotation.get[TestAnnotation, TestAnnotation] should equal(Some(TestAnnotation("Foo"))) + Annotation.get[TestAnnotation, TestAnnotation] should equal( + Some(TestAnnotation("Foo")) + ) } } diff --git a/morpheus-testing/src/test/scala/org/opencypher/morpheus/impl/values/MorpheusLiteralTests.scala b/morpheus-testing/src/test/scala/org/opencypher/morpheus/impl/values/MorpheusLiteralTests.scala index c47402c642..341543c388 100644 --- a/morpheus-testing/src/test/scala/org/opencypher/morpheus/impl/values/MorpheusLiteralTests.scala +++ b/morpheus-testing/src/test/scala/org/opencypher/morpheus/impl/values/MorpheusLiteralTests.scala @@ -39,35 +39,53 @@ import org.typelevel.claimant.Claim class MorpheusLiteralTests extends MorpheusTestSuite with Checkers with ScanGraphInit { val supportedLiteral: Gen[CypherValue] = Gen.oneOf( - homogeneousScalarList, propertyMap, string, boolean, integer, float, const(CypherNull) + homogeneousScalarList, + propertyMap, + string, + boolean, + integer, + float, + const(CypherNull) ) it("round trip for supported literals") { - check(Prop.forAll(supportedLiteral) { v: CypherValue => - val query = s"RETURN ${v.toCypherString} AS result" - val result = morpheus.cypher(query).records.collect.toList - val expected = List(CypherMap("result" -> v)) - Claim(result == expected) - }, minSuccessful(10)) + check( + Prop.forAll(supportedLiteral) { v: CypherValue => + val query = s"RETURN ${v.toCypherString} AS result" + val result = morpheus.cypher(query).records.collect.toList + val expected = List(CypherMap("result" -> v)) + Claim(result == expected) + }, + minSuccessful(10) + ) } it("round trip for nodes") { - check(Prop.forAll(node) { n: Node[CypherInteger] => - val graph = initGraph(s"CREATE ${n.toCypherString}") - val query = s"MATCH (n) RETURN n" - val result = TestNode(graph.cypher(query).records.collect.head("n").cast[Node[_]]) - Claim(result == n) - }, minSuccessful(10)) + check( + Prop.forAll(node) { n: Node[CypherInteger] => + val graph = initGraph(s"CREATE ${n.toCypherString}") + val query = s"MATCH (n) RETURN n" + val result = + TestNode(graph.cypher(query).records.collect.head("n").cast[Node[_]]) + Claim(result == n) + }, + minSuccessful(10) + ) } // TODO: Diagnose and fix error "AnalysisException: Reference 'node_L __ BOOLEAN' is ambiguous, could be: node_L __ BOOLEAN, node_L __ BOOLEAN.;" ignore("round trip for relationships") { - check(Prop.forAll(nodeRelNodePattern()) { p: NodeRelNodePattern[_] => - val graph = initGraph(p.toCreateQuery) - val query = s"MATCH ()-[r]->() RETURN r" - val result = TestRelationship(graph.cypher(query).records.collect.head("r").cast[Relationship[_]]) - Claim(result == p.relationship) - }, minSuccessful(1000)) + check( + Prop.forAll(nodeRelNodePattern()) { p: NodeRelNodePattern[_] => + val graph = initGraph(p.toCreateQuery) + val query = s"MATCH ()-[r]->() RETURN r" + val result = TestRelationship( + graph.cypher(query).records.collect.head("r").cast[Relationship[_]] + ) + Claim(result == p.relationship) + }, + minSuccessful(1000) + ) } } diff --git a/morpheus-testing/src/test/scala/org/opencypher/morpheus/schema/MorpheusSchemaTest.scala b/morpheus-testing/src/test/scala/org/opencypher/morpheus/schema/MorpheusSchemaTest.scala index 3272bcbdb3..cbea083632 100644 --- a/morpheus-testing/src/test/scala/org/opencypher/morpheus/schema/MorpheusSchemaTest.scala +++ b/morpheus-testing/src/test/scala/org/opencypher/morpheus/schema/MorpheusSchemaTest.scala @@ -38,30 +38,38 @@ class MorpheusSchemaTest extends MorpheusTestSuite with GraphConstructionFixture it("constructs schema correctly for unlabeled nodes") { val graph = initGraph("CREATE ({id: 1}), ({id: 2}), ({other: 'foo'}), ()") - graph.schema should equal(PropertyGraphSchema.empty - .withNodePropertyKeys(Set.empty[String], Map("id" -> CTInteger.nullable, "other" -> CTString.nullable)) - .asMorpheus + graph.schema should equal( + PropertyGraphSchema.empty + .withNodePropertyKeys( + Set.empty[String], + Map("id" -> CTInteger.nullable, "other" -> CTString.nullable) + ) + .asMorpheus ) } it("constructs schema correctly for labeled nodes") { - val graph = initGraph("CREATE (:A {id: 1}), (:A {id: 2}), (:B {other: 'foo'})") - - graph.schema should equal(PropertyGraphSchema.empty - .withNodePropertyKeys("A")("id" -> CTInteger) - .withNodePropertyKeys("B")("other" -> CTString) - .asMorpheus + val graph = + initGraph("CREATE (:A {id: 1}), (:A {id: 2}), (:B {other: 'foo'})") + + graph.schema should equal( + PropertyGraphSchema.empty + .withNodePropertyKeys("A")("id" -> CTInteger) + .withNodePropertyKeys("B")("other" -> CTString) + .asMorpheus ) } it("constructs schema correctly for multi-labeled nodes") { - val graph = initGraph("CREATE (:A {id: 1}), (:A:B {id: 2}), (:B {other: 'foo'})") - - graph.schema should equal(PropertyGraphSchema.empty - .withNodePropertyKeys("A")("id" -> CTInteger) - .withNodePropertyKeys("B")("other" -> CTString) - .withNodePropertyKeys("A", "B")("id" -> CTInteger) - .asMorpheus + val graph = + initGraph("CREATE (:A {id: 1}), (:A:B {id: 2}), (:B {other: 'foo'})") + + graph.schema should equal( + PropertyGraphSchema.empty + .withNodePropertyKeys("A")("id" -> CTInteger) + .withNodePropertyKeys("B")("other" -> CTString) + .withNodePropertyKeys("A", "B")("id" -> CTInteger) + .asMorpheus ) } @@ -74,11 +82,15 @@ class MorpheusSchemaTest extends MorpheusTestSuite with GraphConstructionFixture """.stripMargin ) - graph.schema should equal(PropertyGraphSchema.empty - .withNodePropertyKeys(Set.empty[String], PropertyKeys.empty) - .withRelationshipPropertyKeys("FOO")("p" -> CTInteger) - .withRelationshipPropertyKeys("BAR")("p" -> CTInteger, "q" -> CTString.nullable) - .asMorpheus + graph.schema should equal( + PropertyGraphSchema.empty + .withNodePropertyKeys(Set.empty[String], PropertyKeys.empty) + .withRelationshipPropertyKeys("FOO")("p" -> CTInteger) + .withRelationshipPropertyKeys("BAR")( + "p" -> CTInteger, + "q" -> CTString.nullable + ) + .asMorpheus ) } @@ -88,7 +100,9 @@ class MorpheusSchemaTest extends MorpheusTestSuite with GraphConstructionFixture val schema2 = PropertyGraphSchema.empty .withNodePropertyKeys("A")("foo" -> CTString, "bar" -> CTInteger) - the[SchemaException] thrownBy (schema1 ++ schema2).asMorpheus should have message + the[ + SchemaException + ] thrownBy (schema1 ++ schema2).asMorpheus should have message "The property type 'UNION(INTEGER, STRING)' for property 'bar' can not be stored in a Spark column. The unsupported type is specified on label combination [A]." } @@ -98,7 +112,9 @@ class MorpheusSchemaTest extends MorpheusTestSuite with GraphConstructionFixture val schema2 = PropertyGraphSchema.empty .withNodePropertyKeys("A")("foo" -> CTString, "baz" -> CTFloat) - the[SchemaException] thrownBy (schema1 ++ schema2).asMorpheus should have message + the[ + SchemaException + ] thrownBy (schema1 ++ schema2).asMorpheus should have message "The property type 'NUMBER' for property 'baz' can not be stored in a Spark column. The unsupported type is specified on label combination [A]." } @@ -109,7 +125,10 @@ class MorpheusSchemaTest extends MorpheusTestSuite with GraphConstructionFixture it("successfully verifies a valid schema") { val schema = PropertyGraphSchema.empty .withNodePropertyKeys("Person")("name" -> CTString) - .withNodePropertyKeys("Employee")("name" -> CTString, "salary" -> CTInteger) + .withNodePropertyKeys("Employee")( + "name" -> CTString, + "salary" -> CTInteger + ) .withNodePropertyKeys("Dog")("name" -> CTFloat) .withNodePropertyKeys("Pet")("notName" -> CTBoolean) @@ -119,7 +138,10 @@ class MorpheusSchemaTest extends MorpheusTestSuite with GraphConstructionFixture it("fails when verifying schema with conflict on implied labels") { val schema = PropertyGraphSchema.empty .withNodePropertyKeys("Person")("name" -> CTString) - .withNodePropertyKeys("Employee", "Person")("name" -> CTString, "salary" -> CTInteger) + .withNodePropertyKeys("Employee", "Person")( + "name" -> CTString, + "salary" -> CTInteger + ) .withNodePropertyKeys("Dog", "Pet")("name" -> CTFloat) .withNodePropertyKeys("Pet")("name" -> CTBoolean) @@ -130,8 +152,14 @@ class MorpheusSchemaTest extends MorpheusTestSuite with GraphConstructionFixture it("fails when verifying schema with conflict on combined labels") { val schema = PropertyGraphSchema.empty .withNodePropertyKeys("Person")("name" -> CTString) - .withNodePropertyKeys("Employee", "Person")("name" -> CTInteger, "salary" -> CTInteger) - .withNodePropertyKeys("Employee")("name" -> CTInteger, "salary" -> CTInteger) + .withNodePropertyKeys("Employee", "Person")( + "name" -> CTInteger, + "salary" -> CTInteger + ) + .withNodePropertyKeys("Employee")( + "name" -> CTInteger, + "salary" -> CTInteger + ) .withNodePropertyKeys("Dog", "Pet")("name" -> CTFloat) .withNodePropertyKeys("Pet")("notName" -> CTBoolean) diff --git a/okapi-api/src/main/scala/org/opencypher/okapi/api/configuration/Configuration.scala b/okapi-api/src/main/scala/org/opencypher/okapi/api/configuration/Configuration.scala index c96edbd708..4ce534a4ce 100644 --- a/okapi-api/src/main/scala/org/opencypher/okapi/api/configuration/Configuration.scala +++ b/okapi-api/src/main/scala/org/opencypher/okapi/api/configuration/Configuration.scala @@ -30,9 +30,7 @@ import org.opencypher.okapi.impl.configuration.ConfigFlag object Configuration { - /** - * If enabled, the time required for executing query processing phases will be printed. - */ + /** If enabled, the time required for executing query processing phases will be printed. */ object PrintTimings extends ConfigFlag("morpheus.printTimings") } diff --git a/okapi-api/src/main/scala/org/opencypher/okapi/api/graph/CypherResult.scala b/okapi-api/src/main/scala/org/opencypher/okapi/api/graph/CypherResult.scala index 0e1742cfd1..5aa888aadc 100644 --- a/okapi-api/src/main/scala/org/opencypher/okapi/api/graph/CypherResult.scala +++ b/okapi-api/src/main/scala/org/opencypher/okapi/api/graph/CypherResult.scala @@ -41,38 +41,40 @@ trait CypherResult extends CypherPrintable { type Graph <: PropertyGraph /** - * Retrieves the graph if one is returned by the query. - * If the query returns a table, `None` is returned. + * Retrieves the graph if one is returned by the query. If the query returns a table, `None` is + * returned. * - * @return a graph if the query returned one, `None` otherwise + * @return + * a graph if the query returned one, `None` otherwise */ def getGraph: Option[Graph] /** * Retrieves the graph if one is returned by the query, otherwise an exception is thrown. * - * @return graph as returned by the query. + * @return + * graph as returned by the query. */ def graph: Graph = getGraph.get /** - * The table of records if one was returned by the query. - * Returns `None` if the query returned a graph. + * The table of records if one was returned by the query. Returns `None` if the query returned a + * graph. * - * @return a table of records, `None` otherwise. + * @return + * a table of records, `None` otherwise. */ def getRecords: Option[Records] /** * The table of records if one was returned by the query, otherwise an exception is thrown. * - * @return a table of records. + * @return + * a table of records. */ def records: Records = getRecords.get - /** - * API for printable plans. This is used for explaining the execution plan of a Cypher query. - */ + /** API for printable plans. This is used for explaining the execution plan of a Cypher query. */ def plans: CypherQueryPlans } diff --git a/okapi-api/src/main/scala/org/opencypher/okapi/api/graph/CypherSession.scala b/okapi-api/src/main/scala/org/opencypher/okapi/api/graph/CypherSession.scala index d605728eef..1b48bf145f 100644 --- a/okapi-api/src/main/scala/org/opencypher/okapi/api/graph/CypherSession.scala +++ b/okapi-api/src/main/scala/org/opencypher/okapi/api/graph/CypherSession.scala @@ -35,25 +35,28 @@ import org.opencypher.okapi.impl.graph.QGNGenerator import org.opencypher.okapi.impl.io.SessionGraphDataSource /** - * The Cypher Session is the main API for a Cypher-based application. It manages graphs which can be queried using - * Cypher. Graphs can be read from / written to different data sources (e.g. CSV) and also stored in / retrieved from - * the session-local storage. + * The Cypher Session is the main API for a Cypher-based application. It manages graphs which can + * be queried using Cypher. Graphs can be read from / written to different data sources (e.g. CSV) + * and also stored in / retrieved from the session-local storage. */ trait CypherSession { - /** - * Back end specific query result type - */ + /** Back end specific query result type */ type Result <: CypherResult /** * Executes a Cypher query in this session on the current ambient graph. * - * @param query Cypher query to execute - * @param parameters parameters used by the Cypher query - * @param drivingTable seed data that can be accessed from within the query - * @param queryCatalog a map of query-local graphs, this allows to evaluate queries that produce graphs recursively - * @return result of the query + * @param query + * Cypher query to execute + * @param parameters + * parameters used by the Cypher query + * @param drivingTable + * seed data that can be accessed from within the query + * @param queryCatalog + * a map of query-local graphs, this allows to evaluate queries that produce graphs recursively + * @return + * result of the query */ def cypher( query: String, @@ -63,65 +66,83 @@ trait CypherSession { ): Result /** - * Interface through which the user may (de-)register property graph datasources as well as read, write and delete property graphs. + * Interface through which the user may (de-)register property graph datasources as well as read, + * write and delete property graphs. * - * @return session catalog + * @return + * session catalog */ def catalog: PropertyGraphCatalog /** - * Register the given [[org.opencypher.okapi.api.io.PropertyGraphDataSource]] under the specific [[org.opencypher.okapi.api.graph.Namespace]] within the session catalog. + * Register the given [[org.opencypher.okapi.api.io.PropertyGraphDataSource]] under the specific + * [[org.opencypher.okapi.api.graph.Namespace]] within the session catalog. * - * This enables a user to refer to that [[org.opencypher.okapi.api.io.PropertyGraphDataSource]] within a Cypher query. + * This enables a user to refer to that [[org.opencypher.okapi.api.io.PropertyGraphDataSource]] + * within a Cypher query. * - * Note, that it is not allowed to overwrite an already registered [[org.opencypher.okapi.api.graph.Namespace]]. - * Use [[CypherSession#deregisterSource]] first. + * Note, that it is not allowed to overwrite an already registered + * [[org.opencypher.okapi.api.graph.Namespace]]. Use [[CypherSession#deregisterSource]] first. * - * @param namespace namespace for lookup - * @param dataSource property graph data source + * @param namespace + * namespace for lookup + * @param dataSource + * property graph data source */ - def registerSource(namespace: Namespace, dataSource: PropertyGraphDataSource): Unit = + def registerSource( + namespace: Namespace, + dataSource: PropertyGraphDataSource + ): Unit = catalog.register(namespace, dataSource) /** - * De-registers a [[org.opencypher.okapi.api.io.PropertyGraphDataSource]] from the sessions catalog by its given [[org.opencypher.okapi.api.graph.Namespace]]. + * De-registers a [[org.opencypher.okapi.api.io.PropertyGraphDataSource]] from the sessions + * catalog by its given [[org.opencypher.okapi.api.graph.Namespace]]. * - * @param namespace namespace for lookup + * @param namespace + * namespace for lookup */ def deregisterSource(namespace: Namespace): Unit = catalog.deregister(namespace) /** - * @return a new unique qualified graph name + * @return + * a new unique qualified graph name */ def generateQualifiedGraphName: QualifiedGraphName = qgnGenerator.generate /** * Executes a Cypher query in this session, using the argument graph as the ambient graph. * - * The ambient graph is the graph that is used for graph matching and updating, - * unless another graph is explicitly selected by the query. + * The ambient graph is the graph that is used for graph matching and updating, unless another + * graph is explicitly selected by the query. * - * @param graph ambient graph for this query - * @param query Cypher query to execute - * @param parameters parameters used by the Cypher query - * @return result of the query + * @param graph + * ambient graph for this query + * @param query + * Cypher query to execute + * @param parameters + * parameters used by the Cypher query + * @return + * result of the query */ private[opencypher] def cypherOnGraph( graph: PropertyGraph, query: String, parameters: CypherMap = CypherMap.empty, drivingTable: Option[CypherRecords], - queryCatalog: Map[QualifiedGraphName, PropertyGraph]): Result + queryCatalog: Map[QualifiedGraphName, PropertyGraph] + ): Result private val maxSessionGraphId: AtomicLong = new AtomicLong(0) - /** - * A generator for qualified graph names - */ + /** A generator for qualified graph names */ private[opencypher] val qgnGenerator = new QGNGenerator { override def generate: QualifiedGraphName = { - QualifiedGraphName(SessionGraphDataSource.Namespace, GraphName(s"tmp${maxSessionGraphId.incrementAndGet}")) + QualifiedGraphName( + SessionGraphDataSource.Namespace, + GraphName(s"tmp${maxSessionGraphId.incrementAndGet}") + ) } } diff --git a/okapi-api/src/main/scala/org/opencypher/okapi/api/graph/GraphElementType.scala b/okapi-api/src/main/scala/org/opencypher/okapi/api/graph/GraphElementType.scala index 1ec8d646c5..1999a52a98 100644 --- a/okapi-api/src/main/scala/org/opencypher/okapi/api/graph/GraphElementType.scala +++ b/okapi-api/src/main/scala/org/opencypher/okapi/api/graph/GraphElementType.scala @@ -26,37 +26,27 @@ */ package org.opencypher.okapi.api.graph -/** - * This trait represents different types of id keys a graph element can have. - */ +/** This trait represents different types of id keys a graph element can have. */ sealed trait IdKey { def name: String } -/** - * A SourceIdKey represents an id which uniquely identifies it's containing element - */ +/** A SourceIdKey represents an id which uniquely identifies it's containing element */ case object SourceIdKey extends IdKey { override def name: String = "id" } -/** - * A SourceStartNodeKey represents an id which identifies the start node of a relationship - */ +/** A SourceStartNodeKey represents an id which identifies the start node of a relationship */ case object SourceStartNodeKey extends IdKey { override def name: String = "source" } -/** - * A SourceEndNodeKey represents an id which identifies the end node of a relationship - */ +/** A SourceEndNodeKey represents an id which identifies the end node of a relationship */ case object SourceEndNodeKey extends IdKey { override def name: String = "target" } -/** - * Enum trait to distinguish between different graph element types - */ +/** Enum trait to distinguish between different graph element types */ sealed trait GraphElementType { def name: String } diff --git a/okapi-api/src/main/scala/org/opencypher/okapi/api/graph/Pattern.scala b/okapi-api/src/main/scala/org/opencypher/okapi/api/graph/Pattern.scala index ecbe2aa0e2..7d8885004c 100644 --- a/okapi-api/src/main/scala/org/opencypher/okapi/api/graph/Pattern.scala +++ b/okapi-api/src/main/scala/org/opencypher/okapi/api/graph/Pattern.scala @@ -1,29 +1,26 @@ /** - * Copyright (c) 2016-2019 "Neo4j Sweden, AB" [https://neo4j.com] - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * Attribution Notice under the terms of the Apache License 2.0 - * - * This work was created by the collective efforts of the openCypher community. - * Without limiting the terms of Section 6, any Derivative Work that is not - * approved by the public consensus process of the openCypher Implementers Group - * should not be described as “Cypher” (and Cypher® is a registered trademark of - * Neo4j Inc.) or as "openCypher". Extensions by implementers or prototypes or - * proposals for change that have been documented or implemented should only be - * described as "implementation extensions to Cypher" or as "proposed changes to - * Cypher that are not yet approved by the openCypher community". - */ + * Copyright (c) 2016-2019 "Neo4j Sweden, AB" [https://neo4j.com] + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the + * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing permissions and + * limitations under the License. + * + * Attribution Notice under the terms of the Apache License 2.0 + * + * This work was created by the collective efforts of the openCypher community. Without limiting + * the terms of Section 6, any Derivative Work that is not approved by the public consensus process + * of the openCypher Implementers Group should not be described as “Cypher” (and Cypher® is a + * registered trademark of Neo4j Inc.) or as "openCypher". Extensions by implementers or prototypes + * or proposals for change that have been documented or implemented should only be described as + * "implementation extensions to Cypher" or as "proposed changes to Cypher that are not yet + * approved by the openCypher community". + */ package org.opencypher.okapi.api.graph import org.opencypher.okapi.api.graph.Pattern._ @@ -43,8 +40,10 @@ case class Connection( /** * Represents an element within a pattern, e.g. a node or a relationship * - * @param name the elements name - * @param cypherType the elements CypherType + * @param name + * the elements name + * @param cypherType + * the elements CypherType */ case class PatternElement(name: String, cypherType: CypherType) @@ -52,9 +51,7 @@ object Pattern { val DEFAULT_NODE_NAME = "node" val DEFAULT_REL_NAME = "rel" - /** - * Patterns can be ordered by the number of relationships - */ + /** Patterns can be ordered by the number of relationships */ implicit case object PatternOrdering extends Ordering[Pattern] { override def compare( x: Pattern, @@ -63,46 +60,68 @@ object Pattern { } } - sealed trait Pattern { /** * All elements contained in the pattern * - * @return elements contained in the pattern + * @return + * elements contained in the pattern */ def elements: Set[PatternElement] /** * The patterns topology, describing connections between node elements via relationships * - * @return the patterns topology + * @return + * the patterns topology */ def topology: Map[PatternElement, Connection] - //TODO: to support general patterns implement a pattern matching algorithm + // TODO: to support general patterns implement a pattern matching algorithm /** * Tries to find an bijective mapping between from the search pattern into this pattern. * - * If exactMatch is true, then two elements can be mapped if they CypherTypes are equal, - * e.g. both are CTNode with the same label set. Otherwise a mapping exists if the search element - * type is supertype of the target element type. + * If exactMatch is true, then two elements can be mapped if they CypherTypes are equal, e.g. + * both are CTNode with the same label set. Otherwise a mapping exists if the search element type + * is supertype of the target element type. * - * @param searchPattern the pattern for which to find the mapping - * @param exactMatch controls how elements are matched - * @return a bijective mapping between the search pattern and the target pattern + * @param searchPattern + * the pattern for which to find the mapping + * @param exactMatch + * controls how elements are matched + * @return + * a bijective mapping between the search pattern and the target pattern */ - def findMapping(searchPattern: Pattern, exactMatch: Boolean): Option[Map[PatternElement, PatternElement]] = { - if((exactMatch && searchPattern == this) || (!exactMatch && subTypeOf(searchPattern))) { + def findMapping( + searchPattern: Pattern, + exactMatch: Boolean + ): Option[Map[PatternElement, PatternElement]] = { + if ( + (exactMatch && searchPattern == this) || (!exactMatch && subTypeOf( + searchPattern + )) + ) { searchPattern -> this match { case (searchNode: NodePattern, otherNode: NodePattern) => Some(Map(searchNode.nodeElement -> otherNode.nodeElement)) case (searchRel: RelationshipPattern, otherRel: RelationshipPattern) => Some(Map(searchRel.relElement -> otherRel.relElement)) case (search: NodeRelPattern, other: NodeRelPattern) => - Some(Map(search.nodeElement -> other.nodeElement, search.relElement -> other.relElement)) + Some( + Map( + search.nodeElement -> other.nodeElement, + search.relElement -> other.relElement + ) + ) case (search: TripletPattern, other: TripletPattern) => - Some(Map(search.sourceElement -> other.sourceElement, search.relElement -> other.relElement, search.targetElement -> other.targetElement)) + Some( + Map( + search.sourceElement -> other.sourceElement, + search.relElement -> other.relElement, + search.targetElement -> other.targetElement + ) + ) case _ => None } } else { @@ -111,23 +130,28 @@ sealed trait Pattern { } /** - * Returns true if the current pattern is a sub type of the other pattern. - * A pattern is subtype of another if there is an bijective mapping between this patterns - * elements into the other pattern's elements and for every mapping the source element type is - * a subtype of the target element type. + * Returns true if the current pattern is a sub type of the other pattern. A pattern is subtype + * of another if there is an bijective mapping between this patterns elements into the other + * pattern's elements and for every mapping the source element type is a subtype of the target + * element type. * - * @param other reference pattern - * @return true if this pattern is subtype of the reference pattern + * @param other + * reference pattern + * @return + * true if this pattern is subtype of the reference pattern */ def subTypeOf(other: Pattern): Boolean /** - * Returns true if the current pattern is a super type of the other pattern. - * A pattern is super type of another pattern iff the other pattern is subtype of the first pattern. + * Returns true if the current pattern is a super type of the other pattern. A pattern is super + * type of another pattern iff the other pattern is subtype of the first pattern. * - * @see [[org.opencypher.okapi.api.graph.Pattern#subTypeOf]] - * @param other reference - * @return true if this pattern s supertype of the reference pattern + * @see + * [[org.opencypher.okapi.api.graph.Pattern#subTypeOf]] + * @param other + * reference + * @return + * true if this pattern s supertype of the reference pattern */ def superTypeOf(other: Pattern): Boolean = other.subTypeOf(this) } @@ -139,7 +163,8 @@ case class NodePattern(nodeType: CTNode) extends Pattern { override def topology: Map[PatternElement, Connection] = Map.empty override def subTypeOf(other: Pattern): Boolean = other match { - case NodePattern(otherNodeType) => nodeType.withoutGraph.subTypeOf(otherNodeType.withoutGraph) + case NodePattern(otherNodeType) => + nodeType.withoutGraph.subTypeOf(otherNodeType.withoutGraph) case _ => false } } @@ -151,7 +176,8 @@ case class RelationshipPattern(relType: CTRelationship) extends Pattern { override def topology: Map[PatternElement, Connection] = Map.empty override def subTypeOf(other: Pattern): Boolean = other match { - case RelationshipPattern(otherRelType) => relType.withoutGraph.subTypeOf(otherRelType.withoutGraph) + case RelationshipPattern(otherRelType) => + relType.withoutGraph.subTypeOf(otherRelType.withoutGraph) case _ => false } } @@ -174,14 +200,22 @@ case class NodeRelPattern(nodeType: CTNode, relType: CTRelationship) extends Pat override def subTypeOf(other: Pattern): Boolean = other match { case NodeRelPattern(otherNodeType, otherRelType) => - nodeType.withoutGraph.subTypeOf(otherNodeType.withoutGraph) && relType.withoutGraph.subTypeOf(otherRelType.withoutGraph) + nodeType.withoutGraph.subTypeOf( + otherNodeType.withoutGraph + ) && relType.withoutGraph.subTypeOf(otherRelType.withoutGraph) case _ => false } } -case class TripletPattern(sourceNodeType: CTNode, relType: CTRelationship, targetNodeType: CTNode) extends Pattern { - val sourceElement = PatternElement("source_" + DEFAULT_NODE_NAME, sourceNodeType) - val targetElement = PatternElement("target_" + DEFAULT_NODE_NAME, targetNodeType) +case class TripletPattern( + sourceNodeType: CTNode, + relType: CTRelationship, + targetNodeType: CTNode +) extends Pattern { + val sourceElement = + PatternElement("source_" + DEFAULT_NODE_NAME, sourceNodeType) + val targetElement = + PatternElement("target_" + DEFAULT_NODE_NAME, targetNodeType) val relElement = PatternElement(DEFAULT_REL_NAME, relType) override def elements: Set[PatternElement] = Set( diff --git a/okapi-api/src/main/scala/org/opencypher/okapi/api/graph/PropertyGraph.scala b/okapi-api/src/main/scala/org/opencypher/okapi/api/graph/PropertyGraph.scala index 3e8177f5b1..e34f42bb04 100644 --- a/okapi-api/src/main/scala/org/opencypher/okapi/api/graph/PropertyGraph.scala +++ b/okapi-api/src/main/scala/org/opencypher/okapi/api/graph/PropertyGraph.scala @@ -34,81 +34,107 @@ import org.opencypher.okapi.api.value.CypherValue.CypherMap /** * A Property Graph as defined by the openCypher Property Graph Model. * - * A graph is always tied to and managed by a session. The lifetime of a graph is bounded - * by the session lifetime. + * A graph is always tied to and managed by a session. The lifetime of a graph is bounded by the + * session lifetime. * * A graph always has a schema, which describes the properties of the elements in the graph, * grouped by the labels and relationship types of the elements. * - * @see [[https://github.com/opencypher/openCypher/blob/master/docs/property-graph-model.adoc openCypher Property Graph Model]] + * @see + * [[https://github.com/opencypher/openCypher/blob/master/docs/property-graph-model.adoc openCypher Property Graph Model]] */ trait PropertyGraph { /** * The schema that describes this graph. * - * @return the schema of this graph. + * @return + * the schema of this graph. */ def schema: PropertyGraphSchema /** * The session in which this graph is managed. * - * @return the session of this graph. + * @return + * the session of this graph. */ def session: CypherSession /** * Returns all nodes in this graph with the given [[org.opencypher.okapi.api.types.CTNode]] type. * - * @param name field name for the returned nodes - * @param nodeCypherType node type used for selection - * @param exactLabelMatch return only nodes that have exactly the given labels - * @return table of nodes of the specified type + * @param name + * field name for the returned nodes + * @param nodeCypherType + * node type used for selection + * @param exactLabelMatch + * return only nodes that have exactly the given labels + * @return + * table of nodes of the specified type */ - def nodes(name: String, nodeCypherType: CTNode = CTNode, exactLabelMatch: Boolean = false): CypherRecords + def nodes( + name: String, + nodeCypherType: CTNode = CTNode, + exactLabelMatch: Boolean = false + ): CypherRecords /** - * Returns all relationships in this graph with the given [[org.opencypher.okapi.api.types.CTRelationship]] type. + * Returns all relationships in this graph with the given + * [[org.opencypher.okapi.api.types.CTRelationship]] type. * - * @param name field name for the returned relationships - * @param relCypherType relationship type used for selection - * @return table of relationships of the specified type + * @param name + * field name for the returned relationships + * @param relCypherType + * relationship type used for selection + * @return + * table of relationships of the specified type */ - def relationships(name: String, relCypherType: CTRelationship = CTRelationship): CypherRecords + def relationships( + name: String, + relCypherType: CTRelationship = CTRelationship + ): CypherRecords /** - * Constructs the union of this graph and the argument graphs. Note that the argument graphs have to - * be managed by the same session as this graph. + * Constructs the union of this graph and the argument graphs. Note that the argument graphs have + * to be managed by the same session as this graph. * - * This operation does not merge any nodes or relationships, but simply creates a new graph consisting - * of all nodes and relationships of the argument graphs. + * This operation does not merge any nodes or relationships, but simply creates a new graph + * consisting of all nodes and relationships of the argument graphs. * - * @param others argument graphs with which to union - * @return union of this and the argument graph + * @param others + * argument graphs with which to union + * @return + * union of this and the argument graph */ def unionAll(others: PropertyGraph*): PropertyGraph /** - * Executes a Cypher query in the session that manages this graph, using this graph as the input graph. + * Executes a Cypher query in the session that manages this graph, using this graph as the input + * graph. * - * @param query Cypher query to execute - * @param parameters parameters used by the Cypher query - * @return result of the query. + * @param query + * Cypher query to execute + * @param parameters + * parameters used by the Cypher query + * @return + * result of the query. */ def cypher( query: String, parameters: CypherMap = CypherMap.empty, drivingTable: Option[CypherRecords] = None, queryCatalog: Map[QualifiedGraphName, PropertyGraph] = Map.empty - ): CypherResult = session.cypherOnGraph(this, query, parameters, drivingTable, queryCatalog) + ): CypherResult = + session.cypherOnGraph(this, query, parameters, drivingTable, queryCatalog) /** * Returns all patterns that the graph can provide * - * @return patterns that the graph can provide + * @return + * patterns that the graph can provide */ def patterns: Set[Pattern] = schema.labelCombinations.combos.map(c => NodePattern(CTNode(c))) ++ - schema.relationshipTypes.map(r => RelationshipPattern(CTRelationship(r))) + schema.relationshipTypes.map(r => RelationshipPattern(CTRelationship(r))) } diff --git a/okapi-api/src/main/scala/org/opencypher/okapi/api/graph/PropertyGraphCatalog.scala b/okapi-api/src/main/scala/org/opencypher/okapi/api/graph/PropertyGraphCatalog.scala index b2cdbac868..b1c1105e8d 100644 --- a/okapi-api/src/main/scala/org/opencypher/okapi/api/graph/PropertyGraphCatalog.scala +++ b/okapi-api/src/main/scala/org/opencypher/okapi/api/graph/PropertyGraphCatalog.scala @@ -35,28 +35,32 @@ import org.opencypher.okapi.api.io.PropertyGraphDataSource */ trait PropertyGraphCatalog { - //################################################ + // ################################################ // Property Graph Data Source specific functions - //################################################ + // ################################################ /** * Returns all [[org.opencypher.okapi.api.graph.Namespace]]s registered at this catalog. * - * @return registered namespaces + * @return + * registered namespaces */ def namespaces: Set[Namespace] /** * Returns the namespace associated with the session. * - * @return session namespace + * @return + * session namespace */ def sessionNamespace: Namespace /** * Returns all registered [[org.opencypher.okapi.api.io.PropertyGraphDataSource]]s. * - * @return a map of all PGDS registered at this catalog, keyed by their [[org.opencypher.okapi.api.graph.Namespace]]s. + * @return + * a map of all PGDS registered at this catalog, keyed by their + * [[org.opencypher.okapi.api.graph.Namespace]]s. */ def listSources: Map[Namespace, PropertyGraphDataSource] @@ -64,114 +68,141 @@ trait PropertyGraphCatalog { * Returns the [[org.opencypher.okapi.api.io.PropertyGraphDataSource]] that is registered under * the given [[org.opencypher.okapi.api.graph.Namespace]]. * - * @param namespace namespace for lookup - * @return property graph data source + * @param namespace + * namespace for lookup + * @return + * property graph data source */ def source(namespace: Namespace): PropertyGraphDataSource /** - * Register the given [[org.opencypher.okapi.api.io.PropertyGraphDataSource]] under - * the specific [[org.opencypher.okapi.api.graph.Namespace]] within this catalog. + * Register the given [[org.opencypher.okapi.api.io.PropertyGraphDataSource]] under the specific + * [[org.opencypher.okapi.api.graph.Namespace]] within this catalog. * - * This enables a user to refer to that [[org.opencypher.okapi.api.io.PropertyGraphDataSource]] within a Cypher query. + * This enables a user to refer to that [[org.opencypher.okapi.api.io.PropertyGraphDataSource]] + * within a Cypher query. * - * Note, that it is not allowed to overwrite an already registered [[org.opencypher.okapi.api.graph.Namespace]]. - * Use [[org.opencypher.okapi.api.graph.PropertyGraphCatalog#deregisterSource]] first. + * Note, that it is not allowed to overwrite an already registered + * [[org.opencypher.okapi.api.graph.Namespace]]. Use + * [[org.opencypher.okapi.api.graph.PropertyGraphCatalog#deregisterSource]] first. * - * @param namespace namespace for lookup - * @param dataSource property graph data source + * @param namespace + * namespace for lookup + * @param dataSource + * property graph data source */ def register(namespace: Namespace, dataSource: PropertyGraphDataSource): Unit /** - * De-registers a [[org.opencypher.okapi.api.io.PropertyGraphDataSource]] from the catalog - * by its given [[org.opencypher.okapi.api.graph.Namespace]]. + * De-registers a [[org.opencypher.okapi.api.io.PropertyGraphDataSource]] from the catalog by its + * given [[org.opencypher.okapi.api.graph.Namespace]]. * - * @param namespace namespace for lookup + * @param namespace + * namespace for lookup */ def deregister(namespace: Namespace): Unit - //################################################ + // ################################################ // Property Graph specific functions - //################################################ + // ################################################ /** - * Returns a set of [[org.opencypher.okapi.api.graph.QualifiedGraphName]]s for [[org.opencypher.okapi.api.graph.PropertyGraph]]s - * that can be provided by this catalog. + * Returns a set of [[org.opencypher.okapi.api.graph.QualifiedGraphName]]s for + * [[org.opencypher.okapi.api.graph.PropertyGraph]]s that can be provided by this catalog. * - * @return qualified names of graphs that can be provided + * @return + * qualified names of graphs that can be provided */ def graphNames: Set[QualifiedGraphName] /** - * Returns a set of [[org.opencypher.okapi.api.graph.QualifiedGraphName]]s for stored view queries - * that can be provided by this catalog. + * Returns a set of [[org.opencypher.okapi.api.graph.QualifiedGraphName]]s for stored view + * queries that can be provided by this catalog. * - * @return qualified names of view queries that can be provided + * @return + * qualified names of view queries that can be provided */ def viewNames: Set[QualifiedGraphName] /** - * Returns all the qualified graph names known to this catalog. These identify either graphs stored in property - * graph data sources or view queries stored directly in the catalog. + * Returns all the qualified graph names known to this catalog. These identify either graphs + * stored in property graph data sources or view queries stored directly in the catalog. * - * @return qualified names of graphs and view queries + * @return + * qualified names of graphs and view queries */ def catalogNames: Set[QualifiedGraphName] = graphNames ++ viewNames /** - * Stores the given [[org.opencypher.okapi.api.graph.PropertyGraph]] using - * the [[org.opencypher.okapi.api.io.PropertyGraphDataSource]] registered under - * the [[org.opencypher.okapi.api.graph.Namespace]] of the specified string representation - * of a [[org.opencypher.okapi.api.graph.QualifiedGraphName]]. + * Stores the given [[org.opencypher.okapi.api.graph.PropertyGraph]] using the + * [[org.opencypher.okapi.api.io.PropertyGraphDataSource]] registered under the + * [[org.opencypher.okapi.api.graph.Namespace]] of the specified string representation of a + * [[org.opencypher.okapi.api.graph.QualifiedGraphName]]. * - * @param qualifiedGraphName qualified graph name - * @param graph property graph to store + * @param qualifiedGraphName + * qualified graph name + * @param graph + * property graph to store */ def store(qualifiedGraphName: String, graph: PropertyGraph): Unit = store(QualifiedGraphName(qualifiedGraphName), graph) /** - * Stores the given [[org.opencypher.okapi.api.graph.PropertyGraph]] using - * the [[org.opencypher.okapi.api.io.PropertyGraphDataSource]] registered under - * the [[org.opencypher.okapi.api.graph.Namespace]] of the specified [[org.opencypher.okapi.api.graph.QualifiedGraphName]]. + * Stores the given [[org.opencypher.okapi.api.graph.PropertyGraph]] using the + * [[org.opencypher.okapi.api.io.PropertyGraphDataSource]] registered under the + * [[org.opencypher.okapi.api.graph.Namespace]] of the specified + * [[org.opencypher.okapi.api.graph.QualifiedGraphName]]. * - * @param qualifiedGraphName qualified graph name - * @param graph property graph to store + * @param qualifiedGraphName + * qualified graph name + * @param graph + * property graph to store */ def store(qualifiedGraphName: QualifiedGraphName, graph: PropertyGraph): Unit /** * Stores the given Cypher query as a view with the given qualified graph name in the catalog. - * The view may be parameterized with graph references. Graph reference parameters may only be used - * in FROM clauses in the view definition. + * The view may be parameterized with graph references. Graph reference parameters may only be + * used in FROM clauses in the view definition. * - * @param qualifiedGraphName qualified graph name - * @param parameterNames list of graph reference parameters used in the view query - * @param viewQuery query string for the view definition + * @param qualifiedGraphName + * qualified graph name + * @param parameterNames + * list of graph reference parameters used in the view query + * @param viewQuery + * query string for the view definition */ - def store(qualifiedGraphName: QualifiedGraphName, parameterNames: List[String], viewQuery: String): Unit + def store( + qualifiedGraphName: QualifiedGraphName, + parameterNames: List[String], + viewQuery: String + ): Unit /** - * Removes the [[org.opencypher.okapi.api.graph.PropertyGraph]] with the given qualified graph name. + * Removes the [[org.opencypher.okapi.api.graph.PropertyGraph]] with the given qualified graph + * name. * - * @param qualifiedGraphName name of the graph within the session. + * @param qualifiedGraphName + * name of the graph within the session. */ def dropGraph(qualifiedGraphName: String): Unit = dropGraph(QualifiedGraphName(qualifiedGraphName)) /** - * Removes the [[org.opencypher.okapi.api.graph.PropertyGraph]] with the given qualified name from the data source - * associated with the specified [[org.opencypher.okapi.api.graph.Namespace]]. + * Removes the [[org.opencypher.okapi.api.graph.PropertyGraph]] with the given qualified name + * from the data source associated with the specified + * [[org.opencypher.okapi.api.graph.Namespace]]. * - * @param qualifiedGraphName qualified graph name + * @param qualifiedGraphName + * qualified graph name */ def dropGraph(qualifiedGraphName: QualifiedGraphName): Unit /** * Removes the view with the given qualified graph name from the catalog. * - * @param qualifiedGraphName name of the view + * @param qualifiedGraphName + * name of the view */ def dropView(qualifiedGraphName: String): Unit = dropView(QualifiedGraphName(qualifiedGraphName)) @@ -179,7 +210,8 @@ trait PropertyGraphCatalog { /** * Removes the view with the given qualified graph name from the catalog. * - * @param qualifiedGraphName name of the view + * @param qualifiedGraphName + * name of the view */ def dropView(qualifiedGraphName: QualifiedGraphName): Unit @@ -187,29 +219,38 @@ trait PropertyGraphCatalog { * Returns the [[org.opencypher.okapi.api.graph.PropertyGraph]] that is stored under the given * string representation of a [[org.opencypher.okapi.api.graph.QualifiedGraphName]]. * - * @param qualifiedGraphName qualified graph name - * @return property graph + * @param qualifiedGraphName + * qualified graph name + * @return + * property graph */ def graph(qualifiedGraphName: String): PropertyGraph = graph(QualifiedGraphName(qualifiedGraphName)) /** - * Returns the [[org.opencypher.okapi.api.graph.PropertyGraph]] that is stored at - * the given [[org.opencypher.okapi.api.graph.QualifiedGraphName]]. + * Returns the [[org.opencypher.okapi.api.graph.PropertyGraph]] that is stored at the given + * [[org.opencypher.okapi.api.graph.QualifiedGraphName]]. * - * @param qualifiedGraphName qualified graph name - * @return property graph + * @param qualifiedGraphName + * qualified graph name + * @return + * property graph */ def graph(qualifiedGraphName: QualifiedGraphName): PropertyGraph /** - * Returns the [[org.opencypher.okapi.api.graph.PropertyGraph]] that is created by the passed [[ViewInvocation]]. + * Returns the [[org.opencypher.okapi.api.graph.PropertyGraph]] that is created by the passed + * [[ViewInvocation]]. * * The view is instantiated with `parameters`. * - * @param viewInvocation view invocation to execute - * @return property graph returned by the parameterized view + * @param viewInvocation + * view invocation to execute + * @return + * property graph returned by the parameterized view */ - private[opencypher] def view(viewInvocation: ViewInvocation)(implicit session: CypherSession): PropertyGraph + private[opencypher] def view(viewInvocation: ViewInvocation)(implicit + session: CypherSession + ): PropertyGraph } diff --git a/okapi-api/src/main/scala/org/opencypher/okapi/api/graph/QualifiedGraphName.scala b/okapi-api/src/main/scala/org/opencypher/okapi/api/graph/QualifiedGraphName.scala index 23cd7e7476..136754e651 100644 --- a/okapi-api/src/main/scala/org/opencypher/okapi/api/graph/QualifiedGraphName.scala +++ b/okapi-api/src/main/scala/org/opencypher/okapi/api/graph/QualifiedGraphName.scala @@ -30,20 +30,22 @@ import org.opencypher.okapi.impl.exception.IllegalArgumentException import org.opencypher.okapi.impl.io.SessionGraphDataSource /** - * A graph name is used to address a specific graph within a [[Namespace]] and is used for lookups in the - * [[org.opencypher.okapi.api.graph.CypherSession]]. + * A graph name is used to address a specific graph within a [[Namespace]] and is used for lookups + * in the [[org.opencypher.okapi.api.graph.CypherSession]]. * - * @param value string representing the graph name + * @param value + * string representing the graph name */ case class GraphName(value: String) extends AnyVal { override def toString: String = value } /** - * A namespace is used to address different [[org.opencypher.okapi.api.io.PropertyGraphDataSource]] implementations within a - * [[org.opencypher.okapi.api.graph.CypherSession]]. + * A namespace is used to address different [[org.opencypher.okapi.api.io.PropertyGraphDataSource]] + * implementations within a [[org.opencypher.okapi.api.graph.CypherSession]]. * - * @param value string representing the namespace + * @param value + * string representing the namespace */ case class Namespace(value: String) extends AnyVal { override def toString: String = value @@ -52,30 +54,43 @@ case class Namespace(value: String) extends AnyVal { object QualifiedGraphName { /** - * Returns a [[org.opencypher.okapi.api.graph.QualifiedGraphName]] from its string representation. A qualified graph name consists of a namespace - * part and a graph name part separated by a '.' character. For example, + * Returns a [[org.opencypher.okapi.api.graph.QualifiedGraphName]] from its string + * representation. A qualified graph name consists of a namespace part and a graph name part + * separated by a '.' character. For example, * * {{{ * mynamespace.mygraphname * mynamespace.my.graph.name * }}} * - * are valid qualified graph names. The separation between namespace and graph name is expected to be at the first - * occurring '.'. A graph name may contain an arbitrary number of additional '.' characters. Note that a string - * without any '.' characters is considered to be associated with the [[org.opencypher.okapi.impl.io.SessionGraphDataSource]]. + * are valid qualified graph names. The separation between namespace and graph name is expected + * to be at the first occurring '.'. A graph name may contain an arbitrary number of additional + * '.' characters. Note that a string without any '.' characters is considered to be associated + * with the [[org.opencypher.okapi.impl.io.SessionGraphDataSource]]. * - * @param qualifiedGraphName string representation of a qualified graph name - * @return qualified graph name + * @param qualifiedGraphName + * string representation of a qualified graph name + * @return + * qualified graph name */ - def apply(qualifiedGraphName: String): QualifiedGraphName = apply(splitQgn(qualifiedGraphName)) + def apply(qualifiedGraphName: String): QualifiedGraphName = apply( + splitQgn(qualifiedGraphName) + ) - private[okapi] def splitQgn(qgn: String): List[String] = qgn.split("\\.").toList + private[okapi] def splitQgn(qgn: String): List[String] = + qgn.split("\\.").toList - private[okapi] def apply(parts: List[String]): QualifiedGraphName = parts match { - case Nil => throw IllegalArgumentException("qualified graph name or single graph name") - case head :: Nil => QualifiedGraphName(SessionGraphDataSource.Namespace, GraphName(head)) - case head :: tail => QualifiedGraphName(Namespace(head), GraphName(tail.mkString("."))) - } + private[okapi] def apply(parts: List[String]): QualifiedGraphName = + parts match { + case Nil => + throw IllegalArgumentException( + "qualified graph name or single graph name" + ) + case head :: Nil => + QualifiedGraphName(SessionGraphDataSource.Namespace, GraphName(head)) + case head :: tail => + QualifiedGraphName(Namespace(head), GraphName(tail.mkString("."))) + } } @@ -90,8 +105,10 @@ object QualifiedGraphName { * * Here, {{myNamespace.myGraphName}} represents a qualified graph name. * - * @param namespace namespace part of the qualified graph name - * @param graphName graph name part of the qualified graph name + * @param namespace + * namespace part of the qualified graph name + * @param graphName + * graph name part of the qualified graph name */ case class QualifiedGraphName(namespace: Namespace, graphName: GraphName) { override def toString: String = s"$namespace.$graphName" diff --git a/okapi-api/src/main/scala/org/opencypher/okapi/api/io/PropertyGraphDataSource.scala b/okapi-api/src/main/scala/org/opencypher/okapi/api/io/PropertyGraphDataSource.scala index 493a2bdc52..82660a9336 100644 --- a/okapi-api/src/main/scala/org/opencypher/okapi/api/io/PropertyGraphDataSource.scala +++ b/okapi-api/src/main/scala/org/opencypher/okapi/api/io/PropertyGraphDataSource.scala @@ -30,91 +30,110 @@ import org.opencypher.okapi.api.graph.{GraphName, PropertyGraph} import org.opencypher.okapi.api.schema.PropertyGraphSchema /** - * Property Graph Data Source (PGDS) is used to read and write property graphs, for example from database or - * file systems, memory-based collections, etc. + * Property Graph Data Source (PGDS) is used to read and write property graphs, for example from + * database or file systems, memory-based collections, etc. * - * [[PropertyGraphDataSource]] is the main interface for connecting custom data sources for specific openCypher implementations. + * [[PropertyGraphDataSource]] is the main interface for connecting custom data sources for + * specific openCypher implementations. * - * A PGDS can handle multiple property graphs and distinguishes between them using [[org.opencypher.okapi.api.graph.GraphName]]s. - * Furthermore, a PGDS can be registered at a [[org.opencypher.okapi.api.graph.CypherSession]] using a specific - * [[org.opencypher.okapi.api.graph.Namespace]] which enables accessing a [[org.opencypher.okapi.api.graph.PropertyGraph]] from within a Cypher query. + * A PGDS can handle multiple property graphs and distinguishes between them using + * [[org.opencypher.okapi.api.graph.GraphName]]s. Furthermore, a PGDS can be registered at a + * [[org.opencypher.okapi.api.graph.CypherSession]] using a specific + * [[org.opencypher.okapi.api.graph.Namespace]] which enables accessing a + * [[org.opencypher.okapi.api.graph.PropertyGraph]] from within a Cypher query. */ trait PropertyGraphDataSource { /** - * Returns `true` if the data source can provide a graph for the given [[org.opencypher.okapi.api.graph.GraphName]]. + * Returns `true` if the data source can provide a graph for the given + * [[org.opencypher.okapi.api.graph.GraphName]]. * - * @param name name of the graph within the data source - * @return `true`, iff the graph can be provided + * @param name + * name of the graph within the data source + * @return + * `true`, iff the graph can be provided */ def hasGraph(name: GraphName): Boolean /** * Returns the [[org.opencypher.okapi.api.graph.PropertyGraph]] for the given name. * - * Throws a [[org.opencypher.okapi.impl.exception.GraphNotFoundException]] when that graph cannot be provided. + * Throws a [[org.opencypher.okapi.impl.exception.GraphNotFoundException]] when that graph cannot + * be provided. * - * @param name name of the graph within the data source - * @return property graph + * @param name + * name of the graph within the data source + * @return + * property graph */ def graph(name: GraphName): PropertyGraph /** - * Returns the [[org.opencypher.okapi.api.schema.PropertyGraphSchema]] of the graph that is stored under the given name. + * Returns the [[org.opencypher.okapi.api.schema.PropertyGraphSchema]] of the graph that is + * stored under the given name. * - * This method gives implementers the ability to efficiently retrieve a graph schema from the data source directly. - * For reasons of performance, it is highly recommended to make a schema available through this call. If an efficient - * retrieval is not possible, the call is typically forwarded to the graph using the [[org.opencypher.okapi.api.graph.PropertyGraph#schema]] - * call, which may require materialising the full graph. + * This method gives implementers the ability to efficiently retrieve a graph schema from the + * data source directly. For reasons of performance, it is highly recommended to make a schema + * available through this call. If an efficient retrieval is not possible, the call is typically + * forwarded to the graph using the [[org.opencypher.okapi.api.graph.PropertyGraph#schema]] call, + * which may require materialising the full graph. * * Returns `None` when the schema cannot be provided. * - * @param name name of the graph within the data source - * @return graph schema when available + * @param name + * name of the graph within the data source + * @return + * graph schema when available */ def schema(name: GraphName): Option[PropertyGraphSchema] /** - * Stores the given [[org.opencypher.okapi.api.graph.PropertyGraph]] under the given [[org.opencypher.okapi.api.graph.GraphName]] within the data source. + * Stores the given [[org.opencypher.okapi.api.graph.PropertyGraph]] under the given + * [[org.opencypher.okapi.api.graph.GraphName]] within the data source. * - * If the data source already stores a graph under the given name, a [[org.opencypher.okapi.impl.exception.GraphAlreadyExistsException]] should be thrown. + * If the data source already stores a graph under the given name, a + * [[org.opencypher.okapi.impl.exception.GraphAlreadyExistsException]] should be thrown. * * Throws an [[java.lang.UnsupportedOperationException]] if not supported. * - * @param name name under which the graph shall be stored - * @param graph property graph to store + * @param name + * name under which the graph shall be stored + * @param graph + * property graph to store */ def store(name: GraphName, graph: PropertyGraph): Unit /** - * Deletes the [[org.opencypher.okapi.api.graph.PropertyGraph]] within the data source that is stored under the given [[org.opencypher.okapi.api.graph.GraphName]]. + * Deletes the [[org.opencypher.okapi.api.graph.PropertyGraph]] within the data source that is + * stored under the given [[org.opencypher.okapi.api.graph.GraphName]]. * * Throws an [[java.lang.UnsupportedOperationException]] if not supported. * * This operation will do nothing if the graph is not found. * - * @param name name under which the graph is stored + * @param name + * name under which the graph is stored */ def delete(name: GraphName): Unit /** - * Returns a set of [[org.opencypher.okapi.api.graph.GraphName]]s for [[org.opencypher.okapi.api.graph.PropertyGraph]]s - * that can be provided by this data source. + * Returns a set of [[org.opencypher.okapi.api.graph.GraphName]]s for + * [[org.opencypher.okapi.api.graph.PropertyGraph]]s that can be provided by this data source. * - * For data sources that provide a known set of graphs, this returns the set of all graphs that can be provided. + * For data sources that provide a known set of graphs, this returns the set of all graphs that + * can be provided. * - * For data sources that can construct graphs dynamically, this merely returns the names of the graphs that have - * already been provided and not yet deleted. + * For data sources that can construct graphs dynamically, this merely returns the names of the + * graphs that have already been provided and not yet deleted. * * For every returned name, `hasGraph` is guaranteed to return `true`. * - * @return names of graphs that can be provided + * @return + * names of graphs that can be provided */ def graphNames: Set[GraphName] - /** - * Resets the data source to its initial state including potential caches. - */ + /** Resets the data source to its initial state including potential caches. */ def reset(): Unit = () } diff --git a/okapi-api/src/main/scala/org/opencypher/okapi/api/io/conversion/ElementMapping.scala b/okapi-api/src/main/scala/org/opencypher/okapi/api/io/conversion/ElementMapping.scala index 4d08c3f1bc..9cf3f7f24a 100644 --- a/okapi-api/src/main/scala/org/opencypher/okapi/api/io/conversion/ElementMapping.scala +++ b/okapi-api/src/main/scala/org/opencypher/okapi/api/io/conversion/ElementMapping.scala @@ -35,20 +35,25 @@ object ElementMapping { } /** - * Represents a mapping from a source with key-based access of element components (e.g. a table definition) to a Pattern. - * The purpose of this class is to define a mapping from an external data source to a property graph. + * Represents a mapping from a source with key-based access of element components (e.g. a table + * definition) to a Pattern. The purpose of this class is to define a mapping from an external data + * source to a property graph. * * The [[pattern]] describes the shape of the pattern that is described by this mapping * * The [[idKeys]] describe the mappings for each pattern element, which map the element identifiers * to columns within the source data. * - * The [[properties]] represent mappings for every pattern element from property keys to keys in the source data. - * The retrieved value from the source is expected to be convertible to a valid [[org.opencypher.okapi.api.value.CypherValue]]. + * The [[properties]] represent mappings for every pattern element from property keys to keys in + * the source data. The retrieved value from the source is expected to be convertible to a valid + * [[org.opencypher.okapi.api.value.CypherValue]]. * - * @param pattern the pattern described by this mapping - * @param properties mapping from property key to source property key - * @param idKeys mapping for the key to access the element identifier in the source data + * @param pattern + * the pattern described by this mapping + * @param properties + * mapping from property key to source property key + * @param idKeys + * mapping for the key to access the element identifier in the source data */ case class ElementMapping( pattern: Pattern, @@ -58,23 +63,29 @@ case class ElementMapping( validate() - lazy val allSourceIdKeys: Seq[String] = idKeys.values.flatMap(keyMapping => keyMapping.values).toSeq.sorted + lazy val allSourceIdKeys: Seq[String] = + idKeys.values.flatMap(keyMapping => keyMapping.values).toSeq.sorted - lazy val allSourcePropertyKeys: Seq[String] = properties.values.flatMap(keyMapping => keyMapping.values).toSeq.sorted + lazy val allSourcePropertyKeys: Seq[String] = + properties.values.flatMap(keyMapping => keyMapping.values).toSeq.sorted - lazy val allSourceKeys: Seq[String] = (allSourceIdKeys ++ allSourcePropertyKeys).sorted + lazy val allSourceKeys: Seq[String] = + (allSourceIdKeys ++ allSourcePropertyKeys).sorted protected def validate(): Unit = { val sourceKeys = allSourceKeys if (allSourceKeys.size != sourceKeys.toSet.size) { - val duplicateColumns = sourceKeys.groupBy(identity).filter { case (_, items) => items.size > 1 } + val duplicateColumns = sourceKeys.groupBy(identity).filter { case (_, items) => + items.size > 1 + } throw IllegalArgumentException( "One-to-one mapping from element elements to source keys", - s"Duplicate columns: $duplicateColumns") + s"Duplicate columns: $duplicateColumns" + ) } pattern.elements.foreach { - case e@PatternElement(_, CTRelationship(types, _)) if types.size != 1 => + case e @ PatternElement(_, CTRelationship(types, _)) if types.size != 1 => throw IllegalArgumentException( s"A single implied type for element $e", types @@ -83,4 +94,3 @@ case class ElementMapping( } } } - diff --git a/okapi-api/src/main/scala/org/opencypher/okapi/api/io/conversion/NodeMappingBuilder.scala b/okapi-api/src/main/scala/org/opencypher/okapi/api/io/conversion/NodeMappingBuilder.scala index fa2c04a141..ef66ee4bb3 100644 --- a/okapi-api/src/main/scala/org/opencypher/okapi/api/io/conversion/NodeMappingBuilder.scala +++ b/okapi-api/src/main/scala/org/opencypher/okapi/api/io/conversion/NodeMappingBuilder.scala @@ -30,34 +30,42 @@ import org.opencypher.okapi.api.graph._ import org.opencypher.okapi.api.types.CTNode object NodeMappingBuilder { + /** * Alias for [[withSourceIdKey]]. * - * @param sourceIdKey key to access the node identifier in the source data - * @return node mapping + * @param sourceIdKey + * key to access the node identifier in the source data + * @return + * node mapping */ def on(sourceIdKey: String): NodeMappingBuilder = withSourceIdKey(sourceIdKey) /** - * - * @param sourceIdKey represents a key to the node identifier within the source data. The retrieved value - * from the source data is expected to be a [[Long]] value that is unique among nodes. - * @return node mapping + * @param sourceIdKey + * represents a key to the node identifier within the source data. The retrieved value from the + * source data is expected to be a [[Long]] value that is unique among nodes. + * @return + * node mapping */ def withSourceIdKey(sourceIdKey: String): NodeMappingBuilder = NodeMappingBuilder(sourceIdKey) /** - * Creates a NodeMapping where optional labels and property keys match with their corresponding keys in the source - * data. + * Creates a NodeMapping where optional labels and property keys match with their corresponding + * keys in the source data. * * See [[NodeMappingBuilder]] for further information. * - * @param nodeIdKey key to access the node identifier in the source data - * @param impliedLabels set of node labels - * @param propertyKeys set of property keys - * @return node mapping + * @param nodeIdKey + * key to access the node identifier in the source data + * @param impliedLabels + * set of node labels + * @param propertyKeys + * set of property keys + * @return + * node mapping */ def create( nodeIdKey: String, @@ -65,35 +73,43 @@ object NodeMappingBuilder { propertyKeys: Set[String] = Set.empty ): ElementMapping = { - val mappingWithImpliedLabels = impliedLabels.foldLeft(NodeMappingBuilder.withSourceIdKey(nodeIdKey)) { - (mapping, label) => mapping.withImpliedLabel(label) - } + val mappingWithImpliedLabels = + impliedLabels.foldLeft(NodeMappingBuilder.withSourceIdKey(nodeIdKey)) { (mapping, label) => + mapping.withImpliedLabel(label) + } - propertyKeys.foldLeft(mappingWithImpliedLabels) { - (mapping, property) => mapping.withPropertyKey(property) - }.build + propertyKeys + .foldLeft(mappingWithImpliedLabels) { (mapping, property) => + mapping.withPropertyKey(property) + } + .build } } /** * Builder to build ElementMapping with a [[NodePattern]]. * - * Represents a mapping from a source with key-based access of node components (e.g. a table definition) to a Cypher - * node. The purpose of this class is to define a mapping from an external data source to a property graph. + * Represents a mapping from a source with key-based access of node components (e.g. a table + * definition) to a Cypher node. The purpose of this class is to define a mapping from an external + * data source to a property graph. * * Construct a [[NodeMappingBuilder]] starting with [[NodeMappingBuilder#on]]. * - * The [[nodeIdKey]] represents a key to the node identifier within the source data. The retrieved value from the - * source data is expected to be a [[scala.Long]] value that is unique among nodes. + * The [[nodeIdKey]] represents a key to the node identifier within the source data. The retrieved + * value from the source data is expected to be a [[scala.Long]] value that is unique among nodes. * * The [[impliedNodeLabels]] represent a set of node labels. * - * The [[propertyMapping]] represents a map from node property keys to keys in the source data. The retrieved value - * from the source is expected to be convertible to a valid [[org.opencypher.okapi.api.value.CypherValue]]. + * The [[propertyMapping]] represents a map from node property keys to keys in the source data. The + * retrieved value from the source is expected to be convertible to a valid + * [[org.opencypher.okapi.api.value.CypherValue]]. * - * @param nodeIdKey key to access the node identifier in the source data - * @param impliedNodeLabels set of node labels - * @param propertyMapping mapping from property key to source property key + * @param nodeIdKey + * key to access the node identifier in the source data + * @param impliedNodeLabels + * set of node labels + * @param propertyMapping + * mapping from property key to source property key */ final case class NodeMappingBuilder( nodeIdKey: String, @@ -109,13 +125,19 @@ final case class NodeMappingBuilder( def withImpliedLabel(label: String): NodeMappingBuilder = copy(impliedNodeLabels = impliedNodeLabels + label) - override protected def updatePropertyMapping(updatedPropertyMapping: Map[String, String]): NodeMappingBuilder = + override protected def updatePropertyMapping( + updatedPropertyMapping: Map[String, String] + ): NodeMappingBuilder = copy(propertyMapping = updatedPropertyMapping) override def build: ElementMapping = { val pattern: NodePattern = NodePattern(CTNode(impliedNodeLabels)) - val properties: Map[PatternElement, Map[String, String]] = Map(pattern.nodeElement -> propertyMapping) - val idKeys: Map[PatternElement, Map[IdKey, String]] = Map(pattern.nodeElement -> Map(SourceIdKey -> nodeIdKey)) + val properties: Map[PatternElement, Map[String, String]] = Map( + pattern.nodeElement -> propertyMapping + ) + val idKeys: Map[PatternElement, Map[IdKey, String]] = Map( + pattern.nodeElement -> Map(SourceIdKey -> nodeIdKey) + ) validate() diff --git a/okapi-api/src/main/scala/org/opencypher/okapi/api/io/conversion/RelationshipMappingBuilder.scala b/okapi-api/src/main/scala/org/opencypher/okapi/api/io/conversion/RelationshipMappingBuilder.scala index 57df05dc12..59187cdcb3 100644 --- a/okapi-api/src/main/scala/org/opencypher/okapi/api/io/conversion/RelationshipMappingBuilder.scala +++ b/okapi-api/src/main/scala/org/opencypher/okapi/api/io/conversion/RelationshipMappingBuilder.scala @@ -33,34 +33,49 @@ import org.opencypher.okapi.impl.exception.IllegalArgumentException object RelationshipMappingBuilder { /** - * @param sourceIdKey represents a key to the relationship identifier within the source data. The retrieved value - * from the source data is expected to be a [[scala.Long]] value that is unique among relationships. - * @return incomplete relationship mapping + * @param sourceIdKey + * represents a key to the relationship identifier within the source data. The retrieved value + * from the source data is expected to be a [[scala.Long]] value that is unique among + * relationships. + * @return + * incomplete relationship mapping */ def withSourceIdKey(sourceIdKey: String): MissingSourceStartNodeKey = new MissingSourceStartNodeKey(sourceIdKey) /** - * Alias for [[org.opencypher.okapi.api.io.conversion.RelationshipMappingBuilder#withSourceIdKey]]. + * Alias for + * [[org.opencypher.okapi.api.io.conversion.RelationshipMappingBuilder#withSourceIdKey]]. * - * @param sourceIdKey represents a key to the relationship identifier within the source data. The retrieved value - * from the source data is expected to be a [[scala.Long]] value that is unique among relationships. - * @return incomplete relationship mapping + * @param sourceIdKey + * represents a key to the relationship identifier within the source data. The retrieved value + * from the source data is expected to be a [[scala.Long]] value that is unique among + * relationships. + * @return + * incomplete relationship mapping */ def on(sourceIdKey: String): MissingSourceStartNodeKey = withSourceIdKey(sourceIdKey) /** - * Creates a RelationshipMapping where property keys match with their corresponding keys in the source data. + * Creates a RelationshipMapping where property keys match with their corresponding keys in the + * source data. * - * See [[org.opencypher.okapi.api.io.conversion.RelationshipMappingBuilder]] for further information. + * See [[org.opencypher.okapi.api.io.conversion.RelationshipMappingBuilder]] for further + * information. * - * @param sourceIdKey key to access the node identifier in the source data - * @param sourceStartNodeKey key to access the start node identifier in the source data - * @param sourceEndNodeKey key to access the end node identifier in the source data - * @param relType relationship type - * @param properties property keys - * @return relationship mapping + * @param sourceIdKey + * key to access the node identifier in the source data + * @param sourceStartNodeKey + * key to access the start node identifier in the source data + * @param sourceEndNodeKey + * key to access the end node identifier in the source data + * @param relType + * relationship type + * @param properties + * property keys + * @return + * relationship mapping */ def create( sourceIdKey: String, @@ -75,64 +90,100 @@ object RelationshipMappingBuilder { .withSourceEndNodeKey(sourceEndNodeKey) .withRelType(relType) - properties.foldLeft(intermediateMapping) { - (mapping, property) => mapping.withPropertyKey(property) - }.build + properties + .foldLeft(intermediateMapping) { (mapping, property) => + mapping.withPropertyKey(property) + } + .build } sealed class MissingSourceStartNodeKey(sourceIdKey: String) { + /** - * @param sourceStartNodeKey represents a key to the start node identifier within the source data. The retrieved - * value from the source data is expected to be a [[scala.Long]] value. - * @return incomplete relationship mapping builder + * @param sourceStartNodeKey + * represents a key to the start node identifier within the source data. The retrieved value + * from the source data is expected to be a [[scala.Long]] value. + * @return + * incomplete relationship mapping builder */ - def withSourceStartNodeKey(sourceStartNodeKey: String): MissingSourceEndNodeKey = + def withSourceStartNodeKey( + sourceStartNodeKey: String + ): MissingSourceEndNodeKey = new MissingSourceEndNodeKey(sourceIdKey, sourceStartNodeKey) /** - * Alias for [[org.opencypher.okapi.api.io.conversion.RelationshipMappingBuilder.MissingSourceStartNodeKey#withSourceStartNodeKey]]. + * Alias for + * [[org.opencypher.okapi.api.io.conversion.RelationshipMappingBuilder.MissingSourceStartNodeKey#withSourceStartNodeKey]]. * - * @param sourceStartNodeKey represents a key to the start node identifier within the source data. The retrieved - * value from the source data is expected to be a [[scala.Long]] value. - * @return incomplete relationship mapping builder + * @param sourceStartNodeKey + * represents a key to the start node identifier within the source data. The retrieved value + * from the source data is expected to be a [[scala.Long]] value. + * @return + * incomplete relationship mapping builder */ def from(sourceStartNodeKey: String): MissingSourceEndNodeKey = withSourceStartNodeKey(sourceStartNodeKey) } - sealed class MissingSourceEndNodeKey(sourceIdKey: String, sourceStartNodeKey: String) { + sealed class MissingSourceEndNodeKey( + sourceIdKey: String, + sourceStartNodeKey: String + ) { + /** - * @param sourceEndNodeKey represents a key to the end node identifier within the source data. The retrieved - * value from the source data is expected to be a [[scala.Long]] value. - * @return incomplete relationship mapping builder + * @param sourceEndNodeKey + * represents a key to the end node identifier within the source data. The retrieved value + * from the source data is expected to be a [[scala.Long]] value. + * @return + * incomplete relationship mapping builder */ def withSourceEndNodeKey(sourceEndNodeKey: String): MissingRelTypeMapping = - new MissingRelTypeMapping(sourceIdKey, sourceStartNodeKey, sourceEndNodeKey) + new MissingRelTypeMapping( + sourceIdKey, + sourceStartNodeKey, + sourceEndNodeKey + ) /** * Alias for [[withSourceEndNodeKey]]. * - * @param sourceEndNodeKey represents a key to the end node identifier within the source data. The retrieved - * value from the source data is expected to be a [[scala.Long]] value. - * @return incomplete relationship mapping builder + * @param sourceEndNodeKey + * represents a key to the end node identifier within the source data. The retrieved value + * from the source data is expected to be a [[scala.Long]] value. + * @return + * incomplete relationship mapping builder */ def to(sourceEndNodeKey: String): MissingRelTypeMapping = withSourceEndNodeKey(sourceEndNodeKey) } - sealed class MissingRelTypeMapping(sourceIdKey: String, sourceStartNodeKey: String, sourceEndNodeKey: String) { + sealed class MissingRelTypeMapping( + sourceIdKey: String, + sourceStartNodeKey: String, + sourceEndNodeKey: String + ) { + /** - * @param relType represents the relationship type for all relationships in the source data - * @return relationship mapping builder + * @param relType + * represents the relationship type for all relationships in the source data + * @return + * relationship mapping builder */ def withRelType(relType: String): RelationshipMappingBuilder = - RelationshipMappingBuilder(sourceIdKey, sourceStartNodeKey, sourceEndNodeKey, relType) + RelationshipMappingBuilder( + sourceIdKey, + sourceStartNodeKey, + sourceEndNodeKey, + relType + ) /** * Alias for [[withRelType]]. * - * @param relType represents the relationship type for all relationships in the source data - * @return relationship mapping builder + * @param relType + * represents the relationship type for all relationships in the source data + * @return + * relationship mapping builder */ def relType(relType: String): RelationshipMappingBuilder = withRelType(relType) @@ -143,17 +194,22 @@ object RelationshipMappingBuilder { /** * Builder to build ElementMapping with a [[NodePattern]]. * - * Represents a mapping from a source with key-based access to relationship components (e.g. a table definition) to a - * Cypher relationship. The purpose of this class is to define a mapping from an external data source to a property - * graph. + * Represents a mapping from a source with key-based access to relationship components (e.g. a + * table definition) to a Cypher relationship. The purpose of this class is to define a mapping + * from an external data source to a property graph. * * Construct a [[RelationshipMappingBuilder]] starting with [[RelationshipMappingBuilder#on]]. * - * @param relationshipIdKey key to access the node identifier in the source data - * @param relationshipStartNodeKey key to access the start node identifier in the source data - * @param relationshipEndNodeKey key to access the end node identifier in the source data - * @param relType a relationship type - * @param propertyMapping mapping from property key to source property key + * @param relationshipIdKey + * key to access the node identifier in the source data + * @param relationshipStartNodeKey + * key to access the start node identifier in the source data + * @param relationshipEndNodeKey + * key to access the end node identifier in the source data + * @param relType + * a relationship type + * @param propertyMapping + * mapping from property key to source property key */ final case class RelationshipMappingBuilder( relationshipIdKey: String, @@ -165,15 +221,21 @@ final case class RelationshipMappingBuilder( override type BuilderType = RelationshipMappingBuilder - override protected def updatePropertyMapping(updatedPropertyMapping: Map[String, String]): RelationshipMappingBuilder = + override protected def updatePropertyMapping( + updatedPropertyMapping: Map[String, String] + ): RelationshipMappingBuilder = copy(propertyMapping = updatedPropertyMapping) override def build: ElementMapping = { validate() - val pattern: RelationshipPattern = RelationshipPattern(CTRelationship(relType)) + val pattern: RelationshipPattern = RelationshipPattern( + CTRelationship(relType) + ) - val properties: Map[PatternElement, Map[String, String]] = Map(pattern.relElement -> propertyMapping) + val properties: Map[PatternElement, Map[String, String]] = Map( + pattern.relElement -> propertyMapping + ) val idKeys: Map[PatternElement, Map[IdKey, String]] = Map( pattern.relElement -> Map( SourceIdKey -> relationshipIdKey, @@ -186,12 +248,14 @@ final case class RelationshipMappingBuilder( } override protected def validate(): Unit = { - val idKeys = Set(relationshipIdKey, relationshipStartNodeKey, relationshipEndNodeKey) + val idKeys = + Set(relationshipIdKey, relationshipStartNodeKey, relationshipEndNodeKey) if (idKeys.size != 3) throw IllegalArgumentException( s"id ($relationshipIdKey), start ($relationshipStartNodeKey) and end ($relationshipEndNodeKey) source keys need to be distinct", - s"non-distinct source keys") + s"non-distinct source keys" + ) } } diff --git a/okapi-api/src/main/scala/org/opencypher/okapi/api/io/conversion/SingleElementMappingBuilder.scala b/okapi-api/src/main/scala/org/opencypher/okapi/api/io/conversion/SingleElementMappingBuilder.scala index 36eb1d9bf6..854748fe34 100644 --- a/okapi-api/src/main/scala/org/opencypher/okapi/api/io/conversion/SingleElementMappingBuilder.scala +++ b/okapi-api/src/main/scala/org/opencypher/okapi/api/io/conversion/SingleElementMappingBuilder.scala @@ -1,29 +1,26 @@ /** - * Copyright (c) 2016-2019 "Neo4j Sweden, AB" [https://neo4j.com] - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * Attribution Notice under the terms of the Apache License 2.0 - * - * This work was created by the collective efforts of the openCypher community. - * Without limiting the terms of Section 6, any Derivative Work that is not - * approved by the public consensus process of the openCypher Implementers Group - * should not be described as “Cypher” (and Cypher® is a registered trademark of - * Neo4j Inc.) or as "openCypher". Extensions by implementers or prototypes or - * proposals for change that have been documented or implemented should only be - * described as "implementation extensions to Cypher" or as "proposed changes to - * Cypher that are not yet approved by the openCypher community". - */ + * Copyright (c) 2016-2019 "Neo4j Sweden, AB" [https://neo4j.com] + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the + * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing permissions and + * limitations under the License. + * + * Attribution Notice under the terms of the Apache License 2.0 + * + * This work was created by the collective efforts of the openCypher community. Without limiting + * the terms of Section 6, any Derivative Work that is not approved by the public consensus process + * of the openCypher Implementers Group should not be described as “Cypher” (and Cypher® is a + * registered trademark of Neo4j Inc.) or as "openCypher". Extensions by implementers or prototypes + * or proposals for change that have been documented or implemented should only be described as + * "implementation extensions to Cypher" or as "proposed changes to Cypher that are not yet + * approved by the openCypher community". + */ package org.opencypher.okapi.api.io.conversion import org.opencypher.okapi.impl.exception.IllegalArgumentException @@ -34,7 +31,9 @@ trait SingleElementMappingBuilder { def propertyMapping: Map[String, String] - protected def updatePropertyMapping(propertyMapping: Map[String, String]): BuilderType + protected def updatePropertyMapping( + propertyMapping: Map[String, String] + ): BuilderType def build: ElementMapping @@ -46,17 +45,24 @@ trait SingleElementMappingBuilder { def withPropertyKey(property: String): BuilderType = withPropertyKey(property, property) - def withPropertyKey(propertyKey: String, sourcePropertyKey: String): BuilderType = { + def withPropertyKey( + propertyKey: String, + sourcePropertyKey: String + ): BuilderType = { preventOverwritingProperty(propertyKey) - updatePropertyMapping(propertyMapping.updated(propertyKey, sourcePropertyKey)) + updatePropertyMapping( + propertyMapping.updated(propertyKey, sourcePropertyKey) + ) } def withPropertyKeys(properties: String*): BuilderType = { if (properties.size != properties.toSet.size) - throw IllegalArgumentException("unique propertyKey definitions", - s"given key $properties overwrites existing mapping") + throw IllegalArgumentException( + "unique propertyKey definitions", + s"given key $properties overwrites existing mapping" + ) - withPropertyKeyMappings(properties.map(p => p -> p):_ *) + withPropertyKeyMappings(properties.map(p => p -> p): _*) } def withPropertyKeyMappings(tuples: (String, String)*): BuilderType = { @@ -71,7 +77,9 @@ trait SingleElementMappingBuilder { protected def preventOverwritingProperty(propertyKey: String): Unit = if (propertyMapping.contains(propertyKey)) - throw IllegalArgumentException("unique property key definitions", - s"given key $propertyKey overwrites existing mapping") + throw IllegalArgumentException( + "unique property key definitions", + s"given key $propertyKey overwrites existing mapping" + ) -} \ No newline at end of file +} diff --git a/okapi-api/src/main/scala/org/opencypher/okapi/api/package.scala b/okapi-api/src/main/scala/org/opencypher/okapi/api/package.scala index 2eb4822a59..2c3a7462f6 100644 --- a/okapi-api/src/main/scala/org/opencypher/okapi/api/package.scala +++ b/okapi-api/src/main/scala/org/opencypher/okapi/api/package.scala @@ -30,7 +30,9 @@ import scala.language.postfixOps package object types { - val CTNumber: CTUnion = CTUnion(Set[CypherType](CTFloat, CTInteger, CTBigDecimal)) + val CTNumber: CTUnion = CTUnion( + Set[CypherType](CTFloat, CTInteger, CTBigDecimal) + ) val CTBoolean: CTUnion = CTUnion(Set[CypherType](CTTrue, CTFalse)) @@ -38,9 +40,13 @@ package object types { val CTAny: CTUnion = CTUnion(Set[CypherType](CTAnyMaterial, CTNull)) - val CTTemporalInstant: CTUnion = CTUnion(Set[CypherType](CTLocalDateTime, CTDate)) + val CTTemporalInstant: CTUnion = CTUnion( + Set[CypherType](CTLocalDateTime, CTDate) + ) - val CTTemporal: CTUnion = CTUnion(Set[CypherType](CTLocalDateTime, CTDate, CTDuration)) + val CTTemporal: CTUnion = CTUnion( + Set[CypherType](CTLocalDateTime, CTDate, CTDuration) + ) val CTContainer: CTUnion = CTUnion(Set[CypherType](CTList, CTMap)) diff --git a/okapi-api/src/main/scala/org/opencypher/okapi/api/schema/LabelPropertyMap.scala b/okapi-api/src/main/scala/org/opencypher/okapi/api/schema/LabelPropertyMap.scala index e5011e7d0c..c70d9fa3fc 100644 --- a/okapi-api/src/main/scala/org/opencypher/okapi/api/schema/LabelPropertyMap.scala +++ b/okapi-api/src/main/scala/org/opencypher/okapi/api/schema/LabelPropertyMap.scala @@ -48,72 +48,93 @@ object LabelPropertyMap { val empty: LabelPropertyMap = Map.empty - /** - * Maps (a set of) labels to typed property keys. - */ + /** Maps (a set of) labels to typed property keys. */ implicit class RichLabelPropertyMap(val map: LabelPropertyMap) extends AnyVal { /** * Registers the given property keys to the specified labels. * - * @note This will override any previous binding for the label combination. - * @param labels set of labels - * @param properties property keys for the given set of labels - * @return updated LabelPropertyMap + * @note + * This will override any previous binding for the label combination. + * @param labels + * set of labels + * @param properties + * property keys for the given set of labels + * @return + * updated LabelPropertyMap */ - def register(labels: Set[String], properties: PropertyKeys): LabelPropertyMap = map.updated(labels, properties) + def register( + labels: Set[String], + properties: PropertyKeys + ): LabelPropertyMap = map.updated(labels, properties) /** * Returns the property keys that are associated with the given set of labels. * - * @param labels set of labels - * @return associated property keys + * @param labels + * set of labels + * @return + * associated property keys */ - def properties(labels: Set[String]): PropertyKeys = map.getOrElse(labels, PropertyKeys.empty) + def properties(labels: Set[String]): PropertyKeys = + map.getOrElse(labels, PropertyKeys.empty) /** - * Merges this LabelPropertyMap with the given map. Property keys for label sets that exist in both maps are being - * merged, diverging types are being joined. + * Merges this LabelPropertyMap with the given map. Property keys for label sets that exist in + * both maps are being merged, diverging types are being joined. * - * @param other LabelPropertyMap to merge - * @return merged LabelPropertyMap + * @param other + * LabelPropertyMap to merge + * @return + * merged LabelPropertyMap */ def ++(other: LabelPropertyMap): LabelPropertyMap = map |+| other /** * Returns the label property map with the given label combination `combo` removed. * - * @param combo label combination to remove - * @return updated label property map + * @param combo + * label combination to remove + * @return + * updated label property map */ def -(combo: Set[String]): LabelPropertyMap = map - combo /** - * Returns a LabelPropertyMap that contains all label combinations which include one or more of the specified labels. + * Returns a LabelPropertyMap that contains all label combinations which include one or more of + * the specified labels. * - * @param knownLabels labels for which the properties should be extracted - * @return extracted label property map + * @param knownLabels + * labels for which the properties should be extracted + * @return + * extracted label property map */ - def filterForLabels(knownLabels: Set[String]): LabelPropertyMap = map.filterKeys(_.exists(knownLabels.contains)) + def filterForLabels(knownLabels: Set[String]): LabelPropertyMap = + map.filterKeys(_.exists(knownLabels.contains)) /** * Returns all registered combinations of labels * - * @return all registered label combinations. + * @return + * all registered label combinations. */ def labelCombinations: Set[Set[String]] = map.keySet // utility signatures - def register(labels: String*)(keys: (String, CypherType)*): LabelPropertyMap = register(labels.toSet, keys.toMap) + def register(labels: String*)( + keys: (String, CypherType)* + ): LabelPropertyMap = register(labels.toSet, keys.toMap) - def register(label: String, properties: PropertyKeys): LabelPropertyMap = register(Set(label), properties) + def register(label: String, properties: PropertyKeys): LabelPropertyMap = + register(Set(label), properties) def properties(label: String*): PropertyKeys = properties(Set(label: _*)) - def filterForLabels(labels: String*): LabelPropertyMap = filterForLabels(labels.toSet) + def filterForLabels(labels: String*): LabelPropertyMap = filterForLabels( + labels.toSet + ) } - } diff --git a/okapi-api/src/main/scala/org/opencypher/okapi/api/schema/PropertyGraphSchema.scala b/okapi-api/src/main/scala/org/opencypher/okapi/api/schema/PropertyGraphSchema.scala index 89923b62c5..f36789afe2 100644 --- a/okapi-api/src/main/scala/org/opencypher/okapi/api/schema/PropertyGraphSchema.scala +++ b/okapi-api/src/main/scala/org/opencypher/okapi/api/schema/PropertyGraphSchema.scala @@ -39,8 +39,8 @@ object PropertyGraphSchema { val CURRENT_VERSION: Version = Version("1.0") /** - * Empty Schema. Start with this to construct a new Schema. - * Use the `with*` functions to add information. + * Empty Schema. Start with this to construct a new Schema. Use the `with*` functions to add + * information. */ val empty: PropertyGraphSchema = PropertyGraphSchemaImpl( labelPropertyMap = LabelPropertyMap.empty, @@ -50,64 +50,64 @@ object PropertyGraphSchema { def fromJson(jsonString: String): PropertyGraphSchema = try upickle.default.read[PropertyGraphSchema](jsonString) catch { - case throwable: Throwable if Option(throwable.getCause).exists(_.getClass.getPackageName.startsWith("org.opencypher")) => + case throwable: Throwable + if Option(throwable.getCause).exists( + _.getClass.getPackageName.startsWith("org.opencypher") + ) => throw throwable.getCause } } /** - * The schema of a graph describes what labels and relationship types exist in a graph, including possible combinations - * of the former. A `label combination` is an exact set of labels on a node. The term `known labels` refers to a subset - * of labels that a node might have. + * The schema of a graph describes what labels and relationship types exist in a graph, including + * possible combinations of the former. A `label combination` is an exact set of labels on a node. + * The term `known labels` refers to a subset of labels that a node might have. * - * It also keeps track of properties and their types that appear on different labels, label combinations and - * relationship types. + * It also keeps track of properties and their types that appear on different labels, label + * combinations and relationship types. */ trait PropertyGraphSchema { - /** - * All labels present in this graph. - */ + + /** All labels present in this graph. */ def labels: Set[String] /** - * Returns a mapping from a node label to a set of property keys that together form an element key for that - * label. An element key uniquely identifies a node with the given label. + * Returns a mapping from a node label to a set of property keys that together form an element + * key for that label. An element key uniquely identifies a node with the given label. */ @experimental def nodeKeys: Map[String, Set[String]] - /** - * All relationship types present in this graph. - */ + /** All relationship types present in this graph. */ def relationshipTypes: Set[String] /** - * Returns a mapping from a relationship type to a set of property keys that together form an element key for that - * relationship type. An element key uniquely identifies a relationship with the given type. + * Returns a mapping from a relationship type to a set of property keys that together form an + * element key for that relationship type. An element key uniquely identifies a relationship with + * the given type. */ @experimental def relationshipKeys: Map[String, Set[String]] - /** - * Property keys and their types for node labels. - */ + /** Property keys and their types for node labels. */ def labelPropertyMap: LabelPropertyMap - /** - * Property keys and their types for relationship types. - */ + /** Property keys and their types for relationship types. */ def relTypePropertyMap: RelTypePropertyMap /** - * Returns all schema patterns known to this schema. - * A schema pattern is a constraint over a (node)-[relationship]->(node) pattern which describes possible label and - * relationship type combinations for the source node, the relationship and the target node in the pattern. + * Returns all schema patterns known to this schema. A schema pattern is a constraint over a + * (node)-[relationship]->(node) pattern which describes possible label and relationship type + * combinations for the source node, the relationship and the target node in the pattern. * - * If no explicit schema patterns are defined, this function will return schema patterns for all possible combinations - * between the known label combinations and relationship types. Otherwise only explicit schema patterns will be returned. + * If no explicit schema patterns are defined, this function will return schema patterns for all + * possible combinations between the known label combinations and relationship types. Otherwise + * only explicit schema patterns will be returned. * - * @see {{{org.opencypher.okapi.api.schema.SchemaPattern}}} for more information - * @return schema pattern combinations encoded in this schema + * @see + * {{{org.opencypher.okapi.api.schema.SchemaPattern}}} for more information + * @return + * schema pattern combinations encoded in this schema */ @experimental def schemaPatterns: Set[SchemaPattern] @@ -115,110 +115,119 @@ trait PropertyGraphSchema { /** * Retrieves the user defined schema patterns * - * @return user defines schema patterns + * @return + * user defines schema patterns */ @experimental def explicitSchemaPatterns: Set[SchemaPattern] - /** - * Implied labels for each existing label. - */ + /** Implied labels for each existing label. */ def impliedLabels: ImpliedLabels - /** - * Groups of labels where each group contains possible label combinations. - */ + /** Groups of labels where each group contains possible label combinations. */ def labelCombinations: LabelCombinations - /** - * Given a set of labels that a node definitely has, returns all labels the node _must_ have. - */ + /** Given a set of labels that a node definitely has, returns all labels the node _must_ have. */ def impliedLabels(knownLabels: Set[String]): Set[String] - /** - * Given a set of labels that a node definitely has, returns all labels the node _must_ have. - */ + /** Given a set of labels that a node definitely has, returns all labels the node _must_ have. */ def impliedLabels(knownLabels: String*): Set[String] = { impliedLabels(knownLabels.toSet) } - /** - * Given a label combination, returns its property keys and their types. - */ + /** Given a label combination, returns its property keys and their types. */ def nodePropertyKeys(labelCombination: Set[String]): PropertyKeys /** - * Returns some property type for a property given the known labels of a node. - * Returns none if this property does not appear on nodes with the given label combination. - * Types of conflicting property keys are joined. + * Returns some property type for a property given the known labels of a node. Returns none if + * this property does not appear on nodes with the given label combination. Types of conflicting + * property keys are joined. * - * @param knownLabels known labels for which the property type is checked - * @param key property key - * @return Cypher type of the property on nodes with the given label combination - */ - def nodePropertyKeyType(knownLabels: Set[String], key: String): Option[CypherType] - - /** - * Returns all combinations of labels that exist on a node in the graph. - */ + * @param knownLabels + * known labels for which the property type is checked + * @param key + * property key + * @return + * Cypher type of the property on nodes with the given label combination + */ + def nodePropertyKeyType( + knownLabels: Set[String], + key: String + ): Option[CypherType] + + /** Returns all combinations of labels that exist on a node in the graph. */ def allCombinations: Set[Set[String]] /** - * Given a set of labels that a node definitely has, returns all combinations of labels that the node could possibly have. + * Given a set of labels that a node definitely has, returns all combinations of labels that the + * node could possibly have. */ def combinationsFor(knownLabels: Set[String]): Set[Set[String]] /** - * Returns property keys for the set of label combinations. - * Types of conflicting property keys are joined. + * Returns property keys for the set of label combinations. Types of conflicting property keys + * are joined. * - * @param labelCombinations label combinations to consider - * @return typed property keys, with joined or nullable types for conflicts + * @param labelCombinations + * label combinations to consider + * @return + * typed property keys, with joined or nullable types for conflicts */ - def nodePropertyKeysForCombinations(labelCombinations: Set[Set[String]]): PropertyKeys + def nodePropertyKeysForCombinations( + labelCombinations: Set[Set[String]] + ): PropertyKeys - - /** - * Returns the property schema for a given relationship type - */ + /** Returns the property schema for a given relationship type */ def relationshipPropertyKeys(relType: String): PropertyKeys /** - * Returns some property type for a property given the possible types of a relationship. - * Returns none if this property does not appear on relationships with one of the given types. - * Types of conflicting property keys are joined. + * Returns some property type for a property given the possible types of a relationship. Returns + * none if this property does not appear on relationships with one of the given types. Types of + * conflicting property keys are joined. * - * @param relTypes relationship types for which the property type is checked - * @param key property key - * @return Cypher type of the property on relationships with one of the given types - */ - def relationshipPropertyKeyType(relTypes: Set[String], key: String): Option[CypherType] - - /** - * Returns property keys for the set of known relationship types. - * Types of conflicting property keys are joined. - * The parameter `knownTypes` functions as a predicate, i.e. an empty Set means that every relationship type - * registered in the schema will be considered. + * @param relTypes + * relationship types for which the property type is checked + * @param key + * property key + * @return + * Cypher type of the property on relationships with one of the given types + */ + def relationshipPropertyKeyType( + relTypes: Set[String], + key: String + ): Option[CypherType] + + /** + * Returns property keys for the set of known relationship types. Types of conflicting property + * keys are joined. The parameter `knownTypes` functions as a predicate, i.e. an empty Set means + * that every relationship type registered in the schema will be considered. * - * @param knownTypes types that relationship can have - * @return typed property keys, with joined or nullable types for conflicts + * @param knownTypes + * types that relationship can have + * @return + * typed property keys, with joined or nullable types for conflicts */ def relationshipPropertyKeysForTypes(knownTypes: Set[String]): PropertyKeys /** - * This function returns all schema patterns that are applicable with regards to the specified known labels and - * relationship types. The given labels and relationship types are interpreted similar to how a Cypher MATCH clause - * would interpret them. That is, the label sets are interpreted as a conjunction of label predicates, i.e. labels - * the node must have. The relationship types are interpreted as a disjunction, i.e. the relationship must have - * one of the given types. + * This function returns all schema patterns that are applicable with regards to the specified + * known labels and relationship types. The given labels and relationship types are interpreted + * similar to how a Cypher MATCH clause would interpret them. That is, the label sets are + * interpreted as a conjunction of label predicates, i.e. labels the node must have. The + * relationship types are interpreted as a disjunction, i.e. the relationship must have one of + * the given types. * - * All the schema patterns that match the given descriptions will be retrieved. In particular, if all the inputs are - * empty sets, all the schema patterns will be retrieved. + * All the schema patterns that match the given descriptions will be retrieved. In particular, if + * all the inputs are empty sets, all the schema patterns will be retrieved. * - * @param knownSourceLabels labels required for the source node (AND semantics) - * @param knownRelTypes relationship types possible for the relationship (OR semantics) - * @param knownTargetLabels labels required for the target node (AND semantics) - * @return schema patterns that fulfill the predicates + * @param knownSourceLabels + * labels required for the source node (AND semantics) + * @param knownRelTypes + * relationship types possible for the relationship (OR semantics) + * @param knownTargetLabels + * labels required for the target node (AND semantics) + * @return + * schema patterns that fulfill the predicates */ @experimental def schemaPatternsFor( @@ -228,128 +237,168 @@ trait PropertyGraphSchema { ): Set[SchemaPattern] /** - * Adds information about a label combination and its associated properties to the schema. - * The arguments provided to this method are interpreted as describing a whole piece of information, - * meaning that for a specific instance of the label, the given properties were present in their exact - * given shape. For example, consider + * Adds information about a label combination and its associated properties to the schema. The + * arguments provided to this method are interpreted as describing a whole piece of information, + * meaning that for a specific instance of the label, the given properties were present in their + * exact given shape. For example, consider * * {{{ * val s = schema.withNodePropertyKeys("Foo")("p" -> CTString, "q" -> CTInteger) * val t = s.withNodePropertyKeys("Foo")("p" -> CTString) * }}} * - * The resulting schema (assigned to `t`) will indicate that the type of `q` is CTInteger.nullable, - * as the schema understands that it is possible to map `:Foo` to both sets of properties, and it - * calculates the join of the property types, respectively. + * The resulting schema (assigned to `t`) will indicate that the type of `q` is + * CTInteger.nullable, as the schema understands that it is possible to map `:Foo` to both sets + * of properties, and it calculates the join of the property types, respectively. * - * @param labelCombination the label combination to add to the schema - * @param propertyKeys the typed property keys to associate with the labels - * @return a copy of the Schema with the provided new data - */ - def withNodePropertyKeys(labelCombination: Set[String], propertyKeys: PropertyKeys = PropertyKeys.empty): PropertyGraphSchema + * @param labelCombination + * the label combination to add to the schema + * @param propertyKeys + * the typed property keys to associate with the labels + * @return + * a copy of the Schema with the provided new data + */ + def withNodePropertyKeys( + labelCombination: Set[String], + propertyKeys: PropertyKeys = PropertyKeys.empty + ): PropertyGraphSchema /** - * @see [[org.opencypher.okapi.api.schema.PropertyGraphSchema#withNodePropertyKeys(scala.collection.Seq, scala.collection.Seq)]] + * @see + * [[org.opencypher.okapi.api.schema.PropertyGraphSchema#withNodePropertyKeys(scala.collection.Seq, scala.collection.Seq)]] */ - def withNodePropertyKeys(labelCombination: String*)(propertyKeys: (String, CypherType)*): PropertyGraphSchema = + def withNodePropertyKeys(labelCombination: String*)( + propertyKeys: (String, CypherType)* + ): PropertyGraphSchema = withNodePropertyKeys(labelCombination.toSet, propertyKeys.toMap) /** - * Adds information about a relationship type and its associated properties to the schema. - * The arguments provided to this method are interpreted as describing a whole piece of information, - * meaning that for a specific instance of the relationship type, the given properties were present - * in their exact given shape. For example, consider + * Adds information about a relationship type and its associated properties to the schema. The + * arguments provided to this method are interpreted as describing a whole piece of information, + * meaning that for a specific instance of the relationship type, the given properties were + * present in their exact given shape. For example, consider * * {{{ * val s = schema.withRelationshipPropertyKeys("FOO")("p" -> CTString, "q" -> CTInteger) * val t = s.withRelationshipPropertyKeys("FOO")("p" -> CTString) * }}} * - * The resulting schema (assigned to `t`) will indicate that the type of `q` is CTInteger.nullable, - * as the schema understands that it is possible to map `:FOO` to both sets of properties, and it - * calculates the join of the property types, respectively. + * The resulting schema (assigned to `t`) will indicate that the type of `q` is + * CTInteger.nullable, as the schema understands that it is possible to map `:FOO` to both sets + * of properties, and it calculates the join of the property types, respectively. * - * @param typ the relationship type to add to the schema - * @param keys the properties (name and type) to associate with the relationship type - * @return a copy of the Schema with the provided new data - */ - def withRelationshipPropertyKeys(typ: String, keys: PropertyKeys): PropertyGraphSchema + * @param typ + * the relationship type to add to the schema + * @param keys + * the properties (name and type) to associate with the relationship type + * @return + * a copy of the Schema with the provided new data + */ + def withRelationshipPropertyKeys( + typ: String, + keys: PropertyKeys + ): PropertyGraphSchema /** - * @see [[org.opencypher.okapi.api.schema.PropertyGraphSchema#withRelationshipPropertyKeys(java.lang.String, scala.collection.Seq)]] + * @see + * [[org.opencypher.okapi.api.schema.PropertyGraphSchema#withRelationshipPropertyKeys(java.lang.String, scala.collection.Seq)]] */ def withRelationshipType(relType: String): PropertyGraphSchema = withRelationshipPropertyKeys(relType)() /** - * @see [[org.opencypher.okapi.api.schema.PropertyGraphSchema#withRelationshipPropertyKeys(java.lang.String, scala.collection.Seq)]] + * @see + * [[org.opencypher.okapi.api.schema.PropertyGraphSchema#withRelationshipPropertyKeys(java.lang.String, scala.collection.Seq)]] */ - def withRelationshipPropertyKeys(typ: String)(keys: (String, CypherType)*): PropertyGraphSchema = + def withRelationshipPropertyKeys(typ: String)( + keys: (String, CypherType)* + ): PropertyGraphSchema = withRelationshipPropertyKeys(typ, keys.toMap) /** - * Adds a node key for node label `label`. A node key uniquely identifies a node with the given label. + * Adds a node key for node label `label`. A node key uniquely identifies a node with the given + * label. * - * @note The label for which the key is added has to exist in this schema. + * @note + * The label for which the key is added has to exist in this schema. */ @experimental def withNodeKey(label: String, nodeKey: Set[String]): PropertyGraphSchema /** - * Adds a relationship key for relationship type `relationshipType`. A relationship key uniquely identifies a - * relationship with the given label. + * Adds a relationship key for relationship type `relationshipType`. A relationship key uniquely + * identifies a relationship with the given label. * - * @note The relationship type for which the key is added has to exist in this schema. + * @note + * The relationship type for which the key is added has to exist in this schema. */ @experimental - def withRelationshipKey(relationshipType: String, relationshipKey: Set[String]): PropertyGraphSchema + def withRelationshipKey( + relationshipType: String, + relationshipKey: Set[String] + ): PropertyGraphSchema /** * Adds the given schema patterns to the explicitly defined schema patterns. * - * @note If at least one explicit schema pattern has been defined, only explicit schema patterns will be considered - * part of this schema. - * @see {{{org.opencypher.okapi.api.schema.Schema#schemaPatterns}}} - * @param patterns the patterns to add - * @return schema with added explicit schema patterns + * @note + * If at least one explicit schema pattern has been defined, only explicit schema patterns will + * be considered part of this schema. + * @see + * {{{org.opencypher.okapi.api.schema.Schema#schemaPatterns}}} + * @param patterns + * the patterns to add + * @return + * schema with added explicit schema patterns */ @experimental def withSchemaPatterns(patterns: SchemaPattern*): PropertyGraphSchema /** - * Returns the union of the input schemas. - * Conflicting property key types are resolved into the joined type. + * Returns the union of the input schemas. Conflicting property key types are resolved into the + * joined type. */ def ++(other: PropertyGraphSchema): PropertyGraphSchema /** * Returns this schema with the properties for `combo` removed. * - * @param labelCombination label combination for which properties are removed - * @return updated schema + * @param labelCombination + * label combination for which properties are removed + * @return + * updated schema */ - private[opencypher] def dropPropertiesFor(labelCombination: Set[String]): PropertyGraphSchema + private[opencypher] def dropPropertiesFor( + labelCombination: Set[String] + ): PropertyGraphSchema /** - * Returns the sub-schema for a node scan under the given constraints. - * Labels are interpreted as constraints on the resulting Schema. - * If no labels are specified, this means the resulting node can have any valid label combination. + * Returns the sub-schema for a node scan under the given constraints. Labels are interpreted as + * constraints on the resulting Schema. If no labels are specified, this means the resulting node + * can have any valid label combination. * - * @param knownLabels Specifies the labels that the node is guaranteed to have - * @return sub-schema for `knownLabels` + * @param knownLabels + * Specifies the labels that the node is guaranteed to have + * @return + * sub-schema for `knownLabels` */ private[opencypher] def forNode(knownLabels: Set[String]): PropertyGraphSchema /** * Returns the sub-schema for `relType` * - * @param relType Specifies the type for which the schema is extracted - * @return sub-schema for `relType` + * @param relType + * Specifies the type for which the schema is extracted + * @return + * sub-schema for `relType` */ - private[opencypher] def forRelationship(relType: CTRelationship): PropertyGraphSchema + private[opencypher] def forRelationship( + relType: CTRelationship + ): PropertyGraphSchema /** - * Returns the updated schema, but overwrites any existing node property keys for the given labels. + * Returns the updated schema, but overwrites any existing node property keys for the given + * labels. */ private[opencypher] def withOverwrittenNodePropertyKeys( labelCombination: Set[String], @@ -357,9 +406,13 @@ trait PropertyGraphSchema { ): PropertyGraphSchema /** - * Returns the updated schema, but overwrites any existing relationship property keys for the given type. + * Returns the updated schema, but overwrites any existing relationship property keys for the + * given type. */ - private[opencypher] def withOverwrittenRelationshipPropertyKeys(relType: String, propertyKeys: PropertyKeys): PropertyGraphSchema + private[opencypher] def withOverwrittenRelationshipPropertyKeys( + relType: String, + propertyKeys: PropertyKeys + ): PropertyGraphSchema def toString: String diff --git a/okapi-api/src/main/scala/org/opencypher/okapi/api/schema/RelTypePropertyMap.scala b/okapi-api/src/main/scala/org/opencypher/okapi/api/schema/RelTypePropertyMap.scala index ae88d1ba8c..1f1a8c456c 100644 --- a/okapi-api/src/main/scala/org/opencypher/okapi/api/schema/RelTypePropertyMap.scala +++ b/okapi-api/src/main/scala/org/opencypher/okapi/api/schema/RelTypePropertyMap.scala @@ -45,28 +45,38 @@ object RelTypePropertyMap { map.updated(relType, oldKeys ++ keys) } - def properties(relKey: String): PropertyKeys = map.getOrElse(relKey, Map.empty) + def properties(relKey: String): PropertyKeys = + map.getOrElse(relKey, Map.empty) - def filterForRelTypes(relType: Set[String]): RelTypePropertyMap = map.filterKeys(relType.contains) + def filterForRelTypes(relType: Set[String]): RelTypePropertyMap = + map.filterKeys(relType.contains) def ++(other: RelTypePropertyMap): RelTypePropertyMap = map |+| other // utility signatures - def register(relType: String)(keys: (String, CypherType)*): RelTypePropertyMap = register(relType, keys.toMap) + def register(relType: String)( + keys: (String, CypherType)* + ): RelTypePropertyMap = register(relType, keys.toMap) - def register(relType: String, keys: => Seq[(String, CypherType)]): RelTypePropertyMap = register(relType, keys.toMap) + def register( + relType: String, + keys: => Seq[(String, CypherType)] + ): RelTypePropertyMap = register(relType, keys.toMap) /** * Sets all cypher types of properties that are not common across all labels to nullable. * - * @return updated property key map + * @return + * updated property key map */ def asNullable: RelTypePropertyMap = { val overlap = map.map(_._2.keySet).reduce(_ intersect _) map.map { pair => - pair._1 -> pair._2.map(p2 => p2._1 -> (if (overlap.contains(p2._1)) p2._2 else p2._2.nullable)) + pair._1 -> pair._2.map(p2 => + p2._1 -> (if (overlap.contains(p2._1)) p2._2 else p2._2.nullable) + ) } } diff --git a/okapi-api/src/main/scala/org/opencypher/okapi/api/schema/SchemaPattern.scala b/okapi-api/src/main/scala/org/opencypher/okapi/api/schema/SchemaPattern.scala index eddca22c9b..cf8d8128cf 100644 --- a/okapi-api/src/main/scala/org/opencypher/okapi/api/schema/SchemaPattern.scala +++ b/okapi-api/src/main/scala/org/opencypher/okapi/api/schema/SchemaPattern.scala @@ -26,7 +26,7 @@ */ package org.opencypher.okapi.api.schema -object SchemaPattern{ +object SchemaPattern { def apply( sourceLabel: String, relType: String, @@ -35,26 +35,36 @@ object SchemaPattern{ } /** - * Describes a (node)-[relationship]->(node) triple in a graph as part of the graph's schema. - * A pattern only applies to nodes with the exact label combination defined by `source/targetLabels` + * Describes a (node)-[relationship]->(node) triple in a graph as part of the graph's schema. A + * pattern only applies to nodes with the exact label combination defined by `source/targetLabels` * and relationships with the specified relationship type. * - * @example Given a graph that only contains the following patterns - * {{{(:A)-[r1:REL]->(:C),}}} - * {{{(:B)-[r2:REL]->(:C),}}} - * {{{(:A:B)-[r3:REL]->(:C)}}} + * @example + * Given a graph that only contains the following patterns {{{(:A)-[r1:REL]->(:C),}}} + * {{{(:B)-[r2:REL]->(:C),}}} {{{(:A:B)-[r3:REL]->(:C)}}} * - * then the schema pattern `SchemaPattern(Set("A"), "REL", Set("C"))` would only apply to `r1` - * and the schema pattern `SchemaPattern(Set("A", "B"), "REL", Set("C"))` would only apply to `r3` + * then the schema pattern `SchemaPattern(Set("A"), "REL", Set("C"))` would only apply to `r1` and + * the schema pattern `SchemaPattern(Set("A", "B"), "REL", Set("C"))` would only apply to `r3` * - * @param sourceLabelCombination label combination for source nodes - * @param relType relationship type - * @param targetLabelCombination label combination for target nodes + * @param sourceLabelCombination + * label combination for source nodes + * @param relType + * relationship type + * @param targetLabelCombination + * label combination for target nodes */ -case class SchemaPattern(sourceLabelCombination: Set[String], relType: String, targetLabelCombination: Set[String]) { +case class SchemaPattern( + sourceLabelCombination: Set[String], + relType: String, + targetLabelCombination: Set[String] +) { override def toString: String = { - val sourceLabelString = if(sourceLabelCombination.isEmpty) "" else sourceLabelCombination.mkString(":", ":", "") - val targetLabelString = if(targetLabelCombination.isEmpty) "" else targetLabelCombination.mkString(":", ":", "") + val sourceLabelString = + if (sourceLabelCombination.isEmpty) "" + else sourceLabelCombination.mkString(":", ":", "") + val targetLabelString = + if (targetLabelCombination.isEmpty) "" + else targetLabelCombination.mkString(":", ":", "") s"($sourceLabelString)-[:$relType]->($targetLabelString)" } } diff --git a/okapi-api/src/main/scala/org/opencypher/okapi/api/table/CypherRecords.scala b/okapi-api/src/main/scala/org/opencypher/okapi/api/table/CypherRecords.scala index 9a837a8cf2..2ed14b08fc 100644 --- a/okapi-api/src/main/scala/org/opencypher/okapi/api/table/CypherRecords.scala +++ b/okapi-api/src/main/scala/org/opencypher/okapi/api/table/CypherRecords.scala @@ -29,25 +29,29 @@ package org.opencypher.okapi.api.table import org.opencypher.okapi.api.value.CypherValue.CypherMap /** - * Represents a table of records containing Cypher values. - * Each column (or slot) in this table represents an evaluated Cypher expression. + * Represents a table of records containing Cypher values. Each column (or slot) in this table + * represents an evaluated Cypher expression. */ trait CypherRecords extends CypherTable with CypherPrintable { /** * Consume these records as an iterator. * - * WARNING: This operation may be very expensive as it may have to materialise the full result set. + * WARNING: This operation may be very expensive as it may have to materialise the full result + * set. * - * @note This method may be considerably slower than [[org.opencypher.okapi.api.table.CypherRecords#collect]]. - * Use this method only if collect could outgrow the available driver memory. + * @note + * This method may be considerably slower than + * [[org.opencypher.okapi.api.table.CypherRecords#collect]]. Use this method only if collect + * could outgrow the available driver memory. */ def iterator: Iterator[CypherMap] /** * Consume these records and collect them into an array. * - * WARNING: This operation may be very expensive as it may have to materialise the full result set. + * WARNING: This operation may be very expensive as it may have to materialise the full result + * set. */ def collect: Array[CypherMap] diff --git a/okapi-api/src/main/scala/org/opencypher/okapi/api/table/CypherTable.scala b/okapi-api/src/main/scala/org/opencypher/okapi/api/table/CypherTable.scala index eb16e8a7a7..3781546a3a 100644 --- a/okapi-api/src/main/scala/org/opencypher/okapi/api/table/CypherTable.scala +++ b/okapi-api/src/main/scala/org/opencypher/okapi/api/table/CypherTable.scala @@ -31,12 +31,12 @@ import org.opencypher.okapi.api.value.CypherValue.CypherValue import org.opencypher.okapi.impl.exception.IllegalArgumentException /** - * Represents a table in which each row contains one [[org.opencypher.okapi.api.value.CypherValue]] per column and the - * values in each column have the same Cypher type. + * Represents a table in which each row contains one [[org.opencypher.okapi.api.value.CypherValue]] + * per column and the values in each column have the same Cypher type. * - * This interface is used to access simple Cypher values from a table. When it is implemented with an element mapping - * it can also be used to assemble complex Cypher values such as CypherNode/CypherRelationship that are stored over - * multiple columns in a low-level Cypher table. + * This interface is used to access simple Cypher values from a table. When it is implemented with + * an element mapping it can also be used to assemble complex Cypher values such as + * CypherNode/CypherRelationship that are stored over multiple columns in a low-level Cypher table. */ trait CypherTable { @@ -47,24 +47,16 @@ trait CypherTable { */ def physicalColumns: Seq[String] - /** - * Logical column names in this table as requested by a RETURN statement. - */ + /** Logical column names in this table as requested by a RETURN statement. */ def logicalColumns: Option[Seq[String]] = None - /** - * CypherType of columns stored in this table. - */ + /** CypherType of columns stored in this table. */ def columnType: Map[String, CypherType] - /** - * Iterator over the rows in this table. - */ + /** Iterator over the rows in this table. */ def rows: Iterator[String => CypherValue] - /** - * Number of rows in this Table. - */ + /** Number of rows in this Table. */ def size: Long } @@ -76,18 +68,29 @@ object CypherTable { /** * Checks if the data type of the given column is compatible with the expected type. * - * @param columnKey column to be checked - * @param expectedType excepted data type + * @param columnKey + * column to be checked + * @param expectedType + * excepted data type */ - def verifyColumnType(columnKey: String, expectedType: CypherType, keyDescription: String): Unit = { - val columnType = table.columnType.getOrElse(columnKey, throw IllegalArgumentException( - s"table with column key $columnKey", - s"table with columns ${table.physicalColumns.mkString(", ")}")) + def verifyColumnType( + columnKey: String, + expectedType: CypherType, + keyDescription: String + ): Unit = { + val columnType = table.columnType.getOrElse( + columnKey, + throw IllegalArgumentException( + s"table with column key $columnKey", + s"table with columns ${table.physicalColumns.mkString(", ")}" + ) + ) if (!columnType.material.subTypeOf(expectedType.material)) { throw IllegalArgumentException( s"$keyDescription column `$columnKey` of type $expectedType", - s"incompatible column type $columnType") + s"incompatible column type $columnType" + ) } } } diff --git a/okapi-api/src/main/scala/org/opencypher/okapi/api/types/CypherType.scala b/okapi-api/src/main/scala/org/opencypher/okapi/api/types/CypherType.scala index 72a47911e2..b4cfb5bab0 100644 --- a/okapi-api/src/main/scala/org/opencypher/okapi/api/types/CypherType.scala +++ b/okapi-api/src/main/scala/org/opencypher/okapi/api/types/CypherType.scala @@ -54,7 +54,8 @@ trait CypherType { else if (other.subTypeOf(this)) other else { this -> other match { - case (l: CTNode, r: CTNode) if l.graph == r.graph => CTNode(l.labels ++ r.labels, l.graph) + case (l: CTNode, r: CTNode) if l.graph == r.graph => + CTNode(l.labels ++ r.labels, l.graph) case (l: CTNode, r: CTNode) => CTNode(l.labels ++ r.labels) case (l: CTRelationship, r: CTRelationship) => val types = l.types.intersect(r.types) @@ -62,21 +63,22 @@ trait CypherType { else if (l.graph == r.graph) CTRelationship(types, l.graph) else CTRelationship(types) case (CTList(l), CTList(r)) => CTList(l & r) - case (CTUnion(ls), CTUnion(rs)) => CTUnion({ - for { - l <- ls - r <- rs - } yield l & r - }.toSeq: _*) + case (CTUnion(ls), CTUnion(rs)) => + CTUnion({ + for { + l <- ls + r <- rs + } yield l & r + }.toSeq: _*) case (CTUnion(ls), r) => CTUnion(ls.map(_ & r).toSeq: _*) case (l, CTUnion(rs)) => CTUnion(rs.map(_ & l).toSeq: _*) case (CTMap(pl), CTMap(pr)) => val intersectedProps = (pl.keys ++ pr.keys).map { k => val ct = pl.get(k) -> pr.get(k) match { case (Some(tl), Some(tr)) => tl | tr - case (Some(tl), None) => tl.nullable - case (None, Some(tr)) => tr.nullable - case (None, None) => CTVoid + case (Some(tl), None) => tl.nullable + case (None, Some(tr)) => tr.nullable + case (None, None) => CTVoid } k -> ct }.toMap @@ -100,15 +102,16 @@ trait CypherType { else if (other.subTypeOf(this)) this else { this -> other match { - case (l: CTRelationship, r: CTRelationship) if l.graph == r.graph => CTRelationship(l.types ++ r.types, l.graph) + case (l: CTRelationship, r: CTRelationship) if l.graph == r.graph => + CTRelationship(l.types ++ r.types, l.graph) case (CTBigDecimal(lp, ls), CTBigDecimal(rp, rs)) => val maxScale = Math.max(ls, rs) val maxDiff = Math.max(lp - ls, rp - rs) CTBigDecimal(maxDiff + maxScale, maxScale) case (CTUnion(ls), CTUnion(rs)) => CTUnion(ls ++ rs) - case (CTUnion(ls), r) => CTUnion(r +: ls.toSeq: _*) - case (l, CTUnion(rs)) => CTUnion(l +: rs.toSeq: _*) - case (l, r) => CTUnion(l, r) + case (CTUnion(ls), r) => CTUnion(r +: ls.toSeq: _*) + case (l, CTUnion(rs)) => CTUnion(l +: rs.toSeq: _*) + case (l, r) => CTUnion(l, r) } } } @@ -117,24 +120,29 @@ trait CypherType { def subTypeOf(other: CypherType): Boolean = { this -> other match { - case (CTVoid, _) => true - case (l, r) if l == r => true - case (_, CTAny) => true + case (CTVoid, _) => true + case (l, r) if l == r => true + case (_, CTAny) => true case (_: CTBigDecimal, CTBigDecimal) => true case (CTBigDecimal, _: CTBigDecimal) => false - case (CTBigDecimal(lp, ls), CTBigDecimal(rp, rs)) => (lp <= rp) && (ls <= rs) && (lp - ls <= rp - rs) + case (CTBigDecimal(lp, ls), CTBigDecimal(rp, rs)) => + (lp <= rp) && (ls <= rs) && (lp - ls <= rp - rs) case (l, CTAnyMaterial) if !l.isNullable => true case (_: CTRelationship, CTRelationship) => true - case (_: CTMap, CTMap) => true - case (_: CTNode, CTNode) => true + case (_: CTMap, CTMap) => true + case (_: CTNode, CTNode) => true case (l: CTNode, r: CTNode) - if l != CTNode && l.graph == r.graph && r.labels.subsetOf(l.labels) => true + if l != CTNode && l.graph == r.graph && r.labels.subsetOf(l.labels) => + true case (l: CTRelationship, r: CTRelationship) - if l != CTRelationship && l.graph == r.graph && l.types.subsetOf(r.types) => true + if l != CTRelationship && l.graph == r.graph && l.types.subsetOf( + r.types + ) => + true case (CTUnion(las), r: CTUnion) => las.forall(_.subTypeOf(r)) - case (l, CTUnion(ras)) => ras.exists(l.subTypeOf) - case (CTList(l), CTList(r)) => l.subTypeOf(r) - case (l@CTMap(lps), CTMap(rps)) => + case (l, CTUnion(ras)) => ras.exists(l.subTypeOf) + case (CTList(l), CTList(r)) => l.subTypeOf(r) + case (l @ CTMap(lps), CTMap(rps)) => if (l == CTMap) false else { (lps.keySet ++ rps.keySet).forall { key => @@ -166,11 +174,14 @@ object CypherType { /** * Parses the name of CypherType into the actual CypherType object. * - * @param name string representation of the CypherType + * @param name + * string representation of the CypherType * @return - * @see {{{org.opencypher.okapi.api.types.CypherType#name}}} + * @see + * {{{org.opencypher.okapi.api.types.CypherType#name}}} */ - def fromName(name: String): Option[CypherType] = CypherTypeParser.parseCypherType(name) + def fromName(name: String): Option[CypherType] = + CypherTypeParser.parseCypherType(name) } @@ -187,13 +198,16 @@ object CTMap extends CTMap(Map.empty) { override def canEqual(that: Any): Boolean = that.isInstanceOf[CTMap.type] - def apply(propertyTypes: (String, CypherType)*): CTMap = CTMap(propertyTypes.toMap) + def apply(propertyTypes: (String, CypherType)*): CTMap = CTMap( + propertyTypes.toMap + ) } case class CTMap(properties: Map[String, CypherType] = Map.empty) extends CypherType { - override def containsNullable: Boolean = properties.values.exists(_.containsNullable) + override def containsNullable: Boolean = + properties.values.exists(_.containsNullable) override def name: String = { s"MAP(${properties.map { case (n, t) => s"$n: ${t.name}" }.mkString(", ")})" @@ -223,7 +237,8 @@ case class CTNode( labels: Set[String] = Set.empty, override val graph: Option[QualifiedGraphName] = None ) extends CypherType { - override def withGraph(qgn: QualifiedGraphName): CTNode = copy(graph = Some(qgn)) + override def withGraph(qgn: QualifiedGraphName): CTNode = + copy(graph = Some(qgn)) override def withoutGraph: CTNode = CTNode(labels) override def name: String = @@ -243,7 +258,8 @@ case class CTRelationship( types: Set[String] = Set.empty, override val graph: Option[QualifiedGraphName] = None ) extends CypherType { - override def withGraph(qgn: QualifiedGraphName): CTRelationship = copy(graph = Some(qgn)) + override def withGraph(qgn: QualifiedGraphName): CTRelationship = + copy(graph = Some(qgn)) override def withoutGraph: CTRelationship = CTRelationship(types) override def name: String = { @@ -282,7 +298,10 @@ case object CTDuration extends CypherType case object CTVoid extends CypherType case class CTUnion(alternatives: Set[CypherType]) extends CypherType { - require(!alternatives.exists(_.isInstanceOf[CTUnion]), "Unions need to be flattened") + require( + !alternatives.exists(_.isInstanceOf[CTUnion]), + "Unions need to be flattened" + ) override def isNullable: Boolean = alternatives.contains(CTNull) @@ -297,24 +316,30 @@ case class CTUnion(alternatives: Set[CypherType]) extends CypherType { else s"UNION(${alternatives.mkString(", ")})" } - override def graph: Option[QualifiedGraphName] = alternatives.flatMap(_.graph).headOption + override def graph: Option[QualifiedGraphName] = + alternatives.flatMap(_.graph).headOption } object CTUnion { def apply(ts: CypherType*): CypherType = { - val flattened = ts.flatMap { - case u: CTUnion => u.alternatives - case p => Set(p) - }.distinct.toList + val flattened = ts + .flatMap { + case u: CTUnion => u.alternatives + case p => Set(p) + } + .distinct + .toList // Filter alternatives that are a subtype of another alternative - val filtered = flattened.filter(t => !flattened.exists(o => o != t && t.subTypeOf(o))) + val filtered = + flattened.filter(t => !flattened.exists(o => o != t && t.subTypeOf(o))) filtered match { - case Nil => CTVoid + case Nil => CTVoid case h :: Nil => h - case many if many.contains(CTAnyMaterial) => if (many.contains(CTNull)) CTAny else CTAnyMaterial + case many if many.contains(CTAnyMaterial) => + if (many.contains(CTNull)) CTAny else CTAnyMaterial case many => CTUnion(many.toSet) } } diff --git a/okapi-api/src/main/scala/org/opencypher/okapi/api/types/CypherTypeHelp.scala b/okapi-api/src/main/scala/org/opencypher/okapi/api/types/CypherTypeHelp.scala index 798d7c1df3..1616370f3e 100644 --- a/okapi-api/src/main/scala/org/opencypher/okapi/api/types/CypherTypeHelp.scala +++ b/okapi-api/src/main/scala/org/opencypher/okapi/api/types/CypherTypeHelp.scala @@ -31,7 +31,8 @@ import org.opencypher.okapi.api.types.CypherType.fromName import upickle.default.{readwriter, _} object CypherTypeHelp { - implicit val typeRw: ReadWriter[CypherType] = readwriter[String].bimap[CypherType](_.name, s => fromName(s).get) + implicit val typeRw: ReadWriter[CypherType] = + readwriter[String].bimap[CypherType](_.name, s => fromName(s).get) implicit val joinMonoid: Monoid[CypherType] = new Monoid[CypherType] { override def empty: CypherType = CTVoid diff --git a/okapi-api/src/main/scala/org/opencypher/okapi/api/util/ZeppelinSupport.scala b/okapi-api/src/main/scala/org/opencypher/okapi/api/util/ZeppelinSupport.scala index 94980865aa..9ba1ba4c03 100644 --- a/okapi-api/src/main/scala/org/opencypher/okapi/api/util/ZeppelinSupport.scala +++ b/okapi-api/src/main/scala/org/opencypher/okapi/api/util/ZeppelinSupport.scala @@ -36,24 +36,29 @@ import org.opencypher.okapi.api.value.CypherValue.{Node, Relationship} import ujson._ import scala.util.Random -/** - * Provides helper methods for Apache Zeppelin integration - */ + +/** Provides helper methods for Apache Zeppelin integration */ object ZeppelinSupport { implicit class ResultVisualizer(result: CypherResult) { /** - * Visualizes the result in Zeppelin. - * If the result contains a graph, it is shown as a network (see [[ZeppelinSupport.ZeppelinGraph#printGraph]]). - * If the result contains a tabular result, they are: - * - visualized as a graph if the result only contains nodes and relationships (see [[ZeppelinSupport.ZeppelinRecords#printGraph]]) - * - TODO: visualized as a table if the result contains non element values (see [[ZeppelinSupport.ZeppelinRecords#printTable]]) + * Visualizes the result in Zeppelin. If the result contains a graph, it is shown as a network + * (see [[ZeppelinSupport.ZeppelinGraph#printGraph]]). If the result contains a tabular result, + * they are: + * - visualized as a graph if the result only contains nodes and relationships (see + * [[ZeppelinSupport.ZeppelinRecords#printGraph]]) + * - TODO: visualized as a table if the result contains non element values (see + * [[ZeppelinSupport.ZeppelinRecords#printTable]]) */ - def visualize()(implicit formatValue: Any => String = CypherValue.Format.defaultValueFormatter): Unit = { + def visualize()(implicit + formatValue: Any => String = CypherValue.Format.defaultValueFormatter + ): Unit = { result.getGraph match { case Some(g) => g.printGraph() - case None => result.records.printTable() // TODO find a way to identify results that could be printed as a graph + case None => + result.records + .printTable() // TODO find a way to identify results that could be printed as a graph } } } @@ -76,12 +81,13 @@ object ZeppelinSupport { * Bob\t42 * }}} */ - def printTable()(implicit formatValue: Any => String = CypherValue.Format.defaultValueFormatter): Unit = { + def printTable()(implicit + formatValue: Any => String = CypherValue.Format.defaultValueFormatter + ): Unit = { print(s""" |%table |$toZeppelinTable - |""".stripMargin - ) + |""".stripMargin) } /** @@ -103,9 +109,10 @@ object ZeppelinSupport { * } * }}} */ - def printGraph()(implicit formatValue: Any => String = CypherValue.Format.defaultValueFormatter): Unit = { - print( - s""" + def printGraph()(implicit + formatValue: Any => String = CypherValue.Format.defaultValueFormatter + ): Unit = { + print(s""" |%network |$toZeppelinGraph """.stripMargin) @@ -120,19 +127,21 @@ object ZeppelinSupport { * Bob\t42 * }}} */ - def toZeppelinTable()(implicit formatValue: Any => String = CypherValue.Format.defaultValueFormatter): String = { + def toZeppelinTable()(implicit + formatValue: Any => String = CypherValue.Format.defaultValueFormatter + ): String = { val columns = r.logicalColumns.get s"""${columns.mkString("\t")} - |${ - r.iterator.map { row => - columns.map(row(_).toCypherString).mkString("\t") - }.mkString("\n") - }""".stripMargin + |${r.iterator + .map { row => + columns.map(row(_).toCypherString).mkString("\t") + } + .mkString("\n")}""".stripMargin } /** - * Returns a Zeppelin compatible Json representation of the result. - * Only colums that are nodes or edges are represented. + * Returns a Zeppelin compatible Json representation of the result. Only colums that are nodes + * or edges are represented. * * {{{ * { @@ -144,20 +153,27 @@ object ZeppelinSupport { * } * }}} */ - def toZeppelinGraph()(implicit formatValue: Any => String = CypherValue.Format.defaultValueFormatter): String = { + def toZeppelinGraph()(implicit + formatValue: Any => String = CypherValue.Format.defaultValueFormatter + ): String = { val data = r.collect - val nodeCols = data.headOption.map { row => - row.value.collect { - case (key, _: Node[_]) => key + val nodeCols = data.headOption + .map { row => + row.value.collect { case (key, _: Node[_]) => + key + } } - }.getOrElse(Seq.empty) + .getOrElse(Seq.empty) - val relCols = data.headOption.map { row => - row.value.collect { - case (key, _: Relationship[_]) => key + val relCols = data.headOption + .map { row => + row.value.collect { case (key, _: Relationship[_]) => + key + } } - }.getOrElse(Seq.empty).toSet + .getOrElse(Seq.empty) + .toSet val nodes = data .flatMap { row => nodeCols.map(row(_).cast[Node[_]]) } @@ -174,9 +190,14 @@ object ZeppelinSupport { val labels = nodes.flatMap(_.labels).toSet val types = rels.map(_.relType).toSet - ZeppelinGraph.toZeppelinJson( - nodes.toIterator, rels.toIterator, labels, types - ).render(2) + ZeppelinGraph + .toZeppelinJson( + nodes.toIterator, + rels.toIterator, + labels, + types + ) + .render(2) } } @@ -247,8 +268,10 @@ object ZeppelinSupport { } private object ZeppelinGraph { + /** - * Returns a Zeppelin compatible Json representation of a graph defined by a list of nodes and edges: + * Returns a Zeppelin compatible Json representation of a graph defined by a list of nodes and + * edges: * * {{{ * { @@ -342,8 +365,7 @@ object ZeppelinSupport { * }}} */ def printGraph()(implicit formatValue: Any => String): Unit = { - print( - s""" + print(s""" |%network |${toZeppelinJson.render(2)} """.stripMargin) @@ -364,10 +386,14 @@ object ZeppelinSupport { */ def toZeppelinJson()(implicit formatValue: Any => String): Value = { val nodes = g.nodes("n").iterator.map(m => m("n").cast[Node[_]]) - val rels = g.relationships("r").iterator.map(m => m("r").cast[Relationship[_]]) + val rels = + g.relationships("r").iterator.map(m => m("r").cast[Relationship[_]]) ZeppelinGraph.toZeppelinJson( - nodes, rels, g.schema.labels, g.schema.relationshipTypes + nodes, + rels, + g.schema.labels, + g.schema.relationshipTypes ) } } diff --git a/okapi-api/src/main/scala/org/opencypher/okapi/api/value/CypherValue.scala b/okapi-api/src/main/scala/org/opencypher/okapi/api/value/CypherValue.scala index b1ef10981c..a7915e8f72 100644 --- a/okapi-api/src/main/scala/org/opencypher/okapi/api/value/CypherValue.scala +++ b/okapi-api/src/main/scala/org/opencypher/okapi/api/value/CypherValue.scala @@ -45,55 +45,63 @@ import scala.util.hashing.MurmurHash3 object CypherValue { /** - * Converts a Scala/Java value to a compatible Cypher value, fails if the conversion is not supported. + * Converts a Scala/Java value to a compatible Cypher value, fails if the conversion is not + * supported. * - * @param v value to convert - * @param customConverter additional conversion rules - * @return compatible CypherValue + * @param v + * value to convert + * @param customConverter + * additional conversion rules + * @return + * compatible CypherValue */ - def apply(v: Any)(implicit customConverter: CypherValueConverter = NoopCypherValueConverter): CypherValue = { + def apply(v: Any)(implicit + customConverter: CypherValueConverter = NoopCypherValueConverter + ): CypherValue = { def seqToCypherList(s: Seq[_]): CypherList = s.map(CypherValue(_)).toList - customConverter.convert(v).getOrElse( - v match { - case cv: CypherValue => cv - case null => CypherNull - case jb: java.lang.Byte => jb.toLong - case js: java.lang.Short => js.toLong - case ji: java.lang.Integer => ji.toLong - case jl: java.lang.Long => jl.toLong - case jf: java.lang.Float => jf.toDouble - case jd: java.lang.Double => jd.toDouble - case js: java.lang.String => js.toString - case jb: java.lang.Boolean => jb.booleanValue - case jl: java.util.List[_] => seqToCypherList(jl.toArray) - case dt: java.sql.Date => dt.toLocalDate - case ts: java.sql.Timestamp => ts.toLocalDateTime - case ld: java.time.LocalDate => ld + customConverter + .convert(v) + .getOrElse(v match { + case cv: CypherValue => cv + case null => CypherNull + case jb: java.lang.Byte => jb.toLong + case js: java.lang.Short => js.toLong + case ji: java.lang.Integer => ji.toLong + case jl: java.lang.Long => jl.toLong + case jf: java.lang.Float => jf.toDouble + case jd: java.lang.Double => jd.toDouble + case js: java.lang.String => js.toString + case jb: java.lang.Boolean => jb.booleanValue + case jl: java.util.List[_] => seqToCypherList(jl.toArray) + case dt: java.sql.Date => dt.toLocalDate + case ts: java.sql.Timestamp => ts.toLocalDateTime + case ld: java.time.LocalDate => ld case ldt: java.time.LocalDateTime => ldt - case du: Duration => du - case a: Array[_] => seqToCypherList(a) - case s: Seq[_] => seqToCypherList(s) - case m: Map[_, _] => m.map { case (k, cv) => k.toString -> CypherValue(cv) } - case b: Byte => b.toLong - case s: Short => s.toLong - case i: Int => i.toLong - case l: Long => l - case f: Float => f.toDouble - case d: Double => d - case b: Boolean => b - case b: BigDecimal => b + case du: Duration => du + case a: Array[_] => seqToCypherList(a) + case s: Seq[_] => seqToCypherList(s) + case m: Map[_, _] => + m.map { case (k, cv) => k.toString -> CypherValue(cv) } + case b: Byte => b.toLong + case s: Short => s.toLong + case i: Int => i.toLong + case l: Long => l + case f: Float => f.toDouble + case d: Double => d + case b: Boolean => b + case b: BigDecimal => b case b: java.math.BigDecimal => BigDecimal(b) case invalid => throw IllegalArgumentException( - "a value that can be converted to a Cypher value", s"$invalid of type ${invalid.getClass.getName}") + "a value that can be converted to a Cypher value", + s"$invalid of type ${invalid.getClass.getName}" + ) }) } - /** - * Trait to inject additional CypherValue conversion rules - */ + /** Trait to inject additional CypherValue conversion rules */ trait CypherValueConverter { def convert(v: Any): Option[CypherValue] } @@ -104,8 +112,10 @@ object CypherValue { /** * Converts a Scala/Java value to a compatible Cypher value. * - * @param v value to convert - * @return Some compatible CypherValue or None + * @param v + * value to convert + * @return + * Some compatible CypherValue or None */ def get(v: Any): Option[CypherValue] = { Try(apply(v)).toOption @@ -114,81 +124,79 @@ object CypherValue { /** * Attempts to extract the wrapped value from a CypherValue. * - * @param cv CypherValue to extract from - * @return none or some extracted value. + * @param cv + * CypherValue to extract from + * @return + * none or some extracted value. */ def unapply(cv: CypherValue): Option[Any] = { Option(cv).flatMap(v => Option(v.value)) } object Format { - /** - * Formats a given value to its String representation. - */ + + /** Formats a given value to its String representation. */ implicit def defaultValueFormatter(value: Any): String = value match { - case s: Seq[_] => s.map(defaultValueFormatter).mkString + case s: Seq[_] => s.map(defaultValueFormatter).mkString case a: Array[_] => a.map(defaultValueFormatter).mkString - case b: Byte => "%02X".format(b) - case other => Objects.toString(other) + case b: Byte => "%02X".format(b) + case other => Objects.toString(other) } } - /** - * CypherValue is a wrapper for Scala/Java classes that represent valid Cypher values. - */ + /** CypherValue is a wrapper for Scala/Java classes that represent valid Cypher values. */ sealed trait CypherValue extends Any { + /** - * @return wrapped value + * @return + * wrapped value */ def value: Any def cypherType: CypherType /** - * @return null-safe version of [[value]] + * @return + * null-safe version of [[value]] */ def getValue: Option[Any] /** - * @return unwraps the Cypher value into Scala/Java structures. Unlike [[value]] this is done recursively for the - * Cypher values stored inside of maps and lists. + * @return + * unwraps the Cypher value into Scala/Java structures. Unlike [[value]] this is done + * recursively for the Cypher values stored inside of maps and lists. */ def unwrap: Any /** - * @return true iff the stored value is null. + * @return + * true iff the stored value is null. */ def isNull: Boolean = Objects.isNull(value) - /** - * Safe version of [[cast]] - */ + /** Safe version of [[cast]] */ def as[V: ClassTag]: Option[V] = { this match { case cv: V => Some(cv) case _ => value match { case v: V => Some(v) - case _ => None + case _ => None } } } - /** - * Attempts to cast the Cypher value to `V`, fails when this is not supported. - */ - def cast[V: ClassTag]: V = as[V].getOrElse(throw UnsupportedOperationException( - s"Cannot cast $value of type ${value.getClass.getSimpleName} to ${classTag[V].runtimeClass.getSimpleName}")) - + /** Attempts to cast the Cypher value to `V`, fails when this is not supported. */ + def cast[V: ClassTag]: V = as[V].getOrElse( + throw UnsupportedOperationException( + s"Cannot cast $value of type ${value.getClass.getSimpleName} to ${classTag[V].runtimeClass.getSimpleName}" + ) + ) - /** - * String of the Scala representation of this value. - */ + /** String of the Scala representation of this value. */ override def toString: String = Objects.toString(unwrap) - /** - * Hash code of the Scala representation. - */ + /** Hash code of the Scala representation. */ override def hashCode: Int = Objects.hashCode(unwrap) /** @@ -199,34 +207,34 @@ object CypherValue { override def equals(other: Any): Boolean = { other match { case cv: CypherValue => Objects.equals(unwrap, cv.unwrap) - case _ => false + case _ => false } } /** - * A Cypher string representation. For more information about the exact format of these, please refer to + * A Cypher string representation. For more information about the exact format of these, please + * refer to * [[https://github.com/opencypher/openCypher/tree/master/tck#format-of-the-expected-results the openCypher TCK]]. */ def toCypherString()(implicit formatValue: Any => String): String = { this match { case CypherString(s) => s"'${escape(s)}'" - case CypherList(l) => l.map(_.toCypherString).mkString("[", ", ", "]") + case CypherList(l) => l.map(_.toCypherString).mkString("[", ", ", "]") case CypherMap(m) => m.toSeq .sortBy(_._1) .map { case (k, v) => s"`${escape(k)}`: ${v.toCypherString}" } .mkString("{", ", ", "}") case Relationship(_, _, _, relType, props) => - s"[:`${escape(relType)}`${ - if (props.isEmpty) "" - else s" ${props.toCypherString}" - }]" + s"[:`${escape(relType)}`${if (props.isEmpty) "" + else s" ${props.toCypherString}"}]" case Node(_, labels, props) => val labelString = if (labels.isEmpty) "" else labels.toSeq.sorted.map(escape).mkString(":`", "`:`", "`") - val propertyString = if (props.isEmpty) "" - else s"${props.toCypherString}" + val propertyString = + if (props.isEmpty) "" + else s"${props.toCypherString}" Seq(labelString, propertyString) .filter(_.nonEmpty) .mkString("(", " ", ")") @@ -244,8 +252,8 @@ object CypherValue { private[opencypher] def isOrContainsNull: Boolean = isNull || { this match { case l: CypherList => l.value.exists(_.isOrContainsNull) - case m: CypherMap => m.value.valuesIterator.exists(_.isOrContainsNull) - case _ => false + case m: CypherMap => m.value.valuesIterator.exists(_.isOrContainsNull) + case _ => false } } @@ -265,7 +273,9 @@ object CypherValue { override def cypherType: CypherType = CTString } - implicit class CypherBoolean(val value: Boolean) extends AnyVal with PrimitiveCypherValue[Boolean] { + implicit class CypherBoolean(val value: Boolean) + extends AnyVal + with PrimitiveCypherValue[Boolean] { override def cypherType: CypherType = if (value) CTTrue else CTFalse } @@ -279,27 +289,40 @@ object CypherValue { override def cypherType: CypherType = CTFloat } - implicit class CypherBigDecimal(val value: BigDecimal) extends AnyVal with CypherNumber[BigDecimal] { - override def cypherType: CypherType = CTBigDecimal(value.precision, value.scale) + implicit class CypherBigDecimal(val value: BigDecimal) + extends AnyVal + with CypherNumber[BigDecimal] { + override def cypherType: CypherType = + CTBigDecimal(value.precision, value.scale) } - implicit class CypherLocalDateTime(val value: java.time.LocalDateTime) extends AnyVal with MaterialCypherValue[java.time.LocalDateTime] { + implicit class CypherLocalDateTime(val value: java.time.LocalDateTime) + extends AnyVal + with MaterialCypherValue[java.time.LocalDateTime] { override def unwrap: Any = value override def cypherType: CypherType = CTLocalDateTime } - implicit class CypherDate(val value: java.time.LocalDate) extends AnyVal with MaterialCypherValue[java.time.LocalDate] { + implicit class CypherDate(val value: java.time.LocalDate) + extends AnyVal + with MaterialCypherValue[java.time.LocalDate] { override def unwrap: Any = value override def cypherType: CypherType = CTDate } - implicit class CypherDuration(val value: Duration) extends AnyVal with MaterialCypherValue[Duration] { + implicit class CypherDuration(val value: Duration) + extends AnyVal + with MaterialCypherValue[Duration] { override def unwrap: Any = value override def cypherType: CypherType = CTDuration } - implicit class CypherMap(val value: Map[String, CypherValue]) extends AnyVal with MaterialCypherValue[Map[String, CypherValue]] { - override def unwrap: Map[String, Any] = value.map { case (k, v) => k -> v.unwrap } + implicit class CypherMap(val value: Map[String, CypherValue]) + extends AnyVal + with MaterialCypherValue[Map[String, CypherValue]] { + override def unwrap: Map[String, Any] = value.map { case (k, v) => + k -> v.unwrap + } def isEmpty: Boolean = value.isEmpty @@ -307,7 +330,8 @@ object CypherValue { def get(k: String): Option[CypherValue] = value.get(k) - def getOrElse(k: String, default: CypherValue = CypherNull): CypherValue = value.getOrElse(k, default) + def getOrElse(k: String, default: CypherValue = CypherNull): CypherValue = + value.getOrElse(k, default) def apply(k: String): CypherValue = value.getOrElse(k, CypherNull) @@ -328,9 +352,13 @@ object CypherValue { } - implicit class CypherList(val value: List[CypherValue]) extends AnyVal with MaterialCypherValue[List[CypherValue]] { + implicit class CypherList(val value: List[CypherValue]) + extends AnyVal + with MaterialCypherValue[List[CypherValue]] { override def unwrap: List[Any] = value.map(_.unwrap) - override def cypherType: CypherType = CTList(CTUnion(value.map(_.cypherType): _*)) + override def cypherType: CypherType = CTList( + CTUnion(value.map(_.cypherType): _*) + ) } object CypherList extends UnapplyValue[List[CypherValue], CypherList] { @@ -347,17 +375,26 @@ object CypherValue { def properties: CypherMap override def hashCode: Int = { - MurmurHash3.orderedHash(productIterator, MurmurHash3.stringHash(productPrefix)) + MurmurHash3.orderedHash( + productIterator, + MurmurHash3.stringHash(productPrefix) + ) } override def equals(other: Any): Boolean = other match { case that: Element[_] => - (that canEqual this) && haveEqualValues(this.productIterator, that.productIterator) + (that canEqual this) && haveEqualValues( + this.productIterator, + that.productIterator + ) case _ => false } - protected def haveEqualValues(a: Iterator[Any], b: Iterator[Any]): Boolean = { + protected def haveEqualValues( + a: Iterator[Any], + b: Iterator[Any] + ): Boolean = { while (a.hasNext && b.hasNext) { if (a.next != b.next) return false } @@ -398,12 +435,17 @@ object CypherValue { case 0 => id case 1 => labels case 2 => properties - case other => throw IllegalArgumentException("a valid product index", s"$other") + case other => + throw IllegalArgumentException("a valid product index", s"$other") } override def canEqual(that: Any): Boolean = that.isInstanceOf[Node[_]] - def copy(id: Id = id, labels: Set[String] = labels, properties: CypherMap = properties): I + def copy( + id: Id = id, + labels: Set[String] = labels, + properties: CypherMap = properties + ): I def withLabel(label: String): I = { copy(labels = labels + label) @@ -425,7 +467,10 @@ object CypherValue { } - trait Relationship[Id] extends Element[Id] with MaterialCypherValue[Relationship[Id]] with Product { + trait Relationship[Id] + extends Element[Id] + with MaterialCypherValue[Relationship[Id]] + with Product { override type I <: Relationship[Id] @@ -451,17 +496,20 @@ object CypherValue { case 2 => endId case 3 => relType case 4 => properties - case other => throw IllegalArgumentException("a valid product index", s"$other") + case other => + throw IllegalArgumentException("a valid product index", s"$other") } - override def canEqual(that: Any): Boolean = that.isInstanceOf[Relationship[_]] + override def canEqual(that: Any): Boolean = + that.isInstanceOf[Relationship[_]] def copy( id: Id = id, source: Id = startId, target: Id = endId, relType: String = relType, - properties: CypherMap = properties): I + properties: CypherMap = properties + ): I def withType(relType: String): I = { copy(relType = relType) @@ -479,7 +527,9 @@ object CypherValue { val startIdJsonKey: String = "startId" val endIdJsonKey: String = "endId" - def unapply[Id](r: Relationship[Id]): Option[(Id, Id, Id, String, CypherMap)] = { + def unapply[Id]( + r: Relationship[Id] + ): Option[(Id, Id, Id, String, CypherMap)] = { Option(r).map(rel => (rel.id, rel.startId, rel.endId, rel.relType, rel.properties)) } @@ -491,9 +541,7 @@ object CypherValue { override def getValue: Option[T] = Option(value) } - /** - * A primitive Cypher value is one that does not contain any other Cypher values. - */ + /** A primitive Cypher value is one that does not contain any other Cypher values. */ trait PrimitiveCypherValue[+T] extends Any with MaterialCypherValue[T] { override def unwrap: T = value } @@ -515,9 +563,9 @@ object CypherValue { val context = new MathContext(precision) val bigDecimal = v match { - case i: Int => BigDecimal(i, context) - case l: Long => BigDecimal(l, context) - case f: Float => BigDecimal(f.toDouble, context) + case i: Int => BigDecimal(i, context) + case l: Long => BigDecimal(l, context) + case f: Float => BigDecimal(f.toDouble, context) case d: Double => BigDecimal(d, context) case s: String => BigDecimal(s, context) } diff --git a/okapi-api/src/main/scala/org/opencypher/okapi/api/value/CypherValueHelp.scala b/okapi-api/src/main/scala/org/opencypher/okapi/api/value/CypherValueHelp.scala index deca1876ff..682b216a42 100644 --- a/okapi-api/src/main/scala/org/opencypher/okapi/api/value/CypherValueHelp.scala +++ b/okapi-api/src/main/scala/org/opencypher/okapi/api/value/CypherValueHelp.scala @@ -28,18 +28,34 @@ package org.opencypher.okapi.api.value import org.opencypher.okapi.api.value.CypherValue.Element.{idJsonKey, propertiesJsonKey} import org.opencypher.okapi.api.value.CypherValue.Node.labelsJsonKey -import org.opencypher.okapi.api.value.CypherValue.{CypherBigDecimal, CypherBoolean, CypherFloat, CypherInteger, CypherList, CypherMap, CypherNull, CypherString, CypherValue, Node, Relationship} -import org.opencypher.okapi.api.value.CypherValue.Relationship.{endIdJsonKey, startIdJsonKey, typeJsonKey} +import org.opencypher.okapi.api.value.CypherValue.{ + CypherBigDecimal, + CypherBoolean, + CypherFloat, + CypherInteger, + CypherList, + CypherMap, + CypherNull, + CypherString, + CypherValue, + Node, + Relationship +} +import org.opencypher.okapi.api.value.CypherValue.Relationship.{ + endIdJsonKey, + startIdJsonKey, + typeJsonKey +} import ujson.{Bool, Null, Num, Obj, Str, Value} object CypherValueHelp { def toJson(v: CypherValue)(implicit formatValue: Any => String): Value = { v match { - case CypherNull => Null + case CypherNull => Null case CypherString(s) => Str(s) - case CypherList(l) => l.map(toJson) - case CypherMap(m) => m.mapValues(toJson).toSeq.sortBy(_._1) + case CypherList(l) => l.map(toJson) + case CypherMap(m) => m.mapValues(toJson).toSeq.sortBy(_._1) case Relationship(id, startId, endId, relType, properties) => Obj( idJsonKey -> Str(formatValue(id)), @@ -54,14 +70,15 @@ object CypherValueHelp { labelsJsonKey -> labels.toSeq.sorted.map(Str), propertiesJsonKey -> toJson(properties) ) - case CypherFloat(d) => Num(d) + case CypherFloat(d) => Num(d) case CypherInteger(l) => Str(l.toString) // `Num` would lose precision case CypherBoolean(b) => Bool(b) - case CypherBigDecimal(b) => Obj( - "type" -> Str("BigDecimal"), - "scale" -> Num(b.bigDecimal.scale()), - "precision" -> Num(b.bigDecimal.precision()) - ) + case CypherBigDecimal(b) => + Obj( + "type" -> Str("BigDecimal"), + "scale" -> Num(b.bigDecimal.scale()), + "precision" -> Num(b.bigDecimal.precision()) + ) case other => Str(formatValue(other.value)) } } diff --git a/okapi-api/src/main/scala/org/opencypher/okapi/impl/annotations/experimental.scala b/okapi-api/src/main/scala/org/opencypher/okapi/impl/annotations/experimental.scala index 0c80f9deb0..62d22325bf 100644 --- a/okapi-api/src/main/scala/org/opencypher/okapi/impl/annotations/experimental.scala +++ b/okapi-api/src/main/scala/org/opencypher/okapi/impl/annotations/experimental.scala @@ -29,7 +29,7 @@ package org.opencypher.okapi.impl.annotations import scala.annotation.StaticAnnotation /** - * Experimental methods and classes are not subject to semantic versioning. They might thus be changed or - * removed at any time. + * Experimental methods and classes are not subject to semantic versioning. They might thus be + * changed or removed at any time. */ -final class experimental extends StaticAnnotation \ No newline at end of file +final class experimental extends StaticAnnotation diff --git a/okapi-api/src/main/scala/org/opencypher/okapi/impl/configuration/ConfigOption.scala b/okapi-api/src/main/scala/org/opencypher/okapi/impl/configuration/ConfigOption.scala index 1d825eb41f..298462d71d 100644 --- a/okapi-api/src/main/scala/org/opencypher/okapi/impl/configuration/ConfigOption.scala +++ b/okapi-api/src/main/scala/org/opencypher/okapi/impl/configuration/ConfigOption.scala @@ -28,11 +28,14 @@ package org.opencypher.okapi.impl.configuration import scala.util.Try -abstract class ConfigOption[T](val name: String, val defaultValue: T)(val convert: String => Option[T]) { +abstract class ConfigOption[T](val name: String, val defaultValue: T)( + val convert: String => Option[T] +) { def set(v: String): Unit = System.setProperty(name, v) - def get: T = Option(System.getProperty(name)).flatMap(convert).getOrElse(defaultValue) + def get: T = + Option(System.getProperty(name)).flatMap(convert).getOrElse(defaultValue) override def toString: String = { val padded = name.padTo(25, " ").mkString("") @@ -41,7 +44,7 @@ abstract class ConfigOption[T](val name: String, val defaultValue: T)(val conver } abstract class ConfigFlag(name: String, defaultValue: Boolean = false) - extends ConfigOption[Boolean](name, defaultValue)(s => Try(s.toBoolean).toOption) { + extends ConfigOption[Boolean](name, defaultValue)(s => Try(s.toBoolean).toOption) { def set(): Unit = set(true.toString) @@ -52,7 +55,8 @@ trait ConfigCaching[T] { self: ConfigOption[T] => - override lazy val get: T = Option(System.getProperty(name)).flatMap(convert).getOrElse(defaultValue) + override lazy val get: T = + Option(System.getProperty(name)).flatMap(convert).getOrElse(defaultValue) override def set(v: String): Unit = { System.setProperty(name, v) diff --git a/okapi-api/src/main/scala/org/opencypher/okapi/impl/exception/InternalException.scala b/okapi-api/src/main/scala/org/opencypher/okapi/impl/exception/InternalException.scala index fab4a8e498..686b8509b6 100644 --- a/okapi-api/src/main/scala/org/opencypher/okapi/impl/exception/InternalException.scala +++ b/okapi-api/src/main/scala/org/opencypher/okapi/impl/exception/InternalException.scala @@ -29,37 +29,74 @@ package org.opencypher.okapi.impl.exception import scala.compat.Platform.EOL /** - * Exceptions that are not covered by the TCK. They are related to limitations of a specific Cypher implementation - * or to session or property graph interactions that are not covered by the TCK. + * Exceptions that are not covered by the TCK. They are related to limitations of a specific Cypher + * implementation or to session or property graph interactions that are not covered by the TCK. */ //TODO: Either: 1. Convert to CypherException; 2. Create categories that makes sense in the API module for them; or 3. Move to the internals of the system to which they belong. -abstract class InternalException(msg: String, cause: Option[Throwable] = None) extends RuntimeException(msg, cause.orNull) with Serializable +abstract class InternalException(msg: String, cause: Option[Throwable] = None) + extends RuntimeException(msg, cause.orNull) + with Serializable -final case class SchemaException(msg: String, cause: Option[Throwable] = None) extends InternalException(msg, cause) +final case class SchemaException(msg: String, cause: Option[Throwable] = None) + extends InternalException(msg, cause) -final case class CypherValueException(msg: String, cause: Option[Throwable] = None) extends InternalException(msg, cause) +final case class CypherValueException( + msg: String, + cause: Option[Throwable] = None +) extends InternalException(msg, cause) -final case class NotImplementedException(msg: String, cause: Option[Throwable] = None) extends InternalException(msg, cause) +final case class NotImplementedException( + msg: String, + cause: Option[Throwable] = None +) extends InternalException(msg, cause) -final case class IllegalStateException(msg: String, cause: Option[Throwable] = None) extends InternalException(msg, cause) +final case class IllegalStateException( + msg: String, + cause: Option[Throwable] = None +) extends InternalException(msg, cause) -final case class IllegalArgumentException(expected: Any, actual: Any = "none", explanation: String = "", cause: Option[Throwable] = None) - extends InternalException( - s""" - |${if (explanation.nonEmpty) s"Explanation:$EOL\t$explanation$EOL" else ""} +final case class IllegalArgumentException( + expected: Any, + actual: Any = "none", + explanation: String = "", + cause: Option[Throwable] = None +) extends InternalException( + s""" + |${if (explanation.nonEmpty) s"Explanation:$EOL\t$explanation$EOL" + else ""} |Expected: |\t$expected |Found: - |\t$actual""".stripMargin, cause) + |\t$actual""".stripMargin, + cause + ) -final case class UnsupportedOperationException(msg: String, cause: Option[Throwable] = None) extends InternalException(msg, cause) +final case class UnsupportedOperationException( + msg: String, + cause: Option[Throwable] = None +) extends InternalException(msg, cause) -final case class NoSuitableSignatureForExpr(msg: String, cause: Option[Throwable] = None) extends InternalException(msg, cause) +final case class NoSuitableSignatureForExpr( + msg: String, + cause: Option[Throwable] = None +) extends InternalException(msg, cause) -final case class GraphNotFoundException(msg: String, cause: Option[Throwable] = None) extends InternalException(msg, cause) +final case class GraphNotFoundException( + msg: String, + cause: Option[Throwable] = None +) extends InternalException(msg, cause) -final case class InvalidGraphException(msg: String, cause: Option[Throwable] = None) extends InternalException(msg, cause) +final case class InvalidGraphException( + msg: String, + cause: Option[Throwable] = None +) extends InternalException(msg, cause) -final case class GraphAlreadyExistsException(msg: String, cause: Option[Throwable] = None) extends InternalException(msg, cause) +final case class GraphAlreadyExistsException( + msg: String, + cause: Option[Throwable] = None +) extends InternalException(msg, cause) -final case class ViewAlreadyExistsException(msg: String, cause: Option[Throwable] = None) extends InternalException(msg, cause) +final case class ViewAlreadyExistsException( + msg: String, + cause: Option[Throwable] = None +) extends InternalException(msg, cause) diff --git a/okapi-api/src/main/scala/org/opencypher/okapi/impl/graph/CypherCatalog.scala b/okapi-api/src/main/scala/org/opencypher/okapi/impl/graph/CypherCatalog.scala index d3e6d7e6c6..f20d5e8a54 100644 --- a/okapi-api/src/main/scala/org/opencypher/okapi/impl/graph/CypherCatalog.scala +++ b/okapi-api/src/main/scala/org/opencypher/okapi/impl/graph/CypherCatalog.scala @@ -30,7 +30,11 @@ import org.opencypher.v9_0.ast._ import org.opencypher.okapi.api.graph._ import org.opencypher.okapi.api.io.PropertyGraphDataSource import org.opencypher.okapi.api.value.CypherValue.CypherString -import org.opencypher.okapi.impl.exception.{IllegalArgumentException, UnsupportedOperationException, ViewAlreadyExistsException} +import org.opencypher.okapi.impl.exception.{ + IllegalArgumentException, + UnsupportedOperationException, + ViewAlreadyExistsException +} import org.opencypher.okapi.impl.io.SessionGraphDataSource object CypherCatalog { @@ -42,112 +46,173 @@ object CypherCatalog { } /** - * This is the default implementation of the [[org.opencypher.okapi.api.graph.PropertyGraphCatalog]]. - * It uses a mutable mapping to store the mapping between - * [[org.opencypher.okapi.api.graph.Namespace]]s and [[org.opencypher.okapi.api.io.PropertyGraphDataSource]]s. + * This is the default implementation of the + * [[org.opencypher.okapi.api.graph.PropertyGraphCatalog]]. It uses a mutable mapping to store the + * mapping between [[org.opencypher.okapi.api.graph.Namespace]]s and + * [[org.opencypher.okapi.api.io.PropertyGraphDataSource]]s. * - * By default this catalog mounts a single [[org.opencypher.okapi.impl.io.SessionGraphDataSource]] under the namespace - * [[org.opencypher.okapi.impl.graph.CypherCatalog#sessionNamespace]]. This PGDS is used to store session local graphs. + * By default this catalog mounts a single [[org.opencypher.okapi.impl.io.SessionGraphDataSource]] + * under the namespace [[org.opencypher.okapi.impl.graph.CypherCatalog#sessionNamespace]]. This + * PGDS is used to store session local graphs. */ class CypherCatalog extends PropertyGraphCatalog { /** * The [[org.opencypher.okapi.api.graph.Namespace]] used to store graphs within this session. * - * @return session namespace + * @return + * session namespace */ def sessionNamespace: Namespace = SessionGraphDataSource.Namespace /** - * Stores a mutable mapping between a [[org.opencypher.okapi.api.graph.Namespace]] and the specific - * [[org.opencypher.okapi.api.io.PropertyGraphDataSource]]. + * Stores a mutable mapping between a [[org.opencypher.okapi.api.graph.Namespace]] and the + * specific [[org.opencypher.okapi.api.io.PropertyGraphDataSource]]. * - * This mapping also holds the [[org.opencypher.okapi.impl.io.SessionGraphDataSource]] by default. + * This mapping also holds the [[org.opencypher.okapi.impl.io.SessionGraphDataSource]] by + * default. */ private var dataSourceMapping: Map[Namespace, PropertyGraphDataSource] = Map(sessionNamespace -> new SessionGraphDataSource) - private var viewMapping: Map[QualifiedGraphName, ParameterizedView] = Map.empty + private var viewMapping: Map[QualifiedGraphName, ParameterizedView] = + Map.empty override def namespaces: Set[Namespace] = dataSourceMapping.keySet - override def source(namespace: Namespace): PropertyGraphDataSource = dataSourceMapping.getOrElse(namespace, - throw IllegalArgumentException(s"a data source registered with namespace '$namespace'")) + override def source(namespace: Namespace): PropertyGraphDataSource = + dataSourceMapping.getOrElse( + namespace, + throw IllegalArgumentException( + s"a data source registered with namespace '$namespace'" + ) + ) - override def listSources: Map[Namespace, PropertyGraphDataSource] = dataSourceMapping + override def listSources: Map[Namespace, PropertyGraphDataSource] = + dataSourceMapping override def register( namespace: Namespace, dataSource: PropertyGraphDataSource ): Unit = dataSourceMapping.get(namespace) match { - case Some(p) => throw IllegalArgumentException(s"There is already a data source registered with namespace '$namespace'", p) - case None => dataSourceMapping = dataSourceMapping.updated(namespace, dataSource) + case Some(p) => + throw IllegalArgumentException( + s"There is already a data source registered with namespace '$namespace'", + p + ) + case None => + dataSourceMapping = dataSourceMapping.updated(namespace, dataSource) } override def deregister(namespace: Namespace): Unit = { - if (namespace == sessionNamespace) throw UnsupportedOperationException("de-registering the session data source") + if (namespace == sessionNamespace) + throw UnsupportedOperationException( + "de-registering the session data source" + ) dataSourceMapping.get(namespace) match { case Some(_) => dataSourceMapping = dataSourceMapping - namespace - case None => throw IllegalArgumentException(s"No data source registered with namespace '$namespace'") + case None => + throw IllegalArgumentException( + s"No data source registered with namespace '$namespace'" + ) } } // TODO: Filter empty graph override def graphNames: Set[QualifiedGraphName] = { - dataSourceMapping.flatMap { - case (namespace, pgds) => - pgds.graphNames.map(n => QualifiedGraphName(namespace, n)) + dataSourceMapping.flatMap { case (namespace, pgds) => + pgds.graphNames.map(n => QualifiedGraphName(namespace, n)) }.toSet } override def viewNames: Set[QualifiedGraphName] = viewMapping.keySet - override def store(qualifiedGraphName: QualifiedGraphName, graph: PropertyGraph): Unit = - source(qualifiedGraphName.namespace).store(qualifiedGraphName.graphName, graph) - - override def store(qualifiedGraphName: QualifiedGraphName, parameterNames: List[String], viewQuery: String): Unit = { + override def store( + qualifiedGraphName: QualifiedGraphName, + graph: PropertyGraph + ): Unit = + source(qualifiedGraphName.namespace) + .store(qualifiedGraphName.graphName, graph) + + override def store( + qualifiedGraphName: QualifiedGraphName, + parameterNames: List[String], + viewQuery: String + ): Unit = { val existsAlready = viewMapping.contains(qualifiedGraphName) if (existsAlready) { - throw ViewAlreadyExistsException(s"A view with name `$qualifiedGraphName` already exists") + throw ViewAlreadyExistsException( + s"A view with name `$qualifiedGraphName` already exists" + ) } else { - viewMapping += (qualifiedGraphName -> ParameterizedView(parameterNames, viewQuery)) + viewMapping += (qualifiedGraphName -> ParameterizedView( + parameterNames, + viewQuery + )) } } override def dropGraph(qualifiedGraphName: QualifiedGraphName): Unit = source(qualifiedGraphName.namespace).delete(qualifiedGraphName.graphName) - override def dropView(qualifiedGraphName: QualifiedGraphName): Unit = viewMapping -= qualifiedGraphName + override def dropView(qualifiedGraphName: QualifiedGraphName): Unit = + viewMapping -= qualifiedGraphName override def graph(qualifiedGraphName: QualifiedGraphName): PropertyGraph = source(qualifiedGraphName.namespace).graph(qualifiedGraphName.graphName) - private[opencypher] def view(viewInvocation: ViewInvocation)(implicit session: CypherSession): PropertyGraph = { + private[opencypher] def view( + viewInvocation: ViewInvocation + )(implicit session: CypherSession): PropertyGraph = { val qgn = QualifiedGraphName(viewInvocation.graphName.parts) val viewDefinition = viewMapping.get(qgn) match { case Some(vd) => vd - case None => throw IllegalArgumentException( - s"the name of a stored view${if (viewNames.isEmpty) "" else s" [${viewNames.mkString(", ")}]"}", - s"unknown view name `$qgn`" - ) + case None => + throw IllegalArgumentException( + s"the name of a stored view${if (viewNames.isEmpty) "" + else s" [${viewNames.mkString(", ")}]"}", + s"unknown view name `$qgn`" + ) } - val paramNameValueTuples = viewDefinition.parameterNames.zip(viewInvocation.params) - val (parameterMap, queryLocalGraphs) = paramNameValueTuples.foldLeft(Map.empty[String, CypherString] -> Map.empty[QualifiedGraphName, PropertyGraph]) { - case ((currentParamMap, currentQueryLocalGraphs), (nextName, nextFrom: FromGraph)) => + val paramNameValueTuples = + viewDefinition.parameterNames.zip(viewInvocation.params) + val (parameterMap, queryLocalGraphs) = paramNameValueTuples.foldLeft( + Map.empty[String, CypherString] -> Map + .empty[QualifiedGraphName, PropertyGraph] + ) { + case ( + (currentParamMap, currentQueryLocalGraphs), + (nextName, nextFrom: FromGraph) + ) => nextFrom match { case v: ViewInvocation => // Recursive view evaluation val graph = view(v) val graphQgn = session.generateQualifiedGraphName - currentParamMap.updated(nextName, CypherString(graphQgn.toString)) -> currentQueryLocalGraphs.updated(graphQgn, graph) + currentParamMap.updated( + nextName, + CypherString(graphQgn.toString) + ) -> currentQueryLocalGraphs.updated(graphQgn, graph) case g: GraphLookup => // Simple case, parameter is just passed on - currentParamMap.updated(nextName, CypherString(QualifiedGraphName(g.graphName.parts).toString)) -> currentQueryLocalGraphs + currentParamMap.updated( + nextName, + CypherString(QualifiedGraphName(g.graphName.parts).toString) + ) -> currentQueryLocalGraphs case other => - throw IllegalArgumentException("a graph lookup or a view invocation", other) + throw IllegalArgumentException( + "a graph lookup or a view invocation", + other + ) } } - session.cypher(viewDefinition.viewQuery, parameterMap, queryCatalog = queryLocalGraphs).graph + session + .cypher( + viewDefinition.viewQuery, + parameterMap, + queryCatalog = queryLocalGraphs + ) + .graph } } diff --git a/okapi-api/src/main/scala/org/opencypher/okapi/impl/io/SessionGraphDataSource.scala b/okapi-api/src/main/scala/org/opencypher/okapi/impl/io/SessionGraphDataSource.scala index 404425a5c5..99f80b65c2 100644 --- a/okapi-api/src/main/scala/org/opencypher/okapi/impl/io/SessionGraphDataSource.scala +++ b/okapi-api/src/main/scala/org/opencypher/okapi/impl/io/SessionGraphDataSource.scala @@ -41,17 +41,27 @@ object SessionGraphDataSource { class SessionGraphDataSource() extends PropertyGraphDataSource { - private val graphMap: mutable.Map[GraphName, PropertyGraph] = mutable.Map.empty + private val graphMap: mutable.Map[GraphName, PropertyGraph] = + mutable.Map.empty override def graph(name: GraphName): PropertyGraph = - graphMap.getOrElse(name, throw GraphNotFoundException(s"Session graph with name `$name`.")) + graphMap.getOrElse( + name, + throw GraphNotFoundException(s"Session graph with name `$name`.") + ) - override def schema(name: GraphName): Option[PropertyGraphSchema] = Some(graph(name).schema) + override def schema(name: GraphName): Option[PropertyGraphSchema] = Some( + graph(name).schema + ) - override def store(name: GraphName, graph: PropertyGraph): Unit = graphMap.get(name) match { - case None => graphMap.update(name, graph) - case Some(_) => throw GraphAlreadyExistsException(s"A graph with name $name is already stored in the session.") - } + override def store(name: GraphName, graph: PropertyGraph): Unit = + graphMap.get(name) match { + case None => graphMap.update(name, graph) + case Some(_) => + throw GraphAlreadyExistsException( + s"A graph with name $name is already stored in the session." + ) + } override def delete(name: GraphName): Unit = graphMap.remove(name) diff --git a/okapi-api/src/main/scala/org/opencypher/okapi/impl/schema/ImpliedLabels.scala b/okapi-api/src/main/scala/org/opencypher/okapi/impl/schema/ImpliedLabels.scala index 7542a41f4c..ffc354d45e 100644 --- a/okapi-api/src/main/scala/org/opencypher/okapi/impl/schema/ImpliedLabels.scala +++ b/okapi-api/src/main/scala/org/opencypher/okapi/impl/schema/ImpliedLabels.scala @@ -56,5 +56,6 @@ case class ImpliedLabels(m: Map[String, Set[String]]) { ImpliedLabels(filteredImplications) } - private def implicationsFor(source: String) = m.getOrElse(source, Set.empty) + source + private def implicationsFor(source: String) = + m.getOrElse(source, Set.empty) + source } diff --git a/okapi-api/src/main/scala/org/opencypher/okapi/impl/schema/LabelCombinations.scala b/okapi-api/src/main/scala/org/opencypher/okapi/impl/schema/LabelCombinations.scala index 86a993a81a..0e6c392975 100644 --- a/okapi-api/src/main/scala/org/opencypher/okapi/impl/schema/LabelCombinations.scala +++ b/okapi-api/src/main/scala/org/opencypher/okapi/impl/schema/LabelCombinations.scala @@ -32,16 +32,17 @@ object LabelCombinations { case class LabelCombinations(combos: Set[Set[String]]) { - /** - * Returns all combinations that contain the argument `labels` - */ + /** Returns all combinations that contain the argument `labels` */ def combinationsFor(labels: Set[String]): Set[Set[String]] = combos.filter(labels.subsetOf) def withCombinations(coExistingLabels: String*): LabelCombinations = { - val (lhs, rhs) = combos.partition(labels => coExistingLabels.exists(labels(_))) + val (lhs, rhs) = + combos.partition(labels => coExistingLabels.exists(labels(_))) copy(combos = rhs + (lhs.flatten ++ coExistingLabels)) } - def ++(other: LabelCombinations): LabelCombinations = copy(combos ++ other.combos) + def ++(other: LabelCombinations): LabelCombinations = copy( + combos ++ other.combos + ) } diff --git a/okapi-api/src/main/scala/org/opencypher/okapi/impl/schema/PropertyGraphSchemaImpl.scala b/okapi-api/src/main/scala/org/opencypher/okapi/impl/schema/PropertyGraphSchemaImpl.scala index 566485de5e..42dff01041 100644 --- a/okapi-api/src/main/scala/org/opencypher/okapi/impl/schema/PropertyGraphSchemaImpl.scala +++ b/okapi-api/src/main/scala/org/opencypher/okapi/impl/schema/PropertyGraphSchemaImpl.scala @@ -53,82 +53,103 @@ object PropertyGraphSchemaImpl { val REL_TYPE = "relType" val PROPERTIES = "properties" - - implicit def rw: ReadWriter[PropertyGraphSchema] = readwriter[Value].bimap[PropertyGraphSchema]( - schema => { - val tuples: Seq[(String, Value)] = Seq[(String, Value)]( - VERSION -> writeJs(PropertyGraphSchema.CURRENT_VERSION.toString), - LABEL_PROPERTY_MAP -> writeJs(schema.labelPropertyMap), - REL_TYPE_PROPERTY_MAP -> writeJs(schema.relTypePropertyMap)) ++ { - if (schema.explicitSchemaPatterns.nonEmpty) { - Some(SCHEMA_PATTERNS -> writeJs(schema.explicitSchemaPatterns)) - } else { - None + implicit def rw: ReadWriter[PropertyGraphSchema] = + readwriter[Value].bimap[PropertyGraphSchema]( + schema => { + val tuples: Seq[(String, Value)] = Seq[(String, Value)]( + VERSION -> writeJs(PropertyGraphSchema.CURRENT_VERSION.toString), + LABEL_PROPERTY_MAP -> writeJs(schema.labelPropertyMap), + REL_TYPE_PROPERTY_MAP -> writeJs(schema.relTypePropertyMap) + ) ++ { + if (schema.explicitSchemaPatterns.nonEmpty) { + Some(SCHEMA_PATTERNS -> writeJs(schema.explicitSchemaPatterns)) + } else { + None + } + } ++ { + if (schema.nodeKeys.nonEmpty) { + Some(NODE_KEYS -> writeJs(schema.nodeKeys)) + } else { + None + } + } ++ { + if (schema.relationshipKeys.nonEmpty) { + Some(REL_KEYS -> writeJs(schema.relationshipKeys)) + } else { + None + } } - } ++ { - if (schema.nodeKeys.nonEmpty) { - Some(NODE_KEYS -> writeJs(schema.nodeKeys)) - } else { - None + Obj.from(tuples) + }, + json => { + val versionString: String = json.obj(VERSION) match { + case Str(inner) => inner + case Num(inner) => inner.toString + case other => + throw SchemaException( + s"Expected Version to be a String or a Number but got $other" + ) } - } ++ { - if (schema.relationshipKeys.nonEmpty) { - Some(REL_KEYS -> writeJs(schema.relationshipKeys)) - } else { - None + val version = Version(versionString) + if (!version.compatibleWith(PropertyGraphSchema.CURRENT_VERSION)) + throw SchemaException(s"Incompatible Schema version: $version") + + val labelPropertyMap = + read[LabelPropertyMap](json.obj(LABEL_PROPERTY_MAP)) + val relTypePropertyMap = + read[RelTypePropertyMap](json.obj(REL_TYPE_PROPERTY_MAP)) + val explicitSchemaPatterns = json match { + case Obj(m) if m.keySet.contains(SCHEMA_PATTERNS) => + read[Set[SchemaPattern]](json.obj(SCHEMA_PATTERNS)) + case _ => Set.empty[SchemaPattern] } + val nodeKeys = json match { + case Obj(m) if m.keySet.contains(NODE_KEYS) => + read[Map[String, Set[String]]](json.obj(NODE_KEYS)) + case _ => Map.empty[String, Set[String]] + } + val relKeys = json match { + case Obj(m) if m.keySet.contains(REL_KEYS) => + read[Map[String, Set[String]]](json.obj(REL_KEYS)) + case _ => Map.empty[String, Set[String]] + } + PropertyGraphSchemaImpl( + labelPropertyMap, + relTypePropertyMap, + explicitSchemaPatterns, + nodeKeys, + relKeys + ) } - Obj.from(tuples) - } - , - json => { - val versionString: String = json.obj(VERSION) match { - case Str(inner) => inner - case Num(inner) => inner.toString - case other => throw SchemaException(s"Expected Version to be a String or a Number but got $other") - } - val version = Version(versionString) - if(!version.compatibleWith(PropertyGraphSchema.CURRENT_VERSION)) throw SchemaException(s"Incompatible Schema version: $version") - - val labelPropertyMap = read[LabelPropertyMap](json.obj(LABEL_PROPERTY_MAP)) - val relTypePropertyMap = read[RelTypePropertyMap](json.obj(REL_TYPE_PROPERTY_MAP)) - val explicitSchemaPatterns = json match { - case Obj(m) if m.keySet.contains(SCHEMA_PATTERNS) => read[Set[SchemaPattern]](json.obj(SCHEMA_PATTERNS)) - case _ => Set.empty[SchemaPattern] - } - val nodeKeys = json match { - case Obj(m) if m.keySet.contains(NODE_KEYS) => read[Map[String, Set[String]]](json.obj(NODE_KEYS)) - case _ => Map.empty[String, Set[String]] - } - val relKeys = json match { - case Obj(m) if m.keySet.contains(REL_KEYS) => read[Map[String, Set[String]]](json.obj(REL_KEYS)) - case _ => Map.empty[String, Set[String]] - } - PropertyGraphSchemaImpl(labelPropertyMap, relTypePropertyMap, explicitSchemaPatterns, nodeKeys, relKeys) - } - ) + ) - implicit def lpmRw: ReadWriter[LabelPropertyMap] = readwriter[Value].bimap[LabelPropertyMap]( - labelPropertyMap => - labelPropertyMap.map { - case (labelCombo, propKeys) => Obj(LABELS -> writeJs(labelCombo), PROPERTIES -> writeJs(propKeys)) - }, - json => - json.arr.map { value => - read[Set[String]](value.obj(LABELS)) -> read[PropertyKeys](value.obj(PROPERTIES)) - }.toMap - ) - - implicit def rpmRw: ReadWriter[RelTypePropertyMap] = readwriter[Value].bimap[RelTypePropertyMap]( - relTypePropertyMap => - relTypePropertyMap.map { - case (relType, propKeys) => Obj(REL_TYPE -> writeJs(relType), PROPERTIES -> writeJs(propKeys)) - }, - json => - json.arr.map { value => - read[String](value.obj(REL_TYPE)) -> read[PropertyKeys](value.obj(PROPERTIES)) - }.toMap - ) + implicit def lpmRw: ReadWriter[LabelPropertyMap] = + readwriter[Value].bimap[LabelPropertyMap]( + labelPropertyMap => + labelPropertyMap.map { case (labelCombo, propKeys) => + Obj(LABELS -> writeJs(labelCombo), PROPERTIES -> writeJs(propKeys)) + }, + json => + json.arr.map { value => + read[Set[String]](value.obj(LABELS)) -> read[PropertyKeys]( + value.obj(PROPERTIES) + ) + }.toMap + ) + + implicit def rpmRw: ReadWriter[RelTypePropertyMap] = + readwriter[Value].bimap[RelTypePropertyMap]( + relTypePropertyMap => + relTypePropertyMap.map { case (relType, propKeys) => + Obj(REL_TYPE -> writeJs(relType), PROPERTIES -> writeJs(propKeys)) + }, + json => + json.arr.map { value => + read[String](value.obj(REL_TYPE)) -> read[PropertyKeys]( + value.obj(PROPERTIES) + ) + }.toMap + ) implicit def spRW: ReadWriter[SchemaPattern] = macroRW } @@ -160,14 +181,17 @@ final case class PropertyGraphSchemaImpl( } override lazy val impliedLabels: ImpliedLabels = { - val implications = self.labelCombinations.combos.foldLeft(Map.empty[String, Set[String]]) { - case (currentMap, combo) => combo.foldLeft(currentMap) { - case (innerMap, label) => innerMap.get(label) match { - case Some(innerCombo) => innerMap.updated(label, (innerCombo intersect combo) - label) - case None => innerMap.updated(label, combo - label) - } + val implications = + self.labelCombinations.combos.foldLeft(Map.empty[String, Set[String]]) { + case (currentMap, combo) => + combo.foldLeft(currentMap) { case (innerMap, label) => + innerMap.get(label) match { + case Some(innerCombo) => + innerMap.updated(label, (innerCombo intersect combo) - label) + case None => innerMap.updated(label, combo - label) + } + } } - } ImpliedLabels(implications) } @@ -177,7 +201,8 @@ final case class PropertyGraphSchemaImpl( override def impliedLabels(knownLabels: Set[String]): Set[String] = impliedLabels.transitiveImplicationsFor(knownLabels.intersect(labels)) - override def nodePropertyKeys(labelCombination: Set[String]): PropertyKeys = labelPropertyMap.properties(labelCombination) + override def nodePropertyKeys(labelCombination: Set[String]): PropertyKeys = + labelPropertyMap.properties(labelCombination) override def allCombinations: Set[Set[String]] = combinationsFor(Set.empty) @@ -185,12 +210,17 @@ final case class PropertyGraphSchemaImpl( override def combinationsFor(knownLabels: Set[String]): Set[Set[String]] = labelCombinations.combinationsFor(knownLabels) - override def nodePropertyKeyType(knownLabels: Set[String], key: String): Option[CypherType] = { + override def nodePropertyKeyType( + knownLabels: Set[String], + key: String + ): Option[CypherType] = { val combos = combinationsFor(knownLabels) nodePropertyKeysForCombinations(combos).get(key) } - override def nodePropertyKeysForCombinations(labelCombinations: Set[Set[String]]): PropertyKeys = { + override def nodePropertyKeysForCombinations( + labelCombinations: Set[Set[String]] + ): PropertyKeys = { val allKeys = labelCombinations.toSeq.flatMap(nodePropertyKeys) val propertyKeys = allKeys.groupBy(_._1).mapValues { seq => if (seq.size == labelCombinations.size && seq.distinct.size == 1) { @@ -205,7 +235,10 @@ final case class PropertyGraphSchemaImpl( propertyKeys.view.force } - override def relationshipPropertyKeyType(types: Set[String], key: String): Option[CypherType] = { + override def relationshipPropertyKeyType( + types: Set[String], + key: String + ): Option[CypherType] = { // relationship types have OR semantics: empty set means all types val relevantTypes = if (types.isEmpty) relationshipTypes else types @@ -213,12 +246,15 @@ final case class PropertyGraphSchemaImpl( case (inferred, next) => inferred.join(next.getOrElse(key, CTNull)) } match { case CTNull => None - case tpe => Some(tpe) + case tpe => Some(tpe) } } - override def relationshipPropertyKeysForTypes(knownTypes: Set[String]): PropertyKeys = { - val relevantTypes = if (knownTypes.isEmpty) relationshipTypes else knownTypes + override def relationshipPropertyKeysForTypes( + knownTypes: Set[String] + ): PropertyKeys = { + val relevantTypes = + if (knownTypes.isEmpty) relationshipTypes else knownTypes val allKeys = relevantTypes.toSeq.flatMap(relationshipPropertyKeys) val propertyKeys = allKeys.groupBy(_._1).mapValues { seq => @@ -234,59 +270,83 @@ final case class PropertyGraphSchemaImpl( propertyKeys.view.force } - override def relationshipPropertyKeys(typ: String): PropertyKeys = relTypePropertyMap.properties(typ) + override def relationshipPropertyKeys(typ: String): PropertyKeys = + relTypePropertyMap.properties(typ) override def schemaPatternsFor( knownSourceLabels: Set[String], knownRelTypes: Set[String], knownTargetLabels: Set[String] ): Set[SchemaPattern] = { - val possibleSourcePatterns = schemaPatterns.filter(p => knownSourceLabels.subsetOf(p.sourceLabelCombination)) - val possibleRelTypePatterns = possibleSourcePatterns.filter(p => knownRelTypes.contains(p.relType) || knownRelTypes.isEmpty) - val possibleTargetPatterns = possibleRelTypePatterns.filter(p => knownTargetLabels.subsetOf(p.targetLabelCombination)) + val possibleSourcePatterns = + schemaPatterns.filter(p => knownSourceLabels.subsetOf(p.sourceLabelCombination)) + val possibleRelTypePatterns = + possibleSourcePatterns.filter(p => knownRelTypes.contains(p.relType) || knownRelTypes.isEmpty) + val possibleTargetPatterns = + possibleRelTypePatterns.filter(p => knownTargetLabels.subsetOf(p.targetLabelCombination)) possibleTargetPatterns } - override def withNodePropertyKeys(labelCombination: Set[String], keys: PropertyKeys): PropertyGraphSchema = { + override def withNodePropertyKeys( + labelCombination: Set[String], + keys: PropertyKeys + ): PropertyGraphSchema = { if (labelCombination.exists(_.isEmpty)) throw SchemaException("Labels must be non-empty") - val propertyKeys = if (labelPropertyMap.labelCombinations(labelCombination)) { - computePropertyTypes(labelPropertyMap.properties(labelCombination), keys) - } else { - keys - } + val propertyKeys = + if (labelPropertyMap.labelCombinations(labelCombination)) { + computePropertyTypes( + labelPropertyMap.properties(labelCombination), + keys + ) + } else { + keys + } copy(labelPropertyMap = labelPropertyMap.register(labelCombination, propertyKeys)) } - override def withNodeKey(label: String, nodeKey: Set[String]): PropertyGraphSchema = { + override def withNodeKey( + label: String, + nodeKey: Set[String] + ): PropertyGraphSchema = { if (!labels.contains(label)) { throw SchemaException( s"""|Invalid node key for schema |$pretty |Unknown node label `$label`. - |Should be one of: ${labels.mkString("[", ", ", "]")}""".stripMargin) + |Should be one of: ${labels.mkString("[", ", ", "]")}""".stripMargin + ) } - val propertyKeys = nodePropertyKeysForCombinations(combinationsFor(Set(label))) + val propertyKeys = nodePropertyKeysForCombinations( + combinationsFor(Set(label)) + ) if (!nodeKey.subsetOf(propertyKeys.keySet)) { - throw SchemaException( - s"""|Invalid node key for schema + throw SchemaException(s"""|Invalid node key for schema |$pretty - |Not all combinations that contain `$label` have all the properties for ${nodeKey.mkString("[", ", ", "]")}. - |Available keys: ${propertyKeys.keySet.mkString("[", ", ", "]")}""".stripMargin) + |Not all combinations that contain `$label` have all the properties for ${nodeKey + .mkString("[", ", ", "]")}. + |Available keys: ${propertyKeys.keySet.mkString( + "[", + ", ", + "]" + )}""".stripMargin) } - val nullableProperties = propertyKeys.filter { - case (key, tpe) => nodeKey.contains(key) && tpe.isNullable + val nullableProperties = propertyKeys.filter { case (key, tpe) => + nodeKey.contains(key) && tpe.isNullable } if (nullableProperties.nonEmpty) { - throw SchemaException( - s"""|Invalid node key for schema + throw SchemaException(s"""|Invalid node key for schema |$pretty - |Properties ${nullableProperties.keySet.mkString("[", ", ", "]")} have nullable types. + |Properties ${nullableProperties.keySet.mkString( + "[", + ", ", + "]" + )} have nullable types. |Nullable properties can not be part of a node key.""".stripMargin) } @@ -294,60 +354,79 @@ final case class PropertyGraphSchemaImpl( } - override def withRelationshipKey(relationshipType: String, relationshipKey: Set[String]): PropertyGraphSchema = { + override def withRelationshipKey( + relationshipType: String, + relationshipKey: Set[String] + ): PropertyGraphSchema = { if (!relationshipTypes.contains(relationshipType)) { - throw SchemaException( - s"""|Invalid relationship key for schema + throw SchemaException(s"""|Invalid relationship key for schema |$pretty |Unknown relationship type `$relationshipType`. - |Should be one of: ${relationshipTypes.mkString("[", ", ", "]")}.""".stripMargin) + |Should be one of: ${relationshipTypes.mkString( + "[", + ", ", + "]" + )}.""".stripMargin) } val propertyKeys = relationshipPropertyKeys(relationshipType) if (!relationshipKey.subsetOf(propertyKeys.keySet)) { - throw SchemaException( - s"""|Invalid relationship key for schema + throw SchemaException(s"""|Invalid relationship key for schema |$pretty - |Relationship type `$relationshipType` does not have all the properties for ${relationshipKey.mkString("[", ", ", "]")}. - |Available keys: ${propertyKeys.keySet.mkString("[", ", ", "]")}""".stripMargin) + |Relationship type `$relationshipType` does not have all the properties for ${relationshipKey + .mkString("[", ", ", "]")}. + |Available keys: ${propertyKeys.keySet.mkString( + "[", + ", ", + "]" + )}""".stripMargin) } - val nullableProperties = propertyKeys.filter { - case (key, tpe) => relationshipKey.contains(key) && tpe.isNullable + val nullableProperties = propertyKeys.filter { case (key, tpe) => + relationshipKey.contains(key) && tpe.isNullable } if (nullableProperties.nonEmpty) { throw SchemaException( s"""|Invalid relationship key for schema |$pretty - |Properties ${nullableProperties.keySet.mkString("[", ", ", "]")} have nullable types. - |Nullable properties can not be part of a relationship key.""".stripMargin) + |Properties ${nullableProperties.keySet + .mkString("[", ", ", "]")} have nullable types. + |Nullable properties can not be part of a relationship key.""".stripMargin + ) } copy(relationshipKeys = relationshipKeys.updated(relationshipType, relationshipKey)) } - private def computePropertyTypes(existing: PropertyKeys, input: PropertyKeys): PropertyKeys = { + private def computePropertyTypes( + existing: PropertyKeys, + input: PropertyKeys + ): PropertyKeys = { // Map over input keys to calculate join of type with existing type - val keysWithJoinedTypes = input.map { - case (key, propType) => - val inType = existing.getOrElse(key, CTNull) - key -> propType.join(inType) + val keysWithJoinedTypes = input.map { case (key, propType) => + val inType = existing.getOrElse(key, CTNull) + key -> propType.join(inType) } // Map over the rest of the existing keys to mark them all nullable - val propertiesMarkedOptional = existing.filterKeys(k => !input.contains(k)).foldLeft(keysWithJoinedTypes) { - case (map, (key, propTyp)) => + val propertiesMarkedOptional = existing + .filterKeys(k => !input.contains(k)) + .foldLeft(keysWithJoinedTypes) { case (map, (key, propTyp)) => map.updated(key, propTyp.nullable) - } + } propertiesMarkedOptional } - override def withRelationshipPropertyKeys(typ: String, keys: PropertyKeys): PropertyGraphSchema = { + override def withRelationshipPropertyKeys( + typ: String, + keys: PropertyKeys + ): PropertyGraphSchema = { if (relationshipTypes contains typ) { - val updatedTypes = computePropertyTypes(relTypePropertyMap.properties(typ), keys) + val updatedTypes = + computePropertyTypes(relTypePropertyMap.properties(typ), keys) copy(relTypePropertyMap = relTypePropertyMap.register(typ, updatedTypes.toSeq)) } else { @@ -355,34 +434,59 @@ final case class PropertyGraphSchemaImpl( } } - override def withSchemaPatterns(patterns: SchemaPattern*): PropertyGraphSchema = { + override def withSchemaPatterns( + patterns: SchemaPattern* + ): PropertyGraphSchema = { patterns.foreach { p => - if (!labelCombinations.combos.contains(p.sourceLabelCombination)) throw SchemaException(s"Unknown source node label combination: `${p.sourceLabelCombination}`. Should be one of: ${labelCombinations.combos.mkString("[", ",", "]")}") - if (!relationshipTypes.contains(p.relType)) throw SchemaException(s"Unknown relationship type: `${p.relType}`. Should be one of ${relationshipTypes.mkString("[", ",", "]")}") - if (!labelCombinations.combos.contains(p.targetLabelCombination)) throw SchemaException(s"Unknown target node label combination: `${p.targetLabelCombination}`. Should be one of: ${labelCombinations.combos.mkString("[", ",", "]")}") + if (!labelCombinations.combos.contains(p.sourceLabelCombination)) + throw SchemaException( + s"Unknown source node label combination: `${p.sourceLabelCombination}`. Should be one of: ${labelCombinations.combos + .mkString("[", ",", "]")}" + ) + if (!relationshipTypes.contains(p.relType)) + throw SchemaException( + s"Unknown relationship type: `${p.relType}`. Should be one of ${relationshipTypes + .mkString("[", ",", "]")}" + ) + if (!labelCombinations.combos.contains(p.targetLabelCombination)) + throw SchemaException( + s"Unknown target node label combination: `${p.targetLabelCombination}`. Should be one of: ${labelCombinations.combos + .mkString("[", ",", "]")}" + ) } copy(explicitSchemaPatterns = explicitSchemaPatterns ++ patterns.toSet) } override def ++(other: PropertyGraphSchema): PropertyGraphSchemaImpl = { - val conflictingLabels = labelPropertyMap.labelCombinations intersect other.labelPropertyMap.labelCombinations - val nulledOut = conflictingLabels.foldLeft(Map.empty[Set[String], PropertyKeys]) { - case (acc, next) => - val keys = computePropertyTypes(labelPropertyMap.properties(next), other.labelPropertyMap.properties(next)) + val conflictingLabels = + labelPropertyMap.labelCombinations intersect other.labelPropertyMap.labelCombinations + val nulledOut = + conflictingLabels.foldLeft(Map.empty[Set[String], PropertyKeys]) { case (acc, next) => + val keys = computePropertyTypes( + labelPropertyMap.properties(next), + other.labelPropertyMap.properties(next) + ) acc + (next -> keys) - } - val newLabelPropertyMap = labelPropertyMap |+| other.labelPropertyMap |+| nulledOut - - val conflictingRelTypes = relationshipTypes intersect other.relationshipTypes - val nulledRelProps = conflictingRelTypes.foldLeft(Map.empty[String, PropertyKeys]) { - case (acc, next) => - val keys = computePropertyTypes(relTypePropertyMap.properties(next), other.relTypePropertyMap.properties(next)) + } + val newLabelPropertyMap = + labelPropertyMap |+| other.labelPropertyMap |+| nulledOut + + val conflictingRelTypes = + relationshipTypes intersect other.relationshipTypes + val nulledRelProps = + conflictingRelTypes.foldLeft(Map.empty[String, PropertyKeys]) { case (acc, next) => + val keys = computePropertyTypes( + relTypePropertyMap.properties(next), + other.relTypePropertyMap.properties(next) + ) acc + (next -> keys) - } - val newRelTypePropertyMap = relTypePropertyMap |+| other.relTypePropertyMap |+| nulledRelProps + } + val newRelTypePropertyMap = + relTypePropertyMap |+| other.relTypePropertyMap |+| nulledRelProps - val newExplicitSchemaPatterns = explicitSchemaPatterns ++ other.explicitSchemaPatterns + val newExplicitSchemaPatterns = + explicitSchemaPatterns ++ other.explicitSchemaPatterns val newNodeKeys = nodeKeys |+| other.nodeKeys val newRelationshipKeys = relationshipKeys |+| other.relationshipKeys @@ -399,7 +503,8 @@ final case class PropertyGraphSchemaImpl( def forNode(labelConstraints: Set[String]): PropertyGraphSchema = { val requiredLabels = { val explicitLabels = labelConstraints - val impliedLabels = this.impliedLabels.transitiveImplicationsFor(explicitLabels) + val impliedLabels = + this.impliedLabels.transitiveImplicationsFor(explicitLabels) explicitLabels union impliedLabels } @@ -411,7 +516,8 @@ final case class PropertyGraphSchemaImpl( } // take all label properties that might appear on the possible labels - val newLabelPropertyMap: LabelPropertyMap = this.labelPropertyMap.filterKeys(possibleLabels.contains) + val newLabelPropertyMap: LabelPropertyMap = + this.labelPropertyMap.filterKeys(possibleLabels.contains) // add labels that were specified in the constraints but are not present in source schema val updatedLabelPropertyMap = possibleLabels.foldLeft(newLabelPropertyMap) { @@ -431,10 +537,12 @@ final case class PropertyGraphSchemaImpl( relType.types } - val updatedRelTypePropertyMap = this.relTypePropertyMap.filterForRelTypes(givenRelTypes) - val updatedMap = givenRelTypes.foldLeft(updatedRelTypePropertyMap) { - case (map, givenRelType) => - if (!map.contains(givenRelType)) map.updated(givenRelType, PropertyKeys.empty) else map + val updatedRelTypePropertyMap = + this.relTypePropertyMap.filterForRelTypes(givenRelTypes) + val updatedMap = givenRelTypes.foldLeft(updatedRelTypePropertyMap) { case (map, givenRelType) => + if (!map.contains(givenRelType)) + map.updated(givenRelType, PropertyKeys.empty) + else map } PropertyGraphSchemaImpl( @@ -453,10 +561,12 @@ final case class PropertyGraphSchemaImpl( if (labelPropertyMap.labelCombinations.nonEmpty) { builder.append(s"Node labels {$EOL") labelPropertyMap.labelCombinations.foreach { combo => - val labelStr = if (combo eq Set.empty) "(no label)" else combo.mkString(":", ":", "") + val labelStr = + if (combo eq Set.empty) "(no label)" + else combo.mkString(":", ":", "") builder.append(s"\t$labelStr$EOL") - nodePropertyKeys(combo).foreach { - case (key, typ) => builder.append(s"\t\t$key: $typ$EOL") + nodePropertyKeys(combo).foreach { case (key, typ) => + builder.append(s"\t\t$key: $typ$EOL") } } builder.append(s"}$EOL") @@ -468,7 +578,9 @@ final case class PropertyGraphSchemaImpl( builder.append(s"Implied labels:$EOL") impliedLabels.m.foreach { case (label, implications) if implications.nonEmpty => - builder.append(s":$label -> ${implications.mkString(":", ":", "")}$EOL") + builder.append( + s":$label -> ${implications.mkString(":", ":", "")}$EOL" + ) case _ => } } else { @@ -479,8 +591,8 @@ final case class PropertyGraphSchemaImpl( builder.append(s"Rel types {$EOL") relationshipTypes.foreach { relType => builder.append(s"\t:$relType$EOL") - relationshipPropertyKeys(relType).foreach { - case (key, typ) => builder.append(s"\t\t$key: $typ$EOL") + relationshipPropertyKeys(relType).foreach { case (key, typ) => + builder.append(s"\t\t$key: $typ$EOL") } } builder.append(s"}$EOL") @@ -490,9 +602,7 @@ final case class PropertyGraphSchemaImpl( if (explicitSchemaPatterns.nonEmpty) { builder.append(s"Explicit schema patterns {$EOL") - explicitSchemaPatterns.foreach(p => - builder.append(s"\t$p$EOL") - ) + explicitSchemaPatterns.foreach(p => builder.append(s"\t$p$EOL")) builder.append(s"}$EOL") } @@ -516,5 +626,6 @@ final case class PropertyGraphSchemaImpl( ) = copy(relTypePropertyMap = relTypePropertyMap.register(relType, propertyKeys)) - override def toJson: String = upickle.default.write[PropertyGraphSchema](this, indent = 4) + override def toJson: String = + upickle.default.write[PropertyGraphSchema](this, indent = 4) } diff --git a/okapi-api/src/main/scala/org/opencypher/okapi/impl/table/RecordsPrinter.scala b/okapi-api/src/main/scala/org/opencypher/okapi/impl/table/RecordsPrinter.scala index 9d66fe2b9e..208d092af6 100644 --- a/okapi-api/src/main/scala/org/opencypher/okapi/impl/table/RecordsPrinter.scala +++ b/okapi-api/src/main/scala/org/opencypher/okapi/impl/table/RecordsPrinter.scala @@ -36,14 +36,15 @@ object RecordsPrinter { /** * Prints the given CypherRecords to stdout * - * @param records the records to be printed. + * @param records + * the records to be printed. */ def print(records: CypherRecords)(implicit options: PrintOptions): Unit = { val columns = records.logicalColumns.getOrElse(records.physicalColumns) val rows: Seq[Seq[CypherValue]] = records.collect.map { row => - columns.foldLeft(Seq.empty[CypherValue]) { - case (currentSeq, column) => currentSeq :+ row(column) + columns.foldLeft(Seq.empty[CypherValue]) { case (currentSeq, column) => + currentSeq :+ row(column) } } diff --git a/okapi-api/src/main/scala/org/opencypher/okapi/impl/temporal/Duration.scala b/okapi-api/src/main/scala/org/opencypher/okapi/impl/temporal/Duration.scala index 8ec669777d..dab3fa2e2a 100644 --- a/okapi-api/src/main/scala/org/opencypher/okapi/impl/temporal/Duration.scala +++ b/okapi-api/src/main/scala/org/opencypher/okapi/impl/temporal/Duration.scala @@ -1,28 +1,25 @@ /** * Copyright (c) 2016-2019 "Neo4j Sweden, AB" [https://neo4j.com] * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and + * Unless required by applicable law or agreed to in writing, software distributed under the + * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing permissions and * limitations under the License. * * Attribution Notice under the terms of the Apache License 2.0 * - * This work was created by the collective efforts of the openCypher community. - * Without limiting the terms of Section 6, any Derivative Work that is not - * approved by the public consensus process of the openCypher Implementers Group - * should not be described as “Cypher” (and Cypher® is a registered trademark of - * Neo4j Inc.) or as "openCypher". Extensions by implementers or prototypes or - * proposals for change that have been documented or implemented should only be - * described as "implementation extensions to Cypher" or as "proposed changes to - * Cypher that are not yet approved by the openCypher community". + * This work was created by the collective efforts of the openCypher community. Without limiting + * the terms of Section 6, any Derivative Work that is not approved by the public consensus process + * of the openCypher Implementers Group should not be described as “Cypher” (and Cypher® is a + * registered trademark of Neo4j Inc.) or as "openCypher". Extensions by implementers or prototypes + * or proposals for change that have been documented or implemented should only be described as + * "implementation extensions to Cypher" or as "proposed changes to Cypher that are not yet + * approved by the openCypher community". */ package org.opencypher.okapi.impl.temporal @@ -37,16 +34,26 @@ import org.opencypher.okapi.impl.temporal.TemporalConstants._ /** * Okapi representation of a duration. * - * @param months number of months - * @param days number of days - * @param seconds number of seconds - * @param nanos normalized number of nanoseconds spanning fractions of a second + * @param months + * number of months + * @param days + * number of days + * @param seconds + * number of seconds + * @param nanos + * normalized number of nanoseconds spanning fractions of a second */ -class Duration protected(val months: Long = 0, val days: Long = 0, val seconds: Long = 0, val nanos: Long = 0) - extends Ordered[Duration] { +class Duration protected ( + val months: Long = 0, + val days: Long = 0, + val seconds: Long = 0, + val nanos: Long = 0 +) extends Ordered[Duration] { def toJava: (java.time.Period, java.time.Duration) = { - val period = java.time.Period.of((months / 12).toInt, (months % 12).toInt, days.toInt) - val duration = java.time.Duration.ofSeconds(seconds).plus(nanos, ChronoUnit.NANOS) + val period = + java.time.Period.of((months / 12).toInt, (months % 12).toInt, days.toInt) + val duration = + java.time.Duration.ofSeconds(seconds).plus(nanos, ChronoUnit.NANOS) (period, duration) } @@ -55,7 +62,7 @@ class Duration protected(val months: Long = 0, val days: Long = 0, val seconds: override def equals(o: Any): Boolean = o match { case d: Duration => compare(d) == 0 - case _ => false + case _ => false } /* * Since not every month has the same amount of seconds, we use the average to sum this duration in seconds. @@ -77,9 +84,10 @@ class Duration protected(val months: Long = 0, val days: Long = 0, val seconds: private lazy val COMPARATOR: Comparator[Duration] = { import scala.language.implicitConversions - implicit def toLongFunction[T](f: T => Long): ToLongFunction[T] = new ToLongFunction[T] { - override def applyAsLong(t: T): Long = f(t) - } + implicit def toLongFunction[T](f: T => Long): ToLongFunction[T] = + new ToLongFunction[T] { + override def applyAsLong(t: T): Long = f(t) + } Comparator .comparingLong[Duration]((d: Duration) => d.averageLengthInSeconds) @@ -106,13 +114,21 @@ object Duration { ) def apply( - years: Long = 0, months: Long = 0, weeks: Long = 0, days: Long = 0, - hours: Long = 0, minutes: Long = 0, - seconds: Long = 0, milliseconds: Long = 0, microseconds: Long = 0, nanoseconds: Long = 0 + years: Long = 0, + months: Long = 0, + weeks: Long = 0, + days: Long = 0, + hours: Long = 0, + minutes: Long = 0, + seconds: Long = 0, + milliseconds: Long = 0, + microseconds: Long = 0, + nanoseconds: Long = 0 ): Duration = { val nanoSum = milliseconds * 1000000 + microseconds * 1000 + nanoseconds - val normalizedSeconds = hours * SECONDS_PER_HOUR + minutes * SECONDS_PER_MINUTE + seconds + nanoSum / NANOS_PER_SECOND + val normalizedSeconds = + hours * SECONDS_PER_HOUR + minutes * SECONDS_PER_MINUTE + seconds + nanoSum / NANOS_PER_SECOND val normalizedNanos = nanoSum % NANOS_PER_SECOND new Duration( @@ -124,11 +140,17 @@ object Duration { } def apply(javaDuration: java.time.Duration): Duration = { - Duration(seconds = javaDuration.getSeconds, nanoseconds = javaDuration.getNano) + Duration( + seconds = javaDuration.getSeconds, + nanoseconds = javaDuration.getNano + ) } def apply(period: java.time.Period): Duration = { - Duration(months = period.getYears * 12 + period.getMonths, days = period.getDays) + Duration( + months = period.getYears * 12 + period.getMonths, + days = period.getDays + ) } def apply(map: Map[String, Long]): Duration = { @@ -160,15 +182,27 @@ object Duration { def parse(durationString: String): Duration = { val durationRegex = """^P(\d+Y)?(\d+M)?(\d+W)?(\d+D)?(T(\d+H)?(\d+M)?(\d+(\.\d{1,6})?S)?)?$""" - .r("years", "months", "weeks", "days", "_", "hours", "minutes", "seconds", "_", "_") + .r( + "years", + "months", + "weeks", + "days", + "_", + "hours", + "minutes", + "seconds", + "_", + "_" + ) durationRegex.findFirstMatchIn(durationString) match { case Some(m) => - val superSecondMap = Seq("years", "months", "weeks", "days", "hours", "minutes") - .map(id => id -> m.group(id)) - .filterNot(_._2.isNull) - .toMap - .mapValues(_.dropRight(1).toLong) + val superSecondMap = + Seq("years", "months", "weeks", "days", "hours", "minutes") + .map(id => id -> m.group(id)) + .filterNot(_._2.isNull) + .toMap + .mapValues(_.dropRight(1).toLong) val secondsMap = m.group("seconds") match { case s: String => @@ -189,7 +223,11 @@ object Duration { Duration(superSecondMap ++ secondsMap) - case _ => throw IllegalArgumentException("a valid duration construction string", durationString) + case _ => + throw IllegalArgumentException( + "a valid duration construction string", + durationString + ) } } } diff --git a/okapi-api/src/main/scala/org/opencypher/okapi/impl/temporal/TemporalConstants.scala b/okapi-api/src/main/scala/org/opencypher/okapi/impl/temporal/TemporalConstants.scala index 7f87df4db7..2b05405889 100644 --- a/okapi-api/src/main/scala/org/opencypher/okapi/impl/temporal/TemporalConstants.scala +++ b/okapi-api/src/main/scala/org/opencypher/okapi/impl/temporal/TemporalConstants.scala @@ -1,29 +1,26 @@ /** - * Copyright (c) 2016-2019 "Neo4j Sweden, AB" [https://neo4j.com] - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * Attribution Notice under the terms of the Apache License 2.0 - * - * This work was created by the collective efforts of the openCypher community. - * Without limiting the terms of Section 6, any Derivative Work that is not - * approved by the public consensus process of the openCypher Implementers Group - * should not be described as “Cypher” (and Cypher® is a registered trademark of - * Neo4j Inc.) or as "openCypher". Extensions by implementers or prototypes or - * proposals for change that have been documented or implemented should only be - * described as "implementation extensions to Cypher" or as "proposed changes to - * Cypher that are not yet approved by the openCypher community". - */ + * Copyright (c) 2016-2019 "Neo4j Sweden, AB" [https://neo4j.com] + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the + * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing permissions and + * limitations under the License. + * + * Attribution Notice under the terms of the Apache License 2.0 + * + * This work was created by the collective efforts of the openCypher community. Without limiting + * the terms of Section 6, any Derivative Work that is not approved by the public consensus process + * of the openCypher Implementers Group should not be described as “Cypher” (and Cypher® is a + * registered trademark of Neo4j Inc.) or as "openCypher". Extensions by implementers or prototypes + * or proposals for change that have been documented or implemented should only be described as + * "implementation extensions to Cypher" or as "proposed changes to Cypher that are not yet + * approved by the openCypher community". + */ package org.opencypher.okapi.impl.temporal import java.time.temporal.ChronoUnit.DAYS diff --git a/okapi-api/src/main/scala/org/opencypher/okapi/impl/temporal/TemporalTypesHelper.scala b/okapi-api/src/main/scala/org/opencypher/okapi/impl/temporal/TemporalTypesHelper.scala index 4ef5dbb2cb..317c4a8fc6 100644 --- a/okapi-api/src/main/scala/org/opencypher/okapi/impl/temporal/TemporalTypesHelper.scala +++ b/okapi-api/src/main/scala/org/opencypher/okapi/impl/temporal/TemporalTypesHelper.scala @@ -1,29 +1,26 @@ /** - * Copyright (c) 2016-2019 "Neo4j Sweden, AB" [https://neo4j.com] - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * Attribution Notice under the terms of the Apache License 2.0 - * - * This work was created by the collective efforts of the openCypher community. - * Without limiting the terms of Section 6, any Derivative Work that is not - * approved by the public consensus process of the openCypher Implementers Group - * should not be described as “Cypher” (and Cypher® is a registered trademark of - * Neo4j Inc.) or as "openCypher". Extensions by implementers or prototypes or - * proposals for change that have been documented or implemented should only be - * described as "implementation extensions to Cypher" or as "proposed changes to - * Cypher that are not yet approved by the openCypher community". - */ + * Copyright (c) 2016-2019 "Neo4j Sweden, AB" [https://neo4j.com] + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the + * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing permissions and + * limitations under the License. + * + * Attribution Notice under the terms of the Apache License 2.0 + * + * This work was created by the collective efforts of the openCypher community. Without limiting + * the terms of Section 6, any Derivative Work that is not approved by the public consensus process + * of the openCypher Implementers Group should not be described as “Cypher” (and Cypher® is a + * registered trademark of Neo4j Inc.) or as "openCypher". Extensions by implementers or prototypes + * or proposals for change that have been documented or implemented should only be described as + * "implementation extensions to Cypher" or as "proposed changes to Cypher that are not yet + * approved by the openCypher community". + */ package org.opencypher.okapi.impl.temporal import java.time.format.{DateTimeFormatter, DateTimeFormatterBuilder, SignStyle} @@ -45,7 +42,8 @@ object TemporalTypesHelper { val dateByMonthIdentifiers: Seq[String] = Seq("year", "month", "day") val dateByWeekIdentifiers: Seq[String] = Seq("year", "week", "dayofweek") val dateByOrdinalDayIdentifiers: Seq[String] = Seq("year", "ordinalday") - val dateByQuarterIdentifiers: Seq[String] = Seq("year", "quarter", "dayofquarter") + val dateByQuarterIdentifiers: Seq[String] = + Seq("year", "quarter", "dayofquarter") val timeIdentifiers: Seq[String] = Seq("hour", "minute", "second") val dateFormatters: Seq[DateTimeFormatter] = Seq( @@ -56,12 +54,14 @@ object TemporalTypesHelper { new DateTimeFormatterBuilder().appendPattern("yyyyMMdd").toFormatter, // 2010-10 - new DateTimeFormatterBuilder().appendPattern("yyyy-MM") + new DateTimeFormatterBuilder() + .appendPattern("yyyy-MM") .parseDefaulting(ChronoField.DAY_OF_MONTH, 1) .toFormatter, // 201010 - new DateTimeFormatterBuilder().appendPattern("yyyyMM") + new DateTimeFormatterBuilder() + .appendPattern("yyyyMM") .parseDefaulting(ChronoField.DAY_OF_MONTH, 1) .toFormatter, @@ -69,8 +69,7 @@ object TemporalTypesHelper { DateTimeFormatter.ISO_WEEK_DATE, // 2015W302 - new DateTimeFormatterBuilder() - .parseCaseInsensitive + new DateTimeFormatterBuilder().parseCaseInsensitive .appendValue(IsoFields.WEEK_BASED_YEAR, 4, 10, SignStyle.EXCEEDS_PAD) .appendLiteral("W") .appendValue(IsoFields.WEEK_OF_WEEK_BASED_YEAR, 2) @@ -78,8 +77,7 @@ object TemporalTypesHelper { .toFormatter, // 2015-W30 - new DateTimeFormatterBuilder() - .parseCaseInsensitive + new DateTimeFormatterBuilder().parseCaseInsensitive .appendValue(IsoFields.WEEK_BASED_YEAR, 4, 10, SignStyle.EXCEEDS_PAD) .appendLiteral("-") .appendLiteral("W") @@ -88,8 +86,7 @@ object TemporalTypesHelper { .toFormatter, // 2015W30 - new DateTimeFormatterBuilder() - .parseCaseInsensitive + new DateTimeFormatterBuilder().parseCaseInsensitive .appendValue(IsoFields.WEEK_BASED_YEAR, 4, 10, SignStyle.EXCEEDS_PAD) .appendLiteral("W") .appendValue(IsoFields.WEEK_OF_WEEK_BASED_YEAR, 2) @@ -97,7 +94,8 @@ object TemporalTypesHelper { .toFormatter, // 2015-Q2-60 - new DateTimeFormatterBuilder().appendPattern("yyyy") + new DateTimeFormatterBuilder() + .appendPattern("yyyy") .appendLiteral("-Q") .appendValue(IsoFields.QUARTER_OF_YEAR, 1) .appendLiteral("-") @@ -105,21 +103,24 @@ object TemporalTypesHelper { .toFormatter, // 2015Q260 - new DateTimeFormatterBuilder().appendPattern("yyyy") + new DateTimeFormatterBuilder() + .appendPattern("yyyy") .appendLiteral("Q") .appendValue(IsoFields.QUARTER_OF_YEAR, 1) .appendValue(IsoFields.DAY_OF_QUARTER, 2) .toFormatter, // 2015-Q2 - new DateTimeFormatterBuilder().appendPattern("yyyy") + new DateTimeFormatterBuilder() + .appendPattern("yyyy") .appendLiteral("-Q") .appendValue(IsoFields.QUARTER_OF_YEAR, 1) .parseDefaulting(IsoFields.DAY_OF_QUARTER, 1) .toFormatter, // 2015Q2 - new DateTimeFormatterBuilder().appendPattern("yyyy") + new DateTimeFormatterBuilder() + .appendPattern("yyyy") .appendLiteral("Q") .appendValue(IsoFields.QUARTER_OF_YEAR, 1) .parseDefaulting(IsoFields.DAY_OF_QUARTER, 1) @@ -132,7 +133,8 @@ object TemporalTypesHelper { new DateTimeFormatterBuilder().appendPattern("yyyyDDD").toFormatter, // 2015 - new DateTimeFormatterBuilder().appendPattern("yyyy") + new DateTimeFormatterBuilder() + .appendPattern("yyyy") .parseDefaulting(ChronoField.MONTH_OF_YEAR, 1) .parseDefaulting(ChronoField.DAY_OF_MONTH, 1) .toFormatter @@ -142,7 +144,9 @@ object TemporalTypesHelper { DateTimeFormatter.ISO_LOCAL_TIME, new DateTimeFormatterBuilder().appendPattern("HHmmss.SSS").toFormatter, new DateTimeFormatterBuilder().appendPattern("HHmmss.SSSSSS").toFormatter, - new DateTimeFormatterBuilder().appendPattern("HHmmss.SSSSSSSSS").toFormatter, + new DateTimeFormatterBuilder() + .appendPattern("HHmmss.SSSSSSSSS") + .toFormatter, new DateTimeFormatterBuilder().appendPattern("HHmmss").toFormatter, new DateTimeFormatterBuilder().appendPattern("HH:mm").toFormatter, new DateTimeFormatterBuilder().appendPattern("HHmm").toFormatter, @@ -151,8 +155,8 @@ object TemporalTypesHelper { def parseDate(mapOrString: MapOrString): LocalDate = { mapOrString match { - case Left(map) => parseDateMap(map) - case Right(str)=> parseDateString(str) + case Left(map) => parseDateMap(map) + case Right(str) => parseDateString(str) } } @@ -160,7 +164,6 @@ object TemporalTypesHelper { mapOrString match { case Left(map) => - val date = parseDateMap(map) val time = parseTimeMap(map) LocalDateTime.of(date, time) @@ -171,12 +174,12 @@ object TemporalTypesHelper { val date = parseDateString(dateString) val maybeTime = timeString match { case t :: Nil => Some(parseTimeString(t)) - case _ => None + case _ => None } maybeTime match { case Some(time) => LocalDateTime.of(date, time) - case None => LocalDateTime.of(date, LocalTime.MIN) + case None => LocalDateTime.of(date, LocalTime.MIN) } } } @@ -184,7 +187,11 @@ object TemporalTypesHelper { private def parseDateMap(map: Map[String, Int]): LocalDate = { val sanitizedMap = sanitizeMap(map) - if (!sanitizedMap.contains("year")) throw IllegalArgumentException("the key `year` needs to be set", map.keys.mkString(", ")) + if (!sanitizedMap.contains("year")) + throw IllegalArgumentException( + "the key `year` needs to be set", + map.keys.mkString(", ") + ) if (sanitizedMap.keySet.contains("week")) { checkSignificanceOrder(sanitizedMap, dateByWeekIdentifiers) @@ -192,7 +199,10 @@ object TemporalTypesHelper { LocalDate.MIN .`with`(IsoFields.WEEK_BASED_YEAR, sanitizedMap("year")) .`with`(IsoFields.WEEK_OF_WEEK_BASED_YEAR, sanitizedMap("week")) - .`with`(ChronoField.DAY_OF_WEEK, sanitizedMap.getOrElse("dayofweek", 1).toLong) + .`with`( + ChronoField.DAY_OF_WEEK, + sanitizedMap.getOrElse("dayofweek", 1).toLong + ) } else if (sanitizedMap.keySet.contains("ordinalday")) { checkSignificanceOrder(sanitizedMap, dateByOrdinalDayIdentifiers) @@ -205,13 +215,19 @@ object TemporalTypesHelper { LocalDate.MIN .withYear(sanitizedMap("year")) .`with`(IsoFields.QUARTER_OF_YEAR, sanitizedMap("quarter")) - .`with`(IsoFields.DAY_OF_QUARTER, sanitizedMap.getOrElse("dayofquarter", 1).toLong) + .`with`( + IsoFields.DAY_OF_QUARTER, + sanitizedMap.getOrElse("dayofquarter", 1).toLong + ) - } - else { + } else { checkSignificanceOrder(sanitizedMap, dateByMonthIdentifiers) - LocalDate.of(sanitizedMap("year"), sanitizedMap.getOrElse("month", 1), sanitizedMap.getOrElse("day", 1)) + LocalDate.of( + sanitizedMap("year"), + sanitizedMap.getOrElse("month", 1), + sanitizedMap.getOrElse("day", 1) + ) } } @@ -221,12 +237,13 @@ object TemporalTypesHelper { LocalDate.parse(str, formatter) } match { case Success(date) => Some(date) - case Failure(_) => None + case Failure(_) => None } } matchingDateFormats.find(_.isDefined).flatten match { case Some(matchingDate) => matchingDate - case None => throw IllegalArgumentException("a valid date construction string", str) + case None => + throw IllegalArgumentException("a valid date construction string", str) } } @@ -254,32 +271,37 @@ object TemporalTypesHelper { LocalTime.parse(str, formatter) } match { case Success(date) => Some(date) - case Failure(_) => None + case Failure(_) => None } } matchingTimeFormats.find(_.isDefined).flatten match { case Some(matchingTime) => matchingTime - case None => throw IllegalArgumentException("a valid time construction string", str) + case None => + throw IllegalArgumentException("a valid time construction string", str) } } - private def checkSignificanceOrder(inputMap: Map[String, _], keys: Seq[String]): Unit = { + private def checkSignificanceOrder( + inputMap: Map[String, _], + keys: Seq[String] + ): Unit = { val validOrder = keys .map(inputMap.isDefinedAt) .sliding(2) .forall { case false :: true :: Nil => false - case _ => true + case _ => true } - if (!validOrder) throw IllegalArgumentException( - "a valid significance order", - inputMap.keys.mkString(", "), - "When constructing dates from a map it is forbidden to omit values of higher significance" - ) + if (!validOrder) + throw IllegalArgumentException( + "a valid significance order", + inputMap.keys.mkString(", "), + "When constructing dates from a map it is forbidden to omit values of higher significance" + ) } - def sanitizeMap(map: Map[String, Int]): Map[String, Int] = map.map { - case (key, value) => key.toLowerCase -> value + def sanitizeMap(map: Map[String, Int]): Map[String, Int] = map.map { case (key, value) => + key.toLowerCase -> value } } diff --git a/okapi-api/src/main/scala/org/opencypher/okapi/impl/types/CypherTypeParser.scala b/okapi-api/src/main/scala/org/opencypher/okapi/impl/types/CypherTypeParser.scala index aa242adea8..3ff33e3048 100644 --- a/okapi-api/src/main/scala/org/opencypher/okapi/impl/types/CypherTypeParser.scala +++ b/okapi-api/src/main/scala/org/opencypher/okapi/impl/types/CypherTypeParser.scala @@ -43,7 +43,9 @@ object CypherTypeParser extends Logging { val before = index - math.max(index - 20, 0) val after = math.min(index + 20, extra.input.length) - index val locationPointer = - s"""|\t${extra.input.slice(index - before, index + after).replace('\n', ' ')} + s"""|\t${extra.input + .slice(index - before, index + after) + .replace('\n', ' ')} |\t${"~" * before + "^" + "~" * after} """.stripMargin val msg = @@ -61,33 +63,47 @@ object CypherTypeParser extends Logging { // Basic types def STRING[_: P]: P[CTString.type] = IgnoreCase("STRING").map(_ => CTString) - def INTEGER[_: P]: P[CTInteger.type] = IgnoreCase("INTEGER").map(_ => CTInteger) + def INTEGER[_: P]: P[CTInteger.type] = + IgnoreCase("INTEGER").map(_ => CTInteger) def FLOAT[_: P]: P[CTFloat.type] = IgnoreCase("FLOAT").map(_ => CTFloat) - def NUMBER[_: P]: P[CTNumber.type ] = IgnoreCase("NUMBER").map(_ => CTNumber) - def BOOLEAN[_: P]: P[CTBoolean.type] = IgnoreCase("BOOLEAN").map(_ => CTBoolean) + def NUMBER[_: P]: P[CTNumber.type] = IgnoreCase("NUMBER").map(_ => CTNumber) + def BOOLEAN[_: P]: P[CTBoolean.type] = + IgnoreCase("BOOLEAN").map(_ => CTBoolean) def TRUE[_: P]: P[CTTrue.type] = IgnoreCase("TRUE").map(_ => CTTrue) def FALSE[_: P]: P[CTFalse.type] = IgnoreCase("FALSE").map(_ => CTFalse) - def ANY[_: P]: P[CTAny.type ] = IgnoreCase("ANY?").map(_ => CTAny) - def ANYMATERIAL[_: P]: P[CTAnyMaterial.type] = IgnoreCase("ANY").map(_ => CTAnyMaterial) + def ANY[_: P]: P[CTAny.type] = IgnoreCase("ANY?").map(_ => CTAny) + def ANYMATERIAL[_: P]: P[CTAnyMaterial.type] = + IgnoreCase("ANY").map(_ => CTAnyMaterial) def VOID[_: P]: P[CTVoid.type] = IgnoreCase("VOID").map(_ => CTVoid) def NULL[_: P]: P[CTNull.type] = IgnoreCase("NULL").map(_ => CTNull) def DATE[_: P]: P[CTDate.type] = IgnoreCase("DATE").map(_ => CTDate) - def LOCALDATETIME[_: P]: P[CTLocalDateTime.type] = IgnoreCase("LOCALDATETIME").map(_ => CTLocalDateTime) + def LOCALDATETIME[_: P]: P[CTLocalDateTime.type] = + IgnoreCase("LOCALDATETIME").map(_ => CTLocalDateTime) def BIGDECIMAL[_: P]: P[CTBigDecimal] = - (IgnoreCase("BIGDECIMAL") ~/ "(" ~/ integer ~/ "," ~/ integer ~/ ")").map { case (s, p) => CTBigDecimal(s, p) } + (IgnoreCase("BIGDECIMAL") ~/ "(" ~/ integer ~/ "," ~/ integer ~/ ")").map { case (s, p) => + CTBigDecimal(s, p) + } // element types def NODE[_: P]: P[CTNode] = P( - IgnoreCase("NODE") ~ ("(" ~/ label.rep ~ ")") ~ ("@" ~/ (identifier | ".").rep.!).? + IgnoreCase( + "NODE" + ) ~ ("(" ~/ label.rep ~ ")") ~ ("@" ~/ (identifier | ".").rep.!).? ).map { case (l, mg) => CTNode(l.toSet, mg.map(QualifiedGraphName(_))) } def ANYNODE[_: P]: P[CTNode.type] = P(IgnoreCase("NODE").map(_ => CTNode)) def RELATIONSHIP[_: P]: P[CTRelationship] = P( - IgnoreCase("RELATIONSHIP") ~ ("(" ~/ label.rep(sep = "|") ~/ ")") ~ ("@" ~/ (identifier | ".").rep.!).? - ).map { case (l, mg) => CTRelationship(l.toSet, mg.map(QualifiedGraphName(_))) } + IgnoreCase("RELATIONSHIP") ~ ("(" ~/ label.rep(sep = + "|" + ) ~/ ")") ~ ("@" ~/ (identifier | ".").rep.!).? + ).map { case (l, mg) => + CTRelationship(l.toSet, mg.map(QualifiedGraphName(_))) + } - def ANYRELATIONSHIP[_: P]: P[CTRelationship] = P(IgnoreCase("RELATIONSHIP").map(_ => CTRelationship)) + def ANYRELATIONSHIP[_: P]: P[CTRelationship] = P( + IgnoreCase("RELATIONSHIP").map(_ => CTRelationship) + ) def ELEMENT[_: P]: P[CTUnion] = P(IgnoreCase("ELEMENT").map(_ => CTElement)) @@ -95,13 +111,18 @@ object CypherTypeParser extends Logging { // container types def ANYLIST[_: P]: P[CTList] = P(IgnoreCase("LIST").map(_ => CTList)) - def LIST[_: P]: P[CTList] = P(IgnoreCase("LIST") ~ "(" ~/ cypherType ~/ ")").map(inner => CTList(inner)) + def LIST[_: P]: P[CTList] = + P(IgnoreCase("LIST") ~ "(" ~/ cypherType ~/ ")").map(inner => CTList(inner)) private def mapKey[_: P]: P[String] = P(identifier.! | escapedIdentifier) - private def kvPair[_: P]: P[(String, CypherType)] = P(mapKey ~/ ":" ~/ cypherType) + private def kvPair[_: P]: P[(String, CypherType)] = P( + mapKey ~/ ":" ~/ cypherType + ) def ANYMAP[_: P]: P[CTMap] = P(IgnoreCase("MAP").map(_ => CTMap)) - def MAP[_: P]: P[CTMap] = P(IgnoreCase("MAP") ~ "(" ~/ kvPair.rep(sep = ",") ~/ ")").map { inner => CTMap(inner.toMap) - } + def MAP[_: P]: P[CTMap] = + P(IgnoreCase("MAP") ~ "(" ~/ kvPair.rep(sep = ",") ~/ ")").map { inner => + CTMap(inner.toMap) + } def materialCypherType[_: P]: P[CypherType] = P( STRING | @@ -130,9 +151,11 @@ object CypherTypeParser extends Logging { BIGDECIMAL ) - def cypherType[_: P]: P[CypherType] = P((materialCypherType ~ "?".!.?.map(_.isDefined)).map { - case (ct, isNullable) => if (isNullable) ct.nullable else ct - }) + def cypherType[_: P]: P[CypherType] = P( + (materialCypherType ~ "?".!.?.map(_.isDefined)).map { case (ct, isNullable) => + if (isNullable) ct.nullable else ct + } + ) def cypherTypeFromEntireInput[_: P]: P[CypherType] = Start ~ cypherType ~ End } diff --git a/okapi-api/src/main/scala/org/opencypher/okapi/impl/types/CypherTypeUtils.scala b/okapi-api/src/main/scala/org/opencypher/okapi/impl/types/CypherTypeUtils.scala index 7d34e56865..161e2d9e20 100644 --- a/okapi-api/src/main/scala/org/opencypher/okapi/impl/types/CypherTypeUtils.scala +++ b/okapi-api/src/main/scala/org/opencypher/okapi/impl/types/CypherTypeUtils.scala @@ -33,25 +33,31 @@ object CypherTypeUtils { implicit class RichCypherType(val ct: CypherType) extends AnyVal { - private def notNode() = throw UnsupportedOperationException(s"cannot convert $ct into a CTNode") - private def notRel() = throw UnsupportedOperationException(s"cannot convert $ct into a CTRelationship") + private def notNode() = throw UnsupportedOperationException( + s"cannot convert $ct into a CTNode" + ) + private def notRel() = throw UnsupportedOperationException( + s"cannot convert $ct into a CTRelationship" + ) def toCTNode: CTNode = ct match { case n: CTNode => n - case CTUnion(as) => as.toList.collect { case n: CTNode => n } match { - case n :: Nil => n - // TODO this is Spark specific remove - case _ => notNode() - } + case CTUnion(as) => + as.toList.collect { case n: CTNode => n } match { + case n :: Nil => n + // TODO this is Spark specific remove + case _ => notNode() + } case _ => notNode() } def toCTRelationship: CTRelationship = ct match { case r: CTRelationship => r - case CTUnion(as) => as.toList.collect { case r: CTRelationship => r } match { - case r :: Nil => r - case _ => notRel() - } + case CTUnion(as) => + as.toList.collect { case r: CTRelationship => r } match { + case r :: Nil => r + case _ => notRel() + } case _ => notRel() } } diff --git a/okapi-api/src/main/scala/org/opencypher/okapi/impl/util/JsonUtils.scala b/okapi-api/src/main/scala/org/opencypher/okapi/impl/util/JsonUtils.scala index f736bec745..1644570d02 100644 --- a/okapi-api/src/main/scala/org/opencypher/okapi/impl/util/JsonUtils.scala +++ b/okapi-api/src/main/scala/org/opencypher/okapi/impl/util/JsonUtils.scala @@ -29,22 +29,23 @@ package org.opencypher.okapi.impl.util object JsonUtils { /** - * upickle by default represents Options as json arrays of 0 (None case) or 1 (Some case) elements. This overwrites - * this behaviour and either skips the key-value pair entirely (None case) or just prints `key : value` (Some case). + * upickle by default represents Options as json arrays of 0 (None case) or 1 (Some case) + * elements. This overwrites this behaviour and either skips the key-value pair entirely (None + * case) or just prints `key : value` (Some case). * * Note that this does not support nesting options. */ object FlatOption extends upickle.AttributeTagged { override implicit def OptionWriter[T: Writer]: Writer[Option[T]] = implicitly[Writer[T]].comap[Option[T]] { - case None => null.asInstanceOf[T] + case None => null.asInstanceOf[T] case Some(x) => x } override implicit def OptionReader[T: Reader]: Reader[Option[T]] = - implicitly[Reader[T]].mapNulls{ + implicitly[Reader[T]].mapNulls { case null => None - case x => Some(x) + case x => Some(x) } } } diff --git a/okapi-api/src/main/scala/org/opencypher/okapi/impl/util/ParserUtils.scala b/okapi-api/src/main/scala/org/opencypher/okapi/impl/util/ParserUtils.scala index c47dbc96f5..7d5b85557f 100644 --- a/okapi-api/src/main/scala/org/opencypher/okapi/impl/util/ParserUtils.scala +++ b/okapi-api/src/main/scala/org/opencypher/okapi/impl/util/ParserUtils.scala @@ -32,14 +32,22 @@ import fastparse._ object ParserUtils { def newline[_: P]: P[Unit] = P("\n" | "\r\n" | "\r" | "\f") def invisible[_: P]: P[Unit] = P(" " | "\t" | newline) - def comment[_: P]: P[Unit] = P("--" ~ (!newline ~ AnyChar).rep ~ (newline | &(End))) - implicit val whitespace: P[_] => P[Unit] = { implicit ctx: ParsingRun[_] => (comment | invisible).rep } + def comment[_: P]: P[Unit] = P( + "--" ~ (!newline ~ AnyChar).rep ~ (newline | &(End)) + ) + implicit val whitespace: P[_] => P[Unit] = { implicit ctx: ParsingRun[_] => + (comment | invisible).rep + } def keyword[_: P](k: String): P[Unit] = P(IgnoreCase(k)) def digit[_: P]: P[Unit] = P(CharIn("0-9")) def integer[_: P]: P[Int] = P(digit.repX(1).!.map(_.toInt)) def character[_: P]: P[Unit] = P(CharIn("a-zA-Z")) - def identifier[_: P]: P[Unit] = P(character ~~ P(character | digit | "_").repX) - def escapedIdentifier[_: P]: P[String] = P(identifier.! | ("`" ~~ CharsWhile(_ != '`').! ~~ "`")) + def identifier[_: P]: P[Unit] = P( + character ~~ P(character | digit | "_").repX + ) + def escapedIdentifier[_: P]: P[String] = P( + identifier.! | ("`" ~~ CharsWhile(_ != '`').! ~~ "`") + ) def label[_: P]: P[String] = P(":" ~ (identifier.! | escapedIdentifier)) } diff --git a/okapi-api/src/main/scala/org/opencypher/okapi/impl/util/PrintOptions.scala b/okapi-api/src/main/scala/org/opencypher/okapi/impl/util/PrintOptions.scala index 3a222645ba..34de394f57 100644 --- a/okapi-api/src/main/scala/org/opencypher/okapi/impl/util/PrintOptions.scala +++ b/okapi-api/src/main/scala/org/opencypher/okapi/impl/util/PrintOptions.scala @@ -32,16 +32,26 @@ object PrintOptions { private val DEFAULT_MAX_COLUMN_WIDTH: Int = Int.MaxValue implicit lazy val out: PrintOptions = - PrintOptions(stream = Console.out, maxColumnWidth = DEFAULT_MAX_COLUMN_WIDTH) + PrintOptions( + stream = Console.out, + maxColumnWidth = DEFAULT_MAX_COLUMN_WIDTH + ) lazy val err: PrintOptions = - PrintOptions(stream = Console.err, maxColumnWidth = DEFAULT_MAX_COLUMN_WIDTH) + PrintOptions( + stream = Console.err, + maxColumnWidth = DEFAULT_MAX_COLUMN_WIDTH + ) def current(implicit options: PrintOptions): PrintOptions = options } -final case class PrintOptions(stream: PrintStream, maxColumnWidth: Int = PrintOptions.DEFAULT_MAX_COLUMN_WIDTH) { +final case class PrintOptions( + stream: PrintStream, + maxColumnWidth: Int = PrintOptions.DEFAULT_MAX_COLUMN_WIDTH +) { def stream(newStream: PrintStream): PrintOptions = copy(stream = newStream) - def maxColumnWidth(maxColumnWidth: Int): PrintOptions = copy(maxColumnWidth = maxColumnWidth) + def maxColumnWidth(maxColumnWidth: Int): PrintOptions = + copy(maxColumnWidth = maxColumnWidth) } diff --git a/okapi-api/src/main/scala/org/opencypher/okapi/impl/util/StringEncodingUtilities.scala b/okapi-api/src/main/scala/org/opencypher/okapi/impl/util/StringEncodingUtilities.scala index 8754cef30e..069780551f 100644 --- a/okapi-api/src/main/scala/org/opencypher/okapi/impl/util/StringEncodingUtilities.scala +++ b/okapi-api/src/main/scala/org/opencypher/okapi/impl/util/StringEncodingUtilities.scala @@ -34,7 +34,8 @@ object StringEncodingUtilities { val relTypePrefix: String = "relType_" - protected val maxCharactersInHexStringEncoding: Int = 4 // Hex string encoding of a `Char` is up to 4 characters + protected val maxCharactersInHexStringEncoding: Int = + 4 // Hex string encoding of a `Char` is up to 4 characters implicit class CharOps(val c: Char) extends AnyVal { def isAscii: Boolean = c.toInt <= 127 @@ -65,10 +66,11 @@ object StringEncodingUtilities { /** * Encodes special characters in a string. * - * The encoded string contains only ASCII letters, numbers, '_', and '@'. The encoded string is compatible - * with both SQL column names and file paths. + * The encoded string contains only ASCII letters, numbers, '_', and '@'. The encoded string is + * compatible with both SQL column names and file paths. * - * @return encoded string + * @return + * encoded string */ def encodeSpecialCharacters: String = { val sb = new StringBuilder @@ -96,7 +98,8 @@ object StringEncodingUtilities { /** * Recovers the original string from a string encoded with [[encodeSpecialCharacters]]. * - * @return original string + * @return + * original string */ def decodeSpecialCharacters: String = { val sb = new StringBuilder @@ -106,8 +109,10 @@ object StringEncodingUtilities { val charToDecode = s(index) val nextIndex = if (charToDecode == '@') { val encodedHexStringStart = index + 1 - val indexAfterHexStringEnd = encodedHexStringStart + maxCharactersInHexStringEncoding - val hexString = s.substring(encodedHexStringStart, indexAfterHexStringEnd) + val indexAfterHexStringEnd = + encodedHexStringStart + maxCharactersInHexStringEncoding + val hexString = + s.substring(encodedHexStringStart, indexAfterHexStringEnd) sb.append(hexString.parseHex) indexAfterHexStringEnd } else { diff --git a/okapi-api/src/main/scala/org/opencypher/okapi/impl/util/TablePrinter.scala b/okapi-api/src/main/scala/org/opencypher/okapi/impl/util/TablePrinter.scala index e530b1d48d..26d4bebddd 100644 --- a/okapi-api/src/main/scala/org/opencypher/okapi/impl/util/TablePrinter.scala +++ b/okapi-api/src/main/scala/org/opencypher/okapi/impl/util/TablePrinter.scala @@ -31,22 +31,28 @@ object TablePrinter { private val emptyColumns = "(no columns)" private val emptyRow = "(empty row)" - def toTable[T](headerNames: Seq[String], data: Seq[Seq[T]])(implicit toString: T => String = (t: T) => t.toString): String = { + def toTable[T](headerNames: Seq[String], data: Seq[Seq[T]])(implicit + toString: T => String = (t: T) => t.toString + ): String = { val inputRows = headerNames match { - case Nil => Seq(Seq(emptyColumns),Seq(emptyRow)).toList - case _ => headerNames :: data.map(row => row.map(cell => toString(cell))).toList + case Nil => Seq(Seq(emptyColumns), Seq(emptyRow)).toList + case _ => + headerNames :: data.map(row => row.map(cell => toString(cell))).toList } val cellSizes = inputRows.map { row => row.map { cell => cell.length } } val colSizes = cellSizes.transpose.map { cellSizes => cellSizes.max } val rows = inputRows.map { row => - row.zip(colSizes).map { - case (cell, colSize) => (" %" + (-1 * colSize) + "s ").format(cell) - }.mkString("║", "│", "║") + row + .zip(colSizes) + .map { case (cell, colSize) => + (" %" + (-1 * colSize) + "s ").format(cell) + } + .mkString("║", "│", "║") } val separatorFor = rowSeparator(colSizes) _ - val topRow = separatorFor("╔", "═", "╤", "╗") + val topRow = separatorFor("╔", "═", "╤", "╗") val headerRow = separatorFor("╠", "═", "╪", "╣") val bottomRow = separatorFor("╚", "═", "╧", "╝") @@ -57,7 +63,9 @@ object TablePrinter { (header ++ body ++ footer).mkString("", "\n", "\n") } - def rowSeparator(colSizes: Seq[Int])(left: String, mid: String, cross: String, end: String): String = + def rowSeparator( + colSizes: Seq[Int] + )(left: String, mid: String, cross: String, end: String): String = colSizes.map { colSize => mid * (colSize + 2) }.mkString(left, cross, end) def rowCount(rows: Int): String = rows match { diff --git a/okapi-api/src/main/scala/org/opencypher/okapi/impl/util/Version.scala b/okapi-api/src/main/scala/org/opencypher/okapi/impl/util/Version.scala index 5d47018256..7d289fa00a 100644 --- a/okapi-api/src/main/scala/org/opencypher/okapi/impl/util/Version.scala +++ b/okapi-api/src/main/scala/org/opencypher/okapi/impl/util/Version.scala @@ -33,8 +33,13 @@ object Version { def apply(versionString: String): Version = { versionString.split('.').map(i => Try(i.toInt).toOption).toList match { case Some(major) :: Some(minor) :: Nil => Version(major, minor) - case Some(major) :: Nil => Version(major, 0) - case _ => throw IllegalArgumentException("A version of the format major.minor", versionString, "Malformed version") + case Some(major) :: Nil => Version(major, 0) + case _ => + throw IllegalArgumentException( + "A version of the format major.minor", + versionString, + "Malformed version" + ) } } } diff --git a/okapi-api/src/test/scala/org/opencypher/okapi/api/graph/CypherSessionTest.scala b/okapi-api/src/test/scala/org/opencypher/okapi/api/graph/CypherSessionTest.scala index 638f13d93c..b37441ed94 100644 --- a/okapi-api/src/test/scala/org/opencypher/okapi/api/graph/CypherSessionTest.scala +++ b/okapi-api/src/test/scala/org/opencypher/okapi/api/graph/CypherSessionTest.scala @@ -37,19 +37,25 @@ import org.opencypher.okapi.impl.io.SessionGraphDataSource class CypherSessionTest extends ApiBaseTest { it("avoid de-registering the session data source") { - an[org.opencypher.okapi.impl.exception.UnsupportedOperationException] should be thrownBy + an[ + org.opencypher.okapi.impl.exception.UnsupportedOperationException + ] should be thrownBy createSession.deregisterSource(SessionGraphDataSource.Namespace) } it("avoid de-registering a non-registered data source") { - an[IllegalArgumentException] should be thrownBy createSession.deregisterSource(Namespace("foo")) + an[IllegalArgumentException] should be thrownBy createSession + .deregisterSource(Namespace("foo")) } it("avoids registering a data source with an existing namespace") { val session = createSession val namespace = Namespace("foo") session.registerSource(namespace, mock[PropertyGraphDataSource]) - an[IllegalArgumentException] should be thrownBy session.registerSource(namespace, mock[PropertyGraphDataSource]) + an[IllegalArgumentException] should be thrownBy session.registerSource( + namespace, + mock[PropertyGraphDataSource] + ) } it("register data source") { @@ -67,17 +73,25 @@ class CypherSessionTest extends ApiBaseTest { session.registerSource(namespace, dataSource) session.catalog.source(namespace) should equal(dataSource) session.deregisterSource(namespace) - an[IllegalArgumentException] should be thrownBy session.catalog.source(namespace) - an[IllegalArgumentException] should be thrownBy session.catalog.source(namespace) + an[IllegalArgumentException] should be thrownBy session.catalog.source( + namespace + ) + an[IllegalArgumentException] should be thrownBy session.catalog.source( + namespace + ) } it("namespaces") { val session = createSession - session.catalog.namespaces should equal(Set(SessionGraphDataSource.Namespace)) + session.catalog.namespaces should equal( + Set(SessionGraphDataSource.Namespace) + ) val namespace = Namespace("foo") val dataSource = mock[PropertyGraphDataSource] session.registerSource(namespace, dataSource) - session.catalog.namespaces should equal(Set(SessionGraphDataSource.Namespace, namespace)) + session.catalog.namespaces should equal( + Set(SessionGraphDataSource.Namespace, namespace) + ) } private def createSession: CypherSession = new CypherSession { @@ -87,9 +101,16 @@ class CypherSessionTest extends ApiBaseTest { query: String, parameters: CypherMap, drivingTable: Option[CypherRecords], - queryCatalog: Map[QualifiedGraphName, PropertyGraph]): Result = ??? + queryCatalog: Map[QualifiedGraphName, PropertyGraph] + ): Result = ??? - override private[opencypher] def cypherOnGraph(graph: PropertyGraph, query: String, parameters: CypherMap, drivingTable: Option[CypherRecords], queryCatalog: Map[QualifiedGraphName, PropertyGraph]) = ??? + override private[opencypher] def cypherOnGraph( + graph: PropertyGraph, + query: String, + parameters: CypherMap, + drivingTable: Option[CypherRecords], + queryCatalog: Map[QualifiedGraphName, PropertyGraph] + ) = ??? override def generateQualifiedGraphName: QualifiedGraphName = ??? } diff --git a/okapi-api/src/test/scala/org/opencypher/okapi/api/graph/QualifiedGraphNameTest.scala b/okapi-api/src/test/scala/org/opencypher/okapi/api/graph/QualifiedGraphNameTest.scala index 1f7af37f21..a7c7dd69d0 100644 --- a/okapi-api/src/test/scala/org/opencypher/okapi/api/graph/QualifiedGraphNameTest.scala +++ b/okapi-api/src/test/scala/org/opencypher/okapi/api/graph/QualifiedGraphNameTest.scala @@ -31,20 +31,32 @@ import org.opencypher.okapi.impl.io.SessionGraphDataSource.{Namespace => Session class QualifiedGraphNameTest extends ApiBaseTest { - it("apply with string representation containing single namespace and single graph name") { + it( + "apply with string representation containing single namespace and single graph name" + ) { val string = "testNamespace.testGraphName" - QualifiedGraphName(string) should be(QualifiedGraphName(Namespace("testNamespace"), GraphName("testGraphName"))) + QualifiedGraphName(string) should be( + QualifiedGraphName(Namespace("testNamespace"), GraphName("testGraphName")) + ) } - it("apply with string representation containing single namespace and multiple graph name") { + it( + "apply with string representation containing single namespace and multiple graph name" + ) { val string = "testNamespace.test.Graph.Name" - QualifiedGraphName(string) should be(QualifiedGraphName(Namespace("testNamespace"), GraphName("test.Graph.Name"))) + QualifiedGraphName(string) should be( + QualifiedGraphName( + Namespace("testNamespace"), + GraphName("test.Graph.Name") + ) + ) } it("apply with string representation containing single grap name") { val string = "testGraphName" - QualifiedGraphName(string) should be(QualifiedGraphName(SessionNamespace, GraphName("testGraphName"))) + QualifiedGraphName(string) should be( + QualifiedGraphName(SessionNamespace, GraphName("testGraphName")) + ) } - } diff --git a/okapi-api/src/test/scala/org/opencypher/okapi/api/io/conversion/PatternElementMappingTest.scala b/okapi-api/src/test/scala/org/opencypher/okapi/api/io/conversion/PatternElementMappingTest.scala index f58e298b9f..78e6bcb871 100644 --- a/okapi-api/src/test/scala/org/opencypher/okapi/api/io/conversion/PatternElementMappingTest.scala +++ b/okapi-api/src/test/scala/org/opencypher/okapi/api/io/conversion/PatternElementMappingTest.scala @@ -34,11 +34,12 @@ import org.opencypher.okapi.impl.exception.IllegalArgumentException class PatternElementMappingTest extends ApiBaseTest { describe("NodeMappingBuilder") { it("Construct node mapping") { - val given = NodeMappingBuilder.on("id") + val given = NodeMappingBuilder + .on("id") .withImpliedLabel("Person") .withPropertyKey("name") - .withPropertyKey("age" -> "YEARS").build - + .withPropertyKey("age" -> "YEARS") + .build val pattern = NodePattern(CTNode("Person")) val expected = ElementMapping( @@ -55,18 +56,26 @@ class PatternElementMappingTest extends ApiBaseTest { } it("Refuses to overwrite a property with a different mapping") { - raisesIllegalArgument(NodeMappingBuilder.on("sourceKey").withPropertyKey("a" -> "foo").withPropertyKey("a" -> "bar").build) + raisesIllegalArgument( + NodeMappingBuilder + .on("sourceKey") + .withPropertyKey("a" -> "foo") + .withPropertyKey("a" -> "bar") + .build + ) } } describe("RelationshipMappingBuilder") { it("Construct relationship mapping with static type") { - val given = RelationshipMappingBuilder.on("r") + val given = RelationshipMappingBuilder + .on("r") .from("src") .to("dst") .relType("KNOWS") .withPropertyKey("name") - .withPropertyKey("age" -> "YEARS").build + .withPropertyKey("age" -> "YEARS") + .build val pattern = RelationshipPattern(CTRelationship("KNOWS")) val actual = ElementMapping( @@ -75,7 +84,11 @@ class PatternElementMappingTest extends ApiBaseTest { pattern.relElement -> Map("name" -> "name", "age" -> "YEARS") ), Map( - pattern.relElement -> Map(SourceIdKey -> "r", SourceStartNodeKey -> "src", SourceEndNodeKey -> "dst") + pattern.relElement -> Map( + SourceIdKey -> "r", + SourceStartNodeKey -> "src", + SourceEndNodeKey -> "dst" + ) ) ) @@ -89,19 +102,38 @@ class PatternElementMappingTest extends ApiBaseTest { .from("a") .to("b") .relType("KNOWS") - .withPropertyKey("a" -> "foo").withPropertyKey("a" -> "bar") + .withPropertyKey("a" -> "foo") + .withPropertyKey("a" -> "bar") .build ) } - it("Refuses to use the same source key for incompatible types when constructing relationships") { - raisesIllegalArgument(RelationshipMappingBuilder.on("r").from("r").to("b").relType("KNOWS").build) - raisesIllegalArgument(RelationshipMappingBuilder.on("r").from("a").to("r").relType("KNOWS").build) + it( + "Refuses to use the same source key for incompatible types when constructing relationships" + ) { + raisesIllegalArgument( + RelationshipMappingBuilder + .on("r") + .from("r") + .to("b") + .relType("KNOWS") + .build + ) + raisesIllegalArgument( + RelationshipMappingBuilder + .on("r") + .from("a") + .to("r") + .relType("KNOWS") + .build + ) } } describe("validation") { - it("throws an error if relationship elements do not have exactly one type") { + it( + "throws an error if relationship elements do not have exactly one type" + ) { val pattern1 = RelationshipPattern(CTRelationship("Foo", "Bar")) raisesIllegalArgument(ElementMapping.empty(pattern1)) diff --git a/okapi-api/src/test/scala/org/opencypher/okapi/api/schema/LabelCombinationsTest.scala b/okapi-api/src/test/scala/org/opencypher/okapi/api/schema/LabelCombinationsTest.scala index 759191076e..f05a2d4a7b 100644 --- a/okapi-api/src/test/scala/org/opencypher/okapi/api/schema/LabelCombinationsTest.scala +++ b/okapi-api/src/test/scala/org/opencypher/okapi/api/schema/LabelCombinationsTest.scala @@ -32,23 +32,40 @@ import org.opencypher.okapi.impl.schema.LabelCombinations class LabelCombinationsTest extends ApiBaseTest { it("combinationsFor") { - val in = LabelCombinations(Set( - Set("A"), Set("A", "B", "X"), Set("A", "X"), Set("B") - )) + val in = LabelCombinations( + Set( + Set("A"), + Set("A", "B", "X"), + Set("A", "X"), + Set("B") + ) + ) in.combinationsFor(Set.empty) should equal(in.combos) - in.combinationsFor(Set("A")) should equal(Set( - Set("A"), Set("A", "B", "X"), Set("A", "X") - )) - in.combinationsFor(Set("B")) should equal(Set( - Set("B"), Set("A", "B", "X") - )) - in.combinationsFor(Set("A", "X")) should equal(Set( - Set("A", "B", "X"), Set("A", "X") - )) - in.combinationsFor(Set("A", "B")) should equal(Set( - Set("A", "B", "X") - )) + in.combinationsFor(Set("A")) should equal( + Set( + Set("A"), + Set("A", "B", "X"), + Set("A", "X") + ) + ) + in.combinationsFor(Set("B")) should equal( + Set( + Set("B"), + Set("A", "B", "X") + ) + ) + in.combinationsFor(Set("A", "X")) should equal( + Set( + Set("A", "B", "X"), + Set("A", "X") + ) + ) + in.combinationsFor(Set("A", "B")) should equal( + Set( + Set("A", "B", "X") + ) + ) in.combinationsFor(Set("A", "C")) shouldBe empty } diff --git a/okapi-api/src/test/scala/org/opencypher/okapi/api/schema/LabelPropertyMapTest.scala b/okapi-api/src/test/scala/org/opencypher/okapi/api/schema/LabelPropertyMapTest.scala index e3450c6ba8..0788d218ad 100644 --- a/okapi-api/src/test/scala/org/opencypher/okapi/api/schema/LabelPropertyMapTest.scala +++ b/okapi-api/src/test/scala/org/opencypher/okapi/api/schema/LabelPropertyMapTest.scala @@ -37,7 +37,11 @@ class LabelPropertyMapTest extends ApiBaseTest { it("|+|") { val map1 = LabelPropertyMap.empty - .register("A")("name" -> CTString, "age" -> CTInteger, "gender" -> CTString) + .register("A")( + "name" -> CTString, + "age" -> CTInteger, + "gender" -> CTString + ) .register("B")("p" -> CTBoolean) val map2 = LabelPropertyMap.empty @@ -46,7 +50,11 @@ class LabelPropertyMapTest extends ApiBaseTest { map1 |+| map2 should equal( LabelPropertyMap.empty - .register("A")("name" -> CTString, "age" -> CTInteger, "gender" -> CTUnion(CTString, CTTrue, CTFalse)) + .register("A")( + "name" -> CTString, + "age" -> CTInteger, + "gender" -> CTUnion(CTString, CTTrue, CTFalse) + ) .register("B")("p" -> CTBoolean) .register("C")("name" -> CTString) ) @@ -59,12 +67,14 @@ class LabelPropertyMapTest extends ApiBaseTest { .register("B", "A")("foo" -> CTInteger) .register("C")("bar" -> CTInteger) - map.filterForLabels("A") should equal(LabelPropertyMap.empty - .register("A")("name" -> CTString) - .register("A", "B")("foo" -> CTInteger) + map.filterForLabels("A") should equal( + LabelPropertyMap.empty + .register("A")("name" -> CTString) + .register("A", "B")("foo" -> CTInteger) ) - map.filterForLabels("C") should equal(LabelPropertyMap.empty - .register("C")("bar" -> CTInteger) + map.filterForLabels("C") should equal( + LabelPropertyMap.empty + .register("C")("bar" -> CTInteger) ) map.filterForLabels("X") should equal(LabelPropertyMap.empty) map.filterForLabels("A", "B", "C") should equal(map) diff --git a/okapi-api/src/test/scala/org/opencypher/okapi/api/schema/PropertyGraphSchemaTest.scala b/okapi-api/src/test/scala/org/opencypher/okapi/api/schema/PropertyGraphSchemaTest.scala index c5ec783cf8..d483845a64 100644 --- a/okapi-api/src/test/scala/org/opencypher/okapi/api/schema/PropertyGraphSchemaTest.scala +++ b/okapi-api/src/test/scala/org/opencypher/okapi/api/schema/PropertyGraphSchemaTest.scala @@ -35,15 +35,20 @@ import org.opencypher.okapi.impl.util.Version class PropertyGraphSchemaTest extends ApiBaseTest { it("lists of void and others") { - val s1 = PropertyGraphSchema.empty.withNodePropertyKeys("A")("v" -> CTEmptyList) - val s2 = PropertyGraphSchema.empty.withNodePropertyKeys("A")("v" -> CTList(CTString).nullable) + val s1 = + PropertyGraphSchema.empty.withNodePropertyKeys("A")("v" -> CTEmptyList) + val s2 = PropertyGraphSchema.empty.withNodePropertyKeys("A")( + "v" -> CTList(CTString).nullable + ) val joined = s1 ++ s2 joined should equal(s2) } it("should provide all labels") { - PropertyGraphSchema.empty.withNodePropertyKeys("Person")().labels should equal(Set("Person")) + PropertyGraphSchema.empty + .withNodePropertyKeys("Person")() + .labels should equal(Set("Person")) } it("should provide all types") { @@ -54,32 +59,48 @@ class PropertyGraphSchemaTest extends ApiBaseTest { } it("should give correct node property schema") { - val schema = PropertyGraphSchema.empty.withNodePropertyKeys("Person")("name" -> CTString, "age" -> CTInteger) + val schema = PropertyGraphSchema.empty + .withNodePropertyKeys("Person")("name" -> CTString, "age" -> CTInteger) schema.nodePropertyKeys(Set("NotPerson")) shouldBe empty - schema.nodePropertyKeys(Set("Person")) should equal(Map("name" -> CTString, "age" -> CTInteger)) + schema.nodePropertyKeys(Set("Person")) should equal( + Map("name" -> CTString, "age" -> CTInteger) + ) schema.labels should equal(Set("Person")) } it("should give correct relationship property schema") { - val schema = PropertyGraphSchema.empty.withRelationshipPropertyKeys("KNOWS")("since" -> CTInteger, "relative" -> CTBoolean) + val schema = PropertyGraphSchema.empty.withRelationshipPropertyKeys( + "KNOWS" + )("since" -> CTInteger, "relative" -> CTBoolean) schema.relationshipPropertyKeys("NOT_KNOWS") shouldBe empty - schema.relationshipPropertyKeys("KNOWS") should equal(Map("since" -> CTInteger, "relative" -> CTBoolean)) + schema.relationshipPropertyKeys("KNOWS") should equal( + Map("since" -> CTInteger, "relative" -> CTBoolean) + ) schema.relationshipTypes should equal(Set("KNOWS")) } it("should get simple implication correct") { val schema = PropertyGraphSchema.empty .withNodePropertyKeys("Foo", "Bar")("prop" -> CTBoolean) - .withNodePropertyKeys("Person", "Employee")("name" -> CTString, "nbr" -> CTInteger) + .withNodePropertyKeys("Person", "Employee")( + "name" -> CTString, + "nbr" -> CTInteger + ) .withNodePropertyKeys("Person")("name" -> CTString) - .withNodePropertyKeys("Person", "Dog")("name" -> CTString, "reg" -> CTFloat) + .withNodePropertyKeys("Person", "Dog")( + "name" -> CTString, + "reg" -> CTFloat + ) .withNodePropertyKeys("Dog")("reg" -> CTFloat) schema.impliedLabels("Person") shouldBe Set("Person") schema.impliedLabels("Employee") shouldBe Set("Person", "Employee") - schema.impliedLabels("Employee", "Person") shouldBe Set("Person", "Employee") + schema.impliedLabels("Employee", "Person") shouldBe Set( + "Person", + "Employee" + ) schema.impliedLabels("Foo") shouldBe Set("Foo", "Bar") schema.impliedLabels("Bar") shouldBe Set("Foo", "Bar") schema.impliedLabels("Dog") shouldBe Set("Dog") @@ -94,14 +115,37 @@ class PropertyGraphSchemaTest extends ApiBaseTest { .withNodePropertyKeys("Someone")() schema.impliedLabels(Set("Unknown")) shouldBe empty - schema.impliedLabels(Set("Unknown", "Person")) shouldBe Set("Person", "Human", "Someone") + schema.impliedLabels(Set("Unknown", "Person")) shouldBe Set( + "Person", + "Human", + "Someone" + ) schema.impliedLabels(Set("Human")) shouldBe Set("Human") schema.impliedLabels(Set("Someone")) shouldBe Set("Someone") - schema.impliedLabels(Set("Person")) shouldBe Set("Person", "Human", "Someone") - schema.impliedLabels(Set("Person", "Human")) shouldBe Set("Person", "Human", "Someone") - schema.impliedLabels(Set("Person", "Someone")) shouldBe Set("Person", "Human", "Someone") - schema.impliedLabels(Set("Employee")) shouldBe Set("Employee", "Person", "Human", "Someone") - schema.impliedLabels(Set("Employee", "Person")) shouldBe Set("Employee", "Person", "Human", "Someone") + schema + .impliedLabels(Set("Person")) shouldBe Set("Person", "Human", "Someone") + schema.impliedLabels(Set("Person", "Human")) shouldBe Set( + "Person", + "Human", + "Someone" + ) + schema.impliedLabels(Set("Person", "Someone")) shouldBe Set( + "Person", + "Human", + "Someone" + ) + schema.impliedLabels(Set("Employee")) shouldBe Set( + "Employee", + "Person", + "Human", + "Someone" + ) + schema.impliedLabels(Set("Employee", "Person")) shouldBe Set( + "Employee", + "Person", + "Human", + "Someone" + ) schema.labels should equal(Set("Person", "Employee", "Human", "Someone")) } @@ -111,10 +155,18 @@ class PropertyGraphSchemaTest extends ApiBaseTest { .withNodePropertyKeys("Person", "Director")() .withNodePropertyKeys("Employee", "Director")() - schema.combinationsFor(Set("Employee")) should equal(Set(Set("Person", "Employee"), Set("Employee", "Director"))) - schema.combinationsFor(Set("Director")) should equal(Set(Set("Person", "Director"), Set("Employee", "Director"))) - schema.combinationsFor(Set("Person")) should equal(Set(Set("Person", "Employee"), Set("Person", "Director"))) - schema.combinationsFor(Set("Person", "Employee")) should equal(Set(Set("Person", "Employee"))) + schema.combinationsFor(Set("Employee")) should equal( + Set(Set("Person", "Employee"), Set("Employee", "Director")) + ) + schema.combinationsFor(Set("Director")) should equal( + Set(Set("Person", "Director"), Set("Employee", "Director")) + ) + schema.combinationsFor(Set("Person")) should equal( + Set(Set("Person", "Employee"), Set("Person", "Director")) + ) + schema.combinationsFor(Set("Person", "Employee")) should equal( + Set(Set("Person", "Employee")) + ) schema.labels should equal(Set("Person", "Employee", "Director")) } @@ -124,8 +176,12 @@ class PropertyGraphSchemaTest extends ApiBaseTest { .withNodePropertyKeys("Dog", "Pet")() schema.combinationsFor(Set("NotEmployee")) should equal(Set()) - schema.combinationsFor(Set("Employee")) should equal(Set(Set("Person", "Employee"))) - schema.combinationsFor(Set("Person")) should equal(Set(Set("Person", "Employee"))) + schema.combinationsFor(Set("Employee")) should equal( + Set(Set("Person", "Employee")) + ) + schema.combinationsFor(Set("Person")) should equal( + Set(Set("Person", "Employee")) + ) schema.combinationsFor(Set("Dog")) should equal(Set(Set("Dog", "Pet"))) schema.combinationsFor(Set("Pet", "Employee")) should equal(Set()) schema.labels should equal(Set("Person", "Employee", "Dog", "Pet")) @@ -138,13 +194,19 @@ class PropertyGraphSchemaTest extends ApiBaseTest { .withRelationshipPropertyKeys("BAR")("p1" -> CTBoolean) .withRelationshipPropertyKeys("BAR")("p2" -> CTFloat) - schema.nodePropertyKeys(Set("Foo")) should equal(Map("name" -> CTString, "age" -> CTInteger.nullable)) - schema.relationshipPropertyKeys("BAR") should equal(Map("p1" -> CTBoolean.nullable, "p2" -> CTFloat.nullable)) + schema.nodePropertyKeys(Set("Foo")) should equal( + Map("name" -> CTString, "age" -> CTInteger.nullable) + ) + schema.relationshipPropertyKeys("BAR") should equal( + Map("p1" -> CTBoolean.nullable, "p2" -> CTFloat.nullable) + ) } it("combining schemas, separate keys") { - val schema1 = PropertyGraphSchema.empty.withNodePropertyKeys("A")("foo" -> CTString) - val schema2 = PropertyGraphSchema.empty.withNodePropertyKeys("B")("bar" -> CTString) + val schema1 = + PropertyGraphSchema.empty.withNodePropertyKeys("A")("foo" -> CTString) + val schema2 = + PropertyGraphSchema.empty.withNodePropertyKeys("B")("bar" -> CTString) val schema3 = PropertyGraphSchema.empty .withNodePropertyKeys("C")("baz" -> CTString) .withNodePropertyKeys("A", "C")("baz" -> CTString) @@ -156,18 +218,28 @@ class PropertyGraphSchemaTest extends ApiBaseTest { .withNodePropertyKeys("B")("bar" -> CTString) .withNodePropertyKeys("C")("baz" -> CTString) .withNodePropertyKeys("A", "C")("baz" -> CTString) - .withNodePropertyKeys("A", "C", "X")("baz" -> CTString)) + .withNodePropertyKeys("A", "C", "X")("baz" -> CTString) + ) } it("combining schemas, key subset") { val schema1 = PropertyGraphSchema.empty .withNodePropertyKeys("A")("foo" -> CTString, "bar" -> CTString) val schema2 = PropertyGraphSchema.empty - .withNodePropertyKeys("A")("foo" -> CTString, "bar" -> CTString, "baz" -> CTString) + .withNodePropertyKeys("A")( + "foo" -> CTString, + "bar" -> CTString, + "baz" -> CTString + ) schema1 ++ schema2 should equal( PropertyGraphSchema.empty - .withNodePropertyKeys("A")("foo" -> CTString, "bar" -> CTString, "baz" -> CTString.nullable)) + .withNodePropertyKeys("A")( + "foo" -> CTString, + "bar" -> CTString, + "baz" -> CTString.nullable + ) + ) } it("combining schemas, partial key overlap") { @@ -178,7 +250,12 @@ class PropertyGraphSchemaTest extends ApiBaseTest { schema1 ++ schema2 should equal( PropertyGraphSchema.empty - .withNodePropertyKeys("A")("foo" -> CTString, "bar" -> CTString.nullable, "baz" -> CTString.nullable)) + .withNodePropertyKeys("A")( + "foo" -> CTString, + "bar" -> CTString.nullable, + "baz" -> CTString.nullable + ) + ) } it("combining type conflicting schemas should work across nullability") { @@ -189,7 +266,11 @@ class PropertyGraphSchemaTest extends ApiBaseTest { schema1 ++ schema2 should equal( PropertyGraphSchema.empty - .withNodePropertyKeys("A")("foo" -> CTString.nullable, "bar" -> CTString.nullable)) + .withNodePropertyKeys("A")( + "foo" -> CTString.nullable, + "bar" -> CTString.nullable + ) + ) } it("combining schemas with restricting label implications") { @@ -209,13 +290,17 @@ class PropertyGraphSchemaTest extends ApiBaseTest { .withNodePropertyKeys("A", "E", "B", "C")() .withNodePropertyKeys("B", "C", "D")() .withNodePropertyKeys("C", "D")() - .withNodePropertyKeys("B", "F", "C", "D")()) + .withNodePropertyKeys("B", "F", "C", "D")() + ) } it("extract node schema") { val schema = PropertyGraphSchema.empty .withNodePropertyKeys("Person")("name" -> CTString) - .withNodePropertyKeys("Employee", "Person")("name" -> CTString, "salary" -> CTInteger) + .withNodePropertyKeys("Employee", "Person")( + "name" -> CTString, + "salary" -> CTInteger + ) .withNodePropertyKeys("Dog", "Pet")("name" -> CTFloat) .withNodePropertyKeys("Pet")("notName" -> CTBoolean) .withRelationshipPropertyKeys("OWNER")("since" -> CTInteger) @@ -223,7 +308,10 @@ class PropertyGraphSchemaTest extends ApiBaseTest { schema.forNode(Set("Person")) should equal( PropertyGraphSchema.empty .withNodePropertyKeys("Person")("name" -> CTString) - .withNodePropertyKeys("Employee", "Person")("name" -> CTString, "salary" -> CTInteger) + .withNodePropertyKeys("Employee", "Person")( + "name" -> CTString, + "salary" -> CTInteger + ) ) schema.forNode(Set("Dog")) should equal( @@ -243,7 +331,10 @@ class PropertyGraphSchemaTest extends ApiBaseTest { .withNodePropertyKeys("Person", "Employee")("name" -> CTString) .withNodePropertyKeys("Employee")("name" -> CTString) .withRelationshipPropertyKeys("KNOWS")("name" -> CTString) - .withRelationshipPropertyKeys("LOVES")("deeply" -> CTBoolean, "salary" -> CTInteger) + .withRelationshipPropertyKeys("LOVES")( + "deeply" -> CTBoolean, + "salary" -> CTInteger + ) .withRelationshipPropertyKeys("NEEDS")("rating" -> CTFloat) .withNodePropertyKeys("Dog", "Pet")() .withNodePropertyKeys("Pet")() @@ -257,7 +348,10 @@ class PropertyGraphSchemaTest extends ApiBaseTest { schema.forRelationship(CTRelationship) should equal( PropertyGraphSchema.empty .withRelationshipPropertyKeys("KNOWS")("name" -> CTString) - .withRelationshipPropertyKeys("LOVES")("deeply" -> CTBoolean, "salary" -> CTInteger) + .withRelationshipPropertyKeys("LOVES")( + "deeply" -> CTBoolean, + "salary" -> CTInteger + ) .withRelationshipPropertyKeys("NEEDS")("rating" -> CTFloat) .withRelationshipPropertyKeys("OWNER")("since" -> CTInteger) ) @@ -265,7 +359,10 @@ class PropertyGraphSchemaTest extends ApiBaseTest { schema.forRelationship(CTRelationship("KNOWS", "LOVES")) should equal( PropertyGraphSchema.empty .withRelationshipPropertyKeys("KNOWS")("name" -> CTString) - .withRelationshipPropertyKeys("LOVES")("deeply" -> CTBoolean, "salary" -> CTInteger) + .withRelationshipPropertyKeys("LOVES")( + "deeply" -> CTBoolean, + "salary" -> CTInteger + ) ) } @@ -275,24 +372,43 @@ class PropertyGraphSchemaTest extends ApiBaseTest { .withNodePropertyKeys("A")("name" -> CTInteger) schema.nodePropertyKeys(Set.empty) should equal(Map("name" -> CTString)) - schema.nodePropertyKeys(Set.empty[String]) should equal(Map("name" -> CTString)) + schema.nodePropertyKeys(Set.empty[String]) should equal( + Map("name" -> CTString) + ) } it("get node key type with all given semantics") { val schema = PropertyGraphSchema.empty - .withNodePropertyKeys(Set("A"), Map("a" -> CTInteger, "b" -> CTString, "c" -> CTFloat, "d" -> CTFloat.nullable)) + .withNodePropertyKeys( + Set("A"), + Map( + "a" -> CTInteger, + "b" -> CTString, + "c" -> CTFloat, + "d" -> CTFloat.nullable + ) + ) .withNodePropertyKeys(Set.empty[String], Map("a" -> CTString)) schema.nodePropertyKeyType(Set("A"), "a") should equal(Some(CTInteger)) - schema.nodePropertyKeyType(Set.empty[String], "a") should equal(Some(CTUnion(CTString, CTInteger))) - schema.nodePropertyKeyType(Set.empty[String], "b") should equal(Some(CTString.nullable)) + schema.nodePropertyKeyType(Set.empty[String], "a") should equal( + Some(CTUnion(CTString, CTInteger)) + ) + schema.nodePropertyKeyType(Set.empty[String], "b") should equal( + Some(CTString.nullable) + ) schema.nodePropertyKeyType(Set("B"), "b") should equal(None) schema.nodePropertyKeyType(Set("A"), "x") should equal(None) } it("get rel key type") { val schema = PropertyGraphSchema.empty - .withRelationshipPropertyKeys("A")("a" -> CTInteger, "b" -> CTString, "c" -> CTFloat, "d" -> CTFloat.nullable) + .withRelationshipPropertyKeys("A")( + "a" -> CTInteger, + "b" -> CTString, + "c" -> CTFloat, + "d" -> CTFloat.nullable + ) .withRelationshipPropertyKeys("B")( "a" -> CTFloat, "b" -> CTString.nullable, @@ -300,20 +416,48 @@ class PropertyGraphSchemaTest extends ApiBaseTest { ) .withRelationshipType("C") - schema.relationshipPropertyKeyType(Set("A"), "a") should equal(Some(CTInteger)) - schema.relationshipPropertyKeyType(Set("A", "B"), "a") should equal(Some(CTUnion(CTInteger, CTFloat))) - schema.relationshipPropertyKeyType(Set("A", "B"), "b") should equal(Some(CTString.nullable)) - schema.relationshipPropertyKeyType(Set("A", "B", "C"), "c") should equal(Some(CTUnion(CTFloat, CTString).nullable)) + schema.relationshipPropertyKeyType(Set("A"), "a") should equal( + Some(CTInteger) + ) + schema.relationshipPropertyKeyType(Set("A", "B"), "a") should equal( + Some(CTUnion(CTInteger, CTFloat)) + ) + schema.relationshipPropertyKeyType(Set("A", "B"), "b") should equal( + Some(CTString.nullable) + ) + schema.relationshipPropertyKeyType(Set("A", "B", "C"), "c") should equal( + Some(CTUnion(CTFloat, CTString).nullable) + ) schema.relationshipPropertyKeyType(Set("A"), "e") should equal(None) - schema.relationshipPropertyKeyType(Set.empty, "a") should equal(Some(CTUnion(CTInteger, CTFloat).nullable)) + schema.relationshipPropertyKeyType(Set.empty, "a") should equal( + Some(CTUnion(CTInteger, CTFloat).nullable) + ) } it("get all keys") { val schema = PropertyGraphSchema.empty - .withNodePropertyKeys(Set.empty[String], Map("a" -> CTString, "c" -> CTString, "d" -> CTString.nullable, "f" -> CTString)) - .withNodePropertyKeys("A")("b" -> CTInteger, "c" -> CTString, "e" -> CTString, "f" -> CTInteger) - .withNodePropertyKeys("B")("b" -> CTFloat, "c" -> CTString, "e" -> CTInteger, "f" -> CTBoolean) + .withNodePropertyKeys( + Set.empty[String], + Map( + "a" -> CTString, + "c" -> CTString, + "d" -> CTString.nullable, + "f" -> CTString + ) + ) + .withNodePropertyKeys("A")( + "b" -> CTInteger, + "c" -> CTString, + "e" -> CTString, + "f" -> CTInteger + ) + .withNodePropertyKeys("B")( + "b" -> CTFloat, + "c" -> CTString, + "e" -> CTInteger, + "f" -> CTBoolean + ) allNodePropertyKeys(schema) should equal( Map( @@ -322,31 +466,102 @@ class PropertyGraphSchemaTest extends ApiBaseTest { "c" -> CTString, "d" -> CTString.nullable, "e" -> CTUnion(CTString, CTInteger).nullable, - "f" -> CTUnion(CTString, CTInteger, CTTrue, CTFalse))) + "f" -> CTUnion(CTString, CTInteger, CTTrue, CTFalse) + ) + ) } it("get keys for") { val schema = PropertyGraphSchema.empty - .withNodePropertyKeys(Set.empty[String], Map("a" -> CTString, "c" -> CTString, "d" -> CTString.nullable, "f" -> CTString)) - .withNodePropertyKeys("A")("b" -> CTInteger, "c" -> CTString, "e" -> CTString, "f" -> CTInteger) - .withNodePropertyKeys("B")("b" -> CTFloat, "c" -> CTString, "e" -> CTInteger) + .withNodePropertyKeys( + Set.empty[String], + Map( + "a" -> CTString, + "c" -> CTString, + "d" -> CTString.nullable, + "f" -> CTString + ) + ) + .withNodePropertyKeys("A")( + "b" -> CTInteger, + "c" -> CTString, + "e" -> CTString, + "f" -> CTInteger + ) + .withNodePropertyKeys("B")( + "b" -> CTFloat, + "c" -> CTString, + "e" -> CTInteger + ) - schema.nodePropertyKeysForCombinations(Set(Set("A"))) should equal(Map("b" -> CTInteger, "c" -> CTString, "e" -> CTString, "f" -> CTInteger)) - schema.nodePropertyKeysForCombinations(Set(Set("B"))) should equal(Map("b" -> CTFloat, "c" -> CTString, "e" -> CTInteger)) - schema.nodePropertyKeysForCombinations(Set(Set("A"), Set("B"))) should equal(Map("b" -> CTUnion(CTInteger, CTFloat), "c" -> CTString, "e" -> CTUnion(CTString, CTInteger), "f" -> CTInteger.nullable)) + schema.nodePropertyKeysForCombinations(Set(Set("A"))) should equal( + Map("b" -> CTInteger, "c" -> CTString, "e" -> CTString, "f" -> CTInteger) + ) + schema.nodePropertyKeysForCombinations(Set(Set("B"))) should equal( + Map("b" -> CTFloat, "c" -> CTString, "e" -> CTInteger) + ) + schema.nodePropertyKeysForCombinations( + Set(Set("A"), Set("B")) + ) should equal( + Map( + "b" -> CTUnion(CTInteger, CTFloat), + "c" -> CTString, + "e" -> CTUnion(CTString, CTInteger), + "f" -> CTInteger.nullable + ) + ) } it("get keys for label combinations") { val schema = PropertyGraphSchema.empty - .withNodePropertyKeys(Set.empty[String], Map("a" -> CTString, "c" -> CTString, "d" -> CTString.nullable, "f" -> CTString)) - .withNodePropertyKeys("A")("b" -> CTInteger, "c" -> CTString, "e" -> CTString, "f" -> CTInteger) - .withNodePropertyKeys("B")("b" -> CTFloat, "c" -> CTString, "e" -> CTInteger) + .withNodePropertyKeys( + Set.empty[String], + Map( + "a" -> CTString, + "c" -> CTString, + "d" -> CTString.nullable, + "f" -> CTString + ) + ) + .withNodePropertyKeys("A")( + "b" -> CTInteger, + "c" -> CTString, + "e" -> CTString, + "f" -> CTInteger + ) + .withNodePropertyKeys("B")( + "b" -> CTFloat, + "c" -> CTString, + "e" -> CTInteger + ) - schema.nodePropertyKeysForCombinations(Set(Set("A"))) should equal(Map("b" -> CTInteger, "c" -> CTString, "e" -> CTString, "f" -> CTInteger)) - schema.nodePropertyKeysForCombinations(Set(Set("B"))) should equal(Map("b" -> CTFloat, "c" -> CTString, "e" -> CTInteger)) - schema.nodePropertyKeysForCombinations(Set(Set("A"), Set("B"))) should equal(Map("b" -> CTUnion(CTInteger, CTFloat), "c" -> CTString, "e" -> CTUnion(CTString, CTInteger), "f" -> CTInteger.nullable)) - schema.nodePropertyKeysForCombinations(Set(Set("A", "B"))) should equal(Map.empty) - schema.nodePropertyKeysForCombinations(Set(Set.empty[String])) should equal(Map("a" -> CTString, "c" -> CTString, "d" -> CTString.nullable, "f" -> CTString)) + schema.nodePropertyKeysForCombinations(Set(Set("A"))) should equal( + Map("b" -> CTInteger, "c" -> CTString, "e" -> CTString, "f" -> CTInteger) + ) + schema.nodePropertyKeysForCombinations(Set(Set("B"))) should equal( + Map("b" -> CTFloat, "c" -> CTString, "e" -> CTInteger) + ) + schema.nodePropertyKeysForCombinations( + Set(Set("A"), Set("B")) + ) should equal( + Map( + "b" -> CTUnion(CTInteger, CTFloat), + "c" -> CTString, + "e" -> CTUnion(CTString, CTInteger), + "f" -> CTInteger.nullable + ) + ) + schema.nodePropertyKeysForCombinations(Set(Set("A", "B"))) should equal( + Map.empty + ) + schema.nodePropertyKeysForCombinations(Set(Set.empty[String])) should equal( + Map( + "a" -> CTString, + "c" -> CTString, + "d" -> CTString.nullable, + "f" -> CTString + ) + ) } it("isEmpty") { @@ -356,15 +571,25 @@ class PropertyGraphSchemaTest extends ApiBaseTest { empty.isEmpty shouldBe true (empty ++ PropertyGraphSchema.empty).isEmpty shouldBe true - PropertyGraphSchema.empty.withNodePropertyKeys("label")().isEmpty shouldBe false - PropertyGraphSchema.empty.withRelationshipPropertyKeys("type")("name" -> CTFloat).isEmpty shouldBe false + PropertyGraphSchema.empty + .withNodePropertyKeys("label")() + .isEmpty shouldBe false + PropertyGraphSchema.empty + .withRelationshipPropertyKeys("type")("name" -> CTFloat) + .isEmpty shouldBe false } it("should serialize and deserialize a schema") { val schema = PropertyGraphSchema.empty - .withNodePropertyKeys(Set("A"), PropertyKeys("foo" -> CTString, "bar" -> CTList(CTString.nullable))) - .withNodePropertyKeys(Set("A", "B"), PropertyKeys("foo" -> CTString, "bar" -> CTInteger)) + .withNodePropertyKeys( + Set("A"), + PropertyKeys("foo" -> CTString, "bar" -> CTList(CTString.nullable)) + ) + .withNodePropertyKeys( + Set("A", "B"), + PropertyKeys("foo" -> CTString, "bar" -> CTInteger) + ) .withRelationshipPropertyKeys("FOO", PropertyKeys.empty) val serialized = schema.toJson @@ -373,7 +598,9 @@ class PropertyGraphSchemaTest extends ApiBaseTest { } - it("concatenating schemas should make missing relationship properties nullable") { + it( + "concatenating schemas should make missing relationship properties nullable" + ) { val schema1 = PropertyGraphSchema.empty .withRelationshipPropertyKeys("FOO")() @@ -434,8 +661,7 @@ class PropertyGraphSchemaTest extends ApiBaseTest { val serialized = schema.toJson - serialized should equal( - """|{ + serialized should equal("""|{ | "version": "1.0", | "labelPropertyMap": [ | { @@ -472,8 +698,7 @@ class PropertyGraphSchemaTest extends ApiBaseTest { val serialized = schema.toJson - serialized should equal( - """|{ + serialized should equal("""|{ | "version": "1.0", | "labelPropertyMap": [ | { @@ -543,12 +768,14 @@ class PropertyGraphSchemaTest extends ApiBaseTest { .withNodePropertyKeys("B")() .withRelationshipPropertyKeys("REL")() - schema.schemaPatterns should equal(Set( - SchemaPattern(Set("A"), "REL", Set("A")), - SchemaPattern(Set("A"), "REL", Set("B")), - SchemaPattern(Set("B"), "REL", Set("A")), - SchemaPattern(Set("B"), "REL", Set("B")) - )) + schema.schemaPatterns should equal( + Set( + SchemaPattern(Set("A"), "REL", Set("A")), + SchemaPattern(Set("A"), "REL", Set("B")), + SchemaPattern(Set("B"), "REL", Set("A")), + SchemaPattern(Set("B"), "REL", Set("B")) + ) + ) } it("returns the explicit patterns if any were given") { @@ -558,18 +785,26 @@ class PropertyGraphSchemaTest extends ApiBaseTest { .withRelationshipPropertyKeys("REL")() .withSchemaPatterns(SchemaPattern("A", "REL", "B")) - schema.schemaPatterns should equal(Set( - SchemaPattern("A", "REL", "B") - )) + schema.schemaPatterns should equal( + Set( + SchemaPattern("A", "REL", "B") + ) + ) } - it("throws a SchemaException when adding a schema pattern into an empty schema") { + it( + "throws a SchemaException when adding a schema pattern into an empty schema" + ) { a[SchemaException] should be thrownBy { - PropertyGraphSchema.empty.withSchemaPatterns(SchemaPattern("A", "REL", "B")) + PropertyGraphSchema.empty.withSchemaPatterns( + SchemaPattern("A", "REL", "B") + ) } } - it("throws a SchemaException when adding a schema pattern for unknown start node labels") { + it( + "throws a SchemaException when adding a schema pattern for unknown start node labels" + ) { a[SchemaException] should be thrownBy { PropertyGraphSchema.empty .withNodePropertyKeys("A")() @@ -578,7 +813,9 @@ class PropertyGraphSchemaTest extends ApiBaseTest { } } - it("throws a SchemaException when adding a schema pattern for unknown end node labels") { + it( + "throws a SchemaException when adding a schema pattern for unknown end node labels" + ) { a[SchemaException] should be thrownBy { PropertyGraphSchema.empty .withNodePropertyKeys("B")() @@ -587,7 +824,9 @@ class PropertyGraphSchemaTest extends ApiBaseTest { } } - it("throws a SchemaException when adding a schema pattern for unknown rel type") { + it( + "throws a SchemaException when adding a schema pattern for unknown rel type" + ) { a[SchemaException] should be thrownBy { PropertyGraphSchema.empty .withNodePropertyKeys("A")() @@ -602,7 +841,8 @@ class PropertyGraphSchemaTest extends ApiBaseTest { val aRel2CD = SchemaPattern(Set("A"), "REL2", Set("C", "D")) val bRel2CD = SchemaPattern(Set("B"), "REL2", Set("C", "D")) val CDRel1A = SchemaPattern(Set("C", "D"), "REL1", Set("A")) - val emptyRel1Empty = SchemaPattern(Set.empty[String], "REL1", Set.empty[String]) + val emptyRel1Empty = + SchemaPattern(Set.empty[String], "REL1", Set.empty[String]) val schema = PropertyGraphSchema.empty .withNodePropertyKeys("A")() @@ -618,76 +858,108 @@ class PropertyGraphSchemaTest extends ApiBaseTest { .withSchemaPatterns(emptyRel1Empty) it("works when nothing is known") { - schema.schemaPatternsFor(Set.empty, Set.empty, Set.empty) should equal(Set( - aRel1B, - aRel2CD, - bRel2CD, - CDRel1A, - emptyRel1Empty - )) + schema.schemaPatternsFor(Set.empty, Set.empty, Set.empty) should equal( + Set( + aRel1B, + aRel2CD, + bRel2CD, + CDRel1A, + emptyRel1Empty + ) + ) } it("works when only the source node label is known") { - schema.schemaPatternsFor(Set.empty, Set.empty, Set.empty) should equal(Set( - aRel1B, - aRel2CD, - bRel2CD, - CDRel1A, - emptyRel1Empty - )) - - schema.schemaPatternsFor(Set("A"), Set.empty, Set.empty) should equal(Set( - aRel1B, - aRel2CD - )) - - schema.schemaPatternsFor(Set("C"), Set.empty, Set.empty) should equal(Set( - CDRel1A - )) + schema.schemaPatternsFor(Set.empty, Set.empty, Set.empty) should equal( + Set( + aRel1B, + aRel2CD, + bRel2CD, + CDRel1A, + emptyRel1Empty + ) + ) + + schema.schemaPatternsFor(Set("A"), Set.empty, Set.empty) should equal( + Set( + aRel1B, + aRel2CD + ) + ) + + schema.schemaPatternsFor(Set("C"), Set.empty, Set.empty) should equal( + Set( + CDRel1A + ) + ) } it("works when only the target node label is known") { - schema.schemaPatternsFor(Set.empty, Set.empty, Set("A")) should equal(Set( - CDRel1A - )) - - schema.schemaPatternsFor(Set.empty, Set.empty, Set("C")) should equal(Set( - aRel2CD, - bRel2CD - )) + schema.schemaPatternsFor(Set.empty, Set.empty, Set("A")) should equal( + Set( + CDRel1A + ) + ) + + schema.schemaPatternsFor(Set.empty, Set.empty, Set("C")) should equal( + Set( + aRel2CD, + bRel2CD + ) + ) } it("works when only the rel type is known") { - schema.schemaPatternsFor(Set.empty, Set("REL1"), Set.empty) should equal(Set( - aRel1B, - CDRel1A, - emptyRel1Empty - )) - - schema.schemaPatternsFor(Set.empty, Set("REL1", "REL2"), Set.empty) should equal(Set( - aRel1B, - aRel2CD, - bRel2CD, - CDRel1A, - emptyRel1Empty - )) + schema.schemaPatternsFor(Set.empty, Set("REL1"), Set.empty) should equal( + Set( + aRel1B, + CDRel1A, + emptyRel1Empty + ) + ) + + schema.schemaPatternsFor( + Set.empty, + Set("REL1", "REL2"), + Set.empty + ) should equal( + Set( + aRel1B, + aRel2CD, + bRel2CD, + CDRel1A, + emptyRel1Empty + ) + ) } it("works when every thing is known") { - schema.schemaPatternsFor(Set("A"), Set("REL1"), Set("B")) should equal(Set( - aRel1B - )) + schema.schemaPatternsFor(Set("A"), Set("REL1"), Set("B")) should equal( + Set( + aRel1B + ) + ) - schema.schemaPatternsFor(Set("A"), Set("REL2"), Set("C")) should equal(Set( - aRel2CD - )) + schema.schemaPatternsFor(Set("A"), Set("REL2"), Set("C")) should equal( + Set( + aRel2CD + ) + ) schema.schemaPatternsFor(Set("A"), Set("REL1"), Set("C")) shouldBe empty } it("works for no existing labels/types") { - schema.schemaPatternsFor(Set("A", "B"), Set.empty, Set.empty) shouldBe empty - schema.schemaPatternsFor(Set.empty, Set.empty, Set("A", "B")) shouldBe empty + schema.schemaPatternsFor( + Set("A", "B"), + Set.empty, + Set.empty + ) shouldBe empty + schema.schemaPatternsFor( + Set.empty, + Set.empty, + Set("A", "B") + ) shouldBe empty schema.schemaPatternsFor(Set.empty, Set("REL3"), Set.empty) shouldBe empty } } @@ -736,7 +1008,9 @@ class PropertyGraphSchemaTest extends ApiBaseTest { } } - it("fails if a node key refers to a non-existing property key for the label") { + it( + "fails if a node key refers to a non-existing property key for the label" + ) { an[SchemaException] shouldBe thrownBy { PropertyGraphSchema.empty .withNodePropertyKeys("A")("foo" -> CTString) @@ -752,7 +1026,6 @@ class PropertyGraphSchemaTest extends ApiBaseTest { } } - it("fails if a relationship key refers to a non-existing label") { an[SchemaException] shouldBe thrownBy { PropertyGraphSchema.empty @@ -761,7 +1034,9 @@ class PropertyGraphSchemaTest extends ApiBaseTest { } } - it("fails if a relationship key refers to a non-existing property key for the label") { + it( + "fails if a relationship key refers to a non-existing property key for the label" + ) { an[SchemaException] shouldBe thrownBy { PropertyGraphSchema.empty .withRelationshipPropertyKeys("A")("foo" -> CTString) @@ -769,7 +1044,9 @@ class PropertyGraphSchemaTest extends ApiBaseTest { } } - it("fails if a relationship key refers to a nullable property key for the label") { + it( + "fails if a relationship key refers to a nullable property key for the label" + ) { an[SchemaException] shouldBe thrownBy { PropertyGraphSchema.empty .withRelationshipPropertyKeys("A")("foo" -> CTString.nullable) @@ -818,17 +1095,20 @@ class PropertyGraphSchemaTest extends ApiBaseTest { .toSeq .flatten .groupBy(_._1) - .map { - case (k, v) => k -> v.map(_._2) + .map { case (k, v) => + k -> v.map(_._2) } keyToTypes .mapValues(types => types.foldLeft[CypherType](CTVoid)(_ join _)) - .map { - case (key, tpe) => - if (schema.allCombinations.map(schema.nodePropertyKeys).forall(_.get(key).isDefined)) - key -> tpe - else key -> tpe.nullable + .map { case (key, tpe) => + if ( + schema.allCombinations + .map(schema.nodePropertyKeys) + .forall(_.get(key).isDefined) + ) + key -> tpe + else key -> tpe.nullable } } diff --git a/okapi-api/src/test/scala/org/opencypher/okapi/api/schema/RelTypePropertyMapTest.scala b/okapi-api/src/test/scala/org/opencypher/okapi/api/schema/RelTypePropertyMapTest.scala index eba5c63386..f470b092b1 100644 --- a/okapi-api/src/test/scala/org/opencypher/okapi/api/schema/RelTypePropertyMapTest.scala +++ b/okapi-api/src/test/scala/org/opencypher/okapi/api/schema/RelTypePropertyMapTest.scala @@ -37,7 +37,11 @@ class RelTypePropertyMapTest extends ApiBaseTest { it("|+|") { val map1 = RelTypePropertyMap.empty - .register("A")("name" -> CTString, "age" -> CTInteger, "gender" -> CTString) + .register("A")( + "name" -> CTString, + "age" -> CTInteger, + "gender" -> CTString + ) .register("B")("p" -> CTBoolean) val map2 = RelTypePropertyMap.empty @@ -46,7 +50,11 @@ class RelTypePropertyMapTest extends ApiBaseTest { map1 |+| map2 should equal( RelTypePropertyMap.empty - .register("A")("name" -> CTString, "age" -> CTInteger, "gender" -> CTUnion(CTString, CTTrue, CTFalse)) + .register("A")( + "name" -> CTString, + "age" -> CTInteger, + "gender" -> CTUnion(CTString, CTTrue, CTFalse) + ) .register("B")("p" -> CTBoolean) .register("C")("name" -> CTString) ) @@ -67,8 +75,10 @@ class RelTypePropertyMapTest extends ApiBaseTest { .register("A")("name" -> CTString) .register("B")("foo" -> CTInteger) - map.filterForRelTypes(Set("A", "B")) should equal(RelTypePropertyMap.empty - .register("A")("name" -> CTString) - .register("B")("foo" -> CTInteger)) + map.filterForRelTypes(Set("A", "B")) should equal( + RelTypePropertyMap.empty + .register("A")("name" -> CTString) + .register("B")("foo" -> CTInteger) + ) } } diff --git a/okapi-api/src/test/scala/org/opencypher/okapi/api/types/CypherTypesTest.scala b/okapi-api/src/test/scala/org/opencypher/okapi/api/types/CypherTypesTest.scala index d4cd5f0bb4..5e72ac76d7 100644 --- a/okapi-api/src/test/scala/org/opencypher/okapi/api/types/CypherTypesTest.scala +++ b/okapi-api/src/test/scala/org/opencypher/okapi/api/types/CypherTypesTest.scala @@ -155,25 +155,39 @@ class CypherTypesTest extends ApiBaseTest with Checkers { CTNumber -> ("NUMBER" -> "NUMBER?"), CTInteger -> ("INTEGER" -> "INTEGER?"), CTFloat -> ("FLOAT" -> "FLOAT?"), - CTMap(Map("foo" -> CTString, "bar" -> CTInteger)) -> ("MAP(foo: STRING, bar: INTEGER)" -> "MAP(foo: STRING, bar: INTEGER)?"), + CTMap( + Map("foo" -> CTString, "bar" -> CTInteger) + ) -> ("MAP(foo: STRING, bar: INTEGER)" -> "MAP(foo: STRING, bar: INTEGER)?"), CTNode -> ("NODE" -> "NODE?"), CTNode("Person") -> ("NODE(:Person)" -> "NODE(:Person)?"), - CTNode("Person", "Employee") -> ("NODE(:Person:Employee)" -> "NODE(:Person:Employee)?"), - CTNode(Set("Person"), Some(QualifiedGraphName("foo.bar"))) -> ("NODE(:Person) @ foo.bar" -> "NODE(:Person) @ foo.bar?"), + CTNode( + "Person", + "Employee" + ) -> ("NODE(:Person:Employee)" -> "NODE(:Person:Employee)?"), + CTNode( + Set("Person"), + Some(QualifiedGraphName("foo.bar")) + ) -> ("NODE(:Person) @ foo.bar" -> "NODE(:Person) @ foo.bar?"), CTRelationship -> ("RELATIONSHIP" -> "RELATIONSHIP?"), - CTRelationship(Set("KNOWS")) -> ("RELATIONSHIP(:KNOWS)" -> "RELATIONSHIP(:KNOWS)?"), - CTRelationship(Set("KNOWS", "LOVES")) -> ("RELATIONSHIP(:KNOWS|:LOVES)" -> "RELATIONSHIP(:KNOWS|:LOVES)?"), - CTRelationship(Set("KNOWS"), Some(QualifiedGraphName("foo.bar"))) -> ("RELATIONSHIP(:KNOWS) @ foo.bar" -> "RELATIONSHIP(:KNOWS) @ foo.bar?"), + CTRelationship( + Set("KNOWS") + ) -> ("RELATIONSHIP(:KNOWS)" -> "RELATIONSHIP(:KNOWS)?"), + CTRelationship( + Set("KNOWS", "LOVES") + ) -> ("RELATIONSHIP(:KNOWS|:LOVES)" -> "RELATIONSHIP(:KNOWS|:LOVES)?"), + CTRelationship( + Set("KNOWS"), + Some(QualifiedGraphName("foo.bar")) + ) -> ("RELATIONSHIP(:KNOWS) @ foo.bar" -> "RELATIONSHIP(:KNOWS) @ foo.bar?"), CTPath -> ("PATH" -> "PATH?"), CTList(CTInteger) -> ("LIST(INTEGER)" -> "LIST(INTEGER)?"), CTList(CTInteger.nullable) -> ("LIST(INTEGER?)" -> "LIST(INTEGER?)?"), CTDate -> ("DATE" -> "DATE?"), CTLocalDateTime -> ("LOCALDATETIME" -> "LOCALDATETIME?") - ).foreach { - case (t, (materialName, nullableName)) => - t.isNullable shouldBe false - t.name shouldBe materialName - t.nullable.name shouldBe nullableName + ).foreach { case (t, (materialName, nullableName)) => + t.isNullable shouldBe false + t.name shouldBe materialName + t.nullable.name shouldBe nullableName } CTVoid.name shouldBe "VOID" @@ -186,18 +200,32 @@ class CypherTypesTest extends ApiBaseTest with Checkers { CTRelationship("KNOWS").superTypeOf(CTRelationship()) shouldBe false CTRelationship().subTypeOf(CTRelationship("KNOWS")) shouldBe false CTRelationship("KNOWS").superTypeOf(CTRelationship("KNOWS")) shouldBe true - CTRelationship("KNOWS").superTypeOf(CTRelationship("KNOWS", "LOVES")) shouldBe false - CTRelationship("KNOWS", "LOVES").superTypeOf(CTRelationship("LOVES")) shouldBe true + CTRelationship("KNOWS").superTypeOf( + CTRelationship("KNOWS", "LOVES") + ) shouldBe false + CTRelationship("KNOWS", "LOVES").superTypeOf( + CTRelationship("LOVES") + ) shouldBe true CTRelationship("KNOWS").superTypeOf(CTRelationship("NOSE")) shouldBe false } it("RELATIONSHIP? type") { CTRelationship.nullable.superTypeOf(CTRelationship.nullable) shouldBe true - CTRelationship.nullable.superTypeOf(CTRelationship("KNOWS").nullable) shouldBe true - CTRelationship("KNOWS").nullable.superTypeOf(CTRelationship("KNOWS").nullable) shouldBe true - CTRelationship("KNOWS").nullable.superTypeOf(CTRelationship("KNOWS", "LOVES").nullable) shouldBe false - CTRelationship("KNOWS", "LOVES").nullable.superTypeOf(CTRelationship("LOVES").nullable) shouldBe true - CTRelationship("KNOWS").nullable.superTypeOf(CTRelationship("NOSE").nullable) shouldBe false + CTRelationship.nullable.superTypeOf( + CTRelationship("KNOWS").nullable + ) shouldBe true + CTRelationship("KNOWS").nullable.superTypeOf( + CTRelationship("KNOWS").nullable + ) shouldBe true + CTRelationship("KNOWS").nullable.superTypeOf( + CTRelationship("KNOWS", "LOVES").nullable + ) shouldBe false + CTRelationship("KNOWS", "LOVES").nullable.superTypeOf( + CTRelationship("LOVES").nullable + ) shouldBe true + CTRelationship("KNOWS").nullable.superTypeOf( + CTRelationship("NOSE").nullable + ) shouldBe false CTRelationship("FOO").nullable.superTypeOf(CTNull) shouldBe true } @@ -216,9 +244,15 @@ class CypherTypesTest extends ApiBaseTest with Checkers { it("NODE? type") { CTNode.nullable.superTypeOf(CTNode.nullable) shouldBe true CTNode.nullable.superTypeOf(CTNode("Person").nullable) shouldBe true - CTNode("Person").nullable.superTypeOf(CTNode("Person").nullable) shouldBe true - CTNode("Person").nullable.superTypeOf(CTNode("Person", "Employee").nullable) shouldBe true - CTNode("Person", "Employee").nullable.superTypeOf(CTNode("Employee").nullable) shouldBe false + CTNode("Person").nullable.superTypeOf( + CTNode("Person").nullable + ) shouldBe true + CTNode("Person").nullable.superTypeOf( + CTNode("Person", "Employee").nullable + ) shouldBe true + CTNode("Person", "Employee").nullable.superTypeOf( + CTNode("Employee").nullable + ) shouldBe false CTNode("Person").nullable.superTypeOf(CTNode("Foo").nullable) shouldBe false CTNode("Foo").nullable.superTypeOf(CTNull) shouldBe true } @@ -250,8 +284,12 @@ class CypherTypesTest extends ApiBaseTest with Checkers { it("basic type inheritance") { CTNumber superTypeOf CTInteger shouldBe true CTNumber superTypeOf CTFloat shouldBe true - CTMap(Map("foo" -> CTAny, "bar" -> CTInteger)) superTypeOf CTMap(Map("foo" -> CTString, "bar" -> CTInteger)) shouldBe true - CTMap(Map("foo" -> CTAny, "bar" -> CTAny)) superTypeOf CTMap(Map("foo" -> CTString, "bar" -> CTInteger)) shouldBe true + CTMap(Map("foo" -> CTAny, "bar" -> CTInteger)) superTypeOf CTMap( + Map("foo" -> CTString, "bar" -> CTInteger) + ) shouldBe true + CTMap(Map("foo" -> CTAny, "bar" -> CTAny)) superTypeOf CTMap( + Map("foo" -> CTString, "bar" -> CTInteger) + ) shouldBe true CTAnyMaterial superTypeOf CTInteger shouldBe true CTAnyMaterial superTypeOf CTFloat shouldBe true @@ -301,12 +339,20 @@ class CypherTypesTest extends ApiBaseTest with Checkers { CTNumber join CTFloat shouldBe CTNumber CTNumber join CTInteger shouldBe CTNumber CTNumber join CTBigDecimal shouldBe CTNumber - CTNumber join CTString shouldBe CTUnion(CTString, CTInteger, CTFloat, CTBigDecimal) + CTNumber join CTString shouldBe CTUnion( + CTString, + CTInteger, + CTFloat, + CTBigDecimal + ) CTString join CTBoolean shouldBe CTUnion(CTString, CTBoolean) CTAnyMaterial join CTInteger shouldBe CTAnyMaterial - CTList(CTInteger) join CTList(CTFloat) shouldBe CTUnion(CTList(CTInteger), CTList(CTFloat)) + CTList(CTInteger) join CTList(CTFloat) shouldBe CTUnion( + CTList(CTInteger), + CTList(CTFloat) + ) CTList(CTInteger) join CTNode shouldBe CTUnion(CTList(CTInteger), CTNode) CTAnyMaterial join CTVoid shouldBe CTAnyMaterial @@ -328,26 +374,47 @@ class CypherTypesTest extends ApiBaseTest with Checkers { it("join with nullables") { CTInteger join CTFloat.nullable join CTBigDecimal shouldBe CTNumber.nullable CTFloat.nullable join CTInteger.nullable join CTBigDecimal shouldBe CTNumber.nullable - CTNumber.nullable join CTString shouldBe CTUnion(CTString, CTFloat, CTInteger, CTBigDecimal, CTNull) + CTNumber.nullable join CTString shouldBe CTUnion( + CTString, + CTFloat, + CTInteger, + CTBigDecimal, + CTNull + ) - CTString.nullable join CTBoolean.nullable shouldBe CTUnion(CTString, CTNull, CTTrue, CTFalse) + CTString.nullable join CTBoolean.nullable shouldBe CTUnion( + CTString, + CTNull, + CTTrue, + CTFalse + ) CTAnyMaterial join CTInteger.nullable shouldBe CTAnyMaterial.nullable } it("join with labels and types") { CTNode join CTNode("Person") shouldBe CTNode - CTNode("Other") join CTNode("Person") shouldBe CTUnion(CTNode("Other"), CTNode("Person")) + CTNode("Other") join CTNode("Person") shouldBe CTUnion( + CTNode("Other"), + CTNode("Person") + ) CTNode("Person") join CTNode("Person") shouldBe CTNode("Person") - CTNode("L1", "L2", "Lx") join CTNode("L1", "L2", "Ly") shouldBe CTUnion(CTNode("L1", "L2", "Lx"), CTNode("L1", "L2", "Ly")) + CTNode("L1", "L2", "Lx") join CTNode("L1", "L2", "Ly") shouldBe CTUnion( + CTNode("L1", "L2", "Lx"), + CTNode("L1", "L2", "Ly") + ) CTRelationship join CTRelationship("KNOWS") shouldBe CTRelationship - CTRelationship("OTHER") join CTRelationship("KNOWS") shouldBe CTRelationship("KNOWS", "OTHER") - CTRelationship("KNOWS") join CTRelationship("KNOWS") shouldBe CTRelationship("KNOWS") - CTRelationship("T1", "T2", "Tx") join CTRelationship("T1", "T2", "Ty") shouldBe CTRelationship( + CTRelationship("OTHER") join CTRelationship( + "KNOWS" + ) shouldBe CTRelationship("KNOWS", "OTHER") + CTRelationship("KNOWS") join CTRelationship( + "KNOWS" + ) shouldBe CTRelationship("KNOWS") + CTRelationship("T1", "T2", "Tx") join CTRelationship( "T1", "T2", - "Tx", - "Ty") + "Ty" + ) shouldBe CTRelationship("T1", "T2", "Tx", "Ty") } it("meet") { @@ -364,8 +431,12 @@ class CypherTypesTest extends ApiBaseTest with Checkers { CTNode meet CTNode("Person") shouldBe CTNode("Person") - CTMap("age" -> CTInteger) meet CTMap() shouldBe CTMap("age" -> CTInteger.nullable) - CTMap() meet CTMap("age" -> CTInteger) shouldBe CTMap("age" -> CTInteger.nullable) + CTMap("age" -> CTInteger) meet CTMap() shouldBe CTMap( + "age" -> CTInteger.nullable + ) + CTMap() meet CTMap("age" -> CTInteger) shouldBe CTMap( + "age" -> CTInteger.nullable + ) CTMap("age" -> CTInteger) meet CTMap shouldBe CTMap("age" -> CTInteger) } @@ -376,7 +447,9 @@ class CypherTypesTest extends ApiBaseTest with Checkers { CTRelationship("KNOWS") meet CTRelationship shouldBe CTRelationship("KNOWS") CTRelationship("KNOWS") meet CTRelationship("LOVES") shouldBe CTVoid - CTRelationship("KNOWS", "LOVES") meet CTRelationship("LOVES") shouldBe CTRelationship("LOVES") + CTRelationship("KNOWS", "LOVES") meet CTRelationship( + "LOVES" + ) shouldBe CTRelationship("LOVES") } it("type equality for all types") { @@ -445,7 +518,9 @@ class CypherTypesTest extends ApiBaseTest with Checkers { it("can parse maps with escaped keys") { val input = "MAP(`foo bar_my baz`: STRING)" - parseCypherType(input) should equal(Some(CTMap(Map("foo bar_my baz" -> CTString)))) + parseCypherType(input) should equal( + Some(CTMap(Map("foo bar_my baz" -> CTString))) + ) } it("can parse node types with escaped labels") { @@ -455,7 +530,9 @@ class CypherTypesTest extends ApiBaseTest with Checkers { it("can parse relationship types with escaped labels") { val input = "Relationship(:`foo bar_my baz`|:bar)" - parseCypherType(input) should equal(Some(CTRelationship("foo bar_my baz", "bar"))) + parseCypherType(input) should equal( + Some(CTRelationship("foo bar_my baz", "bar")) + ) } it("handles white space") { @@ -467,43 +544,68 @@ class CypherTypesTest extends ApiBaseTest with Checkers { } it("types for literals") { - check(Prop.forAll(any) { v: CypherValue => - (v.cypherType | CTNull).isNullable === true - }, minSuccessful(100)) + check( + Prop.forAll(any) { v: CypherValue => + (v.cypherType | CTNull).isNullable === true + }, + minSuccessful(100) + ) - check(Prop.forAll(any) { v: CypherValue => - (v.cypherType | CTNull).material.isNullable === false - }, minSuccessful(100)) + check( + Prop.forAll(any) { v: CypherValue => + (v.cypherType | CTNull).material.isNullable === false + }, + minSuccessful(100) + ) - check(Prop.forAll(any) { v: CypherValue => - v.cypherType.nullable.isNullable === true - }, minSuccessful(100)) + check( + Prop.forAll(any) { v: CypherValue => + v.cypherType.nullable.isNullable === true + }, + minSuccessful(100) + ) - check(Prop.forAll(any) { v: CypherValue => - v.cypherType.nullable.material.isNullable === false - }, minSuccessful(100)) + check( + Prop.forAll(any) { v: CypherValue => + v.cypherType.nullable.material.isNullable === false + }, + minSuccessful(100) + ) - check(Prop.forAll(any) { v: CypherValue => - (v.cypherType | CTNull) === v.cypherType.nullable - }, minSuccessful(100)) + check( + Prop.forAll(any) { v: CypherValue => + (v.cypherType | CTNull) === v.cypherType.nullable + }, + minSuccessful(100) + ) - check(Prop.forAll(any) { v: CypherValue => - CTUnion(v.cypherType) === v.cypherType - }, minSuccessful(100)) + check( + Prop.forAll(any) { v: CypherValue => + CTUnion(v.cypherType) === v.cypherType + }, + minSuccessful(100) + ) - check(Prop.forAll(any) { v: CypherValue => - CTUnion(v.cypherType, CTNull) === v.cypherType.nullable - }, minSuccessful(100)) + check( + Prop.forAll(any) { v: CypherValue => + CTUnion(v.cypherType, CTNull) === v.cypherType.nullable + }, + minSuccessful(100) + ) } it("intersects map types with different properties") { - CTMap(Map("name" -> CTString)) & CTMap(Map("age" -> CTInteger)) should equal( + CTMap(Map("name" -> CTString)) & CTMap( + Map("age" -> CTInteger) + ) should equal( CTMap(Map("name" -> CTString.nullable, "age" -> CTInteger.nullable)) ) } it("intersects map types with overlapping properties") { - CTMap(Map("name" -> CTString)) & CTMap(Map("name" -> CTInteger)) should equal( + CTMap(Map("name" -> CTString)) & CTMap( + Map("name" -> CTInteger) + ) should equal( CTMap(Map("name" -> (CTString | CTInteger))) ) } diff --git a/okapi-api/src/test/scala/org/opencypher/okapi/api/types/TypeLawsTest.scala b/okapi-api/src/test/scala/org/opencypher/okapi/api/types/TypeLawsTest.scala index 02d0f3c33b..5c15f0966e 100644 --- a/okapi-api/src/test/scala/org/opencypher/okapi/api/types/TypeLawsTest.scala +++ b/okapi-api/src/test/scala/org/opencypher/okapi/api/types/TypeLawsTest.scala @@ -35,14 +35,22 @@ import org.scalatest.matchers.should.Matchers import org.scalatestplus.scalacheck.ScalaCheckDrivenPropertyChecks import org.typelevel.discipline.scalatest.FunSpecDiscipline -class TypeLawsTest extends AnyFunSpec with Matchers with ScalaCheckDrivenPropertyChecks with FunSpecDiscipline { +class TypeLawsTest + extends AnyFunSpec + with Matchers + with ScalaCheckDrivenPropertyChecks + with FunSpecDiscipline { def pickOne[T](gens: List[Gen[T]]): Gen[T] = for { i <- Gen.choose(0, gens.size - 1) t <- gens(i) } yield t - val maybeGraph: Gen[Option[QualifiedGraphName]] = Gen.oneOf(None, Some(QualifiedGraphName("ns.g1")), Some(QualifiedGraphName("ns.g2"))) + val maybeGraph: Gen[Option[QualifiedGraphName]] = Gen.oneOf( + None, + Some(QualifiedGraphName("ns.g1")), + Some(QualifiedGraphName("ns.g2")) + ) val nodeLabel: Gen[String] = Gen.oneOf("A", "B", "C") @@ -102,7 +110,8 @@ class TypeLawsTest extends AnyFunSpec with Matchers with ScalaCheckDrivenPropert fields <- Gen.mapOf(field) } yield CTMap(fields) - val nestedTypes: List[Gen[CypherType]] = List(list, map, Gen.const(CTList), Gen.const(CTMap)) + val nestedTypes: List[Gen[CypherType]] = + List(list, map, Gen.const(CTList), Gen.const(CTMap)) val allTypes: List[Gen[CypherType]] = List(flatTypes, nestedTypes).flatten @@ -122,8 +131,16 @@ class TypeLawsTest extends AnyFunSpec with Matchers with ScalaCheckDrivenPropert def combine(x: CypherType, y: CypherType): CypherType = x & y } - checkAll("CypherType.union", cats.kernel.laws.discipline.MonoidTests[CypherType](unionMonoid).monoid) + checkAll( + "CypherType.union", + cats.kernel.laws.discipline.MonoidTests[CypherType](unionMonoid).monoid + ) - checkAll("CypherType.intersection", cats.kernel.laws.discipline.MonoidTests[CypherType](intersectionMonoid).monoid) + checkAll( + "CypherType.intersection", + cats.kernel.laws.discipline + .MonoidTests[CypherType](intersectionMonoid) + .monoid + ) } diff --git a/okapi-api/src/test/scala/org/opencypher/okapi/api/value/CypherValueTest.scala b/okapi-api/src/test/scala/org/opencypher/okapi/api/value/CypherValueTest.scala index 2cba82ec77..43635db484 100644 --- a/okapi-api/src/test/scala/org/opencypher/okapi/api/value/CypherValueTest.scala +++ b/okapi-api/src/test/scala/org/opencypher/okapi/api/value/CypherValueTest.scala @@ -28,7 +28,15 @@ package org.opencypher.okapi.api.value import org.opencypher.okapi.ApiBaseTest import org.opencypher.okapi.api.value.CypherValue.Format._ -import org.opencypher.okapi.api.value.CypherValue.{CypherBigDecimal, CypherBoolean, CypherFloat, CypherInteger, CypherList, CypherMap, CypherString} +import org.opencypher.okapi.api.value.CypherValue.{ + CypherBigDecimal, + CypherBoolean, + CypherFloat, + CypherInteger, + CypherList, + CypherMap, + CypherString +} import org.opencypher.okapi.api.value.GenCypherValue.{TestNode, TestRelationship} class CypherValueTest extends ApiBaseTest { @@ -45,18 +53,24 @@ class CypherValueTest extends ApiBaseTest { new CypherBigDecimal(BigDecimal("1.2")) -> "1.2" ) - mapping.foreach { - case (input, expected) => input.toCypherString should equal(expected) + mapping.foreach { case (input, expected) => + input.toCypherString should equal(expected) } } it("converts a CypherList") { - CypherList("foo", 123, false).toCypherString should equal("['foo', 123, false]") + CypherList("foo", 123, false).toCypherString should equal( + "['foo', 123, false]" + ) CypherList().toCypherString should equal("[]") } it("converts a CypherMap") { - CypherMap("foo" -> "bar", "foo\\bar" -> 42, "foo\"bar" -> false).toCypherString should equal( + CypherMap( + "foo" -> "bar", + "foo\\bar" -> 42, + "foo\"bar" -> false + ).toCypherString should equal( "{`foo`: 'bar', `foo\\\"bar`: false, `foo\\\\bar`: 42}" ) CypherMap().toCypherString should equal("{}") @@ -64,13 +78,25 @@ class CypherValueTest extends ApiBaseTest { it("converts a CypherRelationship") { val mapping = Map( - TestRelationship(1, 1, 2, "REL", CypherMap("foo" -> 42)) -> "[:`REL` {`foo`: 42}]", + TestRelationship( + 1, + 1, + 2, + "REL", + CypherMap("foo" -> 42) + ) -> "[:`REL` {`foo`: 42}]", TestRelationship(1, 1, 2, "REL") -> "[:`REL`]", - TestRelationship(1, 1, 2, "My'Rel", CypherMap("foo" -> 42)) -> "[:`My\\'Rel` {`foo`: 42}]" + TestRelationship( + 1, + 1, + 2, + "My'Rel", + CypherMap("foo" -> 42) + ) -> "[:`My\\'Rel` {`foo`: 42}]" ) - mapping.foreach { - case (input, expected) => input.toCypherString should equal(expected) + mapping.foreach { case (input, expected) => + input.toCypherString should equal(expected) } } @@ -78,11 +104,15 @@ class CypherValueTest extends ApiBaseTest { val mapping = Map( TestNode(1, Set("A"), CypherMap("foo" -> 42)) -> "(:`A` {`foo`: 42})", TestNode(1) -> "()", - TestNode(1, Set("My\"Node", "My'Node"), CypherMap("foo" -> 42)) -> "(:`My\\\"Node`:`My\\\'Node` {`foo`: 42})" + TestNode( + 1, + Set("My\"Node", "My'Node"), + CypherMap("foo" -> 42) + ) -> "(:`My\\\"Node`:`My\\\'Node` {`foo`: 42})" ) - mapping.foreach { - case (input, expected) => input.toCypherString should equal(expected) + mapping.foreach { case (input, expected) => + input.toCypherString should equal(expected) } } } diff --git a/okapi-api/src/test/scala/org/opencypher/okapi/api/value/GenCypherValue.scala b/okapi-api/src/test/scala/org/opencypher/okapi/api/value/GenCypherValue.scala index 2e55a5e3a1..967700d2d9 100644 --- a/okapi-api/src/test/scala/org/opencypher/okapi/api/value/GenCypherValue.scala +++ b/okapi-api/src/test/scala/org/opencypher/okapi/api/value/GenCypherValue.scala @@ -52,11 +52,13 @@ object GenCypherValue { ) private val bannedChars = Set("'") ++ reservedParboiledChars - private val stringWithoutBanned = arbitrary[String].map(s => s.filterNot(bannedChars.contains)) + private val stringWithoutBanned = + arbitrary[String].map(s => s.filterNot(bannedChars.contains)) val string: Gen[CypherString] = stringWithoutBanned.map(CypherString) - def oneOfSeq[T](gs: Seq[Gen[T]]): Gen[T] = choose(0, gs.size - 1).flatMap(gs(_)) + def oneOfSeq[T](gs: Seq[Gen[T]]): Gen[T] = + choose(0, gs.size - 1).flatMap(gs(_)) val boolean: Gen[CypherBoolean] = arbitrary[Boolean].map(CypherBoolean) val integer: Gen[CypherInteger] = arbitrary[Long].map(CypherInteger) @@ -73,39 +75,56 @@ object GenCypherValue { labelElements <- listOfN(size, label) } yield labelElements.toSet - val scalarGenerators: Seq[Gen[CypherValue]] = List(string, boolean, integer, float) + val scalarGenerators: Seq[Gen[CypherValue]] = + List(string, boolean, integer, float) val scalar: Gen[CypherValue] = oneOfSeq(scalarGenerators) - val scalarOrNull: Gen[CypherValue] = oneOfSeq(scalarGenerators :+ const(CypherNull)) + val scalarOrNull: Gen[CypherValue] = oneOfSeq( + scalarGenerators :+ const(CypherNull) + ) - val homogeneousScalarList: Gen[CypherList] = oneOfSeq(scalarGenerators.map(listWithElementGenerator)) + val homogeneousScalarList: Gen[CypherList] = oneOfSeq( + scalarGenerators.map(listWithElementGenerator) + ) - def listWithElementGenerator(elementGeneratorGenerator: Gen[CypherValue]): Gen[CypherList] = lzy(for { + def listWithElementGenerator( + elementGeneratorGenerator: Gen[CypherValue] + ): Gen[CypherList] = lzy(for { size <- choose(min = 0, max = maxContainerSize) elementGenerator <- elementGeneratorGenerator listElements <- listOfN(size, elementGenerator) } yield listElements) - lazy val any: Gen[CypherValue] = lzy(oneOf(scalarOrNull, map, list, node, relationship)) + lazy val any: Gen[CypherValue] = lzy( + oneOf(scalarOrNull, map, list, node, relationship) + ) lazy val list: Gen[CypherList] = lzy(listWithElementGenerator(any)) - lazy val propertyMap: Gen[CypherMap] = mapWithValueGenerator(oneOfSeq(scalarGenerators :+ homogeneousScalarList)) + lazy val propertyMap: Gen[CypherMap] = mapWithValueGenerator( + oneOfSeq(scalarGenerators :+ homogeneousScalarList) + ) lazy val map: Gen[CypherMap] = lzy(mapWithValueGenerator(any)) - def mapWithValueGenerator(valueGenerator: Gen[CypherValue]): Gen[CypherMap] = lzy(for { - size <- choose(min = 0, max = maxContainerSize) - keyValuePairs <- mapOfN(size, for { - key <- label - value <- valueGenerator - } yield key -> value) - } yield keyValuePairs) + def mapWithValueGenerator(valueGenerator: Gen[CypherValue]): Gen[CypherMap] = + lzy(for { + size <- choose(min = 0, max = maxContainerSize) + keyValuePairs <- mapOfN( + size, + for { + key <- label + value <- valueGenerator + } yield key -> value + ) + } yield keyValuePairs) def singlePropertyMap( keyGenerator: Gen[String] = const("singleProperty"), - valueGenerator: Gen[CypherValue] = oneOfSeq(scalarGenerators :+ homogeneousScalarList) + valueGenerator: Gen[CypherValue] = oneOfSeq( + scalarGenerators :+ homogeneousScalarList + ) ): Gen[CypherMap] = lzy(for { key <- keyGenerator value <- valueGenerator @@ -136,7 +155,8 @@ object GenCypherValue { } yield TestRelationship(id, start, end, relType, ps) } - val relationship: Gen[TestRelationship[CypherInteger]] = relationshipWithIdGenerators(integer, integer, integer) + val relationship: Gen[TestRelationship[CypherInteger]] = + relationshipWithIdGenerators(integer, integer, integer) // TODO: Add date and datetime generators @@ -150,13 +170,19 @@ object GenCypherValue { } } - def nodeRelNodePattern(mapGenerator: Gen[CypherMap] = propertyMap): Gen[NodeRelNodePattern[_]] = { + def nodeRelNodePattern( + mapGenerator: Gen[CypherMap] = propertyMap + ): Gen[NodeRelNodePattern[_]] = { val n1Id = 0L val rId = 0L val n2Id = 1L for { startNode <- nodeWithCustomGenerators(const(n1Id), mapGenerator) - relationship <- relationshipWithIdGenerators(const(rId), const(n1Id), const(n2Id)) + relationship <- relationshipWithIdGenerators( + const(rId), + const(n1Id), + const(n2Id) + ) endNode <- nodeWithCustomGenerators(const(n2Id), mapGenerator) } yield NodeRelNodePattern(startNode, relationship, endNode) } @@ -178,7 +204,8 @@ object GenCypherValue { override def productElement(n: Int): Any = n match { case 0 => labels case 1 => properties - case other => throw IllegalArgumentException("a valid product index", s"$other") + case other => + throw IllegalArgumentException("a valid product index", s"$other") } override def toString = s"${getClass.getSimpleName}($labels, $properties)}" @@ -206,14 +233,16 @@ object GenCypherValue { endId: Id, relType: String, properties: CypherMap - ): TestRelationship[Id] = TestRelationship(id, startId, endId, relType, properties) + ): TestRelationship[Id] = + TestRelationship(id, startId, endId, relType, properties) override def productArity: Int = 2 override def productElement(n: Int): Any = n match { case 0 => relType case 1 => properties - case other => throw IllegalArgumentException("a valid product index", s"$other") + case other => + throw IllegalArgumentException("a valid product index", s"$other") } override def toString = s"${getClass.getSimpleName}($relType, $properties)}" diff --git a/okapi-api/src/test/scala/org/opencypher/okapi/impl/graph/CypherCatalogTest.scala b/okapi-api/src/test/scala/org/opencypher/okapi/impl/graph/CypherCatalogTest.scala index 65fb04591e..54cee8f320 100644 --- a/okapi-api/src/test/scala/org/opencypher/okapi/impl/graph/CypherCatalogTest.scala +++ b/okapi-api/src/test/scala/org/opencypher/okapi/impl/graph/CypherCatalogTest.scala @@ -32,17 +32,23 @@ import org.opencypher.okapi.api.io.PropertyGraphDataSource import org.opencypher.okapi.impl.exception.{GraphNotFoundException, IllegalArgumentException} import org.opencypher.okapi.impl.io.SessionGraphDataSource -class CypherCatalogTest extends ApiBaseTest { +class CypherCatalogTest extends ApiBaseTest { it("avoids retrieving a non-registered data source") { - an[IllegalArgumentException] should be thrownBy new CypherCatalog().source(Namespace("foo")) + an[IllegalArgumentException] should be thrownBy new CypherCatalog().source( + Namespace("foo") + ) } it("avoids retrieving a graph not stored in the session") { - an[GraphNotFoundException] should be thrownBy new CypherCatalog().graph(QualifiedGraphName("foo")) + an[GraphNotFoundException] should be thrownBy new CypherCatalog().graph( + QualifiedGraphName("foo") + ) } it("avoids retrieving a graph from a non-registered data source") { - an[IllegalArgumentException] should be thrownBy new CypherCatalog().graph(QualifiedGraphName(Namespace("foo"), GraphName("bar"))) + an[IllegalArgumentException] should be thrownBy new CypherCatalog().graph( + QualifiedGraphName(Namespace("foo"), GraphName("bar")) + ) } it("returns all available namespaces") { @@ -51,6 +57,8 @@ class CypherCatalogTest extends ApiBaseTest { val namespace = Namespace("foo") val dataSource = mock[PropertyGraphDataSource] catalog.register(namespace, dataSource) - catalog.namespaces should equal(Set(SessionGraphDataSource.Namespace, namespace)) + catalog.namespaces should equal( + Set(SessionGraphDataSource.Namespace, namespace) + ) } } diff --git a/okapi-api/src/test/scala/org/opencypher/okapi/impl/io/SessionGraphDataSourceTest.scala b/okapi-api/src/test/scala/org/opencypher/okapi/impl/io/SessionGraphDataSourceTest.scala index a267f570a3..01a44b2d24 100644 --- a/okapi-api/src/test/scala/org/opencypher/okapi/impl/io/SessionGraphDataSourceTest.scala +++ b/okapi-api/src/test/scala/org/opencypher/okapi/impl/io/SessionGraphDataSourceTest.scala @@ -64,16 +64,20 @@ class SessionGraphDataSourceTest extends ApiBaseTest { it("schema should throw for non-existing graph") { val source = new SessionGraphDataSource val testGraphName = GraphName("test") - a [GraphNotFoundException] shouldBe thrownBy(source.schema(testGraphName)) + a[GraphNotFoundException] shouldBe thrownBy(source.schema(testGraphName)) } it("schema should return schema for existing graph") { val source = new SessionGraphDataSource val testGraphName = GraphName("test") val propertyGraph = mock[PropertyGraph] - when(propertyGraph.schema).thenReturn(PropertyGraphSchema.empty.withRelationshipType("foo")) + when(propertyGraph.schema).thenReturn( + PropertyGraphSchema.empty.withRelationshipType("foo") + ) source.store(testGraphName, propertyGraph) - source.schema(testGraphName).get should be(PropertyGraphSchema.empty.withRelationshipType("foo")) + source.schema(testGraphName).get should be( + PropertyGraphSchema.empty.withRelationshipType("foo") + ) } it("graphNames should return all names of stored graphs") { diff --git a/okapi-api/src/test/scala/org/opencypher/okapi/impl/util/TablePrinterTest.scala b/okapi-api/src/test/scala/org/opencypher/okapi/impl/util/TablePrinterTest.scala index 776b59e381..c42d82a84f 100644 --- a/okapi-api/src/test/scala/org/opencypher/okapi/impl/util/TablePrinterTest.scala +++ b/okapi-api/src/test/scala/org/opencypher/okapi/impl/util/TablePrinterTest.scala @@ -38,8 +38,7 @@ class TablePrinterTest extends ApiBaseTest { val header = Seq.empty val data = Seq.empty - toTable(header, data) should equal( - """|╔══════════════╗ + toTable(header, data) should equal("""|╔══════════════╗ |║ (no columns) ║ |╠══════════════╣ |║ (empty row) ║ @@ -52,8 +51,7 @@ class TablePrinterTest extends ApiBaseTest { val header = Seq("column") val data = Seq.empty - toTable(header, data) should equal( - """|╔════════╗ + toTable(header, data) should equal("""|╔════════╗ |║ column ║ |╚════════╝ |(no rows) @@ -64,8 +62,7 @@ class TablePrinterTest extends ApiBaseTest { val header = Seq("column") val data = Seq(Seq(1)) - toTable(header, data) should equal( - """|╔════════╗ + toTable(header, data) should equal("""|╔════════╗ |║ column ║ |╠════════╣ |║ 1 ║ @@ -78,8 +75,7 @@ class TablePrinterTest extends ApiBaseTest { val header = Seq("column") val data = Seq(Seq(1), Seq(2)) - toTable(header, data) should equal( - """|╔════════╗ + toTable(header, data) should equal("""|╔════════╗ |║ column ║ |╠════════╣ |║ 1 ║ @@ -100,13 +96,21 @@ class TablePrinterTest extends ApiBaseTest { |║ foo │ 42 │ 42.23 │ true ║ |╚════════╧═════════╧═══════╧═════════╝ |(1 row) - |""".stripMargin) + |""".stripMargin + ) } it("prints simple cypher values correctly") { val header = Seq("String", "Integer", "Float", "Boolean") - val data = Seq(Seq(CypherValue("foo"), CypherValue(42), CypherValue(42.23), CypherValue(true))) + val data = Seq( + Seq( + CypherValue("foo"), + CypherValue(42), + CypherValue(42.23), + CypherValue(true) + ) + ) implicit val f: CypherValue => String = v => v.toCypherString @@ -117,7 +121,8 @@ class TablePrinterTest extends ApiBaseTest { |║ 'foo' │ 42 │ 42.23 │ true ║ |╚════════╧═════════╧═══════╧═════════╝ |(1 row) - |""".stripMargin) + |""".stripMargin + ) } it("prints nested cypher values correctly") { @@ -144,13 +149,29 @@ class TablePrinterTest extends ApiBaseTest { case class TestNode(id: Long, labels: Set[String], properties: CypherMap) extends Node[Long] { override type I = TestNode - override def copy(id: Long, labels: Set[String], properties: CypherMap): TestNode = + override def copy( + id: Long, + labels: Set[String], + properties: CypherMap + ): TestNode = copy(id, labels, properties) } - case class TestRelationship(id: Long, startId: Long, endId: Long, relType: String, properties: CypherMap) extends Relationship[Long] { + case class TestRelationship( + id: Long, + startId: Long, + endId: Long, + relType: String, + properties: CypherMap + ) extends Relationship[Long] { override type I = TestRelationship - override def copy(id: Long, source: Long, target: Long, relType: String, properties: CypherMap): TestRelationship = + override def copy( + id: Long, + source: Long, + target: Long, + relType: String, + properties: CypherMap + ): TestRelationship = copy(id, source, target, relType, properties) } } diff --git a/okapi-api/src/test/scala/org/opencypher/okapi/impl/util/VersionTest.scala b/okapi-api/src/test/scala/org/opencypher/okapi/impl/util/VersionTest.scala index b593c4ad57..e12e07917c 100644 --- a/okapi-api/src/test/scala/org/opencypher/okapi/impl/util/VersionTest.scala +++ b/okapi-api/src/test/scala/org/opencypher/okapi/impl/util/VersionTest.scala @@ -32,14 +32,14 @@ import org.opencypher.okapi.impl.exception.IllegalArgumentException class VersionTest extends ApiBaseTest { describe("parsing") { it("parses two valued version numbers") { - Version("1.0") should equal(Version(1,0)) - Version("1.5") should equal(Version(1,5)) - Version("42.21") should equal(Version(42,21)) + Version("1.0") should equal(Version(1, 0)) + Version("1.5") should equal(Version(1, 5)) + Version("42.21") should equal(Version(42, 21)) } it("parses single valued version numbers") { - Version("1") should equal(Version(1,0)) - Version("42") should equal(Version(42,0)) + Version("1") should equal(Version(1, 0)) + Version("42") should equal(Version(42, 0)) } it("throws errors on malformed version string") { diff --git a/okapi-ir/src/main/scala/org/opencypher/okapi/ir/api/CypherStatement.scala b/okapi-ir/src/main/scala/org/opencypher/okapi/ir/api/CypherStatement.scala index 08c23f0a12..6afaaa1063 100644 --- a/okapi-ir/src/main/scala/org/opencypher/okapi/ir/api/CypherStatement.scala +++ b/okapi-ir/src/main/scala/org/opencypher/okapi/ir/api/CypherStatement.scala @@ -35,7 +35,7 @@ sealed trait CypherStatement sealed trait CypherQuery extends Block with CypherStatement final case class SingleQuery( - model: QueryModel + model: QueryModel ) extends CypherQuery { override def after: List[Block] = model.after @@ -61,8 +61,8 @@ final case class UnionQuery( } final case class CreateGraphStatement( - graph: IRGraph, - innerQuery: SingleQuery + graph: IRGraph, + innerQuery: SingleQuery ) extends CypherStatement final case class CreateViewStatement( @@ -71,7 +71,6 @@ final case class CreateViewStatement( innerQueryString: String ) extends CypherStatement - final case class DeleteGraphStatement( qgn: QualifiedGraphName ) extends CypherStatement @@ -79,5 +78,3 @@ final case class DeleteGraphStatement( final case class DeleteViewStatement( qgn: QualifiedGraphName ) extends CypherStatement - - diff --git a/okapi-ir/src/main/scala/org/opencypher/okapi/ir/api/IRElement.scala b/okapi-ir/src/main/scala/org/opencypher/okapi/ir/api/IRElement.scala index 7fe23aabfc..cb6192b14f 100644 --- a/okapi-ir/src/main/scala/org/opencypher/okapi/ir/api/IRElement.scala +++ b/okapi-ir/src/main/scala/org/opencypher/okapi/ir/api/IRElement.scala @@ -37,7 +37,7 @@ import org.opencypher.okapi.ir.api.set.SetItem object IRField { def relTypes(field: IRField): Set[String] = field.cypherType match { case CTRelationship(types, _) => types - case _ => Set.empty + case _ => Set.empty } } @@ -57,10 +57,16 @@ sealed trait IRGraph { object IRCatalogGraph { def apply(name: String, schema: PropertyGraphSchema): IRCatalogGraph = - IRCatalogGraph(QualifiedGraphName(SessionGraphDataSource.Namespace, GraphName(name)), schema) + IRCatalogGraph( + QualifiedGraphName(SessionGraphDataSource.Namespace, GraphName(name)), + schema + ) } -final case class IRCatalogGraph(qualifiedGraphName: QualifiedGraphName, schema: PropertyGraphSchema) extends IRGraph +final case class IRCatalogGraph( + qualifiedGraphName: QualifiedGraphName, + schema: PropertyGraphSchema +) extends IRGraph final case class IRPatternGraph( qualifiedGraphName: QualifiedGraphName, diff --git a/okapi-ir/src/main/scala/org/opencypher/okapi/ir/api/block/AggregationBlock.scala b/okapi-ir/src/main/scala/org/opencypher/okapi/ir/api/block/AggregationBlock.scala index ff259c9173..05b1cbba4b 100644 --- a/okapi-ir/src/main/scala/org/opencypher/okapi/ir/api/block/AggregationBlock.scala +++ b/okapi-ir/src/main/scala/org/opencypher/okapi/ir/api/block/AggregationBlock.scala @@ -30,10 +30,10 @@ import org.opencypher.okapi.ir.api.expr.Expr import org.opencypher.okapi.ir.api.{IRField, IRGraph} final case class AggregationBlock( - after: List[Block], - binds: Aggregations, - group: Set[IRField], - graph: IRGraph + after: List[Block], + binds: Aggregations, + group: Set[IRField], + graph: IRGraph ) extends BasicBlock[Aggregations](BlockType("aggregation")) { override val where: Set[Expr] = Set.empty[Expr] // no filtering in aggregation blocks diff --git a/okapi-ir/src/main/scala/org/opencypher/okapi/ir/api/block/BlockSig.scala b/okapi-ir/src/main/scala/org/opencypher/okapi/ir/api/block/BlockSig.scala index a3d65585fb..b2a91656bc 100644 --- a/okapi-ir/src/main/scala/org/opencypher/okapi/ir/api/block/BlockSig.scala +++ b/okapi-ir/src/main/scala/org/opencypher/okapi/ir/api/block/BlockSig.scala @@ -35,5 +35,6 @@ final case class BlockSig(inputs: Set[IRField], outputs: Set[IRField]) case object BlockSig { val empty = BlockSig(Set.empty, Set.empty) - implicit def signature(pairs: (Set[IRField], Set[IRField])): BlockSig = BlockSig(pairs._1, pairs._2) + implicit def signature(pairs: (Set[IRField], Set[IRField])): BlockSig = + BlockSig(pairs._1, pairs._2) } diff --git a/okapi-ir/src/main/scala/org/opencypher/okapi/ir/api/block/MatchBlock.scala b/okapi-ir/src/main/scala/org/opencypher/okapi/ir/api/block/MatchBlock.scala index ba40b86a3e..db0631bf0a 100644 --- a/okapi-ir/src/main/scala/org/opencypher/okapi/ir/api/block/MatchBlock.scala +++ b/okapi-ir/src/main/scala/org/opencypher/okapi/ir/api/block/MatchBlock.scala @@ -31,9 +31,9 @@ import org.opencypher.okapi.ir.api.expr.Expr import org.opencypher.okapi.ir.api.pattern.Pattern final case class MatchBlock( - after: List[Block], - binds: Pattern, - where: Set[Expr] = Set.empty[Expr], - optional: Boolean, - graph: IRGraph + after: List[Block], + binds: Pattern, + where: Set[Expr] = Set.empty[Expr], + optional: Boolean, + graph: IRGraph ) extends BasicBlock[Pattern](BlockType("match")) diff --git a/okapi-ir/src/main/scala/org/opencypher/okapi/ir/api/block/OrderAndSliceBlock.scala b/okapi-ir/src/main/scala/org/opencypher/okapi/ir/api/block/OrderAndSliceBlock.scala index ee239adf31..5fab7aa44c 100644 --- a/okapi-ir/src/main/scala/org/opencypher/okapi/ir/api/block/OrderAndSliceBlock.scala +++ b/okapi-ir/src/main/scala/org/opencypher/okapi/ir/api/block/OrderAndSliceBlock.scala @@ -30,11 +30,11 @@ import org.opencypher.okapi.ir.api.IRGraph import org.opencypher.okapi.ir.api.expr.Expr final case class OrderAndSliceBlock( - after: List[Block], - orderBy: Seq[SortItem], - skip: Option[Expr], - limit: Option[Expr], - graph: IRGraph + after: List[Block], + orderBy: Seq[SortItem], + skip: Option[Expr], + limit: Option[Expr], + graph: IRGraph ) extends BasicBlock[OrderedFields](BlockType("order-and-slice")) { override val binds: OrderedFields = OrderedFields() override def where: Set[Expr] = Set.empty[Expr] diff --git a/okapi-ir/src/main/scala/org/opencypher/okapi/ir/api/block/ProjectBlock.scala b/okapi-ir/src/main/scala/org/opencypher/okapi/ir/api/block/ProjectBlock.scala index 8267cb7880..a8758d159a 100644 --- a/okapi-ir/src/main/scala/org/opencypher/okapi/ir/api/block/ProjectBlock.scala +++ b/okapi-ir/src/main/scala/org/opencypher/okapi/ir/api/block/ProjectBlock.scala @@ -30,11 +30,11 @@ import org.opencypher.okapi.ir.api._ import org.opencypher.okapi.ir.api.expr.Expr final case class ProjectBlock( - after: List[Block], - binds: Fields = Fields(), - where: Set[Expr] = Set.empty[Expr], - graph: IRGraph, - distinct: Boolean = false + after: List[Block], + binds: Fields = Fields(), + where: Set[Expr] = Set.empty[Expr], + graph: IRGraph, + distinct: Boolean = false ) extends BasicBlock[Fields](BlockType("project")) final case class Fields(items: Map[IRField, Expr] = Map.empty[IRField, Expr]) extends Binds { diff --git a/okapi-ir/src/main/scala/org/opencypher/okapi/ir/api/block/ResultBlock.scala b/okapi-ir/src/main/scala/org/opencypher/okapi/ir/api/block/ResultBlock.scala index 3ebc5d8aeb..f957989359 100644 --- a/okapi-ir/src/main/scala/org/opencypher/okapi/ir/api/block/ResultBlock.scala +++ b/okapi-ir/src/main/scala/org/opencypher/okapi/ir/api/block/ResultBlock.scala @@ -51,13 +51,18 @@ object TableResultBlock { final case class OrderedFields(orderedFields: List[IRField] = List.empty) extends Binds { override def fields: Set[IRField] = orderedFields.toSet - def select(fields: Set[IRField]): OrderedFields = copy(orderedFields = orderedFields.filter(fields.contains)) + def select(fields: Set[IRField]): OrderedFields = + copy(orderedFields = orderedFields.filter(fields.contains)) } object OrderedFields { - def fieldsFrom[E](fields: IRField*): OrderedFields = OrderedFields(fields.toList) + def fieldsFrom[E](fields: IRField*): OrderedFields = OrderedFields( + fields.toList + ) - def unapplySeq(arg: OrderedFields): Option[Seq[IRField]] = Some(arg.orderedFields) + def unapplySeq(arg: OrderedFields): Option[Seq[IRField]] = Some( + arg.orderedFields + ) } final case class GraphResultBlock( diff --git a/okapi-ir/src/main/scala/org/opencypher/okapi/ir/api/block/SourceBlock.scala b/okapi-ir/src/main/scala/org/opencypher/okapi/ir/api/block/SourceBlock.scala index a560820dcc..dcbae193af 100644 --- a/okapi-ir/src/main/scala/org/opencypher/okapi/ir/api/block/SourceBlock.scala +++ b/okapi-ir/src/main/scala/org/opencypher/okapi/ir/api/block/SourceBlock.scala @@ -30,7 +30,7 @@ import org.opencypher.okapi.ir.api.IRGraph import org.opencypher.okapi.ir.api.expr.Expr case class SourceBlock( - graph: IRGraph + graph: IRGraph ) extends BasicBlock[Binds](BlockType("source")) { override def where: Set[Expr] = Set.empty[Expr] override val after: List[Block] = List.empty diff --git a/okapi-ir/src/main/scala/org/opencypher/okapi/ir/api/block/UnwindBlock.scala b/okapi-ir/src/main/scala/org/opencypher/okapi/ir/api/block/UnwindBlock.scala index 9e2b9fadd7..f59766dd5c 100644 --- a/okapi-ir/src/main/scala/org/opencypher/okapi/ir/api/block/UnwindBlock.scala +++ b/okapi-ir/src/main/scala/org/opencypher/okapi/ir/api/block/UnwindBlock.scala @@ -30,9 +30,9 @@ import org.opencypher.okapi.ir.api.expr.Expr import org.opencypher.okapi.ir.api.{IRField, IRGraph} final case class UnwindBlock( - after: List[Block], - binds: UnwoundList, - graph: IRGraph + after: List[Block], + binds: UnwoundList, + graph: IRGraph ) extends BasicBlock[UnwoundList](BlockType("unwind")) { override def where: Set[Expr] = Set.empty[Expr] // never filters } diff --git a/okapi-ir/src/main/scala/org/opencypher/okapi/ir/api/expr/Expr.scala b/okapi-ir/src/main/scala/org/opencypher/okapi/ir/api/expr/Expr.scala index c9d063c025..217d76894b 100644 --- a/okapi-ir/src/main/scala/org/opencypher/okapi/ir/api/expr/Expr.scala +++ b/okapi-ir/src/main/scala/org/opencypher/okapi/ir/api/expr/Expr.scala @@ -47,7 +47,8 @@ object Expr { /** * Describes a Cypher expression. * - * @see [[http://neo4j.com/docs/developer-manual/current/cypher/syntax/expressions/ Cypher Expressions in the Neo4j Manual]] + * @see + * [[http://neo4j.com/docs/developer-manual/current/cypher/syntax/expressions/ Cypher Expressions in the Neo4j Manual]] */ sealed abstract class Expr extends AbstractTreeNode[Expr] { @@ -57,16 +58,19 @@ sealed abstract class Expr extends AbstractTreeNode[Expr] { override def toString = s"$withoutType :: $cypherType" def isElementExpression: Boolean = owner.isDefined + /** - * Returns the node/relationship that this expression is owned by, if it is owned. - * A node/relationship owns its label/key/property mappings + * Returns the node/relationship that this expression is owned by, if it is owned. A + * node/relationship owns its label/key/property mappings */ def owner: Option[Var] = None def withOwner(v: Var): Expr = this def as(alias: Var) = AliasExpr(this, alias) + /** - * When `nullInNullOut` is true, then the expression evaluates to `null`, if any of its inputs evaluate to `null`. + * When `nullInNullOut` is true, then the expression evaluates to `null`, if any of its inputs + * evaluate to `null`. * * Essentially it means that `null` values pass up the evaluation chain from children to parents. */ @@ -106,17 +110,18 @@ sealed trait Var extends Expr { object Var { def unapply(arg: Var): Option[String] = Some(arg.name) def unnamed: CypherType => Var = apply("") - def apply(name: String)(cypherType: CypherType = CTAny): Var = cypherType match { - case n if n.subTypeOf(CTNode.nullable) => NodeVar(name)(n) - case r if r.subTypeOf(CTRelationship.nullable) => RelationshipVar(name)(r) - case _ => SimpleVar(name)(cypherType) - } + def apply(name: String)(cypherType: CypherType = CTAny): Var = + cypherType match { + case n if n.subTypeOf(CTNode.nullable) => NodeVar(name)(n) + case r if r.subTypeOf(CTRelationship.nullable) => RelationshipVar(name)(r) + case _ => SimpleVar(name)(cypherType) + } } -sealed trait TypeValidatedExpr extends Expr{ +sealed trait TypeValidatedExpr extends Expr { val cypherType: CypherType = computeCypherType def exprs: List[Expr] - def propagationType : Option[PropagationType] = None + def propagationType: Option[PropagationType] = None def computeCypherType: CypherType = { val materialTypes = exprs.map(_.cypherType.material) @@ -126,13 +131,18 @@ sealed trait TypeValidatedExpr extends Expr{ case Some(typ) => propagationType match { case Some(NullOrAnyNullable) => childNullPropagatesTo(typ) - case Some(AnyNullable) => if(exprs.exists(_.cypherType.isNullable)) typ.nullable else typ - case Some(AllNullable) => if (exprs.forall(_.cypherType.isNullable)) typ.nullable else typ + case Some(AnyNullable) => + if (exprs.exists(_.cypherType.isNullable)) typ.nullable else typ + case Some(AllNullable) => + if (exprs.forall(_.cypherType.isNullable)) typ.nullable else typ case None => typ } case None => if (children.exists(_.cypherType == CTNull)) CTNull - else throw NoSuitableSignatureForExpr(s"Type signature ${getClass.getSimpleName}($materialTypes) is not supported.") + else + throw NoSuitableSignatureForExpr( + s"Type signature ${getClass.getSimpleName}($materialTypes) is not supported." + ) } } @@ -146,21 +156,25 @@ final case class ListSegment(index: Int, listVar: Var) extends Var with TypeVali override def name: String = s"${listVar.name}($index)" override def exprs: List[Expr] = List(listVar) - override def signature(inputCypherTypes: Seq[CypherType]): Option[CypherType] = inputCypherTypes.head match { + override def signature( + inputCypherTypes: Seq[CypherType] + ): Option[CypherType] = inputCypherTypes.head match { case CTList(inner) => Some(inner.nullable) case CTUnion(options) => options .map { case CTList(inner) => Some(inner) - case CTNull => Some(CTNull) - case _ => None} - .reduceLeft[Option[CypherType]]{ - case (Some(acc),Some(x)) => Some(acc | x) - case (None, _) => None - case (_, None) => None} + case CTNull => Some(CTNull) + case _ => None + } + .reduceLeft[Option[CypherType]] { + case (Some(acc), Some(x)) => Some(acc | x) + case (None, _) => None + case (_, None) => None + } .map(_.nullable) case CTNull => Some(CTNull) - case _ => None + case _ => None } } @@ -171,29 +185,35 @@ final case class NodeVar(name: String)(val cypherType: CypherType = CTNode) exte override def withOwner(expr: Var): NodeVar = expr match { case n: NodeVar => n - case other => other.cypherType match { - case n if n.subTypeOf(CTNode.nullable) => NodeVar(other.name)(n) - case o => throw IllegalArgumentException(CTNode, o) - } + case other => + other.cypherType match { + case n if n.subTypeOf(CTNode.nullable) => NodeVar(other.name)(n) + case o => throw IllegalArgumentException(CTNode, o) + } } } -final case class RelationshipVar(name: String)(val cypherType: CypherType = CTRelationship) extends ReturnItem { +final case class RelationshipVar(name: String)( + val cypherType: CypherType = CTRelationship +) extends ReturnItem { override def owner: Option[Var] = Some(this) override def withOwner(expr: Var): RelationshipVar = expr match { case r: RelationshipVar => r - case other => other.cypherType match { - case r if r.subTypeOf(CTRelationship.nullable) => RelationshipVar(other.name)(r) - case o => throw IllegalArgumentException(CTRelationship, o) - } + case other => + other.cypherType match { + case r if r.subTypeOf(CTRelationship.nullable) => + RelationshipVar(other.name)(r) + case o => throw IllegalArgumentException(CTRelationship, o) + } } } final case class SimpleVar(name: String)(val cypherType: CypherType) extends ReturnItem { override def owner: Option[Var] = Some(this) - override def withOwner(expr: Var): SimpleVar = SimpleVar(expr.name)(expr.cypherType) + override def withOwner(expr: Var): SimpleVar = + SimpleVar(expr.name)(expr.cypherType) } final case class LambdaVar(name: String)(val cypherType: CypherType) extends Var @@ -205,7 +225,7 @@ final case class StartNode(rel: Expr)(val cypherType: CypherType) extends Expr { override def owner: Option[Var] = rel match { case v: Var => Some(v) - case _ => None + case _ => None } override def withOwner(v: Var): StartNode = StartNode(v)(cypherType) @@ -219,7 +239,7 @@ final case class EndNode(rel: Expr)(val cypherType: CypherType) extends Expr { override def owner: Option[Var] = rel match { case v: Var => Some(v) - case _ => None + case _ => None } override def withOwner(v: Var): EndNode = EndNode(v)(cypherType) @@ -231,11 +251,12 @@ object FlattenOps { // TODO: Implement as a rewriter instead implicit class RichExpressions(exprs: Traversable[Expr]) { - /** - * Flattens child expressions - */ - def flattenExprs[E <: Expr : ClassTag]: List[Expr] = { - @tailrec def flattenRec(es: List[Expr], result: Set[Expr] = Set.empty): Set[Expr] = { + /** Flattens child expressions */ + def flattenExprs[E <: Expr: ClassTag]: List[Expr] = { + @tailrec def flattenRec( + es: List[Expr], + result: Set[Expr] = Set.empty + ): Set[Expr] = { es match { case Nil => result case h :: tail => @@ -255,9 +276,9 @@ object FlattenOps { object Ands { def apply[E <: Expr](exprs: Set[E]): Expr = apply(exprs.toSeq: _*) def apply[E <: Expr](exprs: E*): Expr = exprs.flattenExprs[Ands] match { - case Nil => TrueLit + case Nil => TrueLit case one :: Nil => one - case other => Ands(other) + case other => Ands(other) } } @@ -274,16 +295,17 @@ final case class Ands(_exprs: List[Expr]) extends Expr { def exprs: Set[Expr] = _exprs.toSet - override def withoutType = s"ANDS(${_exprs.map(_.withoutType).mkString(", ")})" + override def withoutType = + s"ANDS(${_exprs.map(_.withoutType).mkString(", ")})" } object Ors { def apply[E <: Expr](exprs: Set[E]): Expr = apply(exprs.toSeq: _*) def apply[E <: Expr](exprs: E*): Expr = exprs.flattenExprs[Ors] match { - case Nil => TrueLit + case Nil => TrueLit case one :: Nil => one - case other => Ors(other) + case other => Ors(other) } } @@ -309,7 +331,9 @@ sealed trait PredicateExpression extends TypeValidatedExpr { override def exprs: List[Expr] = List(inner) override def propagationType: Option[PropagationType] = Some(AnyNullable) - override def signature(inputCypherTypes: Seq[CypherType]): Option[CypherType] = Some(CTBoolean) + override def signature( + inputCypherTypes: Seq[CypherType] + ): Option[CypherType] = Some(CTBoolean) } final case class Not(expr: Expr) extends PredicateExpression { @@ -323,15 +347,18 @@ final case class HasLabel(node: Expr, label: Label) extends PredicateExpression override def owner: Option[Var] = node match { case v: Var => Some(v) - case _ => None + case _ => None } override def withOwner(v: Var): HasLabel = HasLabel(v, label) override def withoutType: String = s"${node.withoutType}:${label.name}" - override def signature(inputCypherTypes: Seq[CypherType]): Option[CypherType] = inputCypherTypes.head match { + override def signature( + inputCypherTypes: Seq[CypherType] + ): Option[CypherType] = inputCypherTypes.head match { case CTNode(_, _) => Some(CTBoolean) - case u : CTUnion if u.alternatives.forall(_.subTypeOf(CTNode)) => Some(CTBoolean) + case u: CTUnion if u.alternatives.forall(_.subTypeOf(CTNode)) => + Some(CTBoolean) case _ => None } } @@ -341,15 +368,18 @@ final case class HasType(rel: Expr, relType: RelType) extends PredicateExpressio override def owner: Option[Var] = rel match { case v: Var => Some(v) - case _ => None + case _ => None } override def withOwner(v: Var): HasType = HasType(v, relType) override def withoutType: String = s"${rel.withoutType}:${relType.name}" - override def signature(inputCypherTypes: Seq[CypherType]): Option[CypherType] = inputCypherTypes.head match { - case CTRelationship(_,_) => Some(CTBoolean) - case u : CTUnion if u.alternatives.forall(_.subTypeOf(CTRelationship)) => Some(CTBoolean) + override def signature( + inputCypherTypes: Seq[CypherType] + ): Option[CypherType] = inputCypherTypes.head match { + case CTRelationship(_, _) => Some(CTBoolean) + case u: CTUnion if u.alternatives.forall(_.subTypeOf(CTRelationship)) => + Some(CTBoolean) case _ => None } } @@ -366,88 +396,117 @@ final case class IsNotNull(expr: Expr) extends PredicateExpression { override def withoutType: String = s"type(${expr.withoutType}) IS NOT NULL" } -final case class StartsWith(lhs: Expr, rhs: Expr) extends BinaryPredicate with BinaryStrExprSignature { +final case class StartsWith(lhs: Expr, rhs: Expr) + extends BinaryPredicate + with BinaryStrExprSignature { override def op: String = "STARTS WITH" } -final case class EndsWith(lhs: Expr, rhs: Expr) extends BinaryPredicate with BinaryStrExprSignature { +final case class EndsWith(lhs: Expr, rhs: Expr) + extends BinaryPredicate + with BinaryStrExprSignature { override def op: String = "ENDS WITH" } -final case class Contains(lhs: Expr, rhs: Expr) extends BinaryPredicate with BinaryStrExprSignature { +final case class Contains(lhs: Expr, rhs: Expr) + extends BinaryPredicate + with BinaryStrExprSignature { override def op: String = "CONTAINS" } // Binary expressions sealed trait BinaryStrExprSignature { - def signature(lhsType : CypherType, rhsType : CypherType): Option[CypherType] = lhsType -> rhsType match { - case (CTString, CTString) => Some(CTBoolean) - case _ => None - } + def signature(lhsType: CypherType, rhsType: CypherType): Option[CypherType] = + lhsType -> rhsType match { + case (CTString, CTString) => Some(CTBoolean) + case _ => None + } } sealed trait InequalityExprSignature { - def signature(lhsType: CypherType, rhsType: CypherType): Option[CypherType] = lhsType -> rhsType match { - case (n1, n2) if n1.subTypeOf(CTNumber) && n2.subTypeOf(CTNumber) => Some(CTBoolean) - case (c1, c2) if c1.couldBeSameTypeAs(c2) => Some(CTBoolean) - case _ => Some(CTVoid) - } + def signature(lhsType: CypherType, rhsType: CypherType): Option[CypherType] = + lhsType -> rhsType match { + case (n1, n2) if n1.subTypeOf(CTNumber) && n2.subTypeOf(CTNumber) => + Some(CTBoolean) + case (c1, c2) if c1.couldBeSameTypeAs(c2) => Some(CTBoolean) + case _ => Some(CTVoid) + } } sealed trait BinaryExpr extends TypeValidatedExpr { override final def toString = s"$lhs $op $rhs" - override final def withoutType: String = s"${lhs.withoutType} $op ${rhs.withoutType}" + override final def withoutType: String = + s"${lhs.withoutType} $op ${rhs.withoutType}" def lhs: Expr def rhs: Expr def op: String override def exprs: List[Expr] = List(lhs, rhs) def signature(lhsType: CypherType, rhsType: CypherType): Option[CypherType] - override def signature(inputCypherTypes: Seq[CypherType]): Option[CypherType] = signature(inputCypherTypes.head, inputCypherTypes(1)) + override def signature( + inputCypherTypes: Seq[CypherType] + ): Option[CypherType] = signature(inputCypherTypes.head, inputCypherTypes(1)) } sealed trait BinaryPredicate extends BinaryExpr { - override def propagationType: Option[PropagationType] = Some(NullOrAnyNullable) + override def propagationType: Option[PropagationType] = Some( + NullOrAnyNullable + ) } final case class Equals(lhs: Expr, rhs: Expr) extends BinaryPredicate { override val op = "=" - override def signature(lhsType: CypherType, rhsType: CypherType): Option[CypherType] = Some(CTBoolean) + override def signature( + lhsType: CypherType, + rhsType: CypherType + ): Option[CypherType] = Some(CTBoolean) } -final case class RegexMatch(lhs: Expr, rhs: Expr) extends BinaryPredicate with BinaryStrExprSignature { +final case class RegexMatch(lhs: Expr, rhs: Expr) + extends BinaryPredicate + with BinaryStrExprSignature { override def op: String = "=~" } -final case class LessThan(lhs: Expr, rhs: Expr) extends BinaryPredicate with InequalityExprSignature { +final case class LessThan(lhs: Expr, rhs: Expr) + extends BinaryPredicate + with InequalityExprSignature { override val op = "<" } -final case class LessThanOrEqual(lhs: Expr, rhs: Expr) extends BinaryPredicate with InequalityExprSignature { +final case class LessThanOrEqual(lhs: Expr, rhs: Expr) + extends BinaryPredicate + with InequalityExprSignature { override val op = "<=" } -final case class GreaterThan(lhs: Expr, rhs: Expr) extends BinaryPredicate with InequalityExprSignature { +final case class GreaterThan(lhs: Expr, rhs: Expr) + extends BinaryPredicate + with InequalityExprSignature { override val op = ">" } -final case class GreaterThanOrEqual(lhs: Expr, rhs: Expr) extends BinaryPredicate with InequalityExprSignature { +final case class GreaterThanOrEqual(lhs: Expr, rhs: Expr) + extends BinaryPredicate + with InequalityExprSignature { override val op = ">=" } final case class In(lhs: Expr, rhs: Expr) extends BinaryPredicate { override val op = "IN" - override def computeCypherType: CypherType = lhs.cypherType -> rhs.cypherType match { - case (_, CTEmptyList) => CTFalse - case (CTNull, _) => CTNull - case (l, _) if l.isNullable => CTBoolean.nullable - case (_, CTList(inner)) if inner.isNullable => CTBoolean.nullable - case (l, CTList(inner)) if !l.couldBeSameTypeAs(inner) => CTFalse - case _ => childNullPropagatesTo(CTBoolean) - } + override def computeCypherType: CypherType = + lhs.cypherType -> rhs.cypherType match { + case (_, CTEmptyList) => CTFalse + case (CTNull, _) => CTNull + case (l, _) if l.isNullable => CTBoolean.nullable + case (_, CTList(inner)) if inner.isNullable => CTBoolean.nullable + case (l, CTList(inner)) if !l.couldBeSameTypeAs(inner) => CTFalse + case _ => childNullPropagatesTo(CTBoolean) + } - def signature(lhsType: CypherType, rhsType: CypherType): Option[CypherType] = None //signatures not used as In to special + def signature(lhsType: CypherType, rhsType: CypherType): Option[CypherType] = + None // signatures not used as In to special } sealed trait Property extends Expr { @@ -458,16 +517,19 @@ sealed trait Property extends Expr { override def owner: Option[Var] = propertyOwner match { case v: Var => Some(v) - case _ => None + case _ => None } override def withoutType: String = s"${propertyOwner.withoutType}.${key.name}" } -final case class ElementProperty(propertyOwner: Expr, key: PropertyKey)(val cypherType: CypherType) extends Property { +final case class ElementProperty(propertyOwner: Expr, key: PropertyKey)( + val cypherType: CypherType +) extends Property { - override def withOwner(v: Var): ElementProperty = ElementProperty(v, key)(cypherType) + override def withOwner(v: Var): ElementProperty = + ElementProperty(v, key)(cypherType) } @@ -477,7 +539,9 @@ final case class MapProperty(propertyOwner: Expr, key: PropertyKey) extends Prop case CTMap(inner) => inner.getOrElse(key.name, CTNull) case other => - throw IllegalArgumentException(s"Map property needs to be defined on a map. `$propertyOwner` is of type `$other`.") + throw IllegalArgumentException( + s"Map property needs to be defined on a map. `$propertyOwner` is of type `$other`." + ) } override def withOwner(v: Var): MapProperty = MapProperty(v, key) @@ -496,7 +560,8 @@ final case class LocalDateTimeProperty(propertyOwner: Expr, key: PropertyKey) ex val cypherType: CypherType = CTInteger.asNullableAs(propertyOwner.cypherType) - override def withOwner(v: Var): LocalDateTimeProperty = LocalDateTimeProperty(v, key) + override def withOwner(v: Var): LocalDateTimeProperty = + LocalDateTimeProperty(v, key) } @@ -512,12 +577,19 @@ final case class MapExpression(items: Map[String, Expr]) extends Expr { override def withoutType: String = s"{${items.mapValues(_.withoutType)}}" - override def cypherType: CypherType = CTMap(items.map { case (key, value) => key -> value.cypherType }) + override def cypherType: CypherType = CTMap(items.map { case (key, value) => + key -> value.cypherType + }) } -final case class MapProjection(mapOwner: Var, items: Seq[(String, Expr)], includeAllProps: Boolean) extends Expr { +final case class MapProjection( + mapOwner: Var, + items: Seq[(String, Expr)], + includeAllProps: Boolean +) extends Expr { def cypherType: CypherType = CTMap - def withoutType: String = s"{${items.map(x => x._1 + ":"+ x._2.withoutType)}}" + def withoutType: String = + s"{${items.map(x => x._1 + ":" + x._2.withoutType)}}" } // Arithmetic expressions @@ -534,46 +606,54 @@ final case class Add(lhs: Expr, rhs: Expr) extends ArithmeticExpr { override val op = "+" - override def signature(lhsType: CypherType, rhsType: CypherType): Option[CypherType] = lhsType -> rhsType match { - case (CTVoid, _) | (_, CTVoid) => Some(CTNull) - case (left: CTList, r) => Some(listConcatJoin(left, r)) - case (l, right: CTList) => Some(listConcatJoin(right, l)) + override def signature( + lhsType: CypherType, + rhsType: CypherType + ): Option[CypherType] = lhsType -> rhsType match { + case (CTVoid, _) | (_, CTVoid) => Some(CTNull) + case (left: CTList, r) => Some(listConcatJoin(left, r)) + case (l, right: CTList) => Some(listConcatJoin(right, l)) case (CTString, r) if r.subTypeOf(CTNumber) => Some(CTString) case (l, CTString) if l.subTypeOf(CTNumber) => Some(CTString) - case (CTString, CTString) => Some(CTString) - case (CTDuration, CTDuration) => Some(CTDuration) - case (CTLocalDateTime, CTDuration) => Some(CTLocalDateTime) - case (CTDuration, CTLocalDateTime) => Some(CTLocalDateTime) - case (CTDate, CTDuration) => Some(CTDate) - case (CTDuration, CTDate) => Some(CTDate) - case (CTInteger, CTInteger) => Some(CTInteger) - case (CTFloat, CTInteger) => Some(CTFloat) - case (CTInteger, CTFloat) => Some(CTFloat) - case (CTFloat, CTFloat) => Some(CTFloat) - case (left, right) => BigDecimalSignatures.arithmeticSignature(Addition)(Seq(left, right)) - } - - - def listConcatJoin(lhsType: CTList, rhsType: CypherType): CypherType = lhsType -> rhsType match { - case (CTList(lInner), CTList(rInner)) => CTList(lInner join rInner) - case (CTList(lInner), _) => CTList(lInner join rhsType) - } + case (CTString, CTString) => Some(CTString) + case (CTDuration, CTDuration) => Some(CTDuration) + case (CTLocalDateTime, CTDuration) => Some(CTLocalDateTime) + case (CTDuration, CTLocalDateTime) => Some(CTLocalDateTime) + case (CTDate, CTDuration) => Some(CTDate) + case (CTDuration, CTDate) => Some(CTDate) + case (CTInteger, CTInteger) => Some(CTInteger) + case (CTFloat, CTInteger) => Some(CTFloat) + case (CTInteger, CTFloat) => Some(CTFloat) + case (CTFloat, CTFloat) => Some(CTFloat) + case (left, right) => + BigDecimalSignatures.arithmeticSignature(Addition)(Seq(left, right)) + } + + def listConcatJoin(lhsType: CTList, rhsType: CypherType): CypherType = + lhsType -> rhsType match { + case (CTList(lInner), CTList(rInner)) => CTList(lInner join rInner) + case (CTList(lInner), _) => CTList(lInner join rhsType) + } } final case class Subtract(lhs: Expr, rhs: Expr) extends ArithmeticExpr { override val op = "-" - override def signature(lhsType: CypherType, rhsType: CypherType): Option[CypherType] = lhsType -> rhsType match { - case (CTVoid, _) | (_, CTVoid) => Some(CTNull) - case (CTInteger, CTInteger) => Some(CTInteger) - case (CTFloat, CTFloat) => Some(CTFloat) - case (CTInteger, CTFloat) => Some(CTFloat) - case (CTFloat, CTInteger) => Some(CTFloat) - case (CTDuration, CTDuration) => Some(CTDuration) + override def signature( + lhsType: CypherType, + rhsType: CypherType + ): Option[CypherType] = lhsType -> rhsType match { + case (CTVoid, _) | (_, CTVoid) => Some(CTNull) + case (CTInteger, CTInteger) => Some(CTInteger) + case (CTFloat, CTFloat) => Some(CTFloat) + case (CTInteger, CTFloat) => Some(CTFloat) + case (CTFloat, CTInteger) => Some(CTFloat) + case (CTDuration, CTDuration) => Some(CTDuration) case (CTLocalDateTime, CTDuration) => Some(CTLocalDateTime) - case (CTDate, CTDuration) => Some(CTDate) - case (left, right) => BigDecimalSignatures.arithmeticSignature(Addition)(Seq(left, right)) + case (CTDate, CTDuration) => Some(CTDate) + case (left, right) => + BigDecimalSignatures.arithmeticSignature(Addition)(Seq(left, right)) } } @@ -581,17 +661,21 @@ final case class Multiply(lhs: Expr, rhs: Expr) extends ArithmeticExpr { override val op = "*" - override def signature(lhsType: CypherType, rhsType: CypherType): Option[CypherType] = lhsType -> rhsType match { + override def signature( + lhsType: CypherType, + rhsType: CypherType + ): Option[CypherType] = lhsType -> rhsType match { case (CTVoid, _) | (_, CTVoid) => Some(CTNull) - case (CTInteger, CTInteger) => Some(CTInteger) - case (CTFloat, CTFloat) => Some(CTFloat) - case (CTInteger, CTFloat) => Some(CTFloat) - case (CTFloat, CTInteger) => Some(CTFloat) - case (CTDuration, CTFloat) => Some(CTDuration) - case (CTDuration, CTInteger) => Some(CTDuration) - case (CTFloat, CTDuration) => Some(CTDuration) - case (CTInteger, CTDuration) => Some(CTDuration) - case (left, right) => BigDecimalSignatures.arithmeticSignature(Multiplication)(Seq(left, right)) + case (CTInteger, CTInteger) => Some(CTInteger) + case (CTFloat, CTFloat) => Some(CTFloat) + case (CTInteger, CTFloat) => Some(CTFloat) + case (CTFloat, CTInteger) => Some(CTFloat) + case (CTDuration, CTFloat) => Some(CTDuration) + case (CTDuration, CTInteger) => Some(CTDuration) + case (CTFloat, CTDuration) => Some(CTDuration) + case (CTInteger, CTDuration) => Some(CTDuration) + case (left, right) => + BigDecimalSignatures.arithmeticSignature(Multiplication)(Seq(left, right)) } } @@ -599,15 +683,19 @@ final case class Divide(lhs: Expr, rhs: Expr) extends ArithmeticExpr { override val op = "/" - override def signature(lhsType: CypherType, rhsType: CypherType): Option[CypherType] = lhsType -> rhsType match { + override def signature( + lhsType: CypherType, + rhsType: CypherType + ): Option[CypherType] = lhsType -> rhsType match { case (CTVoid, _) | (_, CTVoid) => Some(CTNull) - case (CTInteger, CTInteger) => Some(CTInteger) - case (CTFloat, CTFloat) => Some(CTFloat) - case (CTInteger, CTFloat) => Some(CTFloat) - case (CTFloat, CTInteger) => Some(CTFloat) - case (CTDuration, CTFloat) => Some(CTDuration) - case (CTDuration, CTInteger) => Some(CTDuration) - case (left, right) => BigDecimalSignatures.arithmeticSignature(Division)(Seq(left, right)) + case (CTInteger, CTInteger) => Some(CTInteger) + case (CTFloat, CTFloat) => Some(CTFloat) + case (CTInteger, CTFloat) => Some(CTFloat) + case (CTFloat, CTInteger) => Some(CTFloat) + case (CTDuration, CTFloat) => Some(CTDuration) + case (CTDuration, CTInteger) => Some(CTDuration) + case (left, right) => + BigDecimalSignatures.arithmeticSignature(Division)(Seq(left, right)) } } @@ -615,27 +703,33 @@ final case class Modulo(lhs: Expr, rhs: Expr) extends ArithmeticExpr { override val op = "%" - override def signature(lhsType: CypherType, rhsType: CypherType): Option[CypherType] = lhsType -> rhsType match { + override def signature( + lhsType: CypherType, + rhsType: CypherType + ): Option[CypherType] = lhsType -> rhsType match { case (CTVoid, _) | (_, CTVoid) => Some(CTNull) - case (CTInteger, CTInteger) => Some(CTInteger) - case (CTFloat, CTFloat) => Some(CTFloat) - case (CTInteger, CTFloat) => Some(CTFloat) - case (CTFloat, CTInteger) => Some(CTFloat) - case _ => None + case (CTInteger, CTInteger) => Some(CTInteger) + case (CTFloat, CTFloat) => Some(CTFloat) + case (CTInteger, CTFloat) => Some(CTFloat) + case (CTFloat, CTInteger) => Some(CTFloat) + case _ => None } } // Functions sealed trait FunctionExpr extends TypeValidatedExpr { override final def toString = s"$name(${exprs.mkString(", ")})" - override final def withoutType = s"$name(${exprs.map(_.withoutType).mkString(", ")})" + override final def withoutType = + s"$name(${exprs.map(_.withoutType).mkString(", ")})" def name: String = this.getClass.getSimpleName.toLowerCase } sealed trait NullaryFunctionExpr extends FunctionExpr { def exprs: List[Expr] = List.empty[Expr] - override def signature(inputCypherTypes: Seq[CypherType]): Option[CypherType] = Some(cypherType) + override def signature( + inputCypherTypes: Seq[CypherType] + ): Option[CypherType] = Some(cypherType) } final case class MonotonicallyIncreasingId() extends NullaryFunctionExpr { @@ -648,30 +742,36 @@ sealed trait UnaryFunctionExpr extends FunctionExpr { def exprs: List[Expr] = List(expr) def signature(inputCypherType: CypherType): Option[CypherType] - def signature(inputCypherTypes: Seq[CypherType]): Option[CypherType] = signature(inputCypherTypes.head) + def signature(inputCypherTypes: Seq[CypherType]): Option[CypherType] = + signature(inputCypherTypes.head) } -final case class Head(expr: Expr) extends UnaryFunctionExpr{ - override def signature(inputCypherType: CypherType): Option[CypherType] = inputCypherType match { - case CTList(inner) => Some(inner) - case _ => None - } +final case class Head(expr: Expr) extends UnaryFunctionExpr { + override def signature(inputCypherType: CypherType): Option[CypherType] = + inputCypherType match { + case CTList(inner) => Some(inner) + case _ => None + } } -final case class Last(expr: Expr) extends UnaryFunctionExpr{ - override def signature(inputCypherType: CypherType): Option[CypherType] = inputCypherType match { - case CTList(inner) => Some(inner) - case _ => None - } +final case class Last(expr: Expr) extends UnaryFunctionExpr { + override def signature(inputCypherType: CypherType): Option[CypherType] = + inputCypherType match { + case CTList(inner) => Some(inner) + case _ => None + } } final case class Id(expr: Expr) extends UnaryFunctionExpr { - override def propagationType: Option[PropagationType] = Some(NullOrAnyNullable) - - def signature(inputCypherType: CypherType): Option[CypherType] = inputCypherType match { - case CTNode(_, _) | CTRelationship(_, _) => Some(CTIdentity) - case _ => None - } + override def propagationType: Option[PropagationType] = Some( + NullOrAnyNullable + ) + + def signature(inputCypherType: CypherType): Option[CypherType] = + inputCypherType match { + case CTNode(_, _) | CTRelationship(_, _) => Some(CTIdentity) + case _ => None + } } object PrefixId { @@ -679,120 +779,152 @@ object PrefixId { } final case class PrefixId(expr: Expr, prefix: GraphIdPrefix) extends UnaryFunctionExpr { - override def propagationType: Option[PropagationType] = Some(NullOrAnyNullable) - - def signature(inputCypherType: CypherType): Option[CypherType] = inputCypherType match { - case CTIdentity => Some(CTIdentity) - case _ => None - } + override def propagationType: Option[PropagationType] = Some( + NullOrAnyNullable + ) + + def signature(inputCypherType: CypherType): Option[CypherType] = + inputCypherType match { + case CTIdentity => Some(CTIdentity) + case _ => None + } } final case class ToId(expr: Expr) extends UnaryFunctionExpr { - override def propagationType: Option[PropagationType] = Some(NullOrAnyNullable) + override def propagationType: Option[PropagationType] = Some( + NullOrAnyNullable + ) - def signature(inputCypherType: CypherType): Option[CypherType] = inputCypherType match { - case CTInteger | CTIdentity => Some(CTIdentity) + def signature(inputCypherType: CypherType): Option[CypherType] = + inputCypherType match { + case CTInteger | CTIdentity => Some(CTIdentity) case x if x.subTypeOf(CTElement) => Some(CTIdentity) - case _ => None - } + case _ => None + } } final case class Labels(expr: Expr) extends UnaryFunctionExpr { - override def propagationType: Option[PropagationType] = Some(NullOrAnyNullable) - - def signature(inputCypherType: CypherType): Option[CypherType] = inputCypherType match { - case CTNode(_, _) => Some(CTList(CTString)) - case _ => None - } + override def propagationType: Option[PropagationType] = Some( + NullOrAnyNullable + ) + + def signature(inputCypherType: CypherType): Option[CypherType] = + inputCypherType match { + case CTNode(_, _) => Some(CTList(CTString)) + case _ => None + } } final case class Type(expr: Expr) extends UnaryFunctionExpr { - override def propagationType: Option[PropagationType] = Some(NullOrAnyNullable) - - def signature(inputCypherType: CypherType): Option[CypherType] = inputCypherType match { - case CTRelationship(_, _) => Some(CTString) - case _ => None - } + override def propagationType: Option[PropagationType] = Some( + NullOrAnyNullable + ) + + def signature(inputCypherType: CypherType): Option[CypherType] = + inputCypherType match { + case CTRelationship(_, _) => Some(CTString) + case _ => None + } } final case class Exists(expr: Expr) extends UnaryFunctionExpr { - def signature(inputCypherType: CypherType): Option[CypherType] = Some(CTBoolean) + def signature(inputCypherType: CypherType): Option[CypherType] = Some( + CTBoolean + ) } final case class Size(expr: Expr) extends UnaryFunctionExpr { - override def propagationType: Option[PropagationType] = Some(NullOrAnyNullable) - - def signature(inputCypherType: CypherType): Option[CypherType] = inputCypherType match { - case CTList(_) | CTString => Some(CTInteger) - case _ => None - } + override def propagationType: Option[PropagationType] = Some( + NullOrAnyNullable + ) + + def signature(inputCypherType: CypherType): Option[CypherType] = + inputCypherType match { + case CTList(_) | CTString => Some(CTInteger) + case _ => None + } } final case class Keys(expr: Expr) extends UnaryFunctionExpr { - override def propagationType: Option[PropagationType] = Some(NullOrAnyNullable) - - def signature(inputCypherType: CypherType): Option[CypherType] = inputCypherType match { - case CTNode(_, _) | CTRelationship(_, _) | CTMap(_) => Some(CTList(CTString)) - case _ => None - } + override def propagationType: Option[PropagationType] = Some( + NullOrAnyNullable + ) + + def signature(inputCypherType: CypherType): Option[CypherType] = + inputCypherType match { + case CTNode(_, _) | CTRelationship(_, _) | CTMap(_) => + Some(CTList(CTString)) + case _ => None + } } final case class StartNodeFunction(expr: Expr) extends UnaryFunctionExpr { - override def propagationType: Option[PropagationType] = Some(NullOrAnyNullable) - - def signature(inputCypherType: CypherType): Option[CypherType] = inputCypherType match { - case CTRelationship(_, _) => Some(CTNode) - case _ => None - } + override def propagationType: Option[PropagationType] = Some( + NullOrAnyNullable + ) + + def signature(inputCypherType: CypherType): Option[CypherType] = + inputCypherType match { + case CTRelationship(_, _) => Some(CTNode) + case _ => None + } } final case class EndNodeFunction(expr: Expr) extends UnaryFunctionExpr { - override def propagationType: Option[PropagationType] = Some(NullOrAnyNullable) - - def signature(inputCypherType: CypherType): Option[CypherType] = inputCypherType match { - case CTRelationship(_, _) => Some(CTNode) - case _ => None - } + override def propagationType: Option[PropagationType] = Some( + NullOrAnyNullable + ) + + def signature(inputCypherType: CypherType): Option[CypherType] = + inputCypherType match { + case CTRelationship(_, _) => Some(CTNode) + case _ => None + } } final case class ToFloat(expr: Expr) extends UnaryFunctionExpr { override def propagationType: Option[PropagationType] = Some(AnyNullable) - def signature(inputCypherType: CypherType): Option[CypherType] = inputCypherType match { - case CTString => Some(CTFloat) + def signature(inputCypherType: CypherType): Option[CypherType] = + inputCypherType match { + case CTString => Some(CTFloat) case x if x.subTypeOf(CTNumber) => Some(CTFloat) - case _ => None + case _ => None } } final case class ToInteger(expr: Expr) extends UnaryFunctionExpr { override def propagationType: Option[PropagationType] = Some(AnyNullable) - def signature(inputCypherType: CypherType): Option[CypherType] = inputCypherType match { - case CTString => Some(CTInteger) - case x if x.subTypeOf(CTNumber) => Some(CTInteger) - case _ => None - } + def signature(inputCypherType: CypherType): Option[CypherType] = + inputCypherType match { + case CTString => Some(CTInteger) + case x if x.subTypeOf(CTNumber) => Some(CTInteger) + case _ => None + } } final case class ToString(expr: Expr) extends UnaryFunctionExpr { override def propagationType: Option[PropagationType] = Some(AnyNullable) - def signature(inputCypherType: CypherType): Option[CypherType] = inputCypherType match { - case CTString => Some(CTString) - case x if x.subTypeOf(CTUnion(CTNumber, CTTemporal, CTBoolean)) => Some(CTString) - case _ => None - } + def signature(inputCypherType: CypherType): Option[CypherType] = + inputCypherType match { + case CTString => Some(CTString) + case x if x.subTypeOf(CTUnion(CTNumber, CTTemporal, CTBoolean)) => + Some(CTString) + case _ => None + } } final case class ToBoolean(expr: Expr) extends UnaryFunctionExpr { override def propagationType: Option[PropagationType] = Some(AnyNullable) - def signature(inputCypherType: CypherType): Option[CypherType] = inputCypherType match { - case CTString => Some(CTBoolean) - case x if x.subTypeOf(CTBoolean) => Some(CTBoolean) - case _ => None - } + def signature(inputCypherType: CypherType): Option[CypherType] = + inputCypherType match { + case CTString => Some(CTBoolean) + case x if x.subTypeOf(CTBoolean) => Some(CTBoolean) + case _ => None + } } object BigDecimal { @@ -801,14 +933,19 @@ object BigDecimal { final case class BigDecimal(expr: Expr, precision: Long, scale: Long) extends UnaryFunctionExpr { if (scale > precision) { - throw IllegalArgumentException("Greater precision than scale", s"precision: $precision, scale: $scale") + throw IllegalArgumentException( + "Greater precision than scale", + s"precision: $precision, scale: $scale" + ) } override def propagationType: Option[PropagationType] = Some(AnyNullable) - def signature(inputCypherType: CypherType): Option[CypherType] = inputCypherType match { - case x if x.subTypeOf(CTNumber) => Some(CTBigDecimal(precision.toInt, scale.toInt)) - case _ => None - } + def signature(inputCypherType: CypherType): Option[CypherType] = + inputCypherType match { + case x if x.subTypeOf(CTNumber) => + Some(CTBigDecimal(precision.toInt, scale.toInt)) + case _ => None + } } final case class Coalesce(exprs: List[Expr]) extends FunctionExpr { @@ -816,7 +953,9 @@ final case class Coalesce(exprs: List[Expr]) extends FunctionExpr { override def propagationType: Option[PropagationType] = Some(AllNullable) - override def signature(inputCypherTypes: Seq[CypherType]): Option[CypherType] = Some(inputCypherTypes.reduceLeft(_ | _)) + override def signature( + inputCypherTypes: Seq[CypherType] + ): Option[CypherType] = Some(inputCypherTypes.reduceLeft(_ | _)) } @@ -830,25 +969,35 @@ final case class Explode(expr: Expr) extends Expr { case CTList(inner) => inner case CTUnion(options) => - options.map { - case CTList(inner) => inner - case CTNull => CTNull - case _ => throw IllegalArgumentException("input cypher type to resolve to a list", expr.cypherType) - }.reduceLeft(_ | _) + options + .map { + case CTList(inner) => inner + case CTNull => CTNull + case _ => + throw IllegalArgumentException( + "input cypher type to resolve to a list", + expr.cypherType + ) + } + .reduceLeft(_ | _) case CTNull => CTVoid - case _ => throw IllegalArgumentException("input cypher type to resolve to a list", expr.cypherType) + case _ => + throw IllegalArgumentException( + "input cypher type to resolve to a list", + expr.cypherType + ) } } -sealed trait UnaryStringFunctionExpr extends UnaryFunctionExpr{ +sealed trait UnaryStringFunctionExpr extends UnaryFunctionExpr { override def propagationType: Option[PropagationType] = Some(AnyNullable) def signature(cypherType: CypherType): Option[CypherType] = cypherType match { case CTString => Some(CTString) - case _ => None + case _ => None } } @@ -862,20 +1011,23 @@ final case class ToUpper(expr: Expr) extends UnaryStringFunctionExpr final case class ToLower(expr: Expr) extends UnaryStringFunctionExpr -final case class Properties(expr: Expr)(override val cypherType: CypherType) extends UnaryFunctionExpr { - //actually not used here as type already checked at ExpressionConverter - override def signature(inputCypherType: CypherType): Option[CypherType] = inputCypherType match { - case CTNode(_, _) | CTRelationship(_, _) | CTMap(_) => Some(CTMap) - case _ => None - } +final case class Properties(expr: Expr)(override val cypherType: CypherType) + extends UnaryFunctionExpr { + // actually not used here as type already checked at ExpressionConverter + override def signature(inputCypherType: CypherType): Option[CypherType] = + inputCypherType match { + case CTNode(_, _) | CTRelationship(_, _) | CTMap(_) => Some(CTMap) + case _ => None + } } -final case class Reverse(expr: Expr) extends UnaryFunctionExpr{ - def signature(inputCypherType: CypherType): Option[CypherType] = inputCypherType match { - case l : CTList => Some(l) - case CTString => Some(CTString) - case _ => None - } +final case class Reverse(expr: Expr) extends UnaryFunctionExpr { + def signature(inputCypherType: CypherType): Option[CypherType] = + inputCypherType match { + case l: CTList => Some(l) + case CTString => Some(CTString) + case _ => None + } } // NAry Function expressions @@ -883,46 +1035,55 @@ final case class Reverse(expr: Expr) extends UnaryFunctionExpr{ final case class Range(from: Expr, to: Expr, o: Option[Expr]) extends FunctionExpr { override def exprs: List[Expr] = o match { case Some(e) => List(from, to, e) - case None => List(from, to) + case None => List(from, to) } - override def signature(inputCypherTypes: Seq[CypherType]): Option[CypherType] = inputCypherTypes match { - case Seq(CTInteger, CTInteger) => Some(CTList(CTInteger)) + override def signature( + inputCypherTypes: Seq[CypherType] + ): Option[CypherType] = inputCypherTypes match { + case Seq(CTInteger, CTInteger) => Some(CTList(CTInteger)) case Seq(CTInteger, CTInteger, CTInteger) => Some(CTList(CTInteger)) - case _ => None + case _ => None } } final case class Replace(original: Expr, search: Expr, replacement: Expr) extends FunctionExpr { override def exprs: List[Expr] = List(original, search, replacement) - override def signature(inputCypherTypes: Seq[CypherType]): Option[CypherType] = inputCypherTypes match { + override def signature( + inputCypherTypes: Seq[CypherType] + ): Option[CypherType] = inputCypherTypes match { case Seq(CTString, CTString, CTString) => Some(CTString) - case _ => None + case _ => None } } final case class Substring(original: Expr, start: Expr, length: Option[Expr]) extends FunctionExpr { override def exprs: List[Expr] = length match { case Some(l) => List(original, start, l) - case None => List(original, start) + case None => List(original, start) } - override def signature(inputCypherTypes: Seq[CypherType]): Option[CypherType] = inputCypherTypes match { - case Seq(CTString, CTInteger) => Some(CTString) + override def signature( + inputCypherTypes: Seq[CypherType] + ): Option[CypherType] = inputCypherTypes match { + case Seq(CTString, CTInteger) => Some(CTString) case Seq(CTString, CTInteger, CTInteger) => Some(CTString) - case _ => None + case _ => None } } final case class Split(original: Expr, delimiter: Expr) extends FunctionExpr { def exprs: List[Expr] = List(original, delimiter) - def signature(inputCypherTypes: Seq[CypherType]): Option[CypherType] = inputCypherTypes match { - case Seq(CTString, CTString) => Some(CTList(CTString)) - case _ => None - } + def signature(inputCypherTypes: Seq[CypherType]): Option[CypherType] = + inputCypherTypes match { + case Seq(CTString, CTString) => Some(CTList(CTString)) + case _ => None + } - override def propagationType: Option[PropagationType] = Some(NullOrAnyNullable) + override def propagationType: Option[PropagationType] = Some( + NullOrAnyNullable + ) } // Bit operators @@ -934,7 +1095,10 @@ final case class ShiftLeft(value: Expr, shiftBits: IntegerLit) extends BinaryExp override def rhs: Expr = shiftBits override def op: String = "<<" - override def signature(lhsType: CypherType, rhsType: CypherType): Option[CypherType] = Some(value.cypherType) + override def signature( + lhsType: CypherType, + rhsType: CypherType + ): Option[CypherType] = Some(value.cypherType) } final case class ShiftRightUnsigned(value: Expr, shiftBits: IntegerLit) extends BinaryExpr { @@ -944,31 +1108,43 @@ final case class ShiftRightUnsigned(value: Expr, shiftBits: IntegerLit) extends override def rhs: Expr = shiftBits override def op: String = ">>>" - override def signature(lhsType: CypherType, rhsType: CypherType): Option[CypherType] = Some(value.cypherType) + override def signature( + lhsType: CypherType, + rhsType: CypherType + ): Option[CypherType] = Some(value.cypherType) } final case class BitwiseAnd(lhs: Expr, rhs: Expr) extends BinaryExpr { override def op: String = "&" - override def signature(lhsType: CypherType, rhsType: CypherType): Option[CypherType] = Some(lhs.cypherType | rhs.cypherType) + override def signature( + lhsType: CypherType, + rhsType: CypherType + ): Option[CypherType] = Some(lhs.cypherType | rhs.cypherType) } final case class BitwiseOr(lhs: Expr, rhs: Expr) extends BinaryExpr { override def op: String = "|" - override def signature(lhsType: CypherType, rhsType: CypherType): Option[CypherType] = Some(lhs.cypherType | rhs.cypherType) + override def signature( + lhsType: CypherType, + rhsType: CypherType + ): Option[CypherType] = Some(lhs.cypherType | rhs.cypherType) } // Mathematical functions -sealed abstract class UnaryMathematicalFunctionExpr(outPutCypherType : CypherType) extends UnaryFunctionExpr { +sealed abstract class UnaryMathematicalFunctionExpr( + outPutCypherType: CypherType +) extends UnaryFunctionExpr { override def propagationType: Option[PropagationType] = Some(AnyNullable) - def signature(inputCypherType: CypherType): Option[CypherType] = inputCypherType match { - case n if n.subTypeOf(CTNumber) => Some(outPutCypherType) - case _ => None - } + def signature(inputCypherType: CypherType): Option[CypherType] = + inputCypherType match { + case n if n.subTypeOf(CTNumber) => Some(outPutCypherType) + case _ => None + } } final case class Sqrt(expr: Expr) extends UnaryMathematicalFunctionExpr(CTFloat) @@ -1012,11 +1188,16 @@ final case class Asin(expr: Expr) extends UnaryMathematicalFunctionExpr(CTFloat) final case class Atan(expr: Expr) extends UnaryMathematicalFunctionExpr(CTFloat) final case class Atan2(expr1: Expr, expr2: Expr) extends FunctionExpr { - override def propagationType: Option[PropagationType] = Some(NullOrAnyNullable) + override def propagationType: Option[PropagationType] = Some( + NullOrAnyNullable + ) override def exprs: List[Expr] = List(expr1, expr2) - override def signature(inputCypherTypes: Seq[CypherType]): Option[CypherType] = inputCypherTypes match { - case Seq(c1, c2) if c1.subTypeOf(CTNumber) && c2.subTypeOf(CTNumber) => Some(CTFloat) + override def signature( + inputCypherTypes: Seq[CypherType] + ): Option[CypherType] = inputCypherTypes match { + case Seq(c1, c2) if c1.subTypeOf(CTNumber) && c2.subTypeOf(CTNumber) => + Some(CTFloat) case _ => None } } @@ -1044,37 +1225,40 @@ case object Timestamp extends NullaryFunctionExpr { // Aggregators sealed trait Aggregator extends TypeValidatedExpr { override def nullInNullOut: Boolean = false - override def propagationType : Option[PropagationType] = Some(AnyNullable) + override def propagationType: Option[PropagationType] = Some(AnyNullable) } -sealed trait NullaryAggregator extends Aggregator{ +sealed trait NullaryAggregator extends Aggregator { def exprs: List[Expr] = List() def signature: Option[CypherType] - def signature(inputCypherTypes: Seq[CypherType]): Option[CypherType] = signature + def signature(inputCypherTypes: Seq[CypherType]): Option[CypherType] = + signature } -sealed trait UnaryAggregator extends Aggregator{ +sealed trait UnaryAggregator extends Aggregator { def expr: Expr def exprs: List[Expr] = List(expr) def signature(inputCypherType: CypherType): Option[CypherType] - def signature(inputCypherTypes: Seq[CypherType]): Option[CypherType] = signature(inputCypherTypes.head) + def signature(inputCypherTypes: Seq[CypherType]): Option[CypherType] = + signature(inputCypherTypes.head) } - -sealed trait NumericAggregator extends UnaryAggregator{ - def signature(inputCypherType: CypherType): Option[CypherType] = inputCypherType match { - case x if CTNumber.superTypeOf(x) => Some(x) - case _ => None - } +sealed trait NumericAggregator extends UnaryAggregator { + def signature(inputCypherType: CypherType): Option[CypherType] = + inputCypherType match { + case x if CTNumber.superTypeOf(x) => Some(x) + case _ => None + } } -sealed trait NumericAndDurationsAggregator extends UnaryAggregator{ - def signature(inputCypherType: CypherType): Option[CypherType] = inputCypherType match { - case CTDuration => Some(CTDuration) - case x if CTNumber.superTypeOf(x) => Some(x) - case _ => None - } +sealed trait NumericAndDurationsAggregator extends UnaryAggregator { + def signature(inputCypherType: CypherType): Option[CypherType] = + inputCypherType match { + case CTDuration => Some(CTDuration) + case x if CTNumber.superTypeOf(x) => Some(x) + case _ => None + } } final case class Avg(expr: Expr) extends NumericAndDurationsAggregator { @@ -1092,43 +1276,53 @@ final case class Count(expr: Expr, distinct: Boolean) extends UnaryAggregator { override def toString = s"count($expr)" override def withoutType: String = s"count(${expr.withoutType})" - def signature(inputCypherType: CypherType): Option[CypherType] = Some(CTInteger) + def signature(inputCypherType: CypherType): Option[CypherType] = Some( + CTInteger + ) } final case class Max(expr: Expr) extends UnaryAggregator { override def toString = s"max($expr)" override def withoutType: String = s"max(${expr.withoutType})" - def signature(inputCypherType: CypherType): Option[CypherType] = Some(inputCypherType) + def signature(inputCypherType: CypherType): Option[CypherType] = Some( + inputCypherType + ) } final case class Min(expr: Expr) extends UnaryAggregator { override def toString = s"min($expr)" override def withoutType: String = s"min(${expr.withoutType})" - def signature(inputCypherType: CypherType): Option[CypherType] = Some(inputCypherType) + def signature(inputCypherType: CypherType): Option[CypherType] = Some( + inputCypherType + ) } final case class PercentileCont(expr: Expr, percentile: Expr) extends Aggregator { override def toString = s"percentileCont($expr, $percentile)" - override def withoutType: String = s"percentileCont(${expr.withoutType}, ${percentile.withoutType})" + override def withoutType: String = + s"percentileCont(${expr.withoutType}, ${percentile.withoutType})" def exprs: List[Expr] = List(expr, percentile) - def signature(inputCypherTypes: Seq[CypherType]): Option[CypherType] = inputCypherTypes match { - case Seq(x, CTFloat) if x.subTypeOf(CTNumber) => Some(CTFloat) - case _ => None - } + def signature(inputCypherTypes: Seq[CypherType]): Option[CypherType] = + inputCypherTypes match { + case Seq(x, CTFloat) if x.subTypeOf(CTNumber) => Some(CTFloat) + case _ => None + } } final case class PercentileDisc(expr: Expr, percentile: Expr) extends Aggregator { override def toString = s"percentileDisc($expr, $percentile)" - override def withoutType: String = s"percentileDisc(${expr.withoutType}, ${percentile.withoutType})" + override def withoutType: String = + s"percentileDisc(${expr.withoutType}, ${percentile.withoutType})" def exprs: List[Expr] = List(expr, percentile) - def signature(inputCypherTypes: Seq[CypherType]): Option[CypherType] = inputCypherTypes match { - case Seq(x, CTFloat) if x.subTypeOf(CTNumber) => Some(x) - case _ => None - } + def signature(inputCypherTypes: Seq[CypherType]): Option[CypherType] = + inputCypherTypes match { + case Seq(x, CTFloat) if x.subTypeOf(CTNumber) => Some(x) + case _ => None + } } final case class StDev(expr: Expr) extends NumericAggregator { @@ -1141,7 +1335,6 @@ final case class StDevP(expr: Expr) extends NumericAggregator { override def withoutType: String = s"stDev(${expr.withoutType})" } - final case class Sum(expr: Expr) extends NumericAndDurationsAggregator { override def toString = s"sum($expr)" override def withoutType: String = s"sum(${expr.withoutType})" @@ -1151,7 +1344,9 @@ final case class Collect(expr: Expr, distinct: Boolean) extends UnaryAggregator override def toString = s"collect($expr)" override def withoutType: String = s"collect(${expr.withoutType})" - def signature(inputCypherType: CypherType): Option[CypherType] = Some(CTList(inputCypherType)) + def signature(inputCypherType: CypherType): Option[CypherType] = Some( + CTList(inputCypherType) + ) } // Literal expressions @@ -1163,21 +1358,30 @@ sealed trait Lit[T] extends Expr { } final case class ListLit(v: List[Expr]) extends Lit[List[Expr]] { - override def cypherType: CypherType = CTList(v.foldLeft(CTVoid: CypherType)(_ | _.cypherType)) + override def cypherType: CypherType = CTList( + v.foldLeft(CTVoid: CypherType)(_ | _.cypherType) + ) } -sealed abstract class ListSlice(maybeFrom: Option[Expr], maybeTo: Option[Expr]) extends TypeValidatedExpr { +sealed abstract class ListSlice(maybeFrom: Option[Expr], maybeTo: Option[Expr]) + extends TypeValidatedExpr { def list: Expr - override def withoutType: String = s"${list.withoutType}[${maybeFrom.map(_.withoutType).getOrElse("")}..${maybeTo.map(_.withoutType).getOrElse("")}]" - - override def propagationType: Option[PropagationType] = Some(NullOrAnyNullable) - def signature(inputCypherTypes: Seq[CypherType]): Option[CypherType] = inputCypherTypes.head match { - case CTList(_) if inputCypherTypes.tail.forall(_.couldBeSameTypeAs(CTInteger)) => Some(list.cypherType) - case _ => None - } + override def withoutType: String = + s"${list.withoutType}[${maybeFrom.map(_.withoutType).getOrElse("")}..${maybeTo.map(_.withoutType).getOrElse("")}]" + + override def propagationType: Option[PropagationType] = Some( + NullOrAnyNullable + ) + def signature(inputCypherTypes: Seq[CypherType]): Option[CypherType] = + inputCypherTypes.head match { + case CTList(_) if inputCypherTypes.tail.forall(_.couldBeSameTypeAs(CTInteger)) => + Some(list.cypherType) + case _ => None + } } -final case class ListSliceFromTo(list: Expr, from: Expr, to: Expr) extends ListSlice(Some(from), Some(to)) { +final case class ListSliceFromTo(list: Expr, from: Expr, to: Expr) + extends ListSlice(Some(from), Some(to)) { def exprs = List(list, from, to) } @@ -1189,7 +1393,12 @@ final case class ListSliceTo(list: Expr, to: Expr) extends ListSlice(None, Some( override def exprs: List[Expr] = List(list, to) } -final case class ListComprehension(variable: Expr, innerPredicate: Option[Expr], extractExpression: Option[Expr], expr : Expr) extends Expr { +final case class ListComprehension( + variable: Expr, + innerPredicate: Option[Expr], + extractExpression: Option[Expr], + expr: Expr +) extends Expr { override def withoutType: String = { val p = innerPredicate.map(" WHERE " + _.withoutType).getOrElse("") val e = extractExpression.map(" | " + _.withoutType).getOrElse("") @@ -1197,63 +1406,91 @@ final case class ListComprehension(variable: Expr, innerPredicate: Option[Expr], } override def cypherType: CypherType = extractExpression match { case Some(x) => CTList(x.cypherType) - case None => expr.cypherType + case None => expr.cypherType } } -final case class ListReduction(accumulator: Expr, variable: Expr, reduceExpr: Expr, initExpr: Expr, list: Expr) extends TypeValidatedExpr { +final case class ListReduction( + accumulator: Expr, + variable: Expr, + reduceExpr: Expr, + initExpr: Expr, + list: Expr +) extends TypeValidatedExpr { override def withoutType: String = { s"reduce(${accumulator.withoutType} = ${initExpr.withoutType}, ${variable.withoutType} IN ${list.withoutType} | ${reduceExpr.withoutType})" } - override def propagationType: Option[PropagationType] = Some(NullOrAnyNullable) - def exprs: List[Expr] = List(initExpr, reduceExpr, list) //only exprs which need to be type-checked - def signature(inputCypherTypes: Seq[CypherType]): Option[CypherType] = inputCypherTypes match { - case Seq(initType: CypherType, reduceStepType: CypherType, CTList(_)) if initType.couldBeSameTypeAs(reduceStepType) => Some(initType) - case _ => None - } + override def propagationType: Option[PropagationType] = Some( + NullOrAnyNullable + ) + def exprs: List[Expr] = + List(initExpr, reduceExpr, list) // only exprs which need to be type-checked + def signature(inputCypherTypes: Seq[CypherType]): Option[CypherType] = + inputCypherTypes match { + case Seq(initType: CypherType, reduceStepType: CypherType, CTList(_)) + if initType.couldBeSameTypeAs(reduceStepType) => + Some(initType) + case _ => None + } } -sealed abstract class IterablePredicateExpr(variable: LambdaVar, predicate: Expr, list: Expr) extends TypeValidatedExpr { - override def propagationType: Option[PropagationType] = Some(NullOrAnyNullable) +sealed abstract class IterablePredicateExpr( + variable: LambdaVar, + predicate: Expr, + list: Expr +) extends TypeValidatedExpr { + override def propagationType: Option[PropagationType] = Some( + NullOrAnyNullable + ) def exprs: List[Expr] = List(variable, predicate, list) - def signature(inputCypherTypes: Seq[CypherType]): Option[CypherType] = inputCypherTypes match { - case Seq(_, CTBoolean, CTList(_)) => Some(CTBoolean) - case _ => None - } + def signature(inputCypherTypes: Seq[CypherType]): Option[CypherType] = + inputCypherTypes match { + case Seq(_, CTBoolean, CTList(_)) => Some(CTBoolean) + case _ => None + } override def withoutType: String = s"($variable IN $list WHERE $predicate)" } -final case class ListFilter(variable: LambdaVar, predicate: Expr, list: Expr) extends TypeValidatedExpr { - override def withoutType: String = s"filter($variable IN $list WHERE $predicate)" +final case class ListFilter(variable: LambdaVar, predicate: Expr, list: Expr) + extends TypeValidatedExpr { + override def withoutType: String = + s"filter($variable IN $list WHERE $predicate)" def exprs: List[Expr] = List(variable, predicate, list) - def signature(inputCypherTypes: Seq[CypherType]): Option[CypherType] = inputCypherTypes match { - case Seq(_, CTBoolean, l: CTList) => Some(l) - case _ => None - } + def signature(inputCypherTypes: Seq[CypherType]): Option[CypherType] = + inputCypherTypes match { + case Seq(_, CTBoolean, l: CTList) => Some(l) + case _ => None + } } -final case class ListAny(variable: LambdaVar, predicate: Expr, list: Expr) extends IterablePredicateExpr(variable, predicate, list) { - override def withoutType: String = s"any(${super.withoutType})" +final case class ListAny(variable: LambdaVar, predicate: Expr, list: Expr) + extends IterablePredicateExpr(variable, predicate, list) { + override def withoutType: String = s"any(${super.withoutType})" } -final case class ListNone(variable: LambdaVar, predicate: Expr, list: Expr) extends IterablePredicateExpr(variable, predicate, list) { - override def withoutType: String = s"none(${super.withoutType})" +final case class ListNone(variable: LambdaVar, predicate: Expr, list: Expr) + extends IterablePredicateExpr(variable, predicate, list) { + override def withoutType: String = s"none(${super.withoutType})" } -final case class ListAll(variable: LambdaVar, predicate: Expr, list: Expr) extends IterablePredicateExpr(variable, predicate, list) { - override def withoutType: String = s"all(${super.withoutType})" +final case class ListAll(variable: LambdaVar, predicate: Expr, list: Expr) + extends IterablePredicateExpr(variable, predicate, list) { + override def withoutType: String = s"all(${super.withoutType})" } -final case class ListSingle(variable: LambdaVar, predicate: Expr, list: Expr) extends IterablePredicateExpr(variable, predicate, list) { - override def withoutType: String = s"single(${super.withoutType})" +final case class ListSingle(variable: LambdaVar, predicate: Expr, list: Expr) + extends IterablePredicateExpr(variable, predicate, list) { + override def withoutType: String = s"single(${super.withoutType})" } +final case class ContainerIndex(container: Expr, index: Expr)( + val cypherType: CypherType +) extends Expr { -final case class ContainerIndex(container: Expr, index: Expr)(val cypherType: CypherType) extends Expr { - - override def withoutType: String = s"${container.withoutType}[${index.withoutType}]" + override def withoutType: String = + s"${container.withoutType}[${index.withoutType}]" } @@ -1269,28 +1506,37 @@ final case class StringLit(v: String) extends Lit[String] { override val cypherType: CypherType = CTString } -sealed abstract class TemporalInstant(expr: Option[Expr], outputCypherType: CypherType) extends FunctionExpr { +sealed abstract class TemporalInstant( + expr: Option[Expr], + outputCypherType: CypherType +) extends FunctionExpr { override def exprs: List[Expr] = expr.toList - override def propagationType: Option[PropagationType] = Some(NullOrAnyNullable) + override def propagationType: Option[PropagationType] = Some( + NullOrAnyNullable + ) - override def signature(inputCypherTypes: Seq[CypherType]): Option[CypherType] = inputCypherTypes.headOption match { - case Some(CTString) | Some(CTMap(_)) => Some(outputCypherType) - case None => Some(outputCypherType) - case _ => None + override def signature( + inputCypherTypes: Seq[CypherType] + ): Option[CypherType] = inputCypherTypes.headOption match { + case Some(CTString) | Some(CTMap(_)) => Some(outputCypherType) + case None => Some(outputCypherType) + case _ => None } } -final case class LocalDateTime(maybeExpr: Option[Expr]) extends TemporalInstant(maybeExpr, CTLocalDateTime) +final case class LocalDateTime(maybeExpr: Option[Expr]) + extends TemporalInstant(maybeExpr, CTLocalDateTime) final case class Date(expr: Option[Expr]) extends TemporalInstant(expr, CTDate) final case class Duration(expr: Expr) extends UnaryFunctionExpr { override def propagationType: Option[PropagationType] = Some(AnyNullable) - override def signature(cypherType: CypherType): Option[CypherType] = cypherType match{ - case CTMap(_) | CTString => Some(CTDuration) - case _ => None - } + override def signature(cypherType: CypherType): Option[CypherType] = + cypherType match { + case CTMap(_) | CTString => Some(CTDuration) + case _ => None + } } @@ -1313,8 +1559,7 @@ case object NullLit extends Lit[Null] { // Pattern Predicate Expression -final case class ExistsPatternExpr(targetField: Var, ir: CypherQuery) - extends Expr { +final case class ExistsPatternExpr(targetField: Var, ir: CypherQuery) extends Expr { override val cypherType: CypherType = CTBoolean @@ -1324,10 +1569,15 @@ final case class ExistsPatternExpr(targetField: Var, ir: CypherQuery) } -final case class CaseExpr(alternatives: List[(Expr, Expr)], default: Option[Expr]) - (val cypherType: CypherType) extends Expr { +final case class CaseExpr( + alternatives: List[(Expr, Expr)], + default: Option[Expr] +)(val cypherType: CypherType) + extends Expr { - override val children: Array[Expr] = (default ++ alternatives.flatMap { case (cond, value) => Seq(cond, value) }).toArray + override val children: Array[Expr] = (default ++ alternatives.flatMap { case (cond, value) => + Seq(cond, value) + }).toArray override def withNewChildren(newChildren: Array[Expr]): CaseExpr = { val hasDefault = newChildren.length % 2 == 1 diff --git a/okapi-ir/src/main/scala/org/opencypher/okapi/ir/api/expr/PropagationType.scala b/okapi-ir/src/main/scala/org/opencypher/okapi/ir/api/expr/PropagationType.scala index ff6800fb5b..15da3ddb03 100644 --- a/okapi-ir/src/main/scala/org/opencypher/okapi/ir/api/expr/PropagationType.scala +++ b/okapi-ir/src/main/scala/org/opencypher/okapi/ir/api/expr/PropagationType.scala @@ -26,9 +26,7 @@ */ package org.opencypher.okapi.ir.api.expr - sealed trait PropagationType case object NullOrAnyNullable extends PropagationType case object AnyNullable extends PropagationType case object AllNullable extends PropagationType - diff --git a/okapi-ir/src/main/scala/org/opencypher/okapi/ir/api/pattern/Connection.scala b/okapi-ir/src/main/scala/org/opencypher/okapi/ir/api/pattern/Connection.scala index 3b1666e824..bdd40c6e95 100644 --- a/okapi-ir/src/main/scala/org/opencypher/okapi/ir/api/pattern/Connection.scala +++ b/okapi-ir/src/main/scala/org/opencypher/okapi/ir/api/pattern/Connection.scala @@ -45,7 +45,8 @@ sealed trait Connection { def target: IRField override def hashCode(): Int = orientation.hash(endpoints, seed) - override def equals(obj: scala.Any): Boolean = super.equals(obj) || (obj != null && equalsIfNotEq(obj)) + override def equals(obj: scala.Any): Boolean = + super.equals(obj) || (obj != null && equalsIfNotEq(obj)) protected def seed: Int protected def equalsIfNotEq(obj: scala.Any): Boolean @@ -89,42 +90,57 @@ sealed trait SingleRelationship extends Connection { final protected override def seed: Int = SingleRelationship.seed } -final case class DirectedRelationship(endpoints: DifferentEndpoints, semanticDirection: SemanticDirection) - extends SingleRelationship with DirectedConnection { +final case class DirectedRelationship( + endpoints: DifferentEndpoints, + semanticDirection: SemanticDirection +) extends SingleRelationship + with DirectedConnection { protected def equalsIfNotEq(obj: scala.Any): Boolean = obj match { - case other: DirectedRelationship => orientation.eqv(endpoints, other.endpoints) + case other: DirectedRelationship => + orientation.eqv(endpoints, other.endpoints) case _ => false } } case object DirectedRelationship { - def apply(source: IRField, target: IRField, semanticDirection: SemanticDirection = OUTGOING): SingleRelationship = Endpoints(source, target) match { + def apply( + source: IRField, + target: IRField, + semanticDirection: SemanticDirection = OUTGOING + ): SingleRelationship = Endpoints(source, target) match { case ends: IdenticalEndpoints => CyclicRelationship(ends) - case ends: DifferentEndpoints => DirectedRelationship(ends, semanticDirection) + case ends: DifferentEndpoints => + DirectedRelationship(ends, semanticDirection) } } final case class UndirectedRelationship(endpoints: DifferentEndpoints) - extends SingleRelationship with UndirectedConnection { + extends SingleRelationship + with UndirectedConnection { protected def equalsIfNotEq(obj: scala.Any): Boolean = obj match { - case other: UndirectedRelationship => orientation.eqv(endpoints, other.endpoints) + case other: UndirectedRelationship => + orientation.eqv(endpoints, other.endpoints) case _ => false } } case object UndirectedRelationship { - def apply(source: IRField, target: IRField): SingleRelationship = Endpoints(source, target) match { - case ends: IdenticalEndpoints => CyclicRelationship(ends) - case ends: DifferentEndpoints => UndirectedRelationship(ends) - } + def apply(source: IRField, target: IRField): SingleRelationship = + Endpoints(source, target) match { + case ends: IdenticalEndpoints => CyclicRelationship(ends) + case ends: DifferentEndpoints => UndirectedRelationship(ends) + } } -final case class CyclicRelationship(endpoints: IdenticalEndpoints) extends SingleRelationship with CyclicConnection { +final case class CyclicRelationship(endpoints: IdenticalEndpoints) + extends SingleRelationship + with CyclicConnection { protected def equalsIfNotEq(obj: scala.Any): Boolean = obj match { - case other: CyclicRelationship => orientation.eqv(endpoints, other.endpoints) + case other: CyclicRelationship => + orientation.eqv(endpoints, other.endpoints) case _ => false } } @@ -147,27 +163,39 @@ final case class DirectedVarLengthRelationship( lower: Int, upper: Option[Int], semanticDirection: SemanticDirection = OUTGOING -) extends VarLengthRelationship with DirectedConnection { +) extends VarLengthRelationship + with DirectedConnection { override protected def equalsIfNotEq(obj: Any): Boolean = obj match { - case other: DirectedVarLengthRelationship => orientation.eqv(endpoints, other.endpoints) + case other: DirectedVarLengthRelationship => + orientation.eqv(endpoints, other.endpoints) case _ => false } } -final case class UndirectedVarLengthRelationship(edgeType: CTRelationship, endpoints: DifferentEndpoints, lower: Int, upper: Option[Int]) extends VarLengthRelationship with UndirectedConnection { +final case class UndirectedVarLengthRelationship( + edgeType: CTRelationship, + endpoints: DifferentEndpoints, + lower: Int, + upper: Option[Int] +) extends VarLengthRelationship + with UndirectedConnection { override protected def equalsIfNotEq(obj: Any): Boolean = obj match { - case other: UndirectedVarLengthRelationship => orientation.eqv(endpoints, other.endpoints) + case other: UndirectedVarLengthRelationship => + orientation.eqv(endpoints, other.endpoints) case _ => false } } case object ConnectionCopier { - def copy(con: Connection, endpoints: DifferentEndpoints) : Connection = con match { - case r: DirectedRelationship => DirectedRelationship(endpoints, r.semanticDirection) - case r: DirectedVarLengthRelationship => r.copy(endpoints = endpoints) - case _: UndirectedRelationship | _: CyclicRelationship => UndirectedRelationship(endpoints) - case r: UndirectedVarLengthRelationship => r.copy(endpoints = endpoints) - } -} \ No newline at end of file + def copy(con: Connection, endpoints: DifferentEndpoints): Connection = + con match { + case r: DirectedRelationship => + DirectedRelationship(endpoints, r.semanticDirection) + case r: DirectedVarLengthRelationship => r.copy(endpoints = endpoints) + case _: UndirectedRelationship | _: CyclicRelationship => + UndirectedRelationship(endpoints) + case r: UndirectedVarLengthRelationship => r.copy(endpoints = endpoints) + } +} diff --git a/okapi-ir/src/main/scala/org/opencypher/okapi/ir/api/pattern/Endpoints.scala b/okapi-ir/src/main/scala/org/opencypher/okapi/ir/api/pattern/Endpoints.scala index bd906bad97..1e1f461397 100644 --- a/okapi-ir/src/main/scala/org/opencypher/okapi/ir/api/pattern/Endpoints.scala +++ b/okapi-ir/src/main/scala/org/opencypher/okapi/ir/api/pattern/Endpoints.scala @@ -37,7 +37,8 @@ sealed trait Endpoints extends Traversable[IRField] { case object Endpoints { def apply(source: IRField, target: IRField): Endpoints = - if (source == target) OneSingleEndpoint(source) else TwoDifferentEndpoints(source, target) + if (source == target) OneSingleEndpoint(source) + else TwoDifferentEndpoints(source, target) implicit def one(field: IRField): IdenticalEndpoints = OneSingleEndpoint(field) @@ -50,7 +51,8 @@ case object Endpoints { private case class OneSingleEndpoint(field: IRField) extends IdenticalEndpoints { override def contains(f: IRField): Boolean = field == f } - private case class TwoDifferentEndpoints(source: IRField, target: IRField) extends DifferentEndpoints { + private case class TwoDifferentEndpoints(source: IRField, target: IRField) + extends DifferentEndpoints { override def flip: TwoDifferentEndpoints = copy(target, source) override def contains(f: IRField): Boolean = f == source || f == target diff --git a/okapi-ir/src/main/scala/org/opencypher/okapi/ir/api/pattern/Orientation.scala b/okapi-ir/src/main/scala/org/opencypher/okapi/ir/api/pattern/Orientation.scala index 755bdf6ce5..ea2e9848e6 100644 --- a/okapi-ir/src/main/scala/org/opencypher/okapi/ir/api/pattern/Orientation.scala +++ b/okapi-ir/src/main/scala/org/opencypher/okapi/ir/api/pattern/Orientation.scala @@ -36,18 +36,23 @@ sealed trait Orientation[E <: Endpoints] extends Eq[E] { object Orientation { case object Directed extends Orientation[DifferentEndpoints] { - override def hash(ends: DifferentEndpoints, seed: Int): Int = MurmurHash3.orderedHash(ends, seed) - override def eqv(x: DifferentEndpoints, y: DifferentEndpoints): Boolean = x.source == y.source && x.target == y.target + override def hash(ends: DifferentEndpoints, seed: Int): Int = + MurmurHash3.orderedHash(ends, seed) + override def eqv(x: DifferentEndpoints, y: DifferentEndpoints): Boolean = + x.source == y.source && x.target == y.target } case object Undirected extends Orientation[DifferentEndpoints] { - override def hash(ends: DifferentEndpoints, seed: Int): Int = MurmurHash3.unorderedHash(ends, seed) + override def hash(ends: DifferentEndpoints, seed: Int): Int = + MurmurHash3.unorderedHash(ends, seed) override def eqv(x: DifferentEndpoints, y: DifferentEndpoints): Boolean = (x.source == y.source && x.target == y.target) || (x.source == y.target && x.target == y.source) } case object Cyclic extends Orientation[IdenticalEndpoints] { - override def hash(ends: IdenticalEndpoints, seed: Int): Int = MurmurHash3.mix(seed, ends.field.hashCode()) - override def eqv(x: IdenticalEndpoints, y: IdenticalEndpoints): Boolean = x.field == y.field + override def hash(ends: IdenticalEndpoints, seed: Int): Int = + MurmurHash3.mix(seed, ends.field.hashCode()) + override def eqv(x: IdenticalEndpoints, y: IdenticalEndpoints): Boolean = + x.field == y.field } } diff --git a/okapi-ir/src/main/scala/org/opencypher/okapi/ir/api/pattern/Pattern.scala b/okapi-ir/src/main/scala/org/opencypher/okapi/ir/api/pattern/Pattern.scala index 956f4be783..02c52e8650 100644 --- a/okapi-ir/src/main/scala/org/opencypher/okapi/ir/api/pattern/Pattern.scala +++ b/okapi-ir/src/main/scala/org/opencypher/okapi/ir/api/pattern/Pattern.scala @@ -38,14 +38,15 @@ import scala.collection.immutable.ListMap case object Pattern { def empty[E]: Pattern = Pattern(fields = Set.empty, topology = ListMap.empty) - def node[E](node: IRField): Pattern = Pattern(fields = Set(node), topology = ListMap.empty) + def node[E](node: IRField): Pattern = + Pattern(fields = Set(node), topology = ListMap.empty) } final case class Pattern( fields: Set[IRField], topology: ListMap[IRField, Connection], properties: Map[IRField, MapExpression] = Map.empty, - baseFields: Map[IRField, IRField]= Map.empty + baseFields: Map[IRField, IRField] = Map.empty ) extends Binds { lazy val nodes: Set[IRField] = getElement(CTNode) @@ -57,7 +58,8 @@ final case class Pattern( /** * Fuse patterns but fail if they disagree in the definitions of elements or connections * - * @return A pattern that contains all elements and connections of their input + * @return + * A pattern that contains all elements and connections of their input */ def ++(other: Pattern): Pattern = { val thisMap = fields.map(f => f.name -> f.cypherType).toMap @@ -65,64 +67,91 @@ final case class Pattern( verifyFieldTypes(thisMap, otherMap) - val conflicts = topology.keySet.intersect(other.topology.keySet).filter(k => topology(k) != other.topology(k)) - if (conflicts.nonEmpty) throw PatternConversionException( - s"Expected disjoint patterns but found conflicting connection for ${conflicts.head}:\n" + - s"${topology(conflicts.head)} and ${other.topology(conflicts.head)}") + val conflicts = topology.keySet + .intersect(other.topology.keySet) + .filter(k => topology(k) != other.topology(k)) + if (conflicts.nonEmpty) + throw PatternConversionException( + s"Expected disjoint patterns but found conflicting connection for ${conflicts.head}:\n" + + s"${topology(conflicts.head)} and ${other.topology(conflicts.head)}" + ) val newTopology = topology ++ other.topology // Base field conflicts are checked by frontend val newBaseFields = baseFields ++ other.baseFields - Pattern(fields ++ other.fields, newTopology, properties ++ other.properties, newBaseFields) + Pattern( + fields ++ other.fields, + newTopology, + properties ++ other.properties, + newBaseFields + ) } - private def verifyFieldTypes(map1: Map[String, CypherType], map2: Map[String, CypherType]): Unit = { + private def verifyFieldTypes( + map1: Map[String, CypherType], + map2: Map[String, CypherType] + ): Unit = { (map1.keySet ++ map2.keySet).foreach { f => map1.get(f) -> map2.get(f) match { case (Some(t1), Some(t2)) => if (t1 != t2) - throw PatternConversionException(s"Expected disjoint patterns but found conflicting elements $f") + throw PatternConversionException( + s"Expected disjoint patterns but found conflicting elements $f" + ) case _ => } } } def connectionsFor(node: IRField): Map[IRField, Connection] = { - topology.filter { - case (_, c) => c.endpoints.contains(node) + topology.filter { case (_, c) => + c.endpoints.contains(node) } } def isEmpty: Boolean = this == Pattern.empty - def withConnection(key: IRField, connection: Connection, propertiesOpt: Option[MapExpression] = None): Pattern = { + def withConnection( + key: IRField, + connection: Connection, + propertiesOpt: Option[MapExpression] = None + ): Pattern = { val withProperties: Pattern = propertiesOpt match { case Some(props) => copy(properties = properties.updated(key, props)) - case None => this + case None => this } - if (topology.get(key).contains(connection)) withProperties else withProperties.copy(topology = topology.updated(key, connection)) + if (topology.get(key).contains(connection)) withProperties + else withProperties.copy(topology = topology.updated(key, connection)) } - def withElement(field: IRField, propertiesOpt: Option[MapExpression] = None): Pattern = { + def withElement( + field: IRField, + propertiesOpt: Option[MapExpression] = None + ): Pattern = { val withProperties: Pattern = propertiesOpt match { case Some(props) => copy(properties = properties.updated(field, props)) - case None => this + case None => this } - if (fields(field)) withProperties else withProperties.copy(fields = fields + field) + if (fields(field)) withProperties + else withProperties.copy(fields = fields + field) } - def withBaseField(field: IRField, baseOpt: Option[IRField]): Pattern = baseOpt match { - case Some(base) if fields.contains(field) => copy(baseFields = baseFields.updated(field, base)) - case _ => this - } + def withBaseField(field: IRField, baseOpt: Option[IRField]): Pattern = + baseOpt match { + case Some(base) if fields.contains(field) => + copy(baseFields = baseFields.updated(field, base)) + case _ => this + } def components: Set[Pattern] = { - val _fields = fields.foldLeft(Map.empty[IRField, Int]) { case (m, f) => m.updated(f, m.size) } - val components = nodes.foldLeft(Map.empty[Int, Pattern]) { - case (m, f) => m.updated(_fields(f), Pattern.node(f)) + val _fields = fields.foldLeft(Map.empty[IRField, Int]) { case (m, f) => + m.updated(f, m.size) + } + val components = nodes.foldLeft(Map.empty[Int, Pattern]) { case (m, f) => + m.updated(_fields(f), Pattern.node(f)) } computeComponents(topology.toSeq, components, _fields.size, _fields) } @@ -134,7 +163,7 @@ final case class Pattern( count: Int, fieldToComponentIndex: Map[IRField, Int] ): Set[Pattern] = input match { - case Seq((field, connection), tail@_*) => + case Seq((field, connection), tail @ _*) => val endpoints = connection.endpoints.toSet val links = endpoints.flatMap(fieldToComponentIndex.get) @@ -146,7 +175,9 @@ final case class Pattern( topology = ListMap(field -> connection) ).withElement(field) val newComponents = components.updated(count, newPattern) - val newFields = endpoints.foldLeft(fieldToComponentIndex) { case (m, endpoint) => m.updated(endpoint, count) } + val newFields = endpoints.foldLeft(fieldToComponentIndex) { case (m, endpoint) => + m.updated(endpoint, count) + } computeComponents(tail, newComponents, newCount, newFields) } else if (links.size == 1) { // Connection should be added to a single, existing component @@ -167,7 +198,8 @@ final case class Pattern( val newComponents = links .foldLeft(components) { case (m, l) => m - l } .updated(newCount, newPattern) - val newFields = fieldToComponentIndex.mapValues(l => if (links(l)) newCount else l) + val newFields = + fieldToComponentIndex.mapValues(l => if (links(l)) newCount else l) computeComponents(tail, newComponents, newCount, newFields) } diff --git a/okapi-ir/src/main/scala/org/opencypher/okapi/ir/impl/ExpressionConverter.scala b/okapi-ir/src/main/scala/org/opencypher/okapi/ir/impl/ExpressionConverter.scala index de67a6b629..2c39bc16df 100644 --- a/okapi-ir/src/main/scala/org/opencypher/okapi/ir/impl/ExpressionConverter.scala +++ b/okapi-ir/src/main/scala/org/opencypher/okapi/ir/impl/ExpressionConverter.scala @@ -34,7 +34,13 @@ import org.opencypher.okapi.ir.api.expr._ import org.opencypher.okapi.ir.impl.SignatureTyping._ import org.opencypher.okapi.ir.impl.parse.{functions => f} import org.opencypher.okapi.ir.impl.typer.SignatureConverter.Signature -import org.opencypher.okapi.ir.impl.typer.{InvalidArgument, InvalidContainerAccess, MissingParameter, UnTypedExpr, WrongNumberOfArguments} +import org.opencypher.okapi.ir.impl.typer.{ + InvalidArgument, + InvalidContainerAccess, + MissingParameter, + UnTypedExpr, + WrongNumberOfArguments +} import org.opencypher.v9_0.expressions.{ExtractScope, functions} import org.opencypher.v9_0.{expressions => ast} @@ -46,34 +52,49 @@ final class ExpressionConverter(context: IRBuilderContext) { private def parameterType(p: ast.Parameter): CypherType = { context.parameters.get(p.name) match { - case None => throw MissingParameter(p.name) + case None => throw MissingParameter(p.name) case Some(param) => param.cypherType } } private def extractLong(expr: Expr): Long = { expr match { - case param: Param => context.parameters(param.name) match { - case CypherInteger(i) => i - case other => throw IllegalArgumentException("a CypherInteger value", other) - } + case param: Param => + context.parameters(param.name) match { + case CypherInteger(i) => i + case other => + throw IllegalArgumentException("a CypherInteger value", other) + } case l: IntegerLit => l.v - case _ => throw IllegalArgumentException("a literal value", expr) + case _ => throw IllegalArgumentException("a literal value", expr) } } - private def convertFilterScope(variable: ast.LogicalVariable, innerPredicate: ast.Expression, list: Expr)(implicit lambdaVars: Map[String, CypherType]) : (LambdaVar, Expr) = { - val listInnerType = list.cypherType match { - case CTList(inner) => inner - case err => throw IllegalArgumentException("a list to step over", err, "Wrong list comprehension type") - } - val lambdaVar = LambdaVar(variable.name) (listInnerType) //todo: use normal convert + match instead? - val updatedLambdaVars = lambdaVars + (variable.name -> listInnerType) + private def convertFilterScope( + variable: ast.LogicalVariable, + innerPredicate: ast.Expression, + list: Expr + )(implicit lambdaVars: Map[String, CypherType]): (LambdaVar, Expr) = { + val listInnerType = list.cypherType match { + case CTList(inner) => inner + case err => + throw IllegalArgumentException( + "a list to step over", + err, + "Wrong list comprehension type" + ) + } + val lambdaVar = LambdaVar(variable.name)( + listInnerType + ) // todo: use normal convert + match instead? + val updatedLambdaVars = lambdaVars + (variable.name -> listInnerType) - lambdaVar -> convert(innerPredicate)(updatedLambdaVars) + lambdaVar -> convert(innerPredicate)(updatedLambdaVars) } - def convert(e: ast.Expression) (implicit lambdaVars: Map[String, CypherType]): Expr = { + def convert( + e: ast.Expression + )(implicit lambdaVars: Map[String, CypherType]): Expr = { lazy val child0: Expr = convert(e.arguments.head) @@ -84,19 +105,21 @@ final class ExpressionConverter(context: IRBuilderContext) { lazy val convertedChildren: List[Expr] = e.arguments.toList.map(convert(_)) e match { - case ast.Variable(name) => lambdaVars.get(name) match { + case ast.Variable(name) => + lambdaVars.get(name) match { case Some(varType) => LambdaVar(name)(varType) - case None => Var(name)(context.knownTypes.getOrElse(e, throw UnTypedExpr(e))) + case None => + Var(name)(context.knownTypes.getOrElse(e, throw UnTypedExpr(e))) } - case p@ast.Parameter(name, _) => Param(name)(parameterType(p)) + case p @ ast.Parameter(name, _) => Param(name)(parameterType(p)) // Literals - case astExpr: ast.IntegerLiteral => IntegerLit(astExpr.value) + case astExpr: ast.IntegerLiteral => IntegerLit(astExpr.value) case astExpr: ast.DecimalDoubleLiteral => FloatLit(astExpr.value) - case ast.StringLiteral(value) => StringLit(value) - case _: ast.True => TrueLit - case _: ast.False => FalseLit - case _: ast.ListLiteral => ListLit(convertedChildren) + case ast.StringLiteral(value) => StringLit(value) + case _: ast.True => TrueLit + case _: ast.False => FalseLit + case _: ast.ListLiteral => ListLit(convertedChildren) case ast.Property(_, ast.PropertyKeyName(name)) => val owner = child0 @@ -111,16 +134,24 @@ final class ExpressionConverter(context: IRBuilderContext) { // User specified label constraints - we can use those for type inference ElementProperty(owner, key)(propertyType) case CTNode(labels, None) => - val propertyType = schema.nodePropertyKeyType(labels, name).getOrElse(CTNull) + val propertyType = + schema.nodePropertyKeyType(labels, name).getOrElse(CTNull) ElementProperty(owner, key)(propertyType) case CTNode(labels, Some(qgn)) => - val propertyType = context.queryLocalCatalog.schema(qgn).nodePropertyKeyType(labels, name).getOrElse(CTNull) + val propertyType = context.queryLocalCatalog + .schema(qgn) + .nodePropertyKeyType(labels, name) + .getOrElse(CTNull) ElementProperty(owner, key)(propertyType) case CTRelationship(types, None) => - val propertyType = schema.relationshipPropertyKeyType(types, name).getOrElse(CTNull) + val propertyType = + schema.relationshipPropertyKeyType(types, name).getOrElse(CTNull) ElementProperty(owner, key)(propertyType) case CTRelationship(types, Some(qgn)) => - val propertyType = context.queryLocalCatalog.schema(qgn).relationshipPropertyKeyType(types, name).getOrElse(CTNull) + val propertyType = context.queryLocalCatalog + .schema(qgn) + .relationshipPropertyKeyType(types, name) + .getOrElse(CTNull) ElementProperty(owner, key)(propertyType) case _: CTMap => MapProperty(owner, key) @@ -135,57 +166,63 @@ final class ExpressionConverter(context: IRBuilderContext) { // Predicates case _: ast.Ands => Ands(convertedChildren: _*) - case _: ast.Ors => Ors(convertedChildren: _*) - case _: ast.Xor => Ors(Ands(child0, Not(child1)), Ands(Not(child0), child1)) - case ast.HasLabels(_, labels) => Ands(labels.map(l => HasLabel(child0, Label(l.name))).toSet) + case _: ast.Ors => Ors(convertedChildren: _*) + case _: ast.Xor => + Ors(Ands(child0, Not(child1)), Ands(Not(child0), child1)) + case ast.HasLabels(_, labels) => + Ands(labels.map(l => HasLabel(child0, Label(l.name))).toSet) case _: ast.Not => Not(child0) - case ast.Equals(f: ast.FunctionInvocation, s: ast.StringLiteral) if f.function == functions.Type => + case ast.Equals(f: ast.FunctionInvocation, s: ast.StringLiteral) + if f.function == functions.Type => HasType(convert(f.args.head), RelType(s.value)) - case _: ast.Equals => Equals(child0, child1) - case _: ast.LessThan => LessThan(child0, child1) - case _: ast.LessThanOrEqual => LessThanOrEqual(child0, child1) - case _: ast.GreaterThan => GreaterThan(child0, child1) + case _: ast.Equals => Equals(child0, child1) + case _: ast.LessThan => LessThan(child0, child1) + case _: ast.LessThanOrEqual => LessThanOrEqual(child0, child1) + case _: ast.GreaterThan => GreaterThan(child0, child1) case _: ast.GreaterThanOrEqual => GreaterThanOrEqual(child0, child1) - case _: ast.In => In(child0, child1) - case _: ast.IsNull => IsNull(child0) - case _: ast.IsNotNull => IsNotNull(child0) - case _: ast.StartsWith => StartsWith(child0, child1) - case _: ast.EndsWith => EndsWith(child0, child1) - case _: ast.Contains => Contains(child0, child1) + case _: ast.In => In(child0, child1) + case _: ast.IsNull => IsNull(child0) + case _: ast.IsNotNull => IsNotNull(child0) + case _: ast.StartsWith => StartsWith(child0, child1) + case _: ast.EndsWith => EndsWith(child0, child1) + case _: ast.Contains => Contains(child0, child1) // Arithmetics - case _: ast.Add => Add(child0, child1) + case _: ast.Add => Add(child0, child1) case _: ast.Subtract => Subtract(child0, child1) case _: ast.Multiply => Multiply(child0, child1) - case _: ast.Divide => Divide(child0, child1) - case _: ast.Modulo => Modulo(child0, child1) - - case funcInv: ast.FunctionInvocation => funcInv.function match { - case functions.Id => Id(child0) - case functions.Labels => Labels(child0) - case functions.Type => Type(child0) - case functions.Avg => Avg(child0) - case functions.Max => Max(child0) - case functions.Min => Min(child0) + case _: ast.Divide => Divide(child0, child1) + case _: ast.Modulo => Modulo(child0, child1) + + case funcInv: ast.FunctionInvocation => + funcInv.function match { + case functions.Id => Id(child0) + case functions.Labels => Labels(child0) + case functions.Type => Type(child0) + case functions.Avg => Avg(child0) + case functions.Max => Max(child0) + case functions.Min => Min(child0) case functions.PercentileCont => PercentileCont(child0, child1) case functions.PercentileDisc => PercentileDisc(child0, child1) - case functions.StdDev => StDev(child0) - case functions.StdDevP => StDevP(child0) - case functions.Sum => Sum(child0) - case functions.Count => Count(child0, funcInv.distinct) - case functions.Collect => Collect(child0, funcInv.distinct) - case functions.Exists => Exists(child0) - case functions.Size => Size(child0) - case functions.Keys => Keys(child0) - case functions.StartNode => StartNodeFunction(child0) - case functions.EndNode => EndNodeFunction(child0) - case functions.ToFloat => ToFloat(child0) - case functions.ToInteger => ToInteger(child0) - case functions.ToString => ToString(child0) - case functions.ToBoolean => ToBoolean(child0) - case functions.Coalesce => + case functions.StdDev => StDev(child0) + case functions.StdDevP => StDevP(child0) + case functions.Sum => Sum(child0) + case functions.Count => Count(child0, funcInv.distinct) + case functions.Collect => Collect(child0, funcInv.distinct) + case functions.Exists => Exists(child0) + case functions.Size => Size(child0) + case functions.Keys => Keys(child0) + case functions.StartNode => StartNodeFunction(child0) + case functions.EndNode => EndNodeFunction(child0) + case functions.ToFloat => ToFloat(child0) + case functions.ToInteger => ToInteger(child0) + case functions.ToString => ToString(child0) + case functions.ToBoolean => ToBoolean(child0) + case functions.Coalesce => // Special optimisation for coalesce using short-circuit logic - convertedChildren.map(_.cypherType).indexWhere(!_.isNullable) match { + convertedChildren + .map(_.cypherType) + .indexWhere(!_.isNullable) match { case 0 => // first argument is non-nullable; just use it directly without coalesce convertedChildren.head @@ -197,87 +234,110 @@ final class ExpressionConverter(context: IRBuilderContext) { val relevantArgs = convertedChildren.slice(0, other + 1) Coalesce(relevantArgs) } - case functions.Range => Range(child0, child1, convertedChildren.lift(2)) - case functions.Substring => Substring(child0, child1, convertedChildren.lift(2)) + case functions.Range => + Range(child0, child1, convertedChildren.lift(2)) + case functions.Substring => + Substring(child0, child1, convertedChildren.lift(2)) case functions.Split => Split(child0, child1) - case functions.Left => Substring(child0, IntegerLit(0), convertedChildren.lift(1)) - case functions.Right => Substring(child0, Subtract(Multiply(IntegerLit(-1), child1), IntegerLit(1)), None) + case functions.Left => + Substring(child0, IntegerLit(0), convertedChildren.lift(1)) + case functions.Right => + Substring( + child0, + Subtract(Multiply(IntegerLit(-1), child1), IntegerLit(1)), + None + ) case functions.Replace => Replace(child0, child1, child2) case functions.Reverse => Reverse(child0) - case functions.Trim => Trim(child0) - case functions.LTrim => LTrim(child0) - case functions.RTrim => RTrim(child0) + case functions.Trim => Trim(child0) + case functions.LTrim => LTrim(child0) + case functions.RTrim => RTrim(child0) case functions.ToUpper => ToUpper(child0) case functions.ToLower => ToLower(child0) case functions.Properties => val outType = child0.cypherType.material match { case CTVoid => CTNull case CTNode(labels, _) => - CTMap(schema.nodePropertyKeysForCombinations(schema.combinationsFor(labels))) + CTMap( + schema.nodePropertyKeysForCombinations( + schema.combinationsFor(labels) + ) + ) case CTRelationship(types, _) => CTMap(schema.relationshipPropertyKeysForTypes(types)) case m: CTMap => m - case _ => throw InvalidArgument(funcInv, funcInv.args(0)) + case _ => throw InvalidArgument(funcInv, funcInv.args(0)) } Properties(child0)(outType) - //List-access functions + // List-access functions case functions.Last => Last(child0) case functions.Head => Head(child0) case functions.Tail => ListSliceFrom(child0, IntegerLit(1)) // Logarithmic functions - case functions.Sqrt => Sqrt(child0) - case functions.Log => Log(child0) + case functions.Sqrt => Sqrt(child0) + case functions.Log => Log(child0) case functions.Log10 => Log10(child0) - case functions.Exp => Exp(child0) - case functions.E => E - case functions.Pi => Pi + case functions.Exp => Exp(child0) + case functions.E => E + case functions.Pi => Pi // Numeric functions - case functions.Abs => Abs(child0) - case functions.Ceil => Ceil(child0) + case functions.Abs => Abs(child0) + case functions.Ceil => Ceil(child0) case functions.Floor => Floor(child0) - case functions.Rand => Rand + case functions.Rand => Rand case functions.Round => Round(child0) - case functions.Sign => Sign(child0) + case functions.Sign => Sign(child0) // Trigonometric functions - case functions.Acos => Acos(child0) - case functions.Asin => Asin(child0) - case functions.Atan => Atan(child0) - case functions.Atan2 => Atan2(child0, child1) - case functions.Cos => Cos(child0) - case functions.Cot => Cot(child0) - case functions.Degrees => Degrees(child0) + case functions.Acos => Acos(child0) + case functions.Asin => Asin(child0) + case functions.Atan => Atan(child0) + case functions.Atan2 => Atan2(child0, child1) + case functions.Cos => Cos(child0) + case functions.Cot => Cot(child0) + case functions.Degrees => Degrees(child0) case functions.Haversin => Haversin(child0) - case functions.Radians => Radians(child0) - case functions.Sin => Sin(child0) - case functions.Tan => Tan(child0) + case functions.Radians => Radians(child0) + case functions.Sin => Sin(child0) + case functions.Tan => Tan(child0) // Match by name - case functions.UnresolvedFunction => funcInv.name match { - // Time functions - case f.Timestamp.name => Timestamp - case f.LocalDateTime.name => LocalDateTime(convertedChildren.headOption) - case f.Date.name => Date(convertedChildren.headOption) - case f.Duration.name => Duration(child0) - case BigDecimal.name => - e.checkNbrArgs(3, convertedChildren.length) - BigDecimal(child0, extractLong(child1), extractLong(child2)) - case name => throw NotImplementedException(s"Support for converting function '$name' is not yet implemented") - } + case functions.UnresolvedFunction => + funcInv.name match { + // Time functions + case f.Timestamp.name => Timestamp + case f.LocalDateTime.name => + LocalDateTime(convertedChildren.headOption) + case f.Date.name => Date(convertedChildren.headOption) + case f.Duration.name => Duration(child0) + case BigDecimal.name => + e.checkNbrArgs(3, convertedChildren.length) + BigDecimal(child0, extractLong(child1), extractLong(child2)) + case name => + throw NotImplementedException( + s"Support for converting function '$name' is not yet implemented" + ) + } case a: functions.Function => - throw NotImplementedException(s"Support for converting function '${a.name}' is not yet implemented") + throw NotImplementedException( + s"Support for converting function '${a.name}' is not yet implemented" + ) } case _: ast.CountStar => CountStar // Exists (rewritten Pattern Expressions) - case org.opencypher.okapi.ir.impl.parse.rewriter.ExistsPattern(subquery, trueVar) => + case org.opencypher.okapi.ir.impl.parse.rewriter + .ExistsPattern(subquery, trueVar) => val innerModel = IRBuilder(subquery)(context) match { case sq: SingleQuery => sq - case _ => throw IllegalArgumentException("ExistsPattern only accepts SingleQuery") + case _ => + throw IllegalArgumentException( + "ExistsPattern only accepts SingleQuery" + ) } ExistsPatternExpr( Var(trueVar.name)(CTBoolean), @@ -286,58 +346,107 @@ final class ExpressionConverter(context: IRBuilderContext) { // Case When .. Then .. [Else ..] End case ast.CaseExpression(None, alternatives, default) => - val convertedAlternatives = alternatives.toList.map { case (left, right) => convert(left) -> convert(right) } - val maybeConvertedDefault: Option[Expr] = default.map(expr => convert(expr)) - val possibleTypes = convertedAlternatives.map { case (_, thenExpr) => thenExpr.cypherType } - val defaultCaseType = maybeConvertedDefault.map(_.cypherType).getOrElse(CTNull) + val convertedAlternatives = alternatives.toList.map { case (left, right) => + convert(left) -> convert(right) + } + val maybeConvertedDefault: Option[Expr] = + default.map(expr => convert(expr)) + val possibleTypes = convertedAlternatives.map { case (_, thenExpr) => + thenExpr.cypherType + } + val defaultCaseType = + maybeConvertedDefault.map(_.cypherType).getOrElse(CTNull) val returnType = possibleTypes.foldLeft(defaultCaseType)(_ join _) CaseExpr(convertedAlternatives, maybeConvertedDefault)(returnType) case ast.MapExpression(items) => - val convertedMap = items.map { case (key, value) => key.name -> convert(value) }.toMap + val convertedMap = items.map { case (key, value) => + key.name -> convert(value) + }.toMap MapExpression(convertedMap) // Expression - case ast.ListSlice(list, Some(from), Some(to)) => ListSliceFromTo(convert(list), convert(from), convert(to)) - case ast.ListSlice(list, None, Some(to)) => ListSliceTo(convert(list), convert(to)) - case ast.ListSlice(list, Some(from), None) => ListSliceFrom(convert(list), convert(from)) - - case ast.ListComprehension(ExtractScope(variable, innerPredicate, extractExpression), expr) => + case ast.ListSlice(list, Some(from), Some(to)) => + ListSliceFromTo(convert(list), convert(from), convert(to)) + case ast.ListSlice(list, None, Some(to)) => + ListSliceTo(convert(list), convert(to)) + case ast.ListSlice(list, Some(from), None) => + ListSliceFrom(convert(list), convert(from)) + + case ast.ListComprehension( + ExtractScope(variable, innerPredicate, extractExpression), + expr + ) => val listExpr = convert(expr)(lambdaVars) val listInnerType = listExpr.cypherType match { case CTList(inner) => inner - case err => throw IllegalArgumentException("a list to step over", err, "Wrong list comprehension type") + case err => + throw IllegalArgumentException( + "a list to step over", + err, + "Wrong list comprehension type" + ) } - val updatedLambdaVars: Map[String, CypherType] = lambdaVars + (variable.name -> listInnerType) - ListComprehension(convert(variable)(updatedLambdaVars), innerPredicate.map(convert(_)(updatedLambdaVars)), - extractExpression.map(convert(_)(updatedLambdaVars)), listExpr) + val updatedLambdaVars: Map[String, CypherType] = + lambdaVars + (variable.name -> listInnerType) + ListComprehension( + convert(variable)(updatedLambdaVars), + innerPredicate.map(convert(_)(updatedLambdaVars)), + extractExpression.map(convert(_)(updatedLambdaVars)), + listExpr + ) - case ast.ReduceExpression(ast.ReduceScope(accumulator, variable, reduceExpression), _, list) => + case ast.ReduceExpression( + ast.ReduceScope(accumulator, variable, reduceExpression), + _, + list + ) => val initExpr = child1 - val introduceLambdaVars = Map(accumulator.name -> initExpr.cypherType, variable.name -> initExpr.cypherType) + val introduceLambdaVars = Map( + accumulator.name -> initExpr.cypherType, + variable.name -> initExpr.cypherType + ) val updatedLambdaVars = lambdaVars ++ introduceLambdaVars - ListReduction(convert(accumulator)(updatedLambdaVars), convert(variable)(updatedLambdaVars), - convert(reduceExpression)(updatedLambdaVars), initExpr, convert(list)(updatedLambdaVars)) - - case predExpr: ast.IterablePredicateExpression => predExpr.scope.innerPredicate match { - case Some(innerPredicate) => val (lambdaVar, predicate) = convertFilterScope(predExpr.scope.variable, innerPredicate, child0) - predExpr match { - case _: ast.AnyIterablePredicate => ListAny(lambdaVar, predicate, child0) - case _: ast.AllIterablePredicate => ListAll(lambdaVar, predicate, child0) - case _: ast.NoneIterablePredicate => ListNone(lambdaVar, predicate, child0) - case _: ast.SingleIterablePredicate => ListSingle(lambdaVar, predicate, child0) - } - case None => - predExpr match { - case _ => throw IllegalArgumentException("requires a predicate") //same behaviour as neo4j - } - } + ListReduction( + convert(accumulator)(updatedLambdaVars), + convert(variable)(updatedLambdaVars), + convert(reduceExpression)(updatedLambdaVars), + initExpr, + convert(list)(updatedLambdaVars) + ) + + case predExpr: ast.IterablePredicateExpression => + predExpr.scope.innerPredicate match { + case Some(innerPredicate) => + val (lambdaVar, predicate) = convertFilterScope( + predExpr.scope.variable, + innerPredicate, + child0 + ) + predExpr match { + case _: ast.AnyIterablePredicate => + ListAny(lambdaVar, predicate, child0) + case _: ast.AllIterablePredicate => + ListAll(lambdaVar, predicate, child0) + case _: ast.NoneIterablePredicate => + ListNone(lambdaVar, predicate, child0) + case _: ast.SingleIterablePredicate => + ListSingle(lambdaVar, predicate, child0) + } + case None => + predExpr match { + case _ => + throw IllegalArgumentException( + "requires a predicate" + ) // same behaviour as neo4j + } + } case ast.DesugaredMapProjection(_, items, includeAllProps) => val convertedItems = items.map(x => x.key.name -> convert(x.exp)) val mapOwner = child0 match { case v: Var => v - case err => throw IllegalArgumentException("a Var Expr", err) + case err => throw IllegalArgumentException("a Var Expr", err) } MapProjection(mapOwner, convertedItems, includeAllProps) @@ -351,7 +460,10 @@ final class ExpressionConverter(context: IRBuilderContext) { val key = context.parameters(name).cast[String] innerTypes.getOrElse(key, CTVoid) case ast.StringLiteral(key) => innerTypes.getOrElse(key, CTVoid) - case _ => innerTypes.values.foldLeft(CTVoid: CypherType)(_ join _).nullable + case _ => + innerTypes.values + .foldLeft(CTVoid: CypherType)(_ join _) + .nullable } case _ => throw InvalidContainerAccess(e) } @@ -361,7 +473,6 @@ final class ExpressionConverter(context: IRBuilderContext) { case ast.RegexMatch(lhs, rhs) => RegexMatch(convert(lhs), convert(rhs)) - case _ => throw NotImplementedException(s"Not yet able to convert expression: $e") } @@ -372,20 +483,23 @@ final class ExpressionConverter(context: IRBuilderContext) { object BigDecimalSignatures { /** - * Signature for BigDecimal arithmetics on the type level. The semantics are based on Spark SQL, which in turn - * is based on Hive and SQL Server. See DecimalPrecision in Apache Spark + * Signature for BigDecimal arithmetics on the type level. The semantics are based on Spark SQL, + * which in turn is based on Hive and SQL Server. See DecimalPrecision in Apache Spark * * @param precisionScaleOp * @return */ // TODO change to tuples def arithmeticSignature(precisionScaleOp: PrecisionScaleOp): Signature = { - case Seq(CTBigDecimal(p1, s1), CTBigDecimal(p2, s2)) => Some(CTBigDecimal(precisionScaleOp(p1, s1, p2, s2))) - case Seq(CTBigDecimal(p, s), CTInteger) => Some(CTBigDecimal(precisionScaleOp(p, s, 20, 0))) - case Seq(CTInteger, CTBigDecimal(p, s)) => Some(CTBigDecimal(precisionScaleOp(20, 0, p, s))) + case Seq(CTBigDecimal(p1, s1), CTBigDecimal(p2, s2)) => + Some(CTBigDecimal(precisionScaleOp(p1, s1, p2, s2))) + case Seq(CTBigDecimal(p, s), CTInteger) => + Some(CTBigDecimal(precisionScaleOp(p, s, 20, 0))) + case Seq(CTInteger, CTBigDecimal(p, s)) => + Some(CTBigDecimal(precisionScaleOp(20, 0, p, s))) case Seq(_: CTBigDecimal, CTFloat) => Some(CTFloat) case Seq(CTFloat, _: CTBigDecimal) => Some(CTFloat) - case _ => None + case _ => None } trait PrecisionScaleOp { diff --git a/okapi-ir/src/main/scala/org/opencypher/okapi/ir/impl/IRBuilder.scala b/okapi-ir/src/main/scala/org/opencypher/okapi/ir/impl/IRBuilder.scala index f8cee5aea2..8c51d82e91 100644 --- a/okapi-ir/src/main/scala/org/opencypher/okapi/ir/impl/IRBuilder.scala +++ b/okapi-ir/src/main/scala/org/opencypher/okapi/ir/impl/IRBuilder.scala @@ -33,7 +33,11 @@ import org.opencypher.okapi.api.graph.QualifiedGraphName import org.opencypher.okapi.api.schema.PropertyGraphSchema import org.opencypher.okapi.api.types._ import org.opencypher.okapi.api.value.CypherValue.CypherString -import org.opencypher.okapi.impl.exception.{IllegalArgumentException, IllegalStateException, UnsupportedOperationException} +import org.opencypher.okapi.impl.exception.{ + IllegalArgumentException, + IllegalStateException, + UnsupportedOperationException +} import org.opencypher.okapi.ir.api._ import org.opencypher.okapi.ir.api.block.{SortItem, _} import org.opencypher.okapi.ir.api.expr._ @@ -47,26 +51,32 @@ import org.opencypher.v9_0.ast.QueryPart import org.opencypher.v9_0.util.InputPosition import org.opencypher.v9_0.{ast, expressions => exp} - object IRBuilder extends CompilationStage[ast.Statement, CypherStatement, IRBuilderContext] { - override type Out = Either[IRBuilderError, (Option[CypherStatement], IRBuilderContext)] + override type Out = + Either[IRBuilderError, (Option[CypherStatement], IRBuilderContext)] - override def process(input: ast.Statement)(implicit context: IRBuilderContext): Out = + override def process(input: ast.Statement)(implicit + context: IRBuilderContext + ): Out = buildIR[IRBuilderStack[Option[SingleQuery]]](input).run(context) def getContext(output: Out): IRBuilderContext = getTuple(output)._2 private def getTuple(output: Out) = output match { - case Left(error) => throw IllegalStateException(s"Error during IR construction: $error") - case Right((None, _)) => throw IllegalStateException(s"Failed to construct IR") + case Left(error) => + throw IllegalStateException(s"Error during IR construction: $error") + case Right((None, _)) => + throw IllegalStateException(s"Failed to construct IR") case Right((Some(q), ctx)) => q -> ctx } override def extract(output: Out): CypherStatement = getTuple(output)._1 - private def buildIR[R: _mayFail : _hasContext](s: ast.Statement): Eff[R, Option[CypherStatement]] = + private def buildIR[R: _mayFail: _hasContext]( + s: ast.Statement + ): Eff[R, Option[CypherStatement]] = s match { case ast.Query(_, part) => for { @@ -80,11 +90,20 @@ object IRBuilder extends CompilationStage[ast.Statement, CypherStatement, IRBuil result <- { val maybeSingleQuery = innerQuery.asInstanceOf[Option[SingleQuery]] val schema = maybeSingleQuery.get.model.result match { - case GraphResultBlock(_, irGraph) => context.schemaFor(irGraph.qualifiedGraphName) - case _ => throw IllegalArgumentException("The query in CATALOG CREATE GRAPH must return a graph") + case GraphResultBlock(_, irGraph) => + context.schemaFor(irGraph.qualifiedGraphName) + case _ => + throw IllegalArgumentException( + "The query in CATALOG CREATE GRAPH must return a graph" + ) } val irQgn = QualifiedGraphName(qgn.parts) - val statement = Some(CreateGraphStatement(IRCatalogGraph(irQgn, schema), maybeSingleQuery.get)) + val statement = Some( + CreateGraphStatement( + IRCatalogGraph(irQgn, schema), + maybeSingleQuery.get + ) + ) pure[R, Option[CypherStatement]](statement) } } yield result @@ -93,11 +112,13 @@ object IRBuilder extends CompilationStage[ast.Statement, CypherStatement, IRBuil for { context <- get[R, IRBuilderContext] result <- { - val statement = Some(CreateViewStatement( - QualifiedGraphName(qgn.parts), - params.map(_.name).toList, - innerQueryString - )) + val statement = Some( + CreateViewStatement( + QualifiedGraphName(qgn.parts), + params.map(_.name).toList, + innerQueryString + ) + ) pure[R, Option[CypherStatement]](statement) } } yield result @@ -127,12 +148,16 @@ object IRBuilder extends CompilationStage[ast.Statement, CypherStatement, IRBuil } // TODO: return CypherQuery instead of Option[CypherQuery] - private def convertQueryPart[R: _mayFail : _hasContext](part: ast.QueryPart): Eff[R, Option[CypherQuery]] = { + private def convertQueryPart[R: _mayFail: _hasContext]( + part: ast.QueryPart + ): Eff[R, Option[CypherQuery]] = { part match { case ast.SingleQuery(clauses) => val plannedBlocks = for { context <- get[R, IRBuilderContext] - blocks <- put[R, IRBuilderContext](context.resetRegistry) >> clauses.toList.traverse(convertClause[R]) + blocks <- put[R, IRBuilderContext]( + context.resetRegistry + ) >> clauses.toList.traverse(convertClause[R]) } yield blocks plannedBlocks >> convertRegistry @@ -147,9 +172,10 @@ object IRBuilder extends CompilationStage[ast.Statement, CypherStatement, IRBuil } } - def convertUnion[R: _mayFail : _hasContext]( + def convertUnion[R: _mayFail: _hasContext]( innerPart: QueryPart, - singleQuery: ast.SingleQuery, distinct: Boolean + singleQuery: ast.SingleQuery, + distinct: Boolean ): Eff[R, Option[CypherQuery]] = { for { first <- convertQueryPart(innerPart) @@ -159,7 +185,9 @@ object IRBuilder extends CompilationStage[ast.Statement, CypherStatement, IRBuil } } - private def convertClause[R: _mayFail : _hasContext](c: ast.Clause): Eff[R, List[Block]] = { + private def convertClause[R: _mayFail: _hasContext]( + c: ast.Clause + ): Eff[R, List[Block]] = { c match { @@ -170,8 +198,12 @@ object IRBuilder extends CompilationStage[ast.Statement, CypherStatement, IRBuil val graph = context.instantiateView(v) val generatedQgn = context.qgnGenerator.generate val irGraph = IRCatalogGraph(generatedQgn, graph.schema) - val updatedContext = context.withWorkingGraph(irGraph).registerGraph(generatedQgn, graph) - put[R, IRBuilderContext](updatedContext) >> pure[R, List[Block]](List.empty) + val updatedContext = context + .withWorkingGraph(irGraph) + .registerGraph(generatedQgn, graph) + put[R, IRBuilderContext](updatedContext) >> pure[R, List[Block]]( + List.empty + ) } } yield blocks @@ -181,10 +213,18 @@ object IRBuilder extends CompilationStage[ast.Statement, CypherStatement, IRBuil result <- { val maybeParameterValue = context.parameters.get(p.name) val fromGraph = maybeParameterValue match { - case Some(CypherString(paramValue)) => ast.GraphLookup(ast.CatalogName(QualifiedGraphName.splitQgn(paramValue)))(p.position) - case Some(other) => throw ParsingException( - s"Parameter ${p.name} needs to be of type ${CTString.toString}, was $other") - case None => throw ParsingException(s"No parameter ${p.name} was specified ${p.position}") + case Some(CypherString(paramValue)) => + ast.GraphLookup( + ast.CatalogName(QualifiedGraphName.splitQgn(paramValue)) + )(p.position) + case Some(other) => + throw ParsingException( + s"Parameter ${p.name} needs to be of type ${CTString.toString}, was $other" + ) + case None => + throw ParsingException( + s"No parameter ${p.name} was specified ${p.position}" + ) } convertClause(fromGraph) } @@ -198,7 +238,9 @@ object IRBuilder extends CompilationStage[ast.Statement, CypherStatement, IRBuil val schema = context.schemaFor(irQgn) val irGraph = IRCatalogGraph(irQgn, schema) val updatedContext = context.withWorkingGraph(irGraph) - put[R, IRBuilderContext](updatedContext) >> pure[R, List[Block]](List.empty) + put[R, IRBuilderContext](updatedContext) >> pure[R, List[Block]]( + List.empty + ) } } yield blocks @@ -210,28 +252,50 @@ object IRBuilder extends CompilationStage[ast.Statement, CypherStatement, IRBuil blocks <- { val blockRegistry = context.blockRegistry val after = blockRegistry.lastAdded.toList - val block = MatchBlock(after, pattern, given, optional, context.workingGraph) + val block = + MatchBlock(after, pattern, given, optional, context.workingGraph) val typedOutputs = typedMatchBlock.outputs(block) val updatedRegistry = blockRegistry.register(block) - val updatedContext = context.withBlocks(updatedRegistry).withFields(typedOutputs).registerConnections(pattern.topology) - put[R, IRBuilderContext](updatedContext) >> pure[R, List[Block]](List(block)) + val updatedContext = context + .withBlocks(updatedRegistry) + .withFields(typedOutputs) + .registerConnections(pattern.topology) + put[R, IRBuilderContext](updatedContext) >> pure[R, List[Block]]( + List(block) + ) } } yield blocks - case ast.With(distinct, ast.ReturnItems(_, items), orderBy, skip, limit, where) - if !items.exists(_.expression.containsAggregate) => + case ast.With( + distinct, + ast.ReturnItems(_, items), + orderBy, + skip, + limit, + where + ) if !items.exists(_.expression.containsAggregate) => for { fieldExprs <- items.toList.traverse(convertReturnItem[R]) given <- convertWhere(where) context <- get[R, IRBuilderContext] refs <- { val (projectRef, projectReg) = - registerProjectBlock(context, fieldExprs, given, context.workingGraph, distinct = distinct) + registerProjectBlock( + context, + fieldExprs, + given, + context.workingGraph, + distinct = distinct + ) val appendList = (list: List[Block]) => pure[R, List[Block]](projectRef +: list) - val orderAndSliceBlock = registerOrderAndSliceBlock(orderBy, skip, limit) - val updatedContext = context.updateKnownConnections(fieldExprs).withBlocks(projectReg) - put[R, IRBuilderContext](updatedContext) >> orderAndSliceBlock flatMap appendList + val orderAndSliceBlock = + registerOrderAndSliceBlock(orderBy, skip, limit) + val updatedContext = + context.updateKnownConnections(fieldExprs).withBlocks(projectReg) + put[R, IRBuilderContext]( + updatedContext + ) >> orderAndSliceBlock flatMap appendList } } yield refs @@ -242,15 +306,29 @@ object IRBuilder extends CompilationStage[ast.Statement, CypherStatement, IRBuil blocks <- { val (agg, group) = fieldExprs.partition { case (_, _: Aggregator) => true - case _ => false + case _ => false } - val (projectBlock, updatedRegistry1) = registerProjectBlock(context, group, source = context.workingGraph, distinct = false) + val (projectBlock, updatedRegistry1) = registerProjectBlock( + context, + group, + source = context.workingGraph, + distinct = false + ) val after = updatedRegistry1.lastAdded.toList - val aggregationBlock = AggregationBlock(after, Aggregations(agg.toSet), group.map(_._1).toSet, context.workingGraph) + val aggregationBlock = AggregationBlock( + after, + Aggregations(agg.toSet), + group.map(_._1).toSet, + context.workingGraph + ) val updatedRegistry2 = updatedRegistry1.register(aggregationBlock) - val updatedContext = context.updateKnownConnections(fieldExprs).withBlocks(updatedRegistry2) - put[R, IRBuilderContext](updatedContext) >> pure[R, List[Block]](List(projectBlock, aggregationBlock)) + val updatedContext = context + .updateKnownConnections(fieldExprs) + .withBlocks(updatedRegistry2) + put[R, IRBuilderContext](updatedContext) >> pure[R, List[Block]]( + List(projectBlock, aggregationBlock) + ) } } yield blocks @@ -261,10 +339,16 @@ object IRBuilder extends CompilationStage[ast.Statement, CypherStatement, IRBuil block <- { val (list, item) = tuple val binds: UnwoundList = UnwoundList(list, item) - val block = UnwindBlock(context.blockRegistry.lastAdded.toList, binds, context.workingGraph) + val block = UnwindBlock( + context.blockRegistry.lastAdded.toList, + binds, + context.workingGraph + ) val updatedRegistry = context.blockRegistry.register(block) val updatedContext = context.withBlocks(updatedRegistry) - put[R, IRBuilderContext](updatedContext) >> pure[R, List[Block]](List(block)) + put[R, IRBuilderContext](updatedContext) >> pure[R, List[Block]]( + List(block) + ) } } yield block @@ -274,21 +358,29 @@ object IRBuilder extends CompilationStage[ast.Statement, CypherStatement, IRBuil qgn = context.qgnGenerator.generate - explicitCloneItems <- clones.flatMap(_.items).traverse(convertClone[R](_, qgn)) + explicitCloneItems <- clones + .flatMap(_.items) + .traverse(convertClone[R](_, qgn)) - createPatterns <- creates.map { - case ast.CreateInConstruct(p: exp.Pattern) => p - }.traverse(convertPattern[R](_, Some(qgn))) + createPatterns <- creates + .map { case ast.CreateInConstruct(p: exp.Pattern) => + p + } + .traverse(convertPattern[R](_, Some(qgn))) - setItems <- sets.flatMap { - case ast.SetClause(s) => s - }.traverse(convertSetItem[R]) + setItems <- sets + .flatMap { case ast.SetClause(s) => + s + } + .traverse(convertSetItem[R]) refs <- { - val onGraphs: List[QualifiedGraphName] = on.map(graph => QualifiedGraphName(graph.parts)) - val schemaForOnGraphUnion = onGraphs.foldLeft(PropertyGraphSchema.empty) { case (agg, next) => - agg ++ context.schemaFor(next) - } + val onGraphs: List[QualifiedGraphName] = + on.map(graph => QualifiedGraphName(graph.parts)) + val schemaForOnGraphUnion = + onGraphs.foldLeft(PropertyGraphSchema.empty) { case (agg, next) => + agg ++ context.schemaFor(next) + } // Computing single nodes/rels constructed by CREATEd // TODO: Throw exception if both clone alias and original field name are used in CREATE @@ -299,13 +391,15 @@ object IRBuilder extends CompilationStage[ast.Statement, CypherStatement, IRBuil val aliasedCloneItems = explicitCloneItemMap.filter { case (field, expr: Var) => field.name != expr.name - case _ => false + case _ => false } val cloneContext = context.renameKnownConnections(aliasedCloneItems) // Items from other graphs that are cloned by default val implicitCloneItems = createPattern.fields.filterNot { f => - f.cypherType.graph.get == qgn || explicitCloneItemMap.keys.exists(_.name == f.name) + f.cypherType.graph.get == qgn || explicitCloneItemMap.keys.exists( + _.name == f.name + ) } val implicitCloneItemMap = implicitCloneItems.map { f => // Convert field to clone item @@ -314,62 +408,92 @@ object IRBuilder extends CompilationStage[ast.Statement, CypherStatement, IRBuil val cloneItemMap = implicitCloneItemMap ++ explicitCloneItemMap // Fields inside of CONSTRUCT could have been matched on other graphs than just the workingGraph - val cloneSchema = schemaForElementTypes(context, cloneItemMap.values.map(_.cypherType).toSet) + val cloneSchema = schemaForElementTypes( + context, + cloneItemMap.values.map(_.cypherType).toSet + ) - //Make sure cloned relationships have the correct source and target nodes + // Make sure cloned relationships have the correct source and target nodes cloneItemMap.foreach { case (IRField(constructRelName), _: RelationshipVar) => createPatterns.flatMap(_.topology).foreach { - case (relField@IRField(patternFieldName), con: Connection) if patternFieldName == constructRelName => - val expectedConnection = cloneContext.knownConnections.getOrElse(relField, throw IllegalArgumentException("correct source and target nodes", con, - s"Cloned relationship '$patternFieldName' needs its source and target nodes from the MATCH pattern (could be out of scope after a WITH).")) + case (relField @ IRField(patternFieldName), con: Connection) + if patternFieldName == constructRelName => + val expectedConnection = + cloneContext.knownConnections.getOrElse( + relField, + throw IllegalArgumentException( + "correct source and target nodes", + con, + s"Cloned relationship '$patternFieldName' needs its source and target nodes from the MATCH pattern (could be out of scope after a WITH)." + ) + ) if (expectedConnection != con) - throw IllegalArgumentException(s"Following pattern was expected: \n $expectedConnection", con, - s"Cloned relationships are are only allowed with the corresponding source and target node") + throw IllegalArgumentException( + s"Following pattern was expected: \n $expectedConnection", + con, + s"Cloned relationships are are only allowed with the corresponding source and target node" + ) case _ => () } case _ => () } - // Make sure that there are no dangling relationships // we can currently only clone relationships that are also part of a new pattern cloneItemMap.keys.foreach { cloneFieldAlias => cloneFieldAlias.cypherType match { case _: CTRelationship if !createPattern.fields.contains(cloneFieldAlias) => - throw UnsupportedOperationException(s"Can only clone relationship ${cloneFieldAlias.name} if it is also part of a CREATE pattern") + throw UnsupportedOperationException( + s"Can only clone relationship ${cloneFieldAlias.name} if it is also part of a CREATE pattern" + ) case _ => () } } - val fieldsInNewPattern = createPattern - .fields + val fieldsInNewPattern = createPattern.fields .filterNot(cloneItemMap.contains) val patternSchema = fieldsInNewPattern.foldLeft(cloneSchema) { case (acc, next) => - val newFieldSchema = schemaForNewField(next, createPattern, context) + val newFieldSchema = + schemaForNewField(next, createPattern, context) acc ++ newFieldSchema } - val (patternSchemaWithSetItems, _) = setItems.foldLeft(patternSchema -> Map.empty[Var, CypherType]) { - case ((currentSchema, rewrittenVarTypes), setItem: SetItem) => - setItem match { - case SetLabelItem(variable, labels) => - val (existingLabels, existingQgn) = rewrittenVarTypes.getOrElse(variable, variable.cypherType) match { - case CTNode(ls, qualifiedGraphName) => ls -> qualifiedGraphName - case other => throw UnsupportedOperationException(s"SET label on something that is not a node: $other") - } - val labelsAfterSet = existingLabels ++ labels - val updatedSchema = currentSchema.addLabelsToCombo(labels, existingLabels) - updatedSchema -> rewrittenVarTypes.updated(variable, CTNode(labelsAfterSet, existingQgn)) - case SetPropertyItem(propertyKey, variable, setValue) => - val propertyType = setValue.cypherType - val updatedSchema = currentSchema.addPropertyToElement(propertyKey, propertyType, variable.cypherType) - updatedSchema -> rewrittenVarTypes - } - } + val (patternSchemaWithSetItems, _) = + setItems.foldLeft(patternSchema -> Map.empty[Var, CypherType]) { + case ((currentSchema, rewrittenVarTypes), setItem: SetItem) => + setItem match { + case SetLabelItem(variable, labels) => + val (existingLabels, existingQgn) = rewrittenVarTypes + .getOrElse(variable, variable.cypherType) match { + case CTNode(ls, qualifiedGraphName) => + ls -> qualifiedGraphName + case other => + throw UnsupportedOperationException( + s"SET label on something that is not a node: $other" + ) + } + val labelsAfterSet = existingLabels ++ labels + val updatedSchema = + currentSchema.addLabelsToCombo(labels, existingLabels) + updatedSchema -> rewrittenVarTypes.updated( + variable, + CTNode(labelsAfterSet, existingQgn) + ) + case SetPropertyItem(propertyKey, variable, setValue) => + val propertyType = setValue.cypherType + val updatedSchema = currentSchema.addPropertyToElement( + propertyKey, + propertyType, + variable.cypherType + ) + updatedSchema -> rewrittenVarTypes + } + } - val patternGraphSchema = schemaForOnGraphUnion ++ patternSchemaWithSetItems + val patternGraphSchema = + schemaForOnGraphUnion ++ patternSchemaWithSetItems val patternGraph = IRPatternGraph( qgn, @@ -377,13 +501,16 @@ object IRBuilder extends CompilationStage[ast.Statement, CypherStatement, IRBuil cloneItemMap, createPattern, setItems, - onGraphs) + onGraphs + ) val updatedContext = context.resetKnownConnections .withWorkingGraph(patternGraph) .registerSchema(qgn, patternGraphSchema) .resetKnownTypes - put[R, IRBuilderContext](updatedContext) >> pure[R, List[Block]](List.empty) + put[R, IRBuilderContext](updatedContext) >> pure[R, List[Block]]( + List.empty + ) } } yield refs @@ -394,45 +521,79 @@ object IRBuilder extends CompilationStage[ast.Statement, CypherStatement, IRBuil val after = context.blockRegistry.lastAdded.toList val returns = GraphResultBlock(after, context.workingGraph) val updatedRegistry = context.blockRegistry.register(returns) - put[R, IRBuilderContext](context.resetKnownConnections.withBlocks(updatedRegistry)) >> pure[R, List[Block]](List(returns)) + put[R, IRBuilderContext]( + context.resetKnownConnections.withBlocks(updatedRegistry) + ) >> pure[R, List[Block]](List(returns)) } } yield refs - case ast.Return(distinct, ast.ReturnItems(_, items), orderBy, skip, limit, _) => + case ast.Return( + distinct, + ast.ReturnItems(_, items), + orderBy, + skip, + limit, + _ + ) => for { fieldExprs <- items.toList.traverse(convertReturnItem[R]) context <- get[R, IRBuilderContext] blocks1 <- { val updatedContext = context.resetKnownConnections val (projectRef, projectReg) = - registerProjectBlock(updatedContext, fieldExprs, distinct = distinct, source = context.workingGraph) + registerProjectBlock( + updatedContext, + fieldExprs, + distinct = distinct, + source = context.workingGraph + ) val appendList = (list: List[Block]) => pure[R, List[Block]](projectRef +: list) - val orderAndSliceBlock = registerOrderAndSliceBlock(orderBy, skip, limit) - put[R, IRBuilderContext](updatedContext.withBlocks(projectReg)) >> orderAndSliceBlock flatMap appendList + val orderAndSliceBlock = + registerOrderAndSliceBlock(orderBy, skip, limit) + put[R, IRBuilderContext]( + updatedContext.withBlocks(projectReg) + ) >> orderAndSliceBlock flatMap appendList } context2 <- get[R, IRBuilderContext] blocks2 <- { val rItems = fieldExprs.map(_._1) val orderedFields = OrderedFields(rItems) - val resultBlock = TableResultBlock(List(blocks1.last), orderedFields, context.workingGraph) + val resultBlock = TableResultBlock( + List(blocks1.last), + orderedFields, + context.workingGraph + ) val updatedRegistry = context2.blockRegistry.register(resultBlock) - val updatedContext = context.resetKnownConnections.withBlocks(updatedRegistry) - put[R, IRBuilderContext](updatedContext) >> pure[R, List[Block]](blocks1 :+ resultBlock) + val updatedContext = + context.resetKnownConnections.withBlocks(updatedRegistry) + put[R, IRBuilderContext](updatedContext) >> pure[R, List[Block]]( + blocks1 :+ resultBlock + ) } } yield blocks2 case x => - error(IRBuilderError(s"Clause not yet supported: $x"))(List.empty[Block]) + error(IRBuilderError(s"Clause not yet supported: $x"))( + List.empty[Block] + ) } } - def schemaForElementTypes(context: IRBuilderContext, cypherTypes: Set[CypherType]): PropertyGraphSchema = + def schemaForElementTypes( + context: IRBuilderContext, + cypherTypes: Set[CypherType] + ): PropertyGraphSchema = cypherTypes .map(schemaForElementType(context, _)) .foldLeft(PropertyGraphSchema.empty)(_ ++ _) - def schemaForElementType(context: IRBuilderContext, cypherType: CypherType): PropertyGraphSchema = { - val graphSchema = cypherType.graph.map(context.schemaFor).getOrElse(context.workingGraph.schema) + def schemaForElementType( + context: IRBuilderContext, + cypherType: CypherType + ): PropertyGraphSchema = { + val graphSchema = cypherType.graph + .map(context.schemaFor) + .getOrElse(context.workingGraph.schema) graphSchema.forElementType(cypherType) } @@ -452,7 +613,7 @@ object IRBuilder extends CompilationStage[ast.Statement, CypherStatement, IRBuil projs -> blockRegistry.register(projs) } - private def registerOrderAndSliceBlock[R: _mayFail : _hasContext]( + private def registerOrderAndSliceBlock[R: _mayFail: _hasContext]( orderBy: Option[ast.OrderBy], skip: Option[ast.Skip], limit: Option[ast.Limit] @@ -468,25 +629,35 @@ object IRBuilder extends CompilationStage[ast.Statement, CypherStatement, IRBuil limitExpr <- convertExpr(limit.map(_.expression)) blocks <- { - if (sortItems.isEmpty && skipExpr.isEmpty && limitExpr.isEmpty) pure[R, List[Block]](List()) + if (sortItems.isEmpty && skipExpr.isEmpty && limitExpr.isEmpty) + pure[R, List[Block]](List()) else { val blockRegistry = context.blockRegistry val after = blockRegistry.lastAdded.toList - val orderAndSliceBlock = OrderAndSliceBlock(after, sortItems, skipExpr, limitExpr, context.workingGraph) + val orderAndSliceBlock = OrderAndSliceBlock( + after, + sortItems, + skipExpr, + limitExpr, + context.workingGraph + ) val updatedRegistry = blockRegistry.register(orderAndSliceBlock) - put[R, IRBuilderContext](context.copy(blockRegistry = updatedRegistry)) >> pure[R, List[Block]](List(orderAndSliceBlock)) + put[R, IRBuilderContext]( + context.copy(blockRegistry = updatedRegistry) + ) >> pure[R, List[Block]](List(orderAndSliceBlock)) } } } yield blocks } - private def convertClone[R: _mayFail : _hasContext]( + private def convertClone[R: _mayFail: _hasContext]( item: ast.ReturnItem, qgn: QualifiedGraphName ): Eff[R, (IRField, Expr)] = { - def convert(cypherType: CypherType, name: String): IRField = IRField(name)(cypherType.withGraph(qgn)) + def convert(cypherType: CypherType, name: String): IRField = + IRField(name)(cypherType.withGraph(qgn)) item match { @@ -496,7 +667,10 @@ object IRBuilder extends CompilationStage[ast.Statement, CypherStatement, IRBuil context <- get[R, IRBuilderContext] field <- { val field = convert(expr.cypherType, v.name) - put[R, IRBuilderContext](context.withFields(Set(field))) >> pure[R, IRField](field) + put[R, IRBuilderContext](context.withFields(Set(field))) >> pure[ + R, + IRField + ](field) } } yield field -> expr @@ -506,16 +680,24 @@ object IRBuilder extends CompilationStage[ast.Statement, CypherStatement, IRBuil context <- get[R, IRBuilderContext] field <- { val field = convert(expr.cypherType, name) - put[R, IRBuilderContext](context.withFields(Set(field))) >> pure[R, IRField](field) + put[R, IRBuilderContext](context.withFields(Set(field))) >> pure[ + R, + IRField + ](field) } } yield field -> expr case _ => - throw IllegalArgumentException(s"${ast.AliasedReturnItem.getClass} or ${ast.UnaliasedReturnItem.getClass}", item.getClass) + throw IllegalArgumentException( + s"${ast.AliasedReturnItem.getClass} or ${ast.UnaliasedReturnItem.getClass}", + item.getClass + ) } } - private def convertReturnItem[R: _mayFail : _hasContext](item: ast.ReturnItem): Eff[R, (IRField, Expr)] = item match { + private def convertReturnItem[R: _mayFail: _hasContext]( + item: ast.ReturnItem + ): Eff[R, (IRField, Expr)] = item match { case ast.AliasedReturnItem(e, v) => for { @@ -523,7 +705,10 @@ object IRBuilder extends CompilationStage[ast.Statement, CypherStatement, IRBuil context <- get[R, IRBuilderContext] field <- { val field = IRField(v.name)(expr.cypherType) - put[R, IRBuilderContext](context.withFields(Set(field))) >> pure[R, IRField](field) + put[R, IRBuilderContext](context.withFields(Set(field))) >> pure[ + R, + IRField + ](field) } } yield field -> expr @@ -533,15 +718,21 @@ object IRBuilder extends CompilationStage[ast.Statement, CypherStatement, IRBuil context <- get[R, IRBuilderContext] field <- { val field = IRField(name)(expr.cypherType) - put[R, IRBuilderContext](context.withFields(Set(field))) >> pure[R, IRField](field) + put[R, IRBuilderContext](context.withFields(Set(field))) >> pure[ + R, + IRField + ](field) } } yield field -> expr case _ => - throw IllegalArgumentException(s"${ast.AliasedReturnItem.getClass} or ${ast.UnaliasedReturnItem.getClass}", item.getClass) + throw IllegalArgumentException( + s"${ast.AliasedReturnItem.getClass} or ${ast.UnaliasedReturnItem.getClass}", + item.getClass + ) } - private def convertUnwindItem[R: _mayFail : _hasContext]( + private def convertUnwindItem[R: _mayFail: _hasContext]( list: exp.Expression, variable: exp.Variable ): Eff[R, (Expr, IRField)] = { @@ -560,7 +751,10 @@ object IRBuilder extends CompilationStage[ast.Statement, CypherStatement, IRBuil } field <- { val field = IRField(variable.name)(typ) - put[R, IRBuilderContext](context.withFields(Set(field))) >> pure[R, (Expr, IRField)](expr -> field) + put[R, IRBuilderContext](context.withFields(Set(field))) >> pure[ + R, + (Expr, IRField) + ](expr -> field) } } yield field } @@ -573,36 +767,41 @@ object IRBuilder extends CompilationStage[ast.Statement, CypherStatement, IRBuil context <- get[R, IRBuilderContext] result <- { val pattern = context.convertPattern(p, qgn) - val patternTypes = pattern.fields.foldLeft(context.knownTypes) { - case (acc, f) => acc.updated(exp.Variable(f.name)(InputPosition.NONE), f.cypherType) + val patternTypes = pattern.fields.foldLeft(context.knownTypes) { case (acc, f) => + acc.updated(exp.Variable(f.name)(InputPosition.NONE), f.cypherType) } - put[R, IRBuilderContext](context.copy(knownTypes = patternTypes)) >> pure[R, Pattern](pattern) + put[R, IRBuilderContext]( + context.copy(knownTypes = patternTypes) + ) >> pure[R, Pattern](pattern) } } yield result } - private def convertExpr[R: _mayFail : _hasContext](e: Option[exp.Expression]): Eff[R, Option[Expr]] = + private def convertExpr[R: _mayFail: _hasContext]( + e: Option[exp.Expression] + ): Eff[R, Option[Expr]] = for { context <- get[R, IRBuilderContext] - } yield - e match { - case Some(expr) => Some(context.convertExpression(expr)) - case None => None - } + } yield e match { + case Some(expr) => Some(context.convertExpression(expr)) + case None => None + } private def convertExpr[R: _hasContext](e: exp.Expression): Eff[R, Expr] = for { context <- get[R, IRBuilderContext] } yield context.convertExpression(e) - private def convertWhere[R: _hasContext](where: Option[ast.Where]): Eff[R, Set[Expr]] = where match { + private def convertWhere[R: _hasContext]( + where: Option[ast.Where] + ): Eff[R, Set[Expr]] = where match { case Some(ast.Where(expr)) => for { predicate <- convertExpr(expr) } yield { predicate match { case org.opencypher.okapi.ir.api.expr.Ands(exprs) => exprs.toSet - case e => Set(e) + case e => Set(e) } } @@ -610,17 +809,22 @@ object IRBuilder extends CompilationStage[ast.Statement, CypherStatement, IRBuil pure[R, Set[Expr]](Set.empty) } - private def convertRegistry[R: _mayFail : _hasContext]: Eff[R, Option[CypherQuery]] = + private def convertRegistry[R: _mayFail: _hasContext]: Eff[R, Option[CypherQuery]] = for { context <- get[R, IRBuilderContext] } yield { val blocks = context.blockRegistry - val model = QueryModel(blocks.lastAdded.get.asInstanceOf[ResultBlock], context.parameters) + val model = QueryModel( + blocks.lastAdded.get.asInstanceOf[ResultBlock], + context.parameters + ) Some(SingleQuery(model)) } - private def convertSortItem[R: _mayFail : _hasContext](item: ast.SortItem): Eff[R, SortItem] = { + private def convertSortItem[R: _mayFail: _hasContext]( + item: ast.SortItem + ): Eff[R, SortItem] = { item match { case ast.AscSortItem(astExpr) => for { @@ -633,35 +837,46 @@ object IRBuilder extends CompilationStage[ast.Statement, CypherStatement, IRBuil } } - private def schemaForNewField(field: IRField, pattern: Pattern, context: IRBuilderContext): PropertyGraphSchema = { - val baseFieldSchema = pattern.baseFields.get(field).map { baseNode => - schemaForElementType(context, baseNode.cypherType) - }.getOrElse(PropertyGraphSchema.empty) + private def schemaForNewField( + field: IRField, + pattern: Pattern, + context: IRBuilderContext + ): PropertyGraphSchema = { + val baseFieldSchema = pattern.baseFields + .get(field) + .map { baseNode => + schemaForElementType(context, baseNode.cypherType) + } + .getOrElse(PropertyGraphSchema.empty) - val newPropertyKeys: Map[String, CypherType] = pattern.properties.get(field) + val newPropertyKeys: Map[String, CypherType] = pattern.properties + .get(field) .map(_.items.map(p => p._1 -> p._2.cypherType)) .getOrElse(Map.empty) field.cypherType match { case CTNode(newLabels, _) => - val oldLabelCombosToNewLabelCombos = if (baseFieldSchema.labels.nonEmpty) - baseFieldSchema.allCombinations.map(oldLabels => oldLabels -> (oldLabels ++ newLabels)) - else - Set(Set.empty[String] -> newLabels) + val oldLabelCombosToNewLabelCombos = + if (baseFieldSchema.labels.nonEmpty) + baseFieldSchema.allCombinations.map(oldLabels => oldLabels -> (oldLabels ++ newLabels)) + else + Set(Set.empty[String] -> newLabels) val updatedPropertyKeys = oldLabelCombosToNewLabelCombos.map { - case (oldLabelCombo, newLabelCombo) => newLabelCombo -> (baseFieldSchema.nodePropertyKeys(oldLabelCombo) ++ newPropertyKeys) + case (oldLabelCombo, newLabelCombo) => + newLabelCombo -> (baseFieldSchema.nodePropertyKeys( + oldLabelCombo + ) ++ newPropertyKeys) } updatedPropertyKeys.foldLeft(PropertyGraphSchema.empty) { - case (acc, (labelCombo, propertyKeys)) => acc.withNodePropertyKeys(labelCombo, propertyKeys) + case (acc, (labelCombo, propertyKeys)) => + acc.withNodePropertyKeys(labelCombo, propertyKeys) } // if there is only one relationship type we need to merge all existing types and update them case CTRelationship(newTypes, _) if newTypes.size == 1 => - val possiblePropertyKeys = baseFieldSchema - .relTypePropertyMap - .values + val possiblePropertyKeys = baseFieldSchema.relTypePropertyMap.values .map(_.keySet) .foldLeft(Set.empty[String])(_ ++ _) @@ -671,27 +886,47 @@ object IRBuilder extends CompilationStage[ast.Statement, CypherStatement, IRBuil val updatedPropertyKeys = joinedPropertyKeys ++ newPropertyKeys - PropertyGraphSchema.empty.withRelationshipPropertyKeys(newTypes.head, updatedPropertyKeys) + PropertyGraphSchema.empty.withRelationshipPropertyKeys( + newTypes.head, + updatedPropertyKeys + ) case CTRelationship(newTypes, _) => - val actualTypes = if (newTypes.nonEmpty) newTypes else baseFieldSchema.relationshipTypes - - actualTypes.foldLeft(PropertyGraphSchema.empty) { - case (acc, relType) => acc.withRelationshipPropertyKeys(relType, baseFieldSchema.relationshipPropertyKeys(relType) ++ newPropertyKeys) + val actualTypes = + if (newTypes.nonEmpty) newTypes else baseFieldSchema.relationshipTypes + + actualTypes.foldLeft(PropertyGraphSchema.empty) { case (acc, relType) => + acc.withRelationshipPropertyKeys( + relType, + baseFieldSchema.relationshipPropertyKeys(relType) ++ newPropertyKeys + ) } - case other => throw IllegalArgumentException("CTNode or CTRelationship", other) + case other => + throw IllegalArgumentException("CTNode or CTRelationship", other) } } - private def convertSetItem[R: _hasContext](p: ast.SetItem): Eff[R, SetItem] = { + private def convertSetItem[R: _hasContext]( + p: ast.SetItem + ): Eff[R, SetItem] = { p match { - case ast.SetPropertyItem(exp.LogicalProperty(map: exp.Variable, exp.PropertyKeyName(propertyName)), setValue: exp.Expression) => + case ast.SetPropertyItem( + exp.LogicalProperty( + map: exp.Variable, + exp.PropertyKeyName(propertyName) + ), + setValue: exp.Expression + ) => for { variable <- convertExpr[R](map) convertedSetExpr <- convertExpr[R](setValue) result <- { - val setItem = SetPropertyItem(propertyName, variable.asInstanceOf[Var], convertedSetExpr) + val setItem = SetPropertyItem( + propertyName, + variable.asInstanceOf[Var], + convertedSetExpr + ) pure[R, SetItem](setItem) } } yield result @@ -699,7 +934,8 @@ object IRBuilder extends CompilationStage[ast.Statement, CypherStatement, IRBuil for { variable <- convertExpr[R](expr) result <- { - val setLabel: SetItem = SetLabelItem(variable.asInstanceOf[Var], labels.map(_.name).toSet) + val setLabel: SetItem = + SetLabelItem(variable.asInstanceOf[Var], labels.map(_.name).toSet) pure[R, SetItem](setLabel) } } yield result diff --git a/okapi-ir/src/main/scala/org/opencypher/okapi/ir/impl/IRBuilderContext.scala b/okapi-ir/src/main/scala/org/opencypher/okapi/ir/impl/IRBuilderContext.scala index c2a424de45..164fbbc541 100644 --- a/okapi-ir/src/main/scala/org/opencypher/okapi/ir/impl/IRBuilderContext.scala +++ b/okapi-ir/src/main/scala/org/opencypher/okapi/ir/impl/IRBuilderContext.scala @@ -61,20 +61,29 @@ final case class IRBuilderContext( private lazy val exprConverter = new ExpressionConverter(self) private lazy val patternConverter = new PatternConverter(self) - def convertPattern(p: ast.Pattern, qgn: Option[QualifiedGraphName] = None): Pattern = { - patternConverter.convert(p, knownTypes, qgn.getOrElse(workingGraph.qualifiedGraphName)) + def convertPattern( + p: ast.Pattern, + qgn: Option[QualifiedGraphName] = None + ): Pattern = { + patternConverter.convert( + p, + knownTypes, + qgn.getOrElse(workingGraph.qualifiedGraphName) + ) } - def convertExpression(e: ast.Expression): Expr = exprConverter.convert(e)(lambdaVars = Map()) + def convertExpression(e: ast.Expression): Expr = + exprConverter.convert(e)(lambdaVars = Map()) - def schemaFor(qgn: QualifiedGraphName): PropertyGraphSchema = queryLocalCatalog.schema(qgn) + def schemaFor(qgn: QualifiedGraphName): PropertyGraphSchema = + queryLocalCatalog.schema(qgn) - def withBlocks(reg: BlockRegistry): IRBuilderContext = copy(blockRegistry = reg) + def withBlocks(reg: BlockRegistry): IRBuilderContext = + copy(blockRegistry = reg) def withFields(fields: Set[IRField]): IRBuilderContext = { - val withFieldTypes = fields.foldLeft(knownTypes) { - case (acc, f) => - acc.updated(ast.Variable(f.name)(InputPosition.NONE), f.cypherType) + val withFieldTypes = fields.foldLeft(knownTypes) { case (acc, f) => + acc.updated(ast.Variable(f.name)(InputPosition.NONE), f.cypherType) } copy(knownTypes = withFieldTypes) } @@ -82,21 +91,26 @@ final case class IRBuilderContext( def withWorkingGraph(graph: IRGraph): IRBuilderContext = copy(workingGraph = graph) - def updateKnownConnections(fields: List[(IRField, Expr)]): IRBuilderContext = { + def updateKnownConnections( + fields: List[(IRField, Expr)] + ): IRBuilderContext = { val remainingConnections = knownConnections.flatMap { case (relField: IRField, con: Connection) => val patternFields = Seq(relField, con.source, con.target) val renamedFields = patternFields.flatMap(knownPatternField => { - fields.find { - case (_, expr: Var) => expr.name == knownPatternField.name - case _ => false - }.map { case (field, _) => field } + fields + .find { + case (_, expr: Var) => expr.name == knownPatternField.name + case _ => false + } + .map { case (field, _) => field } }) renamedFields match { case (rel: IRField) :: (src: IRField) :: (target: IRField) :: Nil => val renamedEndPoints = Endpoints.two(src -> target) - val renamedCon: Connection = ConnectionCopier.copy(con, renamedEndPoints) + val renamedCon: Connection = + ConnectionCopier.copy(con, renamedEndPoints) Some(rel -> renamedCon) case _ => None } @@ -106,22 +120,26 @@ final case class IRBuilderContext( } def renameKnownConnections(fields: Map[IRField, Expr]): IRBuilderContext = { - val remainingConnections = knownConnections.map { - case (relField: IRField, con: Connection) => - val patternFields = Seq(relField, con.source, con.target) - val renamedFields = patternFields.map(knownPatternField => { - fields.flatMap { - case (aliasField, expr: Var) if expr.name == knownPatternField.name => Some(aliasField) + val remainingConnections = knownConnections.map { case (relField: IRField, con: Connection) => + val patternFields = Seq(relField, con.source, con.target) + val renamedFields = patternFields.map(knownPatternField => { + fields + .flatMap { + case (aliasField, expr: Var) if expr.name == knownPatternField.name => + Some(aliasField) case _ => None - }.headOption.getOrElse(knownPatternField) - }) - - renamedFields match { - case (rel: IRField) :: (src: IRField) :: (target: IRField) :: Nil => - val renamedEndPoints = Endpoints.two(src -> target) - val renamedCon: Connection = ConnectionCopier.copy(con, renamedEndPoints) - rel -> renamedCon - } + } + .headOption + .getOrElse(knownPatternField) + }) + + renamedFields match { + case (rel: IRField) :: (src: IRField) :: (target: IRField) :: Nil => + val renamedEndPoints = Endpoints.two(src -> target) + val renamedCon: Connection = + ConnectionCopier.copy(con, renamedEndPoints) + rel -> renamedCon + } } copy(knownConnections = remainingConnections) @@ -135,10 +153,16 @@ final case class IRBuilderContext( copy(knownTypes = Map.empty) } - def registerGraph(qgn: QualifiedGraphName, graph: PropertyGraph): IRBuilderContext = + def registerGraph( + qgn: QualifiedGraphName, + graph: PropertyGraph + ): IRBuilderContext = copy(queryLocalCatalog = queryLocalCatalog.withGraph(qgn, graph)) - def registerSchema(qgn: QualifiedGraphName, schema: PropertyGraphSchema): IRBuilderContext = + def registerSchema( + qgn: QualifiedGraphName, + schema: PropertyGraphSchema + ): IRBuilderContext = copy(queryLocalCatalog = queryLocalCatalog.withSchema(qgn, schema)) def resetRegistry: IRBuilderContext = { @@ -146,7 +170,9 @@ final case class IRBuilderContext( copy(blockRegistry = BlockRegistry.empty.register(sourceBlock)) } - def registerConnections(connections: ListMap[IRField, Connection]): IRBuilderContext = { + def registerConnections( + connections: ListMap[IRField, Connection] + ): IRBuilderContext = { copy(knownConnections = knownConnections ++ connections) } } @@ -176,8 +202,11 @@ object IRBuilderContext { updatedRegistry, semState, queryLocalCatalog, - instantiateView) + instantiateView + ) - context.withFields(fieldsFromDrivingTable.map(v => IRField(v.name)(v.cypherType))) + context.withFields( + fieldsFromDrivingTable.map(v => IRField(v.name)(v.cypherType)) + ) } } diff --git a/okapi-ir/src/main/scala/org/opencypher/okapi/ir/impl/PatternConverter.scala b/okapi-ir/src/main/scala/org/opencypher/okapi/ir/impl/PatternConverter.scala index 5457784ac9..07daf5bf4e 100644 --- a/okapi-ir/src/main/scala/org/opencypher/okapi/ir/impl/PatternConverter.scala +++ b/okapi-ir/src/main/scala/org/opencypher/okapi/ir/impl/PatternConverter.scala @@ -63,20 +63,28 @@ final class PatternConverter(irBuilderContext: IRBuilderContext) { qualifiedGraphName: QualifiedGraphName, pattern: Pattern = Pattern.empty ): Pattern = - convertElement(p.element, knownTypes, qualifiedGraphName).runS(pattern).value + convertElement(p.element, knownTypes, qualifiedGraphName) + .runS(pattern) + .value private def convertPattern( p: ast.Pattern, knownTypes: Map[ast.Expression, CypherType], qualifiedGraphName: QualifiedGraphName ): Result[Unit] = - Foldable[List].sequence_[Result, Unit](p.patternParts.toList.map(convertPart(knownTypes, qualifiedGraphName))) + Foldable[List].sequence_[Result, Unit]( + p.patternParts.toList.map(convertPart(knownTypes, qualifiedGraphName)) + ) @tailrec - private def convertPart(knownTypes: Map[ast.Expression, CypherType], qualifiedGraphName: QualifiedGraphName) - (p: ast.PatternPart): Result[Unit] = p match { - case _: ast.AnonymousPatternPart => stomp(convertElement(p.element, knownTypes, qualifiedGraphName)) - case ast.NamedPatternPart(_, part) => convertPart(knownTypes, qualifiedGraphName)(part) + private def convertPart( + knownTypes: Map[ast.Expression, CypherType], + qualifiedGraphName: QualifiedGraphName + )(p: ast.PatternPart): Result[Unit] = p match { + case _: ast.AnonymousPatternPart => + stomp(convertElement(p.element, knownTypes, qualifiedGraphName)) + case ast.NamedPatternPart(_, part) => + convertPart(knownTypes, qualifiedGraphName)(part) } private def convertElement( @@ -86,36 +94,67 @@ final class PatternConverter(irBuilderContext: IRBuilderContext) { ): Result[IRField] = p match { - case np@ast.NodePattern(vOpt, labels: Seq[ast.LabelName], propertiesOpt, baseNodeVar) => + case np @ ast.NodePattern( + vOpt, + labels: Seq[ast.LabelName], + propertiesOpt, + baseNodeVar + ) => // labels within CREATE patterns, e.g. CREATE (a:Foo), labels for MATCH clauses are rewritten to WHERE val patternLabels = labels.map(_.name).toSet val baseNodeCypherTypeOpt = baseNodeVar.map(knownTypes) - val baseNodeLabels = baseNodeCypherTypeOpt.map(_.toCTNode.labels).getOrElse(Set.empty) + val baseNodeLabels = + baseNodeCypherTypeOpt.map(_.toCTNode.labels).getOrElse(Set.empty) // labels defined in outside scope, passed in by IRBuilder - val (knownLabels, qgnOption) = vOpt.flatMap(expr => knownTypes.get(expr)).flatMap { - case n: CTNode => Some(n.labels -> n.graph) - case _ => None - }.getOrElse(Set.empty[String] -> Some(qualifiedGraphName)) + val (knownLabels, qgnOption) = vOpt + .flatMap(expr => knownTypes.get(expr)) + .flatMap { + case n: CTNode => Some(n.labels -> n.graph) + case _ => None + } + .getOrElse(Set.empty[String] -> Some(qualifiedGraphName)) val allLabels = patternLabels ++ knownLabels ++ baseNodeLabels val nodeVar = vOpt match { case Some(v) => Var(v.name)(CTNode(allLabels, qgnOption)) - case None => FreshVariableNamer(np.position.offset, CTNode(allLabels, qgnOption)) + case None => + FreshVariableNamer(np.position.offset, CTNode(allLabels, qgnOption)) } val baseNodeField = baseNodeVar.map(x => IRField(x.name)(knownTypes(x))) for { element <- pure(IRField(nodeVar.name)(nodeVar.cypherType)) - _ <- modify[Pattern](_.withElement(element, extractProperties(propertiesOpt)).withBaseField(element, baseNodeField)) + _ <- modify[Pattern]( + _.withElement(element, extractProperties(propertiesOpt)) + .withBaseField(element, baseNodeField) + ) } yield element - case rc@ast.RelationshipChain(left, ast.RelationshipPattern(eOpt, types, rangeOpt, propertiesOpt, dir, _, baseRelVar), right) => - - val relVar = createRelationshipVar(knownTypes, rc.position.offset, eOpt, types, baseRelVar, qualifiedGraphName) + case rc @ ast.RelationshipChain( + left, + ast.RelationshipPattern( + eOpt, + types, + rangeOpt, + propertiesOpt, + dir, + _, + baseRelVar + ), + right + ) => + val relVar = createRelationshipVar( + knownTypes, + rc.position.offset, + eOpt, + types, + baseRelVar, + qualifiedGraphName + ) val convertedProperties = extractProperties(propertiesOpt) val baseRelField = baseRelVar.map(x => IRField(x.name)(knownTypes(x))) @@ -123,7 +162,12 @@ final class PatternConverter(irBuilderContext: IRBuilderContext) { for { source <- convertElement(left, knownTypes, qualifiedGraphName) target <- convertElement(right, knownTypes, qualifiedGraphName) - rel <- pure(IRField(relVar.name)(if (rangeOpt.isDefined) CTList(relVar.cypherType) else relVar.cypherType)) + rel <- pure( + IRField(relVar.name)( + if (rangeOpt.isDefined) CTList(relVar.cypherType) + else relVar.cypherType + ) + ) _ <- modify[Pattern] { given => val registered = given .withElement(rel) @@ -134,58 +178,114 @@ final class PatternConverter(irBuilderContext: IRBuilderContext) { val lower = range.lower.map(_.value.intValue()).getOrElse(1) val upper = range.upper .map(_.value.intValue()) - .getOrElse(throw NotImplementedException("Support for unbounded var-length not yet implemented")) + .getOrElse( + throw NotImplementedException( + "Support for unbounded var-length not yet implemented" + ) + ) val relType = relVar.cypherType.toCTRelationship Endpoints.apply(source, target) match { case _: IdenticalEndpoints => - throw NotImplementedException("Support for cyclic var-length not yet implemented") + throw NotImplementedException( + "Support for cyclic var-length not yet implemented" + ) case ends: DifferentEndpoints => dir match { case OUTGOING => - registered.withConnection(rel, DirectedVarLengthRelationship(relType, ends, lower, Some(upper), OUTGOING), convertedProperties) + registered.withConnection( + rel, + DirectedVarLengthRelationship( + relType, + ends, + lower, + Some(upper), + OUTGOING + ), + convertedProperties + ) case INCOMING => - registered.withConnection(rel, DirectedVarLengthRelationship(relType, ends.flip, lower, Some(upper), INCOMING), convertedProperties) + registered.withConnection( + rel, + DirectedVarLengthRelationship( + relType, + ends.flip, + lower, + Some(upper), + INCOMING + ), + convertedProperties + ) case BOTH => - registered.withConnection(rel, UndirectedVarLengthRelationship(relType, ends.flip, lower, Some(upper)), convertedProperties) + registered.withConnection( + rel, + UndirectedVarLengthRelationship( + relType, + ends.flip, + lower, + Some(upper) + ), + convertedProperties + ) } } case None => Endpoints.apply(source, target) match { case ends: IdenticalEndpoints => - registered.withConnection(rel, CyclicRelationship(ends), convertedProperties) + registered.withConnection( + rel, + CyclicRelationship(ends), + convertedProperties + ) case ends: DifferentEndpoints => dir match { case OUTGOING => - registered.withConnection(rel, DirectedRelationship(ends, OUTGOING), convertedProperties) + registered.withConnection( + rel, + DirectedRelationship(ends, OUTGOING), + convertedProperties + ) case INCOMING => - registered.withConnection(rel, DirectedRelationship(ends.flip, INCOMING), convertedProperties) + registered.withConnection( + rel, + DirectedRelationship(ends.flip, INCOMING), + convertedProperties + ) case BOTH => - registered.withConnection(rel, UndirectedRelationship(ends), convertedProperties) + registered.withConnection( + rel, + UndirectedRelationship(ends), + convertedProperties + ) } } - case _ => throw NotImplementedException(s"Support for pattern conversion of $rc not yet implemented") + case _ => + throw NotImplementedException( + s"Support for pattern conversion of $rc not yet implemented" + ) } } } yield target case x => - throw NotImplementedException(s"Support for pattern conversion of $x not yet implemented") + throw NotImplementedException( + s"Support for pattern conversion of $x not yet implemented" + ) } private def extractProperties(propertiesOpt: Option[Expression]) = { propertiesOpt.map(irBuilderContext.convertExpression) match { case Some(e: MapExpression) => Some(e) - case Some(other) => throw IllegalArgumentException("MapExpression", other) - case _ => None + case Some(other) => throw IllegalArgumentException("MapExpression", other) + case _ => None } } @@ -201,13 +301,17 @@ final class PatternConverter(irBuilderContext: IRBuilderContext) { val patternTypes = types.map(_.name).toSet val baseRelCypherTypeOpt = baseRelOpt.map(knownTypes) - val baseRelTypes = baseRelCypherTypeOpt.map(_.toCTRelationship.types).getOrElse(Set.empty) + val baseRelTypes = + baseRelCypherTypeOpt.map(_.toCTRelationship.types).getOrElse(Set.empty) // types defined in outside scope, passed in by IRBuilder - val (knownRelTypes, qgnOption) = eOpt.flatMap(expr => knownTypes.get(expr)).flatMap { - case CTRelationship(t, qgn) => Some(t -> qgn) - case _ => None - }.getOrElse(Set.empty[String] -> Some(qualifiedGraphName)) + val (knownRelTypes, qgnOption) = eOpt + .flatMap(expr => knownTypes.get(expr)) + .flatMap { + case CTRelationship(t, qgn) => Some(t -> qgn) + case _ => None + } + .getOrElse(Set.empty[String] -> Some(qualifiedGraphName)) val relTypes = { if (patternTypes.nonEmpty) patternTypes @@ -217,7 +321,8 @@ final class PatternConverter(irBuilderContext: IRBuilderContext) { val rel = eOpt match { case Some(v) => Var(v.name)(CTRelationship(relTypes, qgnOption)) - case None => FreshVariableNamer(offset, CTRelationship(relTypes, qgnOption)) + case None => + FreshVariableNamer(offset, CTRelationship(relTypes, qgnOption)) } rel } diff --git a/okapi-ir/src/main/scala/org/opencypher/okapi/ir/impl/QueryLocalCatalog.scala b/okapi-ir/src/main/scala/org/opencypher/okapi/ir/impl/QueryLocalCatalog.scala index 7484d560f9..bcc776ff8c 100644 --- a/okapi-ir/src/main/scala/org/opencypher/okapi/ir/impl/QueryLocalCatalog.scala +++ b/okapi-ir/src/main/scala/org/opencypher/okapi/ir/impl/QueryLocalCatalog.scala @@ -31,8 +31,8 @@ import org.opencypher.okapi.api.io.PropertyGraphDataSource import org.opencypher.okapi.api.schema.PropertyGraphSchema /** - * Represents a catalog storing the session graphs and schemas of constructed graphs and temporary graphs for - * recursive view queries. + * Represents a catalog storing the session graphs and schemas of constructed graphs and temporary + * graphs for recursive view queries. */ case class QueryLocalCatalog( dataSourceMapping: Map[Namespace, PropertyGraphDataSource], @@ -46,7 +46,7 @@ case class QueryLocalCatalog( case None => registeredGraphs.get(qgn) match { case Some(g) => g.schema - case None => schemaFromDataSource(qgn) + case None => schemaFromDataSource(qgn) } } } @@ -54,25 +54,33 @@ case class QueryLocalCatalog( def graph(qgn: QualifiedGraphName): PropertyGraph = { registeredGraphs.get(qgn) match { case Some(g) => g - case None => dataSourceMapping(qgn.namespace).graph(qgn.graphName) + case None => dataSourceMapping(qgn.namespace).graph(qgn.graphName) } } - private def schemaFromDataSource(qgn: QualifiedGraphName): PropertyGraphSchema = { + private def schemaFromDataSource( + qgn: QualifiedGraphName + ): PropertyGraphSchema = { val dataSource = dataSourceMapping(qgn.namespace) val graphName = qgn.graphName val schema: PropertyGraphSchema = dataSource.schema(graphName) match { case Some(s) => s - case None => dataSource.graph(graphName).schema + case None => dataSource.graph(graphName).schema } schema } - def withGraph(qgn: QualifiedGraphName, graph: PropertyGraph): QueryLocalCatalog = { + def withGraph( + qgn: QualifiedGraphName, + graph: PropertyGraph + ): QueryLocalCatalog = { copy(registeredGraphs = registeredGraphs.updated(qgn, graph)) } - def withSchema(qgn: QualifiedGraphName, schema: PropertyGraphSchema): QueryLocalCatalog = { + def withSchema( + qgn: QualifiedGraphName, + schema: PropertyGraphSchema + ): QueryLocalCatalog = { copy(registeredSchemas = registeredSchemas.updated(qgn, schema)) } } @@ -84,5 +92,6 @@ object QueryLocalCatalog { ): QueryLocalCatalog = QueryLocalCatalog(dataSourceMapping, queryCatalog, Map.empty) - def empty: QueryLocalCatalog = QueryLocalCatalog(Map.empty, Map.empty, Map.empty) + def empty: QueryLocalCatalog = + QueryLocalCatalog(Map.empty, Map.empty, Map.empty) } diff --git a/okapi-ir/src/main/scala/org/opencypher/okapi/ir/impl/exception/IrException.scala b/okapi-ir/src/main/scala/org/opencypher/okapi/ir/impl/exception/IrException.scala index f67e70a9b1..1ac8883447 100644 --- a/okapi-ir/src/main/scala/org/opencypher/okapi/ir/impl/exception/IrException.scala +++ b/okapi-ir/src/main/scala/org/opencypher/okapi/ir/impl/exception/IrException.scala @@ -28,10 +28,12 @@ package org.opencypher.okapi.ir.impl.exception import org.opencypher.okapi.impl.exception.InternalException -abstract class IrException(msg: String, cause: Option[Throwable] = None) extends InternalException(msg, cause) +abstract class IrException(msg: String, cause: Option[Throwable] = None) + extends InternalException(msg, cause) final case class PatternConversionException(msg: String) extends IrException(msg) -final case class TypingException(msg: String, cause: Option[Throwable] = None) extends IrException(msg, cause) +final case class TypingException(msg: String, cause: Option[Throwable] = None) + extends IrException(msg, cause) final case class ParsingException(msg: String) extends IrException(msg) diff --git a/okapi-ir/src/main/scala/org/opencypher/okapi/ir/impl/package.scala b/okapi-ir/src/main/scala/org/opencypher/okapi/ir/impl/package.scala index 2c3ad8220a..74e1090163 100644 --- a/okapi-ir/src/main/scala/org/opencypher/okapi/ir/impl/package.scala +++ b/okapi-ir/src/main/scala/org/opencypher/okapi/ir/impl/package.scala @@ -44,51 +44,83 @@ package object impl { type IRBuilderStack[A] = Fx.fx2[MayFail, HasContext] - implicit final class RichIRBuilderStack[A](val program: Eff[IRBuilderStack[A], A]) { + implicit final class RichIRBuilderStack[A]( + val program: Eff[IRBuilderStack[A], A] + ) { - def run(context: IRBuilderContext): Either[IRBuilderError, (A, IRBuilderContext)] = { + def run( + context: IRBuilderContext + ): Either[IRBuilderError, (A, IRBuilderContext)] = { val stateRun = program.runState(context) val errorRun = stateRun.runEither[IRBuilderError] errorRun.run } } - def error[R: _mayFail : _hasContext, A](err: IRBuilderError)(v: A): Eff[R, A] = + def error[R: _mayFail: _hasContext, A](err: IRBuilderError)(v: A): Eff[R, A] = left[R, IRBuilderError, BlockRegistry](err) >> pure(v) implicit final class RichSchema(schema: PropertyGraphSchema) { - def forElementType(cypherType: CypherType): PropertyGraphSchema = cypherType match { - case CTNode(labels, _) => - schema.forNode(labels) - case r: CTRelationship => - schema.forRelationship(r) - case x => throw IllegalArgumentException("element type", x) - } + def forElementType(cypherType: CypherType): PropertyGraphSchema = + cypherType match { + case CTNode(labels, _) => + schema.forNode(labels) + case r: CTRelationship => + schema.forRelationship(r) + case x => throw IllegalArgumentException("element type", x) + } - def addLabelsToCombo(labels: Set[String], combo: Set[String]): PropertyGraphSchema = { + def addLabelsToCombo( + labels: Set[String], + combo: Set[String] + ): PropertyGraphSchema = { val labelsWithAddition = combo ++ labels schema .dropPropertiesFor(combo) - .withNodePropertyKeys(labelsWithAddition, schema.nodePropertyKeys(combo)) + .withNodePropertyKeys( + labelsWithAddition, + schema.nodePropertyKeys(combo) + ) } - def addPropertyToElement(propertyKey: String, propertyType: CypherType, elementType: CypherType): PropertyGraphSchema = { + def addPropertyToElement( + propertyKey: String, + propertyType: CypherType, + elementType: CypherType + ): PropertyGraphSchema = { elementType match { case CTNode(labels, _) => val allRelevantLabelCombinations = schema.combinationsFor(labels) - val property = if (allRelevantLabelCombinations.size == 1) propertyType else propertyType.nullable + val property = + if (allRelevantLabelCombinations.size == 1) propertyType + else propertyType.nullable allRelevantLabelCombinations.foldLeft(schema) { case (innerCurrentSchema, combo) => - val updatedPropertyKeys = innerCurrentSchema.nodePropertyKeysForCombinations(Set(combo)).updated(propertyKey, property) - innerCurrentSchema.withOverwrittenNodePropertyKeys(combo, updatedPropertyKeys) + val updatedPropertyKeys = innerCurrentSchema + .nodePropertyKeysForCombinations(Set(combo)) + .updated(propertyKey, property) + innerCurrentSchema.withOverwrittenNodePropertyKeys( + combo, + updatedPropertyKeys + ) } case CTRelationship(types, _) => - val typesToUpdate = if (types.isEmpty) schema.relationshipTypes else types + val typesToUpdate = + if (types.isEmpty) schema.relationshipTypes else types typesToUpdate.foldLeft(schema) { case (innerCurrentSchema, relType) => - val updatedPropertyKeys = innerCurrentSchema.relationshipPropertyKeys(relType).updated(propertyKey, propertyType) - innerCurrentSchema.withOverwrittenRelationshipPropertyKeys(relType, updatedPropertyKeys) + val updatedPropertyKeys = innerCurrentSchema + .relationshipPropertyKeys(relType) + .updated(propertyKey, propertyType) + innerCurrentSchema.withOverwrittenRelationshipPropertyKeys( + relType, + updatedPropertyKeys + ) } - case other => throw IllegalArgumentException("node or relationship to set a property on", other) + case other => + throw IllegalArgumentException( + "node or relationship to set a property on", + other + ) } } diff --git a/okapi-ir/src/main/scala/org/opencypher/okapi/ir/impl/parse/BlankBaseContext.scala b/okapi-ir/src/main/scala/org/opencypher/okapi/ir/impl/parse/BlankBaseContext.scala index db09097c5c..a0317ae392 100644 --- a/okapi-ir/src/main/scala/org/opencypher/okapi/ir/impl/parse/BlankBaseContext.scala +++ b/okapi-ir/src/main/scala/org/opencypher/okapi/ir/impl/parse/BlankBaseContext.scala @@ -33,9 +33,11 @@ import org.opencypher.v9_0.util.{CypherException, InputPosition} import scala.reflect.ClassTag abstract class BlankBaseContext extends BaseContext { - override def tracer: CompilationPhaseTracer = CompilationPhaseTracer.NO_TRACING + override def tracer: CompilationPhaseTracer = + CompilationPhaseTracer.NO_TRACING override def notificationLogger: InternalNotificationLogger = devNullLogger - override def exceptionCreator: (String, InputPosition) => CypherException = (_, _) => null + override def exceptionCreator: (String, InputPosition) => CypherException = + (_, _) => null override def monitors: Monitors = new Monitors { override def newMonitor[T <: AnyRef: ClassTag](tags: String*): T = { diff --git a/okapi-ir/src/main/scala/org/opencypher/okapi/ir/impl/parse/CypherParser.scala b/okapi-ir/src/main/scala/org/opencypher/okapi/ir/impl/parse/CypherParser.scala index 6cd0b3be33..5205459ebb 100644 --- a/okapi-ir/src/main/scala/org/opencypher/okapi/ir/impl/parse/CypherParser.scala +++ b/okapi-ir/src/main/scala/org/opencypher/okapi/ir/impl/parse/CypherParser.scala @@ -43,7 +43,9 @@ object CypherParser extends CypherParser { // TODO: Remove when frontend supports CLONE clause val filteredErrors = errors.filterNot(_.msg.contains("already declared")) if (filteredErrors.nonEmpty) { - throw ParsingException(s"Errors during semantic checking: ${filteredErrors.mkString(", ")}") + throw ParsingException( + s"Errors during semantic checking: ${filteredErrors.mkString(", ")}" + ) } } } @@ -51,11 +53,15 @@ object CypherParser extends CypherParser { trait CypherParser { - def apply(query: String)(implicit context: BaseContext): Statement = process(query)._1 + def apply(query: String)(implicit context: BaseContext): Statement = process( + query + )._1 - def process(query: String, drivingTableFields: Set[Var] = Set.empty) - (implicit context: BaseContext): (Statement, Map[String, Any], SemanticState) = { - val fieldsWithFrontendTypes = drivingTableFields.map(v => v.name -> toFrontendType(v.cypherType)).toMap + def process(query: String, drivingTableFields: Set[Var] = Set.empty)(implicit + context: BaseContext + ): (Statement, Map[String, Any], SemanticState) = { + val fieldsWithFrontendTypes = + drivingTableFields.map(v => v.name -> toFrontendType(v.cypherType)).toMap val startState = InitialState(query, None, null, fieldsWithFrontendTypes) val endState = pipeLine.transform(startState, context) val params = endState.extractedParams @@ -67,11 +73,25 @@ trait CypherParser { Parsing.adds(BaseContains[Statement]) andThen SyntaxDeprecationWarnings(V2) andThen OkapiPreparatoryRewriting andThen - SemanticAnalysis(warn = true, SemanticFeature.Cypher10Support, SemanticFeature.MultipleGraphs, SemanticFeature.WithInitialQuerySignature) + SemanticAnalysis( + warn = true, + SemanticFeature.Cypher10Support, + SemanticFeature.MultipleGraphs, + SemanticFeature.WithInitialQuerySignature + ) .adds(BaseContains[SemanticState]) andThen - AstRewriting(RewriterStepSequencer.newPlain, Forced, getDegreeRewriting = false) andThen + AstRewriting( + RewriterStepSequencer.newPlain, + Forced, + getDegreeRewriting = false + ) andThen isolateAggregation andThen - SemanticAnalysis(warn = false, SemanticFeature.Cypher10Support, SemanticFeature.MultipleGraphs, SemanticFeature.WithInitialQuerySignature) andThen + SemanticAnalysis( + warn = false, + SemanticFeature.Cypher10Support, + SemanticFeature.MultipleGraphs, + SemanticFeature.WithInitialQuerySignature + ) andThen Namespacer andThen CNFNormalizer andThen LateAstRewriting andThen diff --git a/okapi-ir/src/main/scala/org/opencypher/okapi/ir/impl/parse/functions/FunctionExtensions.scala b/okapi-ir/src/main/scala/org/opencypher/okapi/ir/impl/parse/functions/FunctionExtensions.scala index 3684bf5843..e03bfcecf2 100644 --- a/okapi-ir/src/main/scala/org/opencypher/okapi/ir/impl/parse/functions/FunctionExtensions.scala +++ b/okapi-ir/src/main/scala/org/opencypher/okapi/ir/impl/parse/functions/FunctionExtensions.scala @@ -35,7 +35,8 @@ case object FunctionExtensions { Timestamp.name -> Timestamp, LocalDateTime.name -> LocalDateTime, Date.name -> Date, - Duration.name -> Duration) + Duration.name -> Duration + ) .map(p => p._1.toLowerCase -> p._2) def get(name: String): Option[Function] = @@ -65,4 +66,3 @@ object CTIdentity extends CypherType { override def parentType: CypherType = CTAny override def toNeoTypeString: String = "IDENTITY" } - diff --git a/okapi-ir/src/main/scala/org/opencypher/okapi/ir/impl/parse/rewriter/OkapiLateRewriting.scala b/okapi-ir/src/main/scala/org/opencypher/okapi/ir/impl/parse/rewriter/OkapiLateRewriting.scala index 2da8e9443a..2b086a1beb 100644 --- a/okapi-ir/src/main/scala/org/opencypher/okapi/ir/impl/parse/rewriter/OkapiLateRewriting.scala +++ b/okapi-ir/src/main/scala/org/opencypher/okapi/ir/impl/parse/rewriter/OkapiLateRewriting.scala @@ -45,24 +45,24 @@ case object OkapiLateRewriting extends Phase[BaseContext, BaseState, BaseState] pushLabelsIntoScans, extractSubqueryFromPatternExpression(context.exceptionCreator), CNFNormalizer.instance(context) - )) + ) + ) // Extract literals of possibly rewritten subqueries // TODO: once this gets into neo4j-frontend, it can be done in literalReplacement val (rewriters, extractedParams) = rewrittenStatement - .treeFold(Seq.empty[(Rewriter, Map[String, Any])]) { - case ep: ExistsPattern => - acc => - (acc :+ literalReplacement(ep.query, Forced), None) + .treeFold(Seq.empty[(Rewriter, Map[String, Any])]) { case ep: ExistsPattern => + acc => (acc :+ literalReplacement(ep.query, Forced), None) } .unzip // rewrite literals - val finalStatement = rewriters.foldLeft(rewrittenStatement) { - case (acc, rewriter) => acc.endoRewrite(rewriter) + val finalStatement = rewriters.foldLeft(rewrittenStatement) { case (acc, rewriter) => + acc.endoRewrite(rewriter) } // merge extracted params - val extractedParameters = extractedParams.foldLeft(Map.empty[String, Any])(_ ++ _) + val extractedParameters = + extractedParams.foldLeft(Map.empty[String, Any])(_ ++ _) from .withStatement(finalStatement) @@ -71,7 +71,8 @@ case object OkapiLateRewriting extends Phase[BaseContext, BaseState, BaseState] override val phase = AST_REWRITE - override val description = "rewrite the AST into a shape that semantic analysis can be performed on" + override val description = + "rewrite the AST into a shape that semantic analysis can be performed on" override def postConditions: Set[Condition] = Set.empty } diff --git a/okapi-ir/src/main/scala/org/opencypher/okapi/ir/impl/parse/rewriter/OkapiPreparatoryRewriting.scala b/okapi-ir/src/main/scala/org/opencypher/okapi/ir/impl/parse/rewriter/OkapiPreparatoryRewriting.scala index 741ed705d1..e17a9643df 100644 --- a/okapi-ir/src/main/scala/org/opencypher/okapi/ir/impl/parse/rewriter/OkapiPreparatoryRewriting.scala +++ b/okapi-ir/src/main/scala/org/opencypher/okapi/ir/impl/parse/rewriter/OkapiPreparatoryRewriting.scala @@ -27,7 +27,10 @@ package org.opencypher.v9_0.frontend.phases import org.opencypher.okapi.ir.impl.parse.rewriter.legacy -import org.opencypher.okapi.ir.impl.parse.rewriter.legacy.{normalizeReturnClauses, normalizeWithClauses} +import org.opencypher.okapi.ir.impl.parse.rewriter.legacy.{ + normalizeReturnClauses, + normalizeWithClauses +} import org.opencypher.v9_0.rewriting.rewriters._ import org.opencypher.v9_0.util.inSequence import org.opencypher.v9_0.frontend.phases.CompilationPhaseTracer.CompilationPhase.AST_REWRITE @@ -36,18 +39,24 @@ case object OkapiPreparatoryRewriting extends Phase[BaseContext, BaseState, Base override def process(from: BaseState, context: BaseContext): BaseState = { - val rewrittenStatement = from.statement().endoRewrite(inSequence( - legacy.normalizeReturnClauses(context.exceptionCreator), - normalizeWithClauses(context.exceptionCreator), - expandCallWhere, - mergeInPredicates)) + val rewrittenStatement = from + .statement() + .endoRewrite( + inSequence( + legacy.normalizeReturnClauses(context.exceptionCreator), + normalizeWithClauses(context.exceptionCreator), + expandCallWhere, + mergeInPredicates + ) + ) from.withStatement(rewrittenStatement) } override val phase = AST_REWRITE - override val description = "rewrite the AST into a shape that semantic analysis can be performed on" + override val description = + "rewrite the AST into a shape that semantic analysis can be performed on" override def postConditions: Set[Condition] = Set.empty } diff --git a/okapi-ir/src/main/scala/org/opencypher/okapi/ir/impl/parse/rewriter/extractSubqueryFromPatternExpression.scala b/okapi-ir/src/main/scala/org/opencypher/okapi/ir/impl/parse/rewriter/extractSubqueryFromPatternExpression.scala index 6ed6fba114..4d9afa6218 100644 --- a/okapi-ir/src/main/scala/org/opencypher/okapi/ir/impl/parse/rewriter/extractSubqueryFromPatternExpression.scala +++ b/okapi-ir/src/main/scala/org/opencypher/okapi/ir/impl/parse/rewriter/extractSubqueryFromPatternExpression.scala @@ -27,14 +27,19 @@ package org.opencypher.okapi.ir.impl.parse.rewriter import org.opencypher.v9_0.ast._ -import org.opencypher.v9_0.ast.semantics.{SemanticCheck, SemanticCheckResult, SemanticCheckableExpression} +import org.opencypher.v9_0.ast.semantics.{ + SemanticCheck, + SemanticCheckResult, + SemanticCheckableExpression +} import org.opencypher.v9_0.expressions._ import org.opencypher.v9_0.expressions.functions.Exists import org.opencypher.v9_0.rewriting.rewriters.{nameMatchPatternElements, normalizeMatchPredicates} import org.opencypher.v9_0.util._ -case class extractSubqueryFromPatternExpression(mkException: (String, InputPosition) => CypherException) - extends Rewriter { +case class extractSubqueryFromPatternExpression( + mkException: (String, InputPosition) => CypherException +) extends Rewriter { def apply(that: AnyRef): AnyRef = instance.apply(that) @@ -47,49 +52,59 @@ case class extractSubqueryFromPatternExpression(mkException: (String, InputPosit * * to * - * WHERE EXISTS { - * MATCH (a)-[e0]->(v0)-[e1]->(v1)... - * WHERE e0:R AND v0.foo = true - * RETURN a, true + * WHERE EXISTS { MATCH (a)-[e0]->(v0)-[e1]->(v1)... WHERE e0:R AND v0.foo = true RETURN a, true * } AND a.age > 20 */ private val instance = topDown(Rewriter.lift { - case f @ FunctionInvocation(_, _, _, IndexedSeq(p : PatternExpression)) if f.function == Exists => + case f @ FunctionInvocation(_, _, _, IndexedSeq(p: PatternExpression)) + if f.function == Exists => rewritePatternExpression(p) - case p : PatternExpression => + case p: PatternExpression => rewritePatternExpression(p) }) private def rewritePatternExpression(p: PatternExpression): ExistsPattern = { val relationshipsPattern = p.pattern val patternPosition: InputPosition = p.position - val newPattern = Pattern(Seq(EveryPath(relationshipsPattern.element)))(patternPosition) + val newPattern = + Pattern(Seq(EveryPath(relationshipsPattern.element)))(patternPosition) - val joinVariables = relationshipsPattern.element.treeFold(Seq.empty[LogicalVariable]) { - case NodePattern(Some(v), _, _, _) => - acc => - (acc :+ v, None) - case RelationshipPattern(Some(v), _, _, _, _, _, _) => - acc => - (acc :+ v, None) - } + val joinVariables = + relationshipsPattern.element.treeFold(Seq.empty[LogicalVariable]) { + case NodePattern(Some(v), _, _, _) => + acc => (acc :+ v, None) + case RelationshipPattern(Some(v), _, _, _, _, _, _) => + acc => (acc :+ v, None) + } - val returnItems = joinVariables.map(v => AliasedReturnItem(v, v)(v.position)) + val returnItems = + joinVariables.map(v => AliasedReturnItem(v, v)(v.position)) - val trueVariable = Variable(UnNamedNameGenerator.name(p.position))(p.position) - val returnItemsWithTrue = returnItems :+ AliasedReturnItem(True()(trueVariable.position), trueVariable)( - trueVariable.position) + val trueVariable = + Variable(UnNamedNameGenerator.name(p.position))(p.position) + val returnItemsWithTrue = returnItems :+ AliasedReturnItem( + True()(trueVariable.position), + trueVariable + )(trueVariable.position) ExistsPattern( Query( None, SingleQuery( Seq( - Match(optional = false, newPattern, Seq.empty, None)(patternPosition) + Match(optional = false, newPattern, Seq.empty, None)( + patternPosition + ) .endoRewrite(nameMatchPatternElements) - .endoRewrite(normalizeMatchPredicates(getDegreeRewriting = false)), - Return(ReturnItems(includeExisting = false, returnItemsWithTrue)(patternPosition))(patternPosition) + .endoRewrite( + normalizeMatchPredicates(getDegreeRewriting = false) + ), + Return( + ReturnItems(includeExisting = false, returnItemsWithTrue)( + patternPosition + ) + )(patternPosition) ) )(patternPosition) )(patternPosition), @@ -98,8 +113,10 @@ case class extractSubqueryFromPatternExpression(mkException: (String, InputPosit } } -case class ExistsPattern(query: Query, targetField: Variable)(val position: InputPosition) - extends Expression +case class ExistsPattern(query: Query, targetField: Variable)( + val position: InputPosition +) extends Expression with SemanticCheckableExpression { - override def semanticCheck(ctx: Expression.SemanticContext): SemanticCheck = SemanticCheckResult.success + override def semanticCheck(ctx: Expression.SemanticContext): SemanticCheck = + SemanticCheckResult.success } diff --git a/okapi-ir/src/main/scala/org/opencypher/okapi/ir/impl/parse/rewriter/legacy/normalizeReturnClauses.scala b/okapi-ir/src/main/scala/org/opencypher/okapi/ir/impl/parse/rewriter/legacy/normalizeReturnClauses.scala index ceb895865a..0961015473 100644 --- a/okapi-ir/src/main/scala/org/opencypher/okapi/ir/impl/parse/rewriter/legacy/normalizeReturnClauses.scala +++ b/okapi-ir/src/main/scala/org/opencypher/okapi/ir/impl/parse/rewriter/legacy/normalizeReturnClauses.scala @@ -33,66 +33,87 @@ import org.opencypher.v9_0.util._ import scala.collection.mutable /** - * This rewriter makes sure that all return items in a RETURN clauses are aliased, and moves - * any ORDER BY to a preceding WITH clause + * This rewriter makes sure that all return items in a RETURN clauses are aliased, and moves any + * ORDER BY to a preceding WITH clause * * Example: * - * MATCH (n) - * RETURN n.foo AS foo, n.bar ORDER BY foo + * MATCH (n) RETURN n.foo AS foo, n.bar ORDER BY foo * * This rewrite will change the query to: * - * MATCH (n) - * WITH n.foo AS ` FRESHIDxx`, n.bar AS ` FRESHIDnn` ORDER BY ` FRESHIDxx` - * RETURN ` FRESHIDxx` AS foo, ` FRESHIDnn` AS `n.bar` + * MATCH (n) WITH n.foo AS ` FRESHIDxx`, n.bar AS ` FRESHIDnn` ORDER BY ` FRESHIDxx` RETURN ` + * FRESHIDxx` AS foo, ` FRESHIDnn` AS `n.bar` */ -case class normalizeReturnClauses(mkException: (String, InputPosition) => CypherException) extends Rewriter { +case class normalizeReturnClauses( + mkException: (String, InputPosition) => CypherException +) extends Rewriter { def apply(that: AnyRef): AnyRef = instance.apply(that) private val clauseRewriter: (Clause => Seq[Clause]) = { - case clause@Return(_, ri@ReturnItems(_, items), None, _, _, _) => + case clause @ Return(_, ri @ ReturnItems(_, items), None, _, _, _) => val aliasedItems = items.map({ case i: AliasedReturnItem => i case i => val newPosition = i.expression.position.bumped() - AliasedReturnItem(i.expression, Variable(i.name)(newPosition))(i.position) + AliasedReturnItem(i.expression, Variable(i.name)(newPosition))( + i.position + ) }) Seq( - clause.copy(returnItems = ri.copy(items = aliasedItems)(ri.position))(clause.position) + clause.copy(returnItems = ri.copy(items = aliasedItems)(ri.position))( + clause.position + ) ) - case clause@Return(distinct, ri: ReturnItems, orderBy, skip, limit, _) => + case clause @ Return(distinct, ri: ReturnItems, orderBy, skip, limit, _) => clause.verifyOrderByAggregationUse((s, i) => throw mkException(s, i)) var rewrites = mutable.Map[Expression, Variable]() - val (aliasProjection, finalProjection) = ri.items.map { - i => - val returnColumn = i.alias match { - case Some(alias) => alias - case None => Variable(i.name)(i.expression.position.bumped()) - } - val newVariable = Variable(FreshIdNameGenerator.name(i.expression.position))(i.expression.position) - // Always update for the return column, so that it has precedence over the expressions (if there are variables with the same name), - // e.g. match (n),(m) return n as m, m as m2 - rewrites += (returnColumn -> newVariable) - // Only update if rewrites does not yet have a mapping for i.expression - rewrites.getOrElseUpdate(i.expression, newVariable) - (AliasedReturnItem(i.expression, newVariable)(i.position), AliasedReturnItem(newVariable.copyId, returnColumn)(i.position)) + val (aliasProjection, finalProjection) = ri.items.map { i => + val returnColumn = i.alias match { + case Some(alias) => alias + case None => Variable(i.name)(i.expression.position.bumped()) + } + val newVariable = Variable( + FreshIdNameGenerator.name(i.expression.position) + )(i.expression.position) + // Always update for the return column, so that it has precedence over the expressions (if there are variables with the same name), + // e.g. match (n),(m) return n as m, m as m2 + rewrites += (returnColumn -> newVariable) + // Only update if rewrites does not yet have a mapping for i.expression + rewrites.getOrElseUpdate(i.expression, newVariable) + ( + AliasedReturnItem(i.expression, newVariable)(i.position), + AliasedReturnItem(newVariable.copyId, returnColumn)(i.position) + ) }.unzip val newOrderBy = orderBy.endoRewrite(topDown(Rewriter.lift { case exp: Expression if rewrites.contains(exp) => rewrites(exp).copyId })) - val introducedVariables = if (ri.includeExisting) aliasProjection.map(_.variable.name).toSet else Set.empty[String] + val introducedVariables = + if (ri.includeExisting) aliasProjection.map(_.variable.name).toSet + else Set.empty[String] Seq( - With(distinct = distinct, returnItems = ri.copy(items = aliasProjection)(ri.position), - orderBy = newOrderBy, skip = skip, limit = limit, where = None)(clause.position), - Return(distinct = false, returnItems = ri.copy(items = finalProjection)(ri.position), - orderBy = None, skip = None, limit = None, excludedNames = introducedVariables)(clause.position) + With( + distinct = distinct, + returnItems = ri.copy(items = aliasProjection)(ri.position), + orderBy = newOrderBy, + skip = skip, + limit = limit, + where = None + )(clause.position), + Return( + distinct = false, + returnItems = ri.copy(items = finalProjection)(ri.position), + orderBy = None, + skip = None, + limit = None, + excludedNames = introducedVariables + )(clause.position) ) case clause => Seq(clause) } - private val instance: Rewriter = bottomUp(Rewriter.lift { - case query@SingleQuery(clauses) => - query.copy(clauses = clauses.flatMap(clauseRewriter))(query.position) + private val instance: Rewriter = bottomUp(Rewriter.lift { case query @ SingleQuery(clauses) => + query.copy(clauses = clauses.flatMap(clauseRewriter))(query.position) }) } diff --git a/okapi-ir/src/main/scala/org/opencypher/okapi/ir/impl/parse/rewriter/legacy/normalizeWithClauses.scala b/okapi-ir/src/main/scala/org/opencypher/okapi/ir/impl/parse/rewriter/legacy/normalizeWithClauses.scala index fdd2c1b0bf..e63dd591bf 100644 --- a/okapi-ir/src/main/scala/org/opencypher/okapi/ir/impl/parse/rewriter/legacy/normalizeWithClauses.scala +++ b/okapi-ir/src/main/scala/org/opencypher/okapi/ir/impl/parse/rewriter/legacy/normalizeWithClauses.scala @@ -31,49 +31,63 @@ import org.opencypher.v9_0.expressions.{Expression, LogicalVariable} import org.opencypher.v9_0.util._ /** - * This rewriter normalizes the scoping structure of a query, ensuring it is able to - * be correctly processed for semantic checking. It makes sure that all return items - * in a WITH clauses are aliased. + * This rewriter normalizes the scoping structure of a query, ensuring it is able to be correctly + * processed for semantic checking. It makes sure that all return items in a WITH clauses are + * aliased. * - * It also replaces expressions and subexpressions in ORDER BY and WHERE - * to use aliases introduced by the WITH, where possible. + * It also replaces expressions and subexpressions in ORDER BY and WHERE to use aliases introduced + * by the WITH, where possible. * * This rewriter depends on normalizeReturnClauses having first been run. * * Example: * - * MATCH n - * WITH n.prop AS prop ORDER BY n.prop DESC - * RETURN prop + * MATCH n WITH n.prop AS prop ORDER BY n.prop DESC RETURN prop * * This rewrite will change the query to: * - * MATCH n - * WITH n.prop AS prop ORDER BY prop DESC - * RETURN prop + * MATCH n WITH n.prop AS prop ORDER BY prop DESC RETURN prop */ -case class normalizeWithClauses(mkException: (String, InputPosition) => CypherException) extends Rewriter { +case class normalizeWithClauses( + mkException: (String, InputPosition) => CypherException +) extends Rewriter { def apply(that: AnyRef): AnyRef = instance.apply(that) private val clauseRewriter: (Clause => Clause) = { // Only alias return items - case clause@With(_, ri: ReturnItems, None, _, _, None) => - val (unaliasedReturnItems, aliasedReturnItems) = partitionReturnItems(ri.items) + case clause @ With(_, ri: ReturnItems, None, _, _, None) => + val (unaliasedReturnItems, aliasedReturnItems) = partitionReturnItems( + ri.items + ) val initialReturnItems = unaliasedReturnItems ++ aliasedReturnItems clause.copy(returnItems = ri.copy(items = initialReturnItems)(ri.position))(clause.position) // Alias return items and rewrite ORDER BY and WHERE - case clause@With(distinct, ri: ReturnItems, orderBy, skip, limit, where) => + case clause @ With( + distinct, + ri: ReturnItems, + orderBy, + skip, + limit, + where + ) => clause.verifyOrderByAggregationUse((s, i) => throw mkException(s, i)) - val (unaliasedReturnItems, aliasedReturnItems) = partitionReturnItems(ri.items) + val (unaliasedReturnItems, aliasedReturnItems) = partitionReturnItems( + ri.items + ) val initialReturnItems = unaliasedReturnItems ++ aliasedReturnItems - val existingAliases = aliasedReturnItems.map(i => i.expression -> i.alias.get.copyId).toMap + val existingAliases = + aliasedReturnItems.map(i => i.expression -> i.alias.get.copyId).toMap val updatedOrderBy = orderBy.map(aliasOrderBy(existingAliases, _)) val updatedWhere = where.map(aliasWhere(existingAliases, _)) - clause.copy(returnItems = ri.copy(items = initialReturnItems)(ri.position), orderBy = updatedOrderBy, where = updatedWhere)(clause.position) + clause.copy( + returnItems = ri.copy(items = initialReturnItems)(ri.position), + orderBy = updatedOrderBy, + where = updatedWhere + )(clause.position) // Not our business case clause => @@ -81,18 +95,27 @@ case class normalizeWithClauses(mkException: (String, InputPosition) => CypherEx } /** - * Aliases return items if possible. Return a tuple of unaliased (because impossible) and - * aliased (because they already were aliases or we just introduced an alias for them) - * return items. + * Aliases return items if possible. Return a tuple of unaliased (because impossible) and aliased + * (because they already were aliases or we just introduced an alias for them) return items. */ - private def partitionReturnItems(returnItems: Seq[ReturnItem]): (Seq[ReturnItem], Seq[AliasedReturnItem]) = - returnItems.foldLeft((Vector.empty[ReturnItem], Vector.empty[AliasedReturnItem])) { - case ((unaliasedItems, aliasedItems), item) => item match { + private def partitionReturnItems( + returnItems: Seq[ReturnItem] + ): (Seq[ReturnItem], Seq[AliasedReturnItem]) = + returnItems.foldLeft( + (Vector.empty[ReturnItem], Vector.empty[AliasedReturnItem]) + ) { case ((unaliasedItems, aliasedItems), item) => + item match { case i: AliasedReturnItem => (unaliasedItems, aliasedItems :+ i) case i if i.alias.isDefined => - (unaliasedItems, aliasedItems :+ AliasedReturnItem(item.expression, item.alias.get.copyId)(item.position)) + ( + unaliasedItems, + aliasedItems :+ AliasedReturnItem( + item.expression, + item.alias.get.copyId + )(item.position) + ) case _ => // Unaliased return items in WITH will be preserved so that semantic check can report them as an error @@ -100,50 +123,66 @@ case class normalizeWithClauses(mkException: (String, InputPosition) => CypherEx } } - /** - * Given a list of existing aliases, this rewrites an OrderBy to use these where possible. - */ - private def aliasOrderBy(existingAliases: Map[Expression, LogicalVariable], originalOrderBy: OrderBy): OrderBy = { - val updatedSortItems = originalOrderBy.sortItems.map { aliasSortItem(existingAliases, _)} + /** Given a list of existing aliases, this rewrites an OrderBy to use these where possible. */ + private def aliasOrderBy( + existingAliases: Map[Expression, LogicalVariable], + originalOrderBy: OrderBy + ): OrderBy = { + val updatedSortItems = originalOrderBy.sortItems.map { + aliasSortItem(existingAliases, _) + } OrderBy(updatedSortItems)(originalOrderBy.position) } - /** - * Given a list of existing aliases, this rewrites a SortItem to use these where possible. - */ - private def aliasSortItem(existingAliases: Map[Expression, LogicalVariable], sortItem: SortItem): SortItem = { + /** Given a list of existing aliases, this rewrites a SortItem to use these where possible. */ + private def aliasSortItem( + existingAliases: Map[Expression, LogicalVariable], + sortItem: SortItem + ): SortItem = { sortItem match { - case AscSortItem(expression) => AscSortItem(aliasExpression(existingAliases, expression))(sortItem.position) - case DescSortItem(expression) => DescSortItem(aliasExpression(existingAliases, expression))(sortItem.position) + case AscSortItem(expression) => + AscSortItem(aliasExpression(existingAliases, expression))( + sortItem.position + ) + case DescSortItem(expression) => + DescSortItem(aliasExpression(existingAliases, expression))( + sortItem.position + ) } } - /** - * Given a list of existing aliases, this rewrites a where to use these where possible. - */ - private def aliasWhere(existingAliases: Map[Expression, LogicalVariable], originalWhere: Where): Where = { - Where(aliasExpression(existingAliases, originalWhere.expression))(originalWhere.position) + /** Given a list of existing aliases, this rewrites a where to use these where possible. */ + private def aliasWhere( + existingAliases: Map[Expression, LogicalVariable], + originalWhere: Where + ): Where = { + Where(aliasExpression(existingAliases, originalWhere.expression))( + originalWhere.position + ) } - /** - * Given a list of existing aliases, this rewrites expressions to use these where possible. - */ - private def aliasExpression(existingAliases: Map[Expression, LogicalVariable], expression: Expression): Expression = { + /** Given a list of existing aliases, this rewrites expressions to use these where possible. */ + private def aliasExpression( + existingAliases: Map[Expression, LogicalVariable], + expression: Expression + ): Expression = { existingAliases.get(expression) match { case Some(alias) => alias.copyId case None => - val newExpression = expression.endoRewrite(topDown(Rewriter.lift { - case subExpression: Expression => - existingAliases.get(subExpression).map(_.copyId).getOrElse(subExpression) - })) + val newExpression = + expression.endoRewrite(topDown(Rewriter.lift { case subExpression: Expression => + existingAliases + .get(subExpression) + .map(_.copyId) + .getOrElse(subExpression) + })) newExpression } } - private val instance: Rewriter = bottomUp(Rewriter.lift { - case query@SingleQuery(clauses) => - query.copy(clauses = clauses.map(clauseRewriter))(query.position) + private val instance: Rewriter = bottomUp(Rewriter.lift { case query @ SingleQuery(clauses) => + query.copy(clauses = clauses.map(clauseRewriter))(query.position) }) } diff --git a/okapi-ir/src/main/scala/org/opencypher/okapi/ir/impl/parse/rewriter/legacy/projectFreshSortExpressions.scala b/okapi-ir/src/main/scala/org/opencypher/okapi/ir/impl/parse/rewriter/legacy/projectFreshSortExpressions.scala index 4db9d46caa..c7a3a5d950 100644 --- a/okapi-ir/src/main/scala/org/opencypher/okapi/ir/impl/parse/rewriter/legacy/projectFreshSortExpressions.scala +++ b/okapi-ir/src/main/scala/org/opencypher/okapi/ir/impl/parse/rewriter/legacy/projectFreshSortExpressions.scala @@ -33,34 +33,45 @@ import org.opencypher.v9_0.util.{Rewriter, bottomUp} case object projectFreshSortExpressions extends Rewriter { override def apply(that: AnyRef): AnyRef = instance(that) private val clauseRewriter: Clause => Seq[Clause] = { - case clause@With(_, _, None, _, _, None) => + case clause @ With(_, _, None, _, _, None) => Seq(clause) - case clause@With(_, ri: ReturnItems, orderBy, skip, limit, where) => + case clause @ With(_, ri: ReturnItems, orderBy, skip, limit, where) => val allAliases = ri.aliases val passedThroughAliases = ri.passedThrough val evaluatedAliases = allAliases -- passedThroughAliases if (evaluatedAliases.isEmpty) { Seq(clause) } else { - val nonItemDependencies = orderBy.map(_.dependencies).getOrElse(Set.empty) ++ - skip.map(_.dependencies).getOrElse(Set.empty) ++ - limit.map(_.dependencies).getOrElse(Set.empty) ++ - where.map(_.dependencies).getOrElse(Set.empty) + val nonItemDependencies = + orderBy.map(_.dependencies).getOrElse(Set.empty) ++ + skip.map(_.dependencies).getOrElse(Set.empty) ++ + limit.map(_.dependencies).getOrElse(Set.empty) ++ + where.map(_.dependencies).getOrElse(Set.empty) val dependenciesFromPreviousScope = nonItemDependencies -- allAliases - val passedItems = dependenciesFromPreviousScope.map(AliasedReturnItem(_)) + val passedItems = + dependenciesFromPreviousScope.map(AliasedReturnItem(_)) val outputItems = allAliases.toIndexedSeq.map(AliasedReturnItem(_)) val result = Seq( - clause.copy(returnItems = ri.mapItems(originalItems => originalItems ++ passedItems), orderBy = None, skip = None, limit = None, where = None)(clause.position), - clause.copy(distinct = false, returnItems = ri.mapItems(_ => outputItems))(clause.position) + clause.copy( + returnItems = ri.mapItems(originalItems => originalItems ++ passedItems), + orderBy = None, + skip = None, + limit = None, + where = None + )(clause.position), + clause.copy( + distinct = false, + returnItems = ri.mapItems(_ => outputItems) + )(clause.position) ) result } case clause => Seq(clause) } - private val rewriter = Rewriter.lift { - case query@SingleQuery(clauses) => - query.copy(clauses = clauses.flatMap(clauseRewriter))(query.position) + private val rewriter = Rewriter.lift { case query @ SingleQuery(clauses) => + query.copy(clauses = clauses.flatMap(clauseRewriter))(query.position) } - private val instance: Rewriter = bottomUp(rewriter, _.isInstanceOf[Expression]) + private val instance: Rewriter = + bottomUp(rewriter, _.isInstanceOf[Expression]) } diff --git a/okapi-ir/src/main/scala/org/opencypher/okapi/ir/impl/parse/rewriter/normalizeCaseExpression.scala b/okapi-ir/src/main/scala/org/opencypher/okapi/ir/impl/parse/rewriter/normalizeCaseExpression.scala index 7b7896540a..f5e9475f5c 100644 --- a/okapi-ir/src/main/scala/org/opencypher/okapi/ir/impl/parse/rewriter/normalizeCaseExpression.scala +++ b/okapi-ir/src/main/scala/org/opencypher/okapi/ir/impl/parse/rewriter/normalizeCaseExpression.scala @@ -50,14 +50,14 @@ case object normalizeCaseExpression extends Rewriter { def apply(that: AnyRef): AnyRef = instance.apply(that) - private val instance = topDown(Rewriter.lift { - case c: CaseExpression => rewriteCase(c) + private val instance = topDown(Rewriter.lift { case c: CaseExpression => + rewriteCase(c) }) private def rewriteCase: CaseExpression => CaseExpression = { - case expr@CaseExpression(Some(inputExpr), alternatives, default) => - val inlineAlternatives = alternatives.map { - case (predicate, action) => Equals(inputExpr, predicate)(predicate.position) -> action + case expr @ CaseExpression(Some(inputExpr), alternatives, default) => + val inlineAlternatives = alternatives.map { case (predicate, action) => + Equals(inputExpr, predicate)(predicate.position) -> action } CaseExpression(None, inlineAlternatives, default)(expr.position) diff --git a/okapi-ir/src/main/scala/org/opencypher/okapi/ir/impl/parse/rewriter/normalizeReturnClauses.scala b/okapi-ir/src/main/scala/org/opencypher/okapi/ir/impl/parse/rewriter/normalizeReturnClauses.scala index 512345cc85..64e732c68f 100644 --- a/okapi-ir/src/main/scala/org/opencypher/okapi/ir/impl/parse/rewriter/normalizeReturnClauses.scala +++ b/okapi-ir/src/main/scala/org/opencypher/okapi/ir/impl/parse/rewriter/normalizeReturnClauses.scala @@ -35,11 +35,21 @@ case object normalizeReturnClauses extends Rewriter { def apply(that: AnyRef): AnyRef = instance.apply(that) private val clauseRewriter: Clause => Seq[Clause] = { - case clause @ Return(distinct, ri @ ReturnItems(_, items), None, skip, limit, _) => + case clause @ Return( + distinct, + ri @ ReturnItems(_, items), + None, + skip, + limit, + _ + ) => val (aliasProjection, finalProjection) = items.map { // avoid aliasing of primitive expressions (i.e. variables and properties) case item @ AliasedReturnItem(Variable(_), Variable(_)) => - val returnItem = UnaliasedReturnItem(item.variable, item.variable.name)(item.position) + val returnItem = + UnaliasedReturnItem(item.variable, item.variable.name)( + item.position + ) (returnItem, returnItem) case item @ AliasedReturnItem(Property(_, _), _) => @@ -58,22 +68,28 @@ case object normalizeReturnClauses extends Rewriter { case None => Variable(item.name)(item.expression.position.bumped()) } - val newVariable = Variable(FreshIdNameGenerator.name(item.expression.position))(item.expression.position) + val newVariable = Variable( + FreshIdNameGenerator.name(item.expression.position) + )(item.expression.position) ( AliasedReturnItem(item.expression, newVariable)(item.position), - AliasedReturnItem(newVariable.copyId, returnColumn)(item.position)) + AliasedReturnItem(newVariable.copyId, returnColumn)(item.position) + ) }.unzip - val introducedVariables = if (ri.includeExisting) aliasProjection.collect { - case AliasedReturnItem(_, variable) => variable.name - }.toSet - else Set.empty[String] + val introducedVariables = + if (ri.includeExisting) aliasProjection.collect { case AliasedReturnItem(_, variable) => + variable.name + }.toSet + else Set.empty[String] - if (aliasProjection.forall { - case _: UnaliasedReturnItem => true - case _ => false - }) { + if ( + aliasProjection.forall { + case _: UnaliasedReturnItem => true + case _ => false + } + ) { Seq(clause) } else { Seq( @@ -91,7 +107,8 @@ case object normalizeReturnClauses extends Rewriter { orderBy = None, skip = None, limit = None, - excludedNames = introducedVariables)(clause.position) + excludedNames = introducedVariables + )(clause.position) ) } @@ -99,9 +116,7 @@ case object normalizeReturnClauses extends Rewriter { Seq(clause) } - private val instance: Rewriter = bottomUp(Rewriter.lift { - case query @ SingleQuery(clauses) => - query.copy(clauses = clauses.flatMap(clauseRewriter))(query.position) + private val instance: Rewriter = bottomUp(Rewriter.lift { case query @ SingleQuery(clauses) => + query.copy(clauses = clauses.flatMap(clauseRewriter))(query.position) }) } - diff --git a/okapi-ir/src/main/scala/org/opencypher/okapi/ir/impl/parse/rewriter/pushLabelsIntoScans.scala b/okapi-ir/src/main/scala/org/opencypher/okapi/ir/impl/parse/rewriter/pushLabelsIntoScans.scala index 7b3ebc9b2f..4562dd9cf2 100644 --- a/okapi-ir/src/main/scala/org/opencypher/okapi/ir/impl/parse/rewriter/pushLabelsIntoScans.scala +++ b/okapi-ir/src/main/scala/org/opencypher/okapi/ir/impl/parse/rewriter/pushLabelsIntoScans.scala @@ -36,64 +36,79 @@ case object pushLabelsIntoScans extends Rewriter { instance(that) } - private val rewriter = Rewriter.lift { - case m: Match => - def containsHasLabel(e: Expression) = e.treeExists { - case _:HasLabels => true - } + private val rewriter = Rewriter.lift { case m: Match => + def containsHasLabel(e: Expression) = e.treeExists { case _: HasLabels => + true + } - val patternLabelMap = m.pattern.treeFold[Map[String, Set[LabelName]]](Map.empty) { + val patternLabelMap = + m.pattern.treeFold[Map[String, Set[LabelName]]](Map.empty) { case NodePattern(Some(Variable(name)), labels, _, _) => acc => { val updatedLabels = acc.getOrElse(name, Set.empty) ++ labels val updated = acc.updated(name, updatedLabels) - val merge = (other: Map[String, Set[LabelName]]) => updated |+| other + val merge = + (other: Map[String, Set[LabelName]]) => updated |+| other updated -> Some(merge) } } - val whereLabelMap = m.where.treeFold[Map[String, Set[LabelName]]](Map.empty) { - case Or(lhs, rhs) if containsHasLabel(lhs) || containsHasLabel(rhs) => acc => acc -> None - case Ors(children) if children.count(containsHasLabel) >= 1 => acc => acc -> None + val whereLabelMap = + m.where.treeFold[Map[String, Set[LabelName]]](Map.empty) { + case Or(lhs, rhs) if containsHasLabel(lhs) || containsHasLabel(rhs) => + acc => acc -> None + case Ors(children) if children.count(containsHasLabel) >= 1 => + acc => acc -> None case Not(child) if containsHasLabel(child) => acc => acc -> None - case Xor(lhs, rhs) if containsHasLabel(lhs) || containsHasLabel(rhs) => acc => acc -> None + case Xor(lhs, rhs) if containsHasLabel(lhs) || containsHasLabel(rhs) => + acc => acc -> None case HasLabels(Variable(name), labels) => acc => { val updatedLabels = acc.getOrElse(name, Set.empty) ++ labels val updated = acc.updated(name, updatedLabels) - val merge = (other: Map[String, Set[LabelName]]) => updated |+| other + val merge = + (other: Map[String, Set[LabelName]]) => updated |+| other updated -> Some(merge) } } - val labelMap = patternLabelMap |+| whereLabelMap - val pattern = m.pattern.endoRewrite(addLabelsToNodePatterns(labelMap)) - val where = m.where.endoRewrite(removeRedundantLabelFilters(labelMap)) match { + val labelMap = patternLabelMap |+| whereLabelMap + val pattern = m.pattern.endoRewrite(addLabelsToNodePatterns(labelMap)) + val where = + m.where.endoRewrite(removeRedundantLabelFilters(labelMap)) match { case Some(Where(_: True)) => None - case other => other + case other => other } - m.copy(pattern = pattern, where = where)(m.position) + m.copy(pattern = pattern, where = where)(m.position) } - private def addLabelsToNodePatterns(labelMap: Map[String, Set[LabelName]]) = bottomUp { - case n @ NodePattern(Some(v), _, properties, base) if labelMap.contains(v.name) => - NodePattern(Some(v), labelMap(v.name).toSeq, properties, base)(n.position) - case other => other - } + private def addLabelsToNodePatterns(labelMap: Map[String, Set[LabelName]]) = + bottomUp { + case n @ NodePattern(Some(v), _, properties, base) if labelMap.contains(v.name) => + NodePattern(Some(v), labelMap(v.name).toSeq, properties, base)( + n.position + ) + case other => other + } - private def removeRedundantLabelFilters(labelMap: Map[String, Set[LabelName]]) = bottomUp { - case h@HasLabels(Variable(name), labels) if labels.toSet subsetOf labelMap.getOrElse(name, Set.empty) => True()(h.position) - case a@And(_: True, _: True) => True()(a.position) - case a@And(_: True, other) => other - case a@And(other, _: True) => other - case a@Ands(exprs) => + private def removeRedundantLabelFilters( + labelMap: Map[String, Set[LabelName]] + ) = bottomUp { + case h @ HasLabels(Variable(name), labels) + if labels.toSet subsetOf labelMap.getOrElse(name, Set.empty) => + True()(h.position) + case a @ And(_: True, _: True) => True()(a.position) + case a @ And(_: True, other) => other + case a @ And(other, _: True) => other + case a @ Ands(exprs) => val keep = exprs.filter { - case _:True => false - case _ => true + case _: True => false + case _ => true } if (keep.isEmpty) True()(a.position) else Ands(keep)(a.position) case other => other } - private val instance: Rewriter = bottomUp(rewriter, _.isInstanceOf[Expression]) + private val instance: Rewriter = + bottomUp(rewriter, _.isInstanceOf[Expression]) } diff --git a/okapi-ir/src/main/scala/org/opencypher/okapi/ir/impl/refactor/instances/ExprBlockInstances.scala b/okapi-ir/src/main/scala/org/opencypher/okapi/ir/impl/refactor/instances/ExprBlockInstances.scala index 0ebdeff64e..a3b96b8d13 100644 --- a/okapi-ir/src/main/scala/org/opencypher/okapi/ir/impl/refactor/instances/ExprBlockInstances.scala +++ b/okapi-ir/src/main/scala/org/opencypher/okapi/ir/impl/refactor/instances/ExprBlockInstances.scala @@ -54,26 +54,27 @@ trait ExprBlockInstances { val opaqueTypedFields = block.binds.fields val predicates = block.where - predicates.foldLeft(opaqueTypedFields) { - case (fields, predicate) => - predicate match { - case HasLabel(node: Var, label) => - fields.map { - case f if f representsNode node => - f.withLabel(label) - case f => f - } - // The below predicate is never present currently - // Possibly it will be if we introduce a rewrite - // Rel types are currently detailed already in pattern conversion - case HasType(rel: Var, _) => - fields.map { - case f if f representsRel rel => - throw NotImplementedException("No support for annotating relationships in IR yet") - case f => f - } - case _ => fields - } + predicates.foldLeft(opaqueTypedFields) { case (fields, predicate) => + predicate match { + case HasLabel(node: Var, label) => + fields.map { + case f if f representsNode node => + f.withLabel(label) + case f => f + } + // The below predicate is never present currently + // Possibly it will be if we introduce a rewrite + // Rel types are currently detailed already in pattern conversion + case HasType(rel: Var, _) => + fields.map { + case f if f representsRel rel => + throw NotImplementedException( + "No support for annotating relationships in IR yet" + ) + case f => f + } + case _ => fields + } } } } diff --git a/okapi-ir/src/main/scala/org/opencypher/okapi/ir/impl/refactor/syntax/BlockSyntax.scala b/okapi-ir/src/main/scala/org/opencypher/okapi/ir/impl/refactor/syntax/BlockSyntax.scala index 6332bc8c9b..35a7319333 100644 --- a/okapi-ir/src/main/scala/org/opencypher/okapi/ir/impl/refactor/syntax/BlockSyntax.scala +++ b/okapi-ir/src/main/scala/org/opencypher/okapi/ir/impl/refactor/syntax/BlockSyntax.scala @@ -32,11 +32,14 @@ import org.opencypher.okapi.ir.api.block.Block import scala.language.implicitConversions trait BlockSyntax { - implicit def typedBlockOps[B <: Block, E]( - block: B)(implicit instance: TypedBlock[B] { type BlockExpr = E }): TypedBlockOps[B, E] = + implicit def typedBlockOps[B <: Block, E](block: B)(implicit + instance: TypedBlock[B] { type BlockExpr = E } + ): TypedBlockOps[B, E] = new TypedBlockOps[B, E](block) } -final class TypedBlockOps[B <: Block, E](block: B)(implicit instance: TypedBlock[B] { type BlockExpr = E }) { +final class TypedBlockOps[B <: Block, E](block: B)(implicit + instance: TypedBlock[B] { type BlockExpr = E } +) { def outputs: Set[IRField] = instance.outputs(block) } diff --git a/okapi-ir/src/main/scala/org/opencypher/okapi/ir/impl/typer/TypeConverter.scala b/okapi-ir/src/main/scala/org/opencypher/okapi/ir/impl/typer/TypeConverter.scala index 601862fcc7..c88f4dec3b 100644 --- a/okapi-ir/src/main/scala/org/opencypher/okapi/ir/impl/typer/TypeConverter.scala +++ b/okapi-ir/src/main/scala/org/opencypher/okapi/ir/impl/typer/TypeConverter.scala @@ -51,10 +51,10 @@ object TypeConverter { case frontend.CTMap => Some(CTMap) case frontend.ListType(inner) => TypeConverter.convert(inner) match { - case None => None + case None => None case Some(t) => Some(CTList(t)) } - case _ => None + case _ => None } } @@ -64,12 +64,16 @@ object SignatureConverter { def apply(input: Seq[CypherType]): Option[CypherType] } - case class FunctionSignature(declaredInput: Seq[CypherType], output: CypherType) extends Signature { + case class FunctionSignature( + declaredInput: Seq[CypherType], + output: CypherType + ) extends Signature { override def apply(givenInput: Seq[CypherType]): Option[CypherType] = { - if (declaredInput.zip(givenInput).forall { - case (sigType, argType) => + if ( + declaredInput.zip(givenInput).forall { case (sigType, argType) => argType.couldBeSameTypeAs(sigType) - }) { + } + ) { Some(output) } else None @@ -97,12 +101,23 @@ object SignatureConverter { def expandWithNulls: FunctionSignatures = include(for { signature <- sigs - alternative <- substitutions(signature.declaredInput, 1, signature.declaredInput.size)(_ => CTNull) + alternative <- substitutions( + signature.declaredInput, + 1, + signature.declaredInput.size + )(_ => CTNull) } yield FunctionSignature(alternative, CTNull)) - def expandWithSubstitutions(old: CypherType, rep: CypherType): FunctionSignatures = include(for { + def expandWithSubstitutions( + old: CypherType, + rep: CypherType + ): FunctionSignatures = include(for { signature <- sigs - alternative <- substitutions(signature.declaredInput, 1, signature.declaredInput.size)(replace(old, rep)) + alternative <- substitutions( + signature.declaredInput, + 1, + signature.declaredInput.size + )(replace(old, rep)) if sigs.forall(_.declaredInput != alternative) } yield FunctionSignature(alternative, signature.output)) @@ -116,11 +131,15 @@ object SignatureConverter { mask <- mask(size, hits).permutations } yield mask - private def substituteMasked[T](seq: Seq[T], mask: Int => Boolean)(sub: T => T) = for { + private def substituteMasked[T](seq: Seq[T], mask: Int => Boolean)( + sub: T => T + ) = for { (orig, i) <- seq.zipWithIndex } yield if (mask(i)) sub(orig) else orig - private def substitutions[T](seq: Seq[T], minSubs: Int, maxSubs: Int)(sub: T => T): Seq[Seq[T]] = + private def substitutions[T](seq: Seq[T], minSubs: Int, maxSubs: Int)( + sub: T => T + ): Seq[Seq[T]] = masks(seq.size, minSubs, maxSubs).map(mask => substituteMasked(seq, mask)(sub)) private def replace[T](old: T, rep: T)(t: T) = diff --git a/okapi-ir/src/main/scala/org/opencypher/okapi/ir/impl/typer/TypeRecorder.scala b/okapi-ir/src/main/scala/org/opencypher/okapi/ir/impl/typer/TypeRecorder.scala index 5f89425d80..fb9b5ac74d 100644 --- a/okapi-ir/src/main/scala/org/opencypher/okapi/ir/impl/typer/TypeRecorder.scala +++ b/okapi-ir/src/main/scala/org/opencypher/okapi/ir/impl/typer/TypeRecorder.scala @@ -33,14 +33,17 @@ import org.opencypher.v9_0.util.Ref import scala.annotation.tailrec -final case class TypeRecorder(recordedTypes: List[(Ref[Expression], CypherType)]) { +final case class TypeRecorder( + recordedTypes: List[(Ref[Expression], CypherType)] +) { def toMap: Map[Ref[Expression], CypherType] = toMap(Map.empty, recordedTypes) @tailrec private def toMap( - m: Map[Ref[Expression], CypherType], - recorded: Seq[(Ref[Expression], CypherType)]): Map[Ref[Expression], CypherType] = + m: Map[Ref[Expression], CypherType], + recorded: Seq[(Ref[Expression], CypherType)] + ): Map[Ref[Expression], CypherType] = recorded.headOption match { case Some((ref, t)) => m.get(ref) match { @@ -55,8 +58,8 @@ final case class TypeRecorder(recordedTypes: List[(Ref[Expression], CypherType)] object TypeRecorder { def from(tuples: List[(Expression, CypherType)]): TypeRecorder = { - TypeRecorder(tuples.map { - case (e, t) => Ref(e) -> t + TypeRecorder(tuples.map { case (e, t) => + Ref(e) -> t }) } diff --git a/okapi-ir/src/main/scala/org/opencypher/okapi/ir/impl/typer/TypeTracker.scala b/okapi-ir/src/main/scala/org/opencypher/okapi/ir/impl/typer/TypeTracker.scala index b9b3bb5638..132c34de9f 100644 --- a/okapi-ir/src/main/scala/org/opencypher/okapi/ir/impl/typer/TypeTracker.scala +++ b/okapi-ir/src/main/scala/org/opencypher/okapi/ir/impl/typer/TypeTracker.scala @@ -37,13 +37,19 @@ object TypeTracker { val empty = TypeTracker(Map.empty) } -case class TypeTracker(map: Map[Expression, CypherType], parameters: Map[String, CypherValue] = Map.empty) { +case class TypeTracker( + map: Map[Expression, CypherType], + parameters: Map[String, CypherValue] = Map.empty +) { - def withParameters(newParameters: Map[String, CypherValue]): TypeTracker = copy(parameters = newParameters) + def withParameters(newParameters: Map[String, CypherValue]): TypeTracker = + copy(parameters = newParameters) def get(e: Expression): Option[CypherType] = map.get(e) - def getParameterType(e: String): Option[CypherType] = parameters.get(e).map(_.cypherType) + def getParameterType(e: String): Option[CypherType] = + parameters.get(e).map(_.cypherType) - def updated(e: Expression, t: CypherType): TypeTracker = copy(map = map.updated(e, t)) + def updated(e: Expression, t: CypherType): TypeTracker = + copy(map = map.updated(e, t)) } diff --git a/okapi-ir/src/main/scala/org/opencypher/okapi/ir/impl/typer/TyperError.scala b/okapi-ir/src/main/scala/org/opencypher/okapi/ir/impl/typer/TyperError.scala index 73c3b28c90..db72b67ece 100644 --- a/okapi-ir/src/main/scala/org/opencypher/okapi/ir/impl/typer/TyperError.scala +++ b/okapi-ir/src/main/scala/org/opencypher/okapi/ir/impl/typer/TyperError.scala @@ -35,7 +35,8 @@ sealed trait TyperError extends Throwable { } case class UnsupportedExpr(expr: Expression) extends TyperError { - override def toString = s"The expression ${expr.show} is not supported by the system" + override def toString = + s"The expression ${expr.show} is not supported by the system" } case class UnTypedExpr(it: Expression) extends TyperError { @@ -46,25 +47,42 @@ case class MissingParameter(it: String) extends TyperError { override def toString = s"Expected a type for $$$it but found none" } -case class NoSuitableSignatureForExpr(it: Expression, argTypes: Seq[CypherType]) extends TyperError { - override def toString = s"No signature for ${it.show} matched input types ${argTypes.mkString("{ ", ", ", " }")}" +case class NoSuitableSignatureForExpr(it: Expression, argTypes: Seq[CypherType]) + extends TyperError { + override def toString = + s"No signature for ${it.show} matched input types ${argTypes.mkString("{ ", ", ", " }")}" } -case class AlreadyTypedExpr(it: Expression, oldTyp: CypherType, newTyp: CypherType) extends TyperError { - override def toString = s"Tried to type ${it.show} with $newTyp but it was already typed as $oldTyp" +case class AlreadyTypedExpr( + it: Expression, + oldTyp: CypherType, + newTyp: CypherType +) extends TyperError { + override def toString = + s"Tried to type ${it.show} with $newTyp but it was already typed as $oldTyp" } case class InvalidContainerAccess(it: Expression) extends TyperError { - override def toString = s"Invalid indexing into a container detected when typing ${it.show}" + override def toString = + s"Invalid indexing into a container detected when typing ${it.show}" } object InvalidType { - def apply(it: Expression, expected: CypherType, actual: CypherType): InvalidType = + def apply( + it: Expression, + expected: CypherType, + actual: CypherType + ): InvalidType = InvalidType(it, Seq(expected), actual) } -case class InvalidType(it: Expression, expected: Seq[CypherType], actual: CypherType) extends TyperError { - override def toString = s"Expected ${it.show} to have $expectedString, but it was of type $actual" +case class InvalidType( + it: Expression, + expected: Seq[CypherType], + actual: CypherType +) extends TyperError { + override def toString = + s"Expected ${it.show} to have $expectedString, but it was of type $actual" private def expectedString = if (expected.size == 1) s"type ${expected.head}" @@ -72,7 +90,8 @@ case class InvalidType(it: Expression, expected: Seq[CypherType], actual: Cypher } case object TypeTrackerScopeError extends TyperError { - override def toString = "Tried to pop scope of type tracker, but it was at top level already" + override def toString = + "Tried to pop scope of type tracker, but it was at top level already" } case class InvalidArgument(expr: Expression, argument: Expression) extends TyperError { @@ -80,5 +99,6 @@ case class InvalidArgument(expr: Expression, argument: Expression) extends Typer } case class WrongNumberOfArguments(expr: Expression, expected: Int, actual: Int) extends TyperError { - override def toString = s"Expected $expected argument(s) for $expr, but got $actual" + override def toString = + s"Expected $expected argument(s) for $expr, but got $actual" } diff --git a/okapi-ir/src/main/scala/org/opencypher/okapi/ir/impl/typer/TyperResult.scala b/okapi-ir/src/main/scala/org/opencypher/okapi/ir/impl/typer/TyperResult.scala index 1b9686eec1..03c72c754f 100644 --- a/okapi-ir/src/main/scala/org/opencypher/okapi/ir/impl/typer/TyperResult.scala +++ b/okapi-ir/src/main/scala/org/opencypher/okapi/ir/impl/typer/TyperResult.scala @@ -26,4 +26,8 @@ */ package org.opencypher.okapi.ir.impl.typer -case class TyperResult[A](value: A, recorder: TypeRecorder, tracker: TypeTracker) +case class TyperResult[A]( + value: A, + recorder: TypeRecorder, + tracker: TypeTracker +) diff --git a/okapi-ir/src/main/scala/org/opencypher/okapi/ir/impl/typer/toFrontendType.scala b/okapi-ir/src/main/scala/org/opencypher/okapi/ir/impl/typer/toFrontendType.scala index 171f37dbcf..d5662b3de6 100644 --- a/okapi-ir/src/main/scala/org/opencypher/okapi/ir/impl/typer/toFrontendType.scala +++ b/okapi-ir/src/main/scala/org/opencypher/okapi/ir/impl/typer/toFrontendType.scala @@ -31,20 +31,23 @@ import org.opencypher.v9_0.util.{symbols => frontend} case object toFrontendType extends (CypherType => frontend.CypherType) { override def apply(in: CypherType): frontend.CypherType = in.material match { - case CTAnyMaterial => frontend.CTAny - case CTInteger => frontend.CTInteger - case CTFloat => frontend.CTFloat - case CTString => frontend.CTString - case CTNode => frontend.CTNode - case CTDate => frontend.CTDate - case CTLocalDateTime => frontend.CTLocalDateTime - case CTDuration => frontend.CTDuration - case CTRelationship => frontend.CTRelationship - case CTPath => frontend.CTPath - case CTMap(_) => frontend.CTMap - case CTList(inner) => frontend.ListType(toFrontendType(inner)) + case CTAnyMaterial => frontend.CTAny + case CTInteger => frontend.CTInteger + case CTFloat => frontend.CTFloat + case CTString => frontend.CTString + case CTNode => frontend.CTNode + case CTDate => frontend.CTDate + case CTLocalDateTime => frontend.CTLocalDateTime + case CTDuration => frontend.CTDuration + case CTRelationship => frontend.CTRelationship + case CTPath => frontend.CTPath + case CTMap(_) => frontend.CTMap + case CTList(inner) => frontend.ListType(toFrontendType(inner)) case b if b.subTypeOf(CTBoolean) => frontend.CTBoolean - case t if t.subTypeOf(CTNumber) => frontend.CTNumber - case x => throw new UnsupportedOperationException(s"Can not convert internal type $x to an openCypher frontend type") + case t if t.subTypeOf(CTNumber) => frontend.CTNumber + case x => + throw new UnsupportedOperationException( + s"Can not convert internal type $x to an openCypher frontend type" + ) } } diff --git a/okapi-ir/src/main/scala/org/opencypher/okapi/ir/impl/util/VarConverters.scala b/okapi-ir/src/main/scala/org/opencypher/okapi/ir/impl/util/VarConverters.scala index 129b5652fd..6e39f001a1 100644 --- a/okapi-ir/src/main/scala/org/opencypher/okapi/ir/impl/util/VarConverters.scala +++ b/okapi-ir/src/main/scala/org/opencypher/okapi/ir/impl/util/VarConverters.scala @@ -57,6 +57,7 @@ object VarConverters { implicit def toField(s: Symbol): IRField = IRField(s.name)() - implicit def toField(t: (Symbol, CypherType)): IRField = IRField(t._1.name)(t._2) + implicit def toField(t: (Symbol, CypherType)): IRField = + IRField(t._1.name)(t._2) } diff --git a/okapi-ir/src/test/scala/org/opencypher/okapi/ir/api/expr/ExprTest.scala b/okapi-ir/src/test/scala/org/opencypher/okapi/ir/api/expr/ExprTest.scala index dc2c271f31..1d30945708 100644 --- a/okapi-ir/src/test/scala/org/opencypher/okapi/ir/api/expr/ExprTest.scala +++ b/okapi-ir/src/test/scala/org/opencypher/okapi/ir/api/expr/ExprTest.scala @@ -82,19 +82,31 @@ class ExprTest extends BaseTestSuite { val number = Var("number")(CTNumber) it("types Coalesce correctly") { - Coalesce(List(a, b)).cypherType should equal(CTUnion(CTNode, CTInteger, CTString)) + Coalesce(List(a, b)).cypherType should equal( + CTUnion(CTNode, CTInteger, CTString) + ) Coalesce(List(b, c)).cypherType should equal(CTUnion(CTInteger, CTString)) - Coalesce(List(a, b, c, d)).cypherType should equal(CTUnion(CTNode, CTInteger, CTString)) + Coalesce(List(a, b, c, d)).cypherType should equal( + CTUnion(CTNode, CTInteger, CTString) + ) - Coalesce(List(d,e)).cypherType should equal(CTUnion(CTInteger, CTString).nullable) + Coalesce(List(d, e)).cypherType should equal( + CTUnion(CTInteger, CTString).nullable + ) } it("types ListSegment correctly") { - ListSegment(3, Var("list")(CTList(CTNode))).cypherType should equalWithTracing( + ListSegment( + 3, + Var("list")(CTList(CTNode)) + ).cypherType should equalWithTracing( CTNode.nullable ) - ListSegment(3, Var("list")(CTUnion(CTList(CTString), CTList(CTInteger)))).cypherType should equalWithTracing( + ListSegment( + 3, + Var("list")(CTUnion(CTList(CTString), CTList(CTInteger))) + ).cypherType should equalWithTracing( CTUnion(CTString, CTInteger).nullable ) @@ -110,13 +122,21 @@ class ExprTest extends BaseTestSuite { "c" -> c, "d" -> d ) - MapExpression(mapFields).cypherType should equalWithTracing(CTMap(mapFields.mapValues(_.cypherType))) + MapExpression(mapFields).cypherType should equalWithTracing( + CTMap(mapFields.mapValues(_.cypherType)) + ) } it("types ListLit correctly") { - ListLit(List(a, b)).cypherType should equal(CTList(CTUnion(CTNode, CTInteger, CTString))) - ListLit(List(b, c)).cypherType should equal(CTList(CTUnion(CTInteger, CTString.nullable))) - ListLit(List(a, b, c, d)).cypherType should equal(CTList(CTUnion(CTNode, CTInteger, CTString).nullable)) + ListLit(List(a, b)).cypherType should equal( + CTList(CTUnion(CTNode, CTInteger, CTString)) + ) + ListLit(List(b, c)).cypherType should equal( + CTList(CTUnion(CTInteger, CTString.nullable)) + ) + ListLit(List(a, b, c, d)).cypherType should equal( + CTList(CTUnion(CTNode, CTInteger, CTString).nullable) + ) } it("types Explode correctly") { @@ -124,7 +144,9 @@ class ExprTest extends BaseTestSuite { CTNode ) - Explode(Var("list")(CTUnion(CTList(CTString), CTList(CTInteger)))).cypherType should equalWithTracing( + Explode( + Var("list")(CTUnion(CTList(CTString), CTList(CTInteger))) + ).cypherType should equalWithTracing( CTUnion(CTString, CTInteger) ) @@ -136,29 +158,29 @@ class ExprTest extends BaseTestSuite { it("types Avg correctly") { Avg(duration).cypherType shouldBe CTDuration Avg(number).cypherType shouldBe CTNumber - an[NoSuitableSignatureForExpr] shouldBe thrownBy {Avg(datetime)} + an[NoSuitableSignatureForExpr] shouldBe thrownBy { Avg(datetime) } } it("types Sum correctly") { Sum(duration).cypherType shouldBe CTDuration Sum(number).cypherType shouldBe CTNumber - an[NoSuitableSignatureForExpr] shouldBe thrownBy {Sum(datetime)} + an[NoSuitableSignatureForExpr] shouldBe thrownBy { Sum(datetime) } } it("types Size correctly") { Size(e).cypherType shouldBe CTInteger.nullable - an[NoSuitableSignatureForExpr] shouldBe thrownBy {Size(datetime)} + an[NoSuitableSignatureForExpr] shouldBe thrownBy { Size(datetime) } } it("types Trim correctly") { Trim(e).cypherType shouldBe CTString.nullable - an[NoSuitableSignatureForExpr] shouldBe thrownBy {Trim(datetime)} + an[NoSuitableSignatureForExpr] shouldBe thrownBy { Trim(datetime) } } it("types Range correctly") { Range(d, d, None).cypherType shouldBe CTList(CTInteger) Range(d, d, Some(d)).cypherType - an[NoSuitableSignatureForExpr] shouldBe thrownBy {Range(e, d, None)} + an[NoSuitableSignatureForExpr] shouldBe thrownBy { Range(e, d, None) } } it("types TemporalInstants correctly") { diff --git a/okapi-ir/src/test/scala/org/opencypher/okapi/ir/api/pattern/ConnectionTest.scala b/okapi-ir/src/test/scala/org/opencypher/okapi/ir/api/pattern/ConnectionTest.scala index fcb2cccf1d..8e8f3dd705 100644 --- a/okapi-ir/src/test/scala/org/opencypher/okapi/ir/api/pattern/ConnectionTest.scala +++ b/okapi-ir/src/test/scala/org/opencypher/okapi/ir/api/pattern/ConnectionTest.scala @@ -40,18 +40,32 @@ class ConnectionTest extends BaseTestSuite { val relType = CTRelationship("FOO") test("SimpleConnection.equals") { - DirectedRelationship(field_a, field_b) shouldNot equal(DirectedRelationship(field_b, field_a)) - DirectedRelationship(field_a, field_a) should equal(DirectedRelationship(field_a, field_a)) - DirectedRelationship(field_a, field_a, OUTGOING) should equal(DirectedRelationship(field_a, field_a, OUTGOING)) - DirectedRelationship(field_a, field_a) shouldNot equal(DirectedRelationship(field_a, field_b)) + DirectedRelationship(field_a, field_b) shouldNot equal( + DirectedRelationship(field_b, field_a) + ) + DirectedRelationship(field_a, field_a) should equal( + DirectedRelationship(field_a, field_a) + ) + DirectedRelationship(field_a, field_a, OUTGOING) should equal( + DirectedRelationship(field_a, field_a, OUTGOING) + ) + DirectedRelationship(field_a, field_a) shouldNot equal( + DirectedRelationship(field_a, field_b) + ) } test("UndirectedConnection.equals") { - UndirectedRelationship(field_a, field_b) should equal(UndirectedRelationship(field_b, field_a)) - UndirectedRelationship(field_c, field_c) should equal(UndirectedRelationship(field_c, field_c)) + UndirectedRelationship(field_a, field_b) should equal( + UndirectedRelationship(field_b, field_a) + ) + UndirectedRelationship(field_c, field_c) should equal( + UndirectedRelationship(field_c, field_c) + ) } test("Mixed equals") { - DirectedRelationship(field_a, field_a) should equal(UndirectedRelationship(field_a, field_a)) + DirectedRelationship(field_a, field_a) should equal( + UndirectedRelationship(field_a, field_a) + ) } } diff --git a/okapi-ir/src/test/scala/org/opencypher/okapi/ir/impl/ExpressionConverterTest.scala b/okapi-ir/src/test/scala/org/opencypher/okapi/ir/impl/ExpressionConverterTest.scala index 229e11e333..b2c3d36fc5 100644 --- a/okapi-ir/src/test/scala/org/opencypher/okapi/ir/impl/ExpressionConverterTest.scala +++ b/okapi-ir/src/test/scala/org/opencypher/okapi/ir/impl/ExpressionConverterTest.scala @@ -45,10 +45,19 @@ import scala.language.implicitConversions class ExpressionConverterTest extends BaseTestSuite with Neo4jAstTestSupport { val baseTypes: Seq[CypherType] = Seq[CypherType]( - CTAny, CTUnion(CTInteger, CTFloat), CTNull, CTVoid, - CTBoolean, CTInteger, CTFloat, CTString, - CTDate, CTLocalDateTime, CTDuration, - CTIdentity, CTPath + CTAny, + CTUnion(CTInteger, CTFloat), + CTNull, + CTVoid, + CTBoolean, + CTInteger, + CTFloat, + CTString, + CTDate, + CTLocalDateTime, + CTDuration, + CTIdentity, + CTPath ) val simple: Seq[(String, CypherType)] = @@ -73,12 +82,12 @@ class ExpressionConverterTest extends BaseTestSuite with Neo4jAstTestSupport { lists.map { case (n, t) => s"${n}_OR_NULL" -> t.nullable }, maps, maps.map { case (n, t) => s"${n}_OR_NULL" -> t.nullable } - ).flatten.map { - case (name, typ) => Var(name)(typ) + ).flatten.map { case (name, typ) => + Var(name)(typ) }.toSet - private val properties = - simple ++ Seq("name" -> CTString, "age" -> CTInteger) + private val properties = + simple ++ Seq("name" -> CTString, "age" -> CTInteger) private val properties2 = simple ++ Seq("name" -> CTBoolean, "age" -> CTFloat) @@ -87,9 +96,9 @@ class ExpressionConverterTest extends BaseTestSuite with Neo4jAstTestSupport { simple ++ Seq("name" -> CTAny, "age" -> CTUnion(CTInteger, CTFloat)) private val schema: PropertyGraphSchema = PropertyGraphSchema.empty - .withNodePropertyKeys("Node")(properties : _*) + .withNodePropertyKeys("Node")(properties: _*) .withRelationshipPropertyKeys("REL")(properties: _*) - .withNodePropertyKeys("Node2")(properties2 : _*) + .withNodePropertyKeys("Node2")(properties2: _*) .withRelationshipPropertyKeys("REL2")(properties2: _*) val testContext: IRBuilderContext = IRBuilderContext.initial( @@ -110,32 +119,57 @@ class ExpressionConverterTest extends BaseTestSuite with Neo4jAstTestSupport { } it("should convert bigdecimal addition") { - val result = convert(parseExpr("bigdecimal(INTEGER, 4, 2) + bigdecimal(INTEGER, 10, 6)")) - result shouldEqual Add(BigDecimal('INTEGER, 4, 2), BigDecimal('INTEGER, 10, 6)) + val result = convert( + parseExpr("bigdecimal(INTEGER, 4, 2) + bigdecimal(INTEGER, 10, 6)") + ) + result shouldEqual Add( + BigDecimal('INTEGER, 4, 2), + BigDecimal('INTEGER, 10, 6) + ) result.cypherType shouldEqual CTBigDecimal(11, 6) } it("should convert bigdecimal subtraction") { - val result = convert(parseExpr("bigdecimal(INTEGER, 4, 2) - bigdecimal(INTEGER, 10, 6)")) - result shouldEqual Subtract(BigDecimal('INTEGER, 4, 2), BigDecimal('INTEGER, 10, 6)) + val result = convert( + parseExpr("bigdecimal(INTEGER, 4, 2) - bigdecimal(INTEGER, 10, 6)") + ) + result shouldEqual Subtract( + BigDecimal('INTEGER, 4, 2), + BigDecimal('INTEGER, 10, 6) + ) result.cypherType shouldEqual CTBigDecimal(11, 6) } it("should convert bigdecimal multiplication") { - val result = convert(parseExpr("bigdecimal(INTEGER, 4, 2) * bigdecimal(INTEGER, 10, 6)")) - result shouldEqual Multiply(BigDecimal('INTEGER, 4, 2), BigDecimal('INTEGER, 10, 6)) + val result = convert( + parseExpr("bigdecimal(INTEGER, 4, 2) * bigdecimal(INTEGER, 10, 6)") + ) + result shouldEqual Multiply( + BigDecimal('INTEGER, 4, 2), + BigDecimal('INTEGER, 10, 6) + ) result.cypherType shouldEqual CTBigDecimal(15, 8) } it("should convert bigdecimal division") { - val result = convert(parseExpr("bigdecimal(INTEGER, 4, 2) / bigdecimal(INTEGER, 10, 6)")) - result shouldEqual Divide(BigDecimal('INTEGER, 4, 2), BigDecimal('INTEGER, 10, 6)) + val result = convert( + parseExpr("bigdecimal(INTEGER, 4, 2) / bigdecimal(INTEGER, 10, 6)") + ) + result shouldEqual Divide( + BigDecimal('INTEGER, 4, 2), + BigDecimal('INTEGER, 10, 6) + ) result.cypherType shouldEqual CTBigDecimal(21, 13) } it("should convert bigdecimal division (magic number 6)") { - val result = convert(parseExpr("bigdecimal(INTEGER, 3, 1) / bigdecimal(INTEGER, 2, 1)")) - result shouldEqual Divide(BigDecimal('INTEGER, 3, 1), BigDecimal('INTEGER, 2, 1)) + val result = convert( + parseExpr("bigdecimal(INTEGER, 3, 1) / bigdecimal(INTEGER, 2, 1)") + ) + result shouldEqual Divide( + BigDecimal('INTEGER, 3, 1), + BigDecimal('INTEGER, 2, 1) + ) result.cypherType shouldEqual CTBigDecimal(9, 6) } @@ -164,38 +198,58 @@ class ExpressionConverterTest extends BaseTestSuite with Neo4jAstTestSupport { } it("should not allow scale to be greater than precision") { - val a = the [IllegalArgumentException] thrownBy convert(parseExpr("bigdecimal(INTEGER, 2, 3)")) - a.getMessage should(include("Greater precision than scale") and include("precision: 2") and include("scale: 3")) + val a = the[IllegalArgumentException] thrownBy convert( + parseExpr("bigdecimal(INTEGER, 2, 3)") + ) + a.getMessage should (include("Greater precision than scale") and include( + "precision: 2" + ) and include("scale: 3")) } } it("should convert CASE") { - convert(parseExpr("CASE WHEN INTEGER > INTEGER THEN INTEGER ELSE FLOAT END")) should equal( - CaseExpr(List((GreaterThan('INTEGER, 'INTEGER), 'INTEGER)), Some('FLOAT))(CTUnion(CTInteger, CTFloat)) + convert( + parseExpr("CASE WHEN INTEGER > INTEGER THEN INTEGER ELSE FLOAT END") + ) should equal( + CaseExpr(List((GreaterThan('INTEGER, 'INTEGER), 'INTEGER)), Some('FLOAT))( + CTUnion(CTInteger, CTFloat) + ) ) - convert(parseExpr("CASE WHEN STRING > STRING_OR_NULL THEN NODE END")) should equal( - CaseExpr(List((GreaterThan('STRING, 'STRING_OR_NULL), 'NODE)), None)(CTNode("Node").nullable) + convert( + parseExpr("CASE WHEN STRING > STRING_OR_NULL THEN NODE END") + ) should equal( + CaseExpr(List((GreaterThan('STRING, 'STRING_OR_NULL), 'NODE)), None)( + CTNode("Node").nullable + ) ) } describe("coalesce") { it("should convert coalesce") { - convert(parseExpr("coalesce(INTEGER_OR_NULL, STRING_OR_NULL, NODE)")) shouldEqual + convert( + parseExpr("coalesce(INTEGER_OR_NULL, STRING_OR_NULL, NODE)") + ) shouldEqual Coalesce(List('INTEGER_OR_NULL, 'STRING_OR_NULL, 'NODE)) } it("should become nullable if nothing is non-null") { - convert(parseExpr("coalesce(INTEGER_OR_NULL, STRING_OR_NULL, NODE_OR_NULL)")) shouldEqual + convert( + parseExpr("coalesce(INTEGER_OR_NULL, STRING_OR_NULL, NODE_OR_NULL)") + ) shouldEqual Coalesce(List('INTEGER_OR_NULL, 'STRING_OR_NULL, 'NODE_OR_NULL)) } it("should not consider arguments past the first non-nullable coalesce") { - convert(parseExpr("coalesce(INTEGER_OR_NULL, FLOAT, NODE, STRING)")) shouldEqual + convert( + parseExpr("coalesce(INTEGER_OR_NULL, FLOAT, NODE, STRING)") + ) shouldEqual Coalesce(List('INTEGER_OR_NULL, 'FLOAT)) } it("should remove coalesce if the first arg is non-nullable") { - convert(parseExpr("coalesce(INTEGER, STRING_OR_NULL, NODE)")) shouldEqual( + convert( + parseExpr("coalesce(INTEGER, STRING_OR_NULL, NODE)") + ) shouldEqual ( toVar('INTEGER), CTInteger ) } @@ -205,7 +259,7 @@ class ExpressionConverterTest extends BaseTestSuite with Neo4jAstTestSupport { // NOTE: pattern version of exists((:A)-->(:B)) is rewritten before IR building it("can convert") { - convert(parseExpr("exists(NODE.name)")) shouldEqual( + convert(parseExpr("exists(NODE.name)")) shouldEqual ( Exists(ElementProperty('NODE, PropertyKey("name"))(CTString)), CTBoolean ) } @@ -213,13 +267,18 @@ class ExpressionConverterTest extends BaseTestSuite with Neo4jAstTestSupport { describe("IN") { it("can convert in predicate and literal list") { - convert(parseExpr("INTEGER IN [INTEGER, INTEGER_OR_NULL, FLOAT]")) shouldEqual( - In('INTEGER, ListLit(List('INTEGER, 'INTEGER_OR_NULL, 'FLOAT))), CTBoolean.nullable + convert( + parseExpr("INTEGER IN [INTEGER, INTEGER_OR_NULL, FLOAT]") + ) shouldEqual ( + In( + 'INTEGER, + ListLit(List('INTEGER, 'INTEGER_OR_NULL, 'FLOAT)) + ), CTBoolean.nullable ) } it("can convert IN for single-element lists") { - convert(parseExpr("STRING IN ['foo']")) shouldEqual( + convert(parseExpr("STRING IN ['foo']")) shouldEqual ( In('STRING, ListLit(List(StringLit("foo")))), CTBoolean ) } @@ -228,7 +287,12 @@ class ExpressionConverterTest extends BaseTestSuite with Neo4jAstTestSupport { it("can convert or predicate") { convert(parseExpr("NODE = NODE_OR_NULL OR STRING_OR_NULL > STRING")) match { case ors @ Ors(inner) => - inner.toSet should equal(Set(GreaterThan('STRING_OR_NULL, 'STRING), Equals('NODE, 'NODE_OR_NULL))) + inner.toSet should equal( + Set( + GreaterThan('STRING_OR_NULL, 'STRING), + Equals('NODE, 'NODE_OR_NULL) + ) + ) ors.cypherType should equal(CTBoolean.nullable) case other => fail(s"Expected an `Ors` Expr, got `$other`") @@ -237,11 +301,11 @@ class ExpressionConverterTest extends BaseTestSuite with Neo4jAstTestSupport { describe("type()") { it("can convert") { - convert(parseExpr("type(REL)")) shouldEqual(Type('REL), CTString) + convert(parseExpr("type(REL)")) shouldEqual (Type('REL), CTString) } it("can convert nullable") { - convert(parseExpr("type(REL_OR_NULL)")) shouldEqual( + convert(parseExpr("type(REL_OR_NULL)")) shouldEqual ( Type('REL_OR_NULL), CTString.nullable ) } @@ -249,17 +313,17 @@ class ExpressionConverterTest extends BaseTestSuite with Neo4jAstTestSupport { describe("count()") { it("can convert") { - convert(parseExpr("count(NODE)")) shouldEqual( + convert(parseExpr("count(NODE)")) shouldEqual ( Count('NODE, distinct = false), CTInteger ) } it("can convert distinct") { - convert(parseExpr("count(distinct INTEGER)")) shouldEqual( + convert(parseExpr("count(distinct INTEGER)")) shouldEqual ( Count('INTEGER, distinct = true), CTInteger ) } it("can convert star") { - convert(parseExpr("count(*)")) shouldEqual( + convert(parseExpr("count(*)")) shouldEqual ( CountStar, CTInteger ) } @@ -268,13 +332,15 @@ class ExpressionConverterTest extends BaseTestSuite with Neo4jAstTestSupport { describe("range") { it("can convert range") { - convert(parseExpr("range(0, 10, 2)")) shouldEqual( - Range(IntegerLit(0), IntegerLit(10), Some(IntegerLit(2))), CTList(CTInteger) + convert(parseExpr("range(0, 10, 2)")) shouldEqual ( + Range(IntegerLit(0), IntegerLit(10), Some(IntegerLit(2))), CTList( + CTInteger + ) ) } it("can convert range with missing step size") { - convert(parseExpr("range(0, 10)")) shouldEqual( + convert(parseExpr("range(0, 10)")) shouldEqual ( Range(IntegerLit(0), IntegerLit(10), None), CTList(CTInteger) ) } @@ -283,38 +349,42 @@ class ExpressionConverterTest extends BaseTestSuite with Neo4jAstTestSupport { describe("substring") { it("can convert substring") { - convert(parseExpr("substring('foobar', 0, 3)")) shouldEqual( - Substring(StringLit("foobar"), IntegerLit(0), Some(IntegerLit(3))), CTString + convert(parseExpr("substring('foobar', 0, 3)")) shouldEqual ( + Substring( + StringLit("foobar"), + IntegerLit(0), + Some(IntegerLit(3)) + ), CTString ) } it("can convert substring with missing length") { - convert(parseExpr("substring('foobar', 0)")) shouldEqual( + convert(parseExpr("substring('foobar', 0)")) shouldEqual ( Substring(StringLit("foobar"), IntegerLit(0), None), CTString ) } } it("can convert less than") { - convert(parseExpr("INTEGER < FLOAT_OR_NULL")) shouldEqual( + convert(parseExpr("INTEGER < FLOAT_OR_NULL")) shouldEqual ( LessThan('INTEGER, 'FLOAT_OR_NULL), CTBoolean.nullable ) } it("can convert less than or equal") { - convert(parseExpr("INTEGER <= FLOAT_OR_NULL")) shouldEqual( + convert(parseExpr("INTEGER <= FLOAT_OR_NULL")) shouldEqual ( LessThanOrEqual('INTEGER, 'FLOAT_OR_NULL), CTBoolean.nullable ) } it("can convert greater than") { - convert(parseExpr("INTEGER > FLOAT_OR_NULL")) shouldEqual( + convert(parseExpr("INTEGER > FLOAT_OR_NULL")) shouldEqual ( GreaterThan('INTEGER, 'FLOAT_OR_NULL), CTBoolean.nullable ) } it("can convert greater than or equal") { - convert(parseExpr("INTEGER >= INTEGER")) shouldEqual( + convert(parseExpr("INTEGER >= INTEGER")) shouldEqual ( GreaterThanOrEqual('INTEGER, 'INTEGER), CTBoolean ) } @@ -344,7 +414,7 @@ class ExpressionConverterTest extends BaseTestSuite with Neo4jAstTestSupport { } it("can convert type function calls used as predicates") { - convert(parseExpr("type(REL) = 'REL_TYPE'")) shouldEqual( + convert(parseExpr("type(REL) = 'REL_TYPE'")) shouldEqual ( HasType('REL, RelType("REL_TYPE")), CTBoolean ) } @@ -366,28 +436,47 @@ class ExpressionConverterTest extends BaseTestSuite with Neo4jAstTestSupport { it("can convert property access") { val convertedNodeProperty = convert(prop("NODE", "age")) convertedNodeProperty.cypherType shouldEqual CTInteger - convertedNodeProperty shouldEqual ElementProperty('NODE, PropertyKey("age"))(CTInteger) + convertedNodeProperty shouldEqual ElementProperty( + 'NODE, + PropertyKey("age") + )(CTInteger) val convertedMapProperty = convert(prop(mapOf("age" -> literal(40)), "age")) convertedMapProperty.cypherType shouldEqual CTInteger convertedMapProperty shouldEqual - MapProperty(MapExpression(Map("age" -> IntegerLit(40))), PropertyKey("age")) + MapProperty( + MapExpression(Map("age" -> IntegerLit(40))), + PropertyKey("age") + ) val convertedDateProperty = convert(prop(function("date"), "year")) convertedDateProperty.cypherType shouldEqual CTInteger - convertedDateProperty shouldEqual DateProperty(Date(None),PropertyKey("year")) + convertedDateProperty shouldEqual DateProperty( + Date(None), + PropertyKey("year") + ) - val convertedLocalDateTimeProperty = convert(prop(function("localdatetime"), "year")) + val convertedLocalDateTimeProperty = + convert(prop(function("localdatetime"), "year")) convertedLocalDateTimeProperty.cypherType shouldEqual CTInteger - convertedLocalDateTimeProperty shouldEqual LocalDateTimeProperty(LocalDateTime(None),PropertyKey("year")) + convertedLocalDateTimeProperty shouldEqual LocalDateTimeProperty( + LocalDateTime(None), + PropertyKey("year") + ) - val convertedDurationProperty = convert(prop(function("duration", literal("PT1M")), "minutes")) + val convertedDurationProperty = + convert(prop(function("duration", literal("PT1M")), "minutes")) convertedDurationProperty.cypherType shouldEqual CTInteger - convertedDurationProperty shouldEqual DurationProperty(Duration(StringLit("PT1M")), PropertyKey("minutes")) + convertedDurationProperty shouldEqual DurationProperty( + Duration(StringLit("PT1M")), + PropertyKey("minutes") + ) } it("can convert equals") { - convert(ast.Equals(varFor("STRING"), varFor("STRING_OR_NULL")) _) shouldEqual( + convert( + ast.Equals(varFor("STRING"), varFor("STRING_OR_NULL")) _ + ) shouldEqual ( Equals('STRING, 'STRING_OR_NULL), CTBoolean.nullable ) } @@ -401,14 +490,20 @@ class ExpressionConverterTest extends BaseTestSuite with Neo4jAstTestSupport { val given = parseExpr("NODE:Person:Duck") convert(given) match { case ands @ Ands(inner) => - inner.toSet should equal( Set(HasLabel('NODE, Label("Person")), HasLabel('NODE, Label("Duck")))) + inner.toSet should equal( + Set( + HasLabel('NODE, Label("Person")), + HasLabel('NODE, Label("Duck")) + ) + ) ands.cypherType should equal(CTBoolean) case other => fail(s"Expected an `Ands` Expr, but got `$other`") } } it("can convert single has-labels") { - val given = ast.HasLabels(varFor("NODE"), Seq(ast.LabelName("Person") _)) _ + val given = + ast.HasLabels(varFor("NODE"), Seq(ast.LabelName("Person") _)) _ convert(given) shouldEqual HasLabel('NODE, Label("Person")) } } @@ -417,24 +512,35 @@ class ExpressionConverterTest extends BaseTestSuite with Neo4jAstTestSupport { val given = ast.Ands( Set( ast.HasLabels(varFor("NODE"), Seq(ast.LabelName("Person") _)) _, - ast.Equals(prop("NODE", "name"), ast.StringLiteral("Mats") _) _)) _ + ast.Equals(prop("NODE", "name"), ast.StringLiteral("Mats") _) _ + ) + ) _ convert(given) match { case ands @ Ands(inner) => - inner.toSet should equal( Set(HasLabel('NODE, Label("Person")), Equals(ElementProperty('NODE, PropertyKey("name"))(CTAnyMaterial), StringLit("Mats")))) + inner.toSet should equal( + Set( + HasLabel('NODE, Label("Person")), + Equals( + ElementProperty('NODE, PropertyKey("name"))(CTAnyMaterial), + StringLit("Mats") + ) + ) + ) ands.cypherType should equal(CTBoolean) case other => fail(s"Expected an `Ands` Expr, but got `$other`") } } it("can convert negation") { - val given = ast.Not(ast.HasLabels(varFor("NODE"), Seq(ast.LabelName("Person") _)) _) _ + val given = + ast.Not(ast.HasLabels(varFor("NODE"), Seq(ast.LabelName("Person") _)) _) _ convert(given) shouldEqual Not(HasLabel('NODE, Label("Person"))) } it("can convert id function") { - convert("id(REL_OR_NULL)") shouldEqual( + convert("id(REL_OR_NULL)") shouldEqual ( Id('REL_OR_NULL), CTIdentity.nullable ) } @@ -442,7 +548,12 @@ class ExpressionConverterTest extends BaseTestSuite with Neo4jAstTestSupport { describe("list comprehension") { val intVar = LambdaVar("x")(CTInteger) it("can convert list comprehension with static mapping") { - convert("[x IN [1,2] | 1]") shouldEqual ListComprehension(intVar, None, Some(IntegerLit(1)), ListLit(List(IntegerLit(1), IntegerLit(2)))) + convert("[x IN [1,2] | 1]") shouldEqual ListComprehension( + intVar, + None, + Some(IntegerLit(1)), + ListLit(List(IntegerLit(1), IntegerLit(2))) + ) } it("can convert list comprehension with unary mapping") { @@ -450,28 +561,41 @@ class ExpressionConverterTest extends BaseTestSuite with Neo4jAstTestSupport { } it("can convert list comprehension with 2 var-calls") { - convert("[x IN [1,2] | x + x * 2]") shouldEqual ListComprehension(intVar, None, Some(Add(intVar, Multiply(intVar, IntegerLit(2)))), ListLit(List(IntegerLit(1), IntegerLit(2)))) + convert("[x IN [1,2] | x + x * 2]") shouldEqual ListComprehension( + intVar, + None, + Some(Add(intVar, Multiply(intVar, IntegerLit(2)))), + ListLit(List(IntegerLit(1), IntegerLit(2))) + ) } it("can convert list comprehension with inner predicate") { - convert("[x IN [1,2] WHERE x < 1 | 1]") shouldEqual ListComprehension(intVar, Some(LessThan(intVar, IntegerLit(1))), Some(IntegerLit(1)), ListLit(List(IntegerLit(1), IntegerLit(2)))) + convert("[x IN [1,2] WHERE x < 1 | 1]") shouldEqual ListComprehension( + intVar, + Some(LessThan(intVar, IntegerLit(1))), + Some(IntegerLit(1)), + ListLit(List(IntegerLit(1), IntegerLit(2))) + ) } } describe("list-access functions") { it("can convert tail()") { - convert("tail([1])") shouldEqual ListSliceFrom(ListLit(List(IntegerLit(1))), IntegerLit(1)) + convert("tail([1])") shouldEqual ListSliceFrom( + ListLit(List(IntegerLit(1))), + IntegerLit(1) + ) } - it("can convert head()"){ + it("can convert head()") { convert("head([1])") shouldEqual Head(ListLit(List(IntegerLit(1)))) } - it("can convert last()"){ + it("can convert last()") { convert("last([1])") shouldEqual Last(ListLit(List(IntegerLit(1)))) } - it("cannot convert list-access functions with non-list argument"){ - a[NoSuitableSignatureForExpr] shouldBe thrownBy(convert("head(1)")) - a[NoSuitableSignatureForExpr] shouldBe thrownBy(convert("tail(1)")) - a[NoSuitableSignatureForExpr] shouldBe thrownBy(convert("last(1)")) + it("cannot convert list-access functions with non-list argument") { + a[NoSuitableSignatureForExpr] shouldBe thrownBy(convert("head(1)")) + a[NoSuitableSignatureForExpr] shouldBe thrownBy(convert("tail(1)")) + a[NoSuitableSignatureForExpr] shouldBe thrownBy(convert("last(1)")) } } @@ -479,19 +603,35 @@ class ExpressionConverterTest extends BaseTestSuite with Neo4jAstTestSupport { val intVar = LambdaVar("x")(CTInteger) it("can convert none()") { - convert("none(x IN [1] WHERE x < 1)") shouldEqual ListNone(intVar, LessThan(intVar, IntegerLit(1)), ListLit(List(IntegerLit(1)))) + convert("none(x IN [1] WHERE x < 1)") shouldEqual ListNone( + intVar, + LessThan(intVar, IntegerLit(1)), + ListLit(List(IntegerLit(1))) + ) } it("can convert all()") { - convert("all(x IN [1] WHERE x < 1)") shouldEqual ListAll(intVar, LessThan(intVar, IntegerLit(1)), ListLit(List(IntegerLit(1)))) + convert("all(x IN [1] WHERE x < 1)") shouldEqual ListAll( + intVar, + LessThan(intVar, IntegerLit(1)), + ListLit(List(IntegerLit(1))) + ) } it("can convert any()") { - convert("any(x IN [1] WHERE x < 1)") shouldEqual ListAny(intVar, LessThan(intVar, IntegerLit(1)), ListLit(List(IntegerLit(1)))) + convert("any(x IN [1] WHERE x < 1)") shouldEqual ListAny( + intVar, + LessThan(intVar, IntegerLit(1)), + ListLit(List(IntegerLit(1))) + ) } it("can convert single()") { - convert("single(x IN [1] WHERE x < 1)") shouldEqual ListSingle(intVar, LessThan(intVar, IntegerLit(1)), ListLit(List(IntegerLit(1)))) + convert("single(x IN [1] WHERE x < 1)") shouldEqual ListSingle( + intVar, + LessThan(intVar, IntegerLit(1)), + ListLit(List(IntegerLit(1))) + ) } } diff --git a/okapi-ir/src/test/scala/org/opencypher/okapi/ir/impl/IrBuilderTest.scala b/okapi-ir/src/test/scala/org/opencypher/okapi/ir/impl/IrBuilderTest.scala index b1b616aba9..73ac8f9e1a 100644 --- a/okapi-ir/src/test/scala/org/opencypher/okapi/ir/impl/IrBuilderTest.scala +++ b/okapi-ir/src/test/scala/org/opencypher/okapi/ir/impl/IrBuilderTest.scala @@ -90,7 +90,9 @@ class IrBuilderTest extends IrTestSuite { clones.keys.size should equal(1) val (b, a) = clones.head a should equal(NodeVar("a")()) - a.asInstanceOf[Var].cypherType.graph should equal(Some(testGraph.qualifiedGraphName)) + a.asInstanceOf[Var].cypherType.graph should equal( + Some(testGraph.qualifiedGraphName) + ) b.cypherType.graph should equal(Some(qgn)) case _ => fail("no matching graph result found") } @@ -105,7 +107,9 @@ class IrBuilderTest extends IrTestSuite { query.asCypherQuery().model.result match { case GraphResultBlock(_, IRPatternGraph(_, schema, _, _, _, _)) => - schema should equal(PropertyGraphSchema.empty.withNodePropertyKeys("A")()) + schema should equal( + PropertyGraphSchema.empty.withNodePropertyKeys("A")() + ) case _ => fail("no matching graph result found") } } @@ -120,7 +124,11 @@ class IrBuilderTest extends IrTestSuite { query.asCypherQuery().model.result match { case GraphResultBlock(_, IRPatternGraph(_, schema, _, _, _, _)) => - schema should equal(PropertyGraphSchema.empty.withNodePropertyKeys("A")().withNodePropertyKeys("B", "C")()) + schema should equal( + PropertyGraphSchema.empty + .withNodePropertyKeys("A")() + .withNodePropertyKeys("B", "C")() + ) case _ => fail("no matching graph result found") } } @@ -135,7 +143,11 @@ class IrBuilderTest extends IrTestSuite { query.asCypherQuery().model.result match { case GraphResultBlock(_, IRPatternGraph(_, schema, _, _, _, _)) => - schema should equal(PropertyGraphSchema.empty.withNodePropertyKeys("A", "D")().withNodePropertyKeys("B", "C")()) + schema should equal( + PropertyGraphSchema.empty + .withNodePropertyKeys("A", "D")() + .withNodePropertyKeys("B", "C")() + ) case _ => fail("no matching graph result found") } } @@ -149,12 +161,16 @@ class IrBuilderTest extends IrTestSuite { query.asCypherQuery().model.result match { case GraphResultBlock(_, IRPatternGraph(_, schema, _, _, _, _)) => - schema should equal(PropertyGraphSchema.empty.withNodePropertyKeys("A", "B", "C")()) + schema should equal( + PropertyGraphSchema.empty.withNodePropertyKeys("A", "B", "C")() + ) case _ => fail("no matching graph result found") } } - it("computes a pattern graph schema correctly - setting 2 different label combinations with overlap") { + it( + "computes a pattern graph schema correctly - setting 2 different label combinations with overlap" + ) { val query = """ |CONSTRUCT @@ -164,12 +180,18 @@ class IrBuilderTest extends IrTestSuite { query.asCypherQuery().model.result match { case GraphResultBlock(_, IRPatternGraph(_, schema, _, _, _, _)) => - schema should equal(PropertyGraphSchema.empty.withNodePropertyKeys("A", "B")().withNodePropertyKeys("A", "C")()) + schema should equal( + PropertyGraphSchema.empty + .withNodePropertyKeys("A", "B")() + .withNodePropertyKeys("A", "C")() + ) case _ => fail("no matching graph result found") } } - it("computes a pattern graph schema correctly - setting 2 equal label combinations") { + it( + "computes a pattern graph schema correctly - setting 2 equal label combinations" + ) { val query = """ |CONSTRUCT @@ -179,7 +201,9 @@ class IrBuilderTest extends IrTestSuite { query.asCypherQuery().model.result match { case GraphResultBlock(_, IRPatternGraph(_, schema, _, _, _, _)) => - schema should equal(PropertyGraphSchema.empty.withNodePropertyKeys("A", "B")()) + schema should equal( + PropertyGraphSchema.empty.withNodePropertyKeys("A", "B")() + ) case _ => fail("no matching graph result found") } } @@ -193,12 +217,18 @@ class IrBuilderTest extends IrTestSuite { query.asCypherQuery().model.result match { case GraphResultBlock(_, IRPatternGraph(_, schema, _, _, _, _)) => - schema should equal(PropertyGraphSchema.empty.withNodePropertyKeys("A")("name" -> CTString)) + schema should equal( + PropertyGraphSchema.empty.withNodePropertyKeys("A")( + "name" -> CTString + ) + ) case _ => fail("no matching graph result found") } } - it("computes a pattern graph schema correctly - setting a node property and a label combination") { + it( + "computes a pattern graph schema correctly - setting a node property and a label combination" + ) { val query = """ |CONSTRUCT @@ -207,7 +237,11 @@ class IrBuilderTest extends IrTestSuite { query.asCypherQuery().model.result match { case GraphResultBlock(_, IRPatternGraph(_, schema, _, _, _, _)) => - schema should equal(PropertyGraphSchema.empty.withNodePropertyKeys("A", "B")("name" -> CTString)) + schema should equal( + PropertyGraphSchema.empty.withNodePropertyKeys("A", "B")( + "name" -> CTString + ) + ) case _ => fail("no matching graph result found") } } @@ -221,7 +255,11 @@ class IrBuilderTest extends IrTestSuite { query.asCypherQuery().model.result match { case GraphResultBlock(_, IRPatternGraph(_, schema, _, _, _, _)) => - schema should equal(PropertyGraphSchema.empty.withNodePropertyKeys(Set.empty[String]).withRelationshipPropertyKeys("R")("level" -> CTString)) + schema should equal( + PropertyGraphSchema.empty + .withNodePropertyKeys(Set.empty[String]) + .withRelationshipPropertyKeys("R")("level" -> CTString) + ) case _ => fail("no matching graph result found") } } @@ -235,7 +273,12 @@ class IrBuilderTest extends IrTestSuite { query.asCypherQuery().model.result match { case GraphResultBlock(_, IRPatternGraph(_, schema, _, _, _, _)) => - schema should equal(PropertyGraphSchema.empty.withNodePropertyKeys(Set("A"), PropertyKeys("category" -> CTString, "ports" -> CTInteger))) + schema should equal( + PropertyGraphSchema.empty.withNodePropertyKeys( + Set("A"), + PropertyKeys("category" -> CTString, "ports" -> CTInteger) + ) + ) case _ => fail("no matching graph result found") } } @@ -261,7 +304,9 @@ class IrBuilderTest extends IrTestSuite { } } - it("computes a pattern graph schema correctly - for copied nodes with unspecified labels") { + it( + "computes a pattern graph schema correctly - for copied nodes with unspecified labels" + ) { val graphName = GraphName("input") val inputSchema = PropertyGraphSchema.empty @@ -283,7 +328,9 @@ class IrBuilderTest extends IrTestSuite { } } - it("computes a pattern graph schema correctly - for copied nodes with additional Label") { + it( + "computes a pattern graph schema correctly - for copied nodes with additional Label" + ) { val graphName = GraphName("input") val inputSchema = PropertyGraphSchema.empty @@ -299,13 +346,20 @@ class IrBuilderTest extends IrTestSuite { query.asCypherQuery(graphName -> inputSchema).model.result match { case GraphResultBlock(_, IRPatternGraph(_, schema, _, _, _, _)) => - schema should equal(PropertyGraphSchema.empty - .withNodePropertyKeys("A", "B")("category" -> CTString, "ports" -> CTInteger)) + schema should equal( + PropertyGraphSchema.empty + .withNodePropertyKeys("A", "B")( + "category" -> CTString, + "ports" -> CTInteger + ) + ) case _ => fail("no matching graph result found") } } - it("computes a pattern graph schema correctly - for copied unspecified nodes with additional Label") { + it( + "computes a pattern graph schema correctly - for copied unspecified nodes with additional Label" + ) { val graphName = GraphName("input") val inputSchema = PropertyGraphSchema.empty @@ -322,15 +376,24 @@ class IrBuilderTest extends IrTestSuite { query.asCypherQuery(graphName -> inputSchema).model.result match { case GraphResultBlock(_, IRPatternGraph(_, schema, _, _, _, _)) => - schema should equal(PropertyGraphSchema.empty - .withNodePropertyKeys("A", "C")("category" -> CTString, "ports" -> CTInteger) - .withNodePropertyKeys("B", "C")("foo" -> CTString, "bar" -> CTInteger) + schema should equal( + PropertyGraphSchema.empty + .withNodePropertyKeys("A", "C")( + "category" -> CTString, + "ports" -> CTInteger + ) + .withNodePropertyKeys("B", "C")( + "foo" -> CTString, + "bar" -> CTInteger + ) ) case _ => fail("no matching graph result found") } } - it("computes a pattern graph schema correctly - for copied nodes with additional properties") { + it( + "computes a pattern graph schema correctly - for copied nodes with additional properties" + ) { val graphName = GraphName("input") val inputSchema = PropertyGraphSchema.empty @@ -346,13 +409,21 @@ class IrBuilderTest extends IrTestSuite { query.asCypherQuery(graphName -> inputSchema).model.result match { case GraphResultBlock(_, IRPatternGraph(_, schema, _, _, _, _)) => - schema should equal(PropertyGraphSchema.empty - .withNodePropertyKeys("A")("category" -> CTString, "ports" -> CTInteger, "memory" -> CTString)) + schema should equal( + PropertyGraphSchema.empty + .withNodePropertyKeys("A")( + "category" -> CTString, + "ports" -> CTInteger, + "memory" -> CTString + ) + ) case _ => fail("no matching graph result found") } } - it("computes a pattern graph schema correctly - for copied nodes with conflicting properties") { + it( + "computes a pattern graph schema correctly - for copied nodes with conflicting properties" + ) { val graphName = GraphName("input") val inputSchema = PropertyGraphSchema.empty @@ -368,18 +439,28 @@ class IrBuilderTest extends IrTestSuite { query.asCypherQuery(graphName -> inputSchema).model.result match { case GraphResultBlock(_, IRPatternGraph(_, schema, _, _, _, _)) => - schema should equal(PropertyGraphSchema.empty - .withNodePropertyKeys("A")("category" -> CTInteger, "ports" -> CTInteger)) + schema should equal( + PropertyGraphSchema.empty + .withNodePropertyKeys("A")( + "category" -> CTInteger, + "ports" -> CTInteger + ) + ) case _ => fail("no matching graph result found") } } - it("computes a pattern graph schema correctly - for copied unspecified nodes with conflicting properties") { + it( + "computes a pattern graph schema correctly - for copied unspecified nodes with conflicting properties" + ) { val graphName = GraphName("input") val inputSchema = PropertyGraphSchema.empty .withNodePropertyKeys("A")("category" -> CTString, "ports" -> CTInteger) - .withNodePropertyKeys("B")("category" -> CTInteger, "ports" -> CTInteger) + .withNodePropertyKeys("B")( + "category" -> CTInteger, + "ports" -> CTInteger + ) val query = """ @@ -391,19 +472,32 @@ class IrBuilderTest extends IrTestSuite { query.asCypherQuery(graphName -> inputSchema).model.result match { case GraphResultBlock(_, IRPatternGraph(_, schema, _, _, _, _)) => - schema should equal(PropertyGraphSchema.empty - .withNodePropertyKeys("A")("category" -> CTInteger, "ports" -> CTInteger) - .withNodePropertyKeys("B")("category" -> CTInteger, "ports" -> CTInteger)) + schema should equal( + PropertyGraphSchema.empty + .withNodePropertyKeys("A")( + "category" -> CTInteger, + "ports" -> CTInteger + ) + .withNodePropertyKeys("B")( + "category" -> CTInteger, + "ports" -> CTInteger + ) + ) case _ => fail("no matching graph result found") } } - it("computes a pattern graph schema correctly - for copied relationships") { + it( + "computes a pattern graph schema correctly - for copied relationships" + ) { val graphName = GraphName("input") val inputSchema = PropertyGraphSchema.empty .withNodePropertyKeys()() - .withRelationshipPropertyKeys("A")("category" -> CTString, "ports" -> CTInteger) + .withRelationshipPropertyKeys("A")( + "category" -> CTString, + "ports" -> CTInteger + ) val query = """ @@ -420,13 +514,21 @@ class IrBuilderTest extends IrTestSuite { } } - it("computes a pattern graph schema correctly - for copied relationships with unspecified type") { + it( + "computes a pattern graph schema correctly - for copied relationships with unspecified type" + ) { val graphName = GraphName("input") val inputSchema = PropertyGraphSchema.empty .withNodePropertyKeys()() - .withRelationshipPropertyKeys("A")("category" -> CTString, "ports" -> CTInteger) - .withRelationshipPropertyKeys("B")("category" -> CTString, "ports" -> CTInteger) + .withRelationshipPropertyKeys("A")( + "category" -> CTString, + "ports" -> CTInteger + ) + .withRelationshipPropertyKeys("B")( + "category" -> CTString, + "ports" -> CTInteger + ) val query = """ @@ -443,14 +545,25 @@ class IrBuilderTest extends IrTestSuite { } } - it("computes a pattern graph schema correctly - for copied relationships with alternative types") { + it( + "computes a pattern graph schema correctly - for copied relationships with alternative types" + ) { val graphName = GraphName("input") val inputSchema = PropertyGraphSchema.empty .withNodePropertyKeys()() - .withRelationshipPropertyKeys("A")("category" -> CTString, "ports" -> CTInteger) - .withRelationshipPropertyKeys("B")("category" -> CTString, "ports" -> CTInteger) - .withRelationshipPropertyKeys("C")("foo" -> CTString, "bar" -> CTInteger) + .withRelationshipPropertyKeys("A")( + "category" -> CTString, + "ports" -> CTInteger + ) + .withRelationshipPropertyKeys("B")( + "category" -> CTString, + "ports" -> CTInteger + ) + .withRelationshipPropertyKeys("C")( + "foo" -> CTString, + "bar" -> CTInteger + ) val query = """ @@ -462,21 +575,33 @@ class IrBuilderTest extends IrTestSuite { query.asCypherQuery(graphName -> inputSchema).model.result match { case GraphResultBlock(_, IRPatternGraph(_, schema, _, _, _, _)) => - schema should equal(PropertyGraphSchema.empty - .withNodePropertyKeys()() - .withRelationshipPropertyKeys("A")("category" -> CTString, "ports" -> CTInteger) - .withRelationshipPropertyKeys("B")("category" -> CTString, "ports" -> CTInteger) + schema should equal( + PropertyGraphSchema.empty + .withNodePropertyKeys()() + .withRelationshipPropertyKeys("A")( + "category" -> CTString, + "ports" -> CTInteger + ) + .withRelationshipPropertyKeys("B")( + "category" -> CTString, + "ports" -> CTInteger + ) ) case _ => fail("no matching graph result found") } } - it("computes a pattern graph schema correctly - for copied relationships with different type") { + it( + "computes a pattern graph schema correctly - for copied relationships with different type" + ) { val graphName = GraphName("input") val inputSchema = PropertyGraphSchema.empty .withNodePropertyKeys()() - .withRelationshipPropertyKeys("A")("category" -> CTString, "ports" -> CTInteger) + .withRelationshipPropertyKeys("A")( + "category" -> CTString, + "ports" -> CTInteger + ) val query = """ @@ -488,15 +613,21 @@ class IrBuilderTest extends IrTestSuite { query.asCypherQuery(graphName -> inputSchema).model.result match { case GraphResultBlock(_, IRPatternGraph(_, schema, _, _, _, _)) => - schema should equal(PropertyGraphSchema.empty - .withNodePropertyKeys()() - .withRelationshipPropertyKeys("B")("category" -> CTString, "ports" -> CTInteger) + schema should equal( + PropertyGraphSchema.empty + .withNodePropertyKeys()() + .withRelationshipPropertyKeys("B")( + "category" -> CTString, + "ports" -> CTInteger + ) ) case _ => fail("no matching graph result found") } } - it("computes a pattern graph schema correctly - for copied relationships with unspecified types and different type") { + it( + "computes a pattern graph schema correctly - for copied relationships with unspecified types and different type" + ) { val graphName = GraphName("input") val inputSchema = PropertyGraphSchema.empty @@ -514,20 +645,29 @@ class IrBuilderTest extends IrTestSuite { query.asCypherQuery(graphName -> inputSchema).model.result match { case GraphResultBlock(_, IRPatternGraph(_, schema, _, _, _, _)) => - schema should equal(PropertyGraphSchema.empty - .withNodePropertyKeys()() - .withRelationshipPropertyKeys("C")("category" -> CTString.nullable, "ports" -> CTInteger.nullable) + schema should equal( + PropertyGraphSchema.empty + .withNodePropertyKeys()() + .withRelationshipPropertyKeys("C")( + "category" -> CTString.nullable, + "ports" -> CTInteger.nullable + ) ) case _ => fail("no matching graph result found") } } - it("computes a pattern graph schema correctly - for copied relationships with additional properties") { + it( + "computes a pattern graph schema correctly - for copied relationships with additional properties" + ) { val graphName = GraphName("input") val inputSchema = PropertyGraphSchema.empty .withNodePropertyKeys()() - .withRelationshipPropertyKeys("A")("category" -> CTString, "ports" -> CTInteger) + .withRelationshipPropertyKeys("A")( + "category" -> CTString, + "ports" -> CTInteger + ) val query = """ @@ -539,19 +679,30 @@ class IrBuilderTest extends IrTestSuite { query.asCypherQuery(graphName -> inputSchema).model.result match { case GraphResultBlock(_, IRPatternGraph(_, schema, _, _, _, _)) => - schema should equal(PropertyGraphSchema.empty - .withNodePropertyKeys()() - .withRelationshipPropertyKeys("A")("category" -> CTString, "ports" -> CTInteger, "memory" -> CTString)) + schema should equal( + PropertyGraphSchema.empty + .withNodePropertyKeys()() + .withRelationshipPropertyKeys("A")( + "category" -> CTString, + "ports" -> CTInteger, + "memory" -> CTString + ) + ) case _ => fail("no matching graph result found") } } - it("computes a pattern graph schema correctly - for copied relationships with conflicting properties") { + it( + "computes a pattern graph schema correctly - for copied relationships with conflicting properties" + ) { val graphName = GraphName("input") val inputSchema = PropertyGraphSchema.empty .withNodePropertyKeys()() - .withRelationshipPropertyKeys("A")("category" -> CTString, "ports" -> CTInteger) + .withRelationshipPropertyKeys("A")( + "category" -> CTString, + "ports" -> CTInteger + ) val query = """ @@ -563,20 +714,33 @@ class IrBuilderTest extends IrTestSuite { query.asCypherQuery(graphName -> inputSchema).model.result match { case GraphResultBlock(_, IRPatternGraph(_, schema, _, _, _, _)) => - schema should equal(PropertyGraphSchema.empty - .withNodePropertyKeys()() - .withRelationshipPropertyKeys("A")("category" -> CTInteger, "ports" -> CTInteger)) + schema should equal( + PropertyGraphSchema.empty + .withNodePropertyKeys()() + .withRelationshipPropertyKeys("A")( + "category" -> CTInteger, + "ports" -> CTInteger + ) + ) case _ => fail("no matching graph result found") } } - it("computes a pattern graph schema correctly - for copied unspecified relationships with conflicting properties") { + it( + "computes a pattern graph schema correctly - for copied unspecified relationships with conflicting properties" + ) { val graphName = GraphName("input") val inputSchema = PropertyGraphSchema.empty .withNodePropertyKeys()() - .withRelationshipPropertyKeys("A")("category" -> CTString, "ports" -> CTInteger) - .withRelationshipPropertyKeys("B")("category" -> CTInteger, "ports" -> CTInteger) + .withRelationshipPropertyKeys("A")( + "category" -> CTString, + "ports" -> CTInteger + ) + .withRelationshipPropertyKeys("B")( + "category" -> CTInteger, + "ports" -> CTInteger + ) val query = """ @@ -588,15 +752,25 @@ class IrBuilderTest extends IrTestSuite { query.asCypherQuery(graphName -> inputSchema).model.result match { case GraphResultBlock(_, IRPatternGraph(_, schema, _, _, _, _)) => - schema should equal(PropertyGraphSchema.empty - .withNodePropertyKeys()() - .withRelationshipPropertyKeys("A")("category" -> CTInteger, "ports" -> CTInteger) - .withRelationshipPropertyKeys("B")("category" -> CTInteger, "ports" -> CTInteger)) + schema should equal( + PropertyGraphSchema.empty + .withNodePropertyKeys()() + .withRelationshipPropertyKeys("A")( + "category" -> CTInteger, + "ports" -> CTInteger + ) + .withRelationshipPropertyKeys("B")( + "category" -> CTInteger, + "ports" -> CTInteger + ) + ) case _ => fail("no matching graph result found") } } - it("computes a pattern graph schema correctly - for copied relationships with unspecified types, different type and updated properties") { + it( + "computes a pattern graph schema correctly - for copied relationships with unspecified types, different type and updated properties" + ) { val graphName = GraphName("input") val inputSchema = PropertyGraphSchema.empty @@ -614,21 +788,34 @@ class IrBuilderTest extends IrTestSuite { query.asCypherQuery(graphName -> inputSchema).model.result match { case GraphResultBlock(_, IRPatternGraph(_, schema, _, _, _, _)) => - schema should equal(PropertyGraphSchema.empty - .withNodePropertyKeys()() - .withRelationshipPropertyKeys("C")("category" -> CTString.nullable, "ports" -> CTInteger.nullable, "memory" -> CTString) + schema should equal( + PropertyGraphSchema.empty + .withNodePropertyKeys()() + .withRelationshipPropertyKeys("C")( + "category" -> CTString.nullable, + "ports" -> CTInteger.nullable, + "memory" -> CTString + ) ) case _ => fail("no matching graph result found") } } - it("computes a pattern graph schema correctly - for copied relationships with alternative types and additional property") { + it( + "computes a pattern graph schema correctly - for copied relationships with alternative types and additional property" + ) { val graphName = GraphName("input") val inputSchema = PropertyGraphSchema.empty .withNodePropertyKeys()() - .withRelationshipPropertyKeys("A")("category" -> CTString, "ports" -> CTInteger) - .withRelationshipPropertyKeys("B")("category" -> CTString, "ports" -> CTInteger) + .withRelationshipPropertyKeys("A")( + "category" -> CTString, + "ports" -> CTInteger + ) + .withRelationshipPropertyKeys("B")( + "category" -> CTString, + "ports" -> CTInteger + ) val query = """ @@ -640,16 +827,27 @@ class IrBuilderTest extends IrTestSuite { query.asCypherQuery(graphName -> inputSchema).model.result match { case GraphResultBlock(_, IRPatternGraph(_, schema, _, _, _, _)) => - schema should equal(PropertyGraphSchema.empty - .withNodePropertyKeys()() - .withRelationshipPropertyKeys("A")("category" -> CTString, "ports" -> CTInteger, "memory" -> CTString) - .withRelationshipPropertyKeys("B")("category" -> CTString, "ports" -> CTInteger, "memory" -> CTString) + schema should equal( + PropertyGraphSchema.empty + .withNodePropertyKeys()() + .withRelationshipPropertyKeys("A")( + "category" -> CTString, + "ports" -> CTInteger, + "memory" -> CTString + ) + .withRelationshipPropertyKeys("B")( + "category" -> CTString, + "ports" -> CTInteger, + "memory" -> CTString + ) ) case _ => fail("no matching graph result found") } } - it("throws an error when a relationships is cloned that is not part of a new pattern") { + it( + "throws an error when a relationships is cloned that is not part of a new pattern" + ) { val query = """ |MATCH ()-[r]->() @@ -661,7 +859,9 @@ class IrBuilderTest extends IrTestSuite { intercept[UnsupportedOperationException](query.asCypherQuery().model) } - it("fails cloning relationships with aliased and newly constructed start and end nodes") { + it( + "fails cloning relationships with aliased and newly constructed start and end nodes" + ) { val query = """ |MATCH (:FOO)-[r:REL]->() @@ -671,7 +871,9 @@ class IrBuilderTest extends IrTestSuite { |RETURN GRAPH """.stripMargin - an[IllegalArgumentException] shouldBe thrownBy(query.asCypherQuery().model.result) + an[IllegalArgumentException] shouldBe thrownBy( + query.asCypherQuery().model.result + ) } it("succeeds cloning relationships with alias #1") { @@ -700,7 +902,9 @@ class IrBuilderTest extends IrTestSuite { query.asCypherQuery().model.result } - it("fails cloning relationships with newly constructed start and end nodes") { + it( + "fails cloning relationships with newly constructed start and end nodes" + ) { val query = """ |MATCH (:FOO)-[r:REL]->() @@ -710,7 +914,9 @@ class IrBuilderTest extends IrTestSuite { |RETURN GRAPH """.stripMargin - an[IllegalArgumentException] shouldBe thrownBy(query.asCypherQuery().model.result) + an[IllegalArgumentException] shouldBe thrownBy( + query.asCypherQuery().model.result + ) } it("fails cloning of relationships with newly constructed end nodes") { @@ -722,7 +928,9 @@ class IrBuilderTest extends IrTestSuite { |RETURN GRAPH """.stripMargin - an[IllegalArgumentException] shouldBe thrownBy(query.asCypherQuery().model.result) + an[IllegalArgumentException] shouldBe thrownBy( + query.asCypherQuery().model.result + ) } it("fails cloning of relationships with newly constructed start nodes") { @@ -734,10 +942,14 @@ class IrBuilderTest extends IrTestSuite { |RETURN GRAPH """.stripMargin - an[IllegalArgumentException] shouldBe thrownBy(query.asCypherQuery().model.result) + an[IllegalArgumentException] shouldBe thrownBy( + query.asCypherQuery().model.result + ) } - it("succeeds pattern with cloned relationship and new unnamed relationship #1") { + it( + "succeeds pattern with cloned relationship and new unnamed relationship #1" + ) { val query = """ |MATCH (a)-[r:REL]->(b) @@ -749,7 +961,9 @@ class IrBuilderTest extends IrTestSuite { query.asCypherQuery().model.result } - it("succeeds pattern with cloned relationship and new unnamed relationship #2") { + it( + "succeeds pattern with cloned relationship and new unnamed relationship #2" + ) { val query = """ |MATCH (a)-[r:REL]->(b)-->(c) @@ -768,7 +982,9 @@ class IrBuilderTest extends IrTestSuite { | CREATE (b)-[e]->(b) |RETURN GRAPH""".stripMargin - an[IllegalArgumentException] shouldBe thrownBy(query.asCypherQuery().model.result) + an[IllegalArgumentException] shouldBe thrownBy( + query.asCypherQuery().model.result + ) } it("fails cloning rel with false src and target nodes #1") { @@ -778,7 +994,9 @@ class IrBuilderTest extends IrTestSuite { | CREATE (b)-[e]->(a) |RETURN GRAPH""".stripMargin - an[IllegalArgumentException] shouldBe thrownBy(query.asCypherQuery().model.result) + an[IllegalArgumentException] shouldBe thrownBy( + query.asCypherQuery().model.result + ) } it("fails cloning rel with false src and target nodes #2") { @@ -788,7 +1006,9 @@ class IrBuilderTest extends IrTestSuite { |CONSTRUCT | CREATE (a)-[e]->(b) |RETURN GRAPH""".stripMargin - an[IllegalArgumentException] shouldBe thrownBy(query.asCypherQuery().model.result) + an[IllegalArgumentException] shouldBe thrownBy( + query.asCypherQuery().model.result + ) } } @@ -801,7 +1021,9 @@ class IrBuilderTest extends IrTestSuite { | CREATE (a)-[e]->(b) |RETURN GRAPH""".stripMargin - an[IllegalArgumentException] shouldBe thrownBy(query.asCypherQuery().model.result) + an[IllegalArgumentException] shouldBe thrownBy( + query.asCypherQuery().model.result + ) } it("WITH using all pattern-elements") { @@ -839,7 +1061,7 @@ class IrBuilderTest extends IrTestSuite { case GraphResultBlock(_, IRPatternGraph(_, _, clones, creates, _, _)) => clones.isEmpty shouldBe true creates.fields.size shouldBe 3 - case _=> fail("no matching graph result found") + case _ => fail("no matching graph result found") } } @@ -863,7 +1085,6 @@ class IrBuilderTest extends IrTestSuite { val inputSchemaB = PropertyGraphSchema.empty .withNodePropertyKeys("A")("category" -> CTString, "ports" -> CTInteger) - val query = """|FROM input1 |MATCH (a)-[e]->(b) @@ -873,57 +1094,67 @@ class IrBuilderTest extends IrTestSuite { | CREATE (a)-[e]->(x) |RETURN GRAPH""".stripMargin - an[IllegalArgumentException] shouldBe thrownBy(query.asCypherQuery(graphNameA -> inputSchemaA, graphNameB -> inputSchemaB).model.result) + an[IllegalArgumentException] shouldBe thrownBy( + query + .asCypherQuery(graphNameA -> inputSchemaA, graphNameB -> inputSchemaB) + .model + .result + ) } } describe("parsing CypherQuery") { test("match node and return it") { - "MATCH (a:Person) RETURN a".asCypherQuery().model.ensureThat { - (model, _) => - val loadBlock = model.findExactlyOne { - case NoWhereBlock(s@SourceBlock(_)) => - s.binds.fields shouldBe empty + "MATCH (a:Person) RETURN a".asCypherQuery().model.ensureThat { (model, _) => + val loadBlock = + model.findExactlyOne { case NoWhereBlock(s @ SourceBlock(_)) => + s.binds.fields shouldBe empty } - val matchBlock = model.findExactlyOne { - case MatchBlock(deps, Pattern(fields, topo, _, _), exprs, _, _) => - deps should equalWithTracing(List(loadBlock)) - fields should equal(Set(toField('a -> CTNode("Person")))) - topo shouldBe empty - exprs should equalWithTracing(Set.empty) + val matchBlock = + model.findExactlyOne { case MatchBlock(deps, Pattern(fields, topo, _, _), exprs, _, _) => + deps should equalWithTracing(List(loadBlock)) + fields should equal(Set(toField('a -> CTNode("Person")))) + topo shouldBe empty + exprs should equalWithTracing(Set.empty) } - val projectBlock = model.findExactlyOne { - case NoWhereBlock(ProjectBlock(deps, Fields(map), _, _, _)) => - deps should equalWithTracing(List(matchBlock)) - map should equal(Map(toField('a) -> toNodeVar('a))) + val projectBlock = + model.findExactlyOne { case NoWhereBlock(ProjectBlock(deps, Fields(map), _, _, _)) => + deps should equalWithTracing(List(matchBlock)) + map should equal(Map(toField('a) -> toNodeVar('a))) } - model.result match { - case NoWhereBlock(TableResultBlock(deps, OrderedFields(List(IRField("a"))), _)) => - deps should equal(List(projectBlock)) - } + model.result match { + case NoWhereBlock( + TableResultBlock(deps, OrderedFields(List(IRField("a"))), _) + ) => + deps should equal(List(projectBlock)) + } - model.dependencies should equalWithTracing( - Set(matchBlock, loadBlock, projectBlock, model.result) - ) + model.dependencies should equalWithTracing( + Set(matchBlock, loadBlock, projectBlock, model.result) + ) } } - it("matches a simple relationship pattern and returns some fields") { - "MATCH (a)-[r]->(b) RETURN b AS otherB, a, r".asCypherQuery().model.ensureThat { - (model, globals) => - val loadBlock = model.findExactlyOne { - case NoWhereBlock(s@SourceBlock(_)) => - s.binds.fields shouldBe empty + "MATCH (a)-[r]->(b) RETURN b AS otherB, a, r" + .asCypherQuery() + .model + .ensureThat { (model, globals) => + val loadBlock = model.findExactlyOne { case NoWhereBlock(s @ SourceBlock(_)) => + s.binds.fields shouldBe empty } val matchBlock = model.findExactlyOne { - case NoWhereBlock(MatchBlock(deps, Pattern(fields, topo, _, _), _, _, _)) => + case NoWhereBlock( + MatchBlock(deps, Pattern(fields, topo, _, _), _, _, _) + ) => deps should equalWithTracing(List(loadBlock)) - fields should equal(Set[IRField]('a -> CTNode, 'b -> CTNode, 'r -> CTRelationship)) + fields should equal( + Set[IRField]('a -> CTNode, 'b -> CTNode, 'r -> CTRelationship) + ) val map = Map(toField('r) -> DirectedRelationship('a, 'b)) topo should equal(map) } @@ -936,30 +1167,39 @@ class IrBuilderTest extends IrTestSuite { toField('a) -> toNodeVar('a), toField('otherB) -> toNodeVar('b), toField('r) -> toRelVar('r) - )) + ) + ) } val resultBlock = model.result.findExactlyOne { - case TableResultBlock(_, OrderedFields(List(IRField("otherB"), IRField("a"), IRField("r"))), _) => + case TableResultBlock( + _, + OrderedFields( + List(IRField("otherB"), IRField("a"), IRField("r")) + ), + _ + ) => } model.dependencies should equalWithTracing( Set(matchBlock, loadBlock, projectBlock, resultBlock) ) - } + } } it("matches node order by name and returns it") { "FROM GRAPH foo MATCH (a:Person) WITH a.name AS name, a.age AS age ORDER BY age RETURN age, name" - .asCypherQuery(GraphName("foo") -> PropertyGraphSchema.empty - .withNodePropertyKeys("Person")( - "name" -> CTString, - "age" -> CTInteger - )).model.ensureThat { - (model, _) => - val loadBlock = model.findExactlyOne { - case NoWhereBlock(s@SourceBlock(_)) => - s.binds.fields shouldBe empty + .asCypherQuery( + GraphName("foo") -> PropertyGraphSchema.empty + .withNodePropertyKeys("Person")( + "name" -> CTString, + "age" -> CTInteger + ) + ) + .model + .ensureThat { (model, _) => + val loadBlock = model.findExactlyOne { case NoWhereBlock(s @ SourceBlock(_)) => + s.binds.fields shouldBe empty } val matchBlock = model.findExactlyOne { @@ -971,59 +1211,84 @@ class IrBuilderTest extends IrTestSuite { } val projectBlock1 = model.findExactlyOne { - case NoWhereBlock(ProjectBlock(deps, Fields(map), _, _, _)) if deps.head == matchBlock => + case NoWhereBlock(ProjectBlock(deps, Fields(map), _, _, _)) + if deps.head == matchBlock => deps should equalWithTracing(List(matchBlock)) map should equal( Map( - toField('name) -> ElementProperty(Var("a")(CTNode), PropertyKey("name"))(CTString), - toField('age) -> ElementProperty(Var("a")(CTNode), PropertyKey("age"))(CTInteger) - )) + toField('name) -> ElementProperty( + Var("a")(CTNode), + PropertyKey("name") + )(CTString), + toField('age) -> ElementProperty( + Var("a")(CTNode), + PropertyKey("age") + )(CTInteger) + ) + ) } val projectBlock2 = model.findExactlyOne { - case NoWhereBlock(ProjectBlock(deps, Fields(map), _, _, _)) if deps.head == projectBlock1 => + case NoWhereBlock(ProjectBlock(deps, Fields(map), _, _, _)) + if deps.head == projectBlock1 => deps should equalWithTracing(List(projectBlock1)) map should equal( Map( toField('age) -> toVar('age), toField('name) -> toVar('name) - )) + ) + ) } val orderByBlock = model.findExactlyOne { - case NoWhereBlock(OrderAndSliceBlock(deps, orderBy, None, None, _)) => + case NoWhereBlock( + OrderAndSliceBlock(deps, orderBy, None, None, _) + ) => val ordered = List(Asc(toVar('age))) orderBy should equal(ordered) deps should equalWithTracing(List(projectBlock2)) } val projectBlock3 = model.findExactlyOne { - case NoWhereBlock(ProjectBlock(deps, Fields(map), _, _, _)) if deps.head == orderByBlock => + case NoWhereBlock(ProjectBlock(deps, Fields(map), _, _, _)) + if deps.head == orderByBlock => deps should equalWithTracing(List(orderByBlock)) map should equal( Map( toField('age) -> toVar('age), toField('name) -> toVar('name) - )) + ) + ) } val resultBlock = model.findExactlyOne { - case TableResultBlock(deps, OrderedFields(List(IRField("age"), IRField("name"))), _) => + case TableResultBlock( + deps, + OrderedFields(List(IRField("age"), IRField("name"))), + _ + ) => deps should equalWithTracing(List(projectBlock3)) } model.dependencies should equalWithTracing( - Set(orderByBlock, projectBlock3, projectBlock2, projectBlock1, matchBlock, loadBlock, resultBlock) + Set( + orderByBlock, + projectBlock3, + projectBlock2, + projectBlock1, + matchBlock, + loadBlock, + resultBlock + ) ) - } + } } } describe("CreateGraphStatement") { it("can parse a CATALOG CREATE GRAPH statement") { - val innerQuery = s"FROM GRAPH ${ - testQualifiedGraphName.toString - } RETURN GRAPH" + val innerQuery = + s"FROM GRAPH ${testQualifiedGraphName.toString} RETURN GRAPH" val query = s""" @@ -1034,8 +1299,12 @@ class IrBuilderTest extends IrTestSuite { val result = query.parseIR[CreateGraphStatement]() - result.innerQuery.model should equalWithTracing(innerQuery.asCypherQuery().model) - result.graph.qualifiedGraphName should equal(QualifiedGraphName(Namespace("session"), GraphName("bar"))) + result.innerQuery.model should equalWithTracing( + innerQuery.asCypherQuery().model + ) + result.graph.qualifiedGraphName should equal( + QualifiedGraphName(Namespace("session"), GraphName("bar")) + ) result.graph.schema should equal(testGraphSchema) } } @@ -1094,7 +1363,11 @@ class IrBuilderTest extends IrTestSuite { query.asCypherQuery().model.result match { case GraphResultBlock(_, IRPatternGraph(_, schema, _, _, _, _)) => - schema should equal(PropertyGraphSchema.empty.withNodePropertyKeys("A")().withNodePropertyKeys("B", "C")()) + schema should equal( + PropertyGraphSchema.empty + .withNodePropertyKeys("A")() + .withNodePropertyKeys("B", "C")() + ) case _ => fail("no matching graph result found") } } @@ -1110,7 +1383,11 @@ class IrBuilderTest extends IrTestSuite { query.asCypherQuery().model.result match { case GraphResultBlock(_, IRPatternGraph(_, schema, _, _, _, _)) => - schema should equal(PropertyGraphSchema.empty.withNodePropertyKeys("A", "D")().withNodePropertyKeys("B", "C")()) + schema should equal( + PropertyGraphSchema.empty + .withNodePropertyKeys("A", "D")() + .withNodePropertyKeys("B", "C")() + ) case _ => fail("no matching graph result found") } } @@ -1126,12 +1403,16 @@ class IrBuilderTest extends IrTestSuite { query.asCypherQuery().model.result match { case GraphResultBlock(_, IRPatternGraph(_, schema, _, _, _, _)) => - schema should equal(PropertyGraphSchema.empty.withNodePropertyKeys("A", "B", "C")()) + schema should equal( + PropertyGraphSchema.empty.withNodePropertyKeys("A", "B", "C")() + ) case _ => fail("no matching graph result found") } } - it("computes a pattern graph schema correctly - 2 creates and 2 sets on same label") { + it( + "computes a pattern graph schema correctly - 2 creates and 2 sets on same label" + ) { val query = """ |CONSTRUCT @@ -1143,12 +1424,18 @@ class IrBuilderTest extends IrTestSuite { query.asCypherQuery().model.result match { case GraphResultBlock(_, IRPatternGraph(_, schema, _, _, _, _)) => - schema should equal(PropertyGraphSchema.empty.withNodePropertyKeys("A", "B")().withNodePropertyKeys("A", "C")()) + schema should equal( + PropertyGraphSchema.empty + .withNodePropertyKeys("A", "B")() + .withNodePropertyKeys("A", "C")() + ) case _ => fail("no matching graph result found") } } - it("computes a pattern graph schema correctly - 2 creates and 2 sets on different labels") { + it( + "computes a pattern graph schema correctly - 2 creates and 2 sets on different labels" + ) { val query = """ |CONSTRUCT @@ -1160,12 +1447,16 @@ class IrBuilderTest extends IrTestSuite { query.asCypherQuery().model.result match { case GraphResultBlock(_, IRPatternGraph(_, schema, _, _, _, _)) => - schema should equal(PropertyGraphSchema.empty.withNodePropertyKeys("A", "B")()) + schema should equal( + PropertyGraphSchema.empty.withNodePropertyKeys("A", "B")() + ) case _ => fail("no matching graph result found") } } - it("computes a pattern graph schema correctly - 1 create and 1 set property") { + it( + "computes a pattern graph schema correctly - 1 create and 1 set property" + ) { val query = """ |CONSTRUCT @@ -1175,12 +1466,18 @@ class IrBuilderTest extends IrTestSuite { query.asCypherQuery().model.result match { case GraphResultBlock(_, IRPatternGraph(_, schema, _, _, _, _)) => - schema should equal(PropertyGraphSchema.empty.withNodePropertyKeys("A")("name" -> CTString)) + schema should equal( + PropertyGraphSchema.empty.withNodePropertyKeys("A")( + "name" -> CTString + ) + ) case _ => fail("no matching graph result found") } } - it("computes a pattern graph schema correctly - 1 create and 1 set property with two labels") { + it( + "computes a pattern graph schema correctly - 1 create and 1 set property with two labels" + ) { val query = """ |CONSTRUCT @@ -1190,12 +1487,18 @@ class IrBuilderTest extends IrTestSuite { query.asCypherQuery().model.result match { case GraphResultBlock(_, IRPatternGraph(_, schema, _, _, _, _)) => - schema should equal(PropertyGraphSchema.empty.withNodePropertyKeys("A", "B")("name" -> CTString)) + schema should equal( + PropertyGraphSchema.empty.withNodePropertyKeys("A", "B")( + "name" -> CTString + ) + ) case _ => fail("no matching graph result found") } } - it("computes a pattern graph schema correctly - 1 create and 1 set rel property") { + it( + "computes a pattern graph schema correctly - 1 create and 1 set rel property" + ) { val query = """ |CONSTRUCT @@ -1205,12 +1508,18 @@ class IrBuilderTest extends IrTestSuite { query.asCypherQuery().model.result match { case GraphResultBlock(_, IRPatternGraph(_, schema, _, _, _, _)) => - schema should equal(PropertyGraphSchema.empty.withNodePropertyKeys(Set.empty[String]).withRelationshipPropertyKeys("R")("level" -> CTString)) + schema should equal( + PropertyGraphSchema.empty + .withNodePropertyKeys(Set.empty[String]) + .withRelationshipPropertyKeys("R")("level" -> CTString) + ) case _ => fail("no matching graph result found") } } - it("computes a pattern graph schema correctly - 1 create and 2 set properties") { + it( + "computes a pattern graph schema correctly - 1 create and 2 set properties" + ) { val query = """ |CONSTRUCT @@ -1221,7 +1530,12 @@ class IrBuilderTest extends IrTestSuite { query.asCypherQuery().model.result match { case GraphResultBlock(_, IRPatternGraph(_, schema, _, _, _, _)) => - schema should equal(PropertyGraphSchema.empty.withNodePropertyKeys(Set("A"), PropertyKeys("category" -> CTString, "ports" -> CTInteger))) + schema should equal( + PropertyGraphSchema.empty.withNodePropertyKeys( + Set("A"), + PropertyKeys("category" -> CTString, "ports" -> CTInteger) + ) + ) case _ => fail("no matching graph result found") } } @@ -1239,7 +1553,9 @@ class IrBuilderTest extends IrTestSuite { query.asCypherQuery().model.result match { case GraphResultBlock(_, IRPatternGraph(_, schema, _, _, _, _)) => - schema should equal(PropertyGraphSchema.empty.withNodePropertyKeys("A", "B")()) + schema should equal( + PropertyGraphSchema.empty.withNodePropertyKeys("A", "B")() + ) case _ => fail("no matching graph result found") } } @@ -1262,7 +1578,8 @@ class IrBuilderTest extends IrTestSuite { implicit class RichModel(model: QueryModel) { - def ensureThat(f: (QueryModel, CypherMap) => Unit): Unit = f(model, model.parameters) + def ensureThat(f: (QueryModel, CypherMap) => Unit): Unit = + f(model, model.parameters) } diff --git a/okapi-ir/src/test/scala/org/opencypher/okapi/ir/impl/IrTestSuite.scala b/okapi-ir/src/test/scala/org/opencypher/okapi/ir/impl/IrTestSuite.scala index a34355b616..3b6f47e716 100644 --- a/okapi-ir/src/test/scala/org/opencypher/okapi/ir/impl/IrTestSuite.scala +++ b/okapi-ir/src/test/scala/org/opencypher/okapi/ir/impl/IrTestSuite.scala @@ -52,7 +52,8 @@ abstract class IrTestSuite extends BaseTestSuite { def project( fields: Fields, after: List[Block] = List(leafBlock), - given: Set[Expr] = Set.empty) = + given: Set[Expr] = Set.empty + ) = ProjectBlock(after, fields, given, testGraph) protected def matchBlock(pattern: Pattern): Block = @@ -68,7 +69,8 @@ abstract class IrTestSuite extends BaseTestSuite { SingleQuery(model) } - case class DummyBlock(override val after: List[Block] = List.empty) extends BasicBlock[DummyBinds[Expr]](BlockType("dummy")) { + case class DummyBlock(override val after: List[Block] = List.empty) + extends BasicBlock[DummyBinds[Expr]](BlockType("dummy")) { override def binds: DummyBinds[Expr] = DummyBinds[Expr]() override def where: Set[Expr] = Set.empty[Expr] @@ -79,17 +81,23 @@ abstract class IrTestSuite extends BaseTestSuite { case class DummyBinds[E](fields: Set[IRField] = Set.empty) extends Binds implicit class RichString(queryText: String) { - def parseIR[T <: CypherStatement : ClassTag](graphsWithSchema: (GraphName, PropertyGraphSchema)*) - (implicit schema: PropertyGraphSchema = PropertyGraphSchema.empty): T = - ir(graphsWithSchema: _ *) match { + def parseIR[T <: CypherStatement: ClassTag]( + graphsWithSchema: (GraphName, PropertyGraphSchema)* + )(implicit schema: PropertyGraphSchema = PropertyGraphSchema.empty): T = + ir(graphsWithSchema: _*) match { case cq: T => cq - case other => throw new IllegalArgumentException(s"Cannot convert $other") + case other => + throw new IllegalArgumentException(s"Cannot convert $other") } - def asCypherQuery(graphsWithSchema: (GraphName, PropertyGraphSchema)*)(implicit schema: PropertyGraphSchema = PropertyGraphSchema.empty): SingleQuery = + def asCypherQuery(graphsWithSchema: (GraphName, PropertyGraphSchema)*)(implicit + schema: PropertyGraphSchema = PropertyGraphSchema.empty + ): SingleQuery = parseIR[SingleQuery](graphsWithSchema: _*) - def ir(graphsWithSchema: (GraphName, PropertyGraphSchema)*)(implicit schema: PropertyGraphSchema = PropertyGraphSchema.empty): CypherStatement = { + def ir(graphsWithSchema: (GraphName, PropertyGraphSchema)*)(implicit + schema: PropertyGraphSchema = PropertyGraphSchema.empty + ): CypherStatement = { val stmt = CypherParser(queryText)(CypherParser.defaultContext) val parameters = Map.empty[String, CypherValue] IRBuilder(stmt)( @@ -99,15 +107,21 @@ abstract class IrTestSuite extends BaseTestSuite { SemanticState.clean, testGraph()(schema), qgnGenerator, - Map.empty.withDefaultValue(testGraphSource(graphsWithSchema :+ (testGraphName -> schema): _*)), + Map.empty.withDefaultValue( + testGraphSource(graphsWithSchema :+ (testGraphName -> schema): _*) + ), _ => ??? - )) + ) + ) } - def irWithParams(params: (String, CypherValue)*)(implicit schema: PropertyGraphSchema = PropertyGraphSchema.empty): CypherStatement = { + def irWithParams(params: (String, CypherValue)*)(implicit + schema: PropertyGraphSchema = PropertyGraphSchema.empty + ): CypherStatement = { val stmt = CypherParser(queryText)(CypherParser.defaultContext) IRBuilder(stmt)( - IRBuilderContext.initial(queryText, + IRBuilderContext.initial( + queryText, params.toMap, SemanticState.clean, testGraph()(schema), diff --git a/okapi-ir/src/test/scala/org/opencypher/okapi/ir/impl/PatternConverterTest.scala b/okapi-ir/src/test/scala/org/opencypher/okapi/ir/impl/PatternConverterTest.scala index 26b3b4d330..ec36b6a802 100644 --- a/okapi-ir/src/test/scala/org/opencypher/okapi/ir/impl/PatternConverterTest.scala +++ b/okapi-ir/src/test/scala/org/opencypher/okapi/ir/impl/PatternConverterTest.scala @@ -256,7 +256,10 @@ class PatternConverterTest extends IrTestSuite { } val converter = new PatternConverter(IRBuilderHelper.emptyIRBuilderContext) - def convert(p: ast.Pattern, knownTypes: Map[ast.Expression, CypherType] = Map.empty): Pattern = + def convert( + p: ast.Pattern, + knownTypes: Map[ast.Expression, CypherType] = Map.empty + ): Pattern = converter.convert(p, knownTypes, testQualifiedGraphName) def parse(exprText: String): ast.Pattern = PatternParser.parse(exprText, None) diff --git a/okapi-ir/src/test/scala/org/opencypher/okapi/ir/impl/RichSchemaTest.scala b/okapi-ir/src/test/scala/org/opencypher/okapi/ir/impl/RichSchemaTest.scala index b6acf33431..52799584a0 100644 --- a/okapi-ir/src/test/scala/org/opencypher/okapi/ir/impl/RichSchemaTest.scala +++ b/okapi-ir/src/test/scala/org/opencypher/okapi/ir/impl/RichSchemaTest.scala @@ -35,56 +35,62 @@ import org.opencypher.okapi.testing.BaseTestSuite import scala.collection.immutable.ListMap class RichSchemaTest extends BaseTestSuite { - describe("fromFields") { - it("can convert fields in a pattern") { - val schema = PropertyGraphSchema.empty - .withNodePropertyKeys("Person")("name" -> CTString) - .withNodePropertyKeys("City")("name" -> CTString, "region" -> CTBoolean) - .withRelationshipPropertyKeys("KNOWS")("since" -> CTFloat.nullable) - .withRelationshipPropertyKeys("BAR")("foo" -> CTInteger) + describe("fromFields") { + it("can convert fields in a pattern") { + val schema = PropertyGraphSchema.empty + .withNodePropertyKeys("Person")("name" -> CTString) + .withNodePropertyKeys("City")("name" -> CTString, "region" -> CTBoolean) + .withRelationshipPropertyKeys("KNOWS")("since" -> CTFloat.nullable) + .withRelationshipPropertyKeys("BAR")("foo" -> CTInteger) - val actual = Pattern( - Set( + val actual = Pattern( + Set( + IRField("n")(CTNode("Person")), + IRField("r")(CTRelationship("BAR")), + IRField("m")(CTNode("Person")) + ), + ListMap( + IRField("r")(CTRelationship("BAR")) -> DirectedRelationship( IRField("n")(CTNode("Person")), - IRField("r")(CTRelationship("BAR")), IRField("m")(CTNode("Person")) - ), - ListMap( - IRField("r")(CTRelationship("BAR")) -> DirectedRelationship(IRField("n")(CTNode("Person")), IRField("m")(CTNode("Person"))) ) - ).fields.map(f => schema.forElementType(f.cypherType)).reduce(_ ++ _) + ) + ).fields.map(f => schema.forElementType(f.cypherType)).reduce(_ ++ _) - val expected = PropertyGraphSchema.empty - .withNodePropertyKeys("Person")("name" -> CTString) - .withRelationshipPropertyKeys("BAR")("foo" -> CTInteger) + val expected = PropertyGraphSchema.empty + .withNodePropertyKeys("Person")("name" -> CTString) + .withRelationshipPropertyKeys("BAR")("foo" -> CTInteger) - actual should be(expected) - } + actual should be(expected) + } - it("can compute a schema when a field is unknown") { - val schema = PropertyGraphSchema.empty - .withNodePropertyKeys("Person")("name" -> CTString) - .withNodePropertyKeys("City")("name" -> CTString, "region" -> CTBoolean) - .withRelationshipPropertyKeys("KNOWS")("since" -> CTFloat.nullable) - .withRelationshipPropertyKeys("BAR")("foo" -> CTInteger) + it("can compute a schema when a field is unknown") { + val schema = PropertyGraphSchema.empty + .withNodePropertyKeys("Person")("name" -> CTString) + .withNodePropertyKeys("City")("name" -> CTString, "region" -> CTBoolean) + .withRelationshipPropertyKeys("KNOWS")("since" -> CTFloat.nullable) + .withRelationshipPropertyKeys("BAR")("foo" -> CTInteger) - val actual = Pattern( - Set( + val actual = Pattern( + Set( + IRField("n")(CTNode("Person")), + IRField("r")(CTRelationship("BAR")), + IRField("m")(CTNode()) + ), + ListMap( + IRField("r")(CTRelationship("BAR")) -> DirectedRelationship( IRField("n")(CTNode("Person")), - IRField("r")(CTRelationship("BAR")), IRField("m")(CTNode()) - ), - ListMap( - IRField("r")(CTRelationship("BAR")) -> DirectedRelationship(IRField("n")(CTNode("Person")), IRField("m")(CTNode())) ) - ).fields.map(f => schema.forElementType(f.cypherType)).reduce(_ ++ _) + ) + ).fields.map(f => schema.forElementType(f.cypherType)).reduce(_ ++ _) - val expected = PropertyGraphSchema.empty - .withNodePropertyKeys("Person")("name" -> CTString) - .withNodePropertyKeys("City")("name" -> CTString, "region" -> CTBoolean) - .withRelationshipPropertyKeys("BAR")("foo" -> CTInteger) + val expected = PropertyGraphSchema.empty + .withNodePropertyKeys("Person")("name" -> CTString) + .withNodePropertyKeys("City")("name" -> CTString, "region" -> CTBoolean) + .withRelationshipPropertyKeys("BAR")("foo" -> CTInteger) - actual should be(expected) - } + actual should be(expected) } + } } diff --git a/okapi-ir/src/test/scala/org/opencypher/okapi/ir/impl/block/TypedMatchBlockTest.scala b/okapi-ir/src/test/scala/org/opencypher/okapi/ir/impl/block/TypedMatchBlockTest.scala index 06f3e2d777..24dde4b74a 100644 --- a/okapi-ir/src/test/scala/org/opencypher/okapi/ir/impl/block/TypedMatchBlockTest.scala +++ b/okapi-ir/src/test/scala/org/opencypher/okapi/ir/impl/block/TypedMatchBlockTest.scala @@ -38,35 +38,44 @@ class TypedMatchBlockTest extends IrTestSuite { implicit val graph: Some[QualifiedGraphName] = Some(testQualifiedGraphName) it("computes detailed types of pattern variables") { - implicit val (block, globals) = matchBlock("MATCH (n:Person:Foo)-[r:TYPE]->(m) RETURN n") + implicit val (block, globals) = + matchBlock("MATCH (n:Person:Foo)-[r:TYPE]->(m) RETURN n") typedMatchBlock.outputs(block).map(_.toTypedTuple) should equal( Set( "n" -> CTNode(Set("Person", "Foo"), Some(testQualifiedGraphName)), "r" -> CTRelationship(Set("TYPE"), Some(testQualifiedGraphName)), "m" -> CTNode(Set.empty[String], Some(testQualifiedGraphName)) - )) + ) + ) } test("computes detailed type of elements also from WHERE clause") { - implicit val (block, globals) = matchBlock("MATCH (n:Person:Foo)-[r:TYPE]->(m) WHERE n:Three RETURN n") + implicit val (block, globals) = + matchBlock("MATCH (n:Person:Foo)-[r:TYPE]->(m) WHERE n:Three RETURN n") typedMatchBlock.outputs(block).map(_.toTypedTuple) should equal( Set( - "n" -> CTNode(Set("Person", "Foo", "Three"), Some(testQualifiedGraphName)), + "n" -> CTNode( + Set("Person", "Foo", "Three"), + Some(testQualifiedGraphName) + ), "r" -> CTRelationship(Set("TYPE"), Some(testQualifiedGraphName)), "m" -> CTNode(Set.empty[String], Some(testQualifiedGraphName)) - )) + ) + ) } // TODO: We need to register the string literal as a relationship type in globals extraction -- is this what we want ignore("computes detailed relationship type from WHERE clause") { - implicit val (block, globals) = matchBlock("MATCH ()-[r]->() WHERE type(r) = 'TYPE' RETURN $noAutoParams") + implicit val (block, globals) = + matchBlock("MATCH ()-[r]->() WHERE type(r) = 'TYPE' RETURN $noAutoParams") typedMatchBlock.outputs(block).map(_.toTypedTuple) should equal( Set( "r" -> CTRelationship("TYPE") - )) + ) + ) } private def matchBlock(singleMatchQuery: String): (MatchBlock, CypherMap) = { diff --git a/okapi-ir/src/test/scala/org/opencypher/okapi/ir/impl/parse/rewriter/normalizeCaseExpressionTest.scala b/okapi-ir/src/test/scala/org/opencypher/okapi/ir/impl/parse/rewriter/normalizeCaseExpressionTest.scala index dd5d2bc084..b22c72a6a0 100644 --- a/okapi-ir/src/test/scala/org/opencypher/okapi/ir/impl/parse/rewriter/normalizeCaseExpressionTest.scala +++ b/okapi-ir/src/test/scala/org/opencypher/okapi/ir/impl/parse/rewriter/normalizeCaseExpressionTest.scala @@ -49,7 +49,8 @@ class normalizeCaseExpressionTest extends BaseTestSuite with RewriterTestSupport | WHEN n.val = "foo" THEN 1 | WHEN n.val = "bar" THEN 2 | END AS val - """.stripMargin) + """.stripMargin + ) } it("should rewrite simple CASE statement with default") { @@ -71,7 +72,8 @@ class normalizeCaseExpressionTest extends BaseTestSuite with RewriterTestSupport | WHEN n.val = "bar" THEN 2 | ELSE 3 | END AS val - """.stripMargin) + """.stripMargin + ) } it("should not rewrite generic CASE statement") { @@ -93,6 +95,7 @@ class normalizeCaseExpressionTest extends BaseTestSuite with RewriterTestSupport | WHEN n.val = "bar" THEN 2 | ELSE 3 | END AS val - """.stripMargin) + """.stripMargin + ) } } diff --git a/okapi-ir/src/test/scala/org/opencypher/okapi/ir/impl/parse/rewriter/normalizeReturnClausesTest.scala b/okapi-ir/src/test/scala/org/opencypher/okapi/ir/impl/parse/rewriter/normalizeReturnClausesTest.scala index b35a4df416..79c3859b6e 100644 --- a/okapi-ir/src/test/scala/org/opencypher/okapi/ir/impl/parse/rewriter/normalizeReturnClausesTest.scala +++ b/okapi-ir/src/test/scala/org/opencypher/okapi/ir/impl/parse/rewriter/normalizeReturnClausesTest.scala @@ -39,7 +39,8 @@ class normalizeReturnClausesTest extends BaseTestSuite with RewriterTestSupport """.stripMargin, """MATCH (n) |RETURN n - """.stripMargin) + """.stripMargin + ) } test("do not rewrite aliased return items of variables") { @@ -49,7 +50,8 @@ class normalizeReturnClausesTest extends BaseTestSuite with RewriterTestSupport """.stripMargin, """MATCH (n) |RETURN n AS foo - """.stripMargin) + """.stripMargin + ) } test("do no rewrite unaliased return item of properties") { @@ -59,7 +61,8 @@ class normalizeReturnClausesTest extends BaseTestSuite with RewriterTestSupport """.stripMargin, """MATCH (n) |RETURN n.prop - """.stripMargin) + """.stripMargin + ) } test("do no rewrite aliased return item of properties") { @@ -98,7 +101,9 @@ class normalizeReturnClausesTest extends BaseTestSuite with RewriterTestSupport ) } - test("introduce WITH clause for unaliased primitive and non-primitive expressions") { + test( + "introduce WITH clause for unaliased primitive and non-primitive expressions" + ) { assertRewrite( """MATCH (n) |RETURN n, n.val, count(n.val) @@ -110,7 +115,9 @@ class normalizeReturnClausesTest extends BaseTestSuite with RewriterTestSupport ) } - test("introduce WITH clause for unaliased, primitive and aliased, non-primitive expressions") { + test( + "introduce WITH clause for unaliased, primitive and aliased, non-primitive expressions" + ) { assertRewrite( """MATCH (n) |RETURN n, n.val, count(n.val) AS foo diff --git a/okapi-ir/src/test/scala/org/opencypher/okapi/ir/impl/parse/rewriter/pushLabelsIntoScansTest.scala b/okapi-ir/src/test/scala/org/opencypher/okapi/ir/impl/parse/rewriter/pushLabelsIntoScansTest.scala index 0eb8202b3b..9976470b85 100644 --- a/okapi-ir/src/test/scala/org/opencypher/okapi/ir/impl/parse/rewriter/pushLabelsIntoScansTest.scala +++ b/okapi-ir/src/test/scala/org/opencypher/okapi/ir/impl/parse/rewriter/pushLabelsIntoScansTest.scala @@ -40,7 +40,8 @@ class pushLabelsIntoScansTest extends BaseTestSuite with RewriterTestSupport { """.stripMargin, """MATCH (n:Foo) |RETURN n - """.stripMargin) + """.stripMargin + ) } test("push labels for multiple nodes into pattern") { @@ -50,7 +51,8 @@ class pushLabelsIntoScansTest extends BaseTestSuite with RewriterTestSupport { """.stripMargin, """MATCH (n:Foo)-[:Rel]->(b:Foo:Bar) |RETURN n - """.stripMargin) + """.stripMargin + ) } test("keep the node label in the pattern") { @@ -60,7 +62,8 @@ class pushLabelsIntoScansTest extends BaseTestSuite with RewriterTestSupport { """.stripMargin, """MATCH (n:Foo) |RETURN n - """.stripMargin) + """.stripMargin + ) } test("push additional labels into pattern") { @@ -71,7 +74,8 @@ class pushLabelsIntoScansTest extends BaseTestSuite with RewriterTestSupport { """.stripMargin, """MATCH (n:Foo:Bar) |RETURN n - """.stripMargin) + """.stripMargin + ) } test("push complex where predicate labels into pattern") { @@ -82,7 +86,8 @@ class pushLabelsIntoScansTest extends BaseTestSuite with RewriterTestSupport { """.stripMargin, """MATCH (n:Bar:Baz) |RETURN n - """.stripMargin) + """.stripMargin + ) } test("push label predicates from complex where predicate into pattern") { @@ -94,7 +99,8 @@ class pushLabelsIntoScansTest extends BaseTestSuite with RewriterTestSupport { """MATCH (n:Bar) |WHERE n.age > 25 |RETURN n - """.stripMargin) + """.stripMargin + ) } test("do not push OR'ed labels into pattern") { @@ -106,7 +112,8 @@ class pushLabelsIntoScansTest extends BaseTestSuite with RewriterTestSupport { """MATCH (n) |WHERE n:Bar OR n:Baz |RETURN n - """.stripMargin) + """.stripMargin + ) } test("do not push ORS'ed labels into pattern") { @@ -118,7 +125,8 @@ class pushLabelsIntoScansTest extends BaseTestSuite with RewriterTestSupport { """MATCH (n) |WHERE n:Foo OR n:Bar OR n:Baz |RETURN n - """.stripMargin) + """.stripMargin + ) } test("do not push NOT'ed labels into pattern") { diff --git a/okapi-ir/src/test/scala/org/opencypher/okapi/ir/impl/typer/TypeRecorderTest.scala b/okapi-ir/src/test/scala/org/opencypher/okapi/ir/impl/typer/TypeRecorderTest.scala index c1ae2844a1..d4d0449b13 100644 --- a/okapi-ir/src/test/scala/org/opencypher/okapi/ir/impl/typer/TypeRecorderTest.scala +++ b/okapi-ir/src/test/scala/org/opencypher/okapi/ir/impl/typer/TypeRecorderTest.scala @@ -37,9 +37,12 @@ class TypeRecorderTest extends BaseTestSuite with AstConstructionTestSupport { test("can convert to map") { val expr1 = True()(pos) val expr2 = True()(pos) - val recorder = TypeRecorder(List(Ref(expr1) -> CTBoolean, Ref(expr2) -> CTString)) + val recorder = + TypeRecorder(List(Ref(expr1) -> CTBoolean, Ref(expr2) -> CTString)) - recorder.toMap should equal(Map(Ref(expr1) -> CTBoolean, Ref(expr2) -> CTString)) + recorder.toMap should equal( + Map(Ref(expr1) -> CTBoolean, Ref(expr2) -> CTString) + ) } } diff --git a/okapi-ir/src/test/scala/org/opencypher/okapi/ir/impl/typer/toFrontendTypeTest.scala b/okapi-ir/src/test/scala/org/opencypher/okapi/ir/impl/typer/toFrontendTypeTest.scala index a4d9624287..af5acac051 100644 --- a/okapi-ir/src/test/scala/org/opencypher/okapi/ir/impl/typer/toFrontendTypeTest.scala +++ b/okapi-ir/src/test/scala/org/opencypher/okapi/ir/impl/typer/toFrontendTypeTest.scala @@ -64,8 +64,12 @@ class toFrontendTypeTest extends BaseTestSuite { test("should convert container types") { CTList(CTInteger) shouldBeConvertedTo frontend.CTList(frontend.CTInteger) - CTList(CTInteger).nullable shouldBeConvertedTo frontend.CTList(frontend.CTInteger) - CTList(CTInteger.nullable) shouldBeConvertedTo frontend.CTList(frontend.CTInteger) + CTList(CTInteger).nullable shouldBeConvertedTo frontend.CTList( + frontend.CTInteger + ) + CTList(CTInteger.nullable) shouldBeConvertedTo frontend.CTList( + frontend.CTInteger + ) CTMap shouldBeConvertedTo frontend.CTMap CTMap.nullable shouldBeConvertedTo frontend.CTMap } diff --git a/okapi-ir/src/test/scala/org/opencypher/okapi/ir/test/package.scala b/okapi-ir/src/test/scala/org/opencypher/okapi/ir/test/package.scala index d263dac5fa..586af5dae3 100644 --- a/okapi-ir/src/test/scala/org/opencypher/okapi/ir/test/package.scala +++ b/okapi-ir/src/test/scala/org/opencypher/okapi/ir/test/package.scala @@ -32,5 +32,6 @@ import org.opencypher.okapi.ir.api.IRCatalogGraph import scala.language.implicitConversions package object test { - implicit def toGraph(s: Symbol): IRCatalogGraph = IRCatalogGraph(s.name, PropertyGraphSchema.empty) + implicit def toGraph(s: Symbol): IRCatalogGraph = + IRCatalogGraph(s.name, PropertyGraphSchema.empty) } diff --git a/okapi-ir/src/test/scala/org/opencypher/okapi/ir/test/support/Neo4jAstTestSupport.scala b/okapi-ir/src/test/scala/org/opencypher/okapi/ir/test/support/Neo4jAstTestSupport.scala index 83585ae801..2fb4876ce8 100644 --- a/okapi-ir/src/test/scala/org/opencypher/okapi/ir/test/support/Neo4jAstTestSupport.scala +++ b/okapi-ir/src/test/scala/org/opencypher/okapi/ir/test/support/Neo4jAstTestSupport.scala @@ -46,12 +46,21 @@ trait Neo4jAstTestSupport extends AstConstructionTestSupport { import Neo4jAstTestSupport.CypherParserWithoutSemanticChecking - def parseQuery(queryText: String): (Statement, Map[String, Any], SemanticState) = - CypherParserWithoutSemanticChecking.process(queryText)(CypherParser.defaultContext) + def parseQuery( + queryText: String + ): (Statement, Map[String, Any], SemanticState) = + CypherParserWithoutSemanticChecking.process(queryText)( + CypherParser.defaultContext + ) implicit def parseExpr(exprText: String): Expression = { - CypherParserWithoutSemanticChecking.process(s"RETURN $exprText")(CypherParser.defaultContext)._1 match { - case Query(_, SingleQuery(Return(_, ReturnItems(_, items), _, _, _, _) :: Nil)) => + CypherParserWithoutSemanticChecking + .process(s"RETURN $exprText")(CypherParser.defaultContext) + ._1 match { + case Query( + _, + SingleQuery(Return(_, ReturnItems(_, items), _, _, _, _) :: Nil) + ) => items.head.expression case _ => throw IllegalArgumentException("an expression", exprText) } @@ -73,14 +82,19 @@ object Neo4jAstTestSupport { object NonThrowingSemanticAnalysis extends SemanticAnalysis(true) { override def process(from: BaseState, context: BaseContext): BaseState = { - val semanticState = NonThrowingChecker.check(from.statement(), context.exceptionCreator) + val semanticState = + NonThrowingChecker.check(from.statement(), context.exceptionCreator) from.withSemanticState(semanticState) } } object NonThrowingChecker { - def check(statement: Statement, mkException: (String, InputPosition) => CypherException): SemanticState = { - val SemanticCheckResult(semanticState, _) = statement.semanticCheck(SemanticState.clean) + def check( + statement: Statement, + mkException: (String, InputPosition) => CypherException + ): SemanticState = { + val SemanticCheckResult(semanticState, _) = + statement.semanticCheck(SemanticState.clean) semanticState } } diff --git a/okapi-ir/src/test/scala/org/opencypher/okapi/ir/test/support/RewriterTestSupport.scala b/okapi-ir/src/test/scala/org/opencypher/okapi/ir/test/support/RewriterTestSupport.scala index 3dbdf37a0b..164027a29d 100644 --- a/okapi-ir/src/test/scala/org/opencypher/okapi/ir/test/support/RewriterTestSupport.scala +++ b/okapi-ir/src/test/scala/org/opencypher/okapi/ir/test/support/RewriterTestSupport.scala @@ -50,11 +50,10 @@ trait RewriterTestSupport extends AstConstructionTestSupport { assert(result === expected, "\n" + originalQuery) } - private def parseForRewriting(queryText: String): Statement = parser.parse(queryText.replace("\r\n", "\n")) + private def parseForRewriting(queryText: String): Statement = + parser.parse(queryText.replace("\r\n", "\n")) private def rewrite(original: Statement): AnyRef = original.rewrite(rewriter) } - - diff --git a/okapi-logical/src/main/scala/org/opencypher/okapi/logical/impl/LogicalOperator.scala b/okapi-logical/src/main/scala/org/opencypher/okapi/logical/impl/LogicalOperator.scala index d1730344f5..f464314157 100644 --- a/okapi-logical/src/main/scala/org/opencypher/okapi/logical/impl/LogicalOperator.scala +++ b/okapi-logical/src/main/scala/org/opencypher/okapi/logical/impl/LogicalOperator.scala @@ -45,7 +45,7 @@ sealed abstract class LogicalOperator extends AbstractTreeNode[LogicalOperator] override def args: Iterator[Any] = super.args.filter { case SolvedQueryModel(_, _) => false - case _ => true + case _ => true } } @@ -66,7 +66,10 @@ sealed trait LogicalGraph { } -case class LogicalCatalogGraph(qualifiedGraphName: QualifiedGraphName, schema: PropertyGraphSchema) extends LogicalGraph { +case class LogicalCatalogGraph( + qualifiedGraphName: QualifiedGraphName, + schema: PropertyGraphSchema +) extends LogicalGraph { override protected def args: String = qualifiedGraphName.toString } @@ -103,7 +106,10 @@ case class ConstructedRelationship( typ: Option[String], baseElement: Option[Var] ) extends ConstructedElement { - require(typ.isDefined || baseElement.isDefined, s"$this: Need to define either the rel type or an equivalence model to construct a relationship") + require( + typ.isDefined || baseElement.isDefined, + s"$this: Need to define either the rel type or an equivalence model to construct a relationship" + ) } sealed abstract class StackingLogicalOperator extends LogicalOperator { @@ -118,8 +124,9 @@ sealed abstract class BinaryLogicalOperator extends LogicalOperator { def rhs: LogicalOperator /** - * Always pick the source graph from the right-hand side, because it works for in-pattern expansions - * and changing of source graphs. This relies on the planner always planning _later_ operators on the rhs. + * Always pick the source graph from the right-hand side, because it works for in-pattern + * expansions and changing of source graphs. This relies on the planner always planning _later_ + * operators on the rhs. */ override def graph: LogicalGraph = rhs.graph } @@ -127,23 +134,37 @@ sealed abstract class BinaryLogicalOperator extends LogicalOperator { sealed abstract class LogicalLeafOperator extends LogicalOperator object PatternScan { - def nodeScan(node: Var, in: LogicalOperator, solved: SolvedQueryModel): PatternScan = { + def nodeScan( + node: Var, + in: LogicalOperator, + solved: SolvedQueryModel + ): PatternScan = { val pattern = NodePattern(node.cypherType.toCTNode) PatternScan(pattern, Map(node -> pattern.nodeElement), in, solved) } } -final case class PatternScan(pattern: Pattern, mapping: Map[Var, PatternElement], in: LogicalOperator, solved: SolvedQueryModel) - extends StackingLogicalOperator { +final case class PatternScan( + pattern: Pattern, + mapping: Map[Var, PatternElement], + in: LogicalOperator, + solved: SolvedQueryModel +) extends StackingLogicalOperator { override val fields: Set[Var] = mapping.keySet } -final case class Distinct(fields: Set[Var], in: LogicalOperator, solved: SolvedQueryModel) - extends StackingLogicalOperator +final case class Distinct( + fields: Set[Var], + in: LogicalOperator, + solved: SolvedQueryModel +) extends StackingLogicalOperator -final case class Filter(expr: Expr, in: LogicalOperator, solved: SolvedQueryModel) - extends StackingLogicalOperator { +final case class Filter( + expr: Expr, + in: LogicalOperator, + solved: SolvedQueryModel +) extends StackingLogicalOperator { // TODO: Add more precise type information based on predicates (?) override val fields: Set[Var] = in.fields @@ -167,8 +188,7 @@ final case class Expand( lhs: LogicalOperator, rhs: LogicalOperator, solved: SolvedQueryModel -) - extends BinaryLogicalOperator +) extends BinaryLogicalOperator with ExpandOperator { override val fields: Set[Var] = lhs.fields ++ rhs.fields + rel @@ -185,11 +205,9 @@ final case class BoundedVarLengthExpand( lhs: LogicalOperator, rhs: LogicalOperator, solved: SolvedQueryModel -) - extends BinaryLogicalOperator +) extends BinaryLogicalOperator with ExpandOperator { - override def rel: Var = list override val fields: Set[Var] = lhs.fields ++ rhs.fields @@ -200,8 +218,7 @@ final case class ValueJoin( rhs: LogicalOperator, predicates: Set[org.opencypher.okapi.ir.api.expr.Equals], solved: SolvedQueryModel -) - extends BinaryLogicalOperator { +) extends BinaryLogicalOperator { override val fields: Set[Var] = lhs.fields ++ rhs.fields } @@ -213,8 +230,7 @@ final case class ExpandInto( direction: Direction, in: LogicalOperator, solved: SolvedQueryModel -) - extends StackingLogicalOperator +) extends StackingLogicalOperator with ExpandOperator { override val fields: Set[Var] = lhs.fields ++ rhs.fields + rel @@ -224,17 +240,24 @@ final case class ExpandInto( def rhs: LogicalOperator = in } -final case class Project(projectExpr: (Expr, Option[Var]), in: LogicalOperator, solved: SolvedQueryModel) - extends StackingLogicalOperator { +final case class Project( + projectExpr: (Expr, Option[Var]), + in: LogicalOperator, + solved: SolvedQueryModel +) extends StackingLogicalOperator { override val fields: Set[Var] = projectExpr._2 match { case Some(v: Var) => in.fields + v - case _ => in.fields + case _ => in.fields } } -final case class Unwind(expr: Expr, field: Var, in: LogicalOperator, solved: SolvedQueryModel) - extends StackingLogicalOperator { +final case class Unwind( + expr: Expr, + field: Var, + in: LogicalOperator, + solved: SolvedQueryModel +) extends StackingLogicalOperator { override val fields: Set[Var] = in.fields + field } @@ -244,8 +267,7 @@ final case class Aggregate( group: Set[Var], in: LogicalOperator, solved: SolvedQueryModel -) - extends StackingLogicalOperator { +) extends StackingLogicalOperator { override val fields: Set[Var] = in.fields ++ aggregations.map(_._1) ++ group } @@ -254,40 +276,53 @@ final case class Select( orderedFields: List[Var], in: LogicalOperator, solved: SolvedQueryModel -) - extends StackingLogicalOperator { +) extends StackingLogicalOperator { override val fields: Set[Var] = orderedFields.toSet } final case class ReturnGraph(in: LogicalOperator, solved: SolvedQueryModel) - extends StackingLogicalOperator with EmptyFields + extends StackingLogicalOperator + with EmptyFields -final case class OrderBy(sortItems: Seq[SortItem], in: LogicalOperator, solved: SolvedQueryModel) - extends StackingLogicalOperator { +final case class OrderBy( + sortItems: Seq[SortItem], + in: LogicalOperator, + solved: SolvedQueryModel +) extends StackingLogicalOperator { override val fields: Set[Var] = in.fields } -final case class Skip(expr: Expr, in: LogicalOperator, solved: SolvedQueryModel) extends StackingLogicalOperator { +final case class Skip(expr: Expr, in: LogicalOperator, solved: SolvedQueryModel) + extends StackingLogicalOperator { override val fields: Set[Var] = in.fields } -final case class Limit(expr: Expr, in: LogicalOperator, solved: SolvedQueryModel) - extends StackingLogicalOperator { +final case class Limit( + expr: Expr, + in: LogicalOperator, + solved: SolvedQueryModel +) extends StackingLogicalOperator { override val fields: Set[Var] = in.fields } -final case class CartesianProduct(lhs: LogicalOperator, rhs: LogicalOperator, solved: SolvedQueryModel) - extends BinaryLogicalOperator { +final case class CartesianProduct( + lhs: LogicalOperator, + rhs: LogicalOperator, + solved: SolvedQueryModel +) extends BinaryLogicalOperator { override val fields: Set[Var] = lhs.fields ++ rhs.fields } -final case class Optional(lhs: LogicalOperator, rhs: LogicalOperator, solved: SolvedQueryModel) - extends BinaryLogicalOperator { +final case class Optional( + lhs: LogicalOperator, + rhs: LogicalOperator, + solved: SolvedQueryModel +) extends BinaryLogicalOperator { override val fields: Set[Var] = lhs.fields ++ rhs.fields } @@ -307,7 +342,10 @@ final case class TabularUnionAll( rhs: LogicalOperator ) extends BinaryLogicalOperator { - assert(lhs.fields == rhs.fields, "Both inputs of TabularUnionAll must have the same fields") + assert( + lhs.fields == rhs.fields, + "Both inputs of TabularUnionAll must have the same fields" + ) override val fields: Set[Var] = lhs.fields override def solved: SolvedQueryModel = lhs.solved ++ rhs.solved } @@ -330,13 +368,22 @@ final case class FromGraph( // TODO: adopt yield for construct override val fields: Set[Var] = graph match { case _: LogicalPatternGraph => Set.empty - case _ => in.fields + case _ => in.fields } } -final case class EmptyRecords(fields: Set[Var], in: LogicalOperator, solved: SolvedQueryModel) - extends StackingLogicalOperator +final case class EmptyRecords( + fields: Set[Var], + in: LogicalOperator, + solved: SolvedQueryModel +) extends StackingLogicalOperator -final case class Start(graph: LogicalGraph, solved: SolvedQueryModel) extends LogicalLeafOperator with EmptyFields +final case class Start(graph: LogicalGraph, solved: SolvedQueryModel) + extends LogicalLeafOperator + with EmptyFields -final case class DrivingTable(graph: LogicalGraph, fields: Set[Var], solved: SolvedQueryModel) extends LogicalLeafOperator +final case class DrivingTable( + graph: LogicalGraph, + fields: Set[Var], + solved: SolvedQueryModel +) extends LogicalLeafOperator diff --git a/okapi-logical/src/main/scala/org/opencypher/okapi/logical/impl/LogicalOperatorProducer.scala b/okapi-logical/src/main/scala/org/opencypher/okapi/logical/impl/LogicalOperatorProducer.scala index 7e394cc17b..455b558ead 100644 --- a/okapi-logical/src/main/scala/org/opencypher/okapi/logical/impl/LogicalOperatorProducer.scala +++ b/okapi-logical/src/main/scala/org/opencypher/okapi/logical/impl/LogicalOperatorProducer.scala @@ -38,49 +38,92 @@ import org.opencypher.okapi.ir.impl.util.VarConverters._ // TODO: Align names with other producers class LogicalOperatorProducer { - def planCartesianProduct(lhs: LogicalOperator, rhs: LogicalOperator): CartesianProduct = { + def planCartesianProduct( + lhs: LogicalOperator, + rhs: LogicalOperator + ): CartesianProduct = { CartesianProduct(lhs, rhs, lhs.solved ++ rhs.solved) } def planValueJoin( - lhs: LogicalOperator, - rhs: LogicalOperator, - predicates: Set[org.opencypher.okapi.ir.api.expr.Equals]): ValueJoin = { - ValueJoin(lhs, rhs, predicates, predicates.foldLeft(lhs.solved ++ rhs.solved) { - case (solved, predicate) => solved.withPredicate(predicate) - }) + lhs: LogicalOperator, + rhs: LogicalOperator, + predicates: Set[org.opencypher.okapi.ir.api.expr.Equals] + ): ValueJoin = { + ValueJoin( + lhs, + rhs, + predicates, + predicates.foldLeft(lhs.solved ++ rhs.solved) { case (solved, predicate) => + solved.withPredicate(predicate) + } + ) } def planBoundedVarLengthExpand( - source: IRField, - r: IRField, - target: IRField, - edgeType: CTRelationship, - direction: Direction, - lower: Int, - upper: Int, - sourcePlan: LogicalOperator, - targetPlan: LogicalOperator): BoundedVarLengthExpand = { + source: IRField, + r: IRField, + target: IRField, + edgeType: CTRelationship, + direction: Direction, + lower: Int, + upper: Int, + sourcePlan: LogicalOperator, + targetPlan: LogicalOperator + ): BoundedVarLengthExpand = { val prevSolved = sourcePlan.solved ++ targetPlan.solved - BoundedVarLengthExpand(source, r, target, edgeType, direction, lower, upper, sourcePlan, targetPlan, prevSolved.withField(r)) + BoundedVarLengthExpand( + source, + r, + target, + edgeType, + direction, + lower, + upper, + sourcePlan, + targetPlan, + prevSolved.withField(r) + ) } def planExpand( - source: IRField, - rel: IRField, - target: IRField, - direction: Direction, - sourcePlan: LogicalOperator, - targetPlan: LogicalOperator): Expand = { + source: IRField, + rel: IRField, + target: IRField, + direction: Direction, + sourcePlan: LogicalOperator, + targetPlan: LogicalOperator + ): Expand = { val prevSolved = sourcePlan.solved ++ targetPlan.solved - Expand(source, rel, target, direction, sourcePlan, targetPlan, prevSolved.solveRelationship(rel)) - } - - def planExpandInto(source: IRField, rel: IRField, target: IRField, direction: Direction, sourcePlan: LogicalOperator): ExpandInto = { - ExpandInto(source, rel, target, direction, sourcePlan, sourcePlan.solved.solveRelationship(rel)) + Expand( + source, + rel, + target, + direction, + sourcePlan, + targetPlan, + prevSolved.solveRelationship(rel) + ) + } + + def planExpandInto( + source: IRField, + rel: IRField, + target: IRField, + direction: Direction, + sourcePlan: LogicalOperator + ): ExpandInto = { + ExpandInto( + source, + rel, + target, + direction, + sourcePlan, + sourcePlan.solved.solveRelationship(rel) + ) } def planNodeScan(node: IRField, prev: LogicalOperator): PatternScan = { @@ -89,11 +132,19 @@ class LogicalOperatorProducer { val nodeType = node.cypherType.toCTNode val solvedWithPredicates = node.cypherType match { - case CTNode(labels, _) => solvedWithField.withPredicates(labels.map(l => HasLabel(node, Label(l))).toSeq:_*) + case CTNode(labels, _) => + solvedWithField.withPredicates( + labels.map(l => HasLabel(node, Label(l))).toSeq: _* + ) case _ => solvedWithField } val pattern = NodePattern(nodeType) - PatternScan(pattern, Map(node.toVar -> pattern.nodeElement), prev, solvedWithPredicates) + PatternScan( + pattern, + Map(node.toVar -> pattern.nodeElement), + prev, + solvedWithPredicates + ) } def planFilter(expr: Expr, prev: LogicalOperator): Filter = { @@ -104,28 +155,51 @@ class LogicalOperatorProducer { Distinct(fields.map(toVar), prev, prev.solved) } - def planOptional(nonOptionalPlan: LogicalOperator, optionalPlan: LogicalOperator): Optional = { + def planOptional( + nonOptionalPlan: LogicalOperator, + optionalPlan: LogicalOperator + ): Optional = { Optional(nonOptionalPlan, optionalPlan, optionalPlan.solved) } def planExistsSubQuery( - expr: ExistsPatternExpr, - matchPlan: LogicalOperator, - patternPlan: LogicalOperator): ExistsSubQuery = { + expr: ExistsPatternExpr, + matchPlan: LogicalOperator, + patternPlan: LogicalOperator + ): ExistsSubQuery = { ExistsSubQuery(expr, matchPlan, patternPlan, matchPlan.solved) } - def aggregate(aggregations: Aggregations, group: Set[IRField], prev: LogicalOperator): Aggregate = { - val transformed: Set[(Var, Aggregator)] = aggregations.pairs.collect { case (field, aggregator: Aggregator) => toVar(field) -> aggregator } + def aggregate( + aggregations: Aggregations, + group: Set[IRField], + prev: LogicalOperator + ): Aggregate = { + val transformed: Set[(Var, Aggregator)] = aggregations.pairs.collect { + case (field, aggregator: Aggregator) => toVar(field) -> aggregator + } - Aggregate(transformed, group.map(toVar), prev, prev.solved.withFields(aggregations.fields.toSeq: _*)) + Aggregate( + transformed, + group.map(toVar), + prev, + prev.solved.withFields(aggregations.fields.toSeq: _*) + ) } - def projectField(expr: Expr, field: IRField, prev: LogicalOperator): Project = { + def projectField( + expr: Expr, + field: IRField, + prev: LogicalOperator + ): Project = { Project(expr -> Some(field), prev, prev.solved.withField(field)) } - def planUnwind(list: Expr, variable: IRField, withList: LogicalOperator): Unwind = { + def planUnwind( + list: Expr, + variable: IRField, + withList: LogicalOperator + ): Unwind = { Unwind(list, variable, withList, withList.solved.withField(variable)) } @@ -145,7 +219,10 @@ class LogicalOperatorProducer { Start(graph, SolvedQueryModel.empty) } - def planStartWithDrivingTable(graph: LogicalGraph, fields: Set[Var]): DrivingTable = { + def planStartWithDrivingTable( + graph: LogicalGraph, + fields: Set[Var] + ): DrivingTable = { val irFields = fields.map { v => IRField(v.name)(v.cypherType) } diff --git a/okapi-logical/src/main/scala/org/opencypher/okapi/logical/impl/LogicalOptimizer.scala b/okapi-logical/src/main/scala/org/opencypher/okapi/logical/impl/LogicalOptimizer.scala index 46f6b20be7..772f1de2d6 100644 --- a/okapi-logical/src/main/scala/org/opencypher/okapi/logical/impl/LogicalOptimizer.scala +++ b/okapi-logical/src/main/scala/org/opencypher/okapi/logical/impl/LogicalOptimizer.scala @@ -36,52 +36,72 @@ import org.opencypher.okapi.trees.{BottomUp, BottomUpWithContext} import scala.util.Try -object LogicalOptimizer extends DirectCompilationStage[LogicalOperator, LogicalOperator, LogicalPlannerContext] { - - override def process(input: LogicalOperator)(implicit context: LogicalPlannerContext): LogicalOperator = { +object LogicalOptimizer + extends DirectCompilationStage[ + LogicalOperator, + LogicalOperator, + LogicalPlannerContext + ] { + + override def process( + input: LogicalOperator + )(implicit context: LogicalPlannerContext): LogicalOperator = { val optimizationRules = Seq( discardScansForNonexistentLabels, replaceCartesianWithValueJoin, replaceScansWithRecognizedPatterns ) - optimizationRules.foldLeft(input) { + optimizationRules.foldLeft(input) { // TODO: Evaluate if multiple rewriters could be fused - case (tree: LogicalOperator, optimizationRule) => BottomUp[LogicalOperator](optimizationRule).transform(tree) + case (tree: LogicalOperator, optimizationRule) => + BottomUp[LogicalOperator](optimizationRule).transform(tree) } } def replaceCartesianWithValueJoin: PartialFunction[LogicalOperator, LogicalOperator] = { - case filter@Filter(e @ CanOptimize(leftField, rightField), in, _) => - val (newChild, rewritten) = BottomUpWithContext[LogicalOperator, Boolean] { - case (CartesianProduct(lhs, rhs, solved), false) if solved.solves(leftField) && solved.solves(rightField) => - val (leftExpr, rightExpr) = if (lhs.solved.solves(leftField)) e.lhs -> e.rhs else e.rhs -> e.lhs - val joinExpr = Equals(leftExpr, rightExpr) - val leftProject = Project(leftExpr -> None, lhs, lhs.solved) - val rightProject = Project(rightExpr -> None, rhs, rhs.solved) - ValueJoin(leftProject, rightProject, Set(joinExpr), solved.withPredicate(joinExpr)) -> true - }.transform(in, context = false) + case filter @ Filter(e @ CanOptimize(leftField, rightField), in, _) => + val (newChild, rewritten) = + BottomUpWithContext[LogicalOperator, Boolean] { + case (CartesianProduct(lhs, rhs, solved), false) + if solved.solves(leftField) && solved.solves(rightField) => + val (leftExpr, rightExpr) = + if (lhs.solved.solves(leftField)) e.lhs -> e.rhs + else e.rhs -> e.lhs + val joinExpr = Equals(leftExpr, rightExpr) + val leftProject = Project(leftExpr -> None, lhs, lhs.solved) + val rightProject = Project(rightExpr -> None, rhs, rhs.solved) + ValueJoin( + leftProject, + rightProject, + Set(joinExpr), + solved.withPredicate(joinExpr) + ) -> true + }.transform(in, context = false) if (rewritten) newChild else filter } - def replaceScansWithRecognizedPatterns(implicit context: LogicalPlannerContext): PartialFunction[LogicalOperator, LogicalOperator] = { - case exp: Expand => - exp.source.cypherType.graph.map { g => - + def replaceScansWithRecognizedPatterns(implicit + context: LogicalPlannerContext + ): PartialFunction[LogicalOperator, LogicalOperator] = { case exp: Expand => + exp.source.cypherType.graph + .map { g => val availablePatterns: Set[Pattern] = Try { context.resolveGraph(g) }.map { g => g.patterns .collect { - case nr: NodeRelPattern => nr + case nr: NodeRelPattern => nr case nrn: TripletPattern => nrn - }.toList + } + .toList .sorted(Pattern.PatternOrdering) .reverse - }.getOrElse(Set.empty).toSet + }.getOrElse(Set.empty) + .toSet - if ( graphProvidesTripletPatternFor(exp, availablePatterns, g, context) ) { + if (graphProvidesTripletPatternFor(exp, availablePatterns, g, context)) { val sourceType = exp.source.cypherType.toCTNode val relType = exp.rel.cypherType.toCTRelationship val targetType = exp.target.cypherType.toCTNode @@ -89,19 +109,38 @@ object LogicalOptimizer extends DirectCompilationStage[LogicalOperator, LogicalO val pattern = TripletPattern(sourceType, relType, targetType) val withPatternScan = replaceScans(exp.rhs, exp.target, pattern) { parent => - val map = Map(exp.source -> pattern.sourceElement, exp.rel -> pattern.relElement, exp.target -> pattern.targetElement) - PatternScan(pattern, map, parent, parent.solved.withFields(map.keySet.map(_.toField.get).toList:_ *)) + val map = Map( + exp.source -> pattern.sourceElement, + exp.rel -> pattern.relElement, + exp.target -> pattern.targetElement + ) + PatternScan( + pattern, + map, + parent, + parent.solved + .withFields(map.keySet.map(_.toField.get).toList: _*) + ) } - replaceScans(exp.lhs, exp.source, pattern){_ => withPatternScan} + replaceScans(exp.lhs, exp.source, pattern) { _ => withPatternScan } - } else if ( graphProvidesNodeRelPatternFor(exp, availablePatterns, g, context) ){ + } else if (graphProvidesNodeRelPatternFor(exp, availablePatterns, g, context)) { val nodeType = exp.source.cypherType.toCTNode val relType = exp.rel.cypherType.toCTRelationship val pattern = NodeRelPattern(nodeType, relType) val withPatternScan = replaceScans(exp.lhs, exp.source, pattern) { parent => - val map = Map(exp.source -> pattern.nodeElement, exp.rel -> pattern.relElement) - PatternScan(pattern, map, parent, parent.solved.withFields(map.keySet.map(_.toField.get).toList:_ *)) + val map = Map( + exp.source -> pattern.nodeElement, + exp.rel -> pattern.relElement + ) + PatternScan( + pattern, + map, + parent, + parent.solved + .withFields(map.keySet.map(_.toField.get).toList: _*) + ) } val joinExpr = Equals(EndNode(exp.rel)(CTNode), exp.target) @@ -110,24 +149,49 @@ object LogicalOptimizer extends DirectCompilationStage[LogicalOperator, LogicalO } else { exp } - }.getOrElse(exp) + } + .getOrElse(exp) } - def replaceScans(subtree: LogicalOperator, varToReplace: Var, pattern: Pattern)(f: LogicalOperator => LogicalOperator):LogicalOperator = { - def rewriter: PartialFunction[LogicalOperator, LogicalOperator] = { - case PatternScan(_: NodePattern, mapping, parent, _) if mapping.keySet.contains(varToReplace) => f(parent) + def replaceScans( + subtree: LogicalOperator, + varToReplace: Var, + pattern: Pattern + )(f: LogicalOperator => LogicalOperator): LogicalOperator = { + def rewriter: PartialFunction[LogicalOperator, LogicalOperator] = { + case PatternScan(_: NodePattern, mapping, parent, _) + if mapping.keySet.contains(varToReplace) => + f(parent) case pScan: PatternScan if pScan.mapping.contains(varToReplace) => - val renamedVarToReplace = Var(varToReplace.name + "_renamed")(varToReplace.cypherType) + val renamedVarToReplace = + Var(varToReplace.name + "_renamed")(varToReplace.cypherType) val replaceOp = f(pScan.in) - val withAliasedVar = Project(varToReplace -> Some(renamedVarToReplace), replaceOp, replaceOp.solved.withFields(renamedVarToReplace.toField.get)) + val withAliasedVar = Project( + varToReplace -> Some(renamedVarToReplace), + replaceOp, + replaceOp.solved.withFields(renamedVarToReplace.toField.get) + ) val toSelect = replaceOp.fields - varToReplace + renamedVarToReplace - val selectOp = Select(toSelect.toList, withAliasedVar, replaceOp.solved.withFields(renamedVarToReplace.toField.get)) + val selectOp = Select( + toSelect.toList, + withAliasedVar, + replaceOp.solved.withFields(renamedVarToReplace.toField.get) + ) val joinExpr = Equals(varToReplace, renamedVarToReplace) - val joinOp = ValueJoin(pScan, selectOp, Set(joinExpr), pScan.solved ++ selectOp.solved) - Select((pScan.mapping.keySet ++ selectOp.fields).toList, joinOp, joinOp.solved) + val joinOp = ValueJoin( + pScan, + selectOp, + Set(joinExpr), + pScan.solved ++ selectOp.solved + ) + Select( + (pScan.mapping.keySet ++ selectOp.fields).toList, + joinOp, + joinOp.solved + ) } BottomUp[LogicalOperator](rewriter).transform(subtree) @@ -142,26 +206,30 @@ object LogicalOptimizer extends DirectCompilationStage[LogicalOperator, LogicalO private implicit class RichExpr(val expr: Expr) extends AnyVal { @scala.annotation.tailrec final def toField: Option[IRField] = expr match { - case v: Var => Some(IRField(v.name)(v.cypherType)) + case v: Var => Some(IRField(v.name)(v.cypherType)) case p: Property => p.propertyOwner.toField - case _ => None + case _ => None } } def discardScansForNonexistentLabels: PartialFunction[LogicalOperator, LogicalOperator] = { - case scan@PatternScan(NodePattern(CTNode(labels, _)), mapping, in, _) => + case scan @ PatternScan(NodePattern(CTNode(labels, _)), mapping, in, _) => def graphSchema = in.graph.schema def emptyRecords = { val fields = mapping.keySet.flatMap { case v: Var => Set(v) - case _ => Set.empty[Var] + case _ => Set.empty[Var] } EmptyRecords(fields, in, scan.solved) } - if ((labels.size == 1 && !graphSchema.labels.contains(labels.head)) || - (labels.size > 1 && !graphSchema.labelCombinations.combos.exists(labels.subsetOf(_)))) { + if ( + (labels.size == 1 && !graphSchema.labels.contains(labels.head)) || + (labels.size > 1 && !graphSchema.labelCombinations.combos.exists( + labels.subsetOf(_) + )) + ) { emptyRecords } else { scan @@ -188,13 +256,18 @@ object LogicalOptimizer extends DirectCompilationStage[LogicalOperator, LogicalO targetCombo <- targetCombos } yield (sourceCombo, relType, targetCombo) - combos.forall { - case(sourceCombo, relType, targetCombo) => - availablePatterns.exists { - case TripletPattern(CTNode(source, _), CTRelationship(rel, _), CTNode(target, _)) => - source == sourceCombo && rel.contains(relType) && target == targetCombo - case _ => false - } + combos.forall { case (sourceCombo, relType, targetCombo) => + availablePatterns.exists { + case TripletPattern( + CTNode(source, _), + CTRelationship(rel, _), + CTNode(target, _) + ) => + source == sourceCombo && rel.contains( + relType + ) && target == targetCombo + case _ => false + } } && combos.nonEmpty } @@ -214,12 +287,12 @@ object LogicalOptimizer extends DirectCompilationStage[LogicalOperator, LogicalO relType <- relTypes } yield (sourceCombo, relType) - combos.forall { - case(sourceCombo, relType) => - availablePatterns.exists { - case NodeRelPattern(CTNode(labels, _), CTRelationship(rel, _)) => labels == sourceCombo && rel.contains(relType) - case _ => false - } + combos.forall { case (sourceCombo, relType) => + availablePatterns.exists { + case NodeRelPattern(CTNode(labels, _), CTRelationship(rel, _)) => + labels == sourceCombo && rel.contains(relType) + case _ => false + } } && combos.nonEmpty } diff --git a/okapi-logical/src/main/scala/org/opencypher/okapi/logical/impl/LogicalPlanner.scala b/okapi-logical/src/main/scala/org/opencypher/okapi/logical/impl/LogicalPlanner.scala index 78d9c5ebca..e063556668 100644 --- a/okapi-logical/src/main/scala/org/opencypher/okapi/logical/impl/LogicalPlanner.scala +++ b/okapi-logical/src/main/scala/org/opencypher/okapi/logical/impl/LogicalPlanner.scala @@ -27,7 +27,12 @@ package org.opencypher.okapi.logical.impl import org.opencypher.okapi.api.types.{CTNode, CTRelationship} -import org.opencypher.okapi.impl.exception.{IllegalArgumentException, IllegalStateException, NotImplementedException, UnsupportedOperationException} +import org.opencypher.okapi.impl.exception.{ + IllegalArgumentException, + IllegalStateException, + NotImplementedException, + UnsupportedOperationException +} import org.opencypher.okapi.ir.api.block._ import org.opencypher.okapi.ir.api.expr._ import org.opencypher.okapi.ir.api.pattern._ @@ -36,15 +41,25 @@ import org.opencypher.okapi.ir.api.util.DirectCompilationStage import org.opencypher.okapi.ir.api.{Label, _} import org.opencypher.okapi.ir.impl.syntax.ExprSyntax._ import org.opencypher.okapi.ir.impl.util.VarConverters._ -import org.opencypher.okapi.logical.impl.exception.{InvalidCypherTypeException, InvalidDependencyException, InvalidPatternException} +import org.opencypher.okapi.logical.impl.exception.{ + InvalidCypherTypeException, + InvalidDependencyException, + InvalidPatternException +} import org.opencypher.v9_0.expressions.SemanticDirection.{INCOMING, OUTGOING} import scala.annotation.tailrec class LogicalPlanner(producer: LogicalOperatorProducer) - extends DirectCompilationStage[CypherQuery, LogicalOperator, LogicalPlannerContext] { - - override def process(ir: CypherQuery)(implicit context: LogicalPlannerContext): LogicalOperator = { + extends DirectCompilationStage[ + CypherQuery, + LogicalOperator, + LogicalPlannerContext + ] { + + override def process( + ir: CypherQuery + )(implicit context: LogicalPlannerContext): LogicalOperator = { ir match { case sq: SingleQuery => planModel(sq.model.result, sq.model) case UnionQuery(left, right, distinct) => @@ -53,28 +68,34 @@ class LogicalPlanner(producer: LogicalOperatorProducer) val isLeftGraph = leftOperator match { case _: ReturnGraph | _: GraphUnionAll => true - case _ => false + case _ => false } val isRightGraph = rightOperator match { case _: ReturnGraph | _: GraphUnionAll => true - case _ => false + case _ => false } (isLeftGraph, isRightGraph) match { case (true, true) => - if (distinct) throw UnsupportedOperationException("Distinct Union between graphs") + if (distinct) + throw UnsupportedOperationException( + "Distinct Union between graphs" + ) else GraphUnionAll(leftOperator, rightOperator) case (false, false) => val union = TabularUnionAll(leftOperator, rightOperator) if (distinct) Distinct(union.fields, union, union.solved) else union - case _ => throw UnsupportedOperationException("Union between graph and table is not supported") + case _ => + throw UnsupportedOperationException( + "Union between graph and table is not supported" + ) } } } - def planModel(block: ResultBlock, model: QueryModel)( - implicit context: LogicalPlannerContext + def planModel(block: ResultBlock, model: QueryModel)(implicit + context: LogicalPlannerContext ): LogicalOperator = { val first = block.after.head // there should only be one, right? @@ -86,12 +107,18 @@ class LogicalPlanner(producer: LogicalOperatorProducer) val fields = t.binds.orderedFields.map(f => Var(f.name)(f.cypherType)) producer.planSelect(fields, plan) case g: GraphResultBlock => - producer.planReturnGraph(producer.planFromGraph(resolveGraph(g.graph, plan.fields), plan)) + producer.planReturnGraph( + producer.planFromGraph(resolveGraph(g.graph, plan.fields), plan) + ) } } - final def planBlock(block: Block, model: QueryModel, plan: Option[LogicalOperator])( - implicit context: LogicalPlannerContext + final def planBlock( + block: Block, + model: QueryModel, + plan: Option[LogicalOperator] + )(implicit + context: LogicalPlannerContext ): LogicalOperator = { if (block.after.isEmpty) { // this is a leaf block, just plan it @@ -111,31 +138,45 @@ class LogicalPlanner(producer: LogicalOperatorProducer) // TODO: refactor to remove illegal state block.after .find(r => !_plan.solved.contains(r)) - .getOrElse(throw IllegalStateException("Found block with unsolved dependencies which cannot be solved.")) + .getOrElse( + throw IllegalStateException( + "Found block with unsolved dependencies which cannot be solved." + ) + ) } val dependency = planBlock(depRef, model, plan) planBlock(block, model, Some(dependency)) } } - def planLeaf(block: Block, model: QueryModel)(implicit context: LogicalPlannerContext): LogicalOperator = { + def planLeaf(block: Block, model: QueryModel)(implicit + context: LogicalPlannerContext + ): LogicalOperator = { block match { case SourceBlock(irGraph: IRCatalogGraph) => val qualifiedGraphName = irGraph.qualifiedGraphName - val logicalGraph = LogicalCatalogGraph(qualifiedGraphName, context.resolveSchema(qualifiedGraphName)) + val logicalGraph = LogicalCatalogGraph( + qualifiedGraphName, + context.resolveSchema(qualifiedGraphName) + ) if (context.inputRecordFields.isEmpty) { producer.planStart(logicalGraph) } else { - producer.planStartWithDrivingTable(logicalGraph, context.inputRecordFields) + producer.planStartWithDrivingTable( + logicalGraph, + context.inputRecordFields + ) } case x => - throw NotImplementedException(s"Support for leaf planning of $x not yet implemented. Tree:\n${x.pretty}") + throw NotImplementedException( + s"Support for leaf planning of $x not yet implemented. Tree:\n${x.pretty}" + ) } } - def planNonLeaf(block: Block, model: QueryModel, plan: LogicalOperator)( - implicit context: LogicalPlannerContext + def planNonLeaf(block: Block, model: QueryModel, plan: LogicalOperator)(implicit + context: LogicalPlannerContext ): LogicalOperator = { block match { case MatchBlock(_, pattern, where, optional, graph) => @@ -146,12 +187,14 @@ class LogicalPlanner(producer: LogicalOperatorProducer) plan match { // If the inner plan is a start, simply rewrite it to start with the required graph case Start(_, solved) => Start(lg, solved) - case _ => planFromGraph(lg, plan) + case _ => planFromGraph(lg, plan) } } // this plans both pattern and filter for convenience -- TODO: split up - val patternPlan = planMatchPattern(inputGraphPlan, pattern, where, graph) - if (optional) producer.planOptional(inputGraphPlan, patternPlan) else patternPlan + val patternPlan = + planMatchPattern(inputGraphPlan, pattern, where, graph) + if (optional) producer.planOptional(inputGraphPlan, patternPlan) + else patternPlan case ProjectBlock(_, Fields(fields), where, _, distinct) => val withFields = planFieldProjections(plan, fields) @@ -163,16 +206,18 @@ class LogicalPlanner(producer: LogicalOperatorProducer) } case OrderAndSliceBlock(_, sortItems, skip, limit, _) => - val orderOp = if (sortItems.nonEmpty) producer.planOrderBy(sortItems, plan) else plan + val orderOp = + if (sortItems.nonEmpty) producer.planOrderBy(sortItems, plan) + else plan val skipOp = skip match { case Some(expr) => producer.planSkip(expr, orderOp) - case None => orderOp + case None => orderOp } limit match { case Some(expr) => producer.planLimit(expr, skipOp) - case None => skipOp + case None => skipOp } case AggregationBlock(_, a, group, _) => @@ -182,43 +227,55 @@ class LogicalPlanner(producer: LogicalOperatorProducer) producer.planUnwind(list, variable, plan) case GraphResultBlock(_, graph) => - producer.planReturnGraph(producer.planFromGraph(resolveGraph(graph, plan.fields), plan)) + producer.planReturnGraph( + producer.planFromGraph(resolveGraph(graph, plan.fields), plan) + ) case x => - throw NotImplementedException(s"Support for logical planning of $x not yet implemented. Tree:\n${x.pretty}") + throw NotImplementedException( + s"Support for logical planning of $x not yet implemented. Tree:\n${x.pretty}" + ) } } - private def planFieldProjections(in: LogicalOperator, exprs: Map[IRField, Expr])( - implicit context: LogicalPlannerContext + private def planFieldProjections( + in: LogicalOperator, + exprs: Map[IRField, Expr] + )(implicit + context: LogicalPlannerContext ): LogicalOperator = { exprs.foldLeft(in) { case (acc, (f, ex: ExistsPatternExpr)) => - val existsPlan = producer.planExistsSubQuery(ex, acc, this (ex.ir)) + val existsPlan = producer.planExistsSubQuery(ex, acc, this(ex.ir)) producer.projectField(existsPlan.expr.targetField, f, existsPlan) case (acc, (f, expr)) => if (acc.solved.solves(f)) { acc } else { - producer.projectField(expr, f, expr.children.foldLeft(acc)((op, e) => planInnerSubquery(e, op))) + producer.projectField( + expr, + f, + expr.children.foldLeft(acc)((op, e) => planInnerSubquery(e, op)) + ) } } } - private def planFilter(in: LogicalOperator, where: Set[Expr])( - implicit context: LogicalPlannerContext + private def planFilter(in: LogicalOperator, where: Set[Expr])(implicit + context: LogicalPlannerContext ): LogicalOperator = { val filtersAndProjections = where.foldLeft(in) { case (acc, ex: ExistsPatternExpr) => - val predicate = producer.planExistsSubQuery(ex, acc, this (ex.ir)) + val predicate = producer.planExistsSubQuery(ex, acc, this(ex.ir)) producer.planFilter(ex, predicate) case (acc, predicate) => - val withInnerExpressions = predicate.children.foldLeft(acc)((acc, e) => planInnerSubquery(e, acc)) + val withInnerExpressions = + predicate.children.foldLeft(acc)((acc, e) => planInnerSubquery(e, acc)) producer.planFilter(predicate, withInnerExpressions) } @@ -226,13 +283,13 @@ class LogicalPlanner(producer: LogicalOperatorProducer) filtersAndProjections } - private def planInnerSubquery(expr: Expr, in: LogicalOperator)( - implicit context: LogicalPlannerContext + private def planInnerSubquery(expr: Expr, in: LogicalOperator)(implicit + context: LogicalPlannerContext ): LogicalOperator = { expr match { case ex: ExistsPatternExpr => - producer.planExistsSubQuery(ex, in, this (ex.ir)) + producer.planExistsSubQuery(ex, in, this(ex.ir)) case _ => expr.children.foldLeft(in)((op, e) => planInnerSubquery(e, op)) @@ -240,8 +297,8 @@ class LogicalPlanner(producer: LogicalOperatorProducer) } } - private def resolveGraph(graph: IRGraph, fieldsInScope: Set[Var])( - implicit context: LogicalPlannerContext + private def resolveGraph(graph: IRGraph, fieldsInScope: Set[Var])(implicit + context: LogicalPlannerContext ): LogicalGraph = { graph match { @@ -256,58 +313,91 @@ class LogicalPlanner(producer: LogicalOperatorProducer) val elementsToCreate = newPatternElements -- clonePatternElements - val clonedVarToInputVar: Map[Var, Var] = p.clones.map { case (clonedField, inputExpression) => - val inputVar = inputExpression match { - case v: Var => v - case other => throw IllegalArgumentException("CLONED expression to be a variable", other) - } - clonedField.toVar -> inputVar + val clonedVarToInputVar: Map[Var, Var] = p.clones.map { + case (clonedField, inputExpression) => + val inputVar = inputExpression match { + case v: Var => v + case other => + throw IllegalArgumentException( + "CLONED expression to be a variable", + other + ) + } + clonedField.toVar -> inputVar } - val newElements: Set[ConstructedElement] = elementsToCreate.map(e => extractConstructedElements(p.creates, e, baseElements.get(e))) + val newElements: Set[ConstructedElement] = + elementsToCreate.map(e => extractConstructedElements(p.creates, e, baseElements.get(e))) val setItems = { - val setPropertyItemsFromCreates = p.creates.properties.flatMap { case (irField, mapExpr) => - val v = irField.toVar - mapExpr.items.map { case (propertyKey, expr) => - SetPropertyItem(propertyKey, v, expr) - } + val setPropertyItemsFromCreates = p.creates.properties.flatMap { + case (irField, mapExpr) => + val v = irField.toVar + mapExpr.items.map { case (propertyKey, expr) => + SetPropertyItem(propertyKey, v, expr) + } } setPropertyItemsFromCreates ++ p.sets }.toList - LogicalPatternGraph(p.schema, clonedVarToInputVar, newElements, setItems, p.onGraphs, p.qualifiedGraphName) + LogicalPatternGraph( + p.schema, + clonedVarToInputVar, + newElements, + setItems, + p.onGraphs, + p.qualifiedGraphName + ) case IRCatalogGraph(qgn, schema) => LogicalCatalogGraph(qgn, schema) } } - private def extractConstructedElements(pattern: Pattern, e: IRField, baseField: Option[Var]) = e.cypherType match { + private def extractConstructedElements( + pattern: Pattern, + e: IRField, + baseField: Option[Var] + ) = e.cypherType match { case CTRelationship(relTypes, _) if relTypes.size <= 1 => val connection = pattern.topology(e) - ConstructedRelationship(e, connection.source, connection.target, relTypes.headOption, baseField) + ConstructedRelationship( + e, + connection.source, + connection.target, + relTypes.headOption, + baseField + ) case CTNode(labels, _) => ConstructedNode(e, labels.map(Label), baseField) case other => - throw InvalidCypherTypeException(s"Expected an element type (CTNode, CTRelationship), got $other") + throw InvalidCypherTypeException( + s"Expected an element type (CTNode, CTRelationship), got $other" + ) } - private def planStart(graph: IRGraph)(implicit context: LogicalPlannerContext): Start = { + private def planStart( + graph: IRGraph + )(implicit context: LogicalPlannerContext): Start = { val logicalGraph: LogicalGraph = resolveGraph(graph, Set.empty) producer.planStart(logicalGraph) } - private def planFromGraph(graph: LogicalGraph, prev: LogicalOperator)( - implicit context: LogicalPlannerContext + private def planFromGraph(graph: LogicalGraph, prev: LogicalOperator)(implicit + context: LogicalPlannerContext ): FromGraph = { producer.planFromGraph(graph, prev) } - private def planMatchPattern(plan: LogicalOperator, pattern: Pattern, where: Set[Expr], graph: IRGraph)( - implicit context: LogicalPlannerContext + private def planMatchPattern( + plan: LogicalOperator, + pattern: Pattern, + where: Set[Expr], + graph: IRGraph + )(implicit + context: LogicalPlannerContext ) = { val components = pattern.components.toSeq if (components.size == 1) { @@ -315,18 +405,23 @@ class LogicalPlanner(producer: LogicalOperatorProducer) val filteredPlan = planFilter(patternPlan, where) filteredPlan } else { - components.foldLeft(plan) { - case (base, component) => - val componentPlan = planComponentPattern(base, component, graph) - val predicates = where.filter(_.canEvaluate(componentPlan.fields)).filterNot(componentPlan.solved.predicates) - val filteredPlan = planFilter(componentPlan, predicates) - filteredPlan + components.foldLeft(plan) { case (base, component) => + val componentPlan = planComponentPattern(base, component, graph) + val predicates = where + .filter(_.canEvaluate(componentPlan.fields)) + .filterNot(componentPlan.solved.predicates) + val filteredPlan = planFilter(componentPlan, predicates) + filteredPlan } } } - private def planComponentPattern(plan: LogicalOperator, pattern: Pattern, graph: IRGraph)( - implicit context: LogicalPlannerContext + private def planComponentPattern( + plan: LogicalOperator, + pattern: Pattern, + graph: IRGraph + )(implicit + context: LogicalPlannerContext ): LogicalOperator = { // find all unsolved nodes from the pattern @@ -346,16 +441,20 @@ class LogicalPlanner(producer: LogicalOperatorProducer) val solved = nodes.intersect(plan.solved.fields) val unsolved = nodes -- solved - val (firstPlan, remaining) = if (solved.isEmpty) { // there is no connection to the previous plan - val field = nodes.head - if (plan.fields.nonEmpty) { // there are already fields in the previous plan, we need to plan a cartesian product - producer.planCartesianProduct(plan, nodePlan(planStart(graph), field)) -> nodes.tail - } else { // there are no previous results, it's safe to plan a node scan - nodePlan(plan, field) -> nodes.tail + val (firstPlan, remaining) = + if (solved.isEmpty) { // there is no connection to the previous plan + val field = nodes.head + if (plan.fields.nonEmpty) { // there are already fields in the previous plan, we need to plan a cartesian product + producer.planCartesianProduct( + plan, + nodePlan(planStart(graph), field) + ) -> nodes.tail + } else { // there are no previous results, it's safe to plan a node scan + nodePlan(plan, field) -> nodes.tail + } + } else { // we can connect to the previous plan + plan -> unsolved } - } else { // we can connect to the previous plan - plan -> unsolved - } val nodePlans: Set[LogicalOperator] = remaining.map { nodePlan(planStart(graph), _) @@ -374,34 +473,66 @@ class LogicalPlanner(producer: LogicalOperatorProducer) ): LogicalOperator = { val allSolved = disconnectedPlans.map(_.solved).reduce(_ ++ _) - val (r, c) = pattern.topology.collectFirst { - case (rel, conn: Connection) if !allSolved.solves(rel) => - rel -> conn - }.getOrElse( - // TODO: exclude case with type system - throw InvalidPatternException("Cannot plan an expansion that has already been solved") - ) - - val sourcePlan = disconnectedPlans.collectFirst { - case p if p.solved.solves(c.source) => p - }.getOrElse(throw InvalidDependencyException("Cannot plan expansion for unsolved source plan")) - val targetPlan = disconnectedPlans.collectFirst { - case p if p.solved.solves(c.target) => p - }.getOrElse(throw InvalidDependencyException("Cannot plan expansion for unsolved target plan")) + val (r, c) = pattern.topology + .collectFirst { + case (rel, conn: Connection) if !allSolved.solves(rel) => + rel -> conn + } + .getOrElse( + // TODO: exclude case with type system + throw InvalidPatternException( + "Cannot plan an expansion that has already been solved" + ) + ) + + val sourcePlan = disconnectedPlans + .collectFirst { + case p if p.solved.solves(c.source) => p + } + .getOrElse( + throw InvalidDependencyException( + "Cannot plan expansion for unsolved source plan" + ) + ) + val targetPlan = disconnectedPlans + .collectFirst { + case p if p.solved.solves(c.target) => p + } + .getOrElse( + throw InvalidDependencyException( + "Cannot plan expansion for unsolved target plan" + ) + ) val expand = c match { case v: VarLengthRelationship if v.upper.nonEmpty => val direction = v match { - case rel: DirectedVarLengthRelationship if rel.semanticDirection == OUTGOING => Outgoing - case rel: DirectedVarLengthRelationship if rel.semanticDirection == INCOMING => Incoming + case rel: DirectedVarLengthRelationship if rel.semanticDirection == OUTGOING => + Outgoing + case rel: DirectedVarLengthRelationship if rel.semanticDirection == INCOMING => + Incoming case _: UndirectedVarLengthRelationship => Undirected } if (v.upper.getOrElse(Integer.MAX_VALUE) < v.lower) { val solved = sourcePlan.solved ++ targetPlan.solved.withField(r) - EmptyRecords(sourcePlan.fields ++ targetPlan.fields, Start(targetPlan.graph, solved), solved) + EmptyRecords( + sourcePlan.fields ++ targetPlan.fields, + Start(targetPlan.graph, solved), + solved + ) } else { - producer.planBoundedVarLengthExpand(c.source, r, c.target, v.edgeType, direction, v.lower, v.upper.get, sourcePlan, targetPlan) + producer.planBoundedVarLengthExpand( + c.source, + r, + c.target, + v.edgeType, + direction, + v.lower, + v.upper.get, + sourcePlan, + targetPlan + ) } case _: UndirectedConnection if sourcePlan == targetPlan => @@ -411,28 +542,58 @@ class LogicalPlanner(producer: LogicalOperatorProducer) case _: CyclicRelationship => producer.planExpandInto(c.source, r, c.target, Outgoing, sourcePlan) - case rel: DirectedRelationship if sourcePlan == targetPlan && rel.semanticDirection == OUTGOING => + case rel: DirectedRelationship + if sourcePlan == targetPlan && rel.semanticDirection == OUTGOING => producer.planExpandInto(c.source, r, c.target, Outgoing, sourcePlan) - case rel: DirectedRelationship if sourcePlan == targetPlan && rel.semanticDirection == INCOMING => + case rel: DirectedRelationship + if sourcePlan == targetPlan && rel.semanticDirection == INCOMING => producer.planExpandInto(c.source, r, c.target, Incoming, sourcePlan) case rel: DirectedRelationship if rel.semanticDirection == OUTGOING => - producer.planExpand(c.source, r, c.target, Outgoing, sourcePlan, targetPlan) + producer.planExpand( + c.source, + r, + c.target, + Outgoing, + sourcePlan, + targetPlan + ) case rel: DirectedRelationship if rel.semanticDirection == INCOMING => - producer.planExpand(c.source, r, c.target, Incoming, sourcePlan, targetPlan) + producer.planExpand( + c.source, + r, + c.target, + Incoming, + sourcePlan, + targetPlan + ) case _: UndirectedConnection => - producer.planExpand(c.source, r, c.target, Undirected, sourcePlan, targetPlan) + producer.planExpand( + c.source, + r, + c.target, + Undirected, + sourcePlan, + targetPlan + ) } if (expand.solved.solves(pattern)) expand - else planExpansions((disconnectedPlans - sourcePlan - targetPlan) + expand, pattern, producer) + else + planExpansions( + (disconnectedPlans - sourcePlan - targetPlan) + expand, + pattern, + producer + ) } - private def nodePlan(plan: LogicalOperator, field: IRField)(implicit context: LogicalPlannerContext) = { + private def nodePlan(plan: LogicalOperator, field: IRField)(implicit + context: LogicalPlannerContext + ) = { producer.planNodeScan(field, plan) } } diff --git a/okapi-logical/src/main/scala/org/opencypher/okapi/logical/impl/LogicalPlannerContext.scala b/okapi-logical/src/main/scala/org/opencypher/okapi/logical/impl/LogicalPlannerContext.scala index 50cee273bb..1cc1fbc3c4 100644 --- a/okapi-logical/src/main/scala/org/opencypher/okapi/logical/impl/LogicalPlannerContext.scala +++ b/okapi-logical/src/main/scala/org/opencypher/okapi/logical/impl/LogicalPlannerContext.scala @@ -39,6 +39,8 @@ final case class LogicalPlannerContext( queryLocalCatalog: QueryLocalCatalog ) { - def resolveGraph(qgn: QualifiedGraphName): PropertyGraph = queryLocalCatalog.graph(qgn) - def resolveSchema(qgn: QualifiedGraphName): PropertyGraphSchema = queryLocalCatalog.schema(qgn) + def resolveGraph(qgn: QualifiedGraphName): PropertyGraph = + queryLocalCatalog.graph(qgn) + def resolveSchema(qgn: QualifiedGraphName): PropertyGraphSchema = + queryLocalCatalog.schema(qgn) } diff --git a/okapi-logical/src/main/scala/org/opencypher/okapi/logical/impl/SolvedQueryModel.scala b/okapi-logical/src/main/scala/org/opencypher/okapi/logical/impl/SolvedQueryModel.scala index 0d795690b9..7caaf682c0 100644 --- a/okapi-logical/src/main/scala/org/opencypher/okapi/logical/impl/SolvedQueryModel.scala +++ b/okapi-logical/src/main/scala/org/opencypher/okapi/logical/impl/SolvedQueryModel.scala @@ -42,8 +42,10 @@ case class SolvedQueryModel( // extension def withField(f: IRField): SolvedQueryModel = copy(fields = fields + f) def withFields(fs: IRField*): SolvedQueryModel = copy(fields = fields ++ fs) - def withPredicate(pred: Expr): SolvedQueryModel = copy(predicates = predicates + pred) - def withPredicates(preds: Expr*): SolvedQueryModel = copy(predicates = predicates ++ preds) + def withPredicate(pred: Expr): SolvedQueryModel = + copy(predicates = predicates + pred) + def withPredicates(preds: Expr*): SolvedQueryModel = + copy(predicates = predicates ++ preds) def ++(other: SolvedQueryModel): SolvedQueryModel = copy(fields ++ other.fields, predicates ++ other.predicates) diff --git a/okapi-logical/src/test/scala/org/opencypher/okapi/logical/impl/IrConstruction.scala b/okapi-logical/src/test/scala/org/opencypher/okapi/logical/impl/IrConstruction.scala index c9c8c4b21f..64ba91e1da 100644 --- a/okapi-logical/src/test/scala/org/opencypher/okapi/logical/impl/IrConstruction.scala +++ b/okapi-logical/src/test/scala/org/opencypher/okapi/logical/impl/IrConstruction.scala @@ -47,15 +47,20 @@ trait IrConstruction { def project( fields: Fields, after: List[Block] = List(leafBlock), - given: Set[Expr] = Set.empty) = + given: Set[Expr] = Set.empty + ) = ProjectBlock(after, fields, given, testGraph) - - private def testGraph()(implicit schema: PropertyGraphSchema = testGraphSchema) = + private def testGraph()(implicit + schema: PropertyGraphSchema = testGraphSchema + ) = IRCatalogGraph(testQualifiedGraphName, schema) protected def leafPlan: Start = - Start(LogicalCatalogGraph(testGraph.qualifiedGraphName, testGraph.schema), SolvedQueryModel.empty) + Start( + LogicalCatalogGraph(testGraph.qualifiedGraphName, testGraph.schema), + SolvedQueryModel.empty + ) protected def irFor(root: Block): SingleQuery = { val result = TableResultBlock( @@ -73,17 +78,23 @@ trait IrConstruction { MatchBlock(List(leafBlock), pattern, Set.empty, false, testGraph) implicit class RichString(queryText: String) { - def parseIR[T <: CypherStatement : ClassTag](graphsWithSchema: (GraphName, PropertyGraphSchema)*) - (implicit schema: PropertyGraphSchema = PropertyGraphSchema.empty): T = - ir(graphsWithSchema: _ *) match { + def parseIR[T <: CypherStatement: ClassTag]( + graphsWithSchema: (GraphName, PropertyGraphSchema)* + )(implicit schema: PropertyGraphSchema = PropertyGraphSchema.empty): T = + ir(graphsWithSchema: _*) match { case cq: T => cq - case other => throw new IllegalArgumentException(s"Cannot convert $other") + case other => + throw new IllegalArgumentException(s"Cannot convert $other") } - def asCypherQuery(graphsWithSchema: (GraphName, PropertyGraphSchema)*)(implicit schema: PropertyGraphSchema = PropertyGraphSchema.empty): SingleQuery = + def asCypherQuery(graphsWithSchema: (GraphName, PropertyGraphSchema)*)(implicit + schema: PropertyGraphSchema = PropertyGraphSchema.empty + ): SingleQuery = parseIR[SingleQuery](graphsWithSchema: _*) - def ir(graphsWithSchema: (GraphName, PropertyGraphSchema)*)(implicit schema: PropertyGraphSchema = PropertyGraphSchema.empty): CypherStatement = { + def ir(graphsWithSchema: (GraphName, PropertyGraphSchema)*)(implicit + schema: PropertyGraphSchema = PropertyGraphSchema.empty + ): CypherStatement = { val stmt = CypherParser(queryText)(CypherParser.defaultContext) val parameters = Map.empty[String, CypherValue] IRBuilder(stmt)( @@ -93,16 +104,21 @@ trait IrConstruction { SemanticState.clean, testGraph()(schema), qgnGenerator, - Map.empty.withDefaultValue(testGraphSource(graphsWithSchema :+ (testGraphName -> schema): _*)), + Map.empty.withDefaultValue( + testGraphSource(graphsWithSchema :+ (testGraphName -> schema): _*) + ), _ => ??? ) ) } - def irWithParams(params: (String, CypherValue)*)(implicit schema: PropertyGraphSchema = PropertyGraphSchema.empty): CypherStatement = { + def irWithParams(params: (String, CypherValue)*)(implicit + schema: PropertyGraphSchema = PropertyGraphSchema.empty + ): CypherStatement = { val stmt = CypherParser(queryText)(CypherParser.defaultContext) IRBuilder(stmt)( - IRBuilderContext.initial(queryText, + IRBuilderContext.initial( + queryText, params.toMap, SemanticState.clean, testGraph()(schema), @@ -114,5 +130,4 @@ trait IrConstruction { } } - } diff --git a/okapi-logical/src/test/scala/org/opencypher/okapi/logical/impl/SolvedQueryModelTest.scala b/okapi-logical/src/test/scala/org/opencypher/okapi/logical/impl/SolvedQueryModelTest.scala index 2559f3dba4..c45cd94246 100644 --- a/okapi-logical/src/test/scala/org/opencypher/okapi/logical/impl/SolvedQueryModelTest.scala +++ b/okapi-logical/src/test/scala/org/opencypher/okapi/logical/impl/SolvedQueryModelTest.scala @@ -48,7 +48,8 @@ class SolvedQueryModelTest extends BaseTestSuite with IrConstruction { } test("contains a block") { - val block = matchBlock(Pattern.empty.withElement('a).withElement('b).withElement('c)) + val block = + matchBlock(Pattern.empty.withElement('a).withElement('b).withElement('c)) val s = SolvedQueryModel.empty.withField('a).withFields('b, 'c) s.contains(block) shouldBe true @@ -71,7 +72,10 @@ class SolvedQueryModelTest extends BaseTestSuite with IrConstruction { test("solves") { val s = SolvedQueryModel.empty.withField('a).withFields('b, 'c) - val p = Pattern.empty.withElement('a -> CTNode).withElement('b -> CTNode).withElement('c -> CTNode) + val p = Pattern.empty + .withElement('a -> CTNode) + .withElement('b -> CTNode) + .withElement('c -> CTNode) s.solves(toField('a)) shouldBe true s.solves(toField('b)) shouldBe true @@ -83,20 +87,29 @@ class SolvedQueryModelTest extends BaseTestSuite with IrConstruction { it("can solve a relationship") { val s = SolvedQueryModel.empty - an [IllegalArgumentException] should be thrownBy s.solveRelationship('a) - s.solveRelationship('r -> CTRelationship) should equal(SolvedQueryModel.empty.withField('r -> CTRelationship)) + an[IllegalArgumentException] should be thrownBy s.solveRelationship('a) + s.solveRelationship('r -> CTRelationship) should equal( + SolvedQueryModel.empty.withField('r -> CTRelationship) + ) s.solveRelationship('r -> CTRelationship("KNOWS")) should equal( SolvedQueryModel.empty .withField('r -> CTRelationship) - .withPredicate(HasType(Var("r")(CTRelationship("KNOWS")), RelType("KNOWS"))) + .withPredicate( + HasType(Var("r")(CTRelationship("KNOWS")), RelType("KNOWS")) + ) ) - s.solveRelationship('r -> CTRelationship("KNOWS", "LOVES", "HATES")) should equal( + s.solveRelationship( + 'r -> CTRelationship("KNOWS", "LOVES", "HATES") + ) should equal( SolvedQueryModel.empty .withField('r -> CTRelationship) - .withPredicate(Ors( - HasType(RelationshipVar("r")(), RelType("KNOWS")), - HasType(RelationshipVar("r")(), RelType("LOVES")), - HasType(RelationshipVar("r")(), RelType("HATES")))) + .withPredicate( + Ors( + HasType(RelationshipVar("r")(), RelType("KNOWS")), + HasType(RelationshipVar("r")(), RelType("LOVES")), + HasType(RelationshipVar("r")(), RelType("HATES")) + ) + ) ) } } diff --git a/okapi-logical/src/test/scala/org/opencypher/okapi/logical/impl/logical/LogicalOptimizerTest.scala b/okapi-logical/src/test/scala/org/opencypher/okapi/logical/impl/logical/LogicalOptimizerTest.scala index b4b7f19e1a..3fc1fcd1c3 100644 --- a/okapi-logical/src/test/scala/org/opencypher/okapi/logical/impl/logical/LogicalOptimizerTest.scala +++ b/okapi-logical/src/test/scala/org/opencypher/okapi/logical/impl/logical/LogicalOptimizerTest.scala @@ -47,7 +47,8 @@ import scala.language.implicitConversions class LogicalOptimizerTest extends BaseTestSuite with IrConstruction with TreeVerificationSupport { val emptySqm: SolvedQueryModel = SolvedQueryModel.empty - val logicalGraph = LogicalCatalogGraph(testQualifiedGraphName, PropertyGraphSchema.empty) + val logicalGraph = + LogicalCatalogGraph(testQualifiedGraphName, PropertyGraphSchema.empty) val emptySchema: PropertyGraphSchema = PropertyGraphSchema.empty val emptyGraph = TestGraph(emptySchema) @@ -56,16 +57,23 @@ class LogicalOptimizerTest extends BaseTestSuite with IrConstruction with TreeVe """|MATCH (a:Animal) |RETURN a""".stripMargin val plan = logicalPlan(query, emptyGraph) - val optimizedLogicalPlan = LogicalOptimizer(plan)(plannerContext(emptyGraph)) + val optimizedLogicalPlan = + LogicalOptimizer(plan)(plannerContext(emptyGraph)) val expected = Select( List(Var("a")(CTNode(Set("Animal")))), EmptyRecords( Set(Var("a")(CTNode(Set("Animal")))), Start(logicalGraph, emptySqm), - SolvedQueryModel(Set(IRField("a")(CTNode(Set("Animal")))), Set(HasLabel(Var("a")(CTNode(Set("Animal"))), Label("Animal")))) + SolvedQueryModel( + Set(IRField("a")(CTNode(Set("Animal")))), + Set(HasLabel(Var("a")(CTNode(Set("Animal"))), Label("Animal"))) + ) ), - SolvedQueryModel(Set(IRField("a")(CTNode)), Set(HasLabel(Var("a")(CTNode), Label("Animal")))) + SolvedQueryModel( + Set(IRField("a")(CTNode)), + Set(HasLabel(Var("a")(CTNode), Label("Animal"))) + ) ) optimizedLogicalPlan should equalWithTracing(expected) @@ -75,11 +83,14 @@ class LogicalOptimizerTest extends BaseTestSuite with IrConstruction with TreeVe val query = """|MATCH (a:Animal:Astronaut) |RETURN a""".stripMargin - val schema = PropertyGraphSchema.empty.withNodePropertyKeys("Animal")().withNodePropertyKeys("Astronaut")() + val schema = PropertyGraphSchema.empty + .withNodePropertyKeys("Animal")() + .withNodePropertyKeys("Astronaut")() val logicalGraph = LogicalCatalogGraph(testQualifiedGraphName, schema) val plan = logicalPlan(query, TestGraph(schema)) - val optimizedLogicalPlan = LogicalOptimizer(plan)(plannerContext(TestGraph(schema))) + val optimizedLogicalPlan = + LogicalOptimizer(plan)(plannerContext(TestGraph(schema))) val expected = Select( List(Var("a")(CTNode(Set("Animal", "Astronaut")))), @@ -89,8 +100,14 @@ class LogicalOptimizerTest extends BaseTestSuite with IrConstruction with TreeVe SolvedQueryModel( Set(IRField("a")(CTNode(Set("Astronaut", "Animal")))), Set( - HasLabel(Var("a")(CTNode(Set("Astronaut", "Animal"))), Label("Astronaut")), - HasLabel(Var("a")(CTNode(Set("Astronaut", "Animal"))), Label("Animal")) + HasLabel( + Var("a")(CTNode(Set("Astronaut", "Animal"))), + Label("Astronaut") + ), + HasLabel( + Var("a")(CTNode(Set("Astronaut", "Animal"))), + Label("Animal") + ) ) ) ), @@ -98,7 +115,9 @@ class LogicalOptimizerTest extends BaseTestSuite with IrConstruction with TreeVe Set(IRField("a")(CTNode)), Set( HasLabel(Var("a")(CTNode), Label("Animal")), - HasLabel(Var("a")(CTNode), Label("Astronaut")))) + HasLabel(Var("a")(CTNode), Label("Astronaut")) + ) + ) ) optimizedLogicalPlan should equalWithTracing(expected) @@ -107,8 +126,14 @@ class LogicalOptimizerTest extends BaseTestSuite with IrConstruction with TreeVe describe("replace cartesian with ValueJoin") { it("should replace cross with value join if filter is present") { - val startA = Start(LogicalCatalogGraph(testQualifiedGraphName, testGraphSchema), SolvedQueryModel.empty) - val startB = Start(LogicalCatalogGraph(testQualifiedGraphName, testGraphSchema), SolvedQueryModel.empty) + val startA = Start( + LogicalCatalogGraph(testQualifiedGraphName, testGraphSchema), + SolvedQueryModel.empty + ) + val startB = Start( + LogicalCatalogGraph(testQualifiedGraphName, testGraphSchema), + SolvedQueryModel.empty + ) val varA = Var("a")(CTNode) val propA = expr.ElementProperty(varA, PropertyKey("name"))(CTString) val varB = Var("b")(CTNode) @@ -117,24 +142,42 @@ class LogicalOptimizerTest extends BaseTestSuite with IrConstruction with TreeVe val irFieldA = IRField(varA.name)(varA.cypherType) val irFieldB = IRField(varB.name)(varB.cypherType) - val scanA = PatternScan.nodeScan(varA, startA, SolvedQueryModel(Set(irFieldA))) - val scanB = PatternScan.nodeScan(varB, startB, SolvedQueryModel(Set(irFieldB))) - val cartesian = CartesianProduct(scanA, scanB, SolvedQueryModel(Set(irFieldA, irFieldB))) - val filter = Filter(equals, cartesian, SolvedQueryModel(Set(irFieldA, irFieldB))) + val scanA = + PatternScan.nodeScan(varA, startA, SolvedQueryModel(Set(irFieldA))) + val scanB = + PatternScan.nodeScan(varB, startB, SolvedQueryModel(Set(irFieldB))) + val cartesian = CartesianProduct( + scanA, + scanB, + SolvedQueryModel(Set(irFieldA, irFieldB)) + ) + val filter = + Filter(equals, cartesian, SolvedQueryModel(Set(irFieldA, irFieldB))) - val optimizedPlan = BottomUp[LogicalOperator](LogicalOptimizer.replaceCartesianWithValueJoin).transform(filter) + val optimizedPlan = BottomUp[LogicalOperator]( + LogicalOptimizer.replaceCartesianWithValueJoin + ).transform(filter) val projectA = Project(propA -> None, scanA, scanA.solved) val projectB = Project(propB -> None, scanB, scanB.solved) - val solved = SolvedQueryModel(Set(irFieldA, irFieldB)).withPredicate(equals) + val solved = + SolvedQueryModel(Set(irFieldA, irFieldB)).withPredicate(equals) val valueJoin = ValueJoin(projectA, projectB, Set(equals), solved) optimizedPlan should equalWithTracing(valueJoin) } - it("should replace cross with value join if filter with flipped predicate is present") { - val startA = Start(LogicalCatalogGraph(testQualifiedGraphName, testGraphSchema), SolvedQueryModel.empty) - val startB = Start(LogicalCatalogGraph(testQualifiedGraphName, testGraphSchema), SolvedQueryModel.empty) + it( + "should replace cross with value join if filter with flipped predicate is present" + ) { + val startA = Start( + LogicalCatalogGraph(testQualifiedGraphName, testGraphSchema), + SolvedQueryModel.empty + ) + val startB = Start( + LogicalCatalogGraph(testQualifiedGraphName, testGraphSchema), + SolvedQueryModel.empty + ) val varA = Var("a")(CTNode) val propA = expr.ElementProperty(varA, PropertyKey("name"))(CTString) val varB = Var("b")(CTNode) @@ -143,17 +186,27 @@ class LogicalOptimizerTest extends BaseTestSuite with IrConstruction with TreeVe val irFieldA = IRField(varA.name)(varA.cypherType) val irFieldB = IRField(varB.name)(varB.cypherType) - val scanA = PatternScan.nodeScan(varA, startA, SolvedQueryModel(Set(irFieldA))) - val scanB = PatternScan.nodeScan(varB, startB, SolvedQueryModel(Set(irFieldB))) - val cartesian = CartesianProduct(scanA, scanB, SolvedQueryModel(Set(irFieldA, irFieldB))) - val filter = Filter(equals, cartesian, SolvedQueryModel(Set(irFieldA, irFieldB))) + val scanA = + PatternScan.nodeScan(varA, startA, SolvedQueryModel(Set(irFieldA))) + val scanB = + PatternScan.nodeScan(varB, startB, SolvedQueryModel(Set(irFieldB))) + val cartesian = CartesianProduct( + scanA, + scanB, + SolvedQueryModel(Set(irFieldA, irFieldB)) + ) + val filter = + Filter(equals, cartesian, SolvedQueryModel(Set(irFieldA, irFieldB))) - val optimizedPlan = BottomUp[LogicalOperator](LogicalOptimizer.replaceCartesianWithValueJoin).transform(filter) + val optimizedPlan = BottomUp[LogicalOperator]( + LogicalOptimizer.replaceCartesianWithValueJoin + ).transform(filter) val flippedEquals = Equals(propA, propB) val projectA = Project(propA -> None, scanA, scanA.solved) val projectB = Project(propB -> None, scanB, scanB.solved) - val solved = SolvedQueryModel(Set(irFieldA, irFieldB)).withPredicate(flippedEquals) + val solved = + SolvedQueryModel(Set(irFieldA, irFieldB)).withPredicate(flippedEquals) val valueJoin = ValueJoin(projectA, projectB, Set(flippedEquals), solved) optimizedPlan should equalWithTracing(valueJoin) @@ -161,25 +214,45 @@ class LogicalOptimizerTest extends BaseTestSuite with IrConstruction with TreeVe it("should replace cross with value join for driving tables") { val nameField = 'name -> CTString - val startDrivingTable = impl.DrivingTable(LogicalCatalogGraph(testQualifiedGraphName, testGraphSchema), Set(nameField), SolvedQueryModel.empty.withField(nameField)) + val startDrivingTable = impl.DrivingTable( + LogicalCatalogGraph(testQualifiedGraphName, testGraphSchema), + Set(nameField), + SolvedQueryModel.empty.withField(nameField) + ) - val startB = Start(LogicalCatalogGraph(testQualifiedGraphName, testGraphSchema), SolvedQueryModel.empty) + val startB = Start( + LogicalCatalogGraph(testQualifiedGraphName, testGraphSchema), + SolvedQueryModel.empty + ) val varB = Var("b")(CTNode) val propB = expr.ElementProperty(varB, PropertyKey("name"))(CTString) val equals = Equals(nameField, propB) val irFieldB = IRField(varB.name)(varB.cypherType) - val scanB = PatternScan.nodeScan(varB, startB, SolvedQueryModel(Set(irFieldB))) - val cartesian = CartesianProduct(startDrivingTable, scanB, SolvedQueryModel(Set(nameField, irFieldB))) - val filter = Filter(equals, cartesian, SolvedQueryModel(Set(nameField, irFieldB))) + val scanB = + PatternScan.nodeScan(varB, startB, SolvedQueryModel(Set(irFieldB))) + val cartesian = CartesianProduct( + startDrivingTable, + scanB, + SolvedQueryModel(Set(nameField, irFieldB)) + ) + val filter = + Filter(equals, cartesian, SolvedQueryModel(Set(nameField, irFieldB))) - val optimizedPlan = BottomUp[LogicalOperator](LogicalOptimizer.replaceCartesianWithValueJoin).transform(filter) + val optimizedPlan = BottomUp[LogicalOperator]( + LogicalOptimizer.replaceCartesianWithValueJoin + ).transform(filter) - val projectName = Project(toVar(nameField) -> None, startDrivingTable, startDrivingTable.solved) + val projectName = Project( + toVar(nameField) -> None, + startDrivingTable, + startDrivingTable.solved + ) val projectB = Project(propB -> None, scanB, scanB.solved) - val solved = SolvedQueryModel(Set(nameField, irFieldB)).withPredicate(equals) + val solved = + SolvedQueryModel(Set(nameField, irFieldB)).withPredicate(equals) val valueJoin = ValueJoin(projectName, projectB, Set(equals), solved) optimizedPlan should equalWithTracing(valueJoin) @@ -193,7 +266,10 @@ class LogicalOptimizerTest extends BaseTestSuite with IrConstruction with TreeVe .withRelationshipPropertyKeys("B")() it("inserts NodeRelPatterns") { - val pattern = NodeRelPattern(CTNode(Set("A"), Some(testQualifiedGraphName)), CTRelationship(Set("B"), Some(testQualifiedGraphName))) + val pattern = NodeRelPattern( + CTNode(Set("A"), Some(testQualifiedGraphName)), + CTRelationship(Set("B"), Some(testQualifiedGraphName)) + ) val graph = TestGraph(schema, Set(pattern)) val plan = logicalPlan( @@ -208,19 +284,25 @@ class LogicalOptimizerTest extends BaseTestSuite with IrConstruction with TreeVe optimizedPlan.exists { case _: Expand => true - case _ => false + case _ => false } should be(false) optimizedPlan.exists { case PatternScan(otherPattern, map, _, _) => pattern == otherPattern && - map == Map(Var("a")(CTNode) -> pattern.nodeElement, Var("b")(CTRelationship) -> pattern.relElement) + map == Map( + Var("a")(CTNode) -> pattern.nodeElement, + Var("b")(CTRelationship) -> pattern.relElement + ) case _ => false } should be(true) } it("inserts connecting NodeRelPatterns") { - val pattern = NodeRelPattern(CTNode(Set("A"), Some(testQualifiedGraphName)), CTRelationship(Set("B"), Some(testQualifiedGraphName))) + val pattern = NodeRelPattern( + CTNode(Set("A"), Some(testQualifiedGraphName)), + CTRelationship(Set("B"), Some(testQualifiedGraphName)) + ) val graph = TestGraph(schema, Set(pattern)) val plan = logicalPlan( @@ -234,20 +316,26 @@ class LogicalOptimizerTest extends BaseTestSuite with IrConstruction with TreeVe optimizedPlan.occourences[PatternScan] should be(4) optimizedPlan.exists { case _: Expand => true - case _ => false + case _ => false } should be(false) optimizedPlan.exists { case PatternScan(otherPattern, map, _, _) => pattern == otherPattern && - map == Map(Var("a")(CTNode) -> pattern.nodeElement, Var("b1")(CTRelationship) -> pattern.relElement) + map == Map( + Var("a")(CTNode) -> pattern.nodeElement, + Var("b1")(CTRelationship) -> pattern.relElement + ) case _ => false } should be(true) optimizedPlan.exists { case PatternScan(otherPattern, map, _, _) => pattern == otherPattern && - map == Map(Var("a")(CTNode) -> pattern.nodeElement, Var("b2")(CTRelationship) -> pattern.relElement) + map == Map( + Var("a")(CTNode) -> pattern.nodeElement, + Var("b2")(CTRelationship) -> pattern.relElement + ) case _ => false } should be(true) } @@ -273,13 +361,17 @@ class LogicalOptimizerTest extends BaseTestSuite with IrConstruction with TreeVe optimizedPlan.exists { case _: Expand => true - case _ => false + case _ => false } should be(false) optimizedPlan.exists { case PatternScan(otherPattern, map, _, _) => pattern == otherPattern && - map == Map(Var("a")(CTNode) -> pattern.sourceElement, Var("b")(CTRelationship) -> pattern.relElement, Var("c")(CTNode) -> pattern.targetElement) + map == Map( + Var("a")(CTNode) -> pattern.sourceElement, + Var("b")(CTRelationship) -> pattern.relElement, + Var("c")(CTNode) -> pattern.targetElement + ) case _ => false } should be(true) } @@ -304,24 +396,34 @@ class LogicalOptimizerTest extends BaseTestSuite with IrConstruction with TreeVe optimizedPlan.exists { case _: Expand => true - case _ => false + case _ => false } should be(false) optimizedPlan.exists { case PatternScan(otherPattern, map, _, _) => pattern == otherPattern && - map == Map(Var("a")(CTNode) -> pattern.sourceElement, Var("b1")(CTRelationship) -> pattern.relElement, Var("c1")(CTNode) -> pattern.targetElement) + map == Map( + Var("a")(CTNode) -> pattern.sourceElement, + Var("b1")(CTRelationship) -> pattern.relElement, + Var("c1")(CTNode) -> pattern.targetElement + ) case _ => false } should be(true) optimizedPlan.exists { case PatternScan(otherPattern, map, _, _) => pattern == otherPattern && - map == Map(Var("a")(CTNode) -> pattern.sourceElement, Var("b2")(CTRelationship) -> pattern.relElement, Var("c2")(CTNode) -> pattern.targetElement) + map == Map( + Var("a")(CTNode) -> pattern.sourceElement, + Var("b2")(CTRelationship) -> pattern.relElement, + Var("c2")(CTNode) -> pattern.targetElement + ) case _ => false } should be(true) } - it("does not insert node rel patterns if not all node label combos are covered") { + it( + "does not insert node rel patterns if not all node label combos are covered" + ) { val pattern = NodeRelPattern(CTNode("Person"), CTRelationship("KNOWS")) val schema = PropertyGraphSchema.empty .withNodePropertyKeys("Person")() @@ -340,7 +442,7 @@ class LogicalOptimizerTest extends BaseTestSuite with IrConstruction with TreeVe optimizedPlan.exists { case PatternScan(_: NodeRelPattern, _, _, _) => true - case _ => false + case _ => false } should be(false) } @@ -363,8 +465,14 @@ class LogicalOptimizerTest extends BaseTestSuite with IrConstruction with TreeVe optimizedPlan.occourences[ValueJoin] should be(0) } - it("does not insert triplet patterns if not all node label combos are covered") { - val pattern = TripletPattern(CTNode("Person"), CTRelationship("KNOWS"), CTNode("Person")) + it( + "does not insert triplet patterns if not all node label combos are covered" + ) { + val pattern = TripletPattern( + CTNode("Person"), + CTRelationship("KNOWS"), + CTNode("Person") + ) val schema = PropertyGraphSchema.empty .withNodePropertyKeys("Person")() .withNodePropertyKeys("Person", "Employee")() @@ -382,12 +490,16 @@ class LogicalOptimizerTest extends BaseTestSuite with IrConstruction with TreeVe optimizedPlan.exists { case PatternScan(_: TripletPattern, _, _, _) => true - case _ => false + case _ => false } should be(false) } it("does not insert triplet patterns if not all rel types are covered") { - val pattern = TripletPattern(CTNode("Person"), CTRelationship("KNOWS"), CTNode("Person")) + val pattern = TripletPattern( + CTNode("Person"), + CTRelationship("KNOWS"), + CTNode("Person") + ) val schema = PropertyGraphSchema.empty .withNodePropertyKeys("Person")() .withRelationshipPropertyKeys("KNOWS")() @@ -405,14 +517,13 @@ class LogicalOptimizerTest extends BaseTestSuite with IrConstruction with TreeVe optimizedPlan.exists { case PatternScan(_: TripletPattern, _, _, _) => true - case _ => false + case _ => false } should be(false) } } def plannerContext(graph: PropertyGraph): LogicalPlannerContext = { - val catalog = QueryLocalCatalog - .empty + val catalog = QueryLocalCatalog.empty .withGraph(testQualifiedGraphName, graph) .withSchema(testQualifiedGraphName, graph.schema) @@ -424,7 +535,10 @@ class LogicalOptimizerTest extends BaseTestSuite with IrConstruction with TreeVe ) } - private def logicalPlan(query: String, graph: PropertyGraph): LogicalOperator = { + private def logicalPlan( + query: String, + graph: PropertyGraph + ): LogicalOperator = { val producer = new LogicalOperatorProducer val logicalPlanner = new LogicalPlanner(producer) val ir = query.asCypherQuery(testGraphName -> graph.schema)(graph.schema) diff --git a/okapi-logical/src/test/scala/org/opencypher/okapi/logical/impl/logical/LogicalPlannerTest.scala b/okapi-logical/src/test/scala/org/opencypher/okapi/logical/impl/logical/LogicalPlannerTest.scala index a953c9377b..097f7989ac 100644 --- a/okapi-logical/src/test/scala/org/opencypher/okapi/logical/impl/logical/LogicalPlannerTest.scala +++ b/okapi-logical/src/test/scala/org/opencypher/okapi/logical/impl/logical/LogicalPlannerTest.scala @@ -51,10 +51,13 @@ class LogicalPlannerTest extends BaseTestSuite with IrConstruction { val irFieldG: IRField = IRField("g")(CTNode("Group")) val irFieldR: IRField = IRField("r")(CTRelationship) - val varA: Var = Var("a")(CTNode(Set("Administrator"), Some(testQualifiedGraphName))) - val varB: Var = Var("b")(CTNode(Set.empty[String], Some(testQualifiedGraphName))) + val varA: Var = + Var("a")(CTNode(Set("Administrator"), Some(testQualifiedGraphName))) + val varB: Var = + Var("b")(CTNode(Set.empty[String], Some(testQualifiedGraphName))) val varG: Var = Var("g")(CTNode(Set("Group"), Some(testQualifiedGraphName))) - val varR: Var = Var("r")(CTRelationship(Set.empty[String], Some(testQualifiedGraphName))) + val varR: Var = + Var("r")(CTRelationship(Set.empty[String], Some(testQualifiedGraphName))) val aLabelPredicate: HasLabel = HasLabel(varA, Label("Administrator")) @@ -67,8 +70,7 @@ class LogicalPlannerTest extends BaseTestSuite with IrConstruction { } it("converts match block") { - val pattern = Pattern - .empty + val pattern = Pattern.empty .withElement(irFieldA) .withElement(irFieldB) .withElement(irFieldR) @@ -76,20 +78,31 @@ class LogicalPlannerTest extends BaseTestSuite with IrConstruction { val block = matchBlock(pattern) - - val scan1 = PatternScan.nodeScan(irFieldA, leafPlan, emptySqm.withField(irFieldA).withPredicate(aLabelPredicate)) - val scan2 = PatternScan.nodeScan(irFieldB, leafPlan, emptySqm.withField(irFieldB)) + val scan1 = PatternScan.nodeScan( + irFieldA, + leafPlan, + emptySqm.withField(irFieldA).withPredicate(aLabelPredicate) + ) + val scan2 = + PatternScan.nodeScan(irFieldB, leafPlan, emptySqm.withField(irFieldB)) val ir = irFor(block) val result = plan(ir) - val expected = Expand(irFieldA, irFieldR, irFieldB, Outgoing, scan1, scan2, SolvedQueryModel(Set(irFieldA, irFieldB, irFieldR), Set(aLabelPredicate))) + val expected = Expand( + irFieldA, + irFieldR, + irFieldB, + Outgoing, + scan1, + scan2, + SolvedQueryModel(Set(irFieldA, irFieldB, irFieldR), Set(aLabelPredicate)) + ) result should equalWithoutResult(expected) } it("converts cyclic match block") { - val pattern = Pattern - .empty + val pattern = Pattern.empty .withElement(irFieldA) .withElement(irFieldR) .withConnection(irFieldR, CyclicRelationship(irFieldA)) @@ -97,34 +110,56 @@ class LogicalPlannerTest extends BaseTestSuite with IrConstruction { val block = matchBlock(pattern) val ir = irFor(block) - val scan = PatternScan.nodeScan(irFieldA, leafPlan, emptySqm.withField(irFieldA).withPredicate(aLabelPredicate)) - val expandInto = ExpandInto(irFieldA, irFieldR, irFieldA, Outgoing, scan, SolvedQueryModel(Set(irFieldA, irFieldR), Set(aLabelPredicate))) + val scan = PatternScan.nodeScan( + irFieldA, + leafPlan, + emptySqm.withField(irFieldA).withPredicate(aLabelPredicate) + ) + val expandInto = ExpandInto( + irFieldA, + irFieldR, + irFieldA, + Outgoing, + scan, + SolvedQueryModel(Set(irFieldA, irFieldR), Set(aLabelPredicate)) + ) plan(ir) should equalWithoutResult(expandInto) } it("converts project block") { - val fields = Fields(Map(toField('a) -> ElementProperty('n, PropertyKey("prop"))(CTFloat))) + val fields = Fields( + Map(toField('a) -> ElementProperty('n, PropertyKey("prop"))(CTFloat)) + ) val block = project(fields) val result = plan(irFor(block)) val expected = Project( - ElementProperty('n, PropertyKey("prop"))(CTFloat) -> Some('a), // n is a dangling reference here + ElementProperty('n, PropertyKey("prop"))(CTFloat) -> Some( + 'a + ), // n is a dangling reference here leafPlan, - emptySqm.withFields('a)) + emptySqm.withFields('a) + ) result should equalWithoutResult(expected) } it("plans query") { - val ir = "MATCH (a:Administrator)-[r]->(g:Group) WHERE g.name = $foo RETURN a.name".irWithParams( - "foo" -> CypherString("test")) + val ir = + "MATCH (a:Administrator)-[r]->(g:Group) WHERE g.name = $foo RETURN a.name" + .irWithParams("foo" -> CypherString("test")) val result = plan(ir) val expected = Project( - ElementProperty(varA, PropertyKey("name"))(CTNull) -> Some(Var("a.name")(CTNull)), + ElementProperty(varA, PropertyKey("name"))(CTNull) -> Some( + Var("a.name")(CTNull) + ), Filter( - Equals(ElementProperty(varG, PropertyKey("name"))(CTNull), Param("foo")(CTString)), + Equals( + ElementProperty(varG, PropertyKey("name"))(CTNull), + Param("foo")(CTString) + ), Expand( varA, varR, @@ -132,12 +167,27 @@ class LogicalPlannerTest extends BaseTestSuite with IrConstruction { Outgoing, PatternScan.nodeScan( varA, - Start(LogicalCatalogGraph(testQualifiedGraphName, PropertyGraphSchema.empty), emptySqm), - SolvedQueryModel(Set(irFieldA), Set(HasLabel(varA, Label("Administrator")))) + Start( + LogicalCatalogGraph( + testQualifiedGraphName, + PropertyGraphSchema.empty + ), + emptySqm + ), + SolvedQueryModel( + Set(irFieldA), + Set(HasLabel(varA, Label("Administrator"))) + ) ), PatternScan.nodeScan( varG, - Start(LogicalCatalogGraph(testQualifiedGraphName, PropertyGraphSchema.empty), emptySqm), + Start( + LogicalCatalogGraph( + testQualifiedGraphName, + PropertyGraphSchema.empty + ), + emptySqm + ), SolvedQueryModel(Set(irFieldG), Set(HasLabel(varG, Label("Group")))) ), SolvedQueryModel( @@ -153,7 +203,10 @@ class LogicalPlannerTest extends BaseTestSuite with IrConstruction { Set( HasLabel(varA, Label("Administrator")), HasLabel(varG, Label("Group")), - Equals(ElementProperty(varG, PropertyKey("name"))(CTNull), Param("foo")(CTString)) + Equals( + ElementProperty(varG, PropertyKey("name"))(CTNull), + Param("foo")(CTString) + ) ) ) ), @@ -162,7 +215,10 @@ class LogicalPlannerTest extends BaseTestSuite with IrConstruction { Set( HasLabel(varA, Label("Administrator")), HasLabel(varG, Label("Group")), - Equals(ElementProperty(varG, PropertyKey("name"))(CTNull), Param("foo")(CTString)) + Equals( + ElementProperty(varG, PropertyKey("name"))(CTNull), + Param("foo")(CTString) + ) ) ) ) @@ -174,15 +230,22 @@ class LogicalPlannerTest extends BaseTestSuite with IrConstruction { .withNodePropertyKeys("Group")("name" -> CTString) .withNodePropertyKeys("Administrator")("name" -> CTFloat) - val ir = "MATCH (a:Administrator)-[r]->(g:Group) WHERE g.name = $foo RETURN a.name".irWithParams( - "foo" -> CypherString("test")) + val ir = + "MATCH (a:Administrator)-[r]->(g:Group) WHERE g.name = $foo RETURN a.name" + .irWithParams("foo" -> CypherString("test")) val result = plan(ir, schema) val expected = Project( - ElementProperty(Var("a")(CTNode(Set("Administrator"))), PropertyKey("name"))(CTFloat) -> Some(Var("a.name")(CTFloat)), + ElementProperty( + Var("a")(CTNode(Set("Administrator"))), + PropertyKey("name") + )(CTFloat) -> Some(Var("a.name")(CTFloat)), Filter( - Equals(ElementProperty(varG, PropertyKey("name"))(CTString), Param("foo")(CTString)), + Equals( + ElementProperty(varG, PropertyKey("name"))(CTString), + Param("foo")(CTString) + ), Expand( varA, Var("r")(CTRelationship), @@ -197,7 +260,10 @@ class LogicalPlannerTest extends BaseTestSuite with IrConstruction { ), emptySqm ), - SolvedQueryModel(Set(irFieldA), Set(HasLabel(varA, Label("Administrator")))) + SolvedQueryModel( + Set(irFieldA), + Set(HasLabel(varA, Label("Administrator"))) + ) ), PatternScan.nodeScan( varG, @@ -210,17 +276,23 @@ class LogicalPlannerTest extends BaseTestSuite with IrConstruction { ), SolvedQueryModel(Set(irFieldG), Set(HasLabel(varG, Label("Group")))) ), - SolvedQueryModel(Set(irFieldA, irFieldG, irFieldR), Set( - HasLabel(varA, Label("Administrator")), - HasLabel(varG, Label("Group")) - )) + SolvedQueryModel( + Set(irFieldA, irFieldG, irFieldR), + Set( + HasLabel(varA, Label("Administrator")), + HasLabel(varG, Label("Group")) + ) + ) ), SolvedQueryModel( Set(irFieldA, irFieldG, irFieldR), Set( HasLabel(varA, Label("Administrator")), HasLabel(varG, Label("Group")), - Equals(ElementProperty(varG, PropertyKey("name"))(CTString), Param("foo")(CTString)) + Equals( + ElementProperty(varG, PropertyKey("name"))(CTString), + Param("foo")(CTString) + ) ) ) ), @@ -229,7 +301,10 @@ class LogicalPlannerTest extends BaseTestSuite with IrConstruction { Set( HasLabel(varA, Label("Administrator")), HasLabel(varG, Label("Group")), - Equals(ElementProperty(varG, PropertyKey("name"))(CTString), Param("foo")(CTString)) + Equals( + ElementProperty(varG, PropertyKey("name"))(CTString), + Param("foo")(CTString) + ) ) ) ) @@ -239,35 +314,52 @@ class LogicalPlannerTest extends BaseTestSuite with IrConstruction { it("plans query with negation") { val ir = - "MATCH (a) WHERE NOT $p1 = $p2 RETURN a.prop".irWithParams("p1" -> CypherInteger(1L), "p2" -> CypherBoolean(true)) + "MATCH (a) WHERE NOT $p1 = $p2 RETURN a.prop".irWithParams( + "p1" -> CypherInteger(1L), + "p2" -> CypherBoolean(true) + ) val result = plan(ir) - val varA2: Var = Var("a")(CTNode(Set.empty[String], Some(testQualifiedGraphName))) + val varA2: Var = + Var("a")(CTNode(Set.empty[String], Some(testQualifiedGraphName))) val expected = Project( - ElementProperty(varA2, PropertyKey("prop"))(CTNull) -> Some(Var("a.prop")(CTNull)), + ElementProperty(varA2, PropertyKey("prop"))(CTNull) -> Some( + Var("a.prop")(CTNull) + ), Filter( Not(Equals(Param("p1")(CTInteger), Param("p2")(CTBoolean))), PatternScan.nodeScan( varA2, - Start(LogicalCatalogGraph(testQualifiedGraphName, PropertyGraphSchema.empty), emptySqm), + Start( + LogicalCatalogGraph( + testQualifiedGraphName, + PropertyGraphSchema.empty + ), + emptySqm + ), SolvedQueryModel(Set(irFieldA), Set()) ), SolvedQueryModel( Set(irFieldA), - Set(Not(Equals(Param("p1")(CTInteger), Param("p2")(CTBoolean))))) + Set(Not(Equals(Param("p1")(CTInteger), Param("p2")(CTBoolean)))) + ) ), SolvedQueryModel( Set(irFieldA, IRField("a.prop")(CTNull)), - Set(Not(Equals(Param("p1")(CTInteger), Param("p2")(CTBoolean))))) + Set(Not(Equals(Param("p1")(CTInteger), Param("p2")(CTBoolean)))) + ) ) result should equalWithoutResult(expected) } private val planner = new LogicalPlanner(new LogicalOperatorProducer) - private def plan(ir: CypherStatement, schema: PropertyGraphSchema = PropertyGraphSchema.empty): LogicalOperator = + private def plan( + ir: CypherStatement, + schema: PropertyGraphSchema = PropertyGraphSchema.empty + ): LogicalOperator = plan(ir, schema, testGraphName -> schema) private def plan( @@ -300,17 +392,20 @@ class LogicalPlannerTest extends BaseTestSuite with IrConstruction { val planMatch = equalWithTracing(in)(plan) val solvedMatch = equalWithTracing(in.solved)(plan.solved) MatchHelper.combine(planMatch, solvedMatch) - case _ => MatchResult(matches = false, "Expected a Select plan on top", "") + case _ => + MatchResult(matches = false, "Expected a Select plan on top", "") } } } - def graphSource(graphWithSchema: (GraphName, PropertyGraphSchema)*): PropertyGraphDataSource = { + def graphSource( + graphWithSchema: (GraphName, PropertyGraphSchema)* + ): PropertyGraphDataSource = { import org.mockito.Mockito.when val graphSource = mock[PropertyGraphDataSource] - graphWithSchema.foreach { - case (graphName, schema) => when(graphSource.schema(graphName)).thenReturn(Some(schema)) + graphWithSchema.foreach { case (graphName, schema) => + when(graphSource.schema(graphName)).thenReturn(Some(schema)) } graphSource diff --git a/okapi-neo4j-io-testing/src/main/scala/org/opencypher/okapi/neo4j/io/testing/Neo4jTestUtils.scala b/okapi-neo4j-io-testing/src/main/scala/org/opencypher/okapi/neo4j/io/testing/Neo4jTestUtils.scala index 54cff31ea2..54082affb4 100644 --- a/okapi-neo4j-io-testing/src/main/scala/org/opencypher/okapi/neo4j/io/testing/Neo4jTestUtils.scala +++ b/okapi-neo4j-io-testing/src/main/scala/org/opencypher/okapi/neo4j/io/testing/Neo4jTestUtils.scala @@ -34,13 +34,17 @@ import scala.collection.JavaConverters._ object Neo4jTestUtils { - case class Neo4jContext(driver: Driver, session: Session, config: Neo4jConfig) { - + case class Neo4jContext( + driver: Driver, + session: Session, + config: Neo4jConfig + ) { private def dropConstraint(desc: String) = { val regexp = """CONSTRAINT ON (.+) ASSERT \(?(.+?)\)? IS NODE KEY""".r val constraint = desc match { - case regexp(label, keys) => s"CONSTRAINT ON $label ASSERT ($keys) IS NODE KEY" + case regexp(label, keys) => + s"CONSTRAINT ON $label ASSERT ($keys) IS NODE KEY" case c => c } execute(s"DROP $constraint").consume() @@ -50,10 +54,12 @@ object Neo4jTestUtils { execute("MATCH (n) DETACH DELETE n") .consume() execute("CALL db.constraints()") - .list(_.get("description").asString()).asScala + .list(_.get("description").asString()) + .asScala .foreach(dropConstraint) execute("CALL db.indexes YIELD description") - .list(_.get(0).asString).asScala + .list(_.get(0).asString) + .asScala .foreach(index => execute(s"DROP $index").consume()) } @@ -72,10 +78,18 @@ object Neo4jTestUtils { } - def connectNeo4j(dataFixture: String = "", uri: String = "bolt://localhost:7687"): Neo4jContext = { + def connectNeo4j( + dataFixture: String = "", + uri: String = "bolt://localhost:7687" + ): Neo4jContext = { val neo4jURI = URI.create(uri) - val config = Neo4jConfig(neo4jURI, user = "neo4j", password = Some("password"), encrypted = false) + val config = Neo4jConfig( + neo4jURI, + user = "neo4j", + password = Some("password"), + encrypted = false + ) val driver = config.driver() val session = driver.session() val neo4jContext = Neo4jContext(driver, session, config) diff --git a/okapi-neo4j-io-testing/src/test/scala/org/opencypher/okapi/neo4j/io/Neo4jHelpersTest.scala b/okapi-neo4j-io-testing/src/test/scala/org/opencypher/okapi/neo4j/io/Neo4jHelpersTest.scala index e9aebbdf28..1a0048e569 100644 --- a/okapi-neo4j-io-testing/src/test/scala/org/opencypher/okapi/neo4j/io/Neo4jHelpersTest.scala +++ b/okapi-neo4j-io-testing/src/test/scala/org/opencypher/okapi/neo4j/io/Neo4jHelpersTest.scala @@ -41,7 +41,9 @@ class Neo4jHelpersTest extends BaseTestSuite { } it("works for multiple label") { - Set("Foo", "Bar", "Baz with Space").cypherLabelPredicate should equal(":`Foo`:`Bar`:`Baz with Space`") + Set("Foo", "Bar", "Baz with Space").cypherLabelPredicate should equal( + ":`Foo`:`Bar`:`Baz with Space`" + ) } } diff --git a/okapi-neo4j-io-testing/src/test/scala/org/opencypher/okapi/neo4j/io/Neo4jWriteBenchmark.scala b/okapi-neo4j-io-testing/src/test/scala/org/opencypher/okapi/neo4j/io/Neo4jWriteBenchmark.scala index fbf81687f8..b9a0e163ad 100644 --- a/okapi-neo4j-io-testing/src/test/scala/org/opencypher/okapi/neo4j/io/Neo4jWriteBenchmark.scala +++ b/okapi-neo4j-io-testing/src/test/scala/org/opencypher/okapi/neo4j/io/Neo4jWriteBenchmark.scala @@ -41,19 +41,32 @@ object Neo4jWriteBenchmark extends App { Some("passwd") ) - def rowToListValue(data: Array[AnyRef]) = Values.value(data.map(Values.value): _*) + def rowToListValue(data: Array[AnyRef]) = + Values.value(data.map(Values.value): _*) private val numberOfNodes = 10000 val inputNodes = (1 to numberOfNodes).map { i => - Array[AnyRef](i.asInstanceOf[AnyRef], i.asInstanceOf[AnyRef], i.toString.asInstanceOf[AnyRef], (i % 2 == 0).asInstanceOf[AnyRef]) + Array[AnyRef]( + i.asInstanceOf[AnyRef], + i.asInstanceOf[AnyRef], + i.toString.asInstanceOf[AnyRef], + (i % 2 == 0).asInstanceOf[AnyRef] + ) } val inputRels = (2 to numberOfNodes + 1).map { i => - Array[AnyRef](i.asInstanceOf[AnyRef], (i - 1).asInstanceOf[AnyRef], i.asInstanceOf[AnyRef], (i % 2 == 0).asInstanceOf[AnyRef]) + Array[AnyRef]( + i.asInstanceOf[AnyRef], + (i - 1).asInstanceOf[AnyRef], + i.asInstanceOf[AnyRef], + (i % 2 == 0).asInstanceOf[AnyRef] + ) } config.withSession { session => - session.run(s"CREATE CONSTRAINT ON (n:Foo) ASSERT n.$metaPropertyKey IS UNIQUE").consume() + session + .run(s"CREATE CONSTRAINT ON (n:Foo) ASSERT n.$metaPropertyKey IS UNIQUE") + .consume() } val timings: Seq[Long] = (1 to 10).map { _ => diff --git a/okapi-neo4j-io-testing/src/test/scala/org/opencypher/okapi/neo4j/io/PatternElementReaderTest.scala b/okapi-neo4j-io-testing/src/test/scala/org/opencypher/okapi/neo4j/io/PatternElementReaderTest.scala index b6fb45331c..ec9933cb16 100644 --- a/okapi-neo4j-io-testing/src/test/scala/org/opencypher/okapi/neo4j/io/PatternElementReaderTest.scala +++ b/okapi-neo4j-io-testing/src/test/scala/org/opencypher/okapi/neo4j/io/PatternElementReaderTest.scala @@ -37,7 +37,10 @@ class PatternElementReaderTest extends BaseTestSuite { .withNodePropertyKeys("A")("foo" -> CTInteger, "bar" -> CTString.nullable) .withNodePropertyKeys("B")() .withNodePropertyKeys(s"${metaPrefix}C")() - .withRelationshipPropertyKeys("TYPE")("foo" -> CTFloat.nullable, "f" -> CTBoolean) + .withRelationshipPropertyKeys("TYPE")( + "foo" -> CTFloat.nullable, + "f" -> CTBoolean + ) .withRelationshipPropertyKeys("TYPE2")() it("constructs flat node queries from schema") { diff --git a/okapi-neo4j-io-testing/src/test/scala/org/opencypher/okapi/neo4j/io/PatternElementWriterTest.scala b/okapi-neo4j-io-testing/src/test/scala/org/opencypher/okapi/neo4j/io/PatternElementWriterTest.scala index de6a1fb983..b85366752d 100644 --- a/okapi-neo4j-io-testing/src/test/scala/org/opencypher/okapi/neo4j/io/PatternElementWriterTest.scala +++ b/okapi-neo4j-io-testing/src/test/scala/org/opencypher/okapi/neo4j/io/PatternElementWriterTest.scala @@ -56,7 +56,12 @@ class PatternElementWriterTest extends BaseTestSuite with Neo4jServerFixture wit ) }.toBag - val result = neo4jConfig.cypherWithNewSession(s"MATCH (n) RETURN n.$metaPropertyKey, n.val1, n.val2, n.val3").map(CypherMap).toBag + val result = neo4jConfig + .cypherWithNewSession( + s"MATCH (n) RETURN n.$metaPropertyKey, n.val1, n.val2, n.val3" + ) + .map(CypherMap) + .toBag result should equal(expected) } @@ -78,31 +83,39 @@ class PatternElementWriterTest extends BaseTestSuite with Neo4jServerFixture wit ) }.toBag - val result = neo4jConfig.cypherWithNewSession(s"MATCH ()-[r]->() RETURN r.$metaPropertyKey, r.val3").map(CypherMap).toBag + val result = neo4jConfig + .cypherWithNewSession( + s"MATCH ()-[r]->() RETURN r.$metaPropertyKey, r.val3" + ) + .map(CypherMap) + .toBag result should equal(expected) } override def dataFixture: String = "" - private def rowToListValue(data: Array[AnyRef]) = Values.value(data.map(Values.value): _*) + private def rowToListValue(data: Array[AnyRef]) = + Values.value(data.map(Values.value): _*) private val numberOfNodes = 10 - val inputNodes: immutable.IndexedSeq[Array[AnyRef]] = (1 to numberOfNodes).map { i => - Array[AnyRef]( - i.asInstanceOf[AnyRef], - i.asInstanceOf[AnyRef], - i.toString.asInstanceOf[AnyRef], - (i % 2 == 0).asInstanceOf[AnyRef], - (i+1).asInstanceOf[AnyRef] - ) - } + val inputNodes: immutable.IndexedSeq[Array[AnyRef]] = + (1 to numberOfNodes).map { i => + Array[AnyRef]( + i.asInstanceOf[AnyRef], + i.asInstanceOf[AnyRef], + i.toString.asInstanceOf[AnyRef], + (i % 2 == 0).asInstanceOf[AnyRef], + (i + 1).asInstanceOf[AnyRef] + ) + } - val inputRels: immutable.IndexedSeq[Array[AnyRef]] = (2 to numberOfNodes).map { i => - Array[AnyRef]( - i.asInstanceOf[AnyRef], - (i - 1).asInstanceOf[AnyRef], - i.asInstanceOf[AnyRef], - (i % 2 == 0).asInstanceOf[AnyRef] - ) - } + val inputRels: immutable.IndexedSeq[Array[AnyRef]] = + (2 to numberOfNodes).map { i => + Array[AnyRef]( + i.asInstanceOf[AnyRef], + (i - 1).asInstanceOf[AnyRef], + i.asInstanceOf[AnyRef], + (i % 2 == 0).asInstanceOf[AnyRef] + ) + } } diff --git a/okapi-neo4j-io/src/main/scala/org/opencypher/okapi/neo4j/io/ElementReader.scala b/okapi-neo4j-io/src/main/scala/org/opencypher/okapi/neo4j/io/ElementReader.scala index 9043c628a6..ce9f2a64f5 100644 --- a/okapi-neo4j-io/src/main/scala/org/opencypher/okapi/neo4j/io/ElementReader.scala +++ b/okapi-neo4j-io/src/main/scala/org/opencypher/okapi/neo4j/io/ElementReader.scala @@ -33,7 +33,11 @@ import org.opencypher.okapi.neo4j.io.Neo4jHelpers._ object ElementReader { - def flatExactLabelQuery(labels: Set[String], schema: PropertyGraphSchema, maybeMetaLabel: Option[String] = None): String ={ + def flatExactLabelQuery( + labels: Set[String], + schema: PropertyGraphSchema, + maybeMetaLabel: Option[String] = None + ): String = { val props = schema.nodePropertyKeys(labels).propertyExtractorString val allLabels = labels ++ maybeMetaLabel val labelCount = allLabels.size @@ -43,7 +47,11 @@ object ElementReader { |RETURN id($elementVarName) AS $idPropertyKey$props""".stripMargin } - def flatRelTypeQuery(relType: String, schema: PropertyGraphSchema, maybeMetaLabel: Option[String] = None): String ={ + def flatRelTypeQuery( + relType: String, + schema: PropertyGraphSchema, + maybeMetaLabel: Option[String] = None + ): String = { val props = schema.relationshipPropertyKeys(relType).propertyExtractorString val metaLabel = maybeMetaLabel.map(_.cypherLabelPredicate).getOrElse("") @@ -53,10 +61,7 @@ object ElementReader { implicit class RichPropertyTypes(val properties: PropertyKeys) extends AnyVal { def propertyExtractorString: String = { - val propertyStrings = properties - .keys - .toList - .sorted + val propertyStrings = properties.keys.toList.sorted .map(k => s"$elementVarName.$k") if (propertyStrings.isEmpty) "" diff --git a/okapi-neo4j-io/src/main/scala/org/opencypher/okapi/neo4j/io/ElementWriter.scala b/okapi-neo4j-io/src/main/scala/org/opencypher/okapi/neo4j/io/ElementWriter.scala index c129c2a304..75db8d6910 100644 --- a/okapi-neo4j-io/src/main/scala/org/opencypher/okapi/neo4j/io/ElementWriter.scala +++ b/okapi-neo4j-io/src/main/scala/org/opencypher/okapi/neo4j/io/ElementWriter.scala @@ -50,15 +50,18 @@ object ElementWriter extends Logging { )(rowToListValue: T => Value): Unit = { val labelString = labels.cypherLabelPredicate - val nodeKeyProperties = nodeKeys.map { nodeKey => - val keyIndex = rowMapping.indexOf(nodeKey) - val parameterMapLookup = s"$ROW_IDENTIFIER[$keyIndex]" - s"`$nodeKey`: $parameterMapLookup" - }.mkString(", ") - - val setStatements = rowMapping - .zipWithIndex - .filterNot { case (propertyKey, _) => propertyKey == null || nodeKeys.contains(propertyKey) } + val nodeKeyProperties = nodeKeys + .map { nodeKey => + val keyIndex = rowMapping.indexOf(nodeKey) + val parameterMapLookup = s"$ROW_IDENTIFIER[$keyIndex]" + s"`$nodeKey`: $parameterMapLookup" + } + .mkString(", ") + + val setStatements = rowMapping.zipWithIndex + .filterNot { case (propertyKey, _) => + propertyKey == null || nodeKeys.contains(propertyKey) + } .map { case (key, i) => s"SET n.$key = $ROW_IDENTIFIER[$i]" } .mkString("\n") @@ -69,7 +72,13 @@ object ElementWriter extends Logging { |$setStatements """.stripMargin - writeElements(nodes, rowMapping, createQ, config, config.mergeNodeBatchSize)(rowToListValue) + writeElements( + nodes, + rowMapping, + createQ, + config, + config.mergeNodeBatchSize + )(rowToListValue) } // TODO: Share more code with `createRelationships` @@ -84,15 +93,18 @@ object ElementWriter extends Logging { relKeys: Set[String] )(rowToListValue: T => Value): Unit = { - val relKeyProperties = relKeys.map { relKey => - val keyIndex = rowMapping.indexOf(relKey) - val parameterMapLookup = s"$ROW_IDENTIFIER[$keyIndex]" - s"`$relKey`: $parameterMapLookup" - }.mkString(", ") + val relKeyProperties = relKeys + .map { relKey => + val keyIndex = rowMapping.indexOf(relKey) + val parameterMapLookup = s"$ROW_IDENTIFIER[$keyIndex]" + s"`$relKey`: $parameterMapLookup" + } + .mkString(", ") - val setStatements = rowMapping - .zipWithIndex - .filterNot { case (propertyKey, _) => propertyKey == null || relKeys.contains(propertyKey) } + val setStatements = rowMapping.zipWithIndex + .filterNot { case (propertyKey, _) => + propertyKey == null || relKeys.contains(propertyKey) + } .map { case (key, i) => s"SET rel.$key = $ROW_IDENTIFIER[$i]" } .mkString("\n") @@ -107,7 +119,13 @@ object ElementWriter extends Logging { |$setStatements """.stripMargin - writeElements(relationships, rowMapping, createQ, config, config.mergeRelationshipBatchSize)(rowToListValue) + writeElements( + relationships, + rowMapping, + createQ, + config, + config.mergeRelationshipBatchSize + )(rowToListValue) } def createNodes[T]( @@ -118,8 +136,7 @@ object ElementWriter extends Logging { )(rowToListValue: T => Value): Unit = { val labelString = labels.cypherLabelPredicate - val setStatements = rowMapping - .zipWithIndex + val setStatements = rowMapping.zipWithIndex .filterNot(_._1 == null) .map { case (key, i) => s"SET n.$key = $ROW_IDENTIFIER[$i]" } .mkString("\n") @@ -131,7 +148,13 @@ object ElementWriter extends Logging { |$setStatements """.stripMargin - writeElements(nodes, rowMapping, createQ, config, config.createNodeBatchSize)(rowToListValue) + writeElements( + nodes, + rowMapping, + createQ, + config, + config.createNodeBatchSize + )(rowToListValue) } def createRelationships[T]( @@ -143,8 +166,7 @@ object ElementWriter extends Logging { relType: String, nodeLabel: Option[String] )(rowToListValue: T => Value): Unit = { - val setStatements = rowMapping - .zipWithIndex + val setStatements = rowMapping.zipWithIndex .filterNot(_._1 == null) .map { case (key, i) => s"SET rel.$key = $ROW_IDENTIFIER[$i]" } .mkString("\n") @@ -160,7 +182,13 @@ object ElementWriter extends Logging { |$setStatements """.stripMargin - writeElements(relationships, rowMapping, createQ, config, config.createRelationshipBatchSize)(rowToListValue) + writeElements( + relationships, + rowMapping, + createQ, + config, + config.createRelationshipBatchSize + )(rowToListValue) } private def writeElements[T]( @@ -180,7 +208,9 @@ object ElementWriter extends Logging { val batch = batches.next() val rowParameters = new Array[Value](batch.size) - batch.zipWithIndex.foreach { case (row, i) => rowParameters(i) = rowToListValue(row) } + batch.zipWithIndex.foreach { case (row, i) => + rowParameters(i) = rowToListValue(row) + } reuseMap.put("batch", Values.value(rowParameters: _*)) @@ -197,18 +227,22 @@ object ElementWriter extends Logging { } match { case Success(_) => () - case Failure(exception: ClientException) if exception.getMessage.contains("already exists") => + case Failure(exception: ClientException) + if exception.getMessage.contains("already exists") => val originalMessage = exception.getMessage - val elementType = if (originalMessage.contains("Node(")) "nodes" else "relationships" + val elementType = + if (originalMessage.contains("Node(")) "nodes" + else "relationships" val duplicateIdRegex = """.+('[0-9a-fA-F]+')$""".r val duplicateId = originalMessage match { case duplicateIdRegex(idString) => idString - case _ => "UNKNOWN" + case _ => "UNKNOWN" } - val message = s"Could not write the graph to Neo4j. The graph you are attempting to write contains at least two $elementType with Morpheus id $duplicateId" + val message = + s"Could not write the graph to Neo4j. The graph you are attempting to write contains at least two $elementType with Morpheus id $duplicateId" throw IllegalStateException(message, Some(exception)) case Failure(e) => throw e diff --git a/okapi-neo4j-io/src/main/scala/org/opencypher/okapi/neo4j/io/MetaLabelSupport.scala b/okapi-neo4j-io/src/main/scala/org/opencypher/okapi/neo4j/io/MetaLabelSupport.scala index 57e731fd82..5d01e8a419 100644 --- a/okapi-neo4j-io/src/main/scala/org/opencypher/okapi/neo4j/io/MetaLabelSupport.scala +++ b/okapi-neo4j-io/src/main/scala/org/opencypher/okapi/neo4j/io/MetaLabelSupport.scala @@ -46,16 +46,21 @@ object MetaLabelSupport { } implicit class RichPropertyKeys(val keys: PropertyKeys) extends AnyVal { - def withoutMetaProperty: PropertyKeys = keys.filterKeys(k => k != metaPropertyKey) + def withoutMetaProperty: PropertyKeys = + keys.filterKeys(k => k != metaPropertyKey) } implicit class LabelPropertyMapWithMetaSupport(val map: LabelPropertyMap) extends AnyVal { - def withoutMetaLabel(metaLabel: String): LabelPropertyMap = map.map { case (k, v) => (k - metaLabel) -> v } - def withoutMetaProperty: LabelPropertyMap = map.mapValues(_.withoutMetaProperty) + def withoutMetaLabel(metaLabel: String): LabelPropertyMap = map.map { case (k, v) => + (k - metaLabel) -> v + } + def withoutMetaProperty: LabelPropertyMap = + map.mapValues(_.withoutMetaProperty) } - implicit class RelTypePropertyMapWithMetaSupport(val map: RelTypePropertyMap) extends AnyVal { - def withoutMetaProperty: RelTypePropertyMap = map.mapValues(_.withoutMetaProperty) + implicit class RelTypePropertyMapWithMetaSupport(val map: RelTypePropertyMap) extends AnyVal { + def withoutMetaProperty: RelTypePropertyMap = + map.mapValues(_.withoutMetaProperty) } } diff --git a/okapi-neo4j-io/src/main/scala/org/opencypher/okapi/neo4j/io/Neo4jConfig.scala b/okapi-neo4j-io/src/main/scala/org/opencypher/okapi/neo4j/io/Neo4jConfig.scala index 03efca456d..51df7d505d 100644 --- a/okapi-neo4j-io/src/main/scala/org/opencypher/okapi/neo4j/io/Neo4jConfig.scala +++ b/okapi-neo4j-io/src/main/scala/org/opencypher/okapi/neo4j/io/Neo4jConfig.scala @@ -43,7 +43,8 @@ case class Neo4jConfig( ) { def driver(): Driver = password match { - case Some(pwd) => GraphDatabase.driver(uri, AuthTokens.basic(user, pwd), boltConfig()) + case Some(pwd) => + GraphDatabase.driver(uri, AuthTokens.basic(user, pwd), boltConfig()) case _ => GraphDatabase.driver(uri, boltConfig()) } diff --git a/okapi-neo4j-io/src/main/scala/org/opencypher/okapi/neo4j/io/Neo4jHelpers.scala b/okapi-neo4j-io/src/main/scala/org/opencypher/okapi/neo4j/io/Neo4jHelpers.scala index 502aaad839..ea8510a65b 100644 --- a/okapi-neo4j-io/src/main/scala/org/opencypher/okapi/neo4j/io/Neo4jHelpers.scala +++ b/okapi-neo4j-io/src/main/scala/org/opencypher/okapi/neo4j/io/Neo4jHelpers.scala @@ -32,29 +32,37 @@ import org.opencypher.okapi.api.value.CypherValue.CypherValue import scala.collection.JavaConverters._ -/** - * Inefficient convenience methods. - */ +/** Inefficient convenience methods. */ object Neo4jHelpers { /** * Executes a Cypher query with a given implicit session and returns the result as a list of maps * that represent rows. * - * @note materializes the entire result in memory, only advisable for small results. - * @note has a lot of overhead per row, because it stores all the header names in each row map + * @note + * materializes the entire result in memory, only advisable for small results. + * @note + * has a lot of overhead per row, because it stores all the header names in each row map * - * @param query Cypher query to execute - * @param session session with which to execute the Cypher query - * @return list of result rows with each row represented as a map + * @param query + * Cypher query to execute + * @param session + * session with which to execute the Cypher query + * @return + * list of result rows with each row represented as a map */ - def cypher(query: String)(implicit session: Session): List[Map[String, CypherValue]] = { - session.run(query).list().asScala.map(_.asMap().asScala.mapValues(CypherValue(_)).toMap).toList + def cypher( + query: String + )(implicit session: Session): List[Map[String, CypherValue]] = { + session + .run(query) + .list() + .asScala + .map(_.asMap().asScala.mapValues(CypherValue(_)).toMap) + .toList } - /** - * This module defines constants that are used for interactions with the Neo4j database - */ + /** This module defines constants that are used for interactions with the Neo4j database */ object Neo4jDefaults { val metaPrefix: String = "___" @@ -74,7 +82,8 @@ object Neo4jHelpers { } implicit class RichLabelSet(val labels: Set[String]) extends AnyVal { - def cypherLabelPredicate: String = if (labels.isEmpty) "" else labels.map(_.cypherLabelPredicate).mkString("") + def cypherLabelPredicate: String = if (labels.isEmpty) "" + else labels.map(_.cypherLabelPredicate).mkString("") } implicit class RichConfig(val config: Neo4jConfig) extends AnyVal { @@ -91,16 +100,22 @@ object Neo4jHelpers { } /** - * Creates a new driver and session just for one Cypher query and returns the result as a list of maps - * that represent rows. + * Creates a new driver and session just for one Cypher query and returns the result as a list + * of maps that represent rows. * - * @note materializes the entire result in memory, only advisable for small results. - * @note has a lot of overhead per row, because it stores all the header names in each row map - * @note when executing several queries in sequence, it is advisable to use a [[RichConfig#withSession]] block - * instead and make several [[RichConfig#cypher]] calls within it. + * @note + * materializes the entire result in memory, only advisable for small results. + * @note + * has a lot of overhead per row, because it stores all the header names in each row map + * @note + * when executing several queries in sequence, it is advisable to use a + * [[RichConfig#withSession]] block instead and make several [[RichConfig#cypher]] calls + * within it. * - * @param query Cypher query to execute - * @return list of result rows with each row represented as a map + * @param query + * Cypher query to execute + * @return + * list of result rows with each row represented as a map */ def cypherWithNewSession(query: String): List[Map[String, CypherValue]] = { withSession(session => cypher(query)(session)) diff --git a/okapi-neo4j-io/src/main/scala/org/opencypher/okapi/neo4j/io/SchemaFromProcedure.scala b/okapi-neo4j-io/src/main/scala/org/opencypher/okapi/neo4j/io/SchemaFromProcedure.scala index 2e2f2bb80a..f1c8aad525 100644 --- a/okapi-neo4j-io/src/main/scala/org/opencypher/okapi/neo4j/io/SchemaFromProcedure.scala +++ b/okapi-neo4j-io/src/main/scala/org/opencypher/okapi/neo4j/io/SchemaFromProcedure.scala @@ -30,7 +30,12 @@ import org.apache.logging.log4j.scala.Logging import org.opencypher.okapi.api.schema.PropertyKeys.PropertyKeys import org.opencypher.okapi.api.schema.{PropertyKeys, PropertyGraphSchema} import org.opencypher.okapi.api.types.CypherType -import org.opencypher.okapi.api.value.CypherValue.{CypherBoolean, CypherList, CypherString, CypherValue} +import org.opencypher.okapi.api.value.CypherValue.{ + CypherBoolean, + CypherList, + CypherString, + CypherValue +} import org.opencypher.okapi.impl.exception.{IllegalArgumentException, SchemaException} import org.opencypher.okapi.neo4j.io.Neo4jHelpers._ @@ -39,27 +44,33 @@ import scala.util.{Failure, Success, Try} object SchemaFromProcedure extends Logging { /** - * Returns the schema for a Neo4j graph. When the schema contains a property that is incompatible with Morpheus, - * an exception is thrown. Please set `omitImportFailures` in order to omit such properties from the schema - * instead. + * Returns the schema for a Neo4j graph. When the schema contains a property that is incompatible + * with Morpheus, an exception is thrown. Please set `omitImportFailures` in order to omit such + * properties from the schema instead. * * This method relies on the built-in Neo4j schema procedures * [[https://neo4j.com/docs/operations-manual/current/reference/procedures/ nodeTypeProperties/relTypeProperties]], * which were introduced in Neo4j versions 3.3.9, 3.4.10, and 3.5.0. * - * @param neo4j configuration for the Neo4j instance - * @param omitImportFailures when set, incompatible properties are omitted from the schema and a warning is logged - * @return schema for the Neo4j graph + * @param neo4j + * configuration for the Neo4j instance + * @param omitImportFailures + * when set, incompatible properties are omitted from the schema and a warning is logged + * @return + * schema for the Neo4j graph */ - def apply(neo4j: Neo4jConfig, omitImportFailures: Boolean): PropertyGraphSchema = { + def apply( + neo4j: Neo4jConfig, + omitImportFailures: Boolean + ): PropertyGraphSchema = { neo4j.withSession { implicit session => - // Checks if the Neo4j instance supports the required schema procedures def areSchemaProceduresSupported: Boolean = { val schemaProcedures = Set(nodeSchemaProcedure, relSchemaProcedure) - val supportedProcedures = cypher("CALL dbms.procedures() YIELD name AS name") - .map(procedure => procedure("name").value.toString) - .toSet + val supportedProcedures = + cypher("CALL dbms.procedures() YIELD name AS name") + .map(procedure => procedure("name").value.toString) + .toSet schemaProcedures.subsetOf(supportedProcedures) } @@ -76,21 +87,36 @@ object SchemaFromProcedure extends Logging { case Success(true) => val nodeRows = cypher(s"CALL $nodeSchemaProcedure") val nodeRowsGroupedByCombo = nodeRows.groupBy(_.labels) - val nodeSchema = nodeRowsGroupedByCombo.foldLeft(PropertyGraphSchema.empty) { case (currentSchema, (combo, rows)) => - currentSchema.withNodePropertyKeys(combo, propertyKeysForRows(rows)) - } + val nodeSchema = + nodeRowsGroupedByCombo.foldLeft(PropertyGraphSchema.empty) { + case (currentSchema, (combo, rows)) => + currentSchema.withNodePropertyKeys( + combo, + propertyKeysForRows(rows) + ) + } val relRows = cypher(s"CALL $relSchemaProcedure") val relRowsGroupedByType = relRows.groupBy(_.relType) - val relSchema = relRowsGroupedByType.foldLeft(PropertyGraphSchema.empty) { case (currentSchema, (tpe, rows)) => - currentSchema.withRelationshipPropertyKeys(tpe, propertyKeysForRows(rows)) - } + val relSchema = + relRowsGroupedByType.foldLeft(PropertyGraphSchema.empty) { + case (currentSchema, (tpe, rows)) => + currentSchema.withRelationshipPropertyKeys( + tpe, + propertyKeysForRows(rows) + ) + } nodeSchema ++ relSchema - case Success(false) => throw SchemaException( - s"""|Your version of Neo4j does not support `$nodeSchemaProcedure` and `$relSchemaProcedure`. + case Success(false) => + throw SchemaException( + s"""|Your version of Neo4j does not support `$nodeSchemaProcedure` and `$relSchemaProcedure`. |Schema procedure support was added by Neo4j versions 3.3.9, 3.4.10, and 3.5.0. - """.stripMargin) + """.stripMargin + ) case Failure(error) => - throw SchemaException(s"Could not retrieve the procedure list from the Neo4j database", Some(error)) + throw SchemaException( + s"Could not retrieve the procedure list from the Neo4j database", + Some(error) + ) } } @@ -99,9 +125,7 @@ object SchemaFromProcedure extends Logging { // Neo4j schema procedure rows are represented as a map from column name strings to Cypher values type Row = Map[String, CypherValue] - /** - * Functions to extract values from a Neo4j schema procedure row. - */ + /** Functions to extract values from a Neo4j schema procedure row. */ implicit private class Neo4jSchemaRow(row: Row) { def propertyKeys(omitImportFailures: Boolean): PropertyKeys = { @@ -115,54 +139,79 @@ object SchemaFromProcedure extends Logging { } } - def neo4jPropertyTypeStrings: List[String] = getValueAsStrings("propertyTypes") + def neo4jPropertyTypeStrings: List[String] = getValueAsStrings( + "propertyTypes" + ) def isRelationshipSchema: Boolean = row.contains("relType") def relType: String = { val relString = row.get("relType") match { case Some(CypherString(s)) => s - case _ => throw IllegalArgumentException("a valid Neo4j relationship schema row with a relationship type", row) + case _ => + throw IllegalArgumentException( + "a valid Neo4j relationship schema row with a relationship type", + row + ) } // Check that relationship type is properly formatted and not the empty string - if (relString.length > 3 && relString.startsWith(":`") && relString.endsWith("`")) { + if ( + relString.length > 3 && relString.startsWith(":`") && relString + .endsWith("`") + ) { relString.substring(2, relString.length - 1) } else { - throw IllegalArgumentException(s"a Neo4j schema `relType` with format :`REL_TYPE_NAME`", relString) + throw IllegalArgumentException( + s"a Neo4j schema `relType` with format :`REL_TYPE_NAME`", + relString + ) } } def labels: Set[String] = getValueAsStrings("nodeLabels").toSet - def propertyName: Option[String] = row.get("propertyName").collect { case CypherString(s) => s } + def propertyName: Option[String] = + row.get("propertyName").collect { case CypherString(s) => s } - def isMandatory: Boolean = row.get("mandatory").collect { case CypherBoolean(b) => b }.getOrElse(false) + def isMandatory: Boolean = row + .get("mandatory") + .collect { case CypherBoolean(b) => b } + .getOrElse(false) /** * Returns the Cypher type from a Neo4j schema row. * - * Morpheus can have at most one property type for a given property on a given label combination. - * If different property types are present on the same property with multiple nodes that have the same - * label combination, then by default a schema exception is thrown. If `omitImportFailures` is set, then the - * problematic property is instead excluded from the schema and a warning is logged. + * Morpheus can have at most one property type for a given property on a given label + * combination. If different property types are present on the same property with multiple + * nodes that have the same label combination, then by default a schema exception is thrown. If + * `omitImportFailures` is set, then the problematic property is instead excluded from the + * schema and a warning is logged. * - * @param omitImportFailures when set, incompatible properties are omitted from the schema and a warning is logged - * @return joined Cypher type of a property on a label combination + * @param omitImportFailures + * when set, incompatible properties are omitted from the schema and a warning is logged + * @return + * joined Cypher type of a property on a label combination */ - private def maybePropertyType(omitImportFailures: Boolean): Option[CypherType] = { - def rowTypeDescription = if (isRelationshipSchema) s"relationship type [:$relType]" else s"node label combination [:${labels.mkString(", ")}]" - def multipleTypes = s"The Neo4j property `${propertyName.getOrElse("")}` on $rowTypeDescription has multiple property types: [${neo4jPropertyTypeStrings.mkString(", ")}]." + private def maybePropertyType( + omitImportFailures: Boolean + ): Option[CypherType] = { + def rowTypeDescription = if (isRelationshipSchema) + s"relationship type [:$relType]" + else s"node label combination [:${labels.mkString(", ")}]" + def multipleTypes = + s"The Neo4j property `${propertyName.getOrElse("")}` on $rowTypeDescription has multiple property types: [${neo4jPropertyTypeStrings + .mkString(", ")}]." neo4jPropertyTypeStrings match { case neo4jTypeString :: Nil => - def unsupportedPropertyType = s"The Neo4j property `${propertyName.getOrElse("")}` on $rowTypeDescription has unsupported property type `$neo4jTypeString`." + def unsupportedPropertyType = + s"The Neo4j property `${propertyName.getOrElse("")}` on $rowTypeDescription has unsupported property type `$neo4jTypeString`." neo4jTypeString.toCypherType match { case None if omitImportFailures => logger.warn(s"$unsupportedPropertyType $propertyNotAddedMessage") None case None => - throw SchemaException( - s"$unsupportedPropertyType $setFlagMessage") + throw SchemaException(s"$unsupportedPropertyType $setFlagMessage") case Some(tpe) => if (isMandatory) Some(tpe) else Some(tpe.nullable) } @@ -179,22 +228,26 @@ object SchemaFromProcedure extends Logging { private def getValueAsStrings(key: String): List[String] = { row.getOrElse[CypherValue](key, CypherList.empty) match { case CypherList(l) => l.collect { case CypherString(s) => s } - case nonList => throw IllegalArgumentException("A Cypher list of strings", nonList) + case nonList => + throw IllegalArgumentException("A Cypher list of strings", nonList) } } } /** - * String replacement to convert Neo4j type string representations to ones that are compatible with Morpheus. + * String replacement to convert Neo4j type string representations to ones that are compatible + * with Morpheus. */ implicit class Neo4jTypeString(val s: String) extends AnyVal { - private def toMorpheusTypeString: String = neo4jTypeMapping.foldLeft(s) { case (currentTypeString, (neo4jType, morpheusType)) => - currentTypeString.replaceAll(neo4jType, morpheusType) + private def toMorpheusTypeString: String = neo4jTypeMapping.foldLeft(s) { + case (currentTypeString, (neo4jType, morpheusType)) => + currentTypeString.replaceAll(neo4jType, morpheusType) } - def toCypherType: Option[CypherType] = CypherType.fromName(toMorpheusTypeString) + def toCypherType: Option[CypherType] = + CypherType.fromName(toMorpheusTypeString) } @@ -204,8 +257,10 @@ object SchemaFromProcedure extends Logging { "Long" -> "Integer" ) - private[neo4j] val propertyNotAddedMessage = "The property was not added to the schema due to the set `omitImportFailures` flag." - private[neo4j] val setFlagMessage = "Set the `omitImportFailures` flag to compute the Neo4j schema without this property." + private[neo4j] val propertyNotAddedMessage = + "The property was not added to the schema due to the set `omitImportFailures` flag." + private[neo4j] val setFlagMessage = + "Set the `omitImportFailures` flag to compute the Neo4j schema without this property." private[neo4j] val nodeSchemaProcedure = "db.schema.nodeTypeProperties" private[neo4j] val relSchemaProcedure = "db.schema.relTypeProperties" diff --git a/okapi-relational/src/main/scala/org/opencypher/okapi/relational/api/graph/RelationalCypherGraph.scala b/okapi-relational/src/main/scala/org/opencypher/okapi/relational/api/graph/RelationalCypherGraph.scala index 54528297bb..0ca0338f40 100644 --- a/okapi-relational/src/main/scala/org/opencypher/okapi/relational/api/graph/RelationalCypherGraph.scala +++ b/okapi-relational/src/main/scala/org/opencypher/okapi/relational/api/graph/RelationalCypherGraph.scala @@ -36,7 +36,10 @@ import org.opencypher.okapi.ir.api.expr.PrefixId.GraphIdPrefix import org.opencypher.okapi.ir.api.expr.{NodeVar, RelationshipVar} import org.opencypher.okapi.ir.impl.util.VarConverters._ import org.opencypher.okapi.relational.api.io.ElementTable -import org.opencypher.okapi.relational.api.planning.{RelationalCypherResult, RelationalRuntimeContext} +import org.opencypher.okapi.relational.api.planning.{ + RelationalCypherResult, + RelationalRuntimeContext +} import org.opencypher.okapi.relational.api.table.{RelationalCypherRecords, Table} import org.opencypher.okapi.relational.impl.graph.{EmptyGraph, PrefixedGraph, ScanGraph, UnionGraph} import org.opencypher.okapi.relational.impl.operators.{RelationalOperator, Select} @@ -50,21 +53,33 @@ trait RelationalCypherGraphFactory[T <: Table[T]] { implicit val session: RelationalCypherSession[T] - private[opencypher] implicit def tableTypeTag: TypeTag[T] = session.tableTypeTag + private[opencypher] implicit def tableTypeTag: TypeTag[T] = + session.tableTypeTag - def prefixedGraph(graph: RelationalCypherGraph[T], prefix: GraphIdPrefix)(implicit context: RelationalRuntimeContext[T]): Graph = + def prefixedGraph(graph: RelationalCypherGraph[T], prefix: GraphIdPrefix)(implicit + context: RelationalRuntimeContext[T] + ): Graph = PrefixedGraph(graph, prefix) - def unionGraph(graphs: RelationalCypherGraph[T]*)(implicit context: RelationalRuntimeContext[T]): Graph = + def unionGraph(graphs: RelationalCypherGraph[T]*)(implicit + context: RelationalRuntimeContext[T] + ): Graph = UnionGraph(graphs.toList) def empty: Graph = EmptyGraph() - def create(nodeTable: ElementTable[T], elementTables: ElementTable[T]*): Graph = { + def create( + nodeTable: ElementTable[T], + elementTables: ElementTable[T]* + ): Graph = { create(None, nodeTable +: elementTables: _*) } - def create(maybeSchema: Option[PropertyGraphSchema], nodeTable: ElementTable[T], elementTables: ElementTable[T]*): Graph = { + def create( + maybeSchema: Option[PropertyGraphSchema], + nodeTable: ElementTable[T], + elementTables: ElementTable[T]* + ): Graph = { create(maybeSchema, nodeTable +: elementTables: _*) } @@ -72,9 +87,12 @@ trait RelationalCypherGraphFactory[T <: Table[T]] { maybeSchema: Option[PropertyGraphSchema], elementTables: ElementTable[T]* ): Graph = { - implicit val runtimeContext: RelationalRuntimeContext[T] = session.basicRuntimeContext() + implicit val runtimeContext: RelationalRuntimeContext[T] = + session.basicRuntimeContext() val allTables = elementTables - val schema = maybeSchema.getOrElse(allTables.map(_.schema).reduce[PropertyGraphSchema](_ ++ _)) + val schema = maybeSchema.getOrElse( + allTables.map(_.schema).reduce[PropertyGraphSchema](_ ++ _) + ) new ScanGraph(allTables, schema) } } @@ -87,7 +105,8 @@ trait RelationalCypherGraph[T <: Table[T]] extends PropertyGraph { override def session: Session - private[opencypher] implicit def tableTypeTag: TypeTag[T] = session.tableTypeTag + private[opencypher] implicit def tableTypeTag: TypeTag[T] = + session.tableTypeTag def cache(): RelationalCypherGraph[T] = { tables.foreach(_.cache()) @@ -96,45 +115,70 @@ trait RelationalCypherGraph[T <: Table[T]] extends PropertyGraph { def tables: Seq[T] - def scanOperator(searchPattern: Pattern, exactLabelMatch: Boolean = false): RelationalOperator[T] + def scanOperator( + searchPattern: Pattern, + exactLabelMatch: Boolean = false + ): RelationalOperator[T] override def cypher( query: String, parameters: CypherValue.CypherMap, drivingTable: Option[CypherRecords], queryCatalog: Map[QualifiedGraphName, PropertyGraph] - ): RelationalCypherResult[T] = session.cypherOnGraph(this, query, parameters, drivingTable, queryCatalog) - - override def nodes(name: String, nodeCypherType: CTNode, exactLabelMatch: Boolean = false): RelationalCypherRecords[T] = { + ): RelationalCypherResult[T] = + session.cypherOnGraph(this, query, parameters, drivingTable, queryCatalog) + + override def nodes( + name: String, + nodeCypherType: CTNode, + exactLabelMatch: Boolean = false + ): RelationalCypherRecords[T] = { val pattern = NodePattern(nodeCypherType) val scan = scanOperator(pattern, exactLabelMatch) val nodeVar = NodeVar(name)(nodeCypherType) - val namedScan = Select(scan.assignScanName(Map(pattern.nodeElement.toVar -> nodeVar)), List(nodeVar)).alignColumnsWithReturnItems + val namedScan = Select( + scan.assignScanName(Map(pattern.nodeElement.toVar -> nodeVar)), + List(nodeVar) + ).alignColumnsWithReturnItems session.records.from(namedScan.header, namedScan.table) } - override def relationships(name: String, relCypherType: CTRelationship): RelationalCypherRecords[T] = { + override def relationships( + name: String, + relCypherType: CTRelationship + ): RelationalCypherRecords[T] = { val pattern = RelationshipPattern(relCypherType) val scan = scanOperator(pattern) val relVar = RelationshipVar(name)(relCypherType) - val namedScan = Select(scan.assignScanName(Map(pattern.relElement.toVar -> relVar)), List(relVar)).alignColumnsWithReturnItems + val namedScan = Select( + scan.assignScanName(Map(pattern.relElement.toVar -> relVar)), + List(relVar) + ).alignColumnsWithReturnItems session.records.from(namedScan.header, namedScan.table) } override def unionAll(others: PropertyGraph*): RelationalCypherGraph[T] = { val otherGraphs: List[RelationalCypherGraph[T]] = others.toList.map { case g: RelationalCypherGraph[T] => g - case _ => throw UnsupportedOperationException("Union all only works on relational graphs") + case _ => + throw UnsupportedOperationException( + "Union all only works on relational graphs" + ) } // TODO: parameterize property graph API with actual graph type to allow for type safe implementations! - val graphAt = (qgn: QualifiedGraphName) => Some(session.catalog.graph(qgn) match { - case g: RelationalCypherGraph[_] => g.asInstanceOf[RelationalCypherGraph[T]] - }) + val graphAt = (qgn: QualifiedGraphName) => + Some(session.catalog.graph(qgn) match { + case g: RelationalCypherGraph[_] => + g.asInstanceOf[RelationalCypherGraph[T]] + }) - implicit val context: RelationalRuntimeContext[T] = RelationalRuntimeContext(graphAt)(session) + implicit val context: RelationalRuntimeContext[T] = + RelationalRuntimeContext(graphAt)(session) - val allGraphs = (this :: otherGraphs).zipWithIndex.map { case (g, i) => session.graphs.prefixedGraph(g, i.toByte) } + val allGraphs = (this :: otherGraphs).zipWithIndex.map { case (g, i) => + session.graphs.prefixedGraph(g, i.toByte) + } session.graphs.unionGraph(allGraphs: _*) } } diff --git a/okapi-relational/src/main/scala/org/opencypher/okapi/relational/api/graph/RelationalCypherSession.scala b/okapi-relational/src/main/scala/org/opencypher/okapi/relational/api/graph/RelationalCypherSession.scala index 6987bb2efe..25b7c0e79d 100644 --- a/okapi-relational/src/main/scala/org/opencypher/okapi/relational/api/graph/RelationalCypherSession.scala +++ b/okapi-relational/src/main/scala/org/opencypher/okapi/relational/api/graph/RelationalCypherSession.scala @@ -41,10 +41,22 @@ import org.opencypher.okapi.ir.impl.parse.CypherParser import org.opencypher.okapi.ir.impl.{IRBuilder, IRBuilderContext, QueryLocalCatalog} import org.opencypher.okapi.logical.api.configuration.LogicalConfiguration.PrintLogicalPlan import org.opencypher.okapi.logical.impl._ -import org.opencypher.okapi.relational.api.configuration.CoraConfiguration.{PrintOptimizedRelationalPlan, PrintQueryExecutionStages, PrintRelationalPlan} +import org.opencypher.okapi.relational.api.configuration.CoraConfiguration.{ + PrintOptimizedRelationalPlan, + PrintQueryExecutionStages, + PrintRelationalPlan +} import org.opencypher.okapi.relational.api.io.ElementTable -import org.opencypher.okapi.relational.api.planning.{RelationalCypherResult, RelationalRuntimeContext} -import org.opencypher.okapi.relational.api.table.{RelationalCypherRecords, RelationalCypherRecordsFactory, RelationalElementTableFactory, Table} +import org.opencypher.okapi.relational.api.planning.{ + RelationalCypherResult, + RelationalRuntimeContext +} +import org.opencypher.okapi.relational.api.table.{ + RelationalCypherRecords, + RelationalCypherRecordsFactory, + RelationalElementTableFactory, + Table +} import org.opencypher.okapi.relational.impl.RelationalConverters._ import org.opencypher.okapi.relational.impl.planning.{RelationalOptimizer, RelationalPlanner} @@ -53,18 +65,18 @@ import scala.reflect.runtime.universe.TypeTag /** * Base class for relational back ends implementing the OKAPI pipeline. * - * The class provides a generic implementation of the necessary steps to execute a Cypher query on a relational back - * end including parsing, IR planning, logical planning and relational planning. + * The class provides a generic implementation of the necessary steps to execute a Cypher query on + * a relational back end including parsing, IR planning, logical planning and relational planning. * - * Implementers need to make sure to provide factories for back end specific record and graph implementations. + * Implementers need to make sure to provide factories for back end specific record and graph + * implementations. * - * @tparam T back end specific [[Table]] implementation + * @tparam T + * back end specific [[Table]] implementation */ -abstract class RelationalCypherSession[T <: Table[T] : TypeTag] extends CypherSession { +abstract class RelationalCypherSession[T <: Table[T]: TypeTag] extends CypherSession { - /** - * Back end specific records type - */ + /** Back end specific records type */ type Records <: RelationalCypherRecords[T] override type Result = RelationalCypherResult[T] @@ -74,25 +86,36 @@ abstract class RelationalCypherSession[T <: Table[T] : TypeTag] extends CypherSe /** * Reads a graph from a sequence of element tables that contains at least one node table. * - * @param nodeTable first parameter to guarantee there is at least one node table - * @param elementTables sequence of node and relationship tables defining the graph - * @return property graph + * @param nodeTable + * first parameter to guarantee there is at least one node table + * @param elementTables + * sequence of node and relationship tables defining the graph + * @return + * property graph */ - def readFrom(nodeTable: ElementTable[T], elementTables: ElementTable[T]*): RelationalCypherGraph[T] = { - graphs.create(nodeTable, elementTables: _ *) + def readFrom( + nodeTable: ElementTable[T], + elementTables: ElementTable[T]* + ): RelationalCypherGraph[T] = { + graphs.create(nodeTable, elementTables: _*) } - /** - * Qualified graph name for the empty graph - */ - private[opencypher] lazy val emptyGraphQgn = QualifiedGraphName(SessionGraphDataSource.Namespace, GraphName("emptyGraph")) + /** Qualified graph name for the empty graph */ + private[opencypher] lazy val emptyGraphQgn = QualifiedGraphName( + SessionGraphDataSource.Namespace, + GraphName("emptyGraph") + ) // Store empty graph in catalog, so operators that start with an empty graph can refer to its QGN - override lazy val catalog: CypherCatalog = CypherCatalog(emptyGraphQgn -> graphs.empty) + override lazy val catalog: CypherCatalog = CypherCatalog( + emptyGraphQgn -> graphs.empty + ) protected val parser: CypherParser = CypherParser - protected val logicalPlanner: LogicalPlanner = new LogicalPlanner(new LogicalOperatorProducer) + protected val logicalPlanner: LogicalPlanner = new LogicalPlanner( + new LogicalOperatorProducer + ) protected val logicalOptimizer: LogicalOptimizer.type = LogicalOptimizer @@ -104,17 +127,24 @@ abstract class RelationalCypherSession[T <: Table[T] : TypeTag] extends CypherSe private[opencypher] def elementTables: RelationalElementTableFactory[T] - private[opencypher] def graphAt(qgn: QualifiedGraphName): Option[RelationalCypherGraph[T]] = - if (catalog.graphNames.contains(qgn)) Some(catalog.graph(qgn).asRelational) else None + private[opencypher] def graphAt( + qgn: QualifiedGraphName + ): Option[RelationalCypherGraph[T]] = + if (catalog.graphNames.contains(qgn)) Some(catalog.graph(qgn).asRelational) + else None - private[opencypher] def basicRuntimeContext(parameters: CypherMap = CypherMap.empty): RelationalRuntimeContext[T] = + private[opencypher] def basicRuntimeContext( + parameters: CypherMap = CypherMap.empty + ): RelationalRuntimeContext[T] = RelationalRuntimeContext(graphAt, parameters = parameters)(this) private[opencypher] def time[R](description: String)(code: => R): R = { if (PrintTimings.isSet) printTiming(description)(code) else code } - private[opencypher] def mountAmbientGraph(ambient: PropertyGraph): IRCatalogGraph = { + private[opencypher] def mountAmbientGraph( + ambient: PropertyGraph + ): IRCatalogGraph = { val qgn = qgnGenerator.generate catalog.store(qgn, ambient) IRCatalogGraph(qgn, ambient.schema) @@ -125,7 +155,8 @@ abstract class RelationalCypherSession[T <: Table[T] : TypeTag] extends CypherSe parameters: CypherMap = CypherMap.empty, drivingTable: Option[CypherRecords] = None, queryCatalog: Map[QualifiedGraphName, PropertyGraph] = Map.empty - ): Result = cypherOnGraph(graphs.empty, query, parameters, drivingTable, queryCatalog) + ): Result = + cypherOnGraph(graphs.empty, query, parameters, drivingTable, queryCatalog) override private[opencypher] def cypherOnGraph( graph: PropertyGraph, @@ -136,16 +167,20 @@ abstract class RelationalCypherSession[T <: Table[T] : TypeTag] extends CypherSe ): Result = { val ambientGraphNew = mountAmbientGraph(graph) - val maybeRelationalRecords: Option[RelationalCypherRecords[T]] = maybeDrivingTable.map(_.asRelational) + val maybeRelationalRecords: Option[RelationalCypherRecords[T]] = + maybeDrivingTable.map(_.asRelational) val inputFields = maybeRelationalRecords match { case Some(inputRecords) => inputRecords.header.vars - case None => Set.empty[Var] + case None => Set.empty[Var] } - val (stmt, extractedLiterals, semState) = time("AST construction")(parser.process(query, inputFields)(CypherParser.defaultContext)) + val (stmt, extractedLiterals, semState) = time("AST construction")( + parser.process(query, inputFields)(CypherParser.defaultContext) + ) - val extractedParameters: CypherMap = extractedLiterals.mapValues(v => CypherValue(v)) + val extractedParameters: CypherMap = + extractedLiterals.mapValues(v => CypherValue(v)) val allParameters = queryParameters ++ extractedParameters logStageProgress("IR translation ...", newLine = false) @@ -161,7 +196,8 @@ abstract class RelationalCypherSession[T <: Table[T] : TypeTag] extends CypherSe inputFields, queryCatalog ) - val irOut = time("IR translation")(IRBuilder.process(stmt)(irBuilderContext)) + val irOut = + time("IR translation")(IRBuilder.process(stmt)(irBuilderContext)) val ir = IRBuilder.extract(irOut) val queryLocalCatalog = IRBuilder.getContext(irOut).queryLocalCatalog @@ -174,10 +210,24 @@ abstract class RelationalCypherSession[T <: Table[T] : TypeTag] extends CypherSe println("IR:") println(cq.pretty) } - planCypherQuery(graph, cq, allParameters, inputFields, maybeRelationalRecords, queryLocalCatalog) + planCypherQuery( + graph, + cq, + allParameters, + inputFields, + maybeRelationalRecords, + queryLocalCatalog + ) case CreateGraphStatement(targetGraph, innerQueryIr) => - val innerResult = planCypherQuery(graph, innerQueryIr, allParameters, inputFields, maybeRelationalRecords, queryLocalCatalog) + val innerResult = planCypherQuery( + graph, + innerQueryIr, + allParameters, + inputFields, + maybeRelationalRecords, + queryLocalCatalog + ) val resultGraph = innerResult.graph catalog.store(targetGraph.qualifiedGraphName, resultGraph) RelationalCypherResult.empty @@ -206,8 +256,14 @@ abstract class RelationalCypherSession[T <: Table[T] : TypeTag] extends CypherSe maybeDrivingTable: Option[RelationalCypherRecords[T]], queryLocalCatalog: QueryLocalCatalog ): Result = { - val logicalPlan = planLogical(cypherQuery, graph, inputFields, queryLocalCatalog) - planRelational(maybeDrivingTable, allParameters, logicalPlan, queryLocalCatalog) + val logicalPlan = + planLogical(cypherQuery, graph, inputFields, queryLocalCatalog) + planRelational( + maybeDrivingTable, + allParameters, + logicalPlan, + queryLocalCatalog + ) } protected def planLogical( @@ -217,8 +273,14 @@ abstract class RelationalCypherSession[T <: Table[T] : TypeTag] extends CypherSe queryLocalCatalog: QueryLocalCatalog ): LogicalOperator = { logStageProgress("Logical planning ...", newLine = false) - val logicalPlannerContext = LogicalPlannerContext(graph.schema, inputFields, catalog.listSources, queryLocalCatalog) - val logicalPlan = time("Logical planning")(logicalPlanner(ir)(logicalPlannerContext)) + val logicalPlannerContext = LogicalPlannerContext( + graph.schema, + inputFields, + catalog.listSources, + queryLocalCatalog + ) + val logicalPlan = + time("Logical planning")(logicalPlanner(ir)(logicalPlannerContext)) logStageProgress("Done!") if (PrintLogicalPlan.isSet) { println("Logical plan:") @@ -226,7 +288,9 @@ abstract class RelationalCypherSession[T <: Table[T] : TypeTag] extends CypherSe } logStageProgress("Logical optimization ...", newLine = false) - val optimizedLogicalPlan = time("Logical optimization")(logicalOptimizer(logicalPlan)(logicalPlannerContext)) + val optimizedLogicalPlan = time("Logical optimization")( + logicalOptimizer(logicalPlan)(logicalPlannerContext) + ) logStageProgress("Done!") if (PrintLogicalPlan.isSet) { println("Optimized logical plan:") @@ -243,12 +307,16 @@ abstract class RelationalCypherSession[T <: Table[T] : TypeTag] extends CypherSe ): Result = { logStageProgress("Relational planning ... ", newLine = false) - def queryLocalGraphAt(qgn: QualifiedGraphName): Option[RelationalCypherGraph[T]] = + def queryLocalGraphAt( + qgn: QualifiedGraphName + ): Option[RelationalCypherGraph[T]] = Some(new RichPropertyGraph(queryLocalCatalog.graph(qgn)).asRelational[T]) - implicit val context: RelationalRuntimeContext[T] = RelationalRuntimeContext(queryLocalGraphAt, maybeDrivingTable, parameters) + implicit val context: RelationalRuntimeContext[T] = + RelationalRuntimeContext(queryLocalGraphAt, maybeDrivingTable, parameters) - val relationalPlan = time("Relational planning")(RelationalPlanner.process(logicalPlan)) + val relationalPlan = + time("Relational planning")(RelationalPlanner.process(logicalPlan)) logStageProgress("Done!") if (PrintRelationalPlan.isSet) { println("Relational plan:") @@ -256,7 +324,9 @@ abstract class RelationalCypherSession[T <: Table[T] : TypeTag] extends CypherSe } logStageProgress("Relational optimization ... ", newLine = false) - val optimizedRelationalPlan = time("Relational optimization")(RelationalOptimizer.process(relationalPlan)) + val optimizedRelationalPlan = time("Relational optimization")( + RelationalOptimizer.process(relationalPlan) + ) logStageProgress("Done!") if (PrintOptimizedRelationalPlan.isSet) { println("Optimized Relational plan:") diff --git a/okapi-relational/src/main/scala/org/opencypher/okapi/relational/api/io/ElementTable.scala b/okapi-relational/src/main/scala/org/opencypher/okapi/relational/api/io/ElementTable.scala index cd3e48e290..85de42aa3d 100644 --- a/okapi-relational/src/main/scala/org/opencypher/okapi/relational/api/io/ElementTable.scala +++ b/okapi-relational/src/main/scala/org/opencypher/okapi/relational/api/io/ElementTable.scala @@ -37,83 +37,109 @@ import org.opencypher.okapi.relational.api.table.{RelationalCypherRecords, Table import org.opencypher.okapi.relational.impl.table.RecordHeader /** - * An element table describes how to map an input data frame to a Property Graph element - * (i.e. nodes or relationships). + * An element table describes how to map an input data frame to a Property Graph element (i.e. + * nodes or relationships). */ trait ElementTable[T <: Table[T]] extends RelationalCypherRecords[T] { verify() def schema: PropertyGraphSchema = { - mapping.pattern.elements.map { element => - element.cypherType match { - case CTNode(impliedTypes, _) => - val propertyKeys = mapping.properties(element).toSeq.map { - case (propertyKey, sourceKey) => propertyKey -> table.columnType(sourceKey) - } - - PropertyGraphSchema.empty.withNodePropertyKeys(impliedTypes.toSeq: _*)(propertyKeys: _*) - - case CTRelationship(relTypes, _) => - - val propertyKeys = mapping.properties(element).toSeq.map { - case (propertyKey, sourceKey) => propertyKey -> table.columnType(sourceKey) - } - - relTypes.foldLeft(PropertyGraphSchema.empty) { - case (partialSchema, relType) => partialSchema.withRelationshipPropertyKeys(relType)(propertyKeys: _*) - } - - case other => throw IllegalArgumentException("an element with type CTNode or CTRelationship", other) + mapping.pattern.elements + .map { element => + element.cypherType match { + case CTNode(impliedTypes, _) => + val propertyKeys = + mapping.properties(element).toSeq.map { case (propertyKey, sourceKey) => + propertyKey -> table.columnType(sourceKey) + } + + PropertyGraphSchema.empty.withNodePropertyKeys( + impliedTypes.toSeq: _* + )(propertyKeys: _*) + + case CTRelationship(relTypes, _) => + val propertyKeys = + mapping.properties(element).toSeq.map { case (propertyKey, sourceKey) => + propertyKey -> table.columnType(sourceKey) + } + + relTypes.foldLeft(PropertyGraphSchema.empty) { case (partialSchema, relType) => + partialSchema.withRelationshipPropertyKeys(relType)( + propertyKeys: _* + ) + } + + case other => + throw IllegalArgumentException( + "an element with type CTNode or CTRelationship", + other + ) + } } - }.reduce(_ ++ _) + .reduce(_ ++ _) } def mapping: ElementMapping def header: RecordHeader = { - mapping.pattern.elements.map { element => - element.cypherType match { - case n :CTNode => - val nodeVar = Var(element.name)(n) - - val idMapping = Map(nodeVar -> mapping.idKeys(element).head._2) - - val propertyMapping = mapping.properties(element).map { - case (key, source) => ElementProperty(nodeVar, PropertyKey(key))(table.columnType(source)) -> source - } - - RecordHeader(idMapping ++ propertyMapping) - - case r :CTRelationship => - val relVar = Var(element.name)(r) - - val idMapping = mapping.idKeys(element).map { - case (SourceIdKey, source) => relVar -> source - case (SourceStartNodeKey, source) => StartNode(relVar)(CTNode) -> source - case (SourceEndNodeKey, source) => EndNode(relVar)(CTNode) -> source - } - - val propertyMapping = mapping.properties(element).map { - case (key, source) => ElementProperty(relVar, PropertyKey(key))(table.columnType(source)) -> source - } - - RecordHeader(idMapping ++ propertyMapping) - - case other => throw IllegalArgumentException("an element with type CTNode or CTRelationship", other) + mapping.pattern.elements + .map { element => + element.cypherType match { + case n: CTNode => + val nodeVar = Var(element.name)(n) + + val idMapping = Map(nodeVar -> mapping.idKeys(element).head._2) + + val propertyMapping = + mapping.properties(element).map { case (key, source) => + ElementProperty(nodeVar, PropertyKey(key))( + table.columnType(source) + ) -> source + } + + RecordHeader(idMapping ++ propertyMapping) + + case r: CTRelationship => + val relVar = Var(element.name)(r) + + val idMapping = mapping.idKeys(element).map { + case (SourceIdKey, source) => relVar -> source + case (SourceStartNodeKey, source) => + StartNode(relVar)(CTNode) -> source + case (SourceEndNodeKey, source) => + EndNode(relVar)(CTNode) -> source + } + + val propertyMapping = + mapping.properties(element).map { case (key, source) => + ElementProperty(relVar, PropertyKey(key))( + table.columnType(source) + ) -> source + } + + RecordHeader(idMapping ++ propertyMapping) + + case other => + throw IllegalArgumentException( + "an element with type CTNode or CTRelationship", + other + ) + } } - }.reduce(_ ++ _) + .reduce(_ ++ _) } protected def verify(): Unit = { - mapping.idKeys.values.toSeq.flatten.foreach { - case (_, column) => table.verifyColumnType(column, CTIdentity, "id key") + mapping.idKeys.values.toSeq.flatten.foreach { case (_, column) => + table.verifyColumnType(column, CTIdentity, "id key") } - if (table.physicalColumns.toSet != mapping.allSourceKeys.toSet) throw IllegalArgumentException( - s"Columns: ${mapping.allSourceKeys.mkString(", ")}", - s"Columns: ${table.physicalColumns.mkString(", ")}", - s"Use Morpheus[Node|Relationship]Table#fromMapping to create a valid ElementTable") + if (table.physicalColumns.toSet != mapping.allSourceKeys.toSet) + throw IllegalArgumentException( + s"Columns: ${mapping.allSourceKeys.mkString(", ")}", + s"Columns: ${table.physicalColumns.mkString(", ")}", + s"Use Morpheus[Node|Relationship]Table#fromMapping to create a valid ElementTable" + ) } } - diff --git a/okapi-relational/src/main/scala/org/opencypher/okapi/relational/api/planning/QueryPlans.scala b/okapi-relational/src/main/scala/org/opencypher/okapi/relational/api/planning/QueryPlans.scala index ef84707747..04f7f2c02d 100644 --- a/okapi-relational/src/main/scala/org/opencypher/okapi/relational/api/planning/QueryPlans.scala +++ b/okapi-relational/src/main/scala/org/opencypher/okapi/relational/api/planning/QueryPlans.scala @@ -34,7 +34,8 @@ import org.opencypher.okapi.trees.TreeNode case class QueryPlans[T <: Table[T]]( logicalPlan: Option[TreeNode[LogicalOperator]], - relationalPlan: Option[TreeNode[RelationalOperator[T]]]) extends CypherQueryPlans { + relationalPlan: Option[TreeNode[RelationalOperator[T]]] +) extends CypherQueryPlans { override def logical: String = logicalPlan.map(_.pretty).getOrElse("") diff --git a/okapi-relational/src/main/scala/org/opencypher/okapi/relational/api/planning/RelationalCypherResult.scala b/okapi-relational/src/main/scala/org/opencypher/okapi/relational/api/planning/RelationalCypherResult.scala index 3189348a97..22662e849e 100644 --- a/okapi-relational/src/main/scala/org/opencypher/okapi/relational/api/planning/RelationalCypherResult.scala +++ b/okapi-relational/src/main/scala/org/opencypher/okapi/relational/api/planning/RelationalCypherResult.scala @@ -31,51 +31,63 @@ import org.opencypher.okapi.impl.util.PrintOptions import org.opencypher.okapi.logical.impl.LogicalOperator import org.opencypher.okapi.relational.api.graph.{RelationalCypherGraph, RelationalCypherSession} import org.opencypher.okapi.relational.api.table.{RelationalCypherRecords, Table} -import org.opencypher.okapi.relational.impl.operators.{GraphUnionAll, RelationalOperator, ReturnGraph} +import org.opencypher.okapi.relational.impl.operators.{ + GraphUnionAll, + RelationalOperator, + ReturnGraph +} import org.opencypher.okapi.relational.impl.planning.RelationalPlanner._ import scala.reflect.runtime.universe.TypeTag -case class RelationalCypherResult[T <: Table[T] : TypeTag]( +case class RelationalCypherResult[T <: Table[T]: TypeTag]( maybeLogical: Option[LogicalOperator], maybeRelational: Option[RelationalOperator[T]] -)(implicit session: RelationalCypherSession[T]) extends CypherResult { +)(implicit session: RelationalCypherSession[T]) + extends CypherResult { override type Records = RelationalCypherRecords[T] override type Graph = RelationalCypherGraph[T] override def getGraph: Option[Graph] = maybeRelational.flatMap { - case r: ReturnGraph[T] => Some(r.graph) + case r: ReturnGraph[T] => Some(r.graph) case g: GraphUnionAll[T] => Some(g.graph) - case _ => None + case _ => None } /** - * Returns records with minimal number of columns and arbitrary column names. - * The column structure is reflected in the RecordHeader. + * Returns records with minimal number of columns and arbitrary column names. The column + * structure is reflected in the RecordHeader. */ def getInternalRecords: Option[Records] = maybeRelational.flatMap { case _: ReturnGraph[T] => None - case relationalOperator => Some(session.records.from( - relationalOperator.header, - relationalOperator.table, - relationalOperator.maybeReturnItems.map(_.map(_.name)))) + case relationalOperator => + Some( + session.records.from( + relationalOperator.header, + relationalOperator.table, + relationalOperator.maybeReturnItems.map(_.map(_.name)) + ) + ) } override def getRecords: Option[Records] = maybeRelational.flatMap { case _: ReturnGraph[T] => None case relationalOperator => val alignedResult = relationalOperator.alignColumnsWithReturnItems - Some(session.records.from( - alignedResult.header, - alignedResult.table, - alignedResult.maybeReturnItems.map(_.map(_.name)))) + Some( + session.records.from( + alignedResult.header, + alignedResult.table, + alignedResult.maybeReturnItems.map(_.map(_.name)) + ) + ) } override def show(implicit options: PrintOptions): Unit = getRecords match { case Some(r) => r.show - case None => options.stream.print("No results") + case None => options.stream.print("No results") } override def plans: QueryPlans[T] = QueryPlans(maybeLogical, maybeRelational) @@ -83,10 +95,12 @@ case class RelationalCypherResult[T <: Table[T] : TypeTag]( object RelationalCypherResult { - def empty[T <: Table[T] : TypeTag](implicit session: RelationalCypherSession[T]): RelationalCypherResult[T] = + def empty[T <: Table[T]: TypeTag](implicit + session: RelationalCypherSession[T] + ): RelationalCypherResult[T] = RelationalCypherResult(None, None) - def apply[T <: Table[T] : TypeTag]( + def apply[T <: Table[T]: TypeTag]( logical: LogicalOperator, relational: RelationalOperator[T] )(implicit session: RelationalCypherSession[T]): RelationalCypherResult[T] = diff --git a/okapi-relational/src/main/scala/org/opencypher/okapi/relational/api/planning/RelationalRuntimeContext.scala b/okapi-relational/src/main/scala/org/opencypher/okapi/relational/api/planning/RelationalRuntimeContext.scala index f5924033a4..6cb40162ab 100644 --- a/okapi-relational/src/main/scala/org/opencypher/okapi/relational/api/planning/RelationalRuntimeContext.scala +++ b/okapi-relational/src/main/scala/org/opencypher/okapi/relational/api/planning/RelationalRuntimeContext.scala @@ -35,26 +35,39 @@ import org.opencypher.okapi.relational.api.table.{RelationalCypherRecords, Table /** * Responsible for tracking the context during the execution of a single query. * - * @param sessionCatalog mapping between graph names and graphs registered in the session catalog - * @param maybeInputRecords optional driving table for the query - * @param parameters query parameters (e.g. constants) needed for expression evaluation - * @param queryLocalCatalog mapping between graph names and graphs created during query execution - * @param session Cypher session - * @tparam T Table type + * @param sessionCatalog + * mapping between graph names and graphs registered in the session catalog + * @param maybeInputRecords + * optional driving table for the query + * @param parameters + * query parameters (e.g. constants) needed for expression evaluation + * @param queryLocalCatalog + * mapping between graph names and graphs created during query execution + * @param session + * Cypher session + * @tparam T + * Table type */ case class RelationalRuntimeContext[T <: Table[T]]( sessionCatalog: QualifiedGraphName => Option[RelationalCypherGraph[T]], maybeInputRecords: Option[RelationalCypherRecords[T]] = None, parameters: CypherMap = CypherMap.empty, - var queryLocalCatalog: Map[QualifiedGraphName, RelationalCypherGraph[T]] = Map.empty[QualifiedGraphName, RelationalCypherGraph[T]] + var queryLocalCatalog: Map[QualifiedGraphName, RelationalCypherGraph[T]] = + Map.empty[QualifiedGraphName, RelationalCypherGraph[T]] )(implicit val session: RelationalCypherSession[T]) { + /** * Returns the graph referenced by the given QualifiedGraphName. * - * @return back-end specific property graph + * @return + * back-end specific property graph */ - def resolveGraph(qgn: QualifiedGraphName): RelationalCypherGraph[T] = queryLocalCatalog.get(qgn) match { - case None => sessionCatalog(qgn).getOrElse(throw IllegalArgumentException(s"a graph at $qgn")) - case Some(g) => g - } + def resolveGraph(qgn: QualifiedGraphName): RelationalCypherGraph[T] = + queryLocalCatalog.get(qgn) match { + case None => + sessionCatalog(qgn).getOrElse( + throw IllegalArgumentException(s"a graph at $qgn") + ) + case Some(g) => g + } } diff --git a/okapi-relational/src/main/scala/org/opencypher/okapi/relational/api/schema/RelationalPropertyGraphSchema.scala b/okapi-relational/src/main/scala/org/opencypher/okapi/relational/api/schema/RelationalPropertyGraphSchema.scala index bb9c040e54..593aec4eba 100644 --- a/okapi-relational/src/main/scala/org/opencypher/okapi/relational/api/schema/RelationalPropertyGraphSchema.scala +++ b/okapi-relational/src/main/scala/org/opencypher/okapi/relational/api/schema/RelationalPropertyGraphSchema.scala @@ -38,20 +38,30 @@ object RelationalPropertyGraphSchema { implicit class PropertyGraphSchemaOps(val schema: PropertyGraphSchema) extends AnyVal { - def headerForElement(element: Var, exactLabelMatch: Boolean = false): RecordHeader = { + def headerForElement( + element: Var, + exactLabelMatch: Boolean = false + ): RecordHeader = { element.cypherType match { - case _: CTNode => schema.headerForNode(element, exactLabelMatch) + case _: CTNode => schema.headerForNode(element, exactLabelMatch) case _: CTRelationship => schema.headerForRelationship(element) - case other => throw IllegalArgumentException("Element", other) + case other => throw IllegalArgumentException("Element", other) } } - def headerForNode(node: Var, exactLabelMatch: Boolean = false): RecordHeader = { + def headerForNode( + node: Var, + exactLabelMatch: Boolean = false + ): RecordHeader = { val labels: Set[String] = node.cypherType.toCTNode.labels headerForNode(node, labels, exactLabelMatch) } - def headerForNode(node: Var, labels: Set[String], exactLabelMatch: Boolean): RecordHeader = { + def headerForNode( + node: Var, + labels: Set[String], + exactLabelMatch: Boolean + ): RecordHeader = { val labelCombos = if (exactLabelMatch) { Set(labels) } else { @@ -60,7 +70,8 @@ object RelationalPropertyGraphSchema { schema.allCombinations } else { // label scan - val impliedLabels = schema.impliedLabels.transitiveImplicationsFor(labels) + val impliedLabels = + schema.impliedLabels.transitiveImplicationsFor(labels) schema.combinationsFor(impliedLabels) } } @@ -69,9 +80,10 @@ object RelationalPropertyGraphSchema { HasLabel(node, Label(label)) } - val propertyExpressions = schema.nodePropertyKeysForCombinations(labelCombos).map { - case (k, t) => ElementProperty(node, PropertyKey(k))(t) - } + val propertyExpressions = + schema.nodePropertyKeysForCombinations(labelCombos).map { case (k, t) => + ElementProperty(node, PropertyKey(k))(t) + } RecordHeader.from(labelExpressions ++ propertyExpressions + node) } @@ -95,20 +107,25 @@ object RelationalPropertyGraphSchema { .groupBy { case (propertyKey, _) => propertyKey } .mapValues { keysWithType => keysWithType.toSeq.unzip match { - case (keys, types) if keys.size == relTypes.size && types.forall(_ == types.head) => types.head + case (keys, types) + if keys.size == relTypes.size && types.forall( + _ == types.head + ) => + types.head case (_, types) => types.head.nullable } } - val propertyExpressions: Set[Expr] = relKeyHeaderProperties.map { - case (k, t) => ElementProperty(rel, PropertyKey(k))(t) + val propertyExpressions: Set[Expr] = relKeyHeaderProperties.map { case (k, t) => + ElementProperty(rel, PropertyKey(k))(t) }.toSet val startNodeExpr = StartNode(rel)(CTNode) val hasTypeExprs = relTypes.map(relType => HasType(rel, RelType(relType))) val endNodeExpr = EndNode(rel)(CTNode) - val relationshipExpressions = hasTypeExprs ++ propertyExpressions + rel + startNodeExpr + endNodeExpr + val relationshipExpressions = + hasTypeExprs ++ propertyExpressions + rel + startNodeExpr + endNodeExpr RecordHeader.from(relationshipExpressions) } diff --git a/okapi-relational/src/main/scala/org/opencypher/okapi/relational/api/table/RelationalCypherRecords.scala b/okapi-relational/src/main/scala/org/opencypher/okapi/relational/api/table/RelationalCypherRecords.scala index 31a9b25fa8..ea3a92595b 100644 --- a/okapi-relational/src/main/scala/org/opencypher/okapi/relational/api/table/RelationalCypherRecords.scala +++ b/okapi-relational/src/main/scala/org/opencypher/okapi/relational/api/table/RelationalCypherRecords.scala @@ -50,7 +50,11 @@ trait RelationalCypherRecordsFactory[T <: Table[T]] { def fromElementTable(elementTable: ElementTable[T]): Records - def from(header: RecordHeader, table: T, returnItems: Option[Seq[String]] = None): Records + def from( + header: RecordHeader, + table: T, + returnItems: Option[Seq[String]] = None + ): Records } trait RelationalCypherRecords[T <: Table[T]] extends CypherRecords { diff --git a/okapi-relational/src/main/scala/org/opencypher/okapi/relational/api/table/Table.scala b/okapi-relational/src/main/scala/org/opencypher/okapi/relational/api/table/Table.scala index f59038c9a5..caff831545 100644 --- a/okapi-relational/src/main/scala/org/opencypher/okapi/relational/api/table/Table.scala +++ b/okapi-relational/src/main/scala/org/opencypher/okapi/relational/api/table/Table.scala @@ -33,28 +33,34 @@ import org.opencypher.okapi.relational.impl.planning.{JoinType, Order} import org.opencypher.okapi.relational.impl.table.RecordHeader /** - * Main abstraction of a relational backend. A table represents a relation in terms of relational algebra and exposes - * relational and additional auxiliary operators. Operators need to be implemented by the specific backend - * (e.g. morpheus-spark-cypher). + * Main abstraction of a relational backend. A table represents a relation in terms of relational + * algebra and exposes relational and additional auxiliary operators. Operators need to be + * implemented by the specific backend (e.g. morpheus-spark-cypher). * - * @tparam T backend-specific representation of that table (e.g. DataFrame for morpheus-spark-cypher) + * @tparam T + * backend-specific representation of that table (e.g. DataFrame for morpheus-spark-cypher) */ trait Table[T <: Table[T]] extends CypherTable { this: T => /** - * If supported by the backend, calling that operator caches the underlying table within the backend runtime. + * If supported by the backend, calling that operator caches the underlying table within the + * backend runtime. * - * @return cached version of that table + * @return + * cached version of that table */ def cache(): T = this /** - * Returns a table containing only the given columns. The column order within the table is aligned with the argument. + * Returns a table containing only the given columns. The column order within the table is + * aligned with the argument. * - * @param cols columns to select - * @return table containing only requested columns + * @param cols + * columns to select + * @return + * table containing only requested columns */ def select(cols: String*): T = { val tuples = cols.zip(cols) @@ -62,116 +68,161 @@ trait Table[T <: Table[T]] extends CypherTable { } /** - * Returns a table containing only the given columns. The column order within the table is aligned with the argument. + * Returns a table containing only the given columns. The column order within the table is + * aligned with the argument. * - * @param cols columns to select and their alias - * @return table containing only requested aliased columns + * @param cols + * columns to select and their alias + * @return + * table containing only requested aliased columns */ def select(col: (String, String), cols: (String, String)*): T /** * Returns a table containing only rows where the given expression evaluates to true. * - * @param expr filter expression - * @param header table record header - * @param parameters query parameters - * @return table with filtered rows + * @param expr + * filter expression + * @param header + * table record header + * @param parameters + * query parameters + * @return + * table with filtered rows */ - def filter(expr: Expr)(implicit header: RecordHeader, parameters: CypherMap): T + def filter( + expr: Expr + )(implicit header: RecordHeader, parameters: CypherMap): T /** * Returns a table with the given columns removed. * - * @param cols columns to drop - * @return table with dropped columns + * @param cols + * columns to drop + * @return + * table with dropped columns */ def drop(cols: String*): T /** - * Joins the current table with the given table on the specified join columns using equi-join semantics. + * Joins the current table with the given table on the specified join columns using equi-join + * semantics. * - * @param other table to join - * @param joinType join type to perform (e.g. inner, outer, ...) - * @param joinCols columns to join the two tables on - * @return joined table + * @param other + * table to join + * @param joinType + * join type to perform (e.g. inner, outer, ...) + * @param joinCols + * columns to join the two tables on + * @return + * joined table */ def join(other: T, joinType: JoinType, joinCols: (String, String)*): T /** - * Computes the union of the current table and the given table. Requires both tables to have identical column layouts. + * Computes the union of the current table and the given table. Requires both tables to have + * identical column layouts. * - * @param other table to union with - * @return union table + * @param other + * table to union with + * @return + * union table */ def unionAll(other: T): T /** * Returns a table that is ordered by the given columns. * - * @param sortItems a sequence of column names and their order (i.e. ascending / descending) - * @return ordered table + * @param sortItems + * a sequence of column names and their order (i.e. ascending / descending) + * @return + * ordered table */ - def orderBy(sortItems: (Expr, Order)*)(implicit header: RecordHeader, parameters: CypherMap): T + def orderBy( + sortItems: (Expr, Order)* + )(implicit header: RecordHeader, parameters: CypherMap): T /** * Returns a table without the first n rows of the current table. * - * @param n number of rows to skip - * @return table with skipped rows + * @param n + * number of rows to skip + * @return + * table with skipped rows */ def skip(n: Long): T /** * Returns a table containing the first n rows of the current table. * - * @param n number of rows to return - * @return table with at most n rows + * @param n + * number of rows to return + * @return + * table with at most n rows */ def limit(n: Long): T /** * Returns a table where each row is unique. * - * @return table with unique rows + * @return + * table with unique rows */ def distinct: T /** * Returns a table where each row is unique with regard to the specified columns * - * @param cols columns to consider when comparing rows - * @return table containing the specific columns and distinct rows + * @param cols + * columns to consider when comparing rows + * @return + * table containing the specific columns and distinct rows */ def distinct(cols: String*): T /** - * Groups the rows within the table by the given query variables. Additionally a set of aggregations can be performed - * on the grouped table. + * Groups the rows within the table by the given query variables. Additionally a set of + * aggregations can be performed on the grouped table. * - * @param by query variables to group by (e.g. (n)), if empty, the whole row is used as grouping key - * @param aggregations set of aggregations functions and the column to store the result in - * @param header table record header - * @param parameters query parameters - * @return table grouped by the given keys and results of possible aggregate functions + * @param by + * query variables to group by (e.g. (n)), if empty, the whole row is used as grouping key + * @param aggregations + * set of aggregations functions and the column to store the result in + * @param header + * table record header + * @param parameters + * query parameters + * @return + * table grouped by the given keys and results of possible aggregate functions */ - def group(by: Set[Var], aggregations: Map[String, Aggregator]) - (implicit header: RecordHeader, parameters: CypherMap): T + def group(by: Set[Var], aggregations: Map[String, Aggregator])(implicit + header: RecordHeader, + parameters: CypherMap + ): T /** - * Returns a table with additional expressions, which are evaluated and stored in the specified columns. + * Returns a table with additional expressions, which are evaluated and stored in the specified + * columns. * - * @note If the column already exists, its contents will be replaced. - * @param columns tuples of expressions to evaluate and corresponding column name - * @param header table record header - * @param parameters query parameters + * @note + * If the column already exists, its contents will be replaced. + * @param columns + * tuples of expressions to evaluate and corresponding column name + * @param header + * table record header + * @param parameters + * query parameters * @return */ - def withColumns(columns: (Expr, String)*)(implicit header: RecordHeader, parameters: CypherMap): T + def withColumns( + columns: (Expr, String)* + )(implicit header: RecordHeader, parameters: CypherMap): T /** * Prints the table to the system console. * - * @param rows number of rows to print + * @param rows + * number of rows to print */ def show(rows: Int = 20): Unit } diff --git a/okapi-relational/src/main/scala/org/opencypher/okapi/relational/impl/RelationalConverters.scala b/okapi-relational/src/main/scala/org/opencypher/okapi/relational/impl/RelationalConverters.scala index 3fe48a8df7..7d05d7340d 100644 --- a/okapi-relational/src/main/scala/org/opencypher/okapi/relational/impl/RelationalConverters.scala +++ b/okapi-relational/src/main/scala/org/opencypher/okapi/relational/impl/RelationalConverters.scala @@ -37,18 +37,28 @@ import scala.reflect.runtime.universe.TypeTag object RelationalConverters { implicit class RichPropertyGraph(val graph: PropertyGraph) extends AnyVal { - def asRelational[T <: Table[T] : TypeTag]: RelationalCypherGraph[T] = graph.asInstanceOf[RelationalCypherGraph[_]] match { - // The cast is necessary since okapi-API does not expose the underlying table types - case relationalGraph: RelationalCypherGraph[_] => relationalGraph.asInstanceOf[RelationalCypherGraph[T]] - case _ => throw UnsupportedOperationException(s"can only handle relational graphs, got $graph") - } + def asRelational[T <: Table[T]: TypeTag]: RelationalCypherGraph[T] = + graph.asInstanceOf[RelationalCypherGraph[_]] match { + // The cast is necessary since okapi-API does not expose the underlying table types + case relationalGraph: RelationalCypherGraph[_] => + relationalGraph.asInstanceOf[RelationalCypherGraph[T]] + case _ => + throw UnsupportedOperationException( + s"can only handle relational graphs, got $graph" + ) + } } implicit class RichCypherRecords(val records: CypherRecords) extends AnyVal { - def asRelational[T <: Table[T] : TypeTag]: RelationalCypherRecords[T] = records.asInstanceOf[RelationalCypherRecords[_]] match { - // The cast is necessary since okapi-API does not expose the underlying table types - case relationalRecords: RelationalCypherRecords[_] => relationalRecords.asInstanceOf[RelationalCypherRecords[T]] - case _ => throw UnsupportedOperationException(s"can only handle relational records, got $records") - } + def asRelational[T <: Table[T]: TypeTag]: RelationalCypherRecords[T] = + records.asInstanceOf[RelationalCypherRecords[_]] match { + // The cast is necessary since okapi-API does not expose the underlying table types + case relationalRecords: RelationalCypherRecords[_] => + relationalRecords.asInstanceOf[RelationalCypherRecords[T]] + case _ => + throw UnsupportedOperationException( + s"can only handle relational records, got $records" + ) + } } } diff --git a/okapi-relational/src/main/scala/org/opencypher/okapi/relational/impl/exception/RelationalException.scala b/okapi-relational/src/main/scala/org/opencypher/okapi/relational/impl/exception/RelationalException.scala index 940ab350fe..746a121a76 100644 --- a/okapi-relational/src/main/scala/org/opencypher/okapi/relational/impl/exception/RelationalException.scala +++ b/okapi-relational/src/main/scala/org/opencypher/okapi/relational/impl/exception/RelationalException.scala @@ -33,6 +33,9 @@ abstract class RelationalException(msg: String) extends InternalException(msg) final case class RecordHeaderException(msg: String) extends RelationalException(msg) -final case class DuplicateSourceColumnException(columnName: String, element: Var) - extends RelationalException( - s"The source column '$columnName' is used more than once to describe the mapping of $element") +final case class DuplicateSourceColumnException( + columnName: String, + element: Var +) extends RelationalException( + s"The source column '$columnName' is used more than once to describe the mapping of $element" + ) diff --git a/okapi-relational/src/main/scala/org/opencypher/okapi/relational/impl/graph/EmptyGraph.scala b/okapi-relational/src/main/scala/org/opencypher/okapi/relational/impl/graph/EmptyGraph.scala index 3947d13ebe..de520aebaa 100644 --- a/okapi-relational/src/main/scala/org/opencypher/okapi/relational/impl/graph/EmptyGraph.scala +++ b/okapi-relational/src/main/scala/org/opencypher/okapi/relational/impl/graph/EmptyGraph.scala @@ -37,7 +37,9 @@ import org.opencypher.okapi.relational.impl.table.RecordHeader import scala.reflect.runtime.universe.TypeTag -sealed case class EmptyGraph[T <: Table[T] : TypeTag]()(implicit val session: RelationalCypherSession[T]) extends RelationalCypherGraph[T] { +sealed case class EmptyGraph[T <: Table[T]: TypeTag]()(implicit + val session: RelationalCypherSession[T] +) extends RelationalCypherGraph[T] { override type Session = RelationalCypherSession[T] @@ -49,11 +51,15 @@ sealed case class EmptyGraph[T <: Table[T] : TypeTag]()(implicit val session: Re override def tables: Seq[T] = Seq.empty - override def scanOperator(searchPattern: Pattern, exactLabelMatch: Boolean): RelationalOperator[T] = { - implicit val context: RelationalRuntimeContext[T] = session.basicRuntimeContext() + override def scanOperator( + searchPattern: Pattern, + exactLabelMatch: Boolean + ): RelationalOperator[T] = { + implicit val context: RelationalRuntimeContext[T] = + session.basicRuntimeContext() val scanHeader = searchPattern.elements - .map { e => RecordHeader.from(e.toVar)} + .map { e => RecordHeader.from(e.toVar) } .reduce(_ ++ _) val records = session.records.empty(scanHeader) diff --git a/okapi-relational/src/main/scala/org/opencypher/okapi/relational/impl/graph/PrefixedGraph.scala b/okapi-relational/src/main/scala/org/opencypher/okapi/relational/impl/graph/PrefixedGraph.scala index d6b92c43ff..8604cfc90d 100644 --- a/okapi-relational/src/main/scala/org/opencypher/okapi/relational/impl/graph/PrefixedGraph.scala +++ b/okapi-relational/src/main/scala/org/opencypher/okapi/relational/impl/graph/PrefixedGraph.scala @@ -37,8 +37,11 @@ import org.opencypher.okapi.relational.impl.operators.RelationalOperator import org.opencypher.okapi.relational.impl.planning.RelationalPlanner._ // TODO: This should be a planned tree of physical operators instead of a graph -final case class PrefixedGraph[T <: Table[T]](graph: RelationalCypherGraph[T], prefix: GraphIdPrefix) - (implicit context: RelationalRuntimeContext[T]) extends RelationalCypherGraph[T] { +final case class PrefixedGraph[T <: Table[T]]( + graph: RelationalCypherGraph[T], + prefix: GraphIdPrefix +)(implicit context: RelationalRuntimeContext[T]) + extends RelationalCypherGraph[T] { override implicit val session: RelationalCypherSession[T] = context.session @@ -56,8 +59,10 @@ final case class PrefixedGraph[T <: Table[T]](graph: RelationalCypherGraph[T], p searchPattern: Pattern, exactLabelMatch: Boolean ): RelationalOperator[T] = { - searchPattern.elements.foldLeft(graph.scanOperator(searchPattern, exactLabelMatch)) { - case (acc, patternElement) => acc.prefixVariableId(patternElement.toVar, prefix) + searchPattern.elements.foldLeft( + graph.scanOperator(searchPattern, exactLabelMatch) + ) { case (acc, patternElement) => + acc.prefixVariableId(patternElement.toVar, prefix) } } diff --git a/okapi-relational/src/main/scala/org/opencypher/okapi/relational/impl/graph/ScanGraph.scala b/okapi-relational/src/main/scala/org/opencypher/okapi/relational/impl/graph/ScanGraph.scala index 1a6c60b63d..2cfa534bde 100644 --- a/okapi-relational/src/main/scala/org/opencypher/okapi/relational/impl/graph/ScanGraph.scala +++ b/okapi-relational/src/main/scala/org/opencypher/okapi/relational/impl/graph/ScanGraph.scala @@ -40,9 +40,11 @@ import org.opencypher.okapi.relational.impl.planning.RelationalPlanner._ import scala.reflect.runtime.universe.TypeTag -class ScanGraph[T <: Table[T] : TypeTag](val scans: Seq[ElementTable[T]], val schema: PropertyGraphSchema) - (implicit val session: RelationalCypherSession[T]) - extends RelationalCypherGraph[T] { +class ScanGraph[T <: Table[T]: TypeTag]( + val scans: Seq[ElementTable[T]], + val schema: PropertyGraphSchema +)(implicit val session: RelationalCypherSession[T]) + extends RelationalCypherGraph[T] { validate() @@ -51,29 +53,32 @@ class ScanGraph[T <: Table[T] : TypeTag](val scans: Seq[ElementTable[T]], val sc override type Session = RelationalCypherSession[T] // TODO: ScanGraph should be an operator that gets a set of tables as input - private implicit def runtimeContext: RelationalRuntimeContext[T] = session.basicRuntimeContext() + private implicit def runtimeContext: RelationalRuntimeContext[T] = + session.basicRuntimeContext() override def tables: Seq[T] = scans.map(_.table) // TODO: Express `exactLabelMatch` with type - override def scanOperator(searchPattern: Pattern, exactLabelMatch: Boolean): RelationalOperator[T] = { + override def scanOperator( + searchPattern: Pattern, + exactLabelMatch: Boolean + ): RelationalOperator[T] = { val selectedScans = scansForType(searchPattern, exactLabelMatch) - val alignedElementTableOps = selectedScans.map { - case (scan, embedding) => - embedding.foldLeft(scan) { - case (acc, (targetElement, inputElement)) => - val inputElementExpressions = scan.header.expressionsFor(inputElement.toVar) - val targetHeader = acc.header -- inputElementExpressions ++ schema.headerForElement(targetElement.toVar, exactLabelMatch) + val alignedElementTableOps = selectedScans.map { case (scan, embedding) => + embedding.foldLeft(scan) { case (acc, (targetElement, inputElement)) => + val inputElementExpressions = + scan.header.expressionsFor(inputElement.toVar) + val targetHeader = acc.header -- inputElementExpressions ++ schema + .headerForElement(targetElement.toVar, exactLabelMatch) - acc.alignWith(inputElement.toVar, targetElement.toVar, targetHeader) - } + acc.alignWith(inputElement.toVar, targetElement.toVar, targetHeader) + } } alignedElementTableOps.toList match { case Nil => - val scanHeader = searchPattern - .elements + val scanHeader = searchPattern.elements .map { e => schema.headerForElement(e.toVar) } .reduce(_ ++ _) @@ -90,7 +95,8 @@ class ScanGraph[T <: Table[T] : TypeTag](val scans: Seq[ElementTable[T]], val sc searchPattern: Pattern, exactLabelMatch: Boolean ): Seq[(RelationalOperator[T], Map[PatternElement, PatternElement])] = { - val qgn = searchPattern.elements.head.cypherType.graph.getOrElse(session.emptyGraphQgn) + val qgn = searchPattern.elements.head.cypherType.graph + .getOrElse(session.emptyGraphQgn) val selectedScans = scans.flatMap { scan => val scanPattern = scan.mapping.pattern @@ -99,8 +105,8 @@ class ScanGraph[T <: Table[T] : TypeTag](val scans: Seq[ElementTable[T]], val sc .map(embedding => scan -> embedding) } - selectedScans.map { - case (scan, embedding) => Start(qgn, scan) -> embedding + selectedScans.map { case (scan, embedding) => + Start(qgn, scan) -> embedding } } @@ -108,18 +114,17 @@ class ScanGraph[T <: Table[T] : TypeTag](val scans: Seq[ElementTable[T]], val sc scan.mapping.pattern }.toSet - override def toString = s"ScanGraph(${ - scans.map(_.mapping.pattern).mkString(", ") - })" + override def toString = + s"ScanGraph(${scans.map(_.mapping.pattern).mkString(", ")})" def validate(): Unit = { schema.labelCombinations.combos.foreach { combo => val hasScan = patterns.exists { case NodePattern(nodeType) => nodeType.labels == combo - case _ => false + case _ => false } - if(!hasScan) { + if (!hasScan) { throw IllegalArgumentException( s"a scan with NodePattern for label combination $combo", patterns @@ -130,10 +135,10 @@ class ScanGraph[T <: Table[T] : TypeTag](val scans: Seq[ElementTable[T]], val sc schema.relationshipTypes.foreach { relType => val hasScan = patterns.exists { case RelationshipPattern(relTypes) => relTypes.types.head == relType - case _ => false + case _ => false } - if(!hasScan) { + if (!hasScan) { throw IllegalArgumentException( s"a scan with a RelationshipPattern for relationship type $relType", patterns diff --git a/okapi-relational/src/main/scala/org/opencypher/okapi/relational/impl/graph/UnionGraph.scala b/okapi-relational/src/main/scala/org/opencypher/okapi/relational/impl/graph/UnionGraph.scala index 338d6ff3e1..e6dbcc704c 100644 --- a/okapi-relational/src/main/scala/org/opencypher/okapi/relational/impl/graph/UnionGraph.scala +++ b/okapi-relational/src/main/scala/org/opencypher/okapi/relational/impl/graph/UnionGraph.scala @@ -43,8 +43,10 @@ import org.opencypher.okapi.relational.impl.planning.RelationalPlanner._ import scala.reflect.runtime.universe.TypeTag // TODO: This should be a planned tree of physical operators instead of a graph -final case class UnionGraph[T <: Table[T] : TypeTag](graphs: List[RelationalCypherGraph[T]]) - (implicit context: RelationalRuntimeContext[T]) extends RelationalCypherGraph[T] { +final case class UnionGraph[T <: Table[T]: TypeTag]( + graphs: List[RelationalCypherGraph[T]] +)(implicit context: RelationalRuntimeContext[T]) + extends RelationalCypherGraph[T] { // TODO: We could be better here by also keeping patterns for which we know that they cover parts of the schema // that only the respective subgraph supplies @@ -63,7 +65,8 @@ final case class UnionGraph[T <: Table[T] : TypeTag](graphs: List[RelationalCyph override def tables: Seq[T] = graphs.flatMap(_.tables) - override lazy val schema: PropertyGraphSchema = graphs.map(g => g.schema).foldLeft(PropertyGraphSchema.empty)(_ ++ _) + override lazy val schema: PropertyGraphSchema = + graphs.map(g => g.schema).foldLeft(PropertyGraphSchema.empty)(_ ++ _) override def toString = s"UnionGraph(graphs=[${graphs.mkString(",")}])" @@ -73,7 +76,6 @@ final case class UnionGraph[T <: Table[T] : TypeTag](graphs: List[RelationalCyph ): RelationalOperator[T] = { val alignedElementTableOps = graphs.flatMap { graph => - val isEmptyScan = searchPattern.elements.map(_.cypherType).exists { case CTNode(knownLabels, _) if knownLabels.isEmpty => graph.schema.allCombinations.isEmpty @@ -83,20 +85,23 @@ final case class UnionGraph[T <: Table[T] : TypeTag](graphs: List[RelationalCyph graph.schema.relationshipTypes.isEmpty case CTRelationship(types, _) => graph.schema.relTypePropertyMap.filterForRelTypes(types).isEmpty - case other => throw UnsupportedOperationException(s"Cannot scan on $other") + case other => + throw UnsupportedOperationException(s"Cannot scan on $other") } if (isEmptyScan) { None } else { val selectedScan = graph.scanOperator(searchPattern, exactLabelMatch) - val alignedScanOp = searchPattern.elements.foldLeft(selectedScan) { - - case (acc, element) => - val inputElementExpressions = selectedScan.header.expressionsFor(element.toVar) - val targetHeader = acc.header -- inputElementExpressions ++ schema.headerForElement(element.toVar) - - acc.alignWith(element.toVar, element.toVar, targetHeader) + val alignedScanOp = searchPattern.elements.foldLeft(selectedScan) { case (acc, element) => + val inputElementExpressions = + selectedScan.header.expressionsFor(element.toVar) + val targetHeader = + acc.header -- inputElementExpressions ++ schema.headerForElement( + element.toVar + ) + + acc.alignWith(element.toVar, element.toVar, targetHeader) } Some(alignedScanOp) } @@ -104,8 +109,7 @@ final case class UnionGraph[T <: Table[T] : TypeTag](graphs: List[RelationalCyph alignedElementTableOps match { case Nil => - val scanHeader = searchPattern - .elements + val scanHeader = searchPattern.elements .map { e => schema.headerForElement(e.toVar) } .reduce(_ ++ _) Start.fromEmptyGraph(session.records.empty(scanHeader)) diff --git a/okapi-relational/src/main/scala/org/opencypher/okapi/relational/impl/operators/RelationalOperator.scala b/okapi-relational/src/main/scala/org/opencypher/okapi/relational/impl/operators/RelationalOperator.scala index c586b3397b..37662be913 100644 --- a/okapi-relational/src/main/scala/org/opencypher/okapi/relational/impl/operators/RelationalOperator.scala +++ b/okapi-relational/src/main/scala/org/opencypher/okapi/relational/impl/operators/RelationalOperator.scala @@ -45,7 +45,8 @@ import org.opencypher.okapi.trees.AbstractTreeNode import scala.reflect.runtime.universe.TypeTag -abstract class RelationalOperator[T <: Table[T] : TypeTag] extends AbstractTreeNode[RelationalOperator[T]] { +abstract class RelationalOperator[T <: Table[T]: TypeTag] + extends AbstractTreeNode[RelationalOperator[T]] { def header: RecordHeader = children.head.header @@ -61,8 +62,9 @@ abstract class RelationalOperator[T <: Table[T] : TypeTag] extends AbstractTreeN def maybeReturnItems: Option[Seq[Var]] = children.head.maybeReturnItems - protected def resolve(qualifiedGraphName: QualifiedGraphName) - (implicit context: RelationalRuntimeContext[T]): RelationalCypherGraph[T] = + protected def resolve(qualifiedGraphName: QualifiedGraphName)(implicit + context: RelationalRuntimeContext[T] + ): RelationalCypherGraph[T] = context.resolveGraph(qualifiedGraphName) def table: T = { @@ -79,7 +81,9 @@ abstract class RelationalOperator[T <: Table[T] : TypeTag] extends AbstractTreeN if (duplicateColumns.nonEmpty) throw IllegalArgumentException( s"${getClass.getSimpleName}: a table with distinct columns", - s"a table with duplicate columns: ${initialDataColumns.sorted.mkString("[", ", ", "]")}") + s"a table with duplicate columns: ${initialDataColumns.sorted + .mkString("[", ", ", "]")}" + ) // Verify that all header column names exist in the data val headerColumnNames = header.columns @@ -87,9 +91,13 @@ abstract class RelationalOperator[T <: Table[T] : TypeTag] extends AbstractTreeN val missingTableColumns = headerColumnNames -- dataColumnNames if (missingTableColumns.nonEmpty) { throw IllegalArgumentException( - s"${getClass.getSimpleName}: table with columns ${header.columns.toSeq.sorted.mkString("\n[", ", ", "]\n")}", - s"""|table with columns ${dataColumnNames.toSeq.sorted.mkString("\n[", ", ", "]\n")} - |column(s) ${missingTableColumns.mkString(", ")} are missing in the table + s"${getClass.getSimpleName}: table with columns ${header.columns.toSeq.sorted + .mkString("\n[", ", ", "]\n")}", + s"""|table with columns ${dataColumnNames.toSeq.sorted + .mkString("\n[", ", ", "]\n")} + |column(s) ${missingTableColumns.mkString( + ", " + )} are missing in the table """.stripMargin ) } @@ -112,10 +120,16 @@ abstract class RelationalOperator[T <: Table[T] : TypeTag] extends AbstractTreeN headerType match { case n if n.subTypeOf(CTNode.nullable) && tableType == CTInteger => - case r if r.subTypeOf(CTRelationship.nullable) && tableType == CTInteger => + case r + if r.subTypeOf( + CTRelationship.nullable + ) && tableType == CTInteger => case _ if tableType == headerType => - case _ => throw IllegalArgumentException( - s"${getClass.getSimpleName}: data matching header type $headerType for expression $expr", tableType) + case _ => + throw IllegalArgumentException( + s"${getClass.getSimpleName}: data matching header type $headerType for expression $expr", + tableType + ) } } } @@ -143,27 +157,34 @@ trait UnitTable[T <: Table[T]] { object Start { - def fromEmptyGraph[T <: Table[T] : TypeTag](records: RelationalCypherRecords[T]) - (implicit context: RelationalRuntimeContext[T]): Start[T] = { + def fromEmptyGraph[T <: Table[T]: TypeTag]( + records: RelationalCypherRecords[T] + )(implicit context: RelationalRuntimeContext[T]): Start[T] = { Start(context.session.emptyGraphQgn, Some(records)) } - def apply[T <: Table[T] : TypeTag](qgn: QualifiedGraphName, records: RelationalCypherRecords[T]) - (implicit context: RelationalRuntimeContext[T]): Start[T] = { + def apply[T <: Table[T]: TypeTag]( + qgn: QualifiedGraphName, + records: RelationalCypherRecords[T] + )(implicit context: RelationalRuntimeContext[T]): Start[T] = { Start(qgn, Some(records)) } } -final case class Start[T <: Table[T] : TypeTag]( +final case class Start[T <: Table[T]: TypeTag]( qgn: QualifiedGraphName, maybeRecords: Option[RelationalCypherRecords[T]] = None -)(implicit override val context: RelationalRuntimeContext[T], override val tt: TypeTag[RelationalOperator[T]]) - extends RelationalOperator[T] { +)(implicit + override val context: RelationalRuntimeContext[T], + override val tt: TypeTag[RelationalOperator[T]] +) extends RelationalOperator[T] { - override lazy val header: RecordHeader = maybeRecords.map(_.header).getOrElse(RecordHeader.empty) + override lazy val header: RecordHeader = + maybeRecords.map(_.header).getOrElse(RecordHeader.empty) - override lazy val _table: T = maybeRecords.map(_.table).getOrElse(session.records.unit().table) + override lazy val _table: T = + maybeRecords.map(_.table).getOrElse(session.records.unit().table) override lazy val graph: RelationalCypherGraph[T] = resolve(qgn) @@ -182,33 +203,37 @@ final case class Start[T <: Table[T] : TypeTag]( // Unary -final case class PrefixGraph[T <: Table[T] : TypeTag]( +final case class PrefixGraph[T <: Table[T]: TypeTag]( in: RelationalOperator[T], prefix: GraphIdPrefix -) extends RelationalOperator[T] with EmptyTable[T] { +) extends RelationalOperator[T] + with EmptyTable[T] { - override lazy val graphName: QualifiedGraphName = QualifiedGraphName(s"${in.graphName}_tempPrefixed_$prefix") + override lazy val graphName: QualifiedGraphName = QualifiedGraphName( + s"${in.graphName}_tempPrefixed_$prefix" + ) - override lazy val graph: RelationalCypherGraph[T] = session.graphs.prefixedGraph(in.graph, prefix) + override lazy val graph: RelationalCypherGraph[T] = + session.graphs.prefixedGraph(in.graph, prefix) } /** - * Cache is a marker operator that indicates that its child operator is used multiple times within the query. + * Cache is a marker operator that indicates that its child operator is used multiple times within + * the query. */ -final case class Cache[T <: Table[T] : TypeTag](in: RelationalOperator[T]) - extends RelationalOperator[T] { +final case class Cache[T <: Table[T]: TypeTag](in: RelationalOperator[T]) + extends RelationalOperator[T] { override lazy val _table: T = in._table.cache() } -final case class SwitchContext[T <: Table[T] : TypeTag]( +final case class SwitchContext[T <: Table[T]: TypeTag]( in: RelationalOperator[T], override val context: RelationalRuntimeContext[T] ) extends RelationalOperator[T] - -final case class Alias[T <: Table[T] : TypeTag]( +final case class Alias[T <: Table[T]: TypeTag]( in: RelationalOperator[T], aliases: Seq[AliasExpr] ) extends RelationalOperator[T] { @@ -216,7 +241,7 @@ final case class Alias[T <: Table[T] : TypeTag]( override lazy val header: RecordHeader = in.header.withAlias(aliases: _*) } -final case class Add[T <: Table[T] : TypeTag]( +final case class Add[T <: Table[T]: TypeTag]( in: RelationalOperator[T], exprs: List[Expr] ) extends RelationalOperator[T] { @@ -226,12 +251,12 @@ final case class Add[T <: Table[T] : TypeTag]( if (aggHeader.contains(expr)) { expr match { case a: AliasExpr => aggHeader.withAlias(a) - case _ => aggHeader + case _ => aggHeader } } else { expr match { case a: AliasExpr => aggHeader.withExpr(a.expr).withAlias(a) - case _ => aggHeader.withExpr(expr) + case _ => aggHeader.withExpr(expr) } } } @@ -243,12 +268,14 @@ final case class Add[T <: Table[T] : TypeTag]( if (physicalAdditions.isEmpty) { in.table } else { - in.table.withColumns(physicalAdditions.map(expr => expr -> header.column(expr)): _*)(header, context.parameters) + in.table.withColumns( + physicalAdditions.map(expr => expr -> header.column(expr)): _* + )(header, context.parameters) } } } -final case class AddInto[T <: Table[T] : TypeTag]( +final case class AddInto[T <: Table[T]: TypeTag]( in: RelationalOperator[T], valueIntoTuples: List[(Expr, Expr)] ) extends RelationalOperator[T] { @@ -258,12 +285,14 @@ final case class AddInto[T <: Table[T] : TypeTag]( } override lazy val _table: T = { - val valuesToColumnNames = valueIntoTuples.map { case (value, into) => value -> header.column(into) } + val valuesToColumnNames = valueIntoTuples.map { case (value, into) => + value -> header.column(into) + } in.table.withColumns(valuesToColumnNames: _*)(header, context.parameters) } } -final case class Drop[E <: Expr, T <: Table[T] : TypeTag]( +final case class Drop[E <: Expr, T <: Table[T]: TypeTag]( in: RelationalOperator[T], exprs: Set[E] ) extends RelationalOperator[T] { @@ -281,23 +310,24 @@ final case class Drop[E <: Expr, T <: Table[T] : TypeTag]( } } -final case class Filter[T <: Table[T] : TypeTag]( +final case class Filter[T <: Table[T]: TypeTag]( in: RelationalOperator[T], expr: Expr ) extends RelationalOperator[T] { - override lazy val _table: T = in.table.filter(expr)(header, context.parameters) + override lazy val _table: T = + in.table.filter(expr)(header, context.parameters) } -final case class ReturnGraph[T <: Table[T] : TypeTag](in: RelationalOperator[T]) - extends RelationalOperator[T] { +final case class ReturnGraph[T <: Table[T]: TypeTag](in: RelationalOperator[T]) + extends RelationalOperator[T] { override lazy val header: RecordHeader = RecordHeader.empty override lazy val _table: T = session.records.empty().table } -final case class Select[T <: Table[T] : TypeTag]( +final case class Select[T <: Table[T]: TypeTag]( in: RelationalOperator[T], expressions: List[Expr], columnRenames: Map[Expr, String] = Map.empty @@ -305,61 +335,72 @@ final case class Select[T <: Table[T] : TypeTag]( private lazy val selectHeader = in.header.select(expressions: _*) - override lazy val header: RecordHeader = selectHeader.withColumnsReplaced(columnRenames) + override lazy val header: RecordHeader = + selectHeader.withColumnsReplaced(columnRenames) private lazy val returnExpressions = expressions.map { case AliasExpr(_, alias) => alias - case other => other + case other => other } override lazy val _table: T = { - val selectExpressions = returnExpressions.flatMap(expr => header.expressionsFor(expr).toSeq.sorted) - val selectColumns = selectExpressions.map { expr => selectHeader.column(expr) -> header.column(expr) }.distinct + val selectExpressions = + returnExpressions.flatMap(expr => header.expressionsFor(expr).toSeq.sorted) + val selectColumns = selectExpressions.map { expr => + selectHeader.column(expr) -> header.column(expr) + }.distinct in.table.select(selectColumns.head, selectColumns.tail: _*) } override lazy val maybeReturnItems: Option[Seq[Var]] = - Some(returnExpressions.flatMap(_.owner).collect { case e: Var => e }.distinct) + Some( + returnExpressions.flatMap(_.owner).collect { case e: Var => e }.distinct + ) } -final case class Distinct[T <: Table[T] : TypeTag]( +final case class Distinct[T <: Table[T]: TypeTag]( in: RelationalOperator[T], fields: Set[Var] ) extends RelationalOperator[T] { - override lazy val _table: T = in.table.distinct(fields.flatMap(header.expressionsFor).map(header.column).toSeq: _*) + override lazy val _table: T = in.table.distinct( + fields.flatMap(header.expressionsFor).map(header.column).toSeq: _* + ) } -final case class Aggregate[T <: Table[T] : TypeTag]( +final case class Aggregate[T <: Table[T]: TypeTag]( in: RelationalOperator[T], group: Set[Var], aggregations: Set[(Var, Aggregator)] ) extends RelationalOperator[T] { - override lazy val header: RecordHeader = in.header.select(group).withExprs(aggregations.map { case (v, _) => v }) + override lazy val header: RecordHeader = + in.header.select(group).withExprs(aggregations.map { case (v, _) => v }) override lazy val _table: T = { - val preparedAggregations = aggregations.map { case (v, agg) => header.column(v) -> agg }.toMap + val preparedAggregations = aggregations.map { case (v, agg) => + header.column(v) -> agg + }.toMap in.table.group(group, preparedAggregations)(in.header, context.parameters) } } -final case class OrderBy[T <: Table[T] : TypeTag]( +final case class OrderBy[T <: Table[T]: TypeTag]( in: RelationalOperator[T], sortItems: Seq[SortItem] ) extends RelationalOperator[T] { override lazy val _table: T = { val tableSortItems = sortItems.map { - case Asc(expr) => expr -> Ascending + case Asc(expr) => expr -> Ascending case Desc(expr) => expr -> Descending } in.table.orderBy(tableSortItems: _*)(header, context.parameters) } } -final case class Skip[T <: Table[T] : TypeTag]( +final case class Skip[T <: Table[T]: TypeTag]( in: RelationalOperator[T], expr: Expr ) extends RelationalOperator[T] { @@ -370,15 +411,16 @@ final case class Skip[T <: Table[T] : TypeTag]( case Param(name) => context.parameters(name) match { case CypherInteger(l) => l - case other => throw IllegalArgumentException("a CypherInteger", other) + case other => throw IllegalArgumentException("a CypherInteger", other) } - case other => throw IllegalArgumentException("an integer literal or parameter", other) + case other => + throw IllegalArgumentException("an integer literal or parameter", other) } in.table.skip(skip) } } -final case class Limit[T <: Table[T] : TypeTag]( +final case class Limit[T <: Table[T]: TypeTag]( in: RelationalOperator[T], expr: Expr ) extends RelationalOperator[T] { @@ -389,7 +431,7 @@ final case class Limit[T <: Table[T] : TypeTag]( case Param(name) => context.parameters(name) match { case CypherInteger(v) => v - case other => throw IllegalArgumentException("a CypherInteger", other) + case other => throw IllegalArgumentException("a CypherInteger", other) } case other => throw IllegalArgumentException("an integer literal", other) } @@ -397,7 +439,7 @@ final case class Limit[T <: Table[T] : TypeTag]( } } -final case class EmptyRecords[T <: Table[T] : TypeTag]( +final case class EmptyRecords[T <: Table[T]: TypeTag]( in: RelationalOperator[T], fields: Set[Var] = Set.empty ) extends RelationalOperator[T] { @@ -407,12 +449,14 @@ final case class EmptyRecords[T <: Table[T] : TypeTag]( override lazy val _table: T = session.records.empty(header).table } -final case class FromCatalogGraph[T <: Table[T] : TypeTag]( +final case class FromCatalogGraph[T <: Table[T]: TypeTag]( in: RelationalOperator[T], logicalGraph: LogicalCatalogGraph ) extends RelationalOperator[T] { - override def graph: RelationalCypherGraph[T] = resolve(logicalGraph.qualifiedGraphName) + override def graph: RelationalCypherGraph[T] = resolve( + logicalGraph.qualifiedGraphName + ) override def graphName: QualifiedGraphName = logicalGraph.qualifiedGraphName @@ -420,35 +464,45 @@ final case class FromCatalogGraph[T <: Table[T] : TypeTag]( // Binary -final case class Join[T <: Table[T] : TypeTag]( +final case class Join[T <: Table[T]: TypeTag]( lhs: RelationalOperator[T], rhs: RelationalOperator[T], joinExprs: Seq[(Expr, Expr)] = Seq.empty, joinType: JoinType ) extends RelationalOperator[T] { - require((lhs.header.expressions intersect rhs.header.expressions).isEmpty, "Join cannot join operators with overlapping expressions") - require((lhs.header.columns intersect rhs.header.columns).isEmpty, "Join cannot join tables with column name collisions") + require( + (lhs.header.expressions intersect rhs.header.expressions).isEmpty, + "Join cannot join operators with overlapping expressions" + ) + require( + (lhs.header.columns intersect rhs.header.columns).isEmpty, + "Join cannot join tables with column name collisions" + ) override lazy val header: RecordHeader = lhs.header join rhs.header override lazy val _table: T = { - val joinCols = joinExprs.map { case (l, r) => header.column(l) -> rhs.header.column(r) } + val joinCols = joinExprs.map { case (l, r) => + header.column(l) -> rhs.header.column(r) + } lhs.table.join(rhs.table, joinType, joinCols: _*) } } /** - * Computes the union of the two input operators. The two inputs must have identical headers. - * This operation does not remove duplicates. + * Computes the union of the two input operators. The two inputs must have identical headers. This + * operation does not remove duplicates. * * The output header of this operation is identical to the input headers. * - * @param lhs the first operand - * @param rhs the second operand + * @param lhs + * the first operand + * @param rhs + * the second operand */ // TODO: rename to UnionByName // TODO: refactor to n-ary operator (i.e. take List[PhysicalOperator] as input) -final case class TabularUnionAll[T <: Table[T] : TypeTag]( +final case class TabularUnionAll[T <: Table[T]: TypeTag]( lhs: RelationalOperator[T], rhs: RelationalOperator[T] ) extends RelationalOperator[T] { @@ -464,11 +518,17 @@ final case class TabularUnionAll[T <: Table[T] : TypeTag]( val sortedRightColumns = rightColumns.sorted.mkString(", ") if (leftColumns.size != rightColumns.size) { - throw IllegalArgumentException("same number of columns", s"left: $sortedLeftColumns\n\tright: $sortedRightColumns") + throw IllegalArgumentException( + "same number of columns", + s"left: $sortedLeftColumns\n\tright: $sortedRightColumns" + ) } if (leftColumns.toSet != rightColumns.toSet) { - throw IllegalArgumentException("same column names", s"left: $sortedLeftColumns\n\tright: $sortedRightColumns") + throw IllegalArgumentException( + "same column names", + s"left: $sortedLeftColumns\n\tright: $sortedRightColumns" + ) } val orderedRhsTable = if (leftColumns != rightColumns) { @@ -481,12 +541,13 @@ final case class TabularUnionAll[T <: Table[T] : TypeTag]( } } -final case class ConstructGraph[T <: Table[T] : TypeTag]( +final case class ConstructGraph[T <: Table[T]: TypeTag]( in: RelationalOperator[T], constructedGraph: RelationalCypherGraph[T], construct: LogicalPatternGraph, override val context: RelationalRuntimeContext[T] -) extends RelationalOperator[T] with UnitTable[T] { +) extends RelationalOperator[T] + with UnitTable[T] { override def maybeReturnItems: Option[Seq[Var]] = None @@ -502,13 +563,15 @@ final case class ConstructGraph[T <: Table[T] : TypeTag]( // N-ary -final case class GraphUnionAll[T <: Table[T] : TypeTag]( +final case class GraphUnionAll[T <: Table[T]: TypeTag]( inputs: NonEmptyList[RelationalOperator[T]], qgn: QualifiedGraphName -) extends RelationalOperator[T] with EmptyTable[T] { +) extends RelationalOperator[T] + with EmptyTable[T] { override lazy val graphName: QualifiedGraphName = qgn - override lazy val graph: RelationalCypherGraph[T] = session.graphs.unionGraph(inputs.map(_.graph).toList: _*) + override lazy val graph: RelationalCypherGraph[T] = + session.graphs.unionGraph(inputs.map(_.graph).toList: _*) } diff --git a/okapi-relational/src/main/scala/org/opencypher/okapi/relational/impl/planning/ConstructGraphPlanner.scala b/okapi-relational/src/main/scala/org/opencypher/okapi/relational/impl/planning/ConstructGraphPlanner.scala index 22472b3d73..3d61e7136b 100644 --- a/okapi-relational/src/main/scala/org/opencypher/okapi/relational/impl/planning/ConstructGraphPlanner.scala +++ b/okapi-relational/src/main/scala/org/opencypher/okapi/relational/impl/planning/ConstructGraphPlanner.scala @@ -51,14 +51,20 @@ import scala.reflect.runtime.universe.TypeTag object ConstructGraphPlanner { - def planConstructGraph[T <: Table[T] : TypeTag](inputTablePlan: RelationalOperator[T], construct: LogicalPatternGraph) - (implicit context: RelationalRuntimeContext[T]): RelationalOperator[T] = { + def planConstructGraph[T <: Table[T]: TypeTag]( + inputTablePlan: RelationalOperator[T], + construct: LogicalPatternGraph + )(implicit context: RelationalRuntimeContext[T]): RelationalOperator[T] = { val prefixes = computePrefixes(construct) val onGraphs = construct.onGraphs.map { qgn => val start = relational.Start[T](qgn) - prefixes.get(qgn).map(p => relational.PrefixGraph(start, p)).getOrElse(start).graph + prefixes + .get(qgn) + .map(p => relational.PrefixGraph(start, p)) + .getOrElse(start) + .graph } val constructTable = initConstructTable(inputTablePlan, prefixes, construct) @@ -72,39 +78,53 @@ object ConstructGraphPlanner { } val constructedGraph = allGraphs match { - case Nil => context.session.graphs.empty + case Nil => context.session.graphs.empty case head :: Nil => head - case several => context.session.graphs.unionGraph(several: _*) + case several => context.session.graphs.unionGraph(several: _*) } - val constructOp = ConstructGraph(constructTable, constructedGraph, construct, context) + val constructOp = + ConstructGraph(constructTable, constructedGraph, construct, context) context.queryLocalCatalog += (construct.qualifiedGraphName -> constructOp.graph) constructOp } - private def computePrefixes(construct: LogicalPatternGraph): Map[QualifiedGraphName, GraphIdPrefix] = { + private def computePrefixes( + construct: LogicalPatternGraph + ): Map[QualifiedGraphName, GraphIdPrefix] = { val onGraphQgns = construct.onGraphs - val cloneGraphQgns = construct.clones.values.flatMap(_.cypherType.graph).toList - val createGraphQgns = if (construct.newElements.nonEmpty) List(construct.qualifiedGraphName) else Nil + val cloneGraphQgns = + construct.clones.values.flatMap(_.cypherType.graph).toList + val createGraphQgns = + if (construct.newElements.nonEmpty) List(construct.qualifiedGraphName) + else Nil val graphQgns = (onGraphQgns ++ cloneGraphQgns ++ createGraphQgns).distinct if (graphQgns.size <= 1) { Map.empty // No prefixes needed when there is at most one graph QGN } else { graphQgns.zipWithIndex.map { case (qgn, i) => // Assign GraphIdPrefix `11111111` to created nodes and relationships - qgn -> (if (qgn == construct.qualifiedGraphName) (-1).toByte else i.toByte) + qgn -> (if (qgn == construct.qualifiedGraphName) (-1).toByte + else i.toByte) }.toMap } } - private def initConstructTable[T <: Table[T] : TypeTag]( + private def initConstructTable[T <: Table[T]: TypeTag]( inputTablePlan: RelationalOperator[T], unionPrefixStrategy: Map[QualifiedGraphName, GraphIdPrefix], construct: LogicalPatternGraph ): RelationalOperator[T] = { - val LogicalPatternGraph(_, clonedVarsToInputVars, createdElements, sets, _, _) = construct + val LogicalPatternGraph( + _, + clonedVarsToInputVars, + createdElements, + sets, + _, + _ + ) = construct // Apply aliases in CLONE to input table in order to create the base table, on which CONSTRUCT happens val aliasClones = clonedVarsToInputVars @@ -114,14 +134,17 @@ object ConstructGraphPlanner { val aliasOp: RelationalOperator[T] = if (aliasClones.isEmpty) { inputTablePlan } else { - relational.Alias(inputTablePlan, aliasClones.map { case (expr, alias) => expr as alias }.toSeq) + relational.Alias( + inputTablePlan, + aliasClones.map { case (expr, alias) => expr as alias }.toSeq + ) } val prefixedBaseTableOp = clonedVarsToInputVars.foldLeft(aliasOp) { case (op, (alias, original)) => unionPrefixStrategy.get(original.cypherType.graph.get) match { case Some(prefix) => op.prefixVariableId(alias, prefix) - case None => op + case None => op } } @@ -130,17 +153,29 @@ object ConstructGraphPlanner { if (createdElements.isEmpty) { prefixedBaseTableOp } else { - val maybeCreatedElementPrefix = unionPrefixStrategy.get(construct.qualifiedGraphName) - val elementsOp = planConstructElements(prefixedBaseTableOp, createdElements, maybeCreatedElementPrefix) + val maybeCreatedElementPrefix = + unionPrefixStrategy.get(construct.qualifiedGraphName) + val elementsOp = planConstructElements( + prefixedBaseTableOp, + createdElements, + maybeCreatedElementPrefix + ) val setValueForExprTuples = sets.flatMap { case SetPropertyItem(propertyKey, v, valueExpr) => - List(valueExpr -> ElementProperty(v, PropertyKey(propertyKey))(valueExpr.cypherType)) + List( + valueExpr -> ElementProperty(v, PropertyKey(propertyKey))( + valueExpr.cypherType + ) + ) case SetLabelItem(v, labels) => labels.toList.map { label => v.cypherType.material match { case _: CTNode => TrueLit -> expr.HasLabel(v, Label(label)) - case other => throw UnsupportedOperationException(s"Cannot set a label on $other") + case other => + throw UnsupportedOperationException( + s"Cannot set a label on $other" + ) } } } @@ -156,7 +191,7 @@ object ConstructGraphPlanner { constructedElementsOp.dropExprSet(varsToRemoveFromTable) } - def planConstructElements[T <: Table[T] : TypeTag]( + def planConstructElements[T <: Table[T]: TypeTag]( inOp: RelationalOperator[T], toCreate: Set[ConstructedElement], maybeCreatedElementIdPrefix: Option[GraphIdPrefix] @@ -172,7 +207,13 @@ object ConstructGraphPlanner { val (_, nodesToCreate) = nodes.foldLeft(0 -> Seq.empty[(Expr, Expr)]) { case ((nextColumnPartitionId, nodeProjections), nextNodeToConstruct) => - (nextColumnPartitionId + 1) -> (nodeProjections ++ computeNodeProjections(inOp, maybeCreatedElementIdPrefix, nextColumnPartitionId, nodes.size, nextNodeToConstruct)) + (nextColumnPartitionId + 1) -> (nodeProjections ++ computeNodeProjections( + inOp, + maybeCreatedElementIdPrefix, + nextColumnPartitionId, + nodes.size, + nextNodeToConstruct + )) } val createdNodesOp = inOp.addInto(nodesToCreate.map(_.swap): _*) @@ -180,10 +221,18 @@ object ConstructGraphPlanner { val (_, relsToCreate) = rels.foldLeft(0 -> Seq.empty[(Expr, Expr)]) { case ((nextColumnPartitionId, relProjections), nextRelToConstruct) => (nextColumnPartitionId + 1) -> - (relProjections ++ computeRelationshipProjections(createdNodesOp, maybeCreatedElementIdPrefix, nextColumnPartitionId, rels.size, nextRelToConstruct)) + (relProjections ++ computeRelationshipProjections( + createdNodesOp, + maybeCreatedElementIdPrefix, + nextColumnPartitionId, + rels.size, + nextRelToConstruct + )) } - createdNodesOp.addInto(relsToCreate.map { case (into, value) => value -> into }: _*) + createdNodesOp.addInto(relsToCreate.map { case (into, value) => + value -> into + }: _*) } def computeNodeProjections[T <: Table[T]]( @@ -195,19 +244,24 @@ object ConstructGraphPlanner { ): Map[Expr, Expr] = { val idTuple = node.v -> - prefixId(generateId(columnIdPartition, numberOfColumnPartitions), maybeCreatedElementIdPrefix) + prefixId( + generateId(columnIdPartition, numberOfColumnPartitions), + maybeCreatedElementIdPrefix + ) val copiedLabelTuples = node.baseElement match { - case Some(origNode) => copyExpressions(inOp, node.v)(_.labelsFor(origNode)) + case Some(origNode) => + copyExpressions(inOp, node.v)(_.labelsFor(origNode)) case None => Map.empty } - val createdLabelTuples = node.labels.map { - label => HasLabel(node.v, label) -> TrueLit + val createdLabelTuples = node.labels.map { label => + HasLabel(node.v, label) -> TrueLit }.toMap val propertyTuples = node.baseElement match { - case Some(origNode) => copyExpressions(inOp, node.v)(_.propertiesFor(origNode)) + case Some(origNode) => + copyExpressions(inOp, node.v)(_.propertiesFor(origNode)) case None => Map.empty } @@ -224,11 +278,15 @@ object ConstructGraphPlanner { numberOfColumnPartitions: Int, toConstruct: ConstructedRelationship ): Map[Expr, Expr] = { - val ConstructedRelationship(rel, source, target, typOpt, baseRelOpt) = toConstruct + val ConstructedRelationship(rel, source, target, typOpt, baseRelOpt) = + toConstruct // id needs to be generated val idTuple: (Var, Expr) = rel -> - prefixId(generateId(columnIdPartition, numberOfColumnPartitions), maybeCreatedElementIdPrefix) + prefixId( + generateId(columnIdPartition, numberOfColumnPartitions), + maybeCreatedElementIdPrefix + ) // source and target are present: just copy val sourceTuple = { @@ -258,19 +316,27 @@ object ConstructGraphPlanner { propertyTuples ++ typeTuple + idTuple + sourceTuple + targetTuple } - def copyExpressions[E <: Expr, T <: Table[T]](inOp: RelationalOperator[T], targetVar: Var) - (extractor: RecordHeader => Set[E]): Map[Expr, Expr] = { - extractor(inOp.header) - .map(o => o.withOwner(targetVar) -> o) - .toMap + def copyExpressions[E <: Expr, T <: Table[T]]( + inOp: RelationalOperator[T], + targetVar: Var + )(extractor: RecordHeader => Set[E]): Map[Expr, Expr] = { + extractor(inOp.header) + .map(o => o.withOwner(targetVar) -> o) + .toMap } - def prefixId(id: Expr, maybeCreatedElementIdPrefix: Option[GraphIdPrefix]): Expr = { + def prefixId( + id: Expr, + maybeCreatedElementIdPrefix: Option[GraphIdPrefix] + ): Expr = { maybeCreatedElementIdPrefix.map(PrefixId(id, _)).getOrElse(id) } // TODO: improve documentation and add specific tests - def generateId(columnIdPartition: Int, numberOfColumnPartitions: Int): Expr = { + def generateId( + columnIdPartition: Int, + numberOfColumnPartitions: Int + ): Expr = { val columnPartitionBits = math.log(numberOfColumnPartitions).floor.toInt + 1 val totalIdSpaceBits = 33 val columnIdShift = totalIdSpaceBits - columnPartitionBits @@ -284,11 +350,12 @@ object ConstructGraphPlanner { } /** - * Given the construction table this method extracts all possible scans over that table - * and compiles them to a ScanGraph. Note that cloned elements that are also present in an `ON GRAPH` are not - * included in the ScanGraph to avoid duplication. This improves scan performance as fewer DISTINCTs are needed. + * Given the construction table this method extracts all possible scans over that table and + * compiles them to a ScanGraph. Note that cloned elements that are also present in an `ON GRAPH` + * are not included in the ScanGraph to avoid duplication. This improves scan performance as + * fewer DISTINCTs are needed. */ - private def extractScanGraph[T <: Table[T] : TypeTag]( + private def extractScanGraph[T <: Table[T]: TypeTag]( logicalPatternGraph: LogicalPatternGraph, inputOp: RelationalOperator[T] )(implicit context: RelationalRuntimeContext[T]): RelationalCypherGraph[T] = { @@ -296,47 +363,65 @@ object ConstructGraphPlanner { val schema = logicalPatternGraph.schema // extract label sets - val setLabelItems = logicalPatternGraph.sets.collect { - case SetLabelItem(variable, labels) => variable -> labels - }.groupBy { - case (variable, _) => variable - }.map { - case (variable, list) => variable -> list.flatMap(_._2).toSet - } + val setLabelItems = logicalPatternGraph.sets + .collect { case SetLabelItem(variable, labels) => + variable -> labels + } + .groupBy { case (variable, _) => + variable + } + .map { case (variable, list) => + variable -> list.flatMap(_._2).toSet + } // Compute Scans from constructed and cloned elements - val createdElementScanTypes = scanTypesFromCreatedElements(logicalPatternGraph, setLabelItems) - val clonedElementScanTypes = scanTypesFromClonedElements(logicalPatternGraph, setLabelItems) + val createdElementScanTypes = + scanTypesFromCreatedElements(logicalPatternGraph, setLabelItems) + val clonedElementScanTypes = + scanTypesFromClonedElements(logicalPatternGraph, setLabelItems) val scansForCreated = createScans(createdElementScanTypes, inputOp, schema) - val scansForCloned = createScans(clonedElementScanTypes, inputOp, schema, distinct = true) + val scansForCloned = + createScans(clonedElementScanTypes, inputOp, schema, distinct = true) - val scanGraphSchema: PropertyGraphSchema = computeScanGraphSchema(schema, createdElementScanTypes, clonedElementScanTypes) + val scanGraphSchema: PropertyGraphSchema = computeScanGraphSchema( + schema, + createdElementScanTypes, + clonedElementScanTypes + ) // Construct the scan graph - context.session.graphs.create(Some(scanGraphSchema), scansForCreated ++ scansForCloned: _*) + context.session.graphs + .create(Some(scanGraphSchema), scansForCreated ++ scansForCloned: _*) } - /** - * Computes all scan types that can be created from created elements. - */ - private def scanTypesFromCreatedElements[T <: Table[T] : TypeTag]( + /** Computes all scan types that can be created from created elements. */ + private def scanTypesFromCreatedElements[T <: Table[T]: TypeTag]( logicalPatternGraph: LogicalPatternGraph, setLabels: Map[Var, Set[String]] )(implicit context: RelationalRuntimeContext[T]): Set[(Var, CypherType)] = { logicalPatternGraph.newElements.flatMap { case c: ConstructedNode if c.baseElement.isEmpty => - val allLabels = c.labels.map(_.name) ++ setLabels.getOrElse(c.v, Set.empty) + val allLabels = + c.labels.map(_.name) ++ setLabels.getOrElse(c.v, Set.empty) Seq(c.v -> CTNode(allLabels)) case c: ConstructedNode => c.baseElement.get.cypherType match { case CTNode(baseLabels, Some(sourceGraph)) => val sourceSchema = context.resolveGraph(sourceGraph).schema - val allLabels = c.labels.map(_.name) ++ baseLabels ++ setLabels.getOrElse(c.v, Set.empty) - sourceSchema.forNode(allLabels).allCombinations.map(c.v -> CTNode(_)).toSeq - case other => throw UnsupportedOperationException(s"Cannot construct node scan from $other") + val allLabels = c.labels.map(_.name) ++ baseLabels ++ setLabels + .getOrElse(c.v, Set.empty) + sourceSchema + .forNode(allLabels) + .allCombinations + .map(c.v -> CTNode(_)) + .toSeq + case other => + throw UnsupportedOperationException( + s"Cannot construct node scan from $other" + ) } case c: ConstructedRelationship if c.baseElement.isEmpty => @@ -348,25 +433,32 @@ object ConstructGraphPlanner { val sourceSchema = context.resolveGraph(sourceGraph).schema val possibleTypes = c.typ match { case Some(t) => Set(t) - case _ => baseTypes + case _ => baseTypes } - sourceSchema.forRelationship(CTRelationship(possibleTypes)).relationshipTypes.map(c.v -> CTRelationship(_)).toSeq - case other => throw UnsupportedOperationException(s"Cannot construct relationship scan from $other") + sourceSchema + .forRelationship(CTRelationship(possibleTypes)) + .relationshipTypes + .map(c.v -> CTRelationship(_)) + .toSeq + case other => + throw UnsupportedOperationException( + s"Cannot construct relationship scan from $other" + ) } } } - /** - * Computes all scan types that can be created from cloned elements - */ - private def scanTypesFromClonedElements[T <: Table[T] : TypeTag]( + /** Computes all scan types that can be created from cloned elements */ + private def scanTypesFromClonedElements[T <: Table[T]: TypeTag]( logicalPatternGraph: LogicalPatternGraph, setLabels: Map[Var, Set[String]] )(implicit context: RelationalRuntimeContext[T]): Set[(Var, CypherType)] = { - val clonedElementsToKeep = logicalPatternGraph.clones.filterNot { - case (_, base) => logicalPatternGraph.onGraphs.contains(base.cypherType.graph.get) - }.mapValues(_.cypherType) + val clonedElementsToKeep = logicalPatternGraph.clones + .filterNot { case (_, base) => + logicalPatternGraph.onGraphs.contains(base.cypherType.graph.get) + } + .mapValues(_.cypherType) clonedElementsToKeep.toSeq.flatMap { case (v, CTNode(labels, Some(sourceGraph))) => @@ -374,37 +466,47 @@ object ConstructGraphPlanner { val allLabels = labels ++ setLabels.getOrElse(v, Set.empty) sourceSchema.forNode(allLabels).allCombinations.map(v -> CTNode(_)) - case (v, r@CTRelationship(_, Some(sourceGraph))) => + case (v, r @ CTRelationship(_, Some(sourceGraph))) => val sourceSchema = context.resolveGraph(sourceGraph).schema - sourceSchema.forRelationship(r.toCTRelationship).relationshipTypes.map(v -> CTRelationship(_)).toSeq - - case other => throw UnsupportedOperationException(s"Cannot construct scan from $other") + sourceSchema + .forRelationship(r.toCTRelationship) + .relationshipTypes + .map(v -> CTRelationship(_)) + .toSeq + + case other => + throw UnsupportedOperationException( + s"Cannot construct scan from $other" + ) }.toSet } - /** - * Creates the scans for the given scan types - */ - private def createScans[T <: Table[T] : TypeTag]( + /** Creates the scans for the given scan types */ + private def createScans[T <: Table[T]: TypeTag]( scanElements: Set[(Var, CypherType)], inputOp: RelationalOperator[T], schema: PropertyGraphSchema, distinct: Boolean = false )(implicit context: RelationalRuntimeContext[T]): Seq[ElementTable[T]] = { - val groupedScanElements = scanElements.map { - case (v, CTNode(labels, g)) => - v -> CTNode(labels, g) - case other => other - }.groupBy { - case (_, cypherType) => cypherType - }.map { - case (ct, list) => ct -> list.map(_._1).toSeq - } + val groupedScanElements = scanElements + .map { + case (v, CTNode(labels, g)) => + v -> CTNode(labels, g) + case other => other + } + .groupBy { case (_, cypherType) => + cypherType + } + .map { case (ct, list) => + ct -> list.map(_._1).toSeq + } - groupedScanElements.map { case (ct, vars) => scansForType(ct, vars, inputOp, schema, distinct) }.toSeq + groupedScanElements.map { case (ct, vars) => + scansForType(ct, vars, inputOp, schema, distinct) + }.toSeq } - def scansForType[T <: Table[T] : TypeTag]( + def scansForType[T <: Table[T]: TypeTag]( ct: CypherType, vars: Seq[Var], inputOp: RelationalOperator[T], @@ -419,8 +521,9 @@ object ConstructGraphPlanner { // we need to get rid of the label and rel type columns val dropExprs = ct match { case _: CTRelationship => scanOp.header.typesFor(element) - case _: CTNode => scanOp.header.labelsFor(element) - case other => throw UnsupportedOperationException(s"Cannot create scan for $other") + case _: CTNode => scanOp.header.labelsFor(element) + case other => + throw UnsupportedOperationException(s"Cannot create scan for $other") } scanOp.dropExpressions(dropExprs.toSeq: _*) @@ -432,55 +535,70 @@ object ConstructGraphPlanner { case head :: tail => val targetHeader = head.header val targetElement = head.singleElement - tail.map { op => - op.alignWith(op.singleElement, targetElement, targetHeader) - }.foldLeft(head)(_ unionAll _) - - case _ => throw IllegalArgumentException( - expected = "Non-empty list of scans", - actual = "Empty list", - explanation = "This should never happen, possible planning bug.") + tail + .map { op => + op.alignWith(op.singleElement, targetElement, targetHeader) + } + .foldLeft(head)(_ unionAll _) + + case _ => + throw IllegalArgumentException( + expected = "Non-empty list of scans", + actual = "Empty list", + explanation = "This should never happen, possible planning bug." + ) } - val distinctScans = if (distinct) relational.Distinct(combinedScans, combinedScans.header.vars) else combinedScans + val distinctScans = + if (distinct) + relational.Distinct(combinedScans, combinedScans.header.vars) + else combinedScans distinctScans.elementTable } - private def scanForElementAndType[T <: Table[T] : TypeTag]( + private def scanForElementAndType[T <: Table[T]: TypeTag]( extractionVar: Var, elementType: CypherType, op: RelationalOperator[T], schema: PropertyGraphSchema ): RelationalOperator[T] = { val targetElement = Var.unnamed(elementType) - val targetElementHeader = schema.headerForElement(targetElement, exactLabelMatch = true) + val targetElementHeader = + schema.headerForElement(targetElement, exactLabelMatch = true) val labelOrTypePredicate = elementType match { case CTNode(labels, _) => val labelFilters = op.header.labelsFor(extractionVar).map { - case expr@HasLabel(_, Label(label)) if labels.contains(label) => Equals(expr, TrueLit) + case expr @ HasLabel(_, Label(label)) if labels.contains(label) => + Equals(expr, TrueLit) case expr: HasLabel => Equals(expr, FalseLit) } Ands(labelFilters) case CTRelationship(relTypes, _) => - val relTypeExprs: Set[Expr] = relTypes.map(relType => HasType(extractionVar, RelType(relType))) - val physicalExprs = relTypeExprs intersect op.header.expressionsFor(extractionVar) + val relTypeExprs: Set[Expr] = + relTypes.map(relType => HasType(extractionVar, RelType(relType))) + val physicalExprs = + relTypeExprs intersect op.header.expressionsFor(extractionVar) Ors(physicalExprs.map(expr => Equals(expr, TrueLit))) - case other => throw IllegalArgumentException("CTNode or CTRelationship", other) + case other => + throw IllegalArgumentException("CTNode or CTRelationship", other) } val selected = op.select(extractionVar) val idExprs = op.header.idExpressions(extractionVar).toSeq - val validElementPredicate = Ands(idExprs.map(idExpr => IsNotNull(idExpr)) :+ labelOrTypePredicate: _*) + val validElementPredicate = Ands( + idExprs.map(idExpr => IsNotNull(idExpr)) :+ labelOrTypePredicate: _* + ) val filtered = relational.Filter(selected, validElementPredicate) val inputElement = filtered.singleElement - val alignedScan = filtered.alignWith(inputElement, targetElement, targetElementHeader) + val alignedScan = + filtered.alignWith(inputElement, targetElement, targetElementHeader) alignedScan } @@ -490,25 +608,32 @@ object ConstructGraphPlanner { createdElementScanTypes: Set[(Var, CypherType)], clonedElementScanTypes: Set[(Var, CypherType)] ): PropertyGraphSchema = { - val (nodeTypes, relTypes) = (createdElementScanTypes ++ clonedElementScanTypes).partition { - case (_, _: CTNode) => true - case _ => false - } + val (nodeTypes, relTypes) = + (createdElementScanTypes ++ clonedElementScanTypes).partition { + case (_, _: CTNode) => true + case _ => false + } - val scanGraphNodeLabelCombos = nodeTypes.collect { - case (_, CTNode(labels, _)) => labels + val scanGraphNodeLabelCombos = nodeTypes.collect { case (_, CTNode(labels, _)) => + labels } - val scanGraphRelTypes = relTypes.collect { - case (_, CTRelationship(types, _)) => types + val scanGraphRelTypes = relTypes.collect { case (_, CTRelationship(types, _)) => + types }.flatten PropertyGraphSchema.empty - .foldLeftOver(scanGraphNodeLabelCombos) { - case (acc, labelCombo) => acc.withNodePropertyKeys(labelCombo, baseSchema.nodePropertyKeys(labelCombo)) + .foldLeftOver(scanGraphNodeLabelCombos) { case (acc, labelCombo) => + acc.withNodePropertyKeys( + labelCombo, + baseSchema.nodePropertyKeys(labelCombo) + ) } - .foldLeftOver(scanGraphRelTypes) { - case (acc, typ) => acc.withRelationshipPropertyKeys(typ, baseSchema.relationshipPropertyKeys(typ)) + .foldLeftOver(scanGraphRelTypes) { case (acc, typ) => + acc.withRelationshipPropertyKeys( + typ, + baseSchema.relationshipPropertyKeys(typ) + ) } } } diff --git a/okapi-relational/src/main/scala/org/opencypher/okapi/relational/impl/planning/PhysicalConstants.scala b/okapi-relational/src/main/scala/org/opencypher/okapi/relational/impl/planning/PhysicalConstants.scala index dfdc726d07..b50b594efb 100644 --- a/okapi-relational/src/main/scala/org/opencypher/okapi/relational/impl/planning/PhysicalConstants.scala +++ b/okapi-relational/src/main/scala/org/opencypher/okapi/relational/impl/planning/PhysicalConstants.scala @@ -36,5 +36,5 @@ case object CrossJoin extends JoinType sealed trait Order -case object Ascending extends Order -case object Descending extends Order +case object Ascending extends Order +case object Descending extends Order diff --git a/okapi-relational/src/main/scala/org/opencypher/okapi/relational/impl/planning/RelationalOptimizer.scala b/okapi-relational/src/main/scala/org/opencypher/okapi/relational/impl/planning/RelationalOptimizer.scala index 237b55492a..0dc4bcb235 100644 --- a/okapi-relational/src/main/scala/org/opencypher/okapi/relational/impl/planning/RelationalOptimizer.scala +++ b/okapi-relational/src/main/scala/org/opencypher/okapi/relational/impl/planning/RelationalOptimizer.scala @@ -34,16 +34,20 @@ import scala.reflect.runtime.universe.TypeTag object RelationalOptimizer { - def process[T <: Table[T] : TypeTag](input: RelationalOperator[T]): RelationalOperator[T] = { + def process[T <: Table[T]: TypeTag]( + input: RelationalOperator[T] + ): RelationalOperator[T] = { InsertCachingOperators(input) } object InsertCachingOperators { - def apply[T <: Table[T] : TypeTag](input: RelationalOperator[T]): RelationalOperator[T] = { + def apply[T <: Table[T]: TypeTag]( + input: RelationalOperator[T] + ): RelationalOperator[T] = { val replacements = calculateReplacementMap(input).filterKeys { case _: Start[T] => false - case _ => true + case _ => true } val nodesToReplace = replacements.keySet @@ -51,41 +55,50 @@ object RelationalOptimizer { TopDown[RelationalOperator[T]] { case cache: Cache[T] => cache case parent if (parent.childrenAsSet intersect nodesToReplace).nonEmpty => - val newChildren = parent.children.map(c => replacements.getOrElse(c, c)) + val newChildren = + parent.children.map(c => replacements.getOrElse(c, c)) parent.withNewChildren(newChildren) }.transform(input) } - private def calculateReplacementMap[T <: Table[T] : TypeTag](input: RelationalOperator[T]): Map[RelationalOperator[T], RelationalOperator[T]] = { + private def calculateReplacementMap[T <: Table[T]: TypeTag]( + input: RelationalOperator[T] + ): Map[RelationalOperator[T], RelationalOperator[T]] = { val opCounts = identifyDuplicates(input) - val opsByHeight = opCounts.keys.toSeq.sortWith((a, b) => a.height > b.height) - val (opsToCache, _) = opsByHeight.foldLeft(Set.empty[RelationalOperator[T]] -> opCounts) { (agg, currentOp) => - agg match { - case (currentOpsToCache, currentCounts) => - val currentOpCount = currentCounts(currentOp) - if (currentOpCount > 1) { - val updatedOps = currentOpsToCache + currentOp - // We're traversing `opsByHeight` from largest to smallest query sub-tree. - // We pick the trees with the largest height for caching first, and then reduce the duplicate count - // for the sub-trees of the cached tree by the number of times the parent tree appears. - // The idea behind this is that if the parent was already cached, there is no need to additionally - // cache all its children (unless they're used with a different parent somewhere else). - val updatedCounts = currentCounts.map { - case (op, count) => op -> (if (currentOp.containsTree(op)) count - currentOpCount else count) + val opsByHeight = + opCounts.keys.toSeq.sortWith((a, b) => a.height > b.height) + val (opsToCache, _) = + opsByHeight.foldLeft(Set.empty[RelationalOperator[T]] -> opCounts) { (agg, currentOp) => + agg match { + case (currentOpsToCache, currentCounts) => + val currentOpCount = currentCounts(currentOp) + if (currentOpCount > 1) { + val updatedOps = currentOpsToCache + currentOp + // We're traversing `opsByHeight` from largest to smallest query sub-tree. + // We pick the trees with the largest height for caching first, and then reduce the duplicate count + // for the sub-trees of the cached tree by the number of times the parent tree appears. + // The idea behind this is that if the parent was already cached, there is no need to additionally + // cache all its children (unless they're used with a different parent somewhere else). + val updatedCounts = currentCounts.map { case (op, count) => + op -> (if (currentOp.containsTree(op)) + count - currentOpCount + else count) + } + updatedOps -> updatedCounts + } else { + currentOpsToCache -> currentCounts } - updatedOps -> updatedCounts - } else { - currentOpsToCache -> currentCounts - } + } } - } opsToCache.map(op => op -> Cache[T](op)).toMap } - private def identifyDuplicates[T <: Table[T]](input: RelationalOperator[T]): Map[RelationalOperator[T], Int] = { + private def identifyDuplicates[T <: Table[T]]( + input: RelationalOperator[T] + ): Map[RelationalOperator[T], Int] = { input - .foldLeft(Map.empty[RelationalOperator[T], Int].withDefaultValue(0)) { - case (agg, op) => agg.updated(op, agg(op) + 1) + .foldLeft(Map.empty[RelationalOperator[T], Int].withDefaultValue(0)) { case (agg, op) => + agg.updated(op, agg(op) + 1) } .filter(_._2 > 1) } diff --git a/okapi-relational/src/main/scala/org/opencypher/okapi/relational/impl/planning/RelationalPlanner.scala b/okapi-relational/src/main/scala/org/opencypher/okapi/relational/impl/planning/RelationalPlanner.scala index 090f860122..1a17f1f6e2 100644 --- a/okapi-relational/src/main/scala/org/opencypher/okapi/relational/impl/planning/RelationalPlanner.scala +++ b/okapi-relational/src/main/scala/org/opencypher/okapi/relational/impl/planning/RelationalPlanner.scala @@ -30,7 +30,11 @@ import cats.data.NonEmptyList import org.opencypher.okapi.api.graph._ import org.opencypher.okapi.api.io.conversion.{NodeMappingBuilder, RelationshipMappingBuilder} import org.opencypher.okapi.api.types._ -import org.opencypher.okapi.impl.exception.{NotImplementedException, SchemaException, UnsupportedOperationException} +import org.opencypher.okapi.impl.exception.{ + NotImplementedException, + SchemaException, + UnsupportedOperationException +} import org.opencypher.okapi.impl.types.CypherTypeUtils._ import org.opencypher.okapi.ir.api.block.SortItem import org.opencypher.okapi.ir.api.expr.PrefixId.GraphIdPrefix @@ -52,8 +56,9 @@ import scala.reflect.runtime.universe.TypeTag object RelationalPlanner { // TODO: rename to 'plan' - def process[T <: Table[T] : TypeTag](input: LogicalOperator) - (implicit context: RelationalRuntimeContext[T]): RelationalOperator[T] = { + def process[T <: Table[T]: TypeTag]( + input: LogicalOperator + )(implicit context: RelationalRuntimeContext[T]): RelationalOperator[T] = { implicit val session: CypherSession = context.session @@ -62,7 +67,6 @@ object RelationalPlanner { process[T](lhs).join(process[T](rhs), Seq.empty, CrossJoin) case logical.Select(fields, in, _) => - val inOp = process[T](in) val selectExpressions = fields @@ -78,8 +82,8 @@ object RelationalPlanner { maybeAlias match { case Some(alias) if containsExpr => inOp.alias(expr as alias) - case Some(alias) => inOp.add(expr as alias) - case None => inOp.add(expr) + case Some(alias) => inOp.add(expr as alias) + case None => inOp.add(expr) } case logical.EmptyRecords(fields, in, _) => @@ -87,13 +91,15 @@ object RelationalPlanner { case logical.Start(graph, _) => relational.Start(graph.qualifiedGraphName) - case logical.DrivingTable(graph, _, _) => relational.Start(graph.qualifiedGraphName, context.maybeInputRecords) + case logical.DrivingTable(graph, _, _) => + relational.Start(graph.qualifiedGraphName, context.maybeInputRecords) case logical.FromGraph(graph, in, _) => val inOp = process[T](in) graph match { case g: LogicalCatalogGraph => relational.FromCatalogGraph(inOp, g) - case construct: LogicalPatternGraph => planConstructGraph(inOp, construct) + case construct: LogicalPatternGraph => + planConstructGraph(inOp, construct) } case logical.Unwind(list, item, in, _) => @@ -108,7 +114,8 @@ object RelationalPlanner { mapping ) - case logical.Aggregate(aggregations, group, in, _) => relational.Aggregate(process[T](in), group, aggregations) + case logical.Aggregate(aggregations, group, in, _) => + relational.Aggregate(process[T](in), group, aggregations) case logical.Filter(expr, in, _) => process[T](in).filter(expr) @@ -127,7 +134,15 @@ object RelationalPlanner { process[T](left).graphUnionAll(process[T](right)) // TODO: This needs to be a ternary operator taking source, rels and target records instead of just source and target and planning rels only at the physical layer - case logical.Expand(source, rel, target, direction, sourceOp, targetOp, _) => + case logical.Expand( + source, + rel, + target, + direction, + sourceOp, + targetOp, + _ + ) => val first = process[T](sourceOp) val third = process[T](targetOp) @@ -144,22 +159,28 @@ object RelationalPlanner { direction match { case Outgoing => - val tempResult = first.join(second, Seq(source -> startNode), InnerJoin) + val tempResult = + first.join(second, Seq(source -> startNode), InnerJoin) tempResult.join(third, Seq(endNode -> target), InnerJoin) case org.opencypher.okapi.logical.impl.Incoming => - val tempResult = third.join(second, Seq(target -> endNode), InnerJoin) + val tempResult = + third.join(second, Seq(target -> endNode), InnerJoin) tempResult.join(first, Seq(startNode -> source), InnerJoin) case Undirected => - val tempOutgoing = first.join(second, Seq(source -> startNode), InnerJoin) - val outgoing = tempOutgoing.join(third, Seq(endNode -> target), InnerJoin) + val tempOutgoing = + first.join(second, Seq(source -> startNode), InnerJoin) + val outgoing = + tempOutgoing.join(third, Seq(endNode -> target), InnerJoin) val filterExpression = Not(Equals(startNode, endNode)) val relsWithoutLoops = second.filter(filterExpression) - val tempIncoming = first.join(relsWithoutLoops, Seq(source -> endNode), InnerJoin) - val incoming = tempIncoming.join(third, Seq(startNode -> target), InnerJoin) + val tempIncoming = + first.join(relsWithoutLoops, Seq(source -> endNode), InnerJoin) + val incoming = + tempIncoming.join(third, Seq(startNode -> target), InnerJoin) relational.TabularUnionAll(outgoing, incoming) } @@ -180,16 +201,38 @@ object RelationalPlanner { direction match { case Outgoing | Incoming => - in.join(relationships, Seq(source -> startNode, target -> endNode), InnerJoin) + in.join( + relationships, + Seq(source -> startNode, target -> endNode), + InnerJoin + ) case Undirected => - val outgoing = in.join(relationships, Seq(source -> startNode, target -> endNode), InnerJoin) - val incoming = in.join(relationships, Seq(target -> startNode, source -> endNode), InnerJoin) + val outgoing = in.join( + relationships, + Seq(source -> startNode, target -> endNode), + InnerJoin + ) + val incoming = in.join( + relationships, + Seq(target -> startNode, source -> endNode), + InnerJoin + ) relational.TabularUnionAll(outgoing, incoming) } - case logical.BoundedVarLengthExpand(source, list, target, edgeScanType, direction, lower, upper, sourceOp, targetOp, _) => - + case logical.BoundedVarLengthExpand( + source, + list, + target, + edgeScanType, + direction, + lower, + upper, + sourceOp, + targetOp, + _ + ) => val edgeScan = Var(list.name)(edgeScanType) val edgePattern = RelationshipPattern(edgeScanType) @@ -204,17 +247,33 @@ object RelationalPlanner { val planner = direction match { // TODO: verify that var length is able to traverse in different directions - case Outgoing | Incoming => new DirectedVarLengthExpandPlanner[T]( - source, list, edgeScan, target, - lower, upper, - sourceOp, edgeScanOp, targetOp, - isExpandInto) - - case Undirected => new UndirectedVarLengthExpandPlanner[T]( - source, list, edgeScan, target, - lower, upper, - sourceOp, edgeScanOp, targetOp, - isExpandInto) + case Outgoing | Incoming => + new DirectedVarLengthExpandPlanner[T]( + source, + list, + edgeScan, + target, + lower, + upper, + sourceOp, + edgeScanOp, + targetOp, + isExpandInto + ) + + case Undirected => + new UndirectedVarLengthExpandPlanner[T]( + source, + list, + edgeScan, + target, + lower, + upper, + sourceOp, + edgeScanOp, + targetOp, + isExpandInto + ) } planner.plan @@ -222,7 +281,6 @@ object RelationalPlanner { case logical.Optional(lhs, rhs, _) => planOptional(lhs, rhs) case logical.ExistsSubQuery(predicateField, lhs, rhs, _) => - val leftResult = process[T](lhs) val rightResult = process[T](rhs) @@ -238,9 +296,14 @@ object RelationalPlanner { val exprsToRemove = joinExprs.flatMap(v => rightHeader.ownedBy(v)) val reducedRhsData = rightWithAliases.dropExprSet(exprsToRemove) // 3. Compute distinct rows in rhs - val distinctRhsData = relational.Distinct(reducedRhsData, renameExprs.map(_.alias)) + val distinctRhsData = + relational.Distinct(reducedRhsData, renameExprs.map(_.alias)) // 4. Join lhs and prepared rhs using a left outer join - val joinedData = leftResult.join(distinctRhsData, renameExprs.map(a => a.expr -> a.alias).toSeq, LeftOuterJoin) + val joinedData = leftResult.join( + distinctRhsData, + renameExprs.map(a => a.expr -> a.alias).toSeq, + LeftOuterJoin + ) // 5. If at least one rhs join column is not null, the sub-query exists and true is projected to the target expression val targetExpr = renameExprs.head.alias joinedData.addInto(IsNotNull(targetExpr) -> predicateField.targetField) @@ -256,11 +319,12 @@ object RelationalPlanner { case logical.ReturnGraph(in, _) => relational.ReturnGraph(process[T](in)) - case other => throw NotImplementedException(s"Physical planning of operator $other") + case other => + throw NotImplementedException(s"Physical planning of operator $other") } } - def planScan[T <: Table[T] : TypeTag]( + def planScan[T <: Table[T]: TypeTag]( maybeInOp: Option[RelationalOperator[T]], logicalGraph: LogicalGraph, scanPattern: Pattern, @@ -268,14 +332,15 @@ object RelationalPlanner { )(implicit context: RelationalRuntimeContext[T]): RelationalOperator[T] = { val inOp = maybeInOp match { case Some(relationalOp) => relationalOp - case _ => relational.Start(logicalGraph.qualifiedGraphName) + case _ => relational.Start(logicalGraph.qualifiedGraphName) } val graph = logicalGraph match { case _: LogicalCatalogGraph => inOp.context.resolveGraph(logicalGraph.qualifiedGraphName) case p: LogicalPatternGraph => - inOp.context.queryLocalCatalog.getOrElse(p.qualifiedGraphName, planConstructGraph(inOp, p).graph) + inOp.context.queryLocalCatalog + .getOrElse(p.qualifiedGraphName, planConstructGraph(inOp, p).graph) } val scanOp = graph.scanOperator(scanPattern) @@ -286,8 +351,11 @@ object RelationalPlanner { } } - if (!validScan) throw SchemaException(s"Expected the scan to include Variables for all elements of ${scanPattern.elements}" + - s" but got ${scanOp.header.elementVars}") + if (!validScan) + throw SchemaException( + s"Expected the scan to include Variables for all elements of ${scanPattern.elements}" + + s" but got ${scanOp.header.elementVars}" + ) scanOp .assignScanName(varPatternElementMapping.mapValues(_.toVar).map(_.swap)) @@ -295,8 +363,10 @@ object RelationalPlanner { } // TODO: process operator outside of def - private def planOptional[T <: Table[T] : TypeTag](lhs: LogicalOperator, rhs: LogicalOperator) - (implicit context: RelationalRuntimeContext[T]): RelationalOperator[T] = { + private def planOptional[T <: Table[T]: TypeTag]( + lhs: LogicalOperator, + rhs: LogicalOperator + )(implicit context: RelationalRuntimeContext[T]): RelationalOperator[T] = { val lhsOp = process[T](lhs) val rhsOp = process[T](rhs) @@ -306,7 +376,8 @@ object RelationalPlanner { def generateUniqueName = s"tmp${System.nanoTime}" // 1. Compute expressions between left and right side - val commonExpressions = lhsHeader.expressions.intersect(rhsHeader.expressions) + val commonExpressions = + lhsHeader.expressions.intersect(rhsHeader.expressions) val joinExprs = commonExpressions.collect { case v: Var => v } val otherExpressions = commonExpressions -- joinExprs @@ -317,38 +388,50 @@ object RelationalPlanner { val rhsWithDropped = relational.Drop(rhsOp, expressionsToRemove) // 3. Rename the join expressions on the right hand side, in order to make them distinguishable after the join - val joinExprRenames = joinExprs.map(e => e as Var(generateUniqueName)(e.cypherType)) + val joinExprRenames = + joinExprs.map(e => e as Var(generateUniqueName)(e.cypherType)) val rhsWithAlias = relational.Alias(rhsWithDropped, joinExprRenames.toSeq) - val rhsJoinReady = relational.Drop(rhsWithAlias, joinExprs.collect { case e: Expr => e }) + val rhsJoinReady = + relational.Drop(rhsWithAlias, joinExprs.collect { case e: Expr => e }) // 4. Left outer join the left side and the processed right side - val joined = lhsOp.join(rhsJoinReady, joinExprRenames.map(a => a.expr -> a.alias).toSeq, LeftOuterJoin) + val joined = lhsOp.join( + rhsJoinReady, + joinExprRenames.map(a => a.expr -> a.alias).toSeq, + LeftOuterJoin + ) // 5. Select the resulting header expressions relational.Select(joined, joined.header.expressions.toList) } - implicit class RelationalOperatorOps[T <: Table[T] : TypeTag](op: RelationalOperator[T]) { + implicit class RelationalOperatorOps[T <: Table[T]: TypeTag]( + op: RelationalOperator[T] + ) { private implicit def context: RelationalRuntimeContext[T] = op.context - def select(expressions: Expr*): RelationalOperator[T] = relational.Select(op, expressions.toList) + def select(expressions: Expr*): RelationalOperator[T] = + relational.Select(op, expressions.toList) def filter(expression: Expr): RelationalOperator[T] = { if (expression == TrueLit) { op } else if (expression.cypherType == CTNull) { - relational.Start.fromEmptyGraph(context.session.records.empty(op.header)) + relational.Start.fromEmptyGraph( + context.session.records.empty(op.header) + ) } else { relational.Filter(op, expression) } } /** - * Renames physical columns to given header expression names. - * Ensures that there is a physical column for each return item, i.e. aliases lead to duplicate physical columns. + * Renames physical columns to given header expression names. Ensures that there is a physical + * column for each return item, i.e. aliases lead to duplicate physical columns. */ def alignColumnsWithReturnItems: RelationalOperator[T] = { - val selectExprs = op.maybeReturnItems.getOrElse(List.empty) + val selectExprs = op.maybeReturnItems + .getOrElse(List.empty) .flatMap(op.header.expressionsFor) .toList @@ -359,35 +442,58 @@ object RelationalPlanner { relational.Select(op, selectExprs, renames) } - def renameColumns(columnRenames: Map[Expr, String]): RelationalOperator[T] = { - if (columnRenames.isEmpty) op else relational.Select(op, op.header.expressions.toList, columnRenames) + def renameColumns( + columnRenames: Map[Expr, String] + ): RelationalOperator[T] = { + if (columnRenames.isEmpty) op + else relational.Select(op, op.header.expressions.toList, columnRenames) } - def join(other: RelationalOperator[T], joinExprs: Seq[(Expr, Expr)], joinType: JoinType): RelationalOperator[T] = { - relational.Join(op, other.withDisjointColumnNames(op.header), joinExprs, joinType) + def join( + other: RelationalOperator[T], + joinExprs: Seq[(Expr, Expr)], + joinType: JoinType + ): RelationalOperator[T] = { + relational.Join( + op, + other.withDisjointColumnNames(op.header), + joinExprs, + joinType + ) } def graphUnionAll(other: RelationalOperator[T]): RelationalOperator[T] = { - relational.GraphUnionAll(NonEmptyList(op, List(other)), QualifiedGraphName("UnionAllGraph")) + relational.GraphUnionAll( + NonEmptyList(op, List(other)), + QualifiedGraphName("UnionAllGraph") + ) } def unionAll(other: RelationalOperator[T]): RelationalOperator[T] = { val combinedHeader = op.header union other.header // rename all columns to make sure we have no conflicts - val targetHeader = RecordHeader.empty.withExprs(combinedHeader.expressions) + val targetHeader = + RecordHeader.empty.withExprs(combinedHeader.expressions) val elementVars = targetHeader.nodeVars ++ targetHeader.relationshipVars - val opWithAlignedElements = elementVars.foldLeft(op) { - case (acc, elementVar) => acc.alignExpressions(elementVar, elementVar, targetHeader) - }.alignColumnNames(targetHeader) + val opWithAlignedElements = elementVars + .foldLeft(op) { case (acc, elementVar) => + acc.alignExpressions(elementVar, elementVar, targetHeader) + } + .alignColumnNames(targetHeader) - val otherWithAlignedElements = elementVars.foldLeft(other) { - case (acc, elementVar) => acc.alignExpressions(elementVar, elementVar, targetHeader) - }.alignColumnNames(targetHeader) + val otherWithAlignedElements = elementVars + .foldLeft(other) { case (acc, elementVar) => + acc.alignExpressions(elementVar, elementVar, targetHeader) + } + .alignColumnNames(targetHeader) - relational.TabularUnionAll(opWithAlignedElements, otherWithAlignedElements) + relational.TabularUnionAll( + opWithAlignedElements, + otherWithAlignedElements + ) } def add(values: Expr*): RelationalOperator[T] = { @@ -411,43 +517,68 @@ object RelationalPlanner { def alias(aliases: AliasExpr*): RelationalOperator[T] = Alias(op, aliases) - def alias(aliases: Set[AliasExpr]): RelationalOperator[T] = alias(aliases.toSeq: _*) + def alias(aliases: Set[AliasExpr]): RelationalOperator[T] = alias( + aliases.toSeq: _* + ) // Only works with single element tables def assignScanName(mapping: Map[Var, Var]): RelationalOperator[T] = { - val aliases = mapping.map { - case (from, to) => AliasExpr(from, to) + val aliases = mapping.map { case (from, to) => + AliasExpr(from, to) } op.select(aliases.toList: _*) } - def switchContext(context: RelationalRuntimeContext[T]): RelationalOperator[T] = { + def switchContext( + context: RelationalRuntimeContext[T] + ): RelationalOperator[T] = { SwitchContext(op, context) } - def prefixVariableId(v: Var, prefix: GraphIdPrefix): RelationalOperator[T] = { - val prefixedIds = op.header.idExpressions(v).map(exprToPrefix => PrefixId(ToId(exprToPrefix), prefix) -> exprToPrefix) + def prefixVariableId( + v: Var, + prefix: GraphIdPrefix + ): RelationalOperator[T] = { + val prefixedIds = op.header + .idExpressions(v) + .map(exprToPrefix => PrefixId(ToId(exprToPrefix), prefix) -> exprToPrefix) op.addInto(prefixedIds.toSeq: _*) } - def alignWith(inputElement: Var, targetElement: Var, targetHeader: RecordHeader): RelationalOperator[T] = { - op.alignExpressions(inputElement, targetElement, targetHeader).alignColumnNames(targetHeader) + def alignWith( + inputElement: Var, + targetElement: Var, + targetHeader: RecordHeader + ): RelationalOperator[T] = { + op.alignExpressions(inputElement, targetElement, targetHeader) + .alignColumnNames(targetHeader) } // TODO: element needs to contain all labels/relTypes: all case needs to be explicitly expanded with the schema /** - * Aligns a single element within the operator with the given target element in the target header. + * Aligns a single element within the operator with the given target element in the target + * header. * - * @param inputVar the variable of the element that should be aligned - * @param targetVar the variable of the reference element - * @param targetHeader the header describing the desired state - * @return operator with aligned element + * @param inputVar + * the variable of the element that should be aligned + * @param targetVar + * the variable of the reference element + * @param targetHeader + * the header describing the desired state + * @return + * operator with aligned element */ - def alignExpressions(inputVar: Var, targetVar: Var, targetHeader: RecordHeader): RelationalOperator[T] = { + def alignExpressions( + inputVar: Var, + targetVar: Var, + targetHeader: RecordHeader + ): RelationalOperator[T] = { - val targetHeaderLabels = targetHeader.labelsFor(targetVar).map(_.label.name) - val targetHeaderTypes = targetHeader.typesFor(targetVar).map(_.relType.name) + val targetHeaderLabels = + targetHeader.labelsFor(targetVar).map(_.label.name) + val targetHeaderTypes = + targetHeader.typesFor(targetVar).map(_.relType.name) // Labels/RelTypes that do not need to be added val existingLabels = op.header.labelsFor(inputVar).map(_.label.name) @@ -460,79 +591,113 @@ object RelationalPlanner { val renamedElement = op.select(toRetain.toSeq: _*) // Drop expressions that are not in the target header - val dropExpressions = renamedElement.header.expressions -- targetHeader.expressions + val dropExpressions = + renamedElement.header.expressions -- targetHeader.expressions val withDroppedExpressions = renamedElement.dropExprSet(dropExpressions) // Fill in missing true label columns val trueLabels = inputVar.cypherType match { - case CTNode(labels, _) => (targetHeaderLabels intersect labels) -- existingLabels + case CTNode(labels, _) => + (targetHeaderLabels intersect labels) -- existingLabels case _ => Set.empty } val withTrueLabels = withDroppedExpressions.addInto( - trueLabels.map(label => TrueLit -> HasLabel(targetVar, Label(label))).toSeq: _* + trueLabels + .map(label => TrueLit -> HasLabel(targetVar, Label(label))) + .toSeq: _* ) // Fill in missing false label columns val falseLabels = targetVar.cypherType match { - case n if n.subTypeOf(CTNode.nullable) => targetHeaderLabels -- trueLabels -- existingLabels + case n if n.subTypeOf(CTNode.nullable) => + targetHeaderLabels -- trueLabels -- existingLabels case _ => Set.empty } val withFalseLabels = withTrueLabels.addInto( - falseLabels.map(label => FalseLit -> HasLabel(targetVar, Label(label))).toSeq: _* + falseLabels + .map(label => FalseLit -> HasLabel(targetVar, Label(label))) + .toSeq: _* ) // Fill in missing true relType columns val trueRelTypes = inputVar.cypherType match { - case CTRelationship(relTypes, _) => (targetHeaderTypes intersect relTypes) -- existingRelTypes + case CTRelationship(relTypes, _) => + (targetHeaderTypes intersect relTypes) -- existingRelTypes case _ => Set.empty } val withTrueRelTypes = withFalseLabels.addInto( - trueRelTypes.map(relType => TrueLit -> HasType(targetVar, RelType(relType))).toSeq: _* + trueRelTypes + .map(relType => TrueLit -> HasType(targetVar, RelType(relType))) + .toSeq: _* ) // Fill in missing false relType columns val falseRelTypes = targetVar.cypherType match { - case r if r.subTypeOf(CTRelationship.nullable) => targetHeaderTypes -- trueRelTypes -- existingRelTypes + case r if r.subTypeOf(CTRelationship.nullable) => + targetHeaderTypes -- trueRelTypes -- existingRelTypes case _ => Set.empty } val withFalseRelTypes = withTrueRelTypes.addInto( - falseRelTypes.map(relType => FalseLit -> HasType(targetVar, RelType(relType))).toSeq: _* + falseRelTypes + .map(relType => FalseLit -> HasType(targetVar, RelType(relType))) + .toSeq: _* ) // Fill in missing properties - val missingProperties = targetHeader.propertiesFor(targetVar) -- withFalseRelTypes.header.propertiesFor(targetVar) + val missingProperties = targetHeader.propertiesFor( + targetVar + ) -- withFalseRelTypes.header.propertiesFor(targetVar) val withProperties = withFalseRelTypes.addInto( missingProperties.map(propertyExpr => NullLit -> propertyExpr).toSeq: _* ) import Expr._ - assert(targetHeader.expressionsFor(targetVar) == withProperties.header.expressionsFor(targetVar), + assert( + targetHeader.expressionsFor(targetVar) == withProperties.header + .expressionsFor(targetVar), s"""Expected header expressions for $targetVar: - |\t${targetHeader.expressionsFor(targetVar).toSeq.sorted.mkString(", ")}, + |\t${targetHeader + .expressionsFor(targetVar) + .toSeq + .sorted + .mkString(", ")}, |got - |\t${withProperties.header.expressionsFor(targetVar).toSeq.sorted.mkString(", ")}""".stripMargin) + |\t${withProperties.header + .expressionsFor(targetVar) + .toSeq + .sorted + .mkString(", ")}""".stripMargin + ) withProperties } /** - * Returns an operator with renamed columns such that the operators columns do not overlap with the other header's - * columns. + * Returns an operator with renamed columns such that the operators columns do not overlap with + * the other header's columns. * - * @param otherHeader header from which the column names should be disjoint - * @return operator with disjoint column names + * @param otherHeader + * header from which the column names should be disjoint + * @return + * operator with disjoint column names */ - def withDisjointColumnNames(otherHeader: RecordHeader): RelationalOperator[T] = { + def withDisjointColumnNames( + otherHeader: RecordHeader + ): RelationalOperator[T] = { val header = op.header - val conflictingExpressions = header.expressions.filter(e => otherHeader.columns.contains(header.column(e))) + val conflictingExpressions = + header.expressions.filter(e => otherHeader.columns.contains(header.column(e))) if (conflictingExpressions.isEmpty) { op } else { - val renameMapping = conflictingExpressions.foldLeft(Map.empty[Expr, String]) { - case (acc, nextRename) => - val newColumnName = header.newConflictFreeColumnName(nextRename, otherHeader.columns ++ acc.values) + val renameMapping = + conflictingExpressions.foldLeft(Map.empty[Expr, String]) { case (acc, nextRename) => + val newColumnName = header.newConflictFreeColumnName( + nextRename, + otherHeader.columns ++ acc.values + ) acc + (nextRename -> newColumnName) - } + } op.renameColumns(renameMapping) } } @@ -540,25 +705,34 @@ object RelationalPlanner { /** * Ensures that the column names are aligned with the target header. * - * @note All expressions in the operator header must be present in the target header. - * @param targetHeader the header with which the column names should be aligned with - * @return operator with aligned column names + * @note + * All expressions in the operator header must be present in the target header. + * @param targetHeader + * the header with which the column names should be aligned with + * @return + * operator with aligned column names */ def alignColumnNames(targetHeader: RecordHeader): RelationalOperator[T] = { val exprsNotInTarget = op.header.expressions -- targetHeader.expressions - require(exprsNotInTarget.isEmpty, + require( + exprsNotInTarget.isEmpty, s"""|Column alignment requires for all header expressions to be present in the target header: |Current: ${op.header} |Target: $targetHeader |Missing expressions: ${exprsNotInTarget.mkString(", ")} - """.stripMargin) + """.stripMargin + ) - if (op.header.expressions.forall(expr => op.header.column(expr) == targetHeader.column(expr))) { + if ( + op.header.expressions + .forall(expr => op.header.column(expr) == targetHeader.column(expr)) + ) { op } else { - val columnRenames = op.header.expressions.foldLeft(Map.empty[Expr, String]) { - case (currentMap, expr) => currentMap + (expr -> targetHeader.column(expr)) - } + val columnRenames = + op.header.expressions.foldLeft(Map.empty[Expr, String]) { case (currentMap, expr) => + currentMap + (expr -> targetHeader.column(expr)) + } op.renameColumns(columnRenames) } } @@ -566,8 +740,15 @@ object RelationalPlanner { def singleElement: Var = { op.header.elementVars.toList match { case element :: Nil => element - case Nil => throw SchemaException(s"Operation requires single element table, input contains no elements") - case other => throw SchemaException(s"Operation requires single element table, found ${other.mkString("[", ", ", "]")}") + case Nil => + throw SchemaException( + s"Operation requires single element table, input contains no elements" + ) + case other => + throw SchemaException( + s"Operation requires single element table, found ${other + .mkString("[", ", ", "]")}" + ) } } @@ -576,8 +757,11 @@ object RelationalPlanner { val header = op.header val idCol = header.idColumns(element).head - val properties = header.propertiesFor(element).map(p => p -> header.column(p)) - val propertyMapping = properties.map { case (p: Property, column) => p.key.name -> column } + val properties = + header.propertiesFor(element).map(p => p -> header.column(p)) + val propertyMapping = properties.map { case (p: Property, column) => + p.key.name -> column + } element.cypherType match { case CTNode(labels, _) => @@ -602,7 +786,8 @@ object RelationalPlanner { op.session.elementTables.elementTable(mapping, op.table) - case other => throw UnsupportedOperationException(s"Cannot create scan for $other") + case other => + throw UnsupportedOperationException(s"Cannot create scan for $other") } } } diff --git a/okapi-relational/src/main/scala/org/opencypher/okapi/relational/impl/planning/VarLengthExpandPlanner.scala b/okapi-relational/src/main/scala/org/opencypher/okapi/relational/impl/planning/VarLengthExpandPlanner.scala index 5fc14e400a..e746e830bd 100644 --- a/okapi-relational/src/main/scala/org/opencypher/okapi/relational/impl/planning/VarLengthExpandPlanner.scala +++ b/okapi-relational/src/main/scala/org/opencypher/okapi/relational/impl/planning/VarLengthExpandPlanner.scala @@ -42,7 +42,7 @@ sealed trait ExpandDirection case object Outbound extends ExpandDirection case object Inbound extends ExpandDirection -abstract class VarLengthExpandPlanner[T <: Table[T] : TypeTag] { +abstract class VarLengthExpandPlanner[T <: Table[T]: TypeTag] { def source: Var @@ -77,7 +77,8 @@ abstract class VarLengthExpandPlanner[T <: Table[T] : TypeTag] { /** * Performs the initial expand from the start node * - * @param dir expand direction + * @param dir + * expand direction */ protected def init(dir: ExpandDirection): RelationalOperator[T] = { val startEdgeScanOp: RelationalOperator[T] = physicalEdgeScanOp @@ -87,22 +88,30 @@ abstract class VarLengthExpandPlanner[T <: Table[T] : TypeTag] { // Execute the first expand val edgeJoinExpr = dir match { case Outbound => startEdgeScanOp.header.startNodeFor(startEdgeScan) - case Inbound => startEdgeScanOp.header.endNodeFor(startEdgeScan) + case Inbound => startEdgeScanOp.header.endNodeFor(startEdgeScan) } - physicalSourceOp.join(startEdgeScanOp, - Seq(source -> edgeJoinExpr), - InnerJoin - ).filter(isomorphismFilter(startEdgeScan, physicalSourceOp.header.relationshipElements)) + physicalSourceOp + .join(startEdgeScanOp, Seq(source -> edgeJoinExpr), InnerJoin) + .filter( + isomorphismFilter( + startEdgeScan, + physicalSourceOp.header.relationshipElements + ) + ) } /** * Performs the ith expand. * - * @param i number of the iteration - * @param iterationTable result of the i-1th iteration - * @param directions expansion directions - * @param edgeVars edges already traversed + * @param i + * number of the iteration + * @param iterationTable + * result of the i-1th iteration + * @param directions + * expansion directions + * @param edgeVars + * edges already traversed */ def expand( i: Int, @@ -110,21 +119,30 @@ abstract class VarLengthExpandPlanner[T <: Table[T] : TypeTag] { directions: (ExpandDirection, ExpandDirection), edgeVars: Seq[Var] ): (RelationalOperator[T], Var) = { - val nextEdgeCT = if (i > lower) edgeScan.cypherType.nullable else edgeScan.cypherType + val nextEdgeCT = + if (i > lower) edgeScan.cypherType.nullable else edgeScan.cypherType val nextEdge = ListSegment(i, list) - val aliasedEdgeScanOp = physicalEdgeScanOp. - alias(edgeScan as nextEdge) + val aliasedEdgeScanOp = physicalEdgeScanOp + .alias(edgeScan as nextEdge) .select(nextEdge) val iterationTableHeader = iterationTable.header val nextEdgeScanHeader = aliasedEdgeScanOp.header val joinExpr = directions match { - case (Outbound, Outbound) => iterationTableHeader.endNodeFor(edgeVars.last) -> nextEdgeScanHeader.startNodeFor(nextEdge) - case (Outbound, Inbound) => iterationTableHeader.endNodeFor(edgeVars.last) -> nextEdgeScanHeader.endNodeFor(nextEdge) - case (Inbound, Outbound) => iterationTableHeader.startNodeFor(edgeVars.last) -> nextEdgeScanHeader.endNodeFor(nextEdge) - case (Inbound, Inbound) => iterationTableHeader.startNodeFor(edgeVars.last) -> nextEdgeScanHeader.startNodeFor(nextEdge) + case (Outbound, Outbound) => + iterationTableHeader.endNodeFor(edgeVars.last) -> nextEdgeScanHeader + .startNodeFor(nextEdge) + case (Outbound, Inbound) => + iterationTableHeader.endNodeFor(edgeVars.last) -> nextEdgeScanHeader + .endNodeFor(nextEdge) + case (Inbound, Outbound) => + iterationTableHeader.startNodeFor(edgeVars.last) -> nextEdgeScanHeader + .endNodeFor(nextEdge) + case (Inbound, Inbound) => + iterationTableHeader.startNodeFor(edgeVars.last) -> nextEdgeScanHeader + .startNodeFor(nextEdge) } val expandedOp = iterationTable @@ -136,44 +154,55 @@ abstract class VarLengthExpandPlanner[T <: Table[T] : TypeTag] { /** * Finalize the expansions - * 1. adds paths of length zero if needed - * 2. fills empty columns with null values - * 3. unions paths of different lengths + * 1. adds paths of length zero if needed 2. fills empty columns with null values 3. unions + * paths of different lengths * - * @param paths valid paths + * @param paths + * valid paths */ - protected def finalize(paths: Seq[RelationalOperator[T]]): RelationalOperator[T] = { + protected def finalize( + paths: Seq[RelationalOperator[T]] + ): RelationalOperator[T] = { val targetHeader = paths.maxBy(_.header.columns.size).header // check whether to include paths of length 0 val unalignedOps: Seq[RelationalOperator[T]] = if (lower == 0) { - val zeroLengthExpand: RelationalOperator[T] = copyElement(source, target, targetHeader, physicalSourceOp) + val zeroLengthExpand: RelationalOperator[T] = + copyElement(source, target, targetHeader, physicalSourceOp) if (upper == 0) Seq(zeroLengthExpand) else paths :+ zeroLengthExpand } else paths // fill shorter paths with nulls val alignedOps = unalignedOps.map { expansion => - val nullExpressions = targetHeader.expressions -- expansion.header.expressions + val nullExpressions = + targetHeader.expressions -- expansion.header.expressions - val expWithNullLits = expansion.addInto(nullExpressions.map(expr => NullLit -> expr).toSeq: _*) + val expWithNullLits = expansion.addInto( + nullExpressions.map(expr => NullLit -> expr).toSeq: _* + ) val exprsToRename = nullExpressions.filterNot(expr => expWithNullLits.header.column(expr) == targetHeader.column(expr) ) - val renameTuples = exprsToRename.map(expr => expr -> targetHeader.column(expr)) + val renameTuples = + exprsToRename.map(expr => expr -> targetHeader.column(expr)) expWithNullLits.renameColumns(renameTuples.toMap) } // union expands of different lengths alignedOps .map(op => op.alignColumnNames(targetHeader)) - .reduce((agg: RelationalOperator[T], next: RelationalOperator[T]) => relational.TabularUnionAll(agg, next)) + .reduce((agg: RelationalOperator[T], next: RelationalOperator[T]) => + relational.TabularUnionAll(agg, next) + ) } /** * Creates the isomorphism filter for the given edge list * - * @param rel new edge - * @param candidates candidate edges + * @param rel + * new edge + * @param candidates + * candidate edges */ protected def isomorphismFilter(rel: Var, candidates: Set[Var]): Expr = Ands(candidates.map(e => Not(Equals(e, rel))).toSeq: _*) @@ -181,10 +210,14 @@ abstract class VarLengthExpandPlanner[T <: Table[T] : TypeTag] { /** * Copies the content of a variable into another variable * - * @param from source variable - * @param to target variable - * @param targetHeader target header - * @param physicalOp base operation + * @param from + * source variable + * @param to + * target variable + * @param targetHeader + * target header + * @param physicalOp + * base operation */ protected def copyElement( from: Var, @@ -198,12 +231,17 @@ abstract class VarLengthExpandPlanner[T <: Table[T] : TypeTag] { val sourceChildren = targetHeader.expressionsFor(from) val targetChildren = targetHeader.expressionsFor(correctTarget) - val childMapping: Set[(Expr, Expr)] = sourceChildren.map(expr => expr -> expr.withOwner(correctTarget)) - val missingMapping = (targetChildren -- childMapping.map(_._2) - correctTarget).map { - case l: HasLabel => FalseLit -> l - case p: Property => NullLit -> p - case other => throw RecordHeaderException(s"$correctTarget can only own HasLabel and Property but found $other") - } + val childMapping: Set[(Expr, Expr)] = + sourceChildren.map(expr => expr -> expr.withOwner(correctTarget)) + val missingMapping = + (targetChildren -- childMapping.map(_._2) - correctTarget).map { + case l: HasLabel => FalseLit -> l + case p: Property => NullLit -> p + case other => + throw RecordHeaderException( + s"$correctTarget can only own HasLabel and Property but found $other" + ) + } physicalOp.addInto((childMapping ++ missingMapping).toSeq: _*) } @@ -211,14 +249,21 @@ abstract class VarLengthExpandPlanner[T <: Table[T] : TypeTag] { /** * Joins a given path with it's target node * - * @param path the path - * @param edge the paths last edge - * @param dir expand direction + * @param path + * the path + * @param edge + * the paths last edge + * @param dir + * expand direction */ - protected def addTargetOps(path: RelationalOperator[T], edge: Var, dir: ExpandDirection): RelationalOperator[T] = { + protected def addTargetOps( + path: RelationalOperator[T], + edge: Var, + dir: ExpandDirection + ): RelationalOperator[T] = { val expr = dir match { case Outbound => path.header.endNodeFor(edge) - case Inbound => path.header.startNodeFor(edge) + case Inbound => path.header.startNodeFor(edge) } if (isExpandInto) { @@ -230,7 +275,7 @@ abstract class VarLengthExpandPlanner[T <: Table[T] : TypeTag] { } // TODO: use object instead -class DirectedVarLengthExpandPlanner[T <: Table[T] : TypeTag]( +class DirectedVarLengthExpandPlanner[T <: Table[T]: TypeTag]( override val source: Var, override val list: Var, override val edgeScan: Var, @@ -241,19 +286,23 @@ class DirectedVarLengthExpandPlanner[T <: Table[T] : TypeTag]( override val relEdgeScanOp: RelationalOperator[T], override val targetOp: LogicalOperator, override val isExpandInto: Boolean -)(override implicit val context: RelationalRuntimeContext[T]) extends VarLengthExpandPlanner[T] { +)(override implicit val context: RelationalRuntimeContext[T]) + extends VarLengthExpandPlanner[T] { override def plan: RelationalOperator[T] = { // Iteratively expand beginning from startOp with cacheOp - val expandOps = (2 to upper).foldLeft(Seq(init(Outbound) -> Seq(startEdgeScan))) { - case (acc, i) => + val expandOps = (2 to upper) + .foldLeft(Seq(init(Outbound) -> Seq(startEdgeScan))) { case (acc, i) => val (last, edgeVars) = acc.last val (next, nextEdge) = expand(i, last, Outbound -> Outbound, edgeVars) acc :+ (next -> (edgeVars :+ nextEdge)) - }.filter(_._2.size >= lower) + } + .filter(_._2.size >= lower) // Join target nodes on expand ops - val withTargetOps = expandOps.map { case (op, edges) => addTargetOps(op, edges.last, Outbound) } + val withTargetOps = expandOps.map { case (op, edges) => + addTargetOps(op, edges.last, Outbound) + } finalize(withTargetOps) } @@ -261,7 +310,7 @@ class DirectedVarLengthExpandPlanner[T <: Table[T] : TypeTag]( } // TODO: use object instead -class UndirectedVarLengthExpandPlanner[T <: Table[T] : TypeTag]( +class UndirectedVarLengthExpandPlanner[T <: Table[T]: TypeTag]( override val source: Var, override val list: Var, override val edgeScan: Var, @@ -272,7 +321,8 @@ class UndirectedVarLengthExpandPlanner[T <: Table[T] : TypeTag]( override val relEdgeScanOp: RelationalOperator[T], override val targetOp: LogicalOperator, override val isExpandInto: Boolean -)(override implicit val context: RelationalRuntimeContext[T]) extends VarLengthExpandPlanner[T] { +)(override implicit val context: RelationalRuntimeContext[T]) + extends VarLengthExpandPlanner[T] { override def plan: RelationalOperator[T] = { @@ -280,27 +330,28 @@ class UndirectedVarLengthExpandPlanner[T <: Table[T] : TypeTag]( val inStartOp = init(Inbound) // Iteratively expand beginning from startOp with cacheOp - val expandOps = (2 to upper).foldLeft(Seq((outStartOp -> inStartOp) -> Seq(startEdgeScan))) { - case (acc, i) => + val expandOps = (2 to upper) + .foldLeft(Seq((outStartOp -> inStartOp) -> Seq(startEdgeScan))) { case (acc, i) => val ((last, lastRevered), edgeVars) = acc.last - val (outOut, nextEdge) = expand(i, last, Outbound -> Outbound, edgeVars) + val (outOut, nextEdge) = + expand(i, last, Outbound -> Outbound, edgeVars) val (outIn, _) = expand(i, last, Outbound -> Inbound, edgeVars) val (inOut, _) = expand(i, lastRevered, Inbound -> Outbound, edgeVars) val (inIn, _) = expand(i, lastRevered, Inbound -> Inbound, edgeVars) - val nextOps = relational.TabularUnionAll(outOut, inOut) -> relational.TabularUnionAll(outIn, inIn) + val nextOps = relational.TabularUnionAll(outOut, inOut) -> relational + .TabularUnionAll(outIn, inIn) acc :+ nextOps -> (edgeVars :+ nextEdge) - }.filter(_._2.size >= lower) - + } + .filter(_._2.size >= lower) // Join target nodes on expand ops - val withTargetOps = expandOps.map { - case ((out, in), edges) => - relational.TabularUnionAll( - addTargetOps(out, edges.last, Outbound), - addTargetOps(in, edges.last, Inbound) - ) + val withTargetOps = expandOps.map { case ((out, in), edges) => + relational.TabularUnionAll( + addTargetOps(out, edges.last, Outbound), + addTargetOps(in, edges.last, Inbound) + ) } finalize(withTargetOps) diff --git a/okapi-relational/src/main/scala/org/opencypher/okapi/relational/impl/table/RecordHeader.scala b/okapi-relational/src/main/scala/org/opencypher/okapi/relational/impl/table/RecordHeader.scala index d8029ccf99..c8680d4b22 100644 --- a/okapi-relational/src/main/scala/org/opencypher/okapi/relational/impl/table/RecordHeader.scala +++ b/okapi-relational/src/main/scala/org/opencypher/okapi/relational/impl/table/RecordHeader.scala @@ -38,11 +38,13 @@ object RecordHeader { def empty: RecordHeader = RecordHeader(Map.empty) - def from[T <: Expr](expr: T, exprs: T*): RecordHeader = empty.withExprs(expr, exprs: _*) + def from[T <: Expr](expr: T, exprs: T*): RecordHeader = + empty.withExprs(expr, exprs: _*) def from[T <: Expr](exprs: Set[T]): RecordHeader = empty.withExprs(exprs) - def from[T <: Expr](exprs: Seq[T]): RecordHeader = from(exprs.head, exprs.tail: _*) + def from[T <: Expr](exprs: Seq[T]): RecordHeader = + from(exprs.head, exprs.tail: _*) def from(v: Var): RecordHeader = v.cypherType match { case CTNode(labels, _) => @@ -50,12 +52,14 @@ object RecordHeader { v, labels.map(l => HasLabel(v, Label(l))).toSeq: _* ) - case CTRelationship(types, _) => from( - v, - Seq(StartNode(v)(CTIdentity), EndNode(v)(CTIdentity)) - ++ types.map(t => HasType(v, RelType(t))): _* - ) - case other => throw IllegalArgumentException("A node or relationship variable", other) + case CTRelationship(types, _) => + from( + v, + Seq(StartNode(v)(CTIdentity), EndNode(v)(CTIdentity)) + ++ types.map(t => HasType(v, RelType(t))): _* + ) + case other => + throw IllegalArgumentException("A node or relationship variable", other) } } @@ -69,7 +73,8 @@ case class RecordHeader(exprToColumn: Map[Expr, String]) { def vars: Set[Var] = expressions.flatMap(_.owner).collect { case v: Var => v } - def returnItems: Set[ReturnItem] = expressions.flatMap(_.owner).collect { case r: ReturnItem => r } + def returnItems: Set[ReturnItem] = + expressions.flatMap(_.owner).collect { case r: ReturnItem => r } def columns: Set[String] = exprToColumn.values.toSet @@ -78,14 +83,20 @@ case class RecordHeader(exprToColumn: Map[Expr, String]) { // TODO: should we verify that if the expr exists, that it has the same type and nullability def contains(expr: Expr): Boolean = expr match { case AliasExpr(_, alias) => contains(alias) - case _ => exprToColumn.contains(expr) + case _ => exprToColumn.contains(expr) } def getColumn(expr: Expr): Option[String] = exprToColumn.get(expr) def column(expr: Expr): String = expr match { case AliasExpr(innerExpr, _) => column(innerExpr) - case _ => exprToColumn.getOrElse(expr, throw IllegalArgumentException(s"Header does not contain a column for $expr.\n\t${this.toString}")) + case _ => + exprToColumn.getOrElse( + expr, + throw IllegalArgumentException( + s"Header does not contain a column for $expr.\n\t${this.toString}" + ) + ) } def ownedBy(expr: Var): Set[Expr] = { @@ -93,16 +104,17 @@ case class RecordHeader(exprToColumn: Map[Expr, String]) { members.flatMap { case e: Var if e == expr => Seq(e) - case e: Var => ownedBy(e) + e - case other => Seq(other) + case e: Var => ownedBy(e) + e + case other => Seq(other) } } def expressionsFor(expr: Expr): Set[Expr] = { expr match { - case v: Var => if (exprToColumn.contains(v)) ownedBy(v) + v else ownedBy(v) + case v: Var => + if (exprToColumn.contains(v)) ownedBy(v) + v else ownedBy(v) case e if exprToColumn.contains(e) => Set(e) - case _ => Set.empty + case _ => Set.empty } } @@ -113,12 +125,13 @@ case class RecordHeader(exprToColumn: Map[Expr, String]) { def aliasesFor(expr: Expr): Set[Var] = { val aliasesFromHeader: Set[Var] = getColumn(expr) match { case None => Set.empty - case Some(col) => exprToColumn.collect { case (k: Var, v) if v == col => k }.toSet + case Some(col) => + exprToColumn.collect { case (k: Var, v) if v == col => k }.toSet } val aliasesFromParam: Set[Var] = expr match { case v: Var => Set(v) - case _ => Set.empty + case _ => Set.empty } aliasesFromHeader ++ aliasesFromParam @@ -130,60 +143,61 @@ case class RecordHeader(exprToColumn: Map[Expr, String]) { def idExpressions(): Set[Expr] = { exprToColumn.keySet.collect { - case n if n.cypherType.subTypeOf(CTNode) => n + case n if n.cypherType.subTypeOf(CTNode) => n case r if r.cypherType.subTypeOf(CTRelationship) => r } } - def idExpressions(v: Var): Set[Expr] = idExpressions().filter(_.owner.get == v) + def idExpressions(v: Var): Set[Expr] = + idExpressions().filter(_.owner.get == v) def idColumns(): Set[String] = idExpressions().map(column) def idColumns(v: Var): Set[String] = idExpressions(v).map(column) def labelsFor(n: Var): Set[HasLabel] = { - ownedBy(n).collect { - case l: HasLabel => l + ownedBy(n).collect { case l: HasLabel => + l } } def typesFor(r: Var): Set[HasType] = { - ownedBy(r).collect { - case t: HasType => t + ownedBy(r).collect { case t: HasType => + t } } def startNodeFor(r: Var): StartNode = { - ownedBy(r).collectFirst { - case s: StartNode => s + ownedBy(r).collectFirst { case s: StartNode => + s }.get } def endNodeFor(r: Var): EndNode = { - ownedBy(r).collectFirst { - case e: EndNode => e + ownedBy(r).collectFirst { case e: EndNode => + e }.get } def propertiesFor(v: Var): Set[Property] = { - ownedBy(v).collect { - case p: Property => p + ownedBy(v).collect { case p: Property => + p } } def node(name: Var): Set[Expr] = { exprToColumn.keys.collect { - case n: Var if name == n => n - case h@HasLabel(n: Var, _) if name == n => h - case p@ElementProperty(n: Var, _) if name == n => p + case n: Var if name == n => n + case h @ HasLabel(n: Var, _) if name == n => h + case p @ ElementProperty(n: Var, _) if name == n => p }.toSet } def elementVars: Set[Var] = nodeVars ++ relationshipVars def nodeVars[T >: NodeVar <: Var]: Set[T] = { - exprToColumn.keySet.collect { - case v: NodeVar => v + exprToColumn.keySet.collect { case v: NodeVar => + v } } @@ -194,8 +208,8 @@ case class RecordHeader(exprToColumn: Map[Expr, String]) { } def relationshipVars[T >: RelationshipVar <: Var]: Set[T] = { - exprToColumn.keySet.collect { - case v: RelationshipVar => v + exprToColumn.keySet.collect { case v: RelationshipVar => + v } } @@ -207,13 +221,16 @@ case class RecordHeader(exprToColumn: Map[Expr, String]) { def elementsForType(ct: CypherType, exactMatch: Boolean = false): Set[Var] = { ct match { - case n: CTNode => nodesForType(n, exactMatch) + case n: CTNode => nodesForType(n, exactMatch) case r: CTRelationship => relationshipsForType(r) - case other => throw IllegalArgumentException("Element", other) + case other => throw IllegalArgumentException("Element", other) } } - def nodesForType[T >: NodeVar <: Var](nodeType: CTNode, exactMatch: Boolean = false): Set[T] = { + def nodesForType[T >: NodeVar <: Var]( + nodeType: CTNode, + exactMatch: Boolean = false + ): Set[T] = { // and semantics val requiredLabels = nodeType.labels @@ -221,7 +238,7 @@ case class RecordHeader(exprToColumn: Map[Expr, String]) { val physicalLabels = labelsFor(nodeVar).map(_.label.name) val logicalLabels = nodeVar.cypherType match { case CTNode(labels, _) => labels - case _ => Set.empty[String] + case _ => Set.empty[String] } if (exactMatch) { requiredLabels == (physicalLabels ++ logicalLabels) @@ -231,19 +248,23 @@ case class RecordHeader(exprToColumn: Map[Expr, String]) { } } - def relationshipsForType[T >: RelationshipVar <: Var](relType: CTRelationship): Set[T] = { + def relationshipsForType[T >: RelationshipVar <: Var]( + relType: CTRelationship + ): Set[T] = { // or semantics val possibleTypes = relType.types relationshipVars[T].filter { relVar => - val physicalTypes = typesFor(relVar).map { - case HasType(_, RelType(name)) => name + val physicalTypes = typesFor(relVar).map { case HasType(_, RelType(name)) => + name } val logicalTypes = relVar.cypherType match { case CTRelationship(types, _) => types - case _ => Set.empty[String] + case _ => Set.empty[String] } - possibleTypes.isEmpty || (physicalTypes ++ logicalTypes).exists(possibleTypes.contains) + possibleTypes.isEmpty || (physicalTypes ++ logicalTypes).exists( + possibleTypes.contains + ) } } @@ -262,22 +283,25 @@ case class RecordHeader(exprToColumn: Map[Expr, String]) { case AliasExpr(expr, alias) => expr match { case v: Var => expressionsFor(v).map(_.withOwner(alias)) - case other => other.withOwner(alias) + case other => other.withOwner(alias) } case nonVar => Set(nonVar) } } - val selectMappings = headerWithAliases.exprToColumn.filterKeys(selectExpressions.contains) + val selectMappings = + headerWithAliases.exprToColumn.filterKeys(selectExpressions.contains) RecordHeader(selectMappings) } - def withColumnsReplaced[T <: Expr](replacements: Map[T, String]): RecordHeader = { + def withColumnsReplaced[T <: Expr]( + replacements: Map[T, String] + ): RecordHeader = { RecordHeader(exprToColumn ++ replacements) } def withColumnsRenamed[T <: Expr](renames: Seq[(T, String)]): RecordHeader = { - renames.foldLeft(this) { - case (currentHeader, (expr, newColumn)) => currentHeader.withColumnRenamed(expr, newColumn) + renames.foldLeft(this) { case (currentHeader, (expr, newColumn)) => + currentHeader.withColumnRenamed(expr, newColumn) } } @@ -290,9 +314,13 @@ case class RecordHeader(exprToColumn: Map[Expr, String]) { copy(exprToColumn ++ exprs.map(_ -> newColumn)) } - private[opencypher] def newConflictFreeColumnName(expr: Expr, usedColumnNames: Set[String] = columns): String = { + private[opencypher] def newConflictFreeColumnName( + expr: Expr, + usedColumnNames: Set[String] = columns + ): String = { @tailrec def recConflictFreeColumnName(candidateName: String): String = { - if (usedColumnNames.contains(candidateName)) recConflictFreeColumnName(s"_$candidateName") + if (usedColumnNames.contains(candidateName)) + recConflictFreeColumnName(s"_$candidateName") else candidateName } @@ -306,27 +334,30 @@ case class RecordHeader(exprToColumn: Map[Expr, String]) { def withExpr(expr: Expr): RecordHeader = { expr match { case a: AliasExpr => withAlias(a) - case _ => exprToColumn.get(expr) match { - case Some(_) => this + case _ => + exprToColumn.get(expr) match { + case Some(_) => this - case None => - val newColumnName = newConflictFreeColumnName(expr) + case None => + val newColumnName = newConflictFreeColumnName(expr) - // Aliases for (possible) owner of expr need to be updated as well - val exprsToAdd: Set[Expr] = expr.owner match { - case None => Set(expr) + // Aliases for (possible) owner of expr need to be updated as well + val exprsToAdd: Set[Expr] = expr.owner match { + case None => Set(expr) - case Some(exprOwner) => aliasesFor(exprOwner).map(alias => expr.withOwner(alias)) - } + case Some(exprOwner) => + aliasesFor(exprOwner).map(alias => expr.withOwner(alias)) + } - exprsToAdd.foldLeft(this) { - case (current, e) => current.addExprToColumn(e, newColumnName) - } - } + exprsToAdd.foldLeft(this) { case (current, e) => + current.addExprToColumn(e, newColumnName) + } + } } } - def withExprs[T <: Expr](expr: T, exprs: T*): RecordHeader = (expr +: exprs).foldLeft(this)(_ withExpr _) + def withExprs[T <: Expr](expr: T, exprs: T*): RecordHeader = + (expr +: exprs).foldLeft(this)(_ withExpr _) def withExprs[T <: Expr](exprs: Set[T]): RecordHeader = { if (exprs.isEmpty) { @@ -347,27 +378,43 @@ case class RecordHeader(exprToColumn: Map[Expr, String]) { // Element case case elementExpr: Var if exprToColumn.contains(to) => val withElementExpr = addExprToColumn(alias, exprToColumn(to)) - ownedBy(elementExpr).filterNot(_ == elementExpr).foldLeft(withElementExpr) { - case (current, nextExpr) => current.addExprToColumn(nextExpr.withOwner(alias), exprToColumn(nextExpr)) - } + ownedBy(elementExpr) + .filterNot(_ == elementExpr) + .foldLeft(withElementExpr) { case (current, nextExpr) => + current.addExprToColumn( + nextExpr.withOwner(alias), + exprToColumn(nextExpr) + ) + } // Non-element case - case e if exprToColumn.contains(e) => addExprToColumn(alias, exprToColumn(e)) + case e if exprToColumn.contains(e) => + addExprToColumn(alias, exprToColumn(e)) // No expression to alias - case other => throw IllegalArgumentException(s"An expression in $this", s"Unknown expression $other") + case other => + throw IllegalArgumentException( + s"An expression in $this", + s"Unknown expression $other" + ) } } def join(other: RecordHeader): RecordHeader = { val expressionOverlap = expressions.intersect(other.expressions) if (expressionOverlap.nonEmpty) { - throw IllegalArgumentException("two headers with non overlapping expressions", s"overlapping expressions: $expressionOverlap") + throw IllegalArgumentException( + "two headers with non overlapping expressions", + s"overlapping expressions: $expressionOverlap" + ) } val columnOverlap = columns intersect other.columns if (columnOverlap.nonEmpty) { - throw IllegalArgumentException("two headers with non overlapping columns", s"overlapping columns: $columnOverlap") + throw IllegalArgumentException( + "two headers with non overlapping columns", + s"overlapping columns: $columnOverlap" + ) } this ++ other @@ -376,36 +423,47 @@ case class RecordHeader(exprToColumn: Map[Expr, String]) { def union(other: RecordHeader): RecordHeader = { val varOverlap = vars ++ other.vars if (varOverlap != vars) { - throw IllegalArgumentException("two headers with the same variables", s"$vars and ${other.vars}") + throw IllegalArgumentException( + "two headers with the same variables", + s"$vars and ${other.vars}" + ) } this ++ other } def ++(other: RecordHeader): RecordHeader = { - val result = (exprToColumn ++ other.exprToColumn).map { - case (key, value) => - val leftCT = exprToColumn.keySet.find(_ == key).map(_.cypherType).getOrElse(CTVoid) - val rightCT = other.exprToColumn.keySet.find(_ == key).map(_.cypherType).getOrElse(CTVoid) - - val resultExpr = (key, leftCT, rightCT) match { - case (v: Var, l: CTNode, r: CTNode) => Var(v.name)(l.join(r)) - case (v: Var, l: CTRelationship, r: CTRelationship) => Var(v.name)(l.join(r)) - case (_, l, r) if l.subTypeOf(r) => other.exprToColumn.keySet.collectFirst { case k if k == key => k }.get - case (_, l, r) if r.subTypeOf(l) => key - case _ => throw IllegalArgumentException( + val result = (exprToColumn ++ other.exprToColumn).map { case (key, value) => + val leftCT = + exprToColumn.keySet.find(_ == key).map(_.cypherType).getOrElse(CTVoid) + val rightCT = other.exprToColumn.keySet + .find(_ == key) + .map(_.cypherType) + .getOrElse(CTVoid) + + val resultExpr = (key, leftCT, rightCT) match { + case (v: Var, l: CTNode, r: CTNode) => Var(v.name)(l.join(r)) + case (v: Var, l: CTRelationship, r: CTRelationship) => + Var(v.name)(l.join(r)) + case (_, l, r) if l.subTypeOf(r) => + other.exprToColumn.keySet.collectFirst { case k if k == key => k }.get + case (_, l, r) if r.subTypeOf(l) => key + case _ => + throw IllegalArgumentException( expected = s"Compatible Cypher types for expression $key", actual = s"left type `$leftCT` and right type `$rightCT`" ) - } - resultExpr -> value + } + resultExpr -> value } copy(exprToColumn = result) } def --[T <: Expr](expressions: Set[T]): RecordHeader = { val expressionToRemove = expressions.flatMap(expressionsFor) - val updatedExprToColumn = exprToColumn.filterNot { case (e, _) => expressionToRemove.contains(e) } + val updatedExprToColumn = exprToColumn.filterNot { case (e, _) => + expressionToRemove.contains(e) + } copy(exprToColumn = updatedExprToColumn) } @@ -423,8 +481,7 @@ case class RecordHeader(exprToColumn: Map[Expr, String]) { def pretty: String = { val formatCell: String => String = s => s"'$s'" - val (header, row) = exprToColumn - .toSeq + val (header, row) = exprToColumn.toSeq .sortBy(_._2) .map { case (expr, column) => expr.toString -> column } .unzip @@ -434,4 +491,3 @@ case class RecordHeader(exprToColumn: Map[Expr, String]) { def show(): Unit = println(pretty) } - diff --git a/okapi-relational/src/test/scala/org/opencypher/okapi/relational/api/schema/RelationalPropertyGraphSchemaTest.scala b/okapi-relational/src/test/scala/org/opencypher/okapi/relational/api/schema/RelationalPropertyGraphSchemaTest.scala index 8ac7840317..e5535ae9fd 100644 --- a/okapi-relational/src/test/scala/org/opencypher/okapi/relational/api/schema/RelationalPropertyGraphSchemaTest.scala +++ b/okapi-relational/src/test/scala/org/opencypher/okapi/relational/api/schema/RelationalPropertyGraphSchemaTest.scala @@ -42,11 +42,13 @@ class RelationalPropertyGraphSchemaTest extends BaseTestSuite { val n = Var("n")(CTNode(Set("A", "B"))) - schema.headerForNode(n) should equal(RecordHeader.empty - .withExpr(n) - .withExpr(HasLabel(n, Label("A"))) - .withExpr(HasLabel(n, Label("B"))) - .withExpr(ElementProperty(n, PropertyKey("foo"))(CTBoolean))) + schema.headerForNode(n) should equal( + RecordHeader.empty + .withExpr(n) + .withExpr(HasLabel(n, Label("A"))) + .withExpr(HasLabel(n, Label("B"))) + .withExpr(ElementProperty(n, PropertyKey("foo"))(CTBoolean)) + ) } it("creates a header for a given node and changes nullability if necessary") { @@ -56,12 +58,14 @@ class RelationalPropertyGraphSchemaTest extends BaseTestSuite { val n = Var("n")(CTNode(Set("A"))) - schema.headerForNode(n) should equal(RecordHeader.empty - .withExpr(n) - .withExpr(HasLabel(n, Label("A"))) - .withExpr(HasLabel(n, Label("B"))) - .withExpr(HasLabel(n, Label("C"))) - .withExpr(ElementProperty(n, PropertyKey("foo"))(CTString.nullable))) + schema.headerForNode(n) should equal( + RecordHeader.empty + .withExpr(n) + .withExpr(HasLabel(n, Label("A"))) + .withExpr(HasLabel(n, Label("B"))) + .withExpr(HasLabel(n, Label("C"))) + .withExpr(ElementProperty(n, PropertyKey("foo"))(CTString.nullable)) + ) } it("creates a header for given node with implied labels") { @@ -71,12 +75,14 @@ class RelationalPropertyGraphSchemaTest extends BaseTestSuite { val n = Var("n")(CTNode(Set("A"))) - schema.headerForNode(n) should equal(RecordHeader.empty - .withExpr(n) - .withExpr(HasLabel(n, Label("A"))) - .withExpr(HasLabel(n, Label("B"))) - .withExpr(ElementProperty(n, PropertyKey("foo"))(CTBoolean.nullable)) - .withExpr(ElementProperty(n, PropertyKey("bar"))(CTBoolean.nullable))) + schema.headerForNode(n) should equal( + RecordHeader.empty + .withExpr(n) + .withExpr(HasLabel(n, Label("A"))) + .withExpr(HasLabel(n, Label("B"))) + .withExpr(ElementProperty(n, PropertyKey("foo"))(CTBoolean.nullable)) + .withExpr(ElementProperty(n, PropertyKey("bar"))(CTBoolean.nullable)) + ) } it("creates a header for a given relationship") { @@ -85,11 +91,13 @@ class RelationalPropertyGraphSchemaTest extends BaseTestSuite { val r = Var("r")(CTRelationship("A")) - schema.headerForRelationship(r) should equal(RecordHeader.empty - .withExpr(r) - .withExpr(StartNode(r)(CTNode)) - .withExpr(EndNode(r)(CTNode)) - .withExpr(HasType(r, RelType("A"))) - .withExpr(ElementProperty(r, PropertyKey("foo"))(CTBoolean))) + schema.headerForRelationship(r) should equal( + RecordHeader.empty + .withExpr(r) + .withExpr(StartNode(r)(CTNode)) + .withExpr(EndNode(r)(CTNode)) + .withExpr(HasType(r, RelType("A"))) + .withExpr(ElementProperty(r, PropertyKey("foo"))(CTBoolean)) + ) } } diff --git a/okapi-relational/src/test/scala/org/opencypher/okapi/relational/impl/table/RecordHeaderTest.scala b/okapi-relational/src/test/scala/org/opencypher/okapi/relational/impl/table/RecordHeaderTest.scala index c61800f71c..b8dbb3c618 100644 --- a/okapi-relational/src/test/scala/org/opencypher/okapi/relational/impl/table/RecordHeaderTest.scala +++ b/okapi-relational/src/test/scala/org/opencypher/okapi/relational/impl/table/RecordHeaderTest.scala @@ -56,14 +56,20 @@ class RecordHeaderTest extends BaseTestSuite { val nExprs: Set[Expr] = Set(n, nLabelA, nLabelB, nPropFoo) val mExprs: Set[Expr] = nExprs.map(_.withOwner(m)) val oExprs: Set[Expr] = nExprs.map(_.withOwner(o)) - val nodeListExprs: Set[Expr] = Set(nodeListSegment) ++ Set(nLabelA, nLabelB, nPropFoo).map(_.withOwner(nodeListSegment)) + val nodeListExprs: Set[Expr] = + Set(nodeListSegment) ++ Set(nLabelA, nLabelB, nPropFoo).map( + _.withOwner(nodeListSegment) + ) val rStart: StartNode = StartNode(r)(CTNode) val rEnd: EndNode = EndNode(r)(CTNode) val rRelType: HasType = HasType(r, RelType("R")) val rPropFoo: Property = ElementProperty(r, PropertyKey("foo"))(CTString) val rExprs: Set[Expr] = Set(r, rStart, rEnd, rRelType, rPropFoo) - val relListExprs: Set[Expr] = Set(relListSegment) ++ Set(rStart, rEnd, rRelType, rPropFoo).map(_.withOwner(relListSegment)) + val relListExprs: Set[Expr] = + Set(relListSegment) ++ Set(rStart, rEnd, rRelType, rPropFoo).map( + _.withOwner(relListSegment) + ) val nHeader: RecordHeader = RecordHeader.empty.withExprs(nExprs) val mHeader: RecordHeader = RecordHeader.empty.withExprs(mExprs) @@ -86,8 +92,12 @@ class RecordHeaderTest extends BaseTestSuite { nHeader.withAlias(nPropFoo as s).vars should equalWithTracing(Set(n, s)) } - it("can return vars that are not present in the header, but own an expression in the header") { - RecordHeader.empty.withExpr(nodeListSegment).vars should equal(Set(nodeList)) + it( + "can return vars that are not present in the header, but own an expression in the header" + ) { + RecordHeader.empty.withExpr(nodeListSegment).vars should equal( + Set(nodeList) + ) } it("can return all return items") { @@ -97,8 +107,12 @@ class RecordHeaderTest extends BaseTestSuite { } it("can return all contained columns") { - nHeader.columns should equalWithTracing(nHeader.expressions.map(nHeader.column)) - nHeader.withAlias(n as m).columns should equalWithTracing(nHeader.expressions.map(nHeader.column)) + nHeader.columns should equalWithTracing( + nHeader.expressions.map(nHeader.column) + ) + nHeader.withAlias(n as m).columns should equalWithTracing( + nHeader.expressions.map(nHeader.column) + ) } it("can check if an expression is contained") { @@ -116,7 +130,8 @@ class RecordHeaderTest extends BaseTestSuite { } it("can add element expressions without column collisions") { - val underlineHeader = RecordHeader.empty.withExpr(Var("_")()).withExpr(Var(".")(CTAny)) + val underlineHeader = + RecordHeader.empty.withExpr(Var("_")()).withExpr(Var(".")(CTAny)) underlineHeader.columns.size should be(2) } @@ -128,7 +143,9 @@ class RecordHeaderTest extends BaseTestSuite { it("can return all expressions for a given column") { nHeader.expressionsFor(nHeader.column(n)) should equalWithTracing(Set(n)) - nHeader.withAlias(n as m).expressionsFor(nHeader.column(n)) should equalWithTracing(Set(n, m)) + nHeader + .withAlias(n as m) + .expressionsFor(nHeader.column(n)) should equalWithTracing(Set(n, m)) } it("can correctly handles AliasExpr using withExpr") { @@ -145,7 +162,9 @@ class RecordHeaderTest extends BaseTestSuite { withAlias.ownedBy(m) should equalWithTracing(mExprs) } - it("can add an alias for an element that already exists but with different CypherType") { + it( + "can add an alias for an element that already exists but with different CypherType" + ) { val withAlias = nHeader.withAlias(n as Var(n.name)(CTNode)) withAlias.elementVars.size should be(1) @@ -197,25 +216,26 @@ class RecordHeaderTest extends BaseTestSuite { CTNode("A") -> CTNode("A", "B"), CTRelationship("A") -> CTRelationship("B"), CTRelationship("A") -> CTRelationship("A", "B") - ).foreach { - case (p1Type, p2Type) => - val p1 = Var("p")(p1Type) - val p2 = Var("p")(p2Type) + ).foreach { case (p1Type, p2Type) => + val p1 = Var("p")(p1Type) + val p2 = Var("p")(p2Type) - val p1Header = RecordHeader.empty.withExpr(p1) - val p2Header = RecordHeader.empty.withExpr(p2) + val p1Header = RecordHeader.empty.withExpr(p1) + val p2Header = RecordHeader.empty.withExpr(p2) - val unionHeader = p1Header ++ p2Header + val unionHeader = p1Header ++ p2Header - unionHeader.expressions.size shouldBe 1 - unionHeader.expressions.head.cypherType shouldBe (p1Type join p2Type) + unionHeader.expressions.size shouldBe 1 + unionHeader.expressions.head.cypherType shouldBe (p1Type join p2Type) } } it("can remove expressions") { nHeader -- nExprs should equal(RecordHeader.empty) nHeader -- Set(n) should equal(RecordHeader.empty) - nHeader -- Set(nPropFoo) should equal(RecordHeader.empty.withExpr(n).withExpr(nLabelA).withExpr(nLabelB)) + nHeader -- Set(nPropFoo) should equal( + RecordHeader.empty.withExpr(n).withExpr(nLabelA).withExpr(nLabelB) + ) nHeader -- Set(m) should equal(nHeader) } @@ -268,7 +288,9 @@ class RecordHeaderTest extends BaseTestSuite { nHeader.idExpressions(m) should equalWithTracing(Set.empty) rHeader.idExpressions(r) should equalWithTracing(Set(r, rStart, rEnd)) (nHeader ++ rHeader).idExpressions(n) should equalWithTracing(Set(n)) - (nHeader ++ rHeader).idExpressions(r) should equalWithTracing(Set(r, rStart, rEnd)) + (nHeader ++ rHeader).idExpressions(r) should equalWithTracing( + Set(r, rStart, rEnd) + ) } it("finds all id columns") { @@ -279,11 +301,13 @@ class RecordHeaderTest extends BaseTestSuite { ) val rExtendedHeader = nHeader ++ rHeader - rExtendedHeader.idColumns should equalWithTracing(Set( - rExtendedHeader.column(n), - rExtendedHeader.column(r), - rExtendedHeader.column(rStart), - rExtendedHeader.column(rEnd)) + rExtendedHeader.idColumns should equalWithTracing( + Set( + rExtendedHeader.column(n), + rExtendedHeader.column(r), + rExtendedHeader.column(rStart), + rExtendedHeader.column(rEnd) + ) ) } @@ -295,11 +319,15 @@ class RecordHeaderTest extends BaseTestSuite { ) val rExtendedHeader = nHeader ++ rHeader - rExtendedHeader.idColumns(n) should equalWithTracing(Set(rExtendedHeader.column(n))) - rExtendedHeader.idColumns(r) should equalWithTracing(Set( - rExtendedHeader.column(r), - rExtendedHeader.column(rStart), - rExtendedHeader.column(rEnd)) + rExtendedHeader.idColumns(n) should equalWithTracing( + Set(rExtendedHeader.column(n)) + ) + rExtendedHeader.idColumns(r) should equalWithTracing( + Set( + rExtendedHeader.column(r), + rExtendedHeader.column(rStart), + rExtendedHeader.column(rEnd) + ) ) } @@ -319,9 +347,12 @@ class RecordHeaderTest extends BaseTestSuite { } it("can return transitive members for an element") { - val withSegment = nHeader.withAlias(n as nodeListSegment).select(nodeListSegment) + val withSegment = + nHeader.withAlias(n as nodeListSegment).select(nodeListSegment) - withSegment.ownedBy(nodeList) should equalWithTracing(withSegment.expressions) + withSegment.ownedBy(nodeList) should equalWithTracing( + withSegment.expressions + ) } it("returns labels for a node") { @@ -353,8 +384,12 @@ class RecordHeaderTest extends BaseTestSuite { it("returns all rel elements") { rHeader.relationshipElements should equalWithTracing(Set(r)) nHeader.relationshipElements should equalWithTracing(Set.empty) - (nHeader ++ relListHeader).relationshipElements should equalWithTracing(Set(relListSegment)) - (rHeader ++ relListHeader).relationshipElements should equalWithTracing(Set(r, relListSegment)) + (nHeader ++ relListHeader).relationshipElements should equalWithTracing( + Set(relListSegment) + ) + (rHeader ++ relListHeader).relationshipElements should equalWithTracing( + Set(r, relListSegment) + ) } it("returns all node vars for a given node type") { @@ -364,15 +399,30 @@ class RecordHeaderTest extends BaseTestSuite { } it("returns all node var that match a given node type exactly") { - nHeader.nodesForType(CTNode("A", "B"), exactMatch = true) should equalWithTracing(Set(n)) - nHeader.nodesForType(CTNode("A"), exactMatch = true) should equalWithTracing(Set.empty) - nHeader.nodesForType(CTNode("B"), exactMatch = true) should equalWithTracing(Set.empty) + nHeader.nodesForType( + CTNode("A", "B"), + exactMatch = true + ) should equalWithTracing(Set(n)) + nHeader.nodesForType( + CTNode("A"), + exactMatch = true + ) should equalWithTracing(Set.empty) + nHeader.nodesForType( + CTNode("B"), + exactMatch = true + ) should equalWithTracing(Set.empty) } it("returns all rel vars for a given rel type") { - rHeader.relationshipsForType(CTRelationship("R")) should equalWithTracing(Set(r)) - rHeader.relationshipsForType(CTRelationship("R", "S")) should equalWithTracing(Set(r)) - rHeader.relationshipsForType(CTRelationship("S")) should equalWithTracing(Set.empty) + rHeader.relationshipsForType(CTRelationship("R")) should equalWithTracing( + Set(r) + ) + rHeader.relationshipsForType( + CTRelationship("R", "S") + ) should equalWithTracing(Set(r)) + rHeader.relationshipsForType(CTRelationship("S")) should equalWithTracing( + Set.empty + ) rHeader.relationshipsForType(CTRelationship) should equalWithTracing(Set(r)) } @@ -384,18 +434,26 @@ class RecordHeaderTest extends BaseTestSuite { } it("returns the alias without the original when selecting an alias") { - nHeader.select(Set(n as m)) should equal(nHeader.withAlias(n as m) -- nHeader.expressions) + nHeader.select(Set(n as m)) should equal( + nHeader.withAlias(n as m) -- nHeader.expressions + ) } - it("returns selected element and alias vars and their corresponding columns") { + it( + "returns selected element and alias vars and their corresponding columns" + ) { val s = Var("nPropFoo_Alias")(nPropFoo.cypherType) val aliasHeader = nHeader .withAlias(n as m) .withAlias(nPropFoo as s) - aliasHeader.select(Set(s)) should equal(RecordHeader(Map( - s -> nHeader.column(nPropFoo) - ))) + aliasHeader.select(Set(s)) should equal( + RecordHeader( + Map( + s -> nHeader.column(nPropFoo) + ) + ) + ) aliasHeader.select(Set(n, s)) should equal(nHeader.withAlias(nPropFoo as s)) aliasHeader.select(Set(n, m)) should equal(nHeader.withAlias(n as m)) @@ -408,19 +466,30 @@ class RecordHeaderTest extends BaseTestSuite { val aliasHeader2 = selectHeader1.withAlias(m as o) // WITH m as o val selectHeader2 = aliasHeader2.select(Set[Expr](o)) - selectHeader2.ownedBy(o).map(selectHeader2.column) should equal(nHeader.ownedBy(n).map(nHeader.column)) + selectHeader2.ownedBy(o).map(selectHeader2.column) should equal( + nHeader.ownedBy(n).map(nHeader.column) + ) } it("returns original column names after cascaded select with 1:n aliasing") { - val aliasHeader = nHeader.withAlias(n as m).withAlias(n as o) // WITH n, n AS m, n AS o + val aliasHeader = + nHeader.withAlias(n as m).withAlias(n as o) // WITH n, n AS m, n AS o val selectHeader = aliasHeader.select(Set[Expr](n, m, o)) - selectHeader.ownedBy(n).map(selectHeader.column) should equal(nHeader.ownedBy(n).map(nHeader.column)) - selectHeader.ownedBy(m).map(selectHeader.column) should equal(nHeader.ownedBy(n).map(nHeader.column)) - selectHeader.ownedBy(o).map(selectHeader.column) should equal(nHeader.ownedBy(n).map(nHeader.column)) + selectHeader.ownedBy(n).map(selectHeader.column) should equal( + nHeader.ownedBy(n).map(nHeader.column) + ) + selectHeader.ownedBy(m).map(selectHeader.column) should equal( + nHeader.ownedBy(n).map(nHeader.column) + ) + selectHeader.ownedBy(o).map(selectHeader.column) should equal( + nHeader.ownedBy(n).map(nHeader.column) + ) } - it("returns original column names after cascaded select with property aliases") { + it( + "returns original column names after cascaded select with property aliases" + ) { val s = Var("nPropFoo_Alias")(nPropFoo.cypherType) val t = Var("nPropFoo_Alias")(nPropFoo.cypherType) val aliasHeader1 = nHeader.withAlias(nPropFoo as s) // WITH n.foo AS s @@ -441,13 +510,16 @@ class RecordHeaderTest extends BaseTestSuite { selectHeader2 should equal(nHeader) } - it("supports reusing previously used vars with same name but different type") { + it( + "supports reusing previously used vars with same name but different type" + ) { val n2 = Var("n")(nPropFoo.cypherType) val mPropFoo = nPropFoo.withOwner(m) val aliasHeader1 = nHeader.withAlias(n as m) // WITH n AS m val selectHeader1 = aliasHeader1.select(Set(m)) - val aliasHeader2 = selectHeader1.withAlias(mPropFoo as n2) // WITH m.foo AS n + val aliasHeader2 = + selectHeader1.withAlias(mPropFoo as n2) // WITH m.foo AS n val selectHeader2 = aliasHeader2.select(Set(n2)) selectHeader2.column(n2) should equal(nHeader.column(nPropFoo)) @@ -461,7 +533,8 @@ class RecordHeaderTest extends BaseTestSuite { it("renames aliases columns") { val newColumnName = "newName" - val modifiedHeader = nHeader.withAlias(n as m).withColumnRenamed(nPropFoo, newColumnName) + val modifiedHeader = + nHeader.withAlias(n as m).withColumnRenamed(nPropFoo, newColumnName) modifiedHeader.column(nPropFoo) should equal(newColumnName) modifiedHeader.column(nPropFoo.withOwner(m)) should equal(newColumnName) @@ -470,14 +543,17 @@ class RecordHeaderTest extends BaseTestSuite { it("renames multiple columns") { val newName1 = "foo" val newName2 = "lalala" - val modifiedHeader = nHeader.withColumnsRenamed(Seq(nPropFoo -> newName1, nLabelA -> newName2)) + val modifiedHeader = + nHeader.withColumnsRenamed(Seq(nPropFoo -> newName1, nLabelA -> newName2)) modifiedHeader.column(nPropFoo) should equal(newName1) modifiedHeader.column(nLabelA) should equal(newName2) } it("replaces multiple columns") { val aliasHeader = nHeader.withAlias(n as m) // WITH n AS m - val replaceHeader = aliasHeader.withColumnsReplaced(Map(nPropFoo -> "nFoo", mPropFoo -> "mFoo")) + val replaceHeader = aliasHeader.withColumnsReplaced( + Map(nPropFoo -> "nFoo", mPropFoo -> "mFoo") + ) aliasHeader.columns.size should equal(nExprs.size) replaceHeader.columns.size should equal(nExprs.size + 1) @@ -493,12 +569,14 @@ class RecordHeaderTest extends BaseTestSuite { an[IllegalArgumentException] should be thrownBy nHeader.join(aliased) } - it("joins record headers with overlapping column names and multiple expressions per column") { + it( + "joins record headers with overlapping column names and multiple expressions per column" + ) { val aliased = nHeader.withAlias(n as m).withAlias(n as o).select(m, o) an[IllegalArgumentException] should be thrownBy nHeader.join(aliased) } - it("raises an error when joining header with overlapping expressions"){ + it("raises an error when joining header with overlapping expressions") { intercept[org.opencypher.okapi.impl.exception.IllegalArgumentException] { nHeader join nHeader } diff --git a/okapi-tck/src/main/scala/org/opencypher/okapi/tck/test/AcceptanceTestGenerator.scala b/okapi-tck/src/main/scala/org/opencypher/okapi/tck/test/AcceptanceTestGenerator.scala index ff97ec6f7d..fdb384f83e 100644 --- a/okapi-tck/src/main/scala/org/opencypher/okapi/tck/test/AcceptanceTestGenerator.scala +++ b/okapi-tck/src/main/scala/org/opencypher/okapi/tck/test/AcceptanceTestGenerator.scala @@ -46,22 +46,42 @@ case class AcceptanceTestGenerator( private val escapeStringMarks = "\"\"\"" private val packageNames = Map("white" -> "whiteList", "black" -> "blackList") - //generates test-cases for given scenario names - def generateGivenScenarios(outDir: File, resFiles: Array[File], keyWords: Array[String] = Array.empty): Unit = { + // generates test-cases for given scenario names + def generateGivenScenarios( + outDir: File, + resFiles: Array[File], + keyWords: Array[String] = Array.empty + ): Unit = { setUpDirectories(outDir) val scenarios = getScenarios(resFiles) val wantedWhiteScenarios = scenarios.whiteList.filter(scen => - keyWords.map(keyWord => - scen.name - .contains(keyWord)) - .reduce(_ || _)) + keyWords + .map(keyWord => + scen.name + .contains(keyWord) + ) + .reduce(_ || _) + ) val wantedBlackScenarios = scenarios.blackList.filter(scen => - keyWords.map(keyWord => - scen.name - .contains(keyWord)) - .reduce(_ || _)) - generateClassFile("specialWhiteCases", wantedWhiteScenarios, black = false, outDir) - generateClassFile("specialBlackCases", wantedBlackScenarios, black = true, outDir) + keyWords + .map(keyWord => + scen.name + .contains(keyWord) + ) + .reduce(_ || _) + ) + generateClassFile( + "specialWhiteCases", + wantedWhiteScenarios, + black = false, + outDir + ) + generateClassFile( + "specialBlackCases", + wantedBlackScenarios, + black = true, + outDir + ) } def generateAllScenarios(outDir: File, resFiles: Array[File]): Unit = { @@ -78,11 +98,19 @@ case class AcceptanceTestGenerator( } } - private def generateClassFile(className: String, scenarios: Seq[Scenario], black: Boolean, outDir: File) = { - val packageName = if (black) packageNames.get("black") else packageNames.get("white") + private def generateClassFile( + className: String, + scenarios: Seq[Scenario], + black: Boolean, + outDir: File + ) = { + val packageName = + if (black) packageNames.get("black") else packageNames.get("white") val testCases = scenarios - .filterNot(_.name.equals("Failing on incorrect unicode literal")) //produced test couldn't be compiled + .filterNot( + _.name.equals("Failing on incorrect unicode literal") + ) // produced test couldn't be compiled .map(scenario => generateTest(scenario, black)) .mkString("\n") @@ -141,7 +169,7 @@ case class AcceptanceTestGenerator( file.createNewFile() } - //checks if package directories exists clears them or creates new + // checks if package directories exists clears them or creates new private def setUpDirectories(outDir: File): Unit = { if (outDir.exists() || outDir.mkdir()) packageNames.values.map(packageName => { @@ -149,8 +177,7 @@ case class AcceptanceTestGenerator( if (directory.exists()) { val files = directory.listFiles() files.map(_.delete()) - } - else { + } else { directory.mkdir() } val gitIgnoreFile = new File(outDir + "/" + packageName + "/.gitignore") @@ -158,25 +185,26 @@ case class AcceptanceTestGenerator( writer.println("*") writer.close() gitIgnoreFile.createNewFile() - } - ) + }) } private def getScenarios(resFiles: Array[File]): ScenariosFor = { - //todo: only take .txt or files with no suffix? - val resFileNames = resFiles.map(_.getPath).filterNot(name => name.contains(".feature") || name.contains(".scala")) + // todo: only take .txt or files with no suffix? + val resFileNames = resFiles + .map(_.getPath) + .filterNot(name => name.contains(".feature") || name.contains(".scala")) ScenariosFor(resFileNames: _*) } private def generateTest(scenario: Scenario, black: Boolean): String = { val (initSteps, execSteps) = scenario.steps.partition { case Execute(_, InitQuery, _) => true - case _ => false + case _ => false } - //combine multiple initQueries into one - val initQueries = initSteps.collect { - case Execute(query, InitQuery, _) => s"${alignString(query)}" + // combine multiple initQueries into one + val initQueries = initSteps.collect { case Execute(query, InitQuery, _) => + s"${alignString(query)}" } val initQueryString = @@ -197,7 +225,7 @@ case class AcceptanceTestGenerator( val expectsError = scenario.steps.exists { case _: ExpectError => true - case _ => false + case _ => false } if (black) @@ -206,7 +234,9 @@ case class AcceptanceTestGenerator( | $alignedTestString | }) match{ | case Success(_) => - | throw new RuntimeException("${if (expectsError) "False-positive as probably wrong error" else "A blacklisted scenario works"}") + | throw new RuntimeException("${if (expectsError) + "False-positive as probably wrong error" + else "A blacklisted scenario works"}") | case Failure(_) => | } | } @@ -219,80 +249,122 @@ case class AcceptanceTestGenerator( } private def stepsToString(steps: List[(Step, Int)]): String = { - case class stepContext(ExecQueryStepNr: Option[Int], ParameterStepNr: Option[Int], graphStateStepNr: Option[Int]) + case class stepContext( + ExecQueryStepNr: Option[Int], + ParameterStepNr: Option[Int], + graphStateStepNr: Option[Int] + ) - steps.foldLeft((stepContext(None, None, None), "")) { - case ((context, accString), (Parameters(p, _), stepNr)) => - val stepString = s"val parameter$stepNr = ${tckCypherMapToOkapiCreateString(p)}" - stepContext(context.ExecQueryStepNr, Some(stepNr), context.graphStateStepNr) -> (accString + s"\n $stepString") - case ((context, accString), (_: Measure, stepNr)) => - if (checkSideEffects) { - val stepString = s"val beforeState$stepNr = SideEffectOps.measureState(TCKGraph($graphFactoryName,graph))" - stepContext(context.ExecQueryStepNr, context.ParameterStepNr, Some(stepNr)) -> (accString + s"\n $stepString") - } - else context -> accString - case ((context, accString), (Execute(query, queryType, _), stepNr)) => - queryType match { - case ExecQuery => - val parameters = context.ParameterStepNr match { - case Some(pStepNr) => s", parameter$pStepNr" - case None => "" - } + steps + .foldLeft((stepContext(None, None, None), "")) { + case ((context, accString), (Parameters(p, _), stepNr)) => + val stepString = + s"val parameter$stepNr = ${tckCypherMapToOkapiCreateString(p)}" + stepContext( + context.ExecQueryStepNr, + Some(stepNr), + context.graphStateStepNr + ) -> (accString + s"\n $stepString") + case ((context, accString), (_: Measure, stepNr)) => + if (checkSideEffects) { val stepString = - s"""|lazy val result$stepNr = graph.cypher( + s"val beforeState$stepNr = SideEffectOps.measureState(TCKGraph($graphFactoryName,graph))" + stepContext( + context.ExecQueryStepNr, + context.ParameterStepNr, + Some(stepNr) + ) -> (accString + s"\n $stepString") + } else context -> accString + case ((context, accString), (Execute(query, queryType, _), stepNr)) => + queryType match { + case ExecQuery => + val parameters = context.ParameterStepNr match { + case Some(pStepNr) => s", parameter$pStepNr" + case None => "" + } + val stepString = + s"""|lazy val result$stepNr = graph.cypher( |$escapeStringMarks | ${alignString(query)} |$escapeStringMarks$parameters |) """ - stepContext(Some(stepNr), context.ParameterStepNr, context.graphStateStepNr) -> (accString + s"\n $stepString") - case _ => - //currently no TCK-Tests with side effect queries - throw NotImplementedException("Side Effect Queries specified inside the scenario are not supported yet") - } - case ((context, accString), (ExpectResult(expectedResult: CypherValueRecords, _, sorted), _)) => - val resultNumber = context.ExecQueryStepNr match { - case Some(stepNr) => stepNr - case None => throw new IllegalStateException(s"no query found to compare result $expectedResult") - } + stepContext( + Some(stepNr), + context.ParameterStepNr, + context.graphStateStepNr + ) -> (accString + s"\n $stepString") + case _ => + // currently no TCK-Tests with side effect queries + throw NotImplementedException( + "Side Effect Queries specified inside the scenario are not supported yet" + ) + } + case ( + (context, accString), + (ExpectResult(expectedResult: CypherValueRecords, _, sorted), _) + ) => + val resultNumber = context.ExecQueryStepNr match { + case Some(stepNr) => stepNr + case None => + throw new IllegalStateException( + s"no query found to compare result $expectedResult" + ) + } - val equalMethod = if (sorted) "equals" else "equalsUnordered" - val stepString = - s"""|val result${resultNumber}ValueRecords = convertToTckStrings(result$resultNumber.records).asValueRecords - |val expected${resultNumber}ValueRecords = CypherValueRecords(List(${expectedResult.header.map(escapeString).mkString(",")}), - | List(${expectedResult.rows.map(tckCypherMapToTCKCreateString).mkString(s", \n ")})) + val equalMethod = if (sorted) "equals" else "equalsUnordered" + val stepString = + s"""|val result${resultNumber}ValueRecords = convertToTckStrings(result$resultNumber.records).asValueRecords + |val expected${resultNumber}ValueRecords = CypherValueRecords(List(${expectedResult.header + .map(escapeString) + .mkString(",")}), + | List(${expectedResult.rows + .map(tckCypherMapToTCKCreateString) + .mkString(s", \n ")})) | |result${resultNumber}ValueRecords.$equalMethod(expected${resultNumber}ValueRecords) shouldBe true """ - context -> (accString + s"\n $stepString") - case ((context, accString), (ExpectError(errorType, errorPhase, detail, _), _)) => - val stepNumber = context.ExecQueryStepNr match { - case Some(stepNr) => stepNr - case None => throw new IllegalStateException(s"no query found to check for thrown error $detail") - } - //todo: check for errorType and detail (when corresponding errors exist in Morpheus like SyntaxError, TypeError, ParameterMissing, ...) - //todo: maybe check if they get imported? (or modify specificNamings case class with optional parameter - val stepString = - s""" + context -> (accString + s"\n $stepString") + case ( + (context, accString), + (ExpectError(errorType, errorPhase, detail, _), _) + ) => + val stepNumber = context.ExecQueryStepNr match { + case Some(stepNr) => stepNr + case None => + throw new IllegalStateException( + s"no query found to check for thrown error $detail" + ) + } + // todo: check for errorType and detail (when corresponding errors exist in Morpheus like SyntaxError, TypeError, ParameterMissing, ...) + // todo: maybe check if they get imported? (or modify specificNamings case class with optional parameter + val stepString = + s""" |val errorMessage$stepNumber = an[Exception] shouldBe thrownBy{result$stepNumber} """.stripMargin - context -> (accString + s"\n $stepString") - case ((context, accString), (SideEffects(expected, _), _)) => - val relevantSideEffects = expected.v.filter(_._2 > 0) - val stepString = if (checkSideEffects) { - val contextStep = context.graphStateStepNr match { - case Some(v) => v - case None => throw new IllegalStateException(s"no graph state found to check side effects") - } - s"""|val afterState$contextStep = SideEffectOps.measureState(TCKGraph($graphFactoryName,graph)) - |(beforeState$contextStep diff afterState$contextStep) shouldEqual ${diffToCreateString(expected)} + context -> (accString + s"\n $stepString") + case ((context, accString), (SideEffects(expected, _), _)) => + val relevantSideEffects = expected.v.filter(_._2 > 0) + val stepString = if (checkSideEffects) { + val contextStep = context.graphStateStepNr match { + case Some(v) => v + case None => + throw new IllegalStateException( + s"no graph state found to check side effects" + ) + } + s"""|val afterState$contextStep = SideEffectOps.measureState(TCKGraph($graphFactoryName,graph)) + |(beforeState$contextStep diff afterState$contextStep) shouldEqual ${diffToCreateString( + expected + )} """.stripMargin - } - else if (relevantSideEffects.nonEmpty) s"fail //due to ${relevantSideEffects.mkString(" ")} sideEffects expected" - else "" - context -> (accString + s"\n $stepString") - case ((context, accString), _) => context -> accString - }._2 + } else if (relevantSideEffects.nonEmpty) + s"fail //due to ${relevantSideEffects.mkString(" ")} sideEffects expected" + else "" + context -> (accString + s"\n $stepString") + case ((context, accString), _) => context -> accString + } + ._2 } private def alignString(query: String, tabsNumber: Int = 3): String = { diff --git a/okapi-tck/src/main/scala/org/opencypher/okapi/tck/test/CreateStringGenerator.scala b/okapi-tck/src/main/scala/org/opencypher/okapi/tck/test/CreateStringGenerator.scala index 279732a9c6..15212b7373 100644 --- a/okapi-tck/src/main/scala/org/opencypher/okapi/tck/test/CreateStringGenerator.scala +++ b/okapi-tck/src/main/scala/org/opencypher/okapi/tck/test/CreateStringGenerator.scala @@ -26,34 +26,68 @@ */ package org.opencypher.okapi.tck.test - import org.apache.commons.text.StringEscapeUtils import org.opencypher.okapi.impl.exception.NotImplementedException import org.opencypher.okapi.tck.test.TckToCypherConverter.tckValueToCypherValue -import org.opencypher.okapi.api.value.CypherValue.{CypherList => OKAPICypherList, CypherMap => OKAPICypherMap, CypherNull => OKAPICypherNull, CypherString => OKAPICypherString, CypherValue => OKAPICypherValue} +import org.opencypher.okapi.api.value.CypherValue.{ + CypherList => OKAPICypherList, + CypherMap => OKAPICypherMap, + CypherNull => OKAPICypherNull, + CypherString => OKAPICypherString, + CypherValue => OKAPICypherValue +} import org.opencypher.tools.tck.SideEffectOps.Diff -import org.opencypher.tools.tck.values.{Backward => TCKBackward, CypherBoolean => TCKCypherBoolean, CypherFloat => TCKCypherFloat, CypherInteger => TCKCypherInteger, CypherList => TCKCypherList, CypherNode => TCKCypherNode, CypherNull => TCKCypherNull, CypherOrderedList => TCKCypherOrderedList, CypherPath => TCKCypherPath, CypherProperty => TCKCypherProperty, CypherPropertyMap => TCKCypherPropertyMap, CypherRelationship => TCKCypherRelationship, CypherString => TCKCypherString, CypherUnorderedList => TCKUnorderedList, CypherValue => TCKCypherValue, Forward => TCKForward} +import org.opencypher.tools.tck.values.{ + Backward => TCKBackward, + CypherBoolean => TCKCypherBoolean, + CypherFloat => TCKCypherFloat, + CypherInteger => TCKCypherInteger, + CypherList => TCKCypherList, + CypherNode => TCKCypherNode, + CypherNull => TCKCypherNull, + CypherOrderedList => TCKCypherOrderedList, + CypherPath => TCKCypherPath, + CypherProperty => TCKCypherProperty, + CypherPropertyMap => TCKCypherPropertyMap, + CypherRelationship => TCKCypherRelationship, + CypherString => TCKCypherString, + CypherUnorderedList => TCKUnorderedList, + CypherValue => TCKCypherValue, + Forward => TCKForward +} object CreateStringGenerator { def tckCypherValueToCreateString(value: TCKCypherValue): String = { value match { - case TCKCypherString(v) => s"TCKCypherString(${escapeString(v)})" + case TCKCypherString(v) => s"TCKCypherString(${escapeString(v)})" case TCKCypherInteger(v) => s"TCKCypherInteger(${v}L)" - case TCKCypherFloat(v) => s"TCKCypherFloat($v)" + case TCKCypherFloat(v) => s"TCKCypherFloat($v)" case TCKCypherBoolean(v) => s"TCKCypherBoolean($v)" - case TCKCypherProperty(key, v) => s"""TCKCypherProperty("${escapeString(key)}",${tckCypherValueToCreateString(v)})""" + case TCKCypherProperty(key, v) => + s"""TCKCypherProperty("${escapeString( + key + )}",${tckCypherValueToCreateString(v)})""" case TCKCypherPropertyMap(properties) => - val propertiesCreateString = properties.map { case (key, v) => s"(${escapeString(key)}, ${tckCypherValueToCreateString(v)})" }.mkString(",") + val propertiesCreateString = properties + .map { case (key, v) => + s"(${escapeString(key)}, ${tckCypherValueToCreateString(v)})" + } + .mkString(",") s"TCKCypherPropertyMap(Map($propertiesCreateString))" case l: TCKCypherList => l match { - case TCKCypherOrderedList(elems) => s"TCKCypherOrderedList(List(${elems.map(tckCypherValueToCreateString).mkString(",")}))" + case TCKCypherOrderedList(elems) => + s"TCKCypherOrderedList(List(${elems.map(tckCypherValueToCreateString).mkString(",")}))" case _ => s"TCKCypherValue.apply(${escapeString(l.toString)}, false)" } case TCKCypherNull => "TCKCypherNull" case TCKCypherNode(labels, properties) => - val labelsString = if (labels.isEmpty) "" else s"Set(${labels.map(escapeString).mkString(",")})" - val propertyString = if (properties.properties.isEmpty) "" else s"${tckCypherValueToCreateString(properties)}" + val labelsString = + if (labels.isEmpty) "" + else s"Set(${labels.map(escapeString).mkString(",")})" + val propertyString = + if (properties.properties.isEmpty) "" + else s"${tckCypherValueToCreateString(properties)}" val separatorString = if (properties.properties.isEmpty) "" else if (labels.isEmpty) "properties = " @@ -61,44 +95,61 @@ object CreateStringGenerator { s"TCKCypherNode($labelsString$separatorString$propertyString)" case TCKCypherRelationship(typ, properties) => - val propertyString = if (properties.properties.isEmpty) "" else s", ${tckCypherValueToCreateString(properties)}" + val propertyString = + if (properties.properties.isEmpty) "" + else s", ${tckCypherValueToCreateString(properties)}" s"TCKCypherRelationship(${escapeString(typ)}$propertyString)" case TCKCypherPath(start, connections) => - val connectionsCreateString = connections.map { - case TCKForward(r, n) => s"TCKForward(${tckCypherValueToCreateString(r)},${tckCypherValueToCreateString(n)})" - case TCKBackward(r, n) => s"TCKBackward(${tckCypherValueToCreateString(r)},${tckCypherValueToCreateString(n)})" - }.mkString(",") + val connectionsCreateString = connections + .map { + case TCKForward(r, n) => + s"TCKForward(${tckCypherValueToCreateString(r)},${tckCypherValueToCreateString(n)})" + case TCKBackward(r, n) => + s"TCKBackward(${tckCypherValueToCreateString(r)},${tckCypherValueToCreateString(n)})" + } + .mkString(",") s"TCKCypherPath(${tckCypherValueToCreateString(start)},List($connectionsCreateString))" case other => - throw NotImplementedException(s"Converting Cypher value $value of type `${other.getClass.getSimpleName}`") + throw NotImplementedException( + s"Converting Cypher value $value of type `${other.getClass.getSimpleName}`" + ) } } def cypherValueToCreateString(value: OKAPICypherValue): String = { value match { - case OKAPICypherList(l) => s"List(${l.map(cypherValueToCreateString).mkString(",")})" + case OKAPICypherList(l) => + s"List(${l.map(cypherValueToCreateString).mkString(",")})" case OKAPICypherMap(m) => - val mapElementsString = m.map { - case (key, cv) => s"(${escapeString(key)},${cypherValueToCreateString(cv)})" - }.mkString(",") + val mapElementsString = m + .map { case (key, cv) => + s"(${escapeString(key)},${cypherValueToCreateString(cv)})" + } + .mkString(",") s"CypherMap($mapElementsString)" case OKAPICypherString(s) => escapeString(s) - case OKAPICypherNull => "CypherNull" - case _ => s"${value.getClass.getSimpleName}(${value.unwrap})" + case OKAPICypherNull => "CypherNull" + case _ => s"${value.getClass.getSimpleName}(${value.unwrap})" } } - def tckCypherMapToTCKCreateString(cypherMap: Map[String, TCKCypherValue]): String = { - val mapElementsString = cypherMap.map { - case (key, cypherValue) => escapeString(key) -> tckCypherValueToCreateString(cypherValue) + def tckCypherMapToTCKCreateString( + cypherMap: Map[String, TCKCypherValue] + ): String = { + val mapElementsString = cypherMap.map { case (key, cypherValue) => + escapeString(key) -> tckCypherValueToCreateString(cypherValue) } s"Map(${mapElementsString.mkString(",")})" } - def tckCypherMapToOkapiCreateString(cypherMap: Map[String, TCKCypherValue]): String = { - val mapElementsString = cypherMap.map { - case (key, tckCypherValue) => escapeString(key) -> cypherValueToCreateString(tckValueToCypherValue(tckCypherValue)) + def tckCypherMapToOkapiCreateString( + cypherMap: Map[String, TCKCypherValue] + ): String = { + val mapElementsString = cypherMap.map { case (key, tckCypherValue) => + escapeString(key) -> cypherValueToCreateString( + tckValueToCypherValue(tckCypherValue) + ) } s"CypherMap(${mapElementsString.mkString(",")})" } diff --git a/okapi-tck/src/main/scala/org/opencypher/okapi/tck/test/TCKFixture.scala b/okapi-tck/src/main/scala/org/opencypher/okapi/tck/test/TCKFixture.scala index 68716a6f17..d27959dc32 100644 --- a/okapi-tck/src/main/scala/org/opencypher/okapi/tck/test/TCKFixture.scala +++ b/okapi-tck/src/main/scala/org/opencypher/okapi/tck/test/TCKFixture.scala @@ -39,10 +39,21 @@ import org.opencypher.okapi.tck.test.TCKFixture._ import org.opencypher.okapi.testing.propertygraph.{CreateGraphFactory, CypherTestGraphFactory} import org.opencypher.tools.tck.api._ import org.opencypher.tools.tck.constants.{TCKErrorDetails, TCKErrorPhases, TCKErrorTypes} -import org.opencypher.tools.tck.values.{CypherValue => TCKCypherValue, CypherString => TCKCypherString, CypherList => TCKCypherList, CypherOrderedList => TCKCypherOrderedList, - CypherNode => TCKCypherNode, CypherRelationship => TCKCypherRelationship, - CypherInteger => TCKCypherInteger, CypherFloat => TCKCypherFloat, CypherBoolean => TCKCypherBoolean, CypherProperty => TCKCypherProperty, - CypherPropertyMap => TCKCypherPropertyMap, CypherNull => TCKCypherNull, CypherPath => TCKCypherPath} +import org.opencypher.tools.tck.values.{ + CypherValue => TCKCypherValue, + CypherString => TCKCypherString, + CypherList => TCKCypherList, + CypherOrderedList => TCKCypherOrderedList, + CypherNode => TCKCypherNode, + CypherRelationship => TCKCypherRelationship, + CypherInteger => TCKCypherInteger, + CypherFloat => TCKCypherFloat, + CypherBoolean => TCKCypherBoolean, + CypherProperty => TCKCypherProperty, + CypherPropertyMap => TCKCypherPropertyMap, + CypherNull => TCKCypherNull, + CypherPath => TCKCypherPath +} import org.scalatest.Tag import org.scalatest.prop.TableDrivenPropertyChecks._ @@ -52,20 +63,21 @@ import scala.util.{Failure, Success, Try} // needs to be a val because of a TCK bug (scenarios can only be read once) object TCKFixture { // TODO: enable flaky test once new TCk release is there - val scenarios: Seq[Scenario] = CypherTCK.allTckScenarios.filterNot(_.name == "Limit to two hits") + val scenarios: Seq[Scenario] = + CypherTCK.allTckScenarios.filterNot(_.name == "Limit to two hits") - def printAll(): Unit = scenarios - .enumerateScenarioOutlines + def printAll(): Unit = scenarios.enumerateScenarioOutlines .map(s => s"""Feature "${s.featureName}": Scenario "${s.name}"""") .distinct .sorted .foreach(println) implicit class Scenarios(scenarios: Seq[Scenario]) { + /** - * Scenario outlines are parameterised scenarios that all have the same name, but different parameters. - * Because test names need to be unique, we enumerate these scenarios and put the enumeration into the - * scenario name to make those names unique. + * Scenario outlines are parameterised scenarios that all have the same name, but different + * parameters. Because test names need to be unique, we enumerate these scenarios and put the + * enumeration into the scenario name to make those names unique. */ def enumerateScenarioOutlines: Seq[Scenario] = { scenarios.groupBy(_.toString).flatMap { case (_, nameCollisionGroup) => @@ -81,29 +93,61 @@ object TCKFixture { } } -case class TCKGraph[C <: CypherSession](testGraphFactory: CypherTestGraphFactory[C], graph: PropertyGraph) - (implicit OKAPI: C) extends Graph { +case class TCKGraph[C <: CypherSession]( + testGraphFactory: CypherTestGraphFactory[C], + graph: PropertyGraph +)(implicit OKAPI: C) + extends Graph { - override def execute(query: String, params: Map[String, TCKCypherValue], queryType: QueryType): (Graph, Result) = { + override def execute( + query: String, + params: Map[String, TCKCypherValue], + queryType: QueryType + ): (Graph, Result) = { queryType match { case InitQuery => - val propertyGraph = testGraphFactory(CreateGraphFactory(query, params.mapValues(TckToCypherConverter.tckValueToCypherValue))) + val propertyGraph = testGraphFactory( + CreateGraphFactory( + query, + params.mapValues(TckToCypherConverter.tckValueToCypherValue) + ) + ) copy(graph = propertyGraph) -> CypherValueRecords.empty case SideEffectQuery => // this one is tricky, not sure how can do it without Cypher this -> CypherValueRecords.empty case ExecQuery => // mapValues is lazy, so we force it for debug purposes - val result = Try(graph.cypher(query, params.mapValues(TckToCypherConverter.tckValueToCypherValue).view.force)) + val result = Try( + graph.cypher( + query, + params + .mapValues(TckToCypherConverter.tckValueToCypherValue) + .view + .force + ) + ) result match { - case Success(r) => this -> CypherToTCKConverter.convertToTckStrings(r.records) + case Success(r) => + this -> CypherToTCKConverter.convertToTckStrings(r.records) case Failure(e) => - val phase = TCKErrorPhases.RUNTIME // We have no way to detect errors during compile time yet + val phase = + TCKErrorPhases.RUNTIME // We have no way to detect errors during compile time yet e match { - case _: TypingException => this -> - ExecutionFailed(TCKErrorTypes.TYPE_ERROR, phase, TCKErrorDetails.INVALID_ARGUMENT_VALUE) - case ex: NotImplementedException => throw new RuntimeException(s"Unsupported feature in $query", ex) - case _ => throw new RuntimeException(s"Unknown engine failure for query: $query", e) + case _: TypingException => + this -> + ExecutionFailed( + TCKErrorTypes.TYPE_ERROR, + phase, + TCKErrorDetails.INVALID_ARGUMENT_VALUE + ) + case ex: NotImplementedException => + throw new RuntimeException(s"Unsupported feature in $query", ex) + case _ => + throw new RuntimeException( + s"Unknown engine failure for query: $query", + e + ) } } } @@ -114,22 +158,19 @@ case class ScenariosFor(blacklist: Set[String]) { def whiteList = Table( "scenario", - scenarios - .enumerateScenarioOutlines - .filterNot(s => blacklist.contains(s.toString())) - : _* + scenarios.enumerateScenarioOutlines + .filterNot(s => blacklist.contains(s.toString())): _* ) def blackList = Table( "scenario", - scenarios - .enumerateScenarioOutlines - .filter(s => blacklist.contains(s.toString())) - : _* + scenarios.enumerateScenarioOutlines + .filter(s => blacklist.contains(s.toString())): _* ) def get(name: String): Seq[Scenario] = scenarios.filter(s => s.name == name) - def getPrefix(prefix: String): Seq[Scenario] = scenarios.filter(s => s.name.startsWith(prefix)) + def getPrefix(prefix: String): Seq[Scenario] = + scenarios.filter(s => s.name.startsWith(prefix)) } object ScenariosFor { @@ -154,24 +195,30 @@ object Tags { } object TckToCypherConverter { - def tckValueToCypherValue(cypherValue: TCKCypherValue): CypherValue = cypherValue match { - case TCKCypherString(v) => CypherValue(v) - case TCKCypherInteger(v) => CypherValue(v) - case TCKCypherFloat(v) => CypherValue(v) - case TCKCypherBoolean(v) => CypherValue(v) - case TCKCypherProperty(key, value) => CypherMap(key -> tckValueToCypherValue(value)) - case TCKCypherPropertyMap(properties) => CypherMap(properties.mapValues(tckValueToCypherValue)) - case l: TCKCypherList => CypherList(l.elements.map(tckValueToCypherValue)) - case TCKCypherNull => CypherValue(null) - case other => - throw NotImplementedException(s"Converting Cypher value $cypherValue of type `${other.getClass.getSimpleName}`") - } + def tckValueToCypherValue(cypherValue: TCKCypherValue): CypherValue = + cypherValue match { + case TCKCypherString(v) => CypherValue(v) + case TCKCypherInteger(v) => CypherValue(v) + case TCKCypherFloat(v) => CypherValue(v) + case TCKCypherBoolean(v) => CypherValue(v) + case TCKCypherProperty(key, value) => + CypherMap(key -> tckValueToCypherValue(value)) + case TCKCypherPropertyMap(properties) => + CypherMap(properties.mapValues(tckValueToCypherValue)) + case l: TCKCypherList => CypherList(l.elements.map(tckValueToCypherValue)) + case TCKCypherNull => CypherValue(null) + case other => + throw NotImplementedException( + s"Converting Cypher value $cypherValue of type `${other.getClass.getSimpleName}`" + ) + } } object CypherToTCKConverter { def convertToTckStrings(records: CypherRecords): StringRecords = { - val header = records.logicalColumns.getOrElse(records.physicalColumns).toList + val header = + records.logicalColumns.getOrElse(records.physicalColumns).toList val rows: List[Map[String, String]] = records.collect.map { cypherMap: CypherMap => cypherMap.keys.map(k => k -> cypherMap(k).toTCKString).toMap }.toList @@ -182,23 +229,22 @@ object CypherToTCKConverter { def toTCKString: String = { value match { case CypherString(s) => s"'${escape(s)}'" - case CypherList(l) => l.map(_.toTCKString).mkString("[", ", ", "]") + case CypherList(l) => l.map(_.toTCKString).mkString("[", ", ", "]") case CypherMap(m) => m.toSeq .sortBy(_._1) .map { case (k, v) => s"$k: ${v.toTCKString}" } .mkString("{", ", ", "}") case Relationship(_, _, _, relType, props) => - s"[:$relType${ - if (props.isEmpty) "" - else s" ${props.toTCKString}" - }]" + s"[:$relType${if (props.isEmpty) "" + else s" ${props.toTCKString}"}]" case Node(_, labels, props) => val labelString = if (labels.isEmpty) "" else labels.toSeq.sorted.mkString(":", ":", "") - val propertyString = if (props.isEmpty) "" - else s"${props.toTCKString}" + val propertyString = + if (props.isEmpty) "" + else s"${props.toTCKString}" Seq(labelString, propertyString) .filter(_.nonEmpty) .mkString("(", " ", ")") diff --git a/okapi-testing/src/main/scala/org/opencypher/okapi/testing/AsCode.scala b/okapi-testing/src/main/scala/org/opencypher/okapi/testing/AsCode.scala index 17c9dc508c..e5fb06726f 100644 --- a/okapi-testing/src/main/scala/org/opencypher/okapi/testing/AsCode.scala +++ b/okapi-testing/src/main/scala/org/opencypher/okapi/testing/AsCode.scala @@ -29,14 +29,16 @@ package org.opencypher.okapi.testing import org.opencypher.okapi.api.types.CypherType /** - * Returns a string that can be pasted as an object definition for standard case classes, - * some other products, collections and objects. + * Returns a string that can be pasted as an object definition for standard case classes, some + * other products, collections and objects. */ // TODO: remove once MatchHelper can use scalatest again object AsCode { implicit class ImplicitAsCode(a: Any) { - def asCode(implicit specialMappings: PartialFunction[Any, String] = Map.empty): String = { + def asCode(implicit + specialMappings: PartialFunction[Any, String] = Map.empty + ): String = { anyAsCode(a)(specialMappings) } } @@ -49,7 +51,9 @@ object AsCode { anyAsCode(a)(specialMappings) } - private def anyAsCode(a: Any)(implicit specialMappings: PartialFunction[Any, String] = Map.empty): String = { + private def anyAsCode(a: Any)(implicit + specialMappings: PartialFunction[Any, String] = Map.empty + ): String = { if (specialMappings.isDefinedAt(a)) specialMappings(a) else { a match { @@ -70,8 +74,9 @@ object AsCode { } } - private def traversableAsCode(t: Traversable[_])( - implicit specialMappings: PartialFunction[Any, String] = Map.empty): String = { + private def traversableAsCode(t: Traversable[_])(implicit + specialMappings: PartialFunction[Any, String] = Map.empty + ): String = { if (specialMappings.isDefinedAt(t)) specialMappings(t) else { val elementString = t.map(anyAsCode).mkString(", ") @@ -90,7 +95,9 @@ object AsCode { } } - private def productAsCode(p: Product)(implicit specialMappings: PartialFunction[Any, String] = Map.empty): String = { + private def productAsCode(p: Product)(implicit + specialMappings: PartialFunction[Any, String] = Map.empty + ): String = { if (specialMappings.isDefinedAt(p)) specialMappings(p) else { if (p.productIterator.isEmpty) { diff --git a/okapi-testing/src/main/scala/org/opencypher/okapi/testing/BaseTestFixture.scala b/okapi-testing/src/main/scala/org/opencypher/okapi/testing/BaseTestFixture.scala index d198f42333..e2589eface 100644 --- a/okapi-testing/src/main/scala/org/opencypher/okapi/testing/BaseTestFixture.scala +++ b/okapi-testing/src/main/scala/org/opencypher/okapi/testing/BaseTestFixture.scala @@ -29,5 +29,5 @@ package org.opencypher.okapi.testing import org.scalatest.{BeforeAndAfterAll, BeforeAndAfterEach} trait BaseTestFixture extends BeforeAndAfterEach with BeforeAndAfterAll { - self: BaseTestSuite => + self: BaseTestSuite => } diff --git a/okapi-testing/src/main/scala/org/opencypher/okapi/testing/BaseTestSuite.scala b/okapi-testing/src/main/scala/org/opencypher/okapi/testing/BaseTestSuite.scala index ddea5792a1..38075e9513 100644 --- a/okapi-testing/src/main/scala/org/opencypher/okapi/testing/BaseTestSuite.scala +++ b/okapi-testing/src/main/scala/org/opencypher/okapi/testing/BaseTestSuite.scala @@ -40,7 +40,11 @@ import org.scalatest.matchers.should.Matchers import scala.collection.convert.DecorateAsJava import scala.util.Random -abstract class BaseTestSuite extends AnyFunSpec with Matchers with MockitoSugar with DecorateAsJava { +abstract class BaseTestSuite + extends AnyFunSpec + with Matchers + with MockitoSugar + with DecorateAsJava { /* Shared test objects */ val testNamespace = Namespace("testNamespace") @@ -48,20 +52,24 @@ abstract class BaseTestSuite extends AnyFunSpec with Matchers with MockitoSugar val testGraphSchema: PropertyGraphSchema = PropertyGraphSchema.empty val testQualifiedGraphName = QualifiedGraphName(testNamespace, testGraphName) val qgnGenerator: QGNGenerator = new QGNGenerator { - override def generate: QualifiedGraphName = QualifiedGraphName(s"session.#${(Random.nextInt & Int.MaxValue) % 100}") + override def generate: QualifiedGraphName = QualifiedGraphName( + s"session.#${(Random.nextInt & Int.MaxValue) % 100}" + ) } - def testGraphSource(graphsWithSchema: (GraphName, PropertyGraphSchema)*): PropertyGraphDataSource = { + def testGraphSource( + graphsWithSchema: (GraphName, PropertyGraphSchema)* + ): PropertyGraphDataSource = { val gs = mock[PropertyGraphDataSource] - graphsWithSchema.foreach { - case (graphName, schema) => when(gs.schema(graphName)).thenReturn(Some(schema)) + graphsWithSchema.foreach { case (graphName, schema) => + when(gs.schema(graphName)).thenReturn(Some(schema)) } gs } - /** - * Wraps an 'it' call for convenience - */ - def test(name: String, tags: Tag*)(testFun: => Any /* Assertion */ )(implicit pos: source.Position): Unit = + /** Wraps an 'it' call for convenience */ + def test(name: String, tags: Tag*)(testFun: => Any /* Assertion */ )(implicit + pos: source.Position + ): Unit = it(name, tags: _*)(testFun) } diff --git a/okapi-testing/src/main/scala/org/opencypher/okapi/testing/MatchHelper.scala b/okapi-testing/src/main/scala/org/opencypher/okapi/testing/MatchHelper.scala index daf31c00dc..4ee9ed3c77 100644 --- a/okapi-testing/src/main/scala/org/opencypher/okapi/testing/MatchHelper.scala +++ b/okapi-testing/src/main/scala/org/opencypher/okapi/testing/MatchHelper.scala @@ -36,9 +36,8 @@ object MatchHelper { // Returns a successful or the first failed match if there is one. def combine(m: MatchResult*): MatchResult = { - m.foldLeft(success) { - case (aggr, next) => - if (aggr.matches) next else aggr + m.foldLeft(success) { case (aggr, next) => + if (aggr.matches) next else aggr } } @@ -46,13 +45,15 @@ object MatchHelper { if (m.matches) { m } else { - MatchResult(false, m.rawFailureMessage + s"\nTRACE: $traceInfo", m.rawNegatedFailureMessage) + MatchResult( + false, + m.rawFailureMessage + s"\nTRACE: $traceInfo", + m.rawNegatedFailureMessage + ) } } - /** - * A substitute for the ScalaTest `equal` matcher that crashes on some of our trees. - */ + /** A substitute for the ScalaTest `equal` matcher that crashes on some of our trees. */ case class equalWithTracing(right: Any) extends Matcher[Any] { override def apply(left: Any): MatchResult = { equalAny(left, right) @@ -68,14 +69,17 @@ object MatchHelper { } else { left match { case p: Product => equalProduct(p, right.asInstanceOf[Product]) - case t: Seq[_] => equalTraversable(t, t.asInstanceOf[Seq[_]]) - case _ => failure(s"${left.asCode} did not equal ${right.asCode}") + case t: Seq[_] => equalTraversable(t, t.asInstanceOf[Seq[_]]) + case _ => failure(s"${left.asCode} did not equal ${right.asCode}") } } } } - private def equalTraversable(left: Traversable[_], right: Traversable[_]): MatchResult = { + private def equalTraversable( + left: Traversable[_], + right: Traversable[_] + ): MatchResult = { if (left == right) { success } else { @@ -87,11 +91,12 @@ object MatchHelper { combine( left.toSeq .zip(right.toSeq) - .map { - case (l, r) => - equalAny(l, r) - }: _*), - left.asCode) + .map { case (l, r) => + equalAny(l, r) + }: _* + ), + left.asCode + ) } } } @@ -105,17 +110,20 @@ object MatchHelper { failure(s"${left.asCode} does not equal ${right.asCode}") } else { if (left.productIterator.size != right.productIterator.size) { - failure(s"Product of ${left.asCode} does not equal ${right.asCode}}") + failure( + s"Product of ${left.asCode} does not equal ${right.asCode}}" + ) } else { addTrace( combine( left.productIterator.toSeq .zip(right.productIterator.toSeq) - .map { - case (l, r) => - equalAny(l, r) - }: _*), - left.asCode) + .map { case (l, r) => + equalAny(l, r) + }: _* + ), + left.asCode + ) } } } @@ -134,9 +142,12 @@ object MatchHelper { */ case class referenceEqual(right: AnyRef) extends Matcher[AnyRef] { override def apply(left: AnyRef): MatchResult = { - MatchResult(if (left == null) right == null else left.eq(right), s"$left was not the same instance as $right", "") + MatchResult( + if (left == null) right == null else left.eq(right), + s"$left was not the same instance as $right", + "" + ) } } - } diff --git a/okapi-testing/src/main/scala/org/opencypher/okapi/testing/PGDSAcceptanceTest.scala b/okapi-testing/src/main/scala/org/opencypher/okapi/testing/PGDSAcceptanceTest.scala index 04c3838b69..79227f014d 100644 --- a/okapi-testing/src/main/scala/org/opencypher/okapi/testing/PGDSAcceptanceTest.scala +++ b/okapi-testing/src/main/scala/org/opencypher/okapi/testing/PGDSAcceptanceTest.scala @@ -43,7 +43,9 @@ trait PGDSAcceptanceTest[Session <: CypherSession, Graph <: PropertyGraph] { self: BaseTestSuite => object Scenario { - def apply(name: String, initGraph: GraphName)(test: TestContext => Unit): Scenario = { + def apply(name: String, initGraph: GraphName)( + test: TestContext => Unit + ): Scenario = { Scenario(name, List(initGraph))(test) } } @@ -53,11 +55,11 @@ trait PGDSAcceptanceTest[Session <: CypherSession, Graph <: PropertyGraph] { initGraphs: List[GraphName] = Nil )(val test: TestContext => Unit) - abstract class TestContextFactory { self => - def initializeContext(graphNames: List[GraphName]): TestContext = TestContext(initSession, initPgds(graphNames)) + def initializeContext(graphNames: List[GraphName]): TestContext = + TestContext(initSession, initPgds(graphNames)) def initPgds(graphNames: List[GraphName]): PropertyGraphDataSource @@ -83,17 +85,22 @@ trait PGDSAcceptanceTest[Session <: CypherSession, Graph <: PropertyGraph] { case class TestContext(session: Session, pgds: PropertyGraphDataSource) - lazy val graph: Map[GraphName, Graph] = testCreateGraphStatements.mapValues(initGraph) + lazy val graph: Map[GraphName, Graph] = + testCreateGraphStatements.mapValues(initGraph) def allScenarios: List[Scenario] = cypher10Scenarios def initGraph(createStatements: String): Graph - def executeScenariosWithContext(scenarios: List[Scenario], contextFactory: TestContextFactory): Unit = { + def executeScenariosWithContext( + scenarios: List[Scenario], + contextFactory: TestContextFactory + ): Unit = { val scenarioTable = Table("Scenario", scenarios: _*) forAll(scenarioTable) { scenario => test(s"[$contextFactory] ${scenario.name}") { - val ctx: TestContext = contextFactory.initializeContext(scenario.initGraphs) + val ctx: TestContext = + contextFactory.initializeContext(scenario.initGraphs) Try(scenario.test(ctx)) match { case Success(_) => contextFactory.releaseContext(ctx) @@ -149,11 +156,16 @@ trait PGDSAcceptanceTest[Session <: CypherSession, Graph <: PropertyGraph] { session.registerSource(namespace, ctx.pgds) } - def executeQuery(query: String, parameters: CypherMap = CypherMap.empty)(implicit ctx: TestContext): Session#Result = { + def executeQuery(query: String, parameters: CypherMap = CypherMap.empty)(implicit + ctx: TestContext + ): Session#Result = { session.cypher(query, parameters) } - def expectRecordsAnyOrder(result: Session#Result, expectedRecords: CypherMap*): Unit = { + def expectRecordsAnyOrder( + result: Session#Result, + expectedRecords: CypherMap* + ): Unit = { result.records.iterator.toBag should equal( expectedRecords.toBag ) @@ -166,16 +178,18 @@ trait PGDSAcceptanceTest[Session <: CypherSession, Graph <: PropertyGraph] { session.catalog.source(ns).hasGraph(g1) shouldBe true session.catalog.source(ns).hasGraph(GraphName("foo")) shouldBe false }, - - Scenario("API: PropertyGraphDataSource.hasGraph", List(g1, g3, g4)) { implicit ctx: TestContext => - registerPgds(ns) - session.catalog.source(ns).hasGraph(g1) shouldBe true - session.catalog.source(ns).hasGraph(g3) shouldBe true - session.catalog.source(ns).hasGraph(g4) shouldBe true - session.catalog.source(ns).hasGraph(GraphName("foo")) shouldBe false + Scenario("API: PropertyGraphDataSource.hasGraph", List(g1, g3, g4)) { + implicit ctx: TestContext => + registerPgds(ns) + session.catalog.source(ns).hasGraph(g1) shouldBe true + session.catalog.source(ns).hasGraph(g3) shouldBe true + session.catalog.source(ns).hasGraph(g4) shouldBe true + session.catalog.source(ns).hasGraph(GraphName("foo")) shouldBe false }, - - Scenario("API: PropertyGraphDataSource.hasGraph after reset", List(g1, g3, g4)) { implicit ctx: TestContext => + Scenario( + "API: PropertyGraphDataSource.hasGraph after reset", + List(g1, g3, g4) + ) { implicit ctx: TestContext => registerPgds(ns) session.catalog.source(ns).hasGraph(g1) shouldBe true session.catalog.source(ns).hasGraph(g3) shouldBe true @@ -187,15 +201,17 @@ trait PGDSAcceptanceTest[Session <: CypherSession, Graph <: PropertyGraph] { session.catalog.source(ns).hasGraph(g4) shouldBe true session.catalog.source(ns).hasGraph(GraphName("foo")) shouldBe false }, - - Scenario("API: PropertyGraphDataSource.graphNames", List(g1, g3, g4)) { implicit ctx: TestContext => - registerPgds(ns) - session.catalog.source(ns).graphNames should contain(g1) - session.catalog.source(ns).graphNames should contain(g3) - session.catalog.source(ns).graphNames should contain(g4) + Scenario("API: PropertyGraphDataSource.graphNames", List(g1, g3, g4)) { + implicit ctx: TestContext => + registerPgds(ns) + session.catalog.source(ns).graphNames should contain(g1) + session.catalog.source(ns).graphNames should contain(g3) + session.catalog.source(ns).graphNames should contain(g4) }, - - Scenario("API: PropertyGraphDataSource.graphNames after reset", List(g1, g3, g4)) { implicit ctx: TestContext => + Scenario( + "API: PropertyGraphDataSource.graphNames after reset", + List(g1, g3, g4) + ) { implicit ctx: TestContext => registerPgds(ns) session.catalog.source(ns).graphNames should contain(g1) session.catalog.source(ns).graphNames should contain(g3) @@ -205,15 +221,17 @@ trait PGDSAcceptanceTest[Session <: CypherSession, Graph <: PropertyGraph] { session.catalog.source(ns).graphNames should contain(g3) session.catalog.source(ns).graphNames should contain(g4) }, - - Scenario("API: PropertyGraphDataSource.graph", List(g1, g3, g4)) { implicit ctx: TestContext => - registerPgds(ns) - session.catalog.source(ns).graph(g1) - session.catalog.source(ns).graph(g3) - session.catalog.source(ns).graph(g4) + Scenario("API: PropertyGraphDataSource.graph", List(g1, g3, g4)) { + implicit ctx: TestContext => + registerPgds(ns) + session.catalog.source(ns).graph(g1) + session.catalog.source(ns).graph(g3) + session.catalog.source(ns).graph(g4) }, - - Scenario("API: PropertyGraphDataSource.graph after reset", List(g1, g3, g4)) { implicit ctx: TestContext => + Scenario( + "API: PropertyGraphDataSource.graph after reset", + List(g1, g3, g4) + ) { implicit ctx: TestContext => registerPgds(ns) session.catalog.source(ns).graph(g1) session.catalog.source(ns).graph(g3) @@ -223,58 +241,87 @@ trait PGDSAcceptanceTest[Session <: CypherSession, Graph <: PropertyGraph] { session.catalog.source(ns).graph(g3) session.catalog.source(ns).graph(g4) }, - Scenario("API: Correct schema for graph #1", g1) { implicit ctx: TestContext => registerPgds(ns) val expectedSchema = PropertyGraphSchema.empty .withNodePropertyKeys("A")("name" -> CTString, "date" -> CTDate) - .withNodePropertyKeys("B")("type" -> CTString, "size" -> CTInteger.nullable, "datetime" -> CTLocalDateTime.nullable) - .withNodePropertyKeys("A", "B")("name" -> CTString, "type" -> CTString, "size" -> CTInteger.nullable) + .withNodePropertyKeys("B")( + "type" -> CTString, + "size" -> CTInteger.nullable, + "datetime" -> CTLocalDateTime.nullable + ) + .withNodePropertyKeys("A", "B")( + "name" -> CTString, + "type" -> CTString, + "size" -> CTInteger.nullable + ) .withNodePropertyKeys("C")("name" -> CTString) .withNodePropertyKeys("A", "C")("name" -> CTString) - .withRelationshipPropertyKeys("R")("since" -> CTInteger, "before" -> CTFalse.nullable) + .withRelationshipPropertyKeys("R")( + "since" -> CTInteger, + "before" -> CTFalse.nullable + ) .withRelationshipPropertyKeys("S")("since" -> CTInteger) .withRelationshipPropertyKeys("T")() - val schema = session.catalog.source(ns).schema(g1).getOrElse(session.catalog.source(ns).graph(g1).schema) + val schema = session.catalog + .source(ns) + .schema(g1) + .getOrElse(session.catalog.source(ns).graph(g1).schema) schema.labelPropertyMap should equal(expectedSchema.labelPropertyMap) - schema.relTypePropertyMap should equal(expectedSchema.relTypePropertyMap) + schema.relTypePropertyMap should equal( + expectedSchema.relTypePropertyMap + ) }, - - Scenario("API: PropertyGraphDataSource: correct node/rel count for graph #1", g1) { implicit ctx: TestContext => + Scenario( + "API: PropertyGraphDataSource: correct node/rel count for graph #1", + g1 + ) { implicit ctx: TestContext => registerPgds(ns) session.catalog.source(ns).graph(g1).nodes("n").size shouldBe 7 session.catalog.source(ns).graph(g1).relationships("r").size shouldBe 4 }, - - Scenario("API: PropertyGraphDataSource: correct node/rel count for graph #2", g2) { implicit ctx: TestContext => + Scenario( + "API: PropertyGraphDataSource: correct node/rel count for graph #2", + g2 + ) { implicit ctx: TestContext => registerPgds(ns) session.catalog.source(ns).graph(g2).nodes("n").size shouldBe 1 session.catalog.source(ns).graph(g2).relationships("r").size shouldBe 0 }, - - Scenario("API: PropertyGraphDataSource: correct node/rel count for graph #3", g3) { implicit ctx: TestContext => + Scenario( + "API: PropertyGraphDataSource: correct node/rel count for graph #3", + g3 + ) { implicit ctx: TestContext => registerPgds(ns) session.catalog.source(ns).graph(g3).nodes("n").size shouldBe 1 session.catalog.source(ns).graph(g3).relationships("r").size shouldBe 0 }, - - Scenario("API: PropertyGraphDataSource: unique IDs for graph #5", g5) { implicit ctx: TestContext => - registerPgds(ns) - val result = session.catalog.graph(QualifiedGraphName(ns, g5)).cypher("MATCH (a)-[r]->(b) RETURN DISTINCT id(r)").records - result.size should equal(5) + Scenario("API: PropertyGraphDataSource: unique IDs for graph #5", g5) { + implicit ctx: TestContext => + registerPgds(ns) + val result = session.catalog + .graph(QualifiedGraphName(ns, g5)) + .cypher("MATCH (a)-[r]->(b) RETURN DISTINCT id(r)") + .records + result.size should equal(5) }, - Scenario("API: Cypher query directly on graph #1", g1) { implicit ctx: TestContext => registerPgds(ns) - session.catalog.graph(QualifiedGraphName(ns, g1)).cypher("MATCH (a:A) RETURN a.name").records.iterator.toBag should equal(Bag( - CypherMap("a.name" -> "A"), - CypherMap("a.name" -> "COMBO1"), - CypherMap("a.name" -> "COMBO2"), - CypherMap("a.name" -> "AC") - )) + session.catalog + .graph(QualifiedGraphName(ns, g1)) + .cypher("MATCH (a:A) RETURN a.name") + .records + .iterator + .toBag should equal( + Bag( + CypherMap("a.name" -> "A"), + CypherMap("a.name" -> "COMBO1"), + CypherMap("a.name" -> "COMBO2"), + CypherMap("a.name" -> "AC") + ) + ) }, - Scenario("Cypher query on session", g1) { implicit ctx: TestContext => registerPgds(ns) expectRecordsAnyOrder( @@ -285,7 +332,6 @@ trait PGDSAcceptanceTest[Session <: CypherSession, Graph <: PropertyGraph] { CypherMap("b.type" -> "AB2", "b.size" -> CypherNull) ) }, - Scenario("Scans over multiple labels", g1) { implicit ctx: TestContext => registerPgds(ns) expectRecordsAnyOrder( @@ -299,20 +345,24 @@ trait PGDSAcceptanceTest[Session <: CypherSession, Graph <: PropertyGraph] { CypherMap("n.name" -> CypherNull, "n.size" -> CypherNull) ) }, - Scenario("Multi-hop paths", g1) { implicit ctx: TestContext => registerPgds(ns) expectRecordsAnyOrder( - executeQuery(s"FROM GRAPH $ns.$g1 MATCH (a)-[r1]->(b)-[r2]->(c) RETURN r1.since, r2.since, type(r2)"), + executeQuery( + s"FROM GRAPH $ns.$g1 MATCH (a)-[r1]->(b)-[r2]->(c) RETURN r1.since, r2.since, type(r2)" + ), CypherMap("r1.since" -> 2004, "r2.since" -> 2005, "type(r2)" -> "R"), CypherMap("r1.since" -> 2005, "r2.since" -> 2006, "type(r2)" -> "S") ) }, - Scenario("Initialize with a graph", g1) { implicit ctx: TestContext => registerPgds(ns) val gn = GraphName("storedGraph") - Try(session.cypher(s"CATALOG CREATE GRAPH $ns.$gn { FROM GRAPH $ns.$g1 RETURN GRAPH }")) match { + Try( + session.cypher( + s"CATALOG CREATE GRAPH $ns.$gn { FROM GRAPH $ns.$g1 RETURN GRAPH }" + ) + ) match { case Success(_) => withClue("`hasGraph` needs to return `true` after graph creation") { session.catalog.source(ns).hasGraph(gn) shouldBe true @@ -323,15 +373,13 @@ trait PGDSAcceptanceTest[Session <: CypherSession, Graph <: PropertyGraph] { session.cypher(s"CATALOG CREATE GRAPH $ns.$g1 { RETURN GRAPH }") } case Failure(_: UnsupportedOperationException) => - case Failure(t) => throw t + case Failure(t) => throw t } }, - Scenario("Initialize with a constructed graph", g1) { implicit ctx: TestContext => registerPgds(ns) val gn = GraphName("storedGraph") - Try(session.cypher( - s""" + Try(session.cypher(s""" |CATALOG CREATE GRAPH $ns.$gn { | CONSTRUCT ON $ns.$g1 | CREATE (c:C { name: 'new' }) @@ -339,25 +387,31 @@ trait PGDSAcceptanceTest[Session <: CypherSession, Graph <: PropertyGraph] { |} |""".stripMargin)) match { case Success(_) => - withClue("`hasGraph` needs to return `true` after graph creation") { + withClue( + "`hasGraph` needs to return `true` after graph creation" + ) { session.catalog.source(ns).hasGraph(gn) shouldBe true } - val result = session.cypher(s"FROM GRAPH $ns.$gn MATCH (c:C) RETURN c.name").records.iterator.toBag - result should equal(Bag( - CypherMap("c.name" -> "C"), - CypherMap("c.name" -> "AC"), - CypherMap("c.name" -> "new") - )) + val result = session + .cypher(s"FROM GRAPH $ns.$gn MATCH (c:C) RETURN c.name") + .records + .iterator + .toBag + result should equal( + Bag( + CypherMap("c.name" -> "C"), + CypherMap("c.name" -> "AC"), + CypherMap("c.name" -> "new") + ) + ) case Failure(_: UnsupportedOperationException) => - case Failure(t) => throw t + case Failure(t) => throw t } }, - Scenario("Initialize with nodes without labels") { implicit ctx: TestContext => registerPgds(ns) val gn = GraphName("storedGraph") - Try(session.cypher( - s""" + Try(session.cypher(s""" |CATALOG CREATE GRAPH $ns.$gn { | CONSTRUCT | CREATE ({ no_label_node: true })-[:SOME_REL_TYPE]->(:SOME_LABEL) @@ -365,49 +419,67 @@ trait PGDSAcceptanceTest[Session <: CypherSession, Graph <: PropertyGraph] { |} |""".stripMargin)) match { case Success(_) => - withClue("`hasGraph` needs to return `true` after graph creation") { + withClue( + "`hasGraph` needs to return `true` after graph creation" + ) { session.catalog.source(ns).hasGraph(gn) shouldBe true } - val result = session.cypher( - s"FROM GRAPH $ns.$gn MATCH (d) WHERE size(labels(d))=0 RETURN d.no_label_node").records.iterator.toBag - result should equal(Bag( - CypherMap("d.no_label_node" -> true) - )) + val result = session + .cypher( + s"FROM GRAPH $ns.$gn MATCH (d) WHERE size(labels(d))=0 RETURN d.no_label_node" + ) + .records + .iterator + .toBag + result should equal( + Bag( + CypherMap("d.no_label_node" -> true) + ) + ) case Failure(_: UnsupportedOperationException) => - case Failure(t) => throw t + case Failure(t) => throw t } }, - - Scenario("Store European Latin unicode labels, rel types, property keys, and property values") { implicit + Scenario( + "Store European Latin unicode labels, rel types, property keys, and property values" + ) { + implicit ctx: TestContext => - registerPgds(ns) - val gn = GraphName("storedGraph") - Try(session.cypher( - s""" + registerPgds(ns) + val gn = GraphName("storedGraph") + Try(session.cypher(s""" |CATALOG CREATE GRAPH $ns.$gn { | CONSTRUCT | CREATE (:Āſ { Āſ: 'Āſ' })-[:Āſ]->(:ā) | RETURN GRAPH |} |""".stripMargin)) match { - case Success(_) => - withClue("`hasGraph` needs to return `true` after graph creation") { - session.catalog.source(ns).hasGraph(gn) shouldBe true - } - val result = session.cypher(s"FROM GRAPH $ns.$gn MATCH (c:Āſ)-[:Āſ]-(:ā) RETURN c.Āſ").records.iterator.toBag - result should equal(Bag( - CypherMap("c.Āſ" -> "Āſ") - )) - case Failure(_: UnsupportedOperationException) => - case Failure(t) => throw t - } + case Success(_) => + withClue( + "`hasGraph` needs to return `true` after graph creation" + ) { + session.catalog.source(ns).hasGraph(gn) shouldBe true + } + val result = session + .cypher( + s"FROM GRAPH $ns.$gn MATCH (c:Āſ)-[:Āſ]-(:ā) RETURN c.Āſ" + ) + .records + .iterator + .toBag + result should equal( + Bag( + CypherMap("c.Āſ" -> "Āſ") + ) + ) + case Failure(_: UnsupportedOperationException) => + case Failure(t) => throw t + } }, - Scenario("Property with property key `id`") { implicit ctx: TestContext => registerPgds(ns) val gn = GraphName("storedGraph") - Try(session.cypher( - s""" + Try(session.cypher(s""" |CATALOG CREATE GRAPH $ns.$gn { | CONSTRUCT | CREATE ({ id: 100 }) @@ -416,21 +488,32 @@ trait PGDSAcceptanceTest[Session <: CypherSession, Graph <: PropertyGraph] { |""".stripMargin)) match { case Success(_) => withClue("`hasGraph` needs to return `true` after graph creation") { - session.catalog.source(ns).hasGraph(GraphName(s"$gn")) shouldBe true + session.catalog + .source(ns) + .hasGraph(GraphName(s"$gn")) shouldBe true } - val result = session.cypher(s"FROM GRAPH $ns.$gn MATCH (c) RETURN c.id").records.iterator.toBag - result should equal(Bag( - CypherMap("c.id" -> 100) - )) + val result = session + .cypher(s"FROM GRAPH $ns.$gn MATCH (c) RETURN c.id") + .records + .iterator + .toBag + result should equal( + Bag( + CypherMap("c.id" -> 100) + ) + ) case Failure(_: UnsupportedOperationException) => - case Failure(t) => throw t + case Failure(t) => throw t } }, - Scenario("Store a union graph") { implicit ctx: TestContext => registerPgds(ns) - session.cypher("CATALOG CREATE GRAPH g1 { CONSTRUCT CREATE () RETURN GRAPH }") - session.cypher("CATALOG CREATE GRAPH g2 { CONSTRUCT CREATE () RETURN GRAPH }") + session.cypher( + "CATALOG CREATE GRAPH g1 { CONSTRUCT CREATE () RETURN GRAPH }" + ) + session.cypher( + "CATALOG CREATE GRAPH g2 { CONSTRUCT CREATE () RETURN GRAPH }" + ) val unionGraphName = GraphName("union") val g1 = session.catalog.graph("g1") @@ -446,58 +529,74 @@ trait PGDSAcceptanceTest[Session <: CypherSession, Graph <: PropertyGraph] { session.catalog.source(ns).store(unionGraphName, unionGraph) } match { case Success(_) => - withClue("`graph` needs to return graph with correct node size after storing a union graph") { - session.catalog.source(ns).graph(unionGraphName).nodes("n").size shouldBe 2 + withClue( + "`graph` needs to return graph with correct node size after storing a union graph" + ) { + session.catalog + .source(ns) + .graph(unionGraphName) + .nodes("n") + .size shouldBe 2 } case Failure(_: UnsupportedOperationException) => - case Failure(t) => throw t + case Failure(t) => throw t } }, - Scenario("Repeat CONSTRUCT ON", g1) { implicit ctx: TestContext => registerPgds(ns) val firstConstructedGraphName = GraphName("first") val secondConstructedGraphName = GraphName("second") val graph = session.catalog.source(ns).graph(g1) graph.nodes("n").size shouldBe 7 - val firstConstructedGraph = graph.cypher( - s""" + val firstConstructedGraph = graph + .cypher(s""" |CONSTRUCT | ON $ns.$g1 | CREATE (:A {name: "A"}) | RETURN GRAPH - """.stripMargin).graph + """.stripMargin) + .graph firstConstructedGraph.nodes("n").size shouldBe 8 - val maybeStored = Try(session.catalog.source(ns).store(firstConstructedGraphName, firstConstructedGraph)) + val maybeStored = Try( + session.catalog + .source(ns) + .store(firstConstructedGraphName, firstConstructedGraph) + ) maybeStored match { case Failure(_: UnsupportedOperationException) => - case Failure(t) => throw t + case Failure(t) => throw t case Success(_) => - val retrievedConstructedGraph = session.catalog.source(ns).graph(firstConstructedGraphName) + val retrievedConstructedGraph = + session.catalog.source(ns).graph(firstConstructedGraphName) retrievedConstructedGraph.nodes("n").size shouldBe 8 - val secondConstructedGraph = graph.cypher( - s""" + val secondConstructedGraph = graph + .cypher(s""" |CONSTRUCT | ON $ns.$firstConstructedGraphName | CREATE (:A:B {name: "COMBO", size: 2}) | RETURN GRAPH - """.stripMargin).graph + """.stripMargin) + .graph secondConstructedGraph.nodes("n").size shouldBe 9 - session.catalog.source(ns).store(secondConstructedGraphName, secondConstructedGraph) - val retrievedSecondConstructedGraph = session.catalog.source(ns).graph(secondConstructedGraphName) + session.catalog + .source(ns) + .store(secondConstructedGraphName, secondConstructedGraph) + val retrievedSecondConstructedGraph = + session.catalog.source(ns).graph(secondConstructedGraphName) retrievedSecondConstructedGraph.nodes("n").size shouldBe 9 } }, - Scenario("Drop a graph", g1) { implicit ctx: TestContext => registerPgds(ns) Try(session.cypher(s"CATALOG DROP GRAPH $ns.$g1")) match { case Success(_) => - withClue("`hasGraph` needs to return `false` after graph deletion") { + withClue( + "`hasGraph` needs to return `false` after graph deletion" + ) { session.catalog.source(ns).hasGraph(g1) shouldBe false } case Failure(_: UnsupportedOperationException) => - case Failure(t) => throw t + case Failure(t) => throw t } } ) diff --git a/okapi-testing/src/main/scala/org/opencypher/okapi/testing/TestNameFixture.scala b/okapi-testing/src/main/scala/org/opencypher/okapi/testing/TestNameFixture.scala index 26f5907a81..1b638b769c 100644 --- a/okapi-testing/src/main/scala/org/opencypher/okapi/testing/TestNameFixture.scala +++ b/okapi-testing/src/main/scala/org/opencypher/okapi/testing/TestNameFixture.scala @@ -45,7 +45,6 @@ trait TestNameFixture extends BaseTestSuite { * * returns 'testName'. * - * * @return */ protected def separator: String @@ -59,7 +58,8 @@ trait TestNameFixture extends BaseTestSuite { val name = separatorIndex match { case -1 => testName - case _ => testName.substring(separatorIndex + separator.length).trim.stripMargin + case _ => + testName.substring(separatorIndex + separator.length).trim.stripMargin } __testName = Some(name) try { diff --git a/okapi-testing/src/main/scala/org/opencypher/okapi/testing/propertygraph/CreateQueryParser.scala b/okapi-testing/src/main/scala/org/opencypher/okapi/testing/propertygraph/CreateQueryParser.scala index 557d90a4e6..2616af4ae6 100644 --- a/okapi-testing/src/main/scala/org/opencypher/okapi/testing/propertygraph/CreateQueryParser.scala +++ b/okapi-testing/src/main/scala/org/opencypher/okapi/testing/propertygraph/CreateQueryParser.scala @@ -33,7 +33,13 @@ import cats.data.State import cats.data.State._ import cats.instances.list._ import cats.syntax.all._ -import org.opencypher.okapi.api.value.CypherValue.{CypherBigDecimal, Element, CypherMap, Node, Relationship} +import org.opencypher.okapi.api.value.CypherValue.{ + CypherBigDecimal, + Element, + CypherMap, + Node, + Relationship +} import org.opencypher.okapi.impl.exception.{IllegalArgumentException, UnsupportedOperationException} import org.opencypher.okapi.impl.temporal.TemporalTypesHelper.parseDate import org.opencypher.okapi.impl.temporal.{Duration, TemporalTypesHelper} @@ -52,14 +58,16 @@ import scala.reflect.ClassTag object CreateQueryParser { val defaultContext: BaseContext = new BaseContext { - override def tracer: CompilationPhaseTracer = CompilationPhaseTracer.NO_TRACING + override def tracer: CompilationPhaseTracer = + CompilationPhaseTracer.NO_TRACING override def notificationLogger: InternalNotificationLogger = devNullLogger - override def exceptionCreator: (String, InputPosition) => CypherException = (_, _) => null + override def exceptionCreator: (String, InputPosition) => CypherException = + (_, _) => null override def monitors: Monitors = new Monitors { - override def newMonitor[T <: AnyRef : ClassTag](tags: String*): T = { + override def newMonitor[T <: AnyRef: ClassTag](tags: String*): T = { new AstRewritingMonitor { override def abortedRewriting(obj: AnyRef): Unit = () @@ -86,7 +94,11 @@ object CreateQueryParser { SyntaxDeprecationWarnings(V2) andThen PreparatoryRewriting(V2) andThen SemanticAnalysis(warn = true).adds(BaseContains[SemanticState]) andThen - AstRewriting(RewriterStepSequencer.newPlain, Never, getDegreeRewriting = false) andThen + AstRewriting( + RewriterStepSequencer.newPlain, + Never, + getDegreeRewriting = false + ) andThen SemanticAnalysis(warn = false) andThen Namespacer andThen CNFNormalizer andThen @@ -98,12 +110,16 @@ object CreateGraphFactory extends InMemoryGraphFactory { type Result[A] = State[ParsingContext, A] - def apply(createQuery: String, externalParams: Map[String, Any] = Map.empty): InMemoryTestGraph = { + def apply( + createQuery: String, + externalParams: Map[String, Any] = Map.empty + ): InMemoryTestGraph = { val (ast, params, _) = CreateQueryParser.process(createQuery) val context = ParsingContext.fromParams(params ++ externalParams) ast match { - case Query(_, SingleQuery(clauses)) => processClauses(clauses).runS(context).value.graph + case Query(_, SingleQuery(clauses)) => + processClauses(clauses).runS(context).value.graph } } @@ -131,7 +147,10 @@ object CreateGraphFactory extends InMemoryGraphFactory { _ <- modify[ParsingContext](_.popProtectedScope) } yield () - case other => throw UnsupportedOperationException(s"Processing clause: ${other.name}") + case other => + throw UnsupportedOperationException( + s"Processing clause: ${other.name}" + ) } case _ => pure[ParsingContext, Unit](()) } @@ -140,26 +159,47 @@ object CreateGraphFactory extends InMemoryGraphFactory { def processPattern(pattern: Pattern, merge: Boolean): Result[Unit] = { val parts = pattern.patternParts.map { case EveryPath(element) => element - case other => throw UnsupportedOperationException(s"Processing pattern: ${other.getClass.getSimpleName}") + case other => + throw UnsupportedOperationException( + s"Processing pattern: ${other.getClass.getSimpleName}" + ) } - Foldable[List].sequence_[Result, Element[Long]](parts.toList.map(pe => processPatternElement(pe, merge))) + Foldable[List].sequence_[Result, Element[Long]]( + parts.toList.map(pe => processPatternElement(pe, merge)) + ) } - def processPatternElement(patternElement: ASTNode, merge: Boolean): Result[Element[Long]] = { + def processPatternElement( + patternElement: ASTNode, + merge: Boolean + ): Result[Element[Long]] = { patternElement match { case NodePattern(Some(variable), labels, props, _) => for { properties <- props match { case Some(expr: MapExpression) => extractProperties(expr) - case Some(other) => throw IllegalArgumentException("a NodePattern with MapExpression", other) + case Some(other) => + throw IllegalArgumentException( + "a NodePattern with MapExpression", + other + ) case None => pure[ParsingContext, CypherMap](CypherMap.empty) } node <- inspect[ParsingContext, InMemoryTestNode] { context => context.variableMapping.get(variable.name) match { case Some(n: InMemoryTestNode) => n - case Some(other) => throw IllegalArgumentException(s"a Node for variable ${variable.name}", other) - case None => InMemoryTestNode(context.nextId, labels.map(_.name).toSet, properties) + case Some(other) => + throw IllegalArgumentException( + s"a Node for variable ${variable.name}", + other + ) + case None => + InMemoryTestNode( + context.nextId, + labels.map(_.name).toSet, + properties + ) } } _ <- modify[ParsingContext] { context => @@ -171,25 +211,57 @@ object CreateGraphFactory extends InMemoryGraphFactory { } } yield node - case RelationshipChain(first, RelationshipPattern(Some(variable), relType, None, props, direction, _, _), third) => + case RelationshipChain( + first, + RelationshipPattern( + Some(variable), + relType, + None, + props, + direction, + _, + _ + ), + third + ) => for { source <- processPatternElement(first, merge) sourceId <- pure[ParsingContext, Long](source match { - case n: Node[Long] => n.id + case n: Node[Long] => n.id case r: Relationship[Long] => r.endId }) target <- processPatternElement(third, merge) properties <- props match { case Some(expr: MapExpression) => extractProperties(expr) - case Some(other) => throw IllegalArgumentException("a RelationshipChain with MapExpression", other) + case Some(other) => + throw IllegalArgumentException( + "a RelationshipChain with MapExpression", + other + ) case None => pure[ParsingContext, CypherMap](CypherMap.empty) } rel <- inspect[ParsingContext, InMemoryTestRelationship] { context => if (direction == SemanticDirection.OUTGOING) - InMemoryTestRelationship(context.nextId, sourceId, target.id, relType.head.name, properties) + InMemoryTestRelationship( + context.nextId, + sourceId, + target.id, + relType.head.name, + properties + ) else if (direction == SemanticDirection.INCOMING) - InMemoryTestRelationship(context.nextId, target.id, sourceId, relType.head.name, properties) - else throw IllegalArgumentException("a directed relationship", direction) + InMemoryTestRelationship( + context.nextId, + target.id, + sourceId, + relType.head.name, + properties + ) + else + throw IllegalArgumentException( + "a directed relationship", + direction + ) } _ <- modify[ParsingContext](_.updated(variable.name, rel)) @@ -200,8 +272,8 @@ object CreateGraphFactory extends InMemoryGraphFactory { def extractProperties(expr: MapExpression): Result[CypherMap] = { for { keys <- pure(expr.items.map(_._1.name)) - values <- expr.items.toList.traverse[Result, Any] { - case (_, inner) => processExpr(inner) + values <- expr.items.toList.traverse[Result, Any] { case (_, inner) => + processExpr(inner) } res <- pure(CypherMap(keys.zip(values): _*)) } yield res @@ -210,55 +282,102 @@ object CreateGraphFactory extends InMemoryGraphFactory { def processExpr(expr: Expression): Result[Any] = { for { res <- expr match { - case Parameter(name, _) => inspect[ParsingContext, Any](_.parameter(name)) + case Parameter(name, _) => + inspect[ParsingContext, Any](_.parameter(name)) - case Variable(name) => inspect[ParsingContext, Any](_.variableMapping(name)) + case Variable(name) => + inspect[ParsingContext, Any](_.variableMapping(name)) case l: Literal => pure[ParsingContext, Any](l.value) - case ListLiteral(expressions) => expressions.toList.traverse[Result, Any](processExpr) + case ListLiteral(expressions) => + expressions.toList.traverse[Result, Any](processExpr) case Modulo(lhs, rhs) => for { leftVal <- processExpr(lhs) rightVal <- processExpr(rhs) - res <- pure[ParsingContext, Any](leftVal.asInstanceOf[Long] % rightVal.asInstanceOf[Long]) + res <- pure[ParsingContext, Any]( + leftVal.asInstanceOf[Long] % rightVal.asInstanceOf[Long] + ) } yield res case MapExpression(items) => for { keys <- pure(items.map { case (k, _) => k.name }) - valueTypes <- items.toList.traverse[Result, Any] { case (_, v) => processExpr(v) } + valueTypes <- items.toList.traverse[Result, Any] { case (_, v) => + processExpr(v) + } res <- pure[ParsingContext, Any](keys.zip(valueTypes).toMap) } yield res - case FunctionInvocation(_, FunctionName("date"), _, Seq(dateString: StringLiteral)) => + case FunctionInvocation( + _, + FunctionName("date"), + _, + Seq(dateString: StringLiteral) + ) => pure[ParsingContext, Any](parseDate(Right(dateString.value))) - case FunctionInvocation(_, FunctionName("date"), _, Seq(map: MapExpression)) => + case FunctionInvocation( + _, + FunctionName("date"), + _, + Seq(map: MapExpression) + ) => for { dateMap <- processExpr(map) - res <- pure[ParsingContext, Any](parseDate(Left(dateMap.asInstanceOf[Map[String, Long]].mapValues(_.toInt)))) + res <- pure[ParsingContext, Any]( + parseDate( + Left(dateMap.asInstanceOf[Map[String, Long]].mapValues(_.toInt)) + ) + ) } yield res - case FunctionInvocation(_, FunctionName("localdatetime"), _, Seq(dateString: StringLiteral)) => - pure[ParsingContext, Any](TemporalTypesHelper.parseLocalDateTime(Right(dateString.value))) - - case FunctionInvocation(_, FunctionName("duration"), _, Seq(dateString: StringLiteral)) => + case FunctionInvocation( + _, + FunctionName("localdatetime"), + _, + Seq(dateString: StringLiteral) + ) => + pure[ParsingContext, Any]( + TemporalTypesHelper.parseLocalDateTime(Right(dateString.value)) + ) + + case FunctionInvocation( + _, + FunctionName("duration"), + _, + Seq(dateString: StringLiteral) + ) => pure[ParsingContext, Any](Duration.parse(dateString.value)) - case FunctionInvocation(_, FunctionName("duration"), _, Seq(map: MapExpression)) => + case FunctionInvocation( + _, + FunctionName("duration"), + _, + Seq(map: MapExpression) + ) => for { durationMap <- processExpr(map) - res <- pure[ParsingContext, Any](Duration(durationMap.asInstanceOf[Map[String, Long]])) + res <- pure[ParsingContext, Any]( + Duration(durationMap.asInstanceOf[Map[String, Long]]) + ) } yield res - case FunctionInvocation(_, FunctionName("bigdecimal"), _, Seq(n: Literal, p: NumberLiteral, s: NumberLiteral)) => + case FunctionInvocation( + _, + FunctionName("bigdecimal"), + _, + Seq(n: Literal, p: NumberLiteral, s: NumberLiteral) + ) => for { number <- processExpr(n) precision <- pure[ParsingContext, Int](p.stringVal.toInt) scale <- pure[ParsingContext, Int](s.stringVal.toInt) - res <- pure[ParsingContext, Any](CypherBigDecimal(number, precision, scale)) + res <- pure[ParsingContext, Any]( + CypherBigDecimal(number, precision, scale) + ) } yield res case Property(variable: Variable, propertyKey) => @@ -266,35 +385,58 @@ object CreateGraphFactory extends InMemoryGraphFactory { context.variableMapping(variable.name) match { case a: Element[_] => a.properties(propertyKey.name) case other => - throw UnsupportedOperationException(s"Reading property from a ${other.getClass.getSimpleName}") + throw UnsupportedOperationException( + s"Reading property from a ${other.getClass.getSimpleName}" + ) } }) case other => - throw UnsupportedOperationException(s"Processing expression of type ${other.getClass.getSimpleName}") + throw UnsupportedOperationException( + s"Processing expression of type ${other.getClass.getSimpleName}" + ) } } yield res } def processValues(expr: Expression): Result[List[Any]] = { expr match { - case ListLiteral(expressions) => expressions.toList.traverse[Result, Any](processExpr) + case ListLiteral(expressions) => + expressions.toList.traverse[Result, Any](processExpr) case Variable(name) => inspect[ParsingContext, List[Any]](_.variableMapping(name) match { case l: TraversableOnce[Any] => l.toList - case other => throw IllegalArgumentException(s"a list value for variable $name", other) + case other => + throw IllegalArgumentException( + s"a list value for variable $name", + other + ) }) case Parameter(name, _) => inspect[ParsingContext, List[Any]](_.parameter(name) match { case l: TraversableOnce[Any] => l.toList - case other => throw IllegalArgumentException(s"a list value for parameter $name", other) + case other => + throw IllegalArgumentException( + s"a list value for parameter $name", + other + ) }) - case FunctionInvocation(_, FunctionName("range"), _, Seq(lb: IntegerLiteral, ub: IntegerLiteral)) => - pure[ParsingContext, List[Any]](List.range[Long](lb.value, ub.value + 1)) - - case other => throw UnsupportedOperationException(s"Processing value of type ${other.getClass.getSimpleName}") + case FunctionInvocation( + _, + FunctionName("range"), + _, + Seq(lb: IntegerLiteral, ub: IntegerLiteral) + ) => + pure[ParsingContext, List[Any]]( + List.range[Long](lb.value, ub.value + 1) + ) + + case other => + throw UnsupportedOperationException( + s"Processing value of type ${other.getClass.getSimpleName}" + ) } } } @@ -304,7 +446,8 @@ final case class ParsingContext( variableMapping: Map[String, Any], graph: InMemoryTestGraph, protectedScopes: List[Map[String, Any]], - idGenerator: AtomicLong) { + idGenerator: AtomicLong +) { def nextId: Long = idGenerator.getAndIncrement() @@ -316,18 +459,26 @@ final case class ParsingContext( copy(variableMapping = protectedScopes.head) } - def popProtectedScope: ParsingContext = copy(protectedScopes = protectedScopes.tail) - - def updated(k: String, v: Any, merge: Boolean = false): ParsingContext = v match { - case n: InMemoryTestNode if !merge || !containsNode(n) => - copy(graph = graph.updated(n), variableMapping = variableMapping.updated(k, n)) - - case r: InMemoryTestRelationship if !merge || !containsRel(r) => - copy(graph = graph.updated(r), variableMapping = variableMapping.updated(k, r)) - - case _ => - copy(variableMapping = variableMapping.updated(k, v)) - } + def popProtectedScope: ParsingContext = + copy(protectedScopes = protectedScopes.tail) + + def updated(k: String, v: Any, merge: Boolean = false): ParsingContext = + v match { + case n: InMemoryTestNode if !merge || !containsNode(n) => + copy( + graph = graph.updated(n), + variableMapping = variableMapping.updated(k, n) + ) + + case r: InMemoryTestRelationship if !merge || !containsRel(r) => + copy( + graph = graph.updated(r), + variableMapping = variableMapping.updated(k, r) + ) + + case _ => + copy(variableMapping = variableMapping.updated(k, v)) + } private def containsNode(n: InMemoryTestNode): Boolean = graph.nodes.exists(n.equalsSemantically) @@ -338,5 +489,11 @@ final case class ParsingContext( object ParsingContext { def fromParams(params: Map[String, Any]): ParsingContext = - ParsingContext(params, Map.empty, InMemoryTestGraph.empty, List.empty, new AtomicLong()) + ParsingContext( + params, + Map.empty, + InMemoryTestGraph.empty, + List.empty, + new AtomicLong() + ) } diff --git a/okapi-testing/src/main/scala/org/opencypher/okapi/testing/propertygraph/CypherTestGraphFactory.scala b/okapi-testing/src/main/scala/org/opencypher/okapi/testing/propertygraph/CypherTestGraphFactory.scala index 73587288e7..a73ba350b2 100644 --- a/okapi-testing/src/main/scala/org/opencypher/okapi/testing/propertygraph/CypherTestGraphFactory.scala +++ b/okapi-testing/src/main/scala/org/opencypher/okapi/testing/propertygraph/CypherTestGraphFactory.scala @@ -32,7 +32,10 @@ import org.opencypher.okapi.api.types.CypherType._ trait CypherTestGraphFactory[C <: CypherSession] { - def apply(propertyGraph: InMemoryTestGraph, additionalPattern: Seq[Pattern] = Seq.empty)(implicit session: C): PropertyGraph + def apply( + propertyGraph: InMemoryTestGraph, + additionalPattern: Seq[Pattern] = Seq.empty + )(implicit session: C): PropertyGraph def name: String @@ -40,13 +43,13 @@ trait CypherTestGraphFactory[C <: CypherSession] { def computeSchema(propertyGraph: InMemoryTestGraph): PropertyGraphSchema = { def extractFromNode(n: InMemoryTestNode) = - n.labels -> n.properties.value.map { - case (name, prop) => name -> prop.cypherType + n.labels -> n.properties.value.map { case (name, prop) => + name -> prop.cypherType } def extractFromRel(r: InMemoryTestRelationship) = - r.relType -> r.properties.value.map { - case (name, prop) => name -> prop.cypherType + r.relType -> r.properties.value.map { case (name, prop) => + name -> prop.cypherType } val labelsAndProps = propertyGraph.nodes.map(extractFromNode) @@ -56,8 +59,8 @@ trait CypherTestGraphFactory[C <: CypherSession] { case (acc, (labels, props)) => acc.withNodePropertyKeys(labels, props) } - typesAndProps.foldLeft(schemaWithLabels) { - case (acc, (t, props)) => acc.withRelationshipPropertyKeys(t)(props.toSeq: _*) + typesAndProps.foldLeft(schemaWithLabels) { case (acc, (t, props)) => + acc.withRelationshipPropertyKeys(t)(props.toSeq: _*) } } } diff --git a/okapi-testing/src/main/scala/org/opencypher/okapi/testing/propertygraph/InMemoryTestGraph.scala b/okapi-testing/src/main/scala/org/opencypher/okapi/testing/propertygraph/InMemoryTestGraph.scala index 1fb66e00a4..57d05b3cab 100644 --- a/okapi-testing/src/main/scala/org/opencypher/okapi/testing/propertygraph/InMemoryTestGraph.scala +++ b/okapi-testing/src/main/scala/org/opencypher/okapi/testing/propertygraph/InMemoryTestGraph.scala @@ -34,21 +34,26 @@ trait InMemoryGraph { def getNodeById(id: Long): Option[InMemoryTestNode] = { nodes.collectFirst { - case n : InMemoryTestNode if n.id == id => n + case n: InMemoryTestNode if n.id == id => n } } def getRelationshipById(id: Long): Option[InMemoryTestRelationship] = { relationships.collectFirst { - case r : InMemoryTestRelationship if r.id == id => r + case r: InMemoryTestRelationship if r.id == id => r } } } -case class InMemoryTestGraph(nodes: Seq[InMemoryTestNode], relationships: Seq[InMemoryTestRelationship]) extends InMemoryGraph { - def updated(node: InMemoryTestNode): InMemoryTestGraph = copy(nodes = node +: nodes) +case class InMemoryTestGraph( + nodes: Seq[InMemoryTestNode], + relationships: Seq[InMemoryTestRelationship] +) extends InMemoryGraph { + def updated(node: InMemoryTestNode): InMemoryTestGraph = + copy(nodes = node +: nodes) - def updated(rel: InMemoryTestRelationship): InMemoryTestGraph = copy(relationships = rel +: relationships) + def updated(rel: InMemoryTestRelationship): InMemoryTestGraph = + copy(relationships = rel +: relationships) } object InMemoryTestGraph { @@ -63,7 +68,11 @@ case class InMemoryTestNode( type I = InMemoryTestNode - override def copy(id: Long = id, labels: Set[String] = labels, properties: CypherMap = properties): InMemoryTestNode.this.type = { + override def copy( + id: Long = id, + labels: Set[String] = labels, + properties: CypherMap = properties + ): InMemoryTestNode.this.type = { InMemoryTestNode(id, labels, properties).asInstanceOf[this.type] } @@ -82,8 +91,15 @@ case class InMemoryTestRelationship( type I = InMemoryTestRelationship - override def copy(id: Long = id, source: Long = startId, target: Long = endId, relType: String = relType, properties: CypherMap = properties): InMemoryTestRelationship.this.type = { - InMemoryTestRelationship(id, source, target, relType, properties).asInstanceOf[this.type] + override def copy( + id: Long = id, + source: Long = startId, + target: Long = endId, + relType: String = relType, + properties: CypherMap = properties + ): InMemoryTestRelationship.this.type = { + InMemoryTestRelationship(id, source, target, relType, properties) + .asInstanceOf[this.type] } def equalsSemantically(other: I): Boolean = { @@ -93,5 +109,8 @@ case class InMemoryTestRelationship( } trait InMemoryGraphFactory { - def apply(createQuery: String, parameters: Map[String, Any]): InMemoryTestGraph + def apply( + createQuery: String, + parameters: Map[String, Any] + ): InMemoryTestGraph } diff --git a/okapi-testing/src/main/scala/org/opencypher/okapi/testing/support/TreeVerificationSupport.scala b/okapi-testing/src/main/scala/org/opencypher/okapi/testing/support/TreeVerificationSupport.scala index fd2bb7ef6a..2c0064a990 100644 --- a/okapi-testing/src/main/scala/org/opencypher/okapi/testing/support/TreeVerificationSupport.scala +++ b/okapi-testing/src/main/scala/org/opencypher/okapi/testing/support/TreeVerificationSupport.scala @@ -1,29 +1,26 @@ /** - * Copyright (c) 2016-2019 "Neo4j Sweden, AB" [https://neo4j.com] - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * Attribution Notice under the terms of the Apache License 2.0 - * - * This work was created by the collective efforts of the openCypher community. - * Without limiting the terms of Section 6, any Derivative Work that is not - * approved by the public consensus process of the openCypher Implementers Group - * should not be described as “Cypher” (and Cypher® is a registered trademark of - * Neo4j Inc.) or as "openCypher". Extensions by implementers or prototypes or - * proposals for change that have been documented or implemented should only be - * described as "implementation extensions to Cypher" or as "proposed changes to - * Cypher that are not yet approved by the openCypher community". - */ + * Copyright (c) 2016-2019 "Neo4j Sweden, AB" [https://neo4j.com] + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the + * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing permissions and + * limitations under the License. + * + * Attribution Notice under the terms of the Apache License 2.0 + * + * This work was created by the collective efforts of the openCypher community. Without limiting + * the terms of Section 6, any Derivative Work that is not approved by the public consensus process + * of the openCypher Implementers Group should not be described as “Cypher” (and Cypher® is a + * registered trademark of Neo4j Inc.) or as "openCypher". Extensions by implementers or prototypes + * or proposals for change that have been documented or implemented should only be described as + * "implementation extensions to Cypher" or as "proposed changes to Cypher that are not yet + * approved by the openCypher community". + */ package org.opencypher.okapi.testing.support import org.opencypher.okapi.testing.BaseTestSuite @@ -35,10 +32,10 @@ trait TreeVerificationSupport { self: BaseTestSuite => implicit class TreeNodeTestsUtils[T <: TreeNode[T]](op: T) { - def occourences[B <: TreeNode[T] : ClassTag]: Int = { + def occourences[B <: TreeNode[T]: ClassTag]: Int = { op.transform[Int] { case (_: B, list) => list.sum + 1 - case (_, list) => list.sum + case (_, list) => list.sum } } } diff --git a/okapi-testing/src/test/scala/org/opencypher/okapi/testing/BagTest.scala b/okapi-testing/src/test/scala/org/opencypher/okapi/testing/BagTest.scala index b9217def48..77ee89ac43 100644 --- a/okapi-testing/src/test/scala/org/opencypher/okapi/testing/BagTest.scala +++ b/okapi-testing/src/test/scala/org/opencypher/okapi/testing/BagTest.scala @@ -36,12 +36,14 @@ class BagTest extends BaseTestSuite { CypherMap("p1" -> "a", "p2" -> "b"), CypherMap("p1" -> "b", "p2" -> "a"), CypherMap("p1" -> "b", "p2" -> "b") - ) should equal(Bag( - CypherMap("p2" -> "a", "p1" -> "a"), - CypherMap("p2" -> "b", "p1" -> "a"), - CypherMap("p2" -> "a", "p1" -> "b"), - CypherMap("p2" -> "b", "p1" -> "b") - )) + ) should equal( + Bag( + CypherMap("p2" -> "a", "p1" -> "a"), + CypherMap("p2" -> "b", "p1" -> "a"), + CypherMap("p2" -> "a", "p1" -> "b"), + CypherMap("p2" -> "b", "p1" -> "b") + ) + ) } } diff --git a/okapi-testing/src/test/scala/org/opencypher/okapi/testing/propertygraph/CreateQueryFactoryTest.scala b/okapi-testing/src/test/scala/org/opencypher/okapi/testing/propertygraph/CreateQueryFactoryTest.scala index 9bf359e736..ca5cf71af6 100644 --- a/okapi-testing/src/test/scala/org/opencypher/okapi/testing/propertygraph/CreateQueryFactoryTest.scala +++ b/okapi-testing/src/test/scala/org/opencypher/okapi/testing/propertygraph/CreateQueryFactoryTest.scala @@ -33,127 +33,139 @@ import org.opencypher.okapi.testing.{Bag, BaseTestSuite} class CreateQueryFactoryTest extends BaseTestSuite { test("parse single node create statement") { - val graph = CreateGraphFactory( - """ + val graph = CreateGraphFactory(""" |CREATE (a:Person {name: "Alice"}) """.stripMargin) - graph.nodes should equal(Seq( - InMemoryTestNode(0, Set("Person"), CypherMap("name" -> "Alice")) - )) + graph.nodes should equal( + Seq( + InMemoryTestNode(0, Set("Person"), CypherMap("name" -> "Alice")) + ) + ) graph.relationships should be(Seq.empty) } test("parse multiple nodes in single create statement") { - val graph: InMemoryTestGraph = CreateGraphFactory( - """ + val graph: InMemoryTestGraph = CreateGraphFactory(""" |CREATE (a:Person {name: "Alice"}), (b:Person {name: "Bob"}) """.stripMargin) - graph.nodes.toBag should equal(Bag( - InMemoryTestNode(0, Set("Person"), CypherMap("name" -> "Alice")), - InMemoryTestNode(1, Set("Person"), CypherMap("name" -> "Bob")) - )) + graph.nodes.toBag should equal( + Bag( + InMemoryTestNode(0, Set("Person"), CypherMap("name" -> "Alice")), + InMemoryTestNode(1, Set("Person"), CypherMap("name" -> "Bob")) + ) + ) graph.relationships should be(Seq.empty) } test("parse multiple nodes in separate create statements") { - val graph = CreateGraphFactory( - """ + val graph = CreateGraphFactory(""" |CREATE (a:Person {name: "Alice"}) |CREATE (b:Person {name: "Bob"}) """.stripMargin) - graph.nodes.toBag should equal(Bag( - InMemoryTestNode(0, Set("Person"), CypherMap("name" -> "Alice")), - InMemoryTestNode(1, Set("Person"), CypherMap("name" -> "Bob")) - )) + graph.nodes.toBag should equal( + Bag( + InMemoryTestNode(0, Set("Person"), CypherMap("name" -> "Alice")), + InMemoryTestNode(1, Set("Person"), CypherMap("name" -> "Bob")) + ) + ) graph.relationships should be(Seq.empty) } test("parse multiple nodes connected by relationship") { - val graph = CreateGraphFactory( - """ + val graph = CreateGraphFactory(""" |CREATE (a:Person {name: "Alice"})-[:KNOWS {since: 42}]->(b:Person {name: "Bob"}) """.stripMargin) - graph.nodes.toBag should equal(Bag( - InMemoryTestNode(0, Set("Person"), CypherMap("name" -> "Alice")), - InMemoryTestNode(1, Set("Person"), CypherMap("name" -> "Bob")) - )) - - graph.relationships.toBag should be(Bag( - InMemoryTestRelationship(2, 0, 1, "KNOWS", CypherMap("since" -> 42)) - )) + graph.nodes.toBag should equal( + Bag( + InMemoryTestNode(0, Set("Person"), CypherMap("name" -> "Alice")), + InMemoryTestNode(1, Set("Person"), CypherMap("name" -> "Bob")) + ) + ) + + graph.relationships.toBag should be( + Bag( + InMemoryTestRelationship(2, 0, 1, "KNOWS", CypherMap("since" -> 42)) + ) + ) } test("parse multiple nodes and relationship in separate create statements") { - val graph = CreateGraphFactory( - """ + val graph = CreateGraphFactory(""" |CREATE (a:Person {name: "Alice"}) |CREATE (b:Person {name: "Bob"}) |CREATE (a)-[:KNOWS {since: 42}]->(b) """.stripMargin) - graph.nodes.toBag should equal(Bag( - InMemoryTestNode(0, Set("Person"), CypherMap("name" -> "Alice")), - InMemoryTestNode(1, Set("Person"), CypherMap("name" -> "Bob")) - )) - - graph.relationships.toBag should be(Bag( - InMemoryTestRelationship(2, 0, 1, "KNOWS", CypherMap("since" -> 42)) - )) + graph.nodes.toBag should equal( + Bag( + InMemoryTestNode(0, Set("Person"), CypherMap("name" -> "Alice")), + InMemoryTestNode(1, Set("Person"), CypherMap("name" -> "Bob")) + ) + ) + + graph.relationships.toBag should be( + Bag( + InMemoryTestRelationship(2, 0, 1, "KNOWS", CypherMap("since" -> 42)) + ) + ) } test("simple unwind") { - val graph = CreateGraphFactory( - """ + val graph = CreateGraphFactory(""" |UNWIND [1,2,3] as i |CREATE (a {val: i}) """.stripMargin) - graph.nodes.toBag should equal(Bag( - InMemoryTestNode(0, Set(), CypherMap("val" -> 1)), - InMemoryTestNode(1, Set(), CypherMap("val" -> 2)), - InMemoryTestNode(2, Set(), CypherMap("val" -> 3)) - )) + graph.nodes.toBag should equal( + Bag( + InMemoryTestNode(0, Set(), CypherMap("val" -> 1)), + InMemoryTestNode(1, Set(), CypherMap("val" -> 2)), + InMemoryTestNode(2, Set(), CypherMap("val" -> 3)) + ) + ) graph.relationships.toBag shouldBe empty } test("stacked unwind") { - val graph = CreateGraphFactory( - """ + val graph = CreateGraphFactory(""" |UNWIND [1,2,3] AS i |UNWIND [4] AS j |CREATE (a {val1: i, val2: j}) """.stripMargin) - graph.nodes.toBag should equal(Bag( - InMemoryTestNode(0, Set(), CypherMap("val1" -> 1, "val2" -> 4)), - InMemoryTestNode(1, Set(), CypherMap("val1" -> 2, "val2" -> 4)), - InMemoryTestNode(2, Set(), CypherMap("val1" -> 3, "val2" -> 4)) - )) + graph.nodes.toBag should equal( + Bag( + InMemoryTestNode(0, Set(), CypherMap("val1" -> 1, "val2" -> 4)), + InMemoryTestNode(1, Set(), CypherMap("val1" -> 2, "val2" -> 4)), + InMemoryTestNode(2, Set(), CypherMap("val1" -> 3, "val2" -> 4)) + ) + ) graph.relationships.toBag shouldBe empty } test("unwind with variable reference") { - val graph = CreateGraphFactory( - """ + val graph = CreateGraphFactory(""" |UNWIND [[1,2,3]] AS i |UNWIND i AS j |CREATE (a {val: j}) """.stripMargin) - graph.nodes.toBag should equal(Bag( - InMemoryTestNode(0, Set(), CypherMap("val" -> 1)), - InMemoryTestNode(1, Set(), CypherMap("val" -> 2)), - InMemoryTestNode(2, Set(), CypherMap("val" -> 3)) - )) + graph.nodes.toBag should equal( + Bag( + InMemoryTestNode(0, Set(), CypherMap("val" -> 1)), + InMemoryTestNode(1, Set(), CypherMap("val" -> 2)), + InMemoryTestNode(2, Set(), CypherMap("val" -> 3)) + ) + ) graph.relationships.toBag shouldBe empty } @@ -163,47 +175,69 @@ class CreateQueryFactoryTest extends BaseTestSuite { """ |UNWIND $i AS j |CREATE (a {val: j}) - """.stripMargin, Map("i" -> List(1, 2, 3))) - - graph.nodes.toBag should equal(Bag( - InMemoryTestNode(0, Set(), CypherMap("val" -> 1)), - InMemoryTestNode(1, Set(), CypherMap("val" -> 2)), - InMemoryTestNode(2, Set(), CypherMap("val" -> 3)) - )) + """.stripMargin, + Map("i" -> List(1, 2, 3)) + ) + + graph.nodes.toBag should equal( + Bag( + InMemoryTestNode(0, Set(), CypherMap("val" -> 1)), + InMemoryTestNode(1, Set(), CypherMap("val" -> 2)), + InMemoryTestNode(2, Set(), CypherMap("val" -> 3)) + ) + ) graph.relationships.toBag shouldBe empty } test("create statement with property reference") { - val graph = CreateGraphFactory( - """ + val graph = CreateGraphFactory(""" |CREATE (a:Person {name: "Alice"}) |CREATE (b:Person {name: a.name}) """.stripMargin) - graph.nodes.toBag should equal(Bag( - InMemoryTestNode(0, Set("Person"), CypherMap("name" -> "Alice")), - InMemoryTestNode(1, Set("Person"), CypherMap("name" -> "Alice")) - )) + graph.nodes.toBag should equal( + Bag( + InMemoryTestNode(0, Set("Person"), CypherMap("name" -> "Alice")), + InMemoryTestNode(1, Set("Person"), CypherMap("name" -> "Alice")) + ) + ) graph.relationships should be(Seq.empty) } it("can create nodes with big decimal property") { - val graph = CreateGraphFactory( - """ + val graph = CreateGraphFactory(""" |CREATE ({val: bigdecimal("42", 2, 0)}) |CREATE ({val: bigdecimal(42, 2, 0)}) |CREATE ({val: bigdecimal("42.1", 3, 1)}) |CREATE ({val: bigdecimal(42.1, 3, 1)}) """.stripMargin) - graph.nodes.toBag should equal(Bag( - InMemoryTestNode(0, Set.empty, CypherMap("val" -> CypherBigDecimal("42", 2, 0))), - InMemoryTestNode(1, Set.empty, CypherMap("val" -> CypherBigDecimal("42", 2, 0))), - InMemoryTestNode(2, Set.empty, CypherMap("val" -> CypherBigDecimal("42.1", 3, 1))), - InMemoryTestNode(3, Set.empty, CypherMap("val" -> CypherBigDecimal("42.1", 3, 1))) - )) + graph.nodes.toBag should equal( + Bag( + InMemoryTestNode( + 0, + Set.empty, + CypherMap("val" -> CypherBigDecimal("42", 2, 0)) + ), + InMemoryTestNode( + 1, + Set.empty, + CypherMap("val" -> CypherBigDecimal("42", 2, 0)) + ), + InMemoryTestNode( + 2, + Set.empty, + CypherMap("val" -> CypherBigDecimal("42.1", 3, 1)) + ), + InMemoryTestNode( + 3, + Set.empty, + CypherMap("val" -> CypherBigDecimal("42.1", 3, 1)) + ) + ) + ) graph.relationships should be(Seq.empty) } diff --git a/okapi-trees/src/main/scala/org/opencypher/okapi/trees/AbstractTreeNode.scala b/okapi-trees/src/main/scala/org/opencypher/okapi/trees/AbstractTreeNode.scala index db28c1f06f..e0bf272d7d 100644 --- a/okapi-trees/src/main/scala/org/opencypher/okapi/trees/AbstractTreeNode.scala +++ b/okapi-trees/src/main/scala/org/opencypher/okapi/trees/AbstractTreeNode.scala @@ -33,31 +33,36 @@ import scala.reflect.runtime.currentMirror import scala.reflect.runtime.universe.{Type, TypeTag, typeOf, typeTag} /** - * Class that implements the `children` and `withNewChildren` methods using reflection when implementing - * `TreeNode` with a case class or case object. + * Class that implements the `children` and `withNewChildren` methods using reflection when + * implementing `TreeNode` with a case class or case object. * * This class caches values that are expensive to recompute. * - * The constructor can also contain [[NonEmptyList]]s, [[List]]s, and [[Option]]s that contain children. - * This works as long as there the assignment of children in `withNewChildren` to the different constructor - * parameters can be inferred. + * The constructor can also contain [[NonEmptyList]]s, [[List]]s, and [[Option]]s that contain + * children. This works as long as there the assignment of children in `withNewChildren` to the + * different constructor parameters can be inferred. * * Inferred assignment of new children is done as follows: * - Traverse the constructor arguments from left to right - * - Always try to assign the next child in `newChildren` to a constructor parameter that is a child - * - For constructor parameters that are an `Option` of a child: Assign some next child in `newChildren` + * - Always try to assign the next child in `newChildren` to a constructor parameter that is a + * child + * - For constructor parameters that are an `Option` of a child: Assign some next child in + * `newChildren` * if the child type matches the element type of the Option, assign None otherwise. - * - For constructor parameters that are a `List` of children: Assign children from `newChildren` to the list + * - For constructor parameters that are a `List` of children: Assign children from `newChildren` + * to the list * until the type of a child does not match the element type of the list. * - * It is possible to override the defaults and use custom `children`/`withNewChildren` implementations. + * It is possible to override the defaults and use custom `children`/`withNewChildren` + * implementations. */ -abstract class AbstractTreeNode[T <: AbstractTreeNode[T] : TypeTag] extends TreeNode[T] { +abstract class AbstractTreeNode[T <: AbstractTreeNode[T]: TypeTag] extends TreeNode[T] { self: T => override protected def tt: TypeTag[T] = implicitly[TypeTag[T]] - override implicit protected def ct: ClassTag[T] = ClassTag[T](typeTag[T].mirror.runtimeClass(typeTag[T].tpe)) + override implicit protected def ct: ClassTag[T] = + ClassTag[T](typeTag[T].mirror.runtimeClass(typeTag[T].tpe)) override val children: Array[T] = { if (productIterator.isEmpty) { @@ -65,12 +70,16 @@ abstract class AbstractTreeNode[T <: AbstractTreeNode[T] : TypeTag] extends Tree } else { val copyMethod = AbstractTreeNode.copyMethod(self) lazy val treeType = typeOf[T].erasure - lazy val paramTypes: Seq[Type] = copyMethod.symbol.paramLists.head.map(_.typeSignature).toIndexedSeq + lazy val paramTypes: Seq[Type] = + copyMethod.symbol.paramLists.head.map(_.typeSignature).toIndexedSeq productIterator.toArray.zipWithIndex.flatMap { case (t: T, _) => Some(t) - case (o: Option[_], i) if paramTypes(i).typeArgs.head <:< treeType => o.asInstanceOf[Option[T]] - case (l: List[_], i) if paramTypes(i).typeArgs.head <:< treeType => l.asInstanceOf[List[T]] - case (nel: NonEmptyList[_], i) if paramTypes(i).typeArgs.head <:< treeType => nel.toList.asInstanceOf[List[T]] + case (o: Option[_], i) if paramTypes(i).typeArgs.head <:< treeType => + o.asInstanceOf[Option[T]] + case (l: List[_], i) if paramTypes(i).typeArgs.head <:< treeType => + l.asInstanceOf[List[T]] + case (nel: NonEmptyList[_], i) if paramTypes(i).typeArgs.head <:< treeType => + nel.toList.asInstanceOf[List[T]] case _ => Nil } } @@ -81,7 +90,8 @@ abstract class AbstractTreeNode[T <: AbstractTreeNode[T] : TypeTag] extends Tree self } else { val copyMethod = AbstractTreeNode.copyMethod(self) - val copyMethodParamTypes = copyMethod.symbol.paramLists.flatten.zipWithIndex + val copyMethodParamTypes = + copyMethod.symbol.paramLists.flatten.zipWithIndex val valueAndTypeTuples = copyMethodParamTypes.map { case (param, index) => val value = if (index < productArity) { // Access product element to retrieve the value @@ -91,16 +101,21 @@ abstract class AbstractTreeNode[T <: AbstractTreeNode[T] : TypeTag] extends Tree } value -> param.typeSignature } - val updatedConstructorParams = updateConstructorParams(newChildren, valueAndTypeTuples) + val updatedConstructorParams = + updateConstructorParams(newChildren, valueAndTypeTuples) try { copyMethod(updatedConstructorParams: _*).asInstanceOf[T] } catch { - case e: Exception => throw InvalidConstructorArgument( - s"""|Expected valid constructor arguments for $productPrefix + case e: Exception => + throw InvalidConstructorArgument( + s"""|Expected valid constructor arguments for $productPrefix |Old children: ${children.mkString(", ")} |New children: ${newChildren.mkString(", ")} |Current product: ${productIterator.mkString(", ")} - |Constructor arguments updated with new children: ${updatedConstructorParams.mkString(", ")}.""".stripMargin, Some(e)) + |Constructor arguments updated with new children: ${updatedConstructorParams + .mkString(", ")}.""".stripMargin, + Some(e) + ) } } } @@ -117,11 +132,13 @@ abstract class AbstractTreeNode[T <: AbstractTreeNode[T] : TypeTag] extends Tree childrenAsSet.contains(other) } - @inline final override def map[O <: TreeNode[O] : ClassTag](f: T => O): O = super.map(f) + @inline final override def map[O <: TreeNode[O]: ClassTag](f: T => O): O = + super.map(f) @inline final override def foreach[O](f: T => O): Unit = super.foreach(f) - @inline final override def containsTree(other: T): Boolean = super.containsTree(other) + @inline final override def containsTree(other: T): Boolean = + super.containsTree(other) @inline private final def updateConstructorParams( newChildren: Array[T], @@ -132,40 +149,66 @@ abstract class AbstractTreeNode[T <: AbstractTreeNode[T] : TypeTag] extends Tree currentMirror.reflect(instance).symbol.toType <:< tpe.typeArgs.head } - val (unassignedChildren, constructorParams) = currentValuesAndTypes.foldLeft(newChildren.toList -> Vector.empty[Any]) { - case ((remainingChildren, currentConstructorParams), nextValueAndType) => - nextValueAndType match { - case (c: T, _) => - remainingChildren match { - case Nil => throw new IllegalArgumentException( - s"""|When updating with new children: Did not have a child left to assign to the child that was previously $c - |Inferred constructor parameters so far: ${getClass.getSimpleName}(${currentConstructorParams.mkString(", ")}, ...)""".stripMargin) - case h :: t => t -> (currentConstructorParams :+ h) - } - case (_: Option[_], tpe) if tpe.typeArgs.head <:< typeOf[T] => - val option: Option[T] = remainingChildren.headOption.filter { c => couldBeElementOf(c, tpe) } - remainingChildren.drop(option.size) -> (currentConstructorParams :+ option) - case (_: List[_], tpe) if tpe.typeArgs.head <:< typeOf[T] => - val childrenList: List[T] = remainingChildren.takeWhile { c => couldBeElementOf(c, tpe) } - remainingChildren.drop(childrenList.size) -> (currentConstructorParams :+ childrenList) - case (_: NonEmptyList[_], tpe) if tpe.typeArgs.head <:< typeOf[T] => - val childrenList = NonEmptyList.fromListUnsafe(remainingChildren.takeWhile { c => couldBeElementOf(c, tpe) }) - remainingChildren.drop(childrenList.size) -> (currentConstructorParams :+ childrenList) - case (value, _) => - remainingChildren -> (currentConstructorParams :+ value) - } - } + val (unassignedChildren, constructorParams) = + currentValuesAndTypes.foldLeft(newChildren.toList -> Vector.empty[Any]) { + case ( + (remainingChildren, currentConstructorParams), + nextValueAndType + ) => + nextValueAndType match { + case (c: T, _) => + remainingChildren match { + case Nil => + throw new IllegalArgumentException( + s"""|When updating with new children: Did not have a child left to assign to the child that was previously $c + |Inferred constructor parameters so far: ${getClass.getSimpleName}(${currentConstructorParams + .mkString(", ")}, ...)""".stripMargin + ) + case h :: t => t -> (currentConstructorParams :+ h) + } + case (_: Option[_], tpe) if tpe.typeArgs.head <:< typeOf[T] => + val option: Option[T] = remainingChildren.headOption.filter { c => + couldBeElementOf(c, tpe) + } + remainingChildren.drop( + option.size + ) -> (currentConstructorParams :+ option) + case (_: List[_], tpe) if tpe.typeArgs.head <:< typeOf[T] => + val childrenList: List[T] = remainingChildren.takeWhile { c => + couldBeElementOf(c, tpe) + } + remainingChildren.drop( + childrenList.size + ) -> (currentConstructorParams :+ childrenList) + case (_: NonEmptyList[_], tpe) if tpe.typeArgs.head <:< typeOf[T] => + val childrenList = + NonEmptyList.fromListUnsafe(remainingChildren.takeWhile { c => + couldBeElementOf(c, tpe) + }) + remainingChildren.drop( + childrenList.size + ) -> (currentConstructorParams :+ childrenList) + case (value, _) => + remainingChildren -> (currentConstructorParams :+ value) + } + } if (unassignedChildren.nonEmpty) { throw new IllegalArgumentException( - s"""|Could not assign children [${unassignedChildren.mkString(", ")}] to parameters of ${getClass.getSimpleName} - |Inferred constructor parameters: ${getClass.getSimpleName}(${constructorParams.mkString(", ")})""".stripMargin) + s"""|Could not assign children [${unassignedChildren.mkString( + ", " + )}] to parameters of ${getClass.getSimpleName} + |Inferred constructor parameters: ${getClass.getSimpleName}(${constructorParams + .mkString(", ")})""".stripMargin + ) } constructorParams.toArray } - @inline private final def sameAsCurrentChildren(newChildren: Array[T]): Boolean = { + @inline private final def sameAsCurrentChildren( + newChildren: Array[T] + ): Boolean = { val childrenLength = children.length if (childrenLength != newChildren.length) { false @@ -178,9 +221,7 @@ abstract class AbstractTreeNode[T <: AbstractTreeNode[T] : TypeTag] extends Tree } -/** - * Caches an instance of the copy method per case class type. - */ +/** Caches an instance of the copy method per case class type. */ object AbstractTreeNode { import scala.reflect.runtime.universe @@ -189,9 +230,12 @@ object AbstractTreeNode { // No synchronization required: No problem if a cache entry is lost due to a concurrent write. @volatile private var cachedCopyMethods = Map.empty[Class[_], MethodMirror] - private final lazy val mirror = universe.runtimeMirror(getClass.getClassLoader) + private final lazy val mirror = + universe.runtimeMirror(getClass.getClassLoader) - @inline protected final def copyMethod(instance: AbstractTreeNode[_]): MethodMirror = { + @inline protected final def copyMethod( + instance: AbstractTreeNode[_] + ): MethodMirror = { val instanceClass = instance.getClass cachedCopyMethods.getOrElse( instanceClass, { @@ -202,19 +246,26 @@ object AbstractTreeNode { ) } - @inline private final def reflectCopyMethod(instance: Object): MethodMirror = { + @inline private final def reflectCopyMethod( + instance: Object + ): MethodMirror = { try { val instanceMirror = mirror.reflect(instance) val tpe = instanceMirror.symbol.asType.toType val copyMethodSymbol = tpe.decl(TermName("copy")).asMethod instanceMirror.reflectMethod(copyMethodSymbol) } catch { - case e: Exception => throw new UnsupportedOperationException( - s"Could not reflect the copy method of ${instance.toString.filterNot(_ == '$')}", e) + case e: Exception => + throw new UnsupportedOperationException( + s"Could not reflect the copy method of ${instance.toString.filterNot(_ == '$')}", + e + ) } } } -case class InvalidConstructorArgument(message: String, originalException: Option[Exception] = None) - extends RuntimeException(message, originalException.orNull) +case class InvalidConstructorArgument( + message: String, + originalException: Option[Exception] = None +) extends RuntimeException(message, originalException.orNull) diff --git a/okapi-trees/src/main/scala/org/opencypher/okapi/trees/TreeNode.scala b/okapi-trees/src/main/scala/org/opencypher/okapi/trees/TreeNode.scala index 2c07c89444..193b5bfb68 100644 --- a/okapi-trees/src/main/scala/org/opencypher/okapi/trees/TreeNode.scala +++ b/okapi-trees/src/main/scala/org/opencypher/okapi/trees/TreeNode.scala @@ -36,13 +36,14 @@ import scala.reflect.runtime.universe._ import scala.util.hashing.MurmurHash3 /** - * This is the basic tree node class. Usually it makes more sense to use `AbstractTreeNode`, which uses reflection - * to generate the `children` and `withNewChildren` field/method. Our benchmarks show that manually implementing - * them can result in a speedup of a factor of ~3 for tree rewrites, so in performance critical places it might - * make sense to extend `TreeNode` instead. + * This is the basic tree node class. Usually it makes more sense to use `AbstractTreeNode`, which + * uses reflection to generate the `children` and `withNewChildren` field/method. Our benchmarks + * show that manually implementing them can result in a speedup of a factor of ~3 for tree + * rewrites, so in performance critical places it might make sense to extend `TreeNode` instead. * - * This class uses array operations instead of Scala collections, both for improved performance as well as to save - * stack frames during recursion, which allows it to operate on trees that are several thousand nodes high. + * This class uses array operations instead of Scala collections, both for improved performance as + * well as to save stack frames during recursion, which allows it to operate on trees that are + * several thousand nodes high. */ abstract class TreeNode[T <: TreeNode[T]] extends Product with Traversable[T] { self: T => @@ -91,21 +92,30 @@ abstract class TreeNode[T <: TreeNode[T]] extends Product with Traversable[T] { def isLeaf: Boolean = children.isEmpty - def height: Int = transform[Int] { case (_, childHeights) => (0 :: childHeights).max + 1 } + def height: Int = transform[Int] { case (_, childHeights) => + (0 :: childHeights).max + 1 + } - override def size: Int = transform[Int] { case (_, childSizes) => childSizes.sum + 1 } + override def size: Int = transform[Int] { case (_, childSizes) => + childSizes.sum + 1 + } - def map[O <: TreeNode[O] : ClassTag](f: T => O): O = transform[O] { case (node, transformedChildren) => - f(node).withNewChildren(transformedChildren.toArray) + def map[O <: TreeNode[O]: ClassTag](f: T => O): O = transform[O] { + case (node, transformedChildren) => + f(node).withNewChildren(transformedChildren.toArray) } - override def foreach[O](f: T => O): Unit = transform[O] { case (node, _) => f(node) } + override def foreach[O](f: T => O): Unit = transform[O] { case (node, _) => + f(node) + } /** * Checks if the parameter tree is contained within this tree. A tree always contains itself. * - * @param other other tree - * @return true, iff `other` is contained in that tree + * @param other + * other tree + * @return + * true, iff `other` is contained in that tree */ def containsTree(other: T): Boolean = transform[Boolean] { case (node, childrenContain) => (node == other) || childrenContain.contains(true) @@ -114,8 +124,10 @@ abstract class TreeNode[T <: TreeNode[T]] extends Product with Traversable[T] { /** * Checks if `other` is a direct child of this tree. * - * @param other other tree - * @return true, iff `other` is a direct child of this tree + * @param other + * other tree + * @return + * true, iff `other` is a direct child of this tree */ def containsChild(other: T): Boolean = { children.contains(other) @@ -124,13 +136,18 @@ abstract class TreeNode[T <: TreeNode[T]] extends Product with Traversable[T] { /** * Returns a string-tree representation of the node. * - * @return tree-style representation of that node and all children + * @return + * tree-style representation of that node and all children */ def pretty: String = { val lines = new ArrayBuffer[String] @tailrec - def recTreeToString(toPrint: List[T], prefix: String, stack: List[List[T]]): Unit = { + def recTreeToString( + toPrint: List[T], + prefix: String, + stack: List[List[T]] + ): Unit = { toPrint match { case Nil => stack match { @@ -143,7 +160,11 @@ abstract class TreeNode[T <: TreeNode[T]] extends Product with Traversable[T] { recTreeToString(last.children.toList, s"$prefix ", Nil :: stack) case next :: siblings => lines += s"$prefix╟──${next.toString}" - recTreeToString(next.children.toList, s"$prefix║ ", siblings :: stack) + recTreeToString( + next.children.toList, + s"$prefix║ ", + siblings :: stack + ) } } @@ -151,9 +172,7 @@ abstract class TreeNode[T <: TreeNode[T]] extends Product with Traversable[T] { lines.mkString("\n") } - /** - * Prints a tree representation of the node. - */ + /** Prints a tree representation of the node. */ def show(): Unit = { println(pretty) } @@ -161,16 +180,16 @@ abstract class TreeNode[T <: TreeNode[T]] extends Product with Traversable[T] { /** * Turns all arguments in `args` into a string that describes the arguments. * - * @return argument string + * @return + * argument string */ def argString: String = args.mkString(", ") - /** - * Arguments that should be printed. The default implementation excludes children. - */ + /** Arguments that should be printed. The default implementation excludes children. */ def args: Iterator[Any] = { lazy val treeType = typeOf[T].erasure - currentMirror.reflect(this) + currentMirror + .reflect(this) .symbol .typeSignature .members @@ -181,21 +200,23 @@ abstract class TreeNode[T <: TreeNode[T]] extends Product with Traversable[T] { .map(currentMirror.reflect(this).reflectField) .map(fieldMirror => fieldMirror -> fieldMirror.get) .filter { case (fieldMirror, value) => - def containsChildren: Boolean = fieldMirror.symbol.typeSignature.typeArgs.head <:< treeType + def containsChildren: Boolean = + fieldMirror.symbol.typeSignature.typeArgs.head <:< treeType value match { - case c: T if containsChild(c) => false - case _: Option[_] if containsChildren => false + case c: T if containsChild(c) => false + case _: Option[_] if containsChildren => false case _: NonEmptyList[_] if containsChildren => false - case _: Iterable[_] if containsChildren => false - case _: Array[_] if containsChildren => false - case _ => true + case _: Iterable[_] if containsChildren => false + case _: Array[_] if containsChildren => false + case _ => true } } - .map { case (termSymbol, value) => s"${termSymbol.symbol.name.toString.trim}=$value" } + .map { case (termSymbol, value) => + s"${termSymbol.symbol.name.toString.trim}=$value" + } .reverseIterator } - override def toString = s"${getClass.getSimpleName}${ - if (args.isEmpty) "" else s"(${args.mkString(", ")})" - }" + override def toString = s"${getClass.getSimpleName}${if (args.isEmpty) "" + else s"(${args.mkString(", ")})"}" } diff --git a/okapi-trees/src/main/scala/org/opencypher/okapi/trees/TreeTransformer.scala b/okapi-trees/src/main/scala/org/opencypher/okapi/trees/TreeTransformer.scala index d8d9b921d9..70e377213e 100644 --- a/okapi-trees/src/main/scala/org/opencypher/okapi/trees/TreeTransformer.scala +++ b/okapi-trees/src/main/scala/org/opencypher/okapi/trees/TreeTransformer.scala @@ -28,26 +28,24 @@ package org.opencypher.okapi.trees import scala.reflect.ClassTag - -abstract class TreeTransformer[I <: TreeNode[I] : ClassTag, O] { +abstract class TreeTransformer[I <: TreeNode[I]: ClassTag, O] { def transform(tree: I): O } -abstract class TreeTransformerWithContext[I <: TreeNode[I] : ClassTag, O, C] { +abstract class TreeTransformerWithContext[I <: TreeNode[I]: ClassTag, O, C] { def transform(tree: I, context: C): (O, C) } -abstract class TreeRewriter[T <: TreeNode[T] : ClassTag] extends TreeTransformer[T, T] +abstract class TreeRewriter[T <: TreeNode[T]: ClassTag] extends TreeTransformer[T, T] -abstract class TreeRewriterWithContext[T <: TreeNode[T] : ClassTag, C] extends TreeTransformerWithContext[T, T, C] { +abstract class TreeRewriterWithContext[T <: TreeNode[T]: ClassTag, C] + extends TreeTransformerWithContext[T, T, C] { def transform(tree: T, context: C): (T, C) } - -/** - * Applies the given partial function starting from the leaves of this tree. - */ -case class BottomUp[T <: TreeNode[T] : ClassTag](rule: PartialFunction[T, T]) extends TreeRewriter[T] { +/** Applies the given partial function starting from the leaves of this tree. */ +case class BottomUp[T <: TreeNode[T]: ClassTag](rule: PartialFunction[T, T]) + extends TreeRewriter[T] { def transform(tree: T): T = { val childrenLength = tree.children.length @@ -71,10 +69,13 @@ case class BottomUp[T <: TreeNode[T] : ClassTag](rule: PartialFunction[T, T]) ex } /** - * Applies the given partial function starting from the leaves of this tree. An additional context is being recursively - * passed from the leftmost child to its siblings and eventually to its parent. + * Applies the given partial function starting from the leaves of this tree. An additional context + * is being recursively passed from the leftmost child to its siblings and eventually to its + * parent. */ -case class BottomUpWithContext[T <: TreeNode[T] : ClassTag, C](rule: PartialFunction[(T, C), (T, C)]) extends TreeRewriterWithContext[T, C] { +case class BottomUpWithContext[T <: TreeNode[T]: ClassTag, C]( + rule: PartialFunction[(T, C), (T, C)] +) extends TreeRewriterWithContext[T, C] { def transform(tree: T, context: C): (T, C) = { val childrenLength = tree.children.length @@ -103,9 +104,11 @@ case class BottomUpWithContext[T <: TreeNode[T] : ClassTag, C](rule: PartialFunc /** * Applies the given partial function starting from the root of this tree. * - * @note Note the applied rule cannot insert new parent nodes. + * @note + * Note the applied rule cannot insert new parent nodes. */ -case class TopDown[T <: TreeNode[T] : ClassTag](rule: PartialFunction[T, T]) extends TreeRewriter[T] { +case class TopDown[T <: TreeNode[T]: ClassTag](rule: PartialFunction[T, T]) + extends TreeRewriter[T] { def transform(tree: T): T = { val afterSelf = if (rule.isDefinedAt(tree)) rule(tree) else tree @@ -128,10 +131,8 @@ case class TopDown[T <: TreeNode[T] : ClassTag](rule: PartialFunction[T, T]) ext } -/** - * Applies the given transformation starting from the leaves of this tree. - */ -case class Transform[I <: TreeNode[I] : ClassTag, O]( +/** Applies the given transformation starting from the leaves of this tree. */ +case class Transform[I <: TreeNode[I]: ClassTag, O]( transform: (I, List[O]) => O ) extends TreeTransformer[I, O] { diff --git a/okapi-trees/src/main/scala/org/opencypher/okapi/trees/TreeTransformerStackSafe.scala b/okapi-trees/src/main/scala/org/opencypher/okapi/trees/TreeTransformerStackSafe.scala index 3cdb3cff09..3326aac77f 100644 --- a/okapi-trees/src/main/scala/org/opencypher/okapi/trees/TreeTransformerStackSafe.scala +++ b/okapi-trees/src/main/scala/org/opencypher/okapi/trees/TreeTransformerStackSafe.scala @@ -31,35 +31,25 @@ import cats.data.NonEmptyList import scala.annotation.tailrec import scala.reflect.ClassTag -/** - * Common trait of all classes that represent tree operations for off-stack transformations. - */ +/** Common trait of all classes that represent tree operations for off-stack transformations. */ sealed trait TreeOperation[T <: TreeNode[T], O] -/** - * Represents a child transformation operation during off-stack transformations. - */ +/** Represents a child transformation operation during off-stack transformations. */ case class TransformChildren[I <: TreeNode[I], O]( node: I, transformedChildren: List[O] = List.empty[O] ) extends TreeOperation[I, O] -/** - * Represents a node transformation operation during off-stack transformations. - */ +/** Represents a node transformation operation during off-stack transformations. */ case class TransformNode[I <: TreeNode[I], O]( node: I, transformedChildren: List[O] = List.empty[O] ) extends TreeOperation[I, O] -/** - * Represents a finished transformation during off-stack transformations. - */ +/** Represents a finished transformation during off-stack transformations. */ case class Done[I <: TreeNode[I], O](transformedChildren: List[O]) extends TreeOperation[I, O] -/** - * This is the base-class for stack-safe tree transformations. - */ +/** This is the base-class for stack-safe tree transformations. */ trait TransformerStackSafe[I <: TreeNode[I], O] extends TreeTransformer[I, O] { type NonEmptyStack = NonEmptyList[TreeOperation[I, O]] @@ -84,18 +74,14 @@ trait TransformerStackSafe[I <: TreeNode[I], O] extends TreeTransformer[I, O] { } - /** - * Called on each node when going down the tree. - */ + /** Called on each node when going down the tree. */ def transformChildren( node: I, transformedChildren: List[O], stack: Stack ): NonEmptyStack - /** - * Called on each node when going up the tree. - */ + /** Called on each node when going up the tree. */ def transformNode( node: I, transformedChildren: List[O], @@ -103,28 +89,46 @@ trait TransformerStackSafe[I <: TreeNode[I], O] extends TreeTransformer[I, O] { ): NonEmptyStack @tailrec - protected final def run(stack: NonEmptyList[TreeOperation[I, O]]): O = stack match { - case NonEmptyList(TransformChildren(node, transformedChildren), tail) => run(transformChildren(node, transformedChildren, tail)) - case NonEmptyList(TransformNode(node, transformedChildren), tail) => run(transformNode(node, transformedChildren, tail)) - case NonEmptyList(Done(transformed), tail) => - tail match { - case Nil => transformed match { - case result :: Nil => result - case invalid => throw new IllegalStateException(s"Invalid transformation produced $invalid instead of a single final value.") + protected final def run(stack: NonEmptyList[TreeOperation[I, O]]): O = + stack match { + case NonEmptyList(TransformChildren(node, transformedChildren), tail) => + run(transformChildren(node, transformedChildren, tail)) + case NonEmptyList(TransformNode(node, transformedChildren), tail) => + run(transformNode(node, transformedChildren, tail)) + case NonEmptyList(Done(transformed), tail) => + tail match { + case Nil => + transformed match { + case result :: Nil => result + case invalid => + throw new IllegalStateException( + s"Invalid transformation produced $invalid instead of a single final value." + ) + } + case Done(nextNodes) :: nextTail => + run(nextTail.push(Done(transformed ::: nextNodes))) + case TransformChildren(nextNode, transformedChildren) :: nextTail => + run( + nextTail.push( + TransformChildren(nextNode, transformed ::: transformedChildren) + ) + ) + case TransformNode(nextNode, transformedChildren) :: nextTail => + run( + nextTail.push( + TransformNode(nextNode, transformed ::: transformedChildren) + ) + ) } - case Done(nextNodes) :: nextTail => run(nextTail.push(Done(transformed ::: nextNodes))) - case TransformChildren(nextNode, transformedChildren) :: nextTail => run(nextTail.push(TransformChildren(nextNode, transformed ::: transformedChildren))) - case TransformNode(nextNode, transformedChildren) :: nextTail => run(nextTail.push(TransformNode(nextNode, transformed ::: transformedChildren))) - } - } + } - @inline final override def transform(tree: I): O = run(Stack(TransformChildren(tree))) + @inline final override def transform(tree: I): O = run( + Stack(TransformChildren(tree)) + ) } -/** - * Common parent of [[BottomUpStackSafe]] and [[TopDownStackSafe]] - */ +/** Common parent of [[BottomUpStackSafe]] and [[TopDownStackSafe]] */ trait SameTypeTransformerStackSafe[T <: TreeNode[T]] extends TransformerStackSafe[T, T] { protected val partial: PartialFunction[T, T] @@ -136,25 +140,39 @@ trait SameTypeTransformerStackSafe[T <: TreeNode[T]] extends TransformerStackSaf /** * Applies the given partial function starting from the leafs of this tree. * - * @note This is a stack-safe version of [[BottomUp]]. + * @note + * This is a stack-safe version of [[BottomUp]]. */ -case class BottomUpStackSafe[T <: TreeNode[T] : ClassTag]( +case class BottomUpStackSafe[T <: TreeNode[T]: ClassTag]( partial: PartialFunction[T, T] ) extends SameTypeTransformerStackSafe[T] { - @inline final override def transformChildren(node: T, rewrittenChildren: List[T], stack: Stack): NonEmptyStack = { + @inline final override def transformChildren( + node: T, + rewrittenChildren: List[T], + stack: Stack + ): NonEmptyStack = { if (node.children.isEmpty) { stack.push(Done(rule(node) :: rewrittenChildren)) } else { - node.children.foldLeft(stack.push(TransformNode(node, rewrittenChildren))) { case (currentStack, child) => + node.children.foldLeft( + stack.push(TransformNode(node, rewrittenChildren)) + ) { case (currentStack, child) => currentStack.push(TransformChildren(child)) } } } - @inline final override def transformNode(node: T, rewrittenChildren: List[T], stack: Stack): NonEmptyStack = { - val (currentRewrittenChildren, rewrittenForAncestors) = rewrittenChildren.splitAt(node.children.length) - val rewrittenNode = rule(node.withNewChildren(currentRewrittenChildren.toArray)) + @inline final override def transformNode( + node: T, + rewrittenChildren: List[T], + stack: Stack + ): NonEmptyStack = { + val (currentRewrittenChildren, rewrittenForAncestors) = + rewrittenChildren.splitAt(node.children.length) + val rewrittenNode = rule( + node.withNewChildren(currentRewrittenChildren.toArray) + ) stack.push(Done(rewrittenNode :: rewrittenForAncestors)) } } @@ -162,27 +180,39 @@ case class BottomUpStackSafe[T <: TreeNode[T] : ClassTag]( /** * Applies the given partial function starting from the root of this tree. * - * @note Note the applied rule cannot insert new parent nodes. - * @note This is a stack-safe version of [[TopDown]]. + * @note + * Note the applied rule cannot insert new parent nodes. + * @note + * This is a stack-safe version of [[TopDown]]. */ -case class TopDownStackSafe[T <: TreeNode[T] : ClassTag]( +case class TopDownStackSafe[T <: TreeNode[T]: ClassTag]( partial: PartialFunction[T, T] ) extends SameTypeTransformerStackSafe[T] { - @inline final override def transformChildren(node: T, rewrittenChildren: List[T], stack: Stack): NonEmptyStack = { + @inline final override def transformChildren( + node: T, + rewrittenChildren: List[T], + stack: Stack + ): NonEmptyStack = { val updatedNode = rule(node) if (updatedNode.children.isEmpty) { stack.push(Done(updatedNode :: rewrittenChildren)) } else { - updatedNode.children.foldLeft(stack.push(TransformNode(updatedNode, rewrittenChildren))) { - case (currentStack, child) => - currentStack.push(TransformChildren(child)) + updatedNode.children.foldLeft( + stack.push(TransformNode(updatedNode, rewrittenChildren)) + ) { case (currentStack, child) => + currentStack.push(TransformChildren(child)) } } } - @inline final override def transformNode(node: T, rewrittenChildren: List[T], stack: Stack): NonEmptyStack = { - val (currentRewrittenChildren, rewrittenForAncestors) = rewrittenChildren.splitAt(node.children.length) + @inline final override def transformNode( + node: T, + rewrittenChildren: List[T], + stack: Stack + ): NonEmptyStack = { + val (currentRewrittenChildren, rewrittenForAncestors) = + rewrittenChildren.splitAt(node.children.length) val rewrittenNode = node.withNewChildren(currentRewrittenChildren.toArray) stack.push(Done(rewrittenNode :: rewrittenForAncestors)) } @@ -192,24 +222,36 @@ case class TopDownStackSafe[T <: TreeNode[T] : ClassTag]( /** * Applies the given transformation starting from the leaves of this tree. * - * @note This is a stack-safe version of [[Transform]]. + * @note + * This is a stack-safe version of [[Transform]]. */ -case class TransformStackSafe[I <: TreeNode[I] : ClassTag, O]( +case class TransformStackSafe[I <: TreeNode[I]: ClassTag, O]( transform: (I, List[O]) => O ) extends TransformerStackSafe[I, O] { - @inline final override def transformChildren(node: I, transformedChildren: List[O], stack: Stack): NonEmptyStack = { + @inline final override def transformChildren( + node: I, + transformedChildren: List[O], + stack: Stack + ): NonEmptyStack = { if (node.children.isEmpty) { stack.push(Done(transform(node, List.empty[O]) :: transformedChildren)) } else { - node.children.foldLeft(stack.push(TransformNode(node, transformedChildren))) { case (currentStack, child) => + node.children.foldLeft( + stack.push(TransformNode(node, transformedChildren)) + ) { case (currentStack, child) => currentStack.push(TransformChildren(child)) } } } - @inline final override def transformNode(node: I, transformedChildren: List[O], stack: Stack): NonEmptyStack = { - val (transformedChildrenForCurrentNode, transformedChildrenForAncestors) = transformedChildren.splitAt(node.children.length) + @inline final override def transformNode( + node: I, + transformedChildren: List[O], + stack: Stack + ): NonEmptyStack = { + val (transformedChildrenForCurrentNode, transformedChildrenForAncestors) = + transformedChildren.splitAt(node.children.length) val transformedNode = transform(node, transformedChildrenForCurrentNode) stack.push(Done(transformedNode :: transformedChildrenForAncestors)) } diff --git a/okapi-trees/src/test/scala/org/opencypher/okapi/trees/TreeNodeTest.scala b/okapi-trees/src/test/scala/org/opencypher/okapi/trees/TreeNodeTest.scala index 5248636d74..a97b1a2dde 100644 --- a/okapi-trees/src/test/scala/org/opencypher/okapi/trees/TreeNodeTest.scala +++ b/okapi-trees/src/test/scala/org/opencypher/okapi/trees/TreeNodeTest.scala @@ -73,14 +73,40 @@ class TreeNodeTest extends AnyFunSpec with Matchers { } it("lists of children") { - val addList1 = AddList(NonEmptyList.one(1), Number(1), 2, NonEmptyList.one(Number(2)), NonEmptyList.of[Object]("a", "b")) + val addList1 = AddList( + NonEmptyList.one(1), + Number(1), + 2, + NonEmptyList.one(Number(2)), + NonEmptyList.of[Object]("a", "b") + ) addList1.eval should equal(3) - val addList2 = AddList(NonEmptyList.one(1), Number(1), 2, NonEmptyList.of(Number(2), Number(3)), NonEmptyList.of[Object]("a", "b")) + val addList2 = AddList( + NonEmptyList.one(1), + Number(1), + 2, + NonEmptyList.of(Number(2), Number(3)), + NonEmptyList.of[Object]("a", "b") + ) addList2.eval should equal(6) val addList3 = - AddList(NonEmptyList.one(1), Number(0), 2, NonEmptyList.one(Number(2)), NonEmptyList.of[Object]("a", "b")) + AddList( + NonEmptyList.one(1), + Number(0), + 2, + NonEmptyList.one(Number(2)), + NonEmptyList.of[Object]("a", "b") + ) .withNewChildren(Array(1, 2, 3, 4, 5, 6, 7).map(Number)) - addList3 should equal(AddList(NonEmptyList.one(1), Number(1), 2, NonEmptyList.of(2, 3, 4, 5, 6, 7).map(Number), NonEmptyList.of[Object]("a", "b"))) + addList3 should equal( + AddList( + NonEmptyList.one(1), + Number(1), + 2, + NonEmptyList.of(2, 3, 4, 5, 6, 7).map(Number), + NonEmptyList.of[Object]("a", "b") + ) + ) addList3.eval should equal(28) } @@ -88,7 +114,13 @@ class TreeNodeTest extends AnyFunSpec with Matchers { // Test errors when violating list requirements intercept[IllegalArgumentException] { - val fail = AddList(NonEmptyList.one(1), Number(1), 2, NonEmptyList.one(Number(2)), NonEmptyList.of[Object]("a", "b")) + val fail = AddList( + NonEmptyList.one(1), + Number(1), + 2, + NonEmptyList.one(Number(2)), + NonEmptyList.of[Object]("a", "b") + ) fail.children.toSet should equal(Set(Number(1), Number(2))) fail.withNewChildren(Array(Number(1))) }.getMessage should equal("Cannot create NonEmptyList from empty list") @@ -98,8 +130,8 @@ class TreeNodeTest extends AnyFunSpec with Matchers { it("rewrite") { val addNoops: PartialFunction[CalcExpr, CalcExpr] = { case Add(n1: Number, n2: Number) => Add(NoOp(n1), NoOp(n2)) - case Add(n1: Number, n2) => Add(NoOp(n1), n2) - case Add(n1, n2: Number) => Add(n1, NoOp(n2)) + case Add(n1: Number, n2) => Add(NoOp(n1), n2) + case Add(n1, n2: Number) => Add(n1, NoOp(n2)) } val expected = Add(NoOp(Number(5)), Add(NoOp(Number(4)), NoOp(Number(3)))) @@ -122,16 +154,16 @@ class TreeNodeTest extends AnyFunSpec with Matchers { } it("support relatively high trees without stack overflow") { - val highTree = (1 to 1000).foldLeft(Number(1): CalcExpr) { - case (t, n) => Add(t, Number(n)) + val highTree = (1 to 1000).foldLeft(Number(1): CalcExpr) { case (t, n) => + Add(t, Number(n)) } - val simplified = BottomUp[CalcExpr] { - case Add(Number(n1), Number(n2)) => Number(n1 + n2) + val simplified = BottomUp[CalcExpr] { case Add(Number(n1), Number(n2)) => + Number(n1 + n2) }.transform(highTree) simplified should equal(Number(500501)) - val addNoOpsBeforeLeftAdd: PartialFunction[CalcExpr, CalcExpr] = { - case Add(a: Add, b) => Add(NoOp(a), b) + val addNoOpsBeforeLeftAdd: PartialFunction[CalcExpr, CalcExpr] = { case Add(a: Add, b) => + Add(NoOp(a), b) } val noOpTree = TopDown[CalcExpr] { addNoOpsBeforeLeftAdd @@ -142,8 +174,8 @@ class TreeNodeTest extends AnyFunSpec with Matchers { it("stack safe rewrite") { val addNoops: PartialFunction[CalcExpr, CalcExpr] = { case Add(n1: Number, n2: Number) => Add(NoOp(n1), NoOp(n2)) - case Add(n1: Number, n2) => Add(NoOp(n1), n2) - case Add(n1, n2: Number) => Add(n1, NoOp(n2)) + case Add(n1: Number, n2) => Add(NoOp(n1), n2) + case Add(n1, n2: Number) => Add(n1, NoOp(n2)) } val expected = Add(NoOp(Number(5)), Add(NoOp(Number(4)), NoOp(Number(3)))) @@ -154,15 +186,15 @@ class TreeNodeTest extends AnyFunSpec with Matchers { it("support arbitrarily high high trees with stack safe rewrites") { val height = 50000 - val highTree = (1 to height).foldLeft(Number(1): CalcExpr) { - case (t, n) => Add(t, Number(n)) + val highTree = (1 to height).foldLeft(Number(1): CalcExpr) { case (t, n) => + Add(t, Number(n)) } - BottomUpStackSafe[CalcExpr] { - case Add(Number(n1), Number(n2)) => Number(n1 + n2) + BottomUpStackSafe[CalcExpr] { case Add(Number(n1), Number(n2)) => + Number(n1 + n2) }.transform(highTree) - val addNoOpsBeforeLeftAdd: PartialFunction[CalcExpr, CalcExpr] = { - case Add(a: Add, b) => Add(NoOp(a), b) + val addNoOpsBeforeLeftAdd: PartialFunction[CalcExpr, CalcExpr] = { case Add(a: Add, b) => + Add(NoOp(a), b) } val noOpTree = TopDownStackSafe[CalcExpr] { addNoOpsBeforeLeftAdd @@ -176,7 +208,9 @@ class TreeNodeTest extends AnyFunSpec with Matchers { } it("option and list arg string") { - Dummy(None, List.empty, None, List.empty).argString should equal("print1=None, print2=List()") + Dummy(None, List.empty, None, List.empty).argString should equal( + "print1=None, print2=List()" + ) } it("to string") { @@ -186,8 +220,7 @@ class TreeNodeTest extends AnyFunSpec with Matchers { it("pretty") { val t = Add(Add(Number(4), Number(3)), Add(Number(4), Number(3))) - t.pretty should equal( - """|╙──Add + t.pretty should equal("""|╙──Add | ╟──Add | ║ ╟──Number(v=4) | ║ ╙──Number(v=3) @@ -197,7 +230,9 @@ class TreeNodeTest extends AnyFunSpec with Matchers { } it("copy with the same children returns the same instance") { - calculation.withNewChildren(Array(calculation.left, calculation.right)) should be theSameInstanceAs calculation + calculation.withNewChildren( + Array(calculation.left, calculation.right) + ) should be theSameInstanceAs calculation } it("can infer children for complex case classes") { @@ -211,53 +246,80 @@ class TreeNodeTest extends AnyFunSpec with Matchers { Some(SimpleD()) ) - instance.children.toList should equal(List( - SimpleA(), SimpleA(), SimpleB(), SimpleB(), SimpleD() - )) + instance.children.toList should equal( + List( + SimpleA(), + SimpleA(), + SimpleB(), + SimpleB(), + SimpleD() + ) + ) - instance.withNewChildren(Array( - SimpleA(), SimpleC() - )) should equal(Multi( - NonEmptyList.one(1), - NonEmptyList.of(SimpleA()), - List("A", "B"), - List(), - Some(1L), - Some(SimpleC()), - None - )) + instance.withNewChildren( + Array( + SimpleA(), + SimpleC() + ) + ) should equal( + Multi( + NonEmptyList.one(1), + NonEmptyList.of(SimpleA()), + List("A", "B"), + List(), + Some(1L), + Some(SimpleC()), + None + ) + ) } - it("fails with an understandable error message when running out of children to assign") { + it( + "fails with an understandable error message when running out of children to assign" + ) { val instance = ListBeforeFixed(List.empty, SimpleA()) - instance.children.toList should equal(List( - SimpleA() - )) + instance.children.toList should equal( + List( + SimpleA() + ) + ) intercept[IllegalArgumentException] { - instance.withNewChildren(Array( - SimpleA(), SimpleA() - )) + instance.withNewChildren( + Array( + SimpleA(), + SimpleA() + ) + ) }.getMessage should equal( """|When updating with new children: Did not have a child left to assign to the child that was previously SimpleA - |Inferred constructor parameters so far: ListBeforeFixed(List(SimpleA, SimpleA), ...)""".stripMargin) + |Inferred constructor parameters so far: ListBeforeFixed(List(SimpleA, SimpleA), ...)""".stripMargin + ) } - it("fails with an understandable error message when there are children left over after assignment to constructor arguments") { + it( + "fails with an understandable error message when there are children left over after assignment to constructor arguments" + ) { val instance = SimpleList(List(SimpleA())) - instance.children.toList should equal(List( - SimpleA() - )) + instance.children.toList should equal( + List( + SimpleA() + ) + ) intercept[IllegalArgumentException] { - instance.withNewChildren(Array( - SimpleA(), SimpleB() - )) + instance.withNewChildren( + Array( + SimpleA(), + SimpleB() + ) + ) }.getMessage should equal( """|Could not assign children [SimpleB] to parameters of SimpleList - |Inferred constructor parameters: SimpleList(List(SimpleA))""".stripMargin) + |Inferred constructor parameters: SimpleList(List(SimpleA))""".stripMargin + ) } } @@ -277,7 +339,10 @@ object TreeNodeTest { case object UnsupportedLeaf2 extends UnsupportedTree2 - case class UnsupportedNode2(elems: NonEmptyList[UnsupportedTree2], elem: UnsupportedTree2) extends UnsupportedTree2 + case class UnsupportedNode2( + elems: NonEmptyList[UnsupportedTree2], + elem: UnsupportedTree2 + ) extends UnsupportedTree2 abstract class CalcExpr extends AbstractTreeNode[CalcExpr] { def eval: Int @@ -298,8 +363,7 @@ object TreeNodeTest { dummy2: Int, remaining: NonEmptyList[CalcExpr], dummy3: NonEmptyList[Object] - ) - extends CalcExpr { + ) extends CalcExpr { def eval: Int = first.eval + remaining.map(_.eval).toList.sum } @@ -340,4 +404,4 @@ object TreeNodeTest { maybeC: Option[SimpleC], maybeD: Option[SimpleD] ) extends ComplexExample -} \ No newline at end of file +}