test jsonschema with bad inputs

Author: mr-c

testdata/echo-tool.cwl

@@ -0,0 +1,17 @@
+#!/usr/bin/env cwl-runner
+
+class: CommandLineTool
+cwlVersion: v1.2
+inputs:
+  in:
+    type: Any
+    inputBinding: {}
+outputs:
+  out:
+    type: string
+    outputBinding:
+      glob: out.txt
+      loadContents: true
+      outputEval: $(self[0].contents)
+baseCommand: echo
+stdout: out.txt

testdata/empty.json

@@ -0,0 +1 @@
+{}

testdata/null-expression-echo-job.json

@@ -0,0 +1,3 @@
+{
+    "in": null
+}

testdata/null-expression1-job.json

@@ -0,0 +1,3 @@
+{
+    "i1": null
+}

testdata/null-expression2-tool.cwl

@@ -0,0 +1,14 @@
+#!/usr/bin/env cwl-runner
+
+class: ExpressionTool
+requirements:
+  - class: InlineJavascriptRequirement
+cwlVersion: v1.2
+
+inputs:
+  i1: Any
+
+outputs:
+  output: int
+
+expression: "$({'output': (inputs.i1 == 'the-default' ? 1 : 2)})"

tests/test_inputs_schema_gen.py

@@ -73,7 +73,7 @@ def test_cwl_inputs_to_jsonschema(tool_path: Path, inputs_path: Path) -> None:
         raise SchemaError(f"{inputs_path.name} failed schema validation") from err
 
 
-def test_cwl_inputs_to_jsonschema_fails() -> None:
+def test_cwl_inputs_to_jsonschema_single_fail() -> None:
     """Compare tool schema of param 1 against input schema of param 2."""
     tool_path: Path = TEST_PARAMS[0][0]
     inputs_path: Path = TEST_PARAMS[3][1]
@@ -92,3 +92,54 @@ def test_cwl_inputs_to_jsonschema_fails() -> None:
     # We expect this to fail
     with pytest.raises(ValidationError):
         validate(input_obj, json_schema)
+
+
+BAD_TEST_PARAMS = [
+    # Any without defaults cannot be unspecified
+    (
+        get_path("testdata/null-expression2-tool.cwl"),
+        get_path("testdata/empty.json"),
+        "'i1' is a required property",
+    ),
+    # null to Any type without a default value.
+    (
+        get_path("testdata/null-expression2-tool.cwl"),
+        get_path("testdata/null-expression1-job.json"),
+        "None is not valid under any of the given schemas",
+    ),
+    # Any without defaults, unspecified
+    (
+        get_path("testdata/echo-tool.cwl"),
+        get_path("testdata/null-expression-echo-job.json"),
+        "None is not valid under any of the given schemas",
+    ),
+    # JSON null provided for required input
+    (
+        get_path("testdata/echo-tool.cwl"),
+        get_path("testdata/null-expression1-job.json"),
+        "'in' is a required property",
+    ),
+]
+
+
+@pytest.mark.parametrize("tool_path,inputs_path,exception_message", BAD_TEST_PARAMS)
+def test_cwl_inputs_to_jsonschema_fails(
+    tool_path: Path,
+    inputs_path: Path,
+    exception_message: str,
+) -> None:
+    cwl_obj = load_document_by_uri(tool_path.as_uri())
+
+    logger.info(f"Generating schema for {tool_path.name}")
+    json_schema = cwl_to_jsonschema(cwl_obj)
+
+    logger.info(
+        f"Testing {inputs_path.name} against schema generated for input {tool_path.name}"
+    )
+
+    yaml = YAML()
+
+    input_obj = yaml.load(inputs_path)
+
+    with pytest.raises(ValidationError, match=exception_message):
+        validate(input_obj, json_schema)