fix: correct update of test run results when suite teardown fails or is skipped during suite teardown

Unfortunately, if a test has already failed but it is subsequently skipped in the teardown, the error status of VSCode is not set because the skipped status has a lower priority than the failed status. This is a VSCode problem and I can't change it at the moment.

Author: d-biehl

packages/debugger/src/robotcode/debugger/listeners.py

@@ -60,6 +60,8 @@ def end_suite(self, name: str, attributes: Dict[str, Any]) -> None:
             ),
         )
 
+        self.failed_keywords = None
+
     def start_test(self, name: str, attributes: Dict[str, Any]) -> None:
         self.failed_keywords = None
 
@@ -287,8 +289,36 @@ def enqueue(item: Union[running.TestSuite, running.TestCase]) -> Iterator[str]:
 
         self._event_sended = True
 
-    def end_suite(self, data: running.TestSuite, result: result.TestSuite) -> None:
-        pass
+    def end_suite(self, data: running.TestSuite, suite_result: result.TestSuite) -> None:
+        def report_status(item: Union[result.TestSuite, result.TestCase], message: str) -> None:
+            if isinstance(item, result.TestCase):
+                Debugger.instance().send_event(
+                    self,
+                    Event(
+                        event="robotSetFailed",
+                        body=RobotExecutionEventBody(
+                            type="test",
+                            attributes={
+                                "longname": item.longname,
+                                "status": str(item.status),
+                                "elapsedtime": item.elapsedtime,
+                                "source": str(item.source),
+                                "lineno": item.lineno,
+                                "message": item.message,
+                            },
+                            id=f"{item.source or ''};{item.longname or ''}"
+                            + (f";{item.lineno or 0}" if isinstance(item, result.TestCase) else ""),
+                        ),
+                    ),
+                )
+            if isinstance(item, result.TestSuite):
+                for r in item.suites:
+                    report_status(r, message)
+                for r in item.tests:
+                    report_status(r, message)
+
+        if data.teardown and suite_result.teardown.status in ["FAIL", "SKIP"]:
+            report_status(suite_result, message=suite_result.message)
 
     def start_test(self, data: running.TestCase, result: result.TestCase) -> None:
         pass

vscode-client/testcontrollermanager.ts

@@ -151,7 +151,7 @@ export class TestControllerManager {
     public readonly debugManager: DebugManager,
     public readonly outputChannel: vscode.OutputChannel,
   ) {
-    this.testController = vscode.tests.createTestController("robotCode.RobotFramework", "RobotFramework");
+    this.testController = vscode.tests.createTestController("robotCode.RobotFramework", "Robot Framework");
 
     this.testController.resolveHandler = async (item) => {
       await this.refresh(item);
@@ -264,7 +264,6 @@ export class TestControllerManager {
               this.TestRunExited(event.session.configuration.runId);
               break;
             }
-
             case "robotStarted": {
               this.OnRobotStartedEvent(event.session.configuration.runId, event.body as RobotExecutionEvent);
               break;
@@ -273,6 +272,10 @@ export class TestControllerManager {
               this.OnRobotEndedEvent(event.session.configuration.runId, event.body as RobotExecutionEvent);
               break;
             }
+            case "robotSetFailed": {
+              this.OnRobotSetFailed(event.session.configuration.runId, event.body as RobotExecutionEvent);
+              break;
+            }
             case "robotEnqueued": {
               this.TestItemEnqueued(event.session.configuration.runId, event.body?.items);
               break;
@@ -1202,20 +1205,64 @@ export class TestControllerManager {
     }
   }
 
+  private OnRobotSetFailed(runId: string | undefined, event: RobotExecutionEvent) {
+    switch (event.type) {
+      case "suite":
+      case "test":
+        this.TestItemSetFailed(runId, event);
+        break;
+      default:
+        // do nothing
+        break;
+    }
+  }
+
+  private TestItemSetFailed(runId: string | undefined, event: RobotExecutionEvent) {
+    if (runId === undefined || event.attributes?.longname === undefined) return;
+
+    const run = this.testRuns.get(runId);
+
+    if (run !== undefined) {
+      const item = this.findTestItemById(event.id);
+      if (item) {
+        const message = new vscode.TestMessage(event.attributes?.message ?? "");
+
+        if (event.attributes.source) {
+          message.location = new vscode.Location(
+            vscode.Uri.file(event.attributes.source),
+            new vscode.Range(
+              new vscode.Position((event.attributes.lineno ?? 1) - 1, 0),
+              new vscode.Position(event.attributes.lineno ?? 1, 0),
+            ),
+          );
+        }
+
+        if (event.attributes.status === "SKIP") {
+          run.skipped(item);
+        } else if (event.attributes.status === "FAIL") {
+          run.failed(item, message, event.attributes.elapsedtime);
+        } else if (event.attributes.status === "ERROR") {
+          run.errored(item, message, event.attributes.elapsedtime);
+        }
+      }
+    }
+  }
+
   private TestItemEnded(runId: string | undefined, event: RobotExecutionEvent) {
     if (runId === undefined || event.attributes?.longname === undefined) return;
 
     const run = this.testRuns.get(runId);
 
     if (run !== undefined) {
       const item = this.findTestItemById(event.id);
+
       if (item !== undefined) {
         switch (event.attributes.status) {
           case "PASS":
-            run.passed(item, event.attributes.elapsedtime);
+            if (!item?.canResolveChildren) run.passed(item, event.attributes.elapsedtime);
             break;
           case "SKIP":
-            run.skipped(item);
+            if (!item?.canResolveChildren) run.skipped(item);
             break;
           default:
             {
@@ -1256,10 +1303,12 @@ export class TestControllerManager {
                 messages.push(message);
               }
 
-              if (event.attributes.status === "FAIL") {
-                run.failed(item, messages, event.attributes.elapsedtime);
-              } else {
-                run.errored(item, messages, event.attributes.elapsedtime);
+              if (!item?.canResolveChildren) {
+                if (event.attributes.status === "FAIL") {
+                  run.failed(item, messages, event.attributes.elapsedtime);
+                } else {
+                  run.errored(item, messages, event.attributes.elapsedtime);
+                }
               }
             }
             break;