Fix recursive stopping of slips

.github/actions/list-tests-dir/action.yml

@@ -0,0 +1,25 @@
+name: "List Test Files"
+description: "Build JSON matrix of test files"
+inputs:
+  test_dir:
+    description: "Directory to search for test_*.py files"
+    required: true
+  output_prefix:
+    description: "Optional prefix to prepend to each file entry in the output array"
+    required: false
+    default: ""
+outputs:
+  test_files:
+    description: "JSON array of integration test files (relative to tests/)"
+    value: ${{ steps.set-matrix.outputs.files }}
+
+runs:
+  using: "composite"
+  steps:
+    - id: set-matrix
+      shell: bash
+      run: |
+        # Find files in test_dir/ and optionally prefix entries for consumers.
+        PREFIX="${{ inputs.output_prefix }}"
+        FILES=$(find "${{ inputs.test_dir }}" -name "test_*.py" -printf "%P\n" | sed "s|^|${PREFIX}|" | jq -R -s -c 'split("\n")[:-1]')
+        echo "files=$FILES" >> "$GITHUB_OUTPUT"

.github/workflows/integration-tests.yml

@@ -7,7 +7,20 @@ on:
       - 'develop'
 
 jobs:
+  list-integration-tests:
+    runs-on: ubuntu-latest
+    outputs:
+      test_files: ${{ steps.list.outputs.test_files }}
+    steps:
+      - uses: actions/checkout@v5
+      - id: list
+        uses: ./.github/actions/list-tests-dir
+        with:
+          test_dir: tests/integration
+          output_prefix: integration/
+
   integration-tests:
+    needs: list-integration-tests
     runs-on: ubuntu-22.04
     timeout-minutes: 1800
 
@@ -23,33 +36,28 @@ jobs:
       TF_ENABLE_ONEDNN_OPTS: 0
 
     strategy:
+      fail-fast: false
       matrix:
-        test_file:
-          - test_config_files.py
-          - test_portscans.py
-          - test_dataset.py
-          - test_pcap_dataset.py
-          - test_zeek_dataset.py
-          - test_fides.py
-          - test_iris.py
+        test_file: ${{ fromJson(needs.list-integration-tests.outputs.test_files) }}
 
     steps:
-    - uses: actions/checkout@v5
-      with:
-        ref: ${{ github.ref }}
-        fetch-depth: ''
-
-    - name: Start Redis
-      uses: ./.github/actions/start-redis
-
-    - name: Run Integration Tests for ${{ matrix.test_file }}
-      run: |
-        python3 -m pytest tests/integration_tests/${{ matrix.test_file }} -p no:warnings -vv -s -n 3
-
-    - name: Upload Artifacts
-      if: always()
-      uses: actions/upload-artifact@v5
-      with:
-        name: ${{ matrix.test_file }}-integration-tests-output
-        path: |
-          output/integration_tests
+      - uses: actions/checkout@v5
+        with:
+          ref: ${{ github.ref }}
+          fetch-depth: 0
+
+      - name: Start Redis
+        uses: ./.github/actions/start-redis
+
+      - name: Run Integration Tests for ${{ matrix.test_file }}
+        run: |
+          python3 -m pytest tests/${{ matrix.test_file }} -p no:warnings -vv -s -n 3
+
+      - name: Upload Artifacts
+        if: always()
+        uses: actions/upload-artifact@v5
+        with:
+          # Replaces slashes with underscores for valid artifact naming
+          name: ${{ github.run_id }}-${{ strategy.job-index }}-integration-output
+          path: |
+            output/integration

.github/workflows/unit-tests.yml

@@ -7,110 +7,57 @@ on:
       - 'develop'
 
 jobs:
+  # This job finds all test files and creates a JSON array for the matrix
+  list-tests:
+    runs-on: ubuntu-latest
+    timeout-minutes: 120
+    outputs:
+      test_files: ${{ steps.list.outputs.test_files }}
+    steps:
+      - uses: actions/checkout@v5
+      - id: list
+        uses: ./.github/actions/list-tests-dir
+        with:
+          test_dir: tests/unit
 
   unit-tests:
+    needs: list-tests
     runs-on: ubuntu-22.04
     timeout-minutes: 120
 
+    # Do not stop other tests if one file fails
+    strategy:
+      fail-fast: false
+      matrix:
+        test_file: ${{ fromJson(needs.list-tests.outputs.test_files) }}
+
     container:
       image: stratosphereips/slips_dependencies
       # TensorFlow + multiprocessing + pytest -n can hit /dev/shm limits.
       options: --shm-size=2g
 
-
     env:
       TF_CPP_MIN_LOG_LEVEL: 3
       TF_ENABLE_ONEDNN_OPTS: 0
 
-    strategy:
-      matrix:
-        test_file:
-          - test_input.py
-          - test_main.py
-          - test_conn.py
-          - test_downloaded_file.py
-          - test_ssl.py
-          - test_tunnel.py
-          - test_ssh.py
-          - test_dns.py
-          - test_notice.py
-          - test_software.py
-          - test_smtp.py
-          - test_whitelist.py
-          - test_arp.py
-          - test_arp_poisoner.py
-          - test_arp_filter.py
-          - test_blocking.py
-          - test_unblocker.py
-          - test_flow_handler.py
-          - test_horizontal_portscans.py
-          - test_http_analyzer.py
-          - test_vertical_portscans.py
-          - test_network_discovery.py
-          - test_virustotal.py
-          - test_update_file_manager.py
-          - test_threat_intelligence.py
-          - test_slips_utils.py
-          - test_slips.py
-          - test_profiler.py
-          - test_profiler_worker.py
-          - test_profilers_manager.py
-          - test_leak_detector.py
-          - test_ip_info.py
-          - test_evidence.py
-          - test_asn_info.py
-          - test_urlhaus.py
-          - test_markov_chain.py
-          - test_daemon.py
-          - test_go_director.py
-          - test_notify.py
-          - test_checker.py
-          - test_base_model.py
-          - test_set_evidence.py
-          - test_trustdb.py
-          - test_cesnet.py
-          - test_output.py
-          - test_riskiq.py
-          - test_spamhaus.py
-          - test_scan_detections_db.py
-          - test_circllu.py
-          - test_evidence_handler.py
-          - test_evidence_logger.py
-          - test_evidence_formatter.py
-          - test_alert_handler.py
-          - test_redis_manager.py
-          - test_ioc_handler.py
-          - test_timeline.py
-          - test_database.py
-          - test_symbols_handler.py
-          - test_profile_handler.py
-          - test_process_manager.py
-          - test_metadata_manager.py
-          - test_host_ip_manager.py
-          - test_rnn_cc_detection.py
-          - test_idea_format.py
-          - test_fides_sqlite_db.py
-          - test_fides_module.py
-          - test_fides_queues.py
-          - test_fides_bridge.py
-
     steps:
-    - uses: actions/checkout@v5
-      with:
-        ref: ${{ github.ref }}
-        fetch-depth: 0
-
-    - name: Start Redis
-      uses: ./.github/actions/start-redis
-
-    - name: Run Unit Tests for ${{ matrix.test_file }}
-      run: |
-        python3 -m pytest tests/${{ matrix.test_file }} -p no:warnings -vv -s -n 5
-
-    - name: Upload Artifacts
-      if: always()
-      uses: actions/upload-artifact@v5
-      with:
-        name: test_slips_locally-unit-tests-output
-        path: |
-          output/unit_tests
+      - uses: actions/checkout@v5
+        with:
+          ref: ${{ github.ref }}
+          fetch-depth: 0
+
+      - name: Start Redis
+        uses: ./.github/actions/start-redis
+
+      - name: Run Unit Tests for ${{ matrix.test_file }}
+        run: |
+          # The path is reconstructed here using the matrix variable
+          python3 -m pytest tests/unit/${{ matrix.test_file }} -p no:warnings -vv -s -n 5
+
+      - name: Upload Artifacts
+        if: always()
+        uses: actions/upload-artifact@v5
+        with:
+          name: test_slips-output-${{ strategy.job-index }}
+          path: |
+            output/unit_tests

docs/contributing.md

@@ -155,7 +155,7 @@ It all begins when input.py realizes there's no more flows arriving from the zee
 
 It's a good idea to read the code before checking this graph
 
-<img src="https://raw.githubusercontent.com/stratosphereips/StratosphereLinuxIPS/develop/docs/images/how_slips_stops.jpg"
+<img src="https://raw.githubusercontent.com/stratosphereips/StratosphereLinuxIPS/develop/docs/images/how_slips_stops.jpg">
 
 Evidence Handler is the only process that stops but keeps waiting in memory for new msgs to arrive until all other modules are done. because if any of the modules added an evidence, EvidenceHandler should be up to report and handle it or else it will be discarded.
 Once all modules are done processing, EvidenceHandler is killed by the Process manager.

docs/create_new_module.md

@@ -608,7 +608,7 @@ the channel, make sure that:
 
 Slips has 2 kinds of tests, unit tests and integration tests.
 
-integration tests are in ```tests/integration_tests/```, In there we test all files in our ```dataset/``` dir.
+integration tests are in ```tests/integration/```, In there we test all files in our ```dataset/``` dir.
 
 Before pushing, run the unit tests and integration tests by:
 

docs/iris_module.md

@@ -98,7 +98,7 @@ will be left upon the future developers.
 * ```go test ./...```
 
 ### Integration Testing
-Integration tests are located in ```tests/integration_tests/test_iris.py```.
+Integration tests are located in ```tests/integration/test_iris.py```.
 
 ### Test Messaging
 The scenario that was modeled in this test refers to a common use case.

managers/process_manager.py

@@ -18,10 +18,10 @@
     Process,
     Semaphore,
 )
+from multiprocessing.process import BaseProcess
 from typing import (
     List,
     Tuple,
-    Dict,
 )
 
 from exclusiveprocess import (
@@ -82,11 +82,6 @@ def __init__(self, main):
         self.is_profiler_done_event = Event()
         self.read_config()
 
-    def set_slips_processes(self, children: Dict[str, Process]):
-        # this will be set by main.py if slips is not daemonized,
-        # it'll be set to the children of main.py
-        self.processes = children
-
     def read_config(self):
         self.modules_to_ignore: list = self.main.conf.get_disabled_modules(
             self.main.input_type
@@ -206,7 +201,7 @@ def kill_all_children(self):
         kills all processes that are not done
         in self.processes and prints the name of stopped ones
         """
-        for process in self.processes:
+        for process in self.children:
             process: Process
             module_name: str = self.main.db.get_name_of_module_at(process.pid)
             if not module_name:
@@ -433,7 +428,7 @@ def print_started_module(
     def print_stopped_module(self, module):
         self.stopped_modules.append(module)
 
-        modules_left = len(self.processes) - len(self.stopped_modules)
+        modules_left = len(self.children) - len(self.stopped_modules)
 
         # to vertically align them when printing
         module += " " * (20 - len(module))
@@ -578,14 +573,9 @@ def get_hitlist_in_order(self) -> Tuple[List[Process], List[Process]]:
         # now get the process obj of each pid
         to_kill_first: List[Process] = []
         to_kill_last: List[Process] = []
-        for process in self.processes:
+        for process in self.children:
             if process.pid in pids_to_kill_last:
                 to_kill_last.append(process)
-            elif isinstance(process, multiprocessing.context.ForkProcess):
-                # skips the context manager of output.py, will close
-                # it manually later
-                # once all processes are closed
-                continue
             else:
                 to_kill_first.append(process)
 
@@ -625,14 +615,17 @@ def get_analysis_time(self) -> Tuple[str, str]:
             end_time,
         )
 
-    def stop_slips(self) -> bool:
+    def should_stop_slips(self) -> bool:
         """
         determines whether slips should stop
         based on the following:
         1. is slips still receiving new flows? (checks input.py and
         profiler.py)
         2. did slips the control channel recv the stop_slips
         3. is a debugger present?
+
+        This function NEVER returns True if the input and profiler are
+        still processing.
         """
         if self.should_run_non_stop():
             return False
@@ -778,8 +771,8 @@ def get_print_function(self):
 
     def shutdown_gracefully(self):
         """
-        Wait for all modules to confirm that they're done processing
-        or kill them after 15 mins
+        Waits for all modules to confirm that they're done processing
+        or kills them after 15 mins
         """
         try:
             print = self.get_print_function()
@@ -788,6 +781,9 @@ def shutdown_gracefully(self):
                 print("\n" + "-" * 27)
             print("Stopping Slips")
 
+            self.children: List[BaseProcess] = (
+                multiprocessing.active_children()
+            )
             # by default, max 15 mins (taken from wait_for_modules_to_finish)
             # from this time, all modules should be killed
             method_start_time = time.time()