Added '_Lng_LVCC' files to test dataset and added more checks to ensure that the ISPyB entries all register the denoised datasets in their final state

Author: tieneupin

tests/workflows/clem/test_register_preprocessing_results.py

@@ -47,7 +47,7 @@ def generate_preprocessing_messages(
     # Construct all the datasets to be tested
     datasets: list[tuple[Path, bool, bool, tuple[int, int], float, list[float]]] = [
         (
-            grid_dir / "Overview_1" / "Image_1",
+            grid_dir / "Overview 1" / "Image 1",
             False,
             True,
             (2400, 2400),
@@ -59,22 +59,38 @@ def generate_preprocessing_messages(
     datasets.extend(
         [
             (
-                grid_dir / "TileScan_1" / f"Position_{n}",
+                grid_dir / "TileScan 1" / f"Position {n + 1}",
                 True,
                 False,
                 (2048, 2048),
                 1.6e-7,
                 [0.003, 0.00332768, 0.003, 0.00332768],
             )
-            for n in range(5)
+            for n in range(3)
+        ]
+    )
+    datasets.extend(
+        [
+            (
+                grid_dir / "TileScan 1" / f"Position {n + 1}_Lng_LVCC",
+                True,
+                False,
+                (2048, 2048),
+                1.6e-7,
+                [0.003, 0.00332768, 0.003, 0.00332768],
+            )
+            for n in range(3)
         ]
     )
 
     messages: list[dict[str, Any]] = []
-    for dataset in datasets:
+    for series_path, is_stack, is_montage, shape, pixel_size, extent in datasets:
         # Unpack items from list of dataset parameters
-        series_path = dataset[0]
-        series_name = str(series_path.relative_to(processed_dir)).replace("/", "--")
+        series_name = (
+            str(series_path.relative_to(processed_dir))
+            .replace("/", "--")
+            .replace(" ", "_")
+        )
         metadata = series_path / "metadata" / f"{series_path.stem}.xml"
         metadata.parent.mkdir(parents=True, exist_ok=True)
         metadata.touch(exist_ok=True)
@@ -89,11 +105,6 @@ def generate_preprocessing_messages(
                 thumbnail.parent.mkdir(parents=True)
             thumbnail.touch(exist_ok=True)
         thumbnail_size = (512, 512)
-        is_stack = dataset[1]
-        is_montage = dataset[2]
-        shape = dataset[3]
-        pixel_size = dataset[4]
-        extent = dataset[5]
 
         message = {
             "session_id": session_id,
@@ -373,21 +384,23 @@ def test_run_with_db(
     else:
         assert mock_align_and_merge_call.call_count == len(preprocessing_messages) * 3
 
-    # Both databases should have entries for data collection group, and grid squares
-    # ISPyB database should additionally have an atlas entry
+    # Murfey's DataCollectionGroup should have an entry
     murfey_dcg_search = murfey_db_session.exec(
         sm_select(MurfeyDB.DataCollectionGroup).where(
             MurfeyDB.DataCollectionGroup.session_id == murfey_session.id
         )
     ).all()
     assert len(murfey_dcg_search) == 1
+
+    # GridSquare entries should be half the initial number of entries due to overwrites
     murfey_gs_search = murfey_db_session.exec(
         sm_select(MurfeyDB.GridSquare).where(
             MurfeyDB.GridSquare.session_id == murfey_session.id
         )
     ).all()
-    assert len(murfey_gs_search) == len(preprocessing_messages) - 1
+    assert len(murfey_gs_search) == (len(preprocessing_messages) - 1) // 2
 
+    # ISPyB's DataCollectionGroup should have an entry
     murfey_dcg = murfey_dcg_search[0]
     ispyb_dcg_search = (
         ispyb_db_session.execute(
@@ -400,6 +413,7 @@ def test_run_with_db(
     )
     assert len(ispyb_dcg_search) == 1
 
+    # Atlas should have an entry
     ispyb_dcg = ispyb_dcg_search[0]
     ispyb_atlas_search = (
         ispyb_db_session.execute(
@@ -419,12 +433,13 @@ def test_run_with_db(
     }
     collection_mode = _determine_collection_mode(colors)
 
+    # Atlas color flags and collection mode should be set correctly
     ispyb_atlas = ispyb_atlas_search[0]
-    # Check that the Atlas color flags and collection mode are set correctly
     for flag, value in color_flags.items():
         assert getattr(ispyb_atlas, flag) == value
     assert ispyb_atlas.mode == collection_mode
 
+    # ISPyB's GrridSquare should have half the number of intiail entries
     ispyb_gs_search = (
         ispyb_db_session.execute(
             sa_select(ISPyBDB.GridSquare).where(
@@ -434,9 +449,12 @@ def test_run_with_db(
         .scalars()
         .all()
     )
-    assert len(ispyb_gs_search) == len(preprocessing_messages) - 1
+    assert len(ispyb_gs_search) == (len(preprocessing_messages) - 1) // 2
     for gs in ispyb_gs_search:
-        # Check that the Atlas color flags and collection mode are set correctly
+        # Check that all entries point to the denoised images ("_Lng_LVCC")
+        assert gs.gridSquareImage is not None and "_Lng_LVCC" in gs.gridSquareImage
+
+        # Check that the GridSquare color flags and collection mode are set correctly
         for flag, value in color_flags.items():
             assert getattr(gs, flag) == value
         assert gs.mode == collection_mode