From a6e1ee245a285bd3515b8ce7afec82a919e12b8c Mon Sep 17 00:00:00 2001 From: brandon Date: Thu, 12 Dec 2024 12:05:02 -0800 Subject: [PATCH 1/3] temporary fix for how iqs can be returned --- src/groundlight/internalapi.py | 4 ++-- test/unit/conftest.py | 18 +++++++++++++++++- test/unit/test_internalapi.py | 8 ++++++-- 3 files changed, 25 insertions(+), 5 deletions(-) diff --git a/src/groundlight/internalapi.py b/src/groundlight/internalapi.py index bff192cd..314a673d 100644 --- a/src/groundlight/internalapi.py +++ b/src/groundlight/internalapi.py @@ -65,7 +65,7 @@ def iq_is_confident(iq: ImageQuery, confidence_threshold: float) -> bool: The only subtlety here is that currently confidence of None means human label, which is treated as confident. """ - if not iq.result: + if not iq.result and not iq.result.confidence: return False return iq.result.confidence >= confidence_threshold # type: ignore @@ -74,7 +74,7 @@ def iq_is_answered(iq: ImageQuery) -> bool: """Returns True if the image query has a ML or human label. Placeholder and special labels (out of domain) have confidences exactly 0.5 """ - if not iq.result: + if not iq.result and not iq.result.source: return False if (iq.result.source == Source.STILL_PROCESSING) or (iq.result.source is None): # Should never be None return False diff --git a/test/unit/conftest.py b/test/unit/conftest.py index 472d0c85..4e6f6141 100644 --- a/test/unit/conftest.py +++ b/test/unit/conftest.py @@ -2,7 +2,7 @@ import pytest from groundlight import ExperimentalApi, Groundlight -from model import Detector, ImageQuery +from model import Detector, ImageQuery, ImageQueryTypeEnum, ResultTypeEnum def pytest_configure(config): @@ -46,3 +46,19 @@ def fixture_image_query_no(gl: Groundlight, detector: Detector) -> ImageQuery: @pytest.fixture(name="gl_experimental") def _gl() -> ExperimentalApi: return ExperimentalApi() + +@pytest.fixture(name="initial_iq") +def fixture_initial_iq() -> ImageQuery: + ImageQuery( + id="iq_fakeidhere", + type=ImageQueryTypeEnum.image_query, + created_at=datetime.utcnow(), + query = "Is there a dog?", + detector_id="det_fakeidhere", + result_type=ResultTypeEnum.binary_classification, + result=None, + patience_time=30, + confidence_threshold=0.9, + rois=None, + text=None, + ) \ No newline at end of file diff --git a/test/unit/test_internalapi.py b/test/unit/test_internalapi.py index e6f6fbca..06ab37f1 100644 --- a/test/unit/test_internalapi.py +++ b/test/unit/test_internalapi.py @@ -2,13 +2,17 @@ from groundlight.internalapi import iq_is_answered, iq_is_confident -def test_iq_is_confident(gl_experimental: ExperimentalApi): +def test_iq_is_confident(gl_experimental: ExperimentalApi, initial_iq): det = gl_experimental.get_or_create_detector("Test", "test_query") iq = gl_experimental.ask_async(det, image="test/assets/dog.jpeg") assert not iq_is_confident(iq, 0.9) + assert not iq_is_confident(initial_iq, 0.9) -def test_iq_is_answered(gl_experimental: ExperimentalApi): + +def test_iq_is_answered(gl_experimental: ExperimentalApi, initial_iq): det = gl_experimental.get_or_create_detector("Test", "test_query") iq = gl_experimental.ask_async(det, image="test/assets/dog.jpeg") assert not iq_is_answered(iq) + + assert not iq_is_answered(initial_iq) From 5cd01476c4a1f822b82acdfda129fe008dbdf431 Mon Sep 17 00:00:00 2001 From: Auto-format Bot Date: Thu, 12 Dec 2024 20:06:01 +0000 Subject: [PATCH 2/3] Automatically reformatting code --- test/unit/conftest.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/test/unit/conftest.py b/test/unit/conftest.py index 4e6f6141..dff9565b 100644 --- a/test/unit/conftest.py +++ b/test/unit/conftest.py @@ -47,13 +47,14 @@ def fixture_image_query_no(gl: Groundlight, detector: Detector) -> ImageQuery: def _gl() -> ExperimentalApi: return ExperimentalApi() + @pytest.fixture(name="initial_iq") def fixture_initial_iq() -> ImageQuery: ImageQuery( id="iq_fakeidhere", type=ImageQueryTypeEnum.image_query, created_at=datetime.utcnow(), - query = "Is there a dog?", + query="Is there a dog?", detector_id="det_fakeidhere", result_type=ResultTypeEnum.binary_classification, result=None, @@ -61,4 +62,4 @@ def fixture_initial_iq() -> ImageQuery: confidence_threshold=0.9, rois=None, text=None, - ) \ No newline at end of file + ) From 5fee2b83296a8fec854635e970e88fbfcca2cf2b Mon Sep 17 00:00:00 2001 From: brandon Date: Fri, 13 Dec 2024 13:52:34 -0800 Subject: [PATCH 3/3] fix --- src/groundlight/internalapi.py | 4 ++-- test/unit/conftest.py | 3 ++- test/unit/test_internalapi.py | 5 +++-- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/src/groundlight/internalapi.py b/src/groundlight/internalapi.py index 314a673d..10e1791d 100644 --- a/src/groundlight/internalapi.py +++ b/src/groundlight/internalapi.py @@ -65,7 +65,7 @@ def iq_is_confident(iq: ImageQuery, confidence_threshold: float) -> bool: The only subtlety here is that currently confidence of None means human label, which is treated as confident. """ - if not iq.result and not iq.result.confidence: + if not iq.result or not iq.result.confidence: return False return iq.result.confidence >= confidence_threshold # type: ignore @@ -74,7 +74,7 @@ def iq_is_answered(iq: ImageQuery) -> bool: """Returns True if the image query has a ML or human label. Placeholder and special labels (out of domain) have confidences exactly 0.5 """ - if not iq.result and not iq.result.source: + if not iq.result or not iq.result.source: return False if (iq.result.source == Source.STILL_PROCESSING) or (iq.result.source is None): # Should never be None return False diff --git a/test/unit/conftest.py b/test/unit/conftest.py index 4e6f6141..9ef26616 100644 --- a/test/unit/conftest.py +++ b/test/unit/conftest.py @@ -49,7 +49,8 @@ def _gl() -> ExperimentalApi: @pytest.fixture(name="initial_iq") def fixture_initial_iq() -> ImageQuery: - ImageQuery( + return ImageQuery( + metadata=None, id="iq_fakeidhere", type=ImageQueryTypeEnum.image_query, created_at=datetime.utcnow(), diff --git a/test/unit/test_internalapi.py b/test/unit/test_internalapi.py index 06ab37f1..5f3422de 100644 --- a/test/unit/test_internalapi.py +++ b/test/unit/test_internalapi.py @@ -1,8 +1,9 @@ from groundlight import ExperimentalApi from groundlight.internalapi import iq_is_answered, iq_is_confident +from model import ImageQuery -def test_iq_is_confident(gl_experimental: ExperimentalApi, initial_iq): +def test_iq_is_confident(gl_experimental: ExperimentalApi, initial_iq: ImageQuery): det = gl_experimental.get_or_create_detector("Test", "test_query") iq = gl_experimental.ask_async(det, image="test/assets/dog.jpeg") assert not iq_is_confident(iq, 0.9) @@ -10,7 +11,7 @@ def test_iq_is_confident(gl_experimental: ExperimentalApi, initial_iq): assert not iq_is_confident(initial_iq, 0.9) -def test_iq_is_answered(gl_experimental: ExperimentalApi, initial_iq): +def test_iq_is_answered(gl_experimental: ExperimentalApi, initial_iq: ImageQuery): det = gl_experimental.get_or_create_detector("Test", "test_query") iq = gl_experimental.ask_async(det, image="test/assets/dog.jpeg") assert not iq_is_answered(iq)