From 60fe01d72a4399025897860d16f1e94308aeb5e7 Mon Sep 17 00:00:00 2001 From: Forrest Collman Date: Thu, 20 Nov 2025 13:41:35 -0800 Subject: [PATCH 01/14] add timestamp filter option --- annotationengine/api.py | 18 +++++++++++++++++- requirements.txt | 2 +- 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/annotationengine/api.py b/annotationengine/api.py index 3138031..52086f8 100644 --- a/annotationengine/api.py +++ b/annotationengine/api.py @@ -100,6 +100,14 @@ def handle_invalid_usage(error): location="args", help="whether to only return valid items", ) +query_parser.add_argument( + "timestamp", + type=str, + default=None, + location="args", + required=False, + help="timestamp for filtering results (will default to now if not required)", +) def check_aligned_volume(aligned_volume): @@ -243,11 +251,19 @@ def put(self, aligned_volume_name: str) -> FullMetadataSchema: @api_bp.expect(query_parser) def get(self, aligned_volume_name: str): """Get list of annotation tables for a aligned_volume""" + args = query_parser.parse_args() + timestamp_str = args.get("timestamp", None) + if timestamp_str is None: + timestamp = datetime.datetime.now(datetime.timezone.utc)) + else: + timestamp = datetime.strptime(timestamp_str, '%Y-%m-%d %H:%M:%S.%f') + check_aligned_volume(aligned_volume_name) db = get_db(aligned_volume_name) args = query_parser.parse_args() tables = db.database._get_existing_table_names( - filter_valid=args.get("filter_valid", True) + filter_valid=args.get("filter_valid", True), + filter_timestamp=timestamp ) return tables, 200 diff --git a/requirements.txt b/requirements.txt index ede4846..f74bc14 100644 --- a/requirements.txt +++ b/requirements.txt @@ -29,7 +29,7 @@ decorator==5.1.1 # via ipython deprecated==1.2.13 # via redis -dynamicannotationdb==5.8.1 +dynamicannotationdb==5.14.0 # via -r requirements.in emannotationschemas==5.24.12 # via From 855b588650bc728d0e15601a6b6c354a2ffc6b5b Mon Sep 17 00:00:00 2001 From: Forrest Collman Date: Thu, 20 Nov 2025 13:46:40 -0800 Subject: [PATCH 02/14] bumping dadb version --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index f74bc14..ab77ef7 100644 --- a/requirements.txt +++ b/requirements.txt @@ -29,7 +29,7 @@ decorator==5.1.1 # via ipython deprecated==1.2.13 # via redis -dynamicannotationdb==5.14.0 +dynamicannotationdb==5.14.1 # via -r requirements.in emannotationschemas==5.24.12 # via From 45c428acfc5c9f17e6886b03115d57667130c1ea Mon Sep 17 00:00:00 2001 From: Forrest Collman Date: Thu, 20 Nov 2025 13:55:59 -0800 Subject: [PATCH 03/14] remove excess parentheses --- annotationengine/api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/annotationengine/api.py b/annotationengine/api.py index 52086f8..4be41a9 100644 --- a/annotationengine/api.py +++ b/annotationengine/api.py @@ -254,7 +254,7 @@ def get(self, aligned_volume_name: str): args = query_parser.parse_args() timestamp_str = args.get("timestamp", None) if timestamp_str is None: - timestamp = datetime.datetime.now(datetime.timezone.utc)) + timestamp = datetime.datetime.now(datetime.timezone.utc) else: timestamp = datetime.strptime(timestamp_str, '%Y-%m-%d %H:%M:%S.%f') From 3077c260508aa6ff48ca9d5b68f28b92f8cba2a0 Mon Sep 17 00:00:00 2001 From: Forrest Collman Date: Thu, 4 Dec 2025 13:23:23 -0800 Subject: [PATCH 04/14] fix datetime import --- annotationengine/api.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/annotationengine/api.py b/annotationengine/api.py index 4be41a9..3b3e97b 100644 --- a/annotationengine/api.py +++ b/annotationengine/api.py @@ -16,6 +16,7 @@ from caveclient.auth import AuthClient import werkzeug import traceback +import datetime from annotationengine.aligned_volume import ( get_aligned_volumes, @@ -256,7 +257,7 @@ def get(self, aligned_volume_name: str): if timestamp_str is None: timestamp = datetime.datetime.now(datetime.timezone.utc) else: - timestamp = datetime.strptime(timestamp_str, '%Y-%m-%d %H:%M:%S.%f') + timestamp = datetime.datetime.strptime(timestamp_str, '%Y-%m-%d %H:%M:%S.%f') check_aligned_volume(aligned_volume_name) db = get_db(aligned_volume_name) From 01610c0aa750340313dc4cd020fae4380c1ff8bc Mon Sep 17 00:00:00 2001 From: Forrest Collman Date: Thu, 4 Dec 2025 13:30:04 -0800 Subject: [PATCH 05/14] remove duplicate get_args --- annotationengine/api.py | 1 - 1 file changed, 1 deletion(-) diff --git a/annotationengine/api.py b/annotationengine/api.py index 3b3e97b..e93f199 100644 --- a/annotationengine/api.py +++ b/annotationengine/api.py @@ -261,7 +261,6 @@ def get(self, aligned_volume_name: str): check_aligned_volume(aligned_volume_name) db = get_db(aligned_volume_name) - args = query_parser.parse_args() tables = db.database._get_existing_table_names( filter_valid=args.get("filter_valid", True), filter_timestamp=timestamp From 762bb9f023f951a1a5ff4c79b70dd822d9a729d4 Mon Sep 17 00:00:00 2001 From: Forrest Collman Date: Thu, 4 Dec 2025 13:32:09 -0800 Subject: [PATCH 06/14] fixing timestamp localization --- annotationengine/api.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/annotationengine/api.py b/annotationengine/api.py index e93f199..3277454 100644 --- a/annotationengine/api.py +++ b/annotationengine/api.py @@ -17,6 +17,7 @@ import werkzeug import traceback import datetime +import pytz from annotationengine.aligned_volume import ( get_aligned_volumes, @@ -258,6 +259,7 @@ def get(self, aligned_volume_name: str): timestamp = datetime.datetime.now(datetime.timezone.utc) else: timestamp = datetime.datetime.strptime(timestamp_str, '%Y-%m-%d %H:%M:%S.%f') + timestamp = pytz.utc.localize(timestamp) check_aligned_volume(aligned_volume_name) db = get_db(aligned_volume_name) From eac43d816606b565956bde4eeba1a9f176b69060 Mon Sep 17 00:00:00 2001 From: Forrest Collman Date: Thu, 4 Dec 2025 13:33:56 -0800 Subject: [PATCH 07/14] fixing timestamp parsing --- annotationengine/api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/annotationengine/api.py b/annotationengine/api.py index 3277454..80a9f0c 100644 --- a/annotationengine/api.py +++ b/annotationengine/api.py @@ -258,7 +258,7 @@ def get(self, aligned_volume_name: str): if timestamp_str is None: timestamp = datetime.datetime.now(datetime.timezone.utc) else: - timestamp = datetime.datetime.strptime(timestamp_str, '%Y-%m-%d %H:%M:%S.%f') + timestamp = datetime.datetime.strptime(timestamp_str, '%Y-%m-%dT%H:%M:%S.%f') timestamp = pytz.utc.localize(timestamp) check_aligned_volume(aligned_volume_name) From ae6ef7b572607041a5d979c138ee354aa46b7645 Mon Sep 17 00:00:00 2001 From: Forrest Collman Date: Thu, 19 Feb 2026 13:20:13 -0800 Subject: [PATCH 08/14] bump dadb reqs --- dev.Dockerfile | 4 ++-- dev_requirements.txt | 4 ++-- requirements.in | 2 +- requirements.txt | 10 +--------- 4 files changed, 6 insertions(+), 14 deletions(-) diff --git a/dev.Dockerfile b/dev.Dockerfile index 313ebb3..5174f3a 100644 --- a/dev.Dockerfile +++ b/dev.Dockerfile @@ -4,8 +4,8 @@ RUN git config --global http.sslVerify false && \ mkdir -p /home/nginx/.cloudvolume/secrets && chown -R nginx /home/nginx && usermod -d /home/nginx -s /bin/bash nginx RUN python -m pip install --upgrade pip -COPY dev_requirements.txt /app/. -RUN pip install -r dev_requirements.txt +COPY requirements.txt /app/. +RUN pip install -r requirements.txt COPY timeout.conf /etc/nginx/conf.d/ COPY . /app \ No newline at end of file diff --git a/dev_requirements.txt b/dev_requirements.txt index 23557dc..3fe547c 100644 --- a/dev_requirements.txt +++ b/dev_requirements.txt @@ -31,7 +31,7 @@ deprecated==1.2.13 # via redis dynamicannotationdb==5.2.2 # via -r requirements.in -emannotationschemas==5.1.1 +emannotationschemas==5.24.12 # via # -r requirements.in # dynamicannotationdb @@ -178,7 +178,7 @@ requests==2.27.1 # -r requirements.in # caveclient # middle-auth-client -shapely==1.8.0 +shapely==2.0.3 # via # dynamicannotationdb # emannotationschemas diff --git a/requirements.in b/requirements.in index f451fe4..d553c07 100644 --- a/requirements.in +++ b/requirements.in @@ -1,6 +1,6 @@ Flask<2.0 emannotationschemas>=5.24.12 -dynamicannotationdb>=5.5.1 +dynamicannotationdb>=5.14.1 caveclient>=4.20.2 multiwrapper jsonschema<4.0 diff --git a/requirements.txt b/requirements.txt index ab77ef7..c2cc4e5 100644 --- a/requirements.txt +++ b/requirements.txt @@ -27,8 +27,6 @@ click==7.1.2 # via flask decorator==5.1.1 # via ipython -deprecated==1.2.13 - # via redis dynamicannotationdb==5.14.1 # via -r requirements.in emannotationschemas==5.24.12 @@ -121,9 +119,7 @@ numpy==1.21.5 orderedmultidict==1.0.1 # via furl packaging==21.3 - # via - # geoalchemy2 - # redis + # via geoalchemy2 pandas==1.3.5 # via caveclient parso==0.8.3 @@ -153,8 +149,6 @@ pytz==2021.3 # dynamicannotationdb # flask-restx # pandas -redis==4.1.3 - # via middle-auth-client requests==2.27.1 # via # -r requirements.in @@ -200,7 +194,5 @@ werkzeug==1.0.1 # flask # flask-accepts # flask-restx -wrapt==1.13.3 - # via deprecated wtforms==3.0.1 # via flask-admin From 9c561b504ea7398da91ec20b8b993154172dcff5 Mon Sep 17 00:00:00 2001 From: Forrest Collman Date: Thu, 19 Feb 2026 13:29:43 -0800 Subject: [PATCH 09/14] fixing timestamp parsing --- annotationengine/api.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/annotationengine/api.py b/annotationengine/api.py index 80a9f0c..d1818b5 100644 --- a/annotationengine/api.py +++ b/annotationengine/api.py @@ -258,8 +258,15 @@ def get(self, aligned_volume_name: str): if timestamp_str is None: timestamp = datetime.datetime.now(datetime.timezone.utc) else: - timestamp = datetime.datetime.strptime(timestamp_str, '%Y-%m-%dT%H:%M:%S.%f') - timestamp = pytz.utc.localize(timestamp) + # fromisoformat accepts common ISO 8601 formats with or without microseconds + # (e.g. 2026-02-19T10:30:00 or 2026-02-19T10:30:00.123456) + timestamp = datetime.datetime.fromisoformat( + timestamp_str.replace("Z", "+00:00") + ) + if timestamp.tzinfo is None: + timestamp = pytz.utc.localize(timestamp) + else: + timestamp = timestamp.astimezone(pytz.utc) check_aligned_volume(aligned_volume_name) db = get_db(aligned_volume_name) From e35c6cdd74147fe225b6cc110b81374a3efece27 Mon Sep 17 00:00:00 2001 From: Forrest Collman Date: Thu, 19 Feb 2026 13:30:36 -0800 Subject: [PATCH 10/14] updating python version to 3.10 --- .github/workflows/annotationengine.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/annotationengine.yml b/.github/workflows/annotationengine.yml index 3d738f3..b3aa7b1 100644 --- a/.github/workflows/annotationengine.yml +++ b/.github/workflows/annotationengine.yml @@ -34,10 +34,10 @@ jobs: options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5 steps: - uses: actions/checkout@v2 - - name: Set up Python 3.9 + - name: Set up Python 3.10 uses: actions/setup-python@v2 with: - python-version: 3.9 + python-version: 3.10 - uses: actions/cache@v4 with: path: ~/.cache/pip @@ -48,7 +48,7 @@ jobs: with: auto-update-conda: true auto-activate-base: true - python-version: 3.9 + python-version: 3.10 - name: Install dependencies shell: bash -l {0} run: | From 8b26cc0c839f932b6f8913c31f508bbe94f03d9b Mon Sep 17 00:00:00 2001 From: Forrest Collman Date: Thu, 19 Feb 2026 13:32:21 -0800 Subject: [PATCH 11/14] trying in quotes --- .github/workflows/annotationengine.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/annotationengine.yml b/.github/workflows/annotationengine.yml index b3aa7b1..5708e4e 100644 --- a/.github/workflows/annotationengine.yml +++ b/.github/workflows/annotationengine.yml @@ -37,7 +37,7 @@ jobs: - name: Set up Python 3.10 uses: actions/setup-python@v2 with: - python-version: 3.10 + python-version: "3.10" - uses: actions/cache@v4 with: path: ~/.cache/pip @@ -48,7 +48,7 @@ jobs: with: auto-update-conda: true auto-activate-base: true - python-version: 3.10 + python-version: "3.10" - name: Install dependencies shell: bash -l {0} run: | From 86762c2a09c6fbb2a851212ec5536d470421090a Mon Sep 17 00:00:00 2001 From: Forrest Collman Date: Thu, 19 Feb 2026 13:49:36 -0800 Subject: [PATCH 12/14] adding debug statements to client test creation --- test/conftest.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/test/conftest.py b/test/conftest.py index 75a00cd..f3f0411 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -66,8 +66,11 @@ def client(): # Establish an application context with flask_app.app_context(): db.create_all() + logging.info("yielding testing_client") yield testing_client + logging.info("yielding testing_client") db.drop_all() + logging.info("dropped all tables") @pytest.fixture(scope="module") From 4b9d8abcca99f40e668467c499ad5ed41ac0f23f Mon Sep 17 00:00:00 2001 From: Forrest Collman Date: Thu, 19 Feb 2026 13:52:08 -0800 Subject: [PATCH 13/14] fixing clean up of testing infrastructure --- test/conftest.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/test/conftest.py b/test/conftest.py index f3f0411..bc19a7b 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -68,9 +68,16 @@ def client(): db.create_all() logging.info("yielding testing_client") yield testing_client - logging.info("yielding testing_client") + # --- Cleanup Phase --- + logging.info("Tearing down testing_client") + # 1. Remove the scoped session to prevent "ResourceBusy" errors + db.session.remove() + # 2. Drop the tables db.drop_all() logging.info("dropped all tables") + # 3. Dispose of the engine to kill the connection pool (The Fix) + db.engine.dispose() + logging.info("SQLAlchemy engine disposed") @pytest.fixture(scope="module") From f30b0e6cb2c9e6f719584800092d643946bbae73 Mon Sep 17 00:00:00 2001 From: Forrest Collman Date: Thu, 19 Feb 2026 13:59:44 -0800 Subject: [PATCH 14/14] add more logging --- test/conftest.py | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/test/conftest.py b/test/conftest.py index bc19a7b..b7b1260 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -58,7 +58,8 @@ def test_aligned_volume(): @pytest.fixture(scope="module") def client(): - flask_app = create_app(config_name="testing") + config_name = os.environ.get("FLASK_CONFIGURATION", "testing") + flask_app = create_app(config_name=config_name) test_logger.info("Starting test flask app...") # Create a test client using the Flask application configured for testing @@ -69,15 +70,20 @@ def client(): logging.info("yielding testing_client") yield testing_client # --- Cleanup Phase --- - logging.info("Tearing down testing_client") - # 1. Remove the scoped session to prevent "ResourceBusy" errors - db.session.remove() + logging.info("Tearing down testing_client, committing session") + + db.session.commit() + + logging.info("dropping all tables") # 2. Drop the tables db.drop_all() - logging.info("dropped all tables") + logging.info("removing session") + # 1. Remove the scoped session to prevent "ResourceBusy" errors + db.session.remove() + logging.info("disposing engine") # 3. Dispose of the engine to kill the connection pool (The Fix) db.engine.dispose() - logging.info("SQLAlchemy engine disposed") + logging.info("cleanup complete") @pytest.fixture(scope="module")