From 9b1c769990afaa7c6ef9669197ce252f6a1fca44 Mon Sep 17 00:00:00 2001 From: Nick Macholl Date: Tue, 24 Jun 2025 08:58:53 -0700 Subject: [PATCH 1/5] ADD: Add metadata.get_dataset_range schema ranges --- databento/historical/api/metadata.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/databento/historical/api/metadata.py b/databento/historical/api/metadata.py index 971005c..3869170 100644 --- a/databento/historical/api/metadata.py +++ b/databento/historical/api/metadata.py @@ -243,7 +243,7 @@ def get_dataset_range( Returns ------- - dict[str, str] + dict[str, str | dict[str, str]] The available range for the dataset. """ From ae7646f5a793863d6a1664a6d273e28100e93dd6 Mon Sep 17 00:00:00 2001 From: Nick Macholl Date: Mon, 30 Jun 2025 10:37:26 -0700 Subject: [PATCH 2/5] MOD: Change DBNStore.to_df() timezone type --- CHANGELOG.md | 5 ++++ databento/common/dbnstore.py | 24 ++++++++++------- tests/test_historical_bento.py | 47 +++++++++++++++++++++++++++++++--- 3 files changed, 64 insertions(+), 12 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 61045d2..2eaf14a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,10 @@ # Changelog +## 0.57.2 - TBD + +#### Enhancements +- Changed the `tz` parameter in `DBNStore.to_df()` to accept `datetime.tzinfo` instead of `pytz.BaseTzInfo` explicitly + ## 0.57.1 - 2025-06-17 #### Enhancements diff --git a/databento/common/dbnstore.py b/databento/common/dbnstore.py index 9c2d924..047aef7 100644 --- a/databento/common/dbnstore.py +++ b/databento/common/dbnstore.py @@ -1,10 +1,12 @@ from __future__ import annotations import abc +import datetime import decimal import itertools import logging import warnings +import zoneinfo from collections.abc import Generator from collections.abc import Iterator from collections.abc import Mapping @@ -28,7 +30,6 @@ import pandas as pd import pyarrow as pa import pyarrow.parquet as pq -import pytz import zstandard from databento_dbn import FIXED_PRICE_SCALE from databento_dbn import UNDEF_PRICE @@ -859,7 +860,7 @@ def to_df( pretty_ts: bool = ..., map_symbols: bool = ..., schema: Schema | str | None = ..., - tz: pytz.BaseTzInfo | str = ..., + tz: datetime.tzinfo | str = ..., count: None = ..., ) -> pd.DataFrame: ... @@ -870,7 +871,7 @@ def to_df( pretty_ts: bool = ..., map_symbols: bool = ..., schema: Schema | str | None = ..., - tz: pytz.BaseTzInfo | str = ..., + tz: datetime.tzinfo | str = ..., count: int = ..., ) -> DataFrameIterator: ... @@ -880,8 +881,8 @@ def to_df( pretty_ts: bool = True, map_symbols: bool = True, schema: Schema | str | None = None, - tz: pytz.BaseTzInfo | str | Default[pytz.BaseTzInfo] = Default[pytz.BaseTzInfo]( - pytz.UTC, + tz: datetime.tzinfo | str | Default[datetime.tzinfo] = Default[datetime.tzinfo]( + datetime.timezone.utc, ), count: int | None = None, ) -> pd.DataFrame | DataFrameIterator: @@ -909,7 +910,7 @@ def to_df( schema : Schema or str, optional The DBN schema for the dataframe. This is only required when reading a DBN stream with mixed record types. - tz : pytz.BaseTzInfo or str, default UTC + tz : datetime.tzinfo or str, default UTC If `pretty_ts` is `True`, all timestamps will be converted to the specified timezone. count : int, optional If set, instead of returning a single `DataFrame` a `DataFrameIterator` @@ -939,8 +940,13 @@ def to_df( "A timezone was specified when `pretty_ts` is `False`. Did you mean to set `pretty_ts=True`?", ) - if not isinstance(tz, pytz.BaseTzInfo): - tz = pytz.timezone(tz) + if isinstance(tz, str): + tz = zoneinfo.ZoneInfo(tz) + elif not isinstance(tz, datetime.tzinfo): + raise ValueError( + f"The value {tz!r} is not a valid datetime.tzinfo", + ) + if schema is None: if self.schema is None: raise ValueError("a schema must be specified for mixed DBN data") @@ -1442,7 +1448,7 @@ def __init__( count: int | None, struct_type: type[DBNRecord], instrument_map: InstrumentMap, - tz: pytz.BaseTzInfo, + tz: datetime.tzinfo, price_type: PriceType = PriceType.FLOAT, pretty_ts: bool = True, map_symbols: bool = True, diff --git a/tests/test_historical_bento.py b/tests/test_historical_bento.py index 707aca3..e4c4e5a 100644 --- a/tests/test_historical_bento.py +++ b/tests/test_historical_bento.py @@ -1,6 +1,7 @@ import collections import datetime as dt import decimal +import zoneinfo from io import BytesIO from pathlib import Path from typing import Any @@ -1567,11 +1568,51 @@ def test_dbnstore_to_df_with_timezone( df.reset_index(inplace=True) # Assert - expected_timezone = pytz.timezone(timezone)._utcoffset + expected_timezone = zoneinfo.ZoneInfo(timezone).utcoffset(None) failures = [] struct = SCHEMA_STRUCT_MAP[schema] for field in struct._timestamp_fields: - if df[field].dt.tz._utcoffset != expected_timezone: + if df[field].dt.tz.utcoffset(None) != expected_timezone: + failures.append(field) + + assert not failures + + +@pytest.mark.parametrize( + "timezone", + [ + pytz.timezone("US/Central"), + pytz.timezone("US/Eastern"), + pytz.timezone("Europe/Vienna"), + pytz.timezone("Asia/Dubai"), + pytz.timezone("UTC"), + ], +) +@pytest.mark.parametrize( + "schema", + [pytest.param(schema, id=str(schema)) for schema in Schema.variants()], +) +def test_dbnstore_to_df_with_pytz_timezone( + test_data_path: Callable[[Dataset, Schema], Path], + schema: Schema, + timezone: pytz.BaseTzInfo, +) -> None: + """ + Test that setting the `tz` parameter in `DBNStore.to_df` accepts `pytz` + timezone objects. + """ + # Arrange + dbnstore = DBNStore.from_file(path=test_data_path(Dataset.GLBX_MDP3, schema)) + + # Act + df = dbnstore.to_df(tz=timezone) + df.reset_index(inplace=True) + + # Assert + failures = [] + struct = SCHEMA_STRUCT_MAP[schema] + for field in struct._timestamp_fields: + if df[field].dt.tz != timezone: failures.append(field) assert not failures @@ -1591,7 +1632,7 @@ def test_dbnstore_to_df_with_timezone_pretty_ts_error( with pytest.raises(ValueError): dbnstore.to_df( pretty_ts=False, - tz=pytz.UTC, + tz=zoneinfo.ZoneInfo("UTC"), ) From dc644584589dee7d4510182c960db089b520f655 Mon Sep 17 00:00:00 2001 From: Nick Macholl Date: Fri, 4 Jul 2025 10:51:21 -0700 Subject: [PATCH 3/5] BLD: Allow compatible patches of databento_dbn --- CHANGELOG.md | 3 ++- README.md | 4 ++-- pyproject.toml | 2 +- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2eaf14a..2c3bc70 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,9 +1,10 @@ # Changelog -## 0.57.2 - TBD +## 0.58.0 - TBD #### Enhancements - Changed the `tz` parameter in `DBNStore.to_df()` to accept `datetime.tzinfo` instead of `pytz.BaseTzInfo` explicitly +- Modified the dependency specification for `databento_dbn` to allow for compatible patch versions ## 0.57.1 - 2025-06-17 diff --git a/README.md b/README.md index 917263c..97c7c12 100644 --- a/README.md +++ b/README.md @@ -32,8 +32,8 @@ The library is fully compatible with the latest distribution of Anaconda 3.9 and The minimum dependencies as found in the `pyproject.toml` are also listed below: - python = "^3.9" - aiohttp = "^3.8.3" -- databento-dbn = "0.36.1" -- numpy= ">=1.23.5" +- databento-dbn = "~0.36.1" +- numpy = ">=1.23.5" - pandas = ">=1.5.3" - pip-system-certs = ">=4.0" (Windows only) - pyarrow = ">=13.0.0" diff --git a/pyproject.toml b/pyproject.toml index 73b15f8..3155883 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -32,7 +32,7 @@ aiohttp = [ {version = "^3.8.3", python = "<3.12"}, {version = "^3.9.0", python = "^3.12"} ] -databento-dbn = "0.36.1" +databento-dbn = "~=0.36.1" numpy = [ {version = ">=1.23.5", python = "<3.12"}, {version = ">=1.26.0", python = "^3.12"} From 7b6a96f3f6141660566d43ce02c89a7224b8414d Mon Sep 17 00:00:00 2001 From: Nick Macholl Date: Tue, 8 Jul 2025 09:59:02 -0700 Subject: [PATCH 4/5] MOD: Upgrade databento-dbn to 0.36.2 --- CHANGELOG.md | 2 ++ pyproject.toml | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2c3bc70..8862023 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,8 @@ #### Enhancements - Changed the `tz` parameter in `DBNStore.to_df()` to accept `datetime.tzinfo` instead of `pytz.BaseTzInfo` explicitly - Modified the dependency specification for `databento_dbn` to allow for compatible patch versions +- Upgraded `databento-dbn` to 0.36.2 + - Fixed change in behavior where Python `DBNDecoder.decode()` wouldn't always decode all available data on the first call ## 0.57.1 - 2025-06-17 diff --git a/pyproject.toml b/pyproject.toml index 3155883..e85dd4a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -32,7 +32,7 @@ aiohttp = [ {version = "^3.8.3", python = "<3.12"}, {version = "^3.9.0", python = "^3.12"} ] -databento-dbn = "~=0.36.1" +databento-dbn = "~=0.36.2" numpy = [ {version = ">=1.23.5", python = "<3.12"}, {version = ">=1.26.0", python = "^3.12"} From 6cf45e53fb5b7159a365ea51f675e0f752cbbfdc Mon Sep 17 00:00:00 2001 From: Nick Macholl Date: Tue, 8 Jul 2025 09:56:02 -0700 Subject: [PATCH 5/5] VER: Release 0.58.0 --- CHANGELOG.md | 2 +- databento/version.py | 2 +- pyproject.toml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8862023..6330e20 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,6 @@ # Changelog -## 0.58.0 - TBD +## 0.58.0 - 2025-07-08 #### Enhancements - Changed the `tz` parameter in `DBNStore.to_df()` to accept `datetime.tzinfo` instead of `pytz.BaseTzInfo` explicitly diff --git a/databento/version.py b/databento/version.py index 8685210..45dbfca 100644 --- a/databento/version.py +++ b/databento/version.py @@ -1 +1 @@ -__version__ = "0.57.1" +__version__ = "0.58.0" diff --git a/pyproject.toml b/pyproject.toml index e85dd4a..f00cccc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "databento" -version = "0.57.1" +version = "0.58.0" description = "Official Python client library for Databento" authors = [ "Databento ",