From 5922da9850e90fce535bf75da8cd986b4d96565e Mon Sep 17 00:00:00 2001 From: terraputix Date: Thu, 15 May 2025 18:23:09 +0200 Subject: [PATCH 01/50] wip: om domains --- python/omfiles/om_domains.py | 313 +++++++++++++++++++++++++++++++++++ python/omfiles/utils.py | 17 ++ tests/test_om_domains.py | 183 ++++++++++++++++++++ 3 files changed, 513 insertions(+) create mode 100644 python/omfiles/om_domains.py create mode 100644 python/omfiles/utils.py create mode 100644 tests/test_om_domains.py diff --git a/python/omfiles/om_domains.py b/python/omfiles/om_domains.py new file mode 100644 index 00000000..684dd490 --- /dev/null +++ b/python/omfiles/om_domains.py @@ -0,0 +1,313 @@ +from abc import ABC, abstractmethod +from functools import cached_property +from typing import Optional, Tuple + +import numpy as np + +from omfiles.utils import _modulo_positive + + +class AbstractGrid(ABC): + """ + Abstract base class for weather model grid definitions. + + This defines the interface that all grid implementations must follow. + """ + + @property + @abstractmethod + def grid_type(self) -> str: + """Return the grid type identifier.""" + pass + + @cached_property + @abstractmethod + def latitude(self) -> np.ndarray: + """ + Return the latitude coordinates array. + + Uses cached_property to ensure the array is computed only once. + """ + pass + + @cached_property + @abstractmethod + def longitude(self) -> np.ndarray: + """ + Return the longitude coordinates array. + + Uses cached_property to ensure the array is computed only once. + """ + pass + + @property + @abstractmethod + def shape(self) -> Tuple[int, int]: + """Return the grid shape as (n_lat, n_lon).""" + pass + + @abstractmethod + def findPointXy(self, lat: float, lon: float) -> Optional[Tuple[int, int]]: + """ + Find grid point indices (x, y) for given lat/lon coordinates. + """ + pass + + @abstractmethod + def getCoordinates(self, x: int, y: int) -> Tuple[float, float]: + """ + Get lat/lon coordinates for a given grid point indices. + """ + pass + + + +class RegularLatLonGrid(AbstractGrid): + """ + Regular latitude-longitude grid implementation. + + This represents a standard equirectangular grid with uniform spacing. + """ + + def __init__( + self, + lat_start: float, + lat_steps: int, + lat_step_size: float, + lon_start: float, + lon_steps: int, + lon_step_size: float, + ): + """ + Initialize a regular lat/lon grid. + + Parameters: + ----------- + lat_start : float + Starting latitude value + lat_steps : int + Number of latitude points + lat_step_size : float + Spacing between latitude points + lon_start : float + Starting longitude value + lon_steps : int + Number of longitude points + lon_step_size : float + Spacing between longitude points + """ + self._lat_start = lat_start + self._lat_steps = lat_steps + self._lat_step_size = lat_step_size + self._lon_start = lon_start + self._lon_steps = lon_steps + self._lon_step_size = lon_step_size + + @property + def grid_type(self) -> str: + return "regular_latlon" + + @cached_property + def latitude(self) -> np.ndarray: + """ + Lazily compute and cache the latitude coordinate array. + """ + return np.linspace( + self._lat_start, + self._lat_start + self._lat_step_size * self._lat_steps, + self._lat_steps, + endpoint=False + ) + + @cached_property + def longitude(self) -> np.ndarray: + """ + Lazily compute and cache the longitude coordinate array. + """ + return np.linspace( + self._lon_start, + self._lon_start + self._lon_step_size * self._lon_steps, + self._lon_steps, + endpoint=False + ) + + @property + def shape(self) -> Tuple[int, int]: + return (self._lat_steps, self._lon_steps) + + def findPointXy(self, lat: float, lon: float) -> Optional[Tuple[int, int]]: + """ + Find grid point indices (x, y) for given lat/lon coordinates. + + Parameters: + ----------- + lat : float + Latitude in degrees + lon : float + Longitude in degrees + + Returns: + -------- + tuple or None + (x, y) grid indices if point is in grid, None otherwise + """ + # Calculate raw x and y indices + x = int(round((lon - self._lon_start) / self._lon_step_size)) + y = int(round((lat - self._lat_start) / self._lat_step_size)) + + # Handle wrapping for global grids + xx = _modulo_positive(x, self._lon_steps) if (self._lon_steps * self._lon_step_size) >= 359 else x + yy = _modulo_positive(y, self._lat_steps) if (self._lat_steps * self._lat_step_size) >= 179 else y + + # Check if point is within grid bounds + if yy < 0 or xx < 0 or yy >= self._lat_steps or xx >= self._lon_steps: + return None + + return (xx, yy) + + def getCoordinates(self, x: int, y: int) -> Tuple[float, float]: + """ + Get lat/lon coordinates for a given grid point indices. + + Parameters: + ----------- + x : longitude index + y : latitude index + + Returns: + -------- + tuple + (latitude, longitude) coordinates + """ + + lat = self._lat_start + float(y) * self._lat_step_size + lon = self._lon_start + float(x) * self._lon_step_size + + return (lat, lon) + +class OmDomain: + """ + Class representing a domain configuration for a weather model. + + This class provides metadata and configuration for different + weather model grids used in Open-Meteo. + """ + + def __init__( + self, + name: str, + grid: AbstractGrid, + file_length: int, + temporal_resolution_seconds: int = 3600 + ): + """ + Initialize a domain configuration. + + Parameters: + ----------- + name : str + Name of the domain + grid : AbstractGrid + Grid implementation for this domain + file_length : int + Number of time steps in each file chunk + description : str, optional + Human-readable description of the domain + variables : list, optional + List of variable names available in this domain + temporal_resolution_seconds : int, optional + Time resolution in seconds (default: 3600 = 1 hour) + """ + self.name = name + self.grid = grid + self.file_length = file_length + self.temporal_resolution_seconds = temporal_resolution_seconds + + def time_to_chunk_index(self, timestamp: np.datetime64) -> int: + """ + Convert a timestamp to a chunk index. + + Parameters: + ----------- + timestamp : np.datetime64 + The timestamp to convert + + Returns: + -------- + int + The chunk index containing the timestamp + """ + # Basic implementation - can be extended with domain-specific logic + epoch = np.datetime64('1970-01-01T00:00:00') + seconds_since_epoch = (timestamp - epoch) / np.timedelta64(1, 's') + + # Get temporal resolution from metadata (placeholder - real implementation would get from metadata) + temporal_resolution = 3600 # 1 hour in seconds + + # Calculate chunk index + chunk_index = int(seconds_since_epoch / (self.file_length * temporal_resolution)) + return chunk_index + + def get_chunk_time_range(self, chunk_index: int, temporal_resolution_seconds: int = 3600) -> np.ndarray: + """ + Get the time range covered by a specific chunk. + + Parameters: + ----------- + chunk_index : int + Index of the chunk + temporal_resolution_seconds : int, optional + Time resolution in seconds (default: 3600 = 1 hour) + + Returns: + -------- + np.ndarray + Array of datetime64 objects representing the time points in the chunk + """ + epoch = np.datetime64('1970-01-01T00:00:00') + chunk_start_seconds = chunk_index * self.file_length * temporal_resolution_seconds + start_time = epoch + np.timedelta64(chunk_start_seconds, 's') + + # Create array of time points + time_points = start_time + np.arange(self.file_length) * np.timedelta64(temporal_resolution_seconds, 's') + return time_points + +# - MARK: Create grid instances for supported domains + +# DWD ICON D2 is regularized during download to nx: 1215, ny: 746 points +# https://github.com/open-meteo/open-meteo/blob/1753ebb4966d05f61b17dd5bdf59700788d4a913/Sources/App/Icon/Icon.swift#L154 +_dwd_icon_d2_grid = RegularLatLonGrid( + lat_start=43.18, + lat_steps=746, + lat_step_size=0.02, + lon_start=-3.94, + lon_steps=1215, + lon_step_size=0.02 +) + +# ECMWF IQFS grid is a regular global lat/lon grid, nx: 1440, ny: 721 points +# https://github.com/open-meteo/open-meteo/blob/1753ebb4966d05f61b17dd5bdf59700788d4a913/Sources/App/Ecmwf/EcmwfDomain.swift#L107 +_ecmwf_ifs025_grid = RegularLatLonGrid( + lat_start=-90, + lat_steps=721, + lat_step_size=360/1440, + lon_start=-180, + lon_steps=1440, + lon_step_size=180/(721-1) +) + +DOMAINS: dict[str, OmDomain] = { + 'dwd_icon_d2': OmDomain( + name='dwd_icon_d2', + grid=_dwd_icon_d2_grid, + file_length=121, + temporal_resolution_seconds=3600 # 1 hour + ), + 'ecmwf_ifs025': OmDomain( + name='ecmwf_ifs025', + grid=_ecmwf_ifs025_grid, + file_length=104, + temporal_resolution_seconds=3600 # 1 hour + ), + # Additional domains can be added here +} diff --git a/python/omfiles/utils.py b/python/omfiles/utils.py new file mode 100644 index 00000000..7b404850 --- /dev/null +++ b/python/omfiles/utils.py @@ -0,0 +1,17 @@ +def _modulo_positive(value: int, modulo: int) -> int: + """ + Calculate modulo that always returns positive value. + + Parameters: + ----------- + value : int + Value to calculate modulo for + modulo : int + Modulo value + + Returns: + -------- + int + Positive modulo result + """ + return ((value % modulo) + modulo) % modulo diff --git a/tests/test_om_domains.py b/tests/test_om_domains.py new file mode 100644 index 00000000..291dbc32 --- /dev/null +++ b/tests/test_om_domains.py @@ -0,0 +1,183 @@ +import numpy as np +from omfiles.om_domains import DOMAINS, RegularLatLonGrid + + +def test_regular_grid_findPointXy_inside(): + """Test finding grid points inside the domain.""" + grid = RegularLatLonGrid( + lat_start=0.0, + lat_steps=10, + lat_step_size=1.0, + lon_start=0.0, + lon_steps=20, + lon_step_size=1.0 + ) + + # Test exact grid points + assert grid.findPointXy(5.0, 10.0) == (10, 5) + assert grid.findPointXy(0.0, 0.0) == (0, 0) + assert grid.findPointXy(9.0, 19.0) == (19, 9) + + # Test points that should round to grid points + assert grid.findPointXy(5.1, 10.2) == (10, 5) + assert grid.findPointXy(0.4, 0.4) == (0, 0) + + +def test_regular_grid_findPointXy_outside(): + """Test finding grid points outside the domain.""" + grid = RegularLatLonGrid( + lat_start=0.0, + lat_steps=10, + lat_step_size=1.0, + lon_start=0.0, + lon_steps=20, + lon_step_size=1.0 + ) + + # Test points outside of grid + assert grid.findPointXy(-1.0, 10.0) is None + assert grid.findPointXy(5.0, -1.0) is None + assert grid.findPointXy(10.0, 10.0) is None + assert grid.findPointXy(5.0, 20.0) is None + + +def test_global_grid_wrapping(): + """Test that global grids wrap around at the edges.""" + # Create a global grid (360° longitude, 180° latitude coverage) + global_grid = RegularLatLonGrid( + lat_start=-90.0, + lat_steps=180, + lat_step_size=1.0, + lon_start=-180.0, + lon_steps=360, + lon_step_size=1.0 + ) + + # Test wrapping around the longitude + # Point at longitude 180 should be the same as -180 + assert global_grid.findPointXy(0.0, 180.0) == (0, 90) + assert global_grid.findPointXy(0.0, -180.0) == (0, 90) + + # Test a point beyond the normal range + assert global_grid.findPointXy(0.0, 540.0) == (0, 90) + + +def test_ecmwf_grid(): + """Test the ECMWF IFS grid specifically.""" + ecmwf_grid = DOMAINS["ecmwf_ifs025"].grid + + # Test some known points on the grid + # Point at the prime meridian and equator + assert ecmwf_grid.findPointXy(0.0, 0.0) is not None + + # Point at the North Pole + assert ecmwf_grid.findPointXy(90.0, 0.0) is not None + + # Test some edge points (ensure they are properly handled) + assert ecmwf_grid.findPointXy(-90.0, -180.0) is not None + assert ecmwf_grid.findPointXy(90.0, 180.0) is not None + + # Test wrapping for global grid + # A point at longitude 181 should wrap to longitude -179 + point1 = ecmwf_grid.findPointXy(0.0, 181.0) + point2 = ecmwf_grid.findPointXy(0.0, -179.0) + assert point1 == point2 + + +def test_grid_coordinates(): + """Test getting coordinates from grid indices.""" + grid = RegularLatLonGrid( + lat_start=10.0, + lat_steps=5, + lat_step_size=2.0, + lon_start=100.0, + lon_steps=10, + lon_step_size=5.0 + ) + + # Test exact grid points + assert grid.getCoordinates(0, 0) == (10.0, 100.0) + assert grid.getCoordinates(5, 2) == (14.0, 125.0) + + # Test round-trip conversion + lat, lon = 14.0, 125.0 + x, y = grid.findPointXy(lat, lon) + assert grid.getCoordinates(x, y) == (lat, lon) + + +def test_dwd_icon_d2_grid_points(): + """Test specific points in the DWD ICON D2 grid.""" + dwd_grid = DOMAINS["dwd_icon_d2"].grid + + # Test a point known to be in the domain (Central Europe) + # Berlin coordinates: approx. 52.52°N, 13.40°E + berlin = dwd_grid.findPointXy(52.52, 13.40) + assert berlin is not None + + # Test a point outside the domain (should return None) + # New York coordinates: approx. 40.71°N, -74.01°E + new_york = dwd_grid.findPointXy(40.71, -74.01) + assert new_york is None + + # Test gridpoint to coordinate conversion + if berlin is not None: + x, y = berlin + lat, lon = dwd_grid.getCoordinates(x, y) + # Check that we get close to the original coordinates + assert abs(lat - 52.52) < 0.05 + assert abs(lon - 13.40) < 0.05 + + +def test_cached_property_computation(): + """Test that latitude and longitude arrays are lazily computed.""" + grid = RegularLatLonGrid( + lat_start=0.0, + lat_steps=10, + lat_step_size=1.0, + lon_start=0.0, + lon_steps=20, + lon_step_size=1.0 + ) + + # Access latitude array + lat1 = grid.latitude + # Access it again, should be the cached value + lat2 = grid.latitude + + # Check that we get the same array (same memory) + assert lat1 is lat2 + + +def test_time_to_chunk_index(): + """Test conversion from timestamp to chunk index.""" + domain = DOMAINS["dwd_icon_d2"] + + # Create test timestamp (2023-01-01 12:00:00 UTC) + timestamp = np.datetime64('2023-01-01T12:00:00') + + # Calculate expected chunk index + # Seconds since epoch = (2023-01-01 12:00:00 - 1970-01-01 00:00:00) seconds + # chunk_index = seconds_since_epoch / (file_length * temporal_resolution_seconds) + epoch = np.datetime64('1970-01-01T00:00:00') + seconds_since_epoch = (timestamp - epoch) / np.timedelta64(1, 's') + expected_chunk = int(seconds_since_epoch / (domain.file_length * domain.temporal_resolution_seconds)) + + # Test the time_to_chunk_index function + chunk_index = domain.time_to_chunk_index(timestamp) + assert chunk_index == expected_chunk + + +def test_get_chunk_time_range(): + """Test getting time range for a specific chunk.""" + domain = DOMAINS["dwd_icon_d2"] + + # Test chunk 1000 + chunk_index = 1000 + time_range = domain.get_chunk_time_range(chunk_index) + + # Check that we get the expected number of time points + assert len(time_range) == domain.file_length + + # Check that time points are evenly spaced + time_diff = time_range[1] - time_range[0] + assert time_diff == np.timedelta64(domain.temporal_resolution_seconds, 's') From 5889ce7a5eb53caf26e2822cddb316482f6f81f5 Mon Sep 17 00:00:00 2001 From: terraputix Date: Fri, 16 May 2025 17:15:28 +0200 Subject: [PATCH 02/50] add example for selection of grid points via times and coordinates --- examples/select_by_coordinates.py | 242 ++++++++++++++++++++++++++++++ python/omfiles/om_domains.py | 85 +++++++---- python/omfiles/utils.py | 4 + tests/test_om_domains.py | 12 +- 4 files changed, 311 insertions(+), 32 deletions(-) create mode 100644 examples/select_by_coordinates.py diff --git a/examples/select_by_coordinates.py b/examples/select_by_coordinates.py new file mode 100644 index 00000000..11858255 --- /dev/null +++ b/examples/select_by_coordinates.py @@ -0,0 +1,242 @@ +""" +Example showing how to select data from specific coordinates in an Open-Meteo file stored in S3. + +This script demonstrates how to: +1. Use the OmDomain class to work with weather model domains +2. Find the correct grid point for specific latitude/longitude coordinates +3. Load data from S3 using fsspec +4. Convert the data to an xarray Dataset for analysis +5. Extract time series for the selected coordinates across multiple files +6. Merge timeseries data from multiple chunks + +Usage: + python examples/select_by_coordinates.py + +Requirements: + - fsspec + - s3fs + - xarray + - numpy + - matplotlib (for plotting) + - omfiles +""" + +from datetime import datetime +from typing import Tuple + +import fsspec +import matplotlib.pyplot as plt +import numpy as np +import xarray as xr +from fsspec.implementations.cache_mapper import BasenameCacheMapper +from fsspec.implementations.cached import CachingFileSystem +from omfiles import OmFilePyReader +from omfiles.om_domains import DOMAINS +from s3fs import S3FileSystem +from xarray import Dataset + +# We load data from this Cached Fs-Spec Filesystem +FS = CachingFileSystem( + fs=S3FileSystem(anon=True, default_block_size=256, default_cache_type="none"), + # we keep the cache_check short: If files are modified on the remote, + # but we cache parts of these files locally, we potentially run into crashes/UB + cache_check=60, + block_size=256, + cache_storage="cache", + check_files=False, + cache_mapper=BasenameCacheMapper(directory_levels=3) +) + +def load_chunk_data( + chunk_index: int, + domain_name: str, + variable_name: str, + grid_coords: Tuple[int, int], + fs: fsspec.AbstractFileSystem, + start_date: np.datetime64, + end_date: np.datetime64 +): + """ + Load data for a specific chunk and grid coordinates. + + Parameters: + ----------- + chunk_index : int + Index of the chunk to load + domain_name : str + Name of the domain + variable_name : str + Name of the variable to fetch + grid_coords : Tuple[int, int] + Grid coordinates (x, y) to extract + fs : fsspec.AbstractFileSystem + Filesystem to use for loading data + start_date : np.datetime64 + Start of requested date range + end_date : np.datetime64 + End of requested date range + + Returns: + -------- + Tuple[np.ndarray, np.ndarray] + Tuple containing (time_array, data_array) + """ + domain = DOMAINS[domain_name] + x, y = grid_coords + s3_path = f"openmeteo/data/{domain.name}/{variable_name}/chunk_{chunk_index}.om" + + # Generate time array and check if any times are in our range + chunk_times = domain.get_chunk_time_range(chunk_index) + time_mask = (chunk_times >= start_date) & (chunk_times <= end_date) + if not np.any(time_mask): + return np.array([], dtype='datetime64[s]'), np.array([], dtype=float) + + # Create reader and read data of interest + with OmFilePyReader.from_fsspec(fs, s3_path) as reader: + indices = np.where(time_mask)[0] + time_slice = slice(indices[0], indices[-1] + 1) # +1 to include the end + data = reader[y, x, time_slice] + return chunk_times[time_mask], data + + raise ValueError("Unreachable") # Make Pyright happy... + + +def get_data_for_coordinates( + lat: float, + lon: float, + start_date: datetime, + end_date: datetime, + domain_name: str = 'ecmwf_ifs025', + variable_name: str = 'temperature_2m', +) -> Dataset: + """ + Fetch weather data for specific coordinates across a date range, merging multiple files as needed. + + Parameters: + ----------- + lat : float + Latitude in degrees + lon : float + Longitude in degrees + domain_name : str + Name of the domain to use (must be in omfiles.om_domains.DOMAINS) + variable_name : str + Name of the variable to fetch + start_date : datetime + Start date for the data + end_date : datetime + End date for the data + + Returns: + -------- + xr.Dataset + Dataset containing the requested variable at the specified location + """ + # Get the domain configuration + if domain_name not in DOMAINS: + raise ValueError(f"Unknown domain: {domain_name}. Available domains: {list(DOMAINS.keys())}") + + domain = DOMAINS[domain_name] + + # Find grid coordinates for geographical coordinates + grid_point = domain.grid.findPointXy(lat, lon) + if grid_point is None: + raise ValueError(f"Coordinates ({lat}, {lon}) not found in grid of {domain_name}") + + x, y = grid_point + print(f"Found grid point {grid_point} for coordinates ({lat}, {lon})") + print(f"Fetching data from {start_date} to {end_date}") + + start_timestamp = np.datetime64(start_date) + end_timestamp = np.datetime64(end_date) + + # Find all chunks needed for this date range + chunk_indices = domain.chunks_for_date_range(start_timestamp, end_timestamp) + print(f"Need to fetch {len(chunk_indices)} chunks: {chunk_indices}") + + # Load data from all chunks + all_times = [] + all_data = [] + + for chunk_idx in chunk_indices: + times, data = load_chunk_data( + chunk_idx, + domain_name, + variable_name, + (x, y), + FS, + start_timestamp, + end_timestamp + ) + if len(times) > 0: + all_times.append(times) + all_data.append(data) + + # Concatenate all data + if not all_times: + raise ValueError("Failed to load any data for the specified date range") + + time_array = np.concatenate(all_times) + data_array = np.concatenate(all_data) + + # Create the xarray dataset + ds = xr.Dataset( + data_vars={ + variable_name: (["time"], data_array), + }, + coords={ + "time": time_array, + "latitude": lat, + "longitude": lon, + }, + attrs={ + "domain": domain_name, + "grid_indices": grid_point, + } + ) + return ds + + +if __name__ == "__main__": + # Example coordinates: Paris + latitude = 48.864716 + longitude = 2.349014 + + # Define a date range + start_date = datetime(2025, 4, 16, 12, 0) # 16-04-2025'T'12:00 + end_date = datetime(2025, 5, 18, 12, 0) # 18-05-2025'T'12:00 + + # Create a common figure for both plots + plt.figure(figsize=(12, 6)) + + # Fetch and plot Meteofrance Arpege data + arpege_ds = get_data_for_coordinates( + lat=latitude, + lon=longitude, + start_date=start_date, + end_date=end_date, + domain_name='meteofrance_arpege_europe', + variable_name='temperature_2m', + ) + arpege_ds.temperature_2m.plot(label='METEOFRANCE ARPEGE (Europe)') + + # Fetch and plot ICON D2 data + icon_ds = get_data_for_coordinates( + lat=latitude, + lon=longitude, + start_date=start_date, + end_date=end_date, + domain_name='dwd_icon_d2', + variable_name='temperature_2m', + ) + icon_ds.temperature_2m.plot(label='DWD ICON D2 (Central Europe)') + + # Plot the temperature series + plt.title(f"2m Temperature at {latitude:.2f}N, {longitude:.2f}E") + plt.xlabel("Time") + plt.ylabel("Temperature (°C)") + plt.grid(True) + plt.legend() + plt.tight_layout() + plt.savefig("temperature_comparison.png") + plt.show() diff --git a/python/omfiles/om_domains.py b/python/omfiles/om_domains.py index 684dd490..32f06338 100644 --- a/python/omfiles/om_domains.py +++ b/python/omfiles/om_domains.py @@ -1,10 +1,11 @@ from abc import ABC, abstractmethod +from datetime import datetime from functools import cached_property -from typing import Optional, Tuple +from typing import List, Optional, Tuple import numpy as np -from omfiles.utils import _modulo_positive +from omfiles.utils import EPOCH, _modulo_positive class AbstractGrid(ABC): @@ -61,7 +62,6 @@ def getCoordinates(self, x: int, y: int) -> Tuple[float, float]: pass - class RegularLatLonGrid(AbstractGrid): """ Regular latitude-longitude grid implementation. @@ -211,10 +211,6 @@ def __init__( Grid implementation for this domain file_length : int Number of time steps in each file chunk - description : str, optional - Human-readable description of the domain - variables : list, optional - List of variable names available in this domain temporal_resolution_seconds : int, optional Time resolution in seconds (default: 3600 = 1 hour) """ @@ -225,7 +221,8 @@ def __init__( def time_to_chunk_index(self, timestamp: np.datetime64) -> int: """ - Convert a timestamp to a chunk index. + Convert a timestamp to a chunk index. This depends on the file_length + and the temporal_resolution_seconds of the domain. Parameters: ----------- @@ -237,18 +234,37 @@ def time_to_chunk_index(self, timestamp: np.datetime64) -> int: int The chunk index containing the timestamp """ - # Basic implementation - can be extended with domain-specific logic - epoch = np.datetime64('1970-01-01T00:00:00') - seconds_since_epoch = (timestamp - epoch) / np.timedelta64(1, 's') + seconds_since_epoch = (timestamp - EPOCH) / np.timedelta64(1, 's') + chunk_index = int(seconds_since_epoch / (self.file_length * self.temporal_resolution_seconds)) + return chunk_index - # Get temporal resolution from metadata (placeholder - real implementation would get from metadata) - temporal_resolution = 3600 # 1 hour in seconds + def chunks_for_date_range( + self, + start_timestamp: np.datetime64, + end_timestamp: np.datetime64, + ) -> List[int]: + """ + Find all chunk indices that contain data within the given date range. - # Calculate chunk index - chunk_index = int(seconds_since_epoch / (self.file_length * temporal_resolution)) - return chunk_index + Parameters: + ----------- + start_date : datetime + Start date for the data range + end_date : datetime + End date for the data range + Returns: + -------- + List[int] + List of chunk indices containing data within the date range + """ + # Get chunk indices for start and end dates + start_chunk = self.time_to_chunk_index(start_timestamp) + end_chunk = self.time_to_chunk_index(end_timestamp) - def get_chunk_time_range(self, chunk_index: int, temporal_resolution_seconds: int = 3600) -> np.ndarray: + # Generate list of all chunks between start and end (inclusive) + return list(range(start_chunk, end_chunk +1)) + + def get_chunk_time_range(self, chunk_index: int): """ Get the time range covered by a specific chunk. @@ -256,21 +272,20 @@ def get_chunk_time_range(self, chunk_index: int, temporal_resolution_seconds: in ----------- chunk_index : int Index of the chunk - temporal_resolution_seconds : int, optional - Time resolution in seconds (default: 3600 = 1 hour) Returns: -------- np.ndarray Array of datetime64 objects representing the time points in the chunk """ - epoch = np.datetime64('1970-01-01T00:00:00') - chunk_start_seconds = chunk_index * self.file_length * temporal_resolution_seconds - start_time = epoch + np.timedelta64(chunk_start_seconds, 's') + chunk_start_seconds = chunk_index * self.file_length * self.temporal_resolution_seconds + start_time = EPOCH + np.timedelta64(chunk_start_seconds, 's') - # Create array of time points - time_points = start_time + np.arange(self.file_length) * np.timedelta64(temporal_resolution_seconds, 's') - return time_points + # Generate timestamps at regular intervals from the start time + time_delta = np.timedelta64(self.temporal_resolution_seconds, 's') + # Note: better type inference via list comprehension here + timestamps = np.array([start_time + i * time_delta for i in range(self.file_length)]) + return timestamps # - MARK: Create grid instances for supported domains @@ -296,18 +311,34 @@ def get_chunk_time_range(self, chunk_index: int, temporal_resolution_seconds: in lon_step_size=180/(721-1) ) +# https://github.com/open-meteo/open-meteo/blob/1753ebb4966d05f61b17dd5bdf59700788d4a913/Sources/App/MeteoFrance/MeteoFranceDomain.swift#L348 +_meteofrance_arpege_europe_grid = RegularLatLonGrid( + lat_start=20, + lat_steps=521, + lat_step_size=0.1, + lon_start=-32, + lon_steps=741, + lon_step_size=0.1 +) + DOMAINS: dict[str, OmDomain] = { 'dwd_icon_d2': OmDomain( name='dwd_icon_d2', grid=_dwd_icon_d2_grid, file_length=121, - temporal_resolution_seconds=3600 # 1 hour + temporal_resolution_seconds=3600 ), 'ecmwf_ifs025': OmDomain( name='ecmwf_ifs025', grid=_ecmwf_ifs025_grid, file_length=104, - temporal_resolution_seconds=3600 # 1 hour + temporal_resolution_seconds=3600*3 ), + 'meteofrance_arpege_europe': OmDomain( + name='meteofrance_arpege_europe', + grid=_meteofrance_arpege_europe_grid, + file_length=114+3*24, + temporal_resolution_seconds=3600 + ) # Additional domains can be added here } diff --git a/python/omfiles/utils.py b/python/omfiles/utils.py index 7b404850..b3de924b 100644 --- a/python/omfiles/utils.py +++ b/python/omfiles/utils.py @@ -1,3 +1,7 @@ +import numpy as np + +EPOCH = np.datetime64(0, 's') + def _modulo_positive(value: int, modulo: int) -> int: """ Calculate modulo that always returns positive value. diff --git a/tests/test_om_domains.py b/tests/test_om_domains.py index 291dbc32..bdb53fad 100644 --- a/tests/test_om_domains.py +++ b/tests/test_om_domains.py @@ -68,14 +68,14 @@ def test_ecmwf_grid(): # Test some known points on the grid # Point at the prime meridian and equator - assert ecmwf_grid.findPointXy(0.0, 0.0) is not None + assert ecmwf_grid.findPointXy(0.0, 0.0) == (720, 360) # Point at the North Pole - assert ecmwf_grid.findPointXy(90.0, 0.0) is not None + assert ecmwf_grid.findPointXy(90.0, 0.0) == (720, 720) # Test some edge points (ensure they are properly handled) - assert ecmwf_grid.findPointXy(-90.0, -180.0) is not None - assert ecmwf_grid.findPointXy(90.0, 180.0) is not None + assert ecmwf_grid.findPointXy(-90.0, -180.0) == (0, 0) + assert ecmwf_grid.findPointXy(90.0, 180.0) == (0, 720) # Test wrapping for global grid # A point at longitude 181 should wrap to longitude -179 @@ -101,7 +101,9 @@ def test_grid_coordinates(): # Test round-trip conversion lat, lon = 14.0, 125.0 - x, y = grid.findPointXy(lat, lon) + result = grid.findPointXy(lat, lon) + assert result is not None, f"Could not find grid point for ({lat}, {lon})" + x, y = result assert grid.getCoordinates(x, y) == (lat, lon) From 76522bfaa959025c73a3fb1a6ef454f8b3a98f75 Mon Sep 17 00:00:00 2001 From: terraputix Date: Sat, 17 May 2025 15:26:42 +0200 Subject: [PATCH 03/50] move grids to separate module --- python/omfiles/grids.py | 185 +++++++++++++++++++++++++++++++++++ python/omfiles/om_domains.py | 185 +---------------------------------- 2 files changed, 188 insertions(+), 182 deletions(-) create mode 100644 python/omfiles/grids.py diff --git a/python/omfiles/grids.py b/python/omfiles/grids.py new file mode 100644 index 00000000..b5bb97c2 --- /dev/null +++ b/python/omfiles/grids.py @@ -0,0 +1,185 @@ +from abc import ABC, abstractmethod +from functools import cached_property +from typing import Optional, Tuple + +import numpy as np + +from omfiles.utils import _modulo_positive + + +class AbstractGrid(ABC): + """ + Abstract base class for weather model grid definitions. + + This defines the interface that all grid implementations must follow. + """ + + @property + @abstractmethod + def grid_type(self) -> str: + """Return the grid type identifier.""" + pass + + @cached_property + @abstractmethod + def latitude(self) -> np.ndarray: + """ + Return the latitude coordinates array. + + Uses cached_property to ensure the array is computed only once. + """ + pass + + @cached_property + @abstractmethod + def longitude(self) -> np.ndarray: + """ + Return the longitude coordinates array. + + Uses cached_property to ensure the array is computed only once. + """ + pass + + @property + @abstractmethod + def shape(self) -> Tuple[int, int]: + """Return the grid shape as (n_lat, n_lon).""" + pass + + @abstractmethod + def findPointXy(self, lat: float, lon: float) -> Optional[Tuple[int, int]]: + """ + Find grid point indices (x, y) for given lat/lon coordinates. + """ + pass + + @abstractmethod + def getCoordinates(self, x: int, y: int) -> Tuple[float, float]: + """ + Get lat/lon coordinates for a given grid point indices. + """ + pass + + +class RegularLatLonGrid(AbstractGrid): + """ + Regular latitude-longitude grid implementation. + + This represents a standard equirectangular grid with uniform spacing. + """ + + def __init__( + self, + lat_start: float, + lat_steps: int, + lat_step_size: float, + lon_start: float, + lon_steps: int, + lon_step_size: float, + ): + """ + Initialize a regular lat/lon grid. + + Parameters: + ----------- + lat_start : float + Starting latitude value + lat_steps : int + Number of latitude points + lat_step_size : float + Spacing between latitude points + lon_start : float + Starting longitude value + lon_steps : int + Number of longitude points + lon_step_size : float + Spacing between longitude points + """ + self._lat_start = lat_start + self._lat_steps = lat_steps + self._lat_step_size = lat_step_size + self._lon_start = lon_start + self._lon_steps = lon_steps + self._lon_step_size = lon_step_size + + @property + def grid_type(self) -> str: + return "regular_latlon" + + @cached_property + def latitude(self) -> np.ndarray: + """ + Lazily compute and cache the latitude coordinate array. + """ + return np.linspace( + self._lat_start, + self._lat_start + self._lat_step_size * self._lat_steps, + self._lat_steps, + endpoint=False + ) + + @cached_property + def longitude(self) -> np.ndarray: + """ + Lazily compute and cache the longitude coordinate array. + """ + return np.linspace( + self._lon_start, + self._lon_start + self._lon_step_size * self._lon_steps, + self._lon_steps, + endpoint=False + ) + + @property + def shape(self) -> Tuple[int, int]: + return (self._lat_steps, self._lon_steps) + + def findPointXy(self, lat: float, lon: float) -> Optional[Tuple[int, int]]: + """ + Find grid point indices (x, y) for given lat/lon coordinates. + + Parameters: + ----------- + lat : float + Latitude in degrees + lon : float + Longitude in degrees + + Returns: + -------- + tuple or None + (x, y) grid indices if point is in grid, None otherwise + """ + # Calculate raw x and y indices + x = int(round((lon - self._lon_start) / self._lon_step_size)) + y = int(round((lat - self._lat_start) / self._lat_step_size)) + + # Handle wrapping for global grids + xx = _modulo_positive(x, self._lon_steps) if (self._lon_steps * self._lon_step_size) >= 359 else x + yy = _modulo_positive(y, self._lat_steps) if (self._lat_steps * self._lat_step_size) >= 179 else y + + # Check if point is within grid bounds + if yy < 0 or xx < 0 or yy >= self._lat_steps or xx >= self._lon_steps: + return None + + return (xx, yy) + + def getCoordinates(self, x: int, y: int) -> Tuple[float, float]: + """ + Get lat/lon coordinates for a given grid point indices. + + Parameters: + ----------- + x : longitude index + y : latitude index + + Returns: + -------- + tuple + (latitude, longitude) coordinates + """ + + lat = self._lat_start + float(y) * self._lat_step_size + lon = self._lon_start + float(x) * self._lon_step_size + + return (lat, lon) diff --git a/python/omfiles/om_domains.py b/python/omfiles/om_domains.py index 32f06338..2bf83954 100644 --- a/python/omfiles/om_domains.py +++ b/python/omfiles/om_domains.py @@ -1,190 +1,11 @@ -from abc import ABC, abstractmethod -from datetime import datetime -from functools import cached_property -from typing import List, Optional, Tuple +from typing import List import numpy as np -from omfiles.utils import EPOCH, _modulo_positive +from omfiles.grids import AbstractGrid, RegularLatLonGrid +from omfiles.utils import EPOCH -class AbstractGrid(ABC): - """ - Abstract base class for weather model grid definitions. - - This defines the interface that all grid implementations must follow. - """ - - @property - @abstractmethod - def grid_type(self) -> str: - """Return the grid type identifier.""" - pass - - @cached_property - @abstractmethod - def latitude(self) -> np.ndarray: - """ - Return the latitude coordinates array. - - Uses cached_property to ensure the array is computed only once. - """ - pass - - @cached_property - @abstractmethod - def longitude(self) -> np.ndarray: - """ - Return the longitude coordinates array. - - Uses cached_property to ensure the array is computed only once. - """ - pass - - @property - @abstractmethod - def shape(self) -> Tuple[int, int]: - """Return the grid shape as (n_lat, n_lon).""" - pass - - @abstractmethod - def findPointXy(self, lat: float, lon: float) -> Optional[Tuple[int, int]]: - """ - Find grid point indices (x, y) for given lat/lon coordinates. - """ - pass - - @abstractmethod - def getCoordinates(self, x: int, y: int) -> Tuple[float, float]: - """ - Get lat/lon coordinates for a given grid point indices. - """ - pass - - -class RegularLatLonGrid(AbstractGrid): - """ - Regular latitude-longitude grid implementation. - - This represents a standard equirectangular grid with uniform spacing. - """ - - def __init__( - self, - lat_start: float, - lat_steps: int, - lat_step_size: float, - lon_start: float, - lon_steps: int, - lon_step_size: float, - ): - """ - Initialize a regular lat/lon grid. - - Parameters: - ----------- - lat_start : float - Starting latitude value - lat_steps : int - Number of latitude points - lat_step_size : float - Spacing between latitude points - lon_start : float - Starting longitude value - lon_steps : int - Number of longitude points - lon_step_size : float - Spacing between longitude points - """ - self._lat_start = lat_start - self._lat_steps = lat_steps - self._lat_step_size = lat_step_size - self._lon_start = lon_start - self._lon_steps = lon_steps - self._lon_step_size = lon_step_size - - @property - def grid_type(self) -> str: - return "regular_latlon" - - @cached_property - def latitude(self) -> np.ndarray: - """ - Lazily compute and cache the latitude coordinate array. - """ - return np.linspace( - self._lat_start, - self._lat_start + self._lat_step_size * self._lat_steps, - self._lat_steps, - endpoint=False - ) - - @cached_property - def longitude(self) -> np.ndarray: - """ - Lazily compute and cache the longitude coordinate array. - """ - return np.linspace( - self._lon_start, - self._lon_start + self._lon_step_size * self._lon_steps, - self._lon_steps, - endpoint=False - ) - - @property - def shape(self) -> Tuple[int, int]: - return (self._lat_steps, self._lon_steps) - - def findPointXy(self, lat: float, lon: float) -> Optional[Tuple[int, int]]: - """ - Find grid point indices (x, y) for given lat/lon coordinates. - - Parameters: - ----------- - lat : float - Latitude in degrees - lon : float - Longitude in degrees - - Returns: - -------- - tuple or None - (x, y) grid indices if point is in grid, None otherwise - """ - # Calculate raw x and y indices - x = int(round((lon - self._lon_start) / self._lon_step_size)) - y = int(round((lat - self._lat_start) / self._lat_step_size)) - - # Handle wrapping for global grids - xx = _modulo_positive(x, self._lon_steps) if (self._lon_steps * self._lon_step_size) >= 359 else x - yy = _modulo_positive(y, self._lat_steps) if (self._lat_steps * self._lat_step_size) >= 179 else y - - # Check if point is within grid bounds - if yy < 0 or xx < 0 or yy >= self._lat_steps or xx >= self._lon_steps: - return None - - return (xx, yy) - - def getCoordinates(self, x: int, y: int) -> Tuple[float, float]: - """ - Get lat/lon coordinates for a given grid point indices. - - Parameters: - ----------- - x : longitude index - y : latitude index - - Returns: - -------- - tuple - (latitude, longitude) coordinates - """ - - lat = self._lat_start + float(y) * self._lat_step_size - lon = self._lon_start + float(x) * self._lon_step_size - - return (lat, lon) - class OmDomain: """ Class representing a domain configuration for a weather model. From 1d71317b99c4b348fe9af28c19d1c6b661fa1962 Mon Sep 17 00:00:00 2001 From: terraputix Date: Sat, 17 May 2025 18:11:02 +0200 Subject: [PATCH 04/50] stereographic projection --- python/omfiles/grids.py | 445 ++++++++++++++++++++++++++++++++++++++- tests/test_grids.py | 116 ++++++++++ tests/test_om_domains.py | 139 ++---------- 3 files changed, 577 insertions(+), 123 deletions(-) create mode 100644 tests/test_grids.py diff --git a/python/omfiles/grids.py b/python/omfiles/grids.py index b5bb97c2..5fe5e4b6 100644 --- a/python/omfiles/grids.py +++ b/python/omfiles/grids.py @@ -1,8 +1,9 @@ from abc import ABC, abstractmethod from functools import cached_property -from typing import Optional, Tuple +from typing import Generic, Optional, Tuple, TypeVar, Union, cast import numpy as np +import numpy.typing as npt from omfiles.utils import _modulo_positive @@ -183,3 +184,445 @@ def getCoordinates(self, x: int, y: int) -> Tuple[float, float]: lon = self._lon_start + float(x) * self._lon_step_size return (lat, lon) + +# Type aliases for clarity +FloatType = Union[float, np.floating] +ArrayType = npt.NDArray[np.floating] +CoordType = Union[float, ArrayType] +ReturnUnionType = Union[tuple[ArrayType, ArrayType], tuple[float, float]] + +# Abstract base class instead of Protocol +class AbstractProjection(ABC): + """Base class for projection implementations.""" + + @abstractmethod + def forward(self, latitude: CoordType, longitude: CoordType) -> ReturnUnionType: + """ + Transform from lat/lon coordinates to projected x/y coordinates. + + Handles both scalar and array inputs transparently. + """ + pass + + @abstractmethod + def inverse(self, x: CoordType, y: CoordType) -> ReturnUnionType: + """ + Transform from projected x/y coordinates back to lat/lon. + + Handles both scalar and array inputs transparently. + """ + pass + +class StereographicProjection(AbstractProjection): + """ + Stereographic projection implementation. + + This implements the equations for the stereographic projection + which projects a sphere onto a plane. + https://mathworld.wolfram.com/StereographicProjection.html + """ + + def __init__(self, latitude: float, longitude: float, radius: float = 6371000.0): + """ + Initialize a stereographic projection. + + Parameters: + ----------- + latitude : float + Central latitude in degrees + longitude : float + Central longitude in degrees + radius : float + Radius of Earth in meters (default: 6371000.0) + """ + self.lambda_0: npt.NDArray[np.float32] = np.radians(longitude) + self.sin_phi_1: npt.NDArray[np.float32] = np.sin(np.radians(latitude)) + self.cos_phi_1: npt.NDArray[np.float32] = np.cos(np.radians(latitude)) + self.R = radius + + def forward(self, latitude: CoordType, longitude: CoordType) -> ReturnUnionType: + """ + Transform from lat/lon coordinates to projected x/y coordinates. + + Parameters: + ----------- + latitude : float or array + Latitude in degrees + longitude : float or array + Longitude in degrees + + Returns: + -------- + tuple + (x, y) coordinates in the projection + """ + scalar_input = np.isscalar(latitude) and np.isscalar(longitude) + + lat_arr = np.asarray(latitude, dtype=np.float32) + lon_arr = np.asarray(longitude, dtype=np.float32) + + phi = np.radians(lat_arr) + lambda_ = np.radians(lon_arr) + k = 2 * self.R / (1 + self.sin_phi_1 * np.sin(phi) + + self.cos_phi_1 * np.cos(phi) * np.cos(lambda_ - self.lambda_0)) + x = k * np.cos(phi) * np.sin(lambda_ - self.lambda_0) + y = k * (self.cos_phi_1 * np.sin(phi) - + self.sin_phi_1 * np.cos(phi) * np.cos(lambda_ - self.lambda_0)) + + if scalar_input: + return float(x.item()), float(y.item()) + + return x, y + + def inverse(self, x: CoordType, y: CoordType) -> ReturnUnionType: + """ + Transform from projected x/y coordinates back to lat/lon. + + Parameters: + ----------- + x : float or array + X coordinate in the projection + y : float or array + Y coordinate in the projection + + Returns: + -------- + tuple + (latitude, longitude) in degrees + """ + # Convert inputs to numpy arrays for uniform handling + x_arr = np.asarray(x, dtype=np.float32) + y_arr = np.asarray(y, dtype=np.float32) + + # Calculate distance from origin + p = np.sqrt(x_arr*x_arr + y_arr*y_arr) + + # Initialize output arrays + phi = np.zeros_like(p) + lambda_ = np.zeros_like(p) + + # Handle the origin case + origin = (p == 0) + phi[origin] = np.arcsin(self.sin_phi_1) + lambda_[origin] = self.lambda_0 + + # Handle non-origin points + non_origin = ~origin + if np.any(non_origin): + c = 2 * np.arctan2(p[non_origin], 2*self.R) + phi[non_origin] = np.arcsin(np.cos(c) * self.sin_phi_1 + + (y_arr[non_origin] * np.sin(c) * self.cos_phi_1) / p[non_origin]) + lambda_[non_origin] = self.lambda_0 + np.arctan2( + x_arr[non_origin] * np.sin(c), + p[non_origin] * self.cos_phi_1 * np.cos(c) - y_arr[non_origin] * self.sin_phi_1 * np.sin(c) + ) + + # Convert to degrees + lat = np.degrees(phi) + lon = np.degrees(lambda_) + + return lat, lon + +P = TypeVar('P', bound=AbstractProjection) + + +class ProjectionGrid(AbstractGrid, Generic[P]): + """ + Grid implementation using a projection. + + This represents a grid in a projected coordinate system. + """ + + def __init__( + self, + projection: P, + nx: int, + ny: int, + origin: Tuple[float, float], + dx: float, + dy: float + ): + """ + Initialize a projection grid with all parameters. + + Parameters: + ----------- + projection : Projectable + Projection implementation + nx : int + Number of grid points in x direction + ny : int + Number of grid points in y direction + origin : Tuple[float, float] + Origin coordinates (x, y) of the grid in projection space + dx : float + Grid spacing in x direction in meters + dy : float + Grid spacing in y direction in meters + """ + self.projection = projection + self.nx = nx + self.ny = ny + self.origin = origin + self.dx = dx + self.dy = dy + + @classmethod + def from_bounds( + cls, + nx: int, + ny: int, + lat_range: Tuple[float, float], + lon_range: Tuple[float, float], + projection: P + ) -> 'ProjectionGrid[P]': + """ + Create a projection grid from geographic bounds. + + Parameters: + ----------- + nx : int + Number of grid points in x direction + ny : int + Number of grid points in y direction + lat_range : Tuple[float, float] + Latitude range (min, max) in degrees + lon_range : Tuple[float, float] + Longitude range (min, max) in degrees + projection : Projectable + Projection implementation + + Returns: + -------- + ProjectionGrid + New grid instance + """ + sw = projection.forward(lat_range[0], lon_range[0]) + ne = projection.forward(lat_range[1], lon_range[1]) + origin = cast(tuple[float, float], sw) + dx = (ne[0] - sw[0]) / (nx-1) + dy = (ne[1] - sw[1]) / (ny-1) + return cls(projection, nx, ny, origin, float(dx), float(dy)) + + @classmethod + def from_center( + cls, + nx: int, + ny: int, + center_lat: float, + center_lon: float, + dx: float, + dy: float, + projection: P + ) -> 'ProjectionGrid[P]': + """ + Create a projection grid centered at a geographic location. + + Parameters: + ----------- + nx : int + Number of grid points in x direction + ny : int + Number of grid points in y direction + center_lat : float + Center latitude in degrees + center_lon : float + Center longitude in degrees + dx : float + Grid spacing in x direction in meters + dy : float + Grid spacing in y direction in meters + projection : Projectable + Projection implementation + + Returns: + -------- + ProjectionGrid + New grid instance + """ + center = cast(tuple[float, float], projection.forward(center_lat, center_lon)) + origin_x = center[0] - dx * (nx // 2) + origin_y = center[1] - dy * (ny // 2) + return cls(projection, nx, ny, (origin_x, origin_y), dx, dy) + + @property + def grid_type(self) -> str: + return "projection" + + @cached_property + def latitude(self) -> np.ndarray: + """ + Lazily compute and cache the latitude coordinate array. + """ + # Create meshgrid of coordinates + y_indices, x_indices = np.meshgrid( + np.arange(self.ny), + np.arange(self.nx), + indexing='ij' + ) + + # Convert to projected coordinates + x_coords = x_indices * self.dx + self.origin[0] + y_coords = y_indices * self.dy + self.origin[1] + + # Convert to lat/lon using vectorized inverse method + lat, _ = cast(tuple[ArrayType, ArrayType], self.projection.inverse(x_coords, y_coords)) + return lat + + @cached_property + def longitude(self) -> np.ndarray: + """ + Lazily compute and cache the longitude coordinate array. + """ + # Create meshgrid of coordinates + y_indices, x_indices = np.meshgrid( + np.arange(self.ny), + np.arange(self.nx), + indexing='ij' + ) + + # Convert to projected coordinates + x_coords = x_indices * self.dx + self.origin[0] + y_coords = y_indices * self.dy + self.origin[1] + + # Convert to lat/lon using vectorized inverse method + _, lon = cast(tuple[ArrayType, ArrayType], self.projection.inverse(x_coords, y_coords)) + return lon + + @property + def shape(self) -> Tuple[int, int]: + return (self.ny, self.nx) + + def findPointXy(self, lat: float, lon: float) -> Optional[Tuple[int, int]]: + """ + Find grid point indices (x, y) for given lat/lon coordinates. + + Parameters: + ----------- + lat : float + Latitude in degrees + lon : float + Longitude in degrees + + Returns: + -------- + tuple or None + (x, y) grid indices if point is in grid, None otherwise + """ + pos = cast(tuple[float, float], self.projection.forward(lat, lon)) + x = int(round((pos[0] - self.origin[0]) / self.dx)) + y = int(round((pos[1] - self.origin[1]) / self.dy)) + + if y < 0 or x < 0 or y >= self.ny or x >= self.nx: + return None + + return (x, y) + + def getCoordinates(self, x: int, y: int) -> Tuple[float, float]: + """ + Get lat/lon coordinates for a given grid point indices. + + Parameters: + ----------- + x : int + X index + y : int + Y index + + Returns: + -------- + tuple + (latitude, longitude) coordinates + """ + xcord = float(x) * self.dx + self.origin[0] + ycord = float(y) * self.dy + self.origin[1] + lat, lon = cast(tuple[float, float], self.projection.inverse(xcord, ycord)) + # Normalize longitude to -180 to 180 range + lon = ((lon + 180.0) % 360.0) - 180.0 + return (lat, lon) + + def get_true_north_direction(self) -> np.ndarray: + """ + Calculate angle towards true north for every grid point. + + Returns: + -------- + numpy.ndarray + Array of angles in degrees, 0 = points towards north pole + """ + pos = self.projection.forward(90, 0) # North pole + north_pole_x = (pos[0] - self.origin[0]) / self.dx + north_pole_y = (pos[1] - self.origin[1]) / self.dy + + # Create grid of x, y coordinates + y_indices, x_indices = np.meshgrid( + np.arange(self.ny), + np.arange(self.nx), + indexing='ij' + ) + + # Vectorized calculation of angles + true_north = np.degrees(np.arctan2( + north_pole_x - x_indices, + north_pole_y - y_indices + )) + + return true_north + + def find_box( + self, + lat_min: float, + lat_max: float, + lon_min: float, + lon_max: float + ) -> np.ndarray: + """ + Find indices of grid points within a geographic bounding box. + + Parameters: + ----------- + lat_min : float + Minimum latitude + lat_max : float + Maximum latitude + lon_min : float + Minimum longitude + lon_max : float + Maximum longitude + + Returns: + -------- + numpy.ndarray + Array of grid point indices within the box + """ + sw = self.findPointXy(lat_min, lon_min) + se = self.findPointXy(lat_min, lon_max) + nw = self.findPointXy(lat_max, lon_min) + ne = self.findPointXy(lat_max, lon_max) + + if not all([sw, se, nw, ne]): + return np.array([], dtype=int) + + # Type casting to inform pyright that these variables are not None + sw = cast(Tuple[int, int], sw) + se = cast(Tuple[int, int], se) + nw = cast(Tuple[int, int], nw) + ne = cast(Tuple[int, int], ne) + + x_min = min(sw[0], nw[0]) + x_max = max(se[0], ne[0]) + 1 + y_min = min(sw[1], se[1]) + y_max = max(nw[1], ne[1]) + 1 + + # Create meshgrid of indices + y_indices, x_indices = np.meshgrid( + np.arange(y_min, y_max), + np.arange(x_min, x_max), + indexing='ij' + ) + + # Convert to flat indices + return np.ravel_multi_index( + (y_indices.flatten(), x_indices.flatten()), + (self.ny, self.nx) + ) diff --git a/tests/test_grids.py b/tests/test_grids.py new file mode 100644 index 00000000..feb29317 --- /dev/null +++ b/tests/test_grids.py @@ -0,0 +1,116 @@ +import numpy as np +import pytest +from omfiles.grids import ProjectionGrid, StereographicProjection +from omfiles.om_domains import RegularLatLonGrid + +# Fixtures for grids + +@pytest.fixture +def local_regular_lat_lon_grid(): + return RegularLatLonGrid( + lat_start=0.0, + lat_steps=10, + lat_step_size=1.0, + lon_start=0.0, + lon_steps=20, + lon_step_size=1.0 + ) + +@pytest.fixture +def stereographic_projection(): + projection = StereographicProjection(90.0, 249.0, 6371229.0) + return ProjectionGrid.from_bounds( + nx=935, + ny=824, + lat_range=(18.14503, 45.405453), + lon_range=(217.10745, 349.8256), + projection=projection + ) + +def test_regular_grid_findPointXy_inside(local_regular_lat_lon_grid): + # Test exact grid points + assert local_regular_lat_lon_grid.findPointXy(5.0, 10.0) == (10, 5) + assert local_regular_lat_lon_grid.findPointXy(0.0, 0.0) == (0, 0) + assert local_regular_lat_lon_grid.findPointXy(9.0, 19.0) == (19, 9) + + # Test points that should round to grid points + assert local_regular_lat_lon_grid.findPointXy(5.1, 10.2) == (10, 5) + assert local_regular_lat_lon_grid.findPointXy(0.4, 0.4) == (0, 0) + + +def test_regular_grid_findPointXy_outside(local_regular_lat_lon_grid): + # Test points outside of grid + assert local_regular_lat_lon_grid.findPointXy(-1.0, 10.0) is None + assert local_regular_lat_lon_grid.findPointXy(5.0, -1.0) is None + assert local_regular_lat_lon_grid.findPointXy(10.0, 10.0) is None + assert local_regular_lat_lon_grid.findPointXy(5.0, 20.0) is None + + +def test_global_grid_wrapping(): + # Create a global grid (360° longitude, 180° latitude coverage) + global_grid = RegularLatLonGrid( + lat_start=-90.0, + lat_steps=180, + lat_step_size=1.0, + lon_start=-180.0, + lon_steps=360, + lon_step_size=1.0 + ) + + # Test wrapping around the longitude + # Point at longitude 180 should be the same as -180 + assert global_grid.findPointXy(0.0, 180.0) == (0, 90) + assert global_grid.findPointXy(0.0, -180.0) == (0, 90) + + # Test a point beyond the normal range + assert global_grid.findPointXy(0.0, 540.0) == (0, 90) + +def test_grid_coordinates(local_regular_lat_lon_grid): + # Test exact grid points + assert local_regular_lat_lon_grid.getCoordinates(0, 0) == (0.0, 0.0) + assert local_regular_lat_lon_grid.getCoordinates(5, 2) == (2.0, 5.0) + + # Test round-trip conversion + lat, lon = 8.0, 15.0 + result = local_regular_lat_lon_grid.findPointXy(lat, lon) + assert result is not None, f"Could not find grid point for ({lat}, {lon})" + x, y = result + assert local_regular_lat_lon_grid.getCoordinates(x, y) == (lat, lon) + + +def test_cached_property_computation(local_regular_lat_lon_grid): + lat1 = local_regular_lat_lon_grid.latitude + lat2 = local_regular_lat_lon_grid.latitude + + # Check that we get the same array (same memory) + assert lat1 is lat2 + + +def test_stereographic(stereographic_projection): + #https://github.com/open-meteo/open-meteo/blob/522917b1d6e72a7e6b7d4ae7dfb49b0c556a6992/Tests/AppTests/DataTests.swift#L248 + pos_x, pos_y = stereographic_projection.findPointXy(lat=64.79836, lon=241.40111) + + assert pos_x == 420 + assert pos_y == 468 + + # Get the coordinates back + lat, lon = stereographic_projection.getCoordinates(pos_x, pos_y) + assert abs(lat - 64.79836) < 1e-4 + assert np.mod(abs(lon - 241.40111), 360) < 1e-4 + +def test_grid_properties(stereographic_projection): + assert stereographic_projection.shape == (824, 935) + assert stereographic_projection.grid_type == "projection" + +def test_out_of_bounds(stereographic_projection): + far_point = stereographic_projection.findPointXy(30.0, 120.0) + assert far_point is None + +def test_latitude_longitude_arrays(stereographic_projection): + # Get latitude and longitude arrays + lats = stereographic_projection.latitude + lons = stereographic_projection.longitude + + # Check shapes match the grid + assert lats.shape == (824, 935) + assert lons.shape == (824, 935) diff --git a/tests/test_om_domains.py b/tests/test_om_domains.py index bdb53fad..e370ab71 100644 --- a/tests/test_om_domains.py +++ b/tests/test_om_domains.py @@ -1,110 +1,5 @@ import numpy as np -from omfiles.om_domains import DOMAINS, RegularLatLonGrid - - -def test_regular_grid_findPointXy_inside(): - """Test finding grid points inside the domain.""" - grid = RegularLatLonGrid( - lat_start=0.0, - lat_steps=10, - lat_step_size=1.0, - lon_start=0.0, - lon_steps=20, - lon_step_size=1.0 - ) - - # Test exact grid points - assert grid.findPointXy(5.0, 10.0) == (10, 5) - assert grid.findPointXy(0.0, 0.0) == (0, 0) - assert grid.findPointXy(9.0, 19.0) == (19, 9) - - # Test points that should round to grid points - assert grid.findPointXy(5.1, 10.2) == (10, 5) - assert grid.findPointXy(0.4, 0.4) == (0, 0) - - -def test_regular_grid_findPointXy_outside(): - """Test finding grid points outside the domain.""" - grid = RegularLatLonGrid( - lat_start=0.0, - lat_steps=10, - lat_step_size=1.0, - lon_start=0.0, - lon_steps=20, - lon_step_size=1.0 - ) - - # Test points outside of grid - assert grid.findPointXy(-1.0, 10.0) is None - assert grid.findPointXy(5.0, -1.0) is None - assert grid.findPointXy(10.0, 10.0) is None - assert grid.findPointXy(5.0, 20.0) is None - - -def test_global_grid_wrapping(): - """Test that global grids wrap around at the edges.""" - # Create a global grid (360° longitude, 180° latitude coverage) - global_grid = RegularLatLonGrid( - lat_start=-90.0, - lat_steps=180, - lat_step_size=1.0, - lon_start=-180.0, - lon_steps=360, - lon_step_size=1.0 - ) - - # Test wrapping around the longitude - # Point at longitude 180 should be the same as -180 - assert global_grid.findPointXy(0.0, 180.0) == (0, 90) - assert global_grid.findPointXy(0.0, -180.0) == (0, 90) - - # Test a point beyond the normal range - assert global_grid.findPointXy(0.0, 540.0) == (0, 90) - - -def test_ecmwf_grid(): - """Test the ECMWF IFS grid specifically.""" - ecmwf_grid = DOMAINS["ecmwf_ifs025"].grid - - # Test some known points on the grid - # Point at the prime meridian and equator - assert ecmwf_grid.findPointXy(0.0, 0.0) == (720, 360) - - # Point at the North Pole - assert ecmwf_grid.findPointXy(90.0, 0.0) == (720, 720) - - # Test some edge points (ensure they are properly handled) - assert ecmwf_grid.findPointXy(-90.0, -180.0) == (0, 0) - assert ecmwf_grid.findPointXy(90.0, 180.0) == (0, 720) - - # Test wrapping for global grid - # A point at longitude 181 should wrap to longitude -179 - point1 = ecmwf_grid.findPointXy(0.0, 181.0) - point2 = ecmwf_grid.findPointXy(0.0, -179.0) - assert point1 == point2 - - -def test_grid_coordinates(): - """Test getting coordinates from grid indices.""" - grid = RegularLatLonGrid( - lat_start=10.0, - lat_steps=5, - lat_step_size=2.0, - lon_start=100.0, - lon_steps=10, - lon_step_size=5.0 - ) - - # Test exact grid points - assert grid.getCoordinates(0, 0) == (10.0, 100.0) - assert grid.getCoordinates(5, 2) == (14.0, 125.0) - - # Test round-trip conversion - lat, lon = 14.0, 125.0 - result = grid.findPointXy(lat, lon) - assert result is not None, f"Could not find grid point for ({lat}, {lon})" - x, y = result - assert grid.getCoordinates(x, y) == (lat, lon) +from omfiles.om_domains import DOMAINS def test_dwd_icon_d2_grid_points(): @@ -129,26 +24,26 @@ def test_dwd_icon_d2_grid_points(): assert abs(lat - 52.52) < 0.05 assert abs(lon - 13.40) < 0.05 +def test_ecmwf_grid(): + """Test the ECMWF IFS grid specifically.""" + ecmwf_grid = DOMAINS["ecmwf_ifs025"].grid -def test_cached_property_computation(): - """Test that latitude and longitude arrays are lazily computed.""" - grid = RegularLatLonGrid( - lat_start=0.0, - lat_steps=10, - lat_step_size=1.0, - lon_start=0.0, - lon_steps=20, - lon_step_size=1.0 - ) + # Test some known points on the grid + # Point at the prime meridian and equator + assert ecmwf_grid.findPointXy(0.0, 0.0) == (720, 360) - # Access latitude array - lat1 = grid.latitude - # Access it again, should be the cached value - lat2 = grid.latitude + # Point at the North Pole + assert ecmwf_grid.findPointXy(90.0, 0.0) == (720, 720) - # Check that we get the same array (same memory) - assert lat1 is lat2 + # Test some edge points (ensure they are properly handled) + assert ecmwf_grid.findPointXy(-90.0, -180.0) == (0, 0) + assert ecmwf_grid.findPointXy(90.0, 180.0) == (0, 720) + # Test wrapping for global grid + # A point at longitude 181 should wrap to longitude -179 + point1 = ecmwf_grid.findPointXy(0.0, 181.0) + point2 = ecmwf_grid.findPointXy(0.0, -179.0) + assert point1 == point2 def test_time_to_chunk_index(): """Test conversion from timestamp to chunk index.""" From 28349b9637ad93df4b255c1b2fdc8156d3be8d5f Mon Sep 17 00:00:00 2001 From: terraputix Date: Sun, 18 May 2025 09:53:08 +0200 Subject: [PATCH 05/50] bump minimum numpy version --- .github/actions/test/action.yml | 2 +- pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/actions/test/action.yml b/.github/actions/test/action.yml index 0c780d02..bdc2e16f 100644 --- a/.github/actions/test/action.yml +++ b/.github/actions/test/action.yml @@ -66,7 +66,7 @@ runs: python -m pip install pytest==6.0 psutil WHEEL_PATH=$(ls dist/*.whl) python -m pip install --force-reinstall "$WHEEL_PATH" \ - "numpy==1.20.0" \ + "numpy==1.21.0" \ "fsspec==2023.1.0" \ "s3fs==2023.1.0" \ "xarray==2023.1.0" diff --git a/pyproject.toml b/pyproject.toml index f78bf909..3cc4ec5d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -12,7 +12,7 @@ classifiers = [ ] dynamic = ["version"] dependencies = [ - "numpy>=1.20.0", + "numpy>=1.21.0", "fsspec>=2023.1.0", "s3fs>=2023.1.0", "xarray>=2023.1.0", From d151c0b181381feed01e4421b93d4ee5a4d40401 Mon Sep 17 00:00:00 2001 From: terraputix Date: Sun, 18 May 2025 16:52:46 +0200 Subject: [PATCH 06/50] add all dwd-icon domains --- examples/select_by_coordinates.py | 92 ++++++++++++++++++--------- python/omfiles/om_domains.py | 101 ++++++++++++++++++++++++++++-- 2 files changed, 158 insertions(+), 35 deletions(-) diff --git a/examples/select_by_coordinates.py b/examples/select_by_coordinates.py index 11858255..168b94e9 100644 --- a/examples/select_by_coordinates.py +++ b/examples/select_by_coordinates.py @@ -1,5 +1,5 @@ """ -Example showing how to select data from specific coordinates in an Open-Meteo file stored in S3. +Example showing how to select data from multiple domains in Open-Meteo files stored in S3. This script demonstrates how to: 1. Use the OmDomain class to work with weather model domains @@ -8,6 +8,7 @@ 4. Convert the data to an xarray Dataset for analysis 5. Extract time series for the selected coordinates across multiple files 6. Merge timeseries data from multiple chunks +7. Plot data from multiple domains in a single figure Usage: python examples/select_by_coordinates.py @@ -196,7 +197,6 @@ def get_data_for_coordinates( ) return ds - if __name__ == "__main__": # Example coordinates: Paris latitude = 48.864716 @@ -206,37 +206,69 @@ def get_data_for_coordinates( start_date = datetime(2025, 4, 16, 12, 0) # 16-04-2025'T'12:00 end_date = datetime(2025, 5, 18, 12, 0) # 18-05-2025'T'12:00 - # Create a common figure for both plots + # Variable to fetch + variable = 'temperature_2m' + + print(f"Fetching {variable} data for coordinates: {latitude}N, {longitude}E") + print(f"Date range: {start_date} to {end_date}") + + # Domain display names for nicer legends + display_names = { + 'dwd_icon': 'DWD ICON (Global)', + 'dwd_icon_eu': 'DWD ICON (Europe)', + 'dwd_icon_d2': 'DWD ICON D2 (Central Europe)', + 'ecmwf_ifs025': 'ECMWF IFS (Global)', + 'meteofrance_arpege_europe': 'Météo-France ARPEGE (Europe)' + } + + # Collect data from each domain + domain_data = {} + successful_domains = [] + + # Loop through all domains in the main function + for domain_name in DOMAINS.keys(): + try: + print(f"\nTrying to fetch data from domain: {domain_name}") + ds = get_data_for_coordinates( + lat=latitude, + lon=longitude, + start_date=start_date, + end_date=end_date, + domain_name=domain_name, + variable_name=variable + ) + domain_data[domain_name] = ds + successful_domains.append(domain_name) + print(f"Successfully fetched data from {domain_name}") + except Exception as e: + print(f"Could not fetch data from {domain_name}: {e}") + domain_data[domain_name] = None + + print(f"\nSuccessfully fetched data from {len(successful_domains)} domains: {successful_domains}") + + if not successful_domains: + print("No data could be fetched from any domain. Exiting.") + exit(1) + + # Domain colors for consistent line colors + colors = plt.cm.get_cmap('tab10')(np.linspace(0, 1, len(successful_domains))) + plt.figure(figsize=(12, 6)) - # Fetch and plot Meteofrance Arpege data - arpege_ds = get_data_for_coordinates( - lat=latitude, - lon=longitude, - start_date=start_date, - end_date=end_date, - domain_name='meteofrance_arpege_europe', - variable_name='temperature_2m', - ) - arpege_ds.temperature_2m.plot(label='METEOFRANCE ARPEGE (Europe)') - - # Fetch and plot ICON D2 data - icon_ds = get_data_for_coordinates( - lat=latitude, - lon=longitude, - start_date=start_date, - end_date=end_date, - domain_name='dwd_icon_d2', - variable_name='temperature_2m', - ) - icon_ds.temperature_2m.plot(label='DWD ICON D2 (Central Europe)') + # Plot data from each domain + for i, domain_name in enumerate(successful_domains): + ds = domain_data[domain_name] + label = display_names.get(domain_name, domain_name) + ds[variable].plot(label=label, color=colors[i], linewidth=2) - # Plot the temperature series - plt.title(f"2m Temperature at {latitude:.2f}N, {longitude:.2f}E") + # Enhance the plot + plt.title(f"{variable.replace('_', ' ').title()} at {latitude:.2f}N, {longitude:.2f}E") plt.xlabel("Time") - plt.ylabel("Temperature (°C)") - plt.grid(True) - plt.legend() + plt.ylabel("Temperature (°C)" if variable == 'temperature_2m' else variable) + plt.grid(True, alpha=0.3) + plt.legend(loc='best') plt.tight_layout() - plt.savefig("temperature_comparison.png") + + # Save and show the figure + plt.savefig(f"{variable}_comparison.png", dpi=150) plt.show() diff --git a/python/omfiles/om_domains.py b/python/omfiles/om_domains.py index 2bf83954..3c4fb8cc 100644 --- a/python/omfiles/om_domains.py +++ b/python/omfiles/om_domains.py @@ -110,8 +110,30 @@ def get_chunk_time_range(self, chunk_index: int): # - MARK: Create grid instances for supported domains +# DWD ICON global is regularized during download to nx: 2879, ny: 1441 points +# https://github.com/open-meteo/open-meteo/blob/522917b1d6e72a7e6b7d4ae7dfb49b0c556a6992/Sources/App/Icon/Icon.swift#L146 +_dwd_icon_grid = RegularLatLonGrid( + lat_start=-90, + lat_steps=1441, + lat_step_size=0.125, + lon_start=-180, + lon_steps=2879, + lon_step_size=0.125 +) + +# DWD ICON EU is regularized during download to nx: 1377, ny: 657 points +# https://github.com/open-meteo/open-meteo/blob/522917b1d6e72a7e6b7d4ae7dfb49b0c556a6992/Sources/App/Icon/Icon.swift#L148 +_dwd_icon_eu_grid = RegularLatLonGrid( + lat_start=29.5, + lat_steps=657, + lat_step_size=0.0625, + lon_start=-23.5, + lon_steps=1377, + lon_step_size=0.0625 +) + # DWD ICON D2 is regularized during download to nx: 1215, ny: 746 points -# https://github.com/open-meteo/open-meteo/blob/1753ebb4966d05f61b17dd5bdf59700788d4a913/Sources/App/Icon/Icon.swift#L154 +# https://github.com/open-meteo/open-meteo/blob/522917b1d6e72a7e6b7d4ae7dfb49b0c556a6992/Sources/App/Icon/Icon.swift#L150 _dwd_icon_d2_grid = RegularLatLonGrid( lat_start=43.18, lat_steps=746, @@ -121,8 +143,41 @@ def get_chunk_time_range(self, chunk_index: int): lon_step_size=0.02 ) -# ECMWF IQFS grid is a regular global lat/lon grid, nx: 1440, ny: 721 points -# https://github.com/open-meteo/open-meteo/blob/1753ebb4966d05f61b17dd5bdf59700788d4a913/Sources/App/Ecmwf/EcmwfDomain.swift#L107 +# DWD ICON EPS global is regularized during download to nx: 1439, ny: 721 points +# https://github.com/open-meteo/open-meteo/blob/522917b1d6e72a7e6b7d4ae7dfb49b0c556a6992/Sources/App/Icon/Icon.swift#L153 +_dwd_icon_eps_grid = RegularLatLonGrid( + lat_start=-90, + lat_steps=721, + lat_step_size=0.25, + lon_start=-180, + lon_steps=1439, + lon_step_size=0.25 +) + +# DWD ICON EU EPS is regularized during download to nx: 689, ny: 329 points +# https://github.com/open-meteo/open-meteo/blob/522917b1d6e72a7e6b7d4ae7dfb49b0c556a6992/Sources/App/Icon/Icon.swift#L156 +_dwd_icon_eu_eps_grid = RegularLatLonGrid( + lat_start=29.5, + lat_steps=329, + lat_step_size=0.125, + lon_start=-23.5, + lon_steps=689, + lon_step_size=0.125 +) + +# DWD ICON D2 EPS is regularized during download to nx: 1214, ny: 745 points +# https://github.com/open-meteo/open-meteo/blob/522917b1d6e72a7e6b7d4ae7dfb49b0c556a6992/Sources/App/Icon/Icon.swift#L160 +_dwd_icon_d2_eps_grid = RegularLatLonGrid( + lat_start=43.18, + lat_steps=745, + lat_step_size=0.02, + lon_start=-3.94, + lon_steps=1214, + lon_step_size=0.02 +) + +# ECMWF IFS grid is a regular global lat/lon grid, nx: 1440, ny: 721 points +# https://github.com/open-meteo/open-meteo/blob/522917b1d6e72a7e6b7d4ae7dfb49b0c556a6992/Sources/App/Ecmwf/EcmwfDomain.swift#L105 _ecmwf_ifs025_grid = RegularLatLonGrid( lat_start=-90, lat_steps=721, @@ -132,7 +187,7 @@ def get_chunk_time_range(self, chunk_index: int): lon_step_size=180/(721-1) ) -# https://github.com/open-meteo/open-meteo/blob/1753ebb4966d05f61b17dd5bdf59700788d4a913/Sources/App/MeteoFrance/MeteoFranceDomain.swift#L348 +# https://github.com/open-meteo/open-meteo/blob/522917b1d6e72a7e6b7d4ae7dfb49b0c556a6992/Sources/App/MeteoFrance/MeteoFranceDomain.swift#L341 _meteofrance_arpege_europe_grid = RegularLatLonGrid( lat_start=20, lat_steps=521, @@ -143,10 +198,46 @@ def get_chunk_time_range(self, chunk_index: int): ) DOMAINS: dict[str, OmDomain] = { + 'dwd_icon': OmDomain( + name='dwd_icon', + grid=_dwd_icon_grid, + file_length=180+1+3*24, # From IconDomains.omFileLength for icon case + temporal_resolution_seconds=3600 + ), + 'dwd_icon_eu': OmDomain( + name='dwd_icon_eu', + grid=_dwd_icon_eu_grid, + file_length=120+1+3*24, # From IconDomains.omFileLength for iconEu case + temporal_resolution_seconds=3600 + ), 'dwd_icon_d2': OmDomain( name='dwd_icon_d2', grid=_dwd_icon_d2_grid, - file_length=121, + file_length=48+1+3*24, # From IconDomains.omFileLength for iconD2 case + temporal_resolution_seconds=3600 + ), + 'dwd_icon_d2_15min': OmDomain( + name='dwd_icon_d2_15min', + grid=_dwd_icon_d2_grid, # Uses same grid as dwd_icon_d2 + file_length=48*4+3*24, # From IconDomains.omFileLength for iconD2_15min case + temporal_resolution_seconds=3600//4 # 15 minutes = 3600/4 + ), + 'dwd_icon_eps': OmDomain( + name='dwd_icon_eps', + grid=_dwd_icon_eps_grid, + file_length=180+1+3*24, # Same as non-eps version + temporal_resolution_seconds=3600 + ), + 'dwd_icon_eu_eps': OmDomain( + name='dwd_icon_eu_eps', + grid=_dwd_icon_eu_eps_grid, + file_length=120+1+3*24, # Same as non-eps version + temporal_resolution_seconds=3600 + ), + 'dwd_icon_d2_eps': OmDomain( + name='dwd_icon_d2_eps', + grid=_dwd_icon_d2_eps_grid, + file_length=48+1+3*24, # Same as non-eps version temporal_resolution_seconds=3600 ), 'ecmwf_ifs025': OmDomain( From ccfce08e2dbdcc4175219a61c3ee2301b65e9a81 Mon Sep 17 00:00:00 2001 From: terraputix Date: Sun, 18 May 2025 19:59:16 +0200 Subject: [PATCH 07/50] add arpege domains --- examples/select_by_coordinates.py | 14 ++++-- python/omfiles/om_domains.py | 78 ++++++++++++++++++++++++++++++- 2 files changed, 86 insertions(+), 6 deletions(-) diff --git a/examples/select_by_coordinates.py b/examples/select_by_coordinates.py index 168b94e9..9aca4e31 100644 --- a/examples/select_by_coordinates.py +++ b/examples/select_by_coordinates.py @@ -213,12 +213,16 @@ def get_data_for_coordinates( print(f"Date range: {start_date} to {end_date}") # Domain display names for nicer legends - display_names = { + domains_and_display_names = { 'dwd_icon': 'DWD ICON (Global)', 'dwd_icon_eu': 'DWD ICON (Europe)', 'dwd_icon_d2': 'DWD ICON D2 (Central Europe)', 'ecmwf_ifs025': 'ECMWF IFS (Global)', - 'meteofrance_arpege_europe': 'Météo-France ARPEGE (Europe)' + 'meteofrance_arpege_europe': 'Météo-France ARPEGE (Europe)', + 'meteofrance_arpege_world025': 'Météo-France ARPEGE (Global)', + 'meteofrance_arome_france0025': 'Météo-France AROME (France)', + 'meteofrance_arome_france_hd': 'Météo-France AROME HD (France)', + 'meteofrance_arome_france_hd_15min': 'Météo-France AROME HD 15min (France)', } # Collect data from each domain @@ -226,7 +230,7 @@ def get_data_for_coordinates( successful_domains = [] # Loop through all domains in the main function - for domain_name in DOMAINS.keys(): + for domain_name in domains_and_display_names.keys(): try: print(f"\nTrying to fetch data from domain: {domain_name}") ds = get_data_for_coordinates( @@ -251,14 +255,14 @@ def get_data_for_coordinates( exit(1) # Domain colors for consistent line colors - colors = plt.cm.get_cmap('tab10')(np.linspace(0, 1, len(successful_domains))) + colors = plt.get_cmap('tab10')(np.linspace(0, 1, len(successful_domains))) plt.figure(figsize=(12, 6)) # Plot data from each domain for i, domain_name in enumerate(successful_domains): ds = domain_data[domain_name] - label = display_names.get(domain_name, domain_name) + label = domains_and_display_names[domain_name] ds[variable].plot(label=label, color=colors[i], linewidth=2) # Enhance the plot diff --git a/python/omfiles/om_domains.py b/python/omfiles/om_domains.py index 3c4fb8cc..5aa350e2 100644 --- a/python/omfiles/om_domains.py +++ b/python/omfiles/om_domains.py @@ -187,6 +187,7 @@ def get_chunk_time_range(self, chunk_index: int): lon_step_size=180/(721-1) ) +# Méteo-France ARPEGE Europe grid: nx: 741, ny: 521 points # https://github.com/open-meteo/open-meteo/blob/522917b1d6e72a7e6b7d4ae7dfb49b0c556a6992/Sources/App/MeteoFrance/MeteoFranceDomain.swift#L341 _meteofrance_arpege_europe_grid = RegularLatLonGrid( lat_start=20, @@ -197,6 +198,39 @@ def get_chunk_time_range(self, chunk_index: int): lon_step_size=0.1 ) +# Méteo-France ARPEGE World grid: nx: 1440, ny: 721 points +# https://github.com/open-meteo/open-meteo/blob/522917b1d6e72a7e6b7d4ae7dfb49b0c556a6992/Sources/App/MeteoFrance/MeteoFranceDomain.swift#L343 +_meteofrance_arpege_world025_grid = RegularLatLonGrid( + lat_start=-90, + lat_steps=721, + lat_step_size=0.25, + lon_start=-180, + lon_steps=1440, + lon_step_size=0.25 +) + +# Méteo-France AROME France grid: nx: 1121, ny: 717 points +# https://github.com/open-meteo/open-meteo/blob/522917b1d6e72a7e6b7d4ae7dfb49b0c556a6992/Sources/App/MeteoFrance/MeteoFranceDomain.swift#L345 +_meteofrance_arome_france0025_grid = RegularLatLonGrid( + lat_start=37.5, + lat_steps=717, + lat_step_size=0.025, + lon_start=-12.0, + lon_steps=1121, + lon_step_size=0.025 +) + +# Méteo-France AROME France HD grid: nx: 2801, ny: 1791 points +# https://github.com/open-meteo/open-meteo/blob/522917b1d6e72a7e6b7d4ae7dfb49b0c556a6992/Sources/App/MeteoFrance/MeteoFranceDomain.swift#L347 +_meteofrance_arome_france_hd_grid = RegularLatLonGrid( + lat_start=37.5, + lat_steps=1791, + lat_step_size=0.01, + lon_start=-12.0, + lon_steps=2801, + lon_step_size=0.01 +) + DOMAINS: dict[str, OmDomain] = { 'dwd_icon': OmDomain( name='dwd_icon', @@ -249,8 +283,50 @@ def get_chunk_time_range(self, chunk_index: int): 'meteofrance_arpege_europe': OmDomain( name='meteofrance_arpege_europe', grid=_meteofrance_arpege_europe_grid, - file_length=114+3*24, + file_length=114+3*24, # From MeteoFranceDomain.omFileLength for arpege_europe case temporal_resolution_seconds=3600 + ), + 'meteofrance_arpege_world025': OmDomain( + name='meteofrance_arpege_world025', + grid=_meteofrance_arpege_world025_grid, + file_length=114+4*24, # From MeteoFranceDomain.omFileLength for arpege_world case + temporal_resolution_seconds=3600 + ), + 'meteofrance_arome_france0025': OmDomain( + name='meteofrance_arome_france0025', + grid=_meteofrance_arome_france0025_grid, + file_length=36+3*24, # From MeteoFranceDomain.omFileLength for arome_france case + temporal_resolution_seconds=3600 + ), + 'meteofrance_arome_france_hd': OmDomain( + name='meteofrance_arome_france_hd', + grid=_meteofrance_arome_france_hd_grid, + file_length=36+3*24, # From MeteoFranceDomain.omFileLength for arome_france_hd case + temporal_resolution_seconds=3600 + ), + 'meteofrance_arome_france0025_15min': OmDomain( + name='meteofrance_arome_france0025_15min', + grid=_meteofrance_arome_france0025_grid, # Using the same grid as non-15min version + file_length=24*2, # From MeteoFranceDomain.omFileLength for arome_france_15min case + temporal_resolution_seconds=900 + ), + 'meteofrance_arome_france_hd_15min': OmDomain( + name='meteofrance_arome_france_hd_15min', + grid=_meteofrance_arome_france_hd_grid, # Using the same grid as non-15min version + file_length=24*2, # From MeteoFranceDomain.omFileLength for arome_france_hd_15min case + temporal_resolution_seconds=900 + ), + 'meteofrance_arpege_europe_probabilities': OmDomain( + name='meteofrance_arpege_europe_probabilities', + grid=_meteofrance_arpege_europe_grid, # Using the same grid as non-probabilities version + file_length=(102+4*24)//3, # From MeteoFranceDomain.omFileLength for arpege_europe_probabilities case + temporal_resolution_seconds=3600*3 + ), + 'meteofrance_arpege_world025_probabilities': OmDomain( + name='meteofrance_arpege_world025_probabilities', + grid=_meteofrance_arpege_world025_grid, # Using the same grid as non-probabilities version + file_length=(102+4*24)//3, # From MeteoFranceDomain.omFileLength for arpege_world_probabilities case + temporal_resolution_seconds=3600*3 ) # Additional domains can be added here } From c96170cb38c2c4c64b1f5d90711f82afdce0130b Mon Sep 17 00:00:00 2001 From: terraputix Date: Mon, 19 May 2025 10:56:29 +0200 Subject: [PATCH 08/50] add rotated lat lon grid --- examples/select_by_coordinates.py | 9 +- python/omfiles/grids.py | 175 +++++++++++++++++++++++------- python/omfiles/om_domains.py | 89 ++++++++++++++- src/array_index.rs | 10 +- tests/test_grids.py | 40 ++++++- 5 files changed, 274 insertions(+), 49 deletions(-) diff --git a/examples/select_by_coordinates.py b/examples/select_by_coordinates.py index 9aca4e31..46f84250 100644 --- a/examples/select_by_coordinates.py +++ b/examples/select_by_coordinates.py @@ -202,8 +202,12 @@ def get_data_for_coordinates( latitude = 48.864716 longitude = 2.349014 + # # Example coordinates: Vancouver + # latitude = 49.246 + # longitude =-123.116 + # Define a date range - start_date = datetime(2025, 4, 16, 12, 0) # 16-04-2025'T'12:00 + start_date = datetime(2025, 4, 25, 12, 0) # 25-04-2025'T'12:00 end_date = datetime(2025, 5, 18, 12, 0) # 18-05-2025'T'12:00 # Variable to fetch @@ -223,6 +227,9 @@ def get_data_for_coordinates( 'meteofrance_arome_france0025': 'Météo-France AROME (France)', 'meteofrance_arome_france_hd': 'Météo-France AROME HD (France)', 'meteofrance_arome_france_hd_15min': 'Météo-France AROME HD 15min (France)', + 'gem_global': 'CMC GEM GDPS (Global)', + 'gem_regional': 'CMC GEM RDPS (Regional)', + 'gem_hrdps_continental': 'CMC GEM HRDPS (Continental)', } # Collect data from each domain diff --git a/python/omfiles/grids.py b/python/omfiles/grids.py index 5fe5e4b6..f4a84501 100644 --- a/python/omfiles/grids.py +++ b/python/omfiles/grids.py @@ -213,6 +213,118 @@ def inverse(self, x: CoordType, y: CoordType) -> ReturnUnionType: """ pass +class RotatedLatLonProjection(AbstractProjection): + """ + Rotated lat/lon projection implementation. + + This implements the transformation between regular lat/lon coordinates and + rotated lat/lon coordinates where the pole is shifted to a specified location. + Based on: https://github.com/open-meteo/open-meteo/blob/main/Sources/App/Domains/RotatedLatLon.swift + """ + + def __init__(self, lat_origin: float, lon_origin: float): + """ + Initialize a rotated lat/lon projection. + + Parameters: + ----------- + lat_origin : float + Latitude of origin in degrees + lon_origin : float + Longitude of origin in degrees + """ + # θ: Rotation around y-axis + self.theta = np.radians(90.0 + lat_origin) + # ϕ: Rotation around z-axis + self.phi = np.radians(lon_origin) + + def forward(self, latitude: CoordType, longitude: CoordType) -> ReturnUnionType: + """ + Transform from regular lat/lon to rotated lat/lon coordinates. + + Parameters: + ----------- + latitude : float or array + Latitude in degrees + longitude : float or array + Longitude in degrees + + Returns: + -------- + tuple + (rotated_lat, rotated_lon) in degrees + """ + scalar_input = np.isscalar(latitude) and np.isscalar(longitude) + + lat_arr = np.asarray(latitude, dtype=np.float32) + lon_arr = np.asarray(longitude, dtype=np.float32) + + # Convert to radians + lat_rad = np.radians(lat_arr) + lon_rad = np.radians(lon_arr) + + # Convert to cartesian coordinates + x = np.cos(lon_rad) * np.cos(lat_rad) + y = np.sin(lon_rad) * np.cos(lat_rad) + z = np.sin(lat_rad) + + # Apply rotation + x2 = np.cos(self.theta) * np.cos(self.phi) * x + np.cos(self.theta) * np.sin(self.phi) * y + np.sin(self.theta) * z + y2 = -np.sin(self.phi) * x + np.cos(self.phi) * y + z2 = -np.sin(self.theta) * np.cos(self.phi) * x - np.sin(self.theta) * np.sin(self.phi) * y + np.cos(self.theta) * z + + # Convert back to spherical coordinates + rot_lon = np.degrees(np.arctan2(y2, x2)) + rot_lat = np.degrees(np.arcsin(z2)) + + if scalar_input: + return float(rot_lon.item()), float(rot_lat.item()) + + return rot_lon, rot_lat + + def inverse(self, x: CoordType, y: CoordType) -> ReturnUnionType: + """ + Transform from rotated lat/lon back to regular lat/lon coordinates. + + Parameters: + ----------- + x : float or array + Rotated longitude in degrees + y : float or array + Rotated latitude in degrees + + Returns: + -------- + tuple + (latitude, longitude) in degrees + """ + scalar_input = np.isscalar(x) and np.isscalar(y) + + rot_lon = np.radians(np.asarray(x, dtype=np.float32)) + rot_lat = np.radians(np.asarray(y, dtype=np.float32)) + + theta_neg = -self.theta + phi_neg = -self.phi + + # Quick solution without conversion in cartesian space + lat_rad = np.arcsin( + np.cos(theta_neg) * np.sin(rot_lat) - np.cos(rot_lon) * np.sin(theta_neg) * np.cos(rot_lat) + ) + + lon_rad = np.arctan2( + np.sin(rot_lon), + np.tan(rot_lat) * np.sin(theta_neg) + np.cos(rot_lon) * np.cos(theta_neg) + ) - phi_neg + + lat2 = np.degrees(lat_rad) + lon2 = np.degrees(lon_rad) + + if scalar_input: + return float(lat2.item()), float(lon2.item()) + + return lat2, lon2 + + class StereographicProjection(AbstractProjection): """ Stereographic projection implementation. @@ -290,38 +402,23 @@ def inverse(self, x: CoordType, y: CoordType) -> ReturnUnionType: tuple (latitude, longitude) in degrees """ - # Convert inputs to numpy arrays for uniform handling x_arr = np.asarray(x, dtype=np.float32) y_arr = np.asarray(y, dtype=np.float32) - # Calculate distance from origin p = np.sqrt(x_arr*x_arr + y_arr*y_arr) # Initialize output arrays phi = np.zeros_like(p) lambda_ = np.zeros_like(p) - # Handle the origin case - origin = (p == 0) - phi[origin] = np.arcsin(self.sin_phi_1) - lambda_[origin] = self.lambda_0 - - # Handle non-origin points - non_origin = ~origin - if np.any(non_origin): - c = 2 * np.arctan2(p[non_origin], 2*self.R) - phi[non_origin] = np.arcsin(np.cos(c) * self.sin_phi_1 + - (y_arr[non_origin] * np.sin(c) * self.cos_phi_1) / p[non_origin]) - lambda_[non_origin] = self.lambda_0 + np.arctan2( - x_arr[non_origin] * np.sin(c), - p[non_origin] * self.cos_phi_1 * np.cos(c) - y_arr[non_origin] * self.sin_phi_1 * np.sin(c) - ) - - # Convert to degrees - lat = np.degrees(phi) - lon = np.degrees(lambda_) + c = 2 * np.arctan2(p, 2*self.R) + phi = np.arcsin(np.cos(c) * self.sin_phi_1 + (y_arr * np.sin(c) * self.cos_phi_1) / p) + lambda_ = self.lambda_0 + np.arctan2( + x_arr * np.sin(c), + p * self.cos_phi_1 * np.cos(c) - y_arr * self.sin_phi_1 * np.sin(c) + ) - return lat, lon + return np.degrees(phi), np.degrees(lambda_) P = TypeVar('P', bound=AbstractProjection) @@ -450,9 +547,9 @@ def grid_type(self) -> str: return "projection" @cached_property - def latitude(self) -> np.ndarray: + def _coordinates(self) -> Tuple[np.ndarray, np.ndarray]: """ - Lazily compute and cache the latitude coordinate array. + Lazily compute and cache both latitude and longitude arrays. """ # Create meshgrid of coordinates y_indices, x_indices = np.meshgrid( @@ -466,28 +563,22 @@ def latitude(self) -> np.ndarray: y_coords = y_indices * self.dy + self.origin[1] # Convert to lat/lon using vectorized inverse method - lat, _ = cast(tuple[ArrayType, ArrayType], self.projection.inverse(x_coords, y_coords)) - return lat + lat, lon = cast(tuple[ArrayType, ArrayType], self.projection.inverse(x_coords, y_coords)) + return lat, lon - @cached_property - def longitude(self) -> np.ndarray: + @property + def latitude(self) -> np.ndarray: # type: ignore """ - Lazily compute and cache the longitude coordinate array. + Get the latitude coordinate array. """ - # Create meshgrid of coordinates - y_indices, x_indices = np.meshgrid( - np.arange(self.ny), - np.arange(self.nx), - indexing='ij' - ) - - # Convert to projected coordinates - x_coords = x_indices * self.dx + self.origin[0] - y_coords = y_indices * self.dy + self.origin[1] + return self._coordinates[0] - # Convert to lat/lon using vectorized inverse method - _, lon = cast(tuple[ArrayType, ArrayType], self.projection.inverse(x_coords, y_coords)) - return lon + @property + def longitude(self) -> np.ndarray: # type: ignore + """ + Get the longitude coordinate array. + """ + return self._coordinates[1] @property def shape(self) -> Tuple[int, int]: diff --git a/python/omfiles/om_domains.py b/python/omfiles/om_domains.py index 5aa350e2..5499eda8 100644 --- a/python/omfiles/om_domains.py +++ b/python/omfiles/om_domains.py @@ -2,7 +2,13 @@ import numpy as np -from omfiles.grids import AbstractGrid, RegularLatLonGrid +from omfiles.grids import ( + AbstractGrid, + ProjectionGrid, + RegularLatLonGrid, + RotatedLatLonProjection, + StereographicProjection, +) from omfiles.utils import EPOCH @@ -231,7 +237,82 @@ def get_chunk_time_range(self, chunk_index: int): lon_step_size=0.01 ) +# GEM Global grid: nx: 2400, ny: 1201 points +# https://github.com/open-meteo/open-meteo/blob/522917b1d6e72a7e6b7d4ae7dfb49b0c556a6992/Sources/App/Gem/GemDomain.swift#L139 +_gem_global_grid = RegularLatLonGrid( + lat_start=-90, + lat_steps=1201, + lat_step_size=0.15, + lon_start=-180, + lon_steps=2400, + lon_step_size=0.15 +) + +# GEM Regional grid: Uses Stereographic projection +# https://github.com/open-meteo/open-meteo/blob/522917b1d6e72a7e6b7d4ae7dfb49b0c556a6992/Sources/App/Gem/GemDomain.swift#L141 +_gem_regional_projection = StereographicProjection( + latitude=90, + longitude=249, + radius=6371229 +) +_gem_regional_grid = ProjectionGrid.from_bounds( + nx=935, + ny=824, + lat_range=(18.14503, 45.405453), + lon_range=(217.10745, 349.8256), + projection=_gem_regional_projection +) + +# GEM HRDPS Continental grid: Uses RotatedLatLon projection +# https://github.com/open-meteo/open-meteo/blob/522917b1d6e72a7e6b7d4ae7dfb49b0c556a6992/Sources/App/Gem/GemDomain.swift#L143 +_gem_hrdps_projection = RotatedLatLonProjection( + lat_origin=-36.0885, + lon_origin=245.305 +) +_gem_hrdps_grid = ProjectionGrid.from_bounds( + nx=2540, + ny=1290, + lat_range=(39.626034, 47.876457), + lon_range=(-133.62952, -40.708557), + projection=_gem_hrdps_projection +) + +# GEM Global Ensemble grid: nx: 720, ny: 361 points +# https://github.com/open-meteo/open-meteo/blob/522917b1d6e72a7e6b7d4ae7dfb49b0c556a6992/Sources/App/Gem/GemDomain.swift#L145 +_gem_global_ensemble_grid = RegularLatLonGrid( + lat_start=-90, + lat_steps=361, + lat_step_size=0.5, + lon_start=-180, + lon_steps=720, + lon_step_size=0.5 +) + DOMAINS: dict[str, OmDomain] = { + 'cmc_gem_gdps': OmDomain( + name='cmc_gem_gdps', + grid=_gem_global_grid, + file_length=110, # From GemDomain.omFileLength for gem_global case + temporal_resolution_seconds=3600*3 # 3-hourly data + ), + 'cmc_gem_rdps': OmDomain( + name='cmc_gem_rdps', + grid=_gem_regional_grid, + file_length=78+36, # From GemDomain.omFileLength for gem_regional case + temporal_resolution_seconds=3600 # Hourly data + ), + 'cmc_gem_hrdps': OmDomain( + name='cmc_gem_hrdps', + grid=_gem_hrdps_grid, + file_length=48+36, # From GemDomain.omFileLength for gem_hrdps_continental case + temporal_resolution_seconds=3600 # Hourly data + ), + 'cmc_gem_geps': OmDomain( + name='cmc_gem_geps', + grid=_gem_global_ensemble_grid, + file_length=384//3+48//3, # From GemDomain.omFileLength for gem_global_ensemble case + temporal_resolution_seconds=3600*3 # 3-hourly data + ), 'dwd_icon': OmDomain( name='dwd_icon', grid=_dwd_icon_grid, @@ -330,3 +411,9 @@ def get_chunk_time_range(self, chunk_index: int): ) # Additional domains can be added here } + +# Domain aliases to match the names in GemDomain.swift +DOMAINS['gem_global'] = DOMAINS['cmc_gem_gdps'] +DOMAINS['gem_regional'] = DOMAINS['cmc_gem_rdps'] +DOMAINS['gem_hrdps_continental'] = DOMAINS['cmc_gem_hrdps'] +DOMAINS['gem_global_ensemble'] = DOMAINS['cmc_gem_geps'] diff --git a/src/array_index.rs b/src/array_index.rs index a2d2e8bd..3ae16ad1 100644 --- a/src/array_index.rs +++ b/src/array_index.rs @@ -65,10 +65,12 @@ impl<'py> FromPyObject<'py> for ArrayIndex { impl ArrayIndex { pub fn to_read_range(&self, shape: &Vec) -> PyResult>> { // Input validation - if self.0.len() > shape.len() { - return Err(PyErr::new::( - "Too many indices for array", - )); + if self.0.len() != shape.len() { + return Err(PyErr::new::(format!( + "Array dimensions do not match. Got: {:}, expected: {:}", + shape.len(), + self.0.len() + ))); } let mut ranges = Vec::new(); diff --git a/tests/test_grids.py b/tests/test_grids.py index feb29317..b5d291e4 100644 --- a/tests/test_grids.py +++ b/tests/test_grids.py @@ -1,6 +1,6 @@ import numpy as np import pytest -from omfiles.grids import ProjectionGrid, StereographicProjection +from omfiles.grids import ProjectionGrid, RotatedLatLonProjection, StereographicProjection from omfiles.om_domains import RegularLatLonGrid # Fixtures for grids @@ -27,6 +27,21 @@ def stereographic_projection(): projection=projection ) +@pytest.fixture +def hrdps_projection(): + return RotatedLatLonProjection(lat_origin=-36.0885, lon_origin=245.305) + +@pytest.fixture +def hrdps_grid(hrdps_projection): + from omfiles.grids import ProjectionGrid + return ProjectionGrid.from_bounds( + nx=2540, + ny=1290, + lat_range=(39.626034, 47.876457), + lon_range=(-133.62952, -40.708557), + projection=hrdps_projection + ) + def test_regular_grid_findPointXy_inside(local_regular_lat_lon_grid): # Test exact grid points assert local_regular_lat_lon_grid.findPointXy(5.0, 10.0) == (10, 5) @@ -114,3 +129,26 @@ def test_latitude_longitude_arrays(stereographic_projection): # Check shapes match the grid assert lats.shape == (824, 935) assert lons.shape == (824, 935) + +def test_hrdps_grid(hrdps_grid): + """Test the HRDPS Continental grid with a modified approach""" + test_points = [ + # lat, lon, expected_x, expected_y + (39.626034, -133.62952, 0, 0), # Bottom-left + (27.284597, -66.96642, 2539, 0), # Bottom-right + (38.96126, -73.63256, 2032, 283), # Middle point + (47.876457, -40.708557, 2539, 1289), # Top-right + ] + + for lat, lon, expected_x, expected_y in test_points: + # Test finding grid point + pos = hrdps_grid.findPointXy(lat=lat, lon=lon) + assert pos is not None, f"Could not find point for {lat}, {lon}" + + x, y = pos + assert x == expected_x, f"X mismatch: got {x}, expected {expected_x}" + assert y == expected_y, f"Y mismatch: got {y}, expected {expected_y}" + + lat2, lon2 = hrdps_grid.getCoordinates(x, y) + assert abs(lat2 - lat) < 0.001, f"latitude mismatch: got {lat2}, expected {lat}" + assert abs(lon2 - lon) < 0.001, f"longitude mismatch: got {lon2}, expected {lon}" From fca4bb77ac78872eadeebf9504c8e36355c42fdd Mon Sep 17 00:00:00 2001 From: terraputix Date: Mon, 19 May 2025 11:28:33 +0200 Subject: [PATCH 09/50] add lamberth azimuthal equal area --- python/omfiles/grids.py | 106 ++++++++++++++++++++++++++++++++++++++++ tests/test_grids.py | 43 +++++++++++++++- 2 files changed, 148 insertions(+), 1 deletion(-) diff --git a/python/omfiles/grids.py b/python/omfiles/grids.py index f4a84501..74b54811 100644 --- a/python/omfiles/grids.py +++ b/python/omfiles/grids.py @@ -420,6 +420,112 @@ def inverse(self, x: CoordType, y: CoordType) -> ReturnUnionType: return np.degrees(phi), np.degrees(lambda_) +class LambertAzimuthalEqualAreaProjection(AbstractProjection): + """ + Lambert Azimuthal Equal-Area projection implementation. + + This implements the equations for the Lambert Azimuthal Equal-Area projection + which preserves area but not angles or distances. + https://mathworld.wolfram.com/LambertAzimuthalEqual-AreaProjection.html + """ + + def __init__(self, lambda_0: float, phi_1: float, radius: float = 6371229.0): + """ + Initialize a Lambert Azimuthal Equal-Area projection. + + Parameters: + ----------- + lambda_0 : float + Central longitude in degrees + phi_1 : float + Standard parallel in degrees + radius : float + Radius of Earth in meters (default: 6371229.0) + """ + self.lambda_0 = np.radians(lambda_0) + self.phi_1 = np.radians(phi_1) + self.R = radius + + def forward(self, latitude: CoordType, longitude: CoordType) -> ReturnUnionType: + """ + Transform from lat/lon coordinates to projected x/y coordinates. + + Parameters: + ----------- + latitude : float or array + Latitude in degrees + longitude : float or array + Longitude in degrees + + Returns: + -------- + tuple + (x, y) coordinates in the projection + """ + scalar_input = np.isscalar(latitude) and np.isscalar(longitude) + + lat_arr = np.asarray(latitude, dtype=np.float64) + lon_arr = np.asarray(longitude, dtype=np.float64) + + lambda_ = np.radians(lon_arr) + phi = np.radians(lat_arr) + + k = np.sqrt(2 / (1 + np.sin(self.phi_1) * np.sin(phi) + + np.cos(self.phi_1) * np.cos(phi) * np.cos(lambda_ - self.lambda_0))) + + x = self.R * k * np.cos(phi) * np.sin(lambda_ - self.lambda_0) + y = self.R * k * (np.cos(self.phi_1) * np.sin(phi) - + np.sin(self.phi_1) * np.cos(phi) * np.cos(lambda_ - self.lambda_0)) + + if scalar_input: + return float(x.item()), float(y.item()) + + return x, y + + def inverse(self, x: CoordType, y: CoordType) -> ReturnUnionType: + """ + Transform from projected x/y coordinates back to lat/lon. + + Parameters: + ----------- + x : float or array + X coordinate in the projection + y : float or array + Y coordinate in the projection + + Returns: + -------- + tuple + (latitude, longitude) in degrees + """ + scalar_input = np.isscalar(x) and np.isscalar(y) + + x_arr = np.asarray(x, dtype=np.float64) + y_arr = np.asarray(y, dtype=np.float64) + + x_norm = x_arr / self.R + y_norm = y_arr / self.R + p = np.sqrt(x_norm * x_norm + y_norm * y_norm) + + # Handle the case where p is zero (projection center) + zero_p = (p == 0) + p = np.where(zero_p, np.finfo(np.float32).eps, p) # Avoid division by zero + + c = 2 * np.arcsin(0.5 * p) + phi = np.arcsin(np.cos(c) * np.sin(self.phi_1) + + (y_norm * np.sin(c) * np.cos(self.phi_1)) / p) + lambda_ = self.lambda_0 + np.arctan2( + x_norm * np.sin(c), + p * np.cos(self.phi_1) * np.cos(c) - y_norm * np.sin(self.phi_1) * np.sin(c) + ) + lat = np.degrees(phi) + lon = np.degrees(lambda_) + + if scalar_input: + return float(lat.item()), float(lon.item()) + + return lat, lon + P = TypeVar('P', bound=AbstractProjection) diff --git a/tests/test_grids.py b/tests/test_grids.py index b5d291e4..0d058bf3 100644 --- a/tests/test_grids.py +++ b/tests/test_grids.py @@ -1,6 +1,11 @@ import numpy as np import pytest -from omfiles.grids import ProjectionGrid, RotatedLatLonProjection, StereographicProjection +from omfiles.grids import ( + LambertAzimuthalEqualAreaProjection, + ProjectionGrid, + RotatedLatLonProjection, + StereographicProjection, +) from omfiles.om_domains import RegularLatLonGrid # Fixtures for grids @@ -152,3 +157,39 @@ def test_hrdps_grid(hrdps_grid): lat2, lon2 = hrdps_grid.getCoordinates(x, y) assert abs(lat2 - lat) < 0.001, f"latitude mismatch: got {lat2}, expected {lat}" assert abs(lon2 - lon) < 0.001, f"longitude mismatch: got {lon2}, expected {lon}" + + +def test_lambert_azimuthal_equal_area_projection(): + """ + Test the Lambert Azimuthal Equal-Area projection. + https://github.com/open-meteo/open-meteo/blob/522917b1d6e72a7e6b7d4ae7dfb49b0c556a6992/Tests/AppTests/DataTests.swift#L189 + """ + proj = LambertAzimuthalEqualAreaProjection(lambda_0=-2.5, phi_1=54.9, radius=6371229) + grid = ProjectionGrid( + projection=proj, + nx=1042, + ny=970, + origin=(-1158000, -1036000), + dx=2000, + dy=2000 + ) + + test_lon = 10.620785 + test_lat = 57.745566 + x, y = proj.forward(latitude=test_lat, longitude=test_lon) + assert abs(x - 773650.5058) < 0.0001 # TODO: There are numerical differences with the Swift test case + assert abs(y - 389820.1483) < 0.0001 # TODO: There are numerical differences with the Swift test case + + lat, lon = proj.inverse(x=773650.5, y=389820.06) + assert abs(lon - test_lon) < 0.0001 + assert abs(lat - test_lat) < 0.0001 + + point_xy = grid.findPointXy(lat=test_lat, lon=test_lon) + assert point_xy is not None, "Point not found in grid" + x_idx, y_idx = point_xy + assert x_idx == 966, f"Expected x index to be 966, got {x_idx}" + assert y_idx == 713, f"Expected y index to be 713, got {y_idx}" + + lat2, lon2 = grid.getCoordinates(x_idx, y_idx) + assert abs(lon2 - 10.6271515) < 0.0001, f"Expected longitude to be 10.6271515, got {lon2}" + assert abs(lat2 - 57.746563) < 0.0001, f"Expected latitude to be 57.746563, got {lat2}" From 23ea1729cca02df740cbede6aba7cffc67c7af2f Mon Sep 17 00:00:00 2001 From: terraputix Date: Mon, 19 May 2025 12:31:37 +0200 Subject: [PATCH 10/50] add lambert conformal --- python/omfiles/grids.py | 128 +++++++++++++++++++++++++- tests/test_grids.py | 196 ++++++++++++++++++++++++++++++++++++++-- 2 files changed, 312 insertions(+), 12 deletions(-) diff --git a/python/omfiles/grids.py b/python/omfiles/grids.py index 74b54811..15190417 100644 --- a/python/omfiles/grids.py +++ b/python/omfiles/grids.py @@ -526,6 +526,130 @@ def inverse(self, x: CoordType, y: CoordType) -> ReturnUnionType: return lat, lon +class LambertConformalConicProjection(AbstractProjection): + """ + Lambert Conformal Conic projection implementation. + + This implements the equations for the Lambert Conformal Conic projection, + which preserves angles but not areas or distances. + https://mathworld.wolfram.com/LambertConformalConicProjection.html + https://pubs.usgs.gov/pp/1395/report.pdf page 104 + """ + + def __init__(self, lambda_0: float, phi_0: float, phi_1: float, phi_2: float, radius: float = 6370997): + """ + Initialize a Lambert Conformal Conic projection. + + Parameters: + ----------- + lambda_0 : float + Reference longitude in degrees (LoVInDegrees in grib) + phi_0 : float + Reference latitude in degrees (LaDInDegrees in grib) + phi_1 : float + First standard parallel in degrees (Latin1InDegrees in grib) + phi_2 : float + Second standard parallel in degrees (Latin2InDegrees in grib) + radius : float + Radius of Earth in meters (default: 6370997) + """ + # Normalize lambda_0 to [-180, 180] range + lambda_0_normalized = ((lambda_0 + 180.0) % 360.0) - 180.0 + self.lambda_0 = np.radians(lambda_0_normalized) + + phi_0_rad = np.radians(phi_0) + phi_1_rad = np.radians(phi_1) + phi_2_rad = np.radians(phi_2) + + if phi_1 == phi_2: + self.n = np.sin(phi_1_rad) + else: + self.n = (np.log(np.cos(phi_1_rad) / np.cos(phi_2_rad)) / + np.log(np.tan(np.pi/4 + phi_2_rad/2) / np.tan(np.pi/4 + phi_1_rad/2))) + + self.F = ((np.cos(phi_1_rad) * np.power(np.tan(np.pi/4 + phi_1_rad/2), self.n)) / + self.n) + + self.rho_0 = self.F / np.power(np.tan(np.pi/4 + phi_0_rad/2), self.n) + + # Earth radius + self.R = radius + + def forward(self, latitude: CoordType, longitude: CoordType) -> ReturnUnionType: + """ + Transform from lat/lon coordinates to projected x/y coordinates. + + Parameters: + ----------- + latitude : float or array + Latitude in degrees + longitude : float or array + Longitude in degrees + + Returns: + -------- + tuple + (x, y) coordinates in the projection + """ + scalar_input = np.isscalar(latitude) and np.isscalar(longitude) + + phi = np.radians(np.asarray(latitude, dtype=np.float64)) + lambda_ = np.radians(np.asarray(longitude, dtype=np.float64)) + + # If (λ - λ0) exceeds the range:±: 180°, 360° should be added or subtracted. + theta = self.n * (lambda_ - self.lambda_0) + + rho = self.F / np.power(np.tan(np.pi/4 + phi/2), self.n) + x = self.R * rho * np.sin(theta) + y = self.R * (self.rho_0 - rho * np.cos(theta)) + + if scalar_input: + return float(x.item()), float(y.item()) + + return x, y + + def inverse(self, x: CoordType, y: CoordType) -> ReturnUnionType: + """ + Transform from projected x/y coordinates back to lat/lon. + + Parameters: + ----------- + x : float or array + X coordinate in the projection + y : float or array + Y coordinate in the projection + + Returns: + -------- + tuple + (latitude, longitude) in degrees + """ + scalar_input = np.isscalar(x) and np.isscalar(y) + + x_scaled = np.asarray(x, dtype=np.float64) / self.R + y_scaled = np.asarray(y, dtype=np.float64) / self.R + + theta = np.where(self.n >= 0, + np.arctan2(x_scaled, self.rho_0 - y_scaled), + np.arctan2(-x_scaled, y_scaled - self.rho_0)) + + sign = np.where(self.n > 0, 1, -1) + rho = sign * np.sqrt(np.square(x_scaled) + np.square(self.rho_0 - y_scaled)) + + phi = 2 * np.arctan(np.power(self.F / rho, 1/self.n)) - np.pi/2 + lambda_ = self.lambda_0 + theta / self.n + + lat = np.degrees(phi) + lon = np.degrees(lambda_) + + lon = np.where(lon > 180, lon - 360, lon) + + if scalar_input: + return float(lat.item()), float(lon.item()) + + return lat, lon + + P = TypeVar('P', bound=AbstractProjection) @@ -644,9 +768,7 @@ def from_center( New grid instance """ center = cast(tuple[float, float], projection.forward(center_lat, center_lon)) - origin_x = center[0] - dx * (nx // 2) - origin_y = center[1] - dy * (ny // 2) - return cls(projection, nx, ny, (origin_x, origin_y), dx, dy) + return cls(projection, nx, ny, center, dx, dy) @property def grid_type(self) -> str: diff --git a/tests/test_grids.py b/tests/test_grids.py index 0d058bf3..aef632d7 100644 --- a/tests/test_grids.py +++ b/tests/test_grids.py @@ -2,6 +2,7 @@ import pytest from omfiles.grids import ( LambertAzimuthalEqualAreaProjection, + LambertConformalConicProjection, ProjectionGrid, RotatedLatLonProjection, StereographicProjection, @@ -177,19 +178,196 @@ def test_lambert_azimuthal_equal_area_projection(): test_lon = 10.620785 test_lat = 57.745566 x, y = proj.forward(latitude=test_lat, longitude=test_lon) - assert abs(x - 773650.5058) < 0.0001 # TODO: There are numerical differences with the Swift test case - assert abs(y - 389820.1483) < 0.0001 # TODO: There are numerical differences with the Swift test case + assert abs(x - 773650.5058) < 0.0001 # TODO: There are small numerical differences with the Swift test case + assert abs(y - 389820.1483) < 0.0001 # TODO: There are small numerical differences with the Swift test case - lat, lon = proj.inverse(x=773650.5, y=389820.06) - assert abs(lon - test_lon) < 0.0001 - assert abs(lat - test_lat) < 0.0001 + lat, lon = proj.inverse(x=x, y=y) + assert abs(lon - test_lon) < 0.00001 + assert abs(lat - test_lat) < 0.00001 point_xy = grid.findPointXy(lat=test_lat, lon=test_lon) assert point_xy is not None, "Point not found in grid" x_idx, y_idx = point_xy - assert x_idx == 966, f"Expected x index to be 966, got {x_idx}" - assert y_idx == 713, f"Expected y index to be 713, got {y_idx}" + assert x_idx == 966 + assert y_idx == 713 lat2, lon2 = grid.getCoordinates(x_idx, y_idx) - assert abs(lon2 - 10.6271515) < 0.0001, f"Expected longitude to be 10.6271515, got {lon2}" - assert abs(lat2 - 57.746563) < 0.0001, f"Expected latitude to be 57.746563, got {lat2}" + assert abs(lon2 - 10.6271515) < 0.0001 + assert abs(lat2 - 57.746563) < 0.0001 + + +def test_lambert_conformal(): + """ + Based on Based on: https://github.com/open-meteo/open-meteo/blob/522917b1d6e72a7e6b7d4ae7dfb49b0c556a6992/Tests/AppTests/DataTests.swift#L128 + """ + proj = LambertConformalConicProjection(lambda_0=-97.5, phi_0=0, phi_1=38.5, phi_2=38.5, radius=6370.997) + x, y = proj.forward(latitude=47, longitude=-8) + assert abs(x - 5833.8667) < 0.0001 + assert abs(y - 8632.7338) < 0.0001 + lat, lon = proj.inverse(x=x, y=y) + assert abs(lat - 47) < 0.0001 + assert abs(lon - (-8)) < 0.0001 + + grid = ProjectionGrid.from_bounds( + nx=1799, + ny=1059, + lat_range=(21.138, 47.8424), + lon_range=(-122.72, -60.918), + projection=proj + ) + + point_xy = grid.findPointXy(lat=34, lon=-118) + assert point_xy is not None + x_idx, y_idx = point_xy + flat_idx = y_idx * grid.nx + x_idx + assert flat_idx == 777441 + + lat2, lon2 = grid.getCoordinates(x_idx, y_idx) + assert abs(lat2 - 34) < 0.01 + assert abs(lon2 - (-118)) < 0.1 + + # Test reference grid points + reference_points = [ + (21.137999999999987, 237.28 - 360, 0), + (24.449714395051082, 265.54789437771944 - 360, 10000), + (22.73382904757237, 242.93190409785294 - 360, 20000), + (24.37172305316154, 271.6307003393202 - 360, 30000), + (24.007414634071907, 248.77817290935954 - 360, 40000) + ] + + for lat, lon, expected_idx in reference_points: + point_xy = grid.findPointXy(lat=lat, lon=lon) + assert point_xy is not None + x_idx, y_idx = point_xy + flat_idx = y_idx * grid.nx + x_idx + assert flat_idx == expected_idx + + lat2, lon2 = grid.getCoordinates(x_idx, y_idx) + assert abs(lat2 - lat) < 0.001 + assert abs(lon2 - lon) < 0.001 + + +def test_nbm_grid(): + """ + Test the NBM (National Blend of Models) grid using Lambert Conformal Conic projection. + https://vlab.noaa.gov/web/mdl/nbm-grib2-v4.0 + https://github.com/open-meteo/open-meteo/blob/522917b1d6e72a7e6b7d4ae7dfb49b0c556a6992/Tests/AppTests/DataTests.swift#L94 + """ + # Create projection with appropriate parameters + proj = LambertConformalConicProjection( + lambda_0=265 - 360, phi_0=0, phi_1=25, phi_2=25, radius=6371200 + ) + + # Create grid + grid = ProjectionGrid.from_center( + projection=proj, + nx=2345, + ny=1597, + center_lat=19.229, + center_lon=233.723 - 360, + dx=2539.7, + dy=2539.7 + ) + + # Test forward projection of grid origin + x, y = proj.forward(latitude=19.229, longitude=233.723 - 360) + assert abs(x - (-3271192.6)) < 0.1 + assert abs(y - 2604269.4) < 0.1 + + # Test grid point lookup + point_xy = grid.findPointXy(lat=19.229, lon=233.723 - 360) + assert point_xy is not None + assert point_xy[0] == 0 + assert point_xy[1] == 0 + + # Test reference grid points directly from grib files + reference_points = [ + (21.137999999999987, 237.28 - 360, 117411), + (24.449714395051082, 265.54789437771944 - 360, 188910), + (22.73382904757237, 242.93190409785294 - 360, 180965), + (24.37172305316154, 271.6307003393202 - 360, 196187), + (24.007414634071907, 248.77817290935954 - 360, 232796) + ] + + for lat, lon, expected_idx in reference_points: + point_xy = grid.findPointXy(lat=lat, lon=lon) + assert point_xy is not None + x_idx, y_idx = point_xy + flat_idx = y_idx * grid.nx + x_idx + assert flat_idx == expected_idx + + # Test grid coordinate lookup for specific indices + reference_coords = [ + (0, 19.228992, -126.27699), + (10000, 21.794254, -111.44652), + (20000, 22.806227, -96.18898), + (30000, 22.222015, -80.87921), + (40000, 20.274399, -123.18192) + ] + + for idx, expected_lat, expected_lon in reference_coords: + y_idx = idx // grid.nx + x_idx = idx % grid.nx + lat, lon = grid.getCoordinates(x_idx, y_idx) + assert abs(lat - expected_lat) < 0.001 + assert abs(lon - expected_lon) < 0.001 + + +def test_lambert_conformal_conic_projection(): + """ + Test the Lambert Conformal Conic projection. + Based on: https://github.com/open-meteo/open-meteo/blob/522917b1d6e72a7e6b7d4ae7dfb49b0c556a6992/Tests/AppTests/DataTests.swift#L163 + """ + proj = LambertConformalConicProjection(lambda_0=352, phi_0=55.5, phi_1=55.5, phi_2=55.5, radius=6371229) + + center_lat = 39.671 + center_lon = -25.421997 + + grid = ProjectionGrid.from_center( + nx=1906, + ny=1606, + center_lat=center_lat, + center_lon=center_lon, + dx=2000, + dy=2000, + projection=proj + ) + + # Test forward projection + origin_x, origin_y = proj.forward(latitude=center_lat, longitude=center_lon) + assert abs(origin_x - (-1527524.624)) < 0.001 + assert abs(origin_y - (-1588681.042)) < 0.001 + lat, lon = proj.inverse(origin_x, origin_y) + assert abs(center_lat - lat) < 0.0001 + assert abs(center_lon - lon) < 0.0001 + + # Test another point + test_lat = 39.675304 + test_lon = -25.400146 + x1, y1 = proj.forward(latitude=test_lat, longitude=test_lon) + assert abs(origin_x - x1 - (-1998.358)) < 0.001 + assert abs(origin_y - y1 - (-0.187)) < 0.001 + lat, lon = proj.inverse(x1, y1) + assert abs(test_lat - lat) < 0.0001 + assert abs(test_lon - lon) < 0.0001 + + # Point at index 1 + lat, lon = grid.getCoordinates(1, 0) + assert abs(lat - test_lat) < 0.001 + assert abs(lon - test_lon) < 0.001 + point_idx = grid.findPointXy(lat=test_lat, lon=test_lon) + assert point_idx == (1, 0) + + # Coords(i: 122440, x: 456, y: 64, latitude: 42.18604, longitude: -15.30127) + lat, lon = grid.getCoordinates(456, 64) + assert abs(lat - 42.18604) < 0.001 + assert abs(lon - (-15.30127)) < 0.001 + point_idx = grid.findPointXy(lat=lat, lon=lon) + assert point_idx == (456, 64) + + # Coords(i: 2999780, x: 1642, y: 1573, latitude: 64.943695, longitude: 30.711975) + lat, lon = grid.getCoordinates(1642, 1573) + assert abs(lat - 64.943695) < 0.001 + assert abs(lon - 30.711975) < 0.001 + point_idx = grid.findPointXy(lat=lat, lon=lon) + assert point_idx == (1642, 1573) From 0d2f458668f517a7acb25e57bbf47a519ae648fa Mon Sep 17 00:00:00 2001 From: terraputix Date: Mon, 19 May 2025 12:33:06 +0200 Subject: [PATCH 11/50] fix array index input validation --- src/array_index.rs | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/src/array_index.rs b/src/array_index.rs index 3ae16ad1..a2d2e8bd 100644 --- a/src/array_index.rs +++ b/src/array_index.rs @@ -65,12 +65,10 @@ impl<'py> FromPyObject<'py> for ArrayIndex { impl ArrayIndex { pub fn to_read_range(&self, shape: &Vec) -> PyResult>> { // Input validation - if self.0.len() != shape.len() { - return Err(PyErr::new::(format!( - "Array dimensions do not match. Got: {:}, expected: {:}", - shape.len(), - self.0.len() - ))); + if self.0.len() > shape.len() { + return Err(PyErr::new::( + "Too many indices for array", + )); } let mut ranges = Vec::new(); From bdc98bac6eb3a3e7de4246ba59a028230968097a Mon Sep 17 00:00:00 2001 From: terraputix Date: Mon, 19 May 2025 19:11:45 +0200 Subject: [PATCH 12/50] normalize lon helper method --- python/omfiles/grids.py | 6 +++--- python/omfiles/utils.py | 3 +++ 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/python/omfiles/grids.py b/python/omfiles/grids.py index 15190417..bc219d7e 100644 --- a/python/omfiles/grids.py +++ b/python/omfiles/grids.py @@ -5,7 +5,7 @@ import numpy as np import numpy.typing as npt -from omfiles.utils import _modulo_positive +from omfiles.utils import _modulo_positive, _normalize_longitude class AbstractGrid(ABC): @@ -554,7 +554,7 @@ def __init__(self, lambda_0: float, phi_0: float, phi_1: float, phi_2: float, ra Radius of Earth in meters (default: 6370997) """ # Normalize lambda_0 to [-180, 180] range - lambda_0_normalized = ((lambda_0 + 180.0) % 360.0) - 180.0 + lambda_0_normalized = _normalize_longitude(lambda_0) self.lambda_0 = np.radians(lambda_0_normalized) phi_0_rad = np.radians(phi_0) @@ -857,7 +857,7 @@ def getCoordinates(self, x: int, y: int) -> Tuple[float, float]: ycord = float(y) * self.dy + self.origin[1] lat, lon = cast(tuple[float, float], self.projection.inverse(xcord, ycord)) # Normalize longitude to -180 to 180 range - lon = ((lon + 180.0) % 360.0) - 180.0 + lon = _normalize_longitude(lon) return (lat, lon) def get_true_north_direction(self) -> np.ndarray: diff --git a/python/omfiles/utils.py b/python/omfiles/utils.py index b3de924b..bc45428f 100644 --- a/python/omfiles/utils.py +++ b/python/omfiles/utils.py @@ -19,3 +19,6 @@ def _modulo_positive(value: int, modulo: int) -> int: Positive modulo result """ return ((value % modulo) + modulo) % modulo + +def _normalize_longitude(lon: float) -> float: + return ((lon + 180.0) % 360.0) - 180.0 From 1c878284a9ac4cb6b0cfdb6feff43e6bbe4826f9 Mon Sep 17 00:00:00 2001 From: terraputix Date: Mon, 19 May 2025 19:18:42 +0200 Subject: [PATCH 13/50] tests against proj4 implementations --- .github/actions/test/action.yml | 9 +- pyproject.toml | 6 + tests/test_grids.py | 219 ++++++++++++++++++++++++++++++++ 3 files changed, 228 insertions(+), 6 deletions(-) diff --git a/.github/actions/test/action.yml b/.github/actions/test/action.yml index bdc2e16f..e9db0786 100644 --- a/.github/actions/test/action.yml +++ b/.github/actions/test/action.yml @@ -36,9 +36,8 @@ runs: - name: Run regular tests if: ${{ inputs.test-type == 'standard' && inputs.is-musl != 'true' }} run: | - python -m pip install pytest psutil WHEEL_PATH=$(ls dist/*.whl) - python -m pip install --force-reinstall "$WHEEL_PATH" + python -m pip install --force-reinstall "$WHEEL_PATH[test]" pytest tests/ shell: bash @@ -55,17 +54,15 @@ runs: source .venv/bin/activate pip install --upgrade pip WHEEL_PATH=$(ls dist/*.whl) - pip install --force-reinstall "$WHEEL_PATH" - pip install pytest psutil + pip install --force-reinstall "$WHEEL_PATH[test]" pytest tests/ # Minimum dependencies test - name: Run tests with minimum dependencies if: ${{ inputs.test-type == 'min_deps' }} run: | - python -m pip install pytest==6.0 psutil WHEEL_PATH=$(ls dist/*.whl) - python -m pip install --force-reinstall "$WHEEL_PATH" \ + python -m pip install --force-reinstall "$WHEEL_PATH[test]" \ "numpy==1.21.0" \ "fsspec==2023.1.0" \ "s3fs==2023.1.0" \ diff --git a/pyproject.toml b/pyproject.toml index 3cc4ec5d..bd70a0f5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -22,12 +22,18 @@ dependencies = [ dev = [ "pytest>=6.0", "psutil", + "pyproj", "hidefix", "h5py", "netCDF4", "zarr", "tensorstore", ] +test = [ + "pytest>=6.0", + "psutil", + "pyproj", +] [tool.maturin] python-source = "python" # Python source code is in the `python` directory diff --git a/tests/test_grids.py b/tests/test_grids.py index aef632d7..0c01fc3a 100644 --- a/tests/test_grids.py +++ b/tests/test_grids.py @@ -1,4 +1,5 @@ import numpy as np +import pyproj import pytest from omfiles.grids import ( LambertAzimuthalEqualAreaProjection, @@ -8,6 +9,7 @@ StereographicProjection, ) from omfiles.om_domains import RegularLatLonGrid +from omfiles.utils import _normalize_longitude # Fixtures for grids @@ -371,3 +373,220 @@ def test_lambert_conformal_conic_projection(): assert abs(lon - 30.711975) < 0.001 point_idx = grid.findPointXy(lat=lat, lon=lon) assert point_idx == (1642, 1573) + + +def test_rotated_latlon_against_proj(): + # Create our custom projection + lat_origin = -36.0885 + lon_origin = 245.305 + custom_proj = RotatedLatLonProjection(lat_origin=lat_origin, lon_origin=lon_origin) + + # Create equivalent PROJ projection + proj_string = (f"+proj=ob_tran +o_proj=longlat +o_lat_p={-lat_origin} " + f"+o_lon_p=0.0 +lon_0={lon_origin} +datum=WGS84 +no_defs +type=crs") + proj_proj = pyproj.Proj(proj_string) + + # Test points covering different regions + test_points = [ + (0, 0), # Origin + (45, 45), # Mid-latitude point + (-45, -45), # Mid-latitude point (southern hemisphere) + (10, 50), # Europe + (40, -100), # North America + (50, -170), # Pacific + (-30, 170), # South Pacific + ] + + for lat, lon in test_points: + # Forward transformation using our implementation + custom_x, custom_y = custom_proj.forward(latitude=lat, longitude=lon) + + # Forward transformation using PROJ + # Note: PROJ expects (lon, lat) order, not (lat, lon) + # proj_x, proj_y = proj_proj(np.radians(lon), np.radians(lat)) + proj_x, proj_y = proj_proj(lon, lat) + # The following fix should be available in proj, but something is weird + # with radians/degrees with ob_tran.... + # https://github.com/OSGeo/PROJ/issues/2804 + proj_x = np.degrees(proj_x) + proj_y = np.degrees(proj_y) + + # Compare results - allowing for small differences due to floating point math + # Convert to radians for comparison since our implementation works in radians + assert abs(custom_x - proj_x) < 1e-5, f"X mismatch for ({lat}, {lon}): custom={custom_x}, proj={proj_x}" + assert abs(custom_y - proj_y) < 1e-5, f"Y mismatch for ({lat}, {lon}): custom={custom_y}, proj={proj_y}" + + # Test inverse transformation + custom_lat, custom_lon = custom_proj.inverse(x=custom_x, y=custom_y) + # PROJ expects inverse=True for inverse transform + proj_lon, proj_lat = proj_proj(np.radians(proj_x), np.radians(proj_y), inverse=True) + + # Compare results + assert abs(custom_lat - proj_lat) < 1e-5, f"Lat mismatch for ({custom_x}, {custom_y}): custom={custom_lat}, proj={proj_lat}" + assert abs(np.mod(custom_lon - proj_lon + 180, 360) - 180) < 1e-5, f"Lon mismatch for ({custom_x}, {custom_y}): custom={custom_lon}, proj={proj_lon}" + + +def test_stereographic_against_proj(): + # Create our custom projection + latitude = 90.0 # North pole + longitude = 249.0 + radius = 6371229.0 + custom_proj = StereographicProjection(latitude=latitude, longitude=longitude, radius=radius) + + # Create equivalent PROJ projection + proj_string = (f"+proj=stere +lat_0={latitude} +lon_0={longitude} +k=1 " + f"+x_0=0 +y_0=0 +R={radius} +units=m +no_defs") + proj_proj = pyproj.Proj(proj_string) + + # Test points - staying away from singular points (poles) + test_points = [ + (0, 0), # Equator + (45, 45), # Mid-latitude + (60, -120), # Northern regions + (45, 249), # Along the central meridian + (70, 249), # Along the central meridian + (80, 249), # Along the central meridian + ] + + for lat, lon in test_points: + # Forward transformation using our implementation + custom_x, custom_y = custom_proj.forward(latitude=lat, longitude=lon) + + # Forward transformation using PROJ + # PROJ uses (lon, lat) order + proj_x, proj_y = proj_proj(lon, lat) + + # Compare results (allowing some tolerance due to potential differences in algorithms) + # Stereographic projections can have larger errors for points far from the center + tolerance = 1 # tolerance in meters + assert abs(custom_x - proj_x) < tolerance, f"X mismatch for ({lat}, {lon}): custom={custom_x}, proj={proj_x}" + assert abs(custom_y - proj_y) < tolerance, f"Y mismatch for ({lat}, {lon}): custom={custom_y}, proj={proj_y}" + + # Test inverse transformation + custom_lat, custom_lon = custom_proj.inverse(x=custom_x, y=custom_y) + proj_lon, proj_lat = proj_proj(proj_x, proj_y, inverse=True) + + # Compare results + assert abs(custom_lat - proj_lat) < 1e-5, f"Lat mismatch: custom={custom_lat}, proj={proj_lat}" + custom_lon = _normalize_longitude(custom_lon) + assert abs(custom_lon - proj_lon) < 1e-4, f"Lon mismatch: custom={custom_lon}, proj={proj_lon}" + +def test_lambert_azimuthal_equal_area_against_proj(): + # Create our custom projection + lambda_0 = -2.5 # Central longitude in degrees + phi_1 = 54.9 # Standard parallel/latitude in degrees + radius = 6371229.0 # Earth radius in meters + custom_proj = LambertAzimuthalEqualAreaProjection(lambda_0=lambda_0, phi_1=phi_1, radius=radius) + + # Create equivalent PROJ projection + # For Lambert Azimuthal Equal Area, we use lat_0 for the standard parallel and lon_0 for central longitude + proj_string = (f"+proj=laea +lat_0={phi_1} +lon_0={lambda_0} +x_0=0 +y_0=0 " + f"+R={radius} +units=m +no_defs +type=crs") + proj_proj = pyproj.Proj(proj_string) + + # Test points covering different regions + test_points = [ + (0, 0), # Origin + (54.9, -2.5), # Projection center (should map to 0,0) + (45, 45), # Mid-latitude point + (-45, -45), # Mid-latitude point (southern hemisphere) + (10, 50), # Europe + (40, -100), # North America + (50, -170), # Pacific + (-30, 170), # South Pacific + # Test point from the existing test + (57.745566, 10.620785) + ] + + for lat, lon in test_points: + # Forward transformation using our implementation + custom_x, custom_y = custom_proj.forward(latitude=lat, longitude=lon) + + # Forward transformation using PROJ + # Note: PROJ expects (lon, lat) order, not (lat, lon) + proj_x, proj_y = proj_proj(lon, lat) + + # Compare results - Lambert projections can have larger differences due to algorithmic differences + # Use a reasonable tolerance (e.g., 0.1 meter for a 6.3 million meter radius) + tolerance = 0.1 + assert abs(custom_x - proj_x) < tolerance, f"X mismatch for ({lat}, {lon}): custom={custom_x}, proj={proj_x}" + assert abs(custom_y - proj_y) < tolerance, f"Y mismatch for ({lat}, {lon}): custom={custom_y}, proj={proj_y}" + + # Test inverse transformation (skip points very close to the poles where inverse can be unstable) + if abs(lat) < 89: + custom_lat, custom_lon = custom_proj.inverse(x=custom_x, y=custom_y) + # PROJ expects inverse=True for inverse transform + proj_lon, proj_lat = proj_proj(proj_x, proj_y, inverse=True) + + # Compare results with appropriate tolerance + # For inverse transformations, angular differences can be larger + angular_tolerance = 1e-5 # roughly 0.00001 degrees + assert abs(custom_lat - proj_lat) < angular_tolerance, \ + f"Lat mismatch for ({custom_x}, {custom_y}): custom={custom_lat}, proj={proj_lat}" + + # Handle longitude wraparound for comparison + lon_diff = np.mod(abs(custom_lon - proj_lon), 360) + assert min(lon_diff, 360 - lon_diff) < angular_tolerance, \ + f"Lon mismatch for ({custom_x}, {custom_y}): custom={custom_lon}, proj={proj_lon}" + +def test_lambert_conformal_conic_against_proj(): + # Create our custom projection with parameters from the existing test + lambda_0 = 352 # Reference longitude in degrees + phi_0 = 55.5 # Reference latitude in degrees + phi_1 = 55.5 # First standard parallel in degrees + phi_2 = 55.5 # Second standard parallel in degrees + radius = 6371229.0 # Earth radius in meters + + custom_proj = LambertConformalConicProjection( + lambda_0=lambda_0, phi_0=phi_0, phi_1=phi_1, phi_2=phi_2, radius=radius + ) + + lambda_0_norm = _normalize_longitude(lambda_0) + # Create equivalent PROJ projection + # For Lambert Conformal Conic, we use lat_0, lon_0, lat_1, lat_2 parameters + proj_string = (f"+proj=lcc +lat_0={phi_0} +lon_0={lambda_0_norm} +lat_1={phi_1} +lat_2={phi_2} " + f"+x_0=0 +y_0=0 +R={radius} +units=m +no_defs +type=crs") + proj_proj = pyproj.Proj(proj_string) + + # Test points from the existing test + center_lat = 39.671 + center_lon = -25.421997 + test_points = [ + (center_lat, center_lon), # Center point + (39.675304, -25.400146), # Near the center + (42.18604, -15.30127), # Point from the test (x=456, y=64) + (64.943695, 30.711975), # Point from the test (x=1642, y=1573) + # Additional test points for broader coverage + (0, 0), # Origin + (phi_0, lambda_0_norm), # Projection origin + (45, 0), # Mid-latitude point + (-45, -45), # Southern hemisphere + (10, 50), # Europe + (40, -100), # North America + (50, -170), # Pacific + (-30, 170), # South Pacific + ] + + for lat, lon in test_points: + # Forward transformation using our implementation + custom_x, custom_y = custom_proj.forward(latitude=lat, longitude=lon) + + # Forward transformation using PROJ + # Note: PROJ expects (lon, lat) order, not (lat, lon) + proj_x, proj_y = proj_proj(lon, lat) + tolerance = 0.1 # 0.1 meters for a 6.3 million meter radius is a reasonable precision + assert abs(custom_x - proj_x) < tolerance, f"X mismatch for ({lat}, {lon}): custom={custom_x}, proj={proj_x}" + assert abs(custom_y - proj_y) < tolerance, f"Y mismatch for ({lat}, {lon}): custom={custom_y}, proj={proj_y}" + + # Test inverse transformation + custom_lat, custom_lon = custom_proj.inverse(x=custom_x, y=custom_y) + # PROJ expects inverse=True for inverse transform + proj_lon, proj_lat = proj_proj(proj_x, proj_y, inverse=True) + angular_tolerance = 1e-5 # approximately 0.00001 degrees + assert abs(custom_lat - proj_lat) < angular_tolerance, \ + f"Lat mismatch for ({custom_x}, {custom_y}): custom={custom_lat}, proj={proj_lat}" + + # Handle longitude wraparound for comparison + lon_diff = np.mod(abs(custom_lon - proj_lon), 360) + assert min(lon_diff, 360 - lon_diff) < angular_tolerance, \ + f"Lon mismatch for ({custom_x}, {custom_y}): custom={custom_lon}, proj={proj_lon}" From b9e44226d1c53b37299cc607a06acb99a47de781 Mon Sep 17 00:00:00 2001 From: terraputix Date: Tue, 20 May 2025 11:54:08 +0200 Subject: [PATCH 14/50] proj prejection grids --- python/omfiles/grids.py | 68 ++++++++++- tests/test_grids.py | 262 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 328 insertions(+), 2 deletions(-) diff --git a/python/omfiles/grids.py b/python/omfiles/grids.py index bc219d7e..29e53080 100644 --- a/python/omfiles/grids.py +++ b/python/omfiles/grids.py @@ -683,9 +683,9 @@ def __init__( origin : Tuple[float, float] Origin coordinates (x, y) of the grid in projection space dx : float - Grid spacing in x direction in meters + Grid spacing in x direction dy : float - Grid spacing in y direction in meters + Grid spacing in y direction """ self.projection = projection self.nx = nx @@ -945,3 +945,67 @@ def find_box( (y_indices.flatten(), x_indices.flatten()), (self.ny, self.nx) ) + + +class ProjProjection(AbstractProjection): + """A projection that wraps a proj projection""" + + def __init__(self, proj_string: str): + """Initialize with a proj string or EPSG code + + Parameters + ---------- + proj_string : str + The proj string (e.g. "+proj=lcc +lat_0=50...") or + EPSG code (e.g. "EPSG:4326") + """ + import pyproj + # Create transformer from lat/lon to projection coordinates + self.crs_proj = pyproj.CRS(proj_string) + self.crs_latlon = pyproj.CRS("EPSG:4326") # WGS84 + self.forward_transformer = pyproj.Transformer.from_crs( + self.crs_latlon, + self.crs_proj, + always_xy=True # This ensures lon/lat -> x/y order + ) + self.inverse_transformer = pyproj.Transformer.from_crs( + self.crs_proj, + self.crs_latlon, + always_xy=True + ) + + def forward(self, latitude: CoordType, longitude: CoordType) -> ReturnUnionType: + """Transform from latitude/longitude to projection coordinates + + Parameters + ---------- + latitude : float + Latitude in degrees + longitude : float + Longitude in degrees + + Returns + ------- + tuple[float, float] + The (x, y) coordinates in the projection + """ + x, y = self.forward_transformer.transform(longitude, latitude) + return x, y + + def inverse(self, x: CoordType, y: CoordType) -> ReturnUnionType: + """Transform from projection coordinates to latitude/longitude + + Parameters + ---------- + x : float + X coordinate in the projection + y : float + Y coordinate in the projection + + Returns + ------- + tuple[float, float] + The (latitude, longitude) coordinates in degrees + """ + lon, lat = self.inverse_transformer.transform(x, y) + return lat, lon diff --git a/tests/test_grids.py b/tests/test_grids.py index 0c01fc3a..500420df 100644 --- a/tests/test_grids.py +++ b/tests/test_grids.py @@ -5,6 +5,7 @@ LambertAzimuthalEqualAreaProjection, LambertConformalConicProjection, ProjectionGrid, + ProjProjection, RotatedLatLonProjection, StereographicProjection, ) @@ -590,3 +591,264 @@ def test_lambert_conformal_conic_against_proj(): lon_diff = np.mod(abs(custom_lon - proj_lon), 360) assert min(lon_diff, 360 - lon_diff) < angular_tolerance, \ f"Lon mismatch for ({custom_x}, {custom_y}): custom={custom_lon}, proj={proj_lon}" + +def test_regular_lat_lon_grid_against_proj(): + """Test that RegularLatLonGrid operations match proj equivalent operations""" + # Create a regular lat-lon grid with 1-degree steps + grid = RegularLatLonGrid( + lat_start=-90, + lat_steps=181, # -90 to 90 + lat_step_size=1.0, + lon_start=-180, + lon_steps=360, # -180 to 180 + lon_step_size=1.0 + ) + + # Create proj objects for WGS84 lat/lon + proj_wgs84 = pyproj.Proj(proj='latlong', datum='WGS84') + + # Test points covering different scenarios + test_points: list[tuple[float, float]] = [ + (0, 0), # Origin + (45, 45), # NE quadrant + (-45, -45), # SW quadrant + (45, -45), # SE quadrant + (-45, 45), # NW quadrant + (89, 0), # Near North pole + (-89, 0), # Near South pole + (0, 179), # Near date line (east) + (0, -179), # Near date line (west) + (10, 20), # Random point + (-33, 151), # Sydney + (37, -122), # San Francisco + ] + + for lat, lon in test_points: + # Get grid coordinates using our implementation + grid_sel = grid.findPointXy(lat, lon) + assert type(grid_sel) is tuple + grid_x, grid_y = grid_sel + result_lat, result_lon = grid.getCoordinates(grid_x, grid_y) + + # For a lat/lon grid, proj just keeps the same coordinates + proj_x, proj_y = proj_wgs84(lon, lat) # Note: proj uses (lon, lat) order + proj_lat, proj_lon = proj_wgs84(proj_y, proj_x, inverse=True) # Get back lat/lon from proj + + # We'll check that our forward and inverse transformations are consistent + # and match with proj's (which just returns the original coordinates for this projection) + + # Check roundtrip accuracy + assert abs(result_lat - lat) < 1e-9, f"Lat roundtrip error: original={lat}, result={result_lat}" + + # Normalize longitudes before comparison due to -180/180 wrapping + lon_norm = _normalize_longitude(lon) + result_lon_norm = _normalize_longitude(result_lon) + assert abs(result_lon_norm - lon_norm) < 1e-9, f"Lon roundtrip error: original={lon}, result={result_lon}" + + # Verify agreement with proj + assert abs(lat - proj_lat) < 1e-9 + assert abs(lon_norm - _normalize_longitude(proj_lon)) < 1e-9, f"Lon mismatch with proj at ({lat}, {lon})" + + # Test longitude wrapping behavior + wrap_test_points = [ + (0, 185), # Should wrap to (0, -175) + (0, -185), # Should wrap to (0, 175) + (0, 361), # Should wrap to (0, 1) + (0, -361), # Should wrap to (0, -1) + (45, 540), # Should wrap to (45, -180) + ] + + for lat, lon in wrap_test_points: + # Find grid coordinates for the wrapped point + grid_x, grid_y = grid.findPointXy(lat=lat, lon=lon) + + # Find grid coordinates for the normalized longitude + norm_lon = _normalize_longitude(lon) + norm_grid_x, norm_grid_y = grid.findPointXy(lat=lat, lon=norm_lon) + assert grid_x == norm_grid_x, f"Grid X mismatch for wrapped lon: {lon} vs {norm_lon}" + assert grid_y == norm_grid_y, f"Grid Y mismatch for wrapped lon: {lon} vs {norm_lon}" + + +def test_proj_projection(): + """Test that ProjProjection correctly wraps proj transformations""" + # Test with a Lambert Conformal Conic projection + proj_string = ( + "+proj=lcc +lat_0=55.5 +lon_0=352 +lat_1=55.5 +lat_2=55.5 " + "+x_0=0 +y_0=0 +R=6371229 +units=m +no_defs" + ) + + # Create our projection wrapper + proj = ProjProjection(proj_string) + + # Create a grid using this projection + grid = ProjectionGrid.from_bounds( + nx=100, + ny=100, + lat_range=(39.67, 64.94), + lon_range=(-25.42, 30.71), + projection=proj + ) + + # Test points + test_points = [ + (39.671, -25.421997), # Lower left + (64.943695, 30.711975), # Upper right + (50.0, 0.0), # Middle-ish + ] + + # Create the raw proj transformer for comparison + raw_proj = pyproj.Proj(proj_string) + + for lat, lon in test_points: + # Forward transformation + grid_x, grid_y = proj.forward(lat, lon) + raw_x, raw_y = raw_proj(lon, lat) # Note: raw proj expects (lon, lat) + + # Compare forward results + assert abs(grid_x - raw_x) < 1e-8, f"X mismatch: {grid_x} vs {raw_x}" + assert abs(grid_y - raw_y) < 1e-8, f"Y mismatch: {grid_y} vs {raw_y}" + + # Inverse transformation + back_lat, back_lon = proj.inverse(grid_x, grid_y) + raw_lon, raw_lat = raw_proj(raw_x, raw_y, inverse=True) + + # Compare inverse results + assert abs(back_lat - raw_lat) < 1e-8, f"Lat mismatch: {back_lat} vs {raw_lat}" + assert abs(back_lon - raw_lon) < 1e-8, f"Lon mismatch: {back_lon} vs {raw_lon}" + + # Test roundtrip through the grid + grid_coords = grid.findPointXy(lat=lat, lon=lon) + result_lat, result_lon = grid.getCoordinates(*grid_coords) + + # Results should match within grid resolution + assert abs(result_lat - lat) < grid.dy, f"Grid lat error: {result_lat} vs {lat}" + assert abs(result_lon - lon) < grid.dx, f"Grid lon error: {result_lon} vs {lon}" + +def test_grid_equivalence(): + """Test that a proj-based grid matches the original implementation""" + # Create original LambertConformalConic projection + original_proj = LambertConformalConicProjection( + lambda_0=352, + phi_0=55.5, + phi_1=55.5, + phi_2=55.5, + radius=6371229.0 + ) + + # Create equivalent proj-based projection + proj_proj = ProjProjection( + "+proj=lcc +lat_0=55.5 +lon_0=352 +lat_1=55.5 +lat_2=55.5 " + "+x_0=0 +y_0=0 +R=6371229 +units=m +no_defs" + ) + + # Create grids with both projections + grid_bounds = { + 'nx': 100, + 'ny': 100, + 'lat_range': (39.67, 64.94), + 'lon_range': (-25.42, 30.71), + } + + original_grid = ProjectionGrid.from_bounds( + projection=original_proj, + **grid_bounds + ) + + proj_grid = ProjectionGrid.from_bounds( + projection=proj_proj, + **grid_bounds + ) + + # Test points + test_points = [ + (39.671, -25.421997), # Lower left + (64.943695, 30.711975), # Upper right + (50.0, 0.0), # Middle-ish + (45.0, -10.0), # Random point + (60.0, 20.0), # Random point + ] + + for lat, lon in test_points: + # Compare projection results + orig_x, orig_y = original_proj.forward(lat, lon) + proj_x, proj_y = proj_proj.forward(lat, lon) + + # Results should match within reasonable tolerance + assert abs(orig_x - proj_x) < 1e-3, f"X mismatch: {orig_x} vs {proj_x}" + assert abs(orig_y - proj_y) < 1e-3, f"Y mismatch: {orig_y} vs {proj_y}" + + # Compare grid results + orig_grid_xy = original_grid.findPointXy(lat=lat, lon=lon) + proj_grid_xy = proj_grid.findPointXy(lat=lat, lon=lon) + + # Grid coordinates should match exactly + assert abs(orig_grid_xy[0] - proj_grid_xy[0]) < 1e-8, \ + f"Grid X mismatch: {orig_grid_xy[0]} vs {proj_grid_xy[0]}" + assert abs(orig_grid_xy[1] - proj_grid_xy[1]) < 1e-8, \ + f"Grid Y mismatch: {orig_grid_xy[1]} vs {proj_grid_xy[1]}" + + +def test_grid_equivalence_regular_latlon(): + """Test that a proj-based regular lat-lon grid matches the original implementation""" + # Create a regular lat-lon grid using RegularLatLonGrid + original_grid = RegularLatLonGrid( + lat_start=10.0, + lat_steps=100, + lat_step_size=0.5, + lon_start=-30.0, + lon_steps=120, + lon_step_size=0.5 + ) + + # Create equivalent proj-based regular lat-lon projection + proj_proj = ProjProjection( + "+proj=longlat +datum=WGS84 +no_defs" + ) + + # Create equivalent grid with the proj projection + proj_grid = ProjectionGrid( + projection=proj_proj, + nx=120, + ny=100, + origin=(-30.0, 10.0), + dx=0.5, + dy=0.5 + ) + + # Test points covering various areas within the grid + test_points = [ + (10.0, -30.0), # Lower left corner + (59.5, 29.5), # Upper right corner + (35.0, 0.0), # Middle-ish + (20.0, -15.0), # Random point + (50.0, 20.0), # Random point + (15.25, -25.75), # Point between grid cells + ] + + for lat, lon in test_points: + # Get grid points using both implementations + orig_grid_xy = original_grid.findPointXy(lat=lat, lon=lon) + proj_grid_xy = proj_grid.findPointXy(lat=lat, lon=lon) + + # Both should find the point or both should not find it + assert (orig_grid_xy is None) == (proj_grid_xy is None), \ + f"Inconsistent point finding for ({lat}, {lon})" + + # If point is found, coordinates should match exactly + if orig_grid_xy is not None: + assert abs(orig_grid_xy[0] - proj_grid_xy[0]) < 1e-8, \ + f"Grid X mismatch: {orig_grid_xy[0]} vs {proj_grid_xy[0]}" + assert abs(orig_grid_xy[1] - proj_grid_xy[1]) < 1e-8, \ + f"Grid Y mismatch: {orig_grid_xy[1]} vs {proj_grid_xy[1]}" + + # Test the inverse transformation (getCoordinates) + if orig_grid_xy is not None: + x, y = orig_grid_xy + orig_lat, orig_lon = original_grid.getCoordinates(x, y) + proj_lat, proj_lon = proj_grid.getCoordinates(x, y) + + # Results should match exactly + assert abs(orig_lat - proj_lat) < 1e-8, \ + f"Latitude mismatch: {orig_lat} vs {proj_lat}" + assert abs(orig_lon - proj_lon) < 1e-8, \ + f"Longitude mismatch: {orig_lon} vs {proj_lon}" From 2afb7aacd0e6d61be4b0f9a83b95160d4297a2f4 Mon Sep 17 00:00:00 2001 From: terraputix Date: Fri, 20 Jun 2025 10:59:03 +0200 Subject: [PATCH 15/50] lint --- examples/select_by_coordinates.py | 62 +++--- python/omfiles/grids.py | 186 +++++++--------- python/omfiles/om_domains.py | 287 +++++++++--------------- python/omfiles/utils.py | 4 +- tests/test_grids.py | 353 +++++++++++++----------------- tests/test_om_domains.py | 10 +- 6 files changed, 370 insertions(+), 532 deletions(-) diff --git a/examples/select_by_coordinates.py b/examples/select_by_coordinates.py index 46f84250..c2d3c7a3 100644 --- a/examples/select_by_coordinates.py +++ b/examples/select_by_coordinates.py @@ -45,9 +45,10 @@ block_size=256, cache_storage="cache", check_files=False, - cache_mapper=BasenameCacheMapper(directory_levels=3) + cache_mapper=BasenameCacheMapper(directory_levels=3), ) + def load_chunk_data( chunk_index: int, domain_name: str, @@ -55,7 +56,7 @@ def load_chunk_data( grid_coords: Tuple[int, int], fs: fsspec.AbstractFileSystem, start_date: np.datetime64, - end_date: np.datetime64 + end_date: np.datetime64, ): """ Load data for a specific chunk and grid coordinates. @@ -90,7 +91,7 @@ def load_chunk_data( chunk_times = domain.get_chunk_time_range(chunk_index) time_mask = (chunk_times >= start_date) & (chunk_times <= end_date) if not np.any(time_mask): - return np.array([], dtype='datetime64[s]'), np.array([], dtype=float) + return np.array([], dtype="datetime64[s]"), np.array([], dtype=float) # Create reader and read data of interest with OmFilePyReader.from_fsspec(fs, s3_path) as reader: @@ -99,7 +100,7 @@ def load_chunk_data( data = reader[y, x, time_slice] return chunk_times[time_mask], data - raise ValueError("Unreachable") # Make Pyright happy... + raise ValueError("Unreachable") # Make Pyright happy... def get_data_for_coordinates( @@ -107,8 +108,8 @@ def get_data_for_coordinates( lon: float, start_date: datetime, end_date: datetime, - domain_name: str = 'ecmwf_ifs025', - variable_name: str = 'temperature_2m', + domain_name: str = "ecmwf_ifs025", + variable_name: str = "temperature_2m", ) -> Dataset: """ Fetch weather data for specific coordinates across a date range, merging multiple files as needed. @@ -160,15 +161,7 @@ def get_data_for_coordinates( all_data = [] for chunk_idx in chunk_indices: - times, data = load_chunk_data( - chunk_idx, - domain_name, - variable_name, - (x, y), - FS, - start_timestamp, - end_timestamp - ) + times, data = load_chunk_data(chunk_idx, domain_name, variable_name, (x, y), FS, start_timestamp, end_timestamp) if len(times) > 0: all_times.append(times) all_data.append(data) @@ -193,10 +186,11 @@ def get_data_for_coordinates( attrs={ "domain": domain_name, "grid_indices": grid_point, - } + }, ) return ds + if __name__ == "__main__": # Example coordinates: Paris latitude = 48.864716 @@ -208,28 +202,28 @@ def get_data_for_coordinates( # Define a date range start_date = datetime(2025, 4, 25, 12, 0) # 25-04-2025'T'12:00 - end_date = datetime(2025, 5, 18, 12, 0) # 18-05-2025'T'12:00 + end_date = datetime(2025, 5, 18, 12, 0) # 18-05-2025'T'12:00 # Variable to fetch - variable = 'temperature_2m' + variable = "temperature_2m" print(f"Fetching {variable} data for coordinates: {latitude}N, {longitude}E") print(f"Date range: {start_date} to {end_date}") # Domain display names for nicer legends domains_and_display_names = { - 'dwd_icon': 'DWD ICON (Global)', - 'dwd_icon_eu': 'DWD ICON (Europe)', - 'dwd_icon_d2': 'DWD ICON D2 (Central Europe)', - 'ecmwf_ifs025': 'ECMWF IFS (Global)', - 'meteofrance_arpege_europe': 'Météo-France ARPEGE (Europe)', - 'meteofrance_arpege_world025': 'Météo-France ARPEGE (Global)', - 'meteofrance_arome_france0025': 'Météo-France AROME (France)', - 'meteofrance_arome_france_hd': 'Météo-France AROME HD (France)', - 'meteofrance_arome_france_hd_15min': 'Météo-France AROME HD 15min (France)', - 'gem_global': 'CMC GEM GDPS (Global)', - 'gem_regional': 'CMC GEM RDPS (Regional)', - 'gem_hrdps_continental': 'CMC GEM HRDPS (Continental)', + "dwd_icon": "DWD ICON (Global)", + "dwd_icon_eu": "DWD ICON (Europe)", + "dwd_icon_d2": "DWD ICON D2 (Central Europe)", + "ecmwf_ifs025": "ECMWF IFS (Global)", + "meteofrance_arpege_europe": "Météo-France ARPEGE (Europe)", + "meteofrance_arpege_world025": "Météo-France ARPEGE (Global)", + "meteofrance_arome_france0025": "Météo-France AROME (France)", + "meteofrance_arome_france_hd": "Météo-France AROME HD (France)", + "meteofrance_arome_france_hd_15min": "Météo-France AROME HD 15min (France)", + "gem_global": "CMC GEM GDPS (Global)", + "gem_regional": "CMC GEM RDPS (Regional)", + "gem_hrdps_continental": "CMC GEM HRDPS (Continental)", } # Collect data from each domain @@ -246,7 +240,7 @@ def get_data_for_coordinates( start_date=start_date, end_date=end_date, domain_name=domain_name, - variable_name=variable + variable_name=variable, ) domain_data[domain_name] = ds successful_domains.append(domain_name) @@ -262,7 +256,7 @@ def get_data_for_coordinates( exit(1) # Domain colors for consistent line colors - colors = plt.get_cmap('tab10')(np.linspace(0, 1, len(successful_domains))) + colors = plt.get_cmap("tab10")(np.linspace(0, 1, len(successful_domains))) plt.figure(figsize=(12, 6)) @@ -275,9 +269,9 @@ def get_data_for_coordinates( # Enhance the plot plt.title(f"{variable.replace('_', ' ').title()} at {latitude:.2f}N, {longitude:.2f}E") plt.xlabel("Time") - plt.ylabel("Temperature (°C)" if variable == 'temperature_2m' else variable) + plt.ylabel("Temperature (°C)" if variable == "temperature_2m" else variable) plt.grid(True, alpha=0.3) - plt.legend(loc='best') + plt.legend(loc="best") plt.tight_layout() # Save and show the figure diff --git a/python/omfiles/grids.py b/python/omfiles/grids.py index 29e53080..1ac124b5 100644 --- a/python/omfiles/grids.py +++ b/python/omfiles/grids.py @@ -113,10 +113,7 @@ def latitude(self) -> np.ndarray: Lazily compute and cache the latitude coordinate array. """ return np.linspace( - self._lat_start, - self._lat_start + self._lat_step_size * self._lat_steps, - self._lat_steps, - endpoint=False + self._lat_start, self._lat_start + self._lat_step_size * self._lat_steps, self._lat_steps, endpoint=False ) @cached_property @@ -125,10 +122,7 @@ def longitude(self) -> np.ndarray: Lazily compute and cache the longitude coordinate array. """ return np.linspace( - self._lon_start, - self._lon_start + self._lon_step_size * self._lon_steps, - self._lon_steps, - endpoint=False + self._lon_start, self._lon_start + self._lon_step_size * self._lon_steps, self._lon_steps, endpoint=False ) @property @@ -185,12 +179,14 @@ def getCoordinates(self, x: int, y: int) -> Tuple[float, float]: return (lat, lon) + # Type aliases for clarity FloatType = Union[float, np.floating] ArrayType = npt.NDArray[np.floating] CoordType = Union[float, ArrayType] ReturnUnionType = Union[tuple[ArrayType, ArrayType], tuple[float, float]] + # Abstract base class instead of Protocol class AbstractProjection(ABC): """Base class for projection implementations.""" @@ -213,6 +209,7 @@ def inverse(self, x: CoordType, y: CoordType) -> ReturnUnionType: """ pass + class RotatedLatLonProjection(AbstractProjection): """ Rotated lat/lon projection implementation. @@ -269,9 +266,17 @@ def forward(self, latitude: CoordType, longitude: CoordType) -> ReturnUnionType: z = np.sin(lat_rad) # Apply rotation - x2 = np.cos(self.theta) * np.cos(self.phi) * x + np.cos(self.theta) * np.sin(self.phi) * y + np.sin(self.theta) * z + x2 = ( + np.cos(self.theta) * np.cos(self.phi) * x + + np.cos(self.theta) * np.sin(self.phi) * y + + np.sin(self.theta) * z + ) y2 = -np.sin(self.phi) * x + np.cos(self.phi) * y - z2 = -np.sin(self.theta) * np.cos(self.phi) * x - np.sin(self.theta) * np.sin(self.phi) * y + np.cos(self.theta) * z + z2 = ( + -np.sin(self.theta) * np.cos(self.phi) * x + - np.sin(self.theta) * np.sin(self.phi) * y + + np.cos(self.theta) * z + ) # Convert back to spherical coordinates rot_lon = np.degrees(np.arctan2(y2, x2)) @@ -307,14 +312,12 @@ def inverse(self, x: CoordType, y: CoordType) -> ReturnUnionType: phi_neg = -self.phi # Quick solution without conversion in cartesian space - lat_rad = np.arcsin( - np.cos(theta_neg) * np.sin(rot_lat) - np.cos(rot_lon) * np.sin(theta_neg) * np.cos(rot_lat) - ) + lat_rad = np.arcsin(np.cos(theta_neg) * np.sin(rot_lat) - np.cos(rot_lon) * np.sin(theta_neg) * np.cos(rot_lat)) - lon_rad = np.arctan2( - np.sin(rot_lon), - np.tan(rot_lat) * np.sin(theta_neg) + np.cos(rot_lon) * np.cos(theta_neg) - ) - phi_neg + lon_rad = ( + np.arctan2(np.sin(rot_lon), np.tan(rot_lat) * np.sin(theta_neg) + np.cos(rot_lon) * np.cos(theta_neg)) + - phi_neg + ) lat2 = np.degrees(lat_rad) lon2 = np.degrees(lon_rad) @@ -375,11 +378,13 @@ def forward(self, latitude: CoordType, longitude: CoordType) -> ReturnUnionType: phi = np.radians(lat_arr) lambda_ = np.radians(lon_arr) - k = 2 * self.R / (1 + self.sin_phi_1 * np.sin(phi) + - self.cos_phi_1 * np.cos(phi) * np.cos(lambda_ - self.lambda_0)) + k = ( + 2 + * self.R + / (1 + self.sin_phi_1 * np.sin(phi) + self.cos_phi_1 * np.cos(phi) * np.cos(lambda_ - self.lambda_0)) + ) x = k * np.cos(phi) * np.sin(lambda_ - self.lambda_0) - y = k * (self.cos_phi_1 * np.sin(phi) - - self.sin_phi_1 * np.cos(phi) * np.cos(lambda_ - self.lambda_0)) + y = k * (self.cos_phi_1 * np.sin(phi) - self.sin_phi_1 * np.cos(phi) * np.cos(lambda_ - self.lambda_0)) if scalar_input: return float(x.item()), float(y.item()) @@ -405,21 +410,21 @@ def inverse(self, x: CoordType, y: CoordType) -> ReturnUnionType: x_arr = np.asarray(x, dtype=np.float32) y_arr = np.asarray(y, dtype=np.float32) - p = np.sqrt(x_arr*x_arr + y_arr*y_arr) + p = np.sqrt(x_arr * x_arr + y_arr * y_arr) # Initialize output arrays phi = np.zeros_like(p) lambda_ = np.zeros_like(p) - c = 2 * np.arctan2(p, 2*self.R) + c = 2 * np.arctan2(p, 2 * self.R) phi = np.arcsin(np.cos(c) * self.sin_phi_1 + (y_arr * np.sin(c) * self.cos_phi_1) / p) lambda_ = self.lambda_0 + np.arctan2( - x_arr * np.sin(c), - p * self.cos_phi_1 * np.cos(c) - y_arr * self.sin_phi_1 * np.sin(c) + x_arr * np.sin(c), p * self.cos_phi_1 * np.cos(c) - y_arr * self.sin_phi_1 * np.sin(c) ) return np.degrees(phi), np.degrees(lambda_) + class LambertAzimuthalEqualAreaProjection(AbstractProjection): """ Lambert Azimuthal Equal-Area projection implementation. @@ -470,12 +475,21 @@ def forward(self, latitude: CoordType, longitude: CoordType) -> ReturnUnionType: lambda_ = np.radians(lon_arr) phi = np.radians(lat_arr) - k = np.sqrt(2 / (1 + np.sin(self.phi_1) * np.sin(phi) + - np.cos(self.phi_1) * np.cos(phi) * np.cos(lambda_ - self.lambda_0))) + k = np.sqrt( + 2 + / ( + 1 + + np.sin(self.phi_1) * np.sin(phi) + + np.cos(self.phi_1) * np.cos(phi) * np.cos(lambda_ - self.lambda_0) + ) + ) x = self.R * k * np.cos(phi) * np.sin(lambda_ - self.lambda_0) - y = self.R * k * (np.cos(self.phi_1) * np.sin(phi) - - np.sin(self.phi_1) * np.cos(phi) * np.cos(lambda_ - self.lambda_0)) + y = ( + self.R + * k + * (np.cos(self.phi_1) * np.sin(phi) - np.sin(self.phi_1) * np.cos(phi) * np.cos(lambda_ - self.lambda_0)) + ) if scalar_input: return float(x.item()), float(y.item()) @@ -508,15 +522,13 @@ def inverse(self, x: CoordType, y: CoordType) -> ReturnUnionType: p = np.sqrt(x_norm * x_norm + y_norm * y_norm) # Handle the case where p is zero (projection center) - zero_p = (p == 0) + zero_p = p == 0 p = np.where(zero_p, np.finfo(np.float32).eps, p) # Avoid division by zero c = 2 * np.arcsin(0.5 * p) - phi = np.arcsin(np.cos(c) * np.sin(self.phi_1) + - (y_norm * np.sin(c) * np.cos(self.phi_1)) / p) + phi = np.arcsin(np.cos(c) * np.sin(self.phi_1) + (y_norm * np.sin(c) * np.cos(self.phi_1)) / p) lambda_ = self.lambda_0 + np.arctan2( - x_norm * np.sin(c), - p * np.cos(self.phi_1) * np.cos(c) - y_norm * np.sin(self.phi_1) * np.sin(c) + x_norm * np.sin(c), p * np.cos(self.phi_1) * np.cos(c) - y_norm * np.sin(self.phi_1) * np.sin(c) ) lat = np.degrees(phi) lon = np.degrees(lambda_) @@ -526,6 +538,7 @@ def inverse(self, x: CoordType, y: CoordType) -> ReturnUnionType: return lat, lon + class LambertConformalConicProjection(AbstractProjection): """ Lambert Conformal Conic projection implementation. @@ -564,13 +577,13 @@ def __init__(self, lambda_0: float, phi_0: float, phi_1: float, phi_2: float, ra if phi_1 == phi_2: self.n = np.sin(phi_1_rad) else: - self.n = (np.log(np.cos(phi_1_rad) / np.cos(phi_2_rad)) / - np.log(np.tan(np.pi/4 + phi_2_rad/2) / np.tan(np.pi/4 + phi_1_rad/2))) + self.n = np.log(np.cos(phi_1_rad) / np.cos(phi_2_rad)) / np.log( + np.tan(np.pi / 4 + phi_2_rad / 2) / np.tan(np.pi / 4 + phi_1_rad / 2) + ) - self.F = ((np.cos(phi_1_rad) * np.power(np.tan(np.pi/4 + phi_1_rad/2), self.n)) / - self.n) + self.F = (np.cos(phi_1_rad) * np.power(np.tan(np.pi / 4 + phi_1_rad / 2), self.n)) / self.n - self.rho_0 = self.F / np.power(np.tan(np.pi/4 + phi_0_rad/2), self.n) + self.rho_0 = self.F / np.power(np.tan(np.pi / 4 + phi_0_rad / 2), self.n) # Earth radius self.R = radius @@ -599,7 +612,7 @@ def forward(self, latitude: CoordType, longitude: CoordType) -> ReturnUnionType: # If (λ - λ0) exceeds the range:±: 180°, 360° should be added or subtracted. theta = self.n * (lambda_ - self.lambda_0) - rho = self.F / np.power(np.tan(np.pi/4 + phi/2), self.n) + rho = self.F / np.power(np.tan(np.pi / 4 + phi / 2), self.n) x = self.R * rho * np.sin(theta) y = self.R * (self.rho_0 - rho * np.cos(theta)) @@ -629,14 +642,14 @@ def inverse(self, x: CoordType, y: CoordType) -> ReturnUnionType: x_scaled = np.asarray(x, dtype=np.float64) / self.R y_scaled = np.asarray(y, dtype=np.float64) / self.R - theta = np.where(self.n >= 0, - np.arctan2(x_scaled, self.rho_0 - y_scaled), - np.arctan2(-x_scaled, y_scaled - self.rho_0)) + theta = np.where( + self.n >= 0, np.arctan2(x_scaled, self.rho_0 - y_scaled), np.arctan2(-x_scaled, y_scaled - self.rho_0) + ) sign = np.where(self.n > 0, 1, -1) rho = sign * np.sqrt(np.square(x_scaled) + np.square(self.rho_0 - y_scaled)) - phi = 2 * np.arctan(np.power(self.F / rho, 1/self.n)) - np.pi/2 + phi = 2 * np.arctan(np.power(self.F / rho, 1 / self.n)) - np.pi / 2 lambda_ = self.lambda_0 + theta / self.n lat = np.degrees(phi) @@ -650,7 +663,7 @@ def inverse(self, x: CoordType, y: CoordType) -> ReturnUnionType: return lat, lon -P = TypeVar('P', bound=AbstractProjection) +P = TypeVar("P", bound=AbstractProjection) class ProjectionGrid(AbstractGrid, Generic[P]): @@ -660,15 +673,7 @@ class ProjectionGrid(AbstractGrid, Generic[P]): This represents a grid in a projected coordinate system. """ - def __init__( - self, - projection: P, - nx: int, - ny: int, - origin: Tuple[float, float], - dx: float, - dy: float - ): + def __init__(self, projection: P, nx: int, ny: int, origin: Tuple[float, float], dx: float, dy: float): """ Initialize a projection grid with all parameters. @@ -696,13 +701,8 @@ def __init__( @classmethod def from_bounds( - cls, - nx: int, - ny: int, - lat_range: Tuple[float, float], - lon_range: Tuple[float, float], - projection: P - ) -> 'ProjectionGrid[P]': + cls, nx: int, ny: int, lat_range: Tuple[float, float], lon_range: Tuple[float, float], projection: P + ) -> "ProjectionGrid[P]": """ Create a projection grid from geographic bounds. @@ -727,21 +727,14 @@ def from_bounds( sw = projection.forward(lat_range[0], lon_range[0]) ne = projection.forward(lat_range[1], lon_range[1]) origin = cast(tuple[float, float], sw) - dx = (ne[0] - sw[0]) / (nx-1) - dy = (ne[1] - sw[1]) / (ny-1) + dx = (ne[0] - sw[0]) / (nx - 1) + dy = (ne[1] - sw[1]) / (ny - 1) return cls(projection, nx, ny, origin, float(dx), float(dy)) @classmethod def from_center( - cls, - nx: int, - ny: int, - center_lat: float, - center_lon: float, - dx: float, - dy: float, - projection: P - ) -> 'ProjectionGrid[P]': + cls, nx: int, ny: int, center_lat: float, center_lon: float, dx: float, dy: float, projection: P + ) -> "ProjectionGrid[P]": """ Create a projection grid centered at a geographic location. @@ -780,11 +773,7 @@ def _coordinates(self) -> Tuple[np.ndarray, np.ndarray]: Lazily compute and cache both latitude and longitude arrays. """ # Create meshgrid of coordinates - y_indices, x_indices = np.meshgrid( - np.arange(self.ny), - np.arange(self.nx), - indexing='ij' - ) + y_indices, x_indices = np.meshgrid(np.arange(self.ny), np.arange(self.nx), indexing="ij") # Convert to projected coordinates x_coords = x_indices * self.dx + self.origin[0] @@ -795,14 +784,14 @@ def _coordinates(self) -> Tuple[np.ndarray, np.ndarray]: return lat, lon @property - def latitude(self) -> np.ndarray: # type: ignore + def latitude(self) -> np.ndarray: # type: ignore """ Get the latitude coordinate array. """ return self._coordinates[0] @property - def longitude(self) -> np.ndarray: # type: ignore + def longitude(self) -> np.ndarray: # type: ignore """ Get the longitude coordinate array. """ @@ -874,27 +863,14 @@ def get_true_north_direction(self) -> np.ndarray: north_pole_y = (pos[1] - self.origin[1]) / self.dy # Create grid of x, y coordinates - y_indices, x_indices = np.meshgrid( - np.arange(self.ny), - np.arange(self.nx), - indexing='ij' - ) + y_indices, x_indices = np.meshgrid(np.arange(self.ny), np.arange(self.nx), indexing="ij") # Vectorized calculation of angles - true_north = np.degrees(np.arctan2( - north_pole_x - x_indices, - north_pole_y - y_indices - )) + true_north = np.degrees(np.arctan2(north_pole_x - x_indices, north_pole_y - y_indices)) return true_north - def find_box( - self, - lat_min: float, - lat_max: float, - lon_min: float, - lon_max: float - ) -> np.ndarray: + def find_box(self, lat_min: float, lat_max: float, lon_min: float, lon_max: float) -> np.ndarray: """ Find indices of grid points within a geographic bounding box. @@ -934,17 +910,10 @@ def find_box( y_max = max(nw[1], ne[1]) + 1 # Create meshgrid of indices - y_indices, x_indices = np.meshgrid( - np.arange(y_min, y_max), - np.arange(x_min, x_max), - indexing='ij' - ) + y_indices, x_indices = np.meshgrid(np.arange(y_min, y_max), np.arange(x_min, x_max), indexing="ij") # Convert to flat indices - return np.ravel_multi_index( - (y_indices.flatten(), x_indices.flatten()), - (self.ny, self.nx) - ) + return np.ravel_multi_index((y_indices.flatten(), x_indices.flatten()), (self.ny, self.nx)) class ProjProjection(AbstractProjection): @@ -960,19 +929,16 @@ def __init__(self, proj_string: str): EPSG code (e.g. "EPSG:4326") """ import pyproj + # Create transformer from lat/lon to projection coordinates self.crs_proj = pyproj.CRS(proj_string) self.crs_latlon = pyproj.CRS("EPSG:4326") # WGS84 self.forward_transformer = pyproj.Transformer.from_crs( self.crs_latlon, self.crs_proj, - always_xy=True # This ensures lon/lat -> x/y order - ) - self.inverse_transformer = pyproj.Transformer.from_crs( - self.crs_proj, - self.crs_latlon, - always_xy=True + always_xy=True, # This ensures lon/lat -> x/y order ) + self.inverse_transformer = pyproj.Transformer.from_crs(self.crs_proj, self.crs_latlon, always_xy=True) def forward(self, latitude: CoordType, longitude: CoordType) -> ReturnUnionType: """Transform from latitude/longitude to projection coordinates diff --git a/python/omfiles/om_domains.py b/python/omfiles/om_domains.py index 5499eda8..8e71588a 100644 --- a/python/omfiles/om_domains.py +++ b/python/omfiles/om_domains.py @@ -20,13 +20,7 @@ class OmDomain: weather model grids used in Open-Meteo. """ - def __init__( - self, - name: str, - grid: AbstractGrid, - file_length: int, - temporal_resolution_seconds: int = 3600 - ): + def __init__(self, name: str, grid: AbstractGrid, file_length: int, temporal_resolution_seconds: int = 3600): """ Initialize a domain configuration. @@ -61,7 +55,7 @@ def time_to_chunk_index(self, timestamp: np.datetime64) -> int: int The chunk index containing the timestamp """ - seconds_since_epoch = (timestamp - EPOCH) / np.timedelta64(1, 's') + seconds_since_epoch = (timestamp - EPOCH) / np.timedelta64(1, "s") chunk_index = int(seconds_since_epoch / (self.file_length * self.temporal_resolution_seconds)) return chunk_index @@ -89,7 +83,7 @@ def chunks_for_date_range( end_chunk = self.time_to_chunk_index(end_timestamp) # Generate list of all chunks between start and end (inclusive) - return list(range(start_chunk, end_chunk +1)) + return list(range(start_chunk, end_chunk + 1)) def get_chunk_time_range(self, chunk_index: int): """ @@ -106,80 +100,51 @@ def get_chunk_time_range(self, chunk_index: int): Array of datetime64 objects representing the time points in the chunk """ chunk_start_seconds = chunk_index * self.file_length * self.temporal_resolution_seconds - start_time = EPOCH + np.timedelta64(chunk_start_seconds, 's') + start_time = EPOCH + np.timedelta64(chunk_start_seconds, "s") # Generate timestamps at regular intervals from the start time - time_delta = np.timedelta64(self.temporal_resolution_seconds, 's') + time_delta = np.timedelta64(self.temporal_resolution_seconds, "s") # Note: better type inference via list comprehension here timestamps = np.array([start_time + i * time_delta for i in range(self.file_length)]) return timestamps + # - MARK: Create grid instances for supported domains # DWD ICON global is regularized during download to nx: 2879, ny: 1441 points # https://github.com/open-meteo/open-meteo/blob/522917b1d6e72a7e6b7d4ae7dfb49b0c556a6992/Sources/App/Icon/Icon.swift#L146 _dwd_icon_grid = RegularLatLonGrid( - lat_start=-90, - lat_steps=1441, - lat_step_size=0.125, - lon_start=-180, - lon_steps=2879, - lon_step_size=0.125 + lat_start=-90, lat_steps=1441, lat_step_size=0.125, lon_start=-180, lon_steps=2879, lon_step_size=0.125 ) # DWD ICON EU is regularized during download to nx: 1377, ny: 657 points # https://github.com/open-meteo/open-meteo/blob/522917b1d6e72a7e6b7d4ae7dfb49b0c556a6992/Sources/App/Icon/Icon.swift#L148 _dwd_icon_eu_grid = RegularLatLonGrid( - lat_start=29.5, - lat_steps=657, - lat_step_size=0.0625, - lon_start=-23.5, - lon_steps=1377, - lon_step_size=0.0625 + lat_start=29.5, lat_steps=657, lat_step_size=0.0625, lon_start=-23.5, lon_steps=1377, lon_step_size=0.0625 ) # DWD ICON D2 is regularized during download to nx: 1215, ny: 746 points # https://github.com/open-meteo/open-meteo/blob/522917b1d6e72a7e6b7d4ae7dfb49b0c556a6992/Sources/App/Icon/Icon.swift#L150 _dwd_icon_d2_grid = RegularLatLonGrid( - lat_start=43.18, - lat_steps=746, - lat_step_size=0.02, - lon_start=-3.94, - lon_steps=1215, - lon_step_size=0.02 + lat_start=43.18, lat_steps=746, lat_step_size=0.02, lon_start=-3.94, lon_steps=1215, lon_step_size=0.02 ) # DWD ICON EPS global is regularized during download to nx: 1439, ny: 721 points # https://github.com/open-meteo/open-meteo/blob/522917b1d6e72a7e6b7d4ae7dfb49b0c556a6992/Sources/App/Icon/Icon.swift#L153 _dwd_icon_eps_grid = RegularLatLonGrid( - lat_start=-90, - lat_steps=721, - lat_step_size=0.25, - lon_start=-180, - lon_steps=1439, - lon_step_size=0.25 + lat_start=-90, lat_steps=721, lat_step_size=0.25, lon_start=-180, lon_steps=1439, lon_step_size=0.25 ) # DWD ICON EU EPS is regularized during download to nx: 689, ny: 329 points # https://github.com/open-meteo/open-meteo/blob/522917b1d6e72a7e6b7d4ae7dfb49b0c556a6992/Sources/App/Icon/Icon.swift#L156 _dwd_icon_eu_eps_grid = RegularLatLonGrid( - lat_start=29.5, - lat_steps=329, - lat_step_size=0.125, - lon_start=-23.5, - lon_steps=689, - lon_step_size=0.125 + lat_start=29.5, lat_steps=329, lat_step_size=0.125, lon_start=-23.5, lon_steps=689, lon_step_size=0.125 ) # DWD ICON D2 EPS is regularized during download to nx: 1214, ny: 745 points # https://github.com/open-meteo/open-meteo/blob/522917b1d6e72a7e6b7d4ae7dfb49b0c556a6992/Sources/App/Icon/Icon.swift#L160 _dwd_icon_d2_eps_grid = RegularLatLonGrid( - lat_start=43.18, - lat_steps=745, - lat_step_size=0.02, - lon_start=-3.94, - lon_steps=1214, - lon_step_size=0.02 + lat_start=43.18, lat_steps=745, lat_step_size=0.02, lon_start=-3.94, lon_steps=1214, lon_step_size=0.02 ) # ECMWF IFS grid is a regular global lat/lon grid, nx: 1440, ny: 721 points @@ -187,233 +152,193 @@ def get_chunk_time_range(self, chunk_index: int): _ecmwf_ifs025_grid = RegularLatLonGrid( lat_start=-90, lat_steps=721, - lat_step_size=360/1440, + lat_step_size=360 / 1440, lon_start=-180, lon_steps=1440, - lon_step_size=180/(721-1) + lon_step_size=180 / (721 - 1), ) # Méteo-France ARPEGE Europe grid: nx: 741, ny: 521 points # https://github.com/open-meteo/open-meteo/blob/522917b1d6e72a7e6b7d4ae7dfb49b0c556a6992/Sources/App/MeteoFrance/MeteoFranceDomain.swift#L341 _meteofrance_arpege_europe_grid = RegularLatLonGrid( - lat_start=20, - lat_steps=521, - lat_step_size=0.1, - lon_start=-32, - lon_steps=741, - lon_step_size=0.1 + lat_start=20, lat_steps=521, lat_step_size=0.1, lon_start=-32, lon_steps=741, lon_step_size=0.1 ) # Méteo-France ARPEGE World grid: nx: 1440, ny: 721 points # https://github.com/open-meteo/open-meteo/blob/522917b1d6e72a7e6b7d4ae7dfb49b0c556a6992/Sources/App/MeteoFrance/MeteoFranceDomain.swift#L343 _meteofrance_arpege_world025_grid = RegularLatLonGrid( - lat_start=-90, - lat_steps=721, - lat_step_size=0.25, - lon_start=-180, - lon_steps=1440, - lon_step_size=0.25 + lat_start=-90, lat_steps=721, lat_step_size=0.25, lon_start=-180, lon_steps=1440, lon_step_size=0.25 ) # Méteo-France AROME France grid: nx: 1121, ny: 717 points # https://github.com/open-meteo/open-meteo/blob/522917b1d6e72a7e6b7d4ae7dfb49b0c556a6992/Sources/App/MeteoFrance/MeteoFranceDomain.swift#L345 _meteofrance_arome_france0025_grid = RegularLatLonGrid( - lat_start=37.5, - lat_steps=717, - lat_step_size=0.025, - lon_start=-12.0, - lon_steps=1121, - lon_step_size=0.025 + lat_start=37.5, lat_steps=717, lat_step_size=0.025, lon_start=-12.0, lon_steps=1121, lon_step_size=0.025 ) # Méteo-France AROME France HD grid: nx: 2801, ny: 1791 points # https://github.com/open-meteo/open-meteo/blob/522917b1d6e72a7e6b7d4ae7dfb49b0c556a6992/Sources/App/MeteoFrance/MeteoFranceDomain.swift#L347 _meteofrance_arome_france_hd_grid = RegularLatLonGrid( - lat_start=37.5, - lat_steps=1791, - lat_step_size=0.01, - lon_start=-12.0, - lon_steps=2801, - lon_step_size=0.01 + lat_start=37.5, lat_steps=1791, lat_step_size=0.01, lon_start=-12.0, lon_steps=2801, lon_step_size=0.01 ) # GEM Global grid: nx: 2400, ny: 1201 points # https://github.com/open-meteo/open-meteo/blob/522917b1d6e72a7e6b7d4ae7dfb49b0c556a6992/Sources/App/Gem/GemDomain.swift#L139 _gem_global_grid = RegularLatLonGrid( - lat_start=-90, - lat_steps=1201, - lat_step_size=0.15, - lon_start=-180, - lon_steps=2400, - lon_step_size=0.15 + lat_start=-90, lat_steps=1201, lat_step_size=0.15, lon_start=-180, lon_steps=2400, lon_step_size=0.15 ) # GEM Regional grid: Uses Stereographic projection # https://github.com/open-meteo/open-meteo/blob/522917b1d6e72a7e6b7d4ae7dfb49b0c556a6992/Sources/App/Gem/GemDomain.swift#L141 -_gem_regional_projection = StereographicProjection( - latitude=90, - longitude=249, - radius=6371229 -) +_gem_regional_projection = StereographicProjection(latitude=90, longitude=249, radius=6371229) _gem_regional_grid = ProjectionGrid.from_bounds( nx=935, ny=824, lat_range=(18.14503, 45.405453), lon_range=(217.10745, 349.8256), - projection=_gem_regional_projection + projection=_gem_regional_projection, ) # GEM HRDPS Continental grid: Uses RotatedLatLon projection # https://github.com/open-meteo/open-meteo/blob/522917b1d6e72a7e6b7d4ae7dfb49b0c556a6992/Sources/App/Gem/GemDomain.swift#L143 -_gem_hrdps_projection = RotatedLatLonProjection( - lat_origin=-36.0885, - lon_origin=245.305 -) +_gem_hrdps_projection = RotatedLatLonProjection(lat_origin=-36.0885, lon_origin=245.305) _gem_hrdps_grid = ProjectionGrid.from_bounds( nx=2540, ny=1290, lat_range=(39.626034, 47.876457), lon_range=(-133.62952, -40.708557), - projection=_gem_hrdps_projection + projection=_gem_hrdps_projection, ) # GEM Global Ensemble grid: nx: 720, ny: 361 points # https://github.com/open-meteo/open-meteo/blob/522917b1d6e72a7e6b7d4ae7dfb49b0c556a6992/Sources/App/Gem/GemDomain.swift#L145 _gem_global_ensemble_grid = RegularLatLonGrid( - lat_start=-90, - lat_steps=361, - lat_step_size=0.5, - lon_start=-180, - lon_steps=720, - lon_step_size=0.5 + lat_start=-90, lat_steps=361, lat_step_size=0.5, lon_start=-180, lon_steps=720, lon_step_size=0.5 ) DOMAINS: dict[str, OmDomain] = { - 'cmc_gem_gdps': OmDomain( - name='cmc_gem_gdps', + "cmc_gem_gdps": OmDomain( + name="cmc_gem_gdps", grid=_gem_global_grid, file_length=110, # From GemDomain.omFileLength for gem_global case - temporal_resolution_seconds=3600*3 # 3-hourly data + temporal_resolution_seconds=3600 * 3, # 3-hourly data ), - 'cmc_gem_rdps': OmDomain( - name='cmc_gem_rdps', + "cmc_gem_rdps": OmDomain( + name="cmc_gem_rdps", grid=_gem_regional_grid, - file_length=78+36, # From GemDomain.omFileLength for gem_regional case - temporal_resolution_seconds=3600 # Hourly data + file_length=78 + 36, # From GemDomain.omFileLength for gem_regional case + temporal_resolution_seconds=3600, # Hourly data ), - 'cmc_gem_hrdps': OmDomain( - name='cmc_gem_hrdps', + "cmc_gem_hrdps": OmDomain( + name="cmc_gem_hrdps", grid=_gem_hrdps_grid, - file_length=48+36, # From GemDomain.omFileLength for gem_hrdps_continental case - temporal_resolution_seconds=3600 # Hourly data + file_length=48 + 36, # From GemDomain.omFileLength for gem_hrdps_continental case + temporal_resolution_seconds=3600, # Hourly data ), - 'cmc_gem_geps': OmDomain( - name='cmc_gem_geps', + "cmc_gem_geps": OmDomain( + name="cmc_gem_geps", grid=_gem_global_ensemble_grid, - file_length=384//3+48//3, # From GemDomain.omFileLength for gem_global_ensemble case - temporal_resolution_seconds=3600*3 # 3-hourly data + file_length=384 // 3 + 48 // 3, # From GemDomain.omFileLength for gem_global_ensemble case + temporal_resolution_seconds=3600 * 3, # 3-hourly data ), - 'dwd_icon': OmDomain( - name='dwd_icon', + "dwd_icon": OmDomain( + name="dwd_icon", grid=_dwd_icon_grid, - file_length=180+1+3*24, # From IconDomains.omFileLength for icon case - temporal_resolution_seconds=3600 + file_length=180 + 1 + 3 * 24, # From IconDomains.omFileLength for icon case + temporal_resolution_seconds=3600, ), - 'dwd_icon_eu': OmDomain( - name='dwd_icon_eu', + "dwd_icon_eu": OmDomain( + name="dwd_icon_eu", grid=_dwd_icon_eu_grid, - file_length=120+1+3*24, # From IconDomains.omFileLength for iconEu case - temporal_resolution_seconds=3600 + file_length=120 + 1 + 3 * 24, # From IconDomains.omFileLength for iconEu case + temporal_resolution_seconds=3600, ), - 'dwd_icon_d2': OmDomain( - name='dwd_icon_d2', + "dwd_icon_d2": OmDomain( + name="dwd_icon_d2", grid=_dwd_icon_d2_grid, - file_length=48+1+3*24, # From IconDomains.omFileLength for iconD2 case - temporal_resolution_seconds=3600 + file_length=48 + 1 + 3 * 24, # From IconDomains.omFileLength for iconD2 case + temporal_resolution_seconds=3600, ), - 'dwd_icon_d2_15min': OmDomain( - name='dwd_icon_d2_15min', + "dwd_icon_d2_15min": OmDomain( + name="dwd_icon_d2_15min", grid=_dwd_icon_d2_grid, # Uses same grid as dwd_icon_d2 - file_length=48*4+3*24, # From IconDomains.omFileLength for iconD2_15min case - temporal_resolution_seconds=3600//4 # 15 minutes = 3600/4 + file_length=48 * 4 + 3 * 24, # From IconDomains.omFileLength for iconD2_15min case + temporal_resolution_seconds=3600 // 4, # 15 minutes = 3600/4 ), - 'dwd_icon_eps': OmDomain( - name='dwd_icon_eps', + "dwd_icon_eps": OmDomain( + name="dwd_icon_eps", grid=_dwd_icon_eps_grid, - file_length=180+1+3*24, # Same as non-eps version - temporal_resolution_seconds=3600 + file_length=180 + 1 + 3 * 24, # Same as non-eps version + temporal_resolution_seconds=3600, ), - 'dwd_icon_eu_eps': OmDomain( - name='dwd_icon_eu_eps', + "dwd_icon_eu_eps": OmDomain( + name="dwd_icon_eu_eps", grid=_dwd_icon_eu_eps_grid, - file_length=120+1+3*24, # Same as non-eps version - temporal_resolution_seconds=3600 + file_length=120 + 1 + 3 * 24, # Same as non-eps version + temporal_resolution_seconds=3600, ), - 'dwd_icon_d2_eps': OmDomain( - name='dwd_icon_d2_eps', + "dwd_icon_d2_eps": OmDomain( + name="dwd_icon_d2_eps", grid=_dwd_icon_d2_eps_grid, - file_length=48+1+3*24, # Same as non-eps version - temporal_resolution_seconds=3600 + file_length=48 + 1 + 3 * 24, # Same as non-eps version + temporal_resolution_seconds=3600, ), - 'ecmwf_ifs025': OmDomain( - name='ecmwf_ifs025', - grid=_ecmwf_ifs025_grid, - file_length=104, - temporal_resolution_seconds=3600*3 + "ecmwf_ifs025": OmDomain( + name="ecmwf_ifs025", grid=_ecmwf_ifs025_grid, file_length=104, temporal_resolution_seconds=3600 * 3 ), - 'meteofrance_arpege_europe': OmDomain( - name='meteofrance_arpege_europe', + "meteofrance_arpege_europe": OmDomain( + name="meteofrance_arpege_europe", grid=_meteofrance_arpege_europe_grid, - file_length=114+3*24, # From MeteoFranceDomain.omFileLength for arpege_europe case - temporal_resolution_seconds=3600 + file_length=114 + 3 * 24, # From MeteoFranceDomain.omFileLength for arpege_europe case + temporal_resolution_seconds=3600, ), - 'meteofrance_arpege_world025': OmDomain( - name='meteofrance_arpege_world025', + "meteofrance_arpege_world025": OmDomain( + name="meteofrance_arpege_world025", grid=_meteofrance_arpege_world025_grid, - file_length=114+4*24, # From MeteoFranceDomain.omFileLength for arpege_world case - temporal_resolution_seconds=3600 + file_length=114 + 4 * 24, # From MeteoFranceDomain.omFileLength for arpege_world case + temporal_resolution_seconds=3600, ), - 'meteofrance_arome_france0025': OmDomain( - name='meteofrance_arome_france0025', + "meteofrance_arome_france0025": OmDomain( + name="meteofrance_arome_france0025", grid=_meteofrance_arome_france0025_grid, - file_length=36+3*24, # From MeteoFranceDomain.omFileLength for arome_france case - temporal_resolution_seconds=3600 + file_length=36 + 3 * 24, # From MeteoFranceDomain.omFileLength for arome_france case + temporal_resolution_seconds=3600, ), - 'meteofrance_arome_france_hd': OmDomain( - name='meteofrance_arome_france_hd', + "meteofrance_arome_france_hd": OmDomain( + name="meteofrance_arome_france_hd", grid=_meteofrance_arome_france_hd_grid, - file_length=36+3*24, # From MeteoFranceDomain.omFileLength for arome_france_hd case - temporal_resolution_seconds=3600 + file_length=36 + 3 * 24, # From MeteoFranceDomain.omFileLength for arome_france_hd case + temporal_resolution_seconds=3600, ), - 'meteofrance_arome_france0025_15min': OmDomain( - name='meteofrance_arome_france0025_15min', + "meteofrance_arome_france0025_15min": OmDomain( + name="meteofrance_arome_france0025_15min", grid=_meteofrance_arome_france0025_grid, # Using the same grid as non-15min version - file_length=24*2, # From MeteoFranceDomain.omFileLength for arome_france_15min case - temporal_resolution_seconds=900 + file_length=24 * 2, # From MeteoFranceDomain.omFileLength for arome_france_15min case + temporal_resolution_seconds=900, ), - 'meteofrance_arome_france_hd_15min': OmDomain( - name='meteofrance_arome_france_hd_15min', + "meteofrance_arome_france_hd_15min": OmDomain( + name="meteofrance_arome_france_hd_15min", grid=_meteofrance_arome_france_hd_grid, # Using the same grid as non-15min version - file_length=24*2, # From MeteoFranceDomain.omFileLength for arome_france_hd_15min case - temporal_resolution_seconds=900 + file_length=24 * 2, # From MeteoFranceDomain.omFileLength for arome_france_hd_15min case + temporal_resolution_seconds=900, ), - 'meteofrance_arpege_europe_probabilities': OmDomain( - name='meteofrance_arpege_europe_probabilities', + "meteofrance_arpege_europe_probabilities": OmDomain( + name="meteofrance_arpege_europe_probabilities", grid=_meteofrance_arpege_europe_grid, # Using the same grid as non-probabilities version - file_length=(102+4*24)//3, # From MeteoFranceDomain.omFileLength for arpege_europe_probabilities case - temporal_resolution_seconds=3600*3 + file_length=(102 + 4 * 24) // 3, # From MeteoFranceDomain.omFileLength for arpege_europe_probabilities case + temporal_resolution_seconds=3600 * 3, ), - 'meteofrance_arpege_world025_probabilities': OmDomain( - name='meteofrance_arpege_world025_probabilities', + "meteofrance_arpege_world025_probabilities": OmDomain( + name="meteofrance_arpege_world025_probabilities", grid=_meteofrance_arpege_world025_grid, # Using the same grid as non-probabilities version - file_length=(102+4*24)//3, # From MeteoFranceDomain.omFileLength for arpege_world_probabilities case - temporal_resolution_seconds=3600*3 - ) + file_length=(102 + 4 * 24) // 3, # From MeteoFranceDomain.omFileLength for arpege_world_probabilities case + temporal_resolution_seconds=3600 * 3, + ), # Additional domains can be added here } # Domain aliases to match the names in GemDomain.swift -DOMAINS['gem_global'] = DOMAINS['cmc_gem_gdps'] -DOMAINS['gem_regional'] = DOMAINS['cmc_gem_rdps'] -DOMAINS['gem_hrdps_continental'] = DOMAINS['cmc_gem_hrdps'] -DOMAINS['gem_global_ensemble'] = DOMAINS['cmc_gem_geps'] +DOMAINS["gem_global"] = DOMAINS["cmc_gem_gdps"] +DOMAINS["gem_regional"] = DOMAINS["cmc_gem_rdps"] +DOMAINS["gem_hrdps_continental"] = DOMAINS["cmc_gem_hrdps"] +DOMAINS["gem_global_ensemble"] = DOMAINS["cmc_gem_geps"] diff --git a/python/omfiles/utils.py b/python/omfiles/utils.py index bc45428f..b21780ad 100644 --- a/python/omfiles/utils.py +++ b/python/omfiles/utils.py @@ -1,6 +1,7 @@ import numpy as np -EPOCH = np.datetime64(0, 's') +EPOCH = np.datetime64(0, "s") + def _modulo_positive(value: int, modulo: int) -> int: """ @@ -20,5 +21,6 @@ def _modulo_positive(value: int, modulo: int) -> int: """ return ((value % modulo) + modulo) % modulo + def _normalize_longitude(lon: float) -> float: return ((lon + 180.0) % 360.0) - 180.0 diff --git a/tests/test_grids.py b/tests/test_grids.py index 500420df..fe7b01cc 100644 --- a/tests/test_grids.py +++ b/tests/test_grids.py @@ -14,43 +14,40 @@ # Fixtures for grids + @pytest.fixture def local_regular_lat_lon_grid(): return RegularLatLonGrid( - lat_start=0.0, - lat_steps=10, - lat_step_size=1.0, - lon_start=0.0, - lon_steps=20, - lon_step_size=1.0 + lat_start=0.0, lat_steps=10, lat_step_size=1.0, lon_start=0.0, lon_steps=20, lon_step_size=1.0 ) + @pytest.fixture def stereographic_projection(): projection = StereographicProjection(90.0, 249.0, 6371229.0) return ProjectionGrid.from_bounds( - nx=935, - ny=824, - lat_range=(18.14503, 45.405453), - lon_range=(217.10745, 349.8256), - projection=projection + nx=935, ny=824, lat_range=(18.14503, 45.405453), lon_range=(217.10745, 349.8256), projection=projection ) + @pytest.fixture def hrdps_projection(): return RotatedLatLonProjection(lat_origin=-36.0885, lon_origin=245.305) + @pytest.fixture def hrdps_grid(hrdps_projection): from omfiles.grids import ProjectionGrid + return ProjectionGrid.from_bounds( nx=2540, ny=1290, lat_range=(39.626034, 47.876457), lon_range=(-133.62952, -40.708557), - projection=hrdps_projection + projection=hrdps_projection, ) + def test_regular_grid_findPointXy_inside(local_regular_lat_lon_grid): # Test exact grid points assert local_regular_lat_lon_grid.findPointXy(5.0, 10.0) == (10, 5) @@ -73,12 +70,7 @@ def test_regular_grid_findPointXy_outside(local_regular_lat_lon_grid): def test_global_grid_wrapping(): # Create a global grid (360° longitude, 180° latitude coverage) global_grid = RegularLatLonGrid( - lat_start=-90.0, - lat_steps=180, - lat_step_size=1.0, - lon_start=-180.0, - lon_steps=360, - lon_step_size=1.0 + lat_start=-90.0, lat_steps=180, lat_step_size=1.0, lon_start=-180.0, lon_steps=360, lon_step_size=1.0 ) # Test wrapping around the longitude @@ -89,6 +81,7 @@ def test_global_grid_wrapping(): # Test a point beyond the normal range assert global_grid.findPointXy(0.0, 540.0) == (0, 90) + def test_grid_coordinates(local_regular_lat_lon_grid): # Test exact grid points assert local_regular_lat_lon_grid.getCoordinates(0, 0) == (0.0, 0.0) @@ -111,7 +104,7 @@ def test_cached_property_computation(local_regular_lat_lon_grid): def test_stereographic(stereographic_projection): - #https://github.com/open-meteo/open-meteo/blob/522917b1d6e72a7e6b7d4ae7dfb49b0c556a6992/Tests/AppTests/DataTests.swift#L248 + # https://github.com/open-meteo/open-meteo/blob/522917b1d6e72a7e6b7d4ae7dfb49b0c556a6992/Tests/AppTests/DataTests.swift#L248 pos_x, pos_y = stereographic_projection.findPointXy(lat=64.79836, lon=241.40111) assert pos_x == 420 @@ -122,14 +115,17 @@ def test_stereographic(stereographic_projection): assert abs(lat - 64.79836) < 1e-4 assert np.mod(abs(lon - 241.40111), 360) < 1e-4 + def test_grid_properties(stereographic_projection): assert stereographic_projection.shape == (824, 935) assert stereographic_projection.grid_type == "projection" + def test_out_of_bounds(stereographic_projection): far_point = stereographic_projection.findPointXy(30.0, 120.0) assert far_point is None + def test_latitude_longitude_arrays(stereographic_projection): # Get latitude and longitude arrays lats = stereographic_projection.latitude @@ -139,14 +135,15 @@ def test_latitude_longitude_arrays(stereographic_projection): assert lats.shape == (824, 935) assert lons.shape == (824, 935) + def test_hrdps_grid(hrdps_grid): """Test the HRDPS Continental grid with a modified approach""" test_points = [ # lat, lon, expected_x, expected_y - (39.626034, -133.62952, 0, 0), # Bottom-left - (27.284597, -66.96642, 2539, 0), # Bottom-right - (38.96126, -73.63256, 2032, 283), # Middle point - (47.876457, -40.708557, 2539, 1289), # Top-right + (39.626034, -133.62952, 0, 0), # Bottom-left + (27.284597, -66.96642, 2539, 0), # Bottom-right + (38.96126, -73.63256, 2032, 283), # Middle point + (47.876457, -40.708557, 2539, 1289), # Top-right ] for lat, lon, expected_x, expected_y in test_points: @@ -169,20 +166,13 @@ def test_lambert_azimuthal_equal_area_projection(): https://github.com/open-meteo/open-meteo/blob/522917b1d6e72a7e6b7d4ae7dfb49b0c556a6992/Tests/AppTests/DataTests.swift#L189 """ proj = LambertAzimuthalEqualAreaProjection(lambda_0=-2.5, phi_1=54.9, radius=6371229) - grid = ProjectionGrid( - projection=proj, - nx=1042, - ny=970, - origin=(-1158000, -1036000), - dx=2000, - dy=2000 - ) + grid = ProjectionGrid(projection=proj, nx=1042, ny=970, origin=(-1158000, -1036000), dx=2000, dy=2000) test_lon = 10.620785 test_lat = 57.745566 x, y = proj.forward(latitude=test_lat, longitude=test_lon) - assert abs(x - 773650.5058) < 0.0001 # TODO: There are small numerical differences with the Swift test case - assert abs(y - 389820.1483) < 0.0001 # TODO: There are small numerical differences with the Swift test case + assert abs(x - 773650.5058) < 0.0001 # TODO: There are small numerical differences with the Swift test case + assert abs(y - 389820.1483) < 0.0001 # TODO: There are small numerical differences with the Swift test case lat, lon = proj.inverse(x=x, y=y) assert abs(lon - test_lon) < 0.00001 @@ -212,11 +202,7 @@ def test_lambert_conformal(): assert abs(lon - (-8)) < 0.0001 grid = ProjectionGrid.from_bounds( - nx=1799, - ny=1059, - lat_range=(21.138, 47.8424), - lon_range=(-122.72, -60.918), - projection=proj + nx=1799, ny=1059, lat_range=(21.138, 47.8424), lon_range=(-122.72, -60.918), projection=proj ) point_xy = grid.findPointXy(lat=34, lon=-118) @@ -235,7 +221,7 @@ def test_lambert_conformal(): (24.449714395051082, 265.54789437771944 - 360, 10000), (22.73382904757237, 242.93190409785294 - 360, 20000), (24.37172305316154, 271.6307003393202 - 360, 30000), - (24.007414634071907, 248.77817290935954 - 360, 40000) + (24.007414634071907, 248.77817290935954 - 360, 40000), ] for lat, lon, expected_idx in reference_points: @@ -257,19 +243,11 @@ def test_nbm_grid(): https://github.com/open-meteo/open-meteo/blob/522917b1d6e72a7e6b7d4ae7dfb49b0c556a6992/Tests/AppTests/DataTests.swift#L94 """ # Create projection with appropriate parameters - proj = LambertConformalConicProjection( - lambda_0=265 - 360, phi_0=0, phi_1=25, phi_2=25, radius=6371200 - ) + proj = LambertConformalConicProjection(lambda_0=265 - 360, phi_0=0, phi_1=25, phi_2=25, radius=6371200) # Create grid grid = ProjectionGrid.from_center( - projection=proj, - nx=2345, - ny=1597, - center_lat=19.229, - center_lon=233.723 - 360, - dx=2539.7, - dy=2539.7 + projection=proj, nx=2345, ny=1597, center_lat=19.229, center_lon=233.723 - 360, dx=2539.7, dy=2539.7 ) # Test forward projection of grid origin @@ -289,7 +267,7 @@ def test_nbm_grid(): (24.449714395051082, 265.54789437771944 - 360, 188910), (22.73382904757237, 242.93190409785294 - 360, 180965), (24.37172305316154, 271.6307003393202 - 360, 196187), - (24.007414634071907, 248.77817290935954 - 360, 232796) + (24.007414634071907, 248.77817290935954 - 360, 232796), ] for lat, lon, expected_idx in reference_points: @@ -305,7 +283,7 @@ def test_nbm_grid(): (10000, 21.794254, -111.44652), (20000, 22.806227, -96.18898), (30000, 22.222015, -80.87921), - (40000, 20.274399, -123.18192) + (40000, 20.274399, -123.18192), ] for idx, expected_lat, expected_lon in reference_coords: @@ -327,13 +305,7 @@ def test_lambert_conformal_conic_projection(): center_lon = -25.421997 grid = ProjectionGrid.from_center( - nx=1906, - ny=1606, - center_lat=center_lat, - center_lon=center_lon, - dx=2000, - dy=2000, - projection=proj + nx=1906, ny=1606, center_lat=center_lat, center_lon=center_lon, dx=2000, dy=2000, projection=proj ) # Test forward projection @@ -383,19 +355,21 @@ def test_rotated_latlon_against_proj(): custom_proj = RotatedLatLonProjection(lat_origin=lat_origin, lon_origin=lon_origin) # Create equivalent PROJ projection - proj_string = (f"+proj=ob_tran +o_proj=longlat +o_lat_p={-lat_origin} " - f"+o_lon_p=0.0 +lon_0={lon_origin} +datum=WGS84 +no_defs +type=crs") + proj_string = ( + f"+proj=ob_tran +o_proj=longlat +o_lat_p={-lat_origin} " + f"+o_lon_p=0.0 +lon_0={lon_origin} +datum=WGS84 +no_defs +type=crs" + ) proj_proj = pyproj.Proj(proj_string) # Test points covering different regions test_points = [ - (0, 0), # Origin - (45, 45), # Mid-latitude point - (-45, -45), # Mid-latitude point (southern hemisphere) - (10, 50), # Europe - (40, -100), # North America - (50, -170), # Pacific - (-30, 170), # South Pacific + (0, 0), # Origin + (45, 45), # Mid-latitude point + (-45, -45), # Mid-latitude point (southern hemisphere) + (10, 50), # Europe + (40, -100), # North America + (50, -170), # Pacific + (-30, 170), # South Pacific ] for lat, lon in test_points: @@ -423,8 +397,12 @@ def test_rotated_latlon_against_proj(): proj_lon, proj_lat = proj_proj(np.radians(proj_x), np.radians(proj_y), inverse=True) # Compare results - assert abs(custom_lat - proj_lat) < 1e-5, f"Lat mismatch for ({custom_x}, {custom_y}): custom={custom_lat}, proj={proj_lat}" - assert abs(np.mod(custom_lon - proj_lon + 180, 360) - 180) < 1e-5, f"Lon mismatch for ({custom_x}, {custom_y}): custom={custom_lon}, proj={proj_lon}" + assert abs(custom_lat - proj_lat) < 1e-5, ( + f"Lat mismatch for ({custom_x}, {custom_y}): custom={custom_lat}, proj={proj_lat}" + ) + assert abs(np.mod(custom_lon - proj_lon + 180, 360) - 180) < 1e-5, ( + f"Lon mismatch for ({custom_x}, {custom_y}): custom={custom_lon}, proj={proj_lon}" + ) def test_stereographic_against_proj(): @@ -435,18 +413,17 @@ def test_stereographic_against_proj(): custom_proj = StereographicProjection(latitude=latitude, longitude=longitude, radius=radius) # Create equivalent PROJ projection - proj_string = (f"+proj=stere +lat_0={latitude} +lon_0={longitude} +k=1 " - f"+x_0=0 +y_0=0 +R={radius} +units=m +no_defs") + proj_string = f"+proj=stere +lat_0={latitude} +lon_0={longitude} +k=1 +x_0=0 +y_0=0 +R={radius} +units=m +no_defs" proj_proj = pyproj.Proj(proj_string) # Test points - staying away from singular points (poles) test_points = [ - (0, 0), # Equator - (45, 45), # Mid-latitude + (0, 0), # Equator + (45, 45), # Mid-latitude (60, -120), # Northern regions - (45, 249), # Along the central meridian - (70, 249), # Along the central meridian - (80, 249), # Along the central meridian + (45, 249), # Along the central meridian + (70, 249), # Along the central meridian + (80, 249), # Along the central meridian ] for lat, lon in test_points: @@ -472,31 +449,31 @@ def test_stereographic_against_proj(): custom_lon = _normalize_longitude(custom_lon) assert abs(custom_lon - proj_lon) < 1e-4, f"Lon mismatch: custom={custom_lon}, proj={proj_lon}" + def test_lambert_azimuthal_equal_area_against_proj(): # Create our custom projection lambda_0 = -2.5 # Central longitude in degrees - phi_1 = 54.9 # Standard parallel/latitude in degrees + phi_1 = 54.9 # Standard parallel/latitude in degrees radius = 6371229.0 # Earth radius in meters custom_proj = LambertAzimuthalEqualAreaProjection(lambda_0=lambda_0, phi_1=phi_1, radius=radius) # Create equivalent PROJ projection # For Lambert Azimuthal Equal Area, we use lat_0 for the standard parallel and lon_0 for central longitude - proj_string = (f"+proj=laea +lat_0={phi_1} +lon_0={lambda_0} +x_0=0 +y_0=0 " - f"+R={radius} +units=m +no_defs +type=crs") + proj_string = f"+proj=laea +lat_0={phi_1} +lon_0={lambda_0} +x_0=0 +y_0=0 +R={radius} +units=m +no_defs +type=crs" proj_proj = pyproj.Proj(proj_string) # Test points covering different regions test_points = [ - (0, 0), # Origin - (54.9, -2.5), # Projection center (should map to 0,0) - (45, 45), # Mid-latitude point - (-45, -45), # Mid-latitude point (southern hemisphere) - (10, 50), # Europe - (40, -100), # North America - (50, -170), # Pacific - (-30, 170), # South Pacific + (0, 0), # Origin + (54.9, -2.5), # Projection center (should map to 0,0) + (45, 45), # Mid-latitude point + (-45, -45), # Mid-latitude point (southern hemisphere) + (10, 50), # Europe + (40, -100), # North America + (50, -170), # Pacific + (-30, 170), # South Pacific # Test point from the existing test - (57.745566, 10.620785) + (57.745566, 10.620785), ] for lat, lon in test_points: @@ -522,21 +499,24 @@ def test_lambert_azimuthal_equal_area_against_proj(): # Compare results with appropriate tolerance # For inverse transformations, angular differences can be larger angular_tolerance = 1e-5 # roughly 0.00001 degrees - assert abs(custom_lat - proj_lat) < angular_tolerance, \ + assert abs(custom_lat - proj_lat) < angular_tolerance, ( f"Lat mismatch for ({custom_x}, {custom_y}): custom={custom_lat}, proj={proj_lat}" + ) # Handle longitude wraparound for comparison lon_diff = np.mod(abs(custom_lon - proj_lon), 360) - assert min(lon_diff, 360 - lon_diff) < angular_tolerance, \ + assert min(lon_diff, 360 - lon_diff) < angular_tolerance, ( f"Lon mismatch for ({custom_x}, {custom_y}): custom={custom_lon}, proj={proj_lon}" + ) + def test_lambert_conformal_conic_against_proj(): # Create our custom projection with parameters from the existing test - lambda_0 = 352 # Reference longitude in degrees - phi_0 = 55.5 # Reference latitude in degrees - phi_1 = 55.5 # First standard parallel in degrees - phi_2 = 55.5 # Second standard parallel in degrees - radius = 6371229.0 # Earth radius in meters + lambda_0 = 352 # Reference longitude in degrees + phi_0 = 55.5 # Reference latitude in degrees + phi_1 = 55.5 # First standard parallel in degrees + phi_2 = 55.5 # Second standard parallel in degrees + radius = 6371229.0 # Earth radius in meters custom_proj = LambertConformalConicProjection( lambda_0=lambda_0, phi_0=phi_0, phi_1=phi_1, phi_2=phi_2, radius=radius @@ -545,27 +525,29 @@ def test_lambert_conformal_conic_against_proj(): lambda_0_norm = _normalize_longitude(lambda_0) # Create equivalent PROJ projection # For Lambert Conformal Conic, we use lat_0, lon_0, lat_1, lat_2 parameters - proj_string = (f"+proj=lcc +lat_0={phi_0} +lon_0={lambda_0_norm} +lat_1={phi_1} +lat_2={phi_2} " - f"+x_0=0 +y_0=0 +R={radius} +units=m +no_defs +type=crs") + proj_string = ( + f"+proj=lcc +lat_0={phi_0} +lon_0={lambda_0_norm} +lat_1={phi_1} +lat_2={phi_2} " + f"+x_0=0 +y_0=0 +R={radius} +units=m +no_defs +type=crs" + ) proj_proj = pyproj.Proj(proj_string) # Test points from the existing test center_lat = 39.671 center_lon = -25.421997 test_points = [ - (center_lat, center_lon), # Center point - (39.675304, -25.400146), # Near the center - (42.18604, -15.30127), # Point from the test (x=456, y=64) - (64.943695, 30.711975), # Point from the test (x=1642, y=1573) + (center_lat, center_lon), # Center point + (39.675304, -25.400146), # Near the center + (42.18604, -15.30127), # Point from the test (x=456, y=64) + (64.943695, 30.711975), # Point from the test (x=1642, y=1573) # Additional test points for broader coverage - (0, 0), # Origin - (phi_0, lambda_0_norm), # Projection origin - (45, 0), # Mid-latitude point - (-45, -45), # Southern hemisphere - (10, 50), # Europe - (40, -100), # North America - (50, -170), # Pacific - (-30, 170), # South Pacific + (0, 0), # Origin + (phi_0, lambda_0_norm), # Projection origin + (45, 0), # Mid-latitude point + (-45, -45), # Southern hemisphere + (10, 50), # Europe + (40, -100), # North America + (50, -170), # Pacific + (-30, 170), # South Pacific ] for lat, lon in test_points: @@ -575,7 +557,7 @@ def test_lambert_conformal_conic_against_proj(): # Forward transformation using PROJ # Note: PROJ expects (lon, lat) order, not (lat, lon) proj_x, proj_y = proj_proj(lon, lat) - tolerance = 0.1 # 0.1 meters for a 6.3 million meter radius is a reasonable precision + tolerance = 0.1 # 0.1 meters for a 6.3 million meter radius is a reasonable precision assert abs(custom_x - proj_x) < tolerance, f"X mismatch for ({lat}, {lon}): custom={custom_x}, proj={proj_x}" assert abs(custom_y - proj_y) < tolerance, f"Y mismatch for ({lat}, {lon}): custom={custom_y}, proj={proj_y}" @@ -584,13 +566,16 @@ def test_lambert_conformal_conic_against_proj(): # PROJ expects inverse=True for inverse transform proj_lon, proj_lat = proj_proj(proj_x, proj_y, inverse=True) angular_tolerance = 1e-5 # approximately 0.00001 degrees - assert abs(custom_lat - proj_lat) < angular_tolerance, \ + assert abs(custom_lat - proj_lat) < angular_tolerance, ( f"Lat mismatch for ({custom_x}, {custom_y}): custom={custom_lat}, proj={proj_lat}" + ) # Handle longitude wraparound for comparison lon_diff = np.mod(abs(custom_lon - proj_lon), 360) - assert min(lon_diff, 360 - lon_diff) < angular_tolerance, \ + assert min(lon_diff, 360 - lon_diff) < angular_tolerance, ( f"Lon mismatch for ({custom_x}, {custom_y}): custom={custom_lon}, proj={proj_lon}" + ) + def test_regular_lat_lon_grid_against_proj(): """Test that RegularLatLonGrid operations match proj equivalent operations""" @@ -601,26 +586,26 @@ def test_regular_lat_lon_grid_against_proj(): lat_step_size=1.0, lon_start=-180, lon_steps=360, # -180 to 180 - lon_step_size=1.0 + lon_step_size=1.0, ) # Create proj objects for WGS84 lat/lon - proj_wgs84 = pyproj.Proj(proj='latlong', datum='WGS84') + proj_wgs84 = pyproj.Proj(proj="latlong", datum="WGS84") # Test points covering different scenarios test_points: list[tuple[float, float]] = [ - (0, 0), # Origin - (45, 45), # NE quadrant - (-45, -45), # SW quadrant - (45, -45), # SE quadrant - (-45, 45), # NW quadrant - (89, 0), # Near North pole - (-89, 0), # Near South pole - (0, 179), # Near date line (east) - (0, -179), # Near date line (west) - (10, 20), # Random point - (-33, 151), # Sydney - (37, -122), # San Francisco + (0, 0), # Origin + (45, 45), # NE quadrant + (-45, -45), # SW quadrant + (45, -45), # SE quadrant + (-45, 45), # NW quadrant + (89, 0), # Near North pole + (-89, 0), # Near South pole + (0, 179), # Near date line (east) + (0, -179), # Near date line (west) + (10, 20), # Random point + (-33, 151), # Sydney + (37, -122), # San Francisco ] for lat, lon in test_points: @@ -651,11 +636,11 @@ def test_regular_lat_lon_grid_against_proj(): # Test longitude wrapping behavior wrap_test_points = [ - (0, 185), # Should wrap to (0, -175) - (0, -185), # Should wrap to (0, 175) - (0, 361), # Should wrap to (0, 1) - (0, -361), # Should wrap to (0, -1) - (45, 540), # Should wrap to (45, -180) + (0, 185), # Should wrap to (0, -175) + (0, -185), # Should wrap to (0, 175) + (0, 361), # Should wrap to (0, 1) + (0, -361), # Should wrap to (0, -1) + (45, 540), # Should wrap to (45, -180) ] for lat, lon in wrap_test_points: @@ -672,28 +657,21 @@ def test_regular_lat_lon_grid_against_proj(): def test_proj_projection(): """Test that ProjProjection correctly wraps proj transformations""" # Test with a Lambert Conformal Conic projection - proj_string = ( - "+proj=lcc +lat_0=55.5 +lon_0=352 +lat_1=55.5 +lat_2=55.5 " - "+x_0=0 +y_0=0 +R=6371229 +units=m +no_defs" - ) + proj_string = "+proj=lcc +lat_0=55.5 +lon_0=352 +lat_1=55.5 +lat_2=55.5 +x_0=0 +y_0=0 +R=6371229 +units=m +no_defs" # Create our projection wrapper proj = ProjProjection(proj_string) # Create a grid using this projection grid = ProjectionGrid.from_bounds( - nx=100, - ny=100, - lat_range=(39.67, 64.94), - lon_range=(-25.42, 30.71), - projection=proj + nx=100, ny=100, lat_range=(39.67, 64.94), lon_range=(-25.42, 30.71), projection=proj ) # Test points test_points = [ - (39.671, -25.421997), # Lower left + (39.671, -25.421997), # Lower left (64.943695, 30.711975), # Upper right - (50.0, 0.0), # Middle-ish + (50.0, 0.0), # Middle-ish ] # Create the raw proj transformer for comparison @@ -724,48 +702,36 @@ def test_proj_projection(): assert abs(result_lat - lat) < grid.dy, f"Grid lat error: {result_lat} vs {lat}" assert abs(result_lon - lon) < grid.dx, f"Grid lon error: {result_lon} vs {lon}" + def test_grid_equivalence(): """Test that a proj-based grid matches the original implementation""" # Create original LambertConformalConic projection - original_proj = LambertConformalConicProjection( - lambda_0=352, - phi_0=55.5, - phi_1=55.5, - phi_2=55.5, - radius=6371229.0 - ) + original_proj = LambertConformalConicProjection(lambda_0=352, phi_0=55.5, phi_1=55.5, phi_2=55.5, radius=6371229.0) # Create equivalent proj-based projection proj_proj = ProjProjection( - "+proj=lcc +lat_0=55.5 +lon_0=352 +lat_1=55.5 +lat_2=55.5 " - "+x_0=0 +y_0=0 +R=6371229 +units=m +no_defs" + "+proj=lcc +lat_0=55.5 +lon_0=352 +lat_1=55.5 +lat_2=55.5 +x_0=0 +y_0=0 +R=6371229 +units=m +no_defs" ) # Create grids with both projections grid_bounds = { - 'nx': 100, - 'ny': 100, - 'lat_range': (39.67, 64.94), - 'lon_range': (-25.42, 30.71), + "nx": 100, + "ny": 100, + "lat_range": (39.67, 64.94), + "lon_range": (-25.42, 30.71), } - original_grid = ProjectionGrid.from_bounds( - projection=original_proj, - **grid_bounds - ) + original_grid = ProjectionGrid.from_bounds(projection=original_proj, **grid_bounds) - proj_grid = ProjectionGrid.from_bounds( - projection=proj_proj, - **grid_bounds - ) + proj_grid = ProjectionGrid.from_bounds(projection=proj_proj, **grid_bounds) # Test points test_points = [ - (39.671, -25.421997), # Lower left + (39.671, -25.421997), # Lower left (64.943695, 30.711975), # Upper right - (50.0, 0.0), # Middle-ish - (45.0, -10.0), # Random point - (60.0, 20.0), # Random point + (50.0, 0.0), # Middle-ish + (45.0, -10.0), # Random point + (60.0, 20.0), # Random point ] for lat, lon in test_points: @@ -782,47 +748,31 @@ def test_grid_equivalence(): proj_grid_xy = proj_grid.findPointXy(lat=lat, lon=lon) # Grid coordinates should match exactly - assert abs(orig_grid_xy[0] - proj_grid_xy[0]) < 1e-8, \ - f"Grid X mismatch: {orig_grid_xy[0]} vs {proj_grid_xy[0]}" - assert abs(orig_grid_xy[1] - proj_grid_xy[1]) < 1e-8, \ - f"Grid Y mismatch: {orig_grid_xy[1]} vs {proj_grid_xy[1]}" + assert abs(orig_grid_xy[0] - proj_grid_xy[0]) < 1e-8, f"Grid X mismatch: {orig_grid_xy[0]} vs {proj_grid_xy[0]}" + assert abs(orig_grid_xy[1] - proj_grid_xy[1]) < 1e-8, f"Grid Y mismatch: {orig_grid_xy[1]} vs {proj_grid_xy[1]}" def test_grid_equivalence_regular_latlon(): """Test that a proj-based regular lat-lon grid matches the original implementation""" # Create a regular lat-lon grid using RegularLatLonGrid original_grid = RegularLatLonGrid( - lat_start=10.0, - lat_steps=100, - lat_step_size=0.5, - lon_start=-30.0, - lon_steps=120, - lon_step_size=0.5 + lat_start=10.0, lat_steps=100, lat_step_size=0.5, lon_start=-30.0, lon_steps=120, lon_step_size=0.5 ) # Create equivalent proj-based regular lat-lon projection - proj_proj = ProjProjection( - "+proj=longlat +datum=WGS84 +no_defs" - ) + proj_proj = ProjProjection("+proj=longlat +datum=WGS84 +no_defs") # Create equivalent grid with the proj projection - proj_grid = ProjectionGrid( - projection=proj_proj, - nx=120, - ny=100, - origin=(-30.0, 10.0), - dx=0.5, - dy=0.5 - ) + proj_grid = ProjectionGrid(projection=proj_proj, nx=120, ny=100, origin=(-30.0, 10.0), dx=0.5, dy=0.5) # Test points covering various areas within the grid test_points = [ - (10.0, -30.0), # Lower left corner - (59.5, 29.5), # Upper right corner - (35.0, 0.0), # Middle-ish - (20.0, -15.0), # Random point - (50.0, 20.0), # Random point - (15.25, -25.75), # Point between grid cells + (10.0, -30.0), # Lower left corner + (59.5, 29.5), # Upper right corner + (35.0, 0.0), # Middle-ish + (20.0, -15.0), # Random point + (50.0, 20.0), # Random point + (15.25, -25.75), # Point between grid cells ] for lat, lon in test_points: @@ -831,15 +781,16 @@ def test_grid_equivalence_regular_latlon(): proj_grid_xy = proj_grid.findPointXy(lat=lat, lon=lon) # Both should find the point or both should not find it - assert (orig_grid_xy is None) == (proj_grid_xy is None), \ - f"Inconsistent point finding for ({lat}, {lon})" + assert (orig_grid_xy is None) == (proj_grid_xy is None), f"Inconsistent point finding for ({lat}, {lon})" # If point is found, coordinates should match exactly if orig_grid_xy is not None: - assert abs(orig_grid_xy[0] - proj_grid_xy[0]) < 1e-8, \ + assert abs(orig_grid_xy[0] - proj_grid_xy[0]) < 1e-8, ( f"Grid X mismatch: {orig_grid_xy[0]} vs {proj_grid_xy[0]}" - assert abs(orig_grid_xy[1] - proj_grid_xy[1]) < 1e-8, \ + ) + assert abs(orig_grid_xy[1] - proj_grid_xy[1]) < 1e-8, ( f"Grid Y mismatch: {orig_grid_xy[1]} vs {proj_grid_xy[1]}" + ) # Test the inverse transformation (getCoordinates) if orig_grid_xy is not None: @@ -848,7 +799,5 @@ def test_grid_equivalence_regular_latlon(): proj_lat, proj_lon = proj_grid.getCoordinates(x, y) # Results should match exactly - assert abs(orig_lat - proj_lat) < 1e-8, \ - f"Latitude mismatch: {orig_lat} vs {proj_lat}" - assert abs(orig_lon - proj_lon) < 1e-8, \ - f"Longitude mismatch: {orig_lon} vs {proj_lon}" + assert abs(orig_lat - proj_lat) < 1e-8, f"Latitude mismatch: {orig_lat} vs {proj_lat}" + assert abs(orig_lon - proj_lon) < 1e-8, f"Longitude mismatch: {orig_lon} vs {proj_lon}" diff --git a/tests/test_om_domains.py b/tests/test_om_domains.py index e370ab71..032d61cc 100644 --- a/tests/test_om_domains.py +++ b/tests/test_om_domains.py @@ -24,6 +24,7 @@ def test_dwd_icon_d2_grid_points(): assert abs(lat - 52.52) < 0.05 assert abs(lon - 13.40) < 0.05 + def test_ecmwf_grid(): """Test the ECMWF IFS grid specifically.""" ecmwf_grid = DOMAINS["ecmwf_ifs025"].grid @@ -45,18 +46,19 @@ def test_ecmwf_grid(): point2 = ecmwf_grid.findPointXy(0.0, -179.0) assert point1 == point2 + def test_time_to_chunk_index(): """Test conversion from timestamp to chunk index.""" domain = DOMAINS["dwd_icon_d2"] # Create test timestamp (2023-01-01 12:00:00 UTC) - timestamp = np.datetime64('2023-01-01T12:00:00') + timestamp = np.datetime64("2023-01-01T12:00:00") # Calculate expected chunk index # Seconds since epoch = (2023-01-01 12:00:00 - 1970-01-01 00:00:00) seconds # chunk_index = seconds_since_epoch / (file_length * temporal_resolution_seconds) - epoch = np.datetime64('1970-01-01T00:00:00') - seconds_since_epoch = (timestamp - epoch) / np.timedelta64(1, 's') + epoch = np.datetime64("1970-01-01T00:00:00") + seconds_since_epoch = (timestamp - epoch) / np.timedelta64(1, "s") expected_chunk = int(seconds_since_epoch / (domain.file_length * domain.temporal_resolution_seconds)) # Test the time_to_chunk_index function @@ -77,4 +79,4 @@ def test_get_chunk_time_range(): # Check that time points are evenly spaced time_diff = time_range[1] - time_range[0] - assert time_diff == np.timedelta64(domain.temporal_resolution_seconds, 's') + assert time_diff == np.timedelta64(domain.temporal_resolution_seconds, "s") From 86b6f5e407a393421e2085b8c15f98ab7659c254 Mon Sep 17 00:00:00 2001 From: terraputix Date: Fri, 20 Jun 2025 11:11:01 +0200 Subject: [PATCH 16/50] disable 3.14 tests because of pyproj --- .github/workflows/build-test.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml index 15d678e7..321dcaac 100644 --- a/.github/workflows/build-test.yml +++ b/.github/workflows/build-test.yml @@ -172,12 +172,12 @@ jobs: python-version: "3.13" test-type: standard - - name: Python 3.14-Dev - runner: ubuntu-latest - platform-name: linux-x86_64 - is-musl: false - python-version: "3.14-dev" - test-type: standard + # - name: Python 3.14-Dev + # runner: ubuntu-latest + # platform-name: linux-x86_64 + # is-musl: false + # python-version: "3.14-dev" + # test-type: standard # Minimum dependencies test - name: Min Dependencies From 80138657c22198b92fdb59890b7c5a08de998f9e Mon Sep 17 00:00:00 2001 From: terraputix Date: Wed, 25 Jun 2025 19:24:54 +0200 Subject: [PATCH 17/50] some better tests and typing --- python/omfiles/grids.py | 13 ++--- python/omfiles/types.py | 9 ++++ python/omfiles/utils.py | 4 +- tests/test_grids.py | 83 +++++++++++++++++++++++++++---- uv.lock | 105 +++++++++++++++++++++++++++++++++++++++- 5 files changed, 193 insertions(+), 21 deletions(-) diff --git a/python/omfiles/grids.py b/python/omfiles/grids.py index 1ac124b5..2fcf3140 100644 --- a/python/omfiles/grids.py +++ b/python/omfiles/grids.py @@ -1,10 +1,11 @@ from abc import ABC, abstractmethod from functools import cached_property -from typing import Generic, Optional, Tuple, TypeVar, Union, cast +from typing import Generic, Optional, Tuple, TypeVar, cast import numpy as np import numpy.typing as npt +from omfiles.types import ArrayType, CoordType, ReturnUnionType from omfiles.utils import _modulo_positive, _normalize_longitude @@ -180,13 +181,6 @@ def getCoordinates(self, x: int, y: int) -> Tuple[float, float]: return (lat, lon) -# Type aliases for clarity -FloatType = Union[float, np.floating] -ArrayType = npt.NDArray[np.floating] -CoordType = Union[float, ArrayType] -ReturnUnionType = Union[tuple[ArrayType, ArrayType], tuple[float, float]] - - # Abstract base class instead of Protocol class AbstractProjection(ABC): """Base class for projection implementations.""" @@ -844,9 +838,10 @@ def getCoordinates(self, x: int, y: int) -> Tuple[float, float]: """ xcord = float(x) * self.dx + self.origin[0] ycord = float(y) * self.dy + self.origin[1] - lat, lon = cast(tuple[float, float], self.projection.inverse(xcord, ycord)) + lat, lon = self.projection.inverse(xcord, ycord) # Normalize longitude to -180 to 180 range lon = _normalize_longitude(lon) + lat, lon = cast(tuple[float, float], (lat, lon)) return (lat, lon) def get_true_north_direction(self) -> np.ndarray: diff --git a/python/omfiles/types.py b/python/omfiles/types.py index 776d9681..64ec92da 100644 --- a/python/omfiles/types.py +++ b/python/omfiles/types.py @@ -1,3 +1,6 @@ +import numpy as np +import numpy.typing as npt + try: from types import EllipsisType except ImportError: @@ -7,3 +10,9 @@ # This is from https://github.com/zarr-developers/zarr-python/blob/main/src/zarr/core/indexing.py#L38C1-L40C87 BasicSelector = Union[int, slice, EllipsisType] BasicSelection = Union[BasicSelector, Tuple[Union[int, slice, EllipsisType], ...]] # also used for BlockIndex + +# Type aliases for grids for clarity +FloatType = Union[float, np.floating] +ArrayType = npt.NDArray[np.floating] +CoordType = Union[float, ArrayType] +ReturnUnionType = Union[tuple[ArrayType, ArrayType], tuple[float, float]] diff --git a/python/omfiles/utils.py b/python/omfiles/utils.py index b21780ad..07722a81 100644 --- a/python/omfiles/utils.py +++ b/python/omfiles/utils.py @@ -1,5 +1,7 @@ import numpy as np +from omfiles.types import ArrayType + EPOCH = np.datetime64(0, "s") @@ -22,5 +24,5 @@ def _modulo_positive(value: int, modulo: int) -> int: return ((value % modulo) + modulo) % modulo -def _normalize_longitude(lon: float) -> float: +def _normalize_longitude(lon: ArrayType | float) -> ArrayType | float: return ((lon + 180.0) % 360.0) - 180.0 diff --git a/tests/test_grids.py b/tests/test_grids.py index fe7b01cc..121cc2f3 100644 --- a/tests/test_grids.py +++ b/tests/test_grids.py @@ -1,3 +1,5 @@ +from typing import cast + import numpy as np import pyproj import pytest @@ -442,6 +444,7 @@ def test_stereographic_against_proj(): # Test inverse transformation custom_lat, custom_lon = custom_proj.inverse(x=custom_x, y=custom_y) + proj_lon, proj_lat = proj_proj(proj_x, proj_y, inverse=True) # Compare results @@ -645,16 +648,20 @@ def test_regular_lat_lon_grid_against_proj(): for lat, lon in wrap_test_points: # Find grid coordinates for the wrapped point - grid_x, grid_y = grid.findPointXy(lat=lat, lon=lon) + result = grid.findPointXy(lat, lon) + assert result is not None, "Point not found in grid" + grid_x, grid_y = result # Find grid coordinates for the normalized longitude - norm_lon = _normalize_longitude(lon) - norm_grid_x, norm_grid_y = grid.findPointXy(lat=lat, lon=norm_lon) + norm_lon = cast(float, _normalize_longitude(lon)) + result = grid.findPointXy(lat=lat, lon=norm_lon) + assert result is not None, "Point not found in grid" + norm_grid_x, norm_grid_y = result assert grid_x == norm_grid_x, f"Grid X mismatch for wrapped lon: {lon} vs {norm_lon}" assert grid_y == norm_grid_y, f"Grid Y mismatch for wrapped lon: {lon} vs {norm_lon}" -def test_proj_projection(): +def test_proj_projection_grid(): """Test that ProjProjection correctly wraps proj transformations""" # Test with a Lambert Conformal Conic projection proj_string = "+proj=lcc +lat_0=55.5 +lon_0=352 +lat_1=55.5 +lat_2=55.5 +x_0=0 +y_0=0 +R=6371229 +units=m +no_defs" @@ -696,6 +703,7 @@ def test_proj_projection(): # Test roundtrip through the grid grid_coords = grid.findPointXy(lat=lat, lon=lon) + assert grid_coords is not None, f"Grid coordinates not found for lat={lat}, lon={lon}" result_lat, result_lon = grid.getCoordinates(*grid_coords) # Results should match within grid resolution @@ -703,7 +711,7 @@ def test_proj_projection(): assert abs(result_lon - lon) < grid.dx, f"Grid lon error: {result_lon} vs {lon}" -def test_grid_equivalence(): +def test_grid_equivalence_lcc(): """Test that a proj-based grid matches the original implementation""" # Create original LambertConformalConic projection original_proj = LambertConformalConicProjection(lambda_0=352, phi_0=55.5, phi_1=55.5, phi_2=55.5, radius=6371229.0) @@ -722,7 +730,6 @@ def test_grid_equivalence(): } original_grid = ProjectionGrid.from_bounds(projection=original_proj, **grid_bounds) - proj_grid = ProjectionGrid.from_bounds(projection=proj_proj, **grid_bounds) # Test points @@ -747,6 +754,11 @@ def test_grid_equivalence(): orig_grid_xy = original_grid.findPointXy(lat=lat, lon=lon) proj_grid_xy = proj_grid.findPointXy(lat=lat, lon=lon) + # Both should find the point or both should not find it + assert (orig_grid_xy is None) == (proj_grid_xy is None), f"Inconsistent point finding for ({lat}, {lon})" + if orig_grid_xy is None or proj_grid_xy is None: + return + # Grid coordinates should match exactly assert abs(orig_grid_xy[0] - proj_grid_xy[0]) < 1e-8, f"Grid X mismatch: {orig_grid_xy[0]} vs {proj_grid_xy[0]}" assert abs(orig_grid_xy[1] - proj_grid_xy[1]) < 1e-8, f"Grid Y mismatch: {orig_grid_xy[1]} vs {proj_grid_xy[1]}" @@ -761,8 +773,6 @@ def test_grid_equivalence_regular_latlon(): # Create equivalent proj-based regular lat-lon projection proj_proj = ProjProjection("+proj=longlat +datum=WGS84 +no_defs") - - # Create equivalent grid with the proj projection proj_grid = ProjectionGrid(projection=proj_proj, nx=120, ny=100, origin=(-30.0, 10.0), dx=0.5, dy=0.5) # Test points covering various areas within the grid @@ -780,8 +790,8 @@ def test_grid_equivalence_regular_latlon(): orig_grid_xy = original_grid.findPointXy(lat=lat, lon=lon) proj_grid_xy = proj_grid.findPointXy(lat=lat, lon=lon) - # Both should find the point or both should not find it - assert (orig_grid_xy is None) == (proj_grid_xy is None), f"Inconsistent point finding for ({lat}, {lon})" + # Both should find the point + assert orig_grid_xy is not None and proj_grid_xy is not None, f"Point not found for ({lat}, {lon})" # If point is found, coordinates should match exactly if orig_grid_xy is not None: @@ -801,3 +811,56 @@ def test_grid_equivalence_regular_latlon(): # Results should match exactly assert abs(orig_lat - proj_lat) < 1e-8, f"Latitude mismatch: {orig_lat} vs {proj_lat}" assert abs(orig_lon - proj_lon) < 1e-8, f"Longitude mismatch: {orig_lon} vs {proj_lon}" + + +def test_grid_equivalence_rotated_latlon(): + """Test that a proj-based rotated lat-lon grid matches the original implementation""" + lat_origin = -36.0885 + lon_origin = 245.305 + + original_proj = RotatedLatLonProjection(lat_origin=lat_origin, lon_origin=lon_origin) + proj_proj = ProjProjection( + f"+proj=ob_tran +o_proj=longlat +o_lat_p={-lat_origin} +o_lon_p=0.0 +lon_0={lon_origin} +datum=WGS84 +no_defs +type=crs" + ) + + # Grid bounds could be adjusted to something used in the open-meteo backend + grid_bounds = { + "nx": 100, + "ny": 80, + "lat_range": (-40.0, 40.0), + "lon_range": (200.0, 300.0), + } + + original_grid = ProjectionGrid.from_bounds(projection=original_proj, **grid_bounds) + proj_grid = ProjectionGrid.from_bounds(projection=proj_proj, **grid_bounds) + + # Test points covering various areas within the grid + test_points = [ + (0, 0), # Origin + (45, 45), # Mid-latitude + (-45, -45), # Southern hemisphere + (10, 50), # Europe + (40, -100), # North America + (50, -170), # Pacific + (-30, 170), # South Pacific + ] + + for lat, lon in test_points: + # Compare projection results + orig_x, orig_y = original_proj.forward(lat, lon) + proj_x, proj_y = proj_proj.forward(lat, lon) + assert abs(orig_x - proj_x) < 1e-3, f"X mismatch: {orig_x} vs {proj_x}" + assert abs(orig_y - proj_y) < 1e-3, f"Y mismatch: {orig_y} vs {proj_y}" + + # Compare grid results + orig_grid_xy = original_grid.findPointXy(lat=lat, lon=lon) + proj_grid_xy = proj_grid.findPointXy(lat=lat, lon=lon) + assert orig_grid_xy == proj_grid_xy, f"Grid index mismatch: {orig_grid_xy} vs {proj_grid_xy}" + + # Test the inverse transformation (getCoordinates) + if orig_grid_xy is not None: + x, y = orig_grid_xy + orig_lat, orig_lon = original_grid.getCoordinates(x, y) + proj_lat, proj_lon = proj_grid.getCoordinates(x, y) + assert abs(orig_lat - proj_lat) < 1e-4, f"Latitude mismatch: {orig_lat} vs {proj_lat}" + assert abs(orig_lon - proj_lon) < 1e-4, f"Longitude mismatch: {orig_lon} vs {proj_lon}" diff --git a/uv.lock b/uv.lock index 76559412..5936ac7c 100644 --- a/uv.lock +++ b/uv.lock @@ -1110,14 +1110,22 @@ bench = [ dev = [ { name = "maturin" }, { name = "psutil" }, + { name = "pyproj", version = "3.6.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" }, + { name = "pyproj", version = "3.7.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" }, { name = "pytest" }, { name = "pytest-asyncio" }, ] +test = [ + { name = "psutil" }, + { name = "pyproj", version = "3.6.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" }, + { name = "pyproj", version = "3.7.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" }, + { name = "pytest" }, +] [package.metadata] requires-dist = [ { name = "fsspec", specifier = ">=2023.1.0" }, - { name = "numpy", specifier = ">=1.20.0" }, + { name = "numpy", specifier = ">=1.21.0" }, { name = "s3fs", specifier = ">=2023.1.0" }, { name = "xarray", specifier = ">=2023.1.0" }, ] @@ -1133,9 +1141,15 @@ bench = [ dev = [ { name = "maturin", specifier = ">=1.7,<2.0" }, { name = "psutil" }, + { name = "pyproj" }, { name = "pytest", specifier = ">=6.1" }, { name = "pytest-asyncio" }, ] +test = [ + { name = "psutil" }, + { name = "pyproj" }, + { name = "pytest", specifier = ">=6.0" }, +] [[package]] name = "packaging" @@ -1341,6 +1355,95 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/8a/0b/9fcc47d19c48b59121088dd6da2488a49d5f72dacf8262e2790a1d2c7d15/pygments-2.19.1-py3-none-any.whl", hash = "sha256:9ea1544ad55cecf4b8242fab6dd35a93bbce657034b0611ee383099054ab6d8c", size = 1225293, upload-time = "2025-01-06T17:26:25.553Z" }, ] +[[package]] +name = "pyproj" +version = "3.6.1" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version < '3.10'", +] +dependencies = [ + { name = "certifi", marker = "python_full_version < '3.10'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/7d/84/2b39bbf888c753ea48b40d47511548c77aa03445465c35cc4c4e9649b643/pyproj-3.6.1.tar.gz", hash = "sha256:44aa7c704c2b7d8fb3d483bbf75af6cb2350d30a63b144279a09b75fead501bf", size = 225131, upload-time = "2023-09-21T02:07:51.593Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c5/32/63cf474f4a8d4804b3bdf7c16b8589f38142e8e2f8319dcea27e0bc21a87/pyproj-3.6.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:ab7aa4d9ff3c3acf60d4b285ccec134167a948df02347585fdd934ebad8811b4", size = 6142763, upload-time = "2023-09-21T02:07:12.844Z" }, + { url = "https://files.pythonhosted.org/packages/18/86/2e7cb9de40492f1bafbf11f4c9072edc394509a40b5e4c52f8139546f039/pyproj-3.6.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:4bc0472302919e59114aa140fd7213c2370d848a7249d09704f10f5b062031fe", size = 4877123, upload-time = "2023-09-21T02:10:37.905Z" }, + { url = "https://files.pythonhosted.org/packages/5e/c5/928d5a26995dbefbebd7507d982141cd9153bc7e4392b334fff722c4af12/pyproj-3.6.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5279586013b8d6582e22b6f9e30c49796966770389a9d5b85e25a4223286cd3f", size = 6190576, upload-time = "2023-09-21T02:17:08.637Z" }, + { url = "https://files.pythonhosted.org/packages/f6/2b/b60cf73b0720abca313bfffef34e34f7f7dae23852b2853cf0368d49426b/pyproj-3.6.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:80fafd1f3eb421694857f254a9bdbacd1eb22fc6c24ca74b136679f376f97d35", size = 8328075, upload-time = "2023-09-21T02:07:15.353Z" }, + { url = "https://files.pythonhosted.org/packages/d9/a8/7193f46032636be917bc775506ae987aad72c931b1f691b775ca812a2917/pyproj-3.6.1-cp310-cp310-win32.whl", hash = "sha256:c41e80ddee130450dcb8829af7118f1ab69eaf8169c4bf0ee8d52b72f098dc2f", size = 5635713, upload-time = "2023-09-21T02:07:17.548Z" }, + { url = "https://files.pythonhosted.org/packages/89/8f/27350c8fba71a37cd0d316f100fbd96bf139cc2b5ff1ab0dcbc7ac64010a/pyproj-3.6.1-cp310-cp310-win_amd64.whl", hash = "sha256:db3aedd458e7f7f21d8176f0a1d924f1ae06d725228302b872885a1c34f3119e", size = 6087932, upload-time = "2023-09-21T02:07:19.793Z" }, + { url = "https://files.pythonhosted.org/packages/84/a6/a300c1b14b2112e966e9f90b18f9c13b586bdcf417207cee913ae9005da3/pyproj-3.6.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:ebfbdbd0936e178091309f6cd4fcb4decd9eab12aa513cdd9add89efa3ec2882", size = 6147442, upload-time = "2023-09-21T02:07:21.879Z" }, + { url = "https://files.pythonhosted.org/packages/30/bd/b9bd3761f08754e8dbb34c5a647db2099b348ab5da338e90980caf280e37/pyproj-3.6.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:447db19c7efad70ff161e5e46a54ab9cc2399acebb656b6ccf63e4bc4a04b97a", size = 4880331, upload-time = "2023-09-21T02:10:40.828Z" }, + { url = "https://files.pythonhosted.org/packages/f4/0a/d82aeeb605b5d6870bc72307c3b5e044e632eb7720df8885e144f51a8eac/pyproj-3.6.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e7e13c40183884ec7f94eb8e0f622f08f1d5716150b8d7a134de48c6110fee85", size = 6192425, upload-time = "2023-09-21T02:17:09.049Z" }, + { url = "https://files.pythonhosted.org/packages/64/90/dfe5c00de1ca4dbb82606e79790659d4ed7f0ed8d372bccb3baca2a5abe0/pyproj-3.6.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:65ad699e0c830e2b8565afe42bd58cc972b47d829b2e0e48ad9638386d994915", size = 8571478, upload-time = "2023-09-21T02:07:23.771Z" }, + { url = "https://files.pythonhosted.org/packages/14/6d/ae373629a1723f0db80d7b8c93598b00d9ecb930ed9ebf4f35826a33e97c/pyproj-3.6.1-cp311-cp311-win32.whl", hash = "sha256:8b8acc31fb8702c54625f4d5a2a6543557bec3c28a0ef638778b7ab1d1772132", size = 5634575, upload-time = "2023-09-21T02:07:26.535Z" }, + { url = "https://files.pythonhosted.org/packages/79/95/eb68113c5b5737c342bde1bab92705dabe69c16299c5a122616e50f1fbd6/pyproj-3.6.1-cp311-cp311-win_amd64.whl", hash = "sha256:38a3361941eb72b82bd9a18f60c78b0df8408416f9340521df442cebfc4306e2", size = 6088494, upload-time = "2023-09-21T02:07:28.75Z" }, + { url = "https://files.pythonhosted.org/packages/0b/64/93232511a7906a492b1b7dfdfc17f4e95982d76a24ef4f86d18cfe7ae2c9/pyproj-3.6.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:1e9fbaf920f0f9b4ee62aab832be3ae3968f33f24e2e3f7fbb8c6728ef1d9746", size = 6135280, upload-time = "2023-09-21T02:07:30.911Z" }, + { url = "https://files.pythonhosted.org/packages/10/f2/b550b1f65cc7e51c9116b220b50aade60c439103432a3fd5b12efbc77e15/pyproj-3.6.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:6d227a865356f225591b6732430b1d1781e946893789a609bb34f59d09b8b0f8", size = 4880030, upload-time = "2023-09-21T02:10:43.067Z" }, + { url = "https://files.pythonhosted.org/packages/fe/4b/2f8f6f94643b9fe2083338eff294feda84d916409b5840b7a402d2be93f8/pyproj-3.6.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:83039e5ae04e5afc974f7d25ee0870a80a6bd6b7957c3aca5613ccbe0d3e72bf", size = 6184439, upload-time = "2023-09-21T02:17:43.499Z" }, + { url = "https://files.pythonhosted.org/packages/19/9b/c57569132174786aa3f72275ac306956859a639dad0ce8d95c8411ce8209/pyproj-3.6.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fffb059ba3bced6f6725961ba758649261d85ed6ce670d3e3b0a26e81cf1aa8d", size = 8660747, upload-time = "2023-09-21T02:07:32.586Z" }, + { url = "https://files.pythonhosted.org/packages/0e/ab/1c2159ec757677c5a6b8803f6be45c2b550dc42c84ec4a228dc219849bbb/pyproj-3.6.1-cp312-cp312-win32.whl", hash = "sha256:2d6ff73cc6dbbce3766b6c0bce70ce070193105d8de17aa2470009463682a8eb", size = 5626805, upload-time = "2023-09-21T02:07:35.28Z" }, + { url = "https://files.pythonhosted.org/packages/c7/f3/2f32fe143cd7ba1d4d68f1b6dce9ca402d909cbd5a5830e3a8fa3d1acbbf/pyproj-3.6.1-cp312-cp312-win_amd64.whl", hash = "sha256:7a27151ddad8e1439ba70c9b4b2b617b290c39395fa9ddb7411ebb0eb86d6fb0", size = 6079779, upload-time = "2023-09-21T02:07:37.486Z" }, + { url = "https://files.pythonhosted.org/packages/d7/50/d369bbe62d7a0d1e2cb40bc211da86a3f6e0f3c99f872957a72c3d5492d6/pyproj-3.6.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:4ba1f9b03d04d8cab24d6375609070580a26ce76eaed54631f03bab00a9c737b", size = 6144755, upload-time = "2023-09-21T02:07:39.611Z" }, + { url = "https://files.pythonhosted.org/packages/2c/c2/8d4f61065dfed965e53badd41201ad86a05af0c1bbc75dffb12ef0f5a7dd/pyproj-3.6.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:18faa54a3ca475bfe6255156f2f2874e9a1c8917b0004eee9f664b86ccc513d3", size = 4879187, upload-time = "2023-09-21T02:10:45.519Z" }, + { url = "https://files.pythonhosted.org/packages/31/38/2cf8777cb2d5622a78195e690281b7029098795fde4751aec8128238b8bb/pyproj-3.6.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fd43bd9a9b9239805f406fd82ba6b106bf4838d9ef37c167d3ed70383943ade1", size = 6192339, upload-time = "2023-09-21T02:17:09.942Z" }, + { url = "https://files.pythonhosted.org/packages/97/0a/b1525be9680369cc06dd288e12c59d24d5798b4afcdcf1b0915836e1caa6/pyproj-3.6.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:50100b2726a3ca946906cbaa789dd0749f213abf0cbb877e6de72ca7aa50e1ae", size = 8332638, upload-time = "2023-09-21T02:07:41.777Z" }, + { url = "https://files.pythonhosted.org/packages/8d/e8/e826e0a962f36bd925a933829cf6ef218efe2055db5ea292be40974a929d/pyproj-3.6.1-cp39-cp39-win32.whl", hash = "sha256:9274880263256f6292ff644ca92c46d96aa7e57a75c6df3f11d636ce845a1877", size = 5638159, upload-time = "2023-09-21T02:07:43.49Z" }, + { url = "https://files.pythonhosted.org/packages/43/d0/cbe29a4dcf38ee7e72bf695d0d3f2bee21b4f22ee6cf579ad974de9edfc8/pyproj-3.6.1-cp39-cp39-win_amd64.whl", hash = "sha256:36b64c2cb6ea1cc091f329c5bd34f9c01bb5da8c8e4492c709bda6a09f96808f", size = 6090565, upload-time = "2023-09-21T02:07:45.735Z" }, + { url = "https://files.pythonhosted.org/packages/43/28/e8d2ca71dd56c27cbe668e4226963d61956cded222a2e839e6fec1ab6d82/pyproj-3.6.1-pp39-pypy39_pp73-macosx_10_9_x86_64.whl", hash = "sha256:fd93c1a0c6c4aedc77c0fe275a9f2aba4d59b8acf88cebfc19fe3c430cfabf4f", size = 6034252, upload-time = "2023-09-21T02:07:47.906Z" }, + { url = "https://files.pythonhosted.org/packages/cb/39/1ce27cb86f51a1f5aed3a1617802a6131b59ea78492141d1fbe36722595e/pyproj-3.6.1-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6420ea8e7d2a88cb148b124429fba8cd2e0fae700a2d96eab7083c0928a85110", size = 6386263, upload-time = "2023-09-21T02:07:49.586Z" }, +] + +[[package]] +name = "pyproj" +version = "3.7.1" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version >= '3.13'", + "python_full_version == '3.12.*'", + "python_full_version == '3.11.*'", + "python_full_version == '3.10.*'", +] +dependencies = [ + { name = "certifi", marker = "python_full_version >= '3.10'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/67/10/a8480ea27ea4bbe896c168808854d00f2a9b49f95c0319ddcbba693c8a90/pyproj-3.7.1.tar.gz", hash = "sha256:60d72facd7b6b79853f19744779abcd3f804c4e0d4fa8815469db20c9f640a47", size = 226339, upload-time = "2025-02-16T04:28:46.621Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/25/a3/c4cd4bba5b336075f145fe784fcaf4ef56ffbc979833303303e7a659dda2/pyproj-3.7.1-cp310-cp310-macosx_13_0_x86_64.whl", hash = "sha256:bf09dbeb333c34e9c546364e7df1ff40474f9fddf9e70657ecb0e4f670ff0b0e", size = 6262524, upload-time = "2025-02-16T04:27:19.725Z" }, + { url = "https://files.pythonhosted.org/packages/40/45/4fdf18f4cc1995f1992771d2a51cf186a9d7a8ec973c9693f8453850c707/pyproj-3.7.1-cp310-cp310-macosx_14_0_arm64.whl", hash = "sha256:6575b2e53cc9e3e461ad6f0692a5564b96e7782c28631c7771c668770915e169", size = 4665102, upload-time = "2025-02-16T04:27:24.428Z" }, + { url = "https://files.pythonhosted.org/packages/0c/d2/360eb127380106cee83569954ae696b88a891c804d7a93abe3fbc15f5976/pyproj-3.7.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8cb516ee35ed57789b46b96080edf4e503fdb62dbb2e3c6581e0d6c83fca014b", size = 9432667, upload-time = "2025-02-16T04:27:27.04Z" }, + { url = "https://files.pythonhosted.org/packages/76/a5/c6e11b9a99ce146741fb4d184d5c468446c6d6015b183cae82ac822a6cfa/pyproj-3.7.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1e47c4e93b88d99dd118875ee3ca0171932444cdc0b52d493371b5d98d0f30ee", size = 9259185, upload-time = "2025-02-16T04:27:30.35Z" }, + { url = "https://files.pythonhosted.org/packages/41/56/a3c15c42145797a99363fa0fdb4e9805dccb8b4a76a6d7b2cdf36ebcc2a1/pyproj-3.7.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:3e8d276caeae34fcbe4813855d0d97b9b825bab8d7a8b86d859c24a6213a5a0d", size = 10469103, upload-time = "2025-02-16T04:27:33.542Z" }, + { url = "https://files.pythonhosted.org/packages/ef/73/c9194c2802fefe2a4fd4230bdd5ab083e7604e93c64d0356fa49c363bad6/pyproj-3.7.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:f173f851ee75e54acdaa053382b6825b400cb2085663a9bb073728a59c60aebb", size = 10401391, upload-time = "2025-02-16T04:27:36.051Z" }, + { url = "https://files.pythonhosted.org/packages/c5/1d/ce8bb5b9251b04d7c22d63619bb3db3d2397f79000a9ae05b3fd86a5837e/pyproj-3.7.1-cp310-cp310-win32.whl", hash = "sha256:f550281ed6e5ea88fcf04a7c6154e246d5714be495c50c9e8e6b12d3fb63e158", size = 5869997, upload-time = "2025-02-16T04:27:38.302Z" }, + { url = "https://files.pythonhosted.org/packages/09/6a/ca145467fd2e5b21e3d5b8c2b9645dcfb3b68f08b62417699a1f5689008e/pyproj-3.7.1-cp310-cp310-win_amd64.whl", hash = "sha256:3537668992a709a2e7f068069192138618c00d0ba113572fdd5ee5ffde8222f3", size = 6278581, upload-time = "2025-02-16T04:27:41.051Z" }, + { url = "https://files.pythonhosted.org/packages/ab/0d/63670fc527e664068b70b7cab599aa38b7420dd009bdc29ea257e7f3dfb3/pyproj-3.7.1-cp311-cp311-macosx_13_0_x86_64.whl", hash = "sha256:a94e26c1a4950cea40116775588a2ca7cf56f1f434ff54ee35a84718f3841a3d", size = 6264315, upload-time = "2025-02-16T04:27:44.539Z" }, + { url = "https://files.pythonhosted.org/packages/25/9d/cbaf82cfb290d1f1fa42feb9ba9464013bb3891e40c4199f8072112e4589/pyproj-3.7.1-cp311-cp311-macosx_14_0_arm64.whl", hash = "sha256:263b54ba5004b6b957d55757d846fc5081bc02980caa0279c4fc95fa0fff6067", size = 4666267, upload-time = "2025-02-16T04:27:47.019Z" }, + { url = "https://files.pythonhosted.org/packages/79/53/24f9f9b8918c0550f3ff49ad5de4cf3f0688c9f91ff191476db8979146fe/pyproj-3.7.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f6d6a2ccd5607cd15ef990c51e6f2dd27ec0a741e72069c387088bba3aab60fa", size = 9680510, upload-time = "2025-02-16T04:27:49.239Z" }, + { url = "https://files.pythonhosted.org/packages/3c/ac/12fab74a908d40b63174dc704587febd0729414804bbfd873cabe504ff2d/pyproj-3.7.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8c5dcf24ede53d8abab7d8a77f69ff1936c6a8843ef4fcc574646e4be66e5739", size = 9493619, upload-time = "2025-02-16T04:27:52.65Z" }, + { url = "https://files.pythonhosted.org/packages/c4/45/26311d6437135da2153a178125db5dfb6abce831ce04d10ec207eabac70a/pyproj-3.7.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:3c2e7449840a44ce860d8bea2c6c1c4bc63fa07cba801dcce581d14dcb031a02", size = 10709755, upload-time = "2025-02-16T04:27:55.239Z" }, + { url = "https://files.pythonhosted.org/packages/99/52/4ecd0986f27d0e6c8ee3a7bc5c63da15acd30ac23034f871325b297e61fd/pyproj-3.7.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:0829865c1d3a3543f918b3919dc601eea572d6091c0dd175e1a054db9c109274", size = 10642970, upload-time = "2025-02-16T04:27:58.343Z" }, + { url = "https://files.pythonhosted.org/packages/3f/a5/d3bfc018fc92195a000d1d28acc1f3f1df15ff9f09ece68f45a2636c0134/pyproj-3.7.1-cp311-cp311-win32.whl", hash = "sha256:6181960b4b812e82e588407fe5c9c68ada267c3b084db078f248db5d7f45d18a", size = 5868295, upload-time = "2025-02-16T04:28:01.712Z" }, + { url = "https://files.pythonhosted.org/packages/92/39/ef6f06a5b223dbea308cfcbb7a0f72e7b506aef1850e061b2c73b0818715/pyproj-3.7.1-cp311-cp311-win_amd64.whl", hash = "sha256:5ad0ff443a785d84e2b380869fdd82e6bfc11eba6057d25b4409a9bbfa867970", size = 6279871, upload-time = "2025-02-16T04:28:04.988Z" }, + { url = "https://files.pythonhosted.org/packages/e6/c9/876d4345b8d17f37ac59ebd39f8fa52fc6a6a9891a420f72d050edb6b899/pyproj-3.7.1-cp312-cp312-macosx_13_0_x86_64.whl", hash = "sha256:2781029d90df7f8d431e29562a3f2d8eafdf233c4010d6fc0381858dc7373217", size = 6264087, upload-time = "2025-02-16T04:28:09.036Z" }, + { url = "https://files.pythonhosted.org/packages/ff/e6/5f8691f8c90e7f402cc80a6276eb19d2ec1faa150d5ae2dd9c7b0a254da8/pyproj-3.7.1-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:d61bf8ab04c73c1da08eedaf21a103b72fa5b0a9b854762905f65ff8b375d394", size = 4669628, upload-time = "2025-02-16T04:28:10.944Z" }, + { url = "https://files.pythonhosted.org/packages/42/ec/16475bbb79c1c68845c0a0d9c60c4fb31e61b8a2a20bc18b1a81e81c7f68/pyproj-3.7.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:04abc517a8555d1b05fcee768db3280143fe42ec39fdd926a2feef31631a1f2f", size = 9721415, upload-time = "2025-02-16T04:28:13.342Z" }, + { url = "https://files.pythonhosted.org/packages/b3/a3/448f05b15e318bd6bea9a32cfaf11e886c4ae61fa3eee6e09ed5c3b74bb2/pyproj-3.7.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:084c0a475688f934d386c2ab3b6ce03398a473cd48adfda70d9ab8f87f2394a0", size = 9556447, upload-time = "2025-02-16T04:28:15.818Z" }, + { url = "https://files.pythonhosted.org/packages/6a/ae/bd15fe8d8bd914ead6d60bca7f895a4e6f8ef7e3928295134ff9a7dad14c/pyproj-3.7.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:a20727a23b1e49c7dc7fe3c3df8e56a8a7acdade80ac2f5cca29d7ca5564c145", size = 10758317, upload-time = "2025-02-16T04:28:18.338Z" }, + { url = "https://files.pythonhosted.org/packages/9d/d9/5ccefb8bca925f44256b188a91c31238cae29ab6ee7f53661ecc04616146/pyproj-3.7.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:bf84d766646f1ebd706d883755df4370aaf02b48187cedaa7e4239f16bc8213d", size = 10771259, upload-time = "2025-02-16T04:28:20.822Z" }, + { url = "https://files.pythonhosted.org/packages/2a/7d/31dedff9c35fa703162f922eeb0baa6c44a3288469a5fd88d209e2892f9e/pyproj-3.7.1-cp312-cp312-win32.whl", hash = "sha256:5f0da2711364d7cb9f115b52289d4a9b61e8bca0da57f44a3a9d6fc9bdeb7274", size = 5859914, upload-time = "2025-02-16T04:28:23.303Z" }, + { url = "https://files.pythonhosted.org/packages/3e/47/c6ab03d6564a7c937590cff81a2742b5990f096cce7c1a622d325be340ee/pyproj-3.7.1-cp312-cp312-win_amd64.whl", hash = "sha256:aee664a9d806612af30a19dba49e55a7a78ebfec3e9d198f6a6176e1d140ec98", size = 6273196, upload-time = "2025-02-16T04:28:25.227Z" }, + { url = "https://files.pythonhosted.org/packages/ef/01/984828464c9960036c602753fc0f21f24f0aa9043c18fa3f2f2b66a86340/pyproj-3.7.1-cp313-cp313-macosx_13_0_x86_64.whl", hash = "sha256:5f8d02ef4431dee414d1753d13fa82a21a2f61494737b5f642ea668d76164d6d", size = 6253062, upload-time = "2025-02-16T04:28:27.861Z" }, + { url = "https://files.pythonhosted.org/packages/68/65/6ecdcdc829811a2c160cdfe2f068a009fc572fd4349664f758ccb0853a7c/pyproj-3.7.1-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:0b853ae99bda66cbe24b4ccfe26d70601d84375940a47f553413d9df570065e0", size = 4660548, upload-time = "2025-02-16T04:28:29.526Z" }, + { url = "https://files.pythonhosted.org/packages/67/da/dda94c4490803679230ba4c17a12f151b307a0d58e8110820405ca2d98db/pyproj-3.7.1-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:83db380c52087f9e9bdd8a527943b2e7324f275881125e39475c4f9277bdeec4", size = 9662464, upload-time = "2025-02-16T04:28:31.437Z" }, + { url = "https://files.pythonhosted.org/packages/6f/57/f61b7d22c91ae1d12ee00ac4c0038714e774ebcd851b9133e5f4f930dd40/pyproj-3.7.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b35ed213892e211a3ce2bea002aa1183e1a2a9b79e51bb3c6b15549a831ae528", size = 9497461, upload-time = "2025-02-16T04:28:33.848Z" }, + { url = "https://files.pythonhosted.org/packages/b7/f6/932128236f79d2ac7d39fe1a19667fdf7155d9a81d31fb9472a7a497790f/pyproj-3.7.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:a8b15b0463d1303bab113d1a6af2860a0d79013c3a66fcc5475ce26ef717fd4f", size = 10708869, upload-time = "2025-02-16T04:28:37.34Z" }, + { url = "https://files.pythonhosted.org/packages/1d/0d/07ac7712994454a254c383c0d08aff9916a2851e6512d59da8dc369b1b02/pyproj-3.7.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:87229e42b75e89f4dad6459200f92988c5998dfb093c7c631fb48524c86cd5dc", size = 10729260, upload-time = "2025-02-16T04:28:40.639Z" }, + { url = "https://files.pythonhosted.org/packages/b0/d0/9c604bc72c37ba69b867b6df724d6a5af6789e8c375022c952f65b2af558/pyproj-3.7.1-cp313-cp313-win32.whl", hash = "sha256:d666c3a3faaf3b1d7fc4a544059c4eab9d06f84a604b070b7aa2f318e227798e", size = 5855462, upload-time = "2025-02-16T04:28:42.827Z" }, + { url = "https://files.pythonhosted.org/packages/98/df/68a2b7f5fb6400c64aad82d72bcc4bc531775e62eedff993a77c780defd0/pyproj-3.7.1-cp313-cp313-win_amd64.whl", hash = "sha256:d3caac7473be22b6d6e102dde6c46de73b96bc98334e577dfaee9886f102ea2e", size = 6266573, upload-time = "2025-02-16T04:28:44.727Z" }, +] + [[package]] name = "pytest" version = "8.4.1" From 4fe96589c8e495c4a39d79f8fd20bd96f45189b4 Mon Sep 17 00:00:00 2001 From: terraputix Date: Thu, 26 Jun 2025 09:15:02 +0200 Subject: [PATCH 18/50] fix typing syntax for older python versions --- python/omfiles/utils.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/python/omfiles/utils.py b/python/omfiles/utils.py index 07722a81..7fd09565 100644 --- a/python/omfiles/utils.py +++ b/python/omfiles/utils.py @@ -1,3 +1,5 @@ +from typing import Union + import numpy as np from omfiles.types import ArrayType @@ -24,5 +26,5 @@ def _modulo_positive(value: int, modulo: int) -> int: return ((value % modulo) + modulo) % modulo -def _normalize_longitude(lon: ArrayType | float) -> ArrayType | float: +def _normalize_longitude(lon: Union[ArrayType, float]) -> Union[ArrayType, float]: return ((lon + 180.0) % 360.0) - 180.0 From db61328b825ae08f931e59b94da31f33f7104469 Mon Sep 17 00:00:00 2001 From: terraputix Date: Fri, 1 Aug 2025 20:30:42 +0200 Subject: [PATCH 19/50] fix docstrings --- examples/select_by_coordinates.py | 53 +-- python/omfiles/{utils.py => _utils.py} | 13 +- python/omfiles/grids.py | 463 +++++++++++-------------- python/omfiles/om_domains.py | 58 ++-- python/omfiles/types.py | 4 +- 5 files changed, 245 insertions(+), 346 deletions(-) rename python/omfiles/{utils.py => _utils.py} (71%) diff --git a/examples/select_by_coordinates.py b/examples/select_by_coordinates.py index c2d3c7a3..fe059b18 100644 --- a/examples/select_by_coordinates.py +++ b/examples/select_by_coordinates.py @@ -61,27 +61,17 @@ def load_chunk_data( """ Load data for a specific chunk and grid coordinates. - Parameters: - ----------- - chunk_index : int - Index of the chunk to load - domain_name : str - Name of the domain - variable_name : str - Name of the variable to fetch - grid_coords : Tuple[int, int] - Grid coordinates (x, y) to extract - fs : fsspec.AbstractFileSystem - Filesystem to use for loading data - start_date : np.datetime64 - Start of requested date range - end_date : np.datetime64 - End of requested date range + Args: + chunk_index (int): Index of the chunk to load. + domain_name (str): Name of the domain. + variable_name (str): Name of the variable to fetch. + grid_coords (Tuple[int, int]): Grid coordinates (x, y) to extract. + fs (fsspec.AbstractFileSystem): Filesystem to use for loading data. + start_date (np.datetime64): Start of requested date range. + end_date (np.datetime64): End of requested date range. Returns: - -------- - Tuple[np.ndarray, np.ndarray] - Tuple containing (time_array, data_array) + Tuple[np.ndarray, np.ndarray]: Tuple containing (time_array, data_array). """ domain = DOMAINS[domain_name] x, y = grid_coords @@ -114,25 +104,16 @@ def get_data_for_coordinates( """ Fetch weather data for specific coordinates across a date range, merging multiple files as needed. - Parameters: - ----------- - lat : float - Latitude in degrees - lon : float - Longitude in degrees - domain_name : str - Name of the domain to use (must be in omfiles.om_domains.DOMAINS) - variable_name : str - Name of the variable to fetch - start_date : datetime - Start date for the data - end_date : datetime - End date for the data + Args: + lat (float): Latitude in degrees. + lon (float): Longitude in degrees. + domain_name (str): Name of the domain to use (must be in omfiles.om_domains.DOMAINS). + variable_name (str): Name of the variable to fetch. + start_date (datetime): Start date for the data. + end_date (datetime): End date for the data. Returns: - -------- - xr.Dataset - Dataset containing the requested variable at the specified location + xr.Dataset: Dataset containing the requested variable at the specified location. """ # Get the domain configuration if domain_name not in DOMAINS: diff --git a/python/omfiles/utils.py b/python/omfiles/_utils.py similarity index 71% rename from python/omfiles/utils.py rename to python/omfiles/_utils.py index 7fd09565..ab964121 100644 --- a/python/omfiles/utils.py +++ b/python/omfiles/_utils.py @@ -11,17 +11,12 @@ def _modulo_positive(value: int, modulo: int) -> int: """ Calculate modulo that always returns positive value. - Parameters: - ----------- - value : int - Value to calculate modulo for - modulo : int - Modulo value + Args: + value (int): Value to calculate modulo for. + modulo (int): Modulo value. Returns: - -------- - int - Positive modulo result + int: Positive modulo result. """ return ((value % modulo) + modulo) % modulo diff --git a/python/omfiles/grids.py b/python/omfiles/grids.py index 2fcf3140..dd1e3869 100644 --- a/python/omfiles/grids.py +++ b/python/omfiles/grids.py @@ -1,3 +1,5 @@ +"""Grids used in Open-Meteo files.""" + from abc import ABC, abstractmethod from functools import cached_property from typing import Generic, Optional, Tuple, TypeVar, cast @@ -5,8 +7,8 @@ import numpy as np import numpy.typing as npt +from omfiles._utils import _modulo_positive, _normalize_longitude from omfiles.types import ArrayType, CoordType, ReturnUnionType -from omfiles.utils import _modulo_positive, _normalize_longitude class AbstractGrid(ABC): @@ -28,7 +30,8 @@ def latitude(self) -> np.ndarray: """ Return the latitude coordinates array. - Uses cached_property to ensure the array is computed only once. + Returns: + np.ndarray: Array of latitude coordinates. """ pass @@ -38,20 +41,33 @@ def longitude(self) -> np.ndarray: """ Return the longitude coordinates array. - Uses cached_property to ensure the array is computed only once. + Returns: + np.ndarray: Array of longitude coordinates. """ pass @property @abstractmethod def shape(self) -> Tuple[int, int]: - """Return the grid shape as (n_lat, n_lon).""" + """ + Return the grid shape as (n_lat, n_lon). + + Returns: + Tuple[int, int]: Shape of the grid as (n_lat, n_lon). + """ pass @abstractmethod def findPointXy(self, lat: float, lon: float) -> Optional[Tuple[int, int]]: """ Find grid point indices (x, y) for given lat/lon coordinates. + + Args: + lat (float): Latitude in degrees. + lon (float): Longitude in degrees. + + Returns: + Optional[Tuple[int, int]]: (x, y) grid indices if point is in grid, None otherwise. """ pass @@ -59,6 +75,13 @@ def findPointXy(self, lat: float, lon: float) -> Optional[Tuple[int, int]]: def getCoordinates(self, x: int, y: int) -> Tuple[float, float]: """ Get lat/lon coordinates for a given grid point indices. + + Args: + x (int): Grid x index. + y (int): Grid y index. + + Returns: + Tuple[float, float]: (latitude, longitude) coordinates. """ pass @@ -67,7 +90,7 @@ class RegularLatLonGrid(AbstractGrid): """ Regular latitude-longitude grid implementation. - This represents a standard equirectangular grid with uniform spacing. + Represents a standard equirectangular grid with uniform spacing. """ def __init__( @@ -82,20 +105,13 @@ def __init__( """ Initialize a regular lat/lon grid. - Parameters: - ----------- - lat_start : float - Starting latitude value - lat_steps : int - Number of latitude points - lat_step_size : float - Spacing between latitude points - lon_start : float - Starting longitude value - lon_steps : int - Number of longitude points - lon_step_size : float - Spacing between longitude points + Args: + lat_start (float): Starting latitude value. + lat_steps (int): Number of latitude points. + lat_step_size (float): Spacing between latitude points. + lon_start (float): Starting longitude value. + lon_steps (int): Number of longitude points. + lon_step_size (float): Spacing between longitude points. """ self._lat_start = lat_start self._lat_steps = lat_steps @@ -106,12 +122,21 @@ def __init__( @property def grid_type(self) -> str: + """ + Grid type identifier. + + Returns: + str: The grid type identifier. + """ return "regular_latlon" @cached_property def latitude(self) -> np.ndarray: """ - Lazily compute and cache the latitude coordinate array. + Compute and cache the latitude coordinate array. + + Returns: + np.ndarray: Array of latitude coordinates. """ return np.linspace( self._lat_start, self._lat_start + self._lat_step_size * self._lat_steps, self._lat_steps, endpoint=False @@ -120,7 +145,10 @@ def latitude(self) -> np.ndarray: @cached_property def longitude(self) -> np.ndarray: """ - Lazily compute and cache the longitude coordinate array. + Compute and cache the longitude coordinate array. + + Returns: + np.ndarray: Array of longitude coordinates. """ return np.linspace( self._lon_start, self._lon_start + self._lon_step_size * self._lon_steps, self._lon_steps, endpoint=False @@ -128,23 +156,24 @@ def longitude(self) -> np.ndarray: @property def shape(self) -> Tuple[int, int]: + """ + Grid shape as (n_lat, n_lon). + + Returns: + Tuple[int, int]: Shape of the grid as (n_lat, n_lon). + """ return (self._lat_steps, self._lon_steps) def findPointXy(self, lat: float, lon: float) -> Optional[Tuple[int, int]]: """ Find grid point indices (x, y) for given lat/lon coordinates. - Parameters: - ----------- - lat : float - Latitude in degrees - lon : float - Longitude in degrees + Args: + lat (float): Latitude in degrees. + lon (float): Longitude in degrees. Returns: - -------- - tuple or None - (x, y) grid indices if point is in grid, None otherwise + Optional[Tuple[int, int]]: (x, y) grid indices if point is in grid, None otherwise. """ # Calculate raw x and y indices x = int(round((lon - self._lon_start) / self._lon_step_size)) @@ -164,24 +193,19 @@ def getCoordinates(self, x: int, y: int) -> Tuple[float, float]: """ Get lat/lon coordinates for a given grid point indices. - Parameters: - ----------- - x : longitude index - y : latitude index + Args: + x (int): Longitude index. + y (int): Latitude index. Returns: - -------- - tuple - (latitude, longitude) coordinates + Tuple[float, float]: (latitude, longitude) coordinates. """ - lat = self._lat_start + float(y) * self._lat_step_size lon = self._lon_start + float(x) * self._lon_step_size return (lat, lon) -# Abstract base class instead of Protocol class AbstractProjection(ABC): """Base class for projection implementations.""" @@ -217,12 +241,9 @@ def __init__(self, lat_origin: float, lon_origin: float): """ Initialize a rotated lat/lon projection. - Parameters: - ----------- - lat_origin : float - Latitude of origin in degrees - lon_origin : float - Longitude of origin in degrees + Args: + lat_origin (float): Latitude of origin in degrees. + lon_origin (float): Longitude of origin in degrees. """ # θ: Rotation around y-axis self.theta = np.radians(90.0 + lat_origin) @@ -233,17 +254,12 @@ def forward(self, latitude: CoordType, longitude: CoordType) -> ReturnUnionType: """ Transform from regular lat/lon to rotated lat/lon coordinates. - Parameters: - ----------- - latitude : float or array - Latitude in degrees - longitude : float or array - Longitude in degrees + Args: + latitude (float or array): Latitude in degrees. + longitude (float or array): Longitude in degrees. Returns: - -------- - tuple - (rotated_lat, rotated_lon) in degrees + tuple: (rotated_lat, rotated_lon) in degrees. """ scalar_input = np.isscalar(latitude) and np.isscalar(longitude) @@ -285,17 +301,12 @@ def inverse(self, x: CoordType, y: CoordType) -> ReturnUnionType: """ Transform from rotated lat/lon back to regular lat/lon coordinates. - Parameters: - ----------- - x : float or array - Rotated longitude in degrees - y : float or array - Rotated latitude in degrees + Args: + x (float or array): Rotated longitude in degrees. + y (float or array): Rotated latitude in degrees. Returns: - -------- - tuple - (latitude, longitude) in degrees + tuple: (latitude, longitude) in degrees. """ scalar_input = np.isscalar(x) and np.isscalar(y) @@ -335,14 +346,10 @@ def __init__(self, latitude: float, longitude: float, radius: float = 6371000.0) """ Initialize a stereographic projection. - Parameters: - ----------- - latitude : float - Central latitude in degrees - longitude : float - Central longitude in degrees - radius : float - Radius of Earth in meters (default: 6371000.0) + Args: + latitude (float): Central latitude in degrees. + longitude (float): Central longitude in degrees. + radius (float, optional): Radius of Earth in meters. Defaults to 6371000.0. """ self.lambda_0: npt.NDArray[np.float32] = np.radians(longitude) self.sin_phi_1: npt.NDArray[np.float32] = np.sin(np.radians(latitude)) @@ -353,17 +360,12 @@ def forward(self, latitude: CoordType, longitude: CoordType) -> ReturnUnionType: """ Transform from lat/lon coordinates to projected x/y coordinates. - Parameters: - ----------- - latitude : float or array - Latitude in degrees - longitude : float or array - Longitude in degrees + Args: + latitude (float or array): Latitude in degrees. + longitude (float or array): Longitude in degrees. Returns: - -------- - tuple - (x, y) coordinates in the projection + tuple: (x, y) coordinates in the projection. """ scalar_input = np.isscalar(latitude) and np.isscalar(longitude) @@ -389,17 +391,12 @@ def inverse(self, x: CoordType, y: CoordType) -> ReturnUnionType: """ Transform from projected x/y coordinates back to lat/lon. - Parameters: - ----------- - x : float or array - X coordinate in the projection - y : float or array - Y coordinate in the projection + Args: + x (float or array): X coordinate in the projection. + y (float or array): Y coordinate in the projection. Returns: - -------- - tuple - (latitude, longitude) in degrees + tuple: (latitude, longitude) in degrees. """ x_arr = np.asarray(x, dtype=np.float32) y_arr = np.asarray(y, dtype=np.float32) @@ -423,23 +420,21 @@ class LambertAzimuthalEqualAreaProjection(AbstractProjection): """ Lambert Azimuthal Equal-Area projection implementation. - This implements the equations for the Lambert Azimuthal Equal-Area projection + Implements the equations for the Lambert Azimuthal Equal-Area projection, which preserves area but not angles or distances. - https://mathworld.wolfram.com/LambertAzimuthalEqual-AreaProjection.html + + Reference: + https://mathworld.wolfram.com/LambertAzimuthalEqual-AreaProjection.html """ def __init__(self, lambda_0: float, phi_1: float, radius: float = 6371229.0): """ Initialize a Lambert Azimuthal Equal-Area projection. - Parameters: - ----------- - lambda_0 : float - Central longitude in degrees - phi_1 : float - Standard parallel in degrees - radius : float - Radius of Earth in meters (default: 6371229.0) + Args: + lambda_0 (float): Central longitude in degrees. + phi_1 (float): Standard parallel in degrees. + radius (float, optional): Radius of Earth in meters. Defaults to 6371229.0. """ self.lambda_0 = np.radians(lambda_0) self.phi_1 = np.radians(phi_1) @@ -449,17 +444,12 @@ def forward(self, latitude: CoordType, longitude: CoordType) -> ReturnUnionType: """ Transform from lat/lon coordinates to projected x/y coordinates. - Parameters: - ----------- - latitude : float or array - Latitude in degrees - longitude : float or array - Longitude in degrees + Args: + latitude (float or array): Latitude in degrees. + longitude (float or array): Longitude in degrees. Returns: - -------- - tuple - (x, y) coordinates in the projection + tuple: (x, y) coordinates in the projection. """ scalar_input = np.isscalar(latitude) and np.isscalar(longitude) @@ -494,17 +484,12 @@ def inverse(self, x: CoordType, y: CoordType) -> ReturnUnionType: """ Transform from projected x/y coordinates back to lat/lon. - Parameters: - ----------- - x : float or array - X coordinate in the projection - y : float or array - Y coordinate in the projection + Args: + x (float or array): X coordinate in the projection. + y (float or array): Y coordinate in the projection. Returns: - -------- - tuple - (latitude, longitude) in degrees + tuple: (latitude, longitude) in degrees. """ scalar_input = np.isscalar(x) and np.isscalar(y) @@ -537,28 +522,24 @@ class LambertConformalConicProjection(AbstractProjection): """ Lambert Conformal Conic projection implementation. - This implements the equations for the Lambert Conformal Conic projection, + Implements the equations for the Lambert Conformal Conic projection, which preserves angles but not areas or distances. - https://mathworld.wolfram.com/LambertConformalConicProjection.html - https://pubs.usgs.gov/pp/1395/report.pdf page 104 + + References: + https://mathworld.wolfram.com/LambertConformalConicProjection.html + https://pubs.usgs.gov/pp/1395/report.pdf page 104 """ def __init__(self, lambda_0: float, phi_0: float, phi_1: float, phi_2: float, radius: float = 6370997): """ Initialize a Lambert Conformal Conic projection. - Parameters: - ----------- - lambda_0 : float - Reference longitude in degrees (LoVInDegrees in grib) - phi_0 : float - Reference latitude in degrees (LaDInDegrees in grib) - phi_1 : float - First standard parallel in degrees (Latin1InDegrees in grib) - phi_2 : float - Second standard parallel in degrees (Latin2InDegrees in grib) - radius : float - Radius of Earth in meters (default: 6370997) + Args: + lambda_0 (float): Reference longitude in degrees (LoVInDegrees in grib). + phi_0 (float): Reference latitude in degrees (LaDInDegrees in grib). + phi_1 (float): First standard parallel in degrees (Latin1InDegrees in grib). + phi_2 (float): Second standard parallel in degrees (Latin2InDegrees in grib). + radius (float): Radius of Earth in meters (default: 6370997). """ # Normalize lambda_0 to [-180, 180] range lambda_0_normalized = _normalize_longitude(lambda_0) @@ -586,17 +567,12 @@ def forward(self, latitude: CoordType, longitude: CoordType) -> ReturnUnionType: """ Transform from lat/lon coordinates to projected x/y coordinates. - Parameters: - ----------- - latitude : float or array - Latitude in degrees - longitude : float or array - Longitude in degrees + Args: + latitude (float or array): Latitude in degrees. + longitude (float or array): Longitude in degrees. Returns: - -------- - tuple - (x, y) coordinates in the projection + tuple: (x, y) coordinates in the projection. """ scalar_input = np.isscalar(latitude) and np.isscalar(longitude) @@ -619,17 +595,12 @@ def inverse(self, x: CoordType, y: CoordType) -> ReturnUnionType: """ Transform from projected x/y coordinates back to lat/lon. - Parameters: - ----------- - x : float or array - X coordinate in the projection - y : float or array - Y coordinate in the projection + Args: + x (float or array): X coordinate in the projection. + y (float or array): Y coordinate in the projection. Returns: - -------- - tuple - (latitude, longitude) in degrees + tuple: (latitude, longitude) in degrees. """ scalar_input = np.isscalar(x) and np.isscalar(y) @@ -671,20 +642,13 @@ def __init__(self, projection: P, nx: int, ny: int, origin: Tuple[float, float], """ Initialize a projection grid with all parameters. - Parameters: - ----------- - projection : Projectable - Projection implementation - nx : int - Number of grid points in x direction - ny : int - Number of grid points in y direction - origin : Tuple[float, float] - Origin coordinates (x, y) of the grid in projection space - dx : float - Grid spacing in x direction - dy : float - Grid spacing in y direction + Args: + projection (Projectable): Projection implementation. + nx (int): Number of grid points in x direction. + ny (int): Number of grid points in y direction. + origin (Tuple[float, float]): Origin coordinates (x, y) of the grid in projection space. + dx (float): Grid spacing in x direction. + dy (float): Grid spacing in y direction. """ self.projection = projection self.nx = nx @@ -700,23 +664,15 @@ def from_bounds( """ Create a projection grid from geographic bounds. - Parameters: - ----------- - nx : int - Number of grid points in x direction - ny : int - Number of grid points in y direction - lat_range : Tuple[float, float] - Latitude range (min, max) in degrees - lon_range : Tuple[float, float] - Longitude range (min, max) in degrees - projection : Projectable - Projection implementation + Args: + nx (int): Number of grid points in x direction. + ny (int): Number of grid points in y direction. + lat_range (Tuple[float, float]): Latitude range (min, max) in degrees. + lon_range (Tuple[float, float]): Longitude range (min, max) in degrees. + projection (Projectable): Projection implementation. Returns: - -------- - ProjectionGrid - New grid instance + ProjectionGrid: New grid instance. """ sw = projection.forward(lat_range[0], lon_range[0]) ne = projection.forward(lat_range[1], lon_range[1]) @@ -732,39 +688,33 @@ def from_center( """ Create a projection grid centered at a geographic location. - Parameters: - ----------- - nx : int - Number of grid points in x direction - ny : int - Number of grid points in y direction - center_lat : float - Center latitude in degrees - center_lon : float - Center longitude in degrees - dx : float - Grid spacing in x direction in meters - dy : float - Grid spacing in y direction in meters - projection : Projectable - Projection implementation + Args: + nx (int): Number of grid points in x direction. + ny (int): Number of grid points in y direction. + center_lat (float): Center latitude in degrees. + center_lon (float): Center longitude in degrees. + dx (float): Grid spacing in x direction in meters. + dy (float): Grid spacing in y direction in meters. + projection (Projectable): Projection implementation. Returns: - -------- - ProjectionGrid - New grid instance + ProjectionGrid: New grid instance. """ center = cast(tuple[float, float], projection.forward(center_lat, center_lon)) return cls(projection, nx, ny, center, dx, dy) @property def grid_type(self) -> str: + """Grid type identifier.""" return "projection" @cached_property def _coordinates(self) -> Tuple[np.ndarray, np.ndarray]: """ Lazily compute and cache both latitude and longitude arrays. + + Returns: + Tuple[np.ndarray, np.ndarray]: Arrays of latitude and longitude coordinates. """ # Create meshgrid of coordinates y_indices, x_indices = np.meshgrid(np.arange(self.ny), np.arange(self.nx), indexing="ij") @@ -781,6 +731,9 @@ def _coordinates(self) -> Tuple[np.ndarray, np.ndarray]: def latitude(self) -> np.ndarray: # type: ignore """ Get the latitude coordinate array. + + Returns: + np.ndarray: Array of latitude coordinates. """ return self._coordinates[0] @@ -788,28 +741,32 @@ def latitude(self) -> np.ndarray: # type: ignore def longitude(self) -> np.ndarray: # type: ignore """ Get the longitude coordinate array. + + Returns: + np.ndarray: Array of longitude coordinates. """ return self._coordinates[1] @property def shape(self) -> Tuple[int, int]: + """ + Grid shape as (n_lat, n_lon). + + Returns: + Tuple[int, int]: Shape of the grid as (n_lat, n_lon). + """ return (self.ny, self.nx) def findPointXy(self, lat: float, lon: float) -> Optional[Tuple[int, int]]: """ Find grid point indices (x, y) for given lat/lon coordinates. - Parameters: - ----------- - lat : float - Latitude in degrees - lon : float - Longitude in degrees + Args: + lat (float): Latitude in degrees. + lon (float): Longitude in degrees. Returns: - -------- - tuple or None - (x, y) grid indices if point is in grid, None otherwise + Optional[Tuple[int, int]]: (x, y) grid indices if point is in grid, None otherwise. """ pos = cast(tuple[float, float], self.projection.forward(lat, lon)) x = int(round((pos[0] - self.origin[0]) / self.dx)) @@ -824,17 +781,12 @@ def getCoordinates(self, x: int, y: int) -> Tuple[float, float]: """ Get lat/lon coordinates for a given grid point indices. - Parameters: - ----------- - x : int - X index - y : int - Y index + Args: + x (int): X index. + y (int): Y index. Returns: - -------- - tuple - (latitude, longitude) coordinates + Tuple[float, float]: (latitude, longitude) coordinates. """ xcord = float(x) * self.dx + self.origin[0] ycord = float(y) * self.dy + self.origin[1] @@ -849,9 +801,7 @@ def get_true_north_direction(self) -> np.ndarray: Calculate angle towards true north for every grid point. Returns: - -------- - numpy.ndarray - Array of angles in degrees, 0 = points towards north pole + np.ndarray: Array of angles in degrees, 0 = points towards north pole. """ pos = self.projection.forward(90, 0) # North pole north_pole_x = (pos[0] - self.origin[0]) / self.dx @@ -869,21 +819,14 @@ def find_box(self, lat_min: float, lat_max: float, lon_min: float, lon_max: floa """ Find indices of grid points within a geographic bounding box. - Parameters: - ----------- - lat_min : float - Minimum latitude - lat_max : float - Maximum latitude - lon_min : float - Minimum longitude - lon_max : float - Maximum longitude + Args: + lat_min (float): Minimum latitude. + lat_max (float): Maximum latitude. + lon_min (float): Minimum longitude. + lon_max (float): Maximum longitude. Returns: - -------- - numpy.ndarray - Array of grid point indices within the box + np.ndarray: Array of grid point indices within the box. """ sw = self.findPointXy(lat_min, lon_min) se = self.findPointXy(lat_min, lon_max) @@ -912,16 +855,18 @@ def find_box(self, lat_min: float, lat_max: float, lon_min: float, lon_max: floa class ProjProjection(AbstractProjection): - """A projection that wraps a proj projection""" + """ + A projection that wraps a proj projection. + + """ def __init__(self, proj_string: str): - """Initialize with a proj string or EPSG code + """ + Initialize with a proj string or EPSG code. - Parameters - ---------- - proj_string : str - The proj string (e.g. "+proj=lcc +lat_0=50...") or - EPSG code (e.g. "EPSG:4326") + Args: + proj_string (str): The proj string (e.g. "+proj=lcc +lat_0=50...") or + EPSG code (e.g. "EPSG:4326"). """ import pyproj @@ -936,37 +881,29 @@ def __init__(self, proj_string: str): self.inverse_transformer = pyproj.Transformer.from_crs(self.crs_proj, self.crs_latlon, always_xy=True) def forward(self, latitude: CoordType, longitude: CoordType) -> ReturnUnionType: - """Transform from latitude/longitude to projection coordinates + """ + Transform from latitude/longitude to projection coordinates. - Parameters - ---------- - latitude : float - Latitude in degrees - longitude : float - Longitude in degrees + Args: + latitude (float): Latitude in degrees. + longitude (float): Longitude in degrees. - Returns - ------- - tuple[float, float] - The (x, y) coordinates in the projection + Returns: + tuple[float, float]: The (x, y) coordinates in the projection. """ x, y = self.forward_transformer.transform(longitude, latitude) return x, y def inverse(self, x: CoordType, y: CoordType) -> ReturnUnionType: - """Transform from projection coordinates to latitude/longitude - - Parameters - ---------- - x : float - X coordinate in the projection - y : float - Y coordinate in the projection - - Returns - ------- - tuple[float, float] - The (latitude, longitude) coordinates in degrees + """ + Transform from projection coordinates to latitude/longitude. + + Args: + x (float): X coordinate in the projection. + y (float): Y coordinate in the projection. + + Returns: + tuple[float, float]: The (latitude, longitude) coordinates in degrees. """ lon, lat = self.inverse_transformer.transform(x, y) return lat, lon diff --git a/python/omfiles/om_domains.py b/python/omfiles/om_domains.py index 8e71588a..034b531a 100644 --- a/python/omfiles/om_domains.py +++ b/python/omfiles/om_domains.py @@ -1,7 +1,10 @@ +"""Domains used in Open-Meteo files.""" + from typing import List import numpy as np +from omfiles._utils import EPOCH from omfiles.grids import ( AbstractGrid, ProjectionGrid, @@ -9,7 +12,6 @@ RotatedLatLonProjection, StereographicProjection, ) -from omfiles.utils import EPOCH class OmDomain: @@ -24,16 +26,11 @@ def __init__(self, name: str, grid: AbstractGrid, file_length: int, temporal_res """ Initialize a domain configuration. - Parameters: - ----------- - name : str - Name of the domain - grid : AbstractGrid - Grid implementation for this domain - file_length : int - Number of time steps in each file chunk - temporal_resolution_seconds : int, optional - Time resolution in seconds (default: 3600 = 1 hour) + Args: + name (str): Name of the domain. + grid (AbstractGrid): Grid implementation for this domain. + file_length (int): Number of time steps in each file chunk. + temporal_resolution_seconds (int, optional): Time resolution in seconds. Defaults to 3600 (1 hour). """ self.name = name self.grid = grid @@ -42,18 +39,15 @@ def __init__(self, name: str, grid: AbstractGrid, file_length: int, temporal_res def time_to_chunk_index(self, timestamp: np.datetime64) -> int: """ - Convert a timestamp to a chunk index. This depends on the file_length - and the temporal_resolution_seconds of the domain. + Convert a timestamp to a chunk index. + + This depends on the file_length and the temporal_resolution_seconds of the domain. - Parameters: - ----------- - timestamp : np.datetime64 - The timestamp to convert + Args: + timestamp (np.datetime64): The timestamp to convert. Returns: - -------- - int - The chunk index containing the timestamp + int: The chunk index containing the timestamp. """ seconds_since_epoch = (timestamp - EPOCH) / np.timedelta64(1, "s") chunk_index = int(seconds_since_epoch / (self.file_length * self.temporal_resolution_seconds)) @@ -67,16 +61,12 @@ def chunks_for_date_range( """ Find all chunk indices that contain data within the given date range. - Parameters: - ----------- - start_date : datetime - Start date for the data range - end_date : datetime - End date for the data range + Args: + start_timestamp (np.datetime64): Start timestamp for the data range. + end_timestamp (np.datetime64): End timestamp for the data range. + Returns: - -------- - List[int] - List of chunk indices containing data within the date range + List[int]: List of chunk indices containing data within the date range. """ # Get chunk indices for start and end dates start_chunk = self.time_to_chunk_index(start_timestamp) @@ -89,15 +79,11 @@ def get_chunk_time_range(self, chunk_index: int): """ Get the time range covered by a specific chunk. - Parameters: - ----------- - chunk_index : int - Index of the chunk + Args: + chunk_index (int): Index of the chunk. Returns: - -------- - np.ndarray - Array of datetime64 objects representing the time points in the chunk + np.ndarray: Array of datetime64 objects representing the time points in the chunk. """ chunk_start_seconds = chunk_index * self.file_length * self.temporal_resolution_seconds start_time = EPOCH + np.timedelta64(chunk_start_seconds, "s") diff --git a/python/omfiles/types.py b/python/omfiles/types.py index 1402df4e..904e03ed 100644 --- a/python/omfiles/types.py +++ b/python/omfiles/types.py @@ -1,3 +1,5 @@ +"""Types used throughout the library.""" + import numpy as np import numpy.typing as npt @@ -10,9 +12,7 @@ # This is from https://github.com/zarr-developers/zarr-python/blob/main/src/zarr/core/indexing.py#L38C1-L40C87 BasicSelector = Union[int, slice, EllipsisType] -"""A single index selector for an array dimension: integer, slice, or ellipsis.""" BasicSelection = Union[BasicSelector, Tuple[Union[int, slice, EllipsisType], ...]] -"""A selection for an array: either a single selector or a tuple of selectors (also used for BlockIndex).""" # Type aliases for grids for clarity FloatType = Union[float, np.floating] From 7e6dcd31470f08fc4b28cbf783cd3341a7f84827 Mon Sep 17 00:00:00 2001 From: terraputix Date: Fri, 1 Aug 2025 20:48:30 +0200 Subject: [PATCH 20/50] fix usage of _utils --- tests/test_grids.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_grids.py b/tests/test_grids.py index 121cc2f3..6a6f0fdf 100644 --- a/tests/test_grids.py +++ b/tests/test_grids.py @@ -3,6 +3,7 @@ import numpy as np import pyproj import pytest +from omfiles._utils import _normalize_longitude from omfiles.grids import ( LambertAzimuthalEqualAreaProjection, LambertConformalConicProjection, @@ -12,7 +13,6 @@ StereographicProjection, ) from omfiles.om_domains import RegularLatLonGrid -from omfiles.utils import _normalize_longitude # Fixtures for grids From 9d76639cd22931660d038dd5d4a74cf9e2b7d7bb Mon Sep 17 00:00:00 2001 From: terraputix Date: Mon, 15 Dec 2025 17:31:14 +0100 Subject: [PATCH 21/50] add min version for pyproj --- pyproject.toml | 4 ++-- uv.lock | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 6668b85e..4204b313 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -37,14 +37,14 @@ codec = [ ] xarray = ["xarray>=2023.1.0"] fsspec = ["fsspec>=2023.1.0", "s3fs>=2023.10.0"] -proj = ["pyproj"] +proj = ["pyproj>=3.1.0"] all = [ "zarr>=2.18.2", "numcodecs>=0.12.1", "xarray>=2023.1.0", "fsspec>=2023.10.0", "s3fs>=2023.1.0", - "pyproj" + "pyproj>=3.1.0" ] [dependency-groups] diff --git a/uv.lock b/uv.lock index aa190895..6a6a820b 100644 --- a/uv.lock +++ b/uv.lock @@ -1392,8 +1392,8 @@ requires-dist = [ { name = "numcodecs", marker = "extra == 'all'", specifier = ">=0.12.1" }, { name = "numcodecs", marker = "extra == 'codec'", specifier = ">=0.12.1" }, { name = "numpy", specifier = ">=1.21.0" }, - { name = "pyproj", marker = "extra == 'all'" }, - { name = "pyproj", marker = "extra == 'proj'" }, + { name = "pyproj", marker = "extra == 'all'", specifier = ">=3.1.0" }, + { name = "pyproj", marker = "extra == 'proj'", specifier = ">=3.1.0" }, { name = "s3fs", marker = "extra == 'all'", specifier = ">=2023.1.0" }, { name = "s3fs", marker = "extra == 'fsspec'", specifier = ">=2023.10.0" }, { name = "xarray", marker = "extra == 'all'", specifier = ">=2023.1.0" }, From 4a17a830bbc915f4e5dc107762bce628509bd77b Mon Sep 17 00:00:00 2001 From: terraputix Date: Mon, 12 Jan 2026 20:39:36 +0100 Subject: [PATCH 22/50] fix script and dependencies --- examples/select_by_coordinates.py | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/examples/select_by_coordinates.py b/examples/select_by_coordinates.py index fe059b18..fb0dacbf 100644 --- a/examples/select_by_coordinates.py +++ b/examples/select_by_coordinates.py @@ -1,3 +1,17 @@ +#!/usr/bin/env -S uv run --script +# +# /// script +# requires-python = ">=3.12" +# dependencies = [ +# "omfiles @ git+https://github.com/open-meteo/python-omfiles@be3f68870b9444cb01a6ccfcaa122fe283d485c9", +# "fsspec>=2025.7.0", +# "s3fs", +# "xarray", +# "matplotlib", +# "cartopy", +# ] +# /// + """ Example showing how to select data from multiple domains in Open-Meteo files stored in S3. @@ -31,7 +45,7 @@ import xarray as xr from fsspec.implementations.cache_mapper import BasenameCacheMapper from fsspec.implementations.cached import CachingFileSystem -from omfiles import OmFilePyReader +from omfiles import OmFileReader from omfiles.om_domains import DOMAINS from s3fs import S3FileSystem from xarray import Dataset @@ -84,7 +98,7 @@ def load_chunk_data( return np.array([], dtype="datetime64[s]"), np.array([], dtype=float) # Create reader and read data of interest - with OmFilePyReader.from_fsspec(fs, s3_path) as reader: + with OmFileReader.from_fsspec(fs, s3_path) as reader: indices = np.where(time_mask)[0] time_slice = slice(indices[0], indices[-1] + 1) # +1 to include the end data = reader[y, x, time_slice] From 911ec4cbe7be35dc8e0050658efbce16af0f7f66 Mon Sep 17 00:00:00 2001 From: terraputix Date: Tue, 13 Jan 2026 14:39:52 +0100 Subject: [PATCH 23/50] wip: make domains and grids compatible --- python/omfiles/om_domains.py | 144 +------------------------- python/omfiles/om_grid.py | 192 +++++++++++++++++++++++++++++++++++ 2 files changed, 194 insertions(+), 142 deletions(-) create mode 100644 python/omfiles/om_grid.py diff --git a/python/omfiles/om_domains.py b/python/omfiles/om_domains.py index 034b531a..d03f6c0b 100644 --- a/python/omfiles/om_domains.py +++ b/python/omfiles/om_domains.py @@ -5,35 +5,23 @@ import numpy as np from omfiles._utils import EPOCH -from omfiles.grids import ( - AbstractGrid, - ProjectionGrid, - RegularLatLonGrid, - RotatedLatLonProjection, - StereographicProjection, -) class OmDomain: """ Class representing a domain configuration for a weather model. - - This class provides metadata and configuration for different - weather model grids used in Open-Meteo. """ - def __init__(self, name: str, grid: AbstractGrid, file_length: int, temporal_resolution_seconds: int = 3600): + def __init__(self, name: str, file_length: int, temporal_resolution_seconds: int = 3600): """ Initialize a domain configuration. Args: name (str): Name of the domain. - grid (AbstractGrid): Grid implementation for this domain. file_length (int): Number of time steps in each file chunk. temporal_resolution_seconds (int, optional): Time resolution in seconds. Defaults to 3600 (1 hour). """ self.name = name - self.grid = grid self.file_length = file_length self.temporal_resolution_seconds = temporal_resolution_seconds @@ -95,228 +83,100 @@ def get_chunk_time_range(self, chunk_index: int): return timestamps -# - MARK: Create grid instances for supported domains - -# DWD ICON global is regularized during download to nx: 2879, ny: 1441 points -# https://github.com/open-meteo/open-meteo/blob/522917b1d6e72a7e6b7d4ae7dfb49b0c556a6992/Sources/App/Icon/Icon.swift#L146 -_dwd_icon_grid = RegularLatLonGrid( - lat_start=-90, lat_steps=1441, lat_step_size=0.125, lon_start=-180, lon_steps=2879, lon_step_size=0.125 -) - -# DWD ICON EU is regularized during download to nx: 1377, ny: 657 points -# https://github.com/open-meteo/open-meteo/blob/522917b1d6e72a7e6b7d4ae7dfb49b0c556a6992/Sources/App/Icon/Icon.swift#L148 -_dwd_icon_eu_grid = RegularLatLonGrid( - lat_start=29.5, lat_steps=657, lat_step_size=0.0625, lon_start=-23.5, lon_steps=1377, lon_step_size=0.0625 -) - -# DWD ICON D2 is regularized during download to nx: 1215, ny: 746 points -# https://github.com/open-meteo/open-meteo/blob/522917b1d6e72a7e6b7d4ae7dfb49b0c556a6992/Sources/App/Icon/Icon.swift#L150 -_dwd_icon_d2_grid = RegularLatLonGrid( - lat_start=43.18, lat_steps=746, lat_step_size=0.02, lon_start=-3.94, lon_steps=1215, lon_step_size=0.02 -) - -# DWD ICON EPS global is regularized during download to nx: 1439, ny: 721 points -# https://github.com/open-meteo/open-meteo/blob/522917b1d6e72a7e6b7d4ae7dfb49b0c556a6992/Sources/App/Icon/Icon.swift#L153 -_dwd_icon_eps_grid = RegularLatLonGrid( - lat_start=-90, lat_steps=721, lat_step_size=0.25, lon_start=-180, lon_steps=1439, lon_step_size=0.25 -) - -# DWD ICON EU EPS is regularized during download to nx: 689, ny: 329 points -# https://github.com/open-meteo/open-meteo/blob/522917b1d6e72a7e6b7d4ae7dfb49b0c556a6992/Sources/App/Icon/Icon.swift#L156 -_dwd_icon_eu_eps_grid = RegularLatLonGrid( - lat_start=29.5, lat_steps=329, lat_step_size=0.125, lon_start=-23.5, lon_steps=689, lon_step_size=0.125 -) - -# DWD ICON D2 EPS is regularized during download to nx: 1214, ny: 745 points -# https://github.com/open-meteo/open-meteo/blob/522917b1d6e72a7e6b7d4ae7dfb49b0c556a6992/Sources/App/Icon/Icon.swift#L160 -_dwd_icon_d2_eps_grid = RegularLatLonGrid( - lat_start=43.18, lat_steps=745, lat_step_size=0.02, lon_start=-3.94, lon_steps=1214, lon_step_size=0.02 -) - -# ECMWF IFS grid is a regular global lat/lon grid, nx: 1440, ny: 721 points -# https://github.com/open-meteo/open-meteo/blob/522917b1d6e72a7e6b7d4ae7dfb49b0c556a6992/Sources/App/Ecmwf/EcmwfDomain.swift#L105 -_ecmwf_ifs025_grid = RegularLatLonGrid( - lat_start=-90, - lat_steps=721, - lat_step_size=360 / 1440, - lon_start=-180, - lon_steps=1440, - lon_step_size=180 / (721 - 1), -) - -# Méteo-France ARPEGE Europe grid: nx: 741, ny: 521 points -# https://github.com/open-meteo/open-meteo/blob/522917b1d6e72a7e6b7d4ae7dfb49b0c556a6992/Sources/App/MeteoFrance/MeteoFranceDomain.swift#L341 -_meteofrance_arpege_europe_grid = RegularLatLonGrid( - lat_start=20, lat_steps=521, lat_step_size=0.1, lon_start=-32, lon_steps=741, lon_step_size=0.1 -) - -# Méteo-France ARPEGE World grid: nx: 1440, ny: 721 points -# https://github.com/open-meteo/open-meteo/blob/522917b1d6e72a7e6b7d4ae7dfb49b0c556a6992/Sources/App/MeteoFrance/MeteoFranceDomain.swift#L343 -_meteofrance_arpege_world025_grid = RegularLatLonGrid( - lat_start=-90, lat_steps=721, lat_step_size=0.25, lon_start=-180, lon_steps=1440, lon_step_size=0.25 -) - -# Méteo-France AROME France grid: nx: 1121, ny: 717 points -# https://github.com/open-meteo/open-meteo/blob/522917b1d6e72a7e6b7d4ae7dfb49b0c556a6992/Sources/App/MeteoFrance/MeteoFranceDomain.swift#L345 -_meteofrance_arome_france0025_grid = RegularLatLonGrid( - lat_start=37.5, lat_steps=717, lat_step_size=0.025, lon_start=-12.0, lon_steps=1121, lon_step_size=0.025 -) - -# Méteo-France AROME France HD grid: nx: 2801, ny: 1791 points -# https://github.com/open-meteo/open-meteo/blob/522917b1d6e72a7e6b7d4ae7dfb49b0c556a6992/Sources/App/MeteoFrance/MeteoFranceDomain.swift#L347 -_meteofrance_arome_france_hd_grid = RegularLatLonGrid( - lat_start=37.5, lat_steps=1791, lat_step_size=0.01, lon_start=-12.0, lon_steps=2801, lon_step_size=0.01 -) - -# GEM Global grid: nx: 2400, ny: 1201 points -# https://github.com/open-meteo/open-meteo/blob/522917b1d6e72a7e6b7d4ae7dfb49b0c556a6992/Sources/App/Gem/GemDomain.swift#L139 -_gem_global_grid = RegularLatLonGrid( - lat_start=-90, lat_steps=1201, lat_step_size=0.15, lon_start=-180, lon_steps=2400, lon_step_size=0.15 -) - -# GEM Regional grid: Uses Stereographic projection -# https://github.com/open-meteo/open-meteo/blob/522917b1d6e72a7e6b7d4ae7dfb49b0c556a6992/Sources/App/Gem/GemDomain.swift#L141 -_gem_regional_projection = StereographicProjection(latitude=90, longitude=249, radius=6371229) -_gem_regional_grid = ProjectionGrid.from_bounds( - nx=935, - ny=824, - lat_range=(18.14503, 45.405453), - lon_range=(217.10745, 349.8256), - projection=_gem_regional_projection, -) - -# GEM HRDPS Continental grid: Uses RotatedLatLon projection -# https://github.com/open-meteo/open-meteo/blob/522917b1d6e72a7e6b7d4ae7dfb49b0c556a6992/Sources/App/Gem/GemDomain.swift#L143 -_gem_hrdps_projection = RotatedLatLonProjection(lat_origin=-36.0885, lon_origin=245.305) -_gem_hrdps_grid = ProjectionGrid.from_bounds( - nx=2540, - ny=1290, - lat_range=(39.626034, 47.876457), - lon_range=(-133.62952, -40.708557), - projection=_gem_hrdps_projection, -) - -# GEM Global Ensemble grid: nx: 720, ny: 361 points -# https://github.com/open-meteo/open-meteo/blob/522917b1d6e72a7e6b7d4ae7dfb49b0c556a6992/Sources/App/Gem/GemDomain.swift#L145 -_gem_global_ensemble_grid = RegularLatLonGrid( - lat_start=-90, lat_steps=361, lat_step_size=0.5, lon_start=-180, lon_steps=720, lon_step_size=0.5 -) - DOMAINS: dict[str, OmDomain] = { "cmc_gem_gdps": OmDomain( name="cmc_gem_gdps", - grid=_gem_global_grid, file_length=110, # From GemDomain.omFileLength for gem_global case temporal_resolution_seconds=3600 * 3, # 3-hourly data ), "cmc_gem_rdps": OmDomain( name="cmc_gem_rdps", - grid=_gem_regional_grid, file_length=78 + 36, # From GemDomain.omFileLength for gem_regional case temporal_resolution_seconds=3600, # Hourly data ), "cmc_gem_hrdps": OmDomain( name="cmc_gem_hrdps", - grid=_gem_hrdps_grid, file_length=48 + 36, # From GemDomain.omFileLength for gem_hrdps_continental case temporal_resolution_seconds=3600, # Hourly data ), "cmc_gem_geps": OmDomain( name="cmc_gem_geps", - grid=_gem_global_ensemble_grid, file_length=384 // 3 + 48 // 3, # From GemDomain.omFileLength for gem_global_ensemble case temporal_resolution_seconds=3600 * 3, # 3-hourly data ), "dwd_icon": OmDomain( name="dwd_icon", - grid=_dwd_icon_grid, file_length=180 + 1 + 3 * 24, # From IconDomains.omFileLength for icon case temporal_resolution_seconds=3600, ), "dwd_icon_eu": OmDomain( name="dwd_icon_eu", - grid=_dwd_icon_eu_grid, file_length=120 + 1 + 3 * 24, # From IconDomains.omFileLength for iconEu case temporal_resolution_seconds=3600, ), "dwd_icon_d2": OmDomain( name="dwd_icon_d2", - grid=_dwd_icon_d2_grid, file_length=48 + 1 + 3 * 24, # From IconDomains.omFileLength for iconD2 case temporal_resolution_seconds=3600, ), "dwd_icon_d2_15min": OmDomain( name="dwd_icon_d2_15min", - grid=_dwd_icon_d2_grid, # Uses same grid as dwd_icon_d2 file_length=48 * 4 + 3 * 24, # From IconDomains.omFileLength for iconD2_15min case temporal_resolution_seconds=3600 // 4, # 15 minutes = 3600/4 ), "dwd_icon_eps": OmDomain( name="dwd_icon_eps", - grid=_dwd_icon_eps_grid, file_length=180 + 1 + 3 * 24, # Same as non-eps version temporal_resolution_seconds=3600, ), "dwd_icon_eu_eps": OmDomain( name="dwd_icon_eu_eps", - grid=_dwd_icon_eu_eps_grid, file_length=120 + 1 + 3 * 24, # Same as non-eps version temporal_resolution_seconds=3600, ), "dwd_icon_d2_eps": OmDomain( name="dwd_icon_d2_eps", - grid=_dwd_icon_d2_eps_grid, file_length=48 + 1 + 3 * 24, # Same as non-eps version temporal_resolution_seconds=3600, ), - "ecmwf_ifs025": OmDomain( - name="ecmwf_ifs025", grid=_ecmwf_ifs025_grid, file_length=104, temporal_resolution_seconds=3600 * 3 - ), + "ecmwf_ifs025": OmDomain(name="ecmwf_ifs025", file_length=104, temporal_resolution_seconds=3600 * 3), "meteofrance_arpege_europe": OmDomain( name="meteofrance_arpege_europe", - grid=_meteofrance_arpege_europe_grid, file_length=114 + 3 * 24, # From MeteoFranceDomain.omFileLength for arpege_europe case temporal_resolution_seconds=3600, ), "meteofrance_arpege_world025": OmDomain( name="meteofrance_arpege_world025", - grid=_meteofrance_arpege_world025_grid, file_length=114 + 4 * 24, # From MeteoFranceDomain.omFileLength for arpege_world case temporal_resolution_seconds=3600, ), "meteofrance_arome_france0025": OmDomain( name="meteofrance_arome_france0025", - grid=_meteofrance_arome_france0025_grid, file_length=36 + 3 * 24, # From MeteoFranceDomain.omFileLength for arome_france case temporal_resolution_seconds=3600, ), "meteofrance_arome_france_hd": OmDomain( name="meteofrance_arome_france_hd", - grid=_meteofrance_arome_france_hd_grid, file_length=36 + 3 * 24, # From MeteoFranceDomain.omFileLength for arome_france_hd case temporal_resolution_seconds=3600, ), "meteofrance_arome_france0025_15min": OmDomain( name="meteofrance_arome_france0025_15min", - grid=_meteofrance_arome_france0025_grid, # Using the same grid as non-15min version file_length=24 * 2, # From MeteoFranceDomain.omFileLength for arome_france_15min case temporal_resolution_seconds=900, ), "meteofrance_arome_france_hd_15min": OmDomain( name="meteofrance_arome_france_hd_15min", - grid=_meteofrance_arome_france_hd_grid, # Using the same grid as non-15min version file_length=24 * 2, # From MeteoFranceDomain.omFileLength for arome_france_hd_15min case temporal_resolution_seconds=900, ), "meteofrance_arpege_europe_probabilities": OmDomain( name="meteofrance_arpege_europe_probabilities", - grid=_meteofrance_arpege_europe_grid, # Using the same grid as non-probabilities version file_length=(102 + 4 * 24) // 3, # From MeteoFranceDomain.omFileLength for arpege_europe_probabilities case temporal_resolution_seconds=3600 * 3, ), "meteofrance_arpege_world025_probabilities": OmDomain( name="meteofrance_arpege_world025_probabilities", - grid=_meteofrance_arpege_world025_grid, # Using the same grid as non-probabilities version file_length=(102 + 4 * 24) // 3, # From MeteoFranceDomain.omFileLength for arpege_world_probabilities case temporal_resolution_seconds=3600 * 3, ), diff --git a/python/omfiles/om_grid.py b/python/omfiles/om_grid.py new file mode 100644 index 00000000..2b69abe8 --- /dev/null +++ b/python/omfiles/om_grid.py @@ -0,0 +1,192 @@ +"""An OmGrid provides utilities to transform between geographic coordinates and grid indices.""" + +from dataclasses import dataclass +from typing import Optional, Tuple + +import numpy as np +import numpy.typing as npt +from pyproj import CRS, Transformer + + +@dataclass +class OmMetaJson: + """Class to decode Open-Meteo metadata JSON files.""" + + # chunk_time_length: int # Number of time steps per chunk (file_length) + crs_wkt: str # Coordinate Reference System in Well-Known Text format + # data_end_time: int # Unix timestamp for when data ends + # last_run_availability_time: int # Unix timestamp for last availability + # last_run_initialisation_time: int # Unix timestamp for last initialization + # last_run_modification_time: int # Unix timestamp for last modification + # temporal_resolution_seconds: int # Time resolution in seconds + # update_interval_seconds: int # How often data is updated + + @classmethod + def from_dict(cls, data: dict) -> "OmMetaJson": + """Create instance from dictionary.""" + return cls(**data) + + +class OmGrid: + """Latitude and longitude grid based on cartopy and wkt projection strings.""" + + # lon_grid: npt.NDArray[np.float64] + # lat_grid: npt.NDArray[np.float64] + + def __init__(self, crs_wkt: str, shape: Tuple[int, int]): + """ + Initialize grid from WKT projection string and data shape. + + Args: + crs_wkt: Coordinate Reference System in Well-Known Text format + shape: Grid shape as (ny, nx) - number of points in y and x directions + """ + self.crs = CRS.from_wkt(crs_wkt) + self.wgs84 = CRS.from_epsg(4326) + self.ny, self.nx = shape + + # TODO: Special case for gaussian grids! + + # Transformers for coordinate conversions + self.to_projection = Transformer.from_crs(self.wgs84, self.crs, always_xy=True) + self.to_wgs84 = Transformer.from_crs(self.crs, self.wgs84, always_xy=True) + + # Get projection bounds from area of use + area = self.crs.area_of_use + if area is None: + raise ValueError("CRS does not have an area of use defined") + + # Transform WGS84 bounds to projection space + xmin, ymin = self.to_projection.transform(area.west, area.south) + xmax, ymax = self.to_projection.transform(area.east, area.north) + + self.bounds = (xmin, xmax, ymin, ymax) + self.origin = (xmin, ymin) + + if self.nx <= 1 or self.ny <= 1: + raise ValueError("Invalid grid shape") + + # Calculate grid spacing + self.dx = (xmax - xmin) / (self.nx - 1) + self.dy = (ymax - ymin) / (self.ny - 1) + + @classmethod + def from_meta_json(cls, meta: OmMetaJson, shape: Tuple[int, int]) -> "OmGrid": + """ + Create grid from metadata JSON. + + Args: + meta: Metadata containing CRS WKT string + shape: Grid shape as (ny, nx) + + Returns: + OmGrid instance + """ + return cls(meta.crs_wkt, shape) + + @property + def shape(self) -> Tuple[int, int]: + """Grid shape as (ny, nx).""" + return (self.ny, self.nx) + + @property + def latitude(self) -> npt.NDArray[np.float64]: + """ + Get 2D array of latitude coordinates for all grid points. + + Returns: + Array of shape (ny, nx) with latitude values + """ + if not hasattr(self, "_latitude"): + self._compute_coordinates() + return self._latitude + + @property + def longitude(self) -> npt.NDArray[np.float64]: + """ + Get 2D array of longitude coordinates for all grid points. + + Returns: + Array of shape (ny, nx) with longitude values + """ + if not hasattr(self, "_longitude"): + self._compute_coordinates() + return self._longitude + + def _compute_coordinates(self) -> None: + """Compute and cache latitude/longitude arrays for all grid points.""" + # Create meshgrid of projection coordinates + x_coords = np.linspace(self.origin[0], self.origin[0] + self.dx * (self.nx - 1), self.nx) + y_coords = np.linspace(self.origin[1], self.origin[1] + self.dy * (self.ny - 1), self.ny) + x_grid, y_grid = np.meshgrid(x_coords, y_coords) + + # Transform to WGS84 + lon_grid, lat_grid = self.to_wgs84.transform(x_grid, y_grid) + + self._longitude = lon_grid + self._latitude = lat_grid + + def find_point_xy(self, lat: float, lon: float) -> Optional[Tuple[int, int]]: + """ + Find grid point indices (x, y) for given lat/lon coordinates. + + Args: + lat: Latitude in degrees + lon: Longitude in degrees + + Returns: + (x, y) grid indices if point is in grid bounds, None otherwise + """ + # Transform to projection coordinates + x_proj, y_proj = self.to_projection.transform(lon, lat) + + # Check if point is within bounds + if not (self.bounds[0] <= x_proj <= self.bounds[1] and self.bounds[2] <= y_proj <= self.bounds[3]): + return None + + # Calculate grid indices + x_idx = int(round((x_proj - self.origin[0]) / self.dx)) + y_idx = int(round((y_proj - self.origin[1]) / self.dy)) + + # Validate indices + if not (0 <= x_idx < self.nx and 0 <= y_idx < self.ny): + return None + + return (x_idx, y_idx) + + def get_coordinates(self, x: int, y: int) -> Tuple[float, float]: + """ + Get lat/lon coordinates for given grid point indices. + + Args: + x: Grid x index + y: Grid y index + + Returns: + (latitude, longitude) in degrees + """ + # Calculate projection coordinates + x_proj = self.origin[0] + x * self.dx + y_proj = self.origin[1] + y * self.dy + + # Transform to WGS84 + lon, lat = self.to_wgs84.transform(x_proj, y_proj) + + return (float(lat), float(lon)) + + def get_meshgrid(self) -> Tuple[npt.NDArray[np.float64], npt.NDArray[np.float64]]: + """ + Get meshgrid of projection coordinates. + + Useful for plotting with matplotlib/cartopy. + + Returns: + (lon_grid, lat_grid) arrays of shape (ny, nx) in projection coordinates + """ + x_coords: npt.NDArray[np.float64] = np.linspace( + self.origin[0], self.origin[0] + self.dx * (self.nx - 1), self.nx + ) + y_coords: npt.NDArray[np.float64] = np.linspace( + self.origin[1], self.origin[1] + self.dy * (self.ny - 1), self.ny + ) + return np.meshgrid(x_coords, y_coords) From 8e318030d353ea6a6a837ebd9f3d1ac8c1bc2054 Mon Sep 17 00:00:00 2001 From: terraputix Date: Tue, 13 Jan 2026 15:34:02 +0100 Subject: [PATCH 24/50] get rid of OmDomain for now --- examples/select_by_coordinates.py | 57 ++++++--- python/omfiles/om_domains.py | 190 ------------------------------ python/omfiles/om_grid.py | 83 ++++++++++++- 3 files changed, 118 insertions(+), 212 deletions(-) delete mode 100644 python/omfiles/om_domains.py diff --git a/examples/select_by_coordinates.py b/examples/select_by_coordinates.py index fb0dacbf..af37a633 100644 --- a/examples/select_by_coordinates.py +++ b/examples/select_by_coordinates.py @@ -3,12 +3,11 @@ # /// script # requires-python = ">=3.12" # dependencies = [ -# "omfiles @ git+https://github.com/open-meteo/python-omfiles@be3f68870b9444cb01a6ccfcaa122fe283d485c9", +# "omfiles[proj] @ /home/fred/dev/terraputix/python-omfiles", # "fsspec>=2025.7.0", # "s3fs", # "xarray", # "matplotlib", -# "cartopy", # ] # /// @@ -36,6 +35,7 @@ - omfiles """ +import json from datetime import datetime from typing import Tuple @@ -46,7 +46,7 @@ from fsspec.implementations.cache_mapper import BasenameCacheMapper from fsspec.implementations.cached import CachingFileSystem from omfiles import OmFileReader -from omfiles.om_domains import DOMAINS +from omfiles.om_grid import OmGrid, OmMetaJson from s3fs import S3FileSystem from xarray import Dataset @@ -63,6 +63,21 @@ ) +def get_domain_info(domain_name: str, fs: fsspec.AbstractFileSystem) -> OmMetaJson: + meta_json_path = f"openmeteo/data/{domain_name}/static/meta.json" + meta_dict = json.loads(fs.cat_file(meta_json_path)) + return OmMetaJson.from_dict(meta_dict) + + +def load_variable_dimensions( + chunk_index: int, domain_name: str, variable_name: str, fs: fsspec.AbstractFileSystem +) -> Tuple[int, int, int]: + s3_path = f"openmeteo/data/{domain_name}/{variable_name}/chunk_{chunk_index}.om" + with OmFileReader.from_fsspec(fs, s3_path) as reader: + return reader.shape + raise ValueError(f"Failed to load variable dimensions for chunk {chunk_index}") + + def load_chunk_data( chunk_index: int, domain_name: str, @@ -71,6 +86,7 @@ def load_chunk_data( fs: fsspec.AbstractFileSystem, start_date: np.datetime64, end_date: np.datetime64, + meta: OmMetaJson, ): """ Load data for a specific chunk and grid coordinates. @@ -87,12 +103,11 @@ def load_chunk_data( Returns: Tuple[np.ndarray, np.ndarray]: Tuple containing (time_array, data_array). """ - domain = DOMAINS[domain_name] x, y = grid_coords - s3_path = f"openmeteo/data/{domain.name}/{variable_name}/chunk_{chunk_index}.om" + s3_path = f"openmeteo/data/{domain_name}/{variable_name}/chunk_{chunk_index}.om" # Generate time array and check if any times are in our range - chunk_times = domain.get_chunk_time_range(chunk_index) + chunk_times = meta.get_chunk_time_range(chunk_index) time_mask = (chunk_times >= start_date) & (chunk_times <= end_date) if not np.any(time_mask): return np.array([], dtype="datetime64[s]"), np.array([], dtype=float) @@ -129,14 +144,22 @@ def get_data_for_coordinates( Returns: xr.Dataset: Dataset containing the requested variable at the specified location. """ - # Get the domain configuration - if domain_name not in DOMAINS: - raise ValueError(f"Unknown domain: {domain_name}. Available domains: {list(DOMAINS.keys())}") + meta = get_domain_info(domain_name, FS) + print("domain info: ", meta) + + start_timestamp = np.datetime64(start_date) + end_timestamp = np.datetime64(end_date) + + # Find all chunks needed for this date range + chunk_indices = meta.chunks_for_date_range(start_timestamp, end_timestamp) + print(f"Need to fetch {len(chunk_indices)} chunks: {chunk_indices}") - domain = DOMAINS[domain_name] + # get dimensions of the variable + num_y, num_x, num_t = load_variable_dimensions(chunk_indices[0], domain_name, variable_name, FS) + grid = OmGrid(meta.crs_wkt, (num_y, num_x)) # Find grid coordinates for geographical coordinates - grid_point = domain.grid.findPointXy(lat, lon) + grid_point = grid.find_point_xy(lat, lon) if grid_point is None: raise ValueError(f"Coordinates ({lat}, {lon}) not found in grid of {domain_name}") @@ -148,7 +171,7 @@ def get_data_for_coordinates( end_timestamp = np.datetime64(end_date) # Find all chunks needed for this date range - chunk_indices = domain.chunks_for_date_range(start_timestamp, end_timestamp) + chunk_indices = meta.chunks_for_date_range(start_timestamp, end_timestamp) print(f"Need to fetch {len(chunk_indices)} chunks: {chunk_indices}") # Load data from all chunks @@ -156,7 +179,9 @@ def get_data_for_coordinates( all_data = [] for chunk_idx in chunk_indices: - times, data = load_chunk_data(chunk_idx, domain_name, variable_name, (x, y), FS, start_timestamp, end_timestamp) + times, data = load_chunk_data( + chunk_idx, domain_name, variable_name, (x, y), FS, start_timestamp, end_timestamp, meta + ) if len(times) > 0: all_times.append(times) all_data.append(data) @@ -216,9 +241,9 @@ def get_data_for_coordinates( "meteofrance_arome_france0025": "Météo-France AROME (France)", "meteofrance_arome_france_hd": "Météo-France AROME HD (France)", "meteofrance_arome_france_hd_15min": "Météo-France AROME HD 15min (France)", - "gem_global": "CMC GEM GDPS (Global)", - "gem_regional": "CMC GEM RDPS (Regional)", - "gem_hrdps_continental": "CMC GEM HRDPS (Continental)", + "cmc_gem_gdps": "CMC GEM GDPS (Global)", + "cmc_gem_rdps": "CMC GEM RDPS (Regional)", + "cmc_gem_hrdps": "CMC GEM HRDPS (Continental)", } # Collect data from each domain diff --git a/python/omfiles/om_domains.py b/python/omfiles/om_domains.py deleted file mode 100644 index d03f6c0b..00000000 --- a/python/omfiles/om_domains.py +++ /dev/null @@ -1,190 +0,0 @@ -"""Domains used in Open-Meteo files.""" - -from typing import List - -import numpy as np - -from omfiles._utils import EPOCH - - -class OmDomain: - """ - Class representing a domain configuration for a weather model. - """ - - def __init__(self, name: str, file_length: int, temporal_resolution_seconds: int = 3600): - """ - Initialize a domain configuration. - - Args: - name (str): Name of the domain. - file_length (int): Number of time steps in each file chunk. - temporal_resolution_seconds (int, optional): Time resolution in seconds. Defaults to 3600 (1 hour). - """ - self.name = name - self.file_length = file_length - self.temporal_resolution_seconds = temporal_resolution_seconds - - def time_to_chunk_index(self, timestamp: np.datetime64) -> int: - """ - Convert a timestamp to a chunk index. - - This depends on the file_length and the temporal_resolution_seconds of the domain. - - Args: - timestamp (np.datetime64): The timestamp to convert. - - Returns: - int: The chunk index containing the timestamp. - """ - seconds_since_epoch = (timestamp - EPOCH) / np.timedelta64(1, "s") - chunk_index = int(seconds_since_epoch / (self.file_length * self.temporal_resolution_seconds)) - return chunk_index - - def chunks_for_date_range( - self, - start_timestamp: np.datetime64, - end_timestamp: np.datetime64, - ) -> List[int]: - """ - Find all chunk indices that contain data within the given date range. - - Args: - start_timestamp (np.datetime64): Start timestamp for the data range. - end_timestamp (np.datetime64): End timestamp for the data range. - - Returns: - List[int]: List of chunk indices containing data within the date range. - """ - # Get chunk indices for start and end dates - start_chunk = self.time_to_chunk_index(start_timestamp) - end_chunk = self.time_to_chunk_index(end_timestamp) - - # Generate list of all chunks between start and end (inclusive) - return list(range(start_chunk, end_chunk + 1)) - - def get_chunk_time_range(self, chunk_index: int): - """ - Get the time range covered by a specific chunk. - - Args: - chunk_index (int): Index of the chunk. - - Returns: - np.ndarray: Array of datetime64 objects representing the time points in the chunk. - """ - chunk_start_seconds = chunk_index * self.file_length * self.temporal_resolution_seconds - start_time = EPOCH + np.timedelta64(chunk_start_seconds, "s") - - # Generate timestamps at regular intervals from the start time - time_delta = np.timedelta64(self.temporal_resolution_seconds, "s") - # Note: better type inference via list comprehension here - timestamps = np.array([start_time + i * time_delta for i in range(self.file_length)]) - return timestamps - - -DOMAINS: dict[str, OmDomain] = { - "cmc_gem_gdps": OmDomain( - name="cmc_gem_gdps", - file_length=110, # From GemDomain.omFileLength for gem_global case - temporal_resolution_seconds=3600 * 3, # 3-hourly data - ), - "cmc_gem_rdps": OmDomain( - name="cmc_gem_rdps", - file_length=78 + 36, # From GemDomain.omFileLength for gem_regional case - temporal_resolution_seconds=3600, # Hourly data - ), - "cmc_gem_hrdps": OmDomain( - name="cmc_gem_hrdps", - file_length=48 + 36, # From GemDomain.omFileLength for gem_hrdps_continental case - temporal_resolution_seconds=3600, # Hourly data - ), - "cmc_gem_geps": OmDomain( - name="cmc_gem_geps", - file_length=384 // 3 + 48 // 3, # From GemDomain.omFileLength for gem_global_ensemble case - temporal_resolution_seconds=3600 * 3, # 3-hourly data - ), - "dwd_icon": OmDomain( - name="dwd_icon", - file_length=180 + 1 + 3 * 24, # From IconDomains.omFileLength for icon case - temporal_resolution_seconds=3600, - ), - "dwd_icon_eu": OmDomain( - name="dwd_icon_eu", - file_length=120 + 1 + 3 * 24, # From IconDomains.omFileLength for iconEu case - temporal_resolution_seconds=3600, - ), - "dwd_icon_d2": OmDomain( - name="dwd_icon_d2", - file_length=48 + 1 + 3 * 24, # From IconDomains.omFileLength for iconD2 case - temporal_resolution_seconds=3600, - ), - "dwd_icon_d2_15min": OmDomain( - name="dwd_icon_d2_15min", - file_length=48 * 4 + 3 * 24, # From IconDomains.omFileLength for iconD2_15min case - temporal_resolution_seconds=3600 // 4, # 15 minutes = 3600/4 - ), - "dwd_icon_eps": OmDomain( - name="dwd_icon_eps", - file_length=180 + 1 + 3 * 24, # Same as non-eps version - temporal_resolution_seconds=3600, - ), - "dwd_icon_eu_eps": OmDomain( - name="dwd_icon_eu_eps", - file_length=120 + 1 + 3 * 24, # Same as non-eps version - temporal_resolution_seconds=3600, - ), - "dwd_icon_d2_eps": OmDomain( - name="dwd_icon_d2_eps", - file_length=48 + 1 + 3 * 24, # Same as non-eps version - temporal_resolution_seconds=3600, - ), - "ecmwf_ifs025": OmDomain(name="ecmwf_ifs025", file_length=104, temporal_resolution_seconds=3600 * 3), - "meteofrance_arpege_europe": OmDomain( - name="meteofrance_arpege_europe", - file_length=114 + 3 * 24, # From MeteoFranceDomain.omFileLength for arpege_europe case - temporal_resolution_seconds=3600, - ), - "meteofrance_arpege_world025": OmDomain( - name="meteofrance_arpege_world025", - file_length=114 + 4 * 24, # From MeteoFranceDomain.omFileLength for arpege_world case - temporal_resolution_seconds=3600, - ), - "meteofrance_arome_france0025": OmDomain( - name="meteofrance_arome_france0025", - file_length=36 + 3 * 24, # From MeteoFranceDomain.omFileLength for arome_france case - temporal_resolution_seconds=3600, - ), - "meteofrance_arome_france_hd": OmDomain( - name="meteofrance_arome_france_hd", - file_length=36 + 3 * 24, # From MeteoFranceDomain.omFileLength for arome_france_hd case - temporal_resolution_seconds=3600, - ), - "meteofrance_arome_france0025_15min": OmDomain( - name="meteofrance_arome_france0025_15min", - file_length=24 * 2, # From MeteoFranceDomain.omFileLength for arome_france_15min case - temporal_resolution_seconds=900, - ), - "meteofrance_arome_france_hd_15min": OmDomain( - name="meteofrance_arome_france_hd_15min", - file_length=24 * 2, # From MeteoFranceDomain.omFileLength for arome_france_hd_15min case - temporal_resolution_seconds=900, - ), - "meteofrance_arpege_europe_probabilities": OmDomain( - name="meteofrance_arpege_europe_probabilities", - file_length=(102 + 4 * 24) // 3, # From MeteoFranceDomain.omFileLength for arpege_europe_probabilities case - temporal_resolution_seconds=3600 * 3, - ), - "meteofrance_arpege_world025_probabilities": OmDomain( - name="meteofrance_arpege_world025_probabilities", - file_length=(102 + 4 * 24) // 3, # From MeteoFranceDomain.omFileLength for arpege_world_probabilities case - temporal_resolution_seconds=3600 * 3, - ), - # Additional domains can be added here -} - -# Domain aliases to match the names in GemDomain.swift -DOMAINS["gem_global"] = DOMAINS["cmc_gem_gdps"] -DOMAINS["gem_regional"] = DOMAINS["cmc_gem_rdps"] -DOMAINS["gem_hrdps_continental"] = DOMAINS["cmc_gem_hrdps"] -DOMAINS["gem_global_ensemble"] = DOMAINS["cmc_gem_geps"] diff --git a/python/omfiles/om_grid.py b/python/omfiles/om_grid.py index 2b69abe8..898c6d0a 100644 --- a/python/omfiles/om_grid.py +++ b/python/omfiles/om_grid.py @@ -1,30 +1,101 @@ """An OmGrid provides utilities to transform between geographic coordinates and grid indices.""" -from dataclasses import dataclass -from typing import Optional, Tuple +import json +from dataclasses import dataclass, fields +from typing import List, Optional, Tuple import numpy as np import numpy.typing as npt from pyproj import CRS, Transformer +from omfiles._utils import EPOCH + @dataclass class OmMetaJson: """Class to decode Open-Meteo metadata JSON files.""" - # chunk_time_length: int # Number of time steps per chunk (file_length) + chunk_time_length: int # Number of time steps per chunk (file_length) crs_wkt: str # Coordinate Reference System in Well-Known Text format # data_end_time: int # Unix timestamp for when data ends # last_run_availability_time: int # Unix timestamp for last availability # last_run_initialisation_time: int # Unix timestamp for last initialization # last_run_modification_time: int # Unix timestamp for last modification - # temporal_resolution_seconds: int # Time resolution in seconds + temporal_resolution_seconds: int # Time resolution in seconds # update_interval_seconds: int # How often data is updated @classmethod def from_dict(cls, data: dict) -> "OmMetaJson": - """Create instance from dictionary.""" - return cls(**data) + """Create instance from dictionary, ignoring extra keys.""" + # Get the names of all fields defined in the dataclass + class_fields = {f.name for f in fields(cls)} + + # Filter the input dictionary + filtered_data = {k: v for k, v in data.items() if k in class_fields} + + return cls(**filtered_data) + + @classmethod + def from_metajson_string(cls, metajson_str: str) -> "OmMetaJson": + """Create instance from metajson string.""" + return cls.from_dict(json.loads(metajson_str)) + + def time_to_chunk_index(self, timestamp: np.datetime64) -> int: + """ + Convert a timestamp to a chunk index. + + This depends on the file_length and the temporal_resolution_seconds of the domain. + + Args: + timestamp (np.datetime64): The timestamp to convert. + + Returns: + int: The chunk index containing the timestamp. + """ + seconds_since_epoch = (timestamp - EPOCH) / np.timedelta64(1, "s") + chunk_index = int(seconds_since_epoch / (self.chunk_time_length * self.temporal_resolution_seconds)) + return chunk_index + + def chunks_for_date_range( + self, + start_timestamp: np.datetime64, + end_timestamp: np.datetime64, + ) -> List[int]: + """ + Find all chunk indices that contain data within the given date range. + + Args: + start_timestamp (np.datetime64): Start timestamp for the data range. + end_timestamp (np.datetime64): End timestamp for the data range. + + Returns: + List[int]: List of chunk indices containing data within the date range. + """ + # Get chunk indices for start and end dates + start_chunk = self.time_to_chunk_index(start_timestamp) + end_chunk = self.time_to_chunk_index(end_timestamp) + + # Generate list of all chunks between start and end (inclusive) + return list(range(start_chunk, end_chunk + 1)) + + def get_chunk_time_range(self, chunk_index: int): + """ + Get the time range covered by a specific chunk. + + Args: + chunk_index (int): Index of the chunk. + + Returns: + np.ndarray: Array of datetime64 objects representing the time points in the chunk. + """ + chunk_start_seconds = chunk_index * self.chunk_time_length * self.temporal_resolution_seconds + start_time = EPOCH + np.timedelta64(chunk_start_seconds, "s") + + # Generate timestamps at regular intervals from the start time + time_delta = np.timedelta64(self.temporal_resolution_seconds, "s") + # Note: better type inference via list comprehension here + timestamps = np.array([start_time + i * time_delta for i in range(self.chunk_time_length)]) + return timestamps class OmGrid: From 29b4d8dc781875f69837b990568cf0246884500e Mon Sep 17 00:00:00 2001 From: terraputix Date: Tue, 13 Jan 2026 18:20:09 +0100 Subject: [PATCH 25/50] wip: grid tests --- python/omfiles/om_grid.py | 14 -- tests/test_grids.py | 440 +++++++++++++++++--------------------- tests/test_om_domains.py | 122 +++++------ 3 files changed, 258 insertions(+), 318 deletions(-) diff --git a/python/omfiles/om_grid.py b/python/omfiles/om_grid.py index 898c6d0a..b160f414 100644 --- a/python/omfiles/om_grid.py +++ b/python/omfiles/om_grid.py @@ -141,20 +141,6 @@ def __init__(self, crs_wkt: str, shape: Tuple[int, int]): self.dx = (xmax - xmin) / (self.nx - 1) self.dy = (ymax - ymin) / (self.ny - 1) - @classmethod - def from_meta_json(cls, meta: OmMetaJson, shape: Tuple[int, int]) -> "OmGrid": - """ - Create grid from metadata JSON. - - Args: - meta: Metadata containing CRS WKT string - shape: Grid shape as (ny, nx) - - Returns: - OmGrid instance - """ - return cls(meta.crs_wkt, shape) - @property def shape(self) -> Tuple[int, int]: """Grid shape as (ny, nx).""" diff --git a/tests/test_grids.py b/tests/test_grids.py index 6a6f0fdf..c8752490 100644 --- a/tests/test_grids.py +++ b/tests/test_grids.py @@ -12,16 +12,7 @@ RotatedLatLonProjection, StereographicProjection, ) -from omfiles.om_domains import RegularLatLonGrid - -# Fixtures for grids - - -@pytest.fixture -def local_regular_lat_lon_grid(): - return RegularLatLonGrid( - lat_start=0.0, lat_steps=10, lat_step_size=1.0, lon_start=0.0, lon_steps=20, lon_step_size=1.0 - ) +from omfiles.om_grid import OmGrid @pytest.fixture @@ -32,188 +23,148 @@ def stereographic_projection(): ) +# Fixtures for grids @pytest.fixture -def hrdps_projection(): - return RotatedLatLonProjection(lat_origin=-36.0885, lon_origin=245.305) +def icon_global_grid(): + wkt = 'GEOGCRS["WGS 84",DATUM["World Geodetic System 1984",ELLIPSOID["WGS 84",6378137,298.257223563]],CS[ellipsoidal,2],AXIS["latitude",north],AXIS["longitude",east],ANGLEUNIT["degree",0.0174532925199433]USAGE[SCOPE["grid"],BBOX[-90.0,-180.0,90.0,179.75]]]' + return OmGrid(wkt, (1441, 2879)) @pytest.fixture -def hrdps_grid(hrdps_projection): - from omfiles.grids import ProjectionGrid +def hrdps_grid(): + wkt = 'GEOGCRS["Rotated Lat/Lon",BASEGEOGCRS["WGS 84",DATUM["World Geodetic System 1984",ELLIPSOID["WGS 84",6378137,298.257223563]]],DERIVINGCONVERSION["Rotated Lat/Lon",METHOD["PROJ ob_tran o_proj=longlat"],PARAMETER["o_lon_p",0],PARAMETER["o_lat_p",36.0885],PARAMETER["lon_0",245.305]]CS[ellipsoidal,2],AXIS["latitude",north],AXIS["longitude",east],ANGLEUNIT["degree",0.0174532925199433],USAGE[SCOPE["grid"],BBOX[39.626034,-133.62952,47.87646,-40.708527]]]' + return OmGrid(wkt, (1290, 2540)) - return ProjectionGrid.from_bounds( - nx=2540, - ny=1290, - lat_range=(39.626034, 47.876457), - lon_range=(-133.62952, -40.708557), - projection=hrdps_projection, - ) - - -def test_regular_grid_findPointXy_inside(local_regular_lat_lon_grid): - # Test exact grid points - assert local_regular_lat_lon_grid.findPointXy(5.0, 10.0) == (10, 5) - assert local_regular_lat_lon_grid.findPointXy(0.0, 0.0) == (0, 0) - assert local_regular_lat_lon_grid.findPointXy(9.0, 19.0) == (19, 9) - - # Test points that should round to grid points - assert local_regular_lat_lon_grid.findPointXy(5.1, 10.2) == (10, 5) - assert local_regular_lat_lon_grid.findPointXy(0.4, 0.4) == (0, 0) +@pytest.fixture +def ukmo2_wkt(): + return 'PROJCRS["Lambert Azimuthal Equal-Area",\n BASEGEOGCRS["GCS_Sphere",DATUM["D_Sphere",ELLIPSOID["Sphere",6371229.0,0.0]]],\n CONVERSION["Lambert Azimuthal Equal-Area",\n METHOD["Lambert Azimuthal Equal-Area"],\n PARAMETER["Latitude of natural origin", 54.9],\n PARAMETER["Longitude of natural origin", -2.5],\n PARAMETER["False easting", 0.0],\n PARAMETER["False northing", 0.0]],\n CS[Cartesian,2],\n AXIS["easting",east],\n AXIS["northing",north],\n LENGTHUNIT["metre",1.0],\n USAGE[\n SCOPE["grid"],\n BBOX[44.508755,-17.152863,61.92511,15.352753]]]' -def test_regular_grid_findPointXy_outside(local_regular_lat_lon_grid): - # Test points outside of grid - assert local_regular_lat_lon_grid.findPointXy(-1.0, 10.0) is None - assert local_regular_lat_lon_grid.findPointXy(5.0, -1.0) is None - assert local_regular_lat_lon_grid.findPointXy(10.0, 10.0) is None - assert local_regular_lat_lon_grid.findPointXy(5.0, 20.0) is None +@pytest.fixture +def ukmo2_grid(ukmo2_wkt): + return OmGrid(ukmo2_wkt, (970, 1042)) -def test_global_grid_wrapping(): - # Create a global grid (360° longitude, 180° latitude coverage) - global_grid = RegularLatLonGrid( - lat_start=-90.0, lat_steps=180, lat_step_size=1.0, lon_start=-180.0, lon_steps=360, lon_step_size=1.0 - ) - # Test wrapping around the longitude - # Point at longitude 180 should be the same as -180 - assert global_grid.findPointXy(0.0, 180.0) == (0, 90) - assert global_grid.findPointXy(0.0, -180.0) == (0, 90) +@pytest.fixture +def gfs_nam_conus_grid(): + wkt = 'PROJCRS["Lambert Conic Conformal",\n BASEGEOGCRS["GCS_Sphere",DATUM["D_Sphere",ELLIPSOID["Sphere",6371229.0,0.0]]],\n CONVERSION["Lambert Conic Conformal",\n METHOD["Lambert Conic Conformal (2SP)"],\n PARAMETER["Latitude of 1st standard parallel",38.5],\n PARAMETER["Latitude of 2nd standard parallel",38.5],\n PARAMETER["Latitude of false origin",0.0],\n PARAMETER["Longitude of false origin",-97.5]],\n CS[Cartesian,2],\n AXIS["easting",east],\n AXIS["northing",north],\n LENGTHUNIT["metre",1],\n USAGE[\n SCOPE["grid"],\n BBOX[21.137995,-122.72,47.842403,-60.918]]]' + return OmGrid(wkt, (1059, 1799)) - # Test a point beyond the normal range - assert global_grid.findPointXy(0.0, 540.0) == (0, 90) +def test_regular_grid(icon_global_grid: OmGrid): + assert icon_global_grid.find_point_xy(-90, -180) == (0, 0) + assert icon_global_grid.find_point_xy(-90, 179.75) == (2878, 0) + assert icon_global_grid.find_point_xy(90, -180) == (0, 1440) + assert icon_global_grid.find_point_xy(90, 179.75) == (2878, 1440) + assert icon_global_grid.find_point_xy(0, 0) == (1440, 720) -def test_grid_coordinates(local_regular_lat_lon_grid): - # Test exact grid points - assert local_regular_lat_lon_grid.getCoordinates(0, 0) == (0.0, 0.0) - assert local_regular_lat_lon_grid.getCoordinates(5, 2) == (2.0, 5.0) - # Test round-trip conversion +def test_regular_grid_roundtrip(icon_global_grid: OmGrid): lat, lon = 8.0, 15.0 - result = local_regular_lat_lon_grid.findPointXy(lat, lon) + result = icon_global_grid.find_point_xy(lat, lon) assert result is not None, f"Could not find grid point for ({lat}, {lon})" x, y = result - assert local_regular_lat_lon_grid.getCoordinates(x, y) == (lat, lon) + assert icon_global_grid.get_coordinates(x, y) == (lat, lon) -def test_cached_property_computation(local_regular_lat_lon_grid): - lat1 = local_regular_lat_lon_grid.latitude - lat2 = local_regular_lat_lon_grid.latitude +def test_cached_property_computation(icon_global_grid: OmGrid): + lat1 = icon_global_grid.latitude + lat2 = icon_global_grid.latitude # Check that we get the same array (same memory) assert lat1 is lat2 -def test_stereographic(stereographic_projection): - # https://github.com/open-meteo/open-meteo/blob/522917b1d6e72a7e6b7d4ae7dfb49b0c556a6992/Tests/AppTests/DataTests.swift#L248 - pos_x, pos_y = stereographic_projection.findPointXy(lat=64.79836, lon=241.40111) +# def test_stereographic(stereographic_projection): +# # https://github.com/open-meteo/open-meteo/blob/522917b1d6e72a7e6b7d4ae7dfb49b0c556a6992/Tests/AppTests/DataTests.swift#L248 +# pos_x, pos_y = stereographic_projection.findPointXy(lat=64.79836, lon=241.40111) - assert pos_x == 420 - assert pos_y == 468 +# assert pos_x == 420 +# assert pos_y == 468 - # Get the coordinates back - lat, lon = stereographic_projection.getCoordinates(pos_x, pos_y) - assert abs(lat - 64.79836) < 1e-4 - assert np.mod(abs(lon - 241.40111), 360) < 1e-4 +# # Get the coordinates back +# lat, lon = stereographic_projection.getCoordinates(pos_x, pos_y) +# assert abs(lat - 64.79836) < 1e-4 +# assert np.mod(abs(lon - 241.40111), 360) < 1e-4 -def test_grid_properties(stereographic_projection): - assert stereographic_projection.shape == (824, 935) - assert stereographic_projection.grid_type == "projection" +# def test_grid_properties(stereographic_projection): +# assert stereographic_projection.shape == (824, 935) +# assert stereographic_projection.grid_type == "projection" -def test_out_of_bounds(stereographic_projection): - far_point = stereographic_projection.findPointXy(30.0, 120.0) - assert far_point is None +# def test_out_of_bounds(stereographic_projection): +# far_point = stereographic_projection.findPointXy(30.0, 120.0) +# assert far_point is None -def test_latitude_longitude_arrays(stereographic_projection): - # Get latitude and longitude arrays - lats = stereographic_projection.latitude - lons = stereographic_projection.longitude +# def test_latitude_longitude_arrays(stereographic_projection): +# # Get latitude and longitude arrays +# lats = stereographic_projection.latitude +# lons = stereographic_projection.longitude - # Check shapes match the grid - assert lats.shape == (824, 935) - assert lons.shape == (824, 935) +# # Check shapes match the grid +# assert lats.shape == (824, 935) +# assert lons.shape == (824, 935) -def test_hrdps_grid(hrdps_grid): +def test_hrdps_grid(hrdps_grid: OmGrid): """Test the HRDPS Continental grid with a modified approach""" test_points = [ # lat, lon, expected_x, expected_y (39.626034, -133.62952, 0, 0), # Bottom-left - (27.284597, -66.96642, 2539, 0), # Bottom-right + # FIXME: Bottom-right point is not valid for HRDPS grid + # (27.284597, -66.96642, 2539, 0), # Bottom-right (38.96126, -73.63256, 2032, 283), # Middle point (47.876457, -40.708557, 2539, 1289), # Top-right ] for lat, lon, expected_x, expected_y in test_points: # Test finding grid point - pos = hrdps_grid.findPointXy(lat=lat, lon=lon) + pos = hrdps_grid.find_point_xy(lat=lat, lon=lon) assert pos is not None, f"Could not find point for {lat}, {lon}" x, y = pos assert x == expected_x, f"X mismatch: got {x}, expected {expected_x}" assert y == expected_y, f"Y mismatch: got {y}, expected {expected_y}" - lat2, lon2 = hrdps_grid.getCoordinates(x, y) + lat2, lon2 = hrdps_grid.get_coordinates(x, y) assert abs(lat2 - lat) < 0.001, f"latitude mismatch: got {lat2}, expected {lat}" assert abs(lon2 - lon) < 0.001, f"longitude mismatch: got {lon2}, expected {lon}" -def test_lambert_azimuthal_equal_area_projection(): +def test_lambert_azimuthal_equal_area_projection(ukmo2_grid: OmGrid): """ Test the Lambert Azimuthal Equal-Area projection. https://github.com/open-meteo/open-meteo/blob/522917b1d6e72a7e6b7d4ae7dfb49b0c556a6992/Tests/AppTests/DataTests.swift#L189 """ - proj = LambertAzimuthalEqualAreaProjection(lambda_0=-2.5, phi_1=54.9, radius=6371229) - grid = ProjectionGrid(projection=proj, nx=1042, ny=970, origin=(-1158000, -1036000), dx=2000, dy=2000) - test_lon = 10.620785 test_lat = 57.745566 - x, y = proj.forward(latitude=test_lat, longitude=test_lon) - assert abs(x - 773650.5058) < 0.0001 # TODO: There are small numerical differences with the Swift test case - assert abs(y - 389820.1483) < 0.0001 # TODO: There are small numerical differences with the Swift test case - - lat, lon = proj.inverse(x=x, y=y) - assert abs(lon - test_lon) < 0.00001 - assert abs(lat - test_lat) < 0.00001 - point_xy = grid.findPointXy(lat=test_lat, lon=test_lon) + point_xy = ukmo2_grid.find_point_xy(lat=test_lat, lon=test_lon) assert point_xy is not None, "Point not found in grid" x_idx, y_idx = point_xy assert x_idx == 966 assert y_idx == 713 - lat2, lon2 = grid.getCoordinates(x_idx, y_idx) + lat2, lon2 = ukmo2_grid.get_coordinates(x_idx, y_idx) assert abs(lon2 - 10.6271515) < 0.0001 assert abs(lat2 - 57.746563) < 0.0001 -def test_lambert_conformal(): +def test_lambert_conformal(gfs_nam_conus_grid: OmGrid): """ Based on Based on: https://github.com/open-meteo/open-meteo/blob/522917b1d6e72a7e6b7d4ae7dfb49b0c556a6992/Tests/AppTests/DataTests.swift#L128 """ - proj = LambertConformalConicProjection(lambda_0=-97.5, phi_0=0, phi_1=38.5, phi_2=38.5, radius=6370.997) - x, y = proj.forward(latitude=47, longitude=-8) - assert abs(x - 5833.8667) < 0.0001 - assert abs(y - 8632.7338) < 0.0001 - lat, lon = proj.inverse(x=x, y=y) - assert abs(lat - 47) < 0.0001 - assert abs(lon - (-8)) < 0.0001 - - grid = ProjectionGrid.from_bounds( - nx=1799, ny=1059, lat_range=(21.138, 47.8424), lon_range=(-122.72, -60.918), projection=proj - ) - point_xy = grid.findPointXy(lat=34, lon=-118) + point_xy = gfs_nam_conus_grid.find_point_xy(lat=34, lon=-118) assert point_xy is not None x_idx, y_idx = point_xy - flat_idx = y_idx * grid.nx + x_idx + flat_idx = y_idx * gfs_nam_conus_grid.nx + x_idx assert flat_idx == 777441 - lat2, lon2 = grid.getCoordinates(x_idx, y_idx) + lat2, lon2 = gfs_nam_conus_grid.get_coordinates(x_idx, y_idx) assert abs(lat2 - 34) < 0.01 assert abs(lon2 - (-118)) < 0.1 @@ -227,13 +178,13 @@ def test_lambert_conformal(): ] for lat, lon, expected_idx in reference_points: - point_xy = grid.findPointXy(lat=lat, lon=lon) + point_xy = gfs_nam_conus_grid.find_point_xy(lat=lat, lon=lon) assert point_xy is not None x_idx, y_idx = point_xy - flat_idx = y_idx * grid.nx + x_idx + flat_idx = y_idx * gfs_nam_conus_grid.nx + x_idx assert flat_idx == expected_idx - lat2, lon2 = grid.getCoordinates(x_idx, y_idx) + lat2, lon2 = gfs_nam_conus_grid.get_coordinates(x_idx, y_idx) assert abs(lat2 - lat) < 0.001 assert abs(lon2 - lon) < 0.001 @@ -453,7 +404,7 @@ def test_stereographic_against_proj(): assert abs(custom_lon - proj_lon) < 1e-4, f"Lon mismatch: custom={custom_lon}, proj={proj_lon}" -def test_lambert_azimuthal_equal_area_against_proj(): +def test_lambert_azimuthal_equal_area_against_proj(ukmo2_wkt): # Create our custom projection lambda_0 = -2.5 # Central longitude in degrees phi_1 = 54.9 # Standard parallel/latitude in degrees @@ -464,6 +415,9 @@ def test_lambert_azimuthal_equal_area_against_proj(): # For Lambert Azimuthal Equal Area, we use lat_0 for the standard parallel and lon_0 for central longitude proj_string = f"+proj=laea +lat_0={phi_1} +lon_0={lambda_0} +x_0=0 +y_0=0 +R={radius} +units=m +no_defs +type=crs" proj_proj = pyproj.Proj(proj_string) + print(proj_proj) + proj_proj = pyproj.Proj(ukmo2_wkt) + print(proj_proj) # Test points covering different regions test_points = [ @@ -580,85 +534,85 @@ def test_lambert_conformal_conic_against_proj(): ) -def test_regular_lat_lon_grid_against_proj(): - """Test that RegularLatLonGrid operations match proj equivalent operations""" - # Create a regular lat-lon grid with 1-degree steps - grid = RegularLatLonGrid( - lat_start=-90, - lat_steps=181, # -90 to 90 - lat_step_size=1.0, - lon_start=-180, - lon_steps=360, # -180 to 180 - lon_step_size=1.0, - ) - - # Create proj objects for WGS84 lat/lon - proj_wgs84 = pyproj.Proj(proj="latlong", datum="WGS84") - - # Test points covering different scenarios - test_points: list[tuple[float, float]] = [ - (0, 0), # Origin - (45, 45), # NE quadrant - (-45, -45), # SW quadrant - (45, -45), # SE quadrant - (-45, 45), # NW quadrant - (89, 0), # Near North pole - (-89, 0), # Near South pole - (0, 179), # Near date line (east) - (0, -179), # Near date line (west) - (10, 20), # Random point - (-33, 151), # Sydney - (37, -122), # San Francisco - ] - - for lat, lon in test_points: - # Get grid coordinates using our implementation - grid_sel = grid.findPointXy(lat, lon) - assert type(grid_sel) is tuple - grid_x, grid_y = grid_sel - result_lat, result_lon = grid.getCoordinates(grid_x, grid_y) - - # For a lat/lon grid, proj just keeps the same coordinates - proj_x, proj_y = proj_wgs84(lon, lat) # Note: proj uses (lon, lat) order - proj_lat, proj_lon = proj_wgs84(proj_y, proj_x, inverse=True) # Get back lat/lon from proj - - # We'll check that our forward and inverse transformations are consistent - # and match with proj's (which just returns the original coordinates for this projection) - - # Check roundtrip accuracy - assert abs(result_lat - lat) < 1e-9, f"Lat roundtrip error: original={lat}, result={result_lat}" - - # Normalize longitudes before comparison due to -180/180 wrapping - lon_norm = _normalize_longitude(lon) - result_lon_norm = _normalize_longitude(result_lon) - assert abs(result_lon_norm - lon_norm) < 1e-9, f"Lon roundtrip error: original={lon}, result={result_lon}" - - # Verify agreement with proj - assert abs(lat - proj_lat) < 1e-9 - assert abs(lon_norm - _normalize_longitude(proj_lon)) < 1e-9, f"Lon mismatch with proj at ({lat}, {lon})" - - # Test longitude wrapping behavior - wrap_test_points = [ - (0, 185), # Should wrap to (0, -175) - (0, -185), # Should wrap to (0, 175) - (0, 361), # Should wrap to (0, 1) - (0, -361), # Should wrap to (0, -1) - (45, 540), # Should wrap to (45, -180) - ] - - for lat, lon in wrap_test_points: - # Find grid coordinates for the wrapped point - result = grid.findPointXy(lat, lon) - assert result is not None, "Point not found in grid" - grid_x, grid_y = result - - # Find grid coordinates for the normalized longitude - norm_lon = cast(float, _normalize_longitude(lon)) - result = grid.findPointXy(lat=lat, lon=norm_lon) - assert result is not None, "Point not found in grid" - norm_grid_x, norm_grid_y = result - assert grid_x == norm_grid_x, f"Grid X mismatch for wrapped lon: {lon} vs {norm_lon}" - assert grid_y == norm_grid_y, f"Grid Y mismatch for wrapped lon: {lon} vs {norm_lon}" +# def test_regular_lat_lon_grid_against_proj(): +# """Test that RegularLatLonGrid operations match proj equivalent operations""" +# # Create a regular lat-lon grid with 1-degree steps +# grid = RegularLatLonGrid( +# lat_start=-90, +# lat_steps=181, # -90 to 90 +# lat_step_size=1.0, +# lon_start=-180, +# lon_steps=360, # -180 to 180 +# lon_step_size=1.0, +# ) + +# # Create proj objects for WGS84 lat/lon +# proj_wgs84 = pyproj.Proj(proj="latlong", datum="WGS84") + +# # Test points covering different scenarios +# test_points: list[tuple[float, float]] = [ +# (0, 0), # Origin +# (45, 45), # NE quadrant +# (-45, -45), # SW quadrant +# (45, -45), # SE quadrant +# (-45, 45), # NW quadrant +# (89, 0), # Near North pole +# (-89, 0), # Near South pole +# (0, 179), # Near date line (east) +# (0, -179), # Near date line (west) +# (10, 20), # Random point +# (-33, 151), # Sydney +# (37, -122), # San Francisco +# ] + +# for lat, lon in test_points: +# # Get grid coordinates using our implementation +# grid_sel = grid.findPointXy(lat, lon) +# assert type(grid_sel) is tuple +# grid_x, grid_y = grid_sel +# result_lat, result_lon = grid.getCoordinates(grid_x, grid_y) + +# # For a lat/lon grid, proj just keeps the same coordinates +# proj_x, proj_y = proj_wgs84(lon, lat) # Note: proj uses (lon, lat) order +# proj_lat, proj_lon = proj_wgs84(proj_y, proj_x, inverse=True) # Get back lat/lon from proj + +# # We'll check that our forward and inverse transformations are consistent +# # and match with proj's (which just returns the original coordinates for this projection) + +# # Check roundtrip accuracy +# assert abs(result_lat - lat) < 1e-9, f"Lat roundtrip error: original={lat}, result={result_lat}" + +# # Normalize longitudes before comparison due to -180/180 wrapping +# lon_norm = _normalize_longitude(lon) +# result_lon_norm = _normalize_longitude(result_lon) +# assert abs(result_lon_norm - lon_norm) < 1e-9, f"Lon roundtrip error: original={lon}, result={result_lon}" + +# # Verify agreement with proj +# assert abs(lat - proj_lat) < 1e-9 +# assert abs(lon_norm - _normalize_longitude(proj_lon)) < 1e-9, f"Lon mismatch with proj at ({lat}, {lon})" + +# # Test longitude wrapping behavior +# wrap_test_points = [ +# (0, 185), # Should wrap to (0, -175) +# (0, -185), # Should wrap to (0, 175) +# (0, 361), # Should wrap to (0, 1) +# (0, -361), # Should wrap to (0, -1) +# (45, 540), # Should wrap to (45, -180) +# ] + +# for lat, lon in wrap_test_points: +# # Find grid coordinates for the wrapped point +# result = grid.findPointXy(lat, lon) +# assert result is not None, "Point not found in grid" +# grid_x, grid_y = result + +# # Find grid coordinates for the normalized longitude +# norm_lon = cast(float, _normalize_longitude(lon)) +# result = grid.findPointXy(lat=lat, lon=norm_lon) +# assert result is not None, "Point not found in grid" +# norm_grid_x, norm_grid_y = result +# assert grid_x == norm_grid_x, f"Grid X mismatch for wrapped lon: {lon} vs {norm_lon}" +# assert grid_y == norm_grid_y, f"Grid Y mismatch for wrapped lon: {lon} vs {norm_lon}" def test_proj_projection_grid(): @@ -764,53 +718,53 @@ def test_grid_equivalence_lcc(): assert abs(orig_grid_xy[1] - proj_grid_xy[1]) < 1e-8, f"Grid Y mismatch: {orig_grid_xy[1]} vs {proj_grid_xy[1]}" -def test_grid_equivalence_regular_latlon(): - """Test that a proj-based regular lat-lon grid matches the original implementation""" - # Create a regular lat-lon grid using RegularLatLonGrid - original_grid = RegularLatLonGrid( - lat_start=10.0, lat_steps=100, lat_step_size=0.5, lon_start=-30.0, lon_steps=120, lon_step_size=0.5 - ) - - # Create equivalent proj-based regular lat-lon projection - proj_proj = ProjProjection("+proj=longlat +datum=WGS84 +no_defs") - proj_grid = ProjectionGrid(projection=proj_proj, nx=120, ny=100, origin=(-30.0, 10.0), dx=0.5, dy=0.5) - - # Test points covering various areas within the grid - test_points = [ - (10.0, -30.0), # Lower left corner - (59.5, 29.5), # Upper right corner - (35.0, 0.0), # Middle-ish - (20.0, -15.0), # Random point - (50.0, 20.0), # Random point - (15.25, -25.75), # Point between grid cells - ] - - for lat, lon in test_points: - # Get grid points using both implementations - orig_grid_xy = original_grid.findPointXy(lat=lat, lon=lon) - proj_grid_xy = proj_grid.findPointXy(lat=lat, lon=lon) - - # Both should find the point - assert orig_grid_xy is not None and proj_grid_xy is not None, f"Point not found for ({lat}, {lon})" - - # If point is found, coordinates should match exactly - if orig_grid_xy is not None: - assert abs(orig_grid_xy[0] - proj_grid_xy[0]) < 1e-8, ( - f"Grid X mismatch: {orig_grid_xy[0]} vs {proj_grid_xy[0]}" - ) - assert abs(orig_grid_xy[1] - proj_grid_xy[1]) < 1e-8, ( - f"Grid Y mismatch: {orig_grid_xy[1]} vs {proj_grid_xy[1]}" - ) - - # Test the inverse transformation (getCoordinates) - if orig_grid_xy is not None: - x, y = orig_grid_xy - orig_lat, orig_lon = original_grid.getCoordinates(x, y) - proj_lat, proj_lon = proj_grid.getCoordinates(x, y) - - # Results should match exactly - assert abs(orig_lat - proj_lat) < 1e-8, f"Latitude mismatch: {orig_lat} vs {proj_lat}" - assert abs(orig_lon - proj_lon) < 1e-8, f"Longitude mismatch: {orig_lon} vs {proj_lon}" +# def test_grid_equivalence_regular_latlon(): +# """Test that a proj-based regular lat-lon grid matches the original implementation""" +# # Create a regular lat-lon grid using RegularLatLonGrid +# original_grid = RegularLatLonGrid( +# lat_start=10.0, lat_steps=100, lat_step_size=0.5, lon_start=-30.0, lon_steps=120, lon_step_size=0.5 +# ) + +# # Create equivalent proj-based regular lat-lon projection +# proj_proj = ProjProjection("+proj=longlat +datum=WGS84 +no_defs") +# proj_grid = ProjectionGrid(projection=proj_proj, nx=120, ny=100, origin=(-30.0, 10.0), dx=0.5, dy=0.5) + +# # Test points covering various areas within the grid +# test_points = [ +# (10.0, -30.0), # Lower left corner +# (59.5, 29.5), # Upper right corner +# (35.0, 0.0), # Middle-ish +# (20.0, -15.0), # Random point +# (50.0, 20.0), # Random point +# (15.25, -25.75), # Point between grid cells +# ] + +# for lat, lon in test_points: +# # Get grid points using both implementations +# orig_grid_xy = original_grid.findPointXy(lat=lat, lon=lon) +# proj_grid_xy = proj_grid.findPointXy(lat=lat, lon=lon) + +# # Both should find the point +# assert orig_grid_xy is not None and proj_grid_xy is not None, f"Point not found for ({lat}, {lon})" + +# # If point is found, coordinates should match exactly +# if orig_grid_xy is not None: +# assert abs(orig_grid_xy[0] - proj_grid_xy[0]) < 1e-8, ( +# f"Grid X mismatch: {orig_grid_xy[0]} vs {proj_grid_xy[0]}" +# ) +# assert abs(orig_grid_xy[1] - proj_grid_xy[1]) < 1e-8, ( +# f"Grid Y mismatch: {orig_grid_xy[1]} vs {proj_grid_xy[1]}" +# ) + +# # Test the inverse transformation (getCoordinates) +# if orig_grid_xy is not None: +# x, y = orig_grid_xy +# orig_lat, orig_lon = original_grid.getCoordinates(x, y) +# proj_lat, proj_lon = proj_grid.getCoordinates(x, y) + +# # Results should match exactly +# assert abs(orig_lat - proj_lat) < 1e-8, f"Latitude mismatch: {orig_lat} vs {proj_lat}" +# assert abs(orig_lon - proj_lon) < 1e-8, f"Longitude mismatch: {orig_lon} vs {proj_lon}" def test_grid_equivalence_rotated_latlon(): diff --git a/tests/test_om_domains.py b/tests/test_om_domains.py index 032d61cc..7df39827 100644 --- a/tests/test_om_domains.py +++ b/tests/test_om_domains.py @@ -1,82 +1,82 @@ -import numpy as np -from omfiles.om_domains import DOMAINS +# import numpy as np +# from omfiles.om_domains import DOMAINS -def test_dwd_icon_d2_grid_points(): - """Test specific points in the DWD ICON D2 grid.""" - dwd_grid = DOMAINS["dwd_icon_d2"].grid +# def test_dwd_icon_d2_grid_points(): +# """Test specific points in the DWD ICON D2 grid.""" +# dwd_grid = DOMAINS["dwd_icon_d2"].grid - # Test a point known to be in the domain (Central Europe) - # Berlin coordinates: approx. 52.52°N, 13.40°E - berlin = dwd_grid.findPointXy(52.52, 13.40) - assert berlin is not None +# # Test a point known to be in the domain (Central Europe) +# # Berlin coordinates: approx. 52.52°N, 13.40°E +# berlin = dwd_grid.findPointXy(52.52, 13.40) +# assert berlin is not None - # Test a point outside the domain (should return None) - # New York coordinates: approx. 40.71°N, -74.01°E - new_york = dwd_grid.findPointXy(40.71, -74.01) - assert new_york is None +# # Test a point outside the domain (should return None) +# # New York coordinates: approx. 40.71°N, -74.01°E +# new_york = dwd_grid.findPointXy(40.71, -74.01) +# assert new_york is None - # Test gridpoint to coordinate conversion - if berlin is not None: - x, y = berlin - lat, lon = dwd_grid.getCoordinates(x, y) - # Check that we get close to the original coordinates - assert abs(lat - 52.52) < 0.05 - assert abs(lon - 13.40) < 0.05 +# # Test gridpoint to coordinate conversion +# if berlin is not None: +# x, y = berlin +# lat, lon = dwd_grid.getCoordinates(x, y) +# # Check that we get close to the original coordinates +# assert abs(lat - 52.52) < 0.05 +# assert abs(lon - 13.40) < 0.05 -def test_ecmwf_grid(): - """Test the ECMWF IFS grid specifically.""" - ecmwf_grid = DOMAINS["ecmwf_ifs025"].grid +# def test_ecmwf_grid(): +# """Test the ECMWF IFS grid specifically.""" +# ecmwf_grid = DOMAINS["ecmwf_ifs025"].grid - # Test some known points on the grid - # Point at the prime meridian and equator - assert ecmwf_grid.findPointXy(0.0, 0.0) == (720, 360) +# # Test some known points on the grid +# # Point at the prime meridian and equator +# assert ecmwf_grid.findPointXy(0.0, 0.0) == (720, 360) - # Point at the North Pole - assert ecmwf_grid.findPointXy(90.0, 0.0) == (720, 720) +# # Point at the North Pole +# assert ecmwf_grid.findPointXy(90.0, 0.0) == (720, 720) - # Test some edge points (ensure they are properly handled) - assert ecmwf_grid.findPointXy(-90.0, -180.0) == (0, 0) - assert ecmwf_grid.findPointXy(90.0, 180.0) == (0, 720) +# # Test some edge points (ensure they are properly handled) +# assert ecmwf_grid.findPointXy(-90.0, -180.0) == (0, 0) +# assert ecmwf_grid.findPointXy(90.0, 180.0) == (0, 720) - # Test wrapping for global grid - # A point at longitude 181 should wrap to longitude -179 - point1 = ecmwf_grid.findPointXy(0.0, 181.0) - point2 = ecmwf_grid.findPointXy(0.0, -179.0) - assert point1 == point2 +# # Test wrapping for global grid +# # A point at longitude 181 should wrap to longitude -179 +# point1 = ecmwf_grid.findPointXy(0.0, 181.0) +# point2 = ecmwf_grid.findPointXy(0.0, -179.0) +# assert point1 == point2 -def test_time_to_chunk_index(): - """Test conversion from timestamp to chunk index.""" - domain = DOMAINS["dwd_icon_d2"] +# def test_time_to_chunk_index(): +# """Test conversion from timestamp to chunk index.""" +# domain = DOMAINS["dwd_icon_d2"] - # Create test timestamp (2023-01-01 12:00:00 UTC) - timestamp = np.datetime64("2023-01-01T12:00:00") +# # Create test timestamp (2023-01-01 12:00:00 UTC) +# timestamp = np.datetime64("2023-01-01T12:00:00") - # Calculate expected chunk index - # Seconds since epoch = (2023-01-01 12:00:00 - 1970-01-01 00:00:00) seconds - # chunk_index = seconds_since_epoch / (file_length * temporal_resolution_seconds) - epoch = np.datetime64("1970-01-01T00:00:00") - seconds_since_epoch = (timestamp - epoch) / np.timedelta64(1, "s") - expected_chunk = int(seconds_since_epoch / (domain.file_length * domain.temporal_resolution_seconds)) +# # Calculate expected chunk index +# # Seconds since epoch = (2023-01-01 12:00:00 - 1970-01-01 00:00:00) seconds +# # chunk_index = seconds_since_epoch / (file_length * temporal_resolution_seconds) +# epoch = np.datetime64("1970-01-01T00:00:00") +# seconds_since_epoch = (timestamp - epoch) / np.timedelta64(1, "s") +# expected_chunk = int(seconds_since_epoch / (domain.file_length * domain.temporal_resolution_seconds)) - # Test the time_to_chunk_index function - chunk_index = domain.time_to_chunk_index(timestamp) - assert chunk_index == expected_chunk +# # Test the time_to_chunk_index function +# chunk_index = domain.time_to_chunk_index(timestamp) +# assert chunk_index == expected_chunk -def test_get_chunk_time_range(): - """Test getting time range for a specific chunk.""" - domain = DOMAINS["dwd_icon_d2"] +# def test_get_chunk_time_range(): +# """Test getting time range for a specific chunk.""" +# domain = DOMAINS["dwd_icon_d2"] - # Test chunk 1000 - chunk_index = 1000 - time_range = domain.get_chunk_time_range(chunk_index) +# # Test chunk 1000 +# chunk_index = 1000 +# time_range = domain.get_chunk_time_range(chunk_index) - # Check that we get the expected number of time points - assert len(time_range) == domain.file_length +# # Check that we get the expected number of time points +# assert len(time_range) == domain.file_length - # Check that time points are evenly spaced - time_diff = time_range[1] - time_range[0] - assert time_diff == np.timedelta64(domain.temporal_resolution_seconds, "s") +# # Check that time points are evenly spaced +# time_diff = time_range[1] - time_range[0] +# assert time_diff == np.timedelta64(domain.temporal_resolution_seconds, "s") From ea141c09e2e74154e3f8e99a1a844f6ec3ea1c21 Mon Sep 17 00:00:00 2001 From: terraputix Date: Tue, 13 Jan 2026 19:47:06 +0100 Subject: [PATCH 26/50] more tests --- python/omfiles/om_grid.py | 4 -- tests/test_grids.py | 122 ++++++++++++++++++++------------------ 2 files changed, 65 insertions(+), 61 deletions(-) diff --git a/python/omfiles/om_grid.py b/python/omfiles/om_grid.py index b160f414..440171ef 100644 --- a/python/omfiles/om_grid.py +++ b/python/omfiles/om_grid.py @@ -197,10 +197,6 @@ def find_point_xy(self, lat: float, lon: float) -> Optional[Tuple[int, int]]: # Transform to projection coordinates x_proj, y_proj = self.to_projection.transform(lon, lat) - # Check if point is within bounds - if not (self.bounds[0] <= x_proj <= self.bounds[1] and self.bounds[2] <= y_proj <= self.bounds[3]): - return None - # Calculate grid indices x_idx = int(round((x_proj - self.origin[0]) / self.dx)) y_idx = int(round((y_proj - self.origin[1]) / self.dy)) diff --git a/tests/test_grids.py b/tests/test_grids.py index c8752490..0e3035b7 100644 --- a/tests/test_grids.py +++ b/tests/test_grids.py @@ -15,14 +15,6 @@ from omfiles.om_grid import OmGrid -@pytest.fixture -def stereographic_projection(): - projection = StereographicProjection(90.0, 249.0, 6371229.0) - return ProjectionGrid.from_bounds( - nx=935, ny=824, lat_range=(18.14503, 45.405453), lon_range=(217.10745, 349.8256), projection=projection - ) - - # Fixtures for grids @pytest.fixture def icon_global_grid(): @@ -31,11 +23,17 @@ def icon_global_grid(): @pytest.fixture -def hrdps_grid(): - wkt = 'GEOGCRS["Rotated Lat/Lon",BASEGEOGCRS["WGS 84",DATUM["World Geodetic System 1984",ELLIPSOID["WGS 84",6378137,298.257223563]]],DERIVINGCONVERSION["Rotated Lat/Lon",METHOD["PROJ ob_tran o_proj=longlat"],PARAMETER["o_lon_p",0],PARAMETER["o_lat_p",36.0885],PARAMETER["lon_0",245.305]]CS[ellipsoidal,2],AXIS["latitude",north],AXIS["longitude",east],ANGLEUNIT["degree",0.0174532925199433],USAGE[SCOPE["grid"],BBOX[39.626034,-133.62952,47.87646,-40.708527]]]' +def gem_hrdps_grid(): + wkt = 'GEOGCRS["Rotated Lat/Lon",BASEGEOGCRS["GCS_Sphere",DATUM["D_Sphere",ELLIPSOID["Sphere",6371229.0,0.0]]],DERIVINGCONVERSION["Rotated Lat/Lon",METHOD["PROJ ob_tran o_proj=longlat"],PARAMETER["o_lon_p",0],PARAMETER["o_lat_p",36.0885],PARAMETER["lon_0",245.305]]CS[ellipsoidal,2],AXIS["latitude",north],AXIS["longitude",east],ANGLEUNIT["degree",0.0174532925199433],USAGE[SCOPE["grid"],BBOX[39.626034,-133.62952,47.87646,-40.708527]]]' return OmGrid(wkt, (1290, 2540)) +@pytest.fixture +def gem_regional_grid(): + wkt = 'PROJCRS["Stereographic",\n BASEGEOGCRS["GCS_Sphere",DATUM["D_Sphere",ELLIPSOID["Sphere",6371229.0,0.0]]],\n CONVERSION["Stereographic",\n METHOD["Stereographic"],\n PARAMETER["Latitude of natural origin", 90.0],\n PARAMETER["Longitude of natural origin", 249.0],\n PARAMETER["Scale factor at natural origin", 1.0],\n PARAMETER["False easting", 0.0],\n PARAMETER["False northing", 0.0]],\n CS[Cartesian,2],\n AXIS["easting",east],\n AXIS["northing",north],\n LENGTHUNIT["metre",1.0],\n USAGE[\n SCOPE["grid"],\n BBOX[18.145027,-142.89252,45.40545,-10.174438]]]' + return OmGrid(wkt, (824, 935)) + + @pytest.fixture def ukmo2_wkt(): return 'PROJCRS["Lambert Azimuthal Equal-Area",\n BASEGEOGCRS["GCS_Sphere",DATUM["D_Sphere",ELLIPSOID["Sphere",6371229.0,0.0]]],\n CONVERSION["Lambert Azimuthal Equal-Area",\n METHOD["Lambert Azimuthal Equal-Area"],\n PARAMETER["Latitude of natural origin", 54.9],\n PARAMETER["Longitude of natural origin", -2.5],\n PARAMETER["False easting", 0.0],\n PARAMETER["False northing", 0.0]],\n CS[Cartesian,2],\n AXIS["easting",east],\n AXIS["northing",north],\n LENGTHUNIT["metre",1.0],\n USAGE[\n SCOPE["grid"],\n BBOX[44.508755,-17.152863,61.92511,15.352753]]]' @@ -47,9 +45,23 @@ def ukmo2_grid(ukmo2_wkt): @pytest.fixture -def gfs_nam_conus_grid(): - wkt = 'PROJCRS["Lambert Conic Conformal",\n BASEGEOGCRS["GCS_Sphere",DATUM["D_Sphere",ELLIPSOID["Sphere",6371229.0,0.0]]],\n CONVERSION["Lambert Conic Conformal",\n METHOD["Lambert Conic Conformal (2SP)"],\n PARAMETER["Latitude of 1st standard parallel",38.5],\n PARAMETER["Latitude of 2nd standard parallel",38.5],\n PARAMETER["Latitude of false origin",0.0],\n PARAMETER["Longitude of false origin",-97.5]],\n CS[Cartesian,2],\n AXIS["easting",east],\n AXIS["northing",north],\n LENGTHUNIT["metre",1],\n USAGE[\n SCOPE["grid"],\n BBOX[21.137995,-122.72,47.842403,-60.918]]]' - return OmGrid(wkt, (1059, 1799)) +def gfs_nam_conus_wkt(): + return 'PROJCRS["Lambert Conic Conformal",\n BASEGEOGCRS["GCS_Sphere",DATUM["D_Sphere",ELLIPSOID["Sphere",6371229.0,0.0]]],\n CONVERSION["Lambert Conic Conformal",\n METHOD["Lambert Conic Conformal (2SP)"],\n PARAMETER["Latitude of 1st standard parallel",38.5],\n PARAMETER["Latitude of 2nd standard parallel",38.5],\n PARAMETER["Latitude of false origin",0.0],\n PARAMETER["Longitude of false origin",-97.5]],\n CS[Cartesian,2],\n AXIS["easting",east],\n AXIS["northing",north],\n LENGTHUNIT["metre",1],\n USAGE[\n SCOPE["grid"],\n BBOX[21.137995,-122.72,47.842403,-60.918]]]' + + +@pytest.fixture +def gfs_nam_conus_grid(gfs_nam_conus_wkt): + return OmGrid(gfs_nam_conus_wkt, (1059, 1799)) + + +@pytest.fixture +def dmi_harmoni_europe_wkt(): + return 'PROJCRS["Lambert Conic Conformal",\n BASEGEOGCRS["GCS_Sphere",DATUM["D_Sphere",ELLIPSOID["Sphere",6371229.0,0.0]]],\n CONVERSION["Lambert Conic Conformal",\n METHOD["Lambert Conic Conformal (2SP)"],\n PARAMETER["Latitude of 1st standard parallel",55.5],\n PARAMETER["Latitude of 2nd standard parallel",55.5],\n PARAMETER["Latitude of false origin",55.5],\n PARAMETER["Longitude of false origin",352.0]],\n CS[Cartesian,2],\n AXIS["easting",east],\n AXIS["northing",north],\n LENGTHUNIT["metre",1],\n USAGE[\n SCOPE["grid"],\n BBOX[39.670998,-25.421997,62.667618,40.069855]]]' + + +@pytest.fixture +def dmi_harmoni_europe_grid(dmi_harmoni_europe_wkt): + return OmGrid(dmi_harmoni_europe_wkt, (1606, 1906)) def test_regular_grid(icon_global_grid: OmGrid): @@ -76,60 +88,58 @@ def test_cached_property_computation(icon_global_grid: OmGrid): assert lat1 is lat2 -# def test_stereographic(stereographic_projection): -# # https://github.com/open-meteo/open-meteo/blob/522917b1d6e72a7e6b7d4ae7dfb49b0c556a6992/Tests/AppTests/DataTests.swift#L248 -# pos_x, pos_y = stereographic_projection.findPointXy(lat=64.79836, lon=241.40111) - -# assert pos_x == 420 -# assert pos_y == 468 +def test_stereographic(gem_regional_grid: OmGrid): + # https://github.com/open-meteo/open-meteo/blob/522917b1d6e72a7e6b7d4ae7dfb49b0c556a6992/Tests/AppTests/DataTests.swift#L248 + indices = gem_regional_grid.find_point_xy(lat=64.79836, lon=241.40111) -# # Get the coordinates back -# lat, lon = stereographic_projection.getCoordinates(pos_x, pos_y) -# assert abs(lat - 64.79836) < 1e-4 -# assert np.mod(abs(lon - 241.40111), 360) < 1e-4 + assert indices is not None + pos_x, pos_y = indices + assert pos_x == 420 + assert pos_y == 468 -# def test_grid_properties(stereographic_projection): -# assert stereographic_projection.shape == (824, 935) -# assert stereographic_projection.grid_type == "projection" + # Get the coordinates back + lat, lon = gem_regional_grid.get_coordinates(pos_x, pos_y) + assert abs(lat - 64.79836) < 1e-4 + assert abs(abs(lon - 241.40111) - 360) < 1e-4 -# def test_out_of_bounds(stereographic_projection): -# far_point = stereographic_projection.findPointXy(30.0, 120.0) -# assert far_point is None +def test_stereographic_out_of_bounds(gem_regional_grid: OmGrid): + far_point = gem_regional_grid.find_point_xy(lat=30.0, lon=120.0) + assert far_point is None -# def test_latitude_longitude_arrays(stereographic_projection): -# # Get latitude and longitude arrays -# lats = stereographic_projection.latitude -# lons = stereographic_projection.longitude +def test_stereographic_latitude_longitude_arrays(gem_regional_grid: OmGrid): + # Get latitude and longitude arrays + lats = gem_regional_grid.latitude + lons = gem_regional_grid.longitude -# # Check shapes match the grid -# assert lats.shape == (824, 935) -# assert lons.shape == (824, 935) + # Check shapes match the grid + assert lats.shape == (824, 935) + assert lons.shape == (824, 935) -def test_hrdps_grid(hrdps_grid: OmGrid): +def test_hrdps_grid(gem_hrdps_grid: OmGrid): """Test the HRDPS Continental grid with a modified approach""" test_points = [ # lat, lon, expected_x, expected_y (39.626034, -133.62952, 0, 0), # Bottom-left # FIXME: Bottom-right point is not valid for HRDPS grid - # (27.284597, -66.96642, 2539, 0), # Bottom-right + (27.284597, -66.96642, 2539, 0), # Bottom-right (38.96126, -73.63256, 2032, 283), # Middle point (47.876457, -40.708557, 2539, 1289), # Top-right ] for lat, lon, expected_x, expected_y in test_points: # Test finding grid point - pos = hrdps_grid.find_point_xy(lat=lat, lon=lon) + pos = gem_hrdps_grid.find_point_xy(lat=lat, lon=lon) assert pos is not None, f"Could not find point for {lat}, {lon}" x, y = pos assert x == expected_x, f"X mismatch: got {x}, expected {expected_x}" assert y == expected_y, f"Y mismatch: got {y}, expected {expected_y}" - lat2, lon2 = hrdps_grid.get_coordinates(x, y) + lat2, lon2 = gem_hrdps_grid.get_coordinates(x, y) assert abs(lat2 - lat) < 0.001, f"latitude mismatch: got {lat2}, expected {lat}" assert abs(lon2 - lon) < 0.001, f"longitude mismatch: got {lon2}, expected {lon}" @@ -247,57 +257,55 @@ def test_nbm_grid(): assert abs(lon - expected_lon) < 0.001 -def test_lambert_conformal_conic_projection(): +def test_lambert_conformal_conic_projection(dmi_harmoni_europe_wkt: str, dmi_harmoni_europe_grid: OmGrid): """ Test the Lambert Conformal Conic projection. - Based on: https://github.com/open-meteo/open-meteo/blob/522917b1d6e72a7e6b7d4ae7dfb49b0c556a6992/Tests/AppTests/DataTests.swift#L163 + Based on: https://github.com/open-meteo/open-meteo/blob/7eb49a5dd41e66ac5cf386023a0527eead3104b4/Tests/AppTests/DataTests.swift#L352 """ proj = LambertConformalConicProjection(lambda_0=352, phi_0=55.5, phi_1=55.5, phi_2=55.5, radius=6371229) + proj = pyproj.Transformer.from_crs(pyproj.CRS.from_epsg(4326), pyproj.CRS.from_wkt(dmi_harmoni_europe_wkt)) + inverse_proj = pyproj.Transformer.from_crs(pyproj.CRS.from_wkt(dmi_harmoni_europe_wkt), pyproj.CRS.from_epsg(4326)) + center_lat = 39.671 center_lon = -25.421997 - - grid = ProjectionGrid.from_center( - nx=1906, ny=1606, center_lat=center_lat, center_lon=center_lon, dx=2000, dy=2000, projection=proj - ) - # Test forward projection - origin_x, origin_y = proj.forward(latitude=center_lat, longitude=center_lon) + origin_x, origin_y = proj.transform(center_lat, center_lon) assert abs(origin_x - (-1527524.624)) < 0.001 assert abs(origin_y - (-1588681.042)) < 0.001 - lat, lon = proj.inverse(origin_x, origin_y) + lat, lon = inverse_proj.transform(origin_x, origin_y) assert abs(center_lat - lat) < 0.0001 assert abs(center_lon - lon) < 0.0001 # Test another point test_lat = 39.675304 test_lon = -25.400146 - x1, y1 = proj.forward(latitude=test_lat, longitude=test_lon) + x1, y1 = proj.transform(test_lat, test_lon) assert abs(origin_x - x1 - (-1998.358)) < 0.001 assert abs(origin_y - y1 - (-0.187)) < 0.001 - lat, lon = proj.inverse(x1, y1) + lat, lon = inverse_proj.transform(x1, y1) assert abs(test_lat - lat) < 0.0001 assert abs(test_lon - lon) < 0.0001 # Point at index 1 - lat, lon = grid.getCoordinates(1, 0) + lat, lon = dmi_harmoni_europe_grid.get_coordinates(1, 0) assert abs(lat - test_lat) < 0.001 assert abs(lon - test_lon) < 0.001 - point_idx = grid.findPointXy(lat=test_lat, lon=test_lon) + point_idx = dmi_harmoni_europe_grid.find_point_xy(lat=test_lat, lon=test_lon) assert point_idx == (1, 0) # Coords(i: 122440, x: 456, y: 64, latitude: 42.18604, longitude: -15.30127) - lat, lon = grid.getCoordinates(456, 64) + lat, lon = dmi_harmoni_europe_grid.get_coordinates(456, 64) assert abs(lat - 42.18604) < 0.001 assert abs(lon - (-15.30127)) < 0.001 - point_idx = grid.findPointXy(lat=lat, lon=lon) + point_idx = dmi_harmoni_europe_grid.find_point_xy(lat=lat, lon=lon) assert point_idx == (456, 64) # Coords(i: 2999780, x: 1642, y: 1573, latitude: 64.943695, longitude: 30.711975) - lat, lon = grid.getCoordinates(1642, 1573) + lat, lon = dmi_harmoni_europe_grid.get_coordinates(1642, 1573) assert abs(lat - 64.943695) < 0.001 assert abs(lon - 30.711975) < 0.001 - point_idx = grid.findPointXy(lat=lat, lon=lon) + point_idx = dmi_harmoni_europe_grid.find_point_xy(lat=lat, lon=lon) assert point_idx == (1642, 1573) From 6558f369e9fdf4798be30c726b2f857e05403035 Mon Sep 17 00:00:00 2001 From: terraputix Date: Tue, 13 Jan 2026 19:55:44 +0100 Subject: [PATCH 27/50] cleanup --- tests/test_grids.py | 569 ++------------------------------------------ 1 file changed, 19 insertions(+), 550 deletions(-) diff --git a/tests/test_grids.py b/tests/test_grids.py index 0e3035b7..d2039577 100644 --- a/tests/test_grids.py +++ b/tests/test_grids.py @@ -1,17 +1,5 @@ -from typing import cast - -import numpy as np import pyproj import pytest -from omfiles._utils import _normalize_longitude -from omfiles.grids import ( - LambertAzimuthalEqualAreaProjection, - LambertConformalConicProjection, - ProjectionGrid, - ProjProjection, - RotatedLatLonProjection, - StereographicProjection, -) from omfiles.om_grid import OmGrid @@ -54,6 +42,12 @@ def gfs_nam_conus_grid(gfs_nam_conus_wkt): return OmGrid(gfs_nam_conus_wkt, (1059, 1799)) +@pytest.fixture +def nbm_conus_grid(): + wkt = 'PROJCRS["Lambert Conic Conformal",\n BASEGEOGCRS["GCS_Sphere",DATUM["D_Sphere",ELLIPSOID["Sphere",6371229.0,0.0]]],\n CONVERSION["Lambert Conic Conformal",\n METHOD["Lambert Conic Conformal (2SP)"],\n PARAMETER["Latitude of 1st standard parallel",25.0],\n PARAMETER["Latitude of 2nd standard parallel",25.0],\n PARAMETER["Latitude of false origin",0.0],\n PARAMETER["Longitude of false origin",-95.0]],\n CS[Cartesian,2],\n AXIS["easting",east],\n AXIS["northing",north],\n LENGTHUNIT["metre",1],\n USAGE[\n SCOPE["grid"],\n BBOX[19.228985,-126.27699,54.372913,-59.042786]]]' + return OmGrid(wkt, (1597, 2345)) + + @pytest.fixture def dmi_harmoni_europe_wkt(): return 'PROJCRS["Lambert Conic Conformal",\n BASEGEOGCRS["GCS_Sphere",DATUM["D_Sphere",ELLIPSOID["Sphere",6371229.0,0.0]]],\n CONVERSION["Lambert Conic Conformal",\n METHOD["Lambert Conic Conformal (2SP)"],\n PARAMETER["Latitude of 1st standard parallel",55.5],\n PARAMETER["Latitude of 2nd standard parallel",55.5],\n PARAMETER["Latitude of false origin",55.5],\n PARAMETER["Longitude of false origin",352.0]],\n CS[Cartesian,2],\n AXIS["easting",east],\n AXIS["northing",north],\n LENGTHUNIT["metre",1],\n USAGE[\n SCOPE["grid"],\n BBOX[39.670998,-25.421997,62.667618,40.069855]]]' @@ -199,27 +193,22 @@ def test_lambert_conformal(gfs_nam_conus_grid: OmGrid): assert abs(lon2 - lon) < 0.001 -def test_nbm_grid(): +def test_nbm_grid(nbm_conus_grid: OmGrid): """ Test the NBM (National Blend of Models) grid using Lambert Conformal Conic projection. https://vlab.noaa.gov/web/mdl/nbm-grib2-v4.0 https://github.com/open-meteo/open-meteo/blob/522917b1d6e72a7e6b7d4ae7dfb49b0c556a6992/Tests/AppTests/DataTests.swift#L94 """ - # Create projection with appropriate parameters - proj = LambertConformalConicProjection(lambda_0=265 - 360, phi_0=0, phi_1=25, phi_2=25, radius=6371200) + # # Create projection with appropriate parameters + # proj = LambertConformalConicProjection(lambda_0=265 - 360, phi_0=0, phi_1=25, phi_2=25, radius=6371200) - # Create grid - grid = ProjectionGrid.from_center( - projection=proj, nx=2345, ny=1597, center_lat=19.229, center_lon=233.723 - 360, dx=2539.7, dy=2539.7 - ) - - # Test forward projection of grid origin - x, y = proj.forward(latitude=19.229, longitude=233.723 - 360) - assert abs(x - (-3271192.6)) < 0.1 - assert abs(y - 2604269.4) < 0.1 + # # Test forward projection of grid origin + # x, y = proj.forward(latitude=19.229, longitude=233.723 - 360) + # assert abs(x - (-3271192.6)) < 0.1 + # assert abs(y - 2604269.4) < 0.1 # Test grid point lookup - point_xy = grid.findPointXy(lat=19.229, lon=233.723 - 360) + point_xy = nbm_conus_grid.find_point_xy(lat=19.229, lon=233.723 - 360) assert point_xy is not None assert point_xy[0] == 0 assert point_xy[1] == 0 @@ -234,10 +223,10 @@ def test_nbm_grid(): ] for lat, lon, expected_idx in reference_points: - point_xy = grid.findPointXy(lat=lat, lon=lon) + point_xy = nbm_conus_grid.find_point_xy(lat=lat, lon=lon) assert point_xy is not None x_idx, y_idx = point_xy - flat_idx = y_idx * grid.nx + x_idx + flat_idx = y_idx * nbm_conus_grid.nx + x_idx assert flat_idx == expected_idx # Test grid coordinate lookup for specific indices @@ -250,9 +239,9 @@ def test_nbm_grid(): ] for idx, expected_lat, expected_lon in reference_coords: - y_idx = idx // grid.nx - x_idx = idx % grid.nx - lat, lon = grid.getCoordinates(x_idx, y_idx) + y_idx = idx // nbm_conus_grid.nx + x_idx = idx % nbm_conus_grid.nx + lat, lon = nbm_conus_grid.get_coordinates(x_idx, y_idx) assert abs(lat - expected_lat) < 0.001 assert abs(lon - expected_lon) < 0.001 @@ -262,7 +251,6 @@ def test_lambert_conformal_conic_projection(dmi_harmoni_europe_wkt: str, dmi_har Test the Lambert Conformal Conic projection. Based on: https://github.com/open-meteo/open-meteo/blob/7eb49a5dd41e66ac5cf386023a0527eead3104b4/Tests/AppTests/DataTests.swift#L352 """ - proj = LambertConformalConicProjection(lambda_0=352, phi_0=55.5, phi_1=55.5, phi_2=55.5, radius=6371229) proj = pyproj.Transformer.from_crs(pyproj.CRS.from_epsg(4326), pyproj.CRS.from_wkt(dmi_harmoni_europe_wkt)) inverse_proj = pyproj.Transformer.from_crs(pyproj.CRS.from_wkt(dmi_harmoni_europe_wkt), pyproj.CRS.from_epsg(4326)) @@ -307,522 +295,3 @@ def test_lambert_conformal_conic_projection(dmi_harmoni_europe_wkt: str, dmi_har assert abs(lon - 30.711975) < 0.001 point_idx = dmi_harmoni_europe_grid.find_point_xy(lat=lat, lon=lon) assert point_idx == (1642, 1573) - - -def test_rotated_latlon_against_proj(): - # Create our custom projection - lat_origin = -36.0885 - lon_origin = 245.305 - custom_proj = RotatedLatLonProjection(lat_origin=lat_origin, lon_origin=lon_origin) - - # Create equivalent PROJ projection - proj_string = ( - f"+proj=ob_tran +o_proj=longlat +o_lat_p={-lat_origin} " - f"+o_lon_p=0.0 +lon_0={lon_origin} +datum=WGS84 +no_defs +type=crs" - ) - proj_proj = pyproj.Proj(proj_string) - - # Test points covering different regions - test_points = [ - (0, 0), # Origin - (45, 45), # Mid-latitude point - (-45, -45), # Mid-latitude point (southern hemisphere) - (10, 50), # Europe - (40, -100), # North America - (50, -170), # Pacific - (-30, 170), # South Pacific - ] - - for lat, lon in test_points: - # Forward transformation using our implementation - custom_x, custom_y = custom_proj.forward(latitude=lat, longitude=lon) - - # Forward transformation using PROJ - # Note: PROJ expects (lon, lat) order, not (lat, lon) - # proj_x, proj_y = proj_proj(np.radians(lon), np.radians(lat)) - proj_x, proj_y = proj_proj(lon, lat) - # The following fix should be available in proj, but something is weird - # with radians/degrees with ob_tran.... - # https://github.com/OSGeo/PROJ/issues/2804 - proj_x = np.degrees(proj_x) - proj_y = np.degrees(proj_y) - - # Compare results - allowing for small differences due to floating point math - # Convert to radians for comparison since our implementation works in radians - assert abs(custom_x - proj_x) < 1e-5, f"X mismatch for ({lat}, {lon}): custom={custom_x}, proj={proj_x}" - assert abs(custom_y - proj_y) < 1e-5, f"Y mismatch for ({lat}, {lon}): custom={custom_y}, proj={proj_y}" - - # Test inverse transformation - custom_lat, custom_lon = custom_proj.inverse(x=custom_x, y=custom_y) - # PROJ expects inverse=True for inverse transform - proj_lon, proj_lat = proj_proj(np.radians(proj_x), np.radians(proj_y), inverse=True) - - # Compare results - assert abs(custom_lat - proj_lat) < 1e-5, ( - f"Lat mismatch for ({custom_x}, {custom_y}): custom={custom_lat}, proj={proj_lat}" - ) - assert abs(np.mod(custom_lon - proj_lon + 180, 360) - 180) < 1e-5, ( - f"Lon mismatch for ({custom_x}, {custom_y}): custom={custom_lon}, proj={proj_lon}" - ) - - -def test_stereographic_against_proj(): - # Create our custom projection - latitude = 90.0 # North pole - longitude = 249.0 - radius = 6371229.0 - custom_proj = StereographicProjection(latitude=latitude, longitude=longitude, radius=radius) - - # Create equivalent PROJ projection - proj_string = f"+proj=stere +lat_0={latitude} +lon_0={longitude} +k=1 +x_0=0 +y_0=0 +R={radius} +units=m +no_defs" - proj_proj = pyproj.Proj(proj_string) - - # Test points - staying away from singular points (poles) - test_points = [ - (0, 0), # Equator - (45, 45), # Mid-latitude - (60, -120), # Northern regions - (45, 249), # Along the central meridian - (70, 249), # Along the central meridian - (80, 249), # Along the central meridian - ] - - for lat, lon in test_points: - # Forward transformation using our implementation - custom_x, custom_y = custom_proj.forward(latitude=lat, longitude=lon) - - # Forward transformation using PROJ - # PROJ uses (lon, lat) order - proj_x, proj_y = proj_proj(lon, lat) - - # Compare results (allowing some tolerance due to potential differences in algorithms) - # Stereographic projections can have larger errors for points far from the center - tolerance = 1 # tolerance in meters - assert abs(custom_x - proj_x) < tolerance, f"X mismatch for ({lat}, {lon}): custom={custom_x}, proj={proj_x}" - assert abs(custom_y - proj_y) < tolerance, f"Y mismatch for ({lat}, {lon}): custom={custom_y}, proj={proj_y}" - - # Test inverse transformation - custom_lat, custom_lon = custom_proj.inverse(x=custom_x, y=custom_y) - - proj_lon, proj_lat = proj_proj(proj_x, proj_y, inverse=True) - - # Compare results - assert abs(custom_lat - proj_lat) < 1e-5, f"Lat mismatch: custom={custom_lat}, proj={proj_lat}" - custom_lon = _normalize_longitude(custom_lon) - assert abs(custom_lon - proj_lon) < 1e-4, f"Lon mismatch: custom={custom_lon}, proj={proj_lon}" - - -def test_lambert_azimuthal_equal_area_against_proj(ukmo2_wkt): - # Create our custom projection - lambda_0 = -2.5 # Central longitude in degrees - phi_1 = 54.9 # Standard parallel/latitude in degrees - radius = 6371229.0 # Earth radius in meters - custom_proj = LambertAzimuthalEqualAreaProjection(lambda_0=lambda_0, phi_1=phi_1, radius=radius) - - # Create equivalent PROJ projection - # For Lambert Azimuthal Equal Area, we use lat_0 for the standard parallel and lon_0 for central longitude - proj_string = f"+proj=laea +lat_0={phi_1} +lon_0={lambda_0} +x_0=0 +y_0=0 +R={radius} +units=m +no_defs +type=crs" - proj_proj = pyproj.Proj(proj_string) - print(proj_proj) - proj_proj = pyproj.Proj(ukmo2_wkt) - print(proj_proj) - - # Test points covering different regions - test_points = [ - (0, 0), # Origin - (54.9, -2.5), # Projection center (should map to 0,0) - (45, 45), # Mid-latitude point - (-45, -45), # Mid-latitude point (southern hemisphere) - (10, 50), # Europe - (40, -100), # North America - (50, -170), # Pacific - (-30, 170), # South Pacific - # Test point from the existing test - (57.745566, 10.620785), - ] - - for lat, lon in test_points: - # Forward transformation using our implementation - custom_x, custom_y = custom_proj.forward(latitude=lat, longitude=lon) - - # Forward transformation using PROJ - # Note: PROJ expects (lon, lat) order, not (lat, lon) - proj_x, proj_y = proj_proj(lon, lat) - - # Compare results - Lambert projections can have larger differences due to algorithmic differences - # Use a reasonable tolerance (e.g., 0.1 meter for a 6.3 million meter radius) - tolerance = 0.1 - assert abs(custom_x - proj_x) < tolerance, f"X mismatch for ({lat}, {lon}): custom={custom_x}, proj={proj_x}" - assert abs(custom_y - proj_y) < tolerance, f"Y mismatch for ({lat}, {lon}): custom={custom_y}, proj={proj_y}" - - # Test inverse transformation (skip points very close to the poles where inverse can be unstable) - if abs(lat) < 89: - custom_lat, custom_lon = custom_proj.inverse(x=custom_x, y=custom_y) - # PROJ expects inverse=True for inverse transform - proj_lon, proj_lat = proj_proj(proj_x, proj_y, inverse=True) - - # Compare results with appropriate tolerance - # For inverse transformations, angular differences can be larger - angular_tolerance = 1e-5 # roughly 0.00001 degrees - assert abs(custom_lat - proj_lat) < angular_tolerance, ( - f"Lat mismatch for ({custom_x}, {custom_y}): custom={custom_lat}, proj={proj_lat}" - ) - - # Handle longitude wraparound for comparison - lon_diff = np.mod(abs(custom_lon - proj_lon), 360) - assert min(lon_diff, 360 - lon_diff) < angular_tolerance, ( - f"Lon mismatch for ({custom_x}, {custom_y}): custom={custom_lon}, proj={proj_lon}" - ) - - -def test_lambert_conformal_conic_against_proj(): - # Create our custom projection with parameters from the existing test - lambda_0 = 352 # Reference longitude in degrees - phi_0 = 55.5 # Reference latitude in degrees - phi_1 = 55.5 # First standard parallel in degrees - phi_2 = 55.5 # Second standard parallel in degrees - radius = 6371229.0 # Earth radius in meters - - custom_proj = LambertConformalConicProjection( - lambda_0=lambda_0, phi_0=phi_0, phi_1=phi_1, phi_2=phi_2, radius=radius - ) - - lambda_0_norm = _normalize_longitude(lambda_0) - # Create equivalent PROJ projection - # For Lambert Conformal Conic, we use lat_0, lon_0, lat_1, lat_2 parameters - proj_string = ( - f"+proj=lcc +lat_0={phi_0} +lon_0={lambda_0_norm} +lat_1={phi_1} +lat_2={phi_2} " - f"+x_0=0 +y_0=0 +R={radius} +units=m +no_defs +type=crs" - ) - proj_proj = pyproj.Proj(proj_string) - - # Test points from the existing test - center_lat = 39.671 - center_lon = -25.421997 - test_points = [ - (center_lat, center_lon), # Center point - (39.675304, -25.400146), # Near the center - (42.18604, -15.30127), # Point from the test (x=456, y=64) - (64.943695, 30.711975), # Point from the test (x=1642, y=1573) - # Additional test points for broader coverage - (0, 0), # Origin - (phi_0, lambda_0_norm), # Projection origin - (45, 0), # Mid-latitude point - (-45, -45), # Southern hemisphere - (10, 50), # Europe - (40, -100), # North America - (50, -170), # Pacific - (-30, 170), # South Pacific - ] - - for lat, lon in test_points: - # Forward transformation using our implementation - custom_x, custom_y = custom_proj.forward(latitude=lat, longitude=lon) - - # Forward transformation using PROJ - # Note: PROJ expects (lon, lat) order, not (lat, lon) - proj_x, proj_y = proj_proj(lon, lat) - tolerance = 0.1 # 0.1 meters for a 6.3 million meter radius is a reasonable precision - assert abs(custom_x - proj_x) < tolerance, f"X mismatch for ({lat}, {lon}): custom={custom_x}, proj={proj_x}" - assert abs(custom_y - proj_y) < tolerance, f"Y mismatch for ({lat}, {lon}): custom={custom_y}, proj={proj_y}" - - # Test inverse transformation - custom_lat, custom_lon = custom_proj.inverse(x=custom_x, y=custom_y) - # PROJ expects inverse=True for inverse transform - proj_lon, proj_lat = proj_proj(proj_x, proj_y, inverse=True) - angular_tolerance = 1e-5 # approximately 0.00001 degrees - assert abs(custom_lat - proj_lat) < angular_tolerance, ( - f"Lat mismatch for ({custom_x}, {custom_y}): custom={custom_lat}, proj={proj_lat}" - ) - - # Handle longitude wraparound for comparison - lon_diff = np.mod(abs(custom_lon - proj_lon), 360) - assert min(lon_diff, 360 - lon_diff) < angular_tolerance, ( - f"Lon mismatch for ({custom_x}, {custom_y}): custom={custom_lon}, proj={proj_lon}" - ) - - -# def test_regular_lat_lon_grid_against_proj(): -# """Test that RegularLatLonGrid operations match proj equivalent operations""" -# # Create a regular lat-lon grid with 1-degree steps -# grid = RegularLatLonGrid( -# lat_start=-90, -# lat_steps=181, # -90 to 90 -# lat_step_size=1.0, -# lon_start=-180, -# lon_steps=360, # -180 to 180 -# lon_step_size=1.0, -# ) - -# # Create proj objects for WGS84 lat/lon -# proj_wgs84 = pyproj.Proj(proj="latlong", datum="WGS84") - -# # Test points covering different scenarios -# test_points: list[tuple[float, float]] = [ -# (0, 0), # Origin -# (45, 45), # NE quadrant -# (-45, -45), # SW quadrant -# (45, -45), # SE quadrant -# (-45, 45), # NW quadrant -# (89, 0), # Near North pole -# (-89, 0), # Near South pole -# (0, 179), # Near date line (east) -# (0, -179), # Near date line (west) -# (10, 20), # Random point -# (-33, 151), # Sydney -# (37, -122), # San Francisco -# ] - -# for lat, lon in test_points: -# # Get grid coordinates using our implementation -# grid_sel = grid.findPointXy(lat, lon) -# assert type(grid_sel) is tuple -# grid_x, grid_y = grid_sel -# result_lat, result_lon = grid.getCoordinates(grid_x, grid_y) - -# # For a lat/lon grid, proj just keeps the same coordinates -# proj_x, proj_y = proj_wgs84(lon, lat) # Note: proj uses (lon, lat) order -# proj_lat, proj_lon = proj_wgs84(proj_y, proj_x, inverse=True) # Get back lat/lon from proj - -# # We'll check that our forward and inverse transformations are consistent -# # and match with proj's (which just returns the original coordinates for this projection) - -# # Check roundtrip accuracy -# assert abs(result_lat - lat) < 1e-9, f"Lat roundtrip error: original={lat}, result={result_lat}" - -# # Normalize longitudes before comparison due to -180/180 wrapping -# lon_norm = _normalize_longitude(lon) -# result_lon_norm = _normalize_longitude(result_lon) -# assert abs(result_lon_norm - lon_norm) < 1e-9, f"Lon roundtrip error: original={lon}, result={result_lon}" - -# # Verify agreement with proj -# assert abs(lat - proj_lat) < 1e-9 -# assert abs(lon_norm - _normalize_longitude(proj_lon)) < 1e-9, f"Lon mismatch with proj at ({lat}, {lon})" - -# # Test longitude wrapping behavior -# wrap_test_points = [ -# (0, 185), # Should wrap to (0, -175) -# (0, -185), # Should wrap to (0, 175) -# (0, 361), # Should wrap to (0, 1) -# (0, -361), # Should wrap to (0, -1) -# (45, 540), # Should wrap to (45, -180) -# ] - -# for lat, lon in wrap_test_points: -# # Find grid coordinates for the wrapped point -# result = grid.findPointXy(lat, lon) -# assert result is not None, "Point not found in grid" -# grid_x, grid_y = result - -# # Find grid coordinates for the normalized longitude -# norm_lon = cast(float, _normalize_longitude(lon)) -# result = grid.findPointXy(lat=lat, lon=norm_lon) -# assert result is not None, "Point not found in grid" -# norm_grid_x, norm_grid_y = result -# assert grid_x == norm_grid_x, f"Grid X mismatch for wrapped lon: {lon} vs {norm_lon}" -# assert grid_y == norm_grid_y, f"Grid Y mismatch for wrapped lon: {lon} vs {norm_lon}" - - -def test_proj_projection_grid(): - """Test that ProjProjection correctly wraps proj transformations""" - # Test with a Lambert Conformal Conic projection - proj_string = "+proj=lcc +lat_0=55.5 +lon_0=352 +lat_1=55.5 +lat_2=55.5 +x_0=0 +y_0=0 +R=6371229 +units=m +no_defs" - - # Create our projection wrapper - proj = ProjProjection(proj_string) - - # Create a grid using this projection - grid = ProjectionGrid.from_bounds( - nx=100, ny=100, lat_range=(39.67, 64.94), lon_range=(-25.42, 30.71), projection=proj - ) - - # Test points - test_points = [ - (39.671, -25.421997), # Lower left - (64.943695, 30.711975), # Upper right - (50.0, 0.0), # Middle-ish - ] - - # Create the raw proj transformer for comparison - raw_proj = pyproj.Proj(proj_string) - - for lat, lon in test_points: - # Forward transformation - grid_x, grid_y = proj.forward(lat, lon) - raw_x, raw_y = raw_proj(lon, lat) # Note: raw proj expects (lon, lat) - - # Compare forward results - assert abs(grid_x - raw_x) < 1e-8, f"X mismatch: {grid_x} vs {raw_x}" - assert abs(grid_y - raw_y) < 1e-8, f"Y mismatch: {grid_y} vs {raw_y}" - - # Inverse transformation - back_lat, back_lon = proj.inverse(grid_x, grid_y) - raw_lon, raw_lat = raw_proj(raw_x, raw_y, inverse=True) - - # Compare inverse results - assert abs(back_lat - raw_lat) < 1e-8, f"Lat mismatch: {back_lat} vs {raw_lat}" - assert abs(back_lon - raw_lon) < 1e-8, f"Lon mismatch: {back_lon} vs {raw_lon}" - - # Test roundtrip through the grid - grid_coords = grid.findPointXy(lat=lat, lon=lon) - assert grid_coords is not None, f"Grid coordinates not found for lat={lat}, lon={lon}" - result_lat, result_lon = grid.getCoordinates(*grid_coords) - - # Results should match within grid resolution - assert abs(result_lat - lat) < grid.dy, f"Grid lat error: {result_lat} vs {lat}" - assert abs(result_lon - lon) < grid.dx, f"Grid lon error: {result_lon} vs {lon}" - - -def test_grid_equivalence_lcc(): - """Test that a proj-based grid matches the original implementation""" - # Create original LambertConformalConic projection - original_proj = LambertConformalConicProjection(lambda_0=352, phi_0=55.5, phi_1=55.5, phi_2=55.5, radius=6371229.0) - - # Create equivalent proj-based projection - proj_proj = ProjProjection( - "+proj=lcc +lat_0=55.5 +lon_0=352 +lat_1=55.5 +lat_2=55.5 +x_0=0 +y_0=0 +R=6371229 +units=m +no_defs" - ) - - # Create grids with both projections - grid_bounds = { - "nx": 100, - "ny": 100, - "lat_range": (39.67, 64.94), - "lon_range": (-25.42, 30.71), - } - - original_grid = ProjectionGrid.from_bounds(projection=original_proj, **grid_bounds) - proj_grid = ProjectionGrid.from_bounds(projection=proj_proj, **grid_bounds) - - # Test points - test_points = [ - (39.671, -25.421997), # Lower left - (64.943695, 30.711975), # Upper right - (50.0, 0.0), # Middle-ish - (45.0, -10.0), # Random point - (60.0, 20.0), # Random point - ] - - for lat, lon in test_points: - # Compare projection results - orig_x, orig_y = original_proj.forward(lat, lon) - proj_x, proj_y = proj_proj.forward(lat, lon) - - # Results should match within reasonable tolerance - assert abs(orig_x - proj_x) < 1e-3, f"X mismatch: {orig_x} vs {proj_x}" - assert abs(orig_y - proj_y) < 1e-3, f"Y mismatch: {orig_y} vs {proj_y}" - - # Compare grid results - orig_grid_xy = original_grid.findPointXy(lat=lat, lon=lon) - proj_grid_xy = proj_grid.findPointXy(lat=lat, lon=lon) - - # Both should find the point or both should not find it - assert (orig_grid_xy is None) == (proj_grid_xy is None), f"Inconsistent point finding for ({lat}, {lon})" - if orig_grid_xy is None or proj_grid_xy is None: - return - - # Grid coordinates should match exactly - assert abs(orig_grid_xy[0] - proj_grid_xy[0]) < 1e-8, f"Grid X mismatch: {orig_grid_xy[0]} vs {proj_grid_xy[0]}" - assert abs(orig_grid_xy[1] - proj_grid_xy[1]) < 1e-8, f"Grid Y mismatch: {orig_grid_xy[1]} vs {proj_grid_xy[1]}" - - -# def test_grid_equivalence_regular_latlon(): -# """Test that a proj-based regular lat-lon grid matches the original implementation""" -# # Create a regular lat-lon grid using RegularLatLonGrid -# original_grid = RegularLatLonGrid( -# lat_start=10.0, lat_steps=100, lat_step_size=0.5, lon_start=-30.0, lon_steps=120, lon_step_size=0.5 -# ) - -# # Create equivalent proj-based regular lat-lon projection -# proj_proj = ProjProjection("+proj=longlat +datum=WGS84 +no_defs") -# proj_grid = ProjectionGrid(projection=proj_proj, nx=120, ny=100, origin=(-30.0, 10.0), dx=0.5, dy=0.5) - -# # Test points covering various areas within the grid -# test_points = [ -# (10.0, -30.0), # Lower left corner -# (59.5, 29.5), # Upper right corner -# (35.0, 0.0), # Middle-ish -# (20.0, -15.0), # Random point -# (50.0, 20.0), # Random point -# (15.25, -25.75), # Point between grid cells -# ] - -# for lat, lon in test_points: -# # Get grid points using both implementations -# orig_grid_xy = original_grid.findPointXy(lat=lat, lon=lon) -# proj_grid_xy = proj_grid.findPointXy(lat=lat, lon=lon) - -# # Both should find the point -# assert orig_grid_xy is not None and proj_grid_xy is not None, f"Point not found for ({lat}, {lon})" - -# # If point is found, coordinates should match exactly -# if orig_grid_xy is not None: -# assert abs(orig_grid_xy[0] - proj_grid_xy[0]) < 1e-8, ( -# f"Grid X mismatch: {orig_grid_xy[0]} vs {proj_grid_xy[0]}" -# ) -# assert abs(orig_grid_xy[1] - proj_grid_xy[1]) < 1e-8, ( -# f"Grid Y mismatch: {orig_grid_xy[1]} vs {proj_grid_xy[1]}" -# ) - -# # Test the inverse transformation (getCoordinates) -# if orig_grid_xy is not None: -# x, y = orig_grid_xy -# orig_lat, orig_lon = original_grid.getCoordinates(x, y) -# proj_lat, proj_lon = proj_grid.getCoordinates(x, y) - -# # Results should match exactly -# assert abs(orig_lat - proj_lat) < 1e-8, f"Latitude mismatch: {orig_lat} vs {proj_lat}" -# assert abs(orig_lon - proj_lon) < 1e-8, f"Longitude mismatch: {orig_lon} vs {proj_lon}" - - -def test_grid_equivalence_rotated_latlon(): - """Test that a proj-based rotated lat-lon grid matches the original implementation""" - lat_origin = -36.0885 - lon_origin = 245.305 - - original_proj = RotatedLatLonProjection(lat_origin=lat_origin, lon_origin=lon_origin) - proj_proj = ProjProjection( - f"+proj=ob_tran +o_proj=longlat +o_lat_p={-lat_origin} +o_lon_p=0.0 +lon_0={lon_origin} +datum=WGS84 +no_defs +type=crs" - ) - - # Grid bounds could be adjusted to something used in the open-meteo backend - grid_bounds = { - "nx": 100, - "ny": 80, - "lat_range": (-40.0, 40.0), - "lon_range": (200.0, 300.0), - } - - original_grid = ProjectionGrid.from_bounds(projection=original_proj, **grid_bounds) - proj_grid = ProjectionGrid.from_bounds(projection=proj_proj, **grid_bounds) - - # Test points covering various areas within the grid - test_points = [ - (0, 0), # Origin - (45, 45), # Mid-latitude - (-45, -45), # Southern hemisphere - (10, 50), # Europe - (40, -100), # North America - (50, -170), # Pacific - (-30, 170), # South Pacific - ] - - for lat, lon in test_points: - # Compare projection results - orig_x, orig_y = original_proj.forward(lat, lon) - proj_x, proj_y = proj_proj.forward(lat, lon) - assert abs(orig_x - proj_x) < 1e-3, f"X mismatch: {orig_x} vs {proj_x}" - assert abs(orig_y - proj_y) < 1e-3, f"Y mismatch: {orig_y} vs {proj_y}" - - # Compare grid results - orig_grid_xy = original_grid.findPointXy(lat=lat, lon=lon) - proj_grid_xy = proj_grid.findPointXy(lat=lat, lon=lon) - assert orig_grid_xy == proj_grid_xy, f"Grid index mismatch: {orig_grid_xy} vs {proj_grid_xy}" - - # Test the inverse transformation (getCoordinates) - if orig_grid_xy is not None: - x, y = orig_grid_xy - orig_lat, orig_lon = original_grid.getCoordinates(x, y) - proj_lat, proj_lon = proj_grid.getCoordinates(x, y) - assert abs(orig_lat - proj_lat) < 1e-4, f"Latitude mismatch: {orig_lat} vs {proj_lat}" - assert abs(orig_lon - proj_lon) < 1e-4, f"Longitude mismatch: {orig_lon} vs {proj_lon}" From 3f2a76e78ea52b3242f8be6ac82a1de0d8c57fcf Mon Sep 17 00:00:00 2001 From: terraputix Date: Tue, 13 Jan 2026 19:57:20 +0100 Subject: [PATCH 28/50] more cleanup --- python/omfiles/grids.py | 909 ---------------------------------------- 1 file changed, 909 deletions(-) delete mode 100644 python/omfiles/grids.py diff --git a/python/omfiles/grids.py b/python/omfiles/grids.py deleted file mode 100644 index dd1e3869..00000000 --- a/python/omfiles/grids.py +++ /dev/null @@ -1,909 +0,0 @@ -"""Grids used in Open-Meteo files.""" - -from abc import ABC, abstractmethod -from functools import cached_property -from typing import Generic, Optional, Tuple, TypeVar, cast - -import numpy as np -import numpy.typing as npt - -from omfiles._utils import _modulo_positive, _normalize_longitude -from omfiles.types import ArrayType, CoordType, ReturnUnionType - - -class AbstractGrid(ABC): - """ - Abstract base class for weather model grid definitions. - - This defines the interface that all grid implementations must follow. - """ - - @property - @abstractmethod - def grid_type(self) -> str: - """Return the grid type identifier.""" - pass - - @cached_property - @abstractmethod - def latitude(self) -> np.ndarray: - """ - Return the latitude coordinates array. - - Returns: - np.ndarray: Array of latitude coordinates. - """ - pass - - @cached_property - @abstractmethod - def longitude(self) -> np.ndarray: - """ - Return the longitude coordinates array. - - Returns: - np.ndarray: Array of longitude coordinates. - """ - pass - - @property - @abstractmethod - def shape(self) -> Tuple[int, int]: - """ - Return the grid shape as (n_lat, n_lon). - - Returns: - Tuple[int, int]: Shape of the grid as (n_lat, n_lon). - """ - pass - - @abstractmethod - def findPointXy(self, lat: float, lon: float) -> Optional[Tuple[int, int]]: - """ - Find grid point indices (x, y) for given lat/lon coordinates. - - Args: - lat (float): Latitude in degrees. - lon (float): Longitude in degrees. - - Returns: - Optional[Tuple[int, int]]: (x, y) grid indices if point is in grid, None otherwise. - """ - pass - - @abstractmethod - def getCoordinates(self, x: int, y: int) -> Tuple[float, float]: - """ - Get lat/lon coordinates for a given grid point indices. - - Args: - x (int): Grid x index. - y (int): Grid y index. - - Returns: - Tuple[float, float]: (latitude, longitude) coordinates. - """ - pass - - -class RegularLatLonGrid(AbstractGrid): - """ - Regular latitude-longitude grid implementation. - - Represents a standard equirectangular grid with uniform spacing. - """ - - def __init__( - self, - lat_start: float, - lat_steps: int, - lat_step_size: float, - lon_start: float, - lon_steps: int, - lon_step_size: float, - ): - """ - Initialize a regular lat/lon grid. - - Args: - lat_start (float): Starting latitude value. - lat_steps (int): Number of latitude points. - lat_step_size (float): Spacing between latitude points. - lon_start (float): Starting longitude value. - lon_steps (int): Number of longitude points. - lon_step_size (float): Spacing between longitude points. - """ - self._lat_start = lat_start - self._lat_steps = lat_steps - self._lat_step_size = lat_step_size - self._lon_start = lon_start - self._lon_steps = lon_steps - self._lon_step_size = lon_step_size - - @property - def grid_type(self) -> str: - """ - Grid type identifier. - - Returns: - str: The grid type identifier. - """ - return "regular_latlon" - - @cached_property - def latitude(self) -> np.ndarray: - """ - Compute and cache the latitude coordinate array. - - Returns: - np.ndarray: Array of latitude coordinates. - """ - return np.linspace( - self._lat_start, self._lat_start + self._lat_step_size * self._lat_steps, self._lat_steps, endpoint=False - ) - - @cached_property - def longitude(self) -> np.ndarray: - """ - Compute and cache the longitude coordinate array. - - Returns: - np.ndarray: Array of longitude coordinates. - """ - return np.linspace( - self._lon_start, self._lon_start + self._lon_step_size * self._lon_steps, self._lon_steps, endpoint=False - ) - - @property - def shape(self) -> Tuple[int, int]: - """ - Grid shape as (n_lat, n_lon). - - Returns: - Tuple[int, int]: Shape of the grid as (n_lat, n_lon). - """ - return (self._lat_steps, self._lon_steps) - - def findPointXy(self, lat: float, lon: float) -> Optional[Tuple[int, int]]: - """ - Find grid point indices (x, y) for given lat/lon coordinates. - - Args: - lat (float): Latitude in degrees. - lon (float): Longitude in degrees. - - Returns: - Optional[Tuple[int, int]]: (x, y) grid indices if point is in grid, None otherwise. - """ - # Calculate raw x and y indices - x = int(round((lon - self._lon_start) / self._lon_step_size)) - y = int(round((lat - self._lat_start) / self._lat_step_size)) - - # Handle wrapping for global grids - xx = _modulo_positive(x, self._lon_steps) if (self._lon_steps * self._lon_step_size) >= 359 else x - yy = _modulo_positive(y, self._lat_steps) if (self._lat_steps * self._lat_step_size) >= 179 else y - - # Check if point is within grid bounds - if yy < 0 or xx < 0 or yy >= self._lat_steps or xx >= self._lon_steps: - return None - - return (xx, yy) - - def getCoordinates(self, x: int, y: int) -> Tuple[float, float]: - """ - Get lat/lon coordinates for a given grid point indices. - - Args: - x (int): Longitude index. - y (int): Latitude index. - - Returns: - Tuple[float, float]: (latitude, longitude) coordinates. - """ - lat = self._lat_start + float(y) * self._lat_step_size - lon = self._lon_start + float(x) * self._lon_step_size - - return (lat, lon) - - -class AbstractProjection(ABC): - """Base class for projection implementations.""" - - @abstractmethod - def forward(self, latitude: CoordType, longitude: CoordType) -> ReturnUnionType: - """ - Transform from lat/lon coordinates to projected x/y coordinates. - - Handles both scalar and array inputs transparently. - """ - pass - - @abstractmethod - def inverse(self, x: CoordType, y: CoordType) -> ReturnUnionType: - """ - Transform from projected x/y coordinates back to lat/lon. - - Handles both scalar and array inputs transparently. - """ - pass - - -class RotatedLatLonProjection(AbstractProjection): - """ - Rotated lat/lon projection implementation. - - This implements the transformation between regular lat/lon coordinates and - rotated lat/lon coordinates where the pole is shifted to a specified location. - Based on: https://github.com/open-meteo/open-meteo/blob/main/Sources/App/Domains/RotatedLatLon.swift - """ - - def __init__(self, lat_origin: float, lon_origin: float): - """ - Initialize a rotated lat/lon projection. - - Args: - lat_origin (float): Latitude of origin in degrees. - lon_origin (float): Longitude of origin in degrees. - """ - # θ: Rotation around y-axis - self.theta = np.radians(90.0 + lat_origin) - # ϕ: Rotation around z-axis - self.phi = np.radians(lon_origin) - - def forward(self, latitude: CoordType, longitude: CoordType) -> ReturnUnionType: - """ - Transform from regular lat/lon to rotated lat/lon coordinates. - - Args: - latitude (float or array): Latitude in degrees. - longitude (float or array): Longitude in degrees. - - Returns: - tuple: (rotated_lat, rotated_lon) in degrees. - """ - scalar_input = np.isscalar(latitude) and np.isscalar(longitude) - - lat_arr = np.asarray(latitude, dtype=np.float32) - lon_arr = np.asarray(longitude, dtype=np.float32) - - # Convert to radians - lat_rad = np.radians(lat_arr) - lon_rad = np.radians(lon_arr) - - # Convert to cartesian coordinates - x = np.cos(lon_rad) * np.cos(lat_rad) - y = np.sin(lon_rad) * np.cos(lat_rad) - z = np.sin(lat_rad) - - # Apply rotation - x2 = ( - np.cos(self.theta) * np.cos(self.phi) * x - + np.cos(self.theta) * np.sin(self.phi) * y - + np.sin(self.theta) * z - ) - y2 = -np.sin(self.phi) * x + np.cos(self.phi) * y - z2 = ( - -np.sin(self.theta) * np.cos(self.phi) * x - - np.sin(self.theta) * np.sin(self.phi) * y - + np.cos(self.theta) * z - ) - - # Convert back to spherical coordinates - rot_lon = np.degrees(np.arctan2(y2, x2)) - rot_lat = np.degrees(np.arcsin(z2)) - - if scalar_input: - return float(rot_lon.item()), float(rot_lat.item()) - - return rot_lon, rot_lat - - def inverse(self, x: CoordType, y: CoordType) -> ReturnUnionType: - """ - Transform from rotated lat/lon back to regular lat/lon coordinates. - - Args: - x (float or array): Rotated longitude in degrees. - y (float or array): Rotated latitude in degrees. - - Returns: - tuple: (latitude, longitude) in degrees. - """ - scalar_input = np.isscalar(x) and np.isscalar(y) - - rot_lon = np.radians(np.asarray(x, dtype=np.float32)) - rot_lat = np.radians(np.asarray(y, dtype=np.float32)) - - theta_neg = -self.theta - phi_neg = -self.phi - - # Quick solution without conversion in cartesian space - lat_rad = np.arcsin(np.cos(theta_neg) * np.sin(rot_lat) - np.cos(rot_lon) * np.sin(theta_neg) * np.cos(rot_lat)) - - lon_rad = ( - np.arctan2(np.sin(rot_lon), np.tan(rot_lat) * np.sin(theta_neg) + np.cos(rot_lon) * np.cos(theta_neg)) - - phi_neg - ) - - lat2 = np.degrees(lat_rad) - lon2 = np.degrees(lon_rad) - - if scalar_input: - return float(lat2.item()), float(lon2.item()) - - return lat2, lon2 - - -class StereographicProjection(AbstractProjection): - """ - Stereographic projection implementation. - - This implements the equations for the stereographic projection - which projects a sphere onto a plane. - https://mathworld.wolfram.com/StereographicProjection.html - """ - - def __init__(self, latitude: float, longitude: float, radius: float = 6371000.0): - """ - Initialize a stereographic projection. - - Args: - latitude (float): Central latitude in degrees. - longitude (float): Central longitude in degrees. - radius (float, optional): Radius of Earth in meters. Defaults to 6371000.0. - """ - self.lambda_0: npt.NDArray[np.float32] = np.radians(longitude) - self.sin_phi_1: npt.NDArray[np.float32] = np.sin(np.radians(latitude)) - self.cos_phi_1: npt.NDArray[np.float32] = np.cos(np.radians(latitude)) - self.R = radius - - def forward(self, latitude: CoordType, longitude: CoordType) -> ReturnUnionType: - """ - Transform from lat/lon coordinates to projected x/y coordinates. - - Args: - latitude (float or array): Latitude in degrees. - longitude (float or array): Longitude in degrees. - - Returns: - tuple: (x, y) coordinates in the projection. - """ - scalar_input = np.isscalar(latitude) and np.isscalar(longitude) - - lat_arr = np.asarray(latitude, dtype=np.float32) - lon_arr = np.asarray(longitude, dtype=np.float32) - - phi = np.radians(lat_arr) - lambda_ = np.radians(lon_arr) - k = ( - 2 - * self.R - / (1 + self.sin_phi_1 * np.sin(phi) + self.cos_phi_1 * np.cos(phi) * np.cos(lambda_ - self.lambda_0)) - ) - x = k * np.cos(phi) * np.sin(lambda_ - self.lambda_0) - y = k * (self.cos_phi_1 * np.sin(phi) - self.sin_phi_1 * np.cos(phi) * np.cos(lambda_ - self.lambda_0)) - - if scalar_input: - return float(x.item()), float(y.item()) - - return x, y - - def inverse(self, x: CoordType, y: CoordType) -> ReturnUnionType: - """ - Transform from projected x/y coordinates back to lat/lon. - - Args: - x (float or array): X coordinate in the projection. - y (float or array): Y coordinate in the projection. - - Returns: - tuple: (latitude, longitude) in degrees. - """ - x_arr = np.asarray(x, dtype=np.float32) - y_arr = np.asarray(y, dtype=np.float32) - - p = np.sqrt(x_arr * x_arr + y_arr * y_arr) - - # Initialize output arrays - phi = np.zeros_like(p) - lambda_ = np.zeros_like(p) - - c = 2 * np.arctan2(p, 2 * self.R) - phi = np.arcsin(np.cos(c) * self.sin_phi_1 + (y_arr * np.sin(c) * self.cos_phi_1) / p) - lambda_ = self.lambda_0 + np.arctan2( - x_arr * np.sin(c), p * self.cos_phi_1 * np.cos(c) - y_arr * self.sin_phi_1 * np.sin(c) - ) - - return np.degrees(phi), np.degrees(lambda_) - - -class LambertAzimuthalEqualAreaProjection(AbstractProjection): - """ - Lambert Azimuthal Equal-Area projection implementation. - - Implements the equations for the Lambert Azimuthal Equal-Area projection, - which preserves area but not angles or distances. - - Reference: - https://mathworld.wolfram.com/LambertAzimuthalEqual-AreaProjection.html - """ - - def __init__(self, lambda_0: float, phi_1: float, radius: float = 6371229.0): - """ - Initialize a Lambert Azimuthal Equal-Area projection. - - Args: - lambda_0 (float): Central longitude in degrees. - phi_1 (float): Standard parallel in degrees. - radius (float, optional): Radius of Earth in meters. Defaults to 6371229.0. - """ - self.lambda_0 = np.radians(lambda_0) - self.phi_1 = np.radians(phi_1) - self.R = radius - - def forward(self, latitude: CoordType, longitude: CoordType) -> ReturnUnionType: - """ - Transform from lat/lon coordinates to projected x/y coordinates. - - Args: - latitude (float or array): Latitude in degrees. - longitude (float or array): Longitude in degrees. - - Returns: - tuple: (x, y) coordinates in the projection. - """ - scalar_input = np.isscalar(latitude) and np.isscalar(longitude) - - lat_arr = np.asarray(latitude, dtype=np.float64) - lon_arr = np.asarray(longitude, dtype=np.float64) - - lambda_ = np.radians(lon_arr) - phi = np.radians(lat_arr) - - k = np.sqrt( - 2 - / ( - 1 - + np.sin(self.phi_1) * np.sin(phi) - + np.cos(self.phi_1) * np.cos(phi) * np.cos(lambda_ - self.lambda_0) - ) - ) - - x = self.R * k * np.cos(phi) * np.sin(lambda_ - self.lambda_0) - y = ( - self.R - * k - * (np.cos(self.phi_1) * np.sin(phi) - np.sin(self.phi_1) * np.cos(phi) * np.cos(lambda_ - self.lambda_0)) - ) - - if scalar_input: - return float(x.item()), float(y.item()) - - return x, y - - def inverse(self, x: CoordType, y: CoordType) -> ReturnUnionType: - """ - Transform from projected x/y coordinates back to lat/lon. - - Args: - x (float or array): X coordinate in the projection. - y (float or array): Y coordinate in the projection. - - Returns: - tuple: (latitude, longitude) in degrees. - """ - scalar_input = np.isscalar(x) and np.isscalar(y) - - x_arr = np.asarray(x, dtype=np.float64) - y_arr = np.asarray(y, dtype=np.float64) - - x_norm = x_arr / self.R - y_norm = y_arr / self.R - p = np.sqrt(x_norm * x_norm + y_norm * y_norm) - - # Handle the case where p is zero (projection center) - zero_p = p == 0 - p = np.where(zero_p, np.finfo(np.float32).eps, p) # Avoid division by zero - - c = 2 * np.arcsin(0.5 * p) - phi = np.arcsin(np.cos(c) * np.sin(self.phi_1) + (y_norm * np.sin(c) * np.cos(self.phi_1)) / p) - lambda_ = self.lambda_0 + np.arctan2( - x_norm * np.sin(c), p * np.cos(self.phi_1) * np.cos(c) - y_norm * np.sin(self.phi_1) * np.sin(c) - ) - lat = np.degrees(phi) - lon = np.degrees(lambda_) - - if scalar_input: - return float(lat.item()), float(lon.item()) - - return lat, lon - - -class LambertConformalConicProjection(AbstractProjection): - """ - Lambert Conformal Conic projection implementation. - - Implements the equations for the Lambert Conformal Conic projection, - which preserves angles but not areas or distances. - - References: - https://mathworld.wolfram.com/LambertConformalConicProjection.html - https://pubs.usgs.gov/pp/1395/report.pdf page 104 - """ - - def __init__(self, lambda_0: float, phi_0: float, phi_1: float, phi_2: float, radius: float = 6370997): - """ - Initialize a Lambert Conformal Conic projection. - - Args: - lambda_0 (float): Reference longitude in degrees (LoVInDegrees in grib). - phi_0 (float): Reference latitude in degrees (LaDInDegrees in grib). - phi_1 (float): First standard parallel in degrees (Latin1InDegrees in grib). - phi_2 (float): Second standard parallel in degrees (Latin2InDegrees in grib). - radius (float): Radius of Earth in meters (default: 6370997). - """ - # Normalize lambda_0 to [-180, 180] range - lambda_0_normalized = _normalize_longitude(lambda_0) - self.lambda_0 = np.radians(lambda_0_normalized) - - phi_0_rad = np.radians(phi_0) - phi_1_rad = np.radians(phi_1) - phi_2_rad = np.radians(phi_2) - - if phi_1 == phi_2: - self.n = np.sin(phi_1_rad) - else: - self.n = np.log(np.cos(phi_1_rad) / np.cos(phi_2_rad)) / np.log( - np.tan(np.pi / 4 + phi_2_rad / 2) / np.tan(np.pi / 4 + phi_1_rad / 2) - ) - - self.F = (np.cos(phi_1_rad) * np.power(np.tan(np.pi / 4 + phi_1_rad / 2), self.n)) / self.n - - self.rho_0 = self.F / np.power(np.tan(np.pi / 4 + phi_0_rad / 2), self.n) - - # Earth radius - self.R = radius - - def forward(self, latitude: CoordType, longitude: CoordType) -> ReturnUnionType: - """ - Transform from lat/lon coordinates to projected x/y coordinates. - - Args: - latitude (float or array): Latitude in degrees. - longitude (float or array): Longitude in degrees. - - Returns: - tuple: (x, y) coordinates in the projection. - """ - scalar_input = np.isscalar(latitude) and np.isscalar(longitude) - - phi = np.radians(np.asarray(latitude, dtype=np.float64)) - lambda_ = np.radians(np.asarray(longitude, dtype=np.float64)) - - # If (λ - λ0) exceeds the range:±: 180°, 360° should be added or subtracted. - theta = self.n * (lambda_ - self.lambda_0) - - rho = self.F / np.power(np.tan(np.pi / 4 + phi / 2), self.n) - x = self.R * rho * np.sin(theta) - y = self.R * (self.rho_0 - rho * np.cos(theta)) - - if scalar_input: - return float(x.item()), float(y.item()) - - return x, y - - def inverse(self, x: CoordType, y: CoordType) -> ReturnUnionType: - """ - Transform from projected x/y coordinates back to lat/lon. - - Args: - x (float or array): X coordinate in the projection. - y (float or array): Y coordinate in the projection. - - Returns: - tuple: (latitude, longitude) in degrees. - """ - scalar_input = np.isscalar(x) and np.isscalar(y) - - x_scaled = np.asarray(x, dtype=np.float64) / self.R - y_scaled = np.asarray(y, dtype=np.float64) / self.R - - theta = np.where( - self.n >= 0, np.arctan2(x_scaled, self.rho_0 - y_scaled), np.arctan2(-x_scaled, y_scaled - self.rho_0) - ) - - sign = np.where(self.n > 0, 1, -1) - rho = sign * np.sqrt(np.square(x_scaled) + np.square(self.rho_0 - y_scaled)) - - phi = 2 * np.arctan(np.power(self.F / rho, 1 / self.n)) - np.pi / 2 - lambda_ = self.lambda_0 + theta / self.n - - lat = np.degrees(phi) - lon = np.degrees(lambda_) - - lon = np.where(lon > 180, lon - 360, lon) - - if scalar_input: - return float(lat.item()), float(lon.item()) - - return lat, lon - - -P = TypeVar("P", bound=AbstractProjection) - - -class ProjectionGrid(AbstractGrid, Generic[P]): - """ - Grid implementation using a projection. - - This represents a grid in a projected coordinate system. - """ - - def __init__(self, projection: P, nx: int, ny: int, origin: Tuple[float, float], dx: float, dy: float): - """ - Initialize a projection grid with all parameters. - - Args: - projection (Projectable): Projection implementation. - nx (int): Number of grid points in x direction. - ny (int): Number of grid points in y direction. - origin (Tuple[float, float]): Origin coordinates (x, y) of the grid in projection space. - dx (float): Grid spacing in x direction. - dy (float): Grid spacing in y direction. - """ - self.projection = projection - self.nx = nx - self.ny = ny - self.origin = origin - self.dx = dx - self.dy = dy - - @classmethod - def from_bounds( - cls, nx: int, ny: int, lat_range: Tuple[float, float], lon_range: Tuple[float, float], projection: P - ) -> "ProjectionGrid[P]": - """ - Create a projection grid from geographic bounds. - - Args: - nx (int): Number of grid points in x direction. - ny (int): Number of grid points in y direction. - lat_range (Tuple[float, float]): Latitude range (min, max) in degrees. - lon_range (Tuple[float, float]): Longitude range (min, max) in degrees. - projection (Projectable): Projection implementation. - - Returns: - ProjectionGrid: New grid instance. - """ - sw = projection.forward(lat_range[0], lon_range[0]) - ne = projection.forward(lat_range[1], lon_range[1]) - origin = cast(tuple[float, float], sw) - dx = (ne[0] - sw[0]) / (nx - 1) - dy = (ne[1] - sw[1]) / (ny - 1) - return cls(projection, nx, ny, origin, float(dx), float(dy)) - - @classmethod - def from_center( - cls, nx: int, ny: int, center_lat: float, center_lon: float, dx: float, dy: float, projection: P - ) -> "ProjectionGrid[P]": - """ - Create a projection grid centered at a geographic location. - - Args: - nx (int): Number of grid points in x direction. - ny (int): Number of grid points in y direction. - center_lat (float): Center latitude in degrees. - center_lon (float): Center longitude in degrees. - dx (float): Grid spacing in x direction in meters. - dy (float): Grid spacing in y direction in meters. - projection (Projectable): Projection implementation. - - Returns: - ProjectionGrid: New grid instance. - """ - center = cast(tuple[float, float], projection.forward(center_lat, center_lon)) - return cls(projection, nx, ny, center, dx, dy) - - @property - def grid_type(self) -> str: - """Grid type identifier.""" - return "projection" - - @cached_property - def _coordinates(self) -> Tuple[np.ndarray, np.ndarray]: - """ - Lazily compute and cache both latitude and longitude arrays. - - Returns: - Tuple[np.ndarray, np.ndarray]: Arrays of latitude and longitude coordinates. - """ - # Create meshgrid of coordinates - y_indices, x_indices = np.meshgrid(np.arange(self.ny), np.arange(self.nx), indexing="ij") - - # Convert to projected coordinates - x_coords = x_indices * self.dx + self.origin[0] - y_coords = y_indices * self.dy + self.origin[1] - - # Convert to lat/lon using vectorized inverse method - lat, lon = cast(tuple[ArrayType, ArrayType], self.projection.inverse(x_coords, y_coords)) - return lat, lon - - @property - def latitude(self) -> np.ndarray: # type: ignore - """ - Get the latitude coordinate array. - - Returns: - np.ndarray: Array of latitude coordinates. - """ - return self._coordinates[0] - - @property - def longitude(self) -> np.ndarray: # type: ignore - """ - Get the longitude coordinate array. - - Returns: - np.ndarray: Array of longitude coordinates. - """ - return self._coordinates[1] - - @property - def shape(self) -> Tuple[int, int]: - """ - Grid shape as (n_lat, n_lon). - - Returns: - Tuple[int, int]: Shape of the grid as (n_lat, n_lon). - """ - return (self.ny, self.nx) - - def findPointXy(self, lat: float, lon: float) -> Optional[Tuple[int, int]]: - """ - Find grid point indices (x, y) for given lat/lon coordinates. - - Args: - lat (float): Latitude in degrees. - lon (float): Longitude in degrees. - - Returns: - Optional[Tuple[int, int]]: (x, y) grid indices if point is in grid, None otherwise. - """ - pos = cast(tuple[float, float], self.projection.forward(lat, lon)) - x = int(round((pos[0] - self.origin[0]) / self.dx)) - y = int(round((pos[1] - self.origin[1]) / self.dy)) - - if y < 0 or x < 0 or y >= self.ny or x >= self.nx: - return None - - return (x, y) - - def getCoordinates(self, x: int, y: int) -> Tuple[float, float]: - """ - Get lat/lon coordinates for a given grid point indices. - - Args: - x (int): X index. - y (int): Y index. - - Returns: - Tuple[float, float]: (latitude, longitude) coordinates. - """ - xcord = float(x) * self.dx + self.origin[0] - ycord = float(y) * self.dy + self.origin[1] - lat, lon = self.projection.inverse(xcord, ycord) - # Normalize longitude to -180 to 180 range - lon = _normalize_longitude(lon) - lat, lon = cast(tuple[float, float], (lat, lon)) - return (lat, lon) - - def get_true_north_direction(self) -> np.ndarray: - """ - Calculate angle towards true north for every grid point. - - Returns: - np.ndarray: Array of angles in degrees, 0 = points towards north pole. - """ - pos = self.projection.forward(90, 0) # North pole - north_pole_x = (pos[0] - self.origin[0]) / self.dx - north_pole_y = (pos[1] - self.origin[1]) / self.dy - - # Create grid of x, y coordinates - y_indices, x_indices = np.meshgrid(np.arange(self.ny), np.arange(self.nx), indexing="ij") - - # Vectorized calculation of angles - true_north = np.degrees(np.arctan2(north_pole_x - x_indices, north_pole_y - y_indices)) - - return true_north - - def find_box(self, lat_min: float, lat_max: float, lon_min: float, lon_max: float) -> np.ndarray: - """ - Find indices of grid points within a geographic bounding box. - - Args: - lat_min (float): Minimum latitude. - lat_max (float): Maximum latitude. - lon_min (float): Minimum longitude. - lon_max (float): Maximum longitude. - - Returns: - np.ndarray: Array of grid point indices within the box. - """ - sw = self.findPointXy(lat_min, lon_min) - se = self.findPointXy(lat_min, lon_max) - nw = self.findPointXy(lat_max, lon_min) - ne = self.findPointXy(lat_max, lon_max) - - if not all([sw, se, nw, ne]): - return np.array([], dtype=int) - - # Type casting to inform pyright that these variables are not None - sw = cast(Tuple[int, int], sw) - se = cast(Tuple[int, int], se) - nw = cast(Tuple[int, int], nw) - ne = cast(Tuple[int, int], ne) - - x_min = min(sw[0], nw[0]) - x_max = max(se[0], ne[0]) + 1 - y_min = min(sw[1], se[1]) - y_max = max(nw[1], ne[1]) + 1 - - # Create meshgrid of indices - y_indices, x_indices = np.meshgrid(np.arange(y_min, y_max), np.arange(x_min, x_max), indexing="ij") - - # Convert to flat indices - return np.ravel_multi_index((y_indices.flatten(), x_indices.flatten()), (self.ny, self.nx)) - - -class ProjProjection(AbstractProjection): - """ - A projection that wraps a proj projection. - - """ - - def __init__(self, proj_string: str): - """ - Initialize with a proj string or EPSG code. - - Args: - proj_string (str): The proj string (e.g. "+proj=lcc +lat_0=50...") or - EPSG code (e.g. "EPSG:4326"). - """ - import pyproj - - # Create transformer from lat/lon to projection coordinates - self.crs_proj = pyproj.CRS(proj_string) - self.crs_latlon = pyproj.CRS("EPSG:4326") # WGS84 - self.forward_transformer = pyproj.Transformer.from_crs( - self.crs_latlon, - self.crs_proj, - always_xy=True, # This ensures lon/lat -> x/y order - ) - self.inverse_transformer = pyproj.Transformer.from_crs(self.crs_proj, self.crs_latlon, always_xy=True) - - def forward(self, latitude: CoordType, longitude: CoordType) -> ReturnUnionType: - """ - Transform from latitude/longitude to projection coordinates. - - Args: - latitude (float): Latitude in degrees. - longitude (float): Longitude in degrees. - - Returns: - tuple[float, float]: The (x, y) coordinates in the projection. - """ - x, y = self.forward_transformer.transform(longitude, latitude) - return x, y - - def inverse(self, x: CoordType, y: CoordType) -> ReturnUnionType: - """ - Transform from projection coordinates to latitude/longitude. - - Args: - x (float): X coordinate in the projection. - y (float): Y coordinate in the projection. - - Returns: - tuple[float, float]: The (latitude, longitude) coordinates in degrees. - """ - lon, lat = self.inverse_transformer.transform(x, y) - return lat, lon From 388ea0a386dcb665b04c6c6973d737d8c1cdc0d0 Mon Sep 17 00:00:00 2001 From: terraputix Date: Tue, 13 Jan 2026 20:44:27 +0100 Subject: [PATCH 29/50] test meta and remove domain tests --- tests/test_grids.py | 45 ++++++++++++++++++++++ tests/test_meta.py | 56 +++++++++++++++++++++++++++ tests/test_om_domains.py | 82 ---------------------------------------- 3 files changed, 101 insertions(+), 82 deletions(-) create mode 100644 tests/test_meta.py delete mode 100644 tests/test_om_domains.py diff --git a/tests/test_grids.py b/tests/test_grids.py index d2039577..9f1e2cfe 100644 --- a/tests/test_grids.py +++ b/tests/test_grids.py @@ -295,3 +295,48 @@ def test_lambert_conformal_conic_projection(dmi_harmoni_europe_wkt: str, dmi_har assert abs(lon - 30.711975) < 0.001 point_idx = dmi_harmoni_europe_grid.find_point_xy(lat=lat, lon=lon) assert point_idx == (1642, 1573) + + +# def test_dwd_icon_d2_grid_points(): +# """Test specific points in the DWD ICON D2 grid.""" +# dwd_grid = DOMAINS["dwd_icon_d2"].grid + +# # Test a point known to be in the domain (Central Europe) +# # Berlin coordinates: approx. 52.52°N, 13.40°E +# berlin = dwd_grid.findPointXy(52.52, 13.40) +# assert berlin is not None + +# # Test a point outside the domain (should return None) +# # New York coordinates: approx. 40.71°N, -74.01°E +# new_york = dwd_grid.findPointXy(40.71, -74.01) +# assert new_york is None + +# # Test gridpoint to coordinate conversion +# if berlin is not None: +# x, y = berlin +# lat, lon = dwd_grid.getCoordinates(x, y) +# # Check that we get close to the original coordinates +# assert abs(lat - 52.52) < 0.05 +# assert abs(lon - 13.40) < 0.05 + + +# def test_ecmwf_grid(): +# """Test the ECMWF IFS grid specifically.""" +# ecmwf_grid = DOMAINS["ecmwf_ifs025"].grid + +# # Test some known points on the grid +# # Point at the prime meridian and equator +# assert ecmwf_grid.findPointXy(0.0, 0.0) == (720, 360) + +# # Point at the North Pole +# assert ecmwf_grid.findPointXy(90.0, 0.0) == (720, 720) + +# # Test some edge points (ensure they are properly handled) +# assert ecmwf_grid.findPointXy(-90.0, -180.0) == (0, 0) +# assert ecmwf_grid.findPointXy(90.0, 180.0) == (0, 720) + +# # Test wrapping for global grid +# # A point at longitude 181 should wrap to longitude -179 +# point1 = ecmwf_grid.findPointXy(0.0, 181.0) +# point2 = ecmwf_grid.findPointXy(0.0, -179.0) +# assert point1 == point2 diff --git a/tests/test_meta.py b/tests/test_meta.py new file mode 100644 index 00000000..f3da151c --- /dev/null +++ b/tests/test_meta.py @@ -0,0 +1,56 @@ +import numpy as np +import pytest +import requests +from omfiles.om_grid import OmMetaJson + + +@pytest.fixture +def icon_d2_meta_json() -> str: + # return meta_str + return '{"chunk_time_length":121,"crs_wkt":"GEOGCRS[\\"WGS 84\\",\\n DATUM[\\"World Geodetic System 1984\\",\\n ELLIPSOID[\\"WGS 84\\",6378137,298.257223563]],\\n CS[ellipsoidal,2],\\n AXIS[\\"latitude\\",north],\\n AXIS[\\"longitude\\",east],\\n ANGLEUNIT[\\"degree\\",0.0174532925199433]\\n USAGE[\\n SCOPE[\\"grid\\"],\\n BBOX[43.18,-3.94,58.08,20.339998]]]","data_end_time":1768503600,"last_run_availability_time":1768332519,"last_run_initialisation_time":1768327200,"last_run_modification_time":1768332519,"temporal_resolution_seconds":3600,"update_interval_seconds":10800}' + + +@pytest.fixture +def icon_d2_meta(icon_d2_meta_json: str) -> OmMetaJson: + return OmMetaJson.from_metajson_string(icon_d2_meta_json) + + +def test_meta_json_creation(icon_d2_meta_json: str): + """Test creation of OmMetaJson object from JSON string.""" + meta = OmMetaJson.from_metajson_string(icon_d2_meta_json) + assert meta.chunk_time_length == 121 + + +def test_time_to_chunk_index(icon_d2_meta: OmMetaJson): + """Test conversion from timestamp to chunk index.""" + + # Create test timestamp (2023-01-01 12:00:00 UTC) + timestamp = np.datetime64("2023-01-01T12:00:00") + + # Calculate expected chunk index + # Seconds since epoch = (2023-01-01 12:00:00 - 1970-01-01 00:00:00) seconds + # chunk_index = seconds_since_epoch / (file_length * temporal_resolution_seconds) + epoch = np.datetime64("1970-01-01T00:00:00") + seconds_since_epoch = (timestamp - epoch) / np.timedelta64(1, "s") + expected_chunk = int( + seconds_since_epoch / (icon_d2_meta.chunk_time_length * icon_d2_meta.temporal_resolution_seconds) + ) + + # Test the time_to_chunk_index function + chunk_index = icon_d2_meta.time_to_chunk_index(timestamp) + assert chunk_index == expected_chunk + + +def test_get_chunk_time_range(icon_d2_meta: OmMetaJson): + """Test getting time range for a specific chunk.""" + + # Test chunk 1000 + chunk_index = 1000 + time_range = icon_d2_meta.get_chunk_time_range(chunk_index) + + # Check that we get the expected number of time points + assert len(time_range) == icon_d2_meta.chunk_time_length + + # Check that time points are evenly spaced + time_diff = time_range[1] - time_range[0] + assert time_diff == np.timedelta64(icon_d2_meta.temporal_resolution_seconds, "s") diff --git a/tests/test_om_domains.py b/tests/test_om_domains.py deleted file mode 100644 index 7df39827..00000000 --- a/tests/test_om_domains.py +++ /dev/null @@ -1,82 +0,0 @@ -# import numpy as np -# from omfiles.om_domains import DOMAINS - - -# def test_dwd_icon_d2_grid_points(): -# """Test specific points in the DWD ICON D2 grid.""" -# dwd_grid = DOMAINS["dwd_icon_d2"].grid - -# # Test a point known to be in the domain (Central Europe) -# # Berlin coordinates: approx. 52.52°N, 13.40°E -# berlin = dwd_grid.findPointXy(52.52, 13.40) -# assert berlin is not None - -# # Test a point outside the domain (should return None) -# # New York coordinates: approx. 40.71°N, -74.01°E -# new_york = dwd_grid.findPointXy(40.71, -74.01) -# assert new_york is None - -# # Test gridpoint to coordinate conversion -# if berlin is not None: -# x, y = berlin -# lat, lon = dwd_grid.getCoordinates(x, y) -# # Check that we get close to the original coordinates -# assert abs(lat - 52.52) < 0.05 -# assert abs(lon - 13.40) < 0.05 - - -# def test_ecmwf_grid(): -# """Test the ECMWF IFS grid specifically.""" -# ecmwf_grid = DOMAINS["ecmwf_ifs025"].grid - -# # Test some known points on the grid -# # Point at the prime meridian and equator -# assert ecmwf_grid.findPointXy(0.0, 0.0) == (720, 360) - -# # Point at the North Pole -# assert ecmwf_grid.findPointXy(90.0, 0.0) == (720, 720) - -# # Test some edge points (ensure they are properly handled) -# assert ecmwf_grid.findPointXy(-90.0, -180.0) == (0, 0) -# assert ecmwf_grid.findPointXy(90.0, 180.0) == (0, 720) - -# # Test wrapping for global grid -# # A point at longitude 181 should wrap to longitude -179 -# point1 = ecmwf_grid.findPointXy(0.0, 181.0) -# point2 = ecmwf_grid.findPointXy(0.0, -179.0) -# assert point1 == point2 - - -# def test_time_to_chunk_index(): -# """Test conversion from timestamp to chunk index.""" -# domain = DOMAINS["dwd_icon_d2"] - -# # Create test timestamp (2023-01-01 12:00:00 UTC) -# timestamp = np.datetime64("2023-01-01T12:00:00") - -# # Calculate expected chunk index -# # Seconds since epoch = (2023-01-01 12:00:00 - 1970-01-01 00:00:00) seconds -# # chunk_index = seconds_since_epoch / (file_length * temporal_resolution_seconds) -# epoch = np.datetime64("1970-01-01T00:00:00") -# seconds_since_epoch = (timestamp - epoch) / np.timedelta64(1, "s") -# expected_chunk = int(seconds_since_epoch / (domain.file_length * domain.temporal_resolution_seconds)) - -# # Test the time_to_chunk_index function -# chunk_index = domain.time_to_chunk_index(timestamp) -# assert chunk_index == expected_chunk - - -# def test_get_chunk_time_range(): -# """Test getting time range for a specific chunk.""" -# domain = DOMAINS["dwd_icon_d2"] - -# # Test chunk 1000 -# chunk_index = 1000 -# time_range = domain.get_chunk_time_range(chunk_index) - -# # Check that we get the expected number of time points -# assert len(time_range) == domain.file_length - -# # Check that time points are evenly spaced -# time_diff = time_range[1] - time_range[0] -# assert time_diff == np.timedelta64(domain.temporal_resolution_seconds, "s") From 53ba9badb003c721f6df39d4a46ffc31fc356d10 Mon Sep 17 00:00:00 2001 From: terraputix Date: Tue, 13 Jan 2026 20:48:15 +0100 Subject: [PATCH 30/50] remove unneeded stuff --- python/omfiles/_utils.py | 25 ------------------------- python/omfiles/om_grid.py | 2 +- python/omfiles/types.py | 9 --------- 3 files changed, 1 insertion(+), 35 deletions(-) delete mode 100644 python/omfiles/_utils.py diff --git a/python/omfiles/_utils.py b/python/omfiles/_utils.py deleted file mode 100644 index ab964121..00000000 --- a/python/omfiles/_utils.py +++ /dev/null @@ -1,25 +0,0 @@ -from typing import Union - -import numpy as np - -from omfiles.types import ArrayType - -EPOCH = np.datetime64(0, "s") - - -def _modulo_positive(value: int, modulo: int) -> int: - """ - Calculate modulo that always returns positive value. - - Args: - value (int): Value to calculate modulo for. - modulo (int): Modulo value. - - Returns: - int: Positive modulo result. - """ - return ((value % modulo) + modulo) % modulo - - -def _normalize_longitude(lon: Union[ArrayType, float]) -> Union[ArrayType, float]: - return ((lon + 180.0) % 360.0) - 180.0 diff --git a/python/omfiles/om_grid.py b/python/omfiles/om_grid.py index 440171ef..2f2238f0 100644 --- a/python/omfiles/om_grid.py +++ b/python/omfiles/om_grid.py @@ -8,7 +8,7 @@ import numpy.typing as npt from pyproj import CRS, Transformer -from omfiles._utils import EPOCH +EPOCH = np.datetime64(0, "s") @dataclass diff --git a/python/omfiles/types.py b/python/omfiles/types.py index 904e03ed..25c84f47 100644 --- a/python/omfiles/types.py +++ b/python/omfiles/types.py @@ -1,8 +1,5 @@ """Types used throughout the library.""" -import numpy as np -import numpy.typing as npt - try: from types import EllipsisType except ImportError: @@ -13,9 +10,3 @@ BasicSelector = Union[int, slice, EllipsisType] BasicSelection = Union[BasicSelector, Tuple[Union[int, slice, EllipsisType], ...]] - -# Type aliases for grids for clarity -FloatType = Union[float, np.floating] -ArrayType = npt.NDArray[np.floating] -CoordType = Union[float, ArrayType] -ReturnUnionType = Union[tuple[ArrayType, ArrayType], tuple[float, float]] From 748465a80d4f2347d2ab7537e183835d01349459 Mon Sep 17 00:00:00 2001 From: terraputix Date: Tue, 13 Jan 2026 22:02:34 +0100 Subject: [PATCH 31/50] gaussian grid --- examples/select_by_coordinates.py | 1 + python/omfiles/grids/gaussian.py | 773 ++++++++++++++++++++++++++++++ python/omfiles/grids/regular.py | 154 ++++++ python/omfiles/om_grid.py | 154 ++---- tests/test_grids.py | 118 ++--- 5 files changed, 1013 insertions(+), 187 deletions(-) create mode 100644 python/omfiles/grids/gaussian.py create mode 100644 python/omfiles/grids/regular.py diff --git a/examples/select_by_coordinates.py b/examples/select_by_coordinates.py index af37a633..ccb290f9 100644 --- a/examples/select_by_coordinates.py +++ b/examples/select_by_coordinates.py @@ -236,6 +236,7 @@ def get_data_for_coordinates( "dwd_icon_eu": "DWD ICON (Europe)", "dwd_icon_d2": "DWD ICON D2 (Central Europe)", "ecmwf_ifs025": "ECMWF IFS (Global)", + "ecmwf_ifs": "ECMWF IFS HRES (Global)", "meteofrance_arpege_europe": "Météo-France ARPEGE (Europe)", "meteofrance_arpege_world025": "Météo-France ARPEGE (Global)", "meteofrance_arome_france0025": "Météo-France AROME (France)", diff --git a/python/omfiles/grids/gaussian.py b/python/omfiles/grids/gaussian.py new file mode 100644 index 00000000..2c4ac671 --- /dev/null +++ b/python/omfiles/grids/gaussian.py @@ -0,0 +1,773 @@ +"""Gaussian grid implementation for reduced Gaussian grids like ECMWF IFS.""" + +from typing import Optional, Tuple + +import numpy as np +import numpy.typing as npt + + +class GaussianGrid: + """ + Implementation of reduced Gaussian grids (O1280, O320, N320, N160). + + These grids have varying numbers of longitude points at each latitude line, + with more points near the equator and fewer near the poles. + """ + + # Lookup tables for N-type grids + N320_COUNT_PER_LINE = [ + 18, + 25, + 36, + 40, + 45, + 50, + 60, + 64, + 72, + 72, + 75, + 81, + 90, + 96, + 100, + 108, + 120, + 120, + 125, + 135, + 144, + 144, + 150, + 160, + 180, + 180, + 180, + 192, + 192, + 200, + 216, + 216, + 216, + 225, + 240, + 240, + 240, + 250, + 256, + 270, + 270, + 288, + 288, + 288, + 300, + 300, + 320, + 320, + 320, + 324, + 360, + 360, + 360, + 360, + 360, + 360, + 375, + 375, + 384, + 384, + 400, + 400, + 405, + 432, + 432, + 432, + 432, + 450, + 450, + 450, + 480, + 480, + 480, + 480, + 480, + 486, + 500, + 500, + 500, + 512, + 512, + 540, + 540, + 540, + 540, + 540, + 576, + 576, + 576, + 576, + 576, + 576, + 600, + 600, + 600, + 600, + 640, + 640, + 640, + 640, + 640, + 640, + 640, + 648, + 648, + 675, + 675, + 675, + 675, + 720, + 720, + 720, + 720, + 720, + 720, + 720, + 720, + 720, + 729, + 750, + 750, + 750, + 750, + 768, + 768, + 768, + 768, + 800, + 800, + 800, + 800, + 800, + 800, + 810, + 810, + 864, + 864, + 864, + 864, + 864, + 864, + 864, + 864, + 864, + 864, + 864, + 900, + 900, + 900, + 900, + 900, + 900, + 900, + 900, + 960, + 960, + 960, + 960, + 960, + 960, + 960, + 960, + 960, + 960, + 960, + 960, + 960, + 960, + 972, + 972, + 1000, + 1000, + 1000, + 1000, + 1000, + 1000, + 1000, + 1000, + 1024, + 1024, + 1024, + 1024, + 1024, + 1024, + 1080, + 1080, + 1080, + 1080, + 1080, + 1080, + 1080, + 1080, + 1080, + 1080, + 1080, + 1080, + 1080, + 1080, + 1125, + 1125, + 1125, + 1125, + 1125, + 1125, + 1125, + 1125, + 1125, + 1125, + 1125, + 1125, + 1125, + 1125, + 1152, + 1152, + 1152, + 1152, + 1152, + 1152, + 1152, + 1152, + 1152, + 1200, + 1200, + 1200, + 1200, + 1200, + 1200, + 1200, + 1200, + 1200, + 1200, + 1200, + 1200, + 1200, + 1200, + 1200, + 1200, + 1200, + 1200, + 1215, + 1215, + 1215, + 1215, + 1215, + 1215, + 1215, + 1280, + 1280, + 1280, + 1280, + 1280, + 1280, + 1280, + 1280, + 1280, + 1280, + 1280, + 1280, + 1280, + 1280, + 1280, + 1280, + 1280, + 1280, + 1280, + 1280, + 1280, + 1280, + 1280, + 1280, + 1280, + 1280, + 1280, + 1280, + 1280, + 1280, + 1280, + 1280, + 1280, + 1280, + 1280, + 1280, + 1280, + 1280, + 1280, + 1280, + 1280, + 1280, + 1280, + 1280, + 1280, + 1280, + 1280, + 1280, + 1280, + 1280, + 1280, + 1280, + 1280, + 1280, + 1280, + 1280, + 1280, + 1280, + 1280, + 1280, + 1280, + 1280, + 1280, + 1280, + 1280, + 1280, + 1280, + 1280, + 1280, + 1280, + 1280, + 1280, + 1280, + 1280, + ] + + N160_COUNT_PER_LINE = [ + 18, + 25, + 36, + 40, + 45, + 50, + 60, + 64, + 72, + 72, + 80, + 90, + 90, + 96, + 108, + 120, + 120, + 125, + 128, + 135, + 144, + 150, + 160, + 160, + 180, + 180, + 180, + 192, + 192, + 200, + 216, + 216, + 225, + 225, + 240, + 240, + 243, + 250, + 256, + 270, + 270, + 288, + 288, + 288, + 300, + 300, + 320, + 320, + 320, + 320, + 324, + 360, + 360, + 360, + 360, + 360, + 360, + 375, + 375, + 375, + 384, + 384, + 400, + 400, + 400, + 405, + 432, + 432, + 432, + 432, + 432, + 450, + 450, + 450, + 450, + 480, + 480, + 480, + 480, + 480, + 480, + 480, + 500, + 500, + 500, + 500, + 500, + 512, + 512, + 540, + 540, + 540, + 540, + 540, + 540, + 540, + 540, + 576, + 576, + 576, + 576, + 576, + 576, + 576, + 576, + 576, + 576, + 600, + 600, + 600, + 600, + 600, + 600, + 600, + 600, + 600, + 640, + 640, + 640, + 640, + 640, + 640, + 640, + 640, + 640, + 640, + 640, + 640, + 640, + 640, + 640, + 640, + 640, + 640, + 640, + 640, + 640, + 640, + 640, + 640, + 640, + 640, + 640, + 640, + 640, + 640, + 640, + 640, + 640, + 640, + 640, + 640, + 640, + 640, + 640, + 640, + 640, + 640, + 640, + 640, + ] + + def __init__(self, crs_wkt: str, shape: Tuple[int, int]): + """ + Initialize Gaussian grid from WKT projection string and shape. + + Args: + crs_wkt: WKT string containing grid type information in REMARK field + shape: Grid shape as (ny, nx) where ny=1 and nx is total grid point count + """ + self.ny, self.nx = shape + + if self.ny != 1: + raise ValueError(f"Gaussian grid must have ny=1, got {self.ny}") + + # Parse grid type from WKT REMARK field + self.grid_type = self._parse_grid_type(crs_wkt) + self.latitude_lines = self._get_latitude_lines() + + # Validate grid point count + expected_count = self._calculate_total_points() + if self.nx != expected_count: + raise ValueError(f"Grid point count mismatch: expected {expected_count}, got {self.nx}") + + # Pre-calculate cumulative sums for faster lookups + self._build_integral_table() + + def _parse_grid_type(self, crs_wkt: str) -> str: + """Extract grid type (O1280, O320, N320, N160) from WKT string.""" + if "O1280" in crs_wkt or self.nx == 6599680: + return "O1280" + elif "O320" in crs_wkt or self.nx == 421120: + return "O320" + elif "N320" in crs_wkt or self.nx == 542080: + return "N320" + elif "N160" in crs_wkt or self.nx == 138346: + return "N160" + else: + raise ValueError(f"Unknown Gaussian grid type with {self.nx} points") + + def _get_latitude_lines(self) -> int: + """Get number of latitude lines from pole to equator.""" + if self.grid_type == "O1280": + return 1280 + elif self.grid_type == "O320": + return 320 + elif self.grid_type == "N320": + return 320 + elif self.grid_type == "N160": + return 160 + else: + raise ValueError(f"Unknown grid type: {self.grid_type}") + + def _calculate_total_points(self) -> int: + """Calculate total number of grid points.""" + if self.grid_type in ["O1280", "O320"]: + return 4 * self.latitude_lines * (self.latitude_lines + 9) + elif self.grid_type == "N320": + return 542080 + elif self.grid_type == "N160": + return 138346 + else: + raise ValueError(f"Unknown grid type: {self.grid_type}") + + def _nx_of_y(self, y: int) -> int: + """Get number of longitude points at latitude line y.""" + if self.grid_type in ["O1280", "O320"]: + # O-type grids have analytical formula + if y < self.latitude_lines: + return 20 + y * 4 + else: + return (2 * self.latitude_lines - y - 1) * 4 + 20 + else: + # N-type grids use lookup table + count_per_line = self.N320_COUNT_PER_LINE if self.grid_type == "N320" else self.N160_COUNT_PER_LINE + if y < self.latitude_lines: + return count_per_line[y] + else: + return count_per_line[2 * len(count_per_line) - y - 1] + + def _build_integral_table(self): + """Pre-calculate cumulative sums for faster grid point lookups.""" + self._integral_table = [0] + for y in range(2 * self.latitude_lines): + self._integral_table.append(self._integral_table[-1] + self._nx_of_y(y)) + + def _integral(self, y: int) -> int: + """Get cumulative number of grid points up to (but not including) latitude line y.""" + return self._integral_table[y] + + def _get_pos(self, gridpoint: int) -> Tuple[int, int, int]: + """ + Find latitude line (y) and longitude index (x) for given grid point. + + This matches the Swift getPos method exactly. + + Returns: + (y, x, nx) where nx is number of points on this latitude line + """ + if gridpoint < 0 or gridpoint >= self.nx: + raise ValueError(f"Grid point {gridpoint} out of range [0, {self.nx})") + + if self.grid_type in ["O1280", "O320"]: + # O-type grids use analytical formula + count = self.nx + half_count = count // 2 + + if gridpoint < half_count: + # Northern hemisphere (including equator) + # Solve: gridpoint = 2*y*y + 18*y + y = int((np.sqrt(2 * gridpoint + 81) - 9) / 2) + else: + # Southern hemisphere + # Mirror from the other side + gridpoint_from_end = count - gridpoint - 1 + y_from_end = int((np.sqrt(2 * gridpoint_from_end + 81) - 9) / 2) + y = 2 * self.latitude_lines - 1 - y_from_end + + x = gridpoint - self._integral(y) + nx = self._nx_of_y(y) + return (y, x, nx) + else: + # N-type grids use lookup + count_per_line = self.N320_COUNT_PER_LINE if self.grid_type == "N320" else self.N160_COUNT_PER_LINE + + # Search in northern hemisphere first + cumsum = 0 + for y, n in enumerate(count_per_line): + cumsum += n + if gridpoint < cumsum: + return (y, gridpoint - (cumsum - n), n) + + # Search in southern hemisphere + for y, n in enumerate(reversed(count_per_line)): + cumsum += n + if gridpoint < cumsum: + actual_y = y + len(count_per_line) + return (actual_y, gridpoint - (cumsum - n), n) + + raise ValueError(f"Grid point {gridpoint} not found") + + @property + def shape(self) -> Tuple[int, int]: + """Grid shape as (ny, nx).""" + return (self.ny, self.nx) + + def get_coordinates(self, x: int, y: int) -> Tuple[float, float]: + """ + Get lat/lon coordinates for grid point index. + + For Gaussian grids, y is always 0, and x is the flat grid point index. + + Args: + x: Grid point index (0 to nx-1) + y: Must be 0 for Gaussian grids + + Returns: + (latitude, longitude) in degrees + """ + if y != 0: + raise ValueError(f"Gaussian grid only has y=0, got y={y}") + + return self._get_coordinates_from_gridpoint(x) + + def _get_coordinates_from_gridpoint(self, gridpoint: int) -> Tuple[float, float]: + """Get coordinates from flat grid point index.""" + y, x, nx = self._get_pos(gridpoint) + + # Calculate latitude + dy = 180.0 / (2 * self.latitude_lines + 0.5) + lat = (self.latitude_lines - y - 1) * dy + dy / 2 + + # Calculate longitude + dx = 360.0 / nx + lon = x * dx + + # Normalize longitude to [-180, 180) + if lon >= 180: + lon -= 360 + + return (float(lat), float(lon)) + + def find_point_xy(self, lat: float, lon: float) -> Optional[Tuple[int, int]]: + """ + Find grid point index for given lat/lon coordinates. + + For Gaussian grids, returns (gridpoint, 0) to match the interface. + + Args: + lat: Latitude in degrees + lon: Longitude in degrees + + Returns: + (gridpoint, 0) if point is valid, None otherwise + """ + x_idx, y_idx = self._find_point_xy(lat, lon) + + # Convert to flat grid point index + gridpoint = self._integral(y_idx) + x_idx + return (gridpoint, 0) + + def _find_point_xy(self, lat: float, lon: float) -> Tuple[int, int]: + # Calculate latitude line + dy = 180.0 / (2.0 * self.latitude_lines + 0.5) + + # Note: Limited by -2 because later we add +1 + y_float = self.latitude_lines - 1.0 - ((lat - dy / 2.0) / dy) + y = max(0, min(2 * self.latitude_lines - 2, int(y_float))) + y_upper = y + 1 + + # Get number of longitude points on both lines + nx = self._nx_of_y(y) + nx_upper = self._nx_of_y(y_upper) + + dx = 360.0 / nx + dx_upper = 360.0 / nx_upper + + # Find closest x on both lines + x = int(round(lon / dx)) + x_upper = int(round(lon / dx_upper)) + + # Calculate actual coordinates + point_lat = (self.latitude_lines - y - 1) * dy + dy / 2.0 + point_lon = x * dx + point_lat_upper = (self.latitude_lines - y_upper - 1) * dy + dy / 2.0 + point_lon_upper = x_upper * dx_upper + + # Calculate squared distances + distance = (point_lat - lat) ** 2 + (point_lon - lon) ** 2 + distance_upper = (point_lat_upper - lat) ** 2 + (point_lon_upper - lon) ** 2 + + # Return closest point with proper wrapping + if distance < distance_upper: + return ((x + nx) % nx, y) + else: + return ((x_upper + nx_upper) % nx_upper, y_upper) + + @property + def latitude(self) -> npt.NDArray[np.float64]: + """Get 1D array of latitude coordinates for all grid points.""" + if not hasattr(self, "_latitude"): + self._compute_coordinates() + return self._latitude + + @property + def longitude(self) -> npt.NDArray[np.float64]: + """Get 1D array of longitude coordinates for all grid points.""" + if not hasattr(self, "_longitude"): + self._compute_coordinates() + return self._longitude + + def _compute_coordinates(self) -> None: + """Compute and cache latitude/longitude arrays for all grid points.""" + lats = np.zeros(self.nx, dtype=np.float64) + lons = np.zeros(self.nx, dtype=np.float64) + + for gridpoint in range(self.nx): + lat, lon = self._get_coordinates_from_gridpoint(gridpoint) + lats[gridpoint] = lat + lons[gridpoint] = lon + + self._latitude = lats + self._longitude = lons + + def get_meshgrid(self) -> Tuple[npt.NDArray[np.float64], npt.NDArray[np.float64]]: + """ + Get meshgrid - for Gaussian grids this returns 1D arrays. + + Returns: + (longitude, latitude) 1D arrays + """ + return (self.longitude, self.latitude) diff --git a/python/omfiles/grids/regular.py b/python/omfiles/grids/regular.py new file mode 100644 index 00000000..52b5e56e --- /dev/null +++ b/python/omfiles/grids/regular.py @@ -0,0 +1,154 @@ +"""Regular latitude/longitude or projected grid.""" + +from typing import Optional, Tuple + +import numpy as np +import numpy.typing as npt +from pyproj import CRS, Transformer + + +class RegularGrid: + """Regular latitude/longitude or projected grid.""" + + # lon_grid: npt.NDArray[np.float64] + # lat_grid: npt.NDArray[np.float64] + + def __init__(self, crs_wkt: str, shape: Tuple[int, int]): + """ + Initialize grid from WKT projection string and data shape. + + Args: + crs_wkt: Coordinate Reference System in Well-Known Text format + shape: Grid shape as (ny, nx) - number of points in y and x directions + """ + self.crs = CRS.from_wkt(crs_wkt) + self.wgs84 = CRS.from_epsg(4326) + self.ny, self.nx = shape + + # TODO: Special case for gaussian grids! + + # Transformers for coordinate conversions + self.to_projection = Transformer.from_crs(self.wgs84, self.crs, always_xy=True) + self.to_wgs84 = Transformer.from_crs(self.crs, self.wgs84, always_xy=True) + + # Get projection bounds from area of use + area = self.crs.area_of_use + if area is None: + raise ValueError("CRS does not have an area of use defined") + + # Transform WGS84 bounds to projection space + xmin, ymin = self.to_projection.transform(area.west, area.south) + xmax, ymax = self.to_projection.transform(area.east, area.north) + + self.bounds = (xmin, xmax, ymin, ymax) + self.origin = (xmin, ymin) + + if self.nx <= 1 or self.ny <= 1: + raise ValueError("Invalid grid shape") + + # Calculate grid spacing + self.dx = (xmax - xmin) / (self.nx - 1) + self.dy = (ymax - ymin) / (self.ny - 1) + + @property + def shape(self) -> Tuple[int, int]: + """Grid shape as (ny, nx).""" + return (self.ny, self.nx) + + @property + def latitude(self) -> npt.NDArray[np.float64]: + """ + Get 2D array of latitude coordinates for all grid points. + + Returns: + Array of shape (ny, nx) with latitude values + """ + if not hasattr(self, "_latitude"): + self._compute_coordinates() + return self._latitude + + @property + def longitude(self) -> npt.NDArray[np.float64]: + """ + Get 2D array of longitude coordinates for all grid points. + + Returns: + Array of shape (ny, nx) with longitude values + """ + if not hasattr(self, "_longitude"): + self._compute_coordinates() + return self._longitude + + def _compute_coordinates(self) -> None: + """Compute and cache latitude/longitude arrays for all grid points.""" + # Create meshgrid of projection coordinates + x_coords = np.linspace(self.origin[0], self.origin[0] + self.dx * (self.nx - 1), self.nx) + y_coords = np.linspace(self.origin[1], self.origin[1] + self.dy * (self.ny - 1), self.ny) + x_grid, y_grid = np.meshgrid(x_coords, y_coords) + + # Transform to WGS84 + lon_grid, lat_grid = self.to_wgs84.transform(x_grid, y_grid) + + self._longitude = lon_grid + self._latitude = lat_grid + + def find_point_xy(self, lat: float, lon: float) -> Optional[Tuple[int, int]]: + """ + Find grid point indices (x, y) for given lat/lon coordinates. + + Args: + lat: Latitude in degrees + lon: Longitude in degrees + + Returns: + (x, y) grid indices if point is in grid bounds, None otherwise + """ + # Transform to projection coordinates + x_proj, y_proj = self.to_projection.transform(lon, lat) + + # Calculate grid indices + x_idx = int(round((x_proj - self.origin[0]) / self.dx)) + y_idx = int(round((y_proj - self.origin[1]) / self.dy)) + + # Validate indices + if not (0 <= x_idx < self.nx and 0 <= y_idx < self.ny): + return None + + return (x_idx, y_idx) + + def get_coordinates(self, x: int, y: int) -> Tuple[float, float]: + """ + Get lat/lon coordinates for given grid point indices. + + Args: + x: Grid x index + y: Grid y index + + Returns: + (latitude, longitude) in degrees + """ + # Calculate projection coordinates + x_proj = self.origin[0] + x * self.dx + y_proj = self.origin[1] + y * self.dy + + # Transform to WGS84 + lon, lat = self.to_wgs84.transform(x_proj, y_proj) + + return (float(lat), float(lon)) + + def get_meshgrid(self) -> Tuple[npt.NDArray[np.float64], npt.NDArray[np.float64]]: + """ + Get meshgrid of projection coordinates. + + Useful for plotting with matplotlib/cartopy. + + Returns: + (lon_grid, lat_grid) arrays of shape (ny, nx) in projection coordinates + """ + x_coords: npt.NDArray[np.float64] = np.linspace( + self.origin[0], self.origin[0] + self.dx * (self.nx - 1), self.nx + ) + y_coords: npt.NDArray[np.float64] = np.linspace( + self.origin[1], self.origin[1] + self.dy * (self.ny - 1), self.ny + ) + return np.meshgrid(x_coords, y_coords) diff --git a/python/omfiles/om_grid.py b/python/omfiles/om_grid.py index 2f2238f0..ef7a00d1 100644 --- a/python/omfiles/om_grid.py +++ b/python/omfiles/om_grid.py @@ -8,6 +8,9 @@ import numpy.typing as npt from pyproj import CRS, Transformer +from omfiles.grids.gaussian import GaussianGrid +from omfiles.grids.regular import RegularGrid + EPOCH = np.datetime64(0, "s") @@ -98,11 +101,13 @@ def get_chunk_time_range(self, chunk_index: int): return timestamps -class OmGrid: - """Latitude and longitude grid based on cartopy and wkt projection strings.""" +def _is_gaussian_grid(crs_wkt: str) -> bool: + """Check if WKT string represents a Gaussian grid.""" + return "Reduced Gaussian Grid" in crs_wkt or "Gaussian Grid" in crs_wkt + - # lon_grid: npt.NDArray[np.float64] - # lat_grid: npt.NDArray[np.float64] +class OmGrid: + """Wrapper for grid implementations - automatically delegates to appropriate grid type.""" def __init__(self, crs_wkt: str, shape: Tuple[int, int]): """ @@ -110,136 +115,49 @@ def __init__(self, crs_wkt: str, shape: Tuple[int, int]): Args: crs_wkt: Coordinate Reference System in Well-Known Text format - shape: Grid shape as (ny, nx) - number of points in y and x directions + shape: Grid shape as (ny, nx) """ - self.crs = CRS.from_wkt(crs_wkt) - self.wgs84 = CRS.from_epsg(4326) - self.ny, self.nx = shape - - # TODO: Special case for gaussian grids! - - # Transformers for coordinate conversions - self.to_projection = Transformer.from_crs(self.wgs84, self.crs, always_xy=True) - self.to_wgs84 = Transformer.from_crs(self.crs, self.wgs84, always_xy=True) - - # Get projection bounds from area of use - area = self.crs.area_of_use - if area is None: - raise ValueError("CRS does not have an area of use defined") - - # Transform WGS84 bounds to projection space - xmin, ymin = self.to_projection.transform(area.west, area.south) - xmax, ymax = self.to_projection.transform(area.east, area.north) - - self.bounds = (xmin, xmax, ymin, ymax) - self.origin = (xmin, ymin) - - if self.nx <= 1 or self.ny <= 1: - raise ValueError("Invalid grid shape") - - # Calculate grid spacing - self.dx = (xmax - xmin) / (self.nx - 1) - self.dy = (ymax - ymin) / (self.ny - 1) + # Detect grid type and create appropriate implementation + if _is_gaussian_grid(crs_wkt): + self._grid = GaussianGrid(crs_wkt, shape) + else: + self._grid = RegularGrid(crs_wkt, shape) @property def shape(self) -> Tuple[int, int]: """Grid shape as (ny, nx).""" - return (self.ny, self.nx) + return self._grid.shape @property def latitude(self) -> npt.NDArray[np.float64]: - """ - Get 2D array of latitude coordinates for all grid points. - - Returns: - Array of shape (ny, nx) with latitude values - """ - if not hasattr(self, "_latitude"): - self._compute_coordinates() - return self._latitude + """Get array of latitude coordinates for all grid points.""" + return self._grid.latitude @property def longitude(self) -> npt.NDArray[np.float64]: - """ - Get 2D array of longitude coordinates for all grid points. - - Returns: - Array of shape (ny, nx) with longitude values - """ - if not hasattr(self, "_longitude"): - self._compute_coordinates() - return self._longitude - - def _compute_coordinates(self) -> None: - """Compute and cache latitude/longitude arrays for all grid points.""" - # Create meshgrid of projection coordinates - x_coords = np.linspace(self.origin[0], self.origin[0] + self.dx * (self.nx - 1), self.nx) - y_coords = np.linspace(self.origin[1], self.origin[1] + self.dy * (self.ny - 1), self.ny) - x_grid, y_grid = np.meshgrid(x_coords, y_coords) - - # Transform to WGS84 - lon_grid, lat_grid = self.to_wgs84.transform(x_grid, y_grid) - - self._longitude = lon_grid - self._latitude = lat_grid + """Get array of longitude coordinates for all grid points.""" + return self._grid.longitude def find_point_xy(self, lat: float, lon: float) -> Optional[Tuple[int, int]]: - """ - Find grid point indices (x, y) for given lat/lon coordinates. - - Args: - lat: Latitude in degrees - lon: Longitude in degrees - - Returns: - (x, y) grid indices if point is in grid bounds, None otherwise - """ - # Transform to projection coordinates - x_proj, y_proj = self.to_projection.transform(lon, lat) - - # Calculate grid indices - x_idx = int(round((x_proj - self.origin[0]) / self.dx)) - y_idx = int(round((y_proj - self.origin[1]) / self.dy)) - - # Validate indices - if not (0 <= x_idx < self.nx and 0 <= y_idx < self.ny): - return None - - return (x_idx, y_idx) + """Find grid point indices for given lat/lon coordinates.""" + return self._grid.find_point_xy(lat, lon) def get_coordinates(self, x: int, y: int) -> Tuple[float, float]: - """ - Get lat/lon coordinates for given grid point indices. - - Args: - x: Grid x index - y: Grid y index - - Returns: - (latitude, longitude) in degrees - """ - # Calculate projection coordinates - x_proj = self.origin[0] + x * self.dx - y_proj = self.origin[1] + y * self.dy - - # Transform to WGS84 - lon, lat = self.to_wgs84.transform(x_proj, y_proj) - - return (float(lat), float(lon)) + """Get lat/lon coordinates for given grid point indices.""" + return self._grid.get_coordinates(x, y) def get_meshgrid(self) -> Tuple[npt.NDArray[np.float64], npt.NDArray[np.float64]]: - """ - Get meshgrid of projection coordinates. + """Get meshgrid of coordinates.""" + return self._grid.get_meshgrid() - Useful for plotting with matplotlib/cartopy. + @property + def is_gaussian(self) -> bool: + """Check if this is a Gaussian grid.""" + return isinstance(self._grid, GaussianGrid) - Returns: - (lon_grid, lat_grid) arrays of shape (ny, nx) in projection coordinates - """ - x_coords: npt.NDArray[np.float64] = np.linspace( - self.origin[0], self.origin[0] + self.dx * (self.nx - 1), self.nx - ) - y_coords: npt.NDArray[np.float64] = np.linspace( - self.origin[1], self.origin[1] + self.dy * (self.ny - 1), self.ny - ) - return np.meshgrid(x_coords, y_coords) + @property + def crs(self) -> CRS | None: + """Get the Coordinate Reference System.""" + if isinstance(self._grid, GaussianGrid): + return None + return self._grid.crs diff --git a/tests/test_grids.py b/tests/test_grids.py index 9f1e2cfe..271fad2d 100644 --- a/tests/test_grids.py +++ b/tests/test_grids.py @@ -1,5 +1,6 @@ import pyproj import pytest +from omfiles.grids.gaussian import GaussianGrid from omfiles.om_grid import OmGrid @@ -53,6 +54,16 @@ def dmi_harmoni_europe_wkt(): return 'PROJCRS["Lambert Conic Conformal",\n BASEGEOGCRS["GCS_Sphere",DATUM["D_Sphere",ELLIPSOID["Sphere",6371229.0,0.0]]],\n CONVERSION["Lambert Conic Conformal",\n METHOD["Lambert Conic Conformal (2SP)"],\n PARAMETER["Latitude of 1st standard parallel",55.5],\n PARAMETER["Latitude of 2nd standard parallel",55.5],\n PARAMETER["Latitude of false origin",55.5],\n PARAMETER["Longitude of false origin",352.0]],\n CS[Cartesian,2],\n AXIS["easting",east],\n AXIS["northing",north],\n LENGTHUNIT["metre",1],\n USAGE[\n SCOPE["grid"],\n BBOX[39.670998,-25.421997,62.667618,40.069855]]]' +@pytest.fixture +def ecmwf_ifs_wkt(): + return 'GEOGCRS["Reduced Gaussian Grid",\n DATUM["World Geodetic System 1984",\n ELLIPSOID["WGS 84",6378137,298.257223563]],\n CS[ellipsoidal,2],\n AXIS["latitude",north],\n AXIS["longitude",east],\n ANGLEUNIT["degree",0.0174532925199433],\n REMARK["Reduced Gaussian Grid O1280 (ECMWF)"],\n USAGE[\n SCOPE["grid"],\n BBOX[-90,-180.0,90,180]]]' + + +@pytest.fixture +def ecmwf_ifs_grid(ecmwf_ifs_wkt): + return OmGrid(ecmwf_ifs_wkt, (1, 6599680))._grid + + @pytest.fixture def dmi_harmoni_europe_grid(dmi_harmoni_europe_wkt): return OmGrid(dmi_harmoni_europe_wkt, (1606, 1906)) @@ -165,8 +176,8 @@ def test_lambert_conformal(gfs_nam_conus_grid: OmGrid): point_xy = gfs_nam_conus_grid.find_point_xy(lat=34, lon=-118) assert point_xy is not None x_idx, y_idx = point_xy - flat_idx = y_idx * gfs_nam_conus_grid.nx + x_idx - assert flat_idx == 777441 + assert x_idx == 273 + assert y_idx == 432 lat2, lon2 = gfs_nam_conus_grid.get_coordinates(x_idx, y_idx) assert abs(lat2 - 34) < 0.01 @@ -174,19 +185,19 @@ def test_lambert_conformal(gfs_nam_conus_grid: OmGrid): # Test reference grid points reference_points = [ - (21.137999999999987, 237.28 - 360, 0), - (24.449714395051082, 265.54789437771944 - 360, 10000), - (22.73382904757237, 242.93190409785294 - 360, 20000), - (24.37172305316154, 271.6307003393202 - 360, 30000), - (24.007414634071907, 248.77817290935954 - 360, 40000), + (21.137999999999987, 237.28 - 360, 0, 0), + (24.449714395051082, 265.54789437771944 - 360, 1005, 5), + (22.73382904757237, 242.93190409785294 - 360, 211, 11), + (24.37172305316154, 271.6307003393202 - 360, 1216, 16), + (24.007414634071907, 248.77817290935954 - 360, 422, 22), ] - for lat, lon, expected_idx in reference_points: + for lat, lon, expected_x, expected_y in reference_points: point_xy = gfs_nam_conus_grid.find_point_xy(lat=lat, lon=lon) assert point_xy is not None x_idx, y_idx = point_xy - flat_idx = y_idx * gfs_nam_conus_grid.nx + x_idx - assert flat_idx == expected_idx + assert x_idx == expected_x + assert y_idx == expected_y lat2, lon2 = gfs_nam_conus_grid.get_coordinates(x_idx, y_idx) assert abs(lat2 - lat) < 0.001 @@ -215,32 +226,30 @@ def test_nbm_grid(nbm_conus_grid: OmGrid): # Test reference grid points directly from grib files reference_points = [ - (21.137999999999987, 237.28 - 360, 117411), - (24.449714395051082, 265.54789437771944 - 360, 188910), - (22.73382904757237, 242.93190409785294 - 360, 180965), - (24.37172305316154, 271.6307003393202 - 360, 196187), - (24.007414634071907, 248.77817290935954 - 360, 232796), + (21.137999999999987, 237.28 - 360, 161, 50), + (24.449714395051082, 265.54789437771944 - 360, 1310, 80), + (22.73382904757237, 242.93190409785294 - 360, 400, 77), + (24.37172305316154, 271.6307003393202 - 360, 1552, 83), + (24.007414634071907, 248.77817290935954 - 360, 641, 99), ] - for lat, lon, expected_idx in reference_points: + for lat, lon, expected_x, expected_y in reference_points: point_xy = nbm_conus_grid.find_point_xy(lat=lat, lon=lon) assert point_xy is not None x_idx, y_idx = point_xy - flat_idx = y_idx * nbm_conus_grid.nx + x_idx - assert flat_idx == expected_idx + assert x_idx == expected_x + assert y_idx == expected_y # Test grid coordinate lookup for specific indices reference_coords = [ - (0, 19.228992, -126.27699), - (10000, 21.794254, -111.44652), - (20000, 22.806227, -96.18898), - (30000, 22.222015, -80.87921), - (40000, 20.274399, -123.18192), + (0, 0, 19.228992, -126.27699), + (4, 620, 21.794254, -111.44652), + (8, 1240, 22.806227, -96.18898), + (12, 1860, 22.222015, -80.87921), + (17, 135, 20.274399, -123.18192), ] - for idx, expected_lat, expected_lon in reference_coords: - y_idx = idx // nbm_conus_grid.nx - x_idx = idx % nbm_conus_grid.nx + for y_idx, x_idx, expected_lat, expected_lon in reference_coords: lat, lon = nbm_conus_grid.get_coordinates(x_idx, y_idx) assert abs(lat - expected_lat) < 0.001 assert abs(lon - expected_lon) < 0.001 @@ -297,46 +306,17 @@ def test_lambert_conformal_conic_projection(dmi_harmoni_europe_wkt: str, dmi_har assert point_idx == (1642, 1573) -# def test_dwd_icon_d2_grid_points(): -# """Test specific points in the DWD ICON D2 grid.""" -# dwd_grid = DOMAINS["dwd_icon_d2"].grid - -# # Test a point known to be in the domain (Central Europe) -# # Berlin coordinates: approx. 52.52°N, 13.40°E -# berlin = dwd_grid.findPointXy(52.52, 13.40) -# assert berlin is not None - -# # Test a point outside the domain (should return None) -# # New York coordinates: approx. 40.71°N, -74.01°E -# new_york = dwd_grid.findPointXy(40.71, -74.01) -# assert new_york is None - -# # Test gridpoint to coordinate conversion -# if berlin is not None: -# x, y = berlin -# lat, lon = dwd_grid.getCoordinates(x, y) -# # Check that we get close to the original coordinates -# assert abs(lat - 52.52) < 0.05 -# assert abs(lon - 13.40) < 0.05 - - -# def test_ecmwf_grid(): -# """Test the ECMWF IFS grid specifically.""" -# ecmwf_grid = DOMAINS["ecmwf_ifs025"].grid - -# # Test some known points on the grid -# # Point at the prime meridian and equator -# assert ecmwf_grid.findPointXy(0.0, 0.0) == (720, 360) - -# # Point at the North Pole -# assert ecmwf_grid.findPointXy(90.0, 0.0) == (720, 720) - -# # Test some edge points (ensure they are properly handled) -# assert ecmwf_grid.findPointXy(-90.0, -180.0) == (0, 0) -# assert ecmwf_grid.findPointXy(90.0, 180.0) == (0, 720) - -# # Test wrapping for global grid -# # A point at longitude 181 should wrap to longitude -179 -# point1 = ecmwf_grid.findPointXy(0.0, 181.0) -# point2 = ecmwf_grid.findPointXy(0.0, -179.0) -# assert point1 == point2 +def test_ecmwf_grid(ecmwf_ifs_grid: GaussianGrid): + # https://github.com/open-meteo/open-meteo/blob/7eb49a5dd41e66ac5cf386023a0527eead3104b4/Tests/AppTests/DataTests.swift#L614 + assert ecmwf_ifs_grid._find_point_xy(53.63797, 45) == (261, 517) + assert ecmwf_ifs_grid._find_point_xy(19.229, 233.723 - 360) == (2625, 1006) + assert ecmwf_ifs_grid._find_point_xy(91.0, 342) == (19, 0) + assert ecmwf_ifs_grid._find_point_xy(-91, 342) == (19, 2559) + assert ecmwf_ifs_grid._find_point_xy(-19.229, 233.723 - 360) == (2625, 1553) + + flat_grid_coords = ecmwf_ifs_grid.find_point_xy(89.94619, 0) + assert flat_grid_coords is not None, "Failed to find point" + assert flat_grid_coords == (0, 0) + position = ecmwf_ifs_grid.get_coordinates(flat_grid_coords[1], flat_grid_coords[0]) + assert abs(position[0] - 89.94619) < 0.005 + assert abs(position[1] - 0) < 0.005 From 82665a53f0d80e78b48ef566d04b8af110c7159c Mon Sep 17 00:00:00 2001 From: terraputix Date: Tue, 13 Jan 2026 22:13:07 +0100 Subject: [PATCH 32/50] fix unnecessary import --- tests/test_meta.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/test_meta.py b/tests/test_meta.py index f3da151c..20dc0c82 100644 --- a/tests/test_meta.py +++ b/tests/test_meta.py @@ -1,6 +1,5 @@ import numpy as np import pytest -import requests from omfiles.om_grid import OmMetaJson From bddd83b58d4c644c34d41296ecd8c3939760f24d Mon Sep 17 00:00:00 2001 From: terraputix Date: Wed, 14 Jan 2026 11:44:08 +0100 Subject: [PATCH 33/50] fix for python 3.9 --- python/omfiles/om_grid.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/python/omfiles/om_grid.py b/python/omfiles/om_grid.py index ef7a00d1..57410828 100644 --- a/python/omfiles/om_grid.py +++ b/python/omfiles/om_grid.py @@ -2,11 +2,11 @@ import json from dataclasses import dataclass, fields -from typing import List, Optional, Tuple +from typing import List, Optional, Tuple, Union import numpy as np import numpy.typing as npt -from pyproj import CRS, Transformer +from pyproj import CRS from omfiles.grids.gaussian import GaussianGrid from omfiles.grids.regular import RegularGrid @@ -156,7 +156,7 @@ def is_gaussian(self) -> bool: return isinstance(self._grid, GaussianGrid) @property - def crs(self) -> CRS | None: + def crs(self) -> Union[CRS, None]: """Get the Coordinate Reference System.""" if isinstance(self._grid, GaussianGrid): return None From 07720a4029bbaad83bae3fd363195692e0c95c2c Mon Sep 17 00:00:00 2001 From: terraputix Date: Wed, 14 Jan 2026 12:23:48 +0100 Subject: [PATCH 34/50] meshgrid fixes --- examples/plot_map.py | 20 ++++++++++---------- examples/select_by_coordinates.py | 8 +------- python/omfiles/grids/gaussian.py | 9 ++++----- python/omfiles/grids/regular.py | 15 +++------------ python/omfiles/om_grid.py | 9 ++++++++- 5 files changed, 26 insertions(+), 35 deletions(-) diff --git a/examples/plot_map.py b/examples/plot_map.py index 1cc937c4..b54967dd 100755 --- a/examples/plot_map.py +++ b/examples/plot_map.py @@ -3,7 +3,7 @@ # /// script # requires-python = ">=3.12" # dependencies = [ -# "omfiles==1.0.1", +# "omfiles[proj] @ /home/fred/dev/terraputix/python-omfiles", # "fsspec>=2025.7.0", # "s3fs", # "matplotlib", @@ -17,12 +17,14 @@ import matplotlib.pyplot as plt import numpy as np from omfiles import OmFileReader +from omfiles.om_grid import OmGrid, OmMetaJson +domain_name = "dmi_harmonie_arome_europe" # Example: URI for a spatial data file in the `data_spatial` S3 bucket # See data organization details: https://github.com/open-meteo/open-data?tab=readme-ov-file#data-organization # Note: Spatial data is only retained for 7 days. The example file below may no longer exist. # Please update the URI to match a currently available file. -s3_uri = "s3://openmeteo/data_spatial/dwd_icon/2025/09/23/0000Z/2025-09-30T0000.om" +s3_uri = f"s3://openmeteo/data_spatial/{domain_name}/2026/01/10/0000Z/2026-01-12T0000.om" # The following two incantations are equivalent # @@ -43,7 +45,7 @@ with OmFileReader(backend) as reader: print("reader.is_group", reader.is_group) - child = reader.get_child_by_name("relative_humidity_2m") + child = reader.get_child_by_name("temperature_2m") print("child.name", child.name) # Get the full data array @@ -64,19 +66,17 @@ ax.add_feature(cfeature.LAND, alpha=0.3) # Create coordinate arrays - # Currently, the files don't contain any information about the spatial coordinates, - # so you need to provide these coordinate arrays manually. - height, width = data.shape - lon = np.linspace(-180, 180, width) # Adjust these bounds - lat = np.linspace(-90, 90, height) # Adjust these bounds - lon_grid, lat_grid = np.meshgrid(lon, lat) + num_y, num_x = child.shape + meta = OmMetaJson.from_s3_json_path(f"openmeteo/data/{domain_name}/static/meta.json", backend.fs) + grid = OmGrid(meta.crs_wkt, (num_y, num_x)) + lon_grid, lat_grid = grid.get_meshgrid() # Plot the data im = ax.contourf(lon_grid, lat_grid, data, levels=20, transform=ccrs.PlateCarree(), cmap="viridis") plt.colorbar(im, ax=ax, shrink=0.6, label=child.name) ax.gridlines(draw_labels=True, alpha=0.3) plt.title(f"2D Map: {child.name}") - ax.set_global() + # ax.set_global() plt.tight_layout() output_filename = f"map_{child.name.replace('/', '_')}.png" diff --git a/examples/select_by_coordinates.py b/examples/select_by_coordinates.py index ccb290f9..020f9763 100644 --- a/examples/select_by_coordinates.py +++ b/examples/select_by_coordinates.py @@ -63,12 +63,6 @@ ) -def get_domain_info(domain_name: str, fs: fsspec.AbstractFileSystem) -> OmMetaJson: - meta_json_path = f"openmeteo/data/{domain_name}/static/meta.json" - meta_dict = json.loads(fs.cat_file(meta_json_path)) - return OmMetaJson.from_dict(meta_dict) - - def load_variable_dimensions( chunk_index: int, domain_name: str, variable_name: str, fs: fsspec.AbstractFileSystem ) -> Tuple[int, int, int]: @@ -144,7 +138,7 @@ def get_data_for_coordinates( Returns: xr.Dataset: Dataset containing the requested variable at the specified location. """ - meta = get_domain_info(domain_name, FS) + meta = OmMetaJson.from_s3_json_path(f"openmeteo/data/{domain_name}/static/meta.json", FS) print("domain info: ", meta) start_timestamp = np.datetime64(start_date) diff --git a/python/omfiles/grids/gaussian.py b/python/omfiles/grids/gaussian.py index 2c4ac671..0ef6cd1c 100644 --- a/python/omfiles/grids/gaussian.py +++ b/python/omfiles/grids/gaussian.py @@ -765,9 +765,8 @@ def _compute_coordinates(self) -> None: def get_meshgrid(self) -> Tuple[npt.NDArray[np.float64], npt.NDArray[np.float64]]: """ - Get meshgrid - for Gaussian grids this returns 1D arrays. - - Returns: - (longitude, latitude) 1D arrays + Meshgrids are not meaningful for Gaussian grids, because the grid points are not evenly spaced. """ - return (self.longitude, self.latitude) + raise NotImplementedError( + "Meshgrids are not meaningful for Gaussian grids. Use earthkit.regrid to regrid to a regular grid." + ) diff --git a/python/omfiles/grids/regular.py b/python/omfiles/grids/regular.py index 52b5e56e..e9852040 100644 --- a/python/omfiles/grids/regular.py +++ b/python/omfiles/grids/regular.py @@ -10,9 +10,6 @@ class RegularGrid: """Regular latitude/longitude or projected grid.""" - # lon_grid: npt.NDArray[np.float64] - # lat_grid: npt.NDArray[np.float64] - def __init__(self, crs_wkt: str, shape: Tuple[int, int]): """ Initialize grid from WKT projection string and data shape. @@ -138,17 +135,11 @@ def get_coordinates(self, x: int, y: int) -> Tuple[float, float]: def get_meshgrid(self) -> Tuple[npt.NDArray[np.float64], npt.NDArray[np.float64]]: """ - Get meshgrid of projection coordinates. + Get meshgrid of geographic coordinates. Useful for plotting with matplotlib/cartopy. Returns: - (lon_grid, lat_grid) arrays of shape (ny, nx) in projection coordinates + (lon_grid, lat_grid) arrays of shape (ny, nx) in geographic coordinates """ - x_coords: npt.NDArray[np.float64] = np.linspace( - self.origin[0], self.origin[0] + self.dx * (self.nx - 1), self.nx - ) - y_coords: npt.NDArray[np.float64] = np.linspace( - self.origin[1], self.origin[1] + self.dy * (self.ny - 1), self.ny - ) - return np.meshgrid(x_coords, y_coords) + return (self.longitude, self.latitude) diff --git a/python/omfiles/om_grid.py b/python/omfiles/om_grid.py index 57410828..b73767e0 100644 --- a/python/omfiles/om_grid.py +++ b/python/omfiles/om_grid.py @@ -4,6 +4,7 @@ from dataclasses import dataclass, fields from typing import List, Optional, Tuple, Union +import fsspec import numpy as np import numpy.typing as npt from pyproj import CRS @@ -43,6 +44,12 @@ def from_metajson_string(cls, metajson_str: str) -> "OmMetaJson": """Create instance from metajson string.""" return cls.from_dict(json.loads(metajson_str)) + @classmethod + def from_s3_json_path(cls, s3_json_path: str, fs: fsspec.AbstractFileSystem) -> "OmMetaJson": + """Create instance from S3 JSON path.""" + meta_dict = json.loads(fs.cat_file(s3_json_path)) + return cls.from_dict(meta_dict) + def time_to_chunk_index(self, timestamp: np.datetime64) -> int: """ Convert a timestamp to a chunk index. @@ -147,7 +154,7 @@ def get_coordinates(self, x: int, y: int) -> Tuple[float, float]: return self._grid.get_coordinates(x, y) def get_meshgrid(self) -> Tuple[npt.NDArray[np.float64], npt.NDArray[np.float64]]: - """Get meshgrid of coordinates.""" + """Get meshgrid of geographic coordinates.""" return self._grid.get_meshgrid() @property From 8c42404e176a1951a3e82df2fb209e4ecb39b8fc Mon Sep 17 00:00:00 2001 From: terraputix Date: Wed, 14 Jan 2026 12:52:07 +0100 Subject: [PATCH 35/50] update examples --- examples/select_by_coordinates.py | 194 +++++++++++++----------------- examples/wkt2_crs.py | 73 +++++++++++ 2 files changed, 155 insertions(+), 112 deletions(-) create mode 100644 examples/wkt2_crs.py diff --git a/examples/select_by_coordinates.py b/examples/select_by_coordinates.py index 020f9763..6d3a7c3d 100644 --- a/examples/select_by_coordinates.py +++ b/examples/select_by_coordinates.py @@ -11,31 +11,6 @@ # ] # /// -""" -Example showing how to select data from multiple domains in Open-Meteo files stored in S3. - -This script demonstrates how to: -1. Use the OmDomain class to work with weather model domains -2. Find the correct grid point for specific latitude/longitude coordinates -3. Load data from S3 using fsspec -4. Convert the data to an xarray Dataset for analysis -5. Extract time series for the selected coordinates across multiple files -6. Merge timeseries data from multiple chunks -7. Plot data from multiple domains in a single figure - -Usage: - python examples/select_by_coordinates.py - -Requirements: - - fsspec - - s3fs - - xarray - - numpy - - matplotlib (for plotting) - - omfiles -""" - -import json from datetime import datetime from typing import Tuple @@ -205,90 +180,85 @@ def get_data_for_coordinates( return ds -if __name__ == "__main__": - # Example coordinates: Paris - latitude = 48.864716 - longitude = 2.349014 - - # # Example coordinates: Vancouver - # latitude = 49.246 - # longitude =-123.116 - - # Define a date range - start_date = datetime(2025, 4, 25, 12, 0) # 25-04-2025'T'12:00 - end_date = datetime(2025, 5, 18, 12, 0) # 18-05-2025'T'12:00 - - # Variable to fetch - variable = "temperature_2m" - - print(f"Fetching {variable} data for coordinates: {latitude}N, {longitude}E") - print(f"Date range: {start_date} to {end_date}") - - # Domain display names for nicer legends - domains_and_display_names = { - "dwd_icon": "DWD ICON (Global)", - "dwd_icon_eu": "DWD ICON (Europe)", - "dwd_icon_d2": "DWD ICON D2 (Central Europe)", - "ecmwf_ifs025": "ECMWF IFS (Global)", - "ecmwf_ifs": "ECMWF IFS HRES (Global)", - "meteofrance_arpege_europe": "Météo-France ARPEGE (Europe)", - "meteofrance_arpege_world025": "Météo-France ARPEGE (Global)", - "meteofrance_arome_france0025": "Météo-France AROME (France)", - "meteofrance_arome_france_hd": "Météo-France AROME HD (France)", - "meteofrance_arome_france_hd_15min": "Météo-France AROME HD 15min (France)", - "cmc_gem_gdps": "CMC GEM GDPS (Global)", - "cmc_gem_rdps": "CMC GEM RDPS (Regional)", - "cmc_gem_hrdps": "CMC GEM HRDPS (Continental)", - } - - # Collect data from each domain - domain_data = {} - successful_domains = [] - - # Loop through all domains in the main function - for domain_name in domains_and_display_names.keys(): - try: - print(f"\nTrying to fetch data from domain: {domain_name}") - ds = get_data_for_coordinates( - lat=latitude, - lon=longitude, - start_date=start_date, - end_date=end_date, - domain_name=domain_name, - variable_name=variable, - ) - domain_data[domain_name] = ds - successful_domains.append(domain_name) - print(f"Successfully fetched data from {domain_name}") - except Exception as e: - print(f"Could not fetch data from {domain_name}: {e}") - domain_data[domain_name] = None - - print(f"\nSuccessfully fetched data from {len(successful_domains)} domains: {successful_domains}") - - if not successful_domains: - print("No data could be fetched from any domain. Exiting.") - exit(1) - - # Domain colors for consistent line colors - colors = plt.get_cmap("tab10")(np.linspace(0, 1, len(successful_domains))) - - plt.figure(figsize=(12, 6)) - - # Plot data from each domain - for i, domain_name in enumerate(successful_domains): - ds = domain_data[domain_name] - label = domains_and_display_names[domain_name] - ds[variable].plot(label=label, color=colors[i], linewidth=2) - - # Enhance the plot - plt.title(f"{variable.replace('_', ' ').title()} at {latitude:.2f}N, {longitude:.2f}E") - plt.xlabel("Time") - plt.ylabel("Temperature (°C)" if variable == "temperature_2m" else variable) - plt.grid(True, alpha=0.3) - plt.legend(loc="best") - plt.tight_layout() - - # Save and show the figure - plt.savefig(f"{variable}_comparison.png", dpi=150) - plt.show() +# Example coordinates: Paris +latitude = 48.864716 +longitude = 2.349014 + +# Define a date range +start_date = datetime(2025, 4, 25, 12, 0) # 25-04-2025'T'12:00 +end_date = datetime(2025, 5, 18, 12, 0) # 18-05-2025'T'12:00 + +# Variable to fetch +variable = "temperature_2m" + +print(f"Fetching {variable} data for coordinates: {latitude}N, {longitude}E") +print(f"Date range: {start_date} to {end_date}") + +# Domain display names for nicer legends +domains_and_display_names = { + "dwd_icon": "DWD ICON (Global)", + "dwd_icon_eu": "DWD ICON (Europe)", + "dwd_icon_d2": "DWD ICON D2 (Central Europe)", + "ecmwf_ifs025": "ECMWF IFS (Global)", + "ecmwf_ifs": "ECMWF IFS HRES (Global)", + "meteofrance_arpege_europe": "Météo-France ARPEGE (Europe)", + "meteofrance_arpege_world025": "Météo-France ARPEGE (Global)", + "meteofrance_arome_france0025": "Météo-France AROME (France)", + "meteofrance_arome_france_hd": "Météo-France AROME HD (France)", + "meteofrance_arome_france_hd_15min": "Météo-France AROME HD 15min (France)", + "cmc_gem_gdps": "CMC GEM GDPS (Global)", + "cmc_gem_rdps": "CMC GEM RDPS (Regional)", + "cmc_gem_hrdps": "CMC GEM HRDPS (Continental)", +} + +# Collect data from each domain +domain_data = {} +successful_domains = [] + +# Loop through all domains in the main function +for domain_name in domains_and_display_names.keys(): + try: + print(f"\nTrying to fetch data from domain: {domain_name}") + ds = get_data_for_coordinates( + lat=latitude, + lon=longitude, + start_date=start_date, + end_date=end_date, + domain_name=domain_name, + variable_name=variable, + ) + domain_data[domain_name] = ds + successful_domains.append(domain_name) + print(f"Successfully fetched data from {domain_name}") + except Exception as e: + print(f"Could not fetch data from {domain_name}: {e}") + domain_data[domain_name] = None + +print(f"\nSuccessfully fetched data from {len(successful_domains)} domains: {successful_domains}") + +if not successful_domains: + print("No data could be fetched from any domain. Exiting.") + exit(1) + +# Domain colors for consistent line colors +colors = plt.get_cmap("tab10")(np.linspace(0, 1, len(successful_domains))) + +plt.figure(figsize=(12, 6)) + +# Plot data from each domain +for i, domain_name in enumerate(successful_domains): + ds = domain_data[domain_name] + label = domains_and_display_names[domain_name] + ds[variable].plot(label=label, color=colors[i], linewidth=2) + +# Enhance the plot +plt.title(f"{variable.replace('_', ' ').title()} at {latitude:.2f}N, {longitude:.2f}E") +plt.xlabel("Time") +plt.ylabel("Temperature (°C)" if variable == "temperature_2m" else variable) +plt.grid(True, alpha=0.3) +plt.legend(loc="best") +plt.tight_layout() + +# Save and show the figure +plt.savefig(f"{variable}_comparison.png", dpi=150) +plt.show() diff --git a/examples/wkt2_crs.py b/examples/wkt2_crs.py new file mode 100644 index 00000000..7795c686 --- /dev/null +++ b/examples/wkt2_crs.py @@ -0,0 +1,73 @@ +#!/usr/bin/env -S uv run --script +# +# /// script +# requires-python = ">=3.12" +# dependencies = [ +# "omfiles[proj] @ /home/fred/dev/terraputix/python-omfiles", +# "fsspec>=2025.7.0", +# "s3fs", +# "matplotlib", +# ] +# /// + +import fsspec +import matplotlib.pyplot as plt +import numpy as np +from omfiles import OmFileReader +from omfiles.om_grid import OmGrid + +# Example: URI for a spatial data file in the `data_spatial` S3 bucket +# See data organization details: https://github.com/open-meteo/open-data?tab=readme-ov-file#data-organization +# Note: Spatial data is only retained for 7 days. The example file below may no longer exist. +# Please update the URI to match a currently available file. +# Other models to test: ncep_gfs013 cmc_gem_hrdps cmc_gem_rdps ukmo_uk_deterministic_2km dmi_harmonie_aroma_europe meteofrance_arome_france0025 +s3_uri = "s3://openmeteo/data_spatial/meteoswiss_icon_ch1/2026/01/06/0000Z/2026-01-06T0000.om" + +# Note: This code does not support ECMWF IFS HRES grids (Reduced Gaussian O1280)! + +backend = fsspec.open( + f"blockcache::{s3_uri}", + mode="rb", + s3={"anon": True, "default_block_size": 65536}, + blockcache={"cache_storage": "cache"}, +) +with OmFileReader(backend) as reader: + # Get the full data array and read into regular "data" array + child = reader.get_child_by_name("temperature_2m") + print("child.shape", child.shape) + print("child.chunks", child.chunks) + data = child[:] + + # Setup projection + crs_wkt = reader.get_child_by_name("crs_wkt").read_scalar() + grid = OmGrid(crs_wkt, shape=data.shape) + # Get coordinate meshgrid for plotting + lon_grid, lat_grid = grid.get_meshgrid() + + # Create figure and axis + fig, ax = plt.subplots(figsize=(10, 5)) + + from pyproj import CRS + + crs = CRS.from_wkt(crs_wkt) + + # Simple plot without cartopy + c = ax.pcolormesh(lon_grid, lat_grid, data, cmap="coolwarm", shading="auto") + + # Add colorbar + plt.colorbar(c, ax=ax, orientation="vertical", label="Temperature (°C)") + + # Add labels + ax.set_xlabel("Longitude") + ax.set_ylabel("Latitude") + ax.set_title(f"Temperature Data\nCRS: {crs.name}") + + # Set aspect ratio + ax.set_aspect("equal") + + # Add grid + ax.grid(True, alpha=0.3) + + plt.tight_layout() + plt.savefig("temperature_map_pyproj.png", dpi=150) + print("Plot saved to temperature_map_pyproj.png") From 53dd628dd286d151219ff69e75da9d903d317989 Mon Sep 17 00:00:00 2001 From: terraputix Date: Wed, 14 Jan 2026 13:22:45 +0100 Subject: [PATCH 36/50] better dependency specification --- examples/readme_example.py | 6 ++---- .../regrid_ecmwf_ifs_hres_gaussian_O1280_to_0.1_degree.py | 4 +--- examples/select_by_coordinates.py | 5 +---- examples/spatial_xarray.py | 5 +---- examples/wkt2_crs.py | 7 ++----- 5 files changed, 7 insertions(+), 20 deletions(-) diff --git a/examples/readme_example.py b/examples/readme_example.py index 2a00450b..ecc6c8b5 100644 --- a/examples/readme_example.py +++ b/examples/readme_example.py @@ -3,9 +3,7 @@ # /// script # requires-python = ">=3.12" # dependencies = [ -# "omfiles==1.0.1", -# "fsspec>=2025.7.0", -# "s3fs", +# "omfiles[fsspec]==1.0.1", # ] # /// @@ -17,7 +15,7 @@ # See data organization details: https://github.com/open-meteo/open-data?tab=readme-ov-file#data-organization # Note: Spatial data is only retained for 7 days. The example file below may no longer exist. # Please update the URI to match a currently available file. -s3_uri = "s3://openmeteo/data_spatial/dwd_icon/2025/09/23/0000Z/2025-09-30T0000.om" +s3_uri = "s3://openmeteo/data_spatial/dwd_icon/2026/01/10/0000Z/2026-01-12T0000.om" # Create and open filesystem, wrapping it in a blockcache backend = fsspec.open( diff --git a/examples/regrid_ecmwf_ifs_hres_gaussian_O1280_to_0.1_degree.py b/examples/regrid_ecmwf_ifs_hres_gaussian_O1280_to_0.1_degree.py index 4efb8748..137510a3 100644 --- a/examples/regrid_ecmwf_ifs_hres_gaussian_O1280_to_0.1_degree.py +++ b/examples/regrid_ecmwf_ifs_hres_gaussian_O1280_to_0.1_degree.py @@ -3,9 +3,7 @@ # /// script # requires-python = ">=3.12" # dependencies = [ -# "omfiles>=1.0.1", -# "fsspec>=2025.7.0", -# "s3fs", +# "omfiles[fsspec]>=1.0.1", # "matplotlib", # "cartopy", # "earthkit-regrid==0.5.0", diff --git a/examples/select_by_coordinates.py b/examples/select_by_coordinates.py index 6d3a7c3d..0db89c92 100644 --- a/examples/select_by_coordinates.py +++ b/examples/select_by_coordinates.py @@ -3,10 +3,7 @@ # /// script # requires-python = ">=3.12" # dependencies = [ -# "omfiles[proj] @ /home/fred/dev/terraputix/python-omfiles", -# "fsspec>=2025.7.0", -# "s3fs", -# "xarray", +# "omfiles[proj,fsspec,xarray] @ /home/fred/dev/terraputix/python-omfiles", # "matplotlib", # ] # /// diff --git a/examples/spatial_xarray.py b/examples/spatial_xarray.py index e6295348..041ba787 100644 --- a/examples/spatial_xarray.py +++ b/examples/spatial_xarray.py @@ -3,10 +3,7 @@ # /// script # requires-python = ">=3.12" # dependencies = [ -# "omfiles==1.0.1", -# "fsspec>=2025.7.0", -# "s3fs", -# "xarray", +# "omfiles[fsspec,xarray]==1.0.1", # "matplotlib", # "cartopy", # ] diff --git a/examples/wkt2_crs.py b/examples/wkt2_crs.py index 7795c686..f2a12a9b 100644 --- a/examples/wkt2_crs.py +++ b/examples/wkt2_crs.py @@ -3,9 +3,7 @@ # /// script # requires-python = ">=3.12" # dependencies = [ -# "omfiles[proj] @ /home/fred/dev/terraputix/python-omfiles", -# "fsspec>=2025.7.0", -# "s3fs", +# "omfiles[fsspec,proj] @ /home/fred/dev/terraputix/python-omfiles", # "matplotlib", # ] # /// @@ -15,6 +13,7 @@ import numpy as np from omfiles import OmFileReader from omfiles.om_grid import OmGrid +from pyproj import CRS # Example: URI for a spatial data file in the `data_spatial` S3 bucket # See data organization details: https://github.com/open-meteo/open-data?tab=readme-ov-file#data-organization @@ -47,8 +46,6 @@ # Create figure and axis fig, ax = plt.subplots(figsize=(10, 5)) - from pyproj import CRS - crs = CRS.from_wkt(crs_wkt) # Simple plot without cartopy From 60f75081ca62431a9c30be050448788dd9a91de1 Mon Sep 17 00:00:00 2001 From: terraputix Date: Wed, 14 Jan 2026 13:45:31 +0100 Subject: [PATCH 37/50] simplify script --- examples/select_by_coordinates.py | 27 ++++++++++----------------- 1 file changed, 10 insertions(+), 17 deletions(-) diff --git a/examples/select_by_coordinates.py b/examples/select_by_coordinates.py index 0db89c92..cedc2f0f 100644 --- a/examples/select_by_coordinates.py +++ b/examples/select_by_coordinates.py @@ -25,13 +25,11 @@ # We load data from this Cached Fs-Spec Filesystem FS = CachingFileSystem( fs=S3FileSystem(anon=True, default_block_size=256, default_cache_type="none"), - # we keep the cache_check short: If files are modified on the remote, - # but we cache parts of these files locally, we potentially run into crashes/UB + # TODO: we'd need to verify files do not change on the remote if they still could change cache_check=60, block_size=256, cache_storage="cache", check_files=False, - cache_mapper=BasenameCacheMapper(directory_levels=3), ) @@ -102,7 +100,7 @@ def get_data_for_coordinates( Args: lat (float): Latitude in degrees. lon (float): Longitude in degrees. - domain_name (str): Name of the domain to use (must be in omfiles.om_domains.DOMAINS). + domain_name (str): Name of the domain to use (must have data on AWS S3 under openmeteo/data/{domain_name}/). variable_name (str): Name of the variable to fetch. start_date (datetime): Start date for the data. end_date (datetime): End date for the data. @@ -111,7 +109,6 @@ def get_data_for_coordinates( xr.Dataset: Dataset containing the requested variable at the specified location. """ meta = OmMetaJson.from_s3_json_path(f"openmeteo/data/{domain_name}/static/meta.json", FS) - print("domain info: ", meta) start_timestamp = np.datetime64(start_date) end_timestamp = np.datetime64(end_date) @@ -159,7 +156,7 @@ def get_data_for_coordinates( time_array = np.concatenate(all_times) data_array = np.concatenate(all_data) - # Create the xarray dataset + # Create xarray dataset ds = xr.Dataset( data_vars={ variable_name: (["time"], data_array), @@ -209,8 +206,7 @@ def get_data_for_coordinates( } # Collect data from each domain -domain_data = {} -successful_domains = [] +domain_data: dict[str, xr.Dataset] = {} # Loop through all domains in the main function for domain_name in domains_and_display_names.keys(): @@ -225,28 +221,25 @@ def get_data_for_coordinates( variable_name=variable, ) domain_data[domain_name] = ds - successful_domains.append(domain_name) print(f"Successfully fetched data from {domain_name}") except Exception as e: print(f"Could not fetch data from {domain_name}: {e}") - domain_data[domain_name] = None -print(f"\nSuccessfully fetched data from {len(successful_domains)} domains: {successful_domains}") +print(f"\nSuccessfully fetched data from {len(domain_data)} domains: {domain_data.keys()}") -if not successful_domains: +if len(domain_data) == 0: print("No data could be fetched from any domain. Exiting.") exit(1) # Domain colors for consistent line colors -colors = plt.get_cmap("tab10")(np.linspace(0, 1, len(successful_domains))) +colors = plt.get_cmap("tab10")(np.linspace(0, 1, len(domain_data))) plt.figure(figsize=(12, 6)) # Plot data from each domain -for i, domain_name in enumerate(successful_domains): - ds = domain_data[domain_name] - label = domains_and_display_names[domain_name] - ds[variable].plot(label=label, color=colors[i], linewidth=2) +for i, (domain_name, ds) in enumerate(domain_data.items()): + label = domains_and_display_names.get(domain_name, domain_name) + plt.plot(ds["time"].values, ds[variable].values, label=label, color=colors[i], linewidth=2) # Enhance the plot plt.title(f"{variable.replace('_', ' ').title()} at {latitude:.2f}N, {longitude:.2f}E") From 69e190dca10af2de05f776b79615d7f035ef98f7 Mon Sep 17 00:00:00 2001 From: terraputix Date: Wed, 14 Jan 2026 14:27:48 +0100 Subject: [PATCH 38/50] separate meta implementation for spatial und chunk meta --- python/omfiles/om_grid.py | 102 ++----------------------------- python/omfiles/om_meta.py | 122 ++++++++++++++++++++++++++++++++++++++ tests/test_meta.py | 12 ++-- 3 files changed, 132 insertions(+), 104 deletions(-) create mode 100644 python/omfiles/om_meta.py diff --git a/python/omfiles/om_grid.py b/python/omfiles/om_grid.py index b73767e0..930fcffc 100644 --- a/python/omfiles/om_grid.py +++ b/python/omfiles/om_grid.py @@ -1,10 +1,7 @@ """An OmGrid provides utilities to transform between geographic coordinates and grid indices.""" -import json -from dataclasses import dataclass, fields -from typing import List, Optional, Tuple, Union +from typing import Optional, Tuple, Union -import fsspec import numpy as np import numpy.typing as npt from pyproj import CRS @@ -15,99 +12,6 @@ EPOCH = np.datetime64(0, "s") -@dataclass -class OmMetaJson: - """Class to decode Open-Meteo metadata JSON files.""" - - chunk_time_length: int # Number of time steps per chunk (file_length) - crs_wkt: str # Coordinate Reference System in Well-Known Text format - # data_end_time: int # Unix timestamp for when data ends - # last_run_availability_time: int # Unix timestamp for last availability - # last_run_initialisation_time: int # Unix timestamp for last initialization - # last_run_modification_time: int # Unix timestamp for last modification - temporal_resolution_seconds: int # Time resolution in seconds - # update_interval_seconds: int # How often data is updated - - @classmethod - def from_dict(cls, data: dict) -> "OmMetaJson": - """Create instance from dictionary, ignoring extra keys.""" - # Get the names of all fields defined in the dataclass - class_fields = {f.name for f in fields(cls)} - - # Filter the input dictionary - filtered_data = {k: v for k, v in data.items() if k in class_fields} - - return cls(**filtered_data) - - @classmethod - def from_metajson_string(cls, metajson_str: str) -> "OmMetaJson": - """Create instance from metajson string.""" - return cls.from_dict(json.loads(metajson_str)) - - @classmethod - def from_s3_json_path(cls, s3_json_path: str, fs: fsspec.AbstractFileSystem) -> "OmMetaJson": - """Create instance from S3 JSON path.""" - meta_dict = json.loads(fs.cat_file(s3_json_path)) - return cls.from_dict(meta_dict) - - def time_to_chunk_index(self, timestamp: np.datetime64) -> int: - """ - Convert a timestamp to a chunk index. - - This depends on the file_length and the temporal_resolution_seconds of the domain. - - Args: - timestamp (np.datetime64): The timestamp to convert. - - Returns: - int: The chunk index containing the timestamp. - """ - seconds_since_epoch = (timestamp - EPOCH) / np.timedelta64(1, "s") - chunk_index = int(seconds_since_epoch / (self.chunk_time_length * self.temporal_resolution_seconds)) - return chunk_index - - def chunks_for_date_range( - self, - start_timestamp: np.datetime64, - end_timestamp: np.datetime64, - ) -> List[int]: - """ - Find all chunk indices that contain data within the given date range. - - Args: - start_timestamp (np.datetime64): Start timestamp for the data range. - end_timestamp (np.datetime64): End timestamp for the data range. - - Returns: - List[int]: List of chunk indices containing data within the date range. - """ - # Get chunk indices for start and end dates - start_chunk = self.time_to_chunk_index(start_timestamp) - end_chunk = self.time_to_chunk_index(end_timestamp) - - # Generate list of all chunks between start and end (inclusive) - return list(range(start_chunk, end_chunk + 1)) - - def get_chunk_time_range(self, chunk_index: int): - """ - Get the time range covered by a specific chunk. - - Args: - chunk_index (int): Index of the chunk. - - Returns: - np.ndarray: Array of datetime64 objects representing the time points in the chunk. - """ - chunk_start_seconds = chunk_index * self.chunk_time_length * self.temporal_resolution_seconds - start_time = EPOCH + np.timedelta64(chunk_start_seconds, "s") - - # Generate timestamps at regular intervals from the start time - time_delta = np.timedelta64(self.temporal_resolution_seconds, "s") - # Note: better type inference via list comprehension here - timestamps = np.array([start_time + i * time_delta for i in range(self.chunk_time_length)]) - return timestamps - - def _is_gaussian_grid(crs_wkt: str) -> bool: """Check if WKT string represents a Gaussian grid.""" return "Reduced Gaussian Grid" in crs_wkt or "Gaussian Grid" in crs_wkt @@ -116,7 +20,7 @@ def _is_gaussian_grid(crs_wkt: str) -> bool: class OmGrid: """Wrapper for grid implementations - automatically delegates to appropriate grid type.""" - def __init__(self, crs_wkt: str, shape: Tuple[int, int]): + def __init__(self, crs_wkt: str, shape: tuple[int, ...]): """ Initialize grid from WKT projection string and data shape. @@ -124,6 +28,8 @@ def __init__(self, crs_wkt: str, shape: Tuple[int, int]): crs_wkt: Coordinate Reference System in Well-Known Text format shape: Grid shape as (ny, nx) """ + if not isinstance(shape, tuple) or not all(isinstance(dim, int) for dim in shape) or not len(shape) == 2: + raise ValueError("shape must be a tuple of two integers") # Detect grid type and create appropriate implementation if _is_gaussian_grid(crs_wkt): self._grid = GaussianGrid(crs_wkt, shape) diff --git a/python/omfiles/om_meta.py b/python/omfiles/om_meta.py new file mode 100644 index 00000000..cf580789 --- /dev/null +++ b/python/omfiles/om_meta.py @@ -0,0 +1,122 @@ +"""Representation of Open-Meteo meta.json files.""" + +import json +from dataclasses import dataclass, fields +from typing import List + +try: + from typing import Self # Python 3.11+ +except ImportError: + from typing_extensions import Self # Python < 3.11 + + +import fsspec +import numpy as np + +EPOCH = np.datetime64(0, "s") + + +@dataclass +class OmMetaBase: + """Base class for Open-Meteo metadata.""" + + crs_wkt: str # Coordinate Reference System in Well-Known Text format + + @classmethod + def from_dict(cls, data: dict) -> Self: + """Create instance from dictionary, ignoring extra keys.""" + # fields(cls) correctly identifies fields in the subclass + class_fields = {f.name for f in fields(cls)} + filtered_data = {k: v for k, v in data.items() if k in class_fields} + return cls(**filtered_data) + + @classmethod + def from_metajson_string(cls, metajson_str: str) -> Self: + """Create instance from metajson string.""" + return cls.from_dict(json.loads(metajson_str)) + + @classmethod + def from_s3_json_path(cls, s3_json_path: str, fs: fsspec.AbstractFileSystem) -> Self: + """Create instance from S3 JSON path.""" + meta_dict = json.loads(fs.cat_file(s3_json_path)) + return cls.from_dict(meta_dict) + + +@dataclass +class OmMetaSpatial(OmMetaBase): + """Representation of the meta.json for spatial datasets.""" + + last_modified_time: str # ISO8601 for last modification + reference_time: str # ISO8601 for reference time + valid_times: List[str] # List of valid times in ISO8601 format + variables: List[str] # List of variables in the dataset + + +@dataclass +class OmMetaChunks(OmMetaBase): + """Representation of the meta.json for time oriented chunks.""" + + chunk_time_length: int # Number of time steps per chunk (file_length) + data_end_time: int # Unix timestamp for when data ends + last_run_availability_time: int # Unix timestamp for last availability + last_run_initialisation_time: int # Unix timestamp for last initialization + last_run_modification_time: int # Unix timestamp for last modification + temporal_resolution_seconds: int # Time resolution in seconds + update_interval_seconds: int # How often data is updated + + def time_to_chunk_index(self, timestamp: np.datetime64) -> int: + """ + Convert a timestamp to a chunk index. + + This depends on the file_length and the temporal_resolution_seconds of the domain. + + Args: + timestamp (np.datetime64): The timestamp to convert. + + Returns: + int: The chunk index containing the timestamp. + """ + seconds_since_epoch = (timestamp - EPOCH) / np.timedelta64(1, "s") + chunk_index = int(seconds_since_epoch / (self.chunk_time_length * self.temporal_resolution_seconds)) + return chunk_index + + def chunks_for_date_range( + self, + start_timestamp: np.datetime64, + end_timestamp: np.datetime64, + ) -> List[int]: + """ + Find all chunk indices that contain data within the given date range. + + Args: + start_timestamp (np.datetime64): Start timestamp for the data range. + end_timestamp (np.datetime64): End timestamp for the data range. + + Returns: + List[int]: List of chunk indices containing data within the date range. + """ + # Get chunk indices for start and end dates + start_chunk = self.time_to_chunk_index(start_timestamp) + end_chunk = self.time_to_chunk_index(end_timestamp) + + # Generate list of all chunks between start and end (inclusive) + return list(range(start_chunk, end_chunk + 1)) + + def get_chunk_time_range(self, chunk_index: int): + """ + Get the time range covered by a specific chunk. + + Args: + chunk_index (int): Index of the chunk. + + Returns: + np.ndarray: Array of datetime64 objects representing the time points in the chunk. + """ + chunk_start_seconds = chunk_index * self.chunk_time_length * self.temporal_resolution_seconds + start_time = EPOCH + np.timedelta64(chunk_start_seconds, "s") + + # Generate timestamps at regular intervals from the start time + time_delta = np.timedelta64(self.temporal_resolution_seconds, "s") + # Note: better type inference via list comprehension here + timestamps = np.array([start_time + i * time_delta for i in range(self.chunk_time_length)]) + return timestamps diff --git a/tests/test_meta.py b/tests/test_meta.py index 20dc0c82..80db5a09 100644 --- a/tests/test_meta.py +++ b/tests/test_meta.py @@ -1,6 +1,6 @@ import numpy as np import pytest -from omfiles.om_grid import OmMetaJson +from omfiles.om_meta import OmMetaChunks @pytest.fixture @@ -10,17 +10,17 @@ def icon_d2_meta_json() -> str: @pytest.fixture -def icon_d2_meta(icon_d2_meta_json: str) -> OmMetaJson: - return OmMetaJson.from_metajson_string(icon_d2_meta_json) +def icon_d2_meta(icon_d2_meta_json: str) -> OmMetaChunks: + return OmMetaChunks.from_metajson_string(icon_d2_meta_json) def test_meta_json_creation(icon_d2_meta_json: str): """Test creation of OmMetaJson object from JSON string.""" - meta = OmMetaJson.from_metajson_string(icon_d2_meta_json) + meta = OmMetaChunks.from_metajson_string(icon_d2_meta_json) assert meta.chunk_time_length == 121 -def test_time_to_chunk_index(icon_d2_meta: OmMetaJson): +def test_time_to_chunk_index(icon_d2_meta: OmMetaChunks): """Test conversion from timestamp to chunk index.""" # Create test timestamp (2023-01-01 12:00:00 UTC) @@ -40,7 +40,7 @@ def test_time_to_chunk_index(icon_d2_meta: OmMetaJson): assert chunk_index == expected_chunk -def test_get_chunk_time_range(icon_d2_meta: OmMetaJson): +def test_get_chunk_time_range(icon_d2_meta: OmMetaChunks): """Test getting time range for a specific chunk.""" # Test chunk 1000 From 2105e1ca1cb72bde0a30f2ee9727a8a736a75f83 Mon Sep 17 00:00:00 2001 From: terraputix Date: Wed, 14 Jan 2026 14:28:29 +0100 Subject: [PATCH 39/50] adjust example scripts --- examples/plot_map.py | 11 ++-- examples/select_by_coordinates.py | 100 ++++++++++++++---------------- examples/spatial_xarray.py | 25 ++++---- 3 files changed, 65 insertions(+), 71 deletions(-) diff --git a/examples/plot_map.py b/examples/plot_map.py index b54967dd..30545e62 100755 --- a/examples/plot_map.py +++ b/examples/plot_map.py @@ -17,14 +17,17 @@ import matplotlib.pyplot as plt import numpy as np from omfiles import OmFileReader -from omfiles.om_grid import OmGrid, OmMetaJson +from omfiles.om_grid import OmGrid +from omfiles.om_meta import OmMetaSpatial -domain_name = "dmi_harmonie_arome_europe" +MODEL_DOMAIN = "dmi_harmonie_arome_europe" # Example: URI for a spatial data file in the `data_spatial` S3 bucket # See data organization details: https://github.com/open-meteo/open-data?tab=readme-ov-file#data-organization # Note: Spatial data is only retained for 7 days. The example file below may no longer exist. # Please update the URI to match a currently available file. -s3_uri = f"s3://openmeteo/data_spatial/{domain_name}/2026/01/10/0000Z/2026-01-12T0000.om" +s3_run = f"s3://openmeteo/data_spatial/{MODEL_DOMAIN}/2026/01/10/0000Z/" +s3_uri = f"{s3_run}2026-01-12T0000.om" +s3_meta_json = f"{s3_run}meta.json" # The following two incantations are equivalent # @@ -67,7 +70,7 @@ # Create coordinate arrays num_y, num_x = child.shape - meta = OmMetaJson.from_s3_json_path(f"openmeteo/data/{domain_name}/static/meta.json", backend.fs) + meta = OmMetaSpatial.from_s3_json_path(s3_meta_json, backend.fs) grid = OmGrid(meta.crs_wkt, (num_y, num_x)) lon_grid, lat_grid = grid.get_meshgrid() diff --git a/examples/select_by_coordinates.py b/examples/select_by_coordinates.py index cedc2f0f..06f24660 100644 --- a/examples/select_by_coordinates.py +++ b/examples/select_by_coordinates.py @@ -18,20 +18,11 @@ from fsspec.implementations.cache_mapper import BasenameCacheMapper from fsspec.implementations.cached import CachingFileSystem from omfiles import OmFileReader -from omfiles.om_grid import OmGrid, OmMetaJson +from omfiles.om_grid import OmGrid +from omfiles.om_meta import OmMetaChunks from s3fs import S3FileSystem from xarray import Dataset -# We load data from this Cached Fs-Spec Filesystem -FS = CachingFileSystem( - fs=S3FileSystem(anon=True, default_block_size=256, default_cache_type="none"), - # TODO: we'd need to verify files do not change on the remote if they still could change - cache_check=60, - block_size=256, - cache_storage="cache", - check_files=False, -) - def load_variable_dimensions( chunk_index: int, domain_name: str, variable_name: str, fs: fsspec.AbstractFileSystem @@ -50,7 +41,7 @@ def load_chunk_data( fs: fsspec.AbstractFileSystem, start_date: np.datetime64, end_date: np.datetime64, - meta: OmMetaJson, + meta: OmMetaChunks, ): """ Load data for a specific chunk and grid coordinates. @@ -108,7 +99,7 @@ def get_data_for_coordinates( Returns: xr.Dataset: Dataset containing the requested variable at the specified location. """ - meta = OmMetaJson.from_s3_json_path(f"openmeteo/data/{domain_name}/static/meta.json", FS) + meta = OmMetaChunks.from_s3_json_path(f"openmeteo/data/{domain_name}/static/meta.json", FS) start_timestamp = np.datetime64(start_date) end_timestamp = np.datetime64(end_date) @@ -174,51 +165,53 @@ def get_data_for_coordinates( return ds -# Example coordinates: Paris -latitude = 48.864716 -longitude = 2.349014 - -# Define a date range -start_date = datetime(2025, 4, 25, 12, 0) # 25-04-2025'T'12:00 -end_date = datetime(2025, 5, 18, 12, 0) # 18-05-2025'T'12:00 - -# Variable to fetch -variable = "temperature_2m" - -print(f"Fetching {variable} data for coordinates: {latitude}N, {longitude}E") -print(f"Date range: {start_date} to {end_date}") - -# Domain display names for nicer legends -domains_and_display_names = { - "dwd_icon": "DWD ICON (Global)", - "dwd_icon_eu": "DWD ICON (Europe)", - "dwd_icon_d2": "DWD ICON D2 (Central Europe)", - "ecmwf_ifs025": "ECMWF IFS (Global)", - "ecmwf_ifs": "ECMWF IFS HRES (Global)", - "meteofrance_arpege_europe": "Météo-France ARPEGE (Europe)", - "meteofrance_arpege_world025": "Météo-France ARPEGE (Global)", - "meteofrance_arome_france0025": "Météo-France AROME (France)", - "meteofrance_arome_france_hd": "Météo-France AROME HD (France)", - "meteofrance_arome_france_hd_15min": "Météo-France AROME HD 15min (France)", - "cmc_gem_gdps": "CMC GEM GDPS (Global)", - "cmc_gem_rdps": "CMC GEM RDPS (Regional)", - "cmc_gem_hrdps": "CMC GEM HRDPS (Continental)", -} +# We load data from this Cached Fs-Spec Filesystem +FS = CachingFileSystem( + fs=S3FileSystem(anon=True, default_block_size=256, default_cache_type="none"), + # TODO: we'd need to verify files do not change on the remote if they still could change + cache_check=60, + block_size=256, + cache_storage="cache", + check_files=False, +) +LATITUDE, LONGITUDE = 48.864716, 2.349014 # Paris +START_DATE = datetime(2025, 4, 25, 12, 0) # 25-04-2025'T'12:00 +END_DATE = datetime(2025, 5, 18, 12, 0) # 18-05-2025'T'12:00 +VARIABLE = "temperature_2m" +DOMAINS = [ + "dwd_icon", + "dwd_icon_eu", + "dwd_icon_d2", + "ecmwf_ifs025", + "ecmwf_ifs", + "meteofrance_arpege_europe", + "meteofrance_arpege_world025", + "meteofrance_arome_france0025", + "meteofrance_arome_france_hd", + "meteofrance_arome_france_hd_15min", + "cmc_gem_gdps", + "cmc_gem_rdps", + "cmc_gem_hrdps", +] + +print(f"Fetching {VARIABLE} data for coordinates: {LATITUDE}N, {LONGITUDE}E") +print(f"Date range: {START_DATE} to {END_DATE}") + # Collect data from each domain domain_data: dict[str, xr.Dataset] = {} # Loop through all domains in the main function -for domain_name in domains_and_display_names.keys(): +for domain_name in DOMAINS: try: print(f"\nTrying to fetch data from domain: {domain_name}") ds = get_data_for_coordinates( - lat=latitude, - lon=longitude, - start_date=start_date, - end_date=end_date, + lat=LATITUDE, + lon=LONGITUDE, + start_date=START_DATE, + end_date=END_DATE, domain_name=domain_name, - variable_name=variable, + variable_name=VARIABLE, ) domain_data[domain_name] = ds print(f"Successfully fetched data from {domain_name}") @@ -238,17 +231,16 @@ def get_data_for_coordinates( # Plot data from each domain for i, (domain_name, ds) in enumerate(domain_data.items()): - label = domains_and_display_names.get(domain_name, domain_name) - plt.plot(ds["time"].values, ds[variable].values, label=label, color=colors[i], linewidth=2) + plt.plot(ds["time"].values, ds[VARIABLE].values, label=domain_name, color=colors[i], linewidth=2) # Enhance the plot -plt.title(f"{variable.replace('_', ' ').title()} at {latitude:.2f}N, {longitude:.2f}E") +plt.title(f"{VARIABLE.replace('_', ' ').title()} at {LATITUDE:.2f}N, {LONGITUDE:.2f}E") plt.xlabel("Time") -plt.ylabel("Temperature (°C)" if variable == "temperature_2m" else variable) +plt.ylabel("Temperature (°C)" if VARIABLE == "temperature_2m" else VARIABLE) plt.grid(True, alpha=0.3) plt.legend(loc="best") plt.tight_layout() # Save and show the figure -plt.savefig(f"{variable}_comparison.png", dpi=150) +plt.savefig(f"{VARIABLE}_comparison.png", dpi=150) plt.show() diff --git a/examples/spatial_xarray.py b/examples/spatial_xarray.py index 041ba787..35b7b1d4 100644 --- a/examples/spatial_xarray.py +++ b/examples/spatial_xarray.py @@ -3,7 +3,7 @@ # /// script # requires-python = ">=3.12" # dependencies = [ -# "omfiles[fsspec,xarray]==1.0.1", +# "omfiles[fsspec,xarray,proj]@ /home/fred/dev/terraputix/python-omfiles", # "matplotlib", # "cartopy", # ] @@ -15,32 +15,29 @@ import matplotlib.pyplot as plt import numpy as np import xarray as xr -from omfiles import OmFileReader +from omfiles.om_grid import OmGrid PLOT_VARIABLE = "temperature_2m" -MODEL = "dwd_icon" +MODEL_DOMAIN = "dwd_icon" # Example: URI for a spatial data file in the `data_spatial` S3 bucket # See data organization details: https://github.com/open-meteo/open-data?tab=readme-ov-file#data-organization # Note: Spatial data is only retained for 7 days. The example file below may no longer exist. # Please update the URI to match a currently available file. -s3_spatial_uri = f"s3://openmeteo/data_spatial/{MODEL}/2025/09/23/0000Z/2025-09-30T0000.om" +s3_run = f"s3://openmeteo/data_spatial/{MODEL_DOMAIN}/2026/01/10/0000Z/" +s3_uri = f"{s3_run}2026-01-12T0000.om" +s3_meta_json = f"{s3_run}meta.json" backend = fsspec.open( - f"blockcache::{s3_spatial_uri}", + f"blockcache::{s3_uri}", mode="rb", s3={"anon": True, "default_block_size": 65536}, blockcache={"cache_storage": "cache"}, ) ds = xr.open_dataset(backend, engine="om") # type: ignore +print(ds.attrs) print(ds.variables.keys()) # any of these keys can be used for plotting -ds = ds.rename_dims({"dim0": "lat", "dim1": "lon"}) -# You need to know what exactly the dimensions of the specified domain is referring to. -# Icon is using a global regular grid: -# https://github.com/open-meteo/open-meteo/blob/a4cdae1ad139f9dfa6dd2552c0636c7e572dcb52/Sources/App/Icon/Icon.swift#L146 -ds["lat"] = np.linspace(-90, 90, ds.sizes["lat"], endpoint=True) -ds["lon"] = np.linspace(-180, 180, num=ds.sizes["lon"], endpoint=False) fig = plt.figure(figsize=(12, 8)) ax = ax = plt.axes(projection=ccrs.PlateCarree()) @@ -53,7 +50,9 @@ ax.add_feature(cfeature.RIVERS, alpha=0.3) plot_data = ds[PLOT_VARIABLE] # shape: (lat, lon) -lon2d, lat2d = np.meshgrid(ds["lon"].values, ds["lat"].values) +# Use OmGrid with the crs_wkt attribute to get the lat/lon grid +grid = OmGrid(ds.attrs["crs_wkt"], ds[PLOT_VARIABLE].shape) +lon2d, lat2d = grid.get_meshgrid() min = int(plot_data.min().values) max = int(plot_data.max().values) @@ -72,6 +71,6 @@ ) cb = plt.colorbar(c, ax=ax, orientation="vertical", pad=0.02, aspect=40, shrink=0.8) cb.set_label(PLOT_VARIABLE, fontsize=14) -plt.title(f"{MODEL} {PLOT_VARIABLE}", fontsize=14, fontweight="bold", pad=20) +plt.title(f"{MODEL_DOMAIN} {PLOT_VARIABLE}", fontsize=14, fontweight="bold", pad=20) plt.tight_layout() plt.savefig("xarray_map.png", dpi=300, bbox_inches="tight") From 3159910545bffae1e98e43446ec026ae0e177ec9 Mon Sep 17 00:00:00 2001 From: terraputix Date: Wed, 14 Jan 2026 14:32:18 +0100 Subject: [PATCH 40/50] rename feature to grids --- examples/plot_map.py | 2 +- examples/select_by_coordinates.py | 2 +- examples/spatial_xarray.py | 2 +- examples/wkt2_crs.py | 2 +- pyproject.toml | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/examples/plot_map.py b/examples/plot_map.py index 30545e62..d0313932 100755 --- a/examples/plot_map.py +++ b/examples/plot_map.py @@ -3,7 +3,7 @@ # /// script # requires-python = ">=3.12" # dependencies = [ -# "omfiles[proj] @ /home/fred/dev/terraputix/python-omfiles", +# "omfiles[grids] @ /home/fred/dev/terraputix/python-omfiles", # "fsspec>=2025.7.0", # "s3fs", # "matplotlib", diff --git a/examples/select_by_coordinates.py b/examples/select_by_coordinates.py index 06f24660..3e126f8d 100644 --- a/examples/select_by_coordinates.py +++ b/examples/select_by_coordinates.py @@ -3,7 +3,7 @@ # /// script # requires-python = ">=3.12" # dependencies = [ -# "omfiles[proj,fsspec,xarray] @ /home/fred/dev/terraputix/python-omfiles", +# "omfiles[grids,fsspec,xarray] @ /home/fred/dev/terraputix/python-omfiles", # "matplotlib", # ] # /// diff --git a/examples/spatial_xarray.py b/examples/spatial_xarray.py index 35b7b1d4..eeb70936 100644 --- a/examples/spatial_xarray.py +++ b/examples/spatial_xarray.py @@ -3,7 +3,7 @@ # /// script # requires-python = ">=3.12" # dependencies = [ -# "omfiles[fsspec,xarray,proj]@ /home/fred/dev/terraputix/python-omfiles", +# "omfiles[grids,fsspec,xarray]@ /home/fred/dev/terraputix/python-omfiles", # "matplotlib", # "cartopy", # ] diff --git a/examples/wkt2_crs.py b/examples/wkt2_crs.py index f2a12a9b..ef5f18e4 100644 --- a/examples/wkt2_crs.py +++ b/examples/wkt2_crs.py @@ -3,7 +3,7 @@ # /// script # requires-python = ">=3.12" # dependencies = [ -# "omfiles[fsspec,proj] @ /home/fred/dev/terraputix/python-omfiles", +# "omfiles[fsspec,grids] @ /home/fred/dev/terraputix/python-omfiles", # "matplotlib", # ] # /// diff --git a/pyproject.toml b/pyproject.toml index 4204b313..872261c6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -37,7 +37,7 @@ codec = [ ] xarray = ["xarray>=2023.1.0"] fsspec = ["fsspec>=2023.1.0", "s3fs>=2023.10.0"] -proj = ["pyproj>=3.1.0"] +grids = ["pyproj>=3.1.0"] all = [ "zarr>=2.18.2", "numcodecs>=0.12.1", From f709850899519b2b53ecc32a64f6dde6125ab3c1 Mon Sep 17 00:00:00 2001 From: terraputix Date: Wed, 14 Jan 2026 15:58:14 +0100 Subject: [PATCH 41/50] chunked reader for simple access to chunked files --- examples/select_by_coordinates.py | 186 ++++-------------------------- python/omfiles/chunk_reader.py | 84 ++++++++++++++ python/omfiles/om_grid.py | 6 +- python/omfiles/om_meta.py | 21 +++- python/omfiles/xarray.py | 7 +- 5 files changed, 140 insertions(+), 164 deletions(-) create mode 100644 python/omfiles/chunk_reader.py diff --git a/examples/select_by_coordinates.py b/examples/select_by_coordinates.py index 3e126f8d..bbfa6704 100644 --- a/examples/select_by_coordinates.py +++ b/examples/select_by_coordinates.py @@ -3,7 +3,7 @@ # /// script # requires-python = ">=3.12" # dependencies = [ -# "omfiles[grids,fsspec,xarray] @ /home/fred/dev/terraputix/python-omfiles", +# "omfiles[grids,fsspec] @ /home/fred/dev/terraputix/python-omfiles", # "matplotlib", # ] # /// @@ -11,159 +11,15 @@ from datetime import datetime from typing import Tuple -import fsspec import matplotlib.pyplot as plt import numpy as np -import xarray as xr -from fsspec.implementations.cache_mapper import BasenameCacheMapper +import numpy.typing as npt from fsspec.implementations.cached import CachingFileSystem from omfiles import OmFileReader +from omfiles.chunk_reader import OmFileChunkReader from omfiles.om_grid import OmGrid from omfiles.om_meta import OmMetaChunks from s3fs import S3FileSystem -from xarray import Dataset - - -def load_variable_dimensions( - chunk_index: int, domain_name: str, variable_name: str, fs: fsspec.AbstractFileSystem -) -> Tuple[int, int, int]: - s3_path = f"openmeteo/data/{domain_name}/{variable_name}/chunk_{chunk_index}.om" - with OmFileReader.from_fsspec(fs, s3_path) as reader: - return reader.shape - raise ValueError(f"Failed to load variable dimensions for chunk {chunk_index}") - - -def load_chunk_data( - chunk_index: int, - domain_name: str, - variable_name: str, - grid_coords: Tuple[int, int], - fs: fsspec.AbstractFileSystem, - start_date: np.datetime64, - end_date: np.datetime64, - meta: OmMetaChunks, -): - """ - Load data for a specific chunk and grid coordinates. - - Args: - chunk_index (int): Index of the chunk to load. - domain_name (str): Name of the domain. - variable_name (str): Name of the variable to fetch. - grid_coords (Tuple[int, int]): Grid coordinates (x, y) to extract. - fs (fsspec.AbstractFileSystem): Filesystem to use for loading data. - start_date (np.datetime64): Start of requested date range. - end_date (np.datetime64): End of requested date range. - - Returns: - Tuple[np.ndarray, np.ndarray]: Tuple containing (time_array, data_array). - """ - x, y = grid_coords - s3_path = f"openmeteo/data/{domain_name}/{variable_name}/chunk_{chunk_index}.om" - - # Generate time array and check if any times are in our range - chunk_times = meta.get_chunk_time_range(chunk_index) - time_mask = (chunk_times >= start_date) & (chunk_times <= end_date) - if not np.any(time_mask): - return np.array([], dtype="datetime64[s]"), np.array([], dtype=float) - - # Create reader and read data of interest - with OmFileReader.from_fsspec(fs, s3_path) as reader: - indices = np.where(time_mask)[0] - time_slice = slice(indices[0], indices[-1] + 1) # +1 to include the end - data = reader[y, x, time_slice] - return chunk_times[time_mask], data - - raise ValueError("Unreachable") # Make Pyright happy... - - -def get_data_for_coordinates( - lat: float, - lon: float, - start_date: datetime, - end_date: datetime, - domain_name: str = "ecmwf_ifs025", - variable_name: str = "temperature_2m", -) -> Dataset: - """ - Fetch weather data for specific coordinates across a date range, merging multiple files as needed. - - Args: - lat (float): Latitude in degrees. - lon (float): Longitude in degrees. - domain_name (str): Name of the domain to use (must have data on AWS S3 under openmeteo/data/{domain_name}/). - variable_name (str): Name of the variable to fetch. - start_date (datetime): Start date for the data. - end_date (datetime): End date for the data. - - Returns: - xr.Dataset: Dataset containing the requested variable at the specified location. - """ - meta = OmMetaChunks.from_s3_json_path(f"openmeteo/data/{domain_name}/static/meta.json", FS) - - start_timestamp = np.datetime64(start_date) - end_timestamp = np.datetime64(end_date) - - # Find all chunks needed for this date range - chunk_indices = meta.chunks_for_date_range(start_timestamp, end_timestamp) - print(f"Need to fetch {len(chunk_indices)} chunks: {chunk_indices}") - - # get dimensions of the variable - num_y, num_x, num_t = load_variable_dimensions(chunk_indices[0], domain_name, variable_name, FS) - grid = OmGrid(meta.crs_wkt, (num_y, num_x)) - - # Find grid coordinates for geographical coordinates - grid_point = grid.find_point_xy(lat, lon) - if grid_point is None: - raise ValueError(f"Coordinates ({lat}, {lon}) not found in grid of {domain_name}") - - x, y = grid_point - print(f"Found grid point {grid_point} for coordinates ({lat}, {lon})") - print(f"Fetching data from {start_date} to {end_date}") - - start_timestamp = np.datetime64(start_date) - end_timestamp = np.datetime64(end_date) - - # Find all chunks needed for this date range - chunk_indices = meta.chunks_for_date_range(start_timestamp, end_timestamp) - print(f"Need to fetch {len(chunk_indices)} chunks: {chunk_indices}") - - # Load data from all chunks - all_times = [] - all_data = [] - - for chunk_idx in chunk_indices: - times, data = load_chunk_data( - chunk_idx, domain_name, variable_name, (x, y), FS, start_timestamp, end_timestamp, meta - ) - if len(times) > 0: - all_times.append(times) - all_data.append(data) - - # Concatenate all data - if not all_times: - raise ValueError("Failed to load any data for the specified date range") - - time_array = np.concatenate(all_times) - data_array = np.concatenate(all_data) - - # Create xarray dataset - ds = xr.Dataset( - data_vars={ - variable_name: (["time"], data_array), - }, - coords={ - "time": time_array, - "latitude": lat, - "longitude": lon, - }, - attrs={ - "domain": domain_name, - "grid_indices": grid_point, - }, - ) - return ds - # We load data from this Cached Fs-Spec Filesystem FS = CachingFileSystem( @@ -175,8 +31,8 @@ def get_data_for_coordinates( check_files=False, ) LATITUDE, LONGITUDE = 48.864716, 2.349014 # Paris -START_DATE = datetime(2025, 4, 25, 12, 0) # 25-04-2025'T'12:00 -END_DATE = datetime(2025, 5, 18, 12, 0) # 18-05-2025'T'12:00 +START_DATE = np.datetime64(datetime(2025, 4, 25, 12, 0)) # 25-04-2025'T'12:00 +END_DATE = np.datetime64(datetime(2025, 5, 18, 12, 0)) # 18-05-2025'T'12:00 VARIABLE = "temperature_2m" DOMAINS = [ "dwd_icon", @@ -199,21 +55,29 @@ def get_data_for_coordinates( # Collect data from each domain -domain_data: dict[str, xr.Dataset] = {} +domain_data: dict[str, Tuple[npt.NDArray[np.datetime64], npt.NDArray[np.float64]]] = {} -# Loop through all domains in the main function for domain_name in DOMAINS: try: print(f"\nTrying to fetch data from domain: {domain_name}") - ds = get_data_for_coordinates( - lat=LATITUDE, - lon=LONGITUDE, - start_date=START_DATE, - end_date=END_DATE, - domain_name=domain_name, - variable_name=VARIABLE, + meta = OmMetaChunks.from_s3_json_path(f"openmeteo/data/{domain_name}/static/meta.json", FS) + chunk_reader = OmFileChunkReader( + meta, FS, f"s3://openmeteo/data/{domain_name}/{VARIABLE}", START_DATE, END_DATE ) - domain_data[domain_name] = ds + first = next(chunk_reader.iter_files(), None) + if first is None: + print(f"No data found for domain {domain_name}") + continue + _, s3_path = first + grid: OmGrid | None = None + with OmFileReader.from_fsspec(FS, s3_path) as reader: + grid = meta.get_grid(reader) + + assert grid is not None, "Grid not found" + indices = grid.find_point_xy(LATITUDE, LONGITUDE) + assert indices is not None, "Indices not found" + times, data = chunk_reader.load_chunked_data(indices) + domain_data[domain_name] = times, data print(f"Successfully fetched data from {domain_name}") except Exception as e: print(f"Could not fetch data from {domain_name}: {e}") @@ -230,8 +94,8 @@ def get_data_for_coordinates( plt.figure(figsize=(12, 6)) # Plot data from each domain -for i, (domain_name, ds) in enumerate(domain_data.items()): - plt.plot(ds["time"].values, ds[VARIABLE].values, label=domain_name, color=colors[i], linewidth=2) +for i, (domain_name, (times, data)) in enumerate(domain_data.items()): + plt.plot(times, data, label=domain_name, color=colors[i], linewidth=2) # Enhance the plot plt.title(f"{VARIABLE.replace('_', ' ').title()} at {LATITUDE:.2f}N, {LONGITUDE:.2f}E") diff --git a/python/omfiles/chunk_reader.py b/python/omfiles/chunk_reader.py new file mode 100644 index 00000000..e398c0d8 --- /dev/null +++ b/python/omfiles/chunk_reader.py @@ -0,0 +1,84 @@ +"""Utility class to iterate over chunks of data.""" + +from typing import Tuple + +try: + import fsspec +except ImportError: + raise ImportError("omfiles[fsspec] is required for using the chunk reader.") + +import numpy as np +import numpy.typing as npt + +from omfiles import OmFileReader +from omfiles.om_meta import OmMetaChunks + + +class OmFileChunkReader: + """Utility class to iterate over chunks of data.""" + + def __init__( + self, + om_meta: OmMetaChunks, + fs: fsspec.AbstractFileSystem, + s3_path_to_chunk_files: str, + start_date: np.datetime64, + end_date: np.datetime64, + ): + """ + Initialize the chunk reader. + + Args: + om_meta (OmMetaChunks): Metadata for the OM files. + fs (fsspec.AbstractFileSystem): Filesystem for accessing the OM files. + s3_path_to_chunk_files (str): Path to the chunk files. + start_date (np.datetime64): Start date of the data to load. + end_date (np.datetime64): End date of the data to load. + """ + self.om_meta = om_meta + self.fs = fs + self.s3_path_to_chunk_files = s3_path_to_chunk_files + self.start_date = start_date + self.end_date = end_date + self.chunk_indices = self.om_meta.chunks_for_date_range(start_date, end_date) + + def iter_files(self): + """ + Iterate over the chunk files. + + Yields: + Tuple[int, str]: Chunk index and path to the chunk file. + """ + for chunk_index in self.chunk_indices: + yield chunk_index, f"{self.s3_path_to_chunk_files}/chunk_{chunk_index}.om" + + def load_chunked_data( + self, spatial_index: Tuple[int, int] + ) -> Tuple[npt.NDArray[np.datetime64], npt.NDArray[np.float64]]: + """ + Load data from all chunks. + + Args: + spatial_index (Tuple[int, int]): Spatial index of the data to load. + + Returns: + Tuple[np.ndarray, np.ndarray]: Time array and data array. + """ + all_times = [] + all_data = [] + for chunk_index, s3_path in self.iter_files(): + chunk_times = self.om_meta.get_chunk_time_range(chunk_index) + time_mask = (chunk_times >= self.start_date) & (chunk_times <= self.end_date) + with OmFileReader.from_fsspec(self.fs, s3_path) as reader: + indices = np.where(time_mask)[0] + time_slice = slice(indices[0], indices[-1] + 1) # +1 to include the end + x, y = spatial_index + data = reader[y, x, time_slice] + times = chunk_times[time_mask] + if len(times) > 0: + all_times.append(times) + all_data.append(data) + + time_array = np.concatenate(all_times) + data_array = np.concatenate(all_data) + return time_array, data_array diff --git a/python/omfiles/om_grid.py b/python/omfiles/om_grid.py index 930fcffc..a431f7ab 100644 --- a/python/omfiles/om_grid.py +++ b/python/omfiles/om_grid.py @@ -4,7 +4,11 @@ import numpy as np import numpy.typing as npt -from pyproj import CRS + +try: + from pyproj import CRS +except ImportError: + raise ImportError("omfiles[grids] is required for OmGrid functionality") from omfiles.grids.gaussian import GaussianGrid from omfiles.grids.regular import RegularGrid diff --git a/python/omfiles/om_meta.py b/python/omfiles/om_meta.py index cf580789..e37faed2 100644 --- a/python/omfiles/om_meta.py +++ b/python/omfiles/om_meta.py @@ -2,7 +2,12 @@ import json from dataclasses import dataclass, fields -from typing import List +from typing import TYPE_CHECKING, List + +if TYPE_CHECKING: + from omfiles.om_grid import OmGrid + +from omfiles import OmFileReader try: from typing import Self # Python 3.11+ @@ -41,6 +46,20 @@ def from_s3_json_path(cls, s3_json_path: str, fs: fsspec.AbstractFileSystem) -> meta_dict = json.loads(fs.cat_file(s3_json_path)) return cls.from_dict(meta_dict) + def get_grid(self, reader: OmFileReader) -> "OmGrid": + """Create grid from metadata.""" + try: + from omfiles.om_grid import OmGrid + except ImportError: + raise ImportError("omfiles[grids] is required for grid operations") + """Create grid from metadata.""" + if len(reader.shape) == 2: + return OmGrid(self.crs_wkt, reader.shape) + elif len(reader.shape) == 3: + return OmGrid(self.crs_wkt, reader.shape[:2]) + else: + raise ValueError("Reader shape must be 2D or 3D") + @dataclass class OmMetaSpatial(OmMetaBase): diff --git a/python/omfiles/xarray.py b/python/omfiles/xarray.py index c60257d3..9bee3eef 100644 --- a/python/omfiles/xarray.py +++ b/python/omfiles/xarray.py @@ -4,6 +4,12 @@ from __future__ import annotations import numpy as np + +try: + from xarray.core import indexing +except ImportError: + raise ImportError("omfiles[xarray] is required for Xarray functionality") + from xarray.backends.common import ( AbstractDataStore, BackendArray, @@ -11,7 +17,6 @@ _normalize_path, ) from xarray.backends.store import StoreBackendEntrypoint -from xarray.core import indexing from xarray.core.dataset import Dataset from xarray.core.utils import FrozenDict from xarray.core.variable import Variable From e2f293977c355be11231bb5336e6e2715500f8a0 Mon Sep 17 00:00:00 2001 From: terraputix Date: Wed, 14 Jan 2026 16:01:17 +0100 Subject: [PATCH 42/50] consistent min version --- pyproject.toml | 2 +- uv.lock | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 872261c6..04bae999 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -36,7 +36,7 @@ codec = [ "numcodecs>=0.12.1", ] xarray = ["xarray>=2023.1.0"] -fsspec = ["fsspec>=2023.1.0", "s3fs>=2023.10.0"] +fsspec = ["fsspec>=2023.1.0", "s3fs>=2023.1.0"] grids = ["pyproj>=3.1.0"] all = [ "zarr>=2.18.2", diff --git a/uv.lock b/uv.lock index 6a6a820b..ef5f7fe1 100644 --- a/uv.lock +++ b/uv.lock @@ -1345,7 +1345,7 @@ fsspec = [ { name = "fsspec" }, { name = "s3fs" }, ] -proj = [ +grids = [ { name = "pyproj", version = "3.6.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" }, { name = "pyproj", version = "3.7.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.10.*'" }, { name = "pyproj", version = "3.7.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, @@ -1393,15 +1393,15 @@ requires-dist = [ { name = "numcodecs", marker = "extra == 'codec'", specifier = ">=0.12.1" }, { name = "numpy", specifier = ">=1.21.0" }, { name = "pyproj", marker = "extra == 'all'", specifier = ">=3.1.0" }, - { name = "pyproj", marker = "extra == 'proj'", specifier = ">=3.1.0" }, + { name = "pyproj", marker = "extra == 'grids'", specifier = ">=3.1.0" }, { name = "s3fs", marker = "extra == 'all'", specifier = ">=2023.1.0" }, - { name = "s3fs", marker = "extra == 'fsspec'", specifier = ">=2023.10.0" }, + { name = "s3fs", marker = "extra == 'fsspec'", specifier = ">=2023.1.0" }, { name = "xarray", marker = "extra == 'all'", specifier = ">=2023.1.0" }, { name = "xarray", marker = "extra == 'xarray'", specifier = ">=2023.1.0" }, { name = "zarr", marker = "extra == 'all'", specifier = ">=2.18.2" }, { name = "zarr", marker = "extra == 'codec'", specifier = ">=2.18.2" }, ] -provides-extras = ["codec", "xarray", "fsspec", "proj", "all"] +provides-extras = ["all", "codec", "fsspec", "grids", "xarray"] [package.metadata.requires-dev] dev = [ From 2e17a7ea51e4cd1daa7787f0d6d3dd5877d08677 Mon Sep 17 00:00:00 2001 From: terraputix Date: Wed, 14 Jan 2026 16:19:58 +0100 Subject: [PATCH 43/50] do not require meta.json in plot_map.py --- examples/plot_map.py | 4 +--- examples/spatial_xarray.py | 1 - 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/examples/plot_map.py b/examples/plot_map.py index d0313932..79e4e05a 100755 --- a/examples/plot_map.py +++ b/examples/plot_map.py @@ -27,7 +27,6 @@ # Please update the URI to match a currently available file. s3_run = f"s3://openmeteo/data_spatial/{MODEL_DOMAIN}/2026/01/10/0000Z/" s3_uri = f"{s3_run}2026-01-12T0000.om" -s3_meta_json = f"{s3_run}meta.json" # The following two incantations are equivalent # @@ -70,8 +69,7 @@ # Create coordinate arrays num_y, num_x = child.shape - meta = OmMetaSpatial.from_s3_json_path(s3_meta_json, backend.fs) - grid = OmGrid(meta.crs_wkt, (num_y, num_x)) + grid = OmGrid(reader.get_child_by_name("crs_wkt").read_scalar(), (num_y, num_x)) lon_grid, lat_grid = grid.get_meshgrid() # Plot the data diff --git a/examples/spatial_xarray.py b/examples/spatial_xarray.py index eeb70936..7fbd8032 100644 --- a/examples/spatial_xarray.py +++ b/examples/spatial_xarray.py @@ -26,7 +26,6 @@ # Please update the URI to match a currently available file. s3_run = f"s3://openmeteo/data_spatial/{MODEL_DOMAIN}/2026/01/10/0000Z/" s3_uri = f"{s3_run}2026-01-12T0000.om" -s3_meta_json = f"{s3_run}meta.json" backend = fsspec.open( f"blockcache::{s3_uri}", From 15905c9fe6f11a6a9015cb10d4c70eef5bf0c93b Mon Sep 17 00:00:00 2001 From: terraputix Date: Wed, 14 Jan 2026 16:47:47 +0100 Subject: [PATCH 44/50] delete two files and put in other branch --- examples/select_by_coordinates.py | 110 ------------------------------ python/omfiles/chunk_reader.py | 84 ----------------------- 2 files changed, 194 deletions(-) delete mode 100644 examples/select_by_coordinates.py delete mode 100644 python/omfiles/chunk_reader.py diff --git a/examples/select_by_coordinates.py b/examples/select_by_coordinates.py deleted file mode 100644 index bbfa6704..00000000 --- a/examples/select_by_coordinates.py +++ /dev/null @@ -1,110 +0,0 @@ -#!/usr/bin/env -S uv run --script -# -# /// script -# requires-python = ">=3.12" -# dependencies = [ -# "omfiles[grids,fsspec] @ /home/fred/dev/terraputix/python-omfiles", -# "matplotlib", -# ] -# /// - -from datetime import datetime -from typing import Tuple - -import matplotlib.pyplot as plt -import numpy as np -import numpy.typing as npt -from fsspec.implementations.cached import CachingFileSystem -from omfiles import OmFileReader -from omfiles.chunk_reader import OmFileChunkReader -from omfiles.om_grid import OmGrid -from omfiles.om_meta import OmMetaChunks -from s3fs import S3FileSystem - -# We load data from this Cached Fs-Spec Filesystem -FS = CachingFileSystem( - fs=S3FileSystem(anon=True, default_block_size=256, default_cache_type="none"), - # TODO: we'd need to verify files do not change on the remote if they still could change - cache_check=60, - block_size=256, - cache_storage="cache", - check_files=False, -) -LATITUDE, LONGITUDE = 48.864716, 2.349014 # Paris -START_DATE = np.datetime64(datetime(2025, 4, 25, 12, 0)) # 25-04-2025'T'12:00 -END_DATE = np.datetime64(datetime(2025, 5, 18, 12, 0)) # 18-05-2025'T'12:00 -VARIABLE = "temperature_2m" -DOMAINS = [ - "dwd_icon", - "dwd_icon_eu", - "dwd_icon_d2", - "ecmwf_ifs025", - "ecmwf_ifs", - "meteofrance_arpege_europe", - "meteofrance_arpege_world025", - "meteofrance_arome_france0025", - "meteofrance_arome_france_hd", - "meteofrance_arome_france_hd_15min", - "cmc_gem_gdps", - "cmc_gem_rdps", - "cmc_gem_hrdps", -] - -print(f"Fetching {VARIABLE} data for coordinates: {LATITUDE}N, {LONGITUDE}E") -print(f"Date range: {START_DATE} to {END_DATE}") - - -# Collect data from each domain -domain_data: dict[str, Tuple[npt.NDArray[np.datetime64], npt.NDArray[np.float64]]] = {} - -for domain_name in DOMAINS: - try: - print(f"\nTrying to fetch data from domain: {domain_name}") - meta = OmMetaChunks.from_s3_json_path(f"openmeteo/data/{domain_name}/static/meta.json", FS) - chunk_reader = OmFileChunkReader( - meta, FS, f"s3://openmeteo/data/{domain_name}/{VARIABLE}", START_DATE, END_DATE - ) - first = next(chunk_reader.iter_files(), None) - if first is None: - print(f"No data found for domain {domain_name}") - continue - _, s3_path = first - grid: OmGrid | None = None - with OmFileReader.from_fsspec(FS, s3_path) as reader: - grid = meta.get_grid(reader) - - assert grid is not None, "Grid not found" - indices = grid.find_point_xy(LATITUDE, LONGITUDE) - assert indices is not None, "Indices not found" - times, data = chunk_reader.load_chunked_data(indices) - domain_data[domain_name] = times, data - print(f"Successfully fetched data from {domain_name}") - except Exception as e: - print(f"Could not fetch data from {domain_name}: {e}") - -print(f"\nSuccessfully fetched data from {len(domain_data)} domains: {domain_data.keys()}") - -if len(domain_data) == 0: - print("No data could be fetched from any domain. Exiting.") - exit(1) - -# Domain colors for consistent line colors -colors = plt.get_cmap("tab10")(np.linspace(0, 1, len(domain_data))) - -plt.figure(figsize=(12, 6)) - -# Plot data from each domain -for i, (domain_name, (times, data)) in enumerate(domain_data.items()): - plt.plot(times, data, label=domain_name, color=colors[i], linewidth=2) - -# Enhance the plot -plt.title(f"{VARIABLE.replace('_', ' ').title()} at {LATITUDE:.2f}N, {LONGITUDE:.2f}E") -plt.xlabel("Time") -plt.ylabel("Temperature (°C)" if VARIABLE == "temperature_2m" else VARIABLE) -plt.grid(True, alpha=0.3) -plt.legend(loc="best") -plt.tight_layout() - -# Save and show the figure -plt.savefig(f"{VARIABLE}_comparison.png", dpi=150) -plt.show() diff --git a/python/omfiles/chunk_reader.py b/python/omfiles/chunk_reader.py deleted file mode 100644 index e398c0d8..00000000 --- a/python/omfiles/chunk_reader.py +++ /dev/null @@ -1,84 +0,0 @@ -"""Utility class to iterate over chunks of data.""" - -from typing import Tuple - -try: - import fsspec -except ImportError: - raise ImportError("omfiles[fsspec] is required for using the chunk reader.") - -import numpy as np -import numpy.typing as npt - -from omfiles import OmFileReader -from omfiles.om_meta import OmMetaChunks - - -class OmFileChunkReader: - """Utility class to iterate over chunks of data.""" - - def __init__( - self, - om_meta: OmMetaChunks, - fs: fsspec.AbstractFileSystem, - s3_path_to_chunk_files: str, - start_date: np.datetime64, - end_date: np.datetime64, - ): - """ - Initialize the chunk reader. - - Args: - om_meta (OmMetaChunks): Metadata for the OM files. - fs (fsspec.AbstractFileSystem): Filesystem for accessing the OM files. - s3_path_to_chunk_files (str): Path to the chunk files. - start_date (np.datetime64): Start date of the data to load. - end_date (np.datetime64): End date of the data to load. - """ - self.om_meta = om_meta - self.fs = fs - self.s3_path_to_chunk_files = s3_path_to_chunk_files - self.start_date = start_date - self.end_date = end_date - self.chunk_indices = self.om_meta.chunks_for_date_range(start_date, end_date) - - def iter_files(self): - """ - Iterate over the chunk files. - - Yields: - Tuple[int, str]: Chunk index and path to the chunk file. - """ - for chunk_index in self.chunk_indices: - yield chunk_index, f"{self.s3_path_to_chunk_files}/chunk_{chunk_index}.om" - - def load_chunked_data( - self, spatial_index: Tuple[int, int] - ) -> Tuple[npt.NDArray[np.datetime64], npt.NDArray[np.float64]]: - """ - Load data from all chunks. - - Args: - spatial_index (Tuple[int, int]): Spatial index of the data to load. - - Returns: - Tuple[np.ndarray, np.ndarray]: Time array and data array. - """ - all_times = [] - all_data = [] - for chunk_index, s3_path in self.iter_files(): - chunk_times = self.om_meta.get_chunk_time_range(chunk_index) - time_mask = (chunk_times >= self.start_date) & (chunk_times <= self.end_date) - with OmFileReader.from_fsspec(self.fs, s3_path) as reader: - indices = np.where(time_mask)[0] - time_slice = slice(indices[0], indices[-1] + 1) # +1 to include the end - x, y = spatial_index - data = reader[y, x, time_slice] - times = chunk_times[time_mask] - if len(times) > 0: - all_times.append(times) - all_data.append(data) - - time_array = np.concatenate(all_times) - data_array = np.concatenate(all_data) - return time_array, data_array From 64ba18f24ca3843052e0d43eb3f03eb0672a2b7e Mon Sep 17 00:00:00 2001 From: terraputix Date: Wed, 14 Jan 2026 16:52:23 +0100 Subject: [PATCH 45/50] undo unneccessary changes --- examples/plot_map.py | 6 ++---- examples/spatial_xarray.py | 3 +-- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/examples/plot_map.py b/examples/plot_map.py index 79e4e05a..1f386679 100755 --- a/examples/plot_map.py +++ b/examples/plot_map.py @@ -18,15 +18,13 @@ import numpy as np from omfiles import OmFileReader from omfiles.om_grid import OmGrid -from omfiles.om_meta import OmMetaSpatial MODEL_DOMAIN = "dmi_harmonie_arome_europe" # Example: URI for a spatial data file in the `data_spatial` S3 bucket # See data organization details: https://github.com/open-meteo/open-data?tab=readme-ov-file#data-organization # Note: Spatial data is only retained for 7 days. The example file below may no longer exist. # Please update the URI to match a currently available file. -s3_run = f"s3://openmeteo/data_spatial/{MODEL_DOMAIN}/2026/01/10/0000Z/" -s3_uri = f"{s3_run}2026-01-12T0000.om" +s3_uri = f"s3://openmeteo/data_spatial/{MODEL_DOMAIN}/2026/01/10/0000Z/2026-01-12T0000.om" # The following two incantations are equivalent # @@ -47,7 +45,7 @@ with OmFileReader(backend) as reader: print("reader.is_group", reader.is_group) - child = reader.get_child_by_name("temperature_2m") + child = reader.get_child_by_name("relative_humidity_2m") print("child.name", child.name) # Get the full data array diff --git a/examples/spatial_xarray.py b/examples/spatial_xarray.py index 7fbd8032..8969e2a9 100644 --- a/examples/spatial_xarray.py +++ b/examples/spatial_xarray.py @@ -24,8 +24,7 @@ # See data organization details: https://github.com/open-meteo/open-data?tab=readme-ov-file#data-organization # Note: Spatial data is only retained for 7 days. The example file below may no longer exist. # Please update the URI to match a currently available file. -s3_run = f"s3://openmeteo/data_spatial/{MODEL_DOMAIN}/2026/01/10/0000Z/" -s3_uri = f"{s3_run}2026-01-12T0000.om" +s3_uri = f"s3://openmeteo/data_spatial/{MODEL_DOMAIN}/2026/01/10/0000Z/2026-01-12T0000.om" backend = fsspec.open( f"blockcache::{s3_uri}", From 5e03c298f04ad55571fd2bf50dae0f592bf99223 Mon Sep 17 00:00:00 2001 From: terraputix Date: Wed, 14 Jan 2026 17:18:46 +0100 Subject: [PATCH 46/50] consistency in examples --- examples/plot_map.py | 36 ++++------ examples/readme_example.py | 10 +-- ...f_ifs_hres_gaussian_O1280_to_0.1_degree.py | 16 +++-- examples/spatial_xarray.py | 32 ++++----- examples/wkt2_crs.py | 70 ------------------- 5 files changed, 44 insertions(+), 120 deletions(-) delete mode 100644 examples/wkt2_crs.py diff --git a/examples/plot_map.py b/examples/plot_map.py index 1f386679..13f7bc1f 100755 --- a/examples/plot_map.py +++ b/examples/plot_map.py @@ -3,9 +3,7 @@ # /// script # requires-python = ">=3.12" # dependencies = [ -# "omfiles[grids] @ /home/fred/dev/terraputix/python-omfiles", -# "fsspec>=2025.7.0", -# "s3fs", +# "omfiles[fsspec,grids]>=1.1.0", # "matplotlib", # "cartopy", # ] @@ -20,24 +18,15 @@ from omfiles.om_grid import OmGrid MODEL_DOMAIN = "dmi_harmonie_arome_europe" +VARIABLE = "relative_humidity_2m" # Example: URI for a spatial data file in the `data_spatial` S3 bucket # See data organization details: https://github.com/open-meteo/open-data?tab=readme-ov-file#data-organization # Note: Spatial data is only retained for 7 days. The example file below may no longer exist. # Please update the URI to match a currently available file. -s3_uri = f"s3://openmeteo/data_spatial/{MODEL_DOMAIN}/2026/01/10/0000Z/2026-01-12T0000.om" - -# The following two incantations are equivalent -# -# from fsspec.implementations.cached import CachingFileSystem -# from s3fs import S3FileSystem -# s3_fs = S3FileSystem(anon=True, default_block_size=65536, default_cache_type="none") -# backend = CachingFileSystem( -# fs=s3_fs, cache_check=3600, block_size=65536, cache_storage="cache", check_files=False, same_names=True -# ) -# with OmFileReader.from_fsspec(backend, s3_uri) as reader: +S3_URI = f"s3://openmeteo/data_spatial/{MODEL_DOMAIN}/2026/01/10/0000Z/2026-01-12T0000.om" backend = fsspec.open( - f"blockcache::{s3_uri}", + f"blockcache::{S3_URI}", mode="rb", s3={"anon": True, "default_block_size": 65536}, blockcache={"cache_storage": "cache"}, @@ -45,7 +34,7 @@ with OmFileReader(backend) as reader: print("reader.is_group", reader.is_group) - child = reader.get_child_by_name("relative_humidity_2m") + child = reader.get_child_by_name(VARIABLE) print("child.name", child.name) # Get the full data array @@ -60,8 +49,8 @@ ax = plt.axes(projection=ccrs.PlateCarree()) # use PlateCarree projection # Add map features - ax.add_feature(cfeature.COASTLINE) - ax.add_feature(cfeature.BORDERS) + ax.add_feature(cfeature.COASTLINE, linewidth=0.8) + ax.add_feature(cfeature.BORDERS, linewidth=0.5) ax.add_feature(cfeature.OCEAN, alpha=0.3) ax.add_feature(cfeature.LAND, alpha=0.3) @@ -71,14 +60,17 @@ lon_grid, lat_grid = grid.get_meshgrid() # Plot the data - im = ax.contourf(lon_grid, lat_grid, data, levels=20, transform=ccrs.PlateCarree(), cmap="viridis") - plt.colorbar(im, ax=ax, shrink=0.6, label=child.name) + im = ax.contourf(lon_grid, lat_grid, data, cmap="coolwarm", shading="auto") + plt.colorbar(im, ax=ax, shrink=0.6, label=VARIABLE) + ax.set_xlabel("Longitude") + ax.set_ylabel("Latitude") ax.gridlines(draw_labels=True, alpha=0.3) - plt.title(f"2D Map: {child.name}") + plt.title(f"{MODEL_DOMAIN} {VARIABLE} Map\nCRS: {grid.crs.name}") + ax.grid(True, alpha=0.3) # ax.set_global() plt.tight_layout() - output_filename = f"map_{child.name.replace('/', '_')}.png" + output_filename = f"map_{VARIABLE}.png" plt.savefig(output_filename, dpi=300, bbox_inches="tight") print(f"Plot saved as: {output_filename}") plt.close() diff --git a/examples/readme_example.py b/examples/readme_example.py index ecc6c8b5..570d3cb1 100644 --- a/examples/readme_example.py +++ b/examples/readme_example.py @@ -3,7 +3,7 @@ # /// script # requires-python = ">=3.12" # dependencies = [ -# "omfiles[fsspec]==1.0.1", +# "omfiles[fsspec]>=1.1.0", # ] # /// @@ -11,15 +11,17 @@ import numpy as np from omfiles import OmFileReader +MODEL_DOMAIN = "dwd_icon" +VARIABLE = "temperature_2m" # Example: URI for a spatial data file in the `data_spatial` S3 bucket # See data organization details: https://github.com/open-meteo/open-data?tab=readme-ov-file#data-organization # Note: Spatial data is only retained for 7 days. The example file below may no longer exist. # Please update the URI to match a currently available file. -s3_uri = "s3://openmeteo/data_spatial/dwd_icon/2026/01/10/0000Z/2026-01-12T0000.om" +S3_URI = f"s3://openmeteo/data_spatial/{MODEL_DOMAIN}/2026/01/10/0000Z/2026-01-12T0000.om" # Create and open filesystem, wrapping it in a blockcache backend = fsspec.open( - f"blockcache::{s3_uri}", + f"blockcache::{S3_URI}", mode="rb", s3={"anon": True, "default_block_size": 65536}, # s3 settings blockcache={"cache_storage": "cache"}, # blockcache settings @@ -33,7 +35,7 @@ print(f"root.is_scalar: {root.is_scalar}") # False print(f"root.is_group: {root.is_group}") # True - temperature_reader = root.get_child_by_name("temperature_2m") + temperature_reader = root.get_child_by_name(VARIABLE) print(f"temperature_reader.is_array: {temperature_reader.is_array}") # True print(f"temperature_reader.is_scalar: {temperature_reader.is_scalar}") # False print(f"temperature_reader.is_group: {temperature_reader.is_group}") # False diff --git a/examples/regrid_ecmwf_ifs_hres_gaussian_O1280_to_0.1_degree.py b/examples/regrid_ecmwf_ifs_hres_gaussian_O1280_to_0.1_degree.py index 137510a3..e2fae287 100644 --- a/examples/regrid_ecmwf_ifs_hres_gaussian_O1280_to_0.1_degree.py +++ b/examples/regrid_ecmwf_ifs_hres_gaussian_O1280_to_0.1_degree.py @@ -3,7 +3,7 @@ # /// script # requires-python = ">=3.12" # dependencies = [ -# "omfiles[fsspec]>=1.0.1", +# "omfiles[fsspec]>=1.1.0", # "matplotlib", # "cartopy", # "earthkit-regrid==0.5.0", @@ -18,14 +18,16 @@ from earthkit.regrid import interpolate from omfiles import OmFileReader +MODEL_DOMAIN = "ecmwf_ifs" +VARIABLE = "temperature_2m" # Example: URI for a spatial data file in the `data_spatial` S3 bucket # See data organization details: https://github.com/open-meteo/open-data?tab=readme-ov-file#data-organization # Note: Spatial data is only retained for 7 days. The example file below may no longer exist. # Please update the URI to match a currently available file. -s3_ifs_spatial_uri = f"s3://openmeteo/data_spatial/ecmwf_ifs/2025/10/01/0000Z/2025-10-01T0000.om" +S3_URI = f"s3://openmeteo/data_spatial/{MODEL_DOMAIN}/2025/10/01/0000Z/2025-10-01T0000.om" backend = fsspec.open( - f"blockcache::{s3_ifs_spatial_uri}", + f"blockcache::{S3_URI}", mode="rb", s3={"anon": True, "default_block_size": 65536}, blockcache={"cache_storage": "cache"}, @@ -33,7 +35,7 @@ with OmFileReader(backend) as reader: print("reader.is_group", reader.is_group) - child = reader.get_child_by_name("temperature_2m") + child = reader.get_child_by_name(VARIABLE) print("child.name", child.name) # Get the full data array @@ -53,8 +55,8 @@ ax = plt.axes(projection=ccrs.PlateCarree()) # use PlateCarree projection # Add map features - ax.add_feature(cfeature.COASTLINE) - ax.add_feature(cfeature.BORDERS) + ax.add_feature(cfeature.COASTLINE, linewidth=0.8) + ax.add_feature(cfeature.BORDERS, linewidth=0.5) ax.add_feature(cfeature.OCEAN, alpha=0.3) ax.add_feature(cfeature.LAND, alpha=0.3) @@ -73,7 +75,7 @@ ax.set_global() plt.tight_layout() - output_filename = f"map_ifs_{child.name.replace('/', '_')}.png" + output_filename = f"map_ifs_{VARIABLE}.png" plt.savefig(output_filename, dpi=300, bbox_inches="tight") print(f"Plot saved as: {output_filename}") plt.close() diff --git a/examples/spatial_xarray.py b/examples/spatial_xarray.py index 8969e2a9..2e9cf159 100644 --- a/examples/spatial_xarray.py +++ b/examples/spatial_xarray.py @@ -3,7 +3,7 @@ # /// script # requires-python = ">=3.12" # dependencies = [ -# "omfiles[grids,fsspec,xarray]@ /home/fred/dev/terraputix/python-omfiles", +# "omfiles[fsspec,grids,xarray]>=1.1.0", # "matplotlib", # "cartopy", # ] @@ -17,17 +17,17 @@ import xarray as xr from omfiles.om_grid import OmGrid -PLOT_VARIABLE = "temperature_2m" MODEL_DOMAIN = "dwd_icon" +VARIABLE = "temperature_2m" # Example: URI for a spatial data file in the `data_spatial` S3 bucket # See data organization details: https://github.com/open-meteo/open-data?tab=readme-ov-file#data-organization # Note: Spatial data is only retained for 7 days. The example file below may no longer exist. # Please update the URI to match a currently available file. -s3_uri = f"s3://openmeteo/data_spatial/{MODEL_DOMAIN}/2026/01/10/0000Z/2026-01-12T0000.om" +S3_URI = f"s3://openmeteo/data_spatial/{MODEL_DOMAIN}/2026/01/10/0000Z/2026-01-12T0000.om" backend = fsspec.open( - f"blockcache::{s3_uri}", + f"blockcache::{S3_URI}", mode="rb", s3={"anon": True, "default_block_size": 65536}, blockcache={"cache_storage": "cache"}, @@ -38,37 +38,35 @@ print(ds.variables.keys()) # any of these keys can be used for plotting fig = plt.figure(figsize=(12, 8)) -ax = ax = plt.axes(projection=ccrs.PlateCarree()) +ax = plt.axes(projection=ccrs.PlateCarree()) ax.add_feature(cfeature.COASTLINE, linewidth=0.8) ax.add_feature(cfeature.BORDERS, linewidth=0.5) ax.add_feature(cfeature.OCEAN, alpha=0.3) ax.add_feature(cfeature.LAND, alpha=0.3) -ax.add_feature(cfeature.LAKES, alpha=0.3) -ax.add_feature(cfeature.RIVERS, alpha=0.3) -plot_data = ds[PLOT_VARIABLE] # shape: (lat, lon) +data = ds[VARIABLE] # shape: (lat, lon) # Use OmGrid with the crs_wkt attribute to get the lat/lon grid -grid = OmGrid(ds.attrs["crs_wkt"], ds[PLOT_VARIABLE].shape) +grid = OmGrid(ds.attrs["crs_wkt"], ds[VARIABLE].shape) lon2d, lat2d = grid.get_meshgrid() -min = int(plot_data.min().values) -max = int(plot_data.max().values) +min = int(data.min().values) +max = int(data.max().values) stepsize = int((max - min) / 30) -c = ax.contourf( +im = ax.contourf( lon2d, lat2d, - plot_data, + data, levels=np.arange(min, max, stepsize), - cmap="Spectral_r", # or "RdYlBu_r" + cmap="Spectral_r", vmin=min, vmax=max, transform=ccrs.PlateCarree(), extend="both", ) -cb = plt.colorbar(c, ax=ax, orientation="vertical", pad=0.02, aspect=40, shrink=0.8) -cb.set_label(PLOT_VARIABLE, fontsize=14) -plt.title(f"{MODEL_DOMAIN} {PLOT_VARIABLE}", fontsize=14, fontweight="bold", pad=20) +cb = plt.colorbar(im, ax=ax, orientation="vertical", pad=0.02, aspect=40, shrink=0.8) +cb.set_label(VARIABLE, fontsize=14) +plt.title(f"{MODEL_DOMAIN} {VARIABLE}", fontsize=14, fontweight="bold", pad=20) plt.tight_layout() plt.savefig("xarray_map.png", dpi=300, bbox_inches="tight") diff --git a/examples/wkt2_crs.py b/examples/wkt2_crs.py deleted file mode 100644 index ef5f18e4..00000000 --- a/examples/wkt2_crs.py +++ /dev/null @@ -1,70 +0,0 @@ -#!/usr/bin/env -S uv run --script -# -# /// script -# requires-python = ">=3.12" -# dependencies = [ -# "omfiles[fsspec,grids] @ /home/fred/dev/terraputix/python-omfiles", -# "matplotlib", -# ] -# /// - -import fsspec -import matplotlib.pyplot as plt -import numpy as np -from omfiles import OmFileReader -from omfiles.om_grid import OmGrid -from pyproj import CRS - -# Example: URI for a spatial data file in the `data_spatial` S3 bucket -# See data organization details: https://github.com/open-meteo/open-data?tab=readme-ov-file#data-organization -# Note: Spatial data is only retained for 7 days. The example file below may no longer exist. -# Please update the URI to match a currently available file. -# Other models to test: ncep_gfs013 cmc_gem_hrdps cmc_gem_rdps ukmo_uk_deterministic_2km dmi_harmonie_aroma_europe meteofrance_arome_france0025 -s3_uri = "s3://openmeteo/data_spatial/meteoswiss_icon_ch1/2026/01/06/0000Z/2026-01-06T0000.om" - -# Note: This code does not support ECMWF IFS HRES grids (Reduced Gaussian O1280)! - -backend = fsspec.open( - f"blockcache::{s3_uri}", - mode="rb", - s3={"anon": True, "default_block_size": 65536}, - blockcache={"cache_storage": "cache"}, -) -with OmFileReader(backend) as reader: - # Get the full data array and read into regular "data" array - child = reader.get_child_by_name("temperature_2m") - print("child.shape", child.shape) - print("child.chunks", child.chunks) - data = child[:] - - # Setup projection - crs_wkt = reader.get_child_by_name("crs_wkt").read_scalar() - grid = OmGrid(crs_wkt, shape=data.shape) - # Get coordinate meshgrid for plotting - lon_grid, lat_grid = grid.get_meshgrid() - - # Create figure and axis - fig, ax = plt.subplots(figsize=(10, 5)) - - crs = CRS.from_wkt(crs_wkt) - - # Simple plot without cartopy - c = ax.pcolormesh(lon_grid, lat_grid, data, cmap="coolwarm", shading="auto") - - # Add colorbar - plt.colorbar(c, ax=ax, orientation="vertical", label="Temperature (°C)") - - # Add labels - ax.set_xlabel("Longitude") - ax.set_ylabel("Latitude") - ax.set_title(f"Temperature Data\nCRS: {crs.name}") - - # Set aspect ratio - ax.set_aspect("equal") - - # Add grid - ax.grid(True, alpha=0.3) - - plt.tight_layout() - plt.savefig("temperature_map_pyproj.png", dpi=150) - print("Plot saved to temperature_map_pyproj.png") From 77ad111597754d717dc0ff576fc426ca0f4459cf Mon Sep 17 00:00:00 2001 From: terraputix Date: Wed, 14 Jan 2026 17:43:17 +0100 Subject: [PATCH 47/50] more consistency in examples --- examples/plot_map.py | 19 +++++++++---------- ...f_ifs_hres_gaussian_O1280_to_0.1_degree.py | 12 ++++++------ examples/spatial_xarray.py | 7 +++---- 3 files changed, 18 insertions(+), 20 deletions(-) diff --git a/examples/plot_map.py b/examples/plot_map.py index 13f7bc1f..ad845e5d 100755 --- a/examples/plot_map.py +++ b/examples/plot_map.py @@ -36,17 +36,16 @@ child = reader.get_child_by_name(VARIABLE) print("child.name", child.name) - - # Get the full data array print("child.shape", child.shape) print("child.chunks", child.chunks) + # Get the full data array data = child[:] print(f"Data shape: {data.shape}") print(f"Data range: {np.nanmin(data)} to {np.nanmax(data)}") # Create the plot fig = plt.figure(figsize=(12, 8)) - ax = plt.axes(projection=ccrs.PlateCarree()) # use PlateCarree projection + ax = plt.axes(projection=ccrs.PlateCarree()) # Add map features ax.add_feature(cfeature.COASTLINE, linewidth=0.8) @@ -55,19 +54,19 @@ ax.add_feature(cfeature.LAND, alpha=0.3) # Create coordinate arrays - num_y, num_x = child.shape + num_y, num_x = data.shape grid = OmGrid(reader.get_child_by_name("crs_wkt").read_scalar(), (num_y, num_x)) lon_grid, lat_grid = grid.get_meshgrid() + crs = grid.crs + assert crs is not None, "CRS is None, this should only happen for gaussian grids" # Plot the data - im = ax.contourf(lon_grid, lat_grid, data, cmap="coolwarm", shading="auto") - plt.colorbar(im, ax=ax, shrink=0.6, label=VARIABLE) - ax.set_xlabel("Longitude") - ax.set_ylabel("Latitude") + im = ax.contourf(lon_grid, lat_grid, data, cmap="coolwarm") ax.gridlines(draw_labels=True, alpha=0.3) - plt.title(f"{MODEL_DOMAIN} {VARIABLE} Map\nCRS: {grid.crs.name}") - ax.grid(True, alpha=0.3) + ax.set_aspect("equal") # ax.set_global() + plt.colorbar(im, ax=ax, orientation="vertical", pad=0.05, aspect=40, shrink=0.55, label=VARIABLE) + plt.title(f"{MODEL_DOMAIN} {VARIABLE} Map\nCRS: {crs.name}", fontsize=12, fontweight="bold", pad=16) plt.tight_layout() output_filename = f"map_{VARIABLE}.png" diff --git a/examples/regrid_ecmwf_ifs_hres_gaussian_O1280_to_0.1_degree.py b/examples/regrid_ecmwf_ifs_hres_gaussian_O1280_to_0.1_degree.py index e2fae287..5d864075 100644 --- a/examples/regrid_ecmwf_ifs_hres_gaussian_O1280_to_0.1_degree.py +++ b/examples/regrid_ecmwf_ifs_hres_gaussian_O1280_to_0.1_degree.py @@ -24,7 +24,7 @@ # See data organization details: https://github.com/open-meteo/open-data?tab=readme-ov-file#data-organization # Note: Spatial data is only retained for 7 days. The example file below may no longer exist. # Please update the URI to match a currently available file. -S3_URI = f"s3://openmeteo/data_spatial/{MODEL_DOMAIN}/2025/10/01/0000Z/2025-10-01T0000.om" +S3_URI = f"s3://openmeteo/data_spatial/{MODEL_DOMAIN}/2026/01/10/0000Z/2026-01-12T0000.om" backend = fsspec.open( f"blockcache::{S3_URI}", @@ -52,7 +52,7 @@ # Create plot fig = plt.figure(figsize=(12, 8)) - ax = plt.axes(projection=ccrs.PlateCarree()) # use PlateCarree projection + ax = plt.axes(projection=ccrs.PlateCarree()) # Add map features ax.add_feature(cfeature.COASTLINE, linewidth=0.8) @@ -68,11 +68,11 @@ lon_grid, lat_grid = np.meshgrid(lon, lat) # Plot the data - im = ax.contourf(lon_grid, lat_grid, regridded, levels=20, transform=ccrs.PlateCarree(), cmap="viridis") - plt.colorbar(im, ax=ax, shrink=0.6, label=child.name) - ax.gridlines(draw_labels=True, alpha=0.3) - plt.title(f"2D Map: {child.name}") + im = ax.contourf(lon_grid, lat_grid, regridded, levels=20, cmap="coolwarm") ax.set_global() + ax.gridlines(draw_labels=True, alpha=0.3) + plt.colorbar(im, ax=ax, orientation="vertical", pad=0.05, aspect=40, shrink=0.55, label=VARIABLE) + plt.title(f"{MODEL_DOMAIN} {VARIABLE} Regridded to 0.1° Map", fontsize=12, fontweight="bold", pad=16) plt.tight_layout() output_filename = f"map_ifs_{VARIABLE}.png" diff --git a/examples/spatial_xarray.py b/examples/spatial_xarray.py index 2e9cf159..23088018 100644 --- a/examples/spatial_xarray.py +++ b/examples/spatial_xarray.py @@ -62,11 +62,10 @@ cmap="Spectral_r", vmin=min, vmax=max, - transform=ccrs.PlateCarree(), extend="both", ) -cb = plt.colorbar(im, ax=ax, orientation="vertical", pad=0.02, aspect=40, shrink=0.8) -cb.set_label(VARIABLE, fontsize=14) -plt.title(f"{MODEL_DOMAIN} {VARIABLE}", fontsize=14, fontweight="bold", pad=20) +ax.gridlines(draw_labels=True, alpha=0.3) +plt.colorbar(im, ax=ax, orientation="vertical", pad=0.05, aspect=40, shrink=0.55, label=VARIABLE) +plt.title(f"{MODEL_DOMAIN} {VARIABLE}", fontsize=12, fontweight="bold", pad=16) plt.tight_layout() plt.savefig("xarray_map.png", dpi=300, bbox_inches="tight") From 9d1216ca1ddf452c1e11458b7204baba0cbcfb7c Mon Sep 17 00:00:00 2001 From: terraputix Date: Wed, 14 Jan 2026 17:50:32 +0100 Subject: [PATCH 48/50] rename and restructure --- examples/plot_map.py | 2 +- examples/select_by_coordinates.py | 110 +++++++++++++++++++++++++ examples/spatial_xarray.py | 2 +- python/omfiles/__init__.py | 8 +- python/omfiles/chunk_reader.py | 84 +++++++++++++++++++ python/omfiles/grids/__init__.py | 11 +++ python/omfiles/{ => grids}/om_grid.py | 0 python/omfiles/{om_meta.py => meta.py} | 8 +- tests/test_grids.py | 3 +- tests/test_meta.py | 12 +-- 10 files changed, 219 insertions(+), 21 deletions(-) create mode 100644 examples/select_by_coordinates.py create mode 100644 python/omfiles/chunk_reader.py create mode 100644 python/omfiles/grids/__init__.py rename python/omfiles/{ => grids}/om_grid.py (100%) rename python/omfiles/{om_meta.py => meta.py} (97%) diff --git a/examples/plot_map.py b/examples/plot_map.py index ad845e5d..8287b285 100755 --- a/examples/plot_map.py +++ b/examples/plot_map.py @@ -15,7 +15,7 @@ import matplotlib.pyplot as plt import numpy as np from omfiles import OmFileReader -from omfiles.om_grid import OmGrid +from omfiles.grids import OmGrid MODEL_DOMAIN = "dmi_harmonie_arome_europe" VARIABLE = "relative_humidity_2m" diff --git a/examples/select_by_coordinates.py b/examples/select_by_coordinates.py new file mode 100644 index 00000000..8d79a24c --- /dev/null +++ b/examples/select_by_coordinates.py @@ -0,0 +1,110 @@ +#!/usr/bin/env -S uv run --script +# +# /// script +# requires-python = ">=3.12" +# dependencies = [ +# "omfiles[grids,fsspec] @ /home/fred/dev/terraputix/python-omfiles", +# "matplotlib", +# ] +# /// + +from datetime import datetime +from typing import Tuple + +import matplotlib.pyplot as plt +import numpy as np +import numpy.typing as npt +from fsspec.implementations.cached import CachingFileSystem +from omfiles import OmFileReader +from omfiles.chunk_reader import OmFileChunkReader +from omfiles.grids import OmGrid +from omfiles.meta import OmChunksMeta +from s3fs import S3FileSystem + +# We load data from this Cached Fs-Spec Filesystem +FS = CachingFileSystem( + fs=S3FileSystem(anon=True, default_block_size=256, default_cache_type="none"), + # TODO: we'd need to verify files do not change on the remote if they still could change + cache_check=60, + block_size=256, + cache_storage="cache", + check_files=False, +) +LATITUDE, LONGITUDE = 48.864716, 2.349014 # Paris +START_DATE = np.datetime64(datetime(2025, 4, 25, 12, 0)) # 25-04-2025'T'12:00 +END_DATE = np.datetime64(datetime(2025, 5, 18, 12, 0)) # 18-05-2025'T'12:00 +VARIABLE = "temperature_2m" +DOMAINS = [ + "dwd_icon", + "dwd_icon_eu", + "dwd_icon_d2", + "ecmwf_ifs025", + "ecmwf_ifs", + "meteofrance_arpege_europe", + "meteofrance_arpege_world025", + "meteofrance_arome_france0025", + "meteofrance_arome_france_hd", + "meteofrance_arome_france_hd_15min", + "cmc_gem_gdps", + "cmc_gem_rdps", + "cmc_gem_hrdps", +] + +print(f"Fetching {VARIABLE} data for coordinates: {LATITUDE}N, {LONGITUDE}E") +print(f"Date range: {START_DATE} to {END_DATE}") + + +# Collect data from each domain +domain_data: dict[str, Tuple[npt.NDArray[np.datetime64], npt.NDArray[np.float64]]] = {} + +for domain_name in DOMAINS: + try: + print(f"\nTrying to fetch data from domain: {domain_name}") + meta = OmChunksMeta.from_s3_json_path(f"openmeteo/data/{domain_name}/static/meta.json", FS) + chunk_reader = OmFileChunkReader( + meta, FS, f"s3://openmeteo/data/{domain_name}/{VARIABLE}", START_DATE, END_DATE + ) + first = next(chunk_reader.iter_files(), None) + if first is None: + print(f"No data found for domain {domain_name}") + continue + _, s3_path = first + grid: OmGrid | None = None + with OmFileReader.from_fsspec(FS, s3_path) as reader: + grid = meta.get_grid(reader) + + assert grid is not None, "Grid not found" + indices = grid.find_point_xy(LATITUDE, LONGITUDE) + assert indices is not None, "Indices not found" + times, data = chunk_reader.load_chunked_data(indices) + domain_data[domain_name] = times, data + print(f"Successfully fetched data from {domain_name}") + except Exception as e: + print(f"Could not fetch data from {domain_name}: {e}") + +print(f"\nSuccessfully fetched data from {len(domain_data)} domains: {domain_data.keys()}") + +if len(domain_data) == 0: + print("No data could be fetched from any domain. Exiting.") + exit(1) + +# Domain colors for consistent line colors +colors = plt.get_cmap("tab10")(np.linspace(0, 1, len(domain_data))) + +plt.figure(figsize=(12, 6)) + +# Plot data from each domain +for i, (domain_name, (times, data)) in enumerate(domain_data.items()): + plt.plot(times, data, label=domain_name, color=colors[i], linewidth=2) + +# Enhance the plot +plt.title(f"{VARIABLE.replace('_', ' ').title()} at {LATITUDE:.2f}N, {LONGITUDE:.2f}E") +plt.xlabel("Time") +plt.ylabel("Temperature (°C)" if VARIABLE == "temperature_2m" else VARIABLE) +plt.grid(True, alpha=0.3) +plt.legend(loc="best") +plt.tight_layout() + +# Save and show the figure +plt.savefig(f"{VARIABLE}_comparison.png", dpi=150) +plt.show() diff --git a/examples/spatial_xarray.py b/examples/spatial_xarray.py index 23088018..f23abffd 100644 --- a/examples/spatial_xarray.py +++ b/examples/spatial_xarray.py @@ -15,7 +15,7 @@ import matplotlib.pyplot as plt import numpy as np import xarray as xr -from omfiles.om_grid import OmGrid +from omfiles.grids import OmGrid MODEL_DOMAIN = "dwd_icon" VARIABLE = "temperature_2m" diff --git a/python/omfiles/__init__.py b/python/omfiles/__init__.py index f2338e84..4897a6c3 100644 --- a/python/omfiles/__init__.py +++ b/python/omfiles/__init__.py @@ -5,10 +5,4 @@ _check_cpu_features() -__all__ = [ - "OmFileReader", - "OmFileReaderAsync", - "OmFileWriter", - "OmVariable", - "types", -] +__all__ = ["OmFileReader", "OmFileReaderAsync", "OmFileWriter", "OmVariable", "types"] diff --git a/python/omfiles/chunk_reader.py b/python/omfiles/chunk_reader.py new file mode 100644 index 00000000..ed9b356c --- /dev/null +++ b/python/omfiles/chunk_reader.py @@ -0,0 +1,84 @@ +"""Utility class to iterate over chunks of data.""" + +from typing import Tuple + +try: + import fsspec +except ImportError: + raise ImportError("omfiles[fsspec] is required for using the chunk reader.") + +import numpy as np +import numpy.typing as npt + +from omfiles import OmFileReader +from omfiles.meta import OmChunksMeta + + +class OmFileChunkReader: + """Utility class to iterate over chunks of data.""" + + def __init__( + self, + om_meta: OmChunksMeta, + fs: fsspec.AbstractFileSystem, + s3_path_to_chunk_files: str, + start_date: np.datetime64, + end_date: np.datetime64, + ): + """ + Initialize the chunk reader. + + Args: + om_meta (OmChunksMeta): Metadata for the OM files. + fs (fsspec.AbstractFileSystem): Filesystem for accessing the OM files. + s3_path_to_chunk_files (str): Path to the chunk files. + start_date (np.datetime64): Start date of the data to load. + end_date (np.datetime64): End date of the data to load. + """ + self.om_meta = om_meta + self.fs = fs + self.s3_path_to_chunk_files = s3_path_to_chunk_files + self.start_date = start_date + self.end_date = end_date + self.chunk_indices = self.om_meta.chunks_for_date_range(start_date, end_date) + + def iter_files(self): + """ + Iterate over the chunk files. + + Yields: + Tuple[int, str]: Chunk index and path to the chunk file. + """ + for chunk_index in self.chunk_indices: + yield chunk_index, f"{self.s3_path_to_chunk_files}/chunk_{chunk_index}.om" + + def load_chunked_data( + self, spatial_index: Tuple[int, int] + ) -> Tuple[npt.NDArray[np.datetime64], npt.NDArray[np.float64]]: + """ + Load data from all chunks. + + Args: + spatial_index (Tuple[int, int]): Spatial index of the data to load. + + Returns: + Tuple[np.ndarray, np.ndarray]: Time array and data array. + """ + all_times = [] + all_data = [] + for chunk_index, s3_path in self.iter_files(): + chunk_times = self.om_meta.get_chunk_time_range(chunk_index) + time_mask = (chunk_times >= self.start_date) & (chunk_times <= self.end_date) + with OmFileReader.from_fsspec(self.fs, s3_path) as reader: + indices = np.where(time_mask)[0] + time_slice = slice(indices[0], indices[-1] + 1) # +1 to include the end + x, y = spatial_index + data = reader[y, x, time_slice] + times = chunk_times[time_mask] + if len(times) > 0: + all_times.append(times) + all_data.append(data) + + time_array = np.concatenate(all_times) + data_array = np.concatenate(all_data) + return time_array, data_array diff --git a/python/omfiles/grids/__init__.py b/python/omfiles/grids/__init__.py new file mode 100644 index 00000000..bf245d34 --- /dev/null +++ b/python/omfiles/grids/__init__.py @@ -0,0 +1,11 @@ +"""Provides classes for representing grids in OM files.""" + +from .gaussian import GaussianGrid +from .om_grid import OmGrid +from .regular import RegularGrid + +__all__ = [ + "GaussianGrid", + "OmGrid", + "RegularGrid", +] diff --git a/python/omfiles/om_grid.py b/python/omfiles/grids/om_grid.py similarity index 100% rename from python/omfiles/om_grid.py rename to python/omfiles/grids/om_grid.py diff --git a/python/omfiles/om_meta.py b/python/omfiles/meta.py similarity index 97% rename from python/omfiles/om_meta.py rename to python/omfiles/meta.py index e37faed2..db709663 100644 --- a/python/omfiles/om_meta.py +++ b/python/omfiles/meta.py @@ -5,7 +5,7 @@ from typing import TYPE_CHECKING, List if TYPE_CHECKING: - from omfiles.om_grid import OmGrid + from omfiles.grids import OmGrid from omfiles import OmFileReader @@ -49,7 +49,7 @@ def from_s3_json_path(cls, s3_json_path: str, fs: fsspec.AbstractFileSystem) -> def get_grid(self, reader: OmFileReader) -> "OmGrid": """Create grid from metadata.""" try: - from omfiles.om_grid import OmGrid + from omfiles.grids import OmGrid except ImportError: raise ImportError("omfiles[grids] is required for grid operations") """Create grid from metadata.""" @@ -62,7 +62,7 @@ def get_grid(self, reader: OmFileReader) -> "OmGrid": @dataclass -class OmMetaSpatial(OmMetaBase): +class OmSpatialMeta(OmMetaBase): """Representation of the meta.json for spatial datasets.""" last_modified_time: str # ISO8601 for last modification @@ -72,7 +72,7 @@ class OmMetaSpatial(OmMetaBase): @dataclass -class OmMetaChunks(OmMetaBase): +class OmChunksMeta(OmMetaBase): """Representation of the meta.json for time oriented chunks.""" chunk_time_length: int # Number of time steps per chunk (file_length) diff --git a/tests/test_grids.py b/tests/test_grids.py index 271fad2d..784be889 100644 --- a/tests/test_grids.py +++ b/tests/test_grids.py @@ -1,7 +1,6 @@ import pyproj import pytest -from omfiles.grids.gaussian import GaussianGrid -from omfiles.om_grid import OmGrid +from omfiles.grids import GaussianGrid, OmGrid # Fixtures for grids diff --git a/tests/test_meta.py b/tests/test_meta.py index 80db5a09..9e93ac3f 100644 --- a/tests/test_meta.py +++ b/tests/test_meta.py @@ -1,6 +1,6 @@ import numpy as np import pytest -from omfiles.om_meta import OmMetaChunks +from omfiles.meta import OmChunksMeta @pytest.fixture @@ -10,17 +10,17 @@ def icon_d2_meta_json() -> str: @pytest.fixture -def icon_d2_meta(icon_d2_meta_json: str) -> OmMetaChunks: - return OmMetaChunks.from_metajson_string(icon_d2_meta_json) +def icon_d2_meta(icon_d2_meta_json: str) -> OmChunksMeta: + return OmChunksMeta.from_metajson_string(icon_d2_meta_json) def test_meta_json_creation(icon_d2_meta_json: str): """Test creation of OmMetaJson object from JSON string.""" - meta = OmMetaChunks.from_metajson_string(icon_d2_meta_json) + meta = OmChunksMeta.from_metajson_string(icon_d2_meta_json) assert meta.chunk_time_length == 121 -def test_time_to_chunk_index(icon_d2_meta: OmMetaChunks): +def test_time_to_chunk_index(icon_d2_meta: OmChunksMeta): """Test conversion from timestamp to chunk index.""" # Create test timestamp (2023-01-01 12:00:00 UTC) @@ -40,7 +40,7 @@ def test_time_to_chunk_index(icon_d2_meta: OmMetaChunks): assert chunk_index == expected_chunk -def test_get_chunk_time_range(icon_d2_meta: OmMetaChunks): +def test_get_chunk_time_range(icon_d2_meta: OmChunksMeta): """Test getting time range for a specific chunk.""" # Test chunk 1000 From a97ea849ef81ee446826781b5897aae0e1f2b451 Mon Sep 17 00:00:00 2001 From: terraputix Date: Wed, 14 Jan 2026 17:53:42 +0100 Subject: [PATCH 49/50] remove file from this branch --- examples/select_by_coordinates.py | 110 ------------------------------ python/omfiles/chunk_reader.py | 84 ----------------------- 2 files changed, 194 deletions(-) delete mode 100644 examples/select_by_coordinates.py delete mode 100644 python/omfiles/chunk_reader.py diff --git a/examples/select_by_coordinates.py b/examples/select_by_coordinates.py deleted file mode 100644 index 8d79a24c..00000000 --- a/examples/select_by_coordinates.py +++ /dev/null @@ -1,110 +0,0 @@ -#!/usr/bin/env -S uv run --script -# -# /// script -# requires-python = ">=3.12" -# dependencies = [ -# "omfiles[grids,fsspec] @ /home/fred/dev/terraputix/python-omfiles", -# "matplotlib", -# ] -# /// - -from datetime import datetime -from typing import Tuple - -import matplotlib.pyplot as plt -import numpy as np -import numpy.typing as npt -from fsspec.implementations.cached import CachingFileSystem -from omfiles import OmFileReader -from omfiles.chunk_reader import OmFileChunkReader -from omfiles.grids import OmGrid -from omfiles.meta import OmChunksMeta -from s3fs import S3FileSystem - -# We load data from this Cached Fs-Spec Filesystem -FS = CachingFileSystem( - fs=S3FileSystem(anon=True, default_block_size=256, default_cache_type="none"), - # TODO: we'd need to verify files do not change on the remote if they still could change - cache_check=60, - block_size=256, - cache_storage="cache", - check_files=False, -) -LATITUDE, LONGITUDE = 48.864716, 2.349014 # Paris -START_DATE = np.datetime64(datetime(2025, 4, 25, 12, 0)) # 25-04-2025'T'12:00 -END_DATE = np.datetime64(datetime(2025, 5, 18, 12, 0)) # 18-05-2025'T'12:00 -VARIABLE = "temperature_2m" -DOMAINS = [ - "dwd_icon", - "dwd_icon_eu", - "dwd_icon_d2", - "ecmwf_ifs025", - "ecmwf_ifs", - "meteofrance_arpege_europe", - "meteofrance_arpege_world025", - "meteofrance_arome_france0025", - "meteofrance_arome_france_hd", - "meteofrance_arome_france_hd_15min", - "cmc_gem_gdps", - "cmc_gem_rdps", - "cmc_gem_hrdps", -] - -print(f"Fetching {VARIABLE} data for coordinates: {LATITUDE}N, {LONGITUDE}E") -print(f"Date range: {START_DATE} to {END_DATE}") - - -# Collect data from each domain -domain_data: dict[str, Tuple[npt.NDArray[np.datetime64], npt.NDArray[np.float64]]] = {} - -for domain_name in DOMAINS: - try: - print(f"\nTrying to fetch data from domain: {domain_name}") - meta = OmChunksMeta.from_s3_json_path(f"openmeteo/data/{domain_name}/static/meta.json", FS) - chunk_reader = OmFileChunkReader( - meta, FS, f"s3://openmeteo/data/{domain_name}/{VARIABLE}", START_DATE, END_DATE - ) - first = next(chunk_reader.iter_files(), None) - if first is None: - print(f"No data found for domain {domain_name}") - continue - _, s3_path = first - grid: OmGrid | None = None - with OmFileReader.from_fsspec(FS, s3_path) as reader: - grid = meta.get_grid(reader) - - assert grid is not None, "Grid not found" - indices = grid.find_point_xy(LATITUDE, LONGITUDE) - assert indices is not None, "Indices not found" - times, data = chunk_reader.load_chunked_data(indices) - domain_data[domain_name] = times, data - print(f"Successfully fetched data from {domain_name}") - except Exception as e: - print(f"Could not fetch data from {domain_name}: {e}") - -print(f"\nSuccessfully fetched data from {len(domain_data)} domains: {domain_data.keys()}") - -if len(domain_data) == 0: - print("No data could be fetched from any domain. Exiting.") - exit(1) - -# Domain colors for consistent line colors -colors = plt.get_cmap("tab10")(np.linspace(0, 1, len(domain_data))) - -plt.figure(figsize=(12, 6)) - -# Plot data from each domain -for i, (domain_name, (times, data)) in enumerate(domain_data.items()): - plt.plot(times, data, label=domain_name, color=colors[i], linewidth=2) - -# Enhance the plot -plt.title(f"{VARIABLE.replace('_', ' ').title()} at {LATITUDE:.2f}N, {LONGITUDE:.2f}E") -plt.xlabel("Time") -plt.ylabel("Temperature (°C)" if VARIABLE == "temperature_2m" else VARIABLE) -plt.grid(True, alpha=0.3) -plt.legend(loc="best") -plt.tight_layout() - -# Save and show the figure -plt.savefig(f"{VARIABLE}_comparison.png", dpi=150) -plt.show() diff --git a/python/omfiles/chunk_reader.py b/python/omfiles/chunk_reader.py deleted file mode 100644 index ed9b356c..00000000 --- a/python/omfiles/chunk_reader.py +++ /dev/null @@ -1,84 +0,0 @@ -"""Utility class to iterate over chunks of data.""" - -from typing import Tuple - -try: - import fsspec -except ImportError: - raise ImportError("omfiles[fsspec] is required for using the chunk reader.") - -import numpy as np -import numpy.typing as npt - -from omfiles import OmFileReader -from omfiles.meta import OmChunksMeta - - -class OmFileChunkReader: - """Utility class to iterate over chunks of data.""" - - def __init__( - self, - om_meta: OmChunksMeta, - fs: fsspec.AbstractFileSystem, - s3_path_to_chunk_files: str, - start_date: np.datetime64, - end_date: np.datetime64, - ): - """ - Initialize the chunk reader. - - Args: - om_meta (OmChunksMeta): Metadata for the OM files. - fs (fsspec.AbstractFileSystem): Filesystem for accessing the OM files. - s3_path_to_chunk_files (str): Path to the chunk files. - start_date (np.datetime64): Start date of the data to load. - end_date (np.datetime64): End date of the data to load. - """ - self.om_meta = om_meta - self.fs = fs - self.s3_path_to_chunk_files = s3_path_to_chunk_files - self.start_date = start_date - self.end_date = end_date - self.chunk_indices = self.om_meta.chunks_for_date_range(start_date, end_date) - - def iter_files(self): - """ - Iterate over the chunk files. - - Yields: - Tuple[int, str]: Chunk index and path to the chunk file. - """ - for chunk_index in self.chunk_indices: - yield chunk_index, f"{self.s3_path_to_chunk_files}/chunk_{chunk_index}.om" - - def load_chunked_data( - self, spatial_index: Tuple[int, int] - ) -> Tuple[npt.NDArray[np.datetime64], npt.NDArray[np.float64]]: - """ - Load data from all chunks. - - Args: - spatial_index (Tuple[int, int]): Spatial index of the data to load. - - Returns: - Tuple[np.ndarray, np.ndarray]: Time array and data array. - """ - all_times = [] - all_data = [] - for chunk_index, s3_path in self.iter_files(): - chunk_times = self.om_meta.get_chunk_time_range(chunk_index) - time_mask = (chunk_times >= self.start_date) & (chunk_times <= self.end_date) - with OmFileReader.from_fsspec(self.fs, s3_path) as reader: - indices = np.where(time_mask)[0] - time_slice = slice(indices[0], indices[-1] + 1) # +1 to include the end - x, y = spatial_index - data = reader[y, x, time_slice] - times = chunk_times[time_mask] - if len(times) > 0: - all_times.append(times) - all_data.append(data) - - time_array = np.concatenate(all_times) - data_array = np.concatenate(all_data) - return time_array, data_array From 69a5ca5b33b29d15f6c2e759ea8923f766496470 Mon Sep 17 00:00:00 2001 From: terraputix Date: Wed, 14 Jan 2026 17:55:28 +0100 Subject: [PATCH 50/50] revert formating change --- python/omfiles/__init__.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/python/omfiles/__init__.py b/python/omfiles/__init__.py index 4897a6c3..f2338e84 100644 --- a/python/omfiles/__init__.py +++ b/python/omfiles/__init__.py @@ -5,4 +5,10 @@ _check_cpu_features() -__all__ = ["OmFileReader", "OmFileReaderAsync", "OmFileWriter", "OmVariable", "types"] +__all__ = [ + "OmFileReader", + "OmFileReaderAsync", + "OmFileWriter", + "OmVariable", + "types", +]