From 4c2e6d3153f5caf3f15c8cd15c666fa93520ddbe Mon Sep 17 00:00:00 2001 From: Valeriu Predoi Date: Wed, 7 Jan 2026 13:34:59 +0000 Subject: [PATCH 1/6] exhaustive handling of new numpy 2.4.0 behaviour --- activestorage/active.py | 3 ++- activestorage/storage.py | 10 ++++++++-- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/activestorage/active.py b/activestorage/active.py index fb963f32..dc34cacb 100644 --- a/activestorage/active.py +++ b/activestorage/active.py @@ -116,7 +116,8 @@ def hfix(x): missing_value = ds.attrs.get('missing_value') # see https://github.com/NCAS-CMS/PyActiveStorage/pull/303 if isinstance(missing_value, np.ndarray): - missing_value = missing_value[0] + if missing_value.size == 1: + missing_value = missing_value[0] valid_min = hfix(ds.attrs.get('valid_min')) valid_max = hfix(ds.attrs.get('valid_max')) valid_range = hfix(ds.attrs.get('valid_range')) diff --git a/activestorage/storage.py b/activestorage/storage.py index eee0cabe..2deafab0 100644 --- a/activestorage/storage.py +++ b/activestorage/storage.py @@ -129,10 +129,16 @@ def mask_missing(data, missing): fill_value, missing_value, valid_min, valid_max = missing if fill_value is not None: - data = np.ma.masked_equal(data, fill_value) + if isinstance(fill_value, np.ndarray) or isinstance(fill_value, list): + data = np.ma.masked_where(data == fill_value, data) + else: + data = np.ma.masked_equal(data, fill_value) if missing_value is not None: - data = np.ma.masked_equal(data, missing_value) + if isinstance(missing_value, np.ndarray) or isinstance(missing_value, list): + data = np.ma.masked_where(data == missing_value, data) + else: + data = np.ma.masked_equal(data, missing_value) if valid_max is not None: data = np.ma.masked_greater(data, valid_max) From 571699a7f2fb7d754ffc5d044e75f7ee5943c269 Mon Sep 17 00:00:00 2001 From: Valeriu Predoi Date: Wed, 7 Jan 2026 13:35:10 +0000 Subject: [PATCH 2/6] add test --- tests/unit/test_storage.py | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/tests/unit/test_storage.py b/tests/unit/test_storage.py index 56d3e627..437b3e8f 100644 --- a/tests/unit/test_storage.py +++ b/tests/unit/test_storage.py @@ -6,6 +6,28 @@ import activestorage.storage as st +def test_mask_missing(): + """Test mask missing.""" + missing_1 = ([-900.], np.array([-900.]), None, None) + missing_2 = ([-900., 33.], np.array([-900., 33.]), None, None) + data_1 = np.ma.array( + [[[-900., 33.], [33., -900], [33., 44.]]], + mask=False, + fill_value=-900.0, + dtype=float + ) + data_2 = np.ma.array( + [[[-900., 33.], [33., -900], [33., 44.]]], + mask=False, + fill_value=[-900.0, 33.], + dtype=float + ) + res_1 = st.mask_missing(data_1, missing_1) + print(res_1) + res_2 = st.mask_missing(data_2, missing_2) + print(res_2) + + def test_reduce_chunk(): """Test reduce chunk entirely.""" rfile = "tests/test_data/cesm2_native.nc" From 9877dfc3fc41dc6091729efb229b78e43f39cde9 Mon Sep 17 00:00:00 2001 From: Valeriu Predoi Date: Tue, 20 Jan 2026 12:53:32 +0000 Subject: [PATCH 3/6] account for unbroadcastable arrays --- activestorage/storage.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/activestorage/storage.py b/activestorage/storage.py index 2deafab0..4229b740 100644 --- a/activestorage/storage.py +++ b/activestorage/storage.py @@ -130,7 +130,11 @@ def mask_missing(data, missing): if fill_value is not None: if isinstance(fill_value, np.ndarray) or isinstance(fill_value, list): - data = np.ma.masked_where(data == fill_value, data) + try: + data = np.ma.masked_where(data == fill_value, data) + except ValueError: # not broadcastable + fill_value = fill_value.reshape(data.shape) + data = np.ma.masked_where(data == fill_value, data) else: data = np.ma.masked_equal(data, fill_value) From cd09108f2e7eff326e8f0d04e56a32748d36c85e Mon Sep 17 00:00:00 2001 From: Valeriu Predoi Date: Tue, 20 Jan 2026 12:53:48 +0000 Subject: [PATCH 4/6] add test and fix previous test --- tests/unit/test_storage.py | 29 +++++++++++++++++++++++++++-- 1 file changed, 27 insertions(+), 2 deletions(-) diff --git a/tests/unit/test_storage.py b/tests/unit/test_storage.py index 437b3e8f..868151b8 100644 --- a/tests/unit/test_storage.py +++ b/tests/unit/test_storage.py @@ -23,9 +23,34 @@ def test_mask_missing(): dtype=float ) res_1 = st.mask_missing(data_1, missing_1) - print(res_1) + expected_1 = np.ma.array( + data_1, + mask=[[[True, False], [False, True], [False, False]]] + ) + np.testing.assert_array_equal(res_1, expected_1) res_2 = st.mask_missing(data_2, missing_2) - print(res_2) + expected_2 = np.ma.array( + data_2, + mask=[[[True, True], [False, False], [False, False]]] + ) + np.testing.assert_array_equal(res_2, expected_2) + + +def test_mask_missing_not_broadcastable(): + """Test mask missing when fill_value cant be broadcast to data.""" + data = np.ma.array( + [[[-900., 33.], [33., -900], [33., 44.]]], + mask=False, + fill_value=np.array([[-900.0, 33.], [-900.0, 33.]]), + dtype=float + ) + missing = ([-900., 33.], np.array([-900., 33.]), None, None) + res = st.mask_missing(data, missing) + expected = np.ma.array( + data, + mask=[[[True, True], [False, False], [False, False]]] + ) + np.testing.assert_array_equal(res, expected) def test_reduce_chunk(): From 1f162835ffa928928143b6c6718e8412d5b932f6 Mon Sep 17 00:00:00 2001 From: Valeriu Predoi Date: Tue, 20 Jan 2026 13:33:13 +0000 Subject: [PATCH 5/6] chuck a value error instead --- activestorage/storage.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/activestorage/storage.py b/activestorage/storage.py index 4229b740..498b50da 100644 --- a/activestorage/storage.py +++ b/activestorage/storage.py @@ -130,17 +130,16 @@ def mask_missing(data, missing): if fill_value is not None: if isinstance(fill_value, np.ndarray) or isinstance(fill_value, list): - try: - data = np.ma.masked_where(data == fill_value, data) - except ValueError: # not broadcastable - fill_value = fill_value.reshape(data.shape) - data = np.ma.masked_where(data == fill_value, data) + data = np.ma.masked_where(data == fill_value, data) else: data = np.ma.masked_equal(data, fill_value) if missing_value is not None: if isinstance(missing_value, np.ndarray) or isinstance(missing_value, list): - data = np.ma.masked_where(data == missing_value, data) + try: + data = np.ma.masked_where(data == missing_value, data) + except ValueError: # not broadcastable + raise ValueError("Data and missing_value arrays are not brodcastable!") else: data = np.ma.masked_equal(data, missing_value) From f3cdeb2b5ecfd70cccbbb4270ddab2ddf9594e73 Mon Sep 17 00:00:00 2001 From: Valeriu Predoi Date: Tue, 20 Jan 2026 13:33:22 +0000 Subject: [PATCH 6/6] add tests --- tests/unit/test_storage.py | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/tests/unit/test_storage.py b/tests/unit/test_storage.py index 868151b8..96c4c0ee 100644 --- a/tests/unit/test_storage.py +++ b/tests/unit/test_storage.py @@ -36,15 +36,15 @@ def test_mask_missing(): np.testing.assert_array_equal(res_2, expected_2) -def test_mask_missing_not_broadcastable(): +def test_mask_missing_missing_broadcastable(): """Test mask missing when fill_value cant be broadcast to data.""" data = np.ma.array( [[[-900., 33.], [33., -900], [33., 44.]]], mask=False, - fill_value=np.array([[-900.0, 33.], [-900.0, 33.]]), + fill_value=np.array([-900.0]), dtype=float ) - missing = ([-900., 33.], np.array([-900., 33.]), None, None) + missing = (-900, np.array([-900., 33.]), None, None) res = st.mask_missing(data, missing) expected = np.ma.array( data, @@ -53,6 +53,20 @@ def test_mask_missing_not_broadcastable(): np.testing.assert_array_equal(res, expected) +def test_mask_missing_missing_not_broadcastable(): + """Test mask missing when fill_value cant be broadcast to data.""" + data = np.ma.array( + [[[-900., 33.], [33., -900], [33., 44.]]], + mask=False, + fill_value=np.array([-900.0]), + dtype=float + ) + missing = (-900, np.array([-900., -900., 33.]), None, None) + msg = "Data and missing_value arrays are not brodcastable!" + with pytest.raises(ValueError, match=msg): + st.mask_missing(data, missing) + + def test_reduce_chunk(): """Test reduce chunk entirely.""" rfile = "tests/test_data/cesm2_native.nc"