From 57550913c78de1659c5c686c8ac72d33dd541682 Mon Sep 17 00:00:00 2001 From: Laura Sandoval Date: Thu, 5 Mar 2026 14:54:47 -0700 Subject: [PATCH 01/20] first pass --- imap_processing/glows/l1b/glows_l1b.py | 47 ++++++++ imap_processing/glows/l1b/glows_l1b_data.py | 100 +++++++++++++++++- imap_processing/tests/glows/conftest.py | 6 +- imap_processing/tests/glows/test_glows_l1b.py | 23 ++-- 4 files changed, 167 insertions(+), 9 deletions(-) diff --git a/imap_processing/glows/l1b/glows_l1b.py b/imap_processing/glows/l1b/glows_l1b.py index eb87d3404..6812f0c8e 100644 --- a/imap_processing/glows/l1b/glows_l1b.py +++ b/imap_processing/glows/l1b/glows_l1b.py @@ -13,10 +13,54 @@ DirectEventL1B, HistogramL1B, PipelineSettings, + get_threshold, ) from imap_processing.spice.time import et_to_datetime64, ttj2000ns_to_et +def _update_daily_statistical_error_flag( + output_dataset: xr.Dataset, + pipeline_settings: PipelineSettings, +) -> xr.Dataset: + """ + Update flag index 11 (is_beyond_daily_statistical_error) for all histograms. + + Compares each histogram's total event count against the daily mean. Histograms + that deviate by more than n_sigma from the mean are flagged as bad (0). + + This must be called after all histograms for the day have been processed, since + the daily mean and standard deviation require the full day's data. + + Parameters + ---------- + output_dataset : xr.Dataset + The L1B output dataset from create_l1b_hist_output. + pipeline_settings : PipelineSettings + Pipeline settings containing n_sigma thresholds. + + Returns + ------- + xr.Dataset + The output dataset with flag index 11 updated. + """ + thresholds = pipeline_settings.processing_thresholds + n_sigma_lower = get_threshold(thresholds, "n_sigma_threshold_lower") + n_sigma_upper = get_threshold(thresholds, "n_sigma_threshold_upper") + + counts = output_dataset["number_of_events"].values.astype(float) + daily_mean = np.mean(counts) + daily_std = np.std(counts) + + is_good = (counts >= daily_mean - n_sigma_lower * daily_std) & ( + counts <= daily_mean + n_sigma_upper * daily_std + ) + + # Flag index 11 corresponds to is_beyond_daily_statistical_error. + output_dataset["flags"].values[:, 11] = is_good.astype(np.uint8) + + return output_dataset + + def glows_l1b( input_dataset: xr.Dataset, excluded_regions: xr.Dataset, @@ -81,6 +125,9 @@ def glows_l1b( output_dataset = create_l1b_hist_output( output_dataarrays, input_dataset["epoch"], input_dataset["bins"], cdf_attrs ) + output_dataset = _update_daily_statistical_error_flag( + output_dataset, pipeline_settings + ) return output_dataset diff --git a/imap_processing/glows/l1b/glows_l1b_data.py b/imap_processing/glows/l1b/glows_l1b_data.py index 5300fa43c..08c96fd97 100644 --- a/imap_processing/glows/l1b/glows_l1b_data.py +++ b/imap_processing/glows/l1b/glows_l1b_data.py @@ -653,6 +653,28 @@ def process_direct_events(direct_events: np.ndarray) -> tuple: return times, pulse_lengths +def get_threshold(thresholds: dict, suffix: str) -> float: + """ + Return the threshold value whose key ends with the given suffix. + + Parameters + ---------- + thresholds : dict + Dictionary of threshold values from PipelineSettings.processing_thresholds. + suffix : str + The suffix to match against threshold keys. + + Returns + ------- + float + The matching threshold value, or inf if no match is found. + """ + for k, v in thresholds.items(): + if k.endswith(suffix): + return float(v) + return np.inf + + @dataclass class HistogramL1B: """ @@ -870,7 +892,7 @@ def __post_init__( # is_inside_excluded_region, is_excluded_by_instr_team, # is_suspected_transient] x 3600 bins self.histogram_flag_array = self._compute_histogram_flag_array(day_exclusions) - self.flags = np.ones((FLAG_LENGTH,), dtype=np.uint8) + self.flags = self._compute_flags(pipeline_settings) def update_spice_parameters(self) -> None: """Update SPICE parameters based on the current state.""" @@ -979,6 +1001,82 @@ def deserialize_flags(raw: int) -> np.ndarray[int]: return flags + def _compute_flags(self, pipeline_settings: PipelineSettings) -> np.ndarray: + """ + Compute the 17 bad-time flags for this histogram. + + Flags 0-9 are decoded from the onboard 16-bit flag integer. Flags 10-16 + are computed during ground processing. The convention is 1 = condition + absent (good), 0 = condition present (bad/flag raised). + + Parameters + ---------- + pipeline_settings : PipelineSettings + Pipeline settings containing processing thresholds. + + Returns + ------- + flags : numpy.ndarray + Array of shape (FLAG_LENGTH,) with dtype uint8. 1 = good, 0 = bad. + """ + thresholds = pipeline_settings.processing_thresholds + + # Section 12.3.1 of the Algorithm Document: onboard generated bad-time flags. + # Flags are "stored in a 16-bit integer field. + onboard_flags = ( + 1 - self.deserialize_flags(int(self.flags_set_onboard)) + ).astype(np.uint8) + + # Section 12.3.2 of the Algorithm Document: ground processing flags: flag 1. + # Informs if the histogram was generated on-board or on the ground + # Flag 1 = onboard. + is_generated_on_ground = np.uint8(1 - int(self.is_generated_on_ground)) + + # Section 12.3.2 of the Algorithm Document: ground processing flags: flag 2. + # Comparison of the total numbers of counts in a given block-accumulated histogram with the + # daily average + # Placeholder. + is_beyond_daily_statistical_error = np.uint8(1) + + # Section 12.3.2 of the Algorithm Document: ground processing flags: flag 3-6. + # (1=good, 0=bad). + temp_threshold = get_threshold(thresholds, "std_dev_threshold__celsius_deg") + hv_threshold = get_threshold(thresholds, "std_dev_threshold__volt") + spin_std_threshold = get_threshold(thresholds, "std_dev_threshold__sec") + pulse_threshold = get_threshold(thresholds, "std_dev_threshold__usec") + spin_diff_threshold = get_threshold( + thresholds, "relative_difference_threshold" + ) + + is_temp_ok = np.uint8(self.filter_temperature_std_dev <= temp_threshold) + is_hv_ok = np.uint8(self.hv_voltage_std_dev <= hv_threshold) + is_spin_std_ok = np.uint8(self.spin_period_std_dev <= spin_std_threshold) + is_pulse_ok = np.uint8(self.pulse_length_std_dev <= pulse_threshold) + + spin_period_avg = float(self.spin_period_average) + if spin_period_avg != 0: + spin_diff = abs( + float(self.spin_period_ground_average) - spin_period_avg + ) / abs(spin_period_avg) + else: + spin_diff = np.inf + is_spin_diff_ok = np.uint8(spin_diff <= spin_diff_threshold) + + ground_flags = np.array( + [ + is_generated_on_ground, + is_beyond_daily_statistical_error, + is_temp_ok, + is_hv_ok, + is_spin_std_ok, + is_pulse_ok, + is_spin_diff_ok, + ], + dtype=np.uint8, + ) + + return np.concatenate([onboard_flags, ground_flags]) + def flag_uv_and_excluded(self, exclusions: AncillaryExclusions) -> tuple: """ Create boolean mask where True means bin is within radius of UV source. diff --git a/imap_processing/tests/glows/conftest.py b/imap_processing/tests/glows/conftest.py index c2d573f73..82881f8b2 100644 --- a/imap_processing/tests/glows/conftest.py +++ b/imap_processing/tests/glows/conftest.py @@ -253,8 +253,10 @@ def mock_pipeline_settings(): "active_bad_time_flags": ( ["epoch", "time_flag_index"], np.tile( - [True] * 17, (len(epoch_range), 1) - ), # 17 bad time flags from the JSON + [True, True, True, True, True, True, False, + True, True, True, True, True, True, True, True, True, False], + (len(epoch_range), 1), + ), ), "sunrise_offset": (["epoch"], [0.0] * len(epoch_range)), "sunset_offset": (["epoch"], [0.0] * len(epoch_range)), diff --git a/imap_processing/tests/glows/test_glows_l1b.py b/imap_processing/tests/glows/test_glows_l1b.py index d8a7a6a42..22add65fc 100644 --- a/imap_processing/tests/glows/test_glows_l1b.py +++ b/imap_processing/tests/glows/test_glows_l1b.py @@ -286,6 +286,8 @@ def test_process_histogram( test_hists = np.zeros(3600) # For temp encoded_val = np.single(expected_temp * 2.318 + 69.5454) + # Zero variance -> zero std dev -> all threshold flags pass (1 = good) + zero_variance = np.single(0) pipeline_settings = PipelineSettings( mock_pipeline_settings.sel( @@ -293,25 +295,27 @@ def test_process_histogram( ) ) + # flags_set_onboard = 64 = 0b01000000: bit 6 (is_night) set -> flag[6] = 0 (bad) + # is_generated_on_ground = 1 -> flag[10] = 0 (bad) test_l1b = HistogramL1B( test_hists, "test", 0, 0, 0, - 0, - 0, + 64, # flags_set_onboard: bit 6 (is_night) set + 1, # is_generated_on_ground 0, 3600, 0, encoded_val, + zero_variance, encoded_val, + zero_variance, encoded_val, + zero_variance, encoded_val, - encoded_val, - encoded_val, - encoded_val, - encoded_val, + zero_variance, time_val, time_val, time_val, @@ -321,6 +325,13 @@ def test_process_histogram( pipeline_settings, ) + # All onboard flags good (1) except flag[6] (is_night, bit 6 of 64). + # Flag[10] (is_generated_on_ground) = 0 (bad). All threshold flags = 1 (good). + expected_flags = np.ones(17, dtype=np.uint8) + expected_flags[6] = 0 # is_night: bit 6 of flags_set_onboard=64 is set + expected_flags[10] = 0 # is_generated_on_ground=1 + assert np.array_equal(test_l1b.flags, expected_flags) + output = process_histogram( hist_dataset, mock_ancillary_exclusions, From 6a54c910d51979e3cea1b9ed2295c2e67d00ab97 Mon Sep 17 00:00:00 2001 From: Laura Sandoval Date: Fri, 6 Mar 2026 12:23:18 -0700 Subject: [PATCH 02/20] improve get_descriptor --- imap_processing/glows/l1b/glows_l1b_data.py | 27 ++++++++++--------- .../tests/glows/test_glows_l1b_data.py | 27 +++++++++++++++++++ 2 files changed, 42 insertions(+), 12 deletions(-) diff --git a/imap_processing/glows/l1b/glows_l1b_data.py b/imap_processing/glows/l1b/glows_l1b_data.py index 08c96fd97..367089b26 100644 --- a/imap_processing/glows/l1b/glows_l1b_data.py +++ b/imap_processing/glows/l1b/glows_l1b_data.py @@ -653,26 +653,30 @@ def process_direct_events(direct_events: np.ndarray) -> tuple: return times, pulse_lengths -def get_threshold(thresholds: dict, suffix: str) -> float: +def get_threshold(thresholds: dict, suffix: str) -> float | None: """ Return the threshold value whose key ends with the given suffix. Parameters ---------- thresholds : dict - Dictionary of threshold values from PipelineSettings.processing_thresholds. + Dictionary of threshold values. suffix : str The suffix to match against threshold keys. Returns ------- - float - The matching threshold value, or inf if no match is found. + return_value : float or None + The matching threshold value, or None if no match is found. """ - for k, v in thresholds.items(): - if k.endswith(suffix): - return float(v) - return np.inf + return_value = None + for section in thresholds.values(): + for descriptor, value in section.items(): + if descriptor.endswith(suffix): + return_value = float(value) + break + + return return_value @dataclass @@ -1033,7 +1037,8 @@ def _compute_flags(self, pipeline_settings: PipelineSettings) -> np.ndarray: is_generated_on_ground = np.uint8(1 - int(self.is_generated_on_ground)) # Section 12.3.2 of the Algorithm Document: ground processing flags: flag 2. - # Comparison of the total numbers of counts in a given block-accumulated histogram with the + # Comparison of the total numbers of counts in a + # given block-accumulated histogram with the # daily average # Placeholder. is_beyond_daily_statistical_error = np.uint8(1) @@ -1044,9 +1049,7 @@ def _compute_flags(self, pipeline_settings: PipelineSettings) -> np.ndarray: hv_threshold = get_threshold(thresholds, "std_dev_threshold__volt") spin_std_threshold = get_threshold(thresholds, "std_dev_threshold__sec") pulse_threshold = get_threshold(thresholds, "std_dev_threshold__usec") - spin_diff_threshold = get_threshold( - thresholds, "relative_difference_threshold" - ) + spin_diff_threshold = get_threshold(thresholds, "relative_difference_threshold") is_temp_ok = np.uint8(self.filter_temperature_std_dev <= temp_threshold) is_hv_ok = np.uint8(self.hv_voltage_std_dev <= hv_threshold) diff --git a/imap_processing/tests/glows/test_glows_l1b_data.py b/imap_processing/tests/glows/test_glows_l1b_data.py index b75281915..ebd6bc1df 100644 --- a/imap_processing/tests/glows/test_glows_l1b_data.py +++ b/imap_processing/tests/glows/test_glows_l1b_data.py @@ -12,6 +12,7 @@ DirectEventL1B, HistogramL1B, PipelineSettings, + get_threshold, ) from imap_processing.spice.time import met_to_ttj2000ns from imap_processing.tests.glows.conftest import mock_update_spice_parameters @@ -287,3 +288,29 @@ def test_pipeline_settings_from_flattened_json(): assert len(settings.active_bad_angle_flags) == 4 assert settings.active_bad_angle_flags[3] is False # is_suspected_transient + + +def test_get_threshold(): + "Test get_threshold function." + test_data = { + "filter_based_on_comparison_of_spin_periods": { + "relative_difference_threshold": 7.0e-4 + }, + "filter_based_on_temperature_std_dev": {"std_dev_threshold__celsius_deg": 2.03}, + "filter_based_on_hv_voltage_std_dev": {"std_dev_threshold__volt": 50.0}, + "filter_based_on_spin_period_std_dev": {"std_dev_threshold__sec": 0.033333}, + "filter_based_on_pulse_length_std_dev": {"std_dev_threshold__usec": 1.0}, + } + + expected = [2.03, 50.0, 0.033333, 1.0, 7.0e-4] + description = [ + "std_dev_threshold__celsius_deg", + "std_dev_threshold__volt", + "std_dev_threshold__sec", + "std_dev_threshold__usec", + "relative_difference_threshold", + ] + + for name, exp in zip(description, expected, strict=False): + threshold = get_threshold(test_data, name) + assert threshold == exp From d57800e0e5dff2232cdb0526ab4f316453bf664a Mon Sep 17 00:00:00 2001 From: Laura Sandoval Date: Fri, 6 Mar 2026 13:06:49 -0700 Subject: [PATCH 03/20] update compute_flags --- imap_processing/glows/l1b/glows_l1b_data.py | 27 ++++----------------- 1 file changed, 5 insertions(+), 22 deletions(-) diff --git a/imap_processing/glows/l1b/glows_l1b_data.py b/imap_processing/glows/l1b/glows_l1b_data.py index 367089b26..a33bb63da 100644 --- a/imap_processing/glows/l1b/glows_l1b_data.py +++ b/imap_processing/glows/l1b/glows_l1b_data.py @@ -896,7 +896,7 @@ def __post_init__( # is_inside_excluded_region, is_excluded_by_instr_team, # is_suspected_transient] x 3600 bins self.histogram_flag_array = self._compute_histogram_flag_array(day_exclusions) - self.flags = self._compute_flags(pipeline_settings) + self.flags = self.compute_flags(pipeline_settings) def update_spice_parameters(self) -> None: """Update SPICE parameters based on the current state.""" @@ -1005,14 +1005,10 @@ def deserialize_flags(raw: int) -> np.ndarray[int]: return flags - def _compute_flags(self, pipeline_settings: PipelineSettings) -> np.ndarray: + def compute_flags(self, pipeline_settings: PipelineSettings) -> np.ndarray: """ Compute the 17 bad-time flags for this histogram. - Flags 0-9 are decoded from the onboard 16-bit flag integer. Flags 10-16 - are computed during ground processing. The convention is 1 = condition - absent (good), 0 = condition present (bad/flag raised). - Parameters ---------- pipeline_settings : PipelineSettings @@ -1032,15 +1028,13 @@ def _compute_flags(self, pipeline_settings: PipelineSettings) -> np.ndarray: ).astype(np.uint8) # Section 12.3.2 of the Algorithm Document: ground processing flags: flag 1. - # Informs if the histogram was generated on-board or on the ground + # Informs if the histogram was generated on-board or on the ground. # Flag 1 = onboard. is_generated_on_ground = np.uint8(1 - int(self.is_generated_on_ground)) # Section 12.3.2 of the Algorithm Document: ground processing flags: flag 2. - # Comparison of the total numbers of counts in a - # given block-accumulated histogram with the - # daily average - # Placeholder. + # Checks whether the total count in a given histogram is far from the daily average. + # Placeholder until daily histogram is available in glows_l1b.py. is_beyond_daily_statistical_error = np.uint8(1) # Section 12.3.2 of the Algorithm Document: ground processing flags: flag 3-6. @@ -1049,22 +1043,12 @@ def _compute_flags(self, pipeline_settings: PipelineSettings) -> np.ndarray: hv_threshold = get_threshold(thresholds, "std_dev_threshold__volt") spin_std_threshold = get_threshold(thresholds, "std_dev_threshold__sec") pulse_threshold = get_threshold(thresholds, "std_dev_threshold__usec") - spin_diff_threshold = get_threshold(thresholds, "relative_difference_threshold") is_temp_ok = np.uint8(self.filter_temperature_std_dev <= temp_threshold) is_hv_ok = np.uint8(self.hv_voltage_std_dev <= hv_threshold) is_spin_std_ok = np.uint8(self.spin_period_std_dev <= spin_std_threshold) is_pulse_ok = np.uint8(self.pulse_length_std_dev <= pulse_threshold) - spin_period_avg = float(self.spin_period_average) - if spin_period_avg != 0: - spin_diff = abs( - float(self.spin_period_ground_average) - spin_period_avg - ) / abs(spin_period_avg) - else: - spin_diff = np.inf - is_spin_diff_ok = np.uint8(spin_diff <= spin_diff_threshold) - ground_flags = np.array( [ is_generated_on_ground, @@ -1073,7 +1057,6 @@ def _compute_flags(self, pipeline_settings: PipelineSettings) -> np.ndarray: is_hv_ok, is_spin_std_ok, is_pulse_ok, - is_spin_diff_ok, ], dtype=np.uint8, ) From d82c1ec78acbb9f5bfb6a8cb276a5e40d63402a9 Mon Sep 17 00:00:00 2001 From: Laura Sandoval Date: Fri, 6 Mar 2026 13:54:16 -0700 Subject: [PATCH 04/20] rollback changes --- imap_processing/glows/l1b/glows_l1b.py | 53 +------------------------- 1 file changed, 2 insertions(+), 51 deletions(-) diff --git a/imap_processing/glows/l1b/glows_l1b.py b/imap_processing/glows/l1b/glows_l1b.py index 6812f0c8e..050eec6d4 100644 --- a/imap_processing/glows/l1b/glows_l1b.py +++ b/imap_processing/glows/l1b/glows_l1b.py @@ -13,54 +13,10 @@ DirectEventL1B, HistogramL1B, PipelineSettings, - get_threshold, ) from imap_processing.spice.time import et_to_datetime64, ttj2000ns_to_et -def _update_daily_statistical_error_flag( - output_dataset: xr.Dataset, - pipeline_settings: PipelineSettings, -) -> xr.Dataset: - """ - Update flag index 11 (is_beyond_daily_statistical_error) for all histograms. - - Compares each histogram's total event count against the daily mean. Histograms - that deviate by more than n_sigma from the mean are flagged as bad (0). - - This must be called after all histograms for the day have been processed, since - the daily mean and standard deviation require the full day's data. - - Parameters - ---------- - output_dataset : xr.Dataset - The L1B output dataset from create_l1b_hist_output. - pipeline_settings : PipelineSettings - Pipeline settings containing n_sigma thresholds. - - Returns - ------- - xr.Dataset - The output dataset with flag index 11 updated. - """ - thresholds = pipeline_settings.processing_thresholds - n_sigma_lower = get_threshold(thresholds, "n_sigma_threshold_lower") - n_sigma_upper = get_threshold(thresholds, "n_sigma_threshold_upper") - - counts = output_dataset["number_of_events"].values.astype(float) - daily_mean = np.mean(counts) - daily_std = np.std(counts) - - is_good = (counts >= daily_mean - n_sigma_lower * daily_std) & ( - counts <= daily_mean + n_sigma_upper * daily_std - ) - - # Flag index 11 corresponds to is_beyond_daily_statistical_error. - output_dataset["flags"].values[:, 11] = is_good.astype(np.uint8) - - return output_dataset - - def glows_l1b( input_dataset: xr.Dataset, excluded_regions: xr.Dataset, @@ -125,9 +81,6 @@ def glows_l1b( output_dataset = create_l1b_hist_output( output_dataarrays, input_dataset["epoch"], input_dataset["bins"], cdf_attrs ) - output_dataset = _update_daily_statistical_error_flag( - output_dataset, pipeline_settings - ) return output_dataset @@ -236,9 +189,7 @@ def create_direct_event_l1b(*args) -> tuple: # type: ignore[no-untyped-def] Tuple of values from DirectEventL1B dataclass. """ return tuple( - dataclasses.asdict( - DirectEventL1B(*args, ancillary_parameters) # type: ignore[call-arg] - ).values() + dataclasses.asdict(DirectEventL1B(*args, ancillary_parameters)).values() ) l1b_fields: tuple = xr.apply_ufunc( @@ -332,7 +283,7 @@ def create_histogram_l1b(*args) -> tuple: # type: ignore[no-untyped-def] tuple Tuple of processed L1B data arrays from HistogramL1B.output_data(). """ - return HistogramL1B( # type: ignore[call-arg] + return HistogramL1B( *args, ancillary_exclusions, ancillary_parameters, pipeline_settings ).output_data() From 057b831939c95cd69b0a35aac7eba308f2697d3e Mon Sep 17 00:00:00 2001 From: Laura Sandoval Date: Fri, 6 Mar 2026 14:07:27 -0700 Subject: [PATCH 05/20] revert some changes --- imap_processing/glows/l1b/glows_l1b_data.py | 1 + imap_processing/tests/glows/test_glows_l1b.py | 111 ++++++++++++++---- 2 files changed, 88 insertions(+), 24 deletions(-) diff --git a/imap_processing/glows/l1b/glows_l1b_data.py b/imap_processing/glows/l1b/glows_l1b_data.py index a33bb63da..9fde22c94 100644 --- a/imap_processing/glows/l1b/glows_l1b_data.py +++ b/imap_processing/glows/l1b/glows_l1b_data.py @@ -1035,6 +1035,7 @@ def compute_flags(self, pipeline_settings: PipelineSettings) -> np.ndarray: # Section 12.3.2 of the Algorithm Document: ground processing flags: flag 2. # Checks whether the total count in a given histogram is far from the daily average. # Placeholder until daily histogram is available in glows_l1b.py. + # TODO: this equation needs to be clarified. is_beyond_daily_statistical_error = np.uint8(1) # Section 12.3.2 of the Algorithm Document: ground processing flags: flag 3-6. diff --git a/imap_processing/tests/glows/test_glows_l1b.py b/imap_processing/tests/glows/test_glows_l1b.py index 22add65fc..eb2c06781 100644 --- a/imap_processing/tests/glows/test_glows_l1b.py +++ b/imap_processing/tests/glows/test_glows_l1b.py @@ -286,8 +286,6 @@ def test_process_histogram( test_hists = np.zeros(3600) # For temp encoded_val = np.single(expected_temp * 2.318 + 69.5454) - # Zero variance -> zero std dev -> all threshold flags pass (1 = good) - zero_variance = np.single(0) pipeline_settings = PipelineSettings( mock_pipeline_settings.sel( @@ -295,27 +293,25 @@ def test_process_histogram( ) ) - # flags_set_onboard = 64 = 0b01000000: bit 6 (is_night) set -> flag[6] = 0 (bad) - # is_generated_on_ground = 1 -> flag[10] = 0 (bad) test_l1b = HistogramL1B( test_hists, "test", 0, 0, 0, - 64, # flags_set_onboard: bit 6 (is_night) set - 1, # is_generated_on_ground + 0, + 0, 0, 3600, 0, encoded_val, - zero_variance, encoded_val, - zero_variance, encoded_val, - zero_variance, encoded_val, - zero_variance, + encoded_val, + encoded_val, + encoded_val, + encoded_val, time_val, time_val, time_val, @@ -325,13 +321,6 @@ def test_process_histogram( pipeline_settings, ) - # All onboard flags good (1) except flag[6] (is_night, bit 6 of 64). - # Flag[10] (is_generated_on_ground) = 0 (bad). All threshold flags = 1 (good). - expected_flags = np.ones(17, dtype=np.uint8) - expected_flags[6] = 0 # is_night: bit 6 of flags_set_onboard=64 is set - expected_flags[10] = 0 # is_generated_on_ground=1 - assert np.array_equal(test_l1b.flags, expected_flags) - output = process_histogram( hist_dataset, mock_ancillary_exclusions, @@ -341,6 +330,87 @@ def test_process_histogram( assert len(output) == len(dataclasses.asdict(test_l1b)) +@pytest.mark.parametrize( + "temp_var, hv_var, spin_var, pulse_var, expected_std_flags", + [ + pytest.param( + 0.0, 0.0, 0.0, 0.0, + [1, 1, 1, 1], + id="all_pass", + ), + pytest.param( + # Encoded variances chosen to exceed thresholds after decode_std_dev: + # temp: std_dev > 2.03°C (param_a=255/110, need encoded_var > ~22.1) + # hv: std_dev > 50.0V (param_a=4095/3500, need encoded_var > ~3422) + # spin: std_dev > 0.033333s (param_a=65535/20.9712, need > ~10850) + # pulse: std_dev > 1.0μs (param_a=255/255=1, need encoded_var > 1.0) + 30.0, 3500.0, 11000.0, 2.0, + [0, 0, 0, 0], + id="all_fail", + ), + ], +) +@patch.object( + HistogramL1B, + "flag_uv_and_excluded", + return_value=(np.zeros(3600, dtype=bool), np.zeros(3600, dtype=bool)), +) +@patch.object(HistogramL1B, "update_spice_parameters", autospec=True) +def test_compute_flags_std_dev_thresholds( + mock_spice_function, + mock_flag_uv_and_excluded, + temp_var, + hv_var, + spin_var, + pulse_var, + expected_std_flags, + mock_ancillary_exclusions, + mock_ancillary_parameters, + mock_pipeline_settings, +): + mock_spice_function.side_effect = mock_update_spice_parameters + encoded_val = np.single(100 * 2.318 + 69.5454) + pipeline_settings = PipelineSettings( + mock_pipeline_settings.sel( + epoch=mock_pipeline_settings.epoch[0], method="nearest" + ) + ) + + test_l1b = HistogramL1B( + np.zeros(3600), + "test", + 0, + 0, + 0, + 0, # flags_set_onboard: all good + 0, # is_generated_on_ground: onboard (good) + 0, + 3600, + 0, + encoded_val, + np.single(temp_var), + encoded_val, + np.single(hv_var), + encoded_val, + np.single(spin_var), + encoded_val, + np.single(pulse_var), + 0.0, + 0.0, + 0.0, + 0.0, + mock_ancillary_exclusions, + mock_ancillary_parameters, + pipeline_settings, + ) + + # flags[0:10] = onboard flags (all 1, flags_set_onboard=0) + # flags[10] = is_generated_on_ground (1, is_generated_on_ground=0) + # flags[11] = is_beyond_daily_statistical_error (placeholder, always 1) + # flags[12:16] = std_dev threshold flags (is_temp_ok, is_hv_ok, is_spin_std_ok, is_pulse_ok) + assert list(test_l1b.flags[12:16]) == expected_std_flags + + @patch.object( HistogramL1B, "flag_uv_and_excluded", @@ -627,11 +697,4 @@ def test_hist_spice_output( # (since the 0.05° threshold is exactly half the 0.1° bin spacing. assert np.count_nonzero(region_mask) == 1 - # Test flag_from_mask_dataset using the fixture data - instr_mask = hist_data.flag_from_mask_dataset( - day_exclusions.exclusions_by_instr_team - ) - assert instr_mask.shape == (3600,) - assert np.count_nonzero(instr_mask) == 10 - # TODO: Maxine will validate actual data with GLOWS team From 9b3831ca71b57e541dbf592b234afa4a86bdc29b Mon Sep 17 00:00:00 2001 From: Laura Sandoval Date: Fri, 6 Mar 2026 14:19:13 -0700 Subject: [PATCH 06/20] update --- imap_processing/glows/l1b/glows_l1b_data.py | 11 ++-- imap_processing/tests/glows/test_glows_l1b.py | 54 +++++++------------ 2 files changed, 23 insertions(+), 42 deletions(-) diff --git a/imap_processing/glows/l1b/glows_l1b_data.py b/imap_processing/glows/l1b/glows_l1b_data.py index 9fde22c94..3b2236261 100644 --- a/imap_processing/glows/l1b/glows_l1b_data.py +++ b/imap_processing/glows/l1b/glows_l1b_data.py @@ -670,11 +670,10 @@ def get_threshold(thresholds: dict, suffix: str) -> float | None: The matching threshold value, or None if no match is found. """ return_value = None - for section in thresholds.values(): - for descriptor, value in section.items(): - if descriptor.endswith(suffix): - return_value = float(value) - break + for descriptor, value in thresholds.items(): + if descriptor.endswith(suffix): + return_value = float(value) + break return return_value @@ -1033,7 +1032,7 @@ def compute_flags(self, pipeline_settings: PipelineSettings) -> np.ndarray: is_generated_on_ground = np.uint8(1 - int(self.is_generated_on_ground)) # Section 12.3.2 of the Algorithm Document: ground processing flags: flag 2. - # Checks whether the total count in a given histogram is far from the daily average. + # Checks if total count in a given histogram is far from the daily average. # Placeholder until daily histogram is available in glows_l1b.py. # TODO: this equation needs to be clarified. is_beyond_daily_statistical_error = np.uint8(1) diff --git a/imap_processing/tests/glows/test_glows_l1b.py b/imap_processing/tests/glows/test_glows_l1b.py index eb2c06781..5067d5312 100644 --- a/imap_processing/tests/glows/test_glows_l1b.py +++ b/imap_processing/tests/glows/test_glows_l1b.py @@ -330,45 +330,21 @@ def test_process_histogram( assert len(output) == len(dataclasses.asdict(test_l1b)) -@pytest.mark.parametrize( - "temp_var, hv_var, spin_var, pulse_var, expected_std_flags", - [ - pytest.param( - 0.0, 0.0, 0.0, 0.0, - [1, 1, 1, 1], - id="all_pass", - ), - pytest.param( - # Encoded variances chosen to exceed thresholds after decode_std_dev: - # temp: std_dev > 2.03°C (param_a=255/110, need encoded_var > ~22.1) - # hv: std_dev > 50.0V (param_a=4095/3500, need encoded_var > ~3422) - # spin: std_dev > 0.033333s (param_a=65535/20.9712, need > ~10850) - # pulse: std_dev > 1.0μs (param_a=255/255=1, need encoded_var > 1.0) - 30.0, 3500.0, 11000.0, 2.0, - [0, 0, 0, 0], - id="all_fail", - ), - ], -) @patch.object( HistogramL1B, "flag_uv_and_excluded", return_value=(np.zeros(3600, dtype=bool), np.zeros(3600, dtype=bool)), ) @patch.object(HistogramL1B, "update_spice_parameters", autospec=True) -def test_compute_flags_std_dev_thresholds( +def test_compute_flags( mock_spice_function, mock_flag_uv_and_excluded, - temp_var, - hv_var, - spin_var, - pulse_var, - expected_std_flags, mock_ancillary_exclusions, mock_ancillary_parameters, mock_pipeline_settings, ): mock_spice_function.side_effect = mock_update_spice_parameters + encoded_val = np.single(100 * 2.318 + 69.5454) pipeline_settings = PipelineSettings( mock_pipeline_settings.sel( @@ -382,19 +358,19 @@ def test_compute_flags_std_dev_thresholds( 0, 0, 0, - 0, # flags_set_onboard: all good - 0, # is_generated_on_ground: onboard (good) + 64, # flags_set_onboard: bit 6 (is_night) set + 1, # is_generated_on_ground 0, 3600, 0, encoded_val, - np.single(temp_var), + np.single(30.0), # filter_temperature_variance: exceeds 2.03°C threshold encoded_val, - np.single(hv_var), + np.single(3500.0), # hv_voltage_variance: exceeds 50.0V threshold encoded_val, - np.single(spin_var), + np.single(11000.0), # spin_period_variance: exceeds 0.033333s threshold encoded_val, - np.single(pulse_var), + np.single(2.0), # pulse_length_variance: exceeds 1.0μs threshold 0.0, 0.0, 0.0, @@ -404,11 +380,17 @@ def test_compute_flags_std_dev_thresholds( pipeline_settings, ) - # flags[0:10] = onboard flags (all 1, flags_set_onboard=0) - # flags[10] = is_generated_on_ground (1, is_generated_on_ground=0) + # flags[0:10] = onboard flags (1=good, 0=bad), one per bit of flags_set_onboard + # flags[10] = is_generated_on_ground (1=onboard, 0=ground) # flags[11] = is_beyond_daily_statistical_error (placeholder, always 1) - # flags[12:16] = std_dev threshold flags (is_temp_ok, is_hv_ok, is_spin_std_ok, is_pulse_ok) - assert list(test_l1b.flags[12:16]) == expected_std_flags + # flags[12:16] = std_dev threshold flags + # (is_temp_ok, is_hv_ok, is_spin_std_ok, is_pulse_ok) + assert test_l1b.flags[6] == 0 # is_night + assert test_l1b.flags[10] == 0 # is_generated_on_ground + assert test_l1b.flags[12] == 0 # is_temp_ok + assert test_l1b.flags[13] == 0 # is_hv_ok + assert test_l1b.flags[14] == 0 # is_spin_std_ok + assert test_l1b.flags[15] == 0 # is_pulse_ok @patch.object( From 93c83ecd13260501fe7d15ef73fbb5f266c0c47e Mon Sep 17 00:00:00 2001 From: Laura Sandoval Date: Fri, 6 Mar 2026 14:31:29 -0700 Subject: [PATCH 07/20] update test --- imap_processing/tests/glows/test_glows_l1b.py | 22 +++++++++---------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/imap_processing/tests/glows/test_glows_l1b.py b/imap_processing/tests/glows/test_glows_l1b.py index 5067d5312..721906699 100644 --- a/imap_processing/tests/glows/test_glows_l1b.py +++ b/imap_processing/tests/glows/test_glows_l1b.py @@ -299,19 +299,19 @@ def test_process_histogram( 0, 0, 0, - 0, - 0, + 64, # flags_set_onboard: bit 6 (is_night) set + 1, # is_generated_on_ground 0, 3600, 0, encoded_val, + np.single(30.0), # filter_temperature_variance: exceeds 2.03°C threshold encoded_val, + np.single(3500.0), # hv_voltage_variance: exceeds 50.0V threshold encoded_val, + np.single(11000.0), # spin_period_variance: exceeds 0.033333s threshold encoded_val, - encoded_val, - encoded_val, - encoded_val, - encoded_val, + np.single(2.0), # pulse_length_variance: exceeds 1.0μs threshold time_val, time_val, time_val, @@ -345,6 +345,7 @@ def test_compute_flags( ): mock_spice_function.side_effect = mock_update_spice_parameters + time_val = np.single(1111111.11) encoded_val = np.single(100 * 2.318 + 69.5454) pipeline_settings = PipelineSettings( mock_pipeline_settings.sel( @@ -371,10 +372,10 @@ def test_compute_flags( np.single(11000.0), # spin_period_variance: exceeds 0.033333s threshold encoded_val, np.single(2.0), # pulse_length_variance: exceeds 1.0μs threshold - 0.0, - 0.0, - 0.0, - 0.0, + time_val, + time_val, + time_val, + time_val, mock_ancillary_exclusions, mock_ancillary_parameters, pipeline_settings, @@ -384,7 +385,6 @@ def test_compute_flags( # flags[10] = is_generated_on_ground (1=onboard, 0=ground) # flags[11] = is_beyond_daily_statistical_error (placeholder, always 1) # flags[12:16] = std_dev threshold flags - # (is_temp_ok, is_hv_ok, is_spin_std_ok, is_pulse_ok) assert test_l1b.flags[6] == 0 # is_night assert test_l1b.flags[10] == 0 # is_generated_on_ground assert test_l1b.flags[12] == 0 # is_temp_ok From d4f5adc0684b3f99284b708ceff416cf7967ae0c Mon Sep 17 00:00:00 2001 From: Laura Sandoval Date: Fri, 6 Mar 2026 14:35:54 -0700 Subject: [PATCH 08/20] update --- imap_processing/tests/glows/conftest.py | 6 ++---- imap_processing/tests/glows/test_glows_l1b.py | 12 ++++++------ 2 files changed, 8 insertions(+), 10 deletions(-) diff --git a/imap_processing/tests/glows/conftest.py b/imap_processing/tests/glows/conftest.py index 82881f8b2..c2d573f73 100644 --- a/imap_processing/tests/glows/conftest.py +++ b/imap_processing/tests/glows/conftest.py @@ -253,10 +253,8 @@ def mock_pipeline_settings(): "active_bad_time_flags": ( ["epoch", "time_flag_index"], np.tile( - [True, True, True, True, True, True, False, - True, True, True, True, True, True, True, True, True, False], - (len(epoch_range), 1), - ), + [True] * 17, (len(epoch_range), 1) + ), # 17 bad time flags from the JSON ), "sunrise_offset": (["epoch"], [0.0] * len(epoch_range)), "sunset_offset": (["epoch"], [0.0] * len(epoch_range)), diff --git a/imap_processing/tests/glows/test_glows_l1b.py b/imap_processing/tests/glows/test_glows_l1b.py index 721906699..b51ec82d6 100644 --- a/imap_processing/tests/glows/test_glows_l1b.py +++ b/imap_processing/tests/glows/test_glows_l1b.py @@ -299,19 +299,19 @@ def test_process_histogram( 0, 0, 0, - 64, # flags_set_onboard: bit 6 (is_night) set - 1, # is_generated_on_ground + 0, + 0, 0, 3600, 0, encoded_val, - np.single(30.0), # filter_temperature_variance: exceeds 2.03°C threshold encoded_val, - np.single(3500.0), # hv_voltage_variance: exceeds 50.0V threshold encoded_val, - np.single(11000.0), # spin_period_variance: exceeds 0.033333s threshold encoded_val, - np.single(2.0), # pulse_length_variance: exceeds 1.0μs threshold + encoded_val, + encoded_val, + encoded_val, + encoded_val, time_val, time_val, time_val, From d2d762d669619b0677e46277c5211d5b4fc86373 Mon Sep 17 00:00:00 2001 From: Laura Sandoval Date: Fri, 6 Mar 2026 14:42:28 -0700 Subject: [PATCH 09/20] add 17th flag --- imap_processing/glows/l1b/glows_l1b_data.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/imap_processing/glows/l1b/glows_l1b_data.py b/imap_processing/glows/l1b/glows_l1b_data.py index 3b2236261..556edf5f3 100644 --- a/imap_processing/glows/l1b/glows_l1b_data.py +++ b/imap_processing/glows/l1b/glows_l1b_data.py @@ -1037,7 +1037,7 @@ def compute_flags(self, pipeline_settings: PipelineSettings) -> np.ndarray: # TODO: this equation needs to be clarified. is_beyond_daily_statistical_error = np.uint8(1) - # Section 12.3.2 of the Algorithm Document: ground processing flags: flag 3-6. + # Section 12.3.2 of the Algorithm Document: ground processing flags: flag 3-7. # (1=good, 0=bad). temp_threshold = get_threshold(thresholds, "std_dev_threshold__celsius_deg") hv_threshold = get_threshold(thresholds, "std_dev_threshold__volt") @@ -1049,6 +1049,10 @@ def compute_flags(self, pipeline_settings: PipelineSettings) -> np.ndarray: is_spin_std_ok = np.uint8(self.spin_period_std_dev <= spin_std_threshold) is_pulse_ok = np.uint8(self.pulse_length_std_dev <= pulse_threshold) + # TODO: listed as TBC in Algorithm Document. + # Placeholder for now. + is_beyond_background_error = np.uint8(1) + ground_flags = np.array( [ is_generated_on_ground, @@ -1057,6 +1061,7 @@ def compute_flags(self, pipeline_settings: PipelineSettings) -> np.ndarray: is_hv_ok, is_spin_std_ok, is_pulse_ok, + is_beyond_background_error, ], dtype=np.uint8, ) From 6c5b207039bd75ea4c46c2f323e9d949568d5dc8 Mon Sep 17 00:00:00 2001 From: Laura Sandoval Date: Fri, 6 Mar 2026 14:43:57 -0700 Subject: [PATCH 10/20] revert --- imap_processing/tests/glows/test_glows_l1b.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/imap_processing/tests/glows/test_glows_l1b.py b/imap_processing/tests/glows/test_glows_l1b.py index b51ec82d6..694fb5f86 100644 --- a/imap_processing/tests/glows/test_glows_l1b.py +++ b/imap_processing/tests/glows/test_glows_l1b.py @@ -679,4 +679,11 @@ def test_hist_spice_output( # (since the 0.05° threshold is exactly half the 0.1° bin spacing. assert np.count_nonzero(region_mask) == 1 + # Test flag_from_mask_dataset using the fixture data + instr_mask = hist_data.flag_from_mask_dataset( + day_exclusions.exclusions_by_instr_team + ) + assert instr_mask.shape == (3600,) + assert np.count_nonzero(instr_mask) == 10 + # TODO: Maxine will validate actual data with GLOWS team From c9d38244e9e30351d1bd38374fccec7ed3488cd7 Mon Sep 17 00:00:00 2001 From: Laura Sandoval Date: Fri, 6 Mar 2026 14:46:46 -0700 Subject: [PATCH 11/20] revert --- imap_processing/glows/l1b/glows_l1b.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/imap_processing/glows/l1b/glows_l1b.py b/imap_processing/glows/l1b/glows_l1b.py index 050eec6d4..eb87d3404 100644 --- a/imap_processing/glows/l1b/glows_l1b.py +++ b/imap_processing/glows/l1b/glows_l1b.py @@ -189,7 +189,9 @@ def create_direct_event_l1b(*args) -> tuple: # type: ignore[no-untyped-def] Tuple of values from DirectEventL1B dataclass. """ return tuple( - dataclasses.asdict(DirectEventL1B(*args, ancillary_parameters)).values() + dataclasses.asdict( + DirectEventL1B(*args, ancillary_parameters) # type: ignore[call-arg] + ).values() ) l1b_fields: tuple = xr.apply_ufunc( @@ -283,7 +285,7 @@ def create_histogram_l1b(*args) -> tuple: # type: ignore[no-untyped-def] tuple Tuple of processed L1B data arrays from HistogramL1B.output_data(). """ - return HistogramL1B( + return HistogramL1B( # type: ignore[call-arg] *args, ancillary_exclusions, ancillary_parameters, pipeline_settings ).output_data() From 6d94eb7a96e8c7fcc3d8bb0b772edf0be150ff37 Mon Sep 17 00:00:00 2001 From: Laura Sandoval Date: Mon, 9 Mar 2026 09:40:37 -0600 Subject: [PATCH 12/20] fix test --- imap_processing/glows/l1b/glows_l1b_data.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/imap_processing/glows/l1b/glows_l1b_data.py b/imap_processing/glows/l1b/glows_l1b_data.py index 556edf5f3..d8fb5af35 100644 --- a/imap_processing/glows/l1b/glows_l1b_data.py +++ b/imap_processing/glows/l1b/glows_l1b_data.py @@ -670,10 +670,11 @@ def get_threshold(thresholds: dict, suffix: str) -> float | None: The matching threshold value, or None if no match is found. """ return_value = None - for descriptor, value in thresholds.items(): - if descriptor.endswith(suffix): - return_value = float(value) - break + for section in thresholds.values(): + for descriptor, value in section.items(): + if descriptor.endswith(suffix): + return_value = float(value) + break return return_value From 5c01f587a4343ac162de5257343ab63699478ac2 Mon Sep 17 00:00:00 2001 From: Laura Sandoval Date: Mon, 9 Mar 2026 11:33:22 -0600 Subject: [PATCH 13/20] fix tests --- imap_processing/glows/l1b/glows_l1b_data.py | 9 ++++----- .../tests/glows/test_glows_l1b_data.py | 17 +++++++++-------- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/imap_processing/glows/l1b/glows_l1b_data.py b/imap_processing/glows/l1b/glows_l1b_data.py index d8fb5af35..556edf5f3 100644 --- a/imap_processing/glows/l1b/glows_l1b_data.py +++ b/imap_processing/glows/l1b/glows_l1b_data.py @@ -670,11 +670,10 @@ def get_threshold(thresholds: dict, suffix: str) -> float | None: The matching threshold value, or None if no match is found. """ return_value = None - for section in thresholds.values(): - for descriptor, value in section.items(): - if descriptor.endswith(suffix): - return_value = float(value) - break + for descriptor, value in thresholds.items(): + if descriptor.endswith(suffix): + return_value = float(value) + break return return_value diff --git a/imap_processing/tests/glows/test_glows_l1b_data.py b/imap_processing/tests/glows/test_glows_l1b_data.py index ebd6bc1df..636cdf8df 100644 --- a/imap_processing/tests/glows/test_glows_l1b_data.py +++ b/imap_processing/tests/glows/test_glows_l1b_data.py @@ -292,17 +292,18 @@ def test_pipeline_settings_from_flattened_json(): def test_get_threshold(): "Test get_threshold function." + test_data = { - "filter_based_on_comparison_of_spin_periods": { - "relative_difference_threshold": 7.0e-4 - }, - "filter_based_on_temperature_std_dev": {"std_dev_threshold__celsius_deg": 2.03}, - "filter_based_on_hv_voltage_std_dev": {"std_dev_threshold__volt": 50.0}, - "filter_based_on_spin_period_std_dev": {"std_dev_threshold__sec": 0.033333}, - "filter_based_on_pulse_length_std_dev": {"std_dev_threshold__usec": 1.0}, + "n_sigma_threshold_lower": 3.0, + "n_sigma_threshold_upper": 3.0, + "relative_difference_threshold": 7e-05, + "std_dev_threshold__celsius_deg": 2.03, + "std_dev_threshold__volt": 50.0, + "std_dev_threshold__sec": 0.033333, + "std_dev_threshold__usec": 1.0, } - expected = [2.03, 50.0, 0.033333, 1.0, 7.0e-4] + expected = [2.03, 50.0, 0.033333, 1.0, 7e-5] description = [ "std_dev_threshold__celsius_deg", "std_dev_threshold__volt", From fb07e81d27714bb1df7cca9d65d65f80a9d9b721 Mon Sep 17 00:00:00 2001 From: Laura Sandoval Date: Mon, 9 Mar 2026 13:13:57 -0600 Subject: [PATCH 14/20] fix tests --- imap_processing/tests/glows/test_glows_l1b.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/imap_processing/tests/glows/test_glows_l1b.py b/imap_processing/tests/glows/test_glows_l1b.py index 694fb5f86..b4fa60c4b 100644 --- a/imap_processing/tests/glows/test_glows_l1b.py +++ b/imap_processing/tests/glows/test_glows_l1b.py @@ -385,12 +385,14 @@ def test_compute_flags( # flags[10] = is_generated_on_ground (1=onboard, 0=ground) # flags[11] = is_beyond_daily_statistical_error (placeholder, always 1) # flags[12:16] = std_dev threshold flags + # flags[16] = is_beyond_background assert test_l1b.flags[6] == 0 # is_night assert test_l1b.flags[10] == 0 # is_generated_on_ground assert test_l1b.flags[12] == 0 # is_temp_ok assert test_l1b.flags[13] == 0 # is_hv_ok assert test_l1b.flags[14] == 0 # is_spin_std_ok assert test_l1b.flags[15] == 0 # is_pulse_ok + assert test_l1b.flags[16] == 1 # is_beyond_background @patch.object( From 117ad908e3105f98214dba48ce9d2e47dc9b5658 Mon Sep 17 00:00:00 2001 From: Laura Sandoval Date: Mon, 9 Mar 2026 13:46:44 -0600 Subject: [PATCH 15/20] fix tests --- imap_processing/tests/glows/test_glows_l1b.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/imap_processing/tests/glows/test_glows_l1b.py b/imap_processing/tests/glows/test_glows_l1b.py index b4fa60c4b..09958345d 100644 --- a/imap_processing/tests/glows/test_glows_l1b.py +++ b/imap_processing/tests/glows/test_glows_l1b.py @@ -346,6 +346,7 @@ def test_compute_flags( mock_spice_function.side_effect = mock_update_spice_parameters time_val = np.single(1111111.11) + test_hists = np.zeros(3600) encoded_val = np.single(100 * 2.318 + 69.5454) pipeline_settings = PipelineSettings( mock_pipeline_settings.sel( @@ -354,24 +355,24 @@ def test_compute_flags( ) test_l1b = HistogramL1B( - np.zeros(3600), + test_hists, "test", 0, 0, 0, - 64, # flags_set_onboard: bit 6 (is_night) set - 1, # is_generated_on_ground + 0, + 0, 0, 3600, 0, encoded_val, - np.single(30.0), # filter_temperature_variance: exceeds 2.03°C threshold encoded_val, - np.single(3500.0), # hv_voltage_variance: exceeds 50.0V threshold encoded_val, - np.single(11000.0), # spin_period_variance: exceeds 0.033333s threshold encoded_val, - np.single(2.0), # pulse_length_variance: exceeds 1.0μs threshold + encoded_val, + encoded_val, + encoded_val, + encoded_val, time_val, time_val, time_val, From 4f07e50732788571d6519dfa6f78f0481962612c Mon Sep 17 00:00:00 2001 From: Laura Sandoval Date: Mon, 9 Mar 2026 13:55:40 -0600 Subject: [PATCH 16/20] fix tests --- imap_processing/tests/glows/test_glows_l1b.py | 66 ------------------- 1 file changed, 66 deletions(-) diff --git a/imap_processing/tests/glows/test_glows_l1b.py b/imap_processing/tests/glows/test_glows_l1b.py index 09958345d..d8a7a6a42 100644 --- a/imap_processing/tests/glows/test_glows_l1b.py +++ b/imap_processing/tests/glows/test_glows_l1b.py @@ -330,72 +330,6 @@ def test_process_histogram( assert len(output) == len(dataclasses.asdict(test_l1b)) -@patch.object( - HistogramL1B, - "flag_uv_and_excluded", - return_value=(np.zeros(3600, dtype=bool), np.zeros(3600, dtype=bool)), -) -@patch.object(HistogramL1B, "update_spice_parameters", autospec=True) -def test_compute_flags( - mock_spice_function, - mock_flag_uv_and_excluded, - mock_ancillary_exclusions, - mock_ancillary_parameters, - mock_pipeline_settings, -): - mock_spice_function.side_effect = mock_update_spice_parameters - - time_val = np.single(1111111.11) - test_hists = np.zeros(3600) - encoded_val = np.single(100 * 2.318 + 69.5454) - pipeline_settings = PipelineSettings( - mock_pipeline_settings.sel( - epoch=mock_pipeline_settings.epoch[0], method="nearest" - ) - ) - - test_l1b = HistogramL1B( - test_hists, - "test", - 0, - 0, - 0, - 0, - 0, - 0, - 3600, - 0, - encoded_val, - encoded_val, - encoded_val, - encoded_val, - encoded_val, - encoded_val, - encoded_val, - encoded_val, - time_val, - time_val, - time_val, - time_val, - mock_ancillary_exclusions, - mock_ancillary_parameters, - pipeline_settings, - ) - - # flags[0:10] = onboard flags (1=good, 0=bad), one per bit of flags_set_onboard - # flags[10] = is_generated_on_ground (1=onboard, 0=ground) - # flags[11] = is_beyond_daily_statistical_error (placeholder, always 1) - # flags[12:16] = std_dev threshold flags - # flags[16] = is_beyond_background - assert test_l1b.flags[6] == 0 # is_night - assert test_l1b.flags[10] == 0 # is_generated_on_ground - assert test_l1b.flags[12] == 0 # is_temp_ok - assert test_l1b.flags[13] == 0 # is_hv_ok - assert test_l1b.flags[14] == 0 # is_spin_std_ok - assert test_l1b.flags[15] == 0 # is_pulse_ok - assert test_l1b.flags[16] == 1 # is_beyond_background - - @patch.object( HistogramL1B, "flag_uv_and_excluded", From 5f7125ddbb858b211f0d800df56a9f484816763a Mon Sep 17 00:00:00 2001 From: Laura Sandoval Date: Mon, 9 Mar 2026 14:33:42 -0600 Subject: [PATCH 17/20] update test --- imap_processing/tests/glows/test_glows_l1b.py | 41 +++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/imap_processing/tests/glows/test_glows_l1b.py b/imap_processing/tests/glows/test_glows_l1b.py index d8a7a6a42..384ea1ba6 100644 --- a/imap_processing/tests/glows/test_glows_l1b.py +++ b/imap_processing/tests/glows/test_glows_l1b.py @@ -329,6 +329,47 @@ def test_process_histogram( ) assert len(output) == len(dataclasses.asdict(test_l1b)) + test_l1b = HistogramL1B( + np.zeros(3600), + "test", + 0, + 0, + 0, + 64, # flags_set_onboard: bit 6 (is_night) set + 1, # is_generated_on_ground + 0, + 3600, + 0, + encoded_val, + np.single(30.0), # filter_temperature_variance: exceeds 2.03°C threshold + encoded_val, + np.single(3500.0), # hv_voltage_variance: exceeds 50.0V threshold + encoded_val, + np.single(11000.0), # spin_period_variance: exceeds 0.033333s threshold + encoded_val, + np.single(2.0), # pulse_length_variance: exceeds 1.0μs threshold + time_val, + time_val, + time_val, + time_val, + mock_ancillary_exclusions, + mock_ancillary_parameters, + pipeline_settings, + ) + + # flags[0:10] = onboard flags (1=good, 0=bad), one per bit of flags_set_onboard + # flags[10] = is_generated_on_ground (1=onboard, 0=ground) + # flags[11] = is_beyond_daily_statistical_error (placeholder, always 1) + # flags[12:16] = std_dev threshold flags + # flags[16] = is_beyond_background + assert test_l1b.flags[6] == 0 # is_night + assert test_l1b.flags[10] == 0 # is_generated_on_ground + assert test_l1b.flags[12] == 0 # is_temp_ok + assert test_l1b.flags[13] == 0 # is_hv_ok + assert test_l1b.flags[14] == 0 # is_spin_std_ok + assert test_l1b.flags[15] == 0 # is_pulse_ok + assert test_l1b.flags[16] == 1 # is_beyond_background + @patch.object( HistogramL1B, From 8002dfbbd5d01646107f38021b19a6843f9661eb Mon Sep 17 00:00:00 2001 From: Laura Sandoval Date: Mon, 9 Mar 2026 14:51:22 -0600 Subject: [PATCH 18/20] update test --- imap_processing/tests/glows/test_glows_l1b.py | 28 +++++++++---------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/imap_processing/tests/glows/test_glows_l1b.py b/imap_processing/tests/glows/test_glows_l1b.py index 384ea1ba6..13f097dbe 100644 --- a/imap_processing/tests/glows/test_glows_l1b.py +++ b/imap_processing/tests/glows/test_glows_l1b.py @@ -330,24 +330,24 @@ def test_process_histogram( assert len(output) == len(dataclasses.asdict(test_l1b)) test_l1b = HistogramL1B( - np.zeros(3600), + test_hists, "test", 0, 0, 0, - 64, # flags_set_onboard: bit 6 (is_night) set - 1, # is_generated_on_ground + 0, + 0, 0, 3600, 0, encoded_val, - np.single(30.0), # filter_temperature_variance: exceeds 2.03°C threshold encoded_val, - np.single(3500.0), # hv_voltage_variance: exceeds 50.0V threshold encoded_val, - np.single(11000.0), # spin_period_variance: exceeds 0.033333s threshold encoded_val, - np.single(2.0), # pulse_length_variance: exceeds 1.0μs threshold + encoded_val, + encoded_val, + encoded_val, + encoded_val, time_val, time_val, time_val, @@ -362,13 +362,13 @@ def test_process_histogram( # flags[11] = is_beyond_daily_statistical_error (placeholder, always 1) # flags[12:16] = std_dev threshold flags # flags[16] = is_beyond_background - assert test_l1b.flags[6] == 0 # is_night - assert test_l1b.flags[10] == 0 # is_generated_on_ground - assert test_l1b.flags[12] == 0 # is_temp_ok - assert test_l1b.flags[13] == 0 # is_hv_ok - assert test_l1b.flags[14] == 0 # is_spin_std_ok - assert test_l1b.flags[15] == 0 # is_pulse_ok - assert test_l1b.flags[16] == 1 # is_beyond_background + # assert test_l1b.flags[6] == 0 # is_night + # assert test_l1b.flags[10] == 0 # is_generated_on_ground + # assert test_l1b.flags[12] == 0 # is_temp_ok + # assert test_l1b.flags[13] == 0 # is_hv_ok + # assert test_l1b.flags[14] == 0 # is_spin_std_ok + # assert test_l1b.flags[15] == 0 # is_pulse_ok + # assert test_l1b.flags[16] == 1 # is_beyond_background @patch.object( From 5e25ac5f91978c0dfd7b199c4c9e865923b68cd9 Mon Sep 17 00:00:00 2001 From: Laura Sandoval Date: Mon, 9 Mar 2026 15:01:34 -0600 Subject: [PATCH 19/20] update test --- imap_processing/tests/glows/test_glows_l1b.py | 54 +++++-------------- 1 file changed, 13 insertions(+), 41 deletions(-) diff --git a/imap_processing/tests/glows/test_glows_l1b.py b/imap_processing/tests/glows/test_glows_l1b.py index 13f097dbe..ff2a3bbbf 100644 --- a/imap_processing/tests/glows/test_glows_l1b.py +++ b/imap_processing/tests/glows/test_glows_l1b.py @@ -299,19 +299,19 @@ def test_process_histogram( 0, 0, 0, - 0, - 0, + 64, # flags_set_onboard: bit 6 (is_night) set + 1, # is_generated_on_ground 0, 3600, 0, encoded_val, + np.single(30.0), # filter_temperature_variance: exceeds 2.03°C threshold encoded_val, + np.single(3500.0), # hv_voltage_variance: exceeds 50.0V threshold encoded_val, + np.single(11000.0), # spin_period_variance: exceeds 0.033333s threshold encoded_val, - encoded_val, - encoded_val, - encoded_val, - encoded_val, + np.single(2.0), # pulse_length_variance: exceeds 1.0μs threshold time_val, time_val, time_val, @@ -329,46 +329,18 @@ def test_process_histogram( ) assert len(output) == len(dataclasses.asdict(test_l1b)) - test_l1b = HistogramL1B( - test_hists, - "test", - 0, - 0, - 0, - 0, - 0, - 0, - 3600, - 0, - encoded_val, - encoded_val, - encoded_val, - encoded_val, - encoded_val, - encoded_val, - encoded_val, - encoded_val, - time_val, - time_val, - time_val, - time_val, - mock_ancillary_exclusions, - mock_ancillary_parameters, - pipeline_settings, - ) - # flags[0:10] = onboard flags (1=good, 0=bad), one per bit of flags_set_onboard # flags[10] = is_generated_on_ground (1=onboard, 0=ground) # flags[11] = is_beyond_daily_statistical_error (placeholder, always 1) # flags[12:16] = std_dev threshold flags # flags[16] = is_beyond_background - # assert test_l1b.flags[6] == 0 # is_night - # assert test_l1b.flags[10] == 0 # is_generated_on_ground - # assert test_l1b.flags[12] == 0 # is_temp_ok - # assert test_l1b.flags[13] == 0 # is_hv_ok - # assert test_l1b.flags[14] == 0 # is_spin_std_ok - # assert test_l1b.flags[15] == 0 # is_pulse_ok - # assert test_l1b.flags[16] == 1 # is_beyond_background + assert test_l1b.flags[6] == 0 # is_night + assert test_l1b.flags[10] == 0 # is_generated_on_ground + assert test_l1b.flags[12] == 0 # is_temp_ok + assert test_l1b.flags[13] == 0 # is_hv_ok + assert test_l1b.flags[14] == 0 # is_spin_std_ok + assert test_l1b.flags[15] == 0 # is_pulse_ok + assert test_l1b.flags[16] == 1 # is_beyond_background @patch.object( From 55715283d2d018c7bce47b8dcef2501b17cda1cf Mon Sep 17 00:00:00 2001 From: Laura Sandoval Date: Thu, 12 Mar 2026 10:55:50 -0600 Subject: [PATCH 20/20] pr response --- imap_processing/glows/l1b/glows_l1b_data.py | 59 +++++++++---------- .../tests/glows/test_glows_l1b_data.py | 7 ++- 2 files changed, 32 insertions(+), 34 deletions(-) diff --git a/imap_processing/glows/l1b/glows_l1b_data.py b/imap_processing/glows/l1b/glows_l1b_data.py index 556edf5f3..fafcfa339 100644 --- a/imap_processing/glows/l1b/glows_l1b_data.py +++ b/imap_processing/glows/l1b/glows_l1b_data.py @@ -176,6 +176,28 @@ def __post_init__(self, pipeline_dataset: xr.Dataset) -> None: if "threshold" in var_name.lower() or "limit" in var_name.lower(): self.processing_thresholds[var_name] = pipeline_dataset[var_name].item() + def get_threshold(self, suffix: str) -> float | None: + """ + Return the threshold value whose key ends with the given suffix. + + Parameters + ---------- + suffix : str + The suffix to match against threshold keys. + + Returns + ------- + return_value : float or None + The matching threshold value, or None if no match is found. + """ + return_value = None + for descriptor, value in self.processing_thresholds.items(): + if descriptor.endswith(suffix): + return_value = float(value) + break + + return return_value + @dataclass class AncillaryExclusions: @@ -653,31 +675,6 @@ def process_direct_events(direct_events: np.ndarray) -> tuple: return times, pulse_lengths -def get_threshold(thresholds: dict, suffix: str) -> float | None: - """ - Return the threshold value whose key ends with the given suffix. - - Parameters - ---------- - thresholds : dict - Dictionary of threshold values. - suffix : str - The suffix to match against threshold keys. - - Returns - ------- - return_value : float or None - The matching threshold value, or None if no match is found. - """ - return_value = None - for descriptor, value in thresholds.items(): - if descriptor.endswith(suffix): - return_value = float(value) - break - - return return_value - - @dataclass class HistogramL1B: """ @@ -1018,8 +1015,6 @@ def compute_flags(self, pipeline_settings: PipelineSettings) -> np.ndarray: flags : numpy.ndarray Array of shape (FLAG_LENGTH,) with dtype uint8. 1 = good, 0 = bad. """ - thresholds = pipeline_settings.processing_thresholds - # Section 12.3.1 of the Algorithm Document: onboard generated bad-time flags. # Flags are "stored in a 16-bit integer field. onboard_flags = ( @@ -1039,10 +1034,12 @@ def compute_flags(self, pipeline_settings: PipelineSettings) -> np.ndarray: # Section 12.3.2 of the Algorithm Document: ground processing flags: flag 3-7. # (1=good, 0=bad). - temp_threshold = get_threshold(thresholds, "std_dev_threshold__celsius_deg") - hv_threshold = get_threshold(thresholds, "std_dev_threshold__volt") - spin_std_threshold = get_threshold(thresholds, "std_dev_threshold__sec") - pulse_threshold = get_threshold(thresholds, "std_dev_threshold__usec") + temp_threshold = pipeline_settings.get_threshold( + "std_dev_threshold__celsius_deg" + ) + hv_threshold = pipeline_settings.get_threshold("std_dev_threshold__volt") + spin_std_threshold = pipeline_settings.get_threshold("std_dev_threshold__sec") + pulse_threshold = pipeline_settings.get_threshold("std_dev_threshold__usec") is_temp_ok = np.uint8(self.filter_temperature_std_dev <= temp_threshold) is_hv_ok = np.uint8(self.hv_voltage_std_dev <= hv_threshold) diff --git a/imap_processing/tests/glows/test_glows_l1b_data.py b/imap_processing/tests/glows/test_glows_l1b_data.py index 636cdf8df..740c13436 100644 --- a/imap_processing/tests/glows/test_glows_l1b_data.py +++ b/imap_processing/tests/glows/test_glows_l1b_data.py @@ -12,7 +12,6 @@ DirectEventL1B, HistogramL1B, PipelineSettings, - get_threshold, ) from imap_processing.spice.time import met_to_ttj2000ns from imap_processing.tests.glows.conftest import mock_update_spice_parameters @@ -291,7 +290,7 @@ def test_pipeline_settings_from_flattened_json(): def test_get_threshold(): - "Test get_threshold function." + "Test PipelineSettings.get_threshold method." test_data = { "n_sigma_threshold_lower": 3.0, @@ -302,6 +301,8 @@ def test_get_threshold(): "std_dev_threshold__sec": 0.033333, "std_dev_threshold__usec": 1.0, } + pipeline_dataset = xr.Dataset({k: xr.DataArray(v) for k, v in test_data.items()}) + settings = PipelineSettings(pipeline_dataset) expected = [2.03, 50.0, 0.033333, 1.0, 7e-5] description = [ @@ -313,5 +314,5 @@ def test_get_threshold(): ] for name, exp in zip(description, expected, strict=False): - threshold = get_threshold(test_data, name) + threshold = settings.get_threshold(name) assert threshold == exp