From 4adc2c0c5573863cdff1bd3cc2988299e03c557a Mon Sep 17 00:00:00 2001 From: Luisa Date: Wed, 25 Feb 2026 12:02:10 -0700 Subject: [PATCH 01/11] sort of changes --- .../config/imap_idex_l1b_variable_attrs.yaml | 26 +++++- imap_processing/idex/idex_l1a.py | 7 ++ imap_processing/idex/idex_l1b.py | 48 ++++------- imap_processing/tests/idex/conftest.py | 4 +- imap_processing/tests/idex/test_idex_l1b.py | 80 +++++++++---------- 5 files changed, 89 insertions(+), 76 deletions(-) diff --git a/imap_processing/cdf/config/imap_idex_l1b_variable_attrs.yaml b/imap_processing/cdf/config/imap_idex_l1b_variable_attrs.yaml index 7397b8ebe3..1b1492548a 100644 --- a/imap_processing/cdf/config/imap_idex_l1b_variable_attrs.yaml +++ b/imap_processing/cdf/config/imap_idex_l1b_variable_attrs.yaml @@ -53,16 +53,38 @@ spice_base: &spice_base VALIDMAX: *spice_data_max # <=== Instrument Setting Attributes ===> -trigger_mode: +trigger_mode_lg: <<: *string_base FIELDNAM: Trigger Mode CATDESC: Channel and mode that triggered the event -trigger_level: +trigger_level_lg: <<: *trigger_base FIELDNAM: Trigger Level CATDESC: Threshold signal level that triggered the event +trigger_mode_mg: + <<: *string_base + FIELDNAM: Trigger Mode + CATDESC: Channel and mode that triggered the event + +trigger_level_mg: + <<: *trigger_base + FIELDNAM: Trigger Level + CATDESC: Threshold signal level that triggered the event + + +trigger_mode_hg: + <<: *string_base + FIELDNAM: Trigger Mode + CATDESC: Channel and mode that triggered the event + +trigger_level_hg: + <<: *trigger_base + FIELDNAM: Trigger Level + CATDESC: Threshold signal level that triggered the event + + tof_high: <<: *l1b_tof_base CATDESC: Time of flight waveform on the high-gain channel diff --git a/imap_processing/idex/idex_l1a.py b/imap_processing/idex/idex_l1a.py index 66a41799d8..d09499b7b4 100644 --- a/imap_processing/idex/idex_l1a.py +++ b/imap_processing/idex/idex_l1a.py @@ -130,6 +130,13 @@ def _create_science_dataset(self, science_decom_packet_list: list) -> xr.Dataset Dataset containing processed dust events. """ dust_events = {} + # To handle duplicate packets received from the POC, calculate a unique ID + # for each packet and find the duplicate indexes. Then, only process the unique + # packets to create the dataset. + # packet_ids = [ + # (p["IDX__SCI0TYPE"], p["IDX__SCI0EVTNUM"]) + # for p in science_decom_packet_list + # ] for packet in science_decom_packet_list: if "IDX__SCI0TYPE" in packet: scitype = packet["IDX__SCI0TYPE"] diff --git a/imap_processing/idex/idex_l1b.py b/imap_processing/idex/idex_l1b.py index a11cfb9ecb..4faddd7f0f 100644 --- a/imap_processing/idex/idex_l1b.py +++ b/imap_processing/idex/idex_l1b.py @@ -117,14 +117,7 @@ def idex_l1b(l1a_dataset: xr.Dataset) -> xr.Dataset: # used for calculations yet but are saved in the CDF for reference. spice_data = get_spice_data(l1a_dataset, idex_attrs) - trigger_settings = get_trigger_mode_and_level(l1a_dataset) - if trigger_settings: - trigger_settings["triggerlevel"].attrs = idex_attrs.get_variable_attributes( - "trigger_level" - ) - trigger_settings["triggermode"].attrs = idex_attrs.get_variable_attributes( - "trigger_mode" - ) + trigger_settings = get_trigger_mode_and_level(l1a_dataset, idex_attrs) # Create l1b Dataset prefixes = ["shcoarse", "shfine", "time_high_sample", "time_low_sample"] @@ -225,6 +218,7 @@ def convert_waveforms( def get_trigger_mode_and_level( l1a_dataset: xr.Dataset, + idex_attrs: ImapCdfAttributes, ) -> dict[str, xr.DataArray] | dict: """ Determine the trigger mode and threshold level for each event. @@ -233,6 +227,8 @@ def get_trigger_mode_and_level( ---------- l1a_dataset : xarray.Dataset IDEX L1a dataset containing the six waveform arrays and instrument settings. + idex_attrs : ImapCdfAttributes + CDF attribute manager object. Returns ------- @@ -243,8 +239,8 @@ def get_trigger_mode_and_level( channels = ["lg", "mg", "hg"] # 10 bit mask mask = 0b1111111111 - trigger_modes = [] - trigger_levels = [] + # Initialize a dict to hold the mode labels and threshold levels for each channel + data_dict = {} def compute_trigger_values( trigger_mode: int, trigger_controls: int, gain_channel: str @@ -302,28 +298,18 @@ def compute_trigger_values( vectorize=True, output_dtypes=[object, float], ) - trigger_modes.append(mode_array.rename("trigger_mode")) - trigger_levels.append(level_array.rename("trigger_level")) - - try: # There should be an array of modes and threshold levels for each channel. - # At each index (event) only one of the three arrays should have a value that is - # not 'None' because each event can only be triggered by one channel. - # By merging the three arrays, we get value for each event. - merged_modes = xr.merge([trigger_modes[0], xr.merge(trigger_modes[1:])]) - merged_levels = xr.merge([trigger_levels[0], xr.merge(trigger_levels[1:])]) - - return { - "triggermode": merged_modes.trigger_mode, - "triggerlevel": merged_levels.trigger_level, - } - - except xr.MergeError as e: - raise ValueError( - f"Only one channel can trigger a dust event. Please make sure " - f"there is only one valid trigger value per event. This " - f"caused Merge Error: {e}" - ) from e + # write each of them out as separate variables because there may be + # multiple channels that can trigger an event. The trigger origin variable + # can be used to determine which channel(s) triggered the event. + mode_array.attrs = idex_attrs.get_variable_attributes(f"trigger_mode_{channel}") + data_dict[f"trigger_mode_{channel}"] = mode_array + level_array.attrs = idex_attrs.get_variable_attributes( + f"trigger_level_{channel}" + ) + data_dict[f"trigger_level_{channel}"] = level_array + + return data_dict def get_spice_data( diff --git a/imap_processing/tests/idex/conftest.py b/imap_processing/tests/idex/conftest.py index 86a199e30b..6fa7203e93 100644 --- a/imap_processing/tests/idex/conftest.py +++ b/imap_processing/tests/idex/conftest.py @@ -12,12 +12,12 @@ TEST_DATA_PATH = imap_module_directory / "tests" / "idex" / "test_data" -TEST_L0_FILE_SCI = TEST_DATA_PATH / "imap_idex_l0_raw_20231218_v001.pkts" +TEST_L0_FILE_SCI = TEST_DATA_PATH / "ois_output_12182023_185430.pkts" TEST_L0_FILE_EVT = TEST_DATA_PATH / "imap_idex_l0_raw_20250108_v001.pkts" # 1418 TEST_L0_FILE_CATLST = TEST_DATA_PATH / "imap_idex_l0_raw_20241206_v001.pkts" # 1419 L1A_EXAMPLE_FILE = TEST_DATA_PATH / "idex_l1a_validation_file.h5" -L1B_EXAMPLE_FILE = TEST_DATA_PATH / "idex_l1b_validation_file.h5" +L1B_EXAMPLE_FILE = TEST_DATA_PATH / "ois_output_12182023_184030.h5" L2A_CDF = TEST_DATA_PATH / "imap_idex_l2a_sci-1week_20251017_v001.cdf" L1B_EVT_CDF = TEST_DATA_PATH / "imap_idex_l1b_evt_20250108_v001.cdf" diff --git a/imap_processing/tests/idex/test_idex_l1b.py b/imap_processing/tests/idex/test_idex_l1b.py index 3cf7a76e22..e3b234dbff 100644 --- a/imap_processing/tests/idex/test_idex_l1b.py +++ b/imap_processing/tests/idex/test_idex_l1b.py @@ -15,6 +15,7 @@ get_trigger_mode_and_level, unpack_instrument_settings, ) +from imap_processing.idex.idex_utils import get_idex_attrs from imap_processing.tests.idex import conftest @@ -149,48 +150,45 @@ def test_get_trigger_settings_success(decom_test_data_sci): # correct when the modes and levels vary from event to event decom_test_data_sci["idx__txhdrmgtrigmode"][0] = 1 decom_test_data_sci["idx__txhdrhgtrigmode"][0] = 0 - + idex_attrs = get_idex_attrs("l1b") n_epochs = len(decom_test_data_sci["epoch"]) - trigger_settings = get_trigger_mode_and_level(decom_test_data_sci) - - expected_modes = np.full(n_epochs, "HGThreshold") - expected_modes[0] = "MGThreshold" - expected_levels = np.full(n_epochs, 0.16762) - expected_levels[0] = 1023.0 * 1.13e-2 - - assert (trigger_settings["triggermode"].data == expected_modes).all(), ( - f"The dict entry 'triggermode' values did not match the expected values: " - f"{expected_modes}. Found: {trigger_settings['triggermode'].data}" - ) - - assert (trigger_settings["triggerlevel"].data == expected_levels).all(), ( - f"The dict entry 'triggerlevel' values did not match the expected values: " - f"{expected_levels}. Found: {trigger_settings['triggerlevel'].data}" - ) - - -def test_get_trigger_settings_failure(decom_test_data_sci): - """ - Check that an error is thrown when there are more than one valid trigger for an - event - - Parameters - ---------- - decom_test_data_sci : xarray.Dataset - L1a dataset - """ - decom_test_data_sci["idx__txhdrhgtrigmode"][0] = 1 - decom_test_data_sci["idx__txhdrmgtrigmode"][0] = 2 - - error_ms = ( - "Only one channel can trigger a dust event. Please make sure there is " - "only one valid trigger value per event. This caused Merge Error: " - "conflicting values for variable 'trigger_mode' on objects to be " - "combined. You can skip this check by specifying compat='override'." - ) - - with pytest.raises(ValueError, match=error_ms): - get_trigger_mode_and_level(decom_test_data_sci) + trigger_settings = get_trigger_mode_and_level(decom_test_data_sci, idex_attrs) + + expected_modes_lg = np.full(n_epochs, None) + expected_modes_hg = expected_modes_lg.copy() + expected_modes_hg[1:] = "HGThreshold" + expected_modes_mg = np.full(n_epochs, None) + expected_modes_mg[0] = "MGThreshold" + expected_levels_lg = np.full(n_epochs, np.nan) + expected_levels_hg = expected_levels_lg.copy() + expected_levels_hg[1:] = 0.16762 + expected_levels_mg = expected_levels_lg.copy() + expected_levels_mg[0] = 1023.0 * 1.13e-2 + + var_names = ["trigger_mode_lg", "trigger_mode_mg", "trigger_mode_hg"] + expected_modes = [expected_modes_lg, expected_modes_mg, expected_modes_hg] + for expected_mode, mode_name in zip(expected_modes, var_names, strict=False): + ( + np.testing.assert_array_equal( + trigger_settings[mode_name].data, expected_mode + ), + ( + f"The dict entry {mode_name} values did not match the expected values: " + f"{expected_mode}. Found: {trigger_settings[mode_name].data}" + ), + ) + var_names = ["trigger_level_lg", "trigger_level_mg", "trigger_level_hg"] + expected_levels = [expected_levels_lg, expected_levels_mg, expected_levels_hg] + for expected_level, level_name in zip(expected_levels, var_names, strict=False): + ( + np.testing.assert_array_equal( + trigger_settings[level_name].data, expected_level + ), + ( + f"The dict entry {level_name} values did not match the expected " + f"values: {expected_level}. Found: {trigger_settings[level_name].data}" + ), + ) @pytest.mark.usefixtures("use_fake_spin_data_for_time") From 6be15f5f305433b6cb781dd4e954a907b7d18c77 Mon Sep 17 00:00:00 2001 From: Luisa Date: Thu, 26 Feb 2026 11:55:12 -0700 Subject: [PATCH 02/11] things working sort of --- imap_processing/idex/idex_l1a.py | 24 +++++++++++++++++++-- imap_processing/tests/idex/conftest.py | 3 ++- imap_processing/tests/idex/test_idex_l1b.py | 5 ++++- 3 files changed, 28 insertions(+), 4 deletions(-) diff --git a/imap_processing/idex/idex_l1a.py b/imap_processing/idex/idex_l1a.py index d09499b7b4..04121eecb7 100644 --- a/imap_processing/idex/idex_l1a.py +++ b/imap_processing/idex/idex_l1a.py @@ -464,8 +464,28 @@ def _set_sample_trigger_times( # 20 positions to the right, and the mask (0b1111111111) keeps only the least # significant 10 bits. # TODO use the delay corresponding to the trigger - high_gain_delay = (packet["IDX__TXHDRSAMPDELAY"] >> 22) & 0b1111111111 + # high_gain_delay = (packet["IDX__TXHDRSAMPDELAY"] >> 22) & 0b1111111111 n_blocks = packet["IDX__TXHDRBLOCKS"] + trigger_item = packet["IDX__TXHDRTRIGID"] + + # Account for HS trigger delay + tofdelay = packet["IDX__TXHDRSAMPDELAY"] # last two bits are padding + + # mask to extract 10-bit values + mask = 0b1111111111 + + hgdelay = (tofdelay) & mask # first 10 bits (0-9) + mgdelay = (tofdelay >> 10) & mask # next 10 bits (10-19) + lgdelay = (tofdelay >> 20) & mask # next 10 bits (20-29) + u10 = trigger_item & 0x3FF + if (u10 >> 0) & 1: + delay = hgdelay + elif (u10 >> 1) & 1: + delay = lgdelay + elif (u10 >> 2) & 1: + delay = mgdelay + else: + delay = hgdelay # Retrieve number of low/high sample pre-trigger blocks @@ -489,7 +509,7 @@ def _set_sample_trigger_times( self.HIGH_SAMPLE_RATE * (num_high_sample_pretrigger_blocks + 1) * self.NUMBER_SAMPLES_PER_HIGH_SAMPLE_BLOCK - - self.HIGH_SAMPLE_RATE * high_gain_delay + - self.HIGH_SAMPLE_RATE * delay ) def _parse_high_sample_waveform(self, waveform_raw: str) -> list[int]: diff --git a/imap_processing/tests/idex/conftest.py b/imap_processing/tests/idex/conftest.py index 6fa7203e93..fd973df8ef 100644 --- a/imap_processing/tests/idex/conftest.py +++ b/imap_processing/tests/idex/conftest.py @@ -12,12 +12,13 @@ TEST_DATA_PATH = imap_module_directory / "tests" / "idex" / "test_data" -TEST_L0_FILE_SCI = TEST_DATA_PATH / "ois_output_12182023_185430.pkts" +TEST_L0_FILE_SCI = TEST_DATA_PATH / "imap_idex_l0_raw_20231218_v001.pkts" TEST_L0_FILE_EVT = TEST_DATA_PATH / "imap_idex_l0_raw_20250108_v001.pkts" # 1418 TEST_L0_FILE_CATLST = TEST_DATA_PATH / "imap_idex_l0_raw_20241206_v001.pkts" # 1419 L1A_EXAMPLE_FILE = TEST_DATA_PATH / "idex_l1a_validation_file.h5" L1B_EXAMPLE_FILE = TEST_DATA_PATH / "ois_output_12182023_184030.h5" +# L1B_EXAMPLE_FILE = TEST_DATA_PATH / "idex_l1b_validation_file.h5" L2A_CDF = TEST_DATA_PATH / "imap_idex_l2a_sci-1week_20251017_v001.cdf" L1B_EVT_CDF = TEST_DATA_PATH / "imap_idex_l1b_evt_20250108_v001.cdf" diff --git a/imap_processing/tests/idex/test_idex_l1b.py b/imap_processing/tests/idex/test_idex_l1b.py index e3b234dbff..9934167e19 100644 --- a/imap_processing/tests/idex/test_idex_l1b.py +++ b/imap_processing/tests/idex/test_idex_l1b.py @@ -279,6 +279,10 @@ def test_validate_l1b_idex_data_variables( "VelocityZ", "RightAscension", ] + # select only the first n events + l1b_example_data = l1b_example_data.isel( + event=np.arange(l1b_dataset.sizes["epoch"]) + ) # Compare each corresponding variable for var in l1b_example_data.data_vars: if var not in arrays_to_skip: @@ -289,7 +293,6 @@ def test_validate_l1b_idex_data_variables( f"The array '{cdf_var}' does not equal the expected example array " ) f"'{var}' produced by the IDEX team" - if l1b_dataset[cdf_var].dtype == object: assert (l1b_dataset[cdf_var].data == l1b_example_data[var]).all(), ( warning From b593e3a338f54ba8aef1ad10f9fd44466db50935 Mon Sep 17 00:00:00 2001 From: Luisa Date: Thu, 5 Mar 2026 09:03:05 -0700 Subject: [PATCH 03/11] more investigation --- imap_processing/tests/idex/conftest.py | 2 +- imap_processing/tests/idex/test_idex_l1b.py | 35 +++++++++++++++------ 2 files changed, 27 insertions(+), 10 deletions(-) diff --git a/imap_processing/tests/idex/conftest.py b/imap_processing/tests/idex/conftest.py index fd973df8ef..24965beab5 100644 --- a/imap_processing/tests/idex/conftest.py +++ b/imap_processing/tests/idex/conftest.py @@ -18,7 +18,7 @@ L1A_EXAMPLE_FILE = TEST_DATA_PATH / "idex_l1a_validation_file.h5" L1B_EXAMPLE_FILE = TEST_DATA_PATH / "ois_output_12182023_184030.h5" -# L1B_EXAMPLE_FILE = TEST_DATA_PATH / "idex_l1b_validation_file.h5" +L1B_EXAMPLE_FILE2 = TEST_DATA_PATH / "idex_l1b_validation_file.h5" L2A_CDF = TEST_DATA_PATH / "imap_idex_l2a_sci-1week_20251017_v001.cdf" L1B_EVT_CDF = TEST_DATA_PATH / "imap_idex_l1b_evt_20250108_v001.cdf" diff --git a/imap_processing/tests/idex/test_idex_l1b.py b/imap_processing/tests/idex/test_idex_l1b.py index 9934167e19..15e0d54725 100644 --- a/imap_processing/tests/idex/test_idex_l1b.py +++ b/imap_processing/tests/idex/test_idex_l1b.py @@ -258,6 +258,9 @@ def test_validate_l1b_idex_data_variables( "voltage_3V3_op_ref": "voltage_3p3_op_ref", "voltage_3V3_ref": "voltage_3p3_ref", "voltage_pos3V3_bus": "voltage_pos3p3v_bus", + "HGTriggerLevel": "trigger_level_hg", + "MGTriggerLevel": "trigger_level_mg", + "LGTriggerLevel": "trigger_level_lg", } # The Engineering data is converting to UTC, and the SDC is converting to J2000, @@ -278,6 +281,12 @@ def test_validate_l1b_idex_data_variables( "VelocityY", "VelocityZ", "RightAscension", + # "FIFODelay", + # "FIFODelayMicroseconds", + # "FIFODelay_H", + # "FIFODelay_L", + # "FIFODelay_M", + # "HSPosttriggerBlocks" ] # select only the first n events l1b_example_data = l1b_example_data.isel( @@ -288,22 +297,30 @@ def test_validate_l1b_idex_data_variables( if var not in arrays_to_skip: # Get the corresponding array name cdf_var = match_variables.get(var, var.lower().replace(".", "p")) - warning = ( f"The array '{cdf_var}' does not equal the expected example array " ) f"'{var}' produced by the IDEX team" + try: + l1b_dataset[cdf_var] + except KeyError: + print(f"The variable '{cdf_var}' was not found in the dataset.") + continue if l1b_dataset[cdf_var].dtype == object: assert (l1b_dataset[cdf_var].data == l1b_example_data[var]).all(), ( warning ) else: - ( - np.testing.assert_array_almost_equal( - l1b_dataset[cdf_var].data, - l1b_example_data[var], - decimal=4, - ), - warning, - ) + try: + ( + np.testing.assert_array_almost_equal( + l1b_dataset[cdf_var].data, + l1b_example_data[var], + decimal=4, + ), + warning, + ) + print("variable: ", var, " DID match !!") + except AssertionError: + print("variable: ", var, "did not match") From 008713ba5117f165669044cb78109635c6f47d4e Mon Sep 17 00:00:00 2001 From: Luisa Date: Thu, 5 Mar 2026 15:49:25 -0700 Subject: [PATCH 04/11] triggertime --- imap_processing/idex/idex_l1a.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/imap_processing/idex/idex_l1a.py b/imap_processing/idex/idex_l1a.py index 04121eecb7..6ed8d7eba4 100644 --- a/imap_processing/idex/idex_l1a.py +++ b/imap_processing/idex/idex_l1a.py @@ -505,11 +505,10 @@ def _set_sample_trigger_times( * (num_low_sample_pretrigger_blocks + 1) * self.NUMBER_SAMPLES_PER_LOW_SAMPLE_BLOCK ) - self.high_sample_trigger_time = ( - self.HIGH_SAMPLE_RATE - * (num_high_sample_pretrigger_blocks + 1) - * self.NUMBER_SAMPLES_PER_HIGH_SAMPLE_BLOCK - - self.HIGH_SAMPLE_RATE * delay + self.high_sample_trigger_time = self.HIGH_SAMPLE_RATE * ( + num_high_sample_pretrigger_blocks + 1 + ) * self.NUMBER_SAMPLES_PER_HIGH_SAMPLE_BLOCK - self.HIGH_SAMPLE_RATE * ( + delay - 1 ) def _parse_high_sample_waveform(self, waveform_raw: str) -> list[int]: From a0e04603a9a3cded16c5c6adb0845b64846c3b83 Mon Sep 17 00:00:00 2001 From: Luisa Date: Thu, 5 Mar 2026 15:53:49 -0700 Subject: [PATCH 05/11] matchign --- imap_processing/idex/idex_l1a.py | 4 +-- imap_processing/tests/idex/conftest.py | 17 ++++++++++++- imap_processing/tests/idex/test_idex_l1b.py | 27 +++++++++------------ 3 files changed, 30 insertions(+), 18 deletions(-) diff --git a/imap_processing/idex/idex_l1a.py b/imap_processing/idex/idex_l1a.py index 6ed8d7eba4..fc84043cca 100644 --- a/imap_processing/idex/idex_l1a.py +++ b/imap_processing/idex/idex_l1a.py @@ -582,7 +582,7 @@ def _calc_low_sample_resolution(self, num_samples: int) -> npt.NDArray: time_low_sample_rate_data : numpy.ndarray Low time sample data array. """ - time_low_sample_rate_init = np.linspace(0, num_samples, num_samples) + time_low_sample_rate_init = np.arange(num_samples, dtype=np.float64) time_low_sample_rate_data = ( self.LOW_SAMPLE_RATE * time_low_sample_rate_init - self.low_sample_trigger_time @@ -609,7 +609,7 @@ def _calc_high_sample_resolution(self, num_samples: int) -> npt.NDArray: time_high_sample_rate_data : numpy.ndarray High sample time data array. """ - time_high_sample_rate_init = np.linspace(0, num_samples, num_samples) + time_high_sample_rate_init = np.arange(num_samples, dtype=np.float64) time_high_sample_rate_data = ( self.HIGH_SAMPLE_RATE * time_high_sample_rate_init - self.high_sample_trigger_time diff --git a/imap_processing/tests/idex/conftest.py b/imap_processing/tests/idex/conftest.py index 24965beab5..19ba94f1c0 100644 --- a/imap_processing/tests/idex/conftest.py +++ b/imap_processing/tests/idex/conftest.py @@ -17,7 +17,9 @@ TEST_L0_FILE_CATLST = TEST_DATA_PATH / "imap_idex_l0_raw_20241206_v001.pkts" # 1419 L1A_EXAMPLE_FILE = TEST_DATA_PATH / "idex_l1a_validation_file.h5" -L1B_EXAMPLE_FILE = TEST_DATA_PATH / "ois_output_12182023_184030.h5" +L1B_EXAMPLE_FILE = ( + TEST_DATA_PATH / "imap_idex_l0_raw_20231218_v001.h5" +) # "ois_output_12182023_184030.h5" L1B_EXAMPLE_FILE2 = TEST_DATA_PATH / "idex_l1b_validation_file.h5" L2A_CDF = TEST_DATA_PATH / "imap_idex_l2a_sci-1week_20251017_v001.cdf" @@ -101,6 +103,19 @@ def l1b_example_data(_download_test_data): return load_hdf_file(L1B_EXAMPLE_FILE) +@pytest.fixture +def l1b_example_data_2(_download_test_data): + """ + Pytest fixture to load example L1B data (produced by the IDEX team) for testing. + + Returns + ------- + dict + A dictionary containing the 6 waveform and telemetry arrays + """ + return load_hdf_file(L1B_EXAMPLE_FILE2) + + @pytest.fixture @mock.patch("imap_processing.idex.idex_l1b.get_spice_data") def l1b_dataset(mock_get_spice_data, decom_test_data_sci: xr.Dataset) -> xr.Dataset: diff --git a/imap_processing/tests/idex/test_idex_l1b.py b/imap_processing/tests/idex/test_idex_l1b.py index 15e0d54725..546302ac57 100644 --- a/imap_processing/tests/idex/test_idex_l1b.py +++ b/imap_processing/tests/idex/test_idex_l1b.py @@ -228,7 +228,7 @@ def test_get_spice_data( @pytest.mark.external_test_data def test_validate_l1b_idex_data_variables( - l1b_dataset: xr.Dataset, l1b_example_data: xr.Dataset + l1b_dataset: xr.Dataset, l1b_example_data: xr.Dataset, l1b_example_data_2 ): """ Verify that each of the 6 waveform and telemetry arrays are equal to the @@ -281,12 +281,12 @@ def test_validate_l1b_idex_data_variables( "VelocityY", "VelocityZ", "RightAscension", - # "FIFODelay", - # "FIFODelayMicroseconds", - # "FIFODelay_H", - # "FIFODelay_L", - # "FIFODelay_M", - # "HSPosttriggerBlocks" + "FIFODelay", + "FIFODelayMicroseconds", + "FIFODelay_H", + "FIFODelay_L", + "FIFODelay_M", + "HSPosttriggerBlocks", ] # select only the first n events l1b_example_data = l1b_example_data.isel( @@ -313,14 +313,11 @@ def test_validate_l1b_idex_data_variables( else: try: - ( - np.testing.assert_array_almost_equal( - l1b_dataset[cdf_var].data, - l1b_example_data[var], - decimal=4, - ), - warning, + np.testing.assert_array_almost_equal( + l1b_dataset[cdf_var].data, + np.squeeze(l1b_example_data[var]), + decimal=4, ) print("variable: ", var, " DID match !!") except AssertionError: - print("variable: ", var, "did not match") + print(f"variable: {var} did not match") From 97872531d1b5b0c56fa73a88e9e70d67c548c8f6 Mon Sep 17 00:00:00 2001 From: Luisa Date: Tue, 10 Mar 2026 09:49:41 -0600 Subject: [PATCH 06/11] idex l1b trigger mode/origin/level fix --- .../config/imap_idex_l1b_variable_attrs.yaml | 28 ++++--- imap_processing/idex/idex_l1a.py | 8 +- imap_processing/idex/idex_l1b.py | 79 ++++++++++++++++++- imap_processing/tests/idex/conftest.py | 18 +---- imap_processing/tests/idex/test_idex_l1b.py | 29 ++++--- 5 files changed, 110 insertions(+), 52 deletions(-) diff --git a/imap_processing/cdf/config/imap_idex_l1b_variable_attrs.yaml b/imap_processing/cdf/config/imap_idex_l1b_variable_attrs.yaml index 1b1492548a..128159abd7 100644 --- a/imap_processing/cdf/config/imap_idex_l1b_variable_attrs.yaml +++ b/imap_processing/cdf/config/imap_idex_l1b_variable_attrs.yaml @@ -55,35 +55,39 @@ spice_base: &spice_base # <=== Instrument Setting Attributes ===> trigger_mode_lg: <<: *string_base - FIELDNAM: Trigger Mode - CATDESC: Channel and mode that triggered the event + FIELDNAM: LG Trigger Mode + CATDESC: LG Trigger Mode. trigger_level_lg: <<: *trigger_base - FIELDNAM: Trigger Level - CATDESC: Threshold signal level that triggered the event + FIELDNAM: LG Trigger Level + CATDESC: LG Trigger Level threshold. trigger_mode_mg: <<: *string_base - FIELDNAM: Trigger Mode - CATDESC: Channel and mode that triggered the event + FIELDNAM: MG Trigger Mode + CATDESC: MG Trigger Mode. trigger_level_mg: <<: *trigger_base - FIELDNAM: Trigger Level - CATDESC: Threshold signal level that triggered the event + FIELDNAM: MG Trigger Level + CATDESC: MG Trigger level threshold. trigger_mode_hg: <<: *string_base - FIELDNAM: Trigger Mode - CATDESC: Channel and mode that triggered the event + FIELDNAM: HG Trigger Mode + CATDESC: HG Trigger Mode. trigger_level_hg: <<: *trigger_base - FIELDNAM: Trigger Level - CATDESC: Threshold signal level that triggered the event + FIELDNAM: HG Trigger Level + CATDESC: HG Trigger Level threshold. +trigger_origin: + <<: *string_base + FIELDNAM: Trigger Origin + CATDESC: Trigger Origin of the event. tof_high: <<: *l1b_tof_base diff --git a/imap_processing/idex/idex_l1a.py b/imap_processing/idex/idex_l1a.py index fc84043cca..992643a647 100644 --- a/imap_processing/idex/idex_l1a.py +++ b/imap_processing/idex/idex_l1a.py @@ -463,20 +463,20 @@ def _set_sample_trigger_times( # To extract the high gain bits, the bitwise right shift (>> 20) moves the bits # 20 positions to the right, and the mask (0b1111111111) keeps only the least # significant 10 bits. - # TODO use the delay corresponding to the trigger - # high_gain_delay = (packet["IDX__TXHDRSAMPDELAY"] >> 22) & 0b1111111111 + n_blocks = packet["IDX__TXHDRBLOCKS"] trigger_item = packet["IDX__TXHDRTRIGID"] - # Account for HS trigger delay tofdelay = packet["IDX__TXHDRSAMPDELAY"] # last two bits are padding # mask to extract 10-bit values mask = 0b1111111111 - hgdelay = (tofdelay) & mask # first 10 bits (0-9) + # Determine the delay based on the trigger id. + hgdelay = tofdelay & mask # first 10 bits (0-9) mgdelay = (tofdelay >> 10) & mask # next 10 bits (10-19) lgdelay = (tofdelay >> 20) & mask # next 10 bits (20-29) + u10 = trigger_item & 0x3FF if (u10 >> 0) & 1: delay = hgdelay diff --git a/imap_processing/idex/idex_l1b.py b/imap_processing/idex/idex_l1b.py index 4faddd7f0f..f2153755d3 100644 --- a/imap_processing/idex/idex_l1b.py +++ b/imap_processing/idex/idex_l1b.py @@ -15,10 +15,13 @@ """ import logging -from enum import Enum +from enum import Enum, IntEnum +import numpy as np import pandas as pd import xarray as xr +from numpy._typing import NDArray +from xarray import DataArray from imap_processing import imap_module_directory from imap_processing.cdf.imap_cdf_manager import ImapCdfAttributes @@ -42,6 +45,27 @@ logger = logging.getLogger(__name__) +class TriggerOrigin(IntEnum): + """Enum class for event trigger origins.""" + + HS_ADC0I_TOF_HG = 0 + HS_ADC0Q_TOF_LG = 1 + HS_ADC1Q_TOF_MG = 2 + LS_ADC1_TARGET_HG = 3 + SW_TRIGGER = 4 + EXTERNAL_TRIGGER = 5 + + +TRIGGER_LABELS = { + TriggerOrigin.HS_ADC0I_TOF_HG: "HS ADC0I trigger (TOF HG)", + TriggerOrigin.HS_ADC0Q_TOF_LG: "HS ADC0Q trigger (TOF LG)", + TriggerOrigin.HS_ADC1Q_TOF_MG: "HS ADC1Q trigger (TOF MG)", + TriggerOrigin.LS_ADC1_TARGET_HG: "LS ADC1 trigger (Target HG / low range)", + TriggerOrigin.SW_TRIGGER: "SW trigger", + TriggerOrigin.EXTERNAL_TRIGGER: "external trigger", +} + + class TriggerMode(Enum): """ Enum class for data collection trigger Modes. @@ -118,13 +142,20 @@ def idex_l1b(l1a_dataset: xr.Dataset) -> xr.Dataset: spice_data = get_spice_data(l1a_dataset, idex_attrs) trigger_settings = get_trigger_mode_and_level(l1a_dataset, idex_attrs) - + trigger_origin = get_trigger_origin( + l1a_dataset["idx__txhdrtrigid"].data, idex_attrs + ) # Create l1b Dataset prefixes = ["shcoarse", "shfine", "time_high_sample", "time_low_sample"] - data_vars = processed_vars | waveforms_converted | trigger_settings | spice_data + data_vars = ( + processed_vars + | waveforms_converted + | trigger_settings + | spice_data + | trigger_origin + ) l1b_dataset = setup_dataset(l1a_dataset, prefixes, idex_attrs, data_vars) l1b_dataset.attrs = idex_attrs.get_global_attributes("imap_idex_l1b_sci") - # Convert variables l1b_dataset = convert_raw_to_eu( l1b_dataset, @@ -312,6 +343,46 @@ def compute_trigger_values( return data_dict +def get_trigger_origin( + trigger_id: NDArray, idex_attrs: ImapCdfAttributes +) -> dict[str, DataArray]: + """ + Determine the trigger origin for each event. + + Parameters + ---------- + trigger_id : NDArray + Array of raw trigger ID values from the l1a dataset. The trigger ID is a 32-bit + integer where the lower 10 bits contain information about the trigger origin. + idex_attrs : ImapCdfAttributes + CDF attribute manager object. + + Returns + ------- + xr.DataArray + An array containing the trigger origin for each event. + """ + # extract the lower 10 bits of the trigger ID to get the trigger origin information + trigger_bits = trigger_id & 0x3FF + # For each event, determine which bits are set and get the corresponding trigger + # origin labels + origin_labels = np.array( + [ + [TRIGGER_LABELS[TriggerOrigin(i)] for i in range(6) if (bits >> i) & 1] + for bits in trigger_bits + ], + dtype=object, + ) + return { + "trigger_origin": xr.DataArray( + name="trigger_origin", + data=np.squeeze(origin_labels), + dims="epoch", + attrs=idex_attrs.get_variable_attributes("trigger_origin"), + ) + } + + def get_spice_data( l1a_dataset: xr.Dataset, idex_attrs: ImapCdfAttributes ) -> dict[str, xr.DataArray]: diff --git a/imap_processing/tests/idex/conftest.py b/imap_processing/tests/idex/conftest.py index 19ba94f1c0..35bee8c542 100644 --- a/imap_processing/tests/idex/conftest.py +++ b/imap_processing/tests/idex/conftest.py @@ -17,10 +17,7 @@ TEST_L0_FILE_CATLST = TEST_DATA_PATH / "imap_idex_l0_raw_20241206_v001.pkts" # 1419 L1A_EXAMPLE_FILE = TEST_DATA_PATH / "idex_l1a_validation_file.h5" -L1B_EXAMPLE_FILE = ( - TEST_DATA_PATH / "imap_idex_l0_raw_20231218_v001.h5" -) # "ois_output_12182023_184030.h5" -L1B_EXAMPLE_FILE2 = TEST_DATA_PATH / "idex_l1b_validation_file.h5" +L1B_EXAMPLE_FILE = TEST_DATA_PATH / "imap_idex_l1b_sci_20231218_v001.h5" L2A_CDF = TEST_DATA_PATH / "imap_idex_l2a_sci-1week_20251017_v001.cdf" L1B_EVT_CDF = TEST_DATA_PATH / "imap_idex_l1b_evt_20250108_v001.cdf" @@ -103,19 +100,6 @@ def l1b_example_data(_download_test_data): return load_hdf_file(L1B_EXAMPLE_FILE) -@pytest.fixture -def l1b_example_data_2(_download_test_data): - """ - Pytest fixture to load example L1B data (produced by the IDEX team) for testing. - - Returns - ------- - dict - A dictionary containing the 6 waveform and telemetry arrays - """ - return load_hdf_file(L1B_EXAMPLE_FILE2) - - @pytest.fixture @mock.patch("imap_processing.idex.idex_l1b.get_spice_data") def l1b_dataset(mock_get_spice_data, decom_test_data_sci: xr.Dataset) -> xr.Dataset: diff --git a/imap_processing/tests/idex/test_idex_l1b.py b/imap_processing/tests/idex/test_idex_l1b.py index 546302ac57..d6d4dbd055 100644 --- a/imap_processing/tests/idex/test_idex_l1b.py +++ b/imap_processing/tests/idex/test_idex_l1b.py @@ -228,7 +228,7 @@ def test_get_spice_data( @pytest.mark.external_test_data def test_validate_l1b_idex_data_variables( - l1b_dataset: xr.Dataset, l1b_example_data: xr.Dataset, l1b_example_data_2 + l1b_dataset: xr.Dataset, l1b_example_data: xr.Dataset ): """ Verify that each of the 6 waveform and telemetry arrays are equal to the @@ -261,6 +261,7 @@ def test_validate_l1b_idex_data_variables( "HGTriggerLevel": "trigger_level_hg", "MGTriggerLevel": "trigger_level_mg", "LGTriggerLevel": "trigger_level_lg", + "TriggerOrigin": "trigger_origin", } # The Engineering data is converting to UTC, and the SDC is converting to J2000, @@ -269,7 +270,7 @@ def test_validate_l1b_idex_data_variables( # SPICE data is mocked. arrays_to_skip = [ "Timestamp", - "Epoch", + "epoch", "Pitch", "Roll", "Yaw", @@ -301,23 +302,21 @@ def test_validate_l1b_idex_data_variables( f"The array '{cdf_var}' does not equal the expected example array " ) f"'{var}' produced by the IDEX team" + # TODO remove this block once the IDEX team fixes the l1b validation file. + # They included a lot of extra variables in the current file. try: l1b_dataset[cdf_var] except KeyError: - print(f"The variable '{cdf_var}' was not found in the dataset.") continue if l1b_dataset[cdf_var].dtype == object: - assert (l1b_dataset[cdf_var].data == l1b_example_data[var]).all(), ( - warning - ) + assert ( + l1b_dataset[cdf_var].data == np.squeeze(l1b_example_data[var]) + ).all(), warning else: - try: - np.testing.assert_array_almost_equal( - l1b_dataset[cdf_var].data, - np.squeeze(l1b_example_data[var]), - decimal=4, - ) - print("variable: ", var, " DID match !!") - except AssertionError: - print(f"variable: {var} did not match") + np.testing.assert_array_almost_equal( + l1b_dataset[cdf_var].data, + np.squeeze(l1b_example_data[var]), + decimal=4, + err_msg=warning, + ) From 9f99391cdc05cc8ca4f49bbc20858cb2f3547795 Mon Sep 17 00:00:00 2001 From: Luisa Date: Tue, 10 Mar 2026 09:51:07 -0600 Subject: [PATCH 07/11] remove code --- imap_processing/idex/idex_l1a.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/imap_processing/idex/idex_l1a.py b/imap_processing/idex/idex_l1a.py index 992643a647..b255a4cbc3 100644 --- a/imap_processing/idex/idex_l1a.py +++ b/imap_processing/idex/idex_l1a.py @@ -130,13 +130,6 @@ def _create_science_dataset(self, science_decom_packet_list: list) -> xr.Dataset Dataset containing processed dust events. """ dust_events = {} - # To handle duplicate packets received from the POC, calculate a unique ID - # for each packet and find the duplicate indexes. Then, only process the unique - # packets to create the dataset. - # packet_ids = [ - # (p["IDX__SCI0TYPE"], p["IDX__SCI0EVTNUM"]) - # for p in science_decom_packet_list - # ] for packet in science_decom_packet_list: if "IDX__SCI0TYPE" in packet: scitype = packet["IDX__SCI0TYPE"] From d5c9e2eb431c509133d76266344d33ed50999af6 Mon Sep 17 00:00:00 2001 From: Luisa Date: Tue, 10 Mar 2026 09:54:55 -0600 Subject: [PATCH 08/11] add new validation file to the external data download config --- imap_processing/tests/external_test_data_config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/imap_processing/tests/external_test_data_config.py b/imap_processing/tests/external_test_data_config.py index ec98531234..6d8a2a6f6c 100644 --- a/imap_processing/tests/external_test_data_config.py +++ b/imap_processing/tests/external_test_data_config.py @@ -132,7 +132,7 @@ # IDEX ("idex_l1a_validation_file.h5", "idex/test_data/"), - ("idex_l1b_validation_file.h5", "idex/test_data/"), + ("imap_idex_l1b_sci_20231218_v001.h5", "idex/test_data/"), ("imap_idex_l2a-calibration-curve-yield-params_20250101_v001.csv", "idex/test_data/"), ("imap_idex_l2a-calibration-curve-t-rise_20250101_v001.csv", "idex/test_data/"), From ebfd41377783126715514849c1c1d081e50ac14b Mon Sep 17 00:00:00 2001 From: Luisa Date: Tue, 10 Mar 2026 09:59:02 -0600 Subject: [PATCH 09/11] trigger origin update --- imap_processing/idex/idex_l1b.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/imap_processing/idex/idex_l1b.py b/imap_processing/idex/idex_l1b.py index f2153755d3..a30e88f160 100644 --- a/imap_processing/idex/idex_l1b.py +++ b/imap_processing/idex/idex_l1b.py @@ -351,7 +351,7 @@ def get_trigger_origin( Parameters ---------- - trigger_id : NDArray + trigger_id : numpy.ndarray Array of raw trigger ID values from the l1a dataset. The trigger ID is a 32-bit integer where the lower 10 bits contain information about the trigger origin. idex_attrs : ImapCdfAttributes @@ -359,7 +359,7 @@ def get_trigger_origin( Returns ------- - xr.DataArray + xarray.DataArray An array containing the trigger origin for each event. """ # extract the lower 10 bits of the trigger ID to get the trigger origin information From dd135d954debd420e361022eaf23db6d3714c7ff Mon Sep 17 00:00:00 2001 From: Luisa Date: Tue, 10 Mar 2026 13:12:14 -0600 Subject: [PATCH 10/11] update code --- docs/source/conf.py | 1 + imap_processing/idex/idex_l1a.py | 12 +++-- imap_processing/idex/idex_l1b.py | 13 +++-- imap_processing/tests/idex/test_idex_l1b.py | 58 +++++++++++++++++---- 4 files changed, 64 insertions(+), 20 deletions(-) diff --git a/docs/source/conf.py b/docs/source/conf.py index 181cd50524..5c67d9fa39 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -122,6 +122,7 @@ (r"py:.*", r".*np.ndarray.*"), (r"py:.*", r".*numpy._typing._array_like._ScalarType_co.*"), (r"py:.*", r".*idex.l1a.TRIGGER_DESCRIPTION.*"), + (r"py:.*", r".*idex.l1b.TriggerOrigin.*"), (r"py:.*", r".*idex.l2a.BaselineNoiseTime.*"), (r"py:.*", r".*PacketProperties"), (r"py:.*", r".*.spice.geometry.SpiceBody.*"), diff --git a/imap_processing/idex/idex_l1a.py b/imap_processing/idex/idex_l1a.py index b255a4cbc3..29663f2fb1 100644 --- a/imap_processing/idex/idex_l1a.py +++ b/imap_processing/idex/idex_l1a.py @@ -451,11 +451,13 @@ def _set_sample_trigger_times( """ # Retrieve the number of samples for high gain delay - # packet['IDX__TXHDRSAMPDELAY'] is a 32-bit value, with the last 10 bits - # representing the high gain sample delay and the first 2 bits used for padding. - # To extract the high gain bits, the bitwise right shift (>> 20) moves the bits - # 20 positions to the right, and the mask (0b1111111111) keeps only the least - # significant 10 bits. + # packet['IDX__TXHDRSAMPDELAY'] is a 32-bit value: + # bits0-9: high-gain delay, + # bits10-19: mid-gain delay, + # bits20-29: low-gain delay. + # bits30-31 are padding/reserved. + # Each delay is extracted by right-shifting to align the field, + # then masking with #0b1111111111 (10 bits). n_blocks = packet["IDX__TXHDRBLOCKS"] trigger_item = packet["IDX__TXHDRTRIGID"] diff --git a/imap_processing/idex/idex_l1b.py b/imap_processing/idex/idex_l1b.py index a30e88f160..5824d134a1 100644 --- a/imap_processing/idex/idex_l1b.py +++ b/imap_processing/idex/idex_l1b.py @@ -20,7 +20,7 @@ import numpy as np import pandas as pd import xarray as xr -from numpy._typing import NDArray +from numpy.typing import NDArray from xarray import DataArray from imap_processing import imap_module_directory @@ -359,8 +359,9 @@ def get_trigger_origin( Returns ------- - xarray.DataArray - An array containing the trigger origin for each event. + dict[str, xarray.DataArray] + A dictionary containing the trigger_origin DataArray with the trigger + origin info for each event. """ # extract the lower 10 bits of the trigger ID to get the trigger origin information trigger_bits = trigger_id & 0x3FF @@ -368,11 +369,15 @@ def get_trigger_origin( # origin labels origin_labels = np.array( [ - [TRIGGER_LABELS[TriggerOrigin(i)] for i in range(6) if (bits >> i) & 1] + ", ".join( + [TRIGGER_LABELS[TriggerOrigin(i)] for i in range(6) if (bits >> i) & 1] + ) for bits in trigger_bits ], dtype=object, ) + # Update any events with no trigger bits set to "unknown trigger origin" + origin_labels[origin_labels == ""] = "Unknown trigger origin" return { "trigger_origin": xr.DataArray( name="trigger_origin", diff --git a/imap_processing/tests/idex/test_idex_l1b.py b/imap_processing/tests/idex/test_idex_l1b.py index d6d4dbd055..f113b595a4 100644 --- a/imap_processing/tests/idex/test_idex_l1b.py +++ b/imap_processing/tests/idex/test_idex_l1b.py @@ -11,8 +11,11 @@ from imap_processing.cdf.imap_cdf_manager import ImapCdfAttributes from imap_processing.cdf.utils import write_cdf from imap_processing.idex.idex_l1b import ( + TRIGGER_LABELS, + TriggerOrigin, get_spice_data, get_trigger_mode_and_level, + get_trigger_origin, unpack_instrument_settings, ) from imap_processing.idex.idex_utils import get_idex_attrs @@ -170,11 +173,11 @@ def test_get_trigger_settings_success(decom_test_data_sci): for expected_mode, mode_name in zip(expected_modes, var_names, strict=False): ( np.testing.assert_array_equal( - trigger_settings[mode_name].data, expected_mode - ), - ( - f"The dict entry {mode_name} values did not match the expected values: " - f"{expected_mode}. Found: {trigger_settings[mode_name].data}" + trigger_settings[mode_name].data, + expected_mode, + err_msg=f"The dict entry {mode_name} values did not match the" + f" expected values: {expected_mode}. Found:" + f" {trigger_settings[mode_name].data}", ), ) var_names = ["trigger_level_lg", "trigger_level_mg", "trigger_level_hg"] @@ -182,15 +185,48 @@ def test_get_trigger_settings_success(decom_test_data_sci): for expected_level, level_name in zip(expected_levels, var_names, strict=False): ( np.testing.assert_array_equal( - trigger_settings[level_name].data, expected_level - ), - ( - f"The dict entry {level_name} values did not match the expected " - f"values: {expected_level}. Found: {trigger_settings[level_name].data}" + trigger_settings[level_name].data, + expected_level, + err_msg=f"The dic entry {level_name} values did not match the" + f" expected values: {expected_level}. Found: " + f"{trigger_settings[level_name].data}", ), ) +def test_trigger_origin(): + """Check that the correct labels are produced for trigger origin values""" + + trigger_bits = np.full(10, 6) + origins = get_trigger_origin(trigger_bits, get_idex_attrs("l1b")) + # Bits 1 and 2 should be set for all events + expected_origin = np.full( + 10, + ", ".join([TRIGGER_LABELS[TriggerOrigin(1)], TRIGGER_LABELS[TriggerOrigin(2)]]), + ) + np.testing.assert_array_equal( + origins["trigger_origin"], + expected_origin, + err_msg=f"The trigger origin values did not match the expected values: " + f"{expected_origin}. Found: {origins}", + ) + + +def test_invalid_trigger_origin(): + """Check the labels when there are invalid trigger origin values""" + + trigger_bits = np.full(10, 64) # invalid trigger origin values + origins = get_trigger_origin(trigger_bits, get_idex_attrs("l1b")) + # Bits 1 and 2 should be set for all events + expected_origin = np.full(10, "Unknown trigger origin") + np.testing.assert_array_equal( + origins["trigger_origin"], + expected_origin, + err_msg=f"The trigger origin values did not match the expected values:" + f"{expected_origin}. Found: {origins}", + ) + + @pytest.mark.usefixtures("use_fake_spin_data_for_time") def test_get_spice_data( mock_spice_functions, @@ -300,8 +336,8 @@ def test_validate_l1b_idex_data_variables( cdf_var = match_variables.get(var, var.lower().replace(".", "p")) warning = ( f"The array '{cdf_var}' does not equal the expected example array " + f"'{var}' produced by the IDEX team" ) - f"'{var}' produced by the IDEX team" # TODO remove this block once the IDEX team fixes the l1b validation file. # They included a lot of extra variables in the current file. try: From 76b817b402568a972cc2f4757d53689312726d44 Mon Sep 17 00:00:00 2001 From: Luisa Date: Tue, 10 Mar 2026 13:56:46 -0600 Subject: [PATCH 11/11] pr feedback fix comments --- .../config/imap_idex_l1b_variable_attrs.yaml | 24 +++++++++---------- imap_processing/idex/idex_l1a.py | 18 +++++++------- imap_processing/tests/idex/test_idex_l1a.py | 10 +++++++- 3 files changed, 30 insertions(+), 22 deletions(-) diff --git a/imap_processing/cdf/config/imap_idex_l1b_variable_attrs.yaml b/imap_processing/cdf/config/imap_idex_l1b_variable_attrs.yaml index 128159abd7..0db4a6b75a 100644 --- a/imap_processing/cdf/config/imap_idex_l1b_variable_attrs.yaml +++ b/imap_processing/cdf/config/imap_idex_l1b_variable_attrs.yaml @@ -55,34 +55,34 @@ spice_base: &spice_base # <=== Instrument Setting Attributes ===> trigger_mode_lg: <<: *string_base - FIELDNAM: LG Trigger Mode - CATDESC: LG Trigger Mode. + FIELDNAM: Low Gain Trigger Mode + CATDESC: Low Gain Trigger Mode. trigger_level_lg: <<: *trigger_base - FIELDNAM: LG Trigger Level - CATDESC: LG Trigger Level threshold. + FIELDNAM: Low Gain Trigger Level + CATDESC: Low Gain Trigger Level threshold. trigger_mode_mg: <<: *string_base - FIELDNAM: MG Trigger Mode - CATDESC: MG Trigger Mode. + FIELDNAM: Mid Gain Trigger Mode + CATDESC: Mid Gain Trigger Mode. trigger_level_mg: <<: *trigger_base - FIELDNAM: MG Trigger Level - CATDESC: MG Trigger level threshold. + FIELDNAM: Mid Gain Trigger Level + CATDESC: Mid Gain Trigger level threshold. trigger_mode_hg: <<: *string_base - FIELDNAM: HG Trigger Mode - CATDESC: HG Trigger Mode. + FIELDNAM: High Gain Trigger Mode + CATDESC: High Gain Trigger Mode. trigger_level_hg: <<: *trigger_base - FIELDNAM: HG Trigger Level - CATDESC: HG Trigger Level threshold. + FIELDNAM: High Trigger Level + CATDESC: High Trigger Level threshold. trigger_origin: <<: *string_base diff --git a/imap_processing/idex/idex_l1a.py b/imap_processing/idex/idex_l1a.py index 29663f2fb1..f19a325982 100644 --- a/imap_processing/idex/idex_l1a.py +++ b/imap_processing/idex/idex_l1a.py @@ -462,25 +462,25 @@ def _set_sample_trigger_times( n_blocks = packet["IDX__TXHDRBLOCKS"] trigger_item = packet["IDX__TXHDRTRIGID"] - tofdelay = packet["IDX__TXHDRSAMPDELAY"] # last two bits are padding + tof_delay = packet["IDX__TXHDRSAMPDELAY"] # last two bits are padding # mask to extract 10-bit values - mask = 0b1111111111 + tof_mask = 0b1111111111 # Determine the delay based on the trigger id. - hgdelay = tofdelay & mask # first 10 bits (0-9) - mgdelay = (tofdelay >> 10) & mask # next 10 bits (10-19) - lgdelay = (tofdelay >> 20) & mask # next 10 bits (20-29) + hg_delay = tof_delay & tof_mask # first 10 bits (0-9) + mg_delay = (tof_delay >> 10) & tof_mask # next 10 bits (10-19) + lg_delay = (tof_delay >> 20) & tof_mask # next 10 bits (20-29) u10 = trigger_item & 0x3FF if (u10 >> 0) & 1: - delay = hgdelay + delay = hg_delay elif (u10 >> 1) & 1: - delay = lgdelay + delay = lg_delay elif (u10 >> 2) & 1: - delay = mgdelay + delay = mg_delay else: - delay = hgdelay + delay = hg_delay # Retrieve number of low/high sample pre-trigger blocks diff --git a/imap_processing/tests/idex/test_idex_l1a.py b/imap_processing/tests/idex/test_idex_l1a.py index 6dfee849ff..b80e960a60 100644 --- a/imap_processing/tests/idex/test_idex_l1a.py +++ b/imap_processing/tests/idex/test_idex_l1a.py @@ -160,7 +160,15 @@ def test_validate_l1a_idex_data_variables( # The Engineering data is converting to UTC, and the SDC is converting to J2000, # for 'epoch' and 'Timestamp' so this test is using the raw time value 'SCHOARSE' to # validate time - arrays_to_skip = ["Timestamp", "Epoch", "event"] + # TODO remove the low and high time from this list after the IDEX team produces a + # new l1a h5 file. + arrays_to_skip = [ + "Timestamp", + "Epoch", + "event", + "Time (high sampling)", + "Time (low sampling)", + ] # loop through all keys from the l1a example dict for var in l1a_example_data.variables: