diff --git a/docs/source/conf.py b/docs/source/conf.py index 181cd50524..5c67d9fa39 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -122,6 +122,7 @@ (r"py:.*", r".*np.ndarray.*"), (r"py:.*", r".*numpy._typing._array_like._ScalarType_co.*"), (r"py:.*", r".*idex.l1a.TRIGGER_DESCRIPTION.*"), + (r"py:.*", r".*idex.l1b.TriggerOrigin.*"), (r"py:.*", r".*idex.l2a.BaselineNoiseTime.*"), (r"py:.*", r".*PacketProperties"), (r"py:.*", r".*.spice.geometry.SpiceBody.*"), diff --git a/imap_processing/cdf/config/imap_idex_l1b_variable_attrs.yaml b/imap_processing/cdf/config/imap_idex_l1b_variable_attrs.yaml index 7397b8ebe3..0db4a6b75a 100644 --- a/imap_processing/cdf/config/imap_idex_l1b_variable_attrs.yaml +++ b/imap_processing/cdf/config/imap_idex_l1b_variable_attrs.yaml @@ -53,15 +53,41 @@ spice_base: &spice_base VALIDMAX: *spice_data_max # <=== Instrument Setting Attributes ===> -trigger_mode: +trigger_mode_lg: <<: *string_base - FIELDNAM: Trigger Mode - CATDESC: Channel and mode that triggered the event + FIELDNAM: Low Gain Trigger Mode + CATDESC: Low Gain Trigger Mode. -trigger_level: +trigger_level_lg: <<: *trigger_base - FIELDNAM: Trigger Level - CATDESC: Threshold signal level that triggered the event + FIELDNAM: Low Gain Trigger Level + CATDESC: Low Gain Trigger Level threshold. + +trigger_mode_mg: + <<: *string_base + FIELDNAM: Mid Gain Trigger Mode + CATDESC: Mid Gain Trigger Mode. + +trigger_level_mg: + <<: *trigger_base + FIELDNAM: Mid Gain Trigger Level + CATDESC: Mid Gain Trigger level threshold. + + +trigger_mode_hg: + <<: *string_base + FIELDNAM: High Gain Trigger Mode + CATDESC: High Gain Trigger Mode. + +trigger_level_hg: + <<: *trigger_base + FIELDNAM: High Trigger Level + CATDESC: High Trigger Level threshold. + +trigger_origin: + <<: *string_base + FIELDNAM: Trigger Origin + CATDESC: Trigger Origin of the event. tof_high: <<: *l1b_tof_base diff --git a/imap_processing/idex/idex_l1a.py b/imap_processing/idex/idex_l1a.py index 66a41799d8..f19a325982 100644 --- a/imap_processing/idex/idex_l1a.py +++ b/imap_processing/idex/idex_l1a.py @@ -451,14 +451,36 @@ def _set_sample_trigger_times( """ # Retrieve the number of samples for high gain delay - # packet['IDX__TXHDRSAMPDELAY'] is a 32-bit value, with the last 10 bits - # representing the high gain sample delay and the first 2 bits used for padding. - # To extract the high gain bits, the bitwise right shift (>> 20) moves the bits - # 20 positions to the right, and the mask (0b1111111111) keeps only the least - # significant 10 bits. - # TODO use the delay corresponding to the trigger - high_gain_delay = (packet["IDX__TXHDRSAMPDELAY"] >> 22) & 0b1111111111 + # packet['IDX__TXHDRSAMPDELAY'] is a 32-bit value: + # bits0-9: high-gain delay, + # bits10-19: mid-gain delay, + # bits20-29: low-gain delay. + # bits30-31 are padding/reserved. + # Each delay is extracted by right-shifting to align the field, + # then masking with #0b1111111111 (10 bits). + n_blocks = packet["IDX__TXHDRBLOCKS"] + trigger_item = packet["IDX__TXHDRTRIGID"] + + tof_delay = packet["IDX__TXHDRSAMPDELAY"] # last two bits are padding + + # mask to extract 10-bit values + tof_mask = 0b1111111111 + + # Determine the delay based on the trigger id. + hg_delay = tof_delay & tof_mask # first 10 bits (0-9) + mg_delay = (tof_delay >> 10) & tof_mask # next 10 bits (10-19) + lg_delay = (tof_delay >> 20) & tof_mask # next 10 bits (20-29) + + u10 = trigger_item & 0x3FF + if (u10 >> 0) & 1: + delay = hg_delay + elif (u10 >> 1) & 1: + delay = lg_delay + elif (u10 >> 2) & 1: + delay = mg_delay + else: + delay = hg_delay # Retrieve number of low/high sample pre-trigger blocks @@ -478,11 +500,10 @@ def _set_sample_trigger_times( * (num_low_sample_pretrigger_blocks + 1) * self.NUMBER_SAMPLES_PER_LOW_SAMPLE_BLOCK ) - self.high_sample_trigger_time = ( - self.HIGH_SAMPLE_RATE - * (num_high_sample_pretrigger_blocks + 1) - * self.NUMBER_SAMPLES_PER_HIGH_SAMPLE_BLOCK - - self.HIGH_SAMPLE_RATE * high_gain_delay + self.high_sample_trigger_time = self.HIGH_SAMPLE_RATE * ( + num_high_sample_pretrigger_blocks + 1 + ) * self.NUMBER_SAMPLES_PER_HIGH_SAMPLE_BLOCK - self.HIGH_SAMPLE_RATE * ( + delay - 1 ) def _parse_high_sample_waveform(self, waveform_raw: str) -> list[int]: @@ -556,7 +577,7 @@ def _calc_low_sample_resolution(self, num_samples: int) -> npt.NDArray: time_low_sample_rate_data : numpy.ndarray Low time sample data array. """ - time_low_sample_rate_init = np.linspace(0, num_samples, num_samples) + time_low_sample_rate_init = np.arange(num_samples, dtype=np.float64) time_low_sample_rate_data = ( self.LOW_SAMPLE_RATE * time_low_sample_rate_init - self.low_sample_trigger_time @@ -583,7 +604,7 @@ def _calc_high_sample_resolution(self, num_samples: int) -> npt.NDArray: time_high_sample_rate_data : numpy.ndarray High sample time data array. """ - time_high_sample_rate_init = np.linspace(0, num_samples, num_samples) + time_high_sample_rate_init = np.arange(num_samples, dtype=np.float64) time_high_sample_rate_data = ( self.HIGH_SAMPLE_RATE * time_high_sample_rate_init - self.high_sample_trigger_time diff --git a/imap_processing/idex/idex_l1b.py b/imap_processing/idex/idex_l1b.py index a11cfb9ecb..5824d134a1 100644 --- a/imap_processing/idex/idex_l1b.py +++ b/imap_processing/idex/idex_l1b.py @@ -15,10 +15,13 @@ """ import logging -from enum import Enum +from enum import Enum, IntEnum +import numpy as np import pandas as pd import xarray as xr +from numpy.typing import NDArray +from xarray import DataArray from imap_processing import imap_module_directory from imap_processing.cdf.imap_cdf_manager import ImapCdfAttributes @@ -42,6 +45,27 @@ logger = logging.getLogger(__name__) +class TriggerOrigin(IntEnum): + """Enum class for event trigger origins.""" + + HS_ADC0I_TOF_HG = 0 + HS_ADC0Q_TOF_LG = 1 + HS_ADC1Q_TOF_MG = 2 + LS_ADC1_TARGET_HG = 3 + SW_TRIGGER = 4 + EXTERNAL_TRIGGER = 5 + + +TRIGGER_LABELS = { + TriggerOrigin.HS_ADC0I_TOF_HG: "HS ADC0I trigger (TOF HG)", + TriggerOrigin.HS_ADC0Q_TOF_LG: "HS ADC0Q trigger (TOF LG)", + TriggerOrigin.HS_ADC1Q_TOF_MG: "HS ADC1Q trigger (TOF MG)", + TriggerOrigin.LS_ADC1_TARGET_HG: "LS ADC1 trigger (Target HG / low range)", + TriggerOrigin.SW_TRIGGER: "SW trigger", + TriggerOrigin.EXTERNAL_TRIGGER: "external trigger", +} + + class TriggerMode(Enum): """ Enum class for data collection trigger Modes. @@ -117,21 +141,21 @@ def idex_l1b(l1a_dataset: xr.Dataset) -> xr.Dataset: # used for calculations yet but are saved in the CDF for reference. spice_data = get_spice_data(l1a_dataset, idex_attrs) - trigger_settings = get_trigger_mode_and_level(l1a_dataset) - if trigger_settings: - trigger_settings["triggerlevel"].attrs = idex_attrs.get_variable_attributes( - "trigger_level" - ) - trigger_settings["triggermode"].attrs = idex_attrs.get_variable_attributes( - "trigger_mode" - ) - + trigger_settings = get_trigger_mode_and_level(l1a_dataset, idex_attrs) + trigger_origin = get_trigger_origin( + l1a_dataset["idx__txhdrtrigid"].data, idex_attrs + ) # Create l1b Dataset prefixes = ["shcoarse", "shfine", "time_high_sample", "time_low_sample"] - data_vars = processed_vars | waveforms_converted | trigger_settings | spice_data + data_vars = ( + processed_vars + | waveforms_converted + | trigger_settings + | spice_data + | trigger_origin + ) l1b_dataset = setup_dataset(l1a_dataset, prefixes, idex_attrs, data_vars) l1b_dataset.attrs = idex_attrs.get_global_attributes("imap_idex_l1b_sci") - # Convert variables l1b_dataset = convert_raw_to_eu( l1b_dataset, @@ -225,6 +249,7 @@ def convert_waveforms( def get_trigger_mode_and_level( l1a_dataset: xr.Dataset, + idex_attrs: ImapCdfAttributes, ) -> dict[str, xr.DataArray] | dict: """ Determine the trigger mode and threshold level for each event. @@ -233,6 +258,8 @@ def get_trigger_mode_and_level( ---------- l1a_dataset : xarray.Dataset IDEX L1a dataset containing the six waveform arrays and instrument settings. + idex_attrs : ImapCdfAttributes + CDF attribute manager object. Returns ------- @@ -243,8 +270,8 @@ def get_trigger_mode_and_level( channels = ["lg", "mg", "hg"] # 10 bit mask mask = 0b1111111111 - trigger_modes = [] - trigger_levels = [] + # Initialize a dict to hold the mode labels and threshold levels for each channel + data_dict = {} def compute_trigger_values( trigger_mode: int, trigger_controls: int, gain_channel: str @@ -302,28 +329,63 @@ def compute_trigger_values( vectorize=True, output_dtypes=[object, float], ) - trigger_modes.append(mode_array.rename("trigger_mode")) - trigger_levels.append(level_array.rename("trigger_level")) - - try: # There should be an array of modes and threshold levels for each channel. - # At each index (event) only one of the three arrays should have a value that is - # not 'None' because each event can only be triggered by one channel. - # By merging the three arrays, we get value for each event. - merged_modes = xr.merge([trigger_modes[0], xr.merge(trigger_modes[1:])]) - merged_levels = xr.merge([trigger_levels[0], xr.merge(trigger_levels[1:])]) - - return { - "triggermode": merged_modes.trigger_mode, - "triggerlevel": merged_levels.trigger_level, - } - - except xr.MergeError as e: - raise ValueError( - f"Only one channel can trigger a dust event. Please make sure " - f"there is only one valid trigger value per event. This " - f"caused Merge Error: {e}" - ) from e + # write each of them out as separate variables because there may be + # multiple channels that can trigger an event. The trigger origin variable + # can be used to determine which channel(s) triggered the event. + mode_array.attrs = idex_attrs.get_variable_attributes(f"trigger_mode_{channel}") + data_dict[f"trigger_mode_{channel}"] = mode_array + level_array.attrs = idex_attrs.get_variable_attributes( + f"trigger_level_{channel}" + ) + data_dict[f"trigger_level_{channel}"] = level_array + + return data_dict + + +def get_trigger_origin( + trigger_id: NDArray, idex_attrs: ImapCdfAttributes +) -> dict[str, DataArray]: + """ + Determine the trigger origin for each event. + + Parameters + ---------- + trigger_id : numpy.ndarray + Array of raw trigger ID values from the l1a dataset. The trigger ID is a 32-bit + integer where the lower 10 bits contain information about the trigger origin. + idex_attrs : ImapCdfAttributes + CDF attribute manager object. + + Returns + ------- + dict[str, xarray.DataArray] + A dictionary containing the trigger_origin DataArray with the trigger + origin info for each event. + """ + # extract the lower 10 bits of the trigger ID to get the trigger origin information + trigger_bits = trigger_id & 0x3FF + # For each event, determine which bits are set and get the corresponding trigger + # origin labels + origin_labels = np.array( + [ + ", ".join( + [TRIGGER_LABELS[TriggerOrigin(i)] for i in range(6) if (bits >> i) & 1] + ) + for bits in trigger_bits + ], + dtype=object, + ) + # Update any events with no trigger bits set to "unknown trigger origin" + origin_labels[origin_labels == ""] = "Unknown trigger origin" + return { + "trigger_origin": xr.DataArray( + name="trigger_origin", + data=np.squeeze(origin_labels), + dims="epoch", + attrs=idex_attrs.get_variable_attributes("trigger_origin"), + ) + } def get_spice_data( diff --git a/imap_processing/tests/external_test_data_config.py b/imap_processing/tests/external_test_data_config.py index ec98531234..6d8a2a6f6c 100644 --- a/imap_processing/tests/external_test_data_config.py +++ b/imap_processing/tests/external_test_data_config.py @@ -132,7 +132,7 @@ # IDEX ("idex_l1a_validation_file.h5", "idex/test_data/"), - ("idex_l1b_validation_file.h5", "idex/test_data/"), + ("imap_idex_l1b_sci_20231218_v001.h5", "idex/test_data/"), ("imap_idex_l2a-calibration-curve-yield-params_20250101_v001.csv", "idex/test_data/"), ("imap_idex_l2a-calibration-curve-t-rise_20250101_v001.csv", "idex/test_data/"), diff --git a/imap_processing/tests/idex/conftest.py b/imap_processing/tests/idex/conftest.py index 86a199e30b..35bee8c542 100644 --- a/imap_processing/tests/idex/conftest.py +++ b/imap_processing/tests/idex/conftest.py @@ -17,7 +17,7 @@ TEST_L0_FILE_CATLST = TEST_DATA_PATH / "imap_idex_l0_raw_20241206_v001.pkts" # 1419 L1A_EXAMPLE_FILE = TEST_DATA_PATH / "idex_l1a_validation_file.h5" -L1B_EXAMPLE_FILE = TEST_DATA_PATH / "idex_l1b_validation_file.h5" +L1B_EXAMPLE_FILE = TEST_DATA_PATH / "imap_idex_l1b_sci_20231218_v001.h5" L2A_CDF = TEST_DATA_PATH / "imap_idex_l2a_sci-1week_20251017_v001.cdf" L1B_EVT_CDF = TEST_DATA_PATH / "imap_idex_l1b_evt_20250108_v001.cdf" diff --git a/imap_processing/tests/idex/test_idex_l1a.py b/imap_processing/tests/idex/test_idex_l1a.py index 6dfee849ff..b80e960a60 100644 --- a/imap_processing/tests/idex/test_idex_l1a.py +++ b/imap_processing/tests/idex/test_idex_l1a.py @@ -160,7 +160,15 @@ def test_validate_l1a_idex_data_variables( # The Engineering data is converting to UTC, and the SDC is converting to J2000, # for 'epoch' and 'Timestamp' so this test is using the raw time value 'SCHOARSE' to # validate time - arrays_to_skip = ["Timestamp", "Epoch", "event"] + # TODO remove the low and high time from this list after the IDEX team produces a + # new l1a h5 file. + arrays_to_skip = [ + "Timestamp", + "Epoch", + "event", + "Time (high sampling)", + "Time (low sampling)", + ] # loop through all keys from the l1a example dict for var in l1a_example_data.variables: diff --git a/imap_processing/tests/idex/test_idex_l1b.py b/imap_processing/tests/idex/test_idex_l1b.py index 3cf7a76e22..f113b595a4 100644 --- a/imap_processing/tests/idex/test_idex_l1b.py +++ b/imap_processing/tests/idex/test_idex_l1b.py @@ -11,10 +11,14 @@ from imap_processing.cdf.imap_cdf_manager import ImapCdfAttributes from imap_processing.cdf.utils import write_cdf from imap_processing.idex.idex_l1b import ( + TRIGGER_LABELS, + TriggerOrigin, get_spice_data, get_trigger_mode_and_level, + get_trigger_origin, unpack_instrument_settings, ) +from imap_processing.idex.idex_utils import get_idex_attrs from imap_processing.tests.idex import conftest @@ -149,49 +153,79 @@ def test_get_trigger_settings_success(decom_test_data_sci): # correct when the modes and levels vary from event to event decom_test_data_sci["idx__txhdrmgtrigmode"][0] = 1 decom_test_data_sci["idx__txhdrhgtrigmode"][0] = 0 - + idex_attrs = get_idex_attrs("l1b") n_epochs = len(decom_test_data_sci["epoch"]) - trigger_settings = get_trigger_mode_and_level(decom_test_data_sci) + trigger_settings = get_trigger_mode_and_level(decom_test_data_sci, idex_attrs) + + expected_modes_lg = np.full(n_epochs, None) + expected_modes_hg = expected_modes_lg.copy() + expected_modes_hg[1:] = "HGThreshold" + expected_modes_mg = np.full(n_epochs, None) + expected_modes_mg[0] = "MGThreshold" + expected_levels_lg = np.full(n_epochs, np.nan) + expected_levels_hg = expected_levels_lg.copy() + expected_levels_hg[1:] = 0.16762 + expected_levels_mg = expected_levels_lg.copy() + expected_levels_mg[0] = 1023.0 * 1.13e-2 + + var_names = ["trigger_mode_lg", "trigger_mode_mg", "trigger_mode_hg"] + expected_modes = [expected_modes_lg, expected_modes_mg, expected_modes_hg] + for expected_mode, mode_name in zip(expected_modes, var_names, strict=False): + ( + np.testing.assert_array_equal( + trigger_settings[mode_name].data, + expected_mode, + err_msg=f"The dict entry {mode_name} values did not match the" + f" expected values: {expected_mode}. Found:" + f" {trigger_settings[mode_name].data}", + ), + ) + var_names = ["trigger_level_lg", "trigger_level_mg", "trigger_level_hg"] + expected_levels = [expected_levels_lg, expected_levels_mg, expected_levels_hg] + for expected_level, level_name in zip(expected_levels, var_names, strict=False): + ( + np.testing.assert_array_equal( + trigger_settings[level_name].data, + expected_level, + err_msg=f"The dic entry {level_name} values did not match the" + f" expected values: {expected_level}. Found: " + f"{trigger_settings[level_name].data}", + ), + ) - expected_modes = np.full(n_epochs, "HGThreshold") - expected_modes[0] = "MGThreshold" - expected_levels = np.full(n_epochs, 0.16762) - expected_levels[0] = 1023.0 * 1.13e-2 - assert (trigger_settings["triggermode"].data == expected_modes).all(), ( - f"The dict entry 'triggermode' values did not match the expected values: " - f"{expected_modes}. Found: {trigger_settings['triggermode'].data}" - ) +def test_trigger_origin(): + """Check that the correct labels are produced for trigger origin values""" - assert (trigger_settings["triggerlevel"].data == expected_levels).all(), ( - f"The dict entry 'triggerlevel' values did not match the expected values: " - f"{expected_levels}. Found: {trigger_settings['triggerlevel'].data}" + trigger_bits = np.full(10, 6) + origins = get_trigger_origin(trigger_bits, get_idex_attrs("l1b")) + # Bits 1 and 2 should be set for all events + expected_origin = np.full( + 10, + ", ".join([TRIGGER_LABELS[TriggerOrigin(1)], TRIGGER_LABELS[TriggerOrigin(2)]]), + ) + np.testing.assert_array_equal( + origins["trigger_origin"], + expected_origin, + err_msg=f"The trigger origin values did not match the expected values: " + f"{expected_origin}. Found: {origins}", ) -def test_get_trigger_settings_failure(decom_test_data_sci): - """ - Check that an error is thrown when there are more than one valid trigger for an - event +def test_invalid_trigger_origin(): + """Check the labels when there are invalid trigger origin values""" - Parameters - ---------- - decom_test_data_sci : xarray.Dataset - L1a dataset - """ - decom_test_data_sci["idx__txhdrhgtrigmode"][0] = 1 - decom_test_data_sci["idx__txhdrmgtrigmode"][0] = 2 - - error_ms = ( - "Only one channel can trigger a dust event. Please make sure there is " - "only one valid trigger value per event. This caused Merge Error: " - "conflicting values for variable 'trigger_mode' on objects to be " - "combined. You can skip this check by specifying compat='override'." + trigger_bits = np.full(10, 64) # invalid trigger origin values + origins = get_trigger_origin(trigger_bits, get_idex_attrs("l1b")) + # Bits 1 and 2 should be set for all events + expected_origin = np.full(10, "Unknown trigger origin") + np.testing.assert_array_equal( + origins["trigger_origin"], + expected_origin, + err_msg=f"The trigger origin values did not match the expected values:" + f"{expected_origin}. Found: {origins}", ) - with pytest.raises(ValueError, match=error_ms): - get_trigger_mode_and_level(decom_test_data_sci) - @pytest.mark.usefixtures("use_fake_spin_data_for_time") def test_get_spice_data( @@ -260,6 +294,10 @@ def test_validate_l1b_idex_data_variables( "voltage_3V3_op_ref": "voltage_3p3_op_ref", "voltage_3V3_ref": "voltage_3p3_ref", "voltage_pos3V3_bus": "voltage_pos3p3v_bus", + "HGTriggerLevel": "trigger_level_hg", + "MGTriggerLevel": "trigger_level_mg", + "LGTriggerLevel": "trigger_level_lg", + "TriggerOrigin": "trigger_origin", } # The Engineering data is converting to UTC, and the SDC is converting to J2000, @@ -268,7 +306,7 @@ def test_validate_l1b_idex_data_variables( # SPICE data is mocked. arrays_to_skip = [ "Timestamp", - "Epoch", + "epoch", "Pitch", "Roll", "Yaw", @@ -280,29 +318,41 @@ def test_validate_l1b_idex_data_variables( "VelocityY", "VelocityZ", "RightAscension", + "FIFODelay", + "FIFODelayMicroseconds", + "FIFODelay_H", + "FIFODelay_L", + "FIFODelay_M", + "HSPosttriggerBlocks", ] + # select only the first n events + l1b_example_data = l1b_example_data.isel( + event=np.arange(l1b_dataset.sizes["epoch"]) + ) # Compare each corresponding variable for var in l1b_example_data.data_vars: if var not in arrays_to_skip: # Get the corresponding array name cdf_var = match_variables.get(var, var.lower().replace(".", "p")) - warning = ( f"The array '{cdf_var}' does not equal the expected example array " + f"'{var}' produced by the IDEX team" ) - f"'{var}' produced by the IDEX team" - + # TODO remove this block once the IDEX team fixes the l1b validation file. + # They included a lot of extra variables in the current file. + try: + l1b_dataset[cdf_var] + except KeyError: + continue if l1b_dataset[cdf_var].dtype == object: - assert (l1b_dataset[cdf_var].data == l1b_example_data[var]).all(), ( - warning - ) + assert ( + l1b_dataset[cdf_var].data == np.squeeze(l1b_example_data[var]) + ).all(), warning else: - ( - np.testing.assert_array_almost_equal( - l1b_dataset[cdf_var].data, - l1b_example_data[var], - decimal=4, - ), - warning, + np.testing.assert_array_almost_equal( + l1b_dataset[cdf_var].data, + np.squeeze(l1b_example_data[var]), + decimal=4, + err_msg=warning, )