IDEX l1b Trigger origin, level, and mode fix (#2828)

* sort of changes

* things working sort of

* more investigation

* triggertime

* matchign

* idex l1b trigger mode/origin/level fix

* remove code

* add new validation file to the external data download config

* trigger origin update

* update code

* pr feedback fix comments

Author: lacoak21

docs/source/conf.py

@@ -122,6 +122,7 @@
     (r"py:.*", r".*np.ndarray.*"),
     (r"py:.*", r".*numpy._typing._array_like._ScalarType_co.*"),
     (r"py:.*", r".*idex.l1a.TRIGGER_DESCRIPTION.*"),
+    (r"py:.*", r".*idex.l1b.TriggerOrigin.*"),
     (r"py:.*", r".*idex.l2a.BaselineNoiseTime.*"),
     (r"py:.*", r".*PacketProperties"),
     (r"py:.*", r".*.spice.geometry.SpiceBody.*"),

imap_processing/cdf/config/imap_idex_l1b_variable_attrs.yaml

@@ -53,15 +53,41 @@ spice_base: &spice_base
   VALIDMAX: *spice_data_max
 
 # <=== Instrument Setting Attributes ===>
-trigger_mode:
+trigger_mode_lg:
   <<: *string_base
-  FIELDNAM: Trigger Mode
-  CATDESC: Channel and mode that triggered the event
+  FIELDNAM: Low Gain Trigger Mode
+  CATDESC: Low Gain Trigger Mode.
 
-trigger_level:
+trigger_level_lg:
   <<: *trigger_base
-  FIELDNAM: Trigger Level
-  CATDESC: Threshold signal level that triggered the event
+  FIELDNAM: Low Gain Trigger Level
+  CATDESC: Low Gain Trigger Level threshold.
+
+trigger_mode_mg:
+  <<: *string_base
+  FIELDNAM: Mid Gain Trigger Mode
+  CATDESC: Mid Gain Trigger Mode.
+
+trigger_level_mg:
+  <<: *trigger_base
+  FIELDNAM: Mid Gain Trigger Level
+  CATDESC: Mid Gain Trigger level threshold.
+
+
+trigger_mode_hg:
+  <<: *string_base
+  FIELDNAM: High Gain Trigger Mode
+  CATDESC: High Gain Trigger Mode.
+
+trigger_level_hg:
+  <<: *trigger_base
+  FIELDNAM: High Trigger Level
+  CATDESC: High Trigger Level threshold.
+
+trigger_origin:
+  <<: *string_base
+  FIELDNAM: Trigger Origin
+  CATDESC: Trigger Origin of the event.
 
 tof_high:
   <<: *l1b_tof_base

imap_processing/idex/idex_l1a.py

@@ -458,14 +458,36 @@ def _set_sample_trigger_times(
         """
         # Retrieve the number of samples for high gain delay
 
-        # packet['IDX__TXHDRSAMPDELAY'] is a 32-bit value, with the last 10 bits
-        # representing the high gain sample delay and the first 2 bits used for padding.
-        # To extract the high gain bits, the bitwise right shift (>> 20) moves the bits
-        # 20 positions to the right, and the mask (0b1111111111) keeps only the least
-        # significant 10 bits.
-        # TODO use the delay corresponding to the trigger
-        high_gain_delay = (packet["IDX__TXHDRSAMPDELAY"] >> 22) & 0b1111111111
+        # packet['IDX__TXHDRSAMPDELAY'] is a 32-bit value:
+        # bits0-9: high-gain delay,
+        # bits10-19: mid-gain delay,
+        # bits20-29: low-gain delay.
+        # bits30-31 are padding/reserved.
+        # Each delay is extracted by right-shifting to align the field,
+        # then masking with #0b1111111111 (10 bits).
+
         n_blocks = packet["IDX__TXHDRBLOCKS"]
+        trigger_item = packet["IDX__TXHDRTRIGID"]
+
+        tof_delay = packet["IDX__TXHDRSAMPDELAY"]  # last two bits are padding
+
+        # mask to extract 10-bit values
+        tof_mask = 0b1111111111
+
+        # Determine the delay based on the trigger id.
+        hg_delay = tof_delay & tof_mask  # first 10 bits (0-9)
+        mg_delay = (tof_delay >> 10) & tof_mask  # next 10 bits (10-19)
+        lg_delay = (tof_delay >> 20) & tof_mask  # next 10 bits (20-29)
+
+        u10 = trigger_item & 0x3FF
+        if (u10 >> 0) & 1:
+            delay = hg_delay
+        elif (u10 >> 1) & 1:
+            delay = lg_delay
+        elif (u10 >> 2) & 1:
+            delay = mg_delay
+        else:
+            delay = hg_delay
 
         # Retrieve number of low/high sample pre-trigger blocks
 
@@ -485,11 +507,10 @@ def _set_sample_trigger_times(
             * (num_low_sample_pretrigger_blocks + 1)
             * self.NUMBER_SAMPLES_PER_LOW_SAMPLE_BLOCK
         )
-        self.high_sample_trigger_time = (
-            self.HIGH_SAMPLE_RATE
-            * (num_high_sample_pretrigger_blocks + 1)
-            * self.NUMBER_SAMPLES_PER_HIGH_SAMPLE_BLOCK
-            - self.HIGH_SAMPLE_RATE * high_gain_delay
+        self.high_sample_trigger_time = self.HIGH_SAMPLE_RATE * (
+            num_high_sample_pretrigger_blocks + 1
+        ) * self.NUMBER_SAMPLES_PER_HIGH_SAMPLE_BLOCK - self.HIGH_SAMPLE_RATE * (
+            delay - 1
         )
 
     def _parse_high_sample_waveform(self, waveform_raw: str) -> list[int]:
@@ -563,7 +584,7 @@ def _calc_low_sample_resolution(self, num_samples: int) -> npt.NDArray:
         time_low_sample_rate_data : numpy.ndarray
             Low time sample data array.
         """
-        time_low_sample_rate_init = np.linspace(0, num_samples, num_samples)
+        time_low_sample_rate_init = np.arange(num_samples, dtype=np.float64)
         time_low_sample_rate_data = (
             self.LOW_SAMPLE_RATE * time_low_sample_rate_init
             - self.low_sample_trigger_time
@@ -590,7 +611,7 @@ def _calc_high_sample_resolution(self, num_samples: int) -> npt.NDArray:
         time_high_sample_rate_data : numpy.ndarray
             High sample time data array.
         """
-        time_high_sample_rate_init = np.linspace(0, num_samples, num_samples)
+        time_high_sample_rate_init = np.arange(num_samples, dtype=np.float64)
         time_high_sample_rate_data = (
             self.HIGH_SAMPLE_RATE * time_high_sample_rate_init
             - self.high_sample_trigger_time

imap_processing/idex/idex_l1b.py

@@ -15,10 +15,13 @@
 """
 
 import logging
-from enum import Enum
+from enum import Enum, IntEnum
 
+import numpy as np
 import pandas as pd
 import xarray as xr
+from numpy.typing import NDArray
+from xarray import DataArray
 
 from imap_processing import imap_module_directory
 from imap_processing.cdf.imap_cdf_manager import ImapCdfAttributes
@@ -42,6 +45,27 @@
 logger = logging.getLogger(__name__)
 
 
+class TriggerOrigin(IntEnum):
+    """Enum class for event trigger origins."""
+
+    HS_ADC0I_TOF_HG = 0
+    HS_ADC0Q_TOF_LG = 1
+    HS_ADC1Q_TOF_MG = 2
+    LS_ADC1_TARGET_HG = 3
+    SW_TRIGGER = 4
+    EXTERNAL_TRIGGER = 5
+
+
+TRIGGER_LABELS = {
+    TriggerOrigin.HS_ADC0I_TOF_HG: "HS ADC0I trigger (TOF HG)",
+    TriggerOrigin.HS_ADC0Q_TOF_LG: "HS ADC0Q trigger (TOF LG)",
+    TriggerOrigin.HS_ADC1Q_TOF_MG: "HS ADC1Q trigger (TOF MG)",
+    TriggerOrigin.LS_ADC1_TARGET_HG: "LS ADC1 trigger (Target HG / low range)",
+    TriggerOrigin.SW_TRIGGER: "SW trigger",
+    TriggerOrigin.EXTERNAL_TRIGGER: "external trigger",
+}
+
+
 class TriggerMode(Enum):
     """
     Enum class for data collection trigger Modes.
@@ -117,21 +141,21 @@ def idex_l1b(l1a_dataset: xr.Dataset) -> xr.Dataset:
     # used for calculations yet but are saved in the CDF for reference.
     spice_data = get_spice_data(l1a_dataset, idex_attrs)
 
-    trigger_settings = get_trigger_mode_and_level(l1a_dataset)
-    if trigger_settings:
-        trigger_settings["triggerlevel"].attrs = idex_attrs.get_variable_attributes(
-            "trigger_level"
-        )
-        trigger_settings["triggermode"].attrs = idex_attrs.get_variable_attributes(
-            "trigger_mode"
-        )
-
+    trigger_settings = get_trigger_mode_and_level(l1a_dataset, idex_attrs)
+    trigger_origin = get_trigger_origin(
+        l1a_dataset["idx__txhdrtrigid"].data, idex_attrs
+    )
     # Create l1b Dataset
     prefixes = ["shcoarse", "shfine", "time_high_sample", "time_low_sample"]
-    data_vars = processed_vars | waveforms_converted | trigger_settings | spice_data
+    data_vars = (
+        processed_vars
+        | waveforms_converted
+        | trigger_settings
+        | spice_data
+        | trigger_origin
+    )
     l1b_dataset = setup_dataset(l1a_dataset, prefixes, idex_attrs, data_vars)
     l1b_dataset.attrs = idex_attrs.get_global_attributes("imap_idex_l1b_sci")
-
     # Convert variables
     l1b_dataset = convert_raw_to_eu(
         l1b_dataset,
@@ -225,6 +249,7 @@ def convert_waveforms(
 
 def get_trigger_mode_and_level(
     l1a_dataset: xr.Dataset,
+    idex_attrs: ImapCdfAttributes,
 ) -> dict[str, xr.DataArray] | dict:
     """
     Determine the trigger mode and threshold level for each event.
@@ -233,6 +258,8 @@ def get_trigger_mode_and_level(
     ----------
     l1a_dataset : xarray.Dataset
         IDEX L1a dataset containing the six waveform arrays and instrument settings.
+    idex_attrs : ImapCdfAttributes
+        CDF attribute manager object.
 
     Returns
     -------
@@ -243,8 +270,8 @@ def get_trigger_mode_and_level(
     channels = ["lg", "mg", "hg"]
     # 10 bit mask
     mask = 0b1111111111
-    trigger_modes = []
-    trigger_levels = []
+    # Initialize a dict to hold the mode labels and threshold levels for each channel
+    data_dict = {}
 
     def compute_trigger_values(
         trigger_mode: int, trigger_controls: int, gain_channel: str
@@ -302,28 +329,63 @@ def compute_trigger_values(
             vectorize=True,
             output_dtypes=[object, float],
         )
-        trigger_modes.append(mode_array.rename("trigger_mode"))
-        trigger_levels.append(level_array.rename("trigger_level"))
-
-    try:
         # There should be an array of modes and threshold levels for each channel.
-        # At each index (event) only one of the three arrays should have a value that is
-        # not 'None' because each event can only be triggered by one channel.
-        # By merging the three arrays, we get value for each event.
-        merged_modes = xr.merge([trigger_modes[0], xr.merge(trigger_modes[1:])])
-        merged_levels = xr.merge([trigger_levels[0], xr.merge(trigger_levels[1:])])
-
-        return {
-            "triggermode": merged_modes.trigger_mode,
-            "triggerlevel": merged_levels.trigger_level,
-        }
-
-    except xr.MergeError as e:
-        raise ValueError(
-            f"Only one channel can trigger a dust event. Please make sure "
-            f"there is only one valid trigger value per event. This "
-            f"caused Merge Error: {e}"
-        ) from e
+        # write each of them out as separate variables because there may be
+        # multiple channels that can trigger an event. The trigger origin variable
+        # can be used to determine which channel(s) triggered the event.
+        mode_array.attrs = idex_attrs.get_variable_attributes(f"trigger_mode_{channel}")
+        data_dict[f"trigger_mode_{channel}"] = mode_array
+        level_array.attrs = idex_attrs.get_variable_attributes(
+            f"trigger_level_{channel}"
+        )
+        data_dict[f"trigger_level_{channel}"] = level_array
+
+    return data_dict
+
+
+def get_trigger_origin(
+    trigger_id: NDArray, idex_attrs: ImapCdfAttributes
+) -> dict[str, DataArray]:
+    """
+    Determine the trigger origin for each event.
+
+    Parameters
+    ----------
+    trigger_id : numpy.ndarray
+        Array of raw trigger ID values from the l1a dataset. The trigger ID is a 32-bit
+        integer where the lower 10 bits contain information about the trigger origin.
+    idex_attrs : ImapCdfAttributes
+        CDF attribute manager object.
+
+    Returns
+    -------
+    dict[str, xarray.DataArray]
+        A dictionary containing the trigger_origin DataArray with the trigger
+        origin info for each event.
+    """
+    # extract the lower 10 bits of the trigger ID to get the trigger origin information
+    trigger_bits = trigger_id & 0x3FF
+    # For each event, determine which bits are set and get the corresponding trigger
+    # origin labels
+    origin_labels = np.array(
+        [
+            ", ".join(
+                [TRIGGER_LABELS[TriggerOrigin(i)] for i in range(6) if (bits >> i) & 1]
+            )
+            for bits in trigger_bits
+        ],
+        dtype=object,
+    )
+    # Update any events with no trigger bits set to "unknown trigger origin"
+    origin_labels[origin_labels == ""] = "Unknown trigger origin"
+    return {
+        "trigger_origin": xr.DataArray(
+            name="trigger_origin",
+            data=np.squeeze(origin_labels),
+            dims="epoch",
+            attrs=idex_attrs.get_variable_attributes("trigger_origin"),
+        )
+    }
 
 
 def get_spice_data(

imap_processing/tests/external_test_data_config.py

@@ -134,7 +134,7 @@
 
     # IDEX
     ("idex_l1a_validation_file.h5", "idex/test_data/"),
-    ("idex_l1b_validation_file.h5", "idex/test_data/"),
+    ("imap_idex_l1b_sci_20231218_v001.h5", "idex/test_data/"),
     ("imap_idex_l2a-calibration-curve-yield-params_20250101_v001.csv", "idex/test_data/"),
     ("imap_idex_l2a-calibration-curve-t-rise_20250101_v001.csv", "idex/test_data/"),
 

imap_processing/tests/idex/conftest.py

@@ -17,7 +17,7 @@
 TEST_L0_FILE_CATLST = TEST_DATA_PATH / "imap_idex_l0_raw_20241206_v001.pkts"  # 1419
 
 L1A_EXAMPLE_FILE = TEST_DATA_PATH / "idex_l1a_validation_file.h5"
-L1B_EXAMPLE_FILE = TEST_DATA_PATH / "idex_l1b_validation_file.h5"
+L1B_EXAMPLE_FILE = TEST_DATA_PATH / "imap_idex_l1b_sci_20231218_v001.h5"
 
 L2A_CDF = TEST_DATA_PATH / "imap_idex_l2a_sci-1week_20251017_v001.cdf"
 L1B_EVT_CDF = TEST_DATA_PATH / "imap_idex_l1b_evt_20250108_v001.cdf"

imap_processing/tests/idex/test_idex_l1a.py

@@ -160,7 +160,15 @@ def test_validate_l1a_idex_data_variables(
     # The Engineering data is converting to UTC, and the SDC is converting to J2000,
     # for 'epoch' and 'Timestamp' so this test is using the raw time value 'SCHOARSE' to
     # validate time
-    arrays_to_skip = ["Timestamp", "Epoch", "event"]
+    # TODO remove the low and high time from this list after the IDEX team produces a
+    #  new l1a h5 file.
+    arrays_to_skip = [
+        "Timestamp",
+        "Epoch",
+        "event",
+        "Time (high sampling)",
+        "Time (low sampling)",
+    ]
 
     # loop through all keys from the l1a example dict
     for var in l1a_example_data.variables: