Addressed review in #1247

hed/errors/error_messages.py

@@ -272,7 +272,7 @@ def val_error_def_value_missing(tag):
 
 @hed_tag_error(ValidationErrors.HED_DEF_VALUE_EXTRA, actual_code=ValidationErrors.DEF_INVALID)
 def val_error_def_value_extra(tag):
-    return f"A def tag does not take a placeholder value, but was given one.  Definition: '{tag}"
+    return f"A def tag does not take a placeholder value, but was given one.  Definition: '{tag}'"
 
 
 @hed_tag_error(ValidationErrors.HED_DEF_EXPAND_UNMATCHED, actual_code=ValidationErrors.DEF_EXPAND_INVALID)
@@ -287,7 +287,7 @@ def val_error_def_expand_value_missing(tag):
 
 @hed_tag_error(ValidationErrors.HED_DEF_EXPAND_VALUE_EXTRA, actual_code=ValidationErrors.DEF_EXPAND_INVALID)
 def val_error_def_expand_value_extra(tag):
-    return f"A Def-expand tag does not take a placeholder value, but was given one.  Definition: '{tag}"
+    return f"A Def-expand tag does not take a placeholder value, but was given one.  Definition: '{tag}'"
 
 
 @hed_tag_error(ValidationErrors.HED_TOP_LEVEL_TAG, actual_code=ValidationErrors.TAG_GROUP_ERROR)

hed/errors/exceptions.py

@@ -18,20 +18,8 @@ class HedExceptions:
     INVALID_FILE_FORMAT = "INVALID_FILE_FORMAT"
 
     # These are actual schema issues, not that the file cannot be found or parsed
-    SCHEMA_HEADER_MISSING = "SCHEMA_HEADER_INVALID"
     SCHEMA_HEADER_INVALID = "SCHEMA_HEADER_INVALID"
-    SCHEMA_UNKNOWN_HEADER_ATTRIBUTE = "SCHEMA_HEADER_INVALID"
-
     SCHEMA_LIBRARY_INVALID = "SCHEMA_LIBRARY_INVALID"
-    BAD_HED_LIBRARY_NAME = "SCHEMA_LIBRARY_INVALID"
-    BAD_WITH_STANDARD = "SCHEMA_LIBRARY_INVALID"
-    BAD_WITH_STANDARD_MULTIPLE_VALUES = "SCHEMA_LOAD_FAILED"
-    ROOTED_TAG_INVALID = "SCHEMA_LIBRARY_INVALID"
-    ROOTED_TAG_HAS_PARENT = "SCHEMA_LIBRARY_INVALID"
-    ROOTED_TAG_DOES_NOT_EXIST = "SCHEMA_LIBRARY_INVALID"
-    IN_LIBRARY_IN_UNMERGED = "SCHEMA_LIBRARY_INVALID"
-    INVALID_LIBRARY_PREFIX = "SCHEMA_LIBRARY_INVALID"
-
     SCHEMA_VERSION_INVALID = "SCHEMA_VERSION_INVALID"
     SCHEMA_SECTION_MISSING = "SCHEMA_SECTION_MISSING"
     SCHEMA_INVALID = "SCHEMA_INVALID"
@@ -44,7 +32,6 @@ class HedExceptions:
     WIKI_LINE_INVALID = "WIKI_LINE_INVALID"
     HED_SCHEMA_NODE_NAME_INVALID = "HED_SCHEMA_NODE_NAME_INVALID"
 
-    SCHEMA_DUPLICATE_PREFIX = "SCHEMA_LOAD_FAILED"
     SCHEMA_DUPLICATE_LIBRARY = "SCHEMA_LIBRARY_INVALID"
     BAD_COLUMN_NAMES = "BAD_COLUMN_NAMES"
 

hed/schema/hed_schema_group.py

@@ -43,8 +43,8 @@ def __init__(self, schema_list, name=""):
         schema_prefixes = [hed_schema._namespace for hed_schema in schema_list]
         if len(set(schema_prefixes)) != len(schema_prefixes):
             raise HedFileError(
-                HedExceptions.SCHEMA_DUPLICATE_PREFIX,
-                "Multiple schema share the same tag name_prefix.  This is not allowed.",
+                HedExceptions.SCHEMA_LOAD_FAILED,
+                "Multiple schema share the same tag name_prefix so schema cannot be loaded.",
                 filename=self.name,
             )
         self._schemas = {hed_schema._namespace: hed_schema for hed_schema in schema_list}

hed/schema/schema_header_util.py

@@ -17,9 +17,9 @@ def validate_library_name(library_name):
     """
     for i, character in enumerate(library_name):
         if not character.isalpha():
-            return f"Non alpha character '{character}' at position {i} in '{library_name}'"
+            return f"Non alpha character '{character}' at position {i} in library name '{library_name}'"
         if character.isupper():
-            return f"Non lowercase character '{character}' at position {i} in '{library_name}'"
+            return f"Non lowercase character '{character}' at position {i} in library name '{library_name}'"
 
 
 def validate_version_string(version_string):
@@ -41,7 +41,7 @@ def validate_version_string(version_string):
 
 header_attribute_validators = {
     constants.VERSION_ATTRIBUTE: (validate_version_string, HedExceptions.SCHEMA_VERSION_INVALID),
-    constants.LIBRARY_ATTRIBUTE: (validate_library_name, HedExceptions.BAD_HED_LIBRARY_NAME),
+    constants.LIBRARY_ATTRIBUTE: (validate_library_name, HedExceptions.SCHEMA_LIBRARY_INVALID),
 }
 
 
@@ -61,7 +61,7 @@ def validate_present_attributes(attrib_dict, name):
     """
     if constants.WITH_STANDARD_ATTRIBUTE in attrib_dict and constants.LIBRARY_ATTRIBUTE not in attrib_dict:
         raise HedFileError(
-            HedExceptions.BAD_WITH_STANDARD,
+            HedExceptions.SCHEMA_LIBRARY_INVALID,
             "withStandard header attribute found, but no library attribute is present",
             name,
         )
@@ -93,7 +93,7 @@ def validate_attributes(attrib_dict, name):
                 raise HedFileError(error_code, had_error, name)
         if attribute_name not in valid_header_attributes:
             raise HedFileError(
-                HedExceptions.SCHEMA_UNKNOWN_HEADER_ATTRIBUTE,
+                HedExceptions.SCHEMA_HEADER_INVALID,
                 f"Unknown attribute {attribute_name} found in header line",
                 filename=name,
             )

hed/schema/schema_io/base2schema.py

@@ -63,14 +63,14 @@ def __init__(self, filename, schema_as_string=None, schema=None, file_format=Non
             self.appending_to_schema = True
             if not self._schema.with_standard:
                 raise HedFileError(
-                    HedExceptions.SCHEMA_DUPLICATE_PREFIX,
+                    HedExceptions.SCHEMA_LOAD_FAILED,
                     "Loading multiple normal schemas as a merged one with the same namespace.  "
                     "Ensure schemas have the withStandard header attribute set",
                     self.name,
                 )
             elif with_standard != self._schema.with_standard:
                 raise HedFileError(
-                    HedExceptions.BAD_WITH_STANDARD_MULTIPLE_VALUES,
+                    HedExceptions.SCHEMA_LOAD_FAILED,
                     f"Merging schemas requires same withStandard value ({with_standard} != {self._schema.with_standard}).",
                     self.name,
                 )
@@ -125,7 +125,7 @@ def _load(self):
                 base_version = load_schema_version(self._schema.with_standard, check_prerelease=self.check_prerelease)
             except HedFileError as e:
                 raise HedFileError(
-                    HedExceptions.BAD_WITH_STANDARD,
+                    HedExceptions.SCHEMA_LIBRARY_INVALID,
                     message=f"Cannot load withStandard schema '{self._schema.with_standard}'",
                     filename=e.filename,
                 ) from e
@@ -194,36 +194,36 @@ def find_rooted_entry(tag_entry, schema, loading_merged):
         if rooted_tag is not None:
             if not schema.with_standard:
                 raise HedFileError(
-                    HedExceptions.ROOTED_TAG_INVALID,
+                    HedExceptions.SCHEMA_ATTRIBUTE_INVALID,
                     f"Rooted tag attribute found on '{tag_entry.short_tag_name}' in a standard schema.",
                     schema.name,
                 )
 
             if not isinstance(rooted_tag, str):
                 raise HedFileError(
-                    HedExceptions.ROOTED_TAG_INVALID,
+                    HedExceptions.SCHEMA_LIBRARY_INVALID,
                     f"Rooted tag '{tag_entry.short_tag_name}' is not a string.\"",
                     schema.name,
                 )
 
             if tag_entry.parent_name and not loading_merged:
                 raise HedFileError(
-                    HedExceptions.ROOTED_TAG_INVALID,
+                    HedExceptions.SCHEMA_LIBRARY_INVALID,
                     f"Found rooted tag '{tag_entry.short_tag_name}' as a non root node.",
                     schema.name,
                 )
 
             if not tag_entry.parent_name and loading_merged:
                 raise HedFileError(
-                    HedExceptions.ROOTED_TAG_INVALID,
+                    HedExceptions.SCHEMA_LIBRARY_INVALID,
                     f"Found rooted tag '{tag_entry.short_tag_name}' as a root node in a merged schema.",
                     schema.name,
                 )
 
             rooted_entry = schema.tags.get(rooted_tag)
             if not rooted_entry or rooted_entry.has_attribute(constants.HedKey.InLibrary):
                 raise HedFileError(
-                    HedExceptions.ROOTED_TAG_DOES_NOT_EXIST,
+                    HedExceptions.LIBRARY_SCHEMA_INVALID,
                     f"Rooted tag '{tag_entry.short_tag_name}' not found in paired standard schema",
                     schema.name,
                 )

hed/schema/schema_io/wiki2schema.py

@@ -77,7 +77,7 @@ def _get_header_attributes(self, file_data):
                 hed_attributes = self._get_header_attributes_internal(line[len(wiki_constants.HEADER_LINE_STRING) :])
                 return hed_attributes
         msg = f"First line of file should be HED, instead found: {line}"
-        raise HedFileError(HedExceptions.SCHEMA_HEADER_MISSING, msg, filename=self.name)
+        raise HedFileError(HedExceptions.SCHEMA_HEADER_INVALID, msg, filename=self.name)
 
     def _parse_data(self):
         wiki_lines_by_section = self._split_lines_into_sections(self.input_data)

hed/tools/__init__.py

@@ -20,7 +20,7 @@
 from .bids.bids_util import parse_bids_filename
 
 from .util.data_util import get_new_dataframe, get_value_dict, replace_values, reorder_columns
-from .util.io_util import check_filename, clean_filename, extract_suffix_path, get_file_list, make_path
+from .util.io_util import check_filename, clean_filename, extract_suffix_path, get_file_list
 from .util.io_util import get_path_components
 
 from .analysis.annotation_util import (

hed/tools/analysis/annotation_util.py

@@ -367,37 +367,6 @@ def _flatten_val_col(col_key, col_dict) -> tuple[list[str], list[str], list[str]
     return [col_key], ["n/a"], [description], [tags]
 
 
-# def _get_row_tags(row, description_tag=True):
-#     """ Return the HED string associated with row, possibly without the description.
-#
-#     Parameters:
-#         row (DataSeries):        Pandas data frame containing a row of a tagging spreadsheet.
-#         description_tag (bool):  If True, include any Description tags in the returned string.
-#
-#     Returns:
-#         str:  A HED string extracted from the row.
-#         str:  A string representing the description (without the Description tag).
-#
-#     Notes:
-#         If description_tag is True the entire tag string is included with description.
-#         If there was a description extracted, it is appended to any existing description.
-#
-#     """
-#     remainder, extracted = extract_tags(row['HED'], 'Description/')
-#     if description_tag:
-#         tags = row["HED"]
-#     else:
-#         tags = remainder
-#
-#     if row["description"] != 'n/a':
-#         description = row["description"]
-#     else:
-#         description = ""
-#     if extracted:
-#         description = " ".join([description, extracted])
-#     return tags, description
-
-
 def _get_value_entry(hed_entry, description_entry, description_tag=True):
     """Return a HED dictionary representing a value entry in a HED tagging spreadsheet.
 
@@ -473,24 +442,3 @@ def _update_cat_dict(cat_dict, value_entry, hed_entry, description_entry, descri
         hed_part = cat_dict.get("HED", {})
         hed_part[value_entry] = value_dict["HED"]
         cat_dict["HED"] = hed_part
-
-
-# def _update_remainder(remainder, update_piece):
-#     """ Update remainder with update piece.
-#
-#     Parameters:
-#         remainder (str):      A tag string without trailing comma.
-#         update_piece (str):   A tag string to be appended.
-#
-#     Returns:
-#         str: A concatenation of remainder and update_piece, paying attention to separating commas.
-#
-#     """
-#     if not update_piece:
-#         return remainder
-#     elif not remainder:
-#         return update_piece
-#     elif remainder.endswith(')') or update_piece.startswith('('):
-#         return remainder + update_piece
-#     else:
-#         return remainder + ", " + update_piece

hed/tools/analysis/hed_tag_counts.py

@@ -143,7 +143,7 @@ def merge_tag_dicts(self, other_dict):
                 continue
             for value, val_count in count.value_dict.items():
                 if value in self.tag_dict[tag].value_dict:
-                    self.tag_dict[tag].value_dict[value] = self.tag_dict[tag].value_dict + val_count
+                    self.tag_dict[tag].value_dict[value] = self.tag_dict[tag].value_dict[value] + val_count
                 else:
                     self.tag_dict[tag].value_dict[value] = val_count
 

hed/tools/analysis/hed_type_factors.py

@@ -1,7 +1,7 @@
 """Manager for factor information for a columnar file."""
 
 import pandas as pd
-from hed.errors.exceptions import HedFileError
+from hed.errors.exceptions import HedExceptions, HedFileError
 
 
 class HedTypeFactors:
@@ -58,15 +58,17 @@ def get_factors(self, factor_encoding="one-hot"):
         sum_factors = factors.sum(axis=1)
         if factor_encoding == "categorical" and sum_factors.max() > 1:
             raise HedFileError(
-                "MultipleFactorSameEvent",
+                HedExceptions.BAD_PARAMETERS,
                 f"{self.type_value} has multiple occurrences at index {sum_factors.idxmax()}",
                 "",
             )
         elif factor_encoding == "categorical":
             return self._one_hot_to_categorical(factors, levels)
         else:
-            raise ValueError(
-                "BadFactorEncoding", f"{factor_encoding} is not in the allowed encodings: {str(self.ALLOWED_ENCODINGS)}"
+            raise HedFileError(
+                HedExceptions.BAD_PARAMETERS,
+                f"{factor_encoding} is not in the allowed encodings: {str(self.ALLOWED_ENCODINGS)}",
+                "",
             )
 
     def _one_hot_to_categorical(self, factors, levels):
@@ -77,7 +79,7 @@ def _one_hot_to_categorical(self, factors, levels):
             levels (list):  List of categorical columns to convert.
 
         Return:
-            pd.ataFrame:  Contains one-hot representation of requested levels.
+            pd.DataFrame:  Contains one-hot representation of requested levels.
 
         """
         df = pd.DataFrame("n/a", index=range(len(factors.index)), columns=[self.type_value])

hed/tools/bids/bids_dataset.py

@@ -108,7 +108,7 @@ def validate(self, check_for_warnings=False, schema=None):
             return [
                 {
                     "code": "SCHEMA_LOAD_FAILED",
-                    "message": "BIDS dataset_description.json has invalid HEDVersion and passed schema was invalid}",
+                    "message": "BIDS dataset_description.json has invalid HEDVersion and passed schema was invalid",
                 }
             ]
 

hed/tools/util/io_util.py

@@ -279,25 +279,6 @@ def get_timestamp():
     return now.strftime(TIME_FORMAT)[:-3]
 
 
-def make_path(root_path, sub_path, filename):
-    """Get path for a file, verifying all components exist.
-
-    Parameters:
-        root_path (str):   path of the root directory.
-        sub_path (str):    sub-path relative to the root directory.
-        filename (str):    filename of the file.
-
-    Returns:
-        str: A valid realpath for the specified file.
-
-    Notes: This function is useful for creating files within BIDS datasets.
-
-    """
-
-    dir_path = os.path.realpath(os.path.join(root_path, sub_path))
-    os.makedirs(dir_path, exist_ok=True)
-
-
 def get_task_from_file(file_path):
     """Returns the task name entity from a BIDS-type file path.
 

hed/validator/def_validator.py

@@ -119,9 +119,6 @@ def validate_def_value_units(self, def_tag, hed_validator, allow_placeholders=Fa
 
         # Validate the def name vs the name class
         def_issues = hed_validator._unit_validator._check_value_class(def_tag, tag_label, report_as=None)
-        # def_issues += hed_validator.validate_units(def_tag,
-        #                                            tag_label,
-        #                                            error_code=error_code)
 
         def_contents = def_entry.get_definition(def_tag, placeholder_value=placeholder, return_copy_of_tag=True)
         if def_contents and def_entry.takes_value and hed_validator:

hed/validator/sidecar_validator.py

@@ -14,6 +14,8 @@
 from hed.errors.error_reporter import check_for_any_errors
 from hed.models import df_util
 
+COLUMN_REF_PATTERN = re.compile(r"\{([a-z_\-0-9]+)\}", re.IGNORECASE)
+
 
 # todo: Add/improve validation for definitions being in known columns(right now it just assumes they aren't)
 class SidecarValidator:
@@ -95,7 +97,7 @@ def validate(self, sidecar, extra_def_dicts=None, name=None, error_handler=None)
                 # Only do full string checks on full columns, not partial ref columns.
                 if not is_ref_column:
                     # TODO: Figure out why this pattern is giving lint errors.
-                    refs = re.findall(r"\{([a-z_\-0-9]+)\}", hed_string, re.IGNORECASE)
+                    refs = COLUMN_REF_PATTERN.findall(hed_string)
                     refs_strings = {data.column_name: data.get_hed_strings() for data in sidecar}
                     if "HED" not in refs_strings:
                         refs_strings["HED"] = ["n/a"]
@@ -170,7 +172,7 @@ def _validate_refs(self, sidecar, error_handler):
                         ColumnErrors.MALFORMED_COLUMN_REF, column_name, loc, bad_symbol
                     )
 
-                sub_matches = re.findall(r"\{([a-z_\-0-9]+)\}", hed_string, re.IGNORECASE)
+                sub_matches = COLUMN_REF_PATTERN.findall(hed_string)
                 matches.append(sub_matches)
                 for match in sub_matches:
                     if match not in possible_column_refs:

hed/validator/spreadsheet_validator.py

@@ -20,7 +20,7 @@
 
 
 class SpreadsheetValidator:
-    ONSET_TOLERANCE = 10 - 7
+    ONSET_TOLERANCE = 1e-7
     TEMPORAL_ANCHORS = re.compile(r"|".join(map(re.escape, ["onset", "inset", "offset", "delay"])))
 
     def __init__(self, hed_schema):

hed/validator/util/group_util.py

@@ -42,7 +42,6 @@ def run_tag_level_validators(self, hed_string_obj) -> list[dict]:
         checks = [
             self._check_group_relationships,
             self._duplicate_checker.check_for_duplicates,
-            # self.validate_duration_tags,
         ]
 
         for check in checks:
@@ -125,7 +124,6 @@ def _check_reserved_group_requirements(self, group):
         if len(validation_issues) > 0:
             return validation_issues
 
-        #   validation_errors = self._reserved_checker.check_reserved_duplicates(reserved_tags, group)
         return validation_issues
 
     @staticmethod