fix: improve queryables handling and CQL2 filter property normalization

Combines the following fixes:
- Normalize property names by stripping 'properties.' prefix in CQL2 filter
- Enhance field mapping to handle 'properties.' prefix in Elasticsearch queries
- Fix nested property traversal and improve error messages
- Implement exclusion of fields from queryables based on environment variable
- Enhance exclusion logic for queryables to support top-level fields and properties prefix

Author: Andrzej Pijanowski

stac_fastapi/core/stac_fastapi/core/queryables.py

@@ -25,7 +25,6 @@ def __init__(self, database_logic: Any):
         self._lock = asyncio.Lock()
         self.validation_enabled: bool = False
         self.cache_ttl: int = 1800  # How often to refresh cache (in seconds)
-        self.excluded_queryables: Set[str] = set()
         self.reload_settings()
 
     def reload_settings(self):
@@ -35,17 +34,6 @@ def reload_settings(self):
         )
         self.cache_ttl = int(os.getenv("QUERYABLES_CACHE_TTL", "1800"))
 
-        excluded = os.getenv("EXCLUDED_FROM_QUERYABLES", "")
-        self.excluded_queryables = set()
-        if excluded:
-            for field in excluded.split(","):
-                field = field.strip()
-                if field:
-                    # Remove 'properties.' prefix if present
-                    if field.startswith("properties."):
-                        field = field[11:]
-                    self.excluded_queryables.add(field)
-
     async def _update_cache(self):
         """Update the cache with the latest queryables from the database."""
         if not self.validation_enabled:
@@ -58,9 +46,6 @@ async def _update_cache(self):
             queryables_mapping = await self._db_logic.get_queryables_mapping()
             all_queryables_set = set(queryables_mapping.keys())
 
-            if self.excluded_queryables:
-                all_queryables_set = all_queryables_set - self.excluded_queryables
-
             self._all_queryables = all_queryables_set
 
             self._cache = {"*": list(all_queryables_set)}
@@ -93,18 +78,28 @@ async def validate(self, fields: Set[str]) -> None:
         if invalid_fields:
             raise HTTPException(
                 status_code=400,
-                detail=f"Invalid query fields: {', '.join(invalid_fields)}.",
+                detail=f"Invalid query fields: {', '.join(sorted(invalid_fields))}. "
+                "These fields are not defined in the collection's queryables. "
+                "Use the /queryables endpoint to see available fields.",
             )
 
 
 def get_properties_from_cql2_filter(cql2_filter: Dict[str, Any]) -> Set[str]:
-    """Recursively extract property names from a CQL2 filter."""
+    """Recursively extract property names from a CQL2 filter.
+
+    Property names are normalized by stripping the 'properties.' prefix
+    if present, to match queryables stored without the prefix.
+    """
     props: Set[str] = set()
     if "op" in cql2_filter and "args" in cql2_filter:
         for arg in cql2_filter["args"]:
             if isinstance(arg, dict):
                 if "op" in arg:
                     props.update(get_properties_from_cql2_filter(arg))
                 elif "property" in arg:
-                    props.add(arg["property"])
+                    prop_name = arg["property"]
+                    # Strip 'properties.' prefix if present
+                    if prop_name.startswith("properties."):
+                        prop_name = prop_name[11:]
+                    props.add(prop_name)
     return props

stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/database/mapping.py

@@ -3,14 +3,62 @@
 This module provides functions for working with Elasticsearch/OpenSearch mappings.
 """
 
-from typing import Any, Dict
+import os
+from collections import deque
+from typing import Any, Dict, Set
+
+
+def _get_excluded_from_queryables() -> Set[str]:
+    """Get fields to exclude from queryables endpoint and filtering.
+
+    Reads from EXCLUDED_FROM_QUERYABLES environment variable.
+    Supports comma-separated list of field names.
+
+    For each exclusion pattern, both the original and the version with/without
+    'properties.' prefix are included. This ensures fields are excluded regardless
+    of whether they appear at the top level or under 'properties' in the mapping.
+
+    Example:
+        EXCLUDED_FROM_QUERYABLES="properties.auth:schemes,storage:schemes"
+
+        This will exclude:
+        - properties.auth:schemes (and children like properties.auth:schemes.s3.type)
+        - auth:schemes (and children like auth:schemes.s3.type)
+        - storage:schemes (and children)
+        - properties.storage:schemes (and children)
+
+    Returns:
+        Set[str]: Set of field names to exclude from queryables
+    """
+    excluded = os.getenv("EXCLUDED_FROM_QUERYABLES", "")
+    if not excluded:
+        return set()
+
+    result = set()
+    for field in excluded.split(","):
+        field = field.strip()
+        if not field:
+            continue
+
+        result.add(field)
+
+        if field.startswith("properties."):
+            result.add(field.removeprefix("properties."))
+        else:
+            result.add(f"properties.{field}")
+
+    return result
 
 
 async def get_queryables_mapping_shared(
-    mappings: Dict[str, Dict[str, Any]], collection_id: str = "*"
+    mappings: Dict[str, Dict[str, Any]],
+    collection_id: str = "*",
 ) -> Dict[str, str]:
     """Retrieve mapping of Queryables for search.
 
+    Fields listed in the EXCLUDED_FROM_QUERYABLES environment variable will be
+    excluded from the result, along with their children.
+
     Args:
         mappings (Dict[str, Dict[str, Any]]): The mapping information returned from
             Elasticsearch/OpenSearch client's indices.get_mapping() method.
@@ -20,19 +68,44 @@ async def get_queryables_mapping_shared(
 
     Returns:
         Dict[str, str]: A dictionary containing the Queryables mappings, where keys are
-            field names and values are the corresponding paths in the Elasticsearch/OpenSearch
-            document structure.
+            field names (with 'properties.' prefix removed) and values are the
+            corresponding paths in the Elasticsearch/OpenSearch document structure.
     """
     queryables_mapping = {}
+    excluded = _get_excluded_from_queryables()
+
+    def is_excluded(path: str) -> bool:
+        """Check if the path starts with any excluded prefix."""
+        return any(
+            path == prefix or path.startswith(prefix + ".") for prefix in excluded
+        )
 
     for mapping in mappings.values():
-        fields = mapping["mappings"].get("properties", {})
-        properties = fields.pop("properties", {}).get("properties", {}).keys()
+        mapping_properties = mapping["mappings"].get("properties", {})
+
+        stack: deque[tuple[str, Dict[str, Any]]] = deque(mapping_properties.items())
+
+        while stack:
+            field_fqn, field_def = stack.popleft()
+
+            nested_properties = field_def.get("properties")
+            if nested_properties:
+                stack.extend(
+                    (f"{field_fqn}.{k}", v)
+                    for k, v in nested_properties.items()
+                    if v.get("enabled", True) and not is_excluded(f"{field_fqn}.{k}")
+                )
+
+            field_type = field_def.get("type")
+            if (
+                not field_type
+                or not field_def.get("enabled", True)
+                or is_excluded(field_fqn)
+            ):
+                continue
 
-        for field_key in fields:
-            queryables_mapping[field_key] = field_key
+            field_name = field_fqn.removeprefix("properties.")
 
-        for property_key in properties:
-            queryables_mapping[property_key] = f"properties.{property_key}"
+            queryables_mapping[field_name] = field_fqn
 
     return queryables_mapping

stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/filter/client.py

@@ -26,16 +26,54 @@ def _get_excluded_from_queryables() -> set[str]:
         Reads from EXCLUDED_FROM_QUERYABLES environment variable.
         Supports comma-separated list of field names.
 
+        For each exclusion pattern, both the original and the version with/without
+        'properties.' prefix are included. This ensures fields are excluded regardless
+        of whether they appear at the top level or under 'properties' in the mapping.
+
         Example:
-            EXCLUDED_FROM_QUERYABLES="auth:schemes,storage:schemes"
+            EXCLUDED_FROM_QUERYABLES="properties.auth:schemes,storage:schemes"
 
         Returns:
             Set[str]: Set of field names to exclude from queryables
         """
         excluded = os.getenv("EXCLUDED_FROM_QUERYABLES", "")
         if not excluded:
             return set()
-        return {field.strip() for field in excluded.split(",") if field.strip()}
+
+        result = set()
+        for field in excluded.split(","):
+            field = field.strip()
+            if not field:
+                continue
+
+            result.add(field)
+
+            if field.startswith("properties."):
+                result.add(field.removeprefix("properties."))
+            else:
+                result.add(f"properties.{field}")
+
+        return result
+
+    @staticmethod
+    def _is_excluded(field_fqn: str, excluded: set[str]) -> bool:
+        """Check if a field should be excluded based on prefix matching.
+
+        A field is excluded if:
+        - It exactly matches an exclusion pattern
+        - It starts with an exclusion pattern followed by a dot (nested child)
+
+        Args:
+            field_fqn: Fully qualified field name (e.g., "properties.auth:schemes.s3.type")
+            excluded: Set of exclusion patterns
+
+        Returns:
+            True if field should be excluded, False otherwise
+        """
+        for prefix in excluded:
+            if field_fqn == prefix or field_fqn.startswith(prefix + "."):
+                return True
+        return False
 
     async def get_queryables(
         self,
@@ -92,23 +130,20 @@ async def get_queryables(
         while stack:
             field_fqn, field_def = stack.popleft()
 
-            # Iterate over nested fields
+            if self._is_excluded(field_fqn, excluded_fields):
+                continue
+
             field_properties = field_def.get("properties")
             if field_properties:
                 stack.extend(
                     (f"{field_fqn}.{k}", v)
                     for k, v in field_properties.items()
                     if v.get("enabled", True)
-                    and f"{field_fqn}.{k}" not in excluded_fields
                 )
 
             # Skip non-indexed or disabled fields
             field_type = field_def.get("type")
-            if (
-                not field_type
-                or not field_def.get("enabled", True)
-                or field_fqn in excluded_fields
-            ):
+            if not field_type or not field_def.get("enabled", True):
                 continue
 
             # Fields in Item Properties should be exposed with their un-prefixed names,

stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/filter/transform.py

@@ -22,7 +22,20 @@ def to_es_field(queryables_mapping: Dict[str, Any], field: str) -> str:
     Returns:
         str: The mapped field name suitable for Elasticsearch queries.
     """
-    return queryables_mapping.get(field, field)
+    # First, try to find the field as-is in the mapping
+    if field in queryables_mapping:
+        return queryables_mapping[field]
+
+    # If field has 'properties.' prefix, try without it
+    # This handles cases where users specify 'properties.eo:cloud_cover'
+    # but queryables_mapping uses 'eo:cloud_cover' as the key
+    if field.startswith("properties."):
+        normalized_field = field[11:]  # len("properties.") == 11
+        if normalized_field in queryables_mapping:
+            return queryables_mapping[normalized_field]
+
+    # If not found, return the original field
+    return field
 
 
 def to_es(queryables_mapping: Dict[str, Any], query: Dict[str, Any]) -> Dict[str, Any]:

stac_fastapi/tests/api/test_api_query_validation.py

@@ -30,6 +30,8 @@ def enable_validation():
     client = app_config["client"]
     with mock.patch.dict(os.environ, {"VALIDATE_QUERYABLES": "true"}):
         client.queryables_cache.reload_settings()
+        client.queryables_cache._cache = {}
+        client.queryables_cache._last_updated = 0
         yield
     client.queryables_cache.reload_settings()
 
@@ -114,3 +116,56 @@ async def test_validate_queryables_excluded(app_client, ctx):
         assert resp.status_code == 200
 
     client.queryables_cache.reload_settings()
+
+
+@pytest.mark.asyncio
+async def test_search_get_cql2_text_invalid_param(app_client, ctx):
+    """Test GET /search with an invalid cql2-text filter parameter."""
+    params = {
+        "filter-lang": "cql2-text",
+        "filter": "properties.invalid_param < 5",
+    }
+    resp = await app_client.get("/search", params=params)
+    assert resp.status_code == 400
+    resp_json = resp.json()
+    assert "Invalid query fields: invalid_param" in resp_json["detail"]
+
+
+@pytest.mark.asyncio
+async def test_search_get_cql2_text_valid_param(app_client, ctx):
+    """Test GET /search with a valid cql2-text filter parameter."""
+    params = {
+        "filter-lang": "cql2-text",
+        "filter": "eo:cloud_cover < 10",
+    }
+    resp = await app_client.get("/search", params=params)
+    assert resp.status_code == 200
+
+
+@pytest.mark.asyncio
+async def test_item_collection_get_cql2_text_invalid_param(app_client, ctx):
+    """Test GET /collections/{collection_id}/items with invalid cql2-text filter."""
+    collection_id = ctx.item["collection"]
+    params = {
+        "filter-lang": "cql2-text",
+        "filter": "properties.invalid_param < 5",
+    }
+    resp = await app_client.get(f"/collections/{collection_id}/items", params=params)
+    assert resp.status_code == 400
+    resp_json = resp.json()
+    assert "Invalid query fields: invalid_param" in resp_json["detail"]
+
+
+@pytest.mark.asyncio
+async def test_search_get_cql2_text_with_properties_prefix(app_client, ctx):
+    """Test GET /search with a valid cql2-text filter using properties. prefix.
+
+    This tests the case where users specify 'properties.eo:cloud_cover' instead of
+    just 'eo:cloud_cover'. Both formats should work correctly.
+    """
+    params = {
+        "filter-lang": "cql2-text",
+        "filter": "properties.eo:cloud_cover < 10",
+    }
+    resp = await app_client.get("/search", params=params)
+    assert resp.status_code == 200

stac_fastapi/tests/core/test_queryables.py

@@ -116,3 +116,22 @@ def test_get_properties_from_cql2_filter():
 
     # Empty/invalid
     assert get_properties_from_cql2_filter({}) == set()
+
+
+def test_get_properties_from_cql2_filter_strips_properties_prefix():
+    """Test that 'properties.' prefix is stripped from property names."""
+    # Single property with prefix
+    cql2 = {"op": "<", "args": [{"property": "properties.none"}, 5]}
+    props = get_properties_from_cql2_filter(cql2)
+    assert props == {"none"}
+
+    # Mixed with and without prefix
+    cql2_nested = {
+        "op": "and",
+        "args": [
+            {"op": "=", "args": [{"property": "properties.test"}, "v1"]},
+            {"op": "<", "args": [{"property": "eo:cloud_cover"}, 10]},
+        ],
+    }
+    props = get_properties_from_cql2_filter(cql2_nested)
+    assert props == {"test", "eo:cloud_cover"}