Merge pull request #21 from Learning-Dashboard/hotfix/update-recovery-scripts-and-pattern-detectors Update recovery scripts, improve pattern detectors and taiga API call

datasources/requests/taiga_api_call.py

@@ -2,27 +2,18 @@
 import requests
 from datetime import datetime, timedelta, timezone
 from utils.taiga_token.taiga_auth import get_taiga_token
-from config.credentials_loader import (
-    CredentialsConfigError,
-    ProjectCredentialsNotFoundError,
-    resolve,
-)
-
-from config.settings import TAIGA_API_URL
+from config.credentials_loader import resolve
+from config.settings import TAIGA_API_URL, TAIGA_TOKEN
 
 _CACHE = {}                 # key = (project_id, milestone_id) -> (timestamp, stats)
 _DETAILS_CACHE = {}         # key = (project_id, milestone_id) -> (timestamp, details)
 _USERSTORY_CACHE = {}       # key = (project_id, userstory_id) -> (timestamp, details)
+_US_CUSTOM_ATTR_NAMES_CACHE = {}  # key = project_id -> (timestamp, {attr_id(str): attr_name})
+_TASK_CUSTOM_ATTR_NAMES_CACHE = {}  # key = project_id -> (timestamp, {attr_id(str): attr_name})
 TTL    = timedelta(minutes=1) # Cache time-to-live, set to 5 minutes. Means that if the same request is made within 5 minutes, it will return the cached result instead of making a new API call.
-logger = logging.getLogger(__name__)
-
-_CACHE = {}  # key = (project_id, milestone_id) -> (timestamp, stats)
-MILESTONE_TIMEOUT = (3, 8)
-TAIGA_LOOKUP_ERRORS = (
-    requests.exceptions.RequestException,
-    CredentialsConfigError,
-    ProjectCredentialsNotFoundError,
-)
+log = logging.getLogger(__name__)
+logger = log
+TAIGA_LOOKUP_ERRORS = (requests.RequestException,)
 
 
 def _empty_stats():
@@ -36,18 +27,122 @@
     }
 
 
-
 def _build_taiga_headers(prj: str):
-    """Return the Taiga headers needed for public and private deployments."""
-    if "api.taiga.io" in TAIGA_API_URL:
+    """Return Taiga headers for public, private and SSO deployments."""
+    if TAIGA_TOKEN:
+        return {"Authorization": f"Bearer {TAIGA_TOKEN}"}
+
+    try:
         user = resolve(prj, "taiga_user")
         psw = resolve(prj, "taiga_password")
-        if user and psw:
+    except KeyError:
+        log.warning("No Taiga credentials configured for project %s; using anonymous requests.", prj)
+        return {}
+
+    if user and psw:
+        try:
             token = get_taiga_token(user, psw)
             return {"Authorization": f"Bearer {token}"}
+        except requests.RequestException as exc:
+            log.warning("Failed to fetch Taiga token for project %s: %s", prj, exc)
+            return {}
+
+    log.warning("Incomplete Taiga credentials for project %s; using anonymous requests.", prj)
     return {}
 
 
+def _userstory_custom_attribute_names(project_id: str, prj: str):
+    """Return map of custom attribute id -> name for userstories in a project."""
+    if not project_id:
+        return {}
+
+    key = str(project_id)
+    now = datetime.now(timezone.utc)
+    if key in _US_CUSTOM_ATTR_NAMES_CACHE and now - _US_CUSTOM_ATTR_NAMES_CACHE[key][0] < TTL:
+        return _US_CUSTOM_ATTR_NAMES_CACHE[key][1]
+
+    headers = _build_taiga_headers(prj)
+    url = f"{TAIGA_API_URL}/userstory-custom-attributes"
+    try:
+        r = requests.get(url, params={"project": project_id}, headers=headers, timeout=(1, 5))
+        r.raise_for_status()
+        mapping = {str(item.get("id")): item.get("name") for item in (r.json() or []) if item.get("id") and item.get("name")}
+    except requests.RequestException as exc:
+        log.warning("Failed to fetch userstory custom attribute definitions for project %s: %s", project_id, exc)
+        mapping = {}
+
+    _US_CUSTOM_ATTR_NAMES_CACHE[key] = (now, mapping)
+    return mapping
+
+
+def _userstory_custom_values(project_id: str, userstory_id: str, prj: str):
+    """Fetch custom attribute values for a userstory and map IDs to attribute names."""
+    if not project_id or not userstory_id:
+        return {}
+
+    headers = _build_taiga_headers(prj)
+    url = f"{TAIGA_API_URL}/userstories/custom-attributes-values/{userstory_id}"
+    try:
+        r = requests.get(url, params={"project": project_id}, headers=headers, timeout=(1, 5))
+        r.raise_for_status()
+        raw_values = (r.json() or {}).get("attributes_values") or {}
+    except requests.RequestException as exc:
+        log.warning("Failed to fetch custom values for userstory %s in project %s: %s", userstory_id, project_id, exc)
+        return {}
+
+    names = _userstory_custom_attribute_names(project_id, prj)
+    mapped = {}
+    for attr_id, value in raw_values.items():
+        mapped[names.get(str(attr_id), str(attr_id))] = value
+    return mapped
+
+
+def _task_custom_attribute_names(project_id: str, prj: str):
+    """Return map of custom attribute id -> name for tasks in a project."""
+    if not project_id:
+        return {}
+
+    key = str(project_id)
+    now = datetime.now(timezone.utc)
+    if key in _TASK_CUSTOM_ATTR_NAMES_CACHE and now - _TASK_CUSTOM_ATTR_NAMES_CACHE[key][0] < TTL:
+        return _TASK_CUSTOM_ATTR_NAMES_CACHE[key][1]
+
+    headers = _build_taiga_headers(prj)
+    url = f"{TAIGA_API_URL}/task-custom-attributes"
+    try:
+        r = requests.get(url, params={"project": project_id}, headers=headers, timeout=(1, 5))
+        r.raise_for_status()
+        mapping = {str(item.get("id")): item.get("name") for item in (r.json() or []) if item.get("id") and item.get("name")}
+    except requests.RequestException as exc:
+        log.warning("Failed to fetch task custom attribute definitions for project %s: %s", project_id, exc)
+        mapping = {}
+
+    _TASK_CUSTOM_ATTR_NAMES_CACHE[key] = (now, mapping)
+    return mapping
+
+
+def _task_custom_values(project_id: str, task_id: str, prj: str):
+    """Fetch custom attribute values for a task and map IDs to attribute names."""
+    if not project_id or not task_id:
+        return {}
+
+    headers = _build_taiga_headers(prj)
+    url = f"{TAIGA_API_URL}/tasks/custom-attributes-values/{task_id}"
+    try:
+        r = requests.get(url, params={"project": project_id}, headers=headers, timeout=(1, 5))
+        r.raise_for_status()
+        raw_values = (r.json() or {}).get("attributes_values") or {}
+    except requests.RequestException as exc:
+        log.warning("Failed to fetch custom values for task %s in project %s: %s", task_id, project_id, exc)
+        return {}
+
+    names = _task_custom_attribute_names(project_id, prj)
+    mapped = {}
+    for attr_id, value in raw_values.items():
+        mapped[names.get(str(attr_id), str(attr_id))] = value
+    return mapped
+
+
 def milestone_details(project_id: str, milestone_id: str, prj: str):
     """
     Fetches the milestone metadata from Taiga.
@@ -94,6 +189,7 @@
     """
     Fetches the userstory metadata from Taiga.
     Used as a fallback when task payloads do not include the nested userstory state.
+    Also returns custom_attributes and description for recovery backfill.
     """
     if not project_id or not userstory_id:
         return {}
@@ -120,12 +216,39 @@
         return {}
 
     js = r.json()
+    custom_values = js.get("custom_attributes_values") or _userstory_custom_values(project_id, userstory_id, prj)
     details = {
         "userstory_is_closed": (js.get("status_extra_info") or {}).get("is_closed"),
+        "custom_attributes_values": custom_values or {},
+        "description": js.get("description") or "",
     }
     _USERSTORY_CACHE[key] = (now, details)
     return details
 
+def task_details(project_id: str, task_id: str, prj: str):
+    """
+    Fetches the task metadata from Taiga.
+    Returns custom_attributes with fallback to dedicated endpoint if empty.
+    """
+    if not project_id or not task_id:
+        return {}
+
+    headers = _build_taiga_headers(prj)
+    url = f"{TAIGA_API_URL}/tasks/{task_id}"
+    try:
+        r = requests.get(url, params={"project": project_id}, headers=headers, timeout=(1, 5))
+        r.raise_for_status()
+        js = r.json()
+    except requests.RequestException as exc:
+        log.warning("Failed to fetch task %s in project %s: %s", task_id, project_id, exc)
+        return {}
+
+    custom_values = js.get("custom_attributes_values") or _task_custom_values(project_id, task_id, prj)
+    details = {
+        "custom_attributes_values": custom_values or {},
+    }
+    return details
+
 def milestone_stats(project_id: str, milestone_id: str, prj: str):
     """
     Fetches the statistics of a milestone in a Taiga project.

routes/taiga_routes.py

@@ -72,6 +72,15 @@
             collection_name, delete_key, id, result.deleted_count
         )
 
+        # When a user story is deleted, also remove all its associated tasks
+        if event_type == "userstory":
+            tasks_coll = get_collection(f"taiga_{prj}.tasks")
+            task_result = tasks_coll.delete_many({"userstory_id": id})
+            logger.info(
+                "Cascade-deleted %s task(s) linked to userstory_id=%s in taiga_%s.tasks",
+                task_result.deleted_count, id, prj,
+            )
+
         author_login = raw_payload.get("by", {}).get("username", "unknown")
         logger.info(
             "Notifying LD_EVAL about deleted event: %s for team with external_id: %s with quality_model: %s",
@@ -81,6 +90,9 @@
         )
         try:
             notify_eval_push(event_type, prj, author_login, quality_model)
+            # Force a task re-eval when tasks were cascade-deleted with the user story
+            if event_type == "userstory":
+                notify_eval_push("task", prj, author_login, quality_model)
         except Exception as e:
             logger.error("Error notifying LD_EVAL: %s", e)
             return jsonify({"error": "Failed to notify LD_EVAL"}), 500

tests/test_taiga_routes.py

@@ -105,6 +105,43 @@ def test_delete_action(self, mock_verify, mock_coll, mock_notify, client):
         mock_collection.delete_one.assert_called_once_with({"task_id": 99})
         mock_notify.assert_called_once_with("task", "P", "u", None)
 
+    @patch("routes.taiga_routes.notify_eval_push")
+    @patch("routes.taiga_routes.get_collection")
+    @patch("routes.taiga_routes.verify_taiga_signature", return_value=True)
+    def test_delete_userstory_cascades_tasks(
+        self, mock_verify, mock_coll, mock_notify, client
+    ):
+        payload = {
+            "type": "userstory",
+            "action": "delete",
+            "data": {"id": 55, "project": {"name": "P"}},
+            "by": {"username": "u"},
+        }
+        mock_us_collection = MagicMock()
+        mock_tasks_collection = MagicMock()
+
+        def side_effect(name):
+            if name == "taiga_P.userstories":
+                return mock_us_collection
+            if name == "taiga_P.tasks":
+                return mock_tasks_collection
+            return MagicMock()
+
+        mock_coll.side_effect = side_effect
+
+        resp = client.post(
+            "/webhook/taiga?prj=P",
+            data=json.dumps(payload),
+            content_type="application/json",
+            headers={"X-TAIGA-WEBHOOK-SIGNATURE": "x"},
+        )
+        assert resp.status_code == 200
+        mock_us_collection.delete_one.assert_called_once_with({"userstory_id": 55})
+        mock_tasks_collection.delete_many.assert_called_once_with({"userstory_id": 55})
+        assert mock_notify.call_count == 2
+        mock_notify.assert_any_call("userstory", "P", "u", None)
+        mock_notify.assert_any_call("task", "P", "u", None)
+
     @patch("routes.taiga_routes.get_collection")
     @patch("routes.taiga_routes.verify_taiga_signature", return_value=True)
     def test_delete_no_id_returns_400(self, mock_verify, mock_coll, client):

utils/pattern_detector.py

@@ -9,14 +9,14 @@ class PatternDetector:
     # Patrones regex compilados para optimización
     PATTERNS = [
         # English
-        r"\bas\s+[\w\s]+\s+i\s+want\s+[\w\s,.:;!?-]+\s+so\s+that\s+[\w\s,.:;!?-]+",
-        r"\bas\s+[\w\s]+\s+i\s+want\s+[\w\s,.:;!?-]+\s+to\s+[\w\s,.:;!?-]+",
+        r"\bas\s+[\w\s'àáäâäèéëêìíïîòóöôùúüûñçÀÁÄÂÈÉËÊÌÍÏÎÒÓÖÔÙÚÜÛÑÇ’()\/·,.:;!?-]+\s+i\s+want\s+[\w\s'àáäâäèéëêìíïîòóöôùúüûñçÀÁÄÂÈÉËÊÌÍÏÎÒÓÖÔÙÚÜÛÑÇ’()\/·,.:;!?-]+\s+so\s+that\s+[\w\s'àáäâäèéëêìíïîòóöôùúüûñçÀÁÄÂÈÉËÊÌÍÏÎÒÓÖÔÙÚÜÛÑÇ’()\/·,.:;!?-]+",
+        r"\bas\s+[\w\s'àáäâäèéëêìíïîòóöôùúüûñçÀÁÄÂÈÉËÊÌÍÏÎÒÓÖÔÙÚÜÛÑÇ’()\/·,.:;!?-]+\s+i\s+want\s+[\w\s'àáäâäèéëêìíïîòóöôùúüûñçÀÁÄÂÈÉËÊÌÍÏÎÒÓÖÔÙÚÜÛÑÇ’()\/·,.:;!?-]+\s+to\s+[\w\s'àáäâäèéëêìíïîòóöôùúüûñçÀÁÄÂÈÉËÊÌÍÏÎÒÓÖÔÙÚÜÛÑÇ’()\/·,.:;!?-]+",
 
         # Spanish - COMO...QUIERO...
-        r"\bcomo\s+[\w\s]+\s+quiero\s+[\w\s,.:;!?-]+\s+(?:de\s+manera\s+que|de\s+forma\s+que|para|por|porqu[eé]|porque)\s+[\w\s,.:;!?-]+",
+        r"\bcomo\s+[\w\s'àáäâäèéëêìíïîòóöôùúüûñçÀÁÄÂÈÉËÊÌÍÏÎÒÓÖÔÙÚÜÛÑÇ’()\/·,.:;!?-]+\s+quiero\s+[\w\s'àáäâäèéëêìíïîòóöôùúüûñçÀÁÄÂÈÉËÊÌÍÏÎÒÓÖÔÙÚÜÛÑÇ’()\/·,.:;!?-]+\s+(?:de\s+manera\s+que|de\s+forma\s+que|para|por|porqu[eé]|porque)\s+[\w\s'àáäâäèéëêìíïîòóöôùúüûñçÀÁÄÂÈÉËÊÌÍÏÎÒÓÖÔÙÚÜÛÑÇ’()\/·,.:;!?-]+",
 
         # Catalan - COM...VULL...
-        r"\bcom\s+[\w\s]+\s+vull\s+[\w\s,.:;!?-]+\s+(?:de\s+manera\s+que|de\s+forma\s+que|per|perqu[eè]|perqué)\s+[\w\s,.:;!?-]+",
+        r"\bcom\s+(?:a\s+)?[\w\s'àáäâäèéëêìíïîòóöôùúüûñçÀÁÄÂÈÉËÊÌÍÏÎÒÓÖÔÙÚÜÛÑÇ’()\/·,.:;!?-]+\s+vull\s+[\w\s'àáäâäèéëêìíïîòóöôùúüûñçÀÁÄÂÈÉËÊÌÍÏÎÒÓÖÔÙÚÜÛÑÇ’()\/·,.:;!?-]+\s+(?:de\s+manera\s+que|de\s+forma\s+que|per\s+a\s+poder|per\s+poder|per\s+tal\s+de|per\s+tal\s+d[’']|per|perqu[eè]|perqué)\s*[\w\s'àáäâäèéëêìíïîòóöôùúüûñçÀÁÄÂÈÉËÊÌÍÏÎÒÓÖÔÙÚÜÛÑÇ’()\/·,.:;!?-]+",
     ]
 
     # Compilar patrones una sola vez