Added pre-upgrade validation for N9K-C9408 with more than 6 N9K-X9400-16W LEM's for the bug CSCws82819

aci-preupgrade-validation-script.py

@@ -6142,6 +6142,55 @@ def is_affected_target(ver):
     return Result(result=result, headers=headers, data=data, recommended_action=recommended_action, doc_url=doc_url)
 
 
+@check_wrapper(check_title='N9K-C9408 with more than 5 N9K-X9400-16W LEMs')
+def n9k_c9408_model_lem_count_check(tversion, fabric_nodes, **kwargs):
+    result = PASS
+    headers = ["Node ID", "Switch Model", "LEM Model", "LEM Count"]
+    data = []
+    recommended_action = (
+        "N9K-C9408 switches configured with >5 N9K-X9400-16W LEMs will enter a boot loop if upgraded to impacted release of CSCws82819. Upgrade to Fix version or Use less than 6 LEMS on impacted release"
+    )
+    doc_url = 'https://datacenter.github.io/ACI-Pre-Upgrade-Validation-Script/validations/#n9k-c9408-with-more-than-5-n9k-x9400-16w-lems'
+
+    if not tversion:
+        return Result(result=MANUAL, msg=TVER_MISSING)
+
+    if tversion.older_than("6.1(2f)") or (tversion.newer_than("6.1(5e)") and not tversion.same_as("6.2(1g)")):
+        return Result(result=NA, msg=VER_NOT_AFFECTED)
+
+    affected_nodes = {}
+    for node in fabric_nodes:
+        node_id = node['fabricNode']['attributes']['id']
+        model = node['fabricNode']['attributes']['model']
+        if model == "N9K-C9408":
+            affected_nodes[node_id] = "N9K-C9408"
+
+    if not affected_nodes:
+        return Result(result=PASS, msg='No N9K-C9408 nodes found. Skipping.')
+
+    eqptLC_api = 'eqptLC.json?query-target-filter=eq(eqptLC.model,"N9K-X9400-16W")'
+    eqptLCs = icurl('class', eqptLC_api)
+
+    lem_count_per_node = defaultdict(int)
+    for eqptLC in eqptLCs:
+        dn = eqptLC['eqptLC']['attributes']['dn']
+        dn_match = re.search(node_regex, dn)
+        if not dn_match:
+            continue
+        node_id = dn_match.group("node")
+        if node_id in affected_nodes:
+            lem_count_per_node[node_id] += 1
+
+    for node_id in sorted(affected_nodes, key=int):
+        lem_count = lem_count_per_node[node_id]
+        if lem_count > 5:
+            data.append([node_id, affected_nodes[node_id], "N9K-X9400-16W", lem_count])
+
+    if data:
+        result = FAIL_O
+
+    return Result(result=result, headers=headers, data=data, recommended_action=recommended_action, doc_url=doc_url)
+
 # ---- Script Execution ----
 
 
@@ -6307,6 +6356,7 @@ class CheckManager:
         configpush_shard_check,
         auto_firmware_update_on_switch_check,
         rogue_ep_coop_exception_mac_check,
+        n9k_c9408_model_lem_count_check,
     ]
     ssh_checks = [
         # General
@@ -6478,3 +6528,4 @@ def main(_args=None):
         prints(msg)
         log.error(msg, exc_info=True)
         sys.exit(1)
+

docs/docs/validations.md

@@ -195,6 +195,7 @@ Items                                           | Defect       | This Script
 [Policydist configpushShardCont Crash][d28]     | CSCwp95515   | :white_check_mark: | :no_entry_sign:
 [Auto Firmware Update on Switch Discovery][d29] | CSCwe83941   | :white_check_mark: | :no_entry_sign:
 [Rogue EP Exception List missing on switches][d30] | CSCwp64296   | :white_check_mark: | :no_entry_sign:
+[N9K-C9408 with more than 5 N9K-X9400-16W LEMs][d31] | CSCws82819   | :white_check_mark: | :no_entry_sign:
 
 [d1]: #ep-announce-compatibility
 [d2]: #eventmgr-db-size-defect-susceptibility
@@ -226,7 +227,7 @@ Items                                           | Defect       | This Script
 [d28]: #policydist-configpushshardcont-crash
 [d29]: #auto-firmware-update-on-switch-discovery
 [d30]: #rogue-ep-exception-list-missing-on-switches
-
+[d31]: #n9k-c9408-with-more-than-5-n9k-x9400-16w-lems
 
 ## General Check Details
 
@@ -2683,6 +2684,13 @@ The root cause is that internal objects called `presListener` for Rogue/COOP Exc
 Recommended action: Delete the affected exception list and create it again. If needed, contact Cisco TAC to help recover missing `presListener` objects on APICs.
 
 
+### N9K-C9408 with more than 5 N9K-X9400-16W LEMs
+
+Due to defect [CSCws82819][65], N9K-C9408 switch will experience a boot loop with dt_helper process crash if upgraded to versions 16.1(2f) to 16.1(5) or 16.2(1g) with more than 5 N9K-X9400-16W LEMs installed.
+
+To avoid this issue, please upgrade to fix version or use less than 6 N9K-X9400-16W in one chassis.
+
+
 [0]: https://github.com/datacenter/ACI-Pre-Upgrade-Validation-Script
 [1]: https://www.cisco.com/c/dam/en/us/td/docs/Website/datacenter/apicmatrix/index.html
 [2]: https://www.cisco.com/c/en/us/support/switches/nexus-9000-series-switches/products-release-notes-list.html
@@ -2747,4 +2755,5 @@ Recommended action: Delete the affected exception list and create it again. If n
 [61]: https://www.cisco.com/c/en/us/solutions/collateral/data-center-virtualization/application-centric-infrastructure/white-paper-c11-743951.html#EnablePolicyCompression
 [62]: https://bst.cloudapps.cisco.com/bugsearch/bug/CSCwe83941
 [63]: https://www.cisco.com/c/en/us/td/docs/dcn/aci/apic/all/apic-installation-aci-upgrade-downgrade/Cisco-APIC-Installation-ACI-Upgrade-Downgrade-Guide/m-auto-firmware-update.html
-[64]: https://bst.cloudapps.cisco.com/bugsearch/bug/CSCwp64296
+[64]: https://bst.cloudapps.cisco.com/bugsearch/bug/CSCwp64296
+[65]: https://bst.cloudapps.cisco.com/bugsearch/bug/CSCws82819

tests/checks/n9k_c9408_model_lem_count_check/eqptLC_5_node.json

@@ -0,0 +1,7 @@
+[
+  {"eqptLC": {"attributes": {"dn": "topology/pod-1/node-101/sys/ch/lcslot-1/lc", "model": "N9K-X9400-16W"}}},
+  {"eqptLC": {"attributes": {"dn": "topology/pod-1/node-101/sys/ch/lcslot-2/lc", "model": "N9K-X9400-16W"}}},
+  {"eqptLC": {"attributes": {"dn": "topology/pod-1/node-101/sys/ch/lcslot-3/lc", "model": "N9K-X9400-16W"}}},
+  {"eqptLC": {"attributes": {"dn": "topology/pod-1/node-101/sys/ch/lcslot-4/lc", "model": "N9K-X9400-16W"}}},
+  {"eqptLC": {"attributes": {"dn": "topology/pod-1/node-101/sys/ch/lcslot-5/lc", "model": "N9K-X9400-16W"}}}
+]

tests/checks/n9k_c9408_model_lem_count_check/eqptLC_6_node.json

@@ -0,0 +1,8 @@
+[
+  {"eqptLC": {"attributes": {"dn": "topology/pod-1/node-101/sys/ch/lcslot-1/lc", "model": "N9K-X9400-16W"}}},
+  {"eqptLC": {"attributes": {"dn": "topology/pod-1/node-101/sys/ch/lcslot-2/lc", "model": "N9K-X9400-16W"}}},
+  {"eqptLC": {"attributes": {"dn": "topology/pod-1/node-101/sys/ch/lcslot-3/lc", "model": "N9K-X9400-16W"}}},
+  {"eqptLC": {"attributes": {"dn": "topology/pod-1/node-101/sys/ch/lcslot-4/lc", "model": "N9K-X9400-16W"}}},
+  {"eqptLC": {"attributes": {"dn": "topology/pod-1/node-101/sys/ch/lcslot-5/lc", "model": "N9K-X9400-16W"}}},
+  {"eqptLC": {"attributes": {"dn": "topology/pod-1/node-101/sys/ch/lcslot-6/lc", "model": "N9K-X9400-16W"}}}
+]

tests/checks/n9k_c9408_model_lem_count_check/eqptLC_7_node.json

@@ -0,0 +1,58 @@
+[
+  {
+    "eqptLC": {
+      "attributes": {
+        "dn": "topology/pod-1/node-101/sys/ch/lcslot-1/lc",
+        "model": "N9K-X9400-16W"
+      }
+    }
+  },
+  {
+    "eqptLC": {
+      "attributes": {
+        "dn": "topology/pod-1/node-101/sys/ch/lcslot-2/lc",
+        "model": "N9K-X9400-16W"
+      }
+    }
+  },
+  {
+    "eqptLC": {
+      "attributes": {
+        "dn": "topology/pod-1/node-101/sys/ch/lcslot-3/lc",
+        "model": "N9K-X9400-16W"
+      }
+    }
+  },
+  {
+    "eqptLC": {
+      "attributes": {
+        "dn": "topology/pod-1/node-101/sys/ch/lcslot-4/lc",
+        "model": "N9K-X9400-16W"
+      }
+    }
+  },
+  {
+    "eqptLC": {
+      "attributes": {
+        "dn": "topology/pod-1/node-101/sys/ch/lcslot-5/lc",
+        "model": "N9K-X9400-16W"
+      }
+    }
+  },
+  {
+    "eqptLC": {
+      "attributes": {
+        "dn": "topology/pod-1/node-101/sys/ch/lcslot-6/lc",
+        "model": "N9K-X9400-16W"
+      }
+    }
+  },
+  {
+    "eqptLC": {
+      "attributes": {
+        "dn": "topology/pod-1/node-101/sys/ch/lcslot-7/lc",
+        "model": "N9K-X9400-16W"
+      }
+    }
+  }
+]

tests/checks/n9k_c9408_model_lem_count_check/eqptLC_empty.json

@@ -0,0 +1 @@
+[]

tests/checks/n9k_c9408_model_lem_count_check/eqptLC_mixed.json

@@ -0,0 +1,15 @@
+[
+  {"eqptLC": {"attributes": {"dn": "topology/pod-1/node-101/sys/ch/lcslot-1/lc", "model": "N9K-X9400-16W"}}},
+  {"eqptLC": {"attributes": {"dn": "topology/pod-1/node-101/sys/ch/lcslot-2/lc", "model": "N9K-X9400-16W"}}},
+  {"eqptLC": {"attributes": {"dn": "topology/pod-1/node-101/sys/ch/lcslot-3/lc", "model": "N9K-X9400-16W"}}},
+  {"eqptLC": {"attributes": {"dn": "topology/pod-1/node-101/sys/ch/lcslot-4/lc", "model": "N9K-X9400-16W"}}},
+  {"eqptLC": {"attributes": {"dn": "topology/pod-1/node-101/sys/ch/lcslot-5/lc", "model": "N9K-X9400-16W"}}},
+  {"eqptLC": {"attributes": {"dn": "topology/pod-1/node-101/sys/ch/lcslot-6/lc", "model": "N9K-X9400-16W"}}},
+  {"eqptLC": {"attributes": {"dn": "topology/pod-1/node-102/sys/ch/lcslot-1/lc", "model": "N9K-X9400-16W"}}},
+  {"eqptLC": {"attributes": {"dn": "topology/pod-1/node-102/sys/ch/lcslot-2/lc", "model": "N9K-X9400-16W"}}},
+  {"eqptLC": {"attributes": {"dn": "topology/pod-1/node-102/sys/ch/lcslot-3/lc", "model": "N9K-X9400-16W"}}},
+  {"eqptLC": {"attributes": {"dn": "topology/pod-1/node-102/sys/ch/lcslot-4/lc", "model": "N9K-X9400-16W"}}},
+  {"eqptLC": {"attributes": {"dn": "topology/pod-1/node-102/sys/ch/lcslot-5/lc", "model": "N9K-X9400-16W"}}},
+  {"eqptLC": {"attributes": {"dn": "topology/pod-1/node-102/sys/ch/lcslot-6/lc", "model": "N9K-X9400-16W"}}},
+  {"eqptLC": {"attributes": {"dn": "topology/pod-1/node-201/sys/ch/lcslot-1/lc", "model": "N9K-X9400-16W"}}}
+]

tests/checks/n9k_c9408_model_lem_count_check/fabricNode_mixed.json

@@ -0,0 +1,26 @@
+[
+  {
+    "fabricNode": {
+      "attributes": {
+        "id": "101",
+        "model": "N9K-C9408"
+      }
+    }
+  },
+  {
+    "fabricNode": {
+      "attributes": {
+        "id": "102",
+        "model": "N9K-C9408"
+      }
+    }
+  },
+  {
+    "fabricNode": {
+      "attributes": {
+        "id": "201",
+        "model": "N9K-C93180YC-FX"
+      }
+    }
+  }
+]

tests/checks/n9k_c9408_model_lem_count_check/fabricNode_n9k_c9408.json

@@ -0,0 +1,10 @@
+[
+  {
+    "fabricNode": {
+      "attributes": {
+        "id": "101",
+        "model": "N9K-C9408"
+      }
+    }
+  }
+]

tests/checks/n9k_c9408_model_lem_count_check/fabricNode_no_n9k_c9408.json

@@ -0,0 +1,10 @@
+[
+  {
+    "fabricNode": {
+      "attributes": {
+        "id": "101",
+        "model": "N9K-C93180YC-FX"
+      }
+    }
+  }
+]

tests/checks/n9k_c9408_model_lem_count_check/test_n9k_c9408_model_lem_count_check.py

@@ -0,0 +1,140 @@
+import os
+import pytest
+import logging
+import importlib
+from helpers.utils import read_data
+
+script = importlib.import_module("aci-preupgrade-validation-script")
+
+log = logging.getLogger(__name__)
+dir = os.path.dirname(os.path.abspath(__file__))
+
+test_function = "n9k_c9408_model_lem_count_check"
+
+# icurl queries
+eqptLC_api = 'eqptLC.json?query-target-filter=eq(eqptLC.model,"N9K-X9400-16W")'
+
+
+@pytest.mark.parametrize(
+    "icurl_outputs, tversion, fabric_nodes, expected_result, expected_data, expected_msg",
+    [
+        # tversion missing
+        ({}, None, [], script.MANUAL, [], script.TVER_MISSING),
+        # Version not affected (lower than 6.1(2f))
+        (
+            {eqptLC_api: read_data(dir, "eqptLC_empty.json")},
+            "6.1(2e)",
+            read_data(dir, "fabricNode_n9k_c9408.json"),
+            script.NA,
+            [],
+            script.VER_NOT_AFFECTED,
+        ),
+        # Version not affected (higher than 6.2(1g))
+        (
+            {eqptLC_api: read_data(dir, "eqptLC_empty.json")},
+            "6.2(1h)",
+            read_data(dir, "fabricNode_n9k_c9408.json"),
+            script.NA,
+            [],
+            script.VER_NOT_AFFECTED,
+        ),
+        # Applicable version but no N9K-C9408 nodes found
+        (
+            {eqptLC_api: read_data(dir, "eqptLC_6_node.json")},
+            "6.1(2f)",
+            read_data(dir, "fabricNode_no_n9k_c9408.json"),
+            script.PASS,
+            [],
+            "No N9K-C9408 nodes found. Skipping.",
+        ),
+        # Applicable version, C9408 exists, no LEM entries
+        (
+            {eqptLC_api: read_data(dir, "eqptLC_empty.json")},
+            "6.1(2f)",
+            read_data(dir, "fabricNode_n9k_c9408.json"),
+            script.PASS,
+            [],
+            "",
+        ),
+        # Applicable version, C9408 exists, with <=5 LEMs -> PASS
+        (
+            {eqptLC_api: read_data(dir, "eqptLC_5_node.json")},
+            "6.2(1g)",
+            read_data(dir, "fabricNode_n9k_c9408.json"),
+            script.PASS,
+            [],
+            "",
+        ),
+        # Applicable version with 6 LEMs -> FAIL_O
+        (
+            {eqptLC_api: read_data(dir, "eqptLC_6_node.json")},
+            "6.2(1g)",
+            read_data(dir, "fabricNode_n9k_c9408.json"),
+            script.FAIL_O,
+            [["101", "N9K-C9408", "N9K-X9400-16W", 6]],
+            "",
+        ),
+        # Applicable mid-train version 6.1(5e), less than 6 LEMs on C9408 -> PASS
+        (
+            {eqptLC_api: read_data(dir, "eqptLC_5_node.json")},
+            "6.1(5e)",
+            read_data(dir, "fabricNode_n9k_c9408.json"),
+            script.PASS,
+            [],
+            "",
+        ),
+        # Applicable mid-train version 6.1(5e), 6 LEMs on C9408 -> FAIL_O
+        (
+            {eqptLC_api: read_data(dir, "eqptLC_6_node.json")},
+            "6.1(5e)",
+            read_data(dir, "fabricNode_n9k_c9408.json"),
+            script.FAIL_O,
+            [["101", "N9K-C9408", "N9K-X9400-16W", 6]],
+            "",
+        ),
+        # Version not affected (fixed in 6.1(6)+)
+        (
+            {eqptLC_api: read_data(dir, "eqptLC_6_node.json")},
+            "6.1(6a)",
+            read_data(dir, "fabricNode_n9k_c9408.json"),
+            script.NA,
+            [],
+            script.VER_NOT_AFFECTED,
+        ),
+        # Applicable version, 6 LEMs on C9408 -> FAIL_O
+        (
+            {eqptLC_api: read_data(dir, "eqptLC_6_node.json")},
+            "6.1(2f)",
+            read_data(dir, "fabricNode_n9k_c9408.json"),
+            script.FAIL_O,
+            [["101", "N9K-C9408", "N9K-X9400-16W", 6]],
+            "",
+        ),
+        # Count only C9408 nodes and only matching LEM model
+        (
+            {eqptLC_api: read_data(dir, "eqptLC_mixed.json")},
+            "6.1(3a)",
+            read_data(dir, "fabricNode_mixed.json"),
+            script.FAIL_O,
+            [
+                ["101", "N9K-C9408", "N9K-X9400-16W", 6],
+                ["102", "N9K-C9408", "N9K-X9400-16W", 6],
+            ],
+            "",
+        ),
+    ],
+)
+def test_logic(
+    run_check, mock_icurl, icurl_outputs, tversion, fabric_nodes, expected_result, expected_data, expected_msg
+):
+    result = run_check(
+        tversion=script.AciVersion(tversion) if tversion else None,
+        fabric_nodes=fabric_nodes,
+    )
+
+    assert result.result == expected_result
+    assert result.data == expected_data
+    assert result.msg == expected_msg
+
+    if expected_result == script.FAIL_O:
+        assert "boot loop" in result.recommended_action