diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 567c4954..1ed9cc7d 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -46,10 +46,13 @@ jobs: uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae with: path: .pytest_cache - key: pytest-cache-${{ runner.os }}-py${{ matrix.python-version }}-${{ github.ref_name }}-${{ github.sha }} + key: pytest-cache-${{ runner.os }}-py${{ matrix.python-version }}-${{ github.ref_name }}-${{ + github.sha }} restore-keys: | pytest-cache-${{ runner.os }}-py${{ matrix.python-version }}-${{ github.ref_name }}- - name: Run unit tests run: make test-unit - name: Run entire test suite - run: make test-integration + # run: make test-integration + run: uv run pytest + ./tests/system/test_ai_agentic_test_app.py::TestAgenticApp::test_mcp_server_app_returns_tags diff --git a/tests/system/__init__.py b/tests/system/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/system/test_ai_agentic_test_app.py b/tests/system/test_ai_agentic_test_app.py index f45dae26..1b3e9878 100644 --- a/tests/system/test_ai_agentic_test_app.py +++ b/tests/system/test_ai_agentic_test_app.py @@ -107,6 +107,36 @@ def test_agentic_app_with_remote_tools(self) -> None: app.delete() self.restart_splunk() # app removal requires a restart + def test_mcp_server_app_returns_tags(self) -> None: + pytest.importorskip("langchain_openai") + self.requires_splunk_10_2() + + # Skip test in case the instance does not have a /splunk-mcp-server.tgz file. + try: + resp = self.service.get("agentic_app/has_mcp_app_file") + assert resp.status == 200 + except HTTPError as e: + if e.status == 404: + pytest.skip("Splunk MCP Server App file not found on Splunk instance") + raise + + app = self.service.apps.create( # pyright: ignore[reportUnknownVariableType] + name="/splunk-mcp-server.tgz", filename=True + ) + + try: + resp = self.service.post( + "agentic_app/tool-tags", + body=self.test_llm_settings.model_dump_json(), + ) + assert resp.status == 200 + + body = str(resp.body) # pyright: ignore[reportUnknownArgumentType] + raise Exception(body) + finally: + app.delete() + self.restart_splunk() # App removal requires a restart + def requires_splunk_10_2(self) -> None: if self.service.splunk_version[0] < 10 or self.service.splunk_version[1] < 2: pytest.skip("Python 3.13 not available on splunk < 10.2") diff --git a/tests/system/test_apps/ai_agentic_test_app/bin/indexes.py b/tests/system/test_apps/ai_agentic_test_app/bin/indexes.py index 5dbc8650..2f488557 100644 --- a/tests/system/test_apps/ai_agentic_test_app/bin/indexes.py +++ b/tests/system/test_apps/ai_agentic_test_app/bin/indexes.py @@ -14,12 +14,11 @@ import os import sys +from typing import override sys.path.insert(0, "/splunklib-deps") sys.path.insert(0, os.path.join(os.path.dirname(__file__), "lib")) -from typing import override - from pydantic import BaseModel, Field from splunklib.ai.agent import Agent diff --git a/tests/system/test_apps/ai_agentic_test_app/bin/tool_tags.py b/tests/system/test_apps/ai_agentic_test_app/bin/tool_tags.py new file mode 100644 index 00000000..5e93dc03 --- /dev/null +++ b/tests/system/test_apps/ai_agentic_test_app/bin/tool_tags.py @@ -0,0 +1,63 @@ +# Copyright © 2011-2026 Splunk, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"): you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import os +import sys +from typing import override +from uuid import uuid4 + +sys.path.insert(0, "/splunklib-deps") +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "lib")) + + +from splunklib.ai.agent import ( + _get_splunk_username, # pyright: ignore[reportPrivateUsage] +) +from splunklib.ai.tools import ( + _list_all_tools, # pyright: ignore[reportPrivateUsage] + connect_remote_mcp, +) +from tests.cre_testlib import CRETestHandler + +# BUG: For some reason the CRE process is started with a overridden trust store path, that +# does not exist on the filesystem. As a workaround in such case if it does not exist, +# remove the env, this causes the default CAs to be used instead. +CA_TRUST_STORE = "/opt/splunk/openssl/cert.pem" +if os.environ.get("SSL_CERT_FILE") == CA_TRUST_STORE and not os.path.exists( + CA_TRUST_STORE +): + os.environ["SSL_CERT_FILE"] = "" + +# This handler connects directly to the Splunk MCP Server App and logs the raw +# MCP tool list response, verifying that tool metadata includes tags. + + +class ToolTagsHandler(CRETestHandler): + @override + async def run(self) -> None: + import asyncio + + splunk_username = await asyncio.to_thread( + lambda: _get_splunk_username(self.service) + ) + + async with connect_remote_mcp( + service=self.service, + app_id="ai_agentic_test_app", + trace_id=str(uuid4()), + splunk_username=splunk_username, + ) as session: + assert session is not None, "MCP Server App not available" + raw_tools = await _list_all_tools(session) + self.response.write(f"[{','.join(rt.model_dump_json() for rt in raw_tools)}]") diff --git a/tests/system/test_apps/ai_agentic_test_app/default/restmap.conf b/tests/system/test_apps/ai_agentic_test_app/default/restmap.conf index 5e16d9eb..fbf35ae0 100644 --- a/tests/system/test_apps/ai_agentic_test_app/default/restmap.conf +++ b/tests/system/test_apps/ai_agentic_test_app/default/restmap.conf @@ -15,3 +15,9 @@ match = /agentic_app/has_mcp_app_file scripttype = python handler = mcp_app_file_exists.Handler python.required = 3.13 + +[script:tool_tags] +match = /agentic_app/tool-tags +scripttype = python +handler = tool_tags.ToolTagsHandler +python.required = 3.13