diff --git a/.github/actions/sample-validation-setup/action.yml b/.github/actions/sample-validation-setup/action.yml new file mode 100644 index 0000000000..3736348579 --- /dev/null +++ b/.github/actions/sample-validation-setup/action.yml @@ -0,0 +1,48 @@ +name: Sample Validation Setup +description: Sets up the environment for sample validation (checkout, Node.js, Copilot CLI, Azure login, Python) + +inputs: + azure-client-id: + description: Azure Client ID for OIDC login + required: true + azure-tenant-id: + description: Azure Tenant ID for OIDC login + required: true + azure-subscription-id: + description: Azure Subscription ID for OIDC login + required: true + python-version: + description: The Python version to set up + required: false + default: "3.12" + os: + description: The operating system to set up + required: false + default: "Linux" + +runs: + using: "composite" + steps: + - name: Set up Node.js environment + uses: actions/setup-node@v4 + + - name: Install Copilot CLI + shell: bash + run: npm install -g @github/copilot + + - name: Test Copilot CLI + shell: bash + run: copilot -p "What can you do in one sentence?" + + - name: Azure CLI Login + uses: azure/login@v2 + with: + client-id: ${{ inputs.azure-client-id }} + tenant-id: ${{ inputs.azure-tenant-id }} + subscription-id: ${{ inputs.azure-subscription-id }} + + - name: Set up python and install the project + uses: ./.github/actions/python-setup + with: + python-version: ${{ inputs.python-version }} + os: ${{ inputs.os }} diff --git a/.github/workflows/python-sample-validation.yml b/.github/workflows/python-sample-validation.yml index ba43394483..1ada1ab113 100644 --- a/.github/workflows/python-sample-validation.yml +++ b/.github/workflows/python-sample-validation.yml @@ -8,32 +8,38 @@ on: env: # Configure a constant location for the uv cache UV_CACHE_DIR: /tmp/.uv-cache + # GitHub Copilot configuration + GITHUB_COPILOT_MODEL: claude-opus-4.6 + COPILOT_GITHUB_TOKEN: ${{ secrets.COPILOT_GITHUB_TOKEN }} + +permissions: + contents: read + id-token: write jobs: validate-01-get-started: name: Validate 01-get-started runs-on: ubuntu-latest - permissions: - contents: read + environment: integration env: - # Azure AI configuration for get-started samples - AZURE_AI_PROJECT_ENDPOINT: ${{ secrets.AZURE_AI_PROJECT_ENDPOINT }} - AZURE_AI_MODEL_DEPLOYMENT_NAME: ${{ secrets.AZURE_AI_MODEL_DEPLOYMENT_NAME }} - # GitHub Copilot configuration - GITHUB_COPILOT_MODEL: ${{ vars.GITHUB_COPILOT_MODEL }} + # Required configuration for get-started samples + AZURE_AI_PROJECT_ENDPOINT: ${{ vars.AZURE_AI_PROJECT_ENDPOINT }} + AZURE_OPENAI_RESPONSES_DEPLOYMENT_NAME: ${{ vars.AZUREOPENAI__RESPONSESDEPLOYMENTNAME }} + AZURE_OPENAI_ENDPOINT: ${{ vars.AZUREOPENAI__ENDPOINT }} + AZURE_OPENAI_CHAT_DEPLOYMENT_NAME: ${{ vars.AZUREOPENAI__CHATDEPLOYMENTNAME }} defaults: run: working-directory: python steps: - uses: actions/checkout@v6 - - name: Set up python and install the project - uses: ./.github/actions/python-setup + - name: Setup environment + uses: ./.github/actions/sample-validation-setup with: - python-version: "3.12" + azure-client-id: ${{ secrets.AZURE_CLIENT_ID }} + azure-tenant-id: ${{ secrets.AZURE_TENANT_ID }} + azure-subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} os: ${{ runner.os }} - env: - UV_CACHE_DIR: /tmp/.uv-cache - name: Run sample validation run: | @@ -49,37 +55,34 @@ jobs: validate-02-agents: name: Validate 02-agents runs-on: ubuntu-latest - permissions: - contents: read + environment: integration env: # Azure AI configuration - AZURE_AI_PROJECT_ENDPOINT: ${{ secrets.AZURE_AI_PROJECT_ENDPOINT }} - AZURE_AI_MODEL_DEPLOYMENT_NAME: ${{ secrets.AZURE_AI_MODEL_DEPLOYMENT_NAME }} + AZURE_AI_PROJECT_ENDPOINT: ${{ vars.AZURE_AI_PROJECT_ENDPOINT }} + AZURE_AI_MODEL_DEPLOYMENT_NAME: ${{ vars.AZUREOPENAI__RESPONSESDEPLOYMENTNAME }} # Azure OpenAI configuration - AZURE_OPENAI_ENDPOINT: ${{ secrets.AZURE_OPENAI_ENDPOINT }} - AZURE_OPENAI_CHAT_DEPLOYMENT_NAME: ${{ secrets.AZURE_OPENAI_CHAT_DEPLOYMENT_NAME }} - AZURE_OPENAI_RESPONSES_DEPLOYMENT_NAME: ${{ secrets.AZURE_OPENAI_RESPONSES_DEPLOYMENT_NAME }} + AZURE_OPENAI_ENDPOINT: ${{ vars.AZUREOPENAI__ENDPOINT }} + AZURE_OPENAI_CHAT_DEPLOYMENT_NAME: ${{ vars.AZUREOPENAI__CHATDEPLOYMENTNAME }} + AZURE_OPENAI_RESPONSES_DEPLOYMENT_NAME: ${{ vars.AZUREOPENAI__RESPONSESDEPLOYMENTNAME }} # OpenAI configuration OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} - OPENAI_CHAT_MODEL_ID: ${{ vars.OPENAI_CHAT_MODEL_ID }} - OPENAI_RESPONSES_MODEL_ID: ${{ vars.OPENAI_RESPONSES_MODEL_ID }} + OPENAI_CHAT_MODEL_ID: ${{ vars.OPENAI_CHAT_MODEL_NAME }} + OPENAI_RESPONSES_MODEL_ID: ${{ vars.OPENAI_REASONING_MODEL_NAME }} # Observability ENABLE_INSTRUMENTATION: "true" - # GitHub Copilot configuration - GITHUB_COPILOT_MODEL: ${{ vars.GITHUB_COPILOT_MODEL }} defaults: run: working-directory: python steps: - uses: actions/checkout@v6 - - name: Set up python and install the project - uses: ./.github/actions/python-setup + - name: Setup environment + uses: ./.github/actions/sample-validation-setup with: - python-version: "3.12" + azure-client-id: ${{ secrets.AZURE_CLIENT_ID }} + azure-tenant-id: ${{ secrets.AZURE_TENANT_ID }} + azure-subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} os: ${{ runner.os }} - env: - UV_CACHE_DIR: /tmp/.uv-cache - name: Run sample validation run: | @@ -95,31 +98,28 @@ jobs: validate-03-workflows: name: Validate 03-workflows runs-on: ubuntu-latest - permissions: - contents: read + environment: integration env: # Azure AI configuration - AZURE_AI_PROJECT_ENDPOINT: ${{ secrets.AZURE_AI_PROJECT_ENDPOINT }} - AZURE_AI_MODEL_DEPLOYMENT_NAME: ${{ secrets.AZURE_AI_MODEL_DEPLOYMENT_NAME }} + AZURE_AI_PROJECT_ENDPOINT: ${{ vars.AZURE_AI_PROJECT_ENDPOINT }} + AZURE_AI_MODEL_DEPLOYMENT_NAME: ${{ vars.AZUREOPENAI__RESPONSESDEPLOYMENTNAME }} # Azure OpenAI configuration - AZURE_OPENAI_ENDPOINT: ${{ secrets.AZURE_OPENAI_ENDPOINT }} - AZURE_OPENAI_CHAT_DEPLOYMENT_NAME: ${{ secrets.AZURE_OPENAI_CHAT_DEPLOYMENT_NAME }} - AZURE_OPENAI_RESPONSES_DEPLOYMENT_NAME: ${{ secrets.AZURE_OPENAI_RESPONSES_DEPLOYMENT_NAME }} - # GitHub Copilot configuration - GITHUB_COPILOT_MODEL: ${{ vars.GITHUB_COPILOT_MODEL }} + AZURE_OPENAI_ENDPOINT: ${{ vars.AZUREOPENAI__ENDPOINT }} + AZURE_OPENAI_CHAT_DEPLOYMENT_NAME: ${{ vars.AZUREOPENAI__CHATDEPLOYMENTNAME }} + AZURE_OPENAI_RESPONSES_DEPLOYMENT_NAME: ${{ vars.AZUREOPENAI__RESPONSESDEPLOYMENTNAME }} defaults: run: working-directory: python steps: - uses: actions/checkout@v6 - - name: Set up python and install the project - uses: ./.github/actions/python-setup + - name: Setup environment + uses: ./.github/actions/sample-validation-setup with: - python-version: "3.12" + azure-client-id: ${{ secrets.AZURE_CLIENT_ID }} + azure-tenant-id: ${{ secrets.AZURE_TENANT_ID }} + azure-subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} os: ${{ runner.os }} - env: - UV_CACHE_DIR: /tmp/.uv-cache - name: Run sample validation run: | @@ -134,31 +134,31 @@ jobs: validate-04-hosting: name: Validate 04-hosting + if: false # Temporarily disabled because of sample complexity runs-on: ubuntu-latest - permissions: - contents: read + environment: integration env: # Azure AI configuration - AZURE_AI_PROJECT_ENDPOINT: ${{ secrets.AZURE_AI_PROJECT_ENDPOINT }} - AZURE_AI_MODEL_DEPLOYMENT_NAME: ${{ secrets.AZURE_AI_MODEL_DEPLOYMENT_NAME }} + AZURE_AI_PROJECT_ENDPOINT: ${{ vars.AZURE_AI_PROJECT_ENDPOINT }} + AZURE_AI_MODEL_DEPLOYMENT_NAME: ${{ vars.AZUREOPENAI__RESPONSESDEPLOYMENTNAME }} # Azure OpenAI configuration - AZURE_OPENAI_ENDPOINT: ${{ secrets.AZURE_OPENAI_ENDPOINT }} - AZURE_OPENAI_RESPONSES_DEPLOYMENT_NAME: ${{ secrets.AZURE_OPENAI_RESPONSES_DEPLOYMENT_NAME }} - # GitHub Copilot configuration - GITHUB_COPILOT_MODEL: ${{ vars.GITHUB_COPILOT_MODEL }} + AZURE_OPENAI_ENDPOINT: ${{ vars.AZUREOPENAI__ENDPOINT }} + AZURE_OPENAI_RESPONSES_DEPLOYMENT_NAME: ${{ vars.AZUREOPENAI__RESPONSESDEPLOYMENTNAME }} + # A2A configuration + A2A_AGENT_HOST: http://localhost:5001/ defaults: run: working-directory: python steps: - uses: actions/checkout@v6 - - name: Set up python and install the project - uses: ./.github/actions/python-setup + - name: Setup environment + uses: ./.github/actions/sample-validation-setup with: - python-version: "3.12" + azure-client-id: ${{ secrets.AZURE_CLIENT_ID }} + azure-tenant-id: ${{ secrets.AZURE_TENANT_ID }} + azure-subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} os: ${{ runner.os }} - env: - UV_CACHE_DIR: /tmp/.uv-cache - name: Run sample validation run: | @@ -173,36 +173,36 @@ jobs: validate-05-end-to-end: name: Validate 05-end-to-end + if: false # Temporarily disabled because of sample complexity runs-on: ubuntu-latest - permissions: - contents: read + environment: integration env: # Azure AI configuration - AZURE_AI_PROJECT_ENDPOINT: ${{ secrets.AZURE_AI_PROJECT_ENDPOINT }} - AZURE_AI_MODEL_DEPLOYMENT_NAME: ${{ secrets.AZURE_AI_MODEL_DEPLOYMENT_NAME }} + AZURE_AI_PROJECT_ENDPOINT: ${{ vars.AZURE_AI_PROJECT_ENDPOINT }} + AZURE_AI_MODEL_DEPLOYMENT_NAME: ${{ vars.AZUREOPENAI__RESPONSESDEPLOYMENTNAME }} # Azure OpenAI configuration - AZURE_OPENAI_ENDPOINT: ${{ secrets.AZURE_OPENAI_ENDPOINT }} - AZURE_OPENAI_CHAT_DEPLOYMENT_NAME: ${{ secrets.AZURE_OPENAI_CHAT_DEPLOYMENT_NAME }} - AZURE_OPENAI_RESPONSES_DEPLOYMENT_NAME: ${{ secrets.AZURE_OPENAI_RESPONSES_DEPLOYMENT_NAME }} + AZURE_OPENAI_ENDPOINT: ${{ vars.AZUREOPENAI__ENDPOINT }} + AZURE_OPENAI_CHAT_DEPLOYMENT_NAME: ${{ vars.AZUREOPENAI__CHATDEPLOYMENTNAME }} + AZURE_OPENAI_RESPONSES_DEPLOYMENT_NAME: ${{ vars.AZUREOPENAI__RESPONSESDEPLOYMENTNAME }} # Azure AI Search (for evaluation samples) AZURE_SEARCH_ENDPOINT: ${{ secrets.AZURE_SEARCH_ENDPOINT }} AZURE_SEARCH_API_KEY: ${{ secrets.AZURE_SEARCH_API_KEY }} AZURE_SEARCH_INDEX_NAME: ${{ secrets.AZURE_SEARCH_INDEX_NAME }} - # GitHub Copilot configuration - GITHUB_COPILOT_MODEL: ${{ vars.GITHUB_COPILOT_MODEL }} + # Evaluation sample + AZURE_AI_MODEL_DEPLOYMENT_NAME_WORKFLOW: ${{ vars.AZUREOPENAI__RESPONSESDEPLOYMENTNAME }} defaults: run: working-directory: python steps: - uses: actions/checkout@v6 - - name: Set up python and install the project - uses: ./.github/actions/python-setup + - name: Setup environment + uses: ./.github/actions/sample-validation-setup with: - python-version: "3.12" + azure-client-id: ${{ secrets.AZURE_CLIENT_ID }} + azure-tenant-id: ${{ secrets.AZURE_TENANT_ID }} + azure-subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} os: ${{ runner.os }} - env: - UV_CACHE_DIR: /tmp/.uv-cache - name: Run sample validation run: | @@ -218,30 +218,31 @@ jobs: validate-autogen-migration: name: Validate autogen-migration runs-on: ubuntu-latest - permissions: - contents: read + environment: integration env: # Azure AI configuration - AZURE_AI_PROJECT_ENDPOINT: ${{ secrets.AZURE_AI_PROJECT_ENDPOINT }} - AZURE_AI_MODEL_DEPLOYMENT_NAME: ${{ secrets.AZURE_AI_MODEL_DEPLOYMENT_NAME }} + AZURE_AI_PROJECT_ENDPOINT: ${{ vars.AZURE_AI_PROJECT_ENDPOINT }} + AZURE_AI_MODEL_DEPLOYMENT_NAME: ${{ vars.AZUREOPENAI__RESPONSESDEPLOYMENTNAME }} # Azure OpenAI configuration - AZURE_OPENAI_ENDPOINT: ${{ secrets.AZURE_OPENAI_ENDPOINT }} - AZURE_OPENAI_CHAT_DEPLOYMENT_NAME: ${{ secrets.AZURE_OPENAI_CHAT_DEPLOYMENT_NAME }} - # GitHub Copilot configuration - GITHUB_COPILOT_MODEL: ${{ vars.GITHUB_COPILOT_MODEL }} + AZURE_OPENAI_ENDPOINT: ${{ vars.AZUREOPENAI__ENDPOINT }} + AZURE_OPENAI_CHAT_DEPLOYMENT_NAME: ${{ vars.AZUREOPENAI__CHATDEPLOYMENTNAME }} + # OpenAI configuration + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + OPENAI_CHAT_MODEL_ID: ${{ vars.OPENAI_CHAT_MODEL_NAME }} + OPENAI_RESPONSES_MODEL_ID: ${{ vars.OPENAI_REASONING_MODEL_NAME }} defaults: run: working-directory: python steps: - uses: actions/checkout@v6 - - name: Set up python and install the project - uses: ./.github/actions/python-setup + - name: Setup environment + uses: ./.github/actions/sample-validation-setup with: - python-version: "3.12" + azure-client-id: ${{ secrets.AZURE_CLIENT_ID }} + azure-tenant-id: ${{ secrets.AZURE_TENANT_ID }} + azure-subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} os: ${{ runner.os }} - env: - UV_CACHE_DIR: /tmp/.uv-cache - name: Run sample validation run: | @@ -257,16 +258,15 @@ jobs: validate-semantic-kernel-migration: name: Validate semantic-kernel-migration runs-on: ubuntu-latest - permissions: - contents: read + environment: integration env: # Azure AI configuration - AZURE_AI_PROJECT_ENDPOINT: ${{ secrets.AZURE_AI_PROJECT_ENDPOINT }} - AZURE_AI_MODEL_DEPLOYMENT_NAME: ${{ secrets.AZURE_AI_MODEL_DEPLOYMENT_NAME }} + AZURE_AI_PROJECT_ENDPOINT: ${{ vars.AZURE_AI_PROJECT_ENDPOINT }} + AZURE_AI_MODEL_DEPLOYMENT_NAME: ${{ vars.AZUREOPENAI__RESPONSESDEPLOYMENTNAME }} # Azure OpenAI configuration - AZURE_OPENAI_ENDPOINT: ${{ secrets.AZURE_OPENAI_ENDPOINT }} - AZURE_OPENAI_CHAT_DEPLOYMENT_NAME: ${{ secrets.AZURE_OPENAI_CHAT_DEPLOYMENT_NAME }} - AZURE_OPENAI_RESPONSES_DEPLOYMENT_NAME: ${{ secrets.AZURE_OPENAI_RESPONSES_DEPLOYMENT_NAME }} + AZURE_OPENAI_ENDPOINT: ${{ vars.AZUREOPENAI__ENDPOINT }} + AZURE_OPENAI_CHAT_DEPLOYMENT_NAME: ${{ vars.AZUREOPENAI__CHATDEPLOYMENTNAME }} + AZURE_OPENAI_RESPONSES_DEPLOYMENT_NAME: ${{ vars.AZUREOPENAI__RESPONSESDEPLOYMENTNAME }} # OpenAI configuration OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} OPENAI_CHAT_MODEL_ID: ${{ vars.OPENAI_CHAT_MODEL_ID }} @@ -276,21 +276,19 @@ jobs: COPILOTSTUDIOAGENT__SCHEMANAME: ${{ secrets.COPILOTSTUDIOAGENT__SCHEMANAME }} COPILOTSTUDIOAGENT__TENANTID: ${{ secrets.COPILOTSTUDIOAGENT__TENANTID }} COPILOTSTUDIOAGENT__AGENTAPPID: ${{ secrets.COPILOTSTUDIOAGENT__AGENTAPPID }} - # GitHub Copilot configuration - GITHUB_COPILOT_MODEL: ${{ vars.GITHUB_COPILOT_MODEL }} defaults: run: working-directory: python steps: - uses: actions/checkout@v6 - - name: Set up python and install the project - uses: ./.github/actions/python-setup + - name: Setup environment + uses: ./.github/actions/sample-validation-setup with: - python-version: "3.12" + azure-client-id: ${{ secrets.AZURE_CLIENT_ID }} + azure-tenant-id: ${{ secrets.AZURE_TENANT_ID }} + azure-subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} os: ${{ runner.os }} - env: - UV_CACHE_DIR: /tmp/.uv-cache - name: Run sample validation run: | diff --git a/python/packages/azurefunctions/agent_framework_azurefunctions/_app.py b/python/packages/azurefunctions/agent_framework_azurefunctions/_app.py index 01735e28d1..c7d8552b24 100644 --- a/python/packages/azurefunctions/agent_framework_azurefunctions/_app.py +++ b/python/packages/azurefunctions/agent_framework_azurefunctions/_app.py @@ -612,11 +612,11 @@ def get_agent( context: Durable Functions orchestration context invoking the agent. agent_name: Name of the agent registered on this app. - Raises: - ValueError: If the requested agent has not been registered. - Returns: DurableAIAgent[AgentTask] wrapper bound to the orchestration context. + + Raises: + ValueError: If the requested agent has not been registered. """ normalized_name = str(agent_name) diff --git a/python/packages/core/agent_framework/_types.py b/python/packages/core/agent_framework/_types.py index 37ee9f1138..3df0bb20fb 100644 --- a/python/packages/core/agent_framework/_types.py +++ b/python/packages/core/agent_framework/_types.py @@ -93,13 +93,13 @@ def detect_media_type_from_base64( This will look at the actual data to determine the media_type and not at the URI prefix. Will also not compare those two values. - Raises: - ValueError: If not exactly 1 of data_bytes, data_str, or data_uri is provided, or if base64 decoding fails. - Returns: The detected media type (e.g., 'image/png', 'audio/wav', 'application/pdf') or None if the format is not recognized. + Raises: + ValueError: If not exactly 1 of data_bytes, data_str, or data_uri is provided, or if base64 decoding fails. + Examples: .. code-block:: python @@ -670,6 +670,9 @@ def from_uri( additional_properties: Optional additional properties. raw_representation: Optional raw representation from an underlying implementation. + Returns: + A Content instance with type="data" for data URIs or type="uri" for external URIs. + Raises: ContentError: If the URI is not valid. @@ -693,9 +696,6 @@ def from_uri( raw_base64_string }" ) - - Returns: - A Content instance with type="data" for data URIs or type="uri" for external URIs. """ return cls( **_validate_uri(uri, media_type), diff --git a/python/packages/core/agent_framework/_workflows/_workflow.py b/python/packages/core/agent_framework/_workflows/_workflow.py index cd7dbb4a68..09ba1048cd 100644 --- a/python/packages/core/agent_framework/_workflows/_workflow.py +++ b/python/packages/core/agent_framework/_workflows/_workflow.py @@ -369,7 +369,6 @@ async def _run_workflow_with_tracing( with _framework_event_origin(): pending_status = WorkflowEvent.status(WorkflowRunState.IN_PROGRESS_PENDING_REQUESTS) yield pending_status - # Workflow runs until idle - emit final status based on whether requests are pending if saw_request: with _framework_event_origin(): diff --git a/python/packages/devui/agent_framework_devui/_deployment.py b/python/packages/devui/agent_framework_devui/_deployment.py index 45f99a315a..db2de27ecf 100644 --- a/python/packages/devui/agent_framework_devui/_deployment.py +++ b/python/packages/devui/agent_framework_devui/_deployment.py @@ -92,8 +92,7 @@ async def deploy(self, config: DeploymentConfig, entity_path: Path) -> AsyncGene break # Get event from queue with short timeout - event = await asyncio.wait_for(event_queue.get(), timeout=0.1) - yield event + yield await asyncio.wait_for(event_queue.get(), timeout=0.1) except asyncio.TimeoutError: # No event in queue, continue waiting continue diff --git a/python/pyproject.toml b/python/pyproject.toml index a03123e9a2..e4e45f0290 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -148,6 +148,8 @@ ignore = [ "**/tests/**" = ["D", "INP", "TD", "ERA001", "RUF", "S"] "samples/**" = ["D", "INP", "ERA001", "RUF", "S", "T201", "CPY"] "*.ipynb" = ["CPY", "E501"] +# RUF070: Assignment before yield is intentional - context manager must exit before yielding +"**/agent_framework/_workflows/_workflow.py" = ["RUF070"] [tool.ruff.format] docstring-code-format = true diff --git a/python/samples/_sample_validation/create_dynamic_workflow_executor.py b/python/samples/_sample_validation/create_dynamic_workflow_executor.py index a8fd2011b4..bff720130d 100644 --- a/python/samples/_sample_validation/create_dynamic_workflow_executor.py +++ b/python/samples/_sample_validation/create_dynamic_workflow_executor.py @@ -53,9 +53,10 @@ class BatchCompletion: AgentInstruction = ( "You are validating exactly one Python sample.\n" - "Analyze the sample code and execute it. Determine if it runs successfully, fails, or times out.\n" + "Analyze the sample code and execute it. Based on the execution result, determine if it " + "runs successfully, fails, or times out. Feel free to install any required dependencies.\n" "The sample can be interactive. If it is interactive, respond to the sample when prompted " - "based on your analysis of the code. You do not need to consult human on what to respond\n" + "based on your analysis of the code. You do not need to consult human on what to respond.\n" "Return ONLY valid JSON with this schema:\n" "{\n" ' "status": "success|failure|timeout|error",\n' diff --git a/python/samples/_sample_validation/report.py b/python/samples/_sample_validation/report.py index d6083f44f6..9d02d342d4 100644 --- a/python/samples/_sample_validation/report.py +++ b/python/samples/_sample_validation/report.py @@ -21,6 +21,14 @@ def generate_report(results: list[RunResult]) -> Report: Returns: Report object with aggregated statistics """ + # Sort results: failures, timeouts, errors first, then successes + status_priority = { + RunStatus.FAILURE: 0, + RunStatus.TIMEOUT: 1, + RunStatus.ERROR: 2, + RunStatus.SUCCESS: 3, + } + sorted_results = sorted(results, key=lambda r: status_priority[r.status]) return Report( timestamp=datetime.now(), @@ -29,7 +37,7 @@ def generate_report(results: list[RunResult]) -> Report: failure_count=sum(1 for r in results if r.status == RunStatus.FAILURE), timeout_count=sum(1 for r in results if r.status == RunStatus.TIMEOUT), error_count=sum(1 for r in results if r.status == RunStatus.ERROR), - results=results, + results=sorted_results, ) @@ -84,9 +92,13 @@ def print_summary(report: Report) -> None: print(f" [PASS] Success: {report.success_count}") print(f" [FAIL] Failure: {report.failure_count}") print(f" [TIMEOUT] Timeout: {report.timeout_count}") - print(f" [ERROR] Error: {report.error_count}") + print(f" [ERR] Errors: {report.error_count}") print("=" * 80) + # Print JSON output for GitHub Actions visibility + print("\nJSON Report:") + print(json.dumps(report.to_dict(), indent=2)) + class GenerateReportExecutor(Executor): """Executor that generates the final validation report."""