From 9a7e490fa489f0cc2a5d40e974d10f780e2dc570 Mon Sep 17 00:00:00 2001 From: Tao Chen Date: Tue, 24 Feb 2026 09:35:24 -0800 Subject: [PATCH 01/26] Tuning validate-01-get-started --- .github/workflows/python-sample-validation.yml | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/.github/workflows/python-sample-validation.yml b/.github/workflows/python-sample-validation.yml index ba43394483..816349b2b4 100644 --- a/.github/workflows/python-sample-validation.yml +++ b/.github/workflows/python-sample-validation.yml @@ -16,17 +16,23 @@ jobs: permissions: contents: read env: - # Azure AI configuration for get-started samples - AZURE_AI_PROJECT_ENDPOINT: ${{ secrets.AZURE_AI_PROJECT_ENDPOINT }} - AZURE_AI_MODEL_DEPLOYMENT_NAME: ${{ secrets.AZURE_AI_MODEL_DEPLOYMENT_NAME }} - # GitHub Copilot configuration - GITHUB_COPILOT_MODEL: ${{ vars.GITHUB_COPILOT_MODEL }} + # Required configuration for get-started samples + AZURE_AI_PROJECT_ENDPOINT: ${{ vars.AZURE_AI_PROJECT_ENDPOINT }} + AZURE_OPENAI_RESPONSES_DEPLOYMENT_NAME: ${{ vars.AZUREOPENAI__RESPONSESDEPLOYMENTNAME }} + AZURE_OPENAI_ENDPOINT: ${{ vars.AZUREOPENAI__ENDPOINT }} + AZURE_OPENAI_CHAT_DEPLOYMENT_NAME: ${{ vars.AZUREOPENAI__CHATDEPLOYMENTNAME }} defaults: run: working-directory: python steps: - uses: actions/checkout@v6 + - name: Set up Node.js environment + uses: actions/setup-node@v4 + + - name: Install Copilot CLI + run: npm install -g @github/copilot + - name: Set up python and install the project uses: ./.github/actions/python-setup with: From ac62934ad1d3c2bd7a0e665e2331b1acb890b6af Mon Sep 17 00:00:00 2001 From: Tao Chen Date: Tue, 24 Feb 2026 10:25:38 -0800 Subject: [PATCH 02/26] Add gh token --- .github/workflows/python-sample-validation.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/python-sample-validation.yml b/.github/workflows/python-sample-validation.yml index 816349b2b4..9cb4804243 100644 --- a/.github/workflows/python-sample-validation.yml +++ b/.github/workflows/python-sample-validation.yml @@ -8,6 +8,9 @@ on: env: # Configure a constant location for the uv cache UV_CACHE_DIR: /tmp/.uv-cache + # setting GH_TOKEN for the entire workflow + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + jobs: validate-01-get-started: @@ -38,8 +41,6 @@ jobs: with: python-version: "3.12" os: ${{ runner.os }} - env: - UV_CACHE_DIR: /tmp/.uv-cache - name: Run sample validation run: | From ba570c72f3aedc381f2c93b1700d8cabb846f69d Mon Sep 17 00:00:00 2001 From: Tao Chen Date: Tue, 24 Feb 2026 11:04:15 -0800 Subject: [PATCH 03/26] Add model --- .github/workflows/python-sample-validation.yml | 1 + .../_sample_validation/create_dynamic_workflow_executor.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/python-sample-validation.yml b/.github/workflows/python-sample-validation.yml index 9cb4804243..a80669cbfe 100644 --- a/.github/workflows/python-sample-validation.yml +++ b/.github/workflows/python-sample-validation.yml @@ -10,6 +10,7 @@ env: UV_CACHE_DIR: /tmp/.uv-cache # setting GH_TOKEN for the entire workflow GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GITHUB_COPILOT_MODEL: claude-opus-4.6 jobs: diff --git a/python/samples/_sample_validation/create_dynamic_workflow_executor.py b/python/samples/_sample_validation/create_dynamic_workflow_executor.py index a8fd2011b4..00063681fb 100644 --- a/python/samples/_sample_validation/create_dynamic_workflow_executor.py +++ b/python/samples/_sample_validation/create_dynamic_workflow_executor.py @@ -55,7 +55,7 @@ class BatchCompletion: "You are validating exactly one Python sample.\n" "Analyze the sample code and execute it. Determine if it runs successfully, fails, or times out.\n" "The sample can be interactive. If it is interactive, respond to the sample when prompted " - "based on your analysis of the code. You do not need to consult human on what to respond\n" + "based on your analysis of the code. You do not need to consult human on what to respond.\n" "Return ONLY valid JSON with this schema:\n" "{\n" ' "status": "success|failure|timeout|error",\n' From c1e2e3e82274557dda084c1f6c0c6467930b4df2 Mon Sep 17 00:00:00 2001 From: Tao Chen Date: Tue, 24 Feb 2026 11:46:12 -0800 Subject: [PATCH 04/26] enable debug log --- .github/workflows/python-sample-validation.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/python-sample-validation.yml b/.github/workflows/python-sample-validation.yml index a80669cbfe..6827b0ea26 100644 --- a/.github/workflows/python-sample-validation.yml +++ b/.github/workflows/python-sample-validation.yml @@ -11,6 +11,7 @@ env: # setting GH_TOKEN for the entire workflow GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} GITHUB_COPILOT_MODEL: claude-opus-4.6 + GITHUB_COPILOT_LOG_LEVEL: debug jobs: From e2b7b8009b939d639090ccd5f0a07eeae5d0c25f Mon Sep 17 00:00:00 2001 From: Tao Chen Date: Tue, 24 Feb 2026 11:56:00 -0800 Subject: [PATCH 05/26] bump up timeout for testing purposes --- .../_sample_validation/create_dynamic_workflow_executor.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/samples/_sample_validation/create_dynamic_workflow_executor.py b/python/samples/_sample_validation/create_dynamic_workflow_executor.py index 00063681fb..d1ac14d882 100644 --- a/python/samples/_sample_validation/create_dynamic_workflow_executor.py +++ b/python/samples/_sample_validation/create_dynamic_workflow_executor.py @@ -223,7 +223,7 @@ async def create( id=agent_id, name=agent_id, instructions=AgentInstruction, - default_options={"on_permission_request": prompt_permission, "timeout": 180}, # type: ignore + default_options={"on_permission_request": prompt_permission, "timeout": 600}, # type: ignore ) agents.append(agent) From d54455aebe6721eb9f91a66069615558a08cf2ec Mon Sep 17 00:00:00 2001 From: Tao Chen Date: Tue, 24 Feb 2026 15:35:19 -0800 Subject: [PATCH 06/26] Test cli is working --- .github/workflows/python-sample-validation.yml | 4 ++++ .../_sample_validation/create_dynamic_workflow_executor.py | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/python-sample-validation.yml b/.github/workflows/python-sample-validation.yml index 6827b0ea26..a5ce02bb8b 100644 --- a/.github/workflows/python-sample-validation.yml +++ b/.github/workflows/python-sample-validation.yml @@ -38,6 +38,10 @@ jobs: - name: Install Copilot CLI run: npm install -g @github/copilot + - name: Test Copilot CLI + run: | + copilot -p "What can you do?"" + - name: Set up python and install the project uses: ./.github/actions/python-setup with: diff --git a/python/samples/_sample_validation/create_dynamic_workflow_executor.py b/python/samples/_sample_validation/create_dynamic_workflow_executor.py index d1ac14d882..00063681fb 100644 --- a/python/samples/_sample_validation/create_dynamic_workflow_executor.py +++ b/python/samples/_sample_validation/create_dynamic_workflow_executor.py @@ -223,7 +223,7 @@ async def create( id=agent_id, name=agent_id, instructions=AgentInstruction, - default_options={"on_permission_request": prompt_permission, "timeout": 600}, # type: ignore + default_options={"on_permission_request": prompt_permission, "timeout": 180}, # type: ignore ) agents.append(agent) From c5169e9e70fff04b6b2b2469890c831fddf80dd2 Mon Sep 17 00:00:00 2001 From: Tao Chen Date: Tue, 24 Feb 2026 15:38:13 -0800 Subject: [PATCH 07/26] Fix end quote --- .github/workflows/python-sample-validation.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/python-sample-validation.yml b/.github/workflows/python-sample-validation.yml index a5ce02bb8b..943f39b401 100644 --- a/.github/workflows/python-sample-validation.yml +++ b/.github/workflows/python-sample-validation.yml @@ -40,7 +40,7 @@ jobs: - name: Test Copilot CLI run: | - copilot -p "What can you do?"" + copilot -p "What can you do?" - name: Set up python and install the project uses: ./.github/actions/python-setup From 1cf584318606c8c3ffccdcaa57a8cdcd5b8f16b3 Mon Sep 17 00:00:00 2001 From: Tao Chen Date: Tue, 24 Feb 2026 16:20:17 -0800 Subject: [PATCH 08/26] Run gh auth --- .github/workflows/python-sample-validation.yml | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/.github/workflows/python-sample-validation.yml b/.github/workflows/python-sample-validation.yml index 943f39b401..d1c0ebc0ac 100644 --- a/.github/workflows/python-sample-validation.yml +++ b/.github/workflows/python-sample-validation.yml @@ -8,11 +8,8 @@ on: env: # Configure a constant location for the uv cache UV_CACHE_DIR: /tmp/.uv-cache - # setting GH_TOKEN for the entire workflow - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + # GitHub Copilot configuration GITHUB_COPILOT_MODEL: claude-opus-4.6 - GITHUB_COPILOT_LOG_LEVEL: debug - jobs: validate-01-get-started: @@ -38,6 +35,11 @@ jobs: - name: Install Copilot CLI run: npm install -g @github/copilot + - name: Authenticate Copilot CLI + run: gh auth login --with-token + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + - name: Test Copilot CLI run: | copilot -p "What can you do?" From 3a77431b25398ebb949bc5898de830fce92b422d Mon Sep 17 00:00:00 2001 From: Tao Chen Date: Tue, 24 Feb 2026 16:22:42 -0800 Subject: [PATCH 09/26] Run gh auth trail 2 --- .github/workflows/python-sample-validation.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/python-sample-validation.yml b/.github/workflows/python-sample-validation.yml index d1c0ebc0ac..596c03756b 100644 --- a/.github/workflows/python-sample-validation.yml +++ b/.github/workflows/python-sample-validation.yml @@ -36,9 +36,7 @@ jobs: run: npm install -g @github/copilot - name: Authenticate Copilot CLI - run: gh auth login --with-token - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: gh auth login --with-token < ${{ secrets.GITHUB_TOKEN }} - name: Test Copilot CLI run: | From b43bb87200bdb75cbdddb78236549fd1213e5680 Mon Sep 17 00:00:00 2001 From: Tao Chen Date: Tue, 24 Feb 2026 16:25:31 -0800 Subject: [PATCH 10/26] Run gh auth trail 3 --- .github/workflows/python-sample-validation.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/python-sample-validation.yml b/.github/workflows/python-sample-validation.yml index 596c03756b..9e0df96dd5 100644 --- a/.github/workflows/python-sample-validation.yml +++ b/.github/workflows/python-sample-validation.yml @@ -36,7 +36,7 @@ jobs: run: npm install -g @github/copilot - name: Authenticate Copilot CLI - run: gh auth login --with-token < ${{ secrets.GITHUB_TOKEN }} + run: echo ${{ secrets.GITHUB_TOKEN }} | gh auth login --with-token - name: Test Copilot CLI run: | From c8fce71d386ccbb21bb6c5256708d017f19f9ef9 Mon Sep 17 00:00:00 2001 From: Tao Chen Date: Thu, 26 Feb 2026 08:18:49 -0800 Subject: [PATCH 11/26] Test token --- .github/workflows/python-sample-validation.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/python-sample-validation.yml b/.github/workflows/python-sample-validation.yml index 9e0df96dd5..1139999b51 100644 --- a/.github/workflows/python-sample-validation.yml +++ b/.github/workflows/python-sample-validation.yml @@ -10,6 +10,7 @@ env: UV_CACHE_DIR: /tmp/.uv-cache # GitHub Copilot configuration GITHUB_COPILOT_MODEL: claude-opus-4.6 + COPILOT_GITHUB_TOKEN: ${{ secrets.COPILOT_GITHUB_TOKEN }} jobs: validate-01-get-started: @@ -35,9 +36,6 @@ jobs: - name: Install Copilot CLI run: npm install -g @github/copilot - - name: Authenticate Copilot CLI - run: echo ${{ secrets.GITHUB_TOKEN }} | gh auth login --with-token - - name: Test Copilot CLI run: | copilot -p "What can you do?" From 0e940afd74cc25088db16f298b5b0a980482c21f Mon Sep 17 00:00:00 2001 From: Tao Chen Date: Thu, 26 Feb 2026 08:26:05 -0800 Subject: [PATCH 12/26] Add zcure login --- .github/workflows/python-sample-validation.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.github/workflows/python-sample-validation.yml b/.github/workflows/python-sample-validation.yml index 1139999b51..987de9e583 100644 --- a/.github/workflows/python-sample-validation.yml +++ b/.github/workflows/python-sample-validation.yml @@ -40,6 +40,13 @@ jobs: run: | copilot -p "What can you do?" + - name: Azure CLI Login + uses: azure/login@v2 + with: + client-id: ${{ secrets.AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.AZURE_TENANT_ID }} + subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} + - name: Set up python and install the project uses: ./.github/actions/python-setup with: From 3e983be4414affa8017ea026ae160cec47dd5f7e Mon Sep 17 00:00:00 2001 From: Tao Chen Date: Thu, 26 Feb 2026 08:32:32 -0800 Subject: [PATCH 13/26] Add zcure login 2 --- .github/workflows/python-sample-validation.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/python-sample-validation.yml b/.github/workflows/python-sample-validation.yml index 987de9e583..700b7270d1 100644 --- a/.github/workflows/python-sample-validation.yml +++ b/.github/workflows/python-sample-validation.yml @@ -16,6 +16,7 @@ jobs: validate-01-get-started: name: Validate 01-get-started runs-on: ubuntu-latest + environment: integration permissions: contents: read env: @@ -38,7 +39,7 @@ jobs: - name: Test Copilot CLI run: | - copilot -p "What can you do?" + copilot -p "What can you do in one sentence?" - name: Azure CLI Login uses: azure/login@v2 From de26384dd7062b5bdee9ae3cf234fe7c2e77ca50 Mon Sep 17 00:00:00 2001 From: Tao Chen Date: Thu, 26 Feb 2026 09:32:21 -0800 Subject: [PATCH 14/26] Add zcure login 3 --- .github/workflows/python-sample-validation.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/python-sample-validation.yml b/.github/workflows/python-sample-validation.yml index 700b7270d1..ec72d22dd8 100644 --- a/.github/workflows/python-sample-validation.yml +++ b/.github/workflows/python-sample-validation.yml @@ -12,6 +12,9 @@ env: GITHUB_COPILOT_MODEL: claude-opus-4.6 COPILOT_GITHUB_TOKEN: ${{ secrets.COPILOT_GITHUB_TOKEN }} +permissions: + id-token: write + jobs: validate-01-get-started: name: Validate 01-get-started From 9e91491c70b9275be3573d2985c2301b75926927 Mon Sep 17 00:00:00 2001 From: Tao Chen Date: Thu, 26 Feb 2026 09:40:36 -0800 Subject: [PATCH 15/26] Add zcure login 4 --- .github/workflows/python-sample-validation.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/python-sample-validation.yml b/.github/workflows/python-sample-validation.yml index ec72d22dd8..1fd9dde877 100644 --- a/.github/workflows/python-sample-validation.yml +++ b/.github/workflows/python-sample-validation.yml @@ -13,6 +13,7 @@ env: COPILOT_GITHUB_TOKEN: ${{ secrets.COPILOT_GITHUB_TOKEN }} permissions: + contents: read id-token: write jobs: @@ -20,8 +21,6 @@ jobs: name: Validate 01-get-started runs-on: ubuntu-latest environment: integration - permissions: - contents: read env: # Required configuration for get-started samples AZURE_AI_PROJECT_ENDPOINT: ${{ vars.AZURE_AI_PROJECT_ENDPOINT }} From 067f21b84bde2a9854ce73bcc57226fdd925bbc0 Mon Sep 17 00:00:00 2001 From: Tao Chen Date: Thu, 26 Feb 2026 09:58:44 -0800 Subject: [PATCH 16/26] Extract common actions --- .../sample-validation-setup/action.yml | 51 +++++++ .../workflows/python-sample-validation.yml | 127 ++++++------------ 2 files changed, 92 insertions(+), 86 deletions(-) create mode 100644 .github/actions/sample-validation-setup/action.yml diff --git a/.github/actions/sample-validation-setup/action.yml b/.github/actions/sample-validation-setup/action.yml new file mode 100644 index 0000000000..12164f05c6 --- /dev/null +++ b/.github/actions/sample-validation-setup/action.yml @@ -0,0 +1,51 @@ +name: Sample Validation Setup +description: Sets up the environment for sample validation (checkout, Node.js, Copilot CLI, Azure login, Python) + +inputs: + azure-client-id: + description: Azure Client ID for OIDC login + required: true + azure-tenant-id: + description: Azure Tenant ID for OIDC login + required: true + azure-subscription-id: + description: Azure Subscription ID for OIDC login + required: true + python-version: + description: The Python version to set up + required: false + default: "3.12" + os: + description: The operating system to set up + required: false + default: "Linux" + +runs: + using: "composite" + steps: + - name: Checkout repository + uses: actions/checkout@v6 + + - name: Set up Node.js environment + uses: actions/setup-node@v4 + + - name: Install Copilot CLI + shell: bash + run: npm install -g @github/copilot + + - name: Test Copilot CLI + shell: bash + run: copilot -p "What can you do in one sentence?" + + - name: Azure CLI Login + uses: azure/login@v2 + with: + client-id: ${{ inputs.azure-client-id }} + tenant-id: ${{ inputs.azure-tenant-id }} + subscription-id: ${{ inputs.azure-subscription-id }} + + - name: Set up python and install the project + uses: ./.github/actions/python-setup + with: + python-version: ${{ inputs.python-version }} + os: ${{ inputs.os }} diff --git a/.github/workflows/python-sample-validation.yml b/.github/workflows/python-sample-validation.yml index 1fd9dde877..3230b3dd41 100644 --- a/.github/workflows/python-sample-validation.yml +++ b/.github/workflows/python-sample-validation.yml @@ -31,29 +31,12 @@ jobs: run: working-directory: python steps: - - uses: actions/checkout@v6 - - - name: Set up Node.js environment - uses: actions/setup-node@v4 - - - name: Install Copilot CLI - run: npm install -g @github/copilot - - - name: Test Copilot CLI - run: | - copilot -p "What can you do in one sentence?" - - - name: Azure CLI Login - uses: azure/login@v2 - with: - client-id: ${{ secrets.AZURE_CLIENT_ID }} - tenant-id: ${{ secrets.AZURE_TENANT_ID }} - subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} - - - name: Set up python and install the project - uses: ./.github/actions/python-setup + - name: Setup environment + uses: ./.github/actions/sample-validation-setup with: - python-version: "3.12" + azure-client-id: ${{ secrets.AZURE_CLIENT_ID }} + azure-tenant-id: ${{ secrets.AZURE_TENANT_ID }} + azure-subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} os: ${{ runner.os }} - name: Run sample validation @@ -70,8 +53,7 @@ jobs: validate-02-agents: name: Validate 02-agents runs-on: ubuntu-latest - permissions: - contents: read + environment: integration env: # Azure AI configuration AZURE_AI_PROJECT_ENDPOINT: ${{ secrets.AZURE_AI_PROJECT_ENDPOINT }} @@ -86,21 +68,17 @@ jobs: OPENAI_RESPONSES_MODEL_ID: ${{ vars.OPENAI_RESPONSES_MODEL_ID }} # Observability ENABLE_INSTRUMENTATION: "true" - # GitHub Copilot configuration - GITHUB_COPILOT_MODEL: ${{ vars.GITHUB_COPILOT_MODEL }} defaults: run: working-directory: python steps: - - uses: actions/checkout@v6 - - - name: Set up python and install the project - uses: ./.github/actions/python-setup + - name: Setup environment + uses: ./.github/actions/sample-validation-setup with: - python-version: "3.12" + azure-client-id: ${{ secrets.AZURE_CLIENT_ID }} + azure-tenant-id: ${{ secrets.AZURE_TENANT_ID }} + azure-subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} os: ${{ runner.os }} - env: - UV_CACHE_DIR: /tmp/.uv-cache - name: Run sample validation run: | @@ -116,8 +94,7 @@ jobs: validate-03-workflows: name: Validate 03-workflows runs-on: ubuntu-latest - permissions: - contents: read + environment: integration env: # Azure AI configuration AZURE_AI_PROJECT_ENDPOINT: ${{ secrets.AZURE_AI_PROJECT_ENDPOINT }} @@ -126,21 +103,17 @@ jobs: AZURE_OPENAI_ENDPOINT: ${{ secrets.AZURE_OPENAI_ENDPOINT }} AZURE_OPENAI_CHAT_DEPLOYMENT_NAME: ${{ secrets.AZURE_OPENAI_CHAT_DEPLOYMENT_NAME }} AZURE_OPENAI_RESPONSES_DEPLOYMENT_NAME: ${{ secrets.AZURE_OPENAI_RESPONSES_DEPLOYMENT_NAME }} - # GitHub Copilot configuration - GITHUB_COPILOT_MODEL: ${{ vars.GITHUB_COPILOT_MODEL }} defaults: run: working-directory: python steps: - - uses: actions/checkout@v6 - - - name: Set up python and install the project - uses: ./.github/actions/python-setup + - name: Setup environment + uses: ./.github/actions/sample-validation-setup with: - python-version: "3.12" + azure-client-id: ${{ secrets.AZURE_CLIENT_ID }} + azure-tenant-id: ${{ secrets.AZURE_TENANT_ID }} + azure-subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} os: ${{ runner.os }} - env: - UV_CACHE_DIR: /tmp/.uv-cache - name: Run sample validation run: | @@ -156,8 +129,7 @@ jobs: validate-04-hosting: name: Validate 04-hosting runs-on: ubuntu-latest - permissions: - contents: read + environment: integration env: # Azure AI configuration AZURE_AI_PROJECT_ENDPOINT: ${{ secrets.AZURE_AI_PROJECT_ENDPOINT }} @@ -165,21 +137,17 @@ jobs: # Azure OpenAI configuration AZURE_OPENAI_ENDPOINT: ${{ secrets.AZURE_OPENAI_ENDPOINT }} AZURE_OPENAI_RESPONSES_DEPLOYMENT_NAME: ${{ secrets.AZURE_OPENAI_RESPONSES_DEPLOYMENT_NAME }} - # GitHub Copilot configuration - GITHUB_COPILOT_MODEL: ${{ vars.GITHUB_COPILOT_MODEL }} defaults: run: working-directory: python steps: - - uses: actions/checkout@v6 - - - name: Set up python and install the project - uses: ./.github/actions/python-setup + - name: Setup environment + uses: ./.github/actions/sample-validation-setup with: - python-version: "3.12" + azure-client-id: ${{ secrets.AZURE_CLIENT_ID }} + azure-tenant-id: ${{ secrets.AZURE_TENANT_ID }} + azure-subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} os: ${{ runner.os }} - env: - UV_CACHE_DIR: /tmp/.uv-cache - name: Run sample validation run: | @@ -195,8 +163,7 @@ jobs: validate-05-end-to-end: name: Validate 05-end-to-end runs-on: ubuntu-latest - permissions: - contents: read + environment: integration env: # Azure AI configuration AZURE_AI_PROJECT_ENDPOINT: ${{ secrets.AZURE_AI_PROJECT_ENDPOINT }} @@ -209,21 +176,17 @@ jobs: AZURE_SEARCH_ENDPOINT: ${{ secrets.AZURE_SEARCH_ENDPOINT }} AZURE_SEARCH_API_KEY: ${{ secrets.AZURE_SEARCH_API_KEY }} AZURE_SEARCH_INDEX_NAME: ${{ secrets.AZURE_SEARCH_INDEX_NAME }} - # GitHub Copilot configuration - GITHUB_COPILOT_MODEL: ${{ vars.GITHUB_COPILOT_MODEL }} defaults: run: working-directory: python steps: - - uses: actions/checkout@v6 - - - name: Set up python and install the project - uses: ./.github/actions/python-setup + - name: Setup environment + uses: ./.github/actions/sample-validation-setup with: - python-version: "3.12" + azure-client-id: ${{ secrets.AZURE_CLIENT_ID }} + azure-tenant-id: ${{ secrets.AZURE_TENANT_ID }} + azure-subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} os: ${{ runner.os }} - env: - UV_CACHE_DIR: /tmp/.uv-cache - name: Run sample validation run: | @@ -239,8 +202,7 @@ jobs: validate-autogen-migration: name: Validate autogen-migration runs-on: ubuntu-latest - permissions: - contents: read + environment: integration env: # Azure AI configuration AZURE_AI_PROJECT_ENDPOINT: ${{ secrets.AZURE_AI_PROJECT_ENDPOINT }} @@ -248,21 +210,17 @@ jobs: # Azure OpenAI configuration AZURE_OPENAI_ENDPOINT: ${{ secrets.AZURE_OPENAI_ENDPOINT }} AZURE_OPENAI_CHAT_DEPLOYMENT_NAME: ${{ secrets.AZURE_OPENAI_CHAT_DEPLOYMENT_NAME }} - # GitHub Copilot configuration - GITHUB_COPILOT_MODEL: ${{ vars.GITHUB_COPILOT_MODEL }} defaults: run: working-directory: python steps: - - uses: actions/checkout@v6 - - - name: Set up python and install the project - uses: ./.github/actions/python-setup + - name: Setup environment + uses: ./.github/actions/sample-validation-setup with: - python-version: "3.12" + azure-client-id: ${{ secrets.AZURE_CLIENT_ID }} + azure-tenant-id: ${{ secrets.AZURE_TENANT_ID }} + azure-subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} os: ${{ runner.os }} - env: - UV_CACHE_DIR: /tmp/.uv-cache - name: Run sample validation run: | @@ -278,8 +236,7 @@ jobs: validate-semantic-kernel-migration: name: Validate semantic-kernel-migration runs-on: ubuntu-latest - permissions: - contents: read + environment: integration env: # Azure AI configuration AZURE_AI_PROJECT_ENDPOINT: ${{ secrets.AZURE_AI_PROJECT_ENDPOINT }} @@ -303,15 +260,13 @@ jobs: run: working-directory: python steps: - - uses: actions/checkout@v6 - - - name: Set up python and install the project - uses: ./.github/actions/python-setup + - name: Setup environment + uses: ./.github/actions/sample-validation-setup with: - python-version: "3.12" + azure-client-id: ${{ secrets.AZURE_CLIENT_ID }} + azure-tenant-id: ${{ secrets.AZURE_TENANT_ID }} + azure-subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} os: ${{ runner.os }} - env: - UV_CACHE_DIR: /tmp/.uv-cache - name: Run sample validation run: | From d36859c4a41f8382c4bfcb2c8f779dbdfafe2cc2 Mon Sep 17 00:00:00 2001 From: Tao Chen Date: Thu, 26 Feb 2026 10:01:19 -0800 Subject: [PATCH 17/26] Extract common actions 2 --- .github/actions/sample-validation-setup/action.yml | 3 --- .github/workflows/python-sample-validation.yml | 14 ++++++++++++++ 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/.github/actions/sample-validation-setup/action.yml b/.github/actions/sample-validation-setup/action.yml index 12164f05c6..3736348579 100644 --- a/.github/actions/sample-validation-setup/action.yml +++ b/.github/actions/sample-validation-setup/action.yml @@ -23,9 +23,6 @@ inputs: runs: using: "composite" steps: - - name: Checkout repository - uses: actions/checkout@v6 - - name: Set up Node.js environment uses: actions/setup-node@v4 diff --git a/.github/workflows/python-sample-validation.yml b/.github/workflows/python-sample-validation.yml index 3230b3dd41..7b3e6a1fdf 100644 --- a/.github/workflows/python-sample-validation.yml +++ b/.github/workflows/python-sample-validation.yml @@ -31,6 +31,8 @@ jobs: run: working-directory: python steps: + - uses: actions/checkout@v6 + - name: Setup environment uses: ./.github/actions/sample-validation-setup with: @@ -72,6 +74,8 @@ jobs: run: working-directory: python steps: + - uses: actions/checkout@v6 + - name: Setup environment uses: ./.github/actions/sample-validation-setup with: @@ -107,6 +111,8 @@ jobs: run: working-directory: python steps: + - uses: actions/checkout@v6 + - name: Setup environment uses: ./.github/actions/sample-validation-setup with: @@ -141,6 +147,8 @@ jobs: run: working-directory: python steps: + - uses: actions/checkout@v6 + - name: Setup environment uses: ./.github/actions/sample-validation-setup with: @@ -180,6 +188,8 @@ jobs: run: working-directory: python steps: + - uses: actions/checkout@v6 + - name: Setup environment uses: ./.github/actions/sample-validation-setup with: @@ -214,6 +224,8 @@ jobs: run: working-directory: python steps: + - uses: actions/checkout@v6 + - name: Setup environment uses: ./.github/actions/sample-validation-setup with: @@ -260,6 +272,8 @@ jobs: run: working-directory: python steps: + - uses: actions/checkout@v6 + - name: Setup environment uses: ./.github/actions/sample-validation-setup with: From e643c8558d734429510a5bd1988b24b6dd74031b Mon Sep 17 00:00:00 2001 From: Tao Chen Date: Thu, 26 Feb 2026 10:23:22 -0800 Subject: [PATCH 18/26] Correct env vars --- .../workflows/python-sample-validation.yml | 60 +++++++++---------- .../create_dynamic_workflow_executor.py | 3 +- python/samples/_sample_validation/report.py | 2 +- 3 files changed, 33 insertions(+), 32 deletions(-) diff --git a/.github/workflows/python-sample-validation.yml b/.github/workflows/python-sample-validation.yml index 7b3e6a1fdf..61f539fc2c 100644 --- a/.github/workflows/python-sample-validation.yml +++ b/.github/workflows/python-sample-validation.yml @@ -58,16 +58,16 @@ jobs: environment: integration env: # Azure AI configuration - AZURE_AI_PROJECT_ENDPOINT: ${{ secrets.AZURE_AI_PROJECT_ENDPOINT }} - AZURE_AI_MODEL_DEPLOYMENT_NAME: ${{ secrets.AZURE_AI_MODEL_DEPLOYMENT_NAME }} + AZURE_AI_PROJECT_ENDPOINT: ${{ vars.AZURE_AI_PROJECT_ENDPOINT }} + AZURE_AI_MODEL_DEPLOYMENT_NAME: ${{ vars.AZUREOPENAI__RESPONSESDEPLOYMENTNAME }} # Azure OpenAI configuration - AZURE_OPENAI_ENDPOINT: ${{ secrets.AZURE_OPENAI_ENDPOINT }} - AZURE_OPENAI_CHAT_DEPLOYMENT_NAME: ${{ secrets.AZURE_OPENAI_CHAT_DEPLOYMENT_NAME }} - AZURE_OPENAI_RESPONSES_DEPLOYMENT_NAME: ${{ secrets.AZURE_OPENAI_RESPONSES_DEPLOYMENT_NAME }} + AZURE_OPENAI_ENDPOINT: ${{ vars.AZUREOPENAI__ENDPOINT }} + AZURE_OPENAI_CHAT_DEPLOYMENT_NAME: ${{ vars.AZUREOPENAI__CHATDEPLOYMENTNAME }} + AZURE_OPENAI_RESPONSES_DEPLOYMENT_NAME: ${{ vars.AZUREOPENAI__RESPONSESDEPLOYMENTNAME }} # OpenAI configuration OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} - OPENAI_CHAT_MODEL_ID: ${{ vars.OPENAI_CHAT_MODEL_ID }} - OPENAI_RESPONSES_MODEL_ID: ${{ vars.OPENAI_RESPONSES_MODEL_ID }} + OPENAI_CHAT_MODEL_ID: ${{ vars.OPENAI_CHAT_MODEL_NAME }} + OPENAI_RESPONSES_MODEL_ID: ${{ vars.OPENAI_REASONING_MODEL_NAME }} # Observability ENABLE_INSTRUMENTATION: "true" defaults: @@ -101,12 +101,12 @@ jobs: environment: integration env: # Azure AI configuration - AZURE_AI_PROJECT_ENDPOINT: ${{ secrets.AZURE_AI_PROJECT_ENDPOINT }} - AZURE_AI_MODEL_DEPLOYMENT_NAME: ${{ secrets.AZURE_AI_MODEL_DEPLOYMENT_NAME }} + AZURE_AI_PROJECT_ENDPOINT: ${{ vars.AZURE_AI_PROJECT_ENDPOINT }} + AZURE_AI_MODEL_DEPLOYMENT_NAME: ${{ vars.AZUREOPENAI__RESPONSESDEPLOYMENTNAME }} # Azure OpenAI configuration - AZURE_OPENAI_ENDPOINT: ${{ secrets.AZURE_OPENAI_ENDPOINT }} - AZURE_OPENAI_CHAT_DEPLOYMENT_NAME: ${{ secrets.AZURE_OPENAI_CHAT_DEPLOYMENT_NAME }} - AZURE_OPENAI_RESPONSES_DEPLOYMENT_NAME: ${{ secrets.AZURE_OPENAI_RESPONSES_DEPLOYMENT_NAME }} + AZURE_OPENAI_ENDPOINT: ${{ vars.AZUREOPENAI__ENDPOINT }} + AZURE_OPENAI_CHAT_DEPLOYMENT_NAME: ${{ vars.AZUREOPENAI__CHATDEPLOYMENTNAME }} + AZURE_OPENAI_RESPONSES_DEPLOYMENT_NAME: ${{ vars.AZUREOPENAI__RESPONSESDEPLOYMENTNAME }} defaults: run: working-directory: python @@ -138,11 +138,11 @@ jobs: environment: integration env: # Azure AI configuration - AZURE_AI_PROJECT_ENDPOINT: ${{ secrets.AZURE_AI_PROJECT_ENDPOINT }} - AZURE_AI_MODEL_DEPLOYMENT_NAME: ${{ secrets.AZURE_AI_MODEL_DEPLOYMENT_NAME }} + AZURE_AI_PROJECT_ENDPOINT: ${{ vars.AZURE_AI_PROJECT_ENDPOINT }} + AZURE_AI_MODEL_DEPLOYMENT_NAME: ${{ vars.AZUREOPENAI__RESPONSESDEPLOYMENTNAME }} # Azure OpenAI configuration - AZURE_OPENAI_ENDPOINT: ${{ secrets.AZURE_OPENAI_ENDPOINT }} - AZURE_OPENAI_RESPONSES_DEPLOYMENT_NAME: ${{ secrets.AZURE_OPENAI_RESPONSES_DEPLOYMENT_NAME }} + AZURE_OPENAI_ENDPOINT: ${{ vars.AZUREOPENAI__ENDPOINT }} + AZURE_OPENAI_RESPONSES_DEPLOYMENT_NAME: ${{ vars.AZUREOPENAI__RESPONSESDEPLOYMENTNAME }} defaults: run: working-directory: python @@ -174,12 +174,12 @@ jobs: environment: integration env: # Azure AI configuration - AZURE_AI_PROJECT_ENDPOINT: ${{ secrets.AZURE_AI_PROJECT_ENDPOINT }} - AZURE_AI_MODEL_DEPLOYMENT_NAME: ${{ secrets.AZURE_AI_MODEL_DEPLOYMENT_NAME }} + AZURE_AI_PROJECT_ENDPOINT: ${{ vars.AZURE_AI_PROJECT_ENDPOINT }} + AZURE_AI_MODEL_DEPLOYMENT_NAME: ${{ vars.AZUREOPENAI__RESPONSESDEPLOYMENTNAME }} # Azure OpenAI configuration - AZURE_OPENAI_ENDPOINT: ${{ secrets.AZURE_OPENAI_ENDPOINT }} - AZURE_OPENAI_CHAT_DEPLOYMENT_NAME: ${{ secrets.AZURE_OPENAI_CHAT_DEPLOYMENT_NAME }} - AZURE_OPENAI_RESPONSES_DEPLOYMENT_NAME: ${{ secrets.AZURE_OPENAI_RESPONSES_DEPLOYMENT_NAME }} + AZURE_OPENAI_ENDPOINT: ${{ vars.AZUREOPENAI__ENDPOINT }} + AZURE_OPENAI_CHAT_DEPLOYMENT_NAME: ${{ vars.AZUREOPENAI__CHATDEPLOYMENTNAME }} + AZURE_OPENAI_RESPONSES_DEPLOYMENT_NAME: ${{ vars.AZUREOPENAI__RESPONSESDEPLOYMENTNAME }} # Azure AI Search (for evaluation samples) AZURE_SEARCH_ENDPOINT: ${{ secrets.AZURE_SEARCH_ENDPOINT }} AZURE_SEARCH_API_KEY: ${{ secrets.AZURE_SEARCH_API_KEY }} @@ -215,11 +215,11 @@ jobs: environment: integration env: # Azure AI configuration - AZURE_AI_PROJECT_ENDPOINT: ${{ secrets.AZURE_AI_PROJECT_ENDPOINT }} - AZURE_AI_MODEL_DEPLOYMENT_NAME: ${{ secrets.AZURE_AI_MODEL_DEPLOYMENT_NAME }} + AZURE_AI_PROJECT_ENDPOINT: ${{ vars.AZURE_AI_PROJECT_ENDPOINT }} + AZURE_AI_MODEL_DEPLOYMENT_NAME: ${{ vars.AZUREOPENAI__RESPONSESDEPLOYMENTNAME }} # Azure OpenAI configuration - AZURE_OPENAI_ENDPOINT: ${{ secrets.AZURE_OPENAI_ENDPOINT }} - AZURE_OPENAI_CHAT_DEPLOYMENT_NAME: ${{ secrets.AZURE_OPENAI_CHAT_DEPLOYMENT_NAME }} + AZURE_OPENAI_ENDPOINT: ${{ vars.AZUREOPENAI__ENDPOINT }} + AZURE_OPENAI_CHAT_DEPLOYMENT_NAME: ${{ vars.AZUREOPENAI__CHATDEPLOYMENTNAME }} defaults: run: working-directory: python @@ -251,12 +251,12 @@ jobs: environment: integration env: # Azure AI configuration - AZURE_AI_PROJECT_ENDPOINT: ${{ secrets.AZURE_AI_PROJECT_ENDPOINT }} - AZURE_AI_MODEL_DEPLOYMENT_NAME: ${{ secrets.AZURE_AI_MODEL_DEPLOYMENT_NAME }} + AZURE_AI_PROJECT_ENDPOINT: ${{ vars.AZURE_AI_PROJECT_ENDPOINT }} + AZURE_AI_MODEL_DEPLOYMENT_NAME: ${{ vars.AZUREOPENAI__RESPONSESDEPLOYMENTNAME }} # Azure OpenAI configuration - AZURE_OPENAI_ENDPOINT: ${{ secrets.AZURE_OPENAI_ENDPOINT }} - AZURE_OPENAI_CHAT_DEPLOYMENT_NAME: ${{ secrets.AZURE_OPENAI_CHAT_DEPLOYMENT_NAME }} - AZURE_OPENAI_RESPONSES_DEPLOYMENT_NAME: ${{ secrets.AZURE_OPENAI_RESPONSES_DEPLOYMENT_NAME }} + AZURE_OPENAI_ENDPOINT: ${{ vars.AZUREOPENAI__ENDPOINT }} + AZURE_OPENAI_CHAT_DEPLOYMENT_NAME: ${{ vars.AZUREOPENAI__CHATDEPLOYMENTNAME }} + AZURE_OPENAI_RESPONSES_DEPLOYMENT_NAME: ${{ vars.AZUREOPENAI__RESPONSESDEPLOYMENTNAME }} # OpenAI configuration OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} OPENAI_CHAT_MODEL_ID: ${{ vars.OPENAI_CHAT_MODEL_ID }} diff --git a/python/samples/_sample_validation/create_dynamic_workflow_executor.py b/python/samples/_sample_validation/create_dynamic_workflow_executor.py index 00063681fb..1863dbc9b0 100644 --- a/python/samples/_sample_validation/create_dynamic_workflow_executor.py +++ b/python/samples/_sample_validation/create_dynamic_workflow_executor.py @@ -53,7 +53,8 @@ class BatchCompletion: AgentInstruction = ( "You are validating exactly one Python sample.\n" - "Analyze the sample code and execute it. Determine if it runs successfully, fails, or times out.\n" + "Analyze the sample code and execute it. Based on the execution result, determine if it " + "runs successfully, fails, or times out.\n" "The sample can be interactive. If it is interactive, respond to the sample when prompted " "based on your analysis of the code. You do not need to consult human on what to respond.\n" "Return ONLY valid JSON with this schema:\n" diff --git a/python/samples/_sample_validation/report.py b/python/samples/_sample_validation/report.py index d6083f44f6..9700de4cd0 100644 --- a/python/samples/_sample_validation/report.py +++ b/python/samples/_sample_validation/report.py @@ -84,7 +84,7 @@ def print_summary(report: Report) -> None: print(f" [PASS] Success: {report.success_count}") print(f" [FAIL] Failure: {report.failure_count}") print(f" [TIMEOUT] Timeout: {report.timeout_count}") - print(f" [ERROR] Error: {report.error_count}") + print(f" [ERR] Errors: {report.error_count}") print("=" * 80) From 059697ea43ce462a45f1be711ee5e99ea62135c9 Mon Sep 17 00:00:00 2001 From: Tao Chen Date: Thu, 26 Feb 2026 11:43:51 -0800 Subject: [PATCH 19/26] Print outputs to action console --- .github/workflows/python-sample-validation.yml | 11 +++++++++-- .../create_dynamic_workflow_executor.py | 2 +- python/samples/_sample_validation/report.py | 14 +++++++++++++- 3 files changed, 23 insertions(+), 4 deletions(-) diff --git a/.github/workflows/python-sample-validation.yml b/.github/workflows/python-sample-validation.yml index 61f539fc2c..1524f67f0e 100644 --- a/.github/workflows/python-sample-validation.yml +++ b/.github/workflows/python-sample-validation.yml @@ -134,6 +134,7 @@ jobs: validate-04-hosting: name: Validate 04-hosting + if: false # Temporarily disabled because of sample complexity runs-on: ubuntu-latest environment: integration env: @@ -143,6 +144,8 @@ jobs: # Azure OpenAI configuration AZURE_OPENAI_ENDPOINT: ${{ vars.AZUREOPENAI__ENDPOINT }} AZURE_OPENAI_RESPONSES_DEPLOYMENT_NAME: ${{ vars.AZUREOPENAI__RESPONSESDEPLOYMENTNAME }} + # A2A configuration + A2A_AGENT_HOST: http://localhost:5001/ defaults: run: working-directory: python @@ -184,6 +187,8 @@ jobs: AZURE_SEARCH_ENDPOINT: ${{ secrets.AZURE_SEARCH_ENDPOINT }} AZURE_SEARCH_API_KEY: ${{ secrets.AZURE_SEARCH_API_KEY }} AZURE_SEARCH_INDEX_NAME: ${{ secrets.AZURE_SEARCH_INDEX_NAME }} + # Evaluation sample + AZURE_AI_MODEL_DEPLOYMENT_NAME_WORKFLOW: ${{ vars.AZUREOPENAI__RESPONSESDEPLOYMENTNAME }} defaults: run: working-directory: python @@ -220,6 +225,10 @@ jobs: # Azure OpenAI configuration AZURE_OPENAI_ENDPOINT: ${{ vars.AZUREOPENAI__ENDPOINT }} AZURE_OPENAI_CHAT_DEPLOYMENT_NAME: ${{ vars.AZUREOPENAI__CHATDEPLOYMENTNAME }} + # OpenAI configuration + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + OPENAI_CHAT_MODEL_ID: ${{ vars.OPENAI_CHAT_MODEL_NAME }} + OPENAI_RESPONSES_MODEL_ID: ${{ vars.OPENAI_REASONING_MODEL_NAME }} defaults: run: working-directory: python @@ -266,8 +275,6 @@ jobs: COPILOTSTUDIOAGENT__SCHEMANAME: ${{ secrets.COPILOTSTUDIOAGENT__SCHEMANAME }} COPILOTSTUDIOAGENT__TENANTID: ${{ secrets.COPILOTSTUDIOAGENT__TENANTID }} COPILOTSTUDIOAGENT__AGENTAPPID: ${{ secrets.COPILOTSTUDIOAGENT__AGENTAPPID }} - # GitHub Copilot configuration - GITHUB_COPILOT_MODEL: ${{ vars.GITHUB_COPILOT_MODEL }} defaults: run: working-directory: python diff --git a/python/samples/_sample_validation/create_dynamic_workflow_executor.py b/python/samples/_sample_validation/create_dynamic_workflow_executor.py index 1863dbc9b0..bff720130d 100644 --- a/python/samples/_sample_validation/create_dynamic_workflow_executor.py +++ b/python/samples/_sample_validation/create_dynamic_workflow_executor.py @@ -54,7 +54,7 @@ class BatchCompletion: AgentInstruction = ( "You are validating exactly one Python sample.\n" "Analyze the sample code and execute it. Based on the execution result, determine if it " - "runs successfully, fails, or times out.\n" + "runs successfully, fails, or times out. Feel free to install any required dependencies.\n" "The sample can be interactive. If it is interactive, respond to the sample when prompted " "based on your analysis of the code. You do not need to consult human on what to respond.\n" "Return ONLY valid JSON with this schema:\n" diff --git a/python/samples/_sample_validation/report.py b/python/samples/_sample_validation/report.py index 9700de4cd0..9d02d342d4 100644 --- a/python/samples/_sample_validation/report.py +++ b/python/samples/_sample_validation/report.py @@ -21,6 +21,14 @@ def generate_report(results: list[RunResult]) -> Report: Returns: Report object with aggregated statistics """ + # Sort results: failures, timeouts, errors first, then successes + status_priority = { + RunStatus.FAILURE: 0, + RunStatus.TIMEOUT: 1, + RunStatus.ERROR: 2, + RunStatus.SUCCESS: 3, + } + sorted_results = sorted(results, key=lambda r: status_priority[r.status]) return Report( timestamp=datetime.now(), @@ -29,7 +37,7 @@ def generate_report(results: list[RunResult]) -> Report: failure_count=sum(1 for r in results if r.status == RunStatus.FAILURE), timeout_count=sum(1 for r in results if r.status == RunStatus.TIMEOUT), error_count=sum(1 for r in results if r.status == RunStatus.ERROR), - results=results, + results=sorted_results, ) @@ -87,6 +95,10 @@ def print_summary(report: Report) -> None: print(f" [ERR] Errors: {report.error_count}") print("=" * 80) + # Print JSON output for GitHub Actions visibility + print("\nJSON Report:") + print(json.dumps(report.to_dict(), indent=2)) + class GenerateReportExecutor(Executor): """Executor that generates the final validation report.""" From cd338bd2d6ba30f9d9ce960d399149df33c958ab Mon Sep 17 00:00:00 2001 From: Tao Chen Date: Thu, 26 Feb 2026 13:18:42 -0800 Subject: [PATCH 20/26] Disable end-to-end samples --- .github/workflows/python-sample-validation.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/python-sample-validation.yml b/.github/workflows/python-sample-validation.yml index 1524f67f0e..1ada1ab113 100644 --- a/.github/workflows/python-sample-validation.yml +++ b/.github/workflows/python-sample-validation.yml @@ -173,6 +173,7 @@ jobs: validate-05-end-to-end: name: Validate 05-end-to-end + if: false # Temporarily disabled because of sample complexity runs-on: ubuntu-latest environment: integration env: From 42d74ecda48b9338e18fb2570198b76ba2c96042 Mon Sep 17 00:00:00 2001 From: Tao Chen Date: Thu, 26 Feb 2026 14:09:25 -0800 Subject: [PATCH 21/26] Fix ruff errors --- .../agent_framework_azurefunctions/_app.py | 6 +++--- python/packages/core/agent_framework/_types.py | 12 ++++++------ .../core/agent_framework/_workflows/_workflow.py | 15 +++++---------- .../devui/agent_framework_devui/_deployment.py | 3 +-- 4 files changed, 15 insertions(+), 21 deletions(-) diff --git a/python/packages/azurefunctions/agent_framework_azurefunctions/_app.py b/python/packages/azurefunctions/agent_framework_azurefunctions/_app.py index 01735e28d1..c7d8552b24 100644 --- a/python/packages/azurefunctions/agent_framework_azurefunctions/_app.py +++ b/python/packages/azurefunctions/agent_framework_azurefunctions/_app.py @@ -612,11 +612,11 @@ def get_agent( context: Durable Functions orchestration context invoking the agent. agent_name: Name of the agent registered on this app. - Raises: - ValueError: If the requested agent has not been registered. - Returns: DurableAIAgent[AgentTask] wrapper bound to the orchestration context. + + Raises: + ValueError: If the requested agent has not been registered. """ normalized_name = str(agent_name) diff --git a/python/packages/core/agent_framework/_types.py b/python/packages/core/agent_framework/_types.py index 37ee9f1138..3df0bb20fb 100644 --- a/python/packages/core/agent_framework/_types.py +++ b/python/packages/core/agent_framework/_types.py @@ -93,13 +93,13 @@ def detect_media_type_from_base64( This will look at the actual data to determine the media_type and not at the URI prefix. Will also not compare those two values. - Raises: - ValueError: If not exactly 1 of data_bytes, data_str, or data_uri is provided, or if base64 decoding fails. - Returns: The detected media type (e.g., 'image/png', 'audio/wav', 'application/pdf') or None if the format is not recognized. + Raises: + ValueError: If not exactly 1 of data_bytes, data_str, or data_uri is provided, or if base64 decoding fails. + Examples: .. code-block:: python @@ -670,6 +670,9 @@ def from_uri( additional_properties: Optional additional properties. raw_representation: Optional raw representation from an underlying implementation. + Returns: + A Content instance with type="data" for data URIs or type="uri" for external URIs. + Raises: ContentError: If the URI is not valid. @@ -693,9 +696,6 @@ def from_uri( raw_base64_string }" ) - - Returns: - A Content instance with type="data" for data URIs or type="uri" for external URIs. """ return cls( **_validate_uri(uri, media_type), diff --git a/python/packages/core/agent_framework/_workflows/_workflow.py b/python/packages/core/agent_framework/_workflows/_workflow.py index cd7dbb4a68..ac4f2d1a8c 100644 --- a/python/packages/core/agent_framework/_workflows/_workflow.py +++ b/python/packages/core/agent_framework/_workflows/_workflow.py @@ -333,11 +333,9 @@ async def _run_workflow_with_tracing( span.add_event(OtelAttr.WORKFLOW_STARTED) # Emit explicit start/status events to the stream with _framework_event_origin(): - started = WorkflowEvent.started() - yield started + yield WorkflowEvent.started() with _framework_event_origin(): - in_progress = WorkflowEvent.status(WorkflowRunState.IN_PROGRESS) - yield in_progress + yield WorkflowEvent.status(WorkflowRunState.IN_PROGRESS) # Reset context for a new run if supported if reset_context: @@ -367,18 +365,15 @@ async def _run_workflow_with_tracing( if event.type == "request_info" and not emitted_in_progress_pending: emitted_in_progress_pending = True with _framework_event_origin(): - pending_status = WorkflowEvent.status(WorkflowRunState.IN_PROGRESS_PENDING_REQUESTS) - yield pending_status + yield WorkflowEvent.status(WorkflowRunState.IN_PROGRESS_PENDING_REQUESTS) # Workflow runs until idle - emit final status based on whether requests are pending if saw_request: with _framework_event_origin(): - terminal_status = WorkflowEvent.status(WorkflowRunState.IDLE_WITH_PENDING_REQUESTS) - yield terminal_status + yield WorkflowEvent.status(WorkflowRunState.IDLE_WITH_PENDING_REQUESTS) else: with _framework_event_origin(): - terminal_status = WorkflowEvent.status(WorkflowRunState.IDLE) - yield terminal_status + yield WorkflowEvent.status(WorkflowRunState.IDLE) span.add_event(OtelAttr.WORKFLOW_COMPLETED) except Exception as exc: diff --git a/python/packages/devui/agent_framework_devui/_deployment.py b/python/packages/devui/agent_framework_devui/_deployment.py index 45f99a315a..db2de27ecf 100644 --- a/python/packages/devui/agent_framework_devui/_deployment.py +++ b/python/packages/devui/agent_framework_devui/_deployment.py @@ -92,8 +92,7 @@ async def deploy(self, config: DeploymentConfig, entity_path: Path) -> AsyncGene break # Get event from queue with short timeout - event = await asyncio.wait_for(event_queue.get(), timeout=0.1) - yield event + yield await asyncio.wait_for(event_queue.get(), timeout=0.1) except asyncio.TimeoutError: # No event in queue, continue waiting continue From f9f4d7382c8b1979b28edc13a6bcf415c2d8a347 Mon Sep 17 00:00:00 2001 From: Tao Chen Date: Thu, 26 Feb 2026 14:17:21 -0800 Subject: [PATCH 22/26] Fix ruff errors 2 --- .../packages/core/agent_framework/_workflows/_workflow.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/python/packages/core/agent_framework/_workflows/_workflow.py b/python/packages/core/agent_framework/_workflows/_workflow.py index ac4f2d1a8c..0dc90bcf01 100644 --- a/python/packages/core/agent_framework/_workflows/_workflow.py +++ b/python/packages/core/agent_framework/_workflows/_workflow.py @@ -384,11 +384,9 @@ async def _run_workflow_with_tracing( # Surface structured failure details before propagating exception details = WorkflowErrorDetails.from_exception(exc) with _framework_event_origin(): - failed_event = WorkflowEvent.failed(details) - yield failed_event + yield WorkflowEvent.failed(details) with _framework_event_origin(): - failed_status = WorkflowEvent.status(WorkflowRunState.FAILED) - yield failed_status + yield WorkflowEvent.status(WorkflowRunState.FAILED) span.add_event( name=OtelAttr.WORKFLOW_ERROR, attributes={ From 4995abf41fe3295aace9dbfea25e19789728315f Mon Sep 17 00:00:00 2001 From: Tao Chen Date: Thu, 26 Feb 2026 14:58:37 -0800 Subject: [PATCH 23/26] Revert workflow changes to fix tests --- .../agent_framework/_workflows/_workflow.py | 22 ++++++++++++------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/python/packages/core/agent_framework/_workflows/_workflow.py b/python/packages/core/agent_framework/_workflows/_workflow.py index 0dc90bcf01..09ba1048cd 100644 --- a/python/packages/core/agent_framework/_workflows/_workflow.py +++ b/python/packages/core/agent_framework/_workflows/_workflow.py @@ -333,9 +333,11 @@ async def _run_workflow_with_tracing( span.add_event(OtelAttr.WORKFLOW_STARTED) # Emit explicit start/status events to the stream with _framework_event_origin(): - yield WorkflowEvent.started() + started = WorkflowEvent.started() + yield started with _framework_event_origin(): - yield WorkflowEvent.status(WorkflowRunState.IN_PROGRESS) + in_progress = WorkflowEvent.status(WorkflowRunState.IN_PROGRESS) + yield in_progress # Reset context for a new run if supported if reset_context: @@ -365,15 +367,17 @@ async def _run_workflow_with_tracing( if event.type == "request_info" and not emitted_in_progress_pending: emitted_in_progress_pending = True with _framework_event_origin(): - yield WorkflowEvent.status(WorkflowRunState.IN_PROGRESS_PENDING_REQUESTS) - + pending_status = WorkflowEvent.status(WorkflowRunState.IN_PROGRESS_PENDING_REQUESTS) + yield pending_status # Workflow runs until idle - emit final status based on whether requests are pending if saw_request: with _framework_event_origin(): - yield WorkflowEvent.status(WorkflowRunState.IDLE_WITH_PENDING_REQUESTS) + terminal_status = WorkflowEvent.status(WorkflowRunState.IDLE_WITH_PENDING_REQUESTS) + yield terminal_status else: with _framework_event_origin(): - yield WorkflowEvent.status(WorkflowRunState.IDLE) + terminal_status = WorkflowEvent.status(WorkflowRunState.IDLE) + yield terminal_status span.add_event(OtelAttr.WORKFLOW_COMPLETED) except Exception as exc: @@ -384,9 +388,11 @@ async def _run_workflow_with_tracing( # Surface structured failure details before propagating exception details = WorkflowErrorDetails.from_exception(exc) with _framework_event_origin(): - yield WorkflowEvent.failed(details) + failed_event = WorkflowEvent.failed(details) + yield failed_event with _framework_event_origin(): - yield WorkflowEvent.status(WorkflowRunState.FAILED) + failed_status = WorkflowEvent.status(WorkflowRunState.FAILED) + yield failed_status span.add_event( name=OtelAttr.WORKFLOW_ERROR, attributes={ From d2fca7f64058881b8b70004bf37ceeae1cbefbc2 Mon Sep 17 00:00:00 2001 From: Tao Chen Date: Thu, 26 Feb 2026 15:25:59 -0800 Subject: [PATCH 24/26] Revert workflow changes to fix tests 2 --- .../core/agent_framework/_workflows/_workflow.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/python/packages/core/agent_framework/_workflows/_workflow.py b/python/packages/core/agent_framework/_workflows/_workflow.py index 09ba1048cd..486acc71aa 100644 --- a/python/packages/core/agent_framework/_workflows/_workflow.py +++ b/python/packages/core/agent_framework/_workflows/_workflow.py @@ -333,10 +333,10 @@ async def _run_workflow_with_tracing( span.add_event(OtelAttr.WORKFLOW_STARTED) # Emit explicit start/status events to the stream with _framework_event_origin(): - started = WorkflowEvent.started() + started = WorkflowEvent.started() # ruff: noqa: RUF070 yield started with _framework_event_origin(): - in_progress = WorkflowEvent.status(WorkflowRunState.IN_PROGRESS) + in_progress = WorkflowEvent.status(WorkflowRunState.IN_PROGRESS) # ruff: noqa: RUF070 yield in_progress # Reset context for a new run if supported @@ -367,16 +367,16 @@ async def _run_workflow_with_tracing( if event.type == "request_info" and not emitted_in_progress_pending: emitted_in_progress_pending = True with _framework_event_origin(): - pending_status = WorkflowEvent.status(WorkflowRunState.IN_PROGRESS_PENDING_REQUESTS) + pending_status = WorkflowEvent.status(WorkflowRunState.IN_PROGRESS_PENDING_REQUESTS) # ruff: noqa: RUF070 yield pending_status # Workflow runs until idle - emit final status based on whether requests are pending if saw_request: with _framework_event_origin(): - terminal_status = WorkflowEvent.status(WorkflowRunState.IDLE_WITH_PENDING_REQUESTS) + terminal_status = WorkflowEvent.status(WorkflowRunState.IDLE_WITH_PENDING_REQUESTS) # ruff: noqa: RUF070 yield terminal_status else: with _framework_event_origin(): - terminal_status = WorkflowEvent.status(WorkflowRunState.IDLE) + terminal_status = WorkflowEvent.status(WorkflowRunState.IDLE) # ruff: noqa: RUF070 yield terminal_status span.add_event(OtelAttr.WORKFLOW_COMPLETED) @@ -388,10 +388,10 @@ async def _run_workflow_with_tracing( # Surface structured failure details before propagating exception details = WorkflowErrorDetails.from_exception(exc) with _framework_event_origin(): - failed_event = WorkflowEvent.failed(details) + failed_event = WorkflowEvent.failed(details) # ruff: noqa: RUF070 yield failed_event with _framework_event_origin(): - failed_status = WorkflowEvent.status(WorkflowRunState.FAILED) + failed_status = WorkflowEvent.status(WorkflowRunState.FAILED) # ruff: noqa: RUF070 yield failed_status span.add_event( name=OtelAttr.WORKFLOW_ERROR, From 0e8f1e03449578c8ac7afe1be5955b96661da253 Mon Sep 17 00:00:00 2001 From: Tao Chen Date: Thu, 26 Feb 2026 16:30:24 -0800 Subject: [PATCH 25/26] Revert workflow changes to fix tests 3 --- .../core/agent_framework/_workflows/_workflow.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/python/packages/core/agent_framework/_workflows/_workflow.py b/python/packages/core/agent_framework/_workflows/_workflow.py index 486acc71aa..034447e22c 100644 --- a/python/packages/core/agent_framework/_workflows/_workflow.py +++ b/python/packages/core/agent_framework/_workflows/_workflow.py @@ -333,10 +333,10 @@ async def _run_workflow_with_tracing( span.add_event(OtelAttr.WORKFLOW_STARTED) # Emit explicit start/status events to the stream with _framework_event_origin(): - started = WorkflowEvent.started() # ruff: noqa: RUF070 + started = WorkflowEvent.started() # noqa: RUF070 yield started with _framework_event_origin(): - in_progress = WorkflowEvent.status(WorkflowRunState.IN_PROGRESS) # ruff: noqa: RUF070 + in_progress = WorkflowEvent.status(WorkflowRunState.IN_PROGRESS) # noqa: RUF070 yield in_progress # Reset context for a new run if supported @@ -367,16 +367,16 @@ async def _run_workflow_with_tracing( if event.type == "request_info" and not emitted_in_progress_pending: emitted_in_progress_pending = True with _framework_event_origin(): - pending_status = WorkflowEvent.status(WorkflowRunState.IN_PROGRESS_PENDING_REQUESTS) # ruff: noqa: RUF070 + pending_status = WorkflowEvent.status(WorkflowRunState.IN_PROGRESS_PENDING_REQUESTS) # noqa: RUF070 yield pending_status # Workflow runs until idle - emit final status based on whether requests are pending if saw_request: with _framework_event_origin(): - terminal_status = WorkflowEvent.status(WorkflowRunState.IDLE_WITH_PENDING_REQUESTS) # ruff: noqa: RUF070 + terminal_status = WorkflowEvent.status(WorkflowRunState.IDLE_WITH_PENDING_REQUESTS) # noqa: RUF070 yield terminal_status else: with _framework_event_origin(): - terminal_status = WorkflowEvent.status(WorkflowRunState.IDLE) # ruff: noqa: RUF070 + terminal_status = WorkflowEvent.status(WorkflowRunState.IDLE) # noqa: RUF070 yield terminal_status span.add_event(OtelAttr.WORKFLOW_COMPLETED) @@ -388,10 +388,10 @@ async def _run_workflow_with_tracing( # Surface structured failure details before propagating exception details = WorkflowErrorDetails.from_exception(exc) with _framework_event_origin(): - failed_event = WorkflowEvent.failed(details) # ruff: noqa: RUF070 + failed_event = WorkflowEvent.failed(details) # noqa: RUF070 yield failed_event with _framework_event_origin(): - failed_status = WorkflowEvent.status(WorkflowRunState.FAILED) # ruff: noqa: RUF070 + failed_status = WorkflowEvent.status(WorkflowRunState.FAILED) # noqa: RUF070 yield failed_status span.add_event( name=OtelAttr.WORKFLOW_ERROR, From b08dc0bd20cbfb7003e5acbc2861fe672e9bd2e1 Mon Sep 17 00:00:00 2001 From: Tao Chen Date: Thu, 26 Feb 2026 16:38:59 -0800 Subject: [PATCH 26/26] Revert workflow changes to fix tests 4 --- .../core/agent_framework/_workflows/_workflow.py | 14 +++++++------- python/pyproject.toml | 2 ++ 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/python/packages/core/agent_framework/_workflows/_workflow.py b/python/packages/core/agent_framework/_workflows/_workflow.py index 034447e22c..09ba1048cd 100644 --- a/python/packages/core/agent_framework/_workflows/_workflow.py +++ b/python/packages/core/agent_framework/_workflows/_workflow.py @@ -333,10 +333,10 @@ async def _run_workflow_with_tracing( span.add_event(OtelAttr.WORKFLOW_STARTED) # Emit explicit start/status events to the stream with _framework_event_origin(): - started = WorkflowEvent.started() # noqa: RUF070 + started = WorkflowEvent.started() yield started with _framework_event_origin(): - in_progress = WorkflowEvent.status(WorkflowRunState.IN_PROGRESS) # noqa: RUF070 + in_progress = WorkflowEvent.status(WorkflowRunState.IN_PROGRESS) yield in_progress # Reset context for a new run if supported @@ -367,16 +367,16 @@ async def _run_workflow_with_tracing( if event.type == "request_info" and not emitted_in_progress_pending: emitted_in_progress_pending = True with _framework_event_origin(): - pending_status = WorkflowEvent.status(WorkflowRunState.IN_PROGRESS_PENDING_REQUESTS) # noqa: RUF070 + pending_status = WorkflowEvent.status(WorkflowRunState.IN_PROGRESS_PENDING_REQUESTS) yield pending_status # Workflow runs until idle - emit final status based on whether requests are pending if saw_request: with _framework_event_origin(): - terminal_status = WorkflowEvent.status(WorkflowRunState.IDLE_WITH_PENDING_REQUESTS) # noqa: RUF070 + terminal_status = WorkflowEvent.status(WorkflowRunState.IDLE_WITH_PENDING_REQUESTS) yield terminal_status else: with _framework_event_origin(): - terminal_status = WorkflowEvent.status(WorkflowRunState.IDLE) # noqa: RUF070 + terminal_status = WorkflowEvent.status(WorkflowRunState.IDLE) yield terminal_status span.add_event(OtelAttr.WORKFLOW_COMPLETED) @@ -388,10 +388,10 @@ async def _run_workflow_with_tracing( # Surface structured failure details before propagating exception details = WorkflowErrorDetails.from_exception(exc) with _framework_event_origin(): - failed_event = WorkflowEvent.failed(details) # noqa: RUF070 + failed_event = WorkflowEvent.failed(details) yield failed_event with _framework_event_origin(): - failed_status = WorkflowEvent.status(WorkflowRunState.FAILED) # noqa: RUF070 + failed_status = WorkflowEvent.status(WorkflowRunState.FAILED) yield failed_status span.add_event( name=OtelAttr.WORKFLOW_ERROR, diff --git a/python/pyproject.toml b/python/pyproject.toml index a03123e9a2..e4e45f0290 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -148,6 +148,8 @@ ignore = [ "**/tests/**" = ["D", "INP", "TD", "ERA001", "RUF", "S"] "samples/**" = ["D", "INP", "ERA001", "RUF", "S", "T201", "CPY"] "*.ipynb" = ["CPY", "E501"] +# RUF070: Assignment before yield is intentional - context manager must exit before yielding +"**/agent_framework/_workflows/_workflow.py" = ["RUF070"] [tool.ruff.format] docstring-code-format = true