diff --git a/.github/workflows/E2E.yaml b/.github/workflows/E2E.yaml index f1a038c345..3ae66a0b35 100644 --- a/.github/workflows/E2E.yaml +++ b/.github/workflows/E2E.yaml @@ -232,6 +232,23 @@ jobs: $allScenarios = @(Get-ChildItem -Path (Join-Path $ENV:GITHUB_WORKSPACE "e2eTests/scenarios/*/runtest.ps1") | ForEach-Object { $_.Directory.Name }) $filteredScenarios = $allScenarios | Where-Object { $scenario = $_; $scenariosFilter | ForEach-Object { $scenario -like $_ } } + # Load disabled scenarios from config file + $disabledScenariosConfig = Get-Content -path (Join-Path $ENV:GITHUB_WORKSPACE "e2eTests/disabled-scenarios.json") -encoding UTF8 -raw | ConvertFrom-Json + $disabledScenarios = @($disabledScenariosConfig | ForEach-Object { $_.scenario }) + Write-Host "Disabled scenarios from config: $($disabledScenarios -join ', ')" + + # Filter out disabled scenarios + $beforeFilter = $filteredScenarios.Count + $filteredScenarios = $filteredScenarios | Where-Object { $disabledScenarios -notcontains $_ } + $afterFilter = $filteredScenarios.Count + if ($beforeFilter -ne $afterFilter) { + Write-Host "Filtered out $($beforeFilter - $afterFilter) disabled scenario(s)" + $disabledScenariosConfig | Where-Object { $filteredScenarios -notcontains $_.scenario } | ForEach-Object { + Write-Host " - $($_.scenario): $($_.reason)" + } + } + Write-Host "Scenarios to run: $($filteredScenarios -join ', ')" + $scenariosJson = @{ "matrix" = @{ "include" = @($filteredScenarios | ForEach-Object { @{ "Scenario" = $_ } }) diff --git a/e2eTests/disabled-scenarios.json b/e2eTests/disabled-scenarios.json new file mode 100644 index 0000000000..f95fda8561 --- /dev/null +++ b/e2eTests/disabled-scenarios.json @@ -0,0 +1,6 @@ +[ + { + "scenario": "FederatedCredentials", + "reason": "Azure resource migration work in progress" + } +] diff --git a/e2eTests/e2eTestHelper.psm1 b/e2eTests/e2eTestHelper.psm1 index 3a858baa1c..557cf0fc0a 100644 --- a/e2eTests/e2eTestHelper.psm1 +++ b/e2eTests/e2eTestHelper.psm1 @@ -328,6 +328,141 @@ function SetRepositorySecret { gh secret set $name -b $value --repo $repository } +<# +.SYNOPSIS + Deletes all workflow runs in a repository. + +.DESCRIPTION + Cleans up workflow runs in a GitHub repository by deleting all existing runs. + This is useful for ensuring a clean state before running tests that track specific workflow runs. + +.PARAMETER repository + The full repository name in the format "owner/repo" (e.g., "microsoft/AL-Go"). + +.EXAMPLE + CleanupWorkflowRuns -repository "microsoft/AL-Go" +#> +function CleanupWorkflowRuns { + Param( + [Parameter(Mandatory = $true)] + [string] $repository + ) + + Write-Host -ForegroundColor Yellow "`nCleaning up workflow runs in $repository" + + RefreshToken -repository $repository + + # Get all workflow runs + $runs = invoke-gh api "/repos/$repository/actions/runs?per_page=100" -silent -returnValue | ConvertFrom-Json + + if ($runs.workflow_runs.Count -eq 0) { + Write-Host "No workflow runs found" + return + } + + Write-Host "Deleting $($runs.workflow_runs.Count) workflow runs..." + foreach ($run in $runs.workflow_runs) { + try { + Write-Host "Deleting run $($run.id) ($($run.name) - $($run.status))" + invoke-gh api /repos/$repository/actions/runs/$($run.id) --method DELETE -silent | Out-Null + } + catch { + Write-Host "Warning: Failed to delete run $($run.id): $_" + } + } + Write-Host "Cleanup completed" +} + +<# +.SYNOPSIS + Resets a repository to match the content of a source repository. + +.DESCRIPTION + Clones the target repository, fetches content from a source repository, and performs a hard reset + followed by a force push. This preserves the repository identity while resetting its content to + match the source repository. Useful for ensuring deterministic state in end-to-end tests. + +.PARAMETER repository + The full name of the target repository to reset in the format "owner/repo" (e.g., "microsoft/AL-Go"). + +.PARAMETER sourceRepository + The full name of the source repository to copy content from in the format "owner/repo". + +.PARAMETER branch + The branch name to reset. Defaults to "main". + +.EXAMPLE + ResetRepositoryToSource -repository "microsoft/test-repo" -sourceRepository "microsoft/source-repo" -branch "main" +#> +function ResetRepositoryToSource { + Param( + [Parameter(Mandatory = $true)] + [string] $repository, + [Parameter(Mandatory = $true)] + [string] $sourceRepository, + [string] $branch = "main" + ) + + Write-Host -ForegroundColor Yellow "`nResetting repository $repository to match $sourceRepository" + + RefreshToken -repository $repository + + # Clone the repository locally if not already in it + $tempPath = [System.IO.Path]::GetTempPath() + $repoPath = Join-Path $tempPath ([System.Guid]::NewGuid().ToString()) + New-Item $repoPath -ItemType Directory | Out-Null + + Push-Location $repoPath + try { + Write-Host "Cloning $repository..." + invoke-gh repo clone $repository . + if ($LASTEXITCODE -ne 0) { + throw "Failed to clone repository $repository" + } + + # Fetch the source repository content + Write-Host "Fetching source repository $sourceRepository..." + invoke-git remote add source "https://github.com/$sourceRepository.git" + if ($LASTEXITCODE -ne 0) { + throw "Failed to add remote source for $sourceRepository" + } + + invoke-git fetch source $branch --quiet + if ($LASTEXITCODE -ne 0) { + throw "Failed to fetch branch $branch from source $sourceRepository" + } + + # Reset the current branch to match the source + Write-Host "Resetting $branch to match source/$branch..." + invoke-git checkout $branch --quiet + if ($LASTEXITCODE -ne 0) { + throw "Failed to checkout branch $branch" + } + + invoke-git reset --hard "source/$branch" --quiet + if ($LASTEXITCODE -ne 0) { + throw "Failed to reset branch $branch to source/$branch" + } + + # Force push to update the repository + Write-Host "Force pushing changes..." + invoke-git push origin $branch --force --quiet + if ($LASTEXITCODE -ne 0) { + throw "Failed to push changes to $repository" + } + + Write-Host "Repository reset completed successfully" + } + catch { + Write-Host "Error resetting repository: $_" + throw + } + finally { + Pop-Location + Remove-Item -Path $repoPath -Force -Recurse -ErrorAction SilentlyContinue + } +} + function CreateNewAppInFolder { Param( [string] $folder, diff --git a/e2eTests/scenarios/FederatedCredentials/runtest.ps1 b/e2eTests/scenarios/FederatedCredentials/runtest.ps1 index 3aad793fe8..3c86b09abc 100644 --- a/e2eTests/scenarios/FederatedCredentials/runtest.ps1 +++ b/e2eTests/scenarios/FederatedCredentials/runtest.ps1 @@ -29,18 +29,23 @@ Write-Host -ForegroundColor Yellow @' # This test uses the bcsamples-bingmaps.appsource repository and will deliver a new build of the app to AppSource. # The bcsamples-bingmaps.appsource repository is setup to use an Azure KeyVault for secrets and app signing. # -# During the test, the bcsamples-bingmaps.appsource repository will be copied to a new repository called tmp-bingmaps.appsource. -# tmp-bingmaps.appsource has access to the same Azure KeyVault as bcsamples-bingmaps.appsource using federated credentials. +# The test requires a stable temporary repository called e2e-bingmaps.appsource that must be manually created +# with federated credentials configured before running this test. +# This is required because federated credentials no longer work with repository name-based matching, +# so the repository must remain stable to maintain the federated credential configuration. +# e2e-bingmaps.appsource has access to the same Azure KeyVault as bcsamples-bingmaps.appsource using federated credentials. # The bcSamples-bingmaps.appsource repository is setup for continuous delivery to AppSource -# tmp-bingmaps.appsource also has access to the Entra ID app registration for delivering to AppSource using federated credentials. +# e2e-bingmaps.appsource also has access to the Entra ID app registration for delivering to AppSource using federated credentials. # This test will deliver another build of the latest app version already delivered to AppSource (without go-live) # # This test tests the following scenario: # -# - Create a new repository called tmp-bingmaps.appsource (based on bcsamples-bingmaps.appsource) -# - Update AL-Go System Files in branch main in tmp-bingmaps.appsource -# - Update version numbers in app.json in tmp-bingmaps.appsource in order to not be lower than the version number in AppSource (and not be higher than the next version from bcsamples-bingmaps.appsource) -# - Wait for CI/CD in branch main in repository tmp-bingmaps.appsource +# - Verify that the repository e2e-bingmaps.appsource exists (error out if not) +# - Reset the repository to match bcsamples-bingmaps.appsource for deterministic state +# - Clean up old workflow runs to ensure proper workflow tracking +# - Update AL-Go System Files in branch main in e2e-bingmaps.appsource +# - Update version numbers in app.json in e2e-bingmaps.appsource in order to not be lower than the version number in AppSource (and not be higher than the next version from bcsamples-bingmaps.appsource) +# - Wait for CI/CD in branch main in repository e2e-bingmaps.appsource # - Check that artifacts are created and signed # - Check that the app is delivered to AppSource '@ @@ -55,38 +60,88 @@ if ($linux) { Remove-Module e2eTestHelper -ErrorAction SilentlyContinue Import-Module (Join-Path $PSScriptRoot "..\..\e2eTestHelper.psm1") -DisableNameChecking -$repository = "$githubOwner/tmp-bingmaps.appsource" +$repository = "$githubOwner/e2e-bingmaps.appsource" $template = "https://github.com/$appSourceTemplate" $sourceRepository = 'microsoft/bcsamples-bingmaps.appsource' # E2E test will create a copy of this repository -# Create temp repository from sourceRepository +# Setup authentication and repository SetTokenAndRepository -github:$github -githubOwner $githubOwner -appId $e2eAppId -appKey $e2eAppKey -repository $repository +# Check if the repository already exists +# This repository must exist with federated credentials already configured gh api repos/$repository --method HEAD -if ($LASTEXITCODE -eq 0) { - Write-Host "Repository $repository already exists. Deleting it." - gh repo delete $repository --yes | Out-Host - Start-Sleep -Seconds 30 +if ($LASTEXITCODE -ne 0) { + throw "Repository $repository does not exist. The repository must be created manually with federated credentials configured before running this test." } -CreateAlGoRepository ` - -github:$github ` - -template "https://github.com/$sourceRepository" ` - -repository $repository ` - -addRepoSettings @{"ghTokenWorkflowSecretName" = "e2eghTokenWorkflow" } +# Repository exists - reuse it and reset to source state +# This is required because federated credentials no longer work with repository name-based matching, +# so the repository must remain stable across test runs +Write-Host "Repository $repository exists. Reusing and resetting to match source." +# Reset the repository to match the source repository +ResetRepositoryToSource -repository $repository -sourceRepository $sourceRepository -branch 'main' + +# Clean up workflow runs to ensure proper workflow tracking +CleanupWorkflowRuns -repository $repository + +# Always set/update secrets (they may have changed or repo may have been reset) SetRepositorySecret -repository $repository -name 'Azure_Credentials' -value $azureCredentials +# Re-apply the custom repository settings that were lost during reset +$tempPath = [System.IO.Path]::GetTempPath() +$repoPath = Join-Path $tempPath ([System.Guid]::NewGuid().ToString()) +New-Item $repoPath -ItemType Directory | Out-Null +Push-Location $repoPath +try { + Write-Host "Re-applying repository settings..." + invoke-gh repo clone $repository . + $repoSettingsFile = ".github\AL-Go-Settings.json" + if (Test-Path $repoSettingsFile) { + Add-PropertiesToJsonFile -path $repoSettingsFile -properties @{"ghTokenWorkflowSecretName" = "e2eghTokenWorkflow"} + invoke-git add $repoSettingsFile + invoke-git commit -m "Update repository settings for test" --quiet + invoke-git push --quiet + } + else { + Write-Host "Warning: .github\AL-Go-Settings.json not found after cloning. Settings may not be applied correctly." + } +} +finally { + Pop-Location + Remove-Item -Path $repoPath -Force -Recurse -ErrorAction SilentlyContinue +} + # Upgrade AL-Go System Files to test version -RunUpdateAlGoSystemFiles -directCommit -wait -templateUrl $template -repository $repository | Out-Null +# Capture the run object to ensure we wait for the correct workflow run +$updateRun = RunUpdateAlGoSystemFiles -directCommit -wait -templateUrl $template -repository $repository # Wait for CI/CD to complete +# The Update AL-Go System Files workflow triggers a CI/CD workflow via push event +# We need to wait for the CI/CD workflow that was triggered AFTER the update workflow completed +Write-Host "Waiting for CI/CD workflow to start (triggered by Update AL-Go System Files)..." Start-Sleep -Seconds 60 + +# Get workflow runs that started after the update workflow +# Use created_at for consistent timestamp comparison, and add a small buffer for timing precision +$updateCreatedAt = [DateTime]$updateRun.created_at $runs = invoke-gh api /repos/$repository/actions/runs -silent -returnValue | ConvertFrom-Json -$run = $runs.workflow_runs | Select-Object -First 1 + +# Find the CI/CD workflow run that started after the update workflow was created +$run = $runs.workflow_runs | Where-Object { + $_.event -eq 'push' -and [DateTime]$_.created_at -gt $updateCreatedAt +} | Select-Object -First 1 + +if (-not $run) { + # Fallback to the first workflow run if we can't find one based on timestamp + Write-Host "Warning: Could not find CI/CD run based on timestamp, using first run" + $run = $runs.workflow_runs | Select-Object -First 1 +} + +Write-Host "Waiting for CI/CD workflow run $($run.id) to complete..." WaitWorkflow -repository $repository -runid $run.id -noError -# The CI/CD workflow should fail because the version number of the app in thie repository is lower than the version number in AppSource +# The CI/CD workflow should fail because the version number of the app in the repository is lower than the version number in AppSource # Reason being that major.minor from the original bcsamples-bingmaps.appsource is the same and the build number in the newly created repository is lower than the one in AppSource # This error is expected we will grab the version number from AppSource, add one to revision number (by switching to versioningstrategy 3 in the tmp repo) and use it in the next run $MatchArr = Test-LogContainsFromRun -repository $repository -runid $run.id -jobName 'Deliver to AppSource' -stepName 'Deliver' -expectedText '(?m)^.*The new version number \((\d+(?:\.\d+){3})\) is lower than the existing version number \((\d+(?:\.\d+){3})\) in Partner Center.*$' -isRegEx