From a7e4e33e4d214345adbc93f45abaabfc0f8b2a7f Mon Sep 17 00:00:00 2001 From: Gerald Versluis Date: Tue, 16 Dec 2025 11:03:01 +0100 Subject: [PATCH 01/19] Add PR category detection for UI tests --- eng/pipelines/common/ui-tests.yml | 257 ++++++++++++++++------ eng/scripts/detect-ui-test-categories.ps1 | 107 +++++++++ 2 files changed, 302 insertions(+), 62 deletions(-) create mode 100644 eng/scripts/detect-ui-test-categories.ps1 diff --git a/eng/pipelines/common/ui-tests.yml b/eng/pipelines/common/ui-tests.yml index efdad1964c0c..2f1b84ab8098 100644 --- a/eng/pipelines/common/ui-tests.yml +++ b/eng/pipelines/common/ui-tests.yml @@ -12,32 +12,6 @@ parameters: skipProvisioning: true BuildNativeAOT: false # Parameter to control whether NativeAOT artifacts should be built RunNativeAOT: false # Parameter to control whether NativeAOT UI tests should run - categoryGroupsToTest: - # Make sure that this list is always up-to-date with src/Controls/tests/TestCases.Shared.Tests/UITestCategories.cs - # we might want to improve this somehow depending on how much the categories change over time - - 'Accessibility,ActionSheet,ActivityIndicator,Animation,AppLinks' - - 'Border,BoxView,Brush,Button' - - 'CarouselView' - - 'Cells,CheckBox,ContextActions,CustomRenderers,DatePicker,Dispatcher,DisplayAlert,DisplayPrompt,DragAndDrop' - - 'CollectionView' - - 'Entry' - - 'Editor,Effects,FlyoutPage,Focus,Fonts,Frame,Gestures,GraphicsView' - - 'Image,ImageButton,IndicatorView,InputTransparent,IsEnabled,IsVisible' - - 'Label' - - 'Layout' - - 'Lifecycle,ManualReview,Maps' - - 'ListView' - - 'Navigation' - - 'Page,Performance,Picker,ProgressBar' - - 'RadioButton,RefreshView' - - 'SafeAreaEdges,Shadow' - - 'ScrollView' - - 'SearchBar,Shape,Slider' - - 'SoftInput,Stepper,Switch,SwipeView' - - 'Shell' - - 'TabbedPage,TableView,TimePicker,TitleView,ToolbarItem' - - 'ViewBaseTests,Window' - - 'WebView' projects: - name: name @@ -48,10 +22,64 @@ parameters: mac: /optional/path/to/mac.csproj app: /optional/path/to/app.csproj +variables: + UITestCategoryMatrixDefault: | + { + "Group00": { "CATEGORYGROUP": "Accessibility,ActionSheet,ActivityIndicator,Animation,AppLinks" }, + "Group01": { "CATEGORYGROUP": "Border,BoxView,Brush,Button" }, + "Group02": { "CATEGORYGROUP": "CarouselView" }, + "Group03": { "CATEGORYGROUP": "Cells,CheckBox,ContextActions,CustomRenderers,DatePicker,Dispatcher,DisplayAlert,DisplayPrompt,DragAndDrop" }, + "Group04": { "CATEGORYGROUP": "CollectionView" }, + "Group05": { "CATEGORYGROUP": "Entry" }, + "Group06": { "CATEGORYGROUP": "Editor,Effects,FlyoutPage,Focus,Fonts,Frame,Gestures,GraphicsView" }, + "Group07": { "CATEGORYGROUP": "Image,ImageButton,IndicatorView,InputTransparent,IsEnabled,IsVisible" }, + "Group08": { "CATEGORYGROUP": "Label" }, + "Group09": { "CATEGORYGROUP": "Layout" }, + "Group10": { "CATEGORYGROUP": "Lifecycle,ManualReview,Maps" }, + "Group11": { "CATEGORYGROUP": "ListView" }, + "Group12": { "CATEGORYGROUP": "Navigation" }, + "Group13": { "CATEGORYGROUP": "Page,Performance,Picker,ProgressBar" }, + "Group14": { "CATEGORYGROUP": "RadioButton,RefreshView" }, + "Group15": { "CATEGORYGROUP": "SafeAreaEdges,Shadow" }, + "Group16": { "CATEGORYGROUP": "ScrollView" }, + "Group17": { "CATEGORYGROUP": "SearchBar,Shape,Slider" }, + "Group18": { "CATEGORYGROUP": "SoftInput,Stepper,Switch,SwipeView" }, + "Group19": { "CATEGORYGROUP": "Shell" }, + "Group20": { "CATEGORYGROUP": "TabbedPage,TableView,TimePicker,TitleView,ToolbarItem" }, + "Group21": { "CATEGORYGROUP": "ViewBaseTests,Window" }, + "Group22": { "CATEGORYGROUP": "WebView" } + } + stages: + - stage: discover_ui_test_categories + displayName: Discover UI Test Categories + condition: eq(variables['Build.Reason'], 'PullRequest') + jobs: + - job: detect + displayName: Detect category filters + pool: + vmImage: ubuntu-latest + steps: + - checkout: self + fetchDepth: 0 + - task: PowerShell@2 + name: DetectCategories + displayName: Determine UI test categories for PRs + inputs: + pwsh: true + filePath: $(System.DefaultWorkingDirectory)/eng/scripts/detect-ui-test-categories.ps1 + arguments: '-TargetBranch "$(System.PullRequest.TargetBranch)"' + - stage: build_ui_tests displayName: Build UITests Sample App - dependsOn: [] + dependsOn: + - discover_ui_test_categories + condition: | + or( + ne(variables['Build.Reason'], 'PullRequest'), + eq(dependencies.discover_ui_test_categories.result, 'Succeeded'), + eq(dependencies.discover_ui_test_categories.result, 'Skipped') + ) jobs: - job: build_ui_tests displayName: Build Sample App @@ -67,7 +95,14 @@ stages: - stage: build_ui_tests_coreclr displayName: Build UITests CoreCLR Sample App - dependsOn: [] + dependsOn: + - discover_ui_test_categories + condition: | + or( + ne(variables['Build.Reason'], 'PullRequest'), + eq(dependencies.discover_ui_test_categories.result, 'Succeeded'), + eq(dependencies.discover_ui_test_categories.result, 'Skipped') + ) jobs: - job: build_ui_tests displayName: Build Sample App @@ -85,7 +120,14 @@ stages: - ${{ if eq(parameters.BuildNativeAOT, true) }}: - stage: build_ui_tests_nativeaot displayName: Build UITests NativeAOT Sample App - dependsOn: [] + dependsOn: + - discover_ui_test_categories + condition: | + or( + ne(variables['Build.Reason'], 'PullRequest'), + eq(dependencies.discover_ui_test_categories.result, 'Succeeded'), + eq(dependencies.discover_ui_test_categories.result, 'Skipped') + ) jobs: - job: build_ui_tests displayName: Build Sample App @@ -102,7 +144,14 @@ stages: - stage: build_ui_tests_windows displayName: Build UITests Windows Sample App - dependsOn: [] + dependsOn: + - discover_ui_test_categories + condition: | + or( + ne(variables['Build.Reason'], 'PullRequest'), + eq(dependencies.discover_ui_test_categories.result, 'Succeeded'), + eq(dependencies.discover_ui_test_categories.result, 'Skipped') + ) jobs: - job: build_ui_tests displayName: Build Sample App (Windows) @@ -117,7 +166,20 @@ stages: - stage: android_ui_tests displayName: Android UITests - dependsOn: build_ui_tests + dependsOn: + - discover_ui_test_categories + - build_ui_tests + condition: | + and( + succeeded('build_ui_tests'), + or( + ne(variables['Build.Reason'], 'PullRequest'), + eq(dependencies.discover_ui_test_categories.result, 'Succeeded'), + eq(dependencies.discover_ui_test_categories.result, 'Skipped') + ) + ) + variables: + StageCategoryMatrix: ${{ coalesce(stageDependencies.discover_ui_test_categories.detect.outputs['DetectCategories.UITestCategoryMatrix'], variables['UITestCategoryMatrixDefault']) }} jobs: - ${{ each project in parameters.projects }}: - ${{ if ne(project.android, '') }}: @@ -125,10 +187,7 @@ stages: - ${{ if not(containsValue(project.androidApiLevelsExclude, api)) }}: - job: android_ui_tests_${{ project.name }}_${{ api }} strategy: - matrix: - ${{ each categoryGroup in parameters.categoryGroupsToTest }}: - ${{ categoryGroup }}: - CATEGORYGROUP: ${{ categoryGroup }} + matrix: $[ fromJson(variables['StageCategoryMatrix']) ] timeoutInMinutes: 240 # how long to run the job before automatically cancelling workspace: clean: all @@ -161,7 +220,20 @@ stages: - stage: android_ui_tests_coreclr displayName: Android UITests CoreClr - dependsOn: build_ui_tests_coreclr + dependsOn: + - discover_ui_test_categories + - build_ui_tests_coreclr + condition: | + and( + succeeded('build_ui_tests_coreclr'), + or( + ne(variables['Build.Reason'], 'PullRequest'), + eq(dependencies.discover_ui_test_categories.result, 'Succeeded'), + eq(dependencies.discover_ui_test_categories.result, 'Skipped') + ) + ) + variables: + StageCategoryMatrix: ${{ coalesce(stageDependencies.discover_ui_test_categories.detect.outputs['DetectCategories.UITestCategoryMatrix'], variables['UITestCategoryMatrixDefault']) }} jobs: - ${{ each project in parameters.projects }}: - ${{ if ne(project.android, '') }}: @@ -169,10 +241,7 @@ stages: - ${{ if not(containsValue(project.androidApiLevelsExclude, api)) }}: - job: android_ui_tests_${{ project.name }}_${{ api }} strategy: - matrix: - ${{ each categoryGroup in parameters.categoryGroupsToTest }}: - ${{ categoryGroup }}: - CATEGORYGROUP: ${{ categoryGroup }} + matrix: $[ fromJson(variables['StageCategoryMatrix']) ] timeoutInMinutes: 240 # how long to run the job before automatically cancelling workspace: clean: all @@ -207,7 +276,20 @@ stages: - stage: ios_ui_tests_mono displayName: iOS UITests Mono - dependsOn: build_ui_tests + dependsOn: + - discover_ui_test_categories + - build_ui_tests + condition: | + and( + succeeded('build_ui_tests'), + or( + ne(variables['Build.Reason'], 'PullRequest'), + eq(dependencies.discover_ui_test_categories.result, 'Succeeded'), + eq(dependencies.discover_ui_test_categories.result, 'Skipped') + ) + ) + variables: + StageCategoryMatrix: ${{ coalesce(stageDependencies.discover_ui_test_categories.detect.outputs['DetectCategories.UITestCategoryMatrix'], variables['UITestCategoryMatrixDefault']) }} jobs: - ${{ each project in parameters.projects }}: - ${{ if ne(project.ios, '') }}: @@ -215,10 +297,7 @@ stages: - ${{ if not(containsValue(project.iosVersionsExclude, version)) }}: - job: ios_ui_tests_mono_${{ project.name }}_${{ replace(version, '.', '_') }} strategy: - matrix: - ${{ each categoryGroup in parameters.categoryGroupsToTest }}: - ${{ categoryGroup }}: - CATEGORYGROUP: ${{ categoryGroup }} + matrix: $[ fromJson(variables['StageCategoryMatrix']) ] timeoutInMinutes: ${{ parameters.timeoutInMinutes }} # how long to run the job before automatically cancelling workspace: clean: all @@ -255,7 +334,19 @@ stages: - stage: ios_ui_tests_mono_cv2 displayName: iOS UITests Mono CollectionView2 - dependsOn: build_ui_tests + dependsOn: + - discover_ui_test_categories + - build_ui_tests + condition: | + and( + succeeded('build_ui_tests'), + or( + ne(variables['Build.Reason'], 'PullRequest'), + eq(dependencies.discover_ui_test_categories.result, 'Skipped'), + eq(stageDependencies.discover_ui_test_categories.detect.outputs['DetectCategories.UITestCategoryMatrix'], ''), + contains(stageDependencies.discover_ui_test_categories.detect.outputs['DetectCategories.UITestCategoryMatrix'], '"CollectionView"') + ) + ) jobs: - ${{ each project in parameters.projects }}: - ${{ if ne(project.ios, '') }}: @@ -299,7 +390,19 @@ stages: - stage: ios_ui_tests_mono_carv2 displayName: iOS UITests Mono CarouselView2 - dependsOn: build_ui_tests + dependsOn: + - discover_ui_test_categories + - build_ui_tests + condition: | + and( + succeeded('build_ui_tests'), + or( + ne(variables['Build.Reason'], 'PullRequest'), + eq(dependencies.discover_ui_test_categories.result, 'Skipped'), + eq(stageDependencies.discover_ui_test_categories.detect.outputs['DetectCategories.UITestCategoryMatrix'], ''), + contains(stageDependencies.discover_ui_test_categories.detect.outputs['DetectCategories.UITestCategoryMatrix'], '"CarouselView"') + ) + ) jobs: - ${{ each project in parameters.projects }}: - ${{ if ne(project.ios, '') }}: @@ -344,7 +447,20 @@ stages: - ${{ if and(eq(parameters.BuildNativeAOT, true), eq(parameters.RunNativeAOT, true)) }}: - stage: ios_ui_tests_nativeaot displayName: iOS UITests NativeAOT - dependsOn: build_ui_tests_nativeaot + dependsOn: + - discover_ui_test_categories + - build_ui_tests_nativeaot + condition: | + and( + succeeded('build_ui_tests_nativeaot'), + or( + ne(variables['Build.Reason'], 'PullRequest'), + eq(dependencies.discover_ui_test_categories.result, 'Succeeded'), + eq(dependencies.discover_ui_test_categories.result, 'Skipped') + ) + ) + variables: + StageCategoryMatrix: ${{ coalesce(stageDependencies.discover_ui_test_categories.detect.outputs['DetectCategories.UITestCategoryMatrix'], variables['UITestCategoryMatrixDefault']) }} jobs: - ${{ each project in parameters.projects }}: - ${{ if ne(project.ios, '') }}: @@ -352,10 +468,7 @@ stages: - ${{ if not(containsValue(project.iosVersionsExclude, version)) }}: - job: ios_ui_tests_nativeaot_${{ project.name }}_${{ replace(version, '.', '_') }} strategy: - matrix: - ${{ each categoryGroup in parameters.categoryGroupsToTest }}: - ${{ categoryGroup }}: - CATEGORYGROUP: ${{ categoryGroup }} + matrix: $[ fromJson(variables['StageCategoryMatrix']) ] timeoutInMinutes: ${{ parameters.timeoutInMinutes }} # how long to run the job before automatically cancelling workspace: clean: all @@ -386,16 +499,26 @@ stages: - stage: winui_ui_tests displayName: WinUI UITests - dependsOn: build_ui_tests_windows + dependsOn: + - discover_ui_test_categories + - build_ui_tests_windows + condition: | + and( + succeeded('build_ui_tests_windows'), + or( + ne(variables['Build.Reason'], 'PullRequest'), + eq(dependencies.discover_ui_test_categories.result, 'Succeeded'), + eq(dependencies.discover_ui_test_categories.result, 'Skipped') + ) + ) + variables: + StageCategoryMatrix: ${{ coalesce(stageDependencies.discover_ui_test_categories.detect.outputs['DetectCategories.UITestCategoryMatrix'], variables['UITestCategoryMatrixDefault']) }} jobs: - ${{ each project in parameters.projects }}: - ${{ if ne(project.winui, '') }}: - job: winui_ui_tests_${{ project.name }} strategy: - matrix: - ${{ each categoryGroup in parameters.categoryGroupsToTest }}: - ${{ categoryGroup }}: - CATEGORYGROUP: ${{ categoryGroup }} + matrix: $[ fromJson(variables['StageCategoryMatrix']) ] timeoutInMinutes: ${{ parameters.timeoutInMinutes }} # how long to run the job before automatically cancelling workspace: clean: all @@ -423,16 +546,26 @@ stages: - stage: mac_ui_tests displayName: macOS UITests - dependsOn: build_ui_tests + dependsOn: + - discover_ui_test_categories + - build_ui_tests + condition: | + and( + succeeded('build_ui_tests'), + or( + ne(variables['Build.Reason'], 'PullRequest'), + eq(dependencies.discover_ui_test_categories.result, 'Succeeded'), + eq(dependencies.discover_ui_test_categories.result, 'Skipped') + ) + ) + variables: + StageCategoryMatrix: ${{ coalesce(stageDependencies.discover_ui_test_categories.detect.outputs['DetectCategories.UITestCategoryMatrix'], variables['UITestCategoryMatrixDefault']) }} jobs: - ${{ each project in parameters.projects }}: - ${{ if ne(project.mac, '') }}: - job: mac_ui_tests_${{ project.name }} strategy: - matrix: - ${{ each categoryGroup in parameters.categoryGroupsToTest }}: - ${{ categoryGroup }}: - CATEGORYGROUP: ${{ categoryGroup }} + matrix: $[ fromJson(variables['StageCategoryMatrix']) ] timeoutInMinutes: ${{ parameters.timeoutInMinutes }} # how long to run the job before automatically cancelling workspace: clean: all diff --git a/eng/scripts/detect-ui-test-categories.ps1 b/eng/scripts/detect-ui-test-categories.ps1 new file mode 100644 index 000000000000..39aeef4f2389 --- /dev/null +++ b/eng/scripts/detect-ui-test-categories.ps1 @@ -0,0 +1,107 @@ +[CmdletBinding()] +param( + [string]$TargetBranch, + [string]$TestRoot = "src/Controls/tests/TestCases.Shared.Tests" +) + +$buildReason = $env:BUILD_REASON +if ([string]::IsNullOrWhiteSpace($buildReason)) { + $buildReason = $env:SYSTEM_REASON +} + +if ($buildReason -ne 'PullRequest') { + Write-Host "Build reason '$buildReason' is not PullRequest. Skipping category detection." -ForegroundColor Cyan + return +} + +if ([string]::IsNullOrWhiteSpace($TargetBranch)) { + $TargetBranch = $env:SYSTEM_PULLREQUEST_TARGETBRANCH +} + +if ([string]::IsNullOrWhiteSpace($TargetBranch)) { + Write-Warning "Unable to determine target branch for comparison. Falling back to running all categories." + return +} + +$targetBranch = $TargetBranch -replace '^refs/heads/', '' + +Write-Host "Fetching target branch 'origin/$targetBranch' for diff analysis..." -ForegroundColor Cyan +try { + git fetch origin "$targetBranch" --no-tags --prune --depth=200 | Out-Null +} catch { + Write-Warning "Failed to fetch origin/$targetBranch: $($_.Exception.Message). Falling back to running all categories." + return +} + +$mergeBase = $null +try { + $mergeBase = (git merge-base HEAD "origin/$targetBranch").Trim() +} catch { + Write-Warning "Could not determine merge base with origin/$targetBranch: $($_.Exception.Message). Falling back to running all categories." + return +} + +if ([string]::IsNullOrWhiteSpace($mergeBase)) { + Write-Warning "Merge base calculation returned empty result. Falling back to running all categories." + return +} + +Write-Host "Calculating diff between $mergeBase and HEAD limited to '$TestRoot'..." -ForegroundColor Cyan +$diff = git diff --diff-filter=AMR --unified=0 $mergeBase HEAD -- "$TestRoot" +if ([string]::IsNullOrWhiteSpace($diff)) { + Write-Host "No changes detected under '$TestRoot'. Falling back to default category matrix." -ForegroundColor Cyan + return +} + +$categoryPattern = '^[+]{1}\s*\[Category\((?[^\)]*)\)\]' +$addedCategories = [System.Collections.Generic.HashSet[string]]::new([System.StringComparer]::OrdinalIgnoreCase) + +foreach ($line in $diff -split "`n") { + if ($line -match $categoryPattern) { + $rawValue = $Matches['value'].Trim() + if ([string]::IsNullOrWhiteSpace($rawValue)) { + continue + } + + # Normalize value: UITestCategories.XYZ => XYZ, quoted strings => trimmed text + if ($rawValue -match '^UITestCategories\.(?[A-Za-z0-9_]+)$') { + $category = $Matches['name'] + } elseif ($rawValue -match '^["\'](?[A-Za-z0-9_\- ]+)["\']$') { + $category = $Matches['name'] + } else { + # Attempt to evaluate nameof-style constructs or fallback to raw value + if ($rawValue -match 'nameof\(UITestCategories\.(?[A-Za-z0-9_]+)\)') { + $category = $Matches['name'] + } else { + Write-Warning "Unrecognized category expression '$rawValue'. Falling back to running all categories." + return + } + } + + $category = $category.Trim() + if (-not [string]::IsNullOrWhiteSpace($category)) { + $addedCategories.Add($category) | Out-Null + } + } +} + +if ($addedCategories.Count -eq 0) { + Write-Host "No new Category attributes detected in diff. Using default category matrix." -ForegroundColor Cyan + return +} + +Write-Host "Detected categories from PR changes: $([string]::Join(', ', $addedCategories))" -ForegroundColor Green + +# Build matrix JSON expected by Azure Pipelines strategy matrix (CATEGORYGROUP values) +$matrix = [ordered]@{} +$index = 0 +foreach ($category in ($addedCategories | Sort-Object)) { + $key = "Category_$index" + $matrix[$key] = @{ CATEGORYGROUP = $category } + $index++ +} + +$matrixJson = $matrix | ConvertTo-Json -Depth 5 + +Write-Host "##vso[task.setvariable variable=UITestCategoryMatrix;isOutput=true]$matrixJson" +Write-Host "##vso[task.setvariable variable=UITestCategoryList;isOutput=true]$([string]::Join(',', ($addedCategories | Sort-Object)))" \ No newline at end of file From fdbc7cb4791a83b9ec6ca8a0244edbf785b98b32 Mon Sep 17 00:00:00 2001 From: Gerald Versluis Date: Tue, 16 Dec 2025 11:11:43 +0100 Subject: [PATCH 02/19] Fix default matrix handling in UI test template --- eng/pipelines/common/ui-tests.yml | 42 +++++++++++++------------------ 1 file changed, 18 insertions(+), 24 deletions(-) diff --git a/eng/pipelines/common/ui-tests.yml b/eng/pipelines/common/ui-tests.yml index 2f1b84ab8098..11b1098eef98 100644 --- a/eng/pipelines/common/ui-tests.yml +++ b/eng/pipelines/common/ui-tests.yml @@ -12,18 +12,7 @@ parameters: skipProvisioning: true BuildNativeAOT: false # Parameter to control whether NativeAOT artifacts should be built RunNativeAOT: false # Parameter to control whether NativeAOT UI tests should run - - projects: - - name: name - desc: Human Description - android: /optional/path/to/android.csproj - ios: /optional/path/to/ios.csproj - winui: /optional/path/to/winui.csproj - mac: /optional/path/to/mac.csproj - app: /optional/path/to/app.csproj - -variables: - UITestCategoryMatrixDefault: | + defaultCategoryMatrix: | { "Group00": { "CATEGORYGROUP": "Accessibility,ActionSheet,ActivityIndicator,Animation,AppLinks" }, "Group01": { "CATEGORYGROUP": "Border,BoxView,Brush,Button" }, @@ -50,6 +39,15 @@ variables: "Group22": { "CATEGORYGROUP": "WebView" } } + projects: + - name: name + desc: Human Description + android: /optional/path/to/android.csproj + ios: /optional/path/to/ios.csproj + winui: /optional/path/to/winui.csproj + mac: /optional/path/to/mac.csproj + app: /optional/path/to/app.csproj + stages: - stage: discover_ui_test_categories displayName: Discover UI Test Categories @@ -179,7 +177,7 @@ stages: ) ) variables: - StageCategoryMatrix: ${{ coalesce(stageDependencies.discover_ui_test_categories.detect.outputs['DetectCategories.UITestCategoryMatrix'], variables['UITestCategoryMatrixDefault']) }} + StageCategoryMatrix: ${{ if eq(variables['Build.Reason'], 'PullRequest') }}${{ coalesce(stageDependencies.discover_ui_test_categories.detect.outputs['DetectCategories.UITestCategoryMatrix'], parameters.defaultCategoryMatrix) }}${{ else }}${{ parameters.defaultCategoryMatrix }} jobs: - ${{ each project in parameters.projects }}: - ${{ if ne(project.android, '') }}: @@ -233,7 +231,7 @@ stages: ) ) variables: - StageCategoryMatrix: ${{ coalesce(stageDependencies.discover_ui_test_categories.detect.outputs['DetectCategories.UITestCategoryMatrix'], variables['UITestCategoryMatrixDefault']) }} + StageCategoryMatrix: ${{ if eq(variables['Build.Reason'], 'PullRequest') }}${{ coalesce(stageDependencies.discover_ui_test_categories.detect.outputs['DetectCategories.UITestCategoryMatrix'], parameters.defaultCategoryMatrix) }}${{ else }}${{ parameters.defaultCategoryMatrix }} jobs: - ${{ each project in parameters.projects }}: - ${{ if ne(project.android, '') }}: @@ -289,7 +287,7 @@ stages: ) ) variables: - StageCategoryMatrix: ${{ coalesce(stageDependencies.discover_ui_test_categories.detect.outputs['DetectCategories.UITestCategoryMatrix'], variables['UITestCategoryMatrixDefault']) }} + StageCategoryMatrix: ${{ if eq(variables['Build.Reason'], 'PullRequest') }}${{ coalesce(stageDependencies.discover_ui_test_categories.detect.outputs['DetectCategories.UITestCategoryMatrix'], parameters.defaultCategoryMatrix) }}${{ else }}${{ parameters.defaultCategoryMatrix }} jobs: - ${{ each project in parameters.projects }}: - ${{ if ne(project.ios, '') }}: @@ -337,11 +335,9 @@ stages: dependsOn: - discover_ui_test_categories - build_ui_tests - condition: | - and( + condition: ${{ if ne(variables['Build.Reason'], 'PullRequest') }} and(succeeded('build_ui_tests'), true) ${{ else }} and( succeeded('build_ui_tests'), or( - ne(variables['Build.Reason'], 'PullRequest'), eq(dependencies.discover_ui_test_categories.result, 'Skipped'), eq(stageDependencies.discover_ui_test_categories.detect.outputs['DetectCategories.UITestCategoryMatrix'], ''), contains(stageDependencies.discover_ui_test_categories.detect.outputs['DetectCategories.UITestCategoryMatrix'], '"CollectionView"') @@ -393,11 +389,9 @@ stages: dependsOn: - discover_ui_test_categories - build_ui_tests - condition: | - and( + condition: ${{ if ne(variables['Build.Reason'], 'PullRequest') }} and(succeeded('build_ui_tests'), true) ${{ else }} and( succeeded('build_ui_tests'), or( - ne(variables['Build.Reason'], 'PullRequest'), eq(dependencies.discover_ui_test_categories.result, 'Skipped'), eq(stageDependencies.discover_ui_test_categories.detect.outputs['DetectCategories.UITestCategoryMatrix'], ''), contains(stageDependencies.discover_ui_test_categories.detect.outputs['DetectCategories.UITestCategoryMatrix'], '"CarouselView"') @@ -460,7 +454,7 @@ stages: ) ) variables: - StageCategoryMatrix: ${{ coalesce(stageDependencies.discover_ui_test_categories.detect.outputs['DetectCategories.UITestCategoryMatrix'], variables['UITestCategoryMatrixDefault']) }} + StageCategoryMatrix: ${{ if eq(variables['Build.Reason'], 'PullRequest') }}${{ coalesce(stageDependencies.discover_ui_test_categories.detect.outputs['DetectCategories.UITestCategoryMatrix'], parameters.defaultCategoryMatrix) }}${{ else }}${{ parameters.defaultCategoryMatrix }} jobs: - ${{ each project in parameters.projects }}: - ${{ if ne(project.ios, '') }}: @@ -512,7 +506,7 @@ stages: ) ) variables: - StageCategoryMatrix: ${{ coalesce(stageDependencies.discover_ui_test_categories.detect.outputs['DetectCategories.UITestCategoryMatrix'], variables['UITestCategoryMatrixDefault']) }} + StageCategoryMatrix: ${{ if eq(variables['Build.Reason'], 'PullRequest') }}${{ coalesce(stageDependencies.discover_ui_test_categories.detect.outputs['DetectCategories.UITestCategoryMatrix'], parameters.defaultCategoryMatrix) }}${{ else }}${{ parameters.defaultCategoryMatrix }} jobs: - ${{ each project in parameters.projects }}: - ${{ if ne(project.winui, '') }}: @@ -559,7 +553,7 @@ stages: ) ) variables: - StageCategoryMatrix: ${{ coalesce(stageDependencies.discover_ui_test_categories.detect.outputs['DetectCategories.UITestCategoryMatrix'], variables['UITestCategoryMatrixDefault']) }} + StageCategoryMatrix: ${{ if eq(variables['Build.Reason'], 'PullRequest') }}${{ coalesce(stageDependencies.discover_ui_test_categories.detect.outputs['DetectCategories.UITestCategoryMatrix'], parameters.defaultCategoryMatrix) }}${{ else }}${{ parameters.defaultCategoryMatrix }} jobs: - ${{ each project in parameters.projects }}: - ${{ if ne(project.mac, '') }}: From 068e0e8db143f70241d7945d6ccf4a187007fc57 Mon Sep 17 00:00:00 2001 From: Gerald Versluis Date: Tue, 16 Dec 2025 11:18:22 +0100 Subject: [PATCH 03/19] Fix: use runtime coalesce expression for category matrix --- eng/pipelines/common/ui-tests.yml | 28 ++++++++++++++++------------ 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/eng/pipelines/common/ui-tests.yml b/eng/pipelines/common/ui-tests.yml index 11b1098eef98..a3fdf1e802ba 100644 --- a/eng/pipelines/common/ui-tests.yml +++ b/eng/pipelines/common/ui-tests.yml @@ -177,7 +177,7 @@ stages: ) ) variables: - StageCategoryMatrix: ${{ if eq(variables['Build.Reason'], 'PullRequest') }}${{ coalesce(stageDependencies.discover_ui_test_categories.detect.outputs['DetectCategories.UITestCategoryMatrix'], parameters.defaultCategoryMatrix) }}${{ else }}${{ parameters.defaultCategoryMatrix }} + StageCategoryMatrix: $[ coalesce(stageDependencies.discover_ui_test_categories.detect.outputs['DetectCategories.UITestCategoryMatrix'], '${{ parameters.defaultCategoryMatrix }}') ] jobs: - ${{ each project in parameters.projects }}: - ${{ if ne(project.android, '') }}: @@ -231,7 +231,7 @@ stages: ) ) variables: - StageCategoryMatrix: ${{ if eq(variables['Build.Reason'], 'PullRequest') }}${{ coalesce(stageDependencies.discover_ui_test_categories.detect.outputs['DetectCategories.UITestCategoryMatrix'], parameters.defaultCategoryMatrix) }}${{ else }}${{ parameters.defaultCategoryMatrix }} + StageCategoryMatrix: $[ coalesce(stageDependencies.discover_ui_test_categories.detect.outputs['DetectCategories.UITestCategoryMatrix'], '${{ parameters.defaultCategoryMatrix }}') ] jobs: - ${{ each project in parameters.projects }}: - ${{ if ne(project.android, '') }}: @@ -287,7 +287,7 @@ stages: ) ) variables: - StageCategoryMatrix: ${{ if eq(variables['Build.Reason'], 'PullRequest') }}${{ coalesce(stageDependencies.discover_ui_test_categories.detect.outputs['DetectCategories.UITestCategoryMatrix'], parameters.defaultCategoryMatrix) }}${{ else }}${{ parameters.defaultCategoryMatrix }} + StageCategoryMatrix: $[ coalesce(stageDependencies.discover_ui_test_categories.detect.outputs['DetectCategories.UITestCategoryMatrix'], '${{ parameters.defaultCategoryMatrix }}') ] jobs: - ${{ each project in parameters.projects }}: - ${{ if ne(project.ios, '') }}: @@ -335,12 +335,14 @@ stages: dependsOn: - discover_ui_test_categories - build_ui_tests - condition: ${{ if ne(variables['Build.Reason'], 'PullRequest') }} and(succeeded('build_ui_tests'), true) ${{ else }} and( + condition: | + and( succeeded('build_ui_tests'), or( + ne(variables['Build.Reason'], 'PullRequest'), eq(dependencies.discover_ui_test_categories.result, 'Skipped'), - eq(stageDependencies.discover_ui_test_categories.detect.outputs['DetectCategories.UITestCategoryMatrix'], ''), - contains(stageDependencies.discover_ui_test_categories.detect.outputs['DetectCategories.UITestCategoryMatrix'], '"CollectionView"') + eq(stageDependencies.discover_ui_test_categories.detect.outputs['DetectCategories.UITestCategoryList'], ''), + contains(stageDependencies.discover_ui_test_categories.detect.outputs['DetectCategories.UITestCategoryList'], 'CollectionView') ) ) jobs: @@ -389,12 +391,14 @@ stages: dependsOn: - discover_ui_test_categories - build_ui_tests - condition: ${{ if ne(variables['Build.Reason'], 'PullRequest') }} and(succeeded('build_ui_tests'), true) ${{ else }} and( + condition: | + and( succeeded('build_ui_tests'), or( + ne(variables['Build.Reason'], 'PullRequest'), eq(dependencies.discover_ui_test_categories.result, 'Skipped'), - eq(stageDependencies.discover_ui_test_categories.detect.outputs['DetectCategories.UITestCategoryMatrix'], ''), - contains(stageDependencies.discover_ui_test_categories.detect.outputs['DetectCategories.UITestCategoryMatrix'], '"CarouselView"') + eq(stageDependencies.discover_ui_test_categories.detect.outputs['DetectCategories.UITestCategoryList'], ''), + contains(stageDependencies.discover_ui_test_categories.detect.outputs['DetectCategories.UITestCategoryList'], 'CarouselView') ) ) jobs: @@ -454,7 +458,7 @@ stages: ) ) variables: - StageCategoryMatrix: ${{ if eq(variables['Build.Reason'], 'PullRequest') }}${{ coalesce(stageDependencies.discover_ui_test_categories.detect.outputs['DetectCategories.UITestCategoryMatrix'], parameters.defaultCategoryMatrix) }}${{ else }}${{ parameters.defaultCategoryMatrix }} + StageCategoryMatrix: $[ coalesce(stageDependencies.discover_ui_test_categories.detect.outputs['DetectCategories.UITestCategoryMatrix'], '${{ parameters.defaultCategoryMatrix }}') ] jobs: - ${{ each project in parameters.projects }}: - ${{ if ne(project.ios, '') }}: @@ -506,7 +510,7 @@ stages: ) ) variables: - StageCategoryMatrix: ${{ if eq(variables['Build.Reason'], 'PullRequest') }}${{ coalesce(stageDependencies.discover_ui_test_categories.detect.outputs['DetectCategories.UITestCategoryMatrix'], parameters.defaultCategoryMatrix) }}${{ else }}${{ parameters.defaultCategoryMatrix }} + StageCategoryMatrix: $[ coalesce(stageDependencies.discover_ui_test_categories.detect.outputs['DetectCategories.UITestCategoryMatrix'], '${{ parameters.defaultCategoryMatrix }}') ] jobs: - ${{ each project in parameters.projects }}: - ${{ if ne(project.winui, '') }}: @@ -553,7 +557,7 @@ stages: ) ) variables: - StageCategoryMatrix: ${{ if eq(variables['Build.Reason'], 'PullRequest') }}${{ coalesce(stageDependencies.discover_ui_test_categories.detect.outputs['DetectCategories.UITestCategoryMatrix'], parameters.defaultCategoryMatrix) }}${{ else }}${{ parameters.defaultCategoryMatrix }} + StageCategoryMatrix: $[ coalesce(stageDependencies.discover_ui_test_categories.detect.outputs['DetectCategories.UITestCategoryMatrix'], '${{ parameters.defaultCategoryMatrix }}') ] jobs: - ${{ each project in parameters.projects }}: - ${{ if ne(project.mac, '') }}: From d252f7c5512a9d7865906cd585164e602fd5e216 Mon Sep 17 00:00:00 2001 From: Gerald Versluis Date: Tue, 16 Dec 2025 11:26:52 +0100 Subject: [PATCH 04/19] Fix PowerShell variable interpolation syntax --- eng/scripts/detect-ui-test-categories.ps1 | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/eng/scripts/detect-ui-test-categories.ps1 b/eng/scripts/detect-ui-test-categories.ps1 index 39aeef4f2389..6b114d982b53 100644 --- a/eng/scripts/detect-ui-test-categories.ps1 +++ b/eng/scripts/detect-ui-test-categories.ps1 @@ -25,19 +25,19 @@ if ([string]::IsNullOrWhiteSpace($TargetBranch)) { $targetBranch = $TargetBranch -replace '^refs/heads/', '' -Write-Host "Fetching target branch 'origin/$targetBranch' for diff analysis..." -ForegroundColor Cyan +Write-Host "Fetching target branch 'origin/${targetBranch}' for diff analysis..." -ForegroundColor Cyan try { - git fetch origin "$targetBranch" --no-tags --prune --depth=200 | Out-Null + git fetch origin "${targetBranch}" --no-tags --prune --depth=200 | Out-Null } catch { - Write-Warning "Failed to fetch origin/$targetBranch: $($_.Exception.Message). Falling back to running all categories." + Write-Warning "Failed to fetch origin/${targetBranch}: $($_.Exception.Message). Falling back to running all categories." return } $mergeBase = $null try { - $mergeBase = (git merge-base HEAD "origin/$targetBranch").Trim() + $mergeBase = (git merge-base HEAD "origin/${targetBranch}").Trim() } catch { - Write-Warning "Could not determine merge base with origin/$targetBranch: $($_.Exception.Message). Falling back to running all categories." + Write-Warning "Could not determine merge base with origin/${targetBranch}: $($_.Exception.Message). Falling back to running all categories." return } From 3a1df7be57dddedb782564c020bcbc549f4cc462 Mon Sep 17 00:00:00 2001 From: Gerald Versluis Date: Tue, 16 Dec 2025 11:29:12 +0100 Subject: [PATCH 05/19] Fix regex character class syntax in PowerShell --- eng/scripts/detect-ui-test-categories.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/scripts/detect-ui-test-categories.ps1 b/eng/scripts/detect-ui-test-categories.ps1 index 6b114d982b53..ed577035eb57 100644 --- a/eng/scripts/detect-ui-test-categories.ps1 +++ b/eng/scripts/detect-ui-test-categories.ps1 @@ -66,7 +66,7 @@ foreach ($line in $diff -split "`n") { # Normalize value: UITestCategories.XYZ => XYZ, quoted strings => trimmed text if ($rawValue -match '^UITestCategories\.(?[A-Za-z0-9_]+)$') { $category = $Matches['name'] - } elseif ($rawValue -match '^["\'](?[A-Za-z0-9_\- ]+)["\']$') { + } elseif ($rawValue -match '^["''](?[A-Za-z0-9_ -]+)["'']$') { $category = $Matches['name'] } else { # Attempt to evaluate nameof-style constructs or fallback to raw value From 0104024bef003e40dfb5ff64d0be1886d40e6b04 Mon Sep 17 00:00:00 2001 From: Gerald Versluis Date: Tue, 16 Dec 2025 11:33:41 +0100 Subject: [PATCH 06/19] Add dummy Button test to validate category detection --- .../tests/TestCases.Shared.Tests/Tests/ButtonUITests.cs | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/Controls/tests/TestCases.Shared.Tests/Tests/ButtonUITests.cs b/src/Controls/tests/TestCases.Shared.Tests/Tests/ButtonUITests.cs index 705d07621ec2..4da08977d066 100644 --- a/src/Controls/tests/TestCases.Shared.Tests/Tests/ButtonUITests.cs +++ b/src/Controls/tests/TestCases.Shared.Tests/Tests/ButtonUITests.cs @@ -19,6 +19,15 @@ protected override void NavigateToGallery() App.NavigateToGallery(ButtonGallery); } + // Test change to trigger category detection + [Test] + [Category(UITestCategories.Button)] + public void DummyButtonTest() + { + // Placeholder test for category detection validation + Assert.Pass("Category detection test"); + } + [Test] [Category(UITestCategories.Button)] public void Clicked() From 39032808dc90f1b83d90562911eec52511716551 Mon Sep 17 00:00:00 2001 From: Gerald Versluis Date: Tue, 16 Dec 2025 14:05:24 +0100 Subject: [PATCH 07/19] Add category filtering logic to all test stages - Update android_ui_tests, android_ui_tests_coreclr, ios_ui_tests_mono, ios_ui_tests_nativeaot, winui_ui_tests, and mac_ui_tests stages - Add dependency on discover_ui_test_categories stage - Add stage variables to capture detected categories from discovery - Add Calculate test filter step to each job that: - For PRs with detected categories: filters to only matching categories - For non-PR builds or when no categories detected: runs all categories - Uses _NO_MATCHING_CATEGORY_ filter when no categories match (runs 0 tests) - Update testFilter parameter to use EffectiveTestFilter variable --- eng/pipelines/common/ui-tests.yml | 490 ++++++++++++++++++++---------- 1 file changed, 334 insertions(+), 156 deletions(-) diff --git a/eng/pipelines/common/ui-tests.yml b/eng/pipelines/common/ui-tests.yml index a3fdf1e802ba..9747217bcc51 100644 --- a/eng/pipelines/common/ui-tests.yml +++ b/eng/pipelines/common/ui-tests.yml @@ -12,32 +12,32 @@ parameters: skipProvisioning: true BuildNativeAOT: false # Parameter to control whether NativeAOT artifacts should be built RunNativeAOT: false # Parameter to control whether NativeAOT UI tests should run - defaultCategoryMatrix: | - { - "Group00": { "CATEGORYGROUP": "Accessibility,ActionSheet,ActivityIndicator,Animation,AppLinks" }, - "Group01": { "CATEGORYGROUP": "Border,BoxView,Brush,Button" }, - "Group02": { "CATEGORYGROUP": "CarouselView" }, - "Group03": { "CATEGORYGROUP": "Cells,CheckBox,ContextActions,CustomRenderers,DatePicker,Dispatcher,DisplayAlert,DisplayPrompt,DragAndDrop" }, - "Group04": { "CATEGORYGROUP": "CollectionView" }, - "Group05": { "CATEGORYGROUP": "Entry" }, - "Group06": { "CATEGORYGROUP": "Editor,Effects,FlyoutPage,Focus,Fonts,Frame,Gestures,GraphicsView" }, - "Group07": { "CATEGORYGROUP": "Image,ImageButton,IndicatorView,InputTransparent,IsEnabled,IsVisible" }, - "Group08": { "CATEGORYGROUP": "Label" }, - "Group09": { "CATEGORYGROUP": "Layout" }, - "Group10": { "CATEGORYGROUP": "Lifecycle,ManualReview,Maps" }, - "Group11": { "CATEGORYGROUP": "ListView" }, - "Group12": { "CATEGORYGROUP": "Navigation" }, - "Group13": { "CATEGORYGROUP": "Page,Performance,Picker,ProgressBar" }, - "Group14": { "CATEGORYGROUP": "RadioButton,RefreshView" }, - "Group15": { "CATEGORYGROUP": "SafeAreaEdges,Shadow" }, - "Group16": { "CATEGORYGROUP": "ScrollView" }, - "Group17": { "CATEGORYGROUP": "SearchBar,Shape,Slider" }, - "Group18": { "CATEGORYGROUP": "SoftInput,Stepper,Switch,SwipeView" }, - "Group19": { "CATEGORYGROUP": "Shell" }, - "Group20": { "CATEGORYGROUP": "TabbedPage,TableView,TimePicker,TitleView,ToolbarItem" }, - "Group21": { "CATEGORYGROUP": "ViewBaseTests,Window" }, - "Group22": { "CATEGORYGROUP": "WebView" } - } + categoryGroupsToTest: + # Make sure that this list is always up-to-date with src/Controls/tests/TestCases.Shared.Tests/UITestCategories.cs + # we might want to improve this somehow depending on how much the categories change over time + - 'Accessibility,ActionSheet,ActivityIndicator,Animation,AppLinks' + - 'Border,BoxView,Brush,Button' + - 'CarouselView' + - 'Cells,CheckBox,ContextActions,CustomRenderers,DatePicker,Dispatcher,DisplayAlert,DisplayPrompt,DragAndDrop' + - 'CollectionView' + - 'Entry' + - 'Editor,Effects,FlyoutPage,Focus,Fonts,Frame,Gestures,GraphicsView' + - 'Image,ImageButton,IndicatorView,InputTransparent,IsEnabled,IsVisible' + - 'Label' + - 'Layout' + - 'Lifecycle,ManualReview,Maps' + - 'ListView' + - 'Navigation' + - 'Page,Performance,Picker,ProgressBar' + - 'RadioButton,RefreshView' + - 'SafeAreaEdges,Shadow' + - 'ScrollView' + - 'SearchBar,Shape,Slider' + - 'SoftInput,Stepper,Switch,SwipeView' + - 'Shell' + - 'TabbedPage,TableView,TimePicker,TitleView,ToolbarItem' + - 'ViewBaseTests,Window' + - 'WebView' projects: - name: name @@ -75,8 +75,7 @@ stages: condition: | or( ne(variables['Build.Reason'], 'PullRequest'), - eq(dependencies.discover_ui_test_categories.result, 'Succeeded'), - eq(dependencies.discover_ui_test_categories.result, 'Skipped') + in(dependencies.discover_ui_test_categories.result, 'Succeeded', 'Skipped') ) jobs: - job: build_ui_tests @@ -93,14 +92,7 @@ stages: - stage: build_ui_tests_coreclr displayName: Build UITests CoreCLR Sample App - dependsOn: - - discover_ui_test_categories - condition: | - or( - ne(variables['Build.Reason'], 'PullRequest'), - eq(dependencies.discover_ui_test_categories.result, 'Succeeded'), - eq(dependencies.discover_ui_test_categories.result, 'Skipped') - ) + dependsOn: [] jobs: - job: build_ui_tests displayName: Build Sample App @@ -118,14 +110,7 @@ stages: - ${{ if eq(parameters.BuildNativeAOT, true) }}: - stage: build_ui_tests_nativeaot displayName: Build UITests NativeAOT Sample App - dependsOn: - - discover_ui_test_categories - condition: | - or( - ne(variables['Build.Reason'], 'PullRequest'), - eq(dependencies.discover_ui_test_categories.result, 'Succeeded'), - eq(dependencies.discover_ui_test_categories.result, 'Skipped') - ) + dependsOn: [] jobs: - job: build_ui_tests displayName: Build Sample App @@ -142,14 +127,7 @@ stages: - stage: build_ui_tests_windows displayName: Build UITests Windows Sample App - dependsOn: - - discover_ui_test_categories - condition: | - or( - ne(variables['Build.Reason'], 'PullRequest'), - eq(dependencies.discover_ui_test_categories.result, 'Succeeded'), - eq(dependencies.discover_ui_test_categories.result, 'Skipped') - ) + dependsOn: [] jobs: - job: build_ui_tests displayName: Build Sample App (Windows) @@ -165,19 +143,11 @@ stages: - stage: android_ui_tests displayName: Android UITests dependsOn: - - discover_ui_test_categories - build_ui_tests - condition: | - and( - succeeded('build_ui_tests'), - or( - ne(variables['Build.Reason'], 'PullRequest'), - eq(dependencies.discover_ui_test_categories.result, 'Succeeded'), - eq(dependencies.discover_ui_test_categories.result, 'Skipped') - ) - ) + - discover_ui_test_categories variables: - StageCategoryMatrix: $[ coalesce(stageDependencies.discover_ui_test_categories.detect.outputs['DetectCategories.UITestCategoryMatrix'], '${{ parameters.defaultCategoryMatrix }}') ] + # Capture detected categories from discovery stage (empty if stage was skipped) + detectedCategories: $[ stageDependencies.discover_ui_test_categories.detect.outputs['DetectCategories.UITestCategoryList'] ] jobs: - ${{ each project in parameters.projects }}: - ${{ if ne(project.android, '') }}: @@ -185,7 +155,10 @@ stages: - ${{ if not(containsValue(project.androidApiLevelsExclude, api)) }}: - job: android_ui_tests_${{ project.name }}_${{ api }} strategy: - matrix: $[ fromJson(variables['StageCategoryMatrix']) ] + matrix: + ${{ each categoryGroup in parameters.categoryGroupsToTest }}: + ${{ categoryGroup }}: + CATEGORYGROUP: ${{ categoryGroup }} timeoutInMinutes: 240 # how long to run the job before automatically cancelling workspace: clean: all @@ -194,7 +167,56 @@ stages: variables: REQUIRED_XCODE: $(DEVICETESTS_REQUIRED_XCODE) APPIUM_HOME: $(System.DefaultWorkingDirectory)/.appium/ + # Make detected categories available at job level + DETECTED_CATEGORIES: $(detectedCategories) steps: + # Calculate effective test filter based on detected categories + - pwsh: | + $detected = "$(DETECTED_CATEGORIES)" + $category = "$(CATEGORYGROUP)" + $isPR = "$(Build.Reason)" -eq "PullRequest" + + Write-Host "Build.Reason: $(Build.Reason)" + Write-Host "Is PR: $isPR" + Write-Host "Detected categories: '$detected'" + Write-Host "Current category group: '$category'" + + # Determine effective filter + $effectiveFilter = $category + + if ($isPR -and -not [string]::IsNullOrWhiteSpace($detected)) { + # For PRs with detected categories, find intersection + $categoryList = $category -split "," + $detectedList = $detected -split "," + $matchingCategories = @() + + foreach ($cat in $categoryList) { + $cat = $cat.Trim() + foreach ($det in $detectedList) { + $det = $det.Trim() + if ($cat -eq $det) { + $matchingCategories += $cat + break + } + } + } + + if ($matchingCategories.Count -gt 0) { + $effectiveFilter = $matchingCategories -join "," + Write-Host "Running matching categories: $effectiveFilter" + } + else { + # No match - use a filter that won't match any tests + $effectiveFilter = "_NO_MATCHING_CATEGORY_" + Write-Host "No matching categories - will skip tests" + } + } + else { + Write-Host "Running all categories in group: $effectiveFilter" + } + + Write-Host "##vso[task.setvariable variable=EffectiveTestFilter]$effectiveFilter" + displayName: Calculate test filter - template: ui-tests-steps.yml parameters: platform: android @@ -206,7 +228,7 @@ stages: ${{ if not(eq(api, 27)) }}: device: android-emulator-64_${{ api }} provisionatorChannel: ${{ parameters.provisionatorChannel }} - testFilter: $(CATEGORYGROUP) + testFilter: $(EffectiveTestFilter) skipProvisioning: ${{ parameters.skipProvisioning }} # Collect and publish Android snapshot diffs @@ -219,19 +241,10 @@ stages: - stage: android_ui_tests_coreclr displayName: Android UITests CoreClr dependsOn: - - discover_ui_test_categories - build_ui_tests_coreclr - condition: | - and( - succeeded('build_ui_tests_coreclr'), - or( - ne(variables['Build.Reason'], 'PullRequest'), - eq(dependencies.discover_ui_test_categories.result, 'Succeeded'), - eq(dependencies.discover_ui_test_categories.result, 'Skipped') - ) - ) + - discover_ui_test_categories variables: - StageCategoryMatrix: $[ coalesce(stageDependencies.discover_ui_test_categories.detect.outputs['DetectCategories.UITestCategoryMatrix'], '${{ parameters.defaultCategoryMatrix }}') ] + detectedCategories: $[ stageDependencies.discover_ui_test_categories.detect.outputs['DetectCategories.UITestCategoryList'] ] jobs: - ${{ each project in parameters.projects }}: - ${{ if ne(project.android, '') }}: @@ -239,7 +252,10 @@ stages: - ${{ if not(containsValue(project.androidApiLevelsExclude, api)) }}: - job: android_ui_tests_${{ project.name }}_${{ api }} strategy: - matrix: $[ fromJson(variables['StageCategoryMatrix']) ] + matrix: + ${{ each categoryGroup in parameters.categoryGroupsToTest }}: + ${{ categoryGroup }}: + CATEGORYGROUP: ${{ categoryGroup }} timeoutInMinutes: 240 # how long to run the job before automatically cancelling workspace: clean: all @@ -248,7 +264,49 @@ stages: variables: REQUIRED_XCODE: $(DEVICETESTS_REQUIRED_XCODE) APPIUM_HOME: $(System.DefaultWorkingDirectory)/.appium/ + DETECTED_CATEGORIES: $(detectedCategories) steps: + - pwsh: | + $detected = "$(DETECTED_CATEGORIES)" + $category = "$(CATEGORYGROUP)" + $isPR = "$(Build.Reason)" -eq "PullRequest" + + Write-Host "Detected categories: '$detected'" + Write-Host "Current category group: '$category'" + + $effectiveFilter = $category + + if ($isPR -and -not [string]::IsNullOrWhiteSpace($detected)) { + $categoryList = $category -split "," + $detectedList = $detected -split "," + $matchingCategories = @() + + foreach ($cat in $categoryList) { + $cat = $cat.Trim() + foreach ($det in $detectedList) { + $det = $det.Trim() + if ($cat -eq $det) { + $matchingCategories += $cat + break + } + } + } + + if ($matchingCategories.Count -gt 0) { + $effectiveFilter = $matchingCategories -join "," + Write-Host "Running matching categories: $effectiveFilter" + } + else { + $effectiveFilter = "_NO_MATCHING_CATEGORY_" + Write-Host "No matching categories - will skip tests" + } + } + else { + Write-Host "Running all categories in group: $effectiveFilter" + } + + Write-Host "##vso[task.setvariable variable=EffectiveTestFilter]$effectiveFilter" + displayName: Calculate test filter - template: ui-tests-steps.yml parameters: platform: android @@ -261,7 +319,7 @@ stages: device: android-emulator-64_${{ api }} provisionatorChannel: ${{ parameters.provisionatorChannel }} agentPoolAccessToken: ${{ parameters.agentPoolAccessToken }} - testFilter: $(CATEGORYGROUP) + testFilter: $(EffectiveTestFilter) headless: ${{ parameters.headless }} skipProvisioning: ${{ parameters.skipProvisioning }} runtimeVariant: "CoreCLR" @@ -275,19 +333,10 @@ stages: - stage: ios_ui_tests_mono displayName: iOS UITests Mono dependsOn: - - discover_ui_test_categories - build_ui_tests - condition: | - and( - succeeded('build_ui_tests'), - or( - ne(variables['Build.Reason'], 'PullRequest'), - eq(dependencies.discover_ui_test_categories.result, 'Succeeded'), - eq(dependencies.discover_ui_test_categories.result, 'Skipped') - ) - ) + - discover_ui_test_categories variables: - StageCategoryMatrix: $[ coalesce(stageDependencies.discover_ui_test_categories.detect.outputs['DetectCategories.UITestCategoryMatrix'], '${{ parameters.defaultCategoryMatrix }}') ] + detectedCategories: $[ stageDependencies.discover_ui_test_categories.detect.outputs['DetectCategories.UITestCategoryList'] ] jobs: - ${{ each project in parameters.projects }}: - ${{ if ne(project.ios, '') }}: @@ -295,7 +344,10 @@ stages: - ${{ if not(containsValue(project.iosVersionsExclude, version)) }}: - job: ios_ui_tests_mono_${{ project.name }}_${{ replace(version, '.', '_') }} strategy: - matrix: $[ fromJson(variables['StageCategoryMatrix']) ] + matrix: + ${{ each categoryGroup in parameters.categoryGroupsToTest }}: + ${{ categoryGroup }}: + CATEGORYGROUP: ${{ categoryGroup }} timeoutInMinutes: ${{ parameters.timeoutInMinutes }} # how long to run the job before automatically cancelling workspace: clean: all @@ -304,7 +356,49 @@ stages: variables: REQUIRED_XCODE: $(DEVICETESTS_REQUIRED_XCODE) APPIUM_HOME: $(System.DefaultWorkingDirectory)/.appium/ + DETECTED_CATEGORIES: $(detectedCategories) steps: + - pwsh: | + $detected = "$(DETECTED_CATEGORIES)" + $category = "$(CATEGORYGROUP)" + $isPR = "$(Build.Reason)" -eq "PullRequest" + + Write-Host "Detected categories: '$detected'" + Write-Host "Current category group: '$category'" + + $effectiveFilter = $category + + if ($isPR -and -not [string]::IsNullOrWhiteSpace($detected)) { + $categoryList = $category -split "," + $detectedList = $detected -split "," + $matchingCategories = @() + + foreach ($cat in $categoryList) { + $cat = $cat.Trim() + foreach ($det in $detectedList) { + $det = $det.Trim() + if ($cat -eq $det) { + $matchingCategories += $cat + break + } + } + } + + if ($matchingCategories.Count -gt 0) { + $effectiveFilter = $matchingCategories -join "," + Write-Host "Running matching categories: $effectiveFilter" + } + else { + $effectiveFilter = "_NO_MATCHING_CATEGORY_" + Write-Host "No matching categories - will skip tests" + } + } + else { + Write-Host "Running all categories in group: $effectiveFilter" + } + + Write-Host "##vso[task.setvariable variable=EffectiveTestFilter]$effectiveFilter" + displayName: Calculate test filter - template: ui-tests-steps.yml parameters: platform: ios @@ -320,7 +414,7 @@ stages: device: ios-simulator-64_${{ version }} provisionatorChannel: ${{ parameters.provisionatorChannel }} runtimeVariant : "Mono" - testFilter: $(CATEGORYGROUP) + testFilter: $(EffectiveTestFilter) headless: ${{ parameters.headless }} skipProvisioning: ${{ parameters.skipProvisioning }} @@ -332,19 +426,7 @@ stages: - stage: ios_ui_tests_mono_cv2 displayName: iOS UITests Mono CollectionView2 - dependsOn: - - discover_ui_test_categories - - build_ui_tests - condition: | - and( - succeeded('build_ui_tests'), - or( - ne(variables['Build.Reason'], 'PullRequest'), - eq(dependencies.discover_ui_test_categories.result, 'Skipped'), - eq(stageDependencies.discover_ui_test_categories.detect.outputs['DetectCategories.UITestCategoryList'], ''), - contains(stageDependencies.discover_ui_test_categories.detect.outputs['DetectCategories.UITestCategoryList'], 'CollectionView') - ) - ) + dependsOn: build_ui_tests jobs: - ${{ each project in parameters.projects }}: - ${{ if ne(project.ios, '') }}: @@ -388,19 +470,7 @@ stages: - stage: ios_ui_tests_mono_carv2 displayName: iOS UITests Mono CarouselView2 - dependsOn: - - discover_ui_test_categories - - build_ui_tests - condition: | - and( - succeeded('build_ui_tests'), - or( - ne(variables['Build.Reason'], 'PullRequest'), - eq(dependencies.discover_ui_test_categories.result, 'Skipped'), - eq(stageDependencies.discover_ui_test_categories.detect.outputs['DetectCategories.UITestCategoryList'], ''), - contains(stageDependencies.discover_ui_test_categories.detect.outputs['DetectCategories.UITestCategoryList'], 'CarouselView') - ) - ) + dependsOn: build_ui_tests jobs: - ${{ each project in parameters.projects }}: - ${{ if ne(project.ios, '') }}: @@ -446,19 +516,10 @@ stages: - stage: ios_ui_tests_nativeaot displayName: iOS UITests NativeAOT dependsOn: - - discover_ui_test_categories - build_ui_tests_nativeaot - condition: | - and( - succeeded('build_ui_tests_nativeaot'), - or( - ne(variables['Build.Reason'], 'PullRequest'), - eq(dependencies.discover_ui_test_categories.result, 'Succeeded'), - eq(dependencies.discover_ui_test_categories.result, 'Skipped') - ) - ) + - discover_ui_test_categories variables: - StageCategoryMatrix: $[ coalesce(stageDependencies.discover_ui_test_categories.detect.outputs['DetectCategories.UITestCategoryMatrix'], '${{ parameters.defaultCategoryMatrix }}') ] + detectedCategories: $[ stageDependencies.discover_ui_test_categories.detect.outputs['DetectCategories.UITestCategoryList'] ] jobs: - ${{ each project in parameters.projects }}: - ${{ if ne(project.ios, '') }}: @@ -466,7 +527,10 @@ stages: - ${{ if not(containsValue(project.iosVersionsExclude, version)) }}: - job: ios_ui_tests_nativeaot_${{ project.name }}_${{ replace(version, '.', '_') }} strategy: - matrix: $[ fromJson(variables['StageCategoryMatrix']) ] + matrix: + ${{ each categoryGroup in parameters.categoryGroupsToTest }}: + ${{ categoryGroup }}: + CATEGORYGROUP: ${{ categoryGroup }} timeoutInMinutes: ${{ parameters.timeoutInMinutes }} # how long to run the job before automatically cancelling workspace: clean: all @@ -475,7 +539,49 @@ stages: variables: REQUIRED_XCODE: $(DEVICETESTS_REQUIRED_XCODE) APPIUM_HOME: $(System.DefaultWorkingDirectory)/.appium/ + DETECTED_CATEGORIES: $(detectedCategories) steps: + - pwsh: | + $detected = "$(DETECTED_CATEGORIES)" + $category = "$(CATEGORYGROUP)" + $isPR = "$(Build.Reason)" -eq "PullRequest" + + Write-Host "Detected categories: '$detected'" + Write-Host "Current category group: '$category'" + + $effectiveFilter = $category + + if ($isPR -and -not [string]::IsNullOrWhiteSpace($detected)) { + $categoryList = $category -split "," + $detectedList = $detected -split "," + $matchingCategories = @() + + foreach ($cat in $categoryList) { + $cat = $cat.Trim() + foreach ($det in $detectedList) { + $det = $det.Trim() + if ($cat -eq $det) { + $matchingCategories += $cat + break + } + } + } + + if ($matchingCategories.Count -gt 0) { + $effectiveFilter = $matchingCategories -join "," + Write-Host "Running matching categories: $effectiveFilter" + } + else { + $effectiveFilter = "_NO_MATCHING_CATEGORY_" + Write-Host "No matching categories - will skip tests" + } + } + else { + Write-Host "Running all categories in group: $effectiveFilter" + } + + Write-Host "##vso[task.setvariable variable=EffectiveTestFilter]$effectiveFilter" + displayName: Calculate test filter - template: ui-tests-steps.yml parameters: platform: ios @@ -491,32 +597,26 @@ stages: device: ios-simulator-64_${{ version }} provisionatorChannel: ${{ parameters.provisionatorChannel }} runtimeVariant : "NativeAOT" - testFilter: $(CATEGORYGROUP) + testFilter: $(EffectiveTestFilter) headless: ${{ parameters.headless }} skipProvisioning: ${{ parameters.skipProvisioning }} - stage: winui_ui_tests displayName: WinUI UITests dependsOn: - - discover_ui_test_categories - build_ui_tests_windows - condition: | - and( - succeeded('build_ui_tests_windows'), - or( - ne(variables['Build.Reason'], 'PullRequest'), - eq(dependencies.discover_ui_test_categories.result, 'Succeeded'), - eq(dependencies.discover_ui_test_categories.result, 'Skipped') - ) - ) + - discover_ui_test_categories variables: - StageCategoryMatrix: $[ coalesce(stageDependencies.discover_ui_test_categories.detect.outputs['DetectCategories.UITestCategoryMatrix'], '${{ parameters.defaultCategoryMatrix }}') ] + detectedCategories: $[ stageDependencies.discover_ui_test_categories.detect.outputs['DetectCategories.UITestCategoryList'] ] jobs: - ${{ each project in parameters.projects }}: - ${{ if ne(project.winui, '') }}: - job: winui_ui_tests_${{ project.name }} strategy: - matrix: $[ fromJson(variables['StageCategoryMatrix']) ] + matrix: + ${{ each categoryGroup in parameters.categoryGroupsToTest }}: + ${{ categoryGroup }}: + CATEGORYGROUP: ${{ categoryGroup }} timeoutInMinutes: ${{ parameters.timeoutInMinutes }} # how long to run the job before automatically cancelling workspace: clean: all @@ -524,7 +624,49 @@ stages: pool: ${{ parameters.windowsPool }} variables: APPIUM_HOME: $(System.DefaultWorkingDirectory)\.appium\ + DETECTED_CATEGORIES: $(detectedCategories) steps: + - pwsh: | + $detected = "$(DETECTED_CATEGORIES)" + $category = "$(CATEGORYGROUP)" + $isPR = "$(Build.Reason)" -eq "PullRequest" + + Write-Host "Detected categories: '$detected'" + Write-Host "Current category group: '$category'" + + $effectiveFilter = $category + + if ($isPR -and -not [string]::IsNullOrWhiteSpace($detected)) { + $categoryList = $category -split "," + $detectedList = $detected -split "," + $matchingCategories = @() + + foreach ($cat in $categoryList) { + $cat = $cat.Trim() + foreach ($det in $detectedList) { + $det = $det.Trim() + if ($cat -eq $det) { + $matchingCategories += $cat + break + } + } + } + + if ($matchingCategories.Count -gt 0) { + $effectiveFilter = $matchingCategories -join "," + Write-Host "Running matching categories: $effectiveFilter" + } + else { + $effectiveFilter = "_NO_MATCHING_CATEGORY_" + Write-Host "No matching categories - will skip tests" + } + } + else { + Write-Host "Running all categories in group: $effectiveFilter" + } + + Write-Host "##vso[task.setvariable variable=EffectiveTestFilter]$effectiveFilter" + displayName: Calculate test filter - template: ui-tests-steps.yml parameters: platform: windows @@ -533,7 +675,7 @@ stages: path: ${{ project.winui }} app: ${{ project.app }} provisionatorChannel: ${{ parameters.provisionatorChannel }} - testFilter: $(CATEGORYGROUP) + testFilter: $(EffectiveTestFilter) skipProvisioning: ${{ parameters.skipProvisioning }} # Collect and publish Windows snapshot diffs @@ -545,25 +687,19 @@ stages: - stage: mac_ui_tests displayName: macOS UITests dependsOn: - - discover_ui_test_categories - build_ui_tests - condition: | - and( - succeeded('build_ui_tests'), - or( - ne(variables['Build.Reason'], 'PullRequest'), - eq(dependencies.discover_ui_test_categories.result, 'Succeeded'), - eq(dependencies.discover_ui_test_categories.result, 'Skipped') - ) - ) + - discover_ui_test_categories variables: - StageCategoryMatrix: $[ coalesce(stageDependencies.discover_ui_test_categories.detect.outputs['DetectCategories.UITestCategoryMatrix'], '${{ parameters.defaultCategoryMatrix }}') ] + detectedCategories: $[ stageDependencies.discover_ui_test_categories.detect.outputs['DetectCategories.UITestCategoryList'] ] jobs: - ${{ each project in parameters.projects }}: - ${{ if ne(project.mac, '') }}: - job: mac_ui_tests_${{ project.name }} strategy: - matrix: $[ fromJson(variables['StageCategoryMatrix']) ] + matrix: + ${{ each categoryGroup in parameters.categoryGroupsToTest }}: + ${{ categoryGroup }}: + CATEGORYGROUP: ${{ categoryGroup }} timeoutInMinutes: ${{ parameters.timeoutInMinutes }} # how long to run the job before automatically cancelling workspace: clean: all @@ -572,7 +708,49 @@ stages: variables: REQUIRED_XCODE: $(DEVICETESTS_REQUIRED_XCODE) APPIUM_HOME: $(System.DefaultWorkingDirectory)/.appium/ + DETECTED_CATEGORIES: $(detectedCategories) steps: + - pwsh: | + $detected = "$(DETECTED_CATEGORIES)" + $category = "$(CATEGORYGROUP)" + $isPR = "$(Build.Reason)" -eq "PullRequest" + + Write-Host "Detected categories: '$detected'" + Write-Host "Current category group: '$category'" + + $effectiveFilter = $category + + if ($isPR -and -not [string]::IsNullOrWhiteSpace($detected)) { + $categoryList = $category -split "," + $detectedList = $detected -split "," + $matchingCategories = @() + + foreach ($cat in $categoryList) { + $cat = $cat.Trim() + foreach ($det in $detectedList) { + $det = $det.Trim() + if ($cat -eq $det) { + $matchingCategories += $cat + break + } + } + } + + if ($matchingCategories.Count -gt 0) { + $effectiveFilter = $matchingCategories -join "," + Write-Host "Running matching categories: $effectiveFilter" + } + else { + $effectiveFilter = "_NO_MATCHING_CATEGORY_" + Write-Host "No matching categories - will skip tests" + } + } + else { + Write-Host "Running all categories in group: $effectiveFilter" + } + + Write-Host "##vso[task.setvariable variable=EffectiveTestFilter]$effectiveFilter" + displayName: Calculate test filter - template: ui-tests-steps.yml parameters: platform: catalyst @@ -581,7 +759,7 @@ stages: path: ${{ project.mac }} app: ${{ project.app }} provisionatorChannel: ${{ parameters.provisionatorChannel }} - testFilter: $(CATEGORYGROUP) + testFilter: $(EffectiveTestFilter) skipProvisioning: ${{ parameters.skipProvisioning }} # Collect and publish Mac snapshot diffs From 6179f0e978e2b50744446f59360d72e36eb12080 Mon Sep 17 00:00:00 2001 From: Gerald Versluis Date: Tue, 16 Dec 2025 14:24:05 +0100 Subject: [PATCH 08/19] Fix variable scoping: move stage dependency reference to job level Remove stage-level variables and use direct stage dependency expression in job-level variables instead. This ensures the detected categories are properly accessible within each matrix job. Changed from: stage: variables: detectedCategories: $[ stageDep... ] jobs: - job: variables: DETECTED_CATEGORIES: $(detectedCategories) To: stage: jobs: - job: variables: DETECTED_CATEGORIES: $[ stageDep... ] --- eng/pipelines/common/ui-tests.yml | 27 +++++++-------------------- 1 file changed, 7 insertions(+), 20 deletions(-) diff --git a/eng/pipelines/common/ui-tests.yml b/eng/pipelines/common/ui-tests.yml index 9747217bcc51..70e2bcae2a08 100644 --- a/eng/pipelines/common/ui-tests.yml +++ b/eng/pipelines/common/ui-tests.yml @@ -145,9 +145,6 @@ stages: dependsOn: - build_ui_tests - discover_ui_test_categories - variables: - # Capture detected categories from discovery stage (empty if stage was skipped) - detectedCategories: $[ stageDependencies.discover_ui_test_categories.detect.outputs['DetectCategories.UITestCategoryList'] ] jobs: - ${{ each project in parameters.projects }}: - ${{ if ne(project.android, '') }}: @@ -167,8 +164,8 @@ stages: variables: REQUIRED_XCODE: $(DEVICETESTS_REQUIRED_XCODE) APPIUM_HOME: $(System.DefaultWorkingDirectory)/.appium/ - # Make detected categories available at job level - DETECTED_CATEGORIES: $(detectedCategories) + # Access detected categories directly from stage dependency + DETECTED_CATEGORIES: $[ stageDependencies.discover_ui_test_categories.detect.outputs['DetectCategories.UITestCategoryList'] ] steps: # Calculate effective test filter based on detected categories - pwsh: | @@ -243,8 +240,6 @@ stages: dependsOn: - build_ui_tests_coreclr - discover_ui_test_categories - variables: - detectedCategories: $[ stageDependencies.discover_ui_test_categories.detect.outputs['DetectCategories.UITestCategoryList'] ] jobs: - ${{ each project in parameters.projects }}: - ${{ if ne(project.android, '') }}: @@ -264,7 +259,7 @@ stages: variables: REQUIRED_XCODE: $(DEVICETESTS_REQUIRED_XCODE) APPIUM_HOME: $(System.DefaultWorkingDirectory)/.appium/ - DETECTED_CATEGORIES: $(detectedCategories) + DETECTED_CATEGORIES: $[ stageDependencies.discover_ui_test_categories.detect.outputs['DetectCategories.UITestCategoryList'] ] steps: - pwsh: | $detected = "$(DETECTED_CATEGORIES)" @@ -335,8 +330,6 @@ stages: dependsOn: - build_ui_tests - discover_ui_test_categories - variables: - detectedCategories: $[ stageDependencies.discover_ui_test_categories.detect.outputs['DetectCategories.UITestCategoryList'] ] jobs: - ${{ each project in parameters.projects }}: - ${{ if ne(project.ios, '') }}: @@ -356,7 +349,7 @@ stages: variables: REQUIRED_XCODE: $(DEVICETESTS_REQUIRED_XCODE) APPIUM_HOME: $(System.DefaultWorkingDirectory)/.appium/ - DETECTED_CATEGORIES: $(detectedCategories) + DETECTED_CATEGORIES: $[ stageDependencies.discover_ui_test_categories.detect.outputs['DetectCategories.UITestCategoryList'] ] steps: - pwsh: | $detected = "$(DETECTED_CATEGORIES)" @@ -518,8 +511,6 @@ stages: dependsOn: - build_ui_tests_nativeaot - discover_ui_test_categories - variables: - detectedCategories: $[ stageDependencies.discover_ui_test_categories.detect.outputs['DetectCategories.UITestCategoryList'] ] jobs: - ${{ each project in parameters.projects }}: - ${{ if ne(project.ios, '') }}: @@ -539,7 +530,7 @@ stages: variables: REQUIRED_XCODE: $(DEVICETESTS_REQUIRED_XCODE) APPIUM_HOME: $(System.DefaultWorkingDirectory)/.appium/ - DETECTED_CATEGORIES: $(detectedCategories) + DETECTED_CATEGORIES: $[ stageDependencies.discover_ui_test_categories.detect.outputs['DetectCategories.UITestCategoryList'] ] steps: - pwsh: | $detected = "$(DETECTED_CATEGORIES)" @@ -606,8 +597,6 @@ stages: dependsOn: - build_ui_tests_windows - discover_ui_test_categories - variables: - detectedCategories: $[ stageDependencies.discover_ui_test_categories.detect.outputs['DetectCategories.UITestCategoryList'] ] jobs: - ${{ each project in parameters.projects }}: - ${{ if ne(project.winui, '') }}: @@ -624,7 +613,7 @@ stages: pool: ${{ parameters.windowsPool }} variables: APPIUM_HOME: $(System.DefaultWorkingDirectory)\.appium\ - DETECTED_CATEGORIES: $(detectedCategories) + DETECTED_CATEGORIES: $[ stageDependencies.discover_ui_test_categories.detect.outputs['DetectCategories.UITestCategoryList'] ] steps: - pwsh: | $detected = "$(DETECTED_CATEGORIES)" @@ -689,8 +678,6 @@ stages: dependsOn: - build_ui_tests - discover_ui_test_categories - variables: - detectedCategories: $[ stageDependencies.discover_ui_test_categories.detect.outputs['DetectCategories.UITestCategoryList'] ] jobs: - ${{ each project in parameters.projects }}: - ${{ if ne(project.mac, '') }}: @@ -708,7 +695,7 @@ stages: variables: REQUIRED_XCODE: $(DEVICETESTS_REQUIRED_XCODE) APPIUM_HOME: $(System.DefaultWorkingDirectory)/.appium/ - DETECTED_CATEGORIES: $(detectedCategories) + DETECTED_CATEGORIES: $[ stageDependencies.discover_ui_test_categories.detect.outputs['DetectCategories.UITestCategoryList'] ] steps: - pwsh: | $detected = "$(DETECTED_CATEGORIES)" From a3843b519d116311f4f519f6dcf6e2ba5251d39f Mon Sep 17 00:00:00 2001 From: Gerald Versluis Date: Tue, 16 Dec 2025 14:43:44 +0100 Subject: [PATCH 09/19] Fix: Overwrite CATEGORYGROUP variable instead of creating new one The testFilter parameter uses $(CATEGORYGROUP) which is already wired up correctly in the ui-tests-steps.yml template. Instead of creating a new EffectiveTestFilter variable and trying to pass it, simply overwrite the existing CATEGORYGROUP variable with the filtered value. This leverages the existing parameter passing mechanism that already works. --- eng/pipelines/common/ui-tests.yml | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/eng/pipelines/common/ui-tests.yml b/eng/pipelines/common/ui-tests.yml index 70e2bcae2a08..7e28d61c140a 100644 --- a/eng/pipelines/common/ui-tests.yml +++ b/eng/pipelines/common/ui-tests.yml @@ -212,7 +212,7 @@ stages: Write-Host "Running all categories in group: $effectiveFilter" } - Write-Host "##vso[task.setvariable variable=EffectiveTestFilter]$effectiveFilter" + Write-Host "##vso[task.setvariable variable=CATEGORYGROUP]$effectiveFilter" displayName: Calculate test filter - template: ui-tests-steps.yml parameters: @@ -225,7 +225,7 @@ stages: ${{ if not(eq(api, 27)) }}: device: android-emulator-64_${{ api }} provisionatorChannel: ${{ parameters.provisionatorChannel }} - testFilter: $(EffectiveTestFilter) + testFilter: $(CATEGORYGROUP) skipProvisioning: ${{ parameters.skipProvisioning }} # Collect and publish Android snapshot diffs @@ -300,7 +300,7 @@ stages: Write-Host "Running all categories in group: $effectiveFilter" } - Write-Host "##vso[task.setvariable variable=EffectiveTestFilter]$effectiveFilter" + Write-Host "##vso[task.setvariable variable=CATEGORYGROUP]$effectiveFilter" displayName: Calculate test filter - template: ui-tests-steps.yml parameters: @@ -314,7 +314,7 @@ stages: device: android-emulator-64_${{ api }} provisionatorChannel: ${{ parameters.provisionatorChannel }} agentPoolAccessToken: ${{ parameters.agentPoolAccessToken }} - testFilter: $(EffectiveTestFilter) + testFilter: $(CATEGORYGROUP) headless: ${{ parameters.headless }} skipProvisioning: ${{ parameters.skipProvisioning }} runtimeVariant: "CoreCLR" @@ -390,7 +390,7 @@ stages: Write-Host "Running all categories in group: $effectiveFilter" } - Write-Host "##vso[task.setvariable variable=EffectiveTestFilter]$effectiveFilter" + Write-Host "##vso[task.setvariable variable=CATEGORYGROUP]$effectiveFilter" displayName: Calculate test filter - template: ui-tests-steps.yml parameters: @@ -407,7 +407,7 @@ stages: device: ios-simulator-64_${{ version }} provisionatorChannel: ${{ parameters.provisionatorChannel }} runtimeVariant : "Mono" - testFilter: $(EffectiveTestFilter) + testFilter: $(CATEGORYGROUP) headless: ${{ parameters.headless }} skipProvisioning: ${{ parameters.skipProvisioning }} @@ -571,7 +571,7 @@ stages: Write-Host "Running all categories in group: $effectiveFilter" } - Write-Host "##vso[task.setvariable variable=EffectiveTestFilter]$effectiveFilter" + Write-Host "##vso[task.setvariable variable=CATEGORYGROUP]$effectiveFilter" displayName: Calculate test filter - template: ui-tests-steps.yml parameters: @@ -588,7 +588,7 @@ stages: device: ios-simulator-64_${{ version }} provisionatorChannel: ${{ parameters.provisionatorChannel }} runtimeVariant : "NativeAOT" - testFilter: $(EffectiveTestFilter) + testFilter: $(CATEGORYGROUP) headless: ${{ parameters.headless }} skipProvisioning: ${{ parameters.skipProvisioning }} @@ -654,7 +654,7 @@ stages: Write-Host "Running all categories in group: $effectiveFilter" } - Write-Host "##vso[task.setvariable variable=EffectiveTestFilter]$effectiveFilter" + Write-Host "##vso[task.setvariable variable=CATEGORYGROUP]$effectiveFilter" displayName: Calculate test filter - template: ui-tests-steps.yml parameters: @@ -664,7 +664,7 @@ stages: path: ${{ project.winui }} app: ${{ project.app }} provisionatorChannel: ${{ parameters.provisionatorChannel }} - testFilter: $(EffectiveTestFilter) + testFilter: $(CATEGORYGROUP) skipProvisioning: ${{ parameters.skipProvisioning }} # Collect and publish Windows snapshot diffs @@ -736,7 +736,7 @@ stages: Write-Host "Running all categories in group: $effectiveFilter" } - Write-Host "##vso[task.setvariable variable=EffectiveTestFilter]$effectiveFilter" + Write-Host "##vso[task.setvariable variable=CATEGORYGROUP]$effectiveFilter" displayName: Calculate test filter - template: ui-tests-steps.yml parameters: @@ -746,7 +746,7 @@ stages: path: ${{ project.mac }} app: ${{ project.app }} provisionatorChannel: ${{ parameters.provisionatorChannel }} - testFilter: $(EffectiveTestFilter) + testFilter: $(CATEGORYGROUP) skipProvisioning: ${{ parameters.skipProvisioning }} # Collect and publish Mac snapshot diffs From 9c5113758bcd6b0ae7885a9640e645ef98ae651b Mon Sep 17 00:00:00 2001 From: Gerald Versluis Date: Tue, 16 Dec 2025 15:29:42 +0100 Subject: [PATCH 10/19] Fix: Use EFFECTIVE_FILTER variable - CATEGORYGROUP is readonly Matrix variables are readonly in Azure Pipelines and cannot be overwritten. Create a new EFFECTIVE_FILTER variable instead and use that for testFilter. --- eng/pipelines/common/ui-tests.yml | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/eng/pipelines/common/ui-tests.yml b/eng/pipelines/common/ui-tests.yml index 7e28d61c140a..e0545d1a5783 100644 --- a/eng/pipelines/common/ui-tests.yml +++ b/eng/pipelines/common/ui-tests.yml @@ -212,7 +212,7 @@ stages: Write-Host "Running all categories in group: $effectiveFilter" } - Write-Host "##vso[task.setvariable variable=CATEGORYGROUP]$effectiveFilter" + Write-Host "##vso[task.setvariable variable=EFFECTIVE_FILTER]$effectiveFilter" displayName: Calculate test filter - template: ui-tests-steps.yml parameters: @@ -225,7 +225,7 @@ stages: ${{ if not(eq(api, 27)) }}: device: android-emulator-64_${{ api }} provisionatorChannel: ${{ parameters.provisionatorChannel }} - testFilter: $(CATEGORYGROUP) + testFilter: $(EFFECTIVE_FILTER) skipProvisioning: ${{ parameters.skipProvisioning }} # Collect and publish Android snapshot diffs @@ -300,7 +300,7 @@ stages: Write-Host "Running all categories in group: $effectiveFilter" } - Write-Host "##vso[task.setvariable variable=CATEGORYGROUP]$effectiveFilter" + Write-Host "##vso[task.setvariable variable=EFFECTIVE_FILTER]$effectiveFilter" displayName: Calculate test filter - template: ui-tests-steps.yml parameters: @@ -314,7 +314,7 @@ stages: device: android-emulator-64_${{ api }} provisionatorChannel: ${{ parameters.provisionatorChannel }} agentPoolAccessToken: ${{ parameters.agentPoolAccessToken }} - testFilter: $(CATEGORYGROUP) + testFilter: $(EFFECTIVE_FILTER) headless: ${{ parameters.headless }} skipProvisioning: ${{ parameters.skipProvisioning }} runtimeVariant: "CoreCLR" @@ -390,7 +390,7 @@ stages: Write-Host "Running all categories in group: $effectiveFilter" } - Write-Host "##vso[task.setvariable variable=CATEGORYGROUP]$effectiveFilter" + Write-Host "##vso[task.setvariable variable=EFFECTIVE_FILTER]$effectiveFilter" displayName: Calculate test filter - template: ui-tests-steps.yml parameters: @@ -407,7 +407,7 @@ stages: device: ios-simulator-64_${{ version }} provisionatorChannel: ${{ parameters.provisionatorChannel }} runtimeVariant : "Mono" - testFilter: $(CATEGORYGROUP) + testFilter: $(EFFECTIVE_FILTER) headless: ${{ parameters.headless }} skipProvisioning: ${{ parameters.skipProvisioning }} @@ -571,7 +571,7 @@ stages: Write-Host "Running all categories in group: $effectiveFilter" } - Write-Host "##vso[task.setvariable variable=CATEGORYGROUP]$effectiveFilter" + Write-Host "##vso[task.setvariable variable=EFFECTIVE_FILTER]$effectiveFilter" displayName: Calculate test filter - template: ui-tests-steps.yml parameters: @@ -588,7 +588,7 @@ stages: device: ios-simulator-64_${{ version }} provisionatorChannel: ${{ parameters.provisionatorChannel }} runtimeVariant : "NativeAOT" - testFilter: $(CATEGORYGROUP) + testFilter: $(EFFECTIVE_FILTER) headless: ${{ parameters.headless }} skipProvisioning: ${{ parameters.skipProvisioning }} @@ -654,7 +654,7 @@ stages: Write-Host "Running all categories in group: $effectiveFilter" } - Write-Host "##vso[task.setvariable variable=CATEGORYGROUP]$effectiveFilter" + Write-Host "##vso[task.setvariable variable=EFFECTIVE_FILTER]$effectiveFilter" displayName: Calculate test filter - template: ui-tests-steps.yml parameters: @@ -664,7 +664,7 @@ stages: path: ${{ project.winui }} app: ${{ project.app }} provisionatorChannel: ${{ parameters.provisionatorChannel }} - testFilter: $(CATEGORYGROUP) + testFilter: $(EFFECTIVE_FILTER) skipProvisioning: ${{ parameters.skipProvisioning }} # Collect and publish Windows snapshot diffs @@ -736,7 +736,7 @@ stages: Write-Host "Running all categories in group: $effectiveFilter" } - Write-Host "##vso[task.setvariable variable=CATEGORYGROUP]$effectiveFilter" + Write-Host "##vso[task.setvariable variable=EFFECTIVE_FILTER]$effectiveFilter" displayName: Calculate test filter - template: ui-tests-steps.yml parameters: @@ -746,7 +746,7 @@ stages: path: ${{ project.mac }} app: ${{ project.app }} provisionatorChannel: ${{ parameters.provisionatorChannel }} - testFilter: $(CATEGORYGROUP) + testFilter: $(EFFECTIVE_FILTER) skipProvisioning: ${{ parameters.skipProvisioning }} # Collect and publish Mac snapshot diffs From f6e8f5f198cc8e0849776771883af23a296ea0f1 Mon Sep 17 00:00:00 2001 From: Gerald Versluis Date: Tue, 16 Dec 2025 16:02:21 +0100 Subject: [PATCH 11/19] Simplify: Move category filter logic into ui-tests-steps.yml Adopted the approach from PR #32935: - Pass CATEGORYGROUP, DETECTED_CATEGORIES via env block - Move intersection logic into ui-tests-steps.yml PowerShell - Remove duplicate 'Calculate test filter' steps from all stages - Read variables from environment instead of pipeline expressions This simplifies the pipeline by consolidating the filtering logic in one place rather than repeating it in every test stage. --- eng/pipelines/common/ui-tests-steps.yml | 60 +++++- eng/pipelines/common/ui-tests.yml | 258 ------------------------ 2 files changed, 55 insertions(+), 263 deletions(-) diff --git a/eng/pipelines/common/ui-tests-steps.yml b/eng/pipelines/common/ui-tests-steps.yml index f279e069f783..e75501083a0e 100644 --- a/eng/pipelines/common/ui-tests-steps.yml +++ b/eng/pipelines/common/ui-tests-steps.yml @@ -131,12 +131,59 @@ steps: $testFilter = "" $testConfigrationArgs = "${{ parameters.testConfigurationArgs }}" - - "${{ parameters.testFilter }}".Split(",") | ForEach { - $testFilter += "TestCategory=" + $_ + "|" + + # Get test filter from environment variable (passed from matrix via env block) + # When a dynamic matrix is used, the categoryGroup value is passed via env block + $testFilterParam = $env:CATEGORY_GROUP + $detectedCategories = $env:DETECTED_CATEGORIES + $isPR = $env:BUILD_REASON -eq "PullRequest" + + Write-Host "Build Reason: $env:BUILD_REASON" + Write-Host "Category Group from matrix: '$testFilterParam'" + Write-Host "Detected Categories: '$detectedCategories'" + + # Fallback to parameter for non-matrix builds + if (-not $testFilterParam) { + $testFilterParam = "${{ parameters.testFilter }}" + Write-Host "Using testFilter parameter: '$testFilterParam'" + } + + # For PRs with detected categories, find intersection with current category group + if ($isPR -and -not [string]::IsNullOrWhiteSpace($detectedCategories) -and -not [string]::IsNullOrWhiteSpace($testFilterParam)) { + $categoryList = $testFilterParam -split "," + $detectedList = $detectedCategories -split "," + $matchingCategories = @() + + foreach ($cat in $categoryList) { + $cat = $cat.Trim() + foreach ($det in $detectedList) { + $det = $det.Trim() + if ($cat -eq $det) { + $matchingCategories += $cat + break + } + } + } + + if ($matchingCategories.Count -gt 0) { + $testFilterParam = $matchingCategories -join "," + Write-Host "Running matching categories: $testFilterParam" + } + else { + # No match - use a filter that won't match any tests + $testFilterParam = "_NO_MATCHING_CATEGORY_" + Write-Host "No matching categories in this group - will skip tests" + } + } + + Write-Host "Final test filter: '$testFilterParam'" + + if ($testFilterParam) { + $testFilterParam.Split(",") | ForEach { + $testFilter += "TestCategory=" + $_ + "|" + } + $testFilter = $testFilter.TrimEnd("|") } - - $testFilter = $testFilter.TrimEnd("|") # Cake does not allow empty parameters, so check if our filter is empty before adding it if ($testConfigrationArgs) { @@ -159,6 +206,9 @@ steps: retryCountOnTaskFailure: 1 env: APPIUM_HOME: $(APPIUM_HOME) + CATEGORY_GROUP: $(CATEGORYGROUP) + DETECTED_CATEGORIES: $(DETECTED_CATEGORIES) + BUILD_REASON: $(Build.Reason) - bash: | cat ${BASH_SOURCE[0]} diff --git a/eng/pipelines/common/ui-tests.yml b/eng/pipelines/common/ui-tests.yml index e0545d1a5783..35a36fb4c39c 100644 --- a/eng/pipelines/common/ui-tests.yml +++ b/eng/pipelines/common/ui-tests.yml @@ -167,53 +167,6 @@ stages: # Access detected categories directly from stage dependency DETECTED_CATEGORIES: $[ stageDependencies.discover_ui_test_categories.detect.outputs['DetectCategories.UITestCategoryList'] ] steps: - # Calculate effective test filter based on detected categories - - pwsh: | - $detected = "$(DETECTED_CATEGORIES)" - $category = "$(CATEGORYGROUP)" - $isPR = "$(Build.Reason)" -eq "PullRequest" - - Write-Host "Build.Reason: $(Build.Reason)" - Write-Host "Is PR: $isPR" - Write-Host "Detected categories: '$detected'" - Write-Host "Current category group: '$category'" - - # Determine effective filter - $effectiveFilter = $category - - if ($isPR -and -not [string]::IsNullOrWhiteSpace($detected)) { - # For PRs with detected categories, find intersection - $categoryList = $category -split "," - $detectedList = $detected -split "," - $matchingCategories = @() - - foreach ($cat in $categoryList) { - $cat = $cat.Trim() - foreach ($det in $detectedList) { - $det = $det.Trim() - if ($cat -eq $det) { - $matchingCategories += $cat - break - } - } - } - - if ($matchingCategories.Count -gt 0) { - $effectiveFilter = $matchingCategories -join "," - Write-Host "Running matching categories: $effectiveFilter" - } - else { - # No match - use a filter that won't match any tests - $effectiveFilter = "_NO_MATCHING_CATEGORY_" - Write-Host "No matching categories - will skip tests" - } - } - else { - Write-Host "Running all categories in group: $effectiveFilter" - } - - Write-Host "##vso[task.setvariable variable=EFFECTIVE_FILTER]$effectiveFilter" - displayName: Calculate test filter - template: ui-tests-steps.yml parameters: platform: android @@ -225,7 +178,6 @@ stages: ${{ if not(eq(api, 27)) }}: device: android-emulator-64_${{ api }} provisionatorChannel: ${{ parameters.provisionatorChannel }} - testFilter: $(EFFECTIVE_FILTER) skipProvisioning: ${{ parameters.skipProvisioning }} # Collect and publish Android snapshot diffs @@ -261,47 +213,6 @@ stages: APPIUM_HOME: $(System.DefaultWorkingDirectory)/.appium/ DETECTED_CATEGORIES: $[ stageDependencies.discover_ui_test_categories.detect.outputs['DetectCategories.UITestCategoryList'] ] steps: - - pwsh: | - $detected = "$(DETECTED_CATEGORIES)" - $category = "$(CATEGORYGROUP)" - $isPR = "$(Build.Reason)" -eq "PullRequest" - - Write-Host "Detected categories: '$detected'" - Write-Host "Current category group: '$category'" - - $effectiveFilter = $category - - if ($isPR -and -not [string]::IsNullOrWhiteSpace($detected)) { - $categoryList = $category -split "," - $detectedList = $detected -split "," - $matchingCategories = @() - - foreach ($cat in $categoryList) { - $cat = $cat.Trim() - foreach ($det in $detectedList) { - $det = $det.Trim() - if ($cat -eq $det) { - $matchingCategories += $cat - break - } - } - } - - if ($matchingCategories.Count -gt 0) { - $effectiveFilter = $matchingCategories -join "," - Write-Host "Running matching categories: $effectiveFilter" - } - else { - $effectiveFilter = "_NO_MATCHING_CATEGORY_" - Write-Host "No matching categories - will skip tests" - } - } - else { - Write-Host "Running all categories in group: $effectiveFilter" - } - - Write-Host "##vso[task.setvariable variable=EFFECTIVE_FILTER]$effectiveFilter" - displayName: Calculate test filter - template: ui-tests-steps.yml parameters: platform: android @@ -314,7 +225,6 @@ stages: device: android-emulator-64_${{ api }} provisionatorChannel: ${{ parameters.provisionatorChannel }} agentPoolAccessToken: ${{ parameters.agentPoolAccessToken }} - testFilter: $(EFFECTIVE_FILTER) headless: ${{ parameters.headless }} skipProvisioning: ${{ parameters.skipProvisioning }} runtimeVariant: "CoreCLR" @@ -351,47 +261,6 @@ stages: APPIUM_HOME: $(System.DefaultWorkingDirectory)/.appium/ DETECTED_CATEGORIES: $[ stageDependencies.discover_ui_test_categories.detect.outputs['DetectCategories.UITestCategoryList'] ] steps: - - pwsh: | - $detected = "$(DETECTED_CATEGORIES)" - $category = "$(CATEGORYGROUP)" - $isPR = "$(Build.Reason)" -eq "PullRequest" - - Write-Host "Detected categories: '$detected'" - Write-Host "Current category group: '$category'" - - $effectiveFilter = $category - - if ($isPR -and -not [string]::IsNullOrWhiteSpace($detected)) { - $categoryList = $category -split "," - $detectedList = $detected -split "," - $matchingCategories = @() - - foreach ($cat in $categoryList) { - $cat = $cat.Trim() - foreach ($det in $detectedList) { - $det = $det.Trim() - if ($cat -eq $det) { - $matchingCategories += $cat - break - } - } - } - - if ($matchingCategories.Count -gt 0) { - $effectiveFilter = $matchingCategories -join "," - Write-Host "Running matching categories: $effectiveFilter" - } - else { - $effectiveFilter = "_NO_MATCHING_CATEGORY_" - Write-Host "No matching categories - will skip tests" - } - } - else { - Write-Host "Running all categories in group: $effectiveFilter" - } - - Write-Host "##vso[task.setvariable variable=EFFECTIVE_FILTER]$effectiveFilter" - displayName: Calculate test filter - template: ui-tests-steps.yml parameters: platform: ios @@ -407,7 +276,6 @@ stages: device: ios-simulator-64_${{ version }} provisionatorChannel: ${{ parameters.provisionatorChannel }} runtimeVariant : "Mono" - testFilter: $(EFFECTIVE_FILTER) headless: ${{ parameters.headless }} skipProvisioning: ${{ parameters.skipProvisioning }} @@ -532,47 +400,6 @@ stages: APPIUM_HOME: $(System.DefaultWorkingDirectory)/.appium/ DETECTED_CATEGORIES: $[ stageDependencies.discover_ui_test_categories.detect.outputs['DetectCategories.UITestCategoryList'] ] steps: - - pwsh: | - $detected = "$(DETECTED_CATEGORIES)" - $category = "$(CATEGORYGROUP)" - $isPR = "$(Build.Reason)" -eq "PullRequest" - - Write-Host "Detected categories: '$detected'" - Write-Host "Current category group: '$category'" - - $effectiveFilter = $category - - if ($isPR -and -not [string]::IsNullOrWhiteSpace($detected)) { - $categoryList = $category -split "," - $detectedList = $detected -split "," - $matchingCategories = @() - - foreach ($cat in $categoryList) { - $cat = $cat.Trim() - foreach ($det in $detectedList) { - $det = $det.Trim() - if ($cat -eq $det) { - $matchingCategories += $cat - break - } - } - } - - if ($matchingCategories.Count -gt 0) { - $effectiveFilter = $matchingCategories -join "," - Write-Host "Running matching categories: $effectiveFilter" - } - else { - $effectiveFilter = "_NO_MATCHING_CATEGORY_" - Write-Host "No matching categories - will skip tests" - } - } - else { - Write-Host "Running all categories in group: $effectiveFilter" - } - - Write-Host "##vso[task.setvariable variable=EFFECTIVE_FILTER]$effectiveFilter" - displayName: Calculate test filter - template: ui-tests-steps.yml parameters: platform: ios @@ -588,7 +415,6 @@ stages: device: ios-simulator-64_${{ version }} provisionatorChannel: ${{ parameters.provisionatorChannel }} runtimeVariant : "NativeAOT" - testFilter: $(EFFECTIVE_FILTER) headless: ${{ parameters.headless }} skipProvisioning: ${{ parameters.skipProvisioning }} @@ -615,47 +441,6 @@ stages: APPIUM_HOME: $(System.DefaultWorkingDirectory)\.appium\ DETECTED_CATEGORIES: $[ stageDependencies.discover_ui_test_categories.detect.outputs['DetectCategories.UITestCategoryList'] ] steps: - - pwsh: | - $detected = "$(DETECTED_CATEGORIES)" - $category = "$(CATEGORYGROUP)" - $isPR = "$(Build.Reason)" -eq "PullRequest" - - Write-Host "Detected categories: '$detected'" - Write-Host "Current category group: '$category'" - - $effectiveFilter = $category - - if ($isPR -and -not [string]::IsNullOrWhiteSpace($detected)) { - $categoryList = $category -split "," - $detectedList = $detected -split "," - $matchingCategories = @() - - foreach ($cat in $categoryList) { - $cat = $cat.Trim() - foreach ($det in $detectedList) { - $det = $det.Trim() - if ($cat -eq $det) { - $matchingCategories += $cat - break - } - } - } - - if ($matchingCategories.Count -gt 0) { - $effectiveFilter = $matchingCategories -join "," - Write-Host "Running matching categories: $effectiveFilter" - } - else { - $effectiveFilter = "_NO_MATCHING_CATEGORY_" - Write-Host "No matching categories - will skip tests" - } - } - else { - Write-Host "Running all categories in group: $effectiveFilter" - } - - Write-Host "##vso[task.setvariable variable=EFFECTIVE_FILTER]$effectiveFilter" - displayName: Calculate test filter - template: ui-tests-steps.yml parameters: platform: windows @@ -664,7 +449,6 @@ stages: path: ${{ project.winui }} app: ${{ project.app }} provisionatorChannel: ${{ parameters.provisionatorChannel }} - testFilter: $(EFFECTIVE_FILTER) skipProvisioning: ${{ parameters.skipProvisioning }} # Collect and publish Windows snapshot diffs @@ -697,47 +481,6 @@ stages: APPIUM_HOME: $(System.DefaultWorkingDirectory)/.appium/ DETECTED_CATEGORIES: $[ stageDependencies.discover_ui_test_categories.detect.outputs['DetectCategories.UITestCategoryList'] ] steps: - - pwsh: | - $detected = "$(DETECTED_CATEGORIES)" - $category = "$(CATEGORYGROUP)" - $isPR = "$(Build.Reason)" -eq "PullRequest" - - Write-Host "Detected categories: '$detected'" - Write-Host "Current category group: '$category'" - - $effectiveFilter = $category - - if ($isPR -and -not [string]::IsNullOrWhiteSpace($detected)) { - $categoryList = $category -split "," - $detectedList = $detected -split "," - $matchingCategories = @() - - foreach ($cat in $categoryList) { - $cat = $cat.Trim() - foreach ($det in $detectedList) { - $det = $det.Trim() - if ($cat -eq $det) { - $matchingCategories += $cat - break - } - } - } - - if ($matchingCategories.Count -gt 0) { - $effectiveFilter = $matchingCategories -join "," - Write-Host "Running matching categories: $effectiveFilter" - } - else { - $effectiveFilter = "_NO_MATCHING_CATEGORY_" - Write-Host "No matching categories - will skip tests" - } - } - else { - Write-Host "Running all categories in group: $effectiveFilter" - } - - Write-Host "##vso[task.setvariable variable=EFFECTIVE_FILTER]$effectiveFilter" - displayName: Calculate test filter - template: ui-tests-steps.yml parameters: platform: catalyst @@ -746,7 +489,6 @@ stages: path: ${{ project.mac }} app: ${{ project.app }} provisionatorChannel: ${{ parameters.provisionatorChannel }} - testFilter: $(EFFECTIVE_FILTER) skipProvisioning: ${{ parameters.skipProvisioning }} # Collect and publish Mac snapshot diffs From f2ffe5f326fb0d28896a18072fe28bde2d96fe41 Mon Sep 17 00:00:00 2001 From: Gerald Versluis Date: Tue, 16 Dec 2025 17:00:19 +0100 Subject: [PATCH 12/19] Fix: Exit early when no matching categories instead of running tests When there are no matching categories for a PR, exit with success immediately instead of running tests with a fake filter. This truly skips the test execution rather than running the test framework with no tests matching. --- eng/pipelines/common/ui-tests-steps.yml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/eng/pipelines/common/ui-tests-steps.yml b/eng/pipelines/common/ui-tests-steps.yml index e75501083a0e..25a7ed65f48c 100644 --- a/eng/pipelines/common/ui-tests-steps.yml +++ b/eng/pipelines/common/ui-tests-steps.yml @@ -170,9 +170,11 @@ steps: Write-Host "Running matching categories: $testFilterParam" } else { - # No match - use a filter that won't match any tests - $testFilterParam = "_NO_MATCHING_CATEGORY_" - Write-Host "No matching categories in this group - will skip tests" + # No matching categories - skip test execution entirely + Write-Host "##[section]No matching categories in this group - SKIPPING test execution" + Write-Host "Category group '$env:CATEGORY_GROUP' does not contain any of the detected categories: $detectedCategories" + Write-Host "##vso[task.complete result=Succeeded;]No tests to run for this category group" + exit 0 } } From e6e454216c8883c8507b8b770ed55902e5f1bf33 Mon Sep 17 00:00:00 2001 From: Gerald Versluis Date: Tue, 16 Dec 2025 17:27:34 +0100 Subject: [PATCH 13/19] Skip setup steps when UI test category doesn't match - Added early 'Check if category should run' step in ui-tests-steps.yml - Set SHOULD_RUN_TESTS variable based on category match - Added condition to skip: artifact downloads, clean bot, KVM enable, notification center disable, defaults modify, .NET install, PATH add, screen resolution, node install, Appium install, test execution This saves time for PR builds where only specific test categories are modified, by skipping setup for non-matching category groups. --- eng/pipelines/common/ui-tests-steps.yml | 86 +++++++++++++++++++------ 1 file changed, 65 insertions(+), 21 deletions(-) diff --git a/eng/pipelines/common/ui-tests-steps.yml b/eng/pipelines/common/ui-tests-steps.yml index 25a7ed65f48c..069a7e295285 100644 --- a/eng/pipelines/common/ui-tests-steps.yml +++ b/eng/pipelines/common/ui-tests-steps.yml @@ -15,23 +15,71 @@ parameters: checkoutDirectory: $(System.DefaultWorkingDirectory) steps: +# EARLY CHECK: Determine if this category group should run tests +# This runs FIRST to avoid wasting time on provisioning if no tests will run +- pwsh: | + $testFilterParam = $env:CATEGORY_GROUP + $detectedCategories = $env:DETECTED_CATEGORIES + $isPR = $env:BUILD_REASON -eq "PullRequest" + + Write-Host "=== Early Category Check ===" + Write-Host "Build Reason: $env:BUILD_REASON" + Write-Host "Category Group: '$testFilterParam'" + Write-Host "Detected Categories: '$detectedCategories'" + + $shouldRun = $true + + # For PRs with detected categories, check if this category group has any matches + if ($isPR -and -not [string]::IsNullOrWhiteSpace($detectedCategories) -and -not [string]::IsNullOrWhiteSpace($testFilterParam)) { + $categoryList = $testFilterParam -split "," + $detectedList = $detectedCategories -split "," + $hasMatch = $false + + foreach ($cat in $categoryList) { + $cat = $cat.Trim() + foreach ($det in $detectedList) { + $det = $det.Trim() + if ($cat -eq $det) { + $hasMatch = $true + Write-Host "Match found: '$cat'" + break + } + } + if ($hasMatch) { break } + } + + if (-not $hasMatch) { + $shouldRun = $false + Write-Host "##[warning]No matching categories - SKIPPING all steps for this job" + Write-Host "Category group '$testFilterParam' does not contain any detected categories: $detectedCategories" + } + } + + Write-Host "Should run tests: $shouldRun" + Write-Host "##vso[task.setvariable variable=SHOULD_RUN_TESTS]$shouldRun" + displayName: 'Check if category should run' + env: + CATEGORY_GROUP: $(CATEGORYGROUP) + DETECTED_CATEGORIES: $(DETECTED_CATEGORIES) + BUILD_REASON: $(Build.Reason) + - task: DownloadPipelineArtifact@2 - condition: and(ne('${{ parameters.platform }}' , 'windows'), eq('${{ parameters.runtimeVariant }}' , 'Mono')) + condition: and(succeeded(), eq(variables['SHOULD_RUN_TESTS'], 'True'), ne('${{ parameters.platform }}' , 'windows'), eq('${{ parameters.runtimeVariant }}' , 'Mono')) inputs: artifact: ui-tests-samples - task: DownloadPipelineArtifact@2 - condition: and(ne('${{ parameters.platform }}' , 'windows'), eq('${{ parameters.runtimeVariant }}' , 'NativeAOT')) + condition: and(succeeded(), eq(variables['SHOULD_RUN_TESTS'], 'True'), ne('${{ parameters.platform }}' , 'windows'), eq('${{ parameters.runtimeVariant }}' , 'NativeAOT')) inputs: artifact: ui-tests-samples-nativeaot - task: DownloadPipelineArtifact@2 - condition: and(ne('${{ parameters.platform }}' , 'windows'), eq('${{ parameters.runtimeVariant }}' , 'CoreCLR')) + condition: and(succeeded(), eq(variables['SHOULD_RUN_TESTS'], 'True'), ne('${{ parameters.platform }}' , 'windows'), eq('${{ parameters.runtimeVariant }}' , 'CoreCLR')) inputs: artifact: ui-tests-samples-coreclr - task: DownloadPipelineArtifact@2 - condition: eq('${{ parameters.platform }}' , 'windows') + condition: and(succeeded(), eq(variables['SHOULD_RUN_TESTS'], 'True'), eq('${{ parameters.platform }}' , 'windows')) inputs: artifact: ui-tests-samples-windows @@ -40,6 +88,7 @@ steps: chmod +x $(System.DefaultWorkingDirectory)/eng/scripts/clean-bot.sh $(System.DefaultWorkingDirectory)/eng/scripts/clean-bot.sh displayName: 'Clean bot' + condition: and(succeeded(), eq(variables['SHOULD_RUN_TESTS'], 'True')) continueOnError: true timeoutInMinutes: 60 @@ -50,13 +99,14 @@ steps: sudo udevadm control --reload-rules sudo udevadm trigger --name-match=kvm displayName: Enable KVM - condition: and(succeeded(), eq(variables['Agent.OS'], 'Linux')) + condition: and(succeeded(), eq(variables['SHOULD_RUN_TESTS'], 'True'), eq(variables['Agent.OS'], 'Linux')) - ${{ if eq(parameters.platform, 'catalyst')}}: - bash: | chmod +x $(System.DefaultWorkingDirectory)/eng/scripts/disable-notification-center.sh $(System.DefaultWorkingDirectory)/eng/scripts/disable-notification-center.sh displayName: 'Disable Notification Center' + condition: and(succeeded(), eq(variables['SHOULD_RUN_TESTS'], 'True')) continueOnError: true timeoutInMinutes: 60 @@ -80,7 +130,7 @@ steps: openSslArgs: '' # Do not use legacy openssl for Catalyst builds - task: PowerShell@2 - condition: and(ne('${{ parameters.platform }}', 'windows'), ne('${{ parameters.platform }}', 'android')) + condition: and(succeeded(), eq(variables['SHOULD_RUN_TESTS'], 'True'), ne('${{ parameters.platform }}', 'windows'), ne('${{ parameters.platform }}', 'android')) inputs: targetType: 'inline' script: | @@ -92,6 +142,7 @@ steps: - pwsh: ./build.ps1 --target=dotnet --configuration="${{ parameters.configuration }}" --verbosity=diagnostic displayName: 'Install .NET' + condition: and(succeeded(), eq(variables['SHOULD_RUN_TESTS'], 'True')) retryCountOnTaskFailure: 2 env: DOTNET_TOKEN: $(dotnetbuilds-internal-container-read-token) @@ -99,15 +150,17 @@ steps: - pwsh: echo "##vso[task.prependpath]$(DotNet.Dir)" displayName: 'Add .NET to PATH' + condition: and(succeeded(), eq(variables['SHOULD_RUN_TESTS'], 'True')) # AzDO hosted agents default to 1024x768; set something bigger for Windows UI tests - pwsh: | $scriptPath = Join-Path "$(System.DefaultWorkingDirectory)" "eng" "scripts" "Set-ScreenResolution.ps1" & $scriptPath -Width 1920 -Height 1080 - condition: eq('${{ parameters.platform }}' , 'windows') + condition: and(succeeded(), eq(variables['SHOULD_RUN_TESTS'], 'True'), eq('${{ parameters.platform }}' , 'windows')) displayName: "Set screen resolution" - task: UseNode@1 + condition: and(succeeded(), eq(variables['SHOULD_RUN_TESTS'], 'True')) inputs: version: "20.3.1" displayName: "Install node" @@ -116,6 +169,7 @@ steps: $skipAppiumDoctor = if ($IsMacOS -or $IsLinux) { "true" } else { "false" } dotnet build ./src/Provisioning/Provisioning.csproj -t:ProvisionAppium -p:SkipAppiumDoctor="$skipAppiumDoctor" -bl:"$(LogDirectory)/provision-appium.binlog" displayName: "Install Appium" + condition: and(succeeded(), eq(variables['SHOULD_RUN_TESTS'], 'True')) continueOnError: false retryCountOnTaskFailure: 2 timeoutInMinutes: 10 @@ -133,12 +187,10 @@ steps: $testConfigrationArgs = "${{ parameters.testConfigurationArgs }}" # Get test filter from environment variable (passed from matrix via env block) - # When a dynamic matrix is used, the categoryGroup value is passed via env block $testFilterParam = $env:CATEGORY_GROUP $detectedCategories = $env:DETECTED_CATEGORIES $isPR = $env:BUILD_REASON -eq "PullRequest" - Write-Host "Build Reason: $env:BUILD_REASON" Write-Host "Category Group from matrix: '$testFilterParam'" Write-Host "Detected Categories: '$detectedCategories'" @@ -148,7 +200,7 @@ steps: Write-Host "Using testFilter parameter: '$testFilterParam'" } - # For PRs with detected categories, find intersection with current category group + # For PRs with detected categories, use only matching categories if ($isPR -and -not [string]::IsNullOrWhiteSpace($detectedCategories) -and -not [string]::IsNullOrWhiteSpace($testFilterParam)) { $categoryList = $testFilterParam -split "," $detectedList = $detectedCategories -split "," @@ -165,17 +217,8 @@ steps: } } - if ($matchingCategories.Count -gt 0) { - $testFilterParam = $matchingCategories -join "," - Write-Host "Running matching categories: $testFilterParam" - } - else { - # No matching categories - skip test execution entirely - Write-Host "##[section]No matching categories in this group - SKIPPING test execution" - Write-Host "Category group '$env:CATEGORY_GROUP' does not contain any of the detected categories: $detectedCategories" - Write-Host "##vso[task.complete result=Succeeded;]No tests to run for this category group" - exit 0 - } + $testFilterParam = $matchingCategories -join "," + Write-Host "Running matching categories: $testFilterParam" } Write-Host "Final test filter: '$testFilterParam'" @@ -204,6 +247,7 @@ steps: Invoke-Expression $command displayName: $(Agent.JobName) + condition: and(succeeded(), eq(variables['SHOULD_RUN_TESTS'], 'True')) ${{ if ne(parameters.platform, 'android')}}: retryCountOnTaskFailure: 1 env: From f060cc5fc296fa91cebff225ac14a19004423b18 Mon Sep 17 00:00:00 2001 From: Gerald Versluis Date: Tue, 16 Dec 2025 17:53:43 +0100 Subject: [PATCH 14/19] Add SKIP_PROVISIONING check to provision.yml Skip expensive provisioning steps (JDK, Android SDK, .NET SDK, Xcode, simulator runtimes) when SHOULD_RUN_TESTS is False. This saves significant build time when a UI test category doesn't match the detected categories from the PR changes. Uses a single early check step that sets SKIP_PROVISIONING variable, which is then checked by only the expensive steps. --- eng/pipelines/common/provision.yml | 41 +++++++++++++++++++++++------- 1 file changed, 32 insertions(+), 9 deletions(-) diff --git a/eng/pipelines/common/provision.yml b/eng/pipelines/common/provision.yml index 4b7b3d756b4b..8f19886cf282 100644 --- a/eng/pipelines/common/provision.yml +++ b/eng/pipelines/common/provision.yml @@ -35,6 +35,25 @@ parameters: steps: +################################################## +# Skip Check for UI Test Optimization # +################################################## + +# If SHOULD_RUN_TESTS is explicitly False (set by ui-tests-steps.yml), +# skip all provisioning to save time +- pwsh: | + $shouldRun = $env:SHOULD_RUN_TESTS + Write-Host "SHOULD_RUN_TESTS: '$shouldRun'" + if ($shouldRun -eq 'False') { + Write-Host "##[warning]Skipping all provisioning - no tests will run for this category" + Write-Host "##vso[task.setvariable variable=SKIP_PROVISIONING]true" + } else { + Write-Host "##vso[task.setvariable variable=SKIP_PROVISIONING]false" + } + displayName: 'Check if provisioning should be skipped' + env: + SHOULD_RUN_TESTS: $(SHOULD_RUN_TESTS) + ################################################## # Provisioning Feeds # ################################################## @@ -77,7 +96,7 @@ steps: - ${{ if ne(parameters.skipXcode, 'true') }}: - ${{ if ne(parameters.skipProvisionator, true) }}: - task: xamops.azdevex.provisionator-task.provisionator@3 - condition: and(succeeded(), eq(variables['Agent.OS'], 'Darwin')) + condition: and(succeeded(), ne(variables['SKIP_PROVISIONING'], 'true'), eq(variables['Agent.OS'], 'Darwin')) displayName: 'Provision Xcode' inputs: provisionator_uri: ${{ parameters.provisionatorUri }} @@ -140,7 +159,7 @@ steps: sudo xcodebuild -runFirstLaunch displayName: Select Xcode Version - condition: and(succeeded(), eq(variables['Agent.OS'], 'Darwin')) + condition: and(succeeded(), ne(variables['SKIP_PROVISIONING'], 'true'), eq(variables['Agent.OS'], 'Darwin')) timeoutInMinutes: 30 - ${{ if ne(parameters.skipSimulatorSetup, 'true') }}: @@ -180,7 +199,7 @@ steps: sudo xcodebuild -downloadPlatform iOS fi displayName: Install Simulator Runtimes - condition: and(succeeded(), eq(variables['Agent.OS'], 'Darwin')) + condition: and(succeeded(), ne(variables['SKIP_PROVISIONING'], 'true'), eq(variables['Agent.OS'], 'Darwin')) timeoutInMinutes: 30 # Provision Additional Software @@ -224,6 +243,7 @@ steps: # Provisioning .NET - .NET SDK - task: UseDotNet@2 displayName: 'Use .NET SDK $(DOTNET_VERSION)' + condition: and(succeeded(), ne(variables['SKIP_PROVISIONING'], 'true')) inputs: packageType: sdk version: $(DOTNET_VERSION) @@ -233,18 +253,21 @@ steps: dotnet --list-sdks dotnet --version displayName: 'List all .NET SDKs' + condition: and(succeeded(), ne(variables['SKIP_PROVISIONING'], 'true')) continueOnError: true # Provisioning .NET - Clear all NuGet caches - ${{ if eq(parameters.clearCaches, 'true') }}: - script: dotnet nuget locals all --clear displayName: 'Clear all NuGet caches' + condition: and(succeeded(), ne(variables['SKIP_PROVISIONING'], 'true')) continueOnError: true # Provisioning .NET - .NET tools - script: | dotnet tool restore --verbosity diag displayName: 'Restore .NET Tools' + condition: and(succeeded(), ne(variables['SKIP_PROVISIONING'], 'true')) ################################################## # Provisioning Android # @@ -254,13 +277,13 @@ steps: - ${{ if ne(parameters.skipJdk, 'true') }}: - pwsh: dotnet build -t:ProvisionJdk -bl:"$(LogDirectory)/provision-jdk.binlog" ${{ parameters.checkoutDirectory }}/src/Provisioning/Provisioning.csproj -v:detailed displayName: 'Provision JDK' - condition: succeeded() + condition: and(succeeded(), ne(variables['SKIP_PROVISIONING'], 'true')) # Provisioning Android - Android SDK common packages (eg: cmdline-tools, emulator, platform-tools, build-tools) - ${{ if ne(parameters.skipAndroidCommonSdks, 'true') }}: - pwsh: dotnet build -t:ProvisionAndroidSdkCommonPackages -bl:"$(LogDirectory)/provision-androidsdk-common.binlog" ${{ parameters.checkoutDirectory }}/src/Provisioning/Provisioning.csproj -v:detailed displayName: 'Provision Android SDK - Common Packages' - condition: succeeded() + condition: and(succeeded(), ne(variables['SKIP_PROVISIONING'], 'true')) # Provisioning Android - Android environment variables - ${{ if ne(parameters.skipAndroidCommonSdks, 'true') }}: @@ -273,13 +296,13 @@ steps: echo "##vso[task.setvariable variable=ANDROID_HOME]$preferredSdk" echo "ANDROID_HOME set to '$preferredSdk'" displayName: 'Provision Android SDK - Environment variables' - condition: succeeded() + condition: and(succeeded(), ne(variables['SKIP_PROVISIONING'], 'true')) # Provisioning Android - Android SDK platform APIs (eg: platforms;android-29, platforms;android-30) - ${{ if ne(parameters.skipAndroidPlatformApis, 'true') }}: - pwsh: dotnet build -t:ProvisionAndroidSdkPlatformApiPackages -p:AndroidSdkProvisionDefaultApiLevelsOnly="$Env:AndroidSdkProvisionDefaultApiLevelsOnly" -bl:"$(LogDirectory)/provision-androidsdk-platform-apis.binlog" ${{ parameters.checkoutDirectory }}/src/Provisioning/Provisioning.csproj -v:detailed displayName: 'Provision Android SDK - Platform APIs' - condition: succeeded() + condition: and(succeeded(), ne(variables['SKIP_PROVISIONING'], 'true')) env: AndroidSdkProvisionDefaultApiLevelsOnly: ${{ parameters.onlyAndroidPlatformDefaultApis }} @@ -287,7 +310,7 @@ steps: - ${{ if ne(parameters.skipAndroidEmulatorImages, 'true') }}: - pwsh: dotnet build -t:ProvisionAndroidSdkEmulatorImagePackages -p:AndroidSdkProvisionApiLevel="$Env:AndroidSdkProvisionApiLevel" -bl:"$(LogDirectory)/provision-androidsdk-emulator-images.binlog" ${{ parameters.checkoutDirectory }}/src/Provisioning/Provisioning.csproj -v:detailed displayName: 'Provision Android SDK - Emulator Images' - condition: succeeded() + condition: and(succeeded(), ne(variables['SKIP_PROVISIONING'], 'true')) env: AndroidSdkProvisionApiLevel: ${{ parameters.androidEmulatorApiLevel }} @@ -295,6 +318,6 @@ steps: - ${{ if ne(parameters.skipAndroidCreateAvds, 'true') }}: - pwsh: dotnet build -t:ProvisionAndroidSdkAvdCreateAvds -p:AndroidSdkProvisionApiLevel="$Env:AndroidSdkProvisionApiLevel" -bl:"$(LogDirectory)/provision-androidsdk-create-avds.binlog" ${{ parameters.checkoutDirectory }}/src/Provisioning/Provisioning.csproj -v:detailed displayName: 'Provision Android SDK - Create AVDs' - condition: succeeded() + condition: and(succeeded(), ne(variables['SKIP_PROVISIONING'], 'true')) env: AndroidSdkProvisionApiLevel: ${{ parameters.androidEmulatorApiLevel }} \ No newline at end of file From 07df2404a9a94538137c22cbe076e37136c234c2 Mon Sep 17 00:00:00 2001 From: Gerald Versluis Date: Wed, 17 Dec 2025 14:56:47 +0100 Subject: [PATCH 15/19] Add category detection to CV2/CARV2 stages - Add discover_ui_test_categories dependency to CV2 and CARV2 stages - Add DETECTED_CATEGORIES variable to CV2 and CARV2 jobs - Update early check to use testFilter parameter as fallback when CATEGORYGROUP is not set (for non-matrix stages like CV2/CARV2) Now CV2 will only run if CollectionView is detected in PR changes, and CARV2 will only run if CarouselView is detected. --- eng/pipelines/common/ui-tests-steps.yml | 11 ++++++++++- eng/pipelines/common/ui-tests.yml | 10 ++++++++-- 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/eng/pipelines/common/ui-tests-steps.yml b/eng/pipelines/common/ui-tests-steps.yml index 069a7e295285..1374f6dc6cd6 100644 --- a/eng/pipelines/common/ui-tests-steps.yml +++ b/eng/pipelines/common/ui-tests-steps.yml @@ -19,14 +19,22 @@ steps: # This runs FIRST to avoid wasting time on provisioning if no tests will run - pwsh: | $testFilterParam = $env:CATEGORY_GROUP + $testFilterFallback = $env:TEST_FILTER_PARAM $detectedCategories = $env:DETECTED_CATEGORIES $isPR = $env:BUILD_REASON -eq "PullRequest" Write-Host "=== Early Category Check ===" Write-Host "Build Reason: $env:BUILD_REASON" - Write-Host "Category Group: '$testFilterParam'" + Write-Host "Category Group (from matrix): '$testFilterParam'" + Write-Host "Test Filter (from parameter): '$testFilterFallback'" Write-Host "Detected Categories: '$detectedCategories'" + # Use testFilter parameter as fallback when CATEGORYGROUP is not set (CV2/CARV2 stages) + if ([string]::IsNullOrWhiteSpace($testFilterParam)) { + $testFilterParam = $testFilterFallback + Write-Host "Using testFilter parameter as category: '$testFilterParam'" + } + $shouldRun = $true # For PRs with detected categories, check if this category group has any matches @@ -60,6 +68,7 @@ steps: displayName: 'Check if category should run' env: CATEGORY_GROUP: $(CATEGORYGROUP) + TEST_FILTER_PARAM: ${{ parameters.testFilter }} DETECTED_CATEGORIES: $(DETECTED_CATEGORIES) BUILD_REASON: $(Build.Reason) diff --git a/eng/pipelines/common/ui-tests.yml b/eng/pipelines/common/ui-tests.yml index 35a36fb4c39c..2ceb9077a7ef 100644 --- a/eng/pipelines/common/ui-tests.yml +++ b/eng/pipelines/common/ui-tests.yml @@ -287,7 +287,9 @@ stages: - stage: ios_ui_tests_mono_cv2 displayName: iOS UITests Mono CollectionView2 - dependsOn: build_ui_tests + dependsOn: + - build_ui_tests + - discover_ui_test_categories jobs: - ${{ each project in parameters.projects }}: - ${{ if ne(project.ios, '') }}: @@ -302,6 +304,7 @@ stages: variables: REQUIRED_XCODE: $(DEVICETESTS_REQUIRED_XCODE) APPIUM_HOME: $(System.DefaultWorkingDirectory)/.appium/ + DETECTED_CATEGORIES: $[ stageDependencies.discover_ui_test_categories.detect.outputs['DetectCategories.UITestCategoryList'] ] steps: - template: ui-tests-steps.yml parameters: @@ -331,7 +334,9 @@ stages: - stage: ios_ui_tests_mono_carv2 displayName: iOS UITests Mono CarouselView2 - dependsOn: build_ui_tests + dependsOn: + - build_ui_tests + - discover_ui_test_categories jobs: - ${{ each project in parameters.projects }}: - ${{ if ne(project.ios, '') }}: @@ -346,6 +351,7 @@ stages: variables: REQUIRED_XCODE: $(DEVICETESTS_REQUIRED_XCODE) APPIUM_HOME: $(System.DefaultWorkingDirectory)/.appium/ + DETECTED_CATEGORIES: $[ stageDependencies.discover_ui_test_categories.detect.outputs['DetectCategories.UITestCategoryList'] ] steps: - template: ui-tests-steps.yml parameters: From 7e2ca9461f973ab5ecd10474907012dce57dcbf9 Mon Sep 17 00:00:00 2001 From: Gerald Versluis Date: Wed, 17 Dec 2025 15:28:05 +0100 Subject: [PATCH 16/19] Fix: Detect unexpanded $(CATEGORYGROUP) variable When CATEGORYGROUP variable is not defined (CV2/CARV2 stages without matrix), Azure Pipelines passes the literal string '$(CATEGORYGROUP)' instead of empty. Now we check for both empty AND the literal unexpanded variable string before falling back to testFilter. --- eng/pipelines/common/ui-tests-steps.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/eng/pipelines/common/ui-tests-steps.yml b/eng/pipelines/common/ui-tests-steps.yml index 1374f6dc6cd6..f13808d84537 100644 --- a/eng/pipelines/common/ui-tests-steps.yml +++ b/eng/pipelines/common/ui-tests-steps.yml @@ -29,8 +29,10 @@ steps: Write-Host "Test Filter (from parameter): '$testFilterFallback'" Write-Host "Detected Categories: '$detectedCategories'" - # Use testFilter parameter as fallback when CATEGORYGROUP is not set (CV2/CARV2 stages) - if ([string]::IsNullOrWhiteSpace($testFilterParam)) { + # Use testFilter parameter as fallback when CATEGORYGROUP is not set or not expanded + # When CATEGORYGROUP variable doesn't exist, it shows as literal '$(CATEGORYGROUP)' + $categoryGroupNotSet = [string]::IsNullOrWhiteSpace($testFilterParam) -or $testFilterParam -eq '$(CATEGORYGROUP)' + if ($categoryGroupNotSet -and -not [string]::IsNullOrWhiteSpace($testFilterFallback)) { $testFilterParam = $testFilterFallback Write-Host "Using testFilter parameter as category: '$testFilterParam'" } From 599710f91120c395244c0024415e1815a5f66746 Mon Sep 17 00:00:00 2001 From: Gerald Versluis Date: Thu, 18 Dec 2025 11:42:58 +0100 Subject: [PATCH 17/19] Test: Add dummy Label test for category detection validation --- .../tests/TestCases.Shared.Tests/Tests/LabelUITests.cs | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/Controls/tests/TestCases.Shared.Tests/Tests/LabelUITests.cs b/src/Controls/tests/TestCases.Shared.Tests/Tests/LabelUITests.cs index 0b78392c7b6e..3b8591d38a22 100644 --- a/src/Controls/tests/TestCases.Shared.Tests/Tests/LabelUITests.cs +++ b/src/Controls/tests/TestCases.Shared.Tests/Tests/LabelUITests.cs @@ -71,4 +71,11 @@ public void FontFamily() remote.TapStateButton(); VerifyScreenshot("LabelUITests_FontFamily_FontAwesome"); } + + // Dummy test for PR category detection validation - will be removed + [Test] + public void DummyLabelTest() + { + Assert.Pass("Dummy test to validate PR category detection for Label"); + } } From d5d2dded8636307bcfd4c5f10f57a1790aef1341 Mon Sep 17 00:00:00 2001 From: Gerald Versluis Date: Thu, 18 Dec 2025 12:51:06 +0100 Subject: [PATCH 18/19] Fix: Add explicit Category attribute to dummy Label test --- src/Controls/tests/TestCases.Shared.Tests/Tests/LabelUITests.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Controls/tests/TestCases.Shared.Tests/Tests/LabelUITests.cs b/src/Controls/tests/TestCases.Shared.Tests/Tests/LabelUITests.cs index 3b8591d38a22..455187fc02f6 100644 --- a/src/Controls/tests/TestCases.Shared.Tests/Tests/LabelUITests.cs +++ b/src/Controls/tests/TestCases.Shared.Tests/Tests/LabelUITests.cs @@ -74,6 +74,7 @@ public void FontFamily() // Dummy test for PR category detection validation - will be removed [Test] + [Category(UITestCategories.Label)] public void DummyLabelTest() { Assert.Pass("Dummy test to validate PR category detection for Label"); From f9ec697a56459c6da925bf775227fe26344903a2 Mon Sep 17 00:00:00 2001 From: Gerald Versluis Date: Thu, 18 Dec 2025 13:55:21 +0100 Subject: [PATCH 19/19] Remove dummy Button and Label tests --- .../tests/TestCases.Shared.Tests/Tests/ButtonUITests.cs | 9 --------- .../tests/TestCases.Shared.Tests/Tests/LabelUITests.cs | 8 -------- 2 files changed, 17 deletions(-) diff --git a/src/Controls/tests/TestCases.Shared.Tests/Tests/ButtonUITests.cs b/src/Controls/tests/TestCases.Shared.Tests/Tests/ButtonUITests.cs index 4da08977d066..705d07621ec2 100644 --- a/src/Controls/tests/TestCases.Shared.Tests/Tests/ButtonUITests.cs +++ b/src/Controls/tests/TestCases.Shared.Tests/Tests/ButtonUITests.cs @@ -19,15 +19,6 @@ protected override void NavigateToGallery() App.NavigateToGallery(ButtonGallery); } - // Test change to trigger category detection - [Test] - [Category(UITestCategories.Button)] - public void DummyButtonTest() - { - // Placeholder test for category detection validation - Assert.Pass("Category detection test"); - } - [Test] [Category(UITestCategories.Button)] public void Clicked() diff --git a/src/Controls/tests/TestCases.Shared.Tests/Tests/LabelUITests.cs b/src/Controls/tests/TestCases.Shared.Tests/Tests/LabelUITests.cs index 455187fc02f6..0b78392c7b6e 100644 --- a/src/Controls/tests/TestCases.Shared.Tests/Tests/LabelUITests.cs +++ b/src/Controls/tests/TestCases.Shared.Tests/Tests/LabelUITests.cs @@ -71,12 +71,4 @@ public void FontFamily() remote.TapStateButton(); VerifyScreenshot("LabelUITests_FontFamily_FontAwesome"); } - - // Dummy test for PR category detection validation - will be removed - [Test] - [Category(UITestCategories.Label)] - public void DummyLabelTest() - { - Assert.Pass("Dummy test to validate PR category detection for Label"); - } }