Skip to content

add a 'Run Flaky Tests' step that does not fail the build #1239

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 4 commits into from
Mar 13, 2019
Merged

add a 'Run Flaky Tests' step that does not fail the build #1239

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
3a03808
add a 'Run Flaky Tests' step that does not fail the build
analogrelay Mar 7, 2019
fdd689a
set -ci in flaky test pass to ensure nuget package root is correct
analogrelay Mar 12, 2019
dfd7b4b
publish flaky test results to AzP
analogrelay Mar 12, 2019
d7be876
chmod!!!!!!
analogrelay Mar 12, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions Directory.Build.props
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,9 +17,11 @@
<OutDirName Condition="'$(IsReferenceAssemblyProject)' == 'true'">$(MSBuildProjectName)-ref</OutDirName>
</PropertyGroup>

<Import Project="eng\FlakyTests.BeforeArcade.props" />
<Import Project="eng\Workarounds.BeforeArcade.props" />
<Import Project="Sdk.props" Sdk="Microsoft.DotNet.Arcade.Sdk" />
<Import Project="eng\Workarounds.AfterArcade.props" />
<Import Project="eng\FlakyTests.AfterArcade.props" />

<PropertyGroup Condition="'$(CopyrightMicrosoft)' != ''">
<Copyright>$(CopyrightMicrosoft)</Copyright>
Expand Down
3 changes: 0 additions & 3 deletions Directory.Build.targets
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -47,8 +47,5 @@
<Import Project="Sdk.targets" Sdk="Microsoft.DotNet.Arcade.Sdk" />
<Import Project="eng\Workarounds.AfterArcade.targets" />

<!-- Only needed on AzDO: Flaky Test handling -->
<Import Project="eng\FlakyTests.targets" Condition="'$(BUILD_BUILDNUMBER)' != ''" />

<Import Project="eng\targets\Npm.Common.targets" Condition="'$(MSBuildProjectExtension)' == '.npmproj'" />
</Project>
34 changes: 34 additions & 0 deletions azure-pipelines.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -91,6 +91,9 @@ jobs:
-prepareMachine
$(_BuildArgs)
displayName: Build and Publish
- script: eng\scripts\ci-flaky-tests.cmd
displayName: Run Flaky Tests
continueOnError: true
- powershell: eng\common\msbuild.ps1 eng/repo.targets /t:TagCiBuilds '/p:IsFinalBuild=$(IsFinalBuild)' '/p:CI=true'
displayName: Set CI Tags
condition: eq(variables['_BuildConfig'], 'Release')
Expand All @@ -101,6 +104,14 @@ jobs:
- task: ms.vss-governance-buildtask.governance-build-task-component-detection.ComponentGovernanceComponentDetection@0
displayName: Detect components
condition: and(succeeded(), eq(variables['system.pullrequest.isfork'], false), eq(variables['_BuildConfig'], 'Release'))
- task: PublishTestResults@2
displayName: Publish Flaky Test Results
inputs:
testResultsFormat: 'xUnit'
testResultsFiles: '*.xml'
searchFolder: '$(Build.SourcesDirectory)/artifacts/TestResults/$(_BuildConfig)/Flaky'
continueOnError: true
condition: always()
- task: PublishBuildArtifacts@1
displayName: Upload package artifacts
condition: and(eq(variables['system.pullrequest.isfork'], false), eq(variables['_BuildConfig'], 'Release'))
Expand Down Expand Up @@ -133,9 +144,20 @@ jobs:
--configuration $(_BuildConfig)
--prepareMachine
displayName: Build
- script: eng/scripts/ci-flaky-tests.sh
displayName: Run Flaky Tests
continueOnError: true
- script: eng/scripts/KillProcesses.sh
displayName: Kill processes
condition: always()
- task: PublishTestResults@2
displayName: Publish Flaky Test Results
inputs:
testResultsFormat: 'xUnit'
testResultsFiles: '*.xml'
searchFolder: '$(Build.SourcesDirectory)/artifacts/TestResults/$(_BuildConfig)/Flaky'
continueOnError: true
condition: always()

- job: OSX_10_13
displayName: 'OSX'
Expand All @@ -160,6 +182,18 @@ jobs:
--configuration $(_BuildConfig)
--prepareMachine
displayName: Build
- script: eng/scripts/ci-flaky-tests.sh
displayName: Run Flaky Tests
continueOnError: true
- script: eng/scripts/KillProcesses.sh
displayName: Kill processes
condition: always()
- task: PublishTestResults@2
displayName: Publish Flaky Test Results
inputs:
testResultsFormat: 'xUnit'
testResultsFiles: '*.xml'
searchFolder: '$(Build.SourcesDirectory)/artifacts/TestResults/$(_BuildConfig)/Flaky'
continueOnError: true
condition: always()

7 changes: 7 additions & 0 deletions eng/FlakyTests.AfterArcade.props
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
<Project>
<!-- Override where xUnit logs and results go if we're doing the flaky run -->
<PropertyGroup Condition="'$(RunFlakyTests)' == 'true'">
<ArtifactsLogDir>$(ArtifactsDir)log\$(Configuration)\Flaky\</ArtifactsLogDir>
<ArtifactsTestResultsDir>$(ArtifactsDir)TestResults\$(Configuration)\Flaky\</ArtifactsTestResultsDir>
</PropertyGroup>
</Project>
18 changes: 18 additions & 0 deletions eng/FlakyTests.BeforeArcade.props
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,18 @@
<Project>
<!-- Local Dev Flakiness -->
<PropertyGroup>
<_FlakyRunAdditionalArgs>-trait "Flaky:All=true"</_FlakyRunAdditionalArgs>
<_NonFlakyRunAdditionalArgs>-notrait "Flaky:All=true"</_NonFlakyRunAdditionalArgs>
</PropertyGroup>

<!-- Azure Pipelines Flakiness -->
<PropertyGroup Condition="'$(AGENT_OS)' != ''">
<_FlakyRunAdditionalArgs>$(_FlakyRunAdditionalArgs) -trait "Flaky:AzP:All=true" -trait "Flaky:AzP:OS:$(AGENT_OS)=true"</_FlakyRunAdditionalArgs>
<_NonFlakyRunAdditionalArgs>$(_NonFlakyRunAdditionalArgs) -notrait "Flaky:AzP:All=true" -notrait "Flaky:AzP:OS:$(AGENT_OS)=true"</_NonFlakyRunAdditionalArgs>
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Aren't -trait "Flaky:AzP:All=true" and -notrait "Flaky:AzP:All=true" unnecessary in these two lines?

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Not quite. The logic isn't as simple as just setting the OS-based flag. The Flaky attribute is trying to be a little agnostic as to the "filters" in use (since we may add other pivots like architectures, etc.), so if you say it's flaky on Darwin, like so:

[Flaky("...", AzurePipelines.macOS)]

Then we set the xUnit trait: Flaky:AzP:OS:Darwin=true

If you say it's flaky everywhere, like so:

[Flaky("...", AzurePipelines.All)]

Then we set a different xUnit trait: Flaky:AzP:All=true since we don't want the Flaky attribute logic to have to enumerate every possible pivot and filter.

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I did it this way so that the complexity could live in the build script rather than having to keep updating the Flaky attribute whenever we add new pivots or new pivot pieces. If we wanted to simplify this logic, we'd need Flaky to inject traits representing all possible pivots for which the test is flaky which makes things like AzurePipelines.All tricky.

</PropertyGroup>

<PropertyGroup>
<TestRunnerAdditionalArguments Condition="'$(RunFlakyTests)' == ''">$(_NonFlakyRunAdditionalArgs)</TestRunnerAdditionalArguments>
<TestRunnerAdditionalArguments Condition="'$(RunFlakyTests)' == 'true'">$(_FlakyRunAdditionalArgs)</TestRunnerAdditionalArguments>
</PropertyGroup>
</Project>
33 changes: 0 additions & 33 deletions eng/FlakyTests.targets
View file
Open in desktop

This file was deleted.

3 changes: 3 additions & 0 deletions eng/scripts/ci-flaky-tests.cmd
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
@echo off
powershell -ExecutionPolicy ByPass -NoProfile -command "& """%~dp0..\common\Build.ps1""" -test -ci /p:RunFlakyTests=true %*"
exit /b %ErrorLevel%
16 changes: 16 additions & 0 deletions eng/scripts/ci-flaky-tests.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
#!/usr/bin/env bash

source="${BASH_SOURCE[0]}"

# resolve $SOURCE until the file is no longer a symlink
while [[ -h $source ]]; do
scriptroot="$( cd -P "$( dirname "$source" )" && pwd )"
source="$(readlink "$source")"

# if $source was a relative symlink, we need to resolve it relative to the path where
# the symlink file was located
[[ $source != /* ]] && source="$scriptroot/$source"
done
scriptroot="$( cd -P "$( dirname "$source" )" && pwd )"

. "$scriptroot/../common/build.sh" --ci --test -p:RunFlakyTests=true $@