Skip to content

Commit 706778d

Browse files
analogrelayanalogrelay
authored
Add a separate test pass to run flaky tests that doesn't fail the build (#8486)
Similar to dotnet/extensions#1239.
1 parent f303a55 commit 706778d

File tree

7 files changed

+93
-7
lines changed

7 files changed

+93
-7
lines changed

‎.azure/pipelines/ci.yml‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -300,13 +300,17 @@ jobs:
300300
- powershell: "& ./.azure/pipelines/tools/SetupTestEnvironment.ps1 Setup signalrclienttests.exe"
301301
displayName: Start AppVerifier
302302
afterBuild:
303+
- powershell: "& ./build.ps1 -CI -NoBuild -Test /p:RunFlakyTests=true"
304+
displayName: Run Flaky Tests
305+
continueOnError: true
303306
- powershell: "& ./.azure/pipelines/tools/SetupTestEnvironment.ps1 Shutdown signalrclienttests.exe"
304307
displayName: Stop AppVerifier
305308
condition: always()
306309
artifacts:
307310
- name: Windows_Test_Logs
308311
path: artifacts/logs/
309312
publishOnError: true
313+
310314
- template: jobs/default-build.yml
311315
parameters:
312316
condition: ne(variables['SkipTests'], 'true')
@@ -318,6 +322,10 @@ jobs:
318322
beforeBuild:
319323
- bash: "./eng/scripts/install-nginx-mac.sh"
320324
displayName: Installing Nginx
325+
afterBuild:
326+
- bash: ./build.sh --no-build --ci --test -p:RunFlakyTests=true
327+
displayName: Run Flaky Tests
328+
continueOnError: true
321329
artifacts:
322330
- name: MacOS_Test_Logs
323331
path: artifacts/logs/
@@ -333,6 +341,10 @@ jobs:
333341
beforeBuild:
334342
- bash: "./eng/scripts/install-nginx-linux.sh"
335343
displayName: Installing Nginx
344+
afterBuild:
345+
- bash: ./build.sh --no-build --ci --test -p:RunFlakyTests=true
346+
displayName: Run Flaky Tests
347+
continueOnError: true
336348
artifacts:
337349
- name: Linux_Test_Logs
338350
path: artifacts/logs/

‎.azure/pipelines/jobs/default-build.yml‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -174,6 +174,8 @@ jobs:
174174
testRunner: vstest
175175
testResultsFiles: '**/artifacts/**/*.trx'
176176
mergeTestResults: true
177+
buildConfiguration: $(BuildConfiguration)
178+
buildPlatform: $(AgentOsName)
177179
- ${{ each artifact in parameters.artifacts }}:
178180
- task: PublishBuildArtifacts@1
179181
displayName: Upload artifacts from ${{ artifact.path }}

‎eng/FlakyTests.AfterArcade.props‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
<Project>
2+
<!-- Override where xUnit logs and results go if we're doing the flaky run -->
3+
<PropertyGroup Condition="'$(RunFlakyTests)' == 'true'">
4+
<ArtifactsLogDir>$(ArtifactsDir)log\$(Configuration)\Flaky\</ArtifactsLogDir>
5+
<ArtifactsTestResultsDir>$(ArtifactsDir)TestResults\$(Configuration)\Flaky\</ArtifactsTestResultsDir>
6+
</PropertyGroup>
7+
</Project>

‎eng/FlakyTests.BeforeArcade.props‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
<Project>
2+
<!-- Local Dev Flakiness -->
3+
<PropertyGroup>
4+
<_FlakyRunAdditionalArgs>-trait "Flaky:All=true"</_FlakyRunAdditionalArgs>
5+
<_NonFlakyRunAdditionalArgs>-notrait "Flaky:All=true"</_NonFlakyRunAdditionalArgs>
6+
</PropertyGroup>
7+
8+
<!-- Azure Pipelines Flakiness -->
9+
<PropertyGroup Condition="'$(AGENT_OS)' != ''">
10+
<_FlakyRunAdditionalArgs>$(_FlakyRunAdditionalArgs) -trait "Flaky:AzP:All=true" -trait "Flaky:AzP:OS:$(AGENT_OS)=true"</_FlakyRunAdditionalArgs>
11+
<_NonFlakyRunAdditionalArgs>$(_NonFlakyRunAdditionalArgs) -notrait "Flaky:AzP:All=true" -notrait "Flaky:AzP:OS:$(AGENT_OS)=true"</_NonFlakyRunAdditionalArgs>
12+
</PropertyGroup>
13+
14+
<PropertyGroup>
15+
<TestRunnerAdditionalArguments Condition="'$(RunFlakyTests)' == ''">$(_NonFlakyRunAdditionalArgs)</TestRunnerAdditionalArguments>
16+
<TestRunnerAdditionalArguments Condition="'$(RunFlakyTests)' == 'true'">$(_FlakyRunAdditionalArgs)</TestRunnerAdditionalArguments>
17+
</PropertyGroup>
18+
</Project>

‎eng/helix/vstest/runtests.cmd‎

Lines changed: 30 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,6 @@
1+
REM Disable "!Foo!" expansions because they break the filter syntax
2+
setlocal disableextensions
3+
14
set target=%1
25
set sdkVersion=%2
36
set runtimeVersion=%3
@@ -18,12 +21,35 @@ set HELIX=%helixQueue%
1821

1922
%DOTNET_ROOT%\dotnet vstest %target% -lt >discovered.txt
2023
find /c "Exception thrown" discovered.txt
21-
if %errorlevel% equ 0 (
22-
echo Exception thrown during test discovery.
23-
type discovered.txt
24+
REM "ERRORLEVEL is not %ERRORLEVEL%" https://blogs.msdn.microsoft.com/oldnewthing/20080926-00/?p=20743/
25+
if not errorlevel 1 (
26+
echo Exception thrown during test discovery. 1>&2
27+
type discovered.txt 1>&2
2428
exit 1
2529
)
2630

27-
%DOTNET_ROOT%\dotnet vstest %target% --logger:trx --logger:console;verbosity=normal
31+
set exit_code=0
32+
33+
REM Run non-flaky tests first
34+
REM We need to specify all possible Flaky filters that apply to this environment, because the flaky attribute
35+
REM only puts the explicit filter traits the user provided in
36+
REM Filter syntax: https://github.com/Microsoft/vstest-docs/blob/master/docs/filter.md
37+
set NONFLAKY_FILTER="Flaky:All!=true&Flaky:Helix:All!=true&Flaky:Helix:Queue:All!=true&Flaky:Helix:Queue:%HELIX%!=true"
38+
echo Running non-flaky tests.
39+
%DOTNET_ROOT%\dotnet vstest %target% --logger:trx --logger:console;verbosity=normal --TestCaseFilter:%NONFLAKY_FILTER%
40+
if errorlevel 1 (
41+
echo Failure in non-flaky test 1>&2
42+
set exit_code=1
43+
REM DO NOT EXIT
44+
)
45+
46+
set FLAKY_FILTER="Flaky:All=true|Flaky:Helix:All=true|Flaky:Helix:Queue:All=true|Flaky:Helix:Queue:%HELIX%=true"
47+
echo Running known-flaky tests.
48+
%DOTNET_ROOT%\dotnet vstest %target% --logger:trx --logger:console;verbosity=normal --TestCaseFilter:%FLAKY_FILTER%
49+
if errorlevel 1 (
50+
echo Failure in flaky test 1>&2
51+
REM DO NOT EXIT and DO NOT SET EXIT_CODE to 1
52+
)
2853

54+
exit %exit_code%
2955

‎eng/helix/vstest/runtests.sh‎

Lines changed: 22 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -67,4 +67,25 @@ if grep -q "Exception thrown" discovered.txt; then
6767
exit 1
6868
fi
6969

70-
$DOTNET_ROOT/dotnet vstest $1 --logger:trx
70+
# Run non-flaky tests first
71+
# We need to specify all possible Flaky filters that apply to this environment, because the flaky attribute
72+
# only puts the explicit filter traits the user provided in the flaky attribute
73+
# Filter syntax: https://github.com/Microsoft/vstest-docs/blob/master/docs/filter.md
74+
NONFLAKY_FILTER="Flaky:All!=true&Flaky:Helix:All!=true&Flaky:Helix:Queue:All!=true&Flaky:Helix:Queue:$HELIX!=true"
75+
echo "Running non-flaky tests."
76+
$DOTNET_ROOT/dotnet vstest $1 --logger:trx --TestCaseFilter:"$NONFLAKY_FILTER"
77+
nonflaky_exitcode=$?
78+
if [ $nonflaky_exitcode != 0 ]; then
79+
echo "Non-flaky tests failed!" 1>&2
80+
# DO NOT EXIT
81+
fi
82+
83+
FLAKY_FILTER="Flaky:All=true|Flaky:Helix:All=true|Flaky:Helix:Queue:All=true|Flaky:Helix:Queue:$HELIX=true"
84+
echo "Running known-flaky tests."
85+
$DOTNET_ROOT/dotnet vstest $1 --logger:trx --TestCaseFilter:"$FLAKY_FILTER"
86+
if [ $? != 0 ]; then
87+
echo "Flaky tests failed!" 1>&2
88+
# DO NOT EXIT
89+
fi
90+
91+
exit $nonflaky_exitcode

‎korebuild-lock.txt‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,2 +1,2 @@
1-
version:3.0.0-build-20190306.2
2-
commithash:18c06e0b774622c87560e6f21b97e099307fd10c
1+
version:3.0.0-build-20190314.2
2+
commithash:e3a8a2aae198f1ef26309714ccba6835be2437c3

0 commit comments

Comments
 (0)