Skip to content

Commit af7f7f5

Browse files
pjcollinspjcollins
authored
[ci] Add test job for .NET APK instrumentation (#5862)
The Mono.Android.NET_Tests suite has been moved to a new smoke test job. Additional test configurations have been added for the interpreter and the .aab package format. Timing data for these tests has been moved to a new .csv file that will be processed separately from the non .NET 6 Mono.Android_Tests variants. Additionally, .apk / .aab artifact uploading will now only occur when a previous test step has failed. The xaprepare `Required` scenario has been removed from the apk test jobs as they should only rely on preparing Java.Interop, which can be done separately.
1 parent a916ee0 commit af7f7f5

File tree

9 files changed

+239
-120
lines changed

9 files changed

+239
-120
lines changed

‎build-tools/automation/azure-pipelines.yaml

Lines changed: 130 additions & 47 deletions
Large diffs are not rendered by default.

‎build-tools/automation/yaml-templates/apk-instrumentation.yaml

Lines changed: 28 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -6,31 +6,43 @@ parameters:
66
extraBuildArgs: ""
77
testResultsFormat: NUnit
88
packageType: Apk
9-
artifactName: ""
9+
artifactSource: ""
1010
artifactFolder: ""
11+
useDotNet: false
1112
condition: succeeded()
1213

1314
steps:
14-
- task: MSBuild@1
15-
displayName: run ${{ parameters.testName }}
16-
inputs:
17-
solution: ${{ parameters.project }}
18-
configuration: ${{ parameters.configuration }}
19-
msbuildArguments: >-
20-
/restore
21-
/t:AcquireAndroidTarget,SignAndroidPackage,DeployTest${{ parameters.packageType }}s,CheckAndRecordApkSizes,RunTestApks,UndeployTestApks,RenameApkTestCases,ReportComponentFailures
22-
/bl:$(System.DefaultWorkingDirectory)/bin/Test${{ parameters.configuration }}/run-${{ parameters.testName }}.binlog
23-
${{ parameters.extraBuildArgs }}
24-
condition: ${{ parameters.condition }}
25-
continueOnError: true
15+
- ${{ if eq(parameters.useDotNet, false) }}:
16+
- task: MSBuild@1
17+
displayName: run ${{ parameters.testName }}
18+
inputs:
19+
solution: ${{ parameters.project }}
20+
configuration: ${{ parameters.configuration }}
21+
msbuildArguments: >-
22+
/restore
23+
/t:AcquireAndroidTarget,SignAndroidPackage,DeployTest${{ parameters.packageType }}s,CheckAndRecordApkSizes,RunTestApks,UndeployTestApks,RenameApkTestCases,ReportComponentFailures
24+
/bl:$(System.DefaultWorkingDirectory)/bin/Test${{ parameters.configuration }}/run-${{ parameters.testName }}.binlog
25+
${{ parameters.extraBuildArgs }}
26+
condition: ${{ parameters.condition }}
27+
continueOnError: true
28+
29+
- ${{ if eq(parameters.useDotNet, true) }}:
30+
- template: run-dotnet-preview.yaml
31+
parameters:
32+
displayName: run ${{ parameters.testName }}
33+
project: ${{ parameters.project }}
34+
arguments: >-
35+
-t:AcquireAndroidTarget,SignAndroidPackage,DeployTest${{ parameters.packageType }}s,CheckAndRecordApkSizes,RunTestApks,UndeployTestApks,RenameApkTestCases,ReportComponentFailures
36+
-bl:$(System.DefaultWorkingDirectory)/bin/Test${{ parameters.configuration }}/run-${{ parameters.testName }}.binlog
37+
-v:n -c ${{ parameters.configuration }} ${{ parameters.extraBuildArgs }}
38+
condition: ${{ parameters.condition }}
2639

2740
- script: >
28-
SOURCE="$(System.DefaultWorkingDirectory)/bin/Test${{ parameters.configuration }}/${{ parameters.artifactName }}" &&
2941
DEST="$(Build.ArtifactStagingDirectory)/${{ parameters.artifactFolder }}/" &&
3042
mkdir -p "$DEST" &&
31-
cp "$SOURCE" "$DEST"
43+
cp "${{ parameters.artifactSource }}" "$DEST"
3244
displayName: copy apk/aab
33-
condition: ${{ parameters.condition }}
45+
condition: eq(variables['agent.jobstatus'], 'SucceededWithIssues')
3446
continueOnError: true
3547

3648
- task: PublishTestResults@2

‎build-tools/automation/yaml-templates/run-dotnet-preview.yaml

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
parameters:
2+
command: build
3+
project: ''
4+
arguments: ''
5+
useExitCodeForErrors: false
6+
displayName: run dotnet preview
7+
condition: succeeded()
8+
continueOnError: true
9+
10+
steps:
11+
- powershell: |
12+
if ([Environment]::OSVersion.Platform -eq "Unix") {
13+
$dotnetPath = "$env:HOME/Library/Android/dotnet/dotnet"
14+
if (-not (Test-Path $dotnetPath)) {
15+
$dotnetPath = "$env:HOME/android-toolchain/dotnet/dotnet"
16+
}
17+
} else {
18+
$dotnetPath = "$env:USERPROFILE\android-toolchain\dotnet\dotnet.exe"
19+
}
20+
& $dotnetPath ${{ parameters.command }} ${{ parameters.project }} ${{ parameters.arguments }}
21+
if ([System.Convert]::ToBoolean("${{ parameters.useExitCodeForErrors }}") -and $LASTEXITCODE -ne 0) {
22+
Write-Host "##vso[task.logissue type=error]Test suite had $LASTEXITCODE failure(s)."
23+
Write-Host "##vso[task.complete result=Failed;]"
24+
exit 0
25+
}
26+
displayName: ${{ parameters.displayName }}
27+
condition: ${{ parameters.condition }}
28+
continueOnError: ${{ parameters.continueOnError }}

‎build-tools/automation/yaml-templates/run-msbuild-device-tests.yaml

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -16,8 +16,6 @@ jobs:
1616
cancelTimeoutInMinutes: 5
1717
workspace:
1818
clean: all
19-
variables:
20-
UseDotNet: ${{ eq(parameters.target_framework, 'netcoreapp3.1') }}
2119
steps:
2220
- template: setup-test-environment.yaml
2321
parameters:
@@ -57,7 +55,7 @@ jobs:
5755

5856
- template: run-nunit-tests.yaml
5957
parameters:
60-
useDotNet: $(UseDotNet)
58+
useDotNet: ${{ eq(parameters.target_framework, 'netcoreapp3.1') }}
6159
testRunTitle: MSBuildDeviceIntegration On Device - macOS - ${{ parameters.job_suffix }}
6260
testAssembly: $(System.DefaultWorkingDirectory)/bin/Test$(XA.Build.Configuration)/MSBuildDeviceIntegration/${{ parameters.target_framework }}/MSBuildDeviceIntegration.dll
6361
nunitConsoleExtraArgs: --where "cat != SystemApplication && cat != TimeZoneInfo && cat != SmokeTests ${{ parameters.nunit_categories }}"

‎build-tools/automation/yaml-templates/run-msbuild-mac-tests.yaml

Lines changed: 3 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -18,8 +18,6 @@ jobs:
1818
cancelTimeoutInMinutes: 5
1919
workspace:
2020
clean: all
21-
variables:
22-
UseDotNet: ${{ eq(parameters.target_framework, 'netcoreapp3.1') }}
2321
steps:
2422
- template: setup-test-environment.yaml
2523
parameters:
@@ -38,7 +36,7 @@ jobs:
3836

3937
- template: run-nunit-tests.yaml
4038
parameters:
41-
useDotNet: $(UseDotNet)
39+
useDotNet: ${{ eq(parameters.target_framework, 'netcoreapp3.1') }}
4240
testRunTitle: Xamarin.Android.Build.Tests - macOS-${{ parameters.node_id }} - ${{ parameters.job_suffix }}
4341
testAssembly: $(System.DefaultWorkingDirectory)/bin/Test$(XA.Build.Configuration)/${{ parameters.target_framework }}/Xamarin.Android.Build.Tests.dll
4442
nunitConsoleExtraArgs: --where "cat == Node-${{ parameters.node_id }} ${{ parameters.nunit_categories }}"
@@ -49,7 +47,7 @@ jobs:
4947
- ${{ if eq(parameters.run_extra_tests, true) }}:
5048
- template: run-nunit-tests.yaml
5149
parameters:
52-
useDotNet: $(UseDotNet)
50+
useDotNet: ${{ eq(parameters.target_framework, 'netcoreapp3.1') }}
5351
testRunTitle: Xamarin.Android.Build.Tests - macOS - No Node
5452
testAssembly: $(System.DefaultWorkingDirectory)/bin/Test$(XA.Build.Configuration)/${{ parameters.target_framework }}/Xamarin.Android.Build.Tests.dll
5553
nunitConsoleExtraArgs: --where "cat != Node-1 && cat != Node-2 && cat != Node-3"
@@ -60,7 +58,7 @@ jobs:
6058
- ${{ if eq(parameters.run_extra_tests, true) }}:
6159
- template: run-nunit-tests.yaml
6260
parameters:
63-
useDotNet: $(UseDotNet)
61+
useDotNet: ${{ eq(parameters.target_framework, 'netcoreapp3.1') }}
6462
testRunTitle: Xamarin.Android.Tools.Aidl-Tests - macOS
6563
testAssembly: $(System.DefaultWorkingDirectory)/bin/Test$(XA.Build.Configuration)/${{ parameters.target_framework }}/Xamarin.Android.Tools.Aidl-Tests.dll
6664
testResultsFile: TestResult-Aidl-Tests-macOS-$(XA.Build.Configuration).xml

‎build-tools/automation/yaml-templates/run-msbuild-win-tests.yaml

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,6 @@ jobs:
1717
cancelTimeoutInMinutes: 5
1818
variables:
1919
VSINSTALLDIR: C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise
20-
UseDotNet: ${{ eq(parameters.target_framework, 'netcoreapp3.1') }}
2120
steps:
2221
- script: netsh int ipv4 set global sourceroutingbehavior=drop
2322

@@ -45,7 +44,7 @@ jobs:
4544
# Using all available cores seems to occasionally bog down our machines and cause parallel test execution to slow down dramatically.
4645
- template: run-nunit-tests.yaml
4746
parameters:
48-
useDotNet: $(UseDotNet)
47+
useDotNet: ${{ eq(parameters.target_framework, 'netcoreapp3.1') }}
4948
testRunTitle: Xamarin.Android.Build.Tests - Windows-${{ parameters.node_id }} - ${{ parameters.job_suffix }}
5049
testAssembly: $(System.DefaultWorkingDirectory)\bin\Test$(XA.Build.Configuration)\${{ parameters.target_framework }}\Xamarin.Android.Build.Tests.dll
5150
nunitConsoleExtraArgs: --where "cat == Node-${{ parameters.node_id }} ${{ parameters.nunit_categories }}"
@@ -56,7 +55,7 @@ jobs:
5655
- ${{ if eq(parameters.run_extra_tests, true) }}:
5756
- template: run-nunit-tests.yaml
5857
parameters:
59-
useDotNet: $(UseDotNet)
58+
useDotNet: ${{ eq(parameters.target_framework, 'netcoreapp3.1') }}
6059
testRunTitle: Xamarin.Android.Build.Tests - Windows - No Node
6160
testAssembly: $(System.DefaultWorkingDirectory)\bin\Test$(XA.Build.Configuration)\${{ parameters.target_framework }}\Xamarin.Android.Build.Tests.dll
6261
nunitConsoleExtraArgs: --where "cat != Node-1 && cat != Node-2 && cat != Node-3"
@@ -67,7 +66,7 @@ jobs:
6766
- ${{ if eq(parameters.run_extra_tests, true) }}:
6867
- template: run-nunit-tests.yaml
6968
parameters:
70-
useDotNet: $(UseDotNet)
69+
useDotNet: ${{ eq(parameters.target_framework, 'netcoreapp3.1') }}
7170
testRunTitle: Xamarin.Android.Tools.Aidl-Tests - Windows
7271
testAssembly: $(System.DefaultWorkingDirectory)\bin\Test$(XA.Build.Configuration)\${{ parameters.target_framework }}\Xamarin.Android.Tools.Aidl-Tests.dll
7372
testResultsFile: TestResult-Aidl-Tests-Windows-$(XA.Build.Configuration).xml

‎build-tools/automation/yaml-templates/run-nunit-tests.yaml

Lines changed: 30 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
parameters:
22
nunitConsole: $(System.DefaultWorkingDirectory)/build-tools/scripts/nunit3-console
33
testRunTitle: Xamarin Android Tests
4-
testAssembly: []
4+
testAssembly: ''
55
testResultsFile: TestResult.xml
66
nunitConsoleExtraArgs: ''
77
dotNetTestExtraArgs: ''
@@ -10,36 +10,35 @@ parameters:
1010
condition: succeeded()
1111

1212
steps:
13-
- powershell: |
14-
if ([Environment]::OSVersion.Platform -eq "Unix") {
15-
if ("${{ parameters.useDotNet }}" -eq $true) {
16-
Write-Host '##vso[task.setvariable variable=TestResultsFormat]VSTest'
17-
$dotnetPath = "$env:HOME/Library/Android/dotnet/dotnet"
18-
if (-not (Test-Path $dotnetPath)) {
19-
$dotnetPath = "$env:HOME/android-toolchain/dotnet/dotnet"
20-
}
21-
& $dotnetPath test ${{ parameters.testAssembly }} --results-directory . --logger "trx;LogFileName=${{ parameters.testResultsFile }}" ${{ parameters.dotNetTestExtraArgs }} -- NUnit.NumberOfTestWorkers=${{ parameters.workers }}
22-
} else {
23-
Write-Host '##vso[task.setvariable variable=TestResultsFormat]NUnit'
24-
& ${{ parameters.nunitConsole }} ${{ parameters.testAssembly }} --result ${{ parameters.testResultsFile }} --workers=${{ parameters.workers }} ${{ parameters.nunitConsoleExtraArgs }}
25-
}
26-
} else {
27-
if ("${{ parameters.useDotNet }}" -eq $true) {
28-
Write-Host '##vso[task.setvariable variable=TestResultsFormat]VSTest'
29-
& $env:USERPROFILE\android-toolchain\dotnet\dotnet.exe test ${{ parameters.testAssembly }} --results-directory . --logger "trx;LogFileName=${{ parameters.testResultsFile }}" ${{ parameters.dotNetTestExtraArgs }} -- NUnit.NumberOfTestWorkers=${{ parameters.workers }}
30-
} else {
31-
Write-Host '##vso[task.setvariable variable=TestResultsFormat]NUnit'
32-
& cmd /c '${{ parameters.nunitConsole }}.cmd' ${{ parameters.testAssembly }} --result ${{ parameters.testResultsFile }} --workers=${{ parameters.workers }} ${{ parameters.nunitConsoleExtraArgs }}
33-
}
34-
}
35-
if ($LASTEXITCODE -ne 0) {
36-
Write-Host "##vso[task.logissue type=error]Test suite had $LASTEXITCODE failure(s)."
37-
Write-Host "##vso[task.complete result=Failed;]"
38-
exit 0
39-
}
40-
displayName: run ${{ parameters.testRunTitle }}
41-
condition: ${{ parameters.condition }}
42-
continueOnError: true
13+
- ${{ if eq(parameters.useDotNet, false) }}:
14+
- powershell: |
15+
Write-Host '##vso[task.setvariable variable=TestResultsFormat]NUnit'
16+
if ([Environment]::OSVersion.Platform -eq "Unix") {
17+
& ${{ parameters.nunitConsole }} ${{ parameters.testAssembly }} --result ${{ parameters.testResultsFile }} --workers=${{ parameters.workers }} ${{ parameters.nunitConsoleExtraArgs }}
18+
} else {
19+
& cmd /c '${{ parameters.nunitConsole }}.cmd' ${{ parameters.testAssembly }} --result ${{ parameters.testResultsFile }} --workers=${{ parameters.workers }} ${{ parameters.nunitConsoleExtraArgs }}
20+
}
21+
if ($LASTEXITCODE -ne 0) {
22+
Write-Host "##vso[task.logissue type=error]Test suite had $LASTEXITCODE failure(s)."
23+
Write-Host "##vso[task.complete result=Failed;]"
24+
exit 0
25+
}
26+
displayName: run ${{ parameters.testRunTitle }}
27+
condition: ${{ parameters.condition }}
28+
continueOnError: true
29+
30+
- ${{ if eq(parameters.useDotNet, true) }}:
31+
- powershell: Write-Host '##vso[task.setvariable variable=TestResultsFormat]VSTest'
32+
- template: run-dotnet-preview.yaml
33+
parameters:
34+
command: test
35+
project: ${{ parameters.testAssembly }}
36+
useExitCodeForErrors: true
37+
arguments: >-
38+
--results-directory . --logger "trx;LogFileName=${{ parameters.testResultsFile }}"
39+
${{ parameters.dotNetTestExtraArgs }} -- NUnit.NumberOfTestWorkers=${{ parameters.workers }}
40+
displayName: run ${{ parameters.testRunTitle }}
41+
condition: ${{ parameters.condition }}
4342

4443
- template: kill-processes.yaml
4544

‎tests/Mono.Android-Tests/Directory.Build.targets

Lines changed: 0 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -59,14 +59,4 @@
5959
/>
6060
</Target>
6161

62-
<Target Name="RunDotNetInstrumentation"
63-
DependsOnTargets="GenerateNuGetConfig" >
64-
<ItemGroup>
65-
<_RuntimeTestArgs Include="-p:Configuration=$(Configuration)" />
66-
<_RuntimeTestArgs Include="-t:AcquireAndroidTarget,SignAndroidPackage,DeployTestApks,RunTestApks,UndeployTestApks,RenameApkTestCases,ReportComponentFailures" />
67-
<_RuntimeTestArgs Include="-bl:$(XamarinAndroidSourcePath)bin\Test$(Configuration)\run-Mono.Android.NET-Tests.binlog" />
68-
</ItemGroup>
69-
<Exec Command="$(DotNetPreviewTool) build /v:n @(_RuntimeTestArgs, ' ') &quot;$(MSBuildThisFileDirectory)Runtime-Microsoft.Android.Sdk\Mono.Android.NET-Tests.csproj&quot;" />
70-
</Target>
71-
7262
</Project>

‎tests/Mono.Android-Tests/Runtime-Microsoft.Android.Sdk/Mono.Android.NET-Tests.csproj

Lines changed: 16 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@
1818
<EnableDefaultAndroidResourceItems>false</EnableDefaultAndroidResourceItems>
1919
<EnableDefaultAndroidAssetItems>false</EnableDefaultAndroidAssetItems>
2020
<_MonoAndroidTestPackage>Mono.Android.NET_Tests</_MonoAndroidTestPackage>
21-
<PlotDataLabelSuffix>-NET6</PlotDataLabelSuffix>
21+
<PlotDataLabelSuffix>-$(TestsFlavor)NET6</PlotDataLabelSuffix>
2222
<!-- TODO: Fix excluded tests -->
2323
<ExcludeCategories>DotNetIgnore</ExcludeCategories>
2424
</PropertyGroup>
@@ -34,20 +34,32 @@
3434
<Compile Remove="..\Mono.Data.Sqlite\SqliteTests.cs" />
3535
</ItemGroup>
3636

37-
<ItemGroup>
37+
<ItemGroup Condition=" '$(AndroidPackageFormat)' != 'aab' ">
3838
<TestApk Include="$(OutputPath)$(_MonoAndroidTestPackage)-Signed.apk">
3939
<Package>$(_MonoAndroidTestPackage)</Package>
4040
<InstrumentationType>xamarin.android.runtimetests.NUnitInstrumentation</InstrumentationType>
4141
<ResultsPath>$(XamarinAndroidSourcePath)TestResult-$(_MonoAndroidTestPackage).xml</ResultsPath>
4242
<TimingDefinitionsFilename>$(XamarinAndroidSourcePath)build-tools\scripts\TimingDefinitions.txt</TimingDefinitionsFilename>
43-
<!-- Use default package name to write all Mono.Android-Tests* timing data to the same .csv -->
44-
<TimingResultsFilename>$(XamarinAndroidSourcePath)TestResult-Mono.Android_Tests-times.csv</TimingResultsFilename>
43+
<TimingResultsFilename>$(XamarinAndroidSourcePath)TestResult-$(_MonoAndroidTestPackage)-times.csv</TimingResultsFilename>
4544
<ApkSizesInputFilename>apk-sizes-$(_MonoAndroidTestPackage)-$(Configuration)$(TestsFlavor).txt</ApkSizesInputFilename>
4645
<ApkSizesDefinitionFilename>$(XamarinAndroidSourcePath)build-tools\scripts\ApkSizesDefinitions.txt</ApkSizesDefinitionFilename>
4746
<ApkSizesResultsFilename>$(XamarinAndroidSourcePath)TestResult-$(_MonoAndroidTestPackage)-values-$(Configuration).csv</ApkSizesResultsFilename>
4847
</TestApk>
4948
</ItemGroup>
5049

50+
<ItemGroup Condition=" '$(AndroidPackageFormat)' == 'aab' ">
51+
<TestAab Include="$(OutputPath)$(_MonoAndroidTestPackage)-Signed.aab">
52+
<Package>$(_MonoAndroidTestPackage)</Package>
53+
<InstrumentationType>xamarin.android.runtimetests.NUnitInstrumentation</InstrumentationType>
54+
<ResultsPath>$(XamarinAndroidSourcePath)TestResult-$(_MonoAndroidTestPackage).xml</ResultsPath>
55+
<TimingDefinitionsFilename>$(XamarinAndroidSourcePath)build-tools\scripts\TimingDefinitions.txt</TimingDefinitionsFilename>
56+
<TimingResultsFilename>$(XamarinAndroidSourcePath)TestResult-$(_MonoAndroidTestPackage)-times.csv</TimingResultsFilename>
57+
<ApkSizesInputFilename>apk-sizes-$(_MonoAndroidTestPackage)-$(Configuration)$(TestsFlavor).txt</ApkSizesInputFilename>
58+
<ApkSizesDefinitionFilename>$(XamarinAndroidSourcePath)build-tools\scripts\ApkSizesDefinitions.txt</ApkSizesDefinitionFilename>
59+
<ApkSizesResultsFilename>$(XamarinAndroidSourcePath)TestResult-$(_MonoAndroidTestPackage)-values-$(Configuration).csv</ApkSizesResultsFilename>
60+
</TestAab>
61+
</ItemGroup>
62+
5163
<Import Project="$(XamarinAndroidSourcePath)build-tools\scripts\TestApks.targets" />
5264

5365
</Project>

0 commit comments

Comments
 (0)