[tests] Add .apk test job to azure-pipelines.yaml (#3042)

Adds a `Test` stage to `build-tools/automation/azure-pipelines.yaml`.
At present, this stage only contains one test job, which builds and
runs the following emulator tests:

  * Mono.Android_Tests
  * Mono.Android-Tests (AOT + LLVM)
  * Mono.Android-Tests (Bundle)
  * Mono.Android-Tests (Profiled AOT)
  * Mono.Android_TestsAppBundle
  * Mono.Android_TestsMultiDex
  * Xamarin.Android.JcwGen_Tests
  * Xamarin.Android.Locale_Tests
  * Xamarin.Android.Locale-Tests (AOT)
  * Xamarin.Android.Locale-Tests (Profiled AOT)
  * Xamarin.Android.EmbeddedDSO_Test
  * Xamarin.Forms.Performance.Integration.Droid 
  * Xamarin.Forms.Performance.Integration.Droid (AOT)
  * Xamarin.Forms.Performance.Integration.Droid (Profiled AOT)
  * Xamarin.Forms.Performance.Integration.Droid (Bundle)

Note in particular that the BCL tests are currently absent.
Execution of the BCL Test suite will also need to be added to the
`Test` stage in the future.

Other Fixes:

  * Introductory support has been added for running `.apk` tests
    against a *system* Xamarin.Android installation.

    We can now build and run `Xamarin.Android.JcwGen_Tests`,
    `Mono.Android_Tests`, and
    `Xamarin.Forms.Performance.Integration.Droid` tests against a
    system installation of Xamarin.Android.

  * Produce an error when any test failures occur during
    instrumentation.

    The instrumentation exit code value of `-1` indicates success,
    and any other values returned here should be reported as errors.
    A non `-1` value indicates that crash or test failure occurred.

  * Produce an error if `adb logcat` timing data can't be parsed.

    We'll now error out when parsing `adb logcat` for timing data if
    the running application died, or if we fail to detect application
    startup.

  * Don't always report `adb logcat` stderr as a warning

    `adb` invocations are prone to returning non zero exit codes even
    when they are successful.  This can make our MSBuild output a bit
    noisy [^0] when they are invoked using the `<Adb/>` task.
    The new optional `Adb.WriteOutputAsMessage` property can now be
    provided to the adb task, so that we don't always write messages
    coming from stderr as MSBuild warnings.  This is used in our
    targets which wait for emulator startup to make it easier to
    detect real failures in test instrumentation and emulator startup.

[^0]: Example warnings:

      …/xamarin-android/build-tools/scripts/TestApks.targets(68,5): warning : adb D 04-29 15:10:48  3527 42649 adb_trace.cpp:192] Android Debug Bridge version 1.0.40
      …/xamarin-android/build-tools/scripts/TestApks.targets(68,5): warning : adb D 04-29 15:10:48  3527 42649 adb_trace.cpp:192] Version 28.0.2-5303910
      …/xamarin-android/build-tools/scripts/TestApks.targets(68,5): warning : adb D 04-29 15:10:48  3527 42649 adb_trace.cpp:192] Installed as /Users/peter/android-toolchain/sdk/platform-tools/adb
      …/xamarin-android/build-tools/scripts/TestApks.targets(68,5): warning : adb D 04-29 15:10:48  3527 42649 adb_trace.cpp:192] 
      …/xamarin-android/build-tools/scripts/TestApks.targets(68,5): warning : adb D 04-29 15:10:48  3527 42649 adb_client.cpp:287] adb_query: host-serial:emulator-5570:features
      …/xamarin-android/build-tools/scripts/TestApks.targets(68,5): warning : adb D 04-29 15:10:48  3527 42649 adb_client.cpp:137] _adb_connect: host:version
      …/xamarin-android/build-tools/scripts/TestApks.targets(68,5): warning : adb D 04-29 15:10:48  3527 42649 adb_io.cpp:107] writex: fd=3 len=16 30303063686f73743a76657273696f6e 000chost:version
      …/xamarin-android/build-tools/scripts/TestApks.targets(68,5): warning : adb D 04-29 15:10:48  3527 42649 adb_io.cpp:81] readx: fd=3 wanted=4
      …/xamarin-android/build-tools/scripts/TestApks.targets(68,5): warning : adb D 04-29 15:10:48  3527 42649 adb_io.cpp:97] readx: fd=3 wanted=4 got=4 4f4b4159 OKAY
      …/xamarin-android/build-tools/scripts/TestApks.targets(68,5): warning : adb D 04-29 15:10:48  3527 42649 adb_client.cpp:165] _adb_connect: return fd 3
      …/xamarin-android/build-tools/scripts/TestApks.targets(68,5): warning : adb D 04-29 15:10:48  3527 42649 adb_client.cpp:197] adb_connect: service host-serial:emulator-5570:features
      …/xamarin-android/build-tools/scripts/TestApks.targets(68,5): warning : adb D 04-29 15:10:48  3527 42649 adb_io.cpp:81] readx: fd=3 wanted=4

Author: pjcollins

build-tools/Xamarin.Android.Tools.BootstrapTasks/Xamarin.Android.Tools.BootstrapTasks/Adb.cs

@@ -43,6 +43,7 @@ protected class CommandInfo
 		public                  bool            IgnoreExitCode        { get; set; }
 		public                  int             Timeout               { get; set; } = -1;
 		public                  string[]        EnvironmentVariables  { get; set; }
+		public                  bool            WriteOutputAsMessage  { get; set; } = false;
 
 		[Required]
 		public                  string          ToolPath              { get; set; }
@@ -147,7 +148,7 @@ void OnOutput (string line, bool isStdout, CommandInfo info)
 			lock (linesLock) lines.Add (line);
 
 			if (!info.SuppressMSbuildLog) {
-				if (isStdout)
+				if (isStdout || WriteOutputAsMessage)
 					Log.LogMessage (MessageImportance.Low, line);
 				else
 					Log.LogWarning (line);

build-tools/Xamarin.Android.Tools.BootstrapTasks/Xamarin.Android.Tools.BootstrapTasks/RenameTestCases.cs

@@ -74,6 +74,7 @@ string GetFixedUpPath (string source, string testNameSuffix)
 		{
 			var destFilename = Path.GetFileNameWithoutExtension (source) +
 				(string.IsNullOrWhiteSpace (Configuration) ? "" : "-" + Configuration) +
+				(string.IsNullOrWhiteSpace (TestsFlavor) ? "" : TestsFlavor) +
 				Path.GetExtension (source);
 			var dest = Path.Combine (DestinationFolder, destFilename);
 			return dest;

build-tools/Xamarin.Android.Tools.BootstrapTasks/Xamarin.Android.Tools.BootstrapTasks/RunInstrumentationTests.cs

@@ -14,6 +14,7 @@ public class RunInstrumentationTests : Adb
 		const                   string              TestResultsPathResult       = "INSTRUMENTATION_RESULT: nunit2-results-path=";
 		internal const          string              AdbRestartText              = "daemon not running; starting now at tcp:";
 		internal const          string              AdbCrashErrorText           = "The adb might have crashed and was restarted. ";
+		const                   string              InstrumentationExitCodeName = "INSTRUMENTATION_CODE: ";
 		const                   int                 StateRunInstrumentation     = 0;
 		const                   int                 StateGetLogcat              = 1;
 		const                   int                 StateClearLogcat            = 2;
@@ -41,6 +42,7 @@ public class RunInstrumentationTests : Adb
 		public                  string              LogLevel                    { get; set; }
 
 		int                     currentState = -1;
+		int                     instrumentationExitCode = 99;
 		string                  targetTestResultsPath;
 
 		bool                    adbRestarted;
@@ -68,6 +70,14 @@ public override bool Execute ()
 				return false;
 			}
 
+			if (instrumentationExitCode != -1) {
+				FailedToRun = Component;
+				Log.LogError (
+					$"Instrumentation for component `{Component}` did not exit successfully. " +
+					"Process crashed or test failures occurred!");
+				return false;
+			}
+
 			return !Log.HasLoggedErrors;
 		}
 
@@ -134,11 +144,15 @@ protected override void ProcessStdout (string line)
 			if (currentState != StateRunInstrumentation || String.IsNullOrEmpty (line))
 				return;
 
-			int i = line.IndexOf (TestResultsPathResult, StringComparison.OrdinalIgnoreCase);
-			if (i < 0)
-				return;
+			int testResultIndex = line.IndexOf (TestResultsPathResult, StringComparison.OrdinalIgnoreCase);
+			int exitCodeIndex = line.IndexOf (InstrumentationExitCodeName, StringComparison.OrdinalIgnoreCase);
 
-			targetTestResultsPath = line.Substring (i + TestResultsPathResult.Length).Trim ();
+			if (testResultIndex < 0 && exitCodeIndex < 0)
+				return;
+			else if (testResultIndex >= 0)
+				targetTestResultsPath = line.Substring (testResultIndex + TestResultsPathResult.Length).Trim ();
+			else if (exitCodeIndex >= 0)
+				instrumentationExitCode = int.Parse (line.Substring (exitCodeIndex + InstrumentationExitCodeName.Length).Trim ());
 		}
 	}
 }

build-tools/Xamarin.Android.Tools.BootstrapTasks/result-packaging.targets

@@ -6,8 +6,7 @@
     <_BuildStatusFiles Include="$(XamarinAndroidSourcePath)Configuration.OperatingSystem.props" Condition="Exists('$(XamarinAndroidSourcePath)Configuration.OperatingSystem.props')" />
     <_BuildStatusFiles Include="$(XamarinAndroidSourcePath)Configuration.Override.props" Condition="Exists('$(XamarinAndroidSourcePath)Configuration.Override.props')" />
     <_BuildStatusFiles Include="$(XamarinAndroidSourcePath)bin\Build$(Configuration)\XABuildConfig.cs" Condition="Exists('$(XamarinAndroidSourcePath)bin\Build$(Configuration)\XABuildConfig.cs')" />
-    <_BuildStatusFiles Include="$(XamarinAndroidSourcePath)bin\Build$(Configuration)\msbuild*.binlog" />
-    <_BuildStatusFiles Include="$(XamarinAndroidSourcePath)bin\Build$(Configuration)\bootstrap*.binlog" />
+    <_BuildStatusFiles Include="$(XamarinAndroidSourcePath)bin\Build$(Configuration)\*.binlog" />
     <_BuildStatusFiles Include="$(XamarinAndroidSourcePath)bin\Build$(Configuration)\prepare*.log" />
     <_BuildStatusFiles Include="$(XamarinAndroidSourcePath)bin\Build$(Configuration)\*.mk" />
     <_BuildStatusFiles Include="$(XamarinAndroidSourcePath)bin\Build$(Configuration)\*.projitems" />
@@ -27,7 +26,7 @@
     <_TestResultFiles Include="$(XamarinAndroidSourcePath)bin\Test$(Configuration)\*.log" />
     <_TestResultFiles Include="$(XamarinAndroidSourcePath)bin\Test$(Configuration)\compatibility\*" />
     <_TestResultFiles Include="$(XamarinAndroidSourcePath)bin\Test$(Configuration)\logcat*" />
-    <_TestResultFiles Include="$(XamarinAndroidSourcePath)bin\Test$(Configuration)\msbuild*.binlog*" />
+    <_TestResultFiles Include="$(XamarinAndroidSourcePath)bin\Test$(Configuration)\*.binlog" />
     <_TestResultFiles Include="$(XamarinAndroidSourcePath)bin\Test$(Configuration)\temp\**\*.binlog" />
     <_TestResultFiles Include="$(XamarinAndroidSourcePath)bin\Test$(Configuration)\temp\**\*.log" />
     <_TestResultFiles Include="$(XamarinAndroidSourcePath)bin\Test$(Configuration)\EmbeddedDSO\EmbeddedDSO.build\**\*" />

build-tools/automation/azure-pipelines.yaml

@@ -9,6 +9,7 @@ trigger:
 # Global variables
 variables:
   BundleArtifactName: bundle
+  InstallerArtifactName: unsigned-installers
   AutoProvisionArgs: /p:AutoProvision=True /p:AutoProvisionUsesSudo=True /p:IgnoreMaxMonoVersion=False
   AndroidTargetAbiArgs: >-
     /p:AndroidSupportedTargetJitAbis=armeabi-v7a:arm64-v8a:x86:x86_64
@@ -122,7 +123,7 @@ stages:
     - task: PublishPipelineArtifact@0
       displayName: upload unsigned installers
       inputs:
-        artifactName: unsigned
+        artifactName: $(InstallerArtifactName)
         targetPath: bin/Build$(XA.Build.Configuration)/unsigned-installers
 
     - task: MSBuild@1
@@ -215,7 +216,6 @@ stages:
         testResultsFormat: NUnit
         testResultsFiles: TestResult-*.xml
         testRunTitle: xamarin-android
-        failTaskOnFailedTests: true
       condition: succeededOrFailed()
 
     - task: MSBuild@1
@@ -232,3 +232,206 @@ stages:
         artifactName: win-build-test-results
         targetPath: $(Build.ArtifactStagingDirectory)
       condition: always()
+
+- stage: test
+  displayName: Test
+  dependsOn: mac_build
+  jobs:
+  - job: mac_apk_tests
+    displayName: APK Instrumentation
+    pool: $(XA.Build.Mac.Pool)
+    timeoutInMinutes: 240
+    cancelTimeoutInMinutes: 5
+    workspace:
+      clean: all
+    variables:
+      ApkTestConfiguration: Release
+    steps:
+    - task: DownloadPipelineArtifact@1
+      inputs:
+        artifactName: $(InstallerArtifactName)
+        itemPattern: "*.pkg"
+        downloadPath: $(System.DefaultWorkingDirectory)
+
+    - template: yaml-templates/run-installer.yaml
+
+    - task: MSBuild@1
+      displayName: build xaprepare
+      inputs:
+        solution: build-tools/xaprepare/xaprepare.sln
+        configuration: $(ApkTestConfiguration)
+        msbuildArguments: /t:Restore,Build
+
+    - script: |
+        mono build-tools/xaprepare/xaprepare/bin/$(ApkTestConfiguration)/xaprepare.exe --s=UpdateMono --auto-provision=yes --auto-provision-uses-sudo=yes --no-emoji --run-mode=CI
+        mono build-tools/xaprepare/xaprepare/bin/$(ApkTestConfiguration)/xaprepare.exe --s=Required --auto-provision=yes --auto-provision-uses-sudo=yes --no-emoji --run-mode=CI
+        mono build-tools/xaprepare/xaprepare/bin/$(ApkTestConfiguration)/xaprepare.exe --s=AndroidToolchain --no-emoji --run-mode=CI
+      displayName: provision dependencies
+
+    - task: NuGetCommand@2
+      displayName: nuget restore Xamarin.Android.Tools.sln
+      inputs:
+        restoreSolution: external/xamarin-android-tools/Xamarin.Android.Tools.sln
+
+    - task: MSBuild@1
+      displayName: build Xamarin.Android.Tools.BootstrapTasks.csproj
+      inputs:
+        solution: build-tools/xamarin.Android.Tools.BootstrapTasks/Xamarin.Android.Tools.BootstrapTasks.csproj
+        configuration: $(ApkTestConfiguration)
+        msbuildArguments: /bl:$(System.DefaultWorkingDirectory)/bin/Test$(ApkTestConfiguration)/BootstrapTasks.binlog
+
+    - task: NuGetCommand@2
+      displayName: nuget restore Xamarin.Android-Tests.sln
+      inputs:
+        restoreSolution: Xamarin.Android-Tests.sln
+
+    - template: yaml-templates/apk-instrumentation.yaml
+      parameters:
+        configuration: $(ApkTestConfiguration)
+        testName: Mono.Android_Tests
+        project: src/Mono.Android/Test/Mono.Android-Tests.csproj
+        testResultsFiles: TestResult-Mono.Android_Tests-$(ApkTestConfiguration).xml
+
+    - template: yaml-templates/apk-instrumentation.yaml
+      parameters:
+        configuration: $(ApkTestConfiguration)
+        testName: Mono.Android_Tests-Aot
+        project: src/Mono.Android/Test/Mono.Android-Tests.csproj
+        testResultsFiles: TestResult-Mono.Android_Tests-$(ApkTestConfiguration)-Aot.xml
+        extraBuildArgs: /p:AotAssemblies=True /p:EnableLlvm=True
+
+    - template: yaml-templates/apk-instrumentation.yaml
+      parameters:
+        configuration: $(ApkTestConfiguration)
+        testName: Mono.Android_Tests-Profiled
+        project: src/Mono.Android/Test/Mono.Android-Tests.csproj
+        testResultsFiles: TestResult-Mono.Android_Tests-$(ApkTestConfiguration)-Profiled.xml
+        extraBuildArgs: /p:AndroidEnableProfiledAot=true
+
+    - template: yaml-templates/apk-instrumentation.yaml
+      parameters:
+        configuration: $(ApkTestConfiguration)
+        testName: Mono.Android_Tests-Bundle
+        project: src/Mono.Android/Test/Mono.Android-Tests.csproj
+        testResultsFiles: TestResult-Mono.Android_Tests-$(ApkTestConfiguration)-Bundle.xml
+        extraBuildArgs: /p:BundleAssemblies=true
+
+    - template: yaml-templates/apk-instrumentation.yaml
+      parameters:
+        configuration: $(ApkTestConfiguration)
+        testName: Mono.Android_TestsAppBundle
+        project: tests/Runtime-AppBundle/Mono.Android-TestsAppBundle.csproj
+        testResultsFiles: TestResult-Mono.Android_TestsAppBundle-$(ApkTestConfiguration).xml
+        packageType: Aab
+
+    - template: yaml-templates/apk-instrumentation.yaml
+      parameters:
+        configuration: $(ApkTestConfiguration)
+        testName: Mono.Android_TestsMultiDex
+        project: tests/Runtime-MultiDex/Mono.Android-TestsMultiDex.csproj
+        testResultsFiles: TestResult-Mono.Android_TestsMultiDex-$(ApkTestConfiguration).xml
+
+    - template: yaml-templates/apk-instrumentation.yaml
+      parameters:
+        configuration: $(ApkTestConfiguration)
+        testName: Xamarin.Android.JcwGen_Tests
+        project: tests/CodeGen-Binding/Xamarin.Android.JcwGen-Tests/Xamarin.Android.JcwGen-Tests.csproj
+        testResultsFiles: TestResult-Xamarin.Android.JcwGen_Tests-$(ApkTestConfiguration).xml
+
+    - template: yaml-templates/apk-instrumentation.yaml
+      parameters:
+        configuration: $(ApkTestConfiguration)
+        testName: Xamarin.Android.Locale_Tests
+        project: tests/locales/Xamarin.Android.Locale-Tests/Xamarin.Android.Locale-Tests.csproj
+        testResultsFiles: TestResult-Xamarin.Android.Locale_Tests-$(ApkTestConfiguration).xml
+
+    - template: yaml-templates/apk-instrumentation.yaml
+      parameters:
+        configuration: $(ApkTestConfiguration)
+        testName: Xamarin.Android.Locale_Tests-Aot
+        project: tests/locales/Xamarin.Android.Locale-Tests/Xamarin.Android.Locale-Tests.csproj
+        testResultsFiles: TestResult-Xamarin.Android.Locale_Tests-$(ApkTestConfiguration)-Aot.xml
+        extraBuildArgs: /p:AotAssemblies=True
+
+    - template: yaml-templates/apk-instrumentation.yaml
+      parameters:
+        configuration: $(ApkTestConfiguration)
+        testName: Xamarin.Android.Locale_Tests-Profiled
+        project: tests/locales/Xamarin.Android.Locale-Tests/Xamarin.Android.Locale-Tests.csproj
+        testResultsFiles: TestResult-Xamarin.Android.Locale_Tests-$(ApkTestConfiguration)-Profiled.xml
+        extraBuildArgs: /p:AndroidEnableProfiledAot=true
+
+    - template: yaml-templates/apk-instrumentation.yaml
+      parameters:
+        configuration: $(ApkTestConfiguration)
+        testName: Xamarin.Android.EmbeddedDSO_Test
+        project: tests/EmbeddedDSOs/EmbeddedDSO/EmbeddedDSO.csproj
+        testResultsFiles: TestResult-Xamarin.Android.EmbeddedDSO_Test.nunit-$(ApkTestConfiguration).xml
+
+    - task: MSBuild@1
+      displayName: run Xamarin.Forms-Performance-Integration
+      inputs:
+        solution: tests/Xamarin.Forms-Performance-Integration/Droid/Xamarin.Forms.Performance.Integration.Droid.csproj
+        configuration: $(ApkTestConfiguration)
+        msbuildArguments: >
+          /t:AcquireAndroidTarget,SignAndroidPackage,UndeployTestApks,DeployTestApks,RunTestApks,ReportComponentFailures
+          /bl:$(System.DefaultWorkingDirectory)/bin/Test$(ApkTestConfiguration)/XamarinFormsPerf.binlog
+      condition: succeededOrFailed()
+
+    - task: MSBuild@1
+      displayName: run Xamarin.Forms-Performance-Integration-Aot
+      inputs:
+        solution: tests/Xamarin.Forms-Performance-Integration/Droid/Xamarin.Forms.Performance.Integration.Droid.csproj
+        configuration: $(ApkTestConfiguration)
+        msbuildArguments: >
+          /t:AcquireAndroidTarget,SignAndroidPackage,UndeployTestApks,DeployTestApks,RunTestApks,ReportComponentFailures
+          /bl:$(System.DefaultWorkingDirectory)/bin/Test$(ApkTestConfiguration)/XamarinFormsPerf-Aot.binlog
+          /p:AotAssemblies=true
+      condition: succeededOrFailed()
+
+    - task: MSBuild@1
+      displayName: run Xamarin.Forms-Performance-Integration-Profiled
+      inputs:
+        solution: tests/Xamarin.Forms-Performance-Integration/Droid/Xamarin.Forms.Performance.Integration.Droid.csproj
+        configuration: $(ApkTestConfiguration)
+        msbuildArguments: >
+          /t:AcquireAndroidTarget,SignAndroidPackage,UndeployTestApks,DeployTestApks,RunTestApks,ReportComponentFailures
+          /bl:$(System.DefaultWorkingDirectory)/bin/Test$(ApkTestConfiguration)/XamarinFormsPerf-Profiled.binlog
+          /p:AndroidEnableProfiledAot=true
+      condition: succeededOrFailed()
+
+    - task: MSBuild@1
+      displayName: run Xamarin.Forms-Performance-Integration-Bundle
+      inputs:
+        solution: tests/Xamarin.Forms-Performance-Integration/Droid/Xamarin.Forms.Performance.Integration.Droid.csproj
+        configuration: $(ApkTestConfiguration)
+        msbuildArguments: >
+          /t:AcquireAndroidTarget,SignAndroidPackage,UndeployTestApks,DeployTestApks,RunTestApks,ReportComponentFailures
+          /bl:$(System.DefaultWorkingDirectory)/bin/Test$(ApkTestConfiguration)/XamarinFormsPerf-Bundle.binlog
+          /p:BundleAssemblies=true
+      condition: succeededOrFailed()
+
+    - task: MSBuild@1
+      displayName: shut down emulator
+      inputs:
+        solution: src/Mono.Android/Test/Mono.Android-Tests.csproj
+        configuration: $(ApkTestConfiguration)
+        msbuildArguments: >
+          /t:AcquireAndroidTarget,ReleaseAndroidTarget
+          /bl:$(System.DefaultWorkingDirectory)/bin/Test$(ApkTestConfiguration)/shutdown-emulator.binlog
+      condition: always()
+
+    - task: MSBuild@1
+      displayName: package results
+      inputs:
+        solution: build-tools/xamarin.Android.Tools.BootstrapTasks/Xamarin.Android.Tools.BootstrapTasks.csproj
+        configuration: $(ApkTestConfiguration)
+        msbuildArguments: /t:ZipBuildStatus;ZipTestResults /p:BuildStatusZipOutputPath=$(Build.ArtifactStagingDirectory) /p:TestResultZipOutputPath=$(Build.ArtifactStagingDirectory)
+      condition: always()
+
+    - task: PublishPipelineArtifact@0
+      displayName: upload artifacts
+      inputs:
+        artifactName: mac-apk-test-results
+        targetPath: $(Build.ArtifactStagingDirectory)
+      condition: always()

build-tools/automation/yaml-templates/apk-instrumentation.yaml

@@ -0,0 +1,28 @@
+parameters:
+  configuration: []
+  testName: []
+  project: []
+  testResultsFiles: []
+  extraBuildArgs: ""
+  testResultsFormat: NUnit
+  packageType: Apk
+
+steps:
+- task: MSBuild@1
+  displayName: run ${{ parameters.testName }}
+  inputs:
+    solution: ${{ parameters.project }}
+    configuration: ${{ parameters.configuration }}
+    msbuildArguments: >
+      /t:AcquireAndroidTarget,SignAndroidPackage,UndeployTestApks,DeployTest${{ parameters.packageType }}s,RunTestApks,RenameApkTestCases,ReportComponentFailures
+      /bl:$(System.DefaultWorkingDirectory)/bin/Test${{ parameters.configuration }}/run${{ parameters.testName }}.binlog
+      ${{ parameters.extraBuildArgs }}
+  condition: succeededOrFailed()
+
+- task: PublishTestResults@2
+  displayName: publish ${{ parameters.testName }} results
+  inputs:
+    testResultsFormat: ${{ parameters.testResultsFormat }}
+    testResultsFiles: ${{ parameters.testResultsFiles }}
+    testRunTitle: ${{ parameters.testName }}
+  condition: succeededOrFailed()

build-tools/automation/yaml-templates/run-installer.yaml

@@ -0,0 +1,22 @@
+parameters:
+  artifactDirectory: $(System.DefaultWorkingDirectory)
+
+steps:
+- powershell: |
+    if ([Environment]::OSVersion.Platform -eq "Unix") {
+        $installer = Get-ChildItem -Path "${{ parameters.artifactDirectory }}/*" -Include *.pkg -File
+    } else {
+        $installer = Get-ChildItem -Path "${{ parameters.artifactDirectory }}\*" -Include *.vsix -File
+    }
+    if (![System.IO.File]::Exists($installer)) {
+        throw [System.IO.FileNotFoundException] "Installer not found in $artifactDirectory."
+    }
+    Write-Host "##vso[task.setvariable variable=XA.Provisionator.Args]$installer"
+  displayName: find installer and set provisionator variable
+
+- task: provisionator@2
+  inputs:
+    provisionator_uri: $(provisionator-uri)
+    github_token: $(GitHub.Token)
+    provisioning_script: $(XA.Provisionator.Args)
+    provisioning_extra_args: -vv

build-tools/scripts/Jar.targets

@@ -7,6 +7,8 @@
       <_JavacSourceVersion Condition=" '$(_JavacSourceVersion)' == '' ">1.5</_JavacSourceVersion>
       <_JavacTargetVersion Condition="$(_JdkVersion.StartsWith ('9'))">1.8</_JavacTargetVersion>
       <_JavacTargetVersion Condition=" '$(_JavacTargetVersion)' == '' ">1.6</_JavacTargetVersion>
+      <JarPath Condition=" '$(JarPath)' == '' ">$(JavaSdkDirectory)\bin\jar</JarPath>
+      <JavaCPath Condition=" '$(JavaCPath)' == '' ">$(JavaSdkDirectory)\bin\javac</JavaCPath>
     </PropertyGroup>
   </Target>
   <Target Name="BuildTestJarFile"