Skip to content

Commit 7189f77

Browse files
authored
[ci] Improve TimeZoneInfo Emulator test reliability; Emulator 30.3.5 (#5566)
The TimeZoneInfo Emulator tests have been very unreliable recently. We speculate that this is down to the emulator not running as fast as it used to on the CI system, possibly due to commit 75317db. Improve the reliability of the TimeZoneInfo Emulator tests: Add `[NonParallelizable]` to tests to ensure that they are not run in parallel. These tests are run on an emulator, and For Great Sanity™ we don't want these tests run in parallel *anyway*: the tests involve changing the `persist.sys.timezone` system property, which is effectively a global variable. We can't sanely run these in parallel (unless we spin up multiple emulators…?). Add a `[Retry]` attribute to at least re-run a test once if it fails. This might help if the emulator is running slowly. Add the following commands both before and after each test: RunAdbCommand ($"shell am force-stop --user all {proj.PackageName}"); RunAdbCommand ($"shell am kill --user all {proj.PackageName}"); These commands will force the application to stop, and force any background tasks the application has to stop as well. This is the best way we could find to completely quit an application. While we were expecting the `TimeZoneChanged` notification to fire if the app was already running, it turns out that when setting the date time via `adb`, the broadcast does *not* get sent. The broadcast is only sent when the timezone is changed via the Settings screen. Thus exiting the application completely and forcing a restart is the best option for this particular test scenario. We should investigate to see if we can script the timezone change via the UI so we can test the broadcast, but that should be done later. Run the TimeZoneInfo tests across *4* CI nodes, not 3 nodes, as the previous changes increased execution time enough to cause frequent timeouts. Running across more CI nodes helps reduce the overall time. Finally, bump to Android Emulator v30.3.5: * https://androidstudio.googleblog.com/2020/05/emulator-30013-canary.html * https://androidstudio.googleblog.com/2020/05/emulator-30014-canary.html * https://androidstudio.googleblog.com/2020/05/emulator-30015-canary.html * https://androidstudio.googleblog.com/2020/06/emulator-30016-canary.html * https://androidstudio.googleblog.com/2020/06/emulator-30017-canary.html * https://androidstudio.googleblog.com/2020/06/emulator-30018-canary.html * https://androidstudio.googleblog.com/2020/06/emulator-30019-canary.html * https://androidstudio.googleblog.com/2020/07/emulator-30020-canary.html * https://androidstudio.googleblog.com/2020/07/emulator-30021-canary.html * https://androidstudio.googleblog.com/2020/07/emulator-30022-canary.html * https://androidstudio.googleblog.com/2020/07/emulator-30023-canary.html * https://androidstudio.googleblog.com/2020/08/emulator-30024-canary.html * https://androidstudio.googleblog.com/2020/08/emulator-30025-canary.html * https://androidstudio.googleblog.com/2020/08/emulator-30026-canary.html * https://androidstudio.googleblog.com/2020/08/emulator-3010-canary.html * https://androidstudio.googleblog.com/2020/08/emulator-3011-canary.html * https://androidstudio.googleblog.com/2020/09/emulator-3013-canary.html
1 parent accc846 commit 7189f77

File tree

6 files changed

+60
-16
lines changed

6 files changed

+60
-16
lines changed

Configuration.props

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -161,8 +161,8 @@
161161
<CommandLineToolsFolder Condition=" '$(CommandLineToolsFolder)' == '' ">1.0</CommandLineToolsFolder>
162162
<CommandLineToolsVersion Condition=" '$(CommandLineToolsVersion)' == '' ">6200805_latest</CommandLineToolsVersion>
163163
<CommandLineToolsBinPath Condition=" '$(CommandLineToolsBinPath)' == '' ">$(AndroidSdkFullPath)\cmdline-tools\$(CommandLineToolsFolder)\bin</CommandLineToolsBinPath>
164-
<EmulatorVersion Condition=" '$(EmulatorVersion)' == '' ">6466327</EmulatorVersion>
165-
<EmulatorPkgRevision Condition=" '$(EmulatorPkgRevision)' == '' ">30.0.12</EmulatorPkgRevision>
164+
<EmulatorVersion Condition=" '$(EmulatorVersion)' == '' ">7033400</EmulatorVersion>
165+
<EmulatorPkgRevision Condition=" '$(EmulatorPkgRevision)' == '' ">30.3.5</EmulatorPkgRevision>
166166
<EmulatorToolPath Condition=" '$(EmulatorToolPath)' == '' ">$(AndroidSdkFullPath)\emulator</EmulatorToolPath>
167167
<EmulatorToolExe Condition=" '$(EmulatorToolExe)' == '' ">emulator</EmulatorToolExe>
168168
<NdkBuildPath Condition=" '$(NdkBuildPath)' == '' And '$(HostOS)' != 'Windows' ">$(AndroidNdkDirectory)\ndk-build</NdkBuildPath>

build-tools/Xamarin.Android.Tools.BootstrapTasks/Xamarin.Android.Tools.BootstrapTasks/StartAndroidEmulator.cs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -62,7 +62,7 @@ void Run (string emulator)
6262
return;
6363

6464
var port = string.IsNullOrEmpty (Port) ? "" : $" -port {Port}";
65-
var arguments = $"-verbose -avd {ImageName}{port} -cache-size 512";
65+
var arguments = $"-verbose -no-boot-anim -no-audio -no-snapshot -cache-size 512 -timezone \"Etc/UTC\" -avd {ImageName}{port}";
6666
Log.LogMessage (MessageImportance.Low, $"Tool {emulator} execution started with arguments: {arguments}");
6767
var psi = new ProcessStartInfo () {
6868
FileName = emulator,

build-tools/automation/azure-pipelines.yaml

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -918,6 +918,10 @@ stages:
918918
parameters:
919919
node_id: 3
920920

921+
- template: yaml-templates\run-timezoneinfo-tests.yaml
922+
parameters:
923+
node_id: 4
924+
921925
- stage: bcl_tests
922926
displayName: BCL Emulator Tests
923927
dependsOn: mac_build

build-tools/automation/yaml-templates/run-timezoneinfo-tests.yaml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@ jobs:
88
displayName: macOS-${{ parameters.node_id }}
99
pool:
1010
vmImage: $(HostedMacImage)
11-
timeoutInMinutes: 90
11+
timeoutInMinutes: 120
1212
cancelTimeoutInMinutes: 5
1313
workspace:
1414
clean: all

src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Utilities/DeviceTest.cs

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -125,8 +125,8 @@ protected static bool MonitorAdbLogcat (Func<string, bool> action, string logcat
125125
proc.BeginOutputReadLine ();
126126
TimeSpan time = TimeSpan.FromSeconds (timeout);
127127
while (!stdout_done.IsSet && !didActionSucceed && time.TotalMilliseconds > 0) {
128-
proc.WaitForExit (100);
129-
time -= TimeSpan.FromMilliseconds (100);
128+
proc.WaitForExit (10);
129+
time -= TimeSpan.FromMilliseconds (10);
130130
}
131131
proc.Kill ();
132132
proc.WaitForExit ();

tests/MSBuildDeviceIntegration/Tests/DeploymentTest.cs

Lines changed: 50 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -35,12 +35,18 @@ public void BeforeDeploymentTests ()
3535
if (debuggable != "1") {
3636
Assert.Ignore ("TimeZone tests need to use `su root` and this device does not support that feature. Try using an emulator.");
3737
}
38+
// Disable auto timezone
39+
RunAdbCommand ("shell settings put global auto_time_zone 0");
3840

3941
proj = new XamarinFormsAndroidApplicationProject ();
4042
proj.SetAndroidSupportedAbis ("armeabi-v7a", "x86");
4143
var mainPage = proj.Sources.First (x => x.Include () == "MainPage.xaml.cs");
4244
var source = mainPage.TextContent ().Replace ("InitializeComponent ();", @"InitializeComponent ();
45+
Console.WriteLine ($""TimeZoneInfoNative={Java.Util.TimeZone.Default.ID}"");
4346
Console.WriteLine ($""TimeZoneInfo={TimeZoneInfo.Local.DisplayName}"");
47+
");
48+
source = source.Replace ("Console.WriteLine (\"Button was Clicked!\");", @"Console.WriteLine (""Button was Clicked!"");
49+
Console.WriteLine ($""TimeZoneInfoClick={TimeZoneInfo.Local.DisplayName}"");
4450
");
4551
mainPage.TextContent = () => source;
4652
builder = CreateApkBuilder (Path.Combine ("temp", "DeploymentTests"));
@@ -183,7 +189,7 @@ public void CheckXamarinFormsAppDeploysAndAButtonWorks ()
183189
}, Path.Combine (Root, builder.ProjectDirectory, "button-logcat.log")), "Button Should have been Clicked.");
184190
}
185191

186-
private const int NODE_COUNT = 3;
192+
private const int NODE_COUNT = 4;
187193

188194
static object [] GetTimeZoneTestCases (int node)
189195
{
@@ -203,38 +209,72 @@ static object [] GetTimeZoneTestCases (int node)
203209
return tests.Where (p => tests.IndexOf (p) % NODE_COUNT == node).ToArray ();
204210
}
205211

206-
[Test]
212+
[Test, NonParallelizable]
207213
[TestCaseSource (nameof (GetTimeZoneTestCases), new object [] { 0 })]
208214
[Category ("TimeZoneInfo")]
215+
[Retry (2)]
209216
public void CheckTimeZoneInfoIsCorrectNode1 (string timeZone) => CheckTimeZoneInfoIsCorrect (timeZone);
210217

211-
[Test]
218+
[Test, NonParallelizable]
212219
[TestCaseSource (nameof (GetTimeZoneTestCases), new object [] { 1 })]
213220
[Category ("TimeZoneInfo")]
221+
[Retry (2)]
214222
public void CheckTimeZoneInfoIsCorrectNode2 (string timeZone) => CheckTimeZoneInfoIsCorrect (timeZone);
215223

216-
[Test]
224+
[Test, NonParallelizable]
217225
[TestCaseSource (nameof (GetTimeZoneTestCases), new object [] { 2 })]
218226
[Category ("TimeZoneInfo")]
227+
[Retry (2)]
219228
public void CheckTimeZoneInfoIsCorrectNode3 (string timeZone) => CheckTimeZoneInfoIsCorrect (timeZone);
220229

230+
[Test, NonParallelizable]
231+
[TestCaseSource (nameof (GetTimeZoneTestCases), new object [] { 3 })]
232+
[Category ("TimeZoneInfo")]
233+
[Retry (2)]
234+
public void CheckTimeZoneInfoIsCorrectNode4 (string timeZone) => CheckTimeZoneInfoIsCorrect (timeZone);
235+
221236
public void CheckTimeZoneInfoIsCorrect (string timeZone)
222237
{
223238
AssertHasDevices ();
224239

225240
string currentTimeZone = RunAdbCommand ("shell getprop persist.sys.timezone")?.Trim ();
241+
string deviceTz = string.Empty;
242+
string logFile = Path.Combine (Root, builder.ProjectDirectory, $"startup-logcat-{timeZone.Replace ("/", "-")}.log");
226243
try {
227-
RunAdbCommand ($"shell su root setprop persist.sys.timezone \"{timeZone}\"");
244+
for (int attempt = 0; attempt < 5; attempt++) {
245+
RunAdbCommand ($"shell su root setprop persist.sys.timezone \"{timeZone}\"");
246+
deviceTz = RunAdbCommand ("shell getprop persist.sys.timezone")?.Trim ();
247+
if (deviceTz == timeZone) {
248+
break;
249+
}
250+
}
251+
Assert.AreEqual (timeZone, deviceTz, $"The command to set the device timezone to {timeZone} failed. Current device timezone is {deviceTz}");
252+
ClearAdbLogcat ();
253+
RunAdbCommand ($"shell am force-stop --user all {proj.PackageName}");
254+
RunAdbCommand ($"shell am kill --user all {proj.PackageName}");
255+
WaitFor ((int)TimeSpan.FromSeconds (2).TotalMilliseconds);
228256
ClearAdbLogcat ();
229257
AdbStartActivity ($"{proj.PackageName}/{proj.JavaPackageName}.MainActivity");
230-
Assert.IsTrue (WaitForActivityToStart (proj.PackageName, "MainActivity",
231-
Path.Combine (Root, builder.ProjectDirectory, $"startup-logcat-{timeZone.Replace ("/", "-")}.log")), "Activity should have started");
258+
Assert.IsTrue (WaitForActivityToStart (proj.PackageName, "MainActivity", logFile), "Activity should have started");
259+
string line = "";
260+
string logCatFile = Path.Combine (Root, builder.ProjectDirectory, $"timezone-logcat-{timeZone.Replace ("/", "-")}.log");
261+
ClickButton (proj.PackageName, "myXFButton", "CLICK ME");
232262
Assert.IsTrue (MonitorAdbLogcat ((l) => {
233-
return l.Contains ($"TimeZoneInfo={timeZone}");
234-
}, Path.Combine (Root, builder.ProjectDirectory, $"timezone-logcat-{timeZone.Replace ("/", "-")}.log")), $"TimeZone should have been {timeZone}");
263+
if (l.Contains ("TimeZoneInfoClick=")) {
264+
line = l;
265+
return l.Contains ($"{timeZone}");
266+
}
267+
return false;
268+
}, logCatFile, timeout:30), $"TimeZone should have been {timeZone}. We found : {line}");
235269
} finally {
236-
if (!string.IsNullOrEmpty (currentTimeZone))
270+
RunAdbCommand ($"shell am force-stop --user all {proj.PackageName}");
271+
RunAdbCommand ($"shell am kill --user all {proj.PackageName}");
272+
if (!string.IsNullOrEmpty (currentTimeZone)) {
237273
RunAdbCommand ($"shell su root setprop persist.sys.timezone \"{currentTimeZone}\"");
274+
}
275+
if (File.Exists (logFile)) {
276+
TestContext.AddTestAttachment (logFile);
277+
}
238278
}
239279
}
240280
}

0 commit comments

Comments
 (0)