Skip to content

Commit 8d15fe2

Browse files
lamnhan066lamnhan066
committed
refactor: enhance benchmark tests with improved warmup and measurement strategies
1 parent c654d57 commit 8d15fe2

2 files changed

Lines changed: 154 additions & 73 deletions

File tree

‎benchmark/benchmark_vm_test.dart‎

Lines changed: 76 additions & 37 deletions
Original file line numberDiff line numberDiff line change
@@ -5,13 +5,16 @@
55
library;
66

77
import 'dart:isolate';
8+
import 'dart:math';
89

910
import 'package:isolate_manager/isolate_manager.dart';
1011
import 'package:test/test.dart';
1112

1213
import 'functions.dart';
1314
import 'utils.dart';
1415

16+
const samples = 30;
17+
1518
void main() {
1619
test('benchmark', () async {
1720
printDebug(
@@ -20,44 +23,63 @@ void main() {
2023
);
2124
printDebug(() => '|:-:|-:|-:|-:|-:|-:|-:|');
2225

23-
// Warmup: run each part once to trigger JIT
24-
await warmup(30);
25-
await warmup(33);
26-
await warmup(36);
26+
// Stronger warmup
27+
await warmup(23);
28+
await warmup(26);
29+
await warmup(29);
2730

28-
// Run benchmarks
29-
await benchmarkFor(30);
30-
await benchmarkFor(33);
31-
await benchmarkFor(36);
31+
await benchmarkFor(23);
32+
await benchmarkFor(26);
33+
await benchmarkFor(29);
3234
}, timeout: const Timeout(Duration(seconds: 180)));
3335
}
3436

35-
/// Run one warmup round to reduce JIT overhead.
3637
Future<void> warmup(int fibonacciNumber) async {
37-
await execute(fibonacciNumber, iterations: 5, warmupOnly: true);
38+
final iterations = switch (fibonacciNumber) {
39+
23 => 20,
40+
26 => 10,
41+
29 => 5,
42+
_ => 10,
43+
};
44+
45+
await execute(fibonacciNumber, iterations: iterations, warmupOnly: true);
3846
}
3947

40-
/// Benchmark each method for the given fibonacci number.
4148
Future<void> benchmarkFor(int fibonacciNumber) async {
42-
final result = await execute(fibonacciNumber, iterations: 50);
49+
final iterations = switch (fibonacciNumber) {
50+
23 => 50,
51+
26 => 30,
52+
29 => 10,
53+
_ => 50,
54+
};
55+
56+
final result = await execute(fibonacciNumber, iterations: iterations);
4357
printDebug(() => result);
4458
}
4559

46-
/// Runs different methods with a separate measurement function.
47-
/// If [warmupOnly] is set, no measurement will be taken.
4860
Future<String> execute(
4961
int fibonacciNumber, {
5062
required int iterations,
5163
bool warmupOnly = false,
5264
}) async {
5365
final results = <String, Duration>{};
5466

55-
// Use a function to measure elapsed time for an async action.
56-
Future<Duration> measure(Future<void> Function() action) async {
57-
final watch = Stopwatch()..start();
58-
await action();
59-
watch.stop();
60-
return watch.elapsed;
67+
Future<Duration> measureStable(Future<void> Function() action) async {
68+
final samplesList = <int>[];
69+
70+
for (var i = 0; i < samples; i++) {
71+
final watch = Stopwatch()..start();
72+
await action();
73+
watch.stop();
74+
samplesList.add(watch.elapsedMicroseconds);
75+
}
76+
77+
samplesList.sort();
78+
79+
// Median (more stable than average)
80+
final median = samplesList[samplesList.length ~/ 2];
81+
82+
return Duration(microseconds: median);
6183
}
6284

6385
Future<void> runMainApp() async {
@@ -69,9 +91,11 @@ Future<String> execute(
6991
Future<void> runOneIsolate() async {
7092
final singleIsolate = IsolateManager<int, int>.create(fibonacciRecursive);
7193
await singleIsolate.start();
94+
7295
for (var i = 0; i < iterations; i++) {
7396
await singleIsolate.compute(fibonacciNumber);
7497
}
98+
7599
await singleIsolate.stop();
76100
}
77101

@@ -80,11 +104,14 @@ Future<String> execute(
80104
fibonacciRecursive,
81105
concurrent: 3,
82106
);
107+
83108
await threeIsolates.start();
109+
84110
await Future.wait([
85111
for (int i = 0; i < iterations; i++)
86112
threeIsolates.compute(fibonacciNumber),
87113
]);
114+
88115
await threeIsolates.stop();
89116
}
90117

@@ -106,29 +133,41 @@ Future<String> execute(
106133
}
107134
}
108135

136+
final tests = <String, Future<void> Function()>{
137+
'Main App': runMainApp,
138+
'One Isolate': runOneIsolate,
139+
'Three Isolates': runThreeIsolates,
140+
'IsolateManager.runFunction': runIsolateManagerFunction,
141+
'IsolateManager.run': runIsolateManagerRun,
142+
'Isolate.run': runIsolateRun,
143+
};
144+
109145
if (warmupOnly) {
110-
await runMainApp();
111-
await runOneIsolate();
112-
await runThreeIsolates();
113-
await runIsolateManagerFunction();
114-
await runIsolateManagerRun();
115-
await runIsolateRun();
146+
for (final fn in tests.values) {
147+
await fn();
148+
}
116149
return '';
117150
}
118151

119-
results['Main App'] = await measure(runMainApp);
120-
results['One Isolate'] = await measure(runOneIsolate);
121-
results['Three Isolates'] = await measure(runThreeIsolates);
122-
results['IsolateManager.runFunction'] = await measure(
123-
runIsolateManagerFunction,
124-
);
125-
results['IsolateManager.run'] = await measure(runIsolateManagerRun);
126-
results['Isolate.run'] = await measure(runIsolateRun);
152+
// Shuffle test order to remove bias
153+
final entries = tests.entries.toList()..shuffle(Random());
154+
155+
for (final entry in entries) {
156+
results[entry.key] = await measureStable(entry.value);
157+
}
127158

128-
// Convert durations to average microseconds per iteration
129-
String format(Duration d) => d.inMicroseconds.asThousands();
159+
String format(Duration d) {
160+
final perIteration = d.inMicroseconds ~/ iterations;
161+
return perIteration.asThousands();
162+
}
130163

131-
return '|$fibonacciNumber|${format(results['Main App']!)}|${format(results['One Isolate']!)}|${format(results['Three Isolates']!)}|${format(results['IsolateManager.runFunction']!)}|${format(results['IsolateManager.run']!)}|${format(results['Isolate.run']!)}|';
164+
return '|$fibonacciNumber|'
165+
'${format(results['Main App']!)}|'
166+
'${format(results['One Isolate']!)}|'
167+
'${format(results['Three Isolates']!)}|'
168+
'${format(results['IsolateManager.runFunction']!)}|'
169+
'${format(results['IsolateManager.run']!)}|'
170+
'${format(results['Isolate.run']!)}|';
132171
}
133172

134173
void printDebug(Object? Function() log) {

‎benchmark/benchmark_web_test.dart‎

Lines changed: 78 additions & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -4,12 +4,17 @@
44
@TestOn('chrome')
55
library;
66

7+
import 'dart:async';
8+
import 'dart:math';
9+
710
import 'package:isolate_manager/isolate_manager.dart';
811
import 'package:test/test.dart';
912

1013
import 'functions.dart';
1114
import 'utils.dart';
1215

16+
const samples = 30;
17+
1318
void main() {
1419
IsolateManager.addWorkerMapping(fibonacciRecursive, 'fibonacciRecursive');
1520

@@ -20,44 +25,63 @@ void main() {
2025
);
2126
printDebug(() => '|:-:|-:|-:|-:|-:|-:|-:|');
2227

23-
// Warmup: run each part once to trigger JIT
24-
await warmup(30);
25-
await warmup(33);
26-
await warmup(36);
28+
await warmup(23);
29+
await warmup(26);
30+
await warmup(29);
2731

28-
// Run benchmarks
29-
await benchmarkFor(30);
30-
await benchmarkFor(33);
31-
await benchmarkFor(36);
32+
await benchmarkFor(23);
33+
await benchmarkFor(26);
34+
await benchmarkFor(29);
3235
}, timeout: const Timeout(Duration(seconds: 180)));
3336
}
3437

35-
/// Run one warmup round to reduce JIT overhead.
3638
Future<void> warmup(int fibonacciNumber) async {
37-
await execute(fibonacciNumber, iterations: 5, warmupOnly: true);
39+
final iterations = switch (fibonacciNumber) {
40+
23 => 20,
41+
26 => 10,
42+
29 => 5,
43+
_ => 10,
44+
};
45+
46+
await execute(fibonacciNumber, iterations: iterations, warmupOnly: true);
3847
}
3948

40-
/// Benchmark each method for the given fibonacci number.
4149
Future<void> benchmarkFor(int fibonacciNumber) async {
42-
final result = await execute(fibonacciNumber, iterations: 70);
50+
final iterations = switch (fibonacciNumber) {
51+
23 => 50,
52+
26 => 30,
53+
29 => 10,
54+
_ => 50,
55+
};
56+
57+
final result = await execute(fibonacciNumber, iterations: iterations);
4358
printDebug(() => result);
4459
}
4560

46-
/// Runs different methods with a separate measurement function.
47-
/// If [warmupOnly] is set, no measurement will be taken.
4861
Future<String> execute(
4962
int fibonacciNumber, {
5063
required int iterations,
5164
bool warmupOnly = false,
5265
}) async {
5366
final results = <String, Duration>{};
5467

55-
// Use a function to measure elapsed time for an async action.
56-
Future<Duration> measure(Future<void> Function() action) async {
57-
final watch = Stopwatch()..start();
58-
await action();
59-
watch.stop();
60-
return watch.elapsed;
68+
Future<Duration> measureStable(Future<void> Function() action) async {
69+
final values = <int>[];
70+
71+
for (var i = 0; i < samples; i++) {
72+
await Future<void>.delayed(Duration.zero); // yield event loop
73+
74+
final watch = Stopwatch()..start();
75+
await action();
76+
watch.stop();
77+
78+
values.add(watch.elapsedMicroseconds);
79+
}
80+
81+
values.sort();
82+
final median = values[values.length ~/ 2];
83+
84+
return Duration(microseconds: median);
6185
}
6286

6387
Future<void> runMainApp() async {
@@ -68,10 +92,13 @@ Future<String> execute(
6892

6993
Future<void> runOneIsolate() async {
7094
final singleIsolate = IsolateManager<int, int>.create(fibonacciRecursive);
95+
7196
await singleIsolate.start();
97+
7298
for (var i = 0; i < iterations; i++) {
7399
await singleIsolate.compute(fibonacciNumber);
74100
}
101+
75102
await singleIsolate.stop();
76103
}
77104

@@ -80,11 +107,14 @@ Future<String> execute(
80107
fibonacciRecursive,
81108
concurrent: 3,
82109
);
110+
83111
await threeIsolates.start();
112+
84113
await Future.wait([
85114
for (int i = 0; i < iterations; i++)
86115
threeIsolates.compute(fibonacciNumber),
87116
]);
117+
88118
await threeIsolates.stop();
89119
}
90120

@@ -104,29 +134,41 @@ Future<String> execute(
104134
}
105135
}
106136

107-
if (warmupOnly) {
108-
await runMainApp();
109-
await runOneIsolate();
110-
await runThreeIsolates();
111-
await runIsolateManagerFunction();
112-
await runIsolateManagerRun();
137+
final tests = <String, Future<void> Function()>{
138+
'Main App': runMainApp,
139+
'One Isolate': runOneIsolate,
140+
'Three Isolates': runThreeIsolates,
141+
'IsolateManager.runFunction': runIsolateManagerFunction,
142+
'IsolateManager.run': runIsolateManagerRun,
143+
};
113144

145+
if (warmupOnly) {
146+
for (final fn in tests.values) {
147+
await fn();
148+
}
114149
return '';
115150
}
116151

117-
results['Main App'] = await measure(runMainApp);
118-
results['One Isolate'] = await measure(runOneIsolate);
119-
results['Three Isolates'] = await measure(runThreeIsolates);
120-
results['IsolateManager.runFunction'] = await measure(
121-
runIsolateManagerFunction,
122-
);
123-
results['IsolateManager.run'] = await measure(runIsolateManagerRun);
152+
final entries = tests.entries.toList()..shuffle(Random());
153+
154+
for (final entry in entries) {
155+
results[entry.key] = await measureStable(entry.value);
156+
}
157+
124158
results['Isolate.run'] = Duration.zero;
125159

126-
// Convert durations to average microseconds per iteration
127-
String format(Duration d) => d.inMicroseconds.asThousands();
160+
String format(Duration d) {
161+
final perIteration = d.inMicroseconds ~/ iterations;
162+
return perIteration.asThousands();
163+
}
128164

129-
return '|$fibonacciNumber|${format(results['Main App']!)}|${format(results['One Isolate']!)}|${format(results['Three Isolates']!)}|${format(results['IsolateManager.runFunction']!)}|${format(results['IsolateManager.run']!)}|${format(results['Isolate.run']!)}|';
165+
return '|$fibonacciNumber|'
166+
'${format(results['Main App']!)}|'
167+
'${format(results['One Isolate']!)}|'
168+
'${format(results['Three Isolates']!)}|'
169+
'${format(results['IsolateManager.runFunction']!)}|'
170+
'${format(results['IsolateManager.run']!)}|'
171+
'${format(results['Isolate.run']!)}|';
130172
}
131173

132174
void printDebug(Object? Function() log) {

0 commit comments

Comments
 (0)