From 646f6cccbde0bbb7d37fa082b7bcb31cac4c1cb3 Mon Sep 17 00:00:00 2001 From: Hansong Zhang Date: Mon, 3 Mar 2025 20:52:55 -0800 Subject: [PATCH 1/2] [minibench] Drop outliers from benchmark result Currently the result has large variance from outliers, so only use 80% samples in the middle (trimmean 0.2) --- .../java/org/pytorch/minibench/BenchmarkActivity.java | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/extension/benchmark/android/benchmark/app/src/main/java/org/pytorch/minibench/BenchmarkActivity.java b/extension/benchmark/android/benchmark/app/src/main/java/org/pytorch/minibench/BenchmarkActivity.java index c0856f3e4fe..cc45a61143e 100644 --- a/extension/benchmark/android/benchmark/app/src/main/java/org/pytorch/minibench/BenchmarkActivity.java +++ b/extension/benchmark/android/benchmark/app/src/main/java/org/pytorch/minibench/BenchmarkActivity.java @@ -20,6 +20,7 @@ import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; import java.util.List; import java.util.stream.Collectors; import org.pytorch.executorch.Module; @@ -80,11 +81,18 @@ protected void onPostExecute(Void aVoid) { final List results = new ArrayList<>(); // The list of metrics we have atm includes: // Avg inference latency after N iterations + // Currently the result has large variance from outliers, so only use + // 80% samples in the middle (trimmean 0.2) + Collections.sort(stats.latency); + int resultSize = stats.latency.size(); + List usedLatencyResults = stats.latency.subList( + resultSize / 10, resultSize * 9 / 10); + results.add( new BenchmarkMetric( benchmarkModel, "avg_inference_latency(ms)", - stats.latency.stream().mapToDouble(l -> l).average().orElse(0.0f), + usedLatencyResults.stream().mapToDouble(l -> l).average().orElse(0.0f), 0.0f)); // Model load time results.add( From a622dc1825864326034db3bf9bc8d06f3b02334c Mon Sep 17 00:00:00 2001 From: Hansong Zhang Date: Mon, 3 Mar 2025 21:43:09 -0800 Subject: [PATCH 2/2] format --- .../main/java/org/pytorch/minibench/BenchmarkActivity.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/extension/benchmark/android/benchmark/app/src/main/java/org/pytorch/minibench/BenchmarkActivity.java b/extension/benchmark/android/benchmark/app/src/main/java/org/pytorch/minibench/BenchmarkActivity.java index cc45a61143e..4b2ba56099e 100644 --- a/extension/benchmark/android/benchmark/app/src/main/java/org/pytorch/minibench/BenchmarkActivity.java +++ b/extension/benchmark/android/benchmark/app/src/main/java/org/pytorch/minibench/BenchmarkActivity.java @@ -85,8 +85,8 @@ protected void onPostExecute(Void aVoid) { // 80% samples in the middle (trimmean 0.2) Collections.sort(stats.latency); int resultSize = stats.latency.size(); - List usedLatencyResults = stats.latency.subList( - resultSize / 10, resultSize * 9 / 10); + List usedLatencyResults = + stats.latency.subList(resultSize / 10, resultSize * 9 / 10); results.add( new BenchmarkMetric(