From 5d1fda1bc15e3a2938294ae214d19868a54a747e Mon Sep 17 00:00:00 2001 From: Weilin Xu Date: Tue, 22 Oct 2024 14:40:09 -0700 Subject: [PATCH 1/3] Export experiment duration in seconds in CSV. Signed-off-by: Weilin Xu --- src/anomalib/pipelines/benchmark/job.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/anomalib/pipelines/benchmark/job.py b/src/anomalib/pipelines/benchmark/job.py index ab443cfa8a..de68c75853 100644 --- a/src/anomalib/pipelines/benchmark/job.py +++ b/src/anomalib/pipelines/benchmark/job.py @@ -4,6 +4,7 @@ # SPDX-License-Identifier: Apache-2.0 import logging +import time from datetime import datetime from pathlib import Path from tempfile import TemporaryDirectory @@ -48,6 +49,7 @@ def run( task_id: int | None = None, ) -> dict[str, Any]: """Run the benchmark.""" + start_time = time.time() devices: str | list[int] = "auto" if task_id is not None: devices = [task_id] @@ -61,6 +63,7 @@ def run( ) engine.fit(self.model, self.datamodule) test_results = engine.test(self.model, self.datamodule) + duration = time.time() - start_time # TODO(ashwinvaidya17): Restore throughput # https://github.com/openvinotoolkit/anomalib/issues/2054 output = { @@ -69,6 +72,7 @@ def run( "model": self.model.__class__.__name__, "data": self.datamodule.__class__.__name__, "category": self.datamodule.category, + "duration": int(duration), **test_results[0], } logger.info(f"Completed with result {output}") From ef3a295cd51c80ef134953f27591eef03512e3fd Mon Sep 17 00:00:00 2001 From: Weilin Xu Date: Tue, 22 Oct 2024 14:44:33 -0700 Subject: [PATCH 2/3] Update CHANGELOG Signed-off-by: Weilin Xu --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index b50bf09ecb..7760befc7a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). ### Changed +- Add duration of experiments in seconds in the benchmark CSV result by [mzweilin](https://github.com/mzweilin) in https://github.com/openvinotoolkit/anomalib/pull/2392 + ### Deprecated ### Fixed From 9b2514ea82522471c97d45d0eecccf6f173341a5 Mon Sep 17 00:00:00 2001 From: Weilin Xu Date: Wed, 23 Oct 2024 09:53:24 -0700 Subject: [PATCH 3/3] Log fit and test durations separately. Signed-off-by: Weilin Xu --- src/anomalib/pipelines/benchmark/job.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/src/anomalib/pipelines/benchmark/job.py b/src/anomalib/pipelines/benchmark/job.py index de68c75853..01822d5f29 100644 --- a/src/anomalib/pipelines/benchmark/job.py +++ b/src/anomalib/pipelines/benchmark/job.py @@ -49,7 +49,7 @@ def run( task_id: int | None = None, ) -> dict[str, Any]: """Run the benchmark.""" - start_time = time.time() + job_start_time = time.time() devices: str | list[int] = "auto" if task_id is not None: devices = [task_id] @@ -61,9 +61,16 @@ def run( devices=devices, default_root_dir=temp_dir, ) + fit_start_time = time.time() engine.fit(self.model, self.datamodule) + test_start_time = time.time() test_results = engine.test(self.model, self.datamodule) - duration = time.time() - start_time + job_end_time = time.time() + durations = { + "job_duration": job_end_time - job_start_time, + "fit_duration": test_start_time - fit_start_time, + "test_duration": job_end_time - test_start_time, + } # TODO(ashwinvaidya17): Restore throughput # https://github.com/openvinotoolkit/anomalib/issues/2054 output = { @@ -72,7 +79,7 @@ def run( "model": self.model.__class__.__name__, "data": self.datamodule.__class__.__name__, "category": self.datamodule.category, - "duration": int(duration), + **durations, **test_results[0], } logger.info(f"Completed with result {output}")