skore/skore-hub-project/src/skore_hub_project/metric/timing.py at 78e1510f513ccc34e7b17650b550061df683899b · probabl-ai/skore · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
"""Class definition of the payload used to send a timing metric to ``hub``."""

from __future__ import annotations

from typing import ClassVar, Literal

from skore_hub_project.protocol import CrossValidationReport, EstimatorReport

from .metric import Metric, cast_to_float


class FitTime(Metric[EstimatorReport]):  # noqa: D101
    name: str = "fit_time"
    verbose_name: str = "Fit time (s)"
    greater_is_better: bool = False
    position: int = 1
    data_source: None = None

    def compute(self) -> None:
        """Compute the value of the metric."""
        timings = self.report.metrics.timings()
        fit_time = timings.get("fit_time")

        self.value = cast_to_float(fit_time)


class FitTimeAggregate(Metric[CrossValidationReport]):  # noqa: D101
    # ``report.metrics.timings()``
    #
    #                             mean       std
    # Fit time (s)                 ...       ...
    # Predict time test (s)        ...       ...
    # Predict time train (s)       ...       ...

    aggregate: ClassVar[Literal["mean", "std"]]
    greater_is_better: bool = False
    data_source: None = None

    def compute(self) -> None:
        """Compute the value of the metric."""
        timings = self.report.metrics.timings(aggregate=self.aggregate)

        try:
            fit_times = timings.loc["Fit time (s)"]
        except KeyError:
            self.value = None
        else:
            self.value = cast_to_float(fit_times.iloc[0])


class FitTimeMean(FitTimeAggregate):  # noqa: D101
    aggregate: ClassVar[Literal["mean"]] = "mean"
    name: str = "fit_time_mean"
    verbose_name: str = "Fit time (s) - MEAN"
    position: int = 1


class FitTimeStd(FitTimeAggregate):  # noqa: D101
    aggregate: ClassVar[Literal["std"]] = "std"
    name: str = "fit_time_std"
    verbose_name: str = "Fit time (s) - STD"
    position: None = None


class PredictTime(Metric[EstimatorReport]):  # noqa: D101
    name: str = "predict_time"
    verbose_name: str = "Predict time (s)"
    greater_is_better: bool = False
    position: int = 2

    def compute(self) -> None:
        """Compute the value of the metric."""
        timings = self.report.metrics.timings()
        predict_time = timings.get(f"predict_time_{self.data_source}")

        self.value = cast_to_float(predict_time)


class PredictTimeTrain(PredictTime):  # noqa: D101
    data_source: Literal["train"] = "train"


class PredictTimeTest(PredictTime):  # noqa: D101
    data_source: Literal["test"] = "test"


class PredictTimeAggregate(Metric[CrossValidationReport]):  # noqa: D101
    # ``report.metrics.timings()``
    #
    #                             mean       std
    # Fit time (s)                 ...       ...
    # Predict time test (s)        ...       ...
    # Predict time train (s)       ...       ...

    aggregate: ClassVar[Literal["mean", "std"]]
    greater_is_better: bool = False

    def compute(self) -> None:
        """Compute the value of the metric."""
        timings = self.report.metrics.timings(aggregate=self.aggregate)

        try:
            predict_times = timings.loc[f"Predict time {self.data_source} (s)"]
        except KeyError:
            self.value = None
        else:
            self.value = cast_to_float(predict_times.iloc[0])


class PredictTimeMean(PredictTimeAggregate):  # noqa: D101
    aggregate: ClassVar[Literal["mean"]] = "mean"
    name: str = "predict_time_mean"
    verbose_name: str = "Predict time (s) - MEAN"
    position: int = 2


class PredictTimeTrainMean(PredictTimeMean):  # noqa: D101
    data_source: Literal["train"] = "train"


class PredictTimeTestMean(PredictTimeMean):  # noqa: D101
    data_source: Literal["test"] = "test"


class PredictTimeStd(PredictTimeAggregate):  # noqa: D101
    aggregate: ClassVar[Literal["std"]] = "std"
    name: str = "predict_time_std"
    verbose_name: str = "Predict time (s) - STD"
    position: None = None


class PredictTimeTrainStd(PredictTimeStd):  # noqa: D101
    data_source: Literal["train"] = "train"


class PredictTimeTestStd(PredictTimeStd):  # noqa: D101
    data_source: Literal["test"] = "test"