Skip to content

Commit 67358b2

Browse files
author
Abdul Fatir Ansari
committed
Add metrics csvs
1 parent 0ab0032 commit 67358b2

28 files changed

+501
-0
lines changed

‎scripts/evaluation/agg-relative-score.py

Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,60 @@
1+
import pandas as pd
2+
import typer
3+
from scipy.stats import gmean
4+
from pathlib import Path
5+
6+
app = typer.Typer(pretty_exceptions_enable=False)
7+
DEFAULT_RESULTS_DIR = Path(__file__).parent / "results"
8+
9+
10+
def agg_relative_score(model_csv: Path, baseline_csv: Path):
11+
model_df = pd.read_csv(model_csv).set_index("dataset")
12+
baseline_df = pd.read_csv(baseline_csv).set_index("dataset")
13+
relative_score = model_df.drop("model", axis="columns") / baseline_df.drop(
14+
"model", axis="columns"
15+
)
16+
return relative_score.agg(gmean)
17+
18+
19+
@app.command()
20+
def main(
21+
model_name: str,
22+
baseline_name: str = "seasonal-naive",
23+
results_dir: Path = DEFAULT_RESULTS_DIR,
24+
):
25+
"""
26+
Compute the aggregated relative score as reported in the Chronos paper.
27+
Results will be saved to {results_dir}/{model_name}-agg-rel-scores.csv
28+
29+
Parameters
30+
----------
31+
model_name : str
32+
Name of the model used in the CSV files. The in-domain and zero-shot CSVs
33+
are expected to be named {model_name}-in-domain.csv and {model_name}-zero-shot.csv.
34+
results_dir : Path, optional, default = results/
35+
Directory where results CSVs generated by evaluate.py are stored
36+
"""
37+
38+
in_domain_agg_score_df = agg_relative_score(
39+
results_dir / f"{model_name}-in-domain.csv",
40+
results_dir / f"{baseline_name}-in-domain.csv",
41+
)
42+
in_domain_agg_score_df.name = "value"
43+
in_domain_agg_score_df.index.name = "metric"
44+
45+
zero_shot_agg_score_df = agg_relative_score(
46+
results_dir / f"{model_name}-zero-shot.csv",
47+
results_dir / f"{baseline_name}-zero-shot.csv",
48+
)
49+
zero_shot_agg_score_df.name = "value"
50+
zero_shot_agg_score_df.index.name = "metric"
51+
52+
agg_score_df = pd.concat(
53+
{"in-domain": in_domain_agg_score_df, "zero-shot": zero_shot_agg_score_df},
54+
names=["benchmark"],
55+
)
56+
agg_score_df.to_csv(f"{results_dir}/{model_name}-agg-rel-scores.csv")
57+
58+
59+
if __name__ == "__main__":
60+
app()

‎scripts/evaluation/results/chronos-bolt-base-agg-rel-scores.csv

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
benchmark,metric,value
2+
in-domain,MASE,0.6800133628315155
3+
in-domain,WQL,0.5339263811489279
4+
zero-shot,MASE,0.7914551113353537
5+
zero-shot,WQL,0.6241424984163773

‎scripts/evaluation/results/chronos-bolt-base-in-domain.csv

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
dataset,model,MASE,WQL
2+
electricity_15min,autogluon/chronos-bolt-base,0.41069374835605243,0.0703533790998506
3+
m4_daily,autogluon/chronos-bolt-base,3.205192517121196,0.02110308498174413
4+
m4_hourly,autogluon/chronos-bolt-base,0.8350129849014075,0.025353803894164
5+
m4_monthly,autogluon/chronos-bolt-base,0.9491758928362231,0.09382496106659234
6+
m4_weekly,autogluon/chronos-bolt-base,2.0847827409162742,0.03816605075768161
7+
monash_electricity_hourly,autogluon/chronos-bolt-base,1.254966217685461,0.09442192616975713
8+
monash_electricity_weekly,autogluon/chronos-bolt-base,1.8391546050108039,0.06410971963960499
9+
monash_kdd_cup_2018,autogluon/chronos-bolt-base,0.6405985809360102,0.2509172188706336
10+
monash_london_smart_meters,autogluon/chronos-bolt-base,0.701398572604996,0.3218915088923906
11+
monash_pedestrian_counts,autogluon/chronos-bolt-base,0.2646412642278343,0.18789459806066328
12+
monash_rideshare,autogluon/chronos-bolt-base,0.7695376426829713,0.11637119433040358
13+
monash_temperature_rain,autogluon/chronos-bolt-base,0.8983612698773724,0.6050555216496304
14+
taxi_30min,autogluon/chronos-bolt-base,0.7688908266765317,0.2363178601205094
15+
uber_tlc_daily,autogluon/chronos-bolt-base,0.8231767493519677,0.0926036406916842
16+
uber_tlc_hourly,autogluon/chronos-bolt-base,0.6632193728217927,0.14987786887626975

‎scripts/evaluation/results/chronos-bolt-base-zero-shot.csv

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
dataset,model,MASE,WQL
2+
ETTh,autogluon/chronos-bolt-base,0.7479154031956647,0.07062173821055001
3+
ETTm,autogluon/chronos-bolt-base,0.6334357237512225,0.052261607745858835
4+
dominick,autogluon/chronos-bolt-base,0.8560272479913918,0.3453573743726445
5+
ercot,autogluon/chronos-bolt-base,0.6933217425507392,0.02142183038021456
6+
exchange_rate,autogluon/chronos-bolt-base,1.7095176257412634,0.01200682136751536
7+
m4_quarterly,autogluon/chronos-bolt-base,1.2244670010522907,0.0771066518089854
8+
m4_yearly,autogluon/chronos-bolt-base,3.513752058541554,0.12142798053483984
9+
m5,autogluon/chronos-bolt-base,0.9152230096463854,0.561999688057527
10+
monash_australian_electricity,autogluon/chronos-bolt-base,0.7403239930185613,0.03584034231329335
11+
monash_car_parts,autogluon/chronos-bolt-base,0.8550263912438314,0.9945122291263591
12+
monash_cif_2016,autogluon/chronos-bolt-base,0.9988541862779904,0.016456104842296485
13+
monash_covid_deaths,autogluon/chronos-bolt-base,38.901749109066415,0.047410971217640714
14+
monash_fred_md,autogluon/chronos-bolt-base,0.6468787708795645,0.04185083716355386
15+
monash_hospital,autogluon/chronos-bolt-base,0.6883138394434054,0.057032869931903894
16+
monash_m1_monthly,autogluon/chronos-bolt-base,1.0997677446267855,0.1392311148066238
17+
monash_m1_quarterly,autogluon/chronos-bolt-base,1.7737851980875563,0.1007118219350403
18+
monash_m1_yearly,autogluon/chronos-bolt-base,4.404672537832342,0.1504617654430952
19+
monash_m3_monthly,autogluon/chronos-bolt-base,0.8510696834878182,0.09269673913736748
20+
monash_m3_quarterly,autogluon/chronos-bolt-base,1.2890908822598466,0.07615133571216029
21+
monash_m3_yearly,autogluon/chronos-bolt-base,2.9067097980770082,0.12934285625258413
22+
monash_nn5_weekly,autogluon/chronos-bolt-base,0.9158766337957451,0.08352114810139548
23+
monash_tourism_monthly,autogluon/chronos-bolt-base,1.5283388458731357,0.09026425492612797
24+
monash_tourism_quarterly,autogluon/chronos-bolt-base,1.756127005530011,0.06448060953595125
25+
monash_tourism_yearly,autogluon/chronos-bolt-base,3.691545772463519,0.16548820700844424
26+
monash_traffic,autogluon/chronos-bolt-base,0.7843310867739336,0.23148632068725078
27+
monash_weather,autogluon/chronos-bolt-base,0.8115247139672316,0.13350830777170594
28+
nn5,autogluon/chronos-bolt-base,0.5764084996361287,0.1500519584148468

‎scripts/evaluation/results/chronos-bolt-mini-agg-rel-scores.csv

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
benchmark,metric,value
2+
in-domain,MASE,0.7268373301543752
3+
in-domain,WQL,0.565140251955324
4+
zero-shot,MASE,0.8221798917822493
5+
zero-shot,WQL,0.6441645845380903

‎scripts/evaluation/results/chronos-bolt-mini-in-domain.csv

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
dataset,model,MASE,WQL
2+
electricity_15min,autogluon/chronos-bolt-mini,0.44185193304080733,0.0731477927531107
3+
m4_daily,autogluon/chronos-bolt-mini,3.1342608828747456,0.0206872246743766
4+
m4_hourly,autogluon/chronos-bolt-mini,0.9218285923038745,0.024383114886067574
5+
m4_monthly,autogluon/chronos-bolt-mini,0.9628339921394529,0.09502498697494888
6+
m4_weekly,autogluon/chronos-bolt-mini,2.2330452369879255,0.039393515325238534
7+
monash_electricity_hourly,autogluon/chronos-bolt-mini,1.6195944363428718,0.11468972600782207
8+
monash_electricity_weekly,autogluon/chronos-bolt-mini,1.866105365159433,0.06019900031840434
9+
monash_kdd_cup_2018,autogluon/chronos-bolt-mini,0.74790954883436,0.3012661161484388
10+
monash_london_smart_meters,autogluon/chronos-bolt-mini,0.7187830347765344,0.32984510693830227
11+
monash_pedestrian_counts,autogluon/chronos-bolt-mini,0.308633944815819,0.23331301029432483
12+
monash_rideshare,autogluon/chronos-bolt-mini,0.818948044410056,0.1297966960374544
13+
monash_temperature_rain,autogluon/chronos-bolt-mini,0.9035244443682741,0.605031064086567
14+
taxi_30min,autogluon/chronos-bolt-mini,0.812010120941363,0.25232294549917317
15+
uber_tlc_daily,autogluon/chronos-bolt-mini,0.8507256206478295,0.10101757743084538
16+
uber_tlc_hourly,autogluon/chronos-bolt-mini,0.6685484898085609,0.1515245941548974

‎scripts/evaluation/results/chronos-bolt-mini-zero-shot.csv

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
dataset,model,MASE,WQL
2+
ETTh,autogluon/chronos-bolt-mini,0.8057126710113404,0.07740387596411452
3+
ETTm,autogluon/chronos-bolt-mini,0.6100793941108849,0.05129333450944573
4+
dominick,autogluon/chronos-bolt-mini,0.8664152477208024,0.3499696999160997
5+
ercot,autogluon/chronos-bolt-mini,0.6871250728215426,0.02448804863744021
6+
exchange_rate,autogluon/chronos-bolt-mini,1.3520551553333662,0.00934663373172766
7+
m4_quarterly,autogluon/chronos-bolt-mini,1.2569644266281508,0.07833787023275976
8+
m4_yearly,autogluon/chronos-bolt-mini,3.7611003052413796,0.12931927951165456
9+
m5,autogluon/chronos-bolt-mini,0.9188876472137485,0.5661303206519673
10+
monash_australian_electricity,autogluon/chronos-bolt-mini,0.8823559450287066,0.04493688824488474
11+
monash_car_parts,autogluon/chronos-bolt-mini,0.8604081423647779,1.0041876404811494
12+
monash_cif_2016,autogluon/chronos-bolt-mini,1.0762361363763873,0.017641893717784202
13+
monash_covid_deaths,autogluon/chronos-bolt-mini,38.83915011538576,0.06098317835750057
14+
monash_fred_md,autogluon/chronos-bolt-mini,0.6169859211923081,0.03256236965040934
15+
monash_hospital,autogluon/chronos-bolt-mini,0.6924431064606051,0.05766349075348645
16+
monash_m1_monthly,autogluon/chronos-bolt-mini,1.147893030263777,0.13270222658510553
17+
monash_m1_quarterly,autogluon/chronos-bolt-mini,1.8662100001165818,0.09846363409254102
18+
monash_m1_yearly,autogluon/chronos-bolt-mini,5.319154632748303,0.16167328827180308
19+
monash_m3_monthly,autogluon/chronos-bolt-mini,0.8758452776118432,0.09493431248614057
20+
monash_m3_quarterly,autogluon/chronos-bolt-mini,1.3555175243802005,0.07808062465932723
21+
monash_m3_yearly,autogluon/chronos-bolt-mini,3.605769430055575,0.15711010456482008
22+
monash_nn5_weekly,autogluon/chronos-bolt-mini,0.9347141924977239,0.08522899825844342
23+
monash_tourism_monthly,autogluon/chronos-bolt-mini,1.649587479665881,0.0979648261309891
24+
monash_tourism_quarterly,autogluon/chronos-bolt-mini,1.8471553663088986,0.06501077791766902
25+
monash_tourism_yearly,autogluon/chronos-bolt-mini,3.9932920493826245,0.1743539122097316
26+
monash_traffic,autogluon/chronos-bolt-mini,0.8355442361271347,0.24351051123330386
27+
monash_weather,autogluon/chronos-bolt-mini,0.800013628350165,0.13041050756802045
28+
nn5,autogluon/chronos-bolt-mini,0.611917632501032,0.1570111102680171

‎scripts/evaluation/results/chronos-bolt-small-agg-rel-scores.csv

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
benchmark,metric,value
2+
in-domain,MASE,0.7030801652116672
3+
in-domain,WQL,0.5443547623341555
4+
zero-shot,MASE,0.8192127745093378
5+
zero-shot,WQL,0.6356097843099521

‎scripts/evaluation/results/chronos-bolt-small-in-domain.csv

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
dataset,model,MASE,WQL
2+
electricity_15min,autogluon/chronos-bolt-small,0.44920089250026723,0.08115291306964295
3+
m4_daily,autogluon/chronos-bolt-small,3.201966619014735,0.02143368277732494
4+
m4_hourly,autogluon/chronos-bolt-small,0.8686298207618999,0.020368729287465817
5+
m4_monthly,autogluon/chronos-bolt-small,0.9537717737278778,0.0939247807527992
6+
m4_weekly,autogluon/chronos-bolt-small,2.1236755094789177,0.03785184715517262
7+
monash_electricity_hourly,autogluon/chronos-bolt-small,1.3728906161330452,0.09452411472431674
8+
monash_electricity_weekly,autogluon/chronos-bolt-small,1.8703239487242378,0.06648479071326366
9+
monash_kdd_cup_2018,autogluon/chronos-bolt-small,0.6458631909979771,0.25148489931571666
10+
monash_london_smart_meters,autogluon/chronos-bolt-small,0.7126939688565166,0.326874529903459
11+
monash_pedestrian_counts,autogluon/chronos-bolt-small,0.3015070035798365,0.2285590441093863
12+
monash_rideshare,autogluon/chronos-bolt-small,0.823726965741684,0.12409769473500927
13+
monash_temperature_rain,autogluon/chronos-bolt-small,0.8980348827836525,0.5984819599873311
14+
taxi_30min,autogluon/chronos-bolt-small,0.7597818149895785,0.2348569752311862
15+
uber_tlc_daily,autogluon/chronos-bolt-small,0.8460854328036702,0.09666483354735897
16+
uber_tlc_hourly,autogluon/chronos-bolt-small,0.6662547495017634,0.1524256346268063

‎scripts/evaluation/results/chronos-bolt-small-zero-shot.csv

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
dataset,model,MASE,WQL
2+
ETTh,autogluon/chronos-bolt-small,0.792521748651108,0.07590654063011319
3+
ETTm,autogluon/chronos-bolt-small,0.6209623928936988,0.05056189722606397
4+
dominick,autogluon/chronos-bolt-small,0.8706134610400587,0.34811141409475416
5+
ercot,autogluon/chronos-bolt-small,0.7562857616685997,0.02596064260343696
6+
exchange_rate,autogluon/chronos-bolt-small,1.774835301692689,0.011363548847621512
7+
m4_quarterly,autogluon/chronos-bolt-small,1.2478142413437487,0.07808795122806232
8+
m4_yearly,autogluon/chronos-bolt-small,3.6925595655002574,0.12772564181388502
9+
m5,autogluon/chronos-bolt-small,0.9195435643571084,0.5668430814831332
10+
monash_australian_electricity,autogluon/chronos-bolt-small,0.8128424798841111,0.041509852162861564
11+
monash_car_parts,autogluon/chronos-bolt-small,0.8584574663781737,1.0074689402521324
12+
monash_cif_2016,autogluon/chronos-bolt-small,1.0182471909074982,0.01581964877692293
13+
monash_covid_deaths,autogluon/chronos-bolt-small,36.467595559655145,0.0427382859406882
14+
monash_fred_md,autogluon/chronos-bolt-small,0.6132863794635253,0.03730410577241995
15+
monash_hospital,autogluon/chronos-bolt-small,0.6954489513780618,0.058119864671526154
16+
monash_m1_monthly,autogluon/chronos-bolt-small,1.1277621848099244,0.1335656174632902
17+
monash_m1_quarterly,autogluon/chronos-bolt-small,1.8356144904231688,0.09363028483838018
18+
monash_m1_yearly,autogluon/chronos-bolt-small,5.098146069746402,0.15669928873371905
19+
monash_m3_monthly,autogluon/chronos-bolt-small,0.8685125121306435,0.09396568468255145
20+
monash_m3_quarterly,autogluon/chronos-bolt-small,1.3269103591066727,0.07691022995374203
21+
monash_m3_yearly,autogluon/chronos-bolt-small,3.40993282700627,0.1547639821304127
22+
monash_nn5_weekly,autogluon/chronos-bolt-small,0.9266513350636507,0.08452821221908001
23+
monash_tourism_monthly,autogluon/chronos-bolt-small,1.6106732721197876,0.09362336754317802
24+
monash_tourism_quarterly,autogluon/chronos-bolt-small,1.8357819365308639,0.06734337535269994
25+
monash_tourism_yearly,autogluon/chronos-bolt-small,3.8963100495394194,0.16766064312072784
26+
monash_traffic,autogluon/chronos-bolt-small,0.8598507749866499,0.25173786112983054
27+
monash_weather,autogluon/chronos-bolt-small,0.8020408743877911,0.13258563963844888
28+
nn5,autogluon/chronos-bolt-small,0.5833047644729239,0.15066847836762787

0 commit comments

Comments
 (0)