Fix metric checking for quantile regression (#1187)

* changes for #1186

* news file entry

* fix test name

* Update R/metric-selection.R

Co-authored-by: Hannah Frick <hfrick@users.noreply.github.com>

* changes based on reviewer comments

---------

Co-authored-by: Hannah Frick <hfrick@users.noreply.github.com>

Author: topepo

.Rbuildignore

@@ -21,3 +21,4 @@ _cache$
 ^\.vscode$
 ^[.]?air[.]toml$
 ^\.claude$
+^\.positai$

.gitignore

@@ -13,3 +13,4 @@ tests/testthat/Rplots.pdf
 docs
 revdep
 tests/testthat/_snaps/notes.new.md
+.positai

NEWS.md

@@ -1,5 +1,9 @@
 # tune (development version)
 
+## Bug Fixes
+
+* Resampling and tuning would fail for quantile regression models if they passed a quantile regression metric (#1186)
+
 # tune 2.1.0
 
 * Model tuning has been enabled for quantile regression models. (#1125)

R/metric-selection.R

@@ -330,43 +330,46 @@ check_metrics_arg <- function(mtr_set, wflow, ..., call = rlang::caller_env()) {
     return(mtr_set)
   }
 
+  if (!inherits(mtr_set, "metric_set")) {
+    cli::cli_abort(
+      "The {.arg metrics} argument should have class {.cls metric_set}, not {.cls {class(mtr_set)}}.",
+      call = call
+    )
+  }
+
   is_numeric_metric_set <- inherits(mtr_set, "numeric_metric_set")
   is_class_prob_metric_set <- inherits(mtr_set, "class_prob_metric_set")
   is_surv_metric_set <- inherits(mtr_set, c("survival_metric_set"))
+  is_qnt_metric_set <- inherits(mtr_set, c("quantile_metric_set"))
 
-  if (
-    !is_numeric_metric_set && !is_class_prob_metric_set && !is_surv_metric_set
-  ) {
+  if (mode == "regression" && !is_numeric_metric_set) {
     cli::cli_abort(
-      "The {.arg metrics} argument should be the results of
-                   {.fn yardstick::metric_set}.",
+      "The parsnip model has {.code mode} value of {.val {mode}},
+       but the metric set for a different model mode.",
       call = call
     )
   }
 
-  if (mode == "regression" && !is_numeric_metric_set) {
+  if (mode == "classification" && !is_class_prob_metric_set) {
     cli::cli_abort(
       "The parsnip model has {.code mode} value of {.val {mode}},
-                   but the {.arg metrics} is a metric set for a
-                   different model mode.",
+       but the metric set for a different model mode.",
       call = call
     )
   }
 
-  if (mode == "classification" && !is_class_prob_metric_set) {
+  if (mode == "censored regression" && !is_surv_metric_set) {
     cli::cli_abort(
       "The parsnip model has {.code mode} value of {.val {mode}},
-                   but the {.arg metrics} is a metric set for a
-                   different model mode.",
+       but the metric set for a different model mode.",
       call = call
     )
   }
 
-  if (mode == "censored regression" && !is_surv_metric_set) {
+  if (mode == "quantile regression" && !is_qnt_metric_set) {
     cli::cli_abort(
       "The parsnip model has {.code mode} value of {.val {mode}},
-                   but the {.arg metrics} is a metric set for a
-                   different model mode.",
+       but the metric set for a different model mode.",
       call = call
     )
   }

tests/testthat/_snaps/censored-reg.md

@@ -12,7 +12,7 @@
       tune_grid(spec, Surv(time, status) ~ ., resamples = rs, metrics = reg_mtr)
     Condition
       Error in `tune_grid()`:
-      ! The parsnip model has `mode` value of "censored regression", but the `metrics` is a metric set for a different model mode.
+      ! The parsnip model has `mode` value of "censored regression", but the metric set for a different model mode.
 
 ---
 

tests/testthat/_snaps/metric-args.md

@@ -7,6 +7,14 @@
       - `rmse()`, a numeric metric | direction: minimize
       - `rsq()`, a numeric metric  | direction: maximize
 
+---
+
+    Code
+      check_metrics_arg(rmse, wflow)
+    Condition
+      Error:
+      ! The `metrics` argument should have class <metric_set>, not <numeric_metric/metric/function>.
+
 ---
 
     Code
@@ -21,79 +29,79 @@
       check_metrics_arg(met_cls, wflow)
     Condition
       Error:
-      ! The parsnip model has `mode` value of "regression", but the `metrics` is a metric set for a different model mode.
+      ! The parsnip model has `mode` value of "regression", but the metric set for a different model mode.
 
 ---
 
     Code
       check_metrics_arg(met_mix_int, wflow)
     Condition
       Error:
-      ! The parsnip model has `mode` value of "regression", but the `metrics` is a metric set for a different model mode.
+      ! The parsnip model has `mode` value of "regression", but the metric set for a different model mode.
 
 ---
 
     Code
       fit_resamples(wflow, rs, metrics = met_cls)
     Condition
       Error in `fit_resamples()`:
-      ! The parsnip model has `mode` value of "regression", but the `metrics` is a metric set for a different model mode.
+      ! The parsnip model has `mode` value of "regression", but the metric set for a different model mode.
 
 ---
 
     Code
       fit_resamples(wflow, rs, metrics = met_mix_int)
     Condition
       Error in `fit_resamples()`:
-      ! The parsnip model has `mode` value of "regression", but the `metrics` is a metric set for a different model mode.
+      ! The parsnip model has `mode` value of "regression", but the metric set for a different model mode.
 
 ---
 
     Code
       tune_grid(wflow_tune, rs, metrics = met_cls)
     Condition
       Error in `tune_grid()`:
-      ! The parsnip model has `mode` value of "regression", but the `metrics` is a metric set for a different model mode.
+      ! The parsnip model has `mode` value of "regression", but the metric set for a different model mode.
 
 ---
 
     Code
       tune_grid(wflow_tune, rs, metrics = met_mix_int)
     Condition
       Error in `tune_grid()`:
-      ! The parsnip model has `mode` value of "regression", but the `metrics` is a metric set for a different model mode.
+      ! The parsnip model has `mode` value of "regression", but the metric set for a different model mode.
 
 ---
 
     Code
       tune_bayes(wflow_tune, rs, metrics = met_cls)
     Condition
       Error in `tune_bayes()`:
-      ! The parsnip model has `mode` value of "regression", but the `metrics` is a metric set for a different model mode.
+      ! The parsnip model has `mode` value of "regression", but the metric set for a different model mode.
 
 ---
 
     Code
       tune_bayes(wflow_tune, rs, metrics = met_mix_int)
     Condition
       Error in `tune_bayes()`:
-      ! The parsnip model has `mode` value of "regression", but the `metrics` is a metric set for a different model mode.
+      ! The parsnip model has `mode` value of "regression", but the metric set for a different model mode.
 
 ---
 
     Code
       last_fit(wflow, split, metrics = met_cls)
     Condition
       Error in `last_fit()`:
-      ! The parsnip model has `mode` value of "regression", but the `metrics` is a metric set for a different model mode.
+      ! The parsnip model has `mode` value of "regression", but the metric set for a different model mode.
 
 ---
 
     Code
       last_fit(wflow, split, metrics = met_mix_int)
     Condition
       Error in `last_fit()`:
-      ! The parsnip model has `mode` value of "regression", but the `metrics` is a metric set for a different model mode.
+      ! The parsnip model has `mode` value of "regression", but the metric set for a different model mode.
 
 # metric inputs are checked for classification models
 
@@ -111,7 +119,7 @@
       check_metrics_arg(met_reg, wflow)
     Condition
       Error:
-      ! The parsnip model has `mode` value of "classification", but the `metrics` is a metric set for a different model mode.
+      ! The parsnip model has `mode` value of "classification", but the metric set for a different model mode.
 
 ---
 
@@ -127,71 +135,71 @@
       check_metrics_arg(met_mix_int, wflow)
     Condition
       Error:
-      ! The parsnip model has `mode` value of "classification", but the `metrics` is a metric set for a different model mode.
+      ! The parsnip model has `mode` value of "classification", but the metric set for a different model mode.
 
 ---
 
     Code
       fit_resamples(wflow, rs, metrics = met_reg)
     Condition
       Error in `fit_resamples()`:
-      ! The parsnip model has `mode` value of "classification", but the `metrics` is a metric set for a different model mode.
+      ! The parsnip model has `mode` value of "classification", but the metric set for a different model mode.
 
 ---
 
     Code
       fit_resamples(wflow, rs, metrics = met_mix_int)
     Condition
       Error in `fit_resamples()`:
-      ! The parsnip model has `mode` value of "classification", but the `metrics` is a metric set for a different model mode.
+      ! The parsnip model has `mode` value of "classification", but the metric set for a different model mode.
 
 ---
 
     Code
       tune_grid(wflow_tune, rs, metrics = met_reg)
     Condition
       Error in `tune_grid()`:
-      ! The parsnip model has `mode` value of "classification", but the `metrics` is a metric set for a different model mode.
+      ! The parsnip model has `mode` value of "classification", but the metric set for a different model mode.
 
 ---
 
     Code
       tune_grid(wflow_tune, rs, metrics = met_mix_int)
     Condition
       Error in `tune_grid()`:
-      ! The parsnip model has `mode` value of "classification", but the `metrics` is a metric set for a different model mode.
+      ! The parsnip model has `mode` value of "classification", but the metric set for a different model mode.
 
 ---
 
     Code
       tune_bayes(wflow_tune, rs, metrics = met_reg)
     Condition
       Error in `tune_bayes()`:
-      ! The parsnip model has `mode` value of "classification", but the `metrics` is a metric set for a different model mode.
+      ! The parsnip model has `mode` value of "classification", but the metric set for a different model mode.
 
 ---
 
     Code
       tune_bayes(wflow_tune, rs, metrics = met_mix_int)
     Condition
       Error in `tune_bayes()`:
-      ! The parsnip model has `mode` value of "classification", but the `metrics` is a metric set for a different model mode.
+      ! The parsnip model has `mode` value of "classification", but the metric set for a different model mode.
 
 ---
 
     Code
       last_fit(wflow, split, metrics = met_reg)
     Condition
       Error in `last_fit()`:
-      ! The parsnip model has `mode` value of "classification", but the `metrics` is a metric set for a different model mode.
+      ! The parsnip model has `mode` value of "classification", but the metric set for a different model mode.
 
 ---
 
     Code
       last_fit(wflow, split, metrics = met_mix_int)
     Condition
       Error in `last_fit()`:
-      ! The parsnip model has `mode` value of "classification", but the `metrics` is a metric set for a different model mode.
+      ! The parsnip model has `mode` value of "classification", but the metric set for a different model mode.
 
 # metric inputs are checked for censored regression models
 
@@ -207,15 +215,15 @@
       check_metrics_arg(met_reg, wflow)
     Condition
       Error:
-      ! The parsnip model has `mode` value of "censored regression", but the `metrics` is a metric set for a different model mode.
+      ! The parsnip model has `mode` value of "censored regression", but the metric set for a different model mode.
 
 ---
 
     Code
       check_metrics_arg(met_cls, wflow)
     Condition
       Error:
-      ! The parsnip model has `mode` value of "censored regression", but the `metrics` is a metric set for a different model mode.
+      ! The parsnip model has `mode` value of "censored regression", but the metric set for a different model mode.
 
 ---
 
@@ -231,61 +239,85 @@
       fit_resamples(wflow, rs, metrics = met_cls)
     Condition
       Error in `fit_resamples()`:
-      ! The parsnip model has `mode` value of "censored regression", but the `metrics` is a metric set for a different model mode.
+      ! The parsnip model has `mode` value of "censored regression", but the metric set for a different model mode.
 
 ---
 
     Code
       fit_resamples(wflow, rs, metrics = met_reg)
     Condition
       Error in `fit_resamples()`:
-      ! The parsnip model has `mode` value of "censored regression", but the `metrics` is a metric set for a different model mode.
+      ! The parsnip model has `mode` value of "censored regression", but the metric set for a different model mode.
 
 ---
 
     Code
       tune_grid(wflow_tune, rs, metrics = met_cls)
     Condition
       Error in `tune_grid()`:
-      ! The parsnip model has `mode` value of "censored regression", but the `metrics` is a metric set for a different model mode.
+      ! The parsnip model has `mode` value of "censored regression", but the metric set for a different model mode.
 
 ---
 
     Code
       tune_grid(wflow_tune, rs, metrics = met_reg)
     Condition
       Error in `tune_grid()`:
-      ! The parsnip model has `mode` value of "censored regression", but the `metrics` is a metric set for a different model mode.
+      ! The parsnip model has `mode` value of "censored regression", but the metric set for a different model mode.
 
 ---
 
     Code
       tune_bayes(wflow_tune, rs, metrics = met_cls)
     Condition
       Error in `tune_bayes()`:
-      ! The parsnip model has `mode` value of "censored regression", but the `metrics` is a metric set for a different model mode.
+      ! The parsnip model has `mode` value of "censored regression", but the metric set for a different model mode.
 
 ---
 
     Code
       tune_bayes(wflow_tune, rs, metrics = met_reg)
     Condition
       Error in `tune_bayes()`:
-      ! The parsnip model has `mode` value of "censored regression", but the `metrics` is a metric set for a different model mode.
+      ! The parsnip model has `mode` value of "censored regression", but the metric set for a different model mode.
 
 ---
 
     Code
       last_fit(wflow, split, metrics = met_cls)
     Condition
       Error in `last_fit()`:
-      ! The parsnip model has `mode` value of "censored regression", but the `metrics` is a metric set for a different model mode.
+      ! The parsnip model has `mode` value of "censored regression", but the metric set for a different model mode.
 
 ---
 
     Code
       last_fit(wflow, split, metrics = met_reg)
     Condition
       Error in `last_fit()`:
-      ! The parsnip model has `mode` value of "censored regression", but the `metrics` is a metric set for a different model mode.
+      ! The parsnip model has `mode` value of "censored regression", but the metric set for a different model mode.
+
+# metric inputs are checked for quantile regression models
+
+    Code
+      check_metrics_arg(NULL, wflow)
+    Output
+      A metric set, consisting of:
+      - `weighted_interval_score()`, a quantile metric | direction: minimize
+
+---
+
+    Code
+      check_metrics_arg(metric_set(rmse), wflow)
+    Condition
+      Error:
+      ! The parsnip model has `mode` value of "quantile regression", but the metric set for a different model mode.
+
+---
+
+    Code
+      check_metrics_arg(metric_set(weighted_interval_score), wflow)
+    Output
+      A metric set, consisting of:
+      - `weighted_interval_score()`, a quantile metric | direction: minimize