test_that("Calculations are correct - two class", { # BrierScore(two_class_example |> # dplyr::select(Class1, Class2) |> # as.matrix, two_class_example$truth) expect_equal( brier_class_vec(two_class_example$truth, two_class_example$Class1), 0.10561859, tolerance = 0.01 ) }) test_that("Calculations are correct - multi class", { # With the mclust pakcage, BrierScore(hpc_cv |> select(VF:L) |> as.matrix, hpc_cv$obs) hpc_exp <- 0.21083946 expect_equal( brier_class(hpc_cv, obs, VF:L)[[".estimate"]], hpc_exp, tolerance = 0.01 ) }) test_that("Calculations handles NAs", { hpc_cv$VF[1:10] <- NA expect_equal( brier_class(hpc_cv, obs, VF:L)[[".estimate"]], 0.21143119 ) expect_equal( brier_class(hpc_cv, obs, VF:L, na_rm = FALSE)[[".estimate"]], NA_real_ ) }) test_that("Case weights calculations are correct", { wts <- rep(1, nrow(hpc_cv)) wts[1] <- 5 hpc_wts <- hpc_cv[c(rep(1, 4), seq_len(nrow(hpc_cv))), ] expect_equal( brier_class(hpc_cv, obs, VF:L), brier_class(hpc_wts, obs, VF:L), tolerance = 0.01 ) }) test_that("works with hardhat case weights", { df <- two_class_example imp_wgt <- hardhat::importance_weights(seq_len(nrow(df))) freq_wgt <- hardhat::frequency_weights(seq_len(nrow(df))) expect_no_error( brier_class_vec(df$truth, df$Class1, case_weights = imp_wgt) ) expect_no_error( brier_class_vec(df$truth, df$Class1, case_weights = freq_wgt) ) }) test_that("errors with class_pred input", { skip_if_not_installed("probably") cp_truth <- probably::as_class_pred(two_class_example$truth, which = 1) fct_truth <- two_class_example$truth fct_truth[1] <- NA estimate <- two_class_example$Class1 expect_snapshot( error = TRUE, brier_class_vec(cp_truth, estimate) ) }) test_that("na_rm argument check", { expect_snapshot( error = TRUE, brier_class_vec(1, 1, na_rm = "yes") ) }) test_that("range values are correct", { direction <- metric_direction(brier_class) range <- metric_range(brier_class) perfect <- ifelse(direction == "minimize", range[1], range[2]) worst <- ifelse(direction == "minimize", range[2], range[1]) df <- tibble::tibble( truth = factor(c("a", "a", "a", "b", "b"), levels = c("a", "b")), perfect = c(1, 1, 1, 0, 0), off = c(0.5, 0.5, 0.5, 0.5, 0.5) ) expect_equal(brier_class_vec(df$truth, df$perfect), perfect) if (direction == "minimize") { expect_gt(brier_class_vec(df$truth, df$off), perfect) expect_lte(brier_class_vec(df$truth, df$off), worst) } if (direction == "maximize") { expect_lt(brier_class_vec(df$truth, df$off), perfect) expect_gte(brier_class_vec(df$truth, df$off), worst) } }) test_that("doesn't produce NaN with high valued case weights (#614)", { df <- two_class_example wts <- rep(1, (nrow(df))) wts_high <- wts * 5000 expect_identical( brier_class_vec(df$truth, df$Class1, case_weights = wts), brier_class_vec(df$truth, df$Class1, case_weights = wts_high) ) }) test_that("`event_level = 'second'` works", { df <- two_class_example df_rev <- df df_rev$truth <- stats::relevel(df_rev$truth, "Class2") expect_equal( brier_class_vec(df$truth, df$Class1), brier_class_vec(df_rev$truth, df_rev$Class1, event_level = "second") ) expect_true( brier_class_vec(df_rev$truth, df_rev$Class1, event_level = "first") != brier_class_vec(df_rev$truth, df_rev$Class1, event_level = "second") ) })