Expected `rr1$aggregate()` to equal `rr2$aggregate()`.
Differences:
  `actual`: 0.1016
`expected`: 0.1043

expect_class(assert_measure(m, task = task), "Measure")
macro = msr("classif.fbeta", id = "macro")
micro = msr("classif.fbeta", id = "micro", average = "micro")
test_that("can set beta for fbeta during construction", {
learner = lrn("classif.featureless", predict_type = "prob")
expect_warning(m$score(p, task = task), "properties", fixed = TRUE)
expect_warning(p$score(m, task = task), "properties", fixed = TRUE)
rr = resample(task, learner, rsmp("holdout"))
expect_warning(res <- rr$score(m), "properties", fixed = TRUE)
expect_warning(res <- rr$aggregate(m), "properties", fixed = TRUE)
expect_warning(res <- bmr$score(m), "properties", fixed = TRUE)
expect_warning(res <- bmr$aggregate(m), "properties", fixed = TRUE)
expect_warning(m$score(p, learner = learner), "predict type", fixed = TRUE)
expect_warning(p$score(m, learner = learner), "predict type", fixed = TRUE)
expect_warning(res <- rr$score(m), "predict type", fixed = TRUE)
expect_warning(res <- rr$aggregate(m), "predict type", fixed = TRUE)
expect_warning(res <- bmr$score(m), "predict type", fixed = TRUE)
expect_warning(res <- bmr$aggregate(m), "predict type", fixed = TRUE)
learner = lrn("classif.featureless", predict_sets = "train")
expect_warning(res <- rr$score(m), "predict sets", fixed = TRUE)
expect_warning(res <- rr$aggregate(m), "predict sets", fixed = TRUE)
expect_warning(res <- bmr$score(m), "predict sets", fixed = TRUE)
expect_warning(res <- bmr$aggregate(m), "predict set", fixed = TRUE)
test_that("time_train works with different predict type (#832)", {
rr = resample(tsk("iris"), lrn("classif.debug", predict_type = "prob"), rsmp("holdout"))
rr = resample(tsk("iris"), lrn("classif.debug"), rsmp("holdout"))
test_that("scoring fails when measure requires_model, but model is in marshaled state", {
expect_error(measure$score(pred, learner = learner, task = task),
prediction_no_weights = learner$predict(tsk("iris"), row_ids = c(1, 150))
prediction_learner_weights = learner$predict(iris_weights_learner, row_ids = c(1, 150))  # should behave the same as no weights
prediction_measure_weights = learner$predict(iris_weights_measure, row_ids = c(1, 150))
m = msr("classif.acc", use_weights = "use")
expect_equal(prediction_measure_weights$score(m), c(classif.acc = 1 / 101))
expect_error(prediction_measure_weights$score(m), "'use_weights' was set to 'error'")
expect_error(m$score(prediction_measure_weights), "'use_weights' was set to 'error'")
prediction_no_weights = learner$predict(tsk("iris"), row_ids = c(1, 2, 51, 52, 101, 102))
prediction_learner_weights = learner$predict(iris_weights_learner, row_ids = c(1, 2, 51, 52, 101, 102))  # should behave the same as no weights
prediction_measure_weights = learner$predict(iris_weights_measure, row_ids = c(1, 2, 51, 52, 101, 102))
expect_error(prediction_measure_weights$score(mauc), "cannot be evaluated with weights")
expect_error(mauc$score(prediction_measure_weights), "cannot be evaluated with weights")
expect_error({mauc$use_weights = "use"}, "Must be element of set")
rr_no_weights = resample(tsk("iris"), learner, resampling)
rr_learner_weights = resample(iris_weights_learner, learner, resampling)
rr_measure_weights = resample(iris_weights_measure, learner, resampling)
expect_equal(rr_no_weights$score(m)$classif.acc, c(1 / 3, 3 / 8))
expect_equal(rr_learner_weights$score(m)$classif.acc, c(1 / 3, 3 / 8))
expect_equal(rr_measure_weights$score(m)$classif.acc, c(1 / 111, 3 / 233))
expect_equal(rr_measure_weights$score(m)$classif.acc, c(1 / 3, 3 / 8))
expect_error(rr_measure_weights$score(m), "since 'use_weights' was set to 'error'")
expect_equal(rr_no_weights$score(mauc)$classif.mauc_au1p, c(1, 1) / 2)
expect_equal(rr_learner_weights$score(mauc)$classif.mauc_au1p, c(1, 1) / 2)
expect_equal(rr_measure_weights$score(mauc)$classif.mauc_au1p, c(1, 1) / 2)
expect_error(rr_measure_weights$score(mauc), "cannot be evaluated with weights in\n  .*Task.*since the Measure does not support\n  weights")
expect_equal(rr_no_weights$aggregate(m), c(classif.acc = 1 / 6 + 3 / 16))
expect_equal(m$aggregate(rr_no_weights), c(classif.acc = 1 / 6 + 3 / 16))
expect_equal(rr_learner_weights$aggregate(m), c(classif.acc = 1 / 6 + 3 / 16))
expect_equal(m$aggregate(rr_learner_weights), c(classif.acc = 1 / 6 + 3 / 16))
expect_equal(rr_measure_weights$aggregate(m), c(classif.acc = 1 / 222 + 3 / 466))
expect_equal(m$aggregate(rr_measure_weights), c(classif.acc = 1 / 222 + 3 / 466))
expect_equal(rr_measure_weights$aggregate(m), c(classif.acc = 1 / 6 + 3 / 16))
expect_equal(m$aggregate(rr_measure_weights), c(classif.acc = 1 / 6 + 3 / 16))
expect_error(rr_measure_weights$aggregate(m), "since 'use_weights' was set to 'error'")
expect_error(m$aggregate(rr_measure_weights), "since 'use_weights' was set to 'error'")
expect_error(rr_measure_weights$aggregate(mauc), "cannot be evaluated with weights in\n  .*Task.*since the Measure does not support\n  weights")
expect_error(mauc$aggregate(rr_measure_weights), "cannot be evaluated with weights in\n  .*Task.*since the Measure does not support\n  weights")
expect_equal(rr_no_weights$aggregate(m), c(classif.acc = 5 / 14))
expect_equal(m$aggregate(rr_no_weights), c(classif.acc = 5 / 14))
expect_equal(rr_learner_weights$aggregate(m), c(classif.acc = 5 / 14))
expect_equal(m$aggregate(rr_learner_weights), c(classif.acc = 5 / 14))
expect_equal(rr_measure_weights$aggregate(m), c(classif.acc = 5 / 455))
expect_equal(m$aggregate(rr_measure_weights), c(classif.acc = 5 / 455))
expect_equal(rr_measure_weights$aggregate(m), c(classif.acc = 5 / 14))  # weighs by number of samples
expect_equal(m$aggregate(rr_measure_weights), c(classif.acc = 5 / 14))
test_that("primary iters are respected", {
train_sets = map(1:10, function(i) resampling$train_set(i))
test_sets = map(1:10, function(i) resampling$train_set(i))
r1 = rsmp("custom")$instantiate(task, train_sets = train_sets, test_sets = test_sets)
r2 = rsmp("custom")$instantiate(task, train_sets = train_sets[1:2], test_sets = test_sets[1:2])
r3 = rsmp("custom")$instantiate(task, train_sets = train_sets, test_sets = test_sets)
learner = lrn("classif.rpart", predict_type = "prob")
rr1 = resample(task, learner, r1, store_models = TRUE)
rr2 = resample(task, learner, r2, store_models = TRUE)
rr3 = resample(task, learner, r3, store_models = TRUE)
test_that("no predict_sets required (#1094)", {
rr = resample(tsk("iris"), lrn("classif.debug", validate = 0.3, predict_sets = NULL), rsmp("holdout"))
expect_error({m$predict_sets = NULL}, "Must be a subset")
expect_error({m$predict_sets = "imaginary"}, "Must be a subset")
test_that("measure and prediction type is checked", {
expect_warning(measure$score(pred), "is missing predict type")
super$initialize("test", task_type = "classif", param_set = ps(), properties = "obs_loss")
.score = function(prediction, task, weights = NULL, ...) {
.obs_loss = function(prediction, task, ...) {
expect_numeric(obs_loss, len = task$nrow, any.missing = FALSE)