test_that("ai_generalizability + ai_dstudy run end to end on a crossed design", { skip_if_not_installed("lme4") set.seed(42) d <- expand.grid(case = 1:10, prompt = 1:2, model = 1:2, run = 1:2) d$score <- 5 + rnorm(10, sd = 1)[d$case] + rnorm(nrow(d), sd = 0.4) g <- suppressWarnings(ai_generalizability(d)) expect_s3_class(g, "aiEvalR_gtheory") expect_true("case" %in% g$variance_components$component) ds <- ai_dstudy(g, n_prompt = 2, n_model = 2, n_run = 2) expect_s3_class(ds, "aiEvalR_dstudy") expect_true(ds$phi >= 0 && ds$phi <= 1) expect_true(ds$g_coef >= 0 && ds$g_coef <= 1) }) test_that("ai_dstudy with fewer conditions yields lower or equal Phi", { skip_if_not_installed("lme4") set.seed(7) d <- expand.grid(case = 1:10, prompt = 1:3, model = 1:2, run = 1:2) d$score <- 5 + rnorm(10, sd = 1)[d$case] + rnorm(nrow(d), sd = 0.5) g <- suppressWarnings(ai_generalizability(d)) ds_full <- ai_dstudy(g, n_prompt = 3, n_model = 2, n_run = 2) ds_reduced <- ai_dstudy(g, n_prompt = 1, n_model = 1, n_run = 1) expect_true(ds_reduced$phi <= ds_full$phi) }) test_that("ai_generalizability errors on missing required columns", { skip_if_not_installed("lme4") d <- data.frame(x = 1:5) expect_error(ai_generalizability(d), "Missing column") }) ## ---- print methods, conditional SEM, decision consistency ---------- test_that("print.aiEvalR_gtheory and print.aiEvalR_dstudy run without error", { skip_if_not_installed("lme4") set.seed(30) d <- expand.grid(case = 1:10, prompt = 1:2, model = 1:2, run = 1:2) d$score <- 5 + rnorm(10, sd = 1)[d$case] + rnorm(nrow(d), sd = 0.4) g <- suppressWarnings(ai_generalizability(d)) expect_output(print(g), "Variance Components") ds <- ai_dstudy(g, n_prompt = 2, n_model = 2, n_run = 2) expect_output(print(ds), "D-Study") }) test_that("ai_conditional_sem returns per-bin CSEM as a data frame", { set.seed(31) # rows = prompts, cols = occasions; larger spread at higher score levels responses <- t(sapply(1:40, function(i) { mu <- i / 10 rnorm(4, mean = mu, sd = 0.1 + mu * 0.05) })) out <- ai_conditional_sem(responses, n_bins = 4) expect_s3_class(out, "data.frame") expect_true(all(c("score_bin_mid", "csem", "n") %in% names(out))) expect_true(all(out$csem >= 0)) }) test_that("ai_decision_consistency is perfect when all occasions agree", { # every prompt is clearly on one side of the cutpoint on every occasion responses <- rbind( c(10, 10, 10), c(0, 0, 0), c(9, 9, 9), c(1, 1, 1) ) out <- ai_decision_consistency(responses, cutpoint = 5) expect_equal(out$consistency, 1) expect_equal(out$kappa, 1) }) test_that("ai_decision_consistency detects inconsistent decisions", { # prompt 1 flips across occasions relative to cutpoint 5 responses <- rbind( c(4, 6, 4), c(0, 0, 0), c(10, 10, 10) ) out <- ai_decision_consistency(responses, cutpoint = 5) expect_lt(out$consistency, 1) }) test_that("ai_decision_consistency returns NA kappa with a single occasion", { responses <- matrix(c(10, 0, 9), ncol = 1) out <- ai_decision_consistency(responses, cutpoint = 5) expect_true(is.na(out$kappa)) }) test_that("ai_generalizability errors without lme4-style data columns", { skip_if_not_installed("lme4") expect_error(ai_generalizability(data.frame(score = 1:5, case = 1:5), facets = c("prompt")), "Missing column") })