test_that("freeze retains synthetic headline counts and equivalent leaders", { fixture <- calibration_freeze_fixture() frozen <- freeze_calibration_fixture(fixture) expect_s3_class(frozen, "ahtau_support_rule") expect_identical(frozen$rule_id, "B_p010_n10") expect_identical(frozen$family, "B") expect_equal(frozen$thresholds, list( p_risk_min = 0.10, n_risk_min = 10, p_support_min = 0.95 )) expect_equal(frozen$reliability_counts[c( "n_total", "n_reliable", "n_unreliable", "n_indeterminate" )], data.frame( n_total = 100, n_reliable = 92, n_unreliable = 1, n_indeterminate = 7 )) expect_equal(nrow(frozen$performance), 20) expect_setequal(frozen$selection$maximizing_rule_ids, c("B_p005_n10", "B_p010_n10")) expect_setequal(frozen$selection$equivalent_rule_ids, c("B_p005_n10", "B_p010_n10")) expect_equal(frozen$selected_performance$UPR_conservative, 0) expect_equal(frozen$selected_performance$APR_confirmed, 1) expect_equal(frozen$selection$classification_pattern_id, frozen$selected_performance$classification_pattern_id) }) test_that("existing helpers determine classifications and performance unchanged", { fixture <- calibration_freeze_fixture() expected_diagnostic <- classify_calib_reliability( fixture$diagnostic, availability_lower_column = "p_support_lower", availability_upper_column = "p_support_upper" ) expected_evaluation <- evaluate_calib_rules( expected_diagnostic, fixture$rules, availability_column = "p_support" ) frozen <- freeze_calibration_fixture(fixture) expect_identical(frozen$diagnostic, expected_diagnostic) expect_identical(frozen$performance, expected_evaluation$performance) expect_identical(frozen$pass_matrix, expected_evaluation$pass_matrix) expect_identical(frozen$reliability_counts, count_calib_reliability(expected_diagnostic)) expect_identical(frozen$selected_rule, fixture$rules[fixture$rules$rule_id == fixture$rule_id, , drop = FALSE]) }) test_that("the explicit choice is independent of candidate order", { fixture <- calibration_freeze_fixture() fixture$rules <- fixture$rules[rev(seq_len(nrow(fixture$rules))), ] frozen <- freeze_calibration_fixture(fixture, rule_id = "B_p005_n10") expect_identical(frozen$rule_id, "B_p005_n10") expect_equal(frozen$thresholds$p_risk_min, 0.05) expect_setequal(frozen$selection$maximizing_rule_ids, c("B_p005_n10", "B_p010_n10")) expect_identical(frozen$rationale, fixture$rationale) }) test_that("frozen thresholds are read from the selected rule", { fixture <- calibration_freeze_fixture() fixture$rules <- fixture$rules[3, ] fixture$rules$p_risk_threshold <- 0.12 fixture$rules$n_risk_threshold <- 11 fixture$rules$p_support_threshold <- 0.98 frozen <- freeze_calibration_fixture(fixture) expect_identical(frozen$thresholds, list( p_risk_min = 0.12, n_risk_min = 11, p_support_min = 0.98 )) }) test_that("numerical-only availability cannot affect support calibration", { fixture <- calibration_freeze_fixture() expected <- freeze_calibration_fixture(fixture) fixture$diagnostic$p_est <- 0 fixture$diagnostic$p_est_lower <- 0 fixture$diagnostic$p_est_upper <- 0.01 fixture$diagnostic$p_numeric <- 0 fixture$diagnostic$p_numeric_lower <- 0 fixture$diagnostic$p_numeric_upper <- 0.01 frozen <- freeze_calibration_fixture(fixture) expect_identical(frozen$pass_matrix, expected$pass_matrix) expect_identical(frozen$performance, expected$performance) expect_identical(frozen$diagnostic$reliability_status, expected$diagnostic$reliability_status) expect_identical(frozen$reliability_criteria$availability_column, "p_support") expect_identical(frozen$reliability_criteria$inference_conditioning, "support_valid") fixture$diagnostic$p_support <- NULL expect_error(freeze_calibration_fixture(fixture), "p_support") fixture <- calibration_freeze_fixture() fixture$diagnostic$p_est_lower <- fixture$diagnostic$p_support_lower fixture$diagnostic$p_support_lower <- NULL expect_error(freeze_calibration_fixture(fixture), "p_support_lower") }) test_that("support point estimates determine pass even with numerical availability", { fixture <- calibration_freeze_fixture() fixture$diagnostic$p_numeric <- 1 fixture$diagnostic$p_est <- 1 fixture$diagnostic$p_support[93:100] <- 0.90 fixture$diagnostic$n_support_valid[93:100] <- 900 fixture$diagnostic$p_support_lower[93:100] <- 0.88 fixture$diagnostic$p_support_upper[93:100] <- 0.92 frozen <- freeze_calibration_fixture(fixture) expect_equal(frozen$pass_matrix[, "A_p005"], c(rep(1, 92), rep(0, 8))) expect_equal(frozen$reliability_counts$n_unreliable, 8) expect_true("A_p005" %in% frozen$selection$maximizing_rule_ids) }) test_that("conservative UPR includes Indeterminate settings and uses pass denominator", { fixture <- calibration_freeze_fixture() frozen <- freeze_calibration_fixture(fixture) permissive <- frozen$performance[frozen$performance$rule_id == "A_p005", ] expect_equal(permissive$UPR_optimistic, 1 / 100) expect_equal(permissive$UPR_conservative, 8 / 100) expect_false("A_p005" %in% frozen$selection$eligible_rule_ids) fixture$rules <- fixture$rules[2:3, ] fixture$rules$n_risk_threshold <- 1 expect_error(freeze_calibration_fixture(fixture), "[Uu][Pp][Rr]|eligible|qualif") }) test_that("the conservative UPR cutoff includes exactly 0.05", { fixture <- calibration_freeze_fixture() fixture$diagnostic <- fixture$diagnostic[c(seq_len(19), 94), ] fixture$diagnostic$min_mean_n_risk <- 20 fixture$rules <- fixture$rules[3, ] frozen <- freeze_calibration_fixture(fixture) expect_equal(frozen$selected_performance$UPR_conservative, 0.05) expect_equal(frozen$selected_performance$APR_confirmed, 1) expect_equal(frozen$reliability_criteria$UPR_conservative_max, 0.05) fixture$diagnostic <- fixture$diagnostic[-1, ] expect_error(freeze_calibration_fixture(fixture), "[Uu][Pp][Rr]|eligible|qualif") }) test_that("an eligible rule must also maximize confirmed APR", { fixture <- calibration_freeze_fixture() frozen <- freeze_calibration_fixture(fixture) strict <- frozen$performance[frozen$performance$rule_id == "B_strict_1", ] expect_equal(strict$UPR_conservative, 0) expect_equal(strict$APR_confirmed, 12 / 92) expect_true("B_strict_1" %in% frozen$selection$eligible_rule_ids) expect_error(freeze_calibration_fixture(fixture, rule_id = "B_strict_1"), "maximiz|APR") }) test_that("non-equivalent maximizing ties require review", { fixture <- calibration_freeze_fixture() fixture$diagnostic <- fixture$diagnostic[c(1:4, 93), ] fixture$diagnostic$min_mean_p_risk <- c(0.2, 0.2, 0.1, 0.1, 0.01) fixture$diagnostic$min_mean_n_risk <- c(5, 5, 20, 20, 0) fixture$rules <- data.frame( rule_id = c("relative_leader", "absolute_leader"), family = "B", p_risk_threshold = c(0.15, 0.05), p_support_threshold = 0.95, use_absolute_n = TRUE, n_risk_threshold = c(1, 10) ) fixture$rule_id <- "relative_leader" expect_error(freeze_calibration_fixture(fixture), "non.equivalent|review") expect_error(freeze_calibration_fixture(fixture), "relative_leader") expect_error(freeze_calibration_fixture(fixture), "absolute_leader") }) test_that("zero passes and absent reliable settings cannot qualify", { fixture <- calibration_freeze_fixture() fixture$rules <- fixture$rules[3, ] fixture$rules$n_risk_threshold <- 100 expect_error(freeze_calibration_fixture(fixture), "pass|eligible|qualif") fixture <- calibration_freeze_fixture() fixture$diagnostic$type1_one_lower <- 0.07 fixture$diagnostic$type1_one_upper <- 0.09 expect_error(freeze_calibration_fixture(fixture), "[Rr]eliable|APR|eligible|qualif") }) test_that("selected unsafe or nonpassing candidates are rejected among valid leaders", { fixture <- calibration_freeze_fixture() extra_rule <- fixture$rules[3, ] extra_rule$rule_id <- "unsafe_B" extra_rule$n_risk_threshold <- 1 fixture$rules <- rbind(fixture$rules, extra_rule) expect_error(freeze_calibration_fixture(fixture, rule_id = "unsafe_B"), "ineligible.*UPR") fixture$rules$n_risk_threshold[fixture$rules$rule_id == "unsafe_B"] <- 100 expect_error(freeze_calibration_fixture(fixture, rule_id = "unsafe_B"), "ineligible.*UPR") }) test_that("rule identity and a selection rationale are required", { fixture <- calibration_freeze_fixture() for (invalid in list("missing", "", " ", NA, 3, c("B_p005_n10", "B_p010_n10"))) { expect_error(freeze_calibration_fixture(fixture, rule_id = invalid), "rule_id") } fixture$rule_id <- NULL expect_error(freeze_calibration_fixture(fixture), "rule_id") for (invalid in list("", " ", NA, 3, c("reason", "another"))) { expect_error(freeze_calibration_fixture(rationale = invalid), "rationale") } }) test_that("invalid diagnostic structure and duplicate settings are rejected", { fixture <- calibration_freeze_fixture() expect_error(freeze_calibration_fixture(diagnostic = list()), "diagnostic") expect_error(freeze_calibration_fixture(diagnostic = fixture$diagnostic[0, ]), "diagnostic|setting|row") expect_error(freeze_calibration_fixture(diagnostic = fixture$diagnostic[1, ]), "two|2|row|setting") for (column in c("scenario_id", "tau", "min_mean_p_risk", "min_mean_n_risk", "p_support", "p_support_lower", "p_support_upper", "type1_one_lower", "type1_one_upper", "coverage_lower", "coverage_upper")) { diagnostic <- fixture$diagnostic diagnostic[[column]] <- NULL expect_error(freeze_calibration_fixture(diagnostic = diagnostic), column) } diagnostic <- fixture$diagnostic diagnostic$scenario_id[2] <- diagnostic$scenario_id[1] expect_error(freeze_calibration_fixture(diagnostic = diagnostic), "unique|duplicate|once") diagnostic <- fixture$diagnostic diagnostic$scenario_id[1] <- NA expect_error(freeze_calibration_fixture(diagnostic = diagnostic), "scenario_id") }) test_that("required calibration fields contain one scalar value per row", { fixture <- calibration_freeze_fixture() for (column in c("min_mean_n_risk", "n_sim")) { diagnostic <- fixture$diagnostic diagnostic[[column]] <- matrix(10, nrow = nrow(diagnostic), ncol = 2) expect_error(freeze_calibration_fixture(diagnostic = diagnostic), "one value per row") } for (column in c("p_risk_threshold", "n_risk_threshold")) { rules <- fixture$rules rules[[column]] <- matrix(0.1, nrow = nrow(rules), ncol = 2) expect_error(freeze_calibration_fixture(rules = rules), "one value per row") } }) test_that("risk summaries and support availability must be valid numeric values", { fixture <- calibration_freeze_fixture() for (column in c("min_mean_p_risk", "min_mean_n_risk", "p_support")) { for (invalid in list(NA, Inf, -0.1, "0.2")) { diagnostic <- fixture$diagnostic diagnostic[[column]][1] <- invalid expect_error(freeze_calibration_fixture(diagnostic = diagnostic), column) } } for (column in c("min_mean_p_risk", "p_support")) { diagnostic <- fixture$diagnostic diagnostic[[column]][1] <- 1.1 expect_error(freeze_calibration_fixture(diagnostic = diagnostic), column) } }) test_that("missing confidence bounds retain the helper's indeterminate semantics", { fixture <- calibration_freeze_fixture() fixture$diagnostic$type1_one_lower <- NA fixture$diagnostic$coverage_upper <- NA fixture$diagnostic$p_support_upper <- NA frozen <- freeze_calibration_fixture(fixture) expect_equal(frozen$reliability_counts$n_reliable, 92) expect_equal(frozen$reliability_counts$n_unreliable, 0) expect_equal(frozen$reliability_counts$n_indeterminate, 8) expect_true(is.logical(frozen$diagnostic$type1_one_lower)) expect_true(all(is.na(frozen$diagnostic$type1_one_lower))) fixture <- calibration_freeze_fixture() fixture$diagnostic$type1_one_lower[94] <- NA fixture$diagnostic$type1_one_upper[94] <- NA frozen <- freeze_calibration_fixture(fixture) expect_identical(frozen$diagnostic$reliability_status[94], "Indeterminate") }) test_that("invalid confidence bounds are rejected", { fixture <- calibration_freeze_fixture() for (column in c("p_support_lower", "p_support_upper", "type1_one_lower", "type1_one_upper", "coverage_lower", "coverage_upper")) { for (invalid in list(-0.1, 1.1, Inf, "0.5", TRUE)) { diagnostic <- fixture$diagnostic diagnostic[[column]] <- rep(invalid, nrow(diagnostic)) expect_error(freeze_calibration_fixture(diagnostic = diagnostic), column) } } for (pair in list(c("p_support_lower", "p_support_upper"), c("type1_one_lower", "type1_one_upper"), c("coverage_lower", "coverage_upper"))) { diagnostic <- fixture$diagnostic diagnostic[[pair[1]]][1] <- 0.9 diagnostic[[pair[2]]][1] <- 0.1 expect_error(freeze_calibration_fixture(diagnostic = diagnostic), "lower|upper|interval") } }) test_that("optional simulation counts preserve all-attempt availability", { fixture <- calibration_freeze_fixture() fixture$diagnostic$n_sim <- rep(c(1000, 2000), 50) fixture$diagnostic$n_support_valid <- rep(c(990, 1980), 50) frozen <- freeze_calibration_fixture(fixture) expect_identical(frozen$diagnostic$n_sim, fixture$diagnostic$n_sim) expect_identical(frozen$diagnostic$n_support_valid, fixture$diagnostic$n_support_valid) expect_equal(frozen$selected_performance$n_reliable, 92) fixture$diagnostic$n_sim <- NULL fixture$diagnostic$n_support_valid <- NULL expect_s3_class(freeze_calibration_fixture(fixture), "ahtau_support_rule") for (column in c("n_sim", "n_support_valid")) { diagnostic <- calibration_freeze_fixture()$diagnostic diagnostic[[column]] <- NULL expect_error(freeze_calibration_fixture(diagnostic = diagnostic), "n_sim|n_support_valid") } for (column in c("n_sim", "n_support_valid")) { for (invalid in list(-1, 1.5, NA, Inf, "1000")) { diagnostic <- calibration_freeze_fixture()$diagnostic diagnostic[[column]][1] <- invalid expect_error(freeze_calibration_fixture(diagnostic = diagnostic), "n_sim|n_support_valid|count") } } diagnostic <- calibration_freeze_fixture()$diagnostic diagnostic$n_support_valid[1] <- 1001 expect_error(freeze_calibration_fixture(diagnostic = diagnostic), "n_sim|n_support_valid") diagnostic <- calibration_freeze_fixture()$diagnostic diagnostic$p_support[1] <- 0.98 expect_error(freeze_calibration_fixture(diagnostic = diagnostic), "p_support") }) test_that("candidate rules require complete and unambiguous fields", { fixture <- calibration_freeze_fixture() expect_error(freeze_calibration_fixture(rules = list()), "rules") expect_error(freeze_calibration_fixture(rules = fixture$rules[0, ]), "rules") for (column in names(fixture$rules)) { rules <- fixture$rules rules[[column]] <- NULL expect_error(freeze_calibration_fixture(rules = rules), column) } rules <- fixture$rules rules$rule_id[1] <- rules$rule_id[2] expect_error(freeze_calibration_fixture(rules = rules), "unique|duplicate") for (column in c("rule_id", "family")) { for (invalid in list("", " ", NA)) { rules <- fixture$rules rules[[column]][1] <- invalid expect_error(freeze_calibration_fixture(rules = rules), column) } } for (column in c("p_risk_threshold", "p_support_threshold")) { for (invalid in list(NA, Inf, -0.1, 1.1, "0.5")) { rules <- fixture$rules rules[[column]][1] <- invalid expect_error(freeze_calibration_fixture(rules = rules), column) } } for (invalid in list(NA, 1, "TRUE")) { rules <- fixture$rules rules$use_absolute_n <- rep(invalid, nrow(rules)) expect_error(freeze_calibration_fixture(rules = rules), "use_absolute_n") } for (invalid in list(NA, Inf, -1, "10")) { rules <- fixture$rules rules$n_risk_threshold[3] <- invalid expect_error(freeze_calibration_fixture(rules = rules), "n_risk_threshold") } }) test_that("factor rule labels are supported", { fixture <- calibration_freeze_fixture() fixture$rules$rule_id <- factor(fixture$rules$rule_id) fixture$rules$family <- factor(fixture$rules$family) frozen <- freeze_calibration_fixture(fixture) expect_equal(as.character(frozen$rule_id), "B_p010_n10") expect_equal(as.character(frozen$family), "B") expect_equal(frozen$thresholds$p_risk_min, 0.10) }) test_that("Family A selection identifies the Stage 1 interface limitation", { fixture <- calibration_freeze_fixture() fixture$diagnostic <- fixture$diagnostic[1:3, ] fixture$rules <- fixture$rules[1, ] fixture$rule_id <- "A_p005" expect_error(freeze_calibration_fixture(fixture), "Stage 1|stage1_audit") expect_error(freeze_calibration_fixture(fixture), "absolute|three.criterion") }) test_that("Family A candidates remain in statistical selection", { fixture <- calibration_freeze_fixture() fixture$diagnostic$min_mean_n_risk[1:80] <- 5 fixture$diagnostic$min_mean_p_risk[93:100] <- 0.01 # Family A now passes all 92 Reliable settings while each B rule passes 12. expect_error(freeze_calibration_fixture(fixture), "maximize.*APR") expect_error(freeze_calibration_fixture(fixture), "A_p005") }) test_that("calibration metadata are preserved without replacement", { fixture <- calibration_freeze_fixture() frozen <- freeze_calibration_fixture(fixture) expect_identical(frozen$rationale, fixture$rationale) expect_identical(frozen$calibration_scope, fixture$calibration_scope) expect_identical(frozen$provenance, fixture$provenance) expect_identical(frozen$simulation_settings, fixture$simulation_settings) expect_identical(frozen$diagnostic[names(fixture$diagnostic)], fixture$diagnostic) expect_equal(frozen$reliability_criteria$type1_tolerance, 0.06) expect_equal(frozen$reliability_criteria$coverage_tolerance, 0.94) expect_equal(frozen$reliability_criteria$support_tolerance, 0.95) }) test_that("scope, provenance, settings and support conditioning are explicit", { for (argument in c("calibration_scope", "provenance", "simulation_settings")) { for (invalid in list(list(), list("unnamed"), "not a list")) { fixture <- calibration_freeze_fixture() fixture[[argument]] <- invalid expect_error(freeze_calibration_fixture(fixture), argument) } fixture <- calibration_freeze_fixture() fixture[[argument]] <- NULL expect_error(freeze_calibration_fixture(fixture), argument) } for (invalid in list(NULL, "numeric_valid", "p_numeric", NA)) { fixture <- calibration_freeze_fixture() fixture$provenance$inference_conditioning <- invalid expect_error(freeze_calibration_fixture(fixture), "inference_conditioning|support.valid") } fixture <- calibration_freeze_fixture() fixture$provenance$inference_conditioning <- NULL fixture$provenance$inference_conditioning_for_another_analysis <- "support_valid" expect_error(freeze_calibration_fixture(fixture), "inference_conditioning") }) test_that("reliability tolerances are explicit and applied by the existing helper", { fixture <- calibration_freeze_fixture() frozen <- freeze_calibration_fixture( fixture, type1_tolerance = 0.055, coverage_tolerance = 0.945, support_tolerance = 0.96 ) expected <- classify_calib_reliability( fixture$diagnostic, type1_tolerance = 0.055, coverage_tolerance = 0.945, support_tolerance = 0.96, availability_lower_column = "p_support_lower", availability_upper_column = "p_support_upper" ) expect_identical(frozen$diagnostic, expected) expect_equal(frozen$reliability_criteria$type1_tolerance, 0.055) expect_equal(frozen$reliability_criteria$coverage_tolerance, 0.945) expect_equal(frozen$reliability_criteria$support_tolerance, 0.96) for (argument in c("type1_tolerance", "coverage_tolerance", "support_tolerance")) { for (invalid in list(NA, Inf, -0.1, 1.1, "0.5", c(0.5, 0.6))) { args <- fixture args[[argument]] <- invalid expect_error(do.call(freeze_support_rule, args), argument) } } }) test_that("frozen thresholds work directly in Stage 1 without changing denominators", { frozen <- freeze_calibration_fixture() raw <- data.frame( scenario_id = rep(c("pass", "count_fail", "relative_fail", "support_fail"), each = 2), tau = 3, theta_hat = 0, SE_theta = 1, n_risk_C = c(10, 10, 9, 9, 10, 10, 10, 10), n_risk_T = c(10, 10, 9, 9, 10, 10, 10, 10), p_risk_C = c(0.10, 0.10, 0.2, 0.2, 0.09, 0.09, 0.2, 0.2), p_risk_T = c(0.10, 0.10, 0.2, 0.2, 0.09, 0.09, 0.2, 0.2), reject = 0 ) raw$SE_theta[8] <- NA result <- stage1_audit(raw, thresholds = frozen$thresholds)$scenario_summary result <- result[match(c("pass", "count_fail", "relative_fail", "support_fail"), result$scenario_id), ] expect_identical(names(frozen$thresholds), c("p_risk_min", "n_risk_min", "p_support_min")) expect_equal(result$Stage1_pass, c(TRUE, FALSE, FALSE, FALSE)) expect_equal(result$Stage1_failed_reason, c("none", "low_absolute_support", "low_relative_support", "low_support_availability")) expect_equal(result$n_sim, rep(2, 4)) expect_equal(result$p_support, c(1, 1, 1, 0.5)) expect_true("thresholds" %in% names(formals(find_n_req))) }) test_that("support summaries feed freeze with original denominators and conditional CIs", { fixture <- calibration_freeze_fixture() # Fixed trial-analysis rows; no random draws or Monte Carlo simulation. raw <- data.frame( scenario_id = rep(c("supported", "empty_tail"), each = 100), sim_id = rep(seq_len(100), 2), tau = 3, theta_hat = 0, SE_theta = 1, n_risk_C = c(rep(20, 100), rep(0, 100)), n_risk_T = c(rep(20, 100), rep(0, 100)), p_risk_C = c(rep(0.2, 100), rep(0, 100)), p_risk_T = c(rep(0.2, 100), rep(0, 100)), reject = c(rep(0, 100), rep(NA, 100)), covered = c(rep(1, 100), rep(NA, 100)) ) fixture$diagnostic <- summarize_calibration_support(raw) frozen <- freeze_calibration_fixture(fixture) supported <- match("supported", frozen$diagnostic$scenario_id) empty_tail <- match("empty_tail", frozen$diagnostic$scenario_id) expect_identical(frozen$diagnostic[names(fixture$diagnostic)], fixture$diagnostic) expect_equal(frozen$diagnostic$n_sim, c(100, 100)) expect_equal(frozen$diagnostic$p_numeric, c(1, 1)) expect_equal(frozen$diagnostic$n_support_valid[c(supported, empty_tail)], c(100, 0)) expect_equal(frozen$diagnostic$p_support[c(supported, empty_tail)], c(1, 0)) expect_equal(frozen$diagnostic$type1_one_upper[supported], stats::binom.test(0, 100)$conf.int[2]) expect_equal(frozen$diagnostic$coverage_lower[supported], stats::binom.test(100, 100)$conf.int[1]) expect_true(is.na(frozen$diagnostic$type1_one_upper[empty_tail])) expect_equal(frozen$diagnostic$reliability_status[c(supported, empty_tail)], c("Reliable", "Unreliable")) expect_equal(frozen$selected_performance$UPR_conservative, 0) expect_equal(frozen$selected_performance$APR_confirmed, 1) })