test_that("cindex_harrell: handles edge cases", { # internal function, called through the same namespace via ::: in tests f <- AutoMLR:::cindex_harrell expect_true(is.na(f(1:3, c(1, 0, 1), c(NA, NA, NA)))) expect_true(is.na(f(1:3, c(0, 0, 0), c(1, 2, 3)))) # perfect concordance: higher risk -> shorter time (all events) expect_equal(f(c(10, 20, 30), c(1, 1, 1), c(3, 2, 1)), 1.0) # perfect anti-concordance expect_equal(f(c(10, 20, 30), c(1, 1, 1), c(1, 2, 3)), 0.0) }) test_that("loocv_cindex with a trivial fit_fn runs and returns a valid C-index", { skip_if_not_installed("survival") data("cancer", package = "survival", envir = environment()) lung <- get("lung", envir = environment()) lung <- stats::na.omit(lung) lung$status <- as.integer(lung$status == 2) set.seed(1) idx <- sample(seq_len(nrow(lung)), 40) # tiny subset for speed df <- lung[idx, ] X <- as.matrix(df[, c("age", "sex", "ph.ecog", "ph.karno", "pat.karno", "meal.cal", "wt.loss")]) y <- survival::Surv(df$time, df$status) # Trivial "fit": return the training column means; predict: dot product with # the test row. This is a sanity LOOCV, not a real model. fit_fn <- function(X_tr, y_tr, hparam) colMeans(X_tr) predict_fn <- function(model, X_te) as.numeric(X_te %*% model) res <- loocv_cindex(X, y, fit_fn, predict_fn, seed = 42, verbose = FALSE) expect_named(res, c("cindex", "risk", "n_folds", "n_failed", "errors", "elapsed_sec")) expect_identical(res$n_folds, nrow(X)) expect_identical(res$n_failed, 0L) expect_length(res$risk, nrow(X)) expect_true(is.finite(res$cindex)) expect_true(res$cindex >= 0 && res$cindex <= 1) }) test_that("single-model registry entries fit and predict on lung", { skip_if_not_installed("survival") skip_if_not_installed("glmnet") skip_if_not_installed("CoxBoost") skip_if_not_installed("plsRcox") skip_if_not_installed("superpc") skip_if_not_installed("gbm") skip_if_not_installed("survivalsvm") skip_if_not_installed("quadprog") skip_if_not_installed("randomForestSRC") data("cancer", package = "survival", envir = environment()) lung <- get("lung", envir = environment()) lung <- stats::na.omit(lung) lung$status <- as.integer(lung$status == 2) X <- as.matrix(lung[seq_len(45), c("age", "sex", "ph.ecog", "ph.karno", "pat.karno", "meal.cal", "wt.loss")]) y <- survival::Surv(lung$time[seq_len(45)], lung$status[seq_len(45)]) reg <- get_surv_registry() for (algo_key in list_surv_algorithms()) { spec <- reg[[algo_key]] hparam <- as.list(spec$grid(automlr_parameters())[1L, , drop = FALSE]) model <- spec$fit(X[1:40, , drop = FALSE], y[1:40], hparam) risk <- spec$predict(model, X[41:45, , drop = FALSE]) expect_length(risk, 5L) expect_true(all(is.finite(risk))) } }) test_that("model variants expand algorithm grids enough for >100 two-model combinations", { variants <- list_model_variants() counts <- count_surv_combinations(min_size = 2, max_size = 2) expect_true(nrow(variants) >= 15L) expect_true(nrow(variants) <= 20L) expect_identical(counts$n_candidates, nrow(variants)) expect_true(counts$n_combinations > 100L) expect_false(counts$allow_same_algorithm) expect_lt(counts$n_combinations, count_surv_combinations(min_size = 2, max_size = 2, allow_same_algorithm = TRUE)$n_combinations) expect_true(all(c("candidate_key", "algo_key", "hparam_label") %in% names(variants))) }) test_that("evaluate_algorithm_loocv('lasso_cox') on lung: finite C-index", { skip_if_not_installed("survival") skip_if_not_installed("glmnet") data("cancer", package = "survival", envir = environment()) lung <- get("lung", envir = environment()) lung <- stats::na.omit(lung) lung$status <- as.integer(lung$status == 2) X <- as.matrix(lung[, c("age", "sex", "ph.ecog", "ph.karno", "pat.karno", "meal.cal", "wt.loss")]) y <- survival::Surv(lung$time, lung$status) params <- automlr_parameters() res <- evaluate_algorithm_loocv("lasso_cox", X, y, params = params, verbose = FALSE) expect_identical(res$algo_key, "lasso_cox") expect_true(is.finite(res$cindex), info = sprintf("lasso_cox LOOCV C-index was %s, errors: %d (%s)", res$cindex, res$n_failed, paste(utils::head(res$errors, 2), collapse = " | "))) expect_true(res$cindex >= 0.5, info = sprintf("lasso_cox LOOCV C-index = %.3f (<0.5 suggests sign flip)", res$cindex)) expect_true(res$n_failed <= ceiling(0.05 * res$n_folds), info = sprintf("too many LOOCV failures: %d / %d", res$n_failed, res$n_folds)) }) test_that("fit_surv_ensemble builds a weighted single-model ensemble", { skip_if_not_installed("survival") skip_if_not_installed("glmnet") data("cancer", package = "survival", envir = environment()) lung <- get("lung", envir = environment()) lung <- stats::na.omit(lung) lung$status <- as.integer(lung$status == 2) X <- as.matrix(lung[seq_len(35), c("age", "sex", "ph.ecog", "ph.karno", "pat.karno", "meal.cal", "wt.loss")]) y <- survival::Surv(lung$time[seq_len(35)], lung$status[seq_len(35)]) params <- automlr_parameters(algorithms = "lasso_cox", verbose = FALSE) ens <- fit_surv_ensemble(X[1:30, , drop = FALSE], y[1:30], params = params, algorithms = "lasso_cox", min_models = 1, max_models = 2, verbose = FALSE) expect_s3_class(ens, "automlr_surv_ensemble") expect_equal(sum(ens$weights), 1) risk <- predict(ens, X[31:35, , drop = FALSE]) expect_length(risk, 5L) expect_true(all(is.finite(risk))) report_dir <- file.path(tempdir(), "automlr-report-test") report <- render_surv_report(ens, output_dir = report_dir, top_n = 5) expect_true(file.exists(report)) expect_true(file.exists(file.path(report_dir, "figures", "single_model_cindex.png"))) expect_true(file.exists(file.path(report_dir, "tables", "single_models.csv"))) expect_true(file.exists(file.path(report_dir, "tables", "combinations.csv"))) expect_true(file.exists(file.path(report_dir, "summary_report.md"))) export_dir <- file.path(tempdir(), "automlr-export-test") exported <- export_surv_results(ens, output_dir = export_dir, formats = "png", top_n = 5) expect_true(file.exists(exported$report)) expect_true(file.exists(file.path(export_dir, "figures", "publication", "fig1_all_model_cohort_heatmap.png"))) expect_true(file.exists(file.path(export_dir, "figures", "publication", "fig2_combination_benchmark_matrix.png"))) expect_true(file.exists(file.path(export_dir, "figures", "publication", "fig5_feature_importance_barplot.png"))) expect_true(file.exists(file.path(export_dir, "figures", "publication", "fig6_shap_summary_plot.png"))) expect_true(file.exists(file.path(export_dir, "figures", "publication", "fig7_shap_dependence_plot.png"))) expect_true(file.exists(file.path(export_dir, "figures", "publication", "fig11_model_cindex_forest.png"))) expect_false(file.exists(file.path(export_dir, "figures", "publication", "fig1_model_performance.png"))) expect_false(file.exists(file.path(export_dir, "figures", "publication", "fig2_combination_ranking.png"))) expect_true(file.exists(file.path(export_dir, "tables", "single_models.csv"))) expect_true(file.exists(file.path(export_dir, "tables", "risk_scores.csv"))) expect_true(file.exists(file.path(export_dir, "tables", "all_combination_cohort_cindex.csv"))) expect_true(file.exists(file.path(export_dir, "tables", "all_model_cohort_cindex.csv"))) expect_true(file.exists(file.path(export_dir, "tables", "cohort_km_stats.csv"))) expect_true(file.exists(file.path(export_dir, "tables", "feature_importance.csv"))) expect_true(file.exists(file.path(export_dir, "tables", "shap_approx_contributions.csv"))) expect_true(file.exists(file.path(export_dir, "tables", "risk_prediction_horizon.csv"))) expect_true(file.exists(file.path(export_dir, "tables", "risk_score_nomogram.csv"))) expect_true(file.exists(file.path(export_dir, "tables", "calibration_curve.csv"))) expect_true(file.exists(file.path(export_dir, "tables", "dca_curve.csv"))) expect_true(file.exists(file.path(export_dir, "tables", "model_cindex_forest.csv"))) expect_true(file.exists(file.path(export_dir, "summaries", "summary_report.md"))) expect_true(file.exists(file.path(export_dir, "summaries", "data_summary.md"))) expect_true(file.exists(file.path(export_dir, "summaries", "base_model_summary.md"))) expect_true(file.exists(file.path(export_dir, "summaries", "ensemble_summary.md"))) expect_true(file.exists(file.path(export_dir, "summaries", "explainability_summary.md"))) summary_text <- paste(readLines(file.path(export_dir, "summaries", "summary_report.md"), warn = FALSE), collapse = "\n") expect_true(grepl("AutoMLR Survival Analysis Summary", summary_text, fixed = TRUE)) expect_true(grepl("\u751F\u5B58\u5206\u6790\u7ED3\u679C\u603B\u7ED3", summary_text)) expect_true(grepl("Explainability and Clinical Utility Outputs", summary_text, fixed = TRUE)) expect_true(file.exists(file.path(export_dir, "objects", "fitted_ensemble.rds"))) expect_true(file.exists(file.path(export_dir, "logs", "session_info.txt"))) risk_scores <- utils::read.csv(file.path(export_dir, "tables", "risk_scores.csv")) expect_true(all(c("sample_id", "time", "status", "risk_score", "risk_group") %in% names(risk_scores))) expect_equal(nrow(risk_scores), length(ens$training$risk)) feature_importance <- utils::read.csv(file.path(export_dir, "tables", "feature_importance.csv")) expect_true(all(c("feature", "cindex_drop", "mean_abs_risk_change") %in% names(feature_importance))) }) test_that("fit_surv_ensemble auto-uses automlr_input cohorts for stability", { skip_if_not_installed("survival") skip_if_not_installed("glmnet") data("cancer", package = "survival", envir = environment()) lung <- get("lung", envir = environment()) lung <- stats::na.omit(lung) lung$status <- as.integer(lung$status == 2) lung$cohort <- rep(c("A", "B"), length.out = nrow(lung)) lung$sample_id <- paste0("S", seq_len(nrow(lung))) lung <- lung[seq_len(36), c("sample_id", "cohort", "time", "status", "age", "sex", "ph.ecog", "ph.karno", "pat.karno", "meal.cal", "wt.loss")] prep <- prepare_cohort_input(lung, cohort = "cohort", time = "time", status = "status", id = "sample_id") params <- automlr_parameters(algorithms = "lasso_cox", stability_resamples = 0L, verbose = FALSE) ens <- fit_surv_ensemble(prep, params = params, algorithms = "lasso_cox", min_models = 1, max_models = 1, verbose = FALSE) expect_s3_class(ens, "automlr_surv_ensemble") expect_identical(ens$input$source, "automlr_input") expect_setequal(ens$input$cohorts, c("A", "B")) expect_true("cohort_summary" %in% names(ens$input)) expect_equal(sum(ens$input$cohort_summary$n_events), ens$input$n_events) expect_true("stability_n" %in% names(ens$combination_evaluation$summary)) expect_true(any(ens$combination_evaluation$summary$stability_n > 0)) expected_sample_id <- unlist(lapply(prep$cohorts, `[[`, "sample_id"), use.names = FALSE) expect_equal(ens$training$sample_id, expected_sample_id) export_dir <- file.path(tempdir(), "automlr-cohort-export-test") export_surv_results(ens, output_dir = export_dir, formats = "png", top_n = 5) expect_true(file.exists(file.path(export_dir, "figures", "publication", "fig1_all_model_cohort_heatmap.png"))) expect_true(file.exists(file.path(export_dir, "figures", "publication", "fig2_combination_benchmark_matrix.png"))) expect_true(file.exists(file.path(export_dir, "figures", "publication", "fig3_multi_cohort_km_panel.png"))) expect_false(file.exists(file.path(export_dir, "figures", "publication", "fig3_cohort_stability_heatmap.png"))) expect_false(file.exists(file.path(export_dir, "figures", "publication", "fig4_risk_stratification_km.png"))) expect_false(file.exists(file.path(export_dir, "figures", "publication", "fig7_all_model_cohort_heatmap.png"))) expect_true(file.exists(file.path(export_dir, "tables", "cohort_stability.csv"))) expect_true(file.exists(file.path(export_dir, "tables", "all_combination_cohort_cindex.csv"))) expect_true(file.exists(file.path(export_dir, "tables", "all_model_cohort_cindex.csv"))) expect_true(file.exists(file.path(export_dir, "tables", "cohort_km_stats.csv"))) expect_true(file.exists(file.path(export_dir, "tables", "time_auc.csv"))) expect_true(file.exists(file.path(export_dir, "tables", "time_roc_curve.csv"))) expect_true(file.exists(file.path(export_dir, "tables", "cohort_time_roc_curve.csv"))) expect_true(file.exists(file.path(export_dir, "tables", "feature_importance.csv"))) expect_true(file.exists(file.path(export_dir, "tables", "shap_approx_contributions.csv"))) expect_true(file.exists(file.path(export_dir, "tables", "model_cindex_forest.csv"))) risk_scores <- utils::read.csv(file.path(export_dir, "tables", "risk_scores.csv")) expect_equal(risk_scores$sample_id, expected_sample_id) data_summary <- paste(readLines(file.path(export_dir, "summaries", "data_summary.md"), warn = FALSE), collapse = "\n") expect_true(grepl("Data Preparation Summary", data_summary, fixed = TRUE)) expect_true(grepl("Cohorts included: 2", data_summary, fixed = TRUE)) if (requireNamespace("timeROC", quietly = TRUE)) { expect_true(file.exists(file.path(export_dir, "figures", "publication", "fig4_multi_cohort_time_roc_panel.png"))) expect_false(file.exists(file.path(export_dir, "figures", "publication", "fig6_time_roc_curves.png"))) expect_false(file.exists(file.path(export_dir, "figures", "publication", "fig10_multi_cohort_time_roc_panel.png"))) } }) test_that("evaluate_surv_combinations ranks all-subset risk combinations", { y <- survival::Surv(c(5, 4, 3, 2, 1), c(1, 1, 1, 1, 1)) fake <- structure( list( summary = data.frame( candidate_key = c("m1", "m2", "m3"), algo_key = c("m1", "m2", "m3"), algo_label = c("Model 1", "Model 2", "Model 3"), cindex = c(1.0, 0.8, 0.0), n_folds = 5L, n_failed = 0L, elapsed_sec = 0, stringsAsFactors = FALSE ), results = list( m1 = list(risk = c(1, 2, 3, 4, 5)), m2 = list(risk = c(1, 1, 3, 3, 5)), m3 = list(risk = c(5, 4, 3, 2, 1)) ), params = automlr_parameters(algorithms = c("m1", "m2", "m3")) ), class = "automlr_loocv_set" ) combos <- evaluate_surv_combinations(fake, y, max_size = 2, weight_method = "equal", top_n = Inf) expect_s3_class(combos, "automlr_combination_set") expect_equal(nrow(combos$summary), 6L) expect_equal(combos$summary$combination_key[1], "m1") expect_true(all(c("selection_score", "mean_failure_rate", "mean_risk_sd", "stability_score") %in% names(combos$summary))) expect_true(combos$summary$cindex[1] >= combos$summary$cindex[nrow(combos$summary)]) }) test_that("survival automatic C-index threshold uses requested quantile", { tab <- data.frame(candidate_key = c("m1", "m2", "m3"), cindex = c(0.52, 0.66, 0.81)) expect_equal(recommend_surv_cindex_threshold(tab, auto_quantile = 0.50), 0.66) expect_equal(recommend_surv_cindex_threshold(tab, auto_quantile = 0.75), as.numeric(stats::quantile(tab$cindex, 0.75, names = FALSE))) expect_error(recommend_surv_cindex_threshold(tab, auto_quantile = -0.1)) }) test_that("risk standardization preserves ordering with extreme outliers", { y <- survival::Surv(seq(48, 1), rep(1, 48)) risk <- -seq_len(48) * 1e-36 risk[9] <- -0.466 standardized <- .standardize_risk(risk) expect_identical(order(risk), order(standardized)) expect_equal( cindex_harrell(as.numeric(y[, "time"]), as.integer(y[, "status"]), standardized), cindex_harrell(as.numeric(y[, "time"]), as.integer(y[, "status"]), risk) ) }) test_that("evaluate_surv_combinations can report queue stability diagnostics", { y <- survival::Surv(c(5, 4, 3, 2, 1, 6), c(1, 1, 1, 1, 1, 0)) fake <- structure( list( summary = data.frame( candidate_key = c("m1", "m2"), algo_key = c("m1", "m2"), algo_label = c("Model 1", "Model 2"), cindex = c(0.8, 0.7), n_folds = 6L, n_failed = 0L, stability_score = c(0.8, 0.6), elapsed_sec = 0, stringsAsFactors = FALSE ), results = list( m1 = list(risk = c(1, 2, 3, 4, 5, 1)), m2 = list(risk = c(1, 1, 3, 3, 5, 2)) ), params = automlr_parameters(algorithms = c("m1", "m2"), stability_resamples = 2L) ), class = "automlr_loocv_set" ) combos <- evaluate_surv_combinations( fake, y, min_size = 2, max_size = 2, stability_groups = c("A", "A", "A", "B", "B", "B"), stability_resamples = 2L, weight_method = "cindex_stability", top_n = Inf ) expect_equal(nrow(combos$summary), 1L) expect_true(combos$summary$stability_n >= 1L) expect_true(is.finite(combos$summary$selection_score)) }) test_that("evaluate_surv_combinations excludes same-base variants by default", { y <- survival::Surv(c(5, 4, 3, 2, 1), c(1, 1, 1, 1, 1)) fake <- structure( list( summary = data.frame( candidate_key = c("m1a", "m1b", "m2"), algo_key = c("m1", "m1", "m2"), algo_label = c("Model 1", "Model 1", "Model 2"), cindex = c(0.8, 0.7, 0.75), n_folds = 5L, n_failed = 0L, elapsed_sec = 0, stringsAsFactors = FALSE ), results = list( m1a = list(risk = c(1, 2, 3, 4, 5)), m1b = list(risk = c(1, 1, 3, 4, 5)), m2 = list(risk = c(2, 2, 3, 4, 4)) ), params = automlr_parameters(algorithms = c("m1", "m2")) ), class = "automlr_loocv_set" ) combos <- evaluate_surv_combinations(fake, y, min_size = 2, max_size = 2, weight_method = "equal", top_n = Inf) expect_false("m1a+m1b" %in% combos$summary$combination_key) expect_equal(nrow(combos$summary), 2L) }) test_that("fit_surv_ensemble can choose the best subset strategy", { skip_if_not_installed("survival") skip_if_not_installed("glmnet") data("cancer", package = "survival", envir = environment()) lung <- get("lung", envir = environment()) lung <- stats::na.omit(lung) lung$status <- as.integer(lung$status == 2) X <- as.matrix(lung[seq_len(35), c("age", "sex", "ph.ecog", "ph.karno", "pat.karno", "meal.cal", "wt.loss")]) y <- survival::Surv(lung$time[seq_len(35)], lung$status[seq_len(35)]) algos <- c("lasso_cox", "ridge_cox") params <- automlr_parameters(algorithms = algos, verbose = FALSE) ens <- fit_surv_ensemble(X[1:30, , drop = FALSE], y[1:30], params = params, algorithms = algos, strategy = "best_subset", max_models = 2, verbose = FALSE) expect_s3_class(ens, "automlr_surv_ensemble") expect_identical(ens$strategy, "best_subset") expect_s3_class(ens$combination_evaluation, "automlr_combination_set") expect_true(length(ens$selected_candidates) >= 1L) expect_true(length(ens$selected_candidates) <= 2L) risk <- predict(ens, X[31:35, , drop = FALSE]) expect_length(risk, 5L) expect_true(all(is.finite(risk))) }) test_that("extreme_surv_screen runs apparent top-N and seed search", { skip_if_not_installed("survival") skip_if_not_installed("glmnet") data("cancer", package = "survival", envir = environment()) lung <- get("lung", envir = environment()) lung <- stats::na.omit(lung) lung$status <- as.integer(lung$status == 2) lung$cohort <- rep(c("A", "B"), length.out = nrow(lung)) lung <- lung[seq_len(42), c("cohort", "time", "status", "age", "sex", "ph.ecog", "ph.karno", "pat.karno", "meal.cal", "wt.loss")] prep <- prepare_cohort_input(lung, cohort = "cohort", time = "time", status = "status") algos <- c("lasso_cox", "ridge_cox") params <- automlr_parameters(algorithms = algos, verbose = FALSE) res <- extreme_surv_screen( prep, params = params, algorithms = algos, top_n = 2, seeds = 1:3, min_models = 1, max_models = 2, verbose = FALSE ) expect_s3_class(res, "automlr_extreme_screen") expect_identical(res$input$source, "automlr_input") expect_lte(nrow(res$top_combinations), 2L) expect_equal(nrow(res$seed_search), nrow(res$top_combinations) * 3L) expect_true(all(c("apparent_cindex", "cohort_cindex_mean", "stage1_rank_score", "performance_scope", "performance_note") %in% names(res$combination_summary))) expect_true(all(c("seed", "combination_key", "train_cindex", "validation_cindex", "validation_cohort_cindex_mean", "seed_search_rank", "performance_scope", "performance_note") %in% names(res$seed_search))) expect_identical(unique(res$top_combinations$performance_scope), "optimistic_apparent_full_data") expect_true(grepl("Optimistic apparent performance", res$notes$apparent_performance, fixed = TRUE)) expect_identical(res$settings$rank_by, "apparent_cindex") expect_true(all(diff(res$top_combinations$apparent_cindex) <= 1e-12)) expect_equal(nrow(res$best), 1L) expect_true(is.finite(res$best$validation_cindex) || all(!is.finite(res$seed_search$validation_cindex))) export_dir <- file.path(tempdir(), "automlr-extreme-export-test") exported <- export_extreme_screen_results( res, output_dir = export_dir, formats = "png", top_n = 2, top_seed_rows = 5, dpi = 120 ) expect_true(file.exists(exported$tables["complete_apparent_cindex_results"])) expect_true(file.exists(exported$tables["seed_search"])) expect_true(file.exists(file.path(export_dir, "figures", "fig1_apparent_top_cindex.png"))) expect_true(file.exists(file.path(export_dir, "figures", "fig2_seed_validation_distribution.png"))) expect_true(file.exists(file.path(export_dir, "figures", "fig6_apparent_vs_validation.png"))) expect_true(file.exists(exported$summary_report)) exported_summary <- paste(readLines(exported$summary_report, warn = FALSE), collapse = "\n") expect_true(grepl("\u6781\u9650\u7B5B\u9009", exported_summary)) expect_true(grepl("Extreme Screening Summary", exported_summary, fixed = TRUE)) english_pos <- regexpr("Extreme Screening Summary", exported_summary, fixed = TRUE)[1L] chinese_pos <- regexpr("\u6781\u9650\u7B5B\u9009", exported_summary)[1L] expect_lt(english_pos, chinese_pos) summary_text <- summarize_extreme_screen_results(res, top_n = 2) expect_true(grepl("Best validation row", summary_text, fixed = TRUE)) expect_true(grepl("Apparent Screen", summary_text, fixed = TRUE)) summary_text_en <- summarize_extreme_screen_results(res, top_n = 2, language = "en") expect_false(grepl("\u6781\u9650\u7B5B\u9009", summary_text_en)) })