mlr3机器学习AUC的置信区间提取

如果你在mlr3拿到机器学习的预测数据

ROC 过程原理探索

假设数据

df <- data.frame(A=iris$Sepal.Length,
group=sample(x = c(0,1),size = 150,replace = T))

分组为 0,1 # 变量A为连续性变量

library(pROC)
roc_obj <- roc(df$group,df$A, levels = c(0, 1),ci=T)
auc(roc_obj)
ci.auc(roc_obj)

如果你直接在机器学习拿到预测数据

fit <- lm(df$groupdf$A)
pre_df <- predict(fit,df)

roc_obj_2 <- roc(df$group, pre_df, levels = c(0, 1),ci=T)
auc(roc_obj_2)

实测demo

# 在train的性能
prediction_train_rf = learner_rf$predict(task, row_ids = train_id)
prediction_train_rf$confusion
prediction_train_rf$score(msr("classif.auc"))
p_rf_train=autoplot(prediction_train_rf,type = "roc")
p_rf_train

# 在test的性能
prediction_test_rf = learner_rf$predict(task, row_ids = test_id)
prediction_test_rf$confusion
prediction_test_rf$score(msr("classif.auc"))
p_rf_test=autoplot(prediction_test_rf,type = "roc")
p_rf_test


# train的AUC和CI,多ROC线准备
library(pROC)
# train的AUC和CI,多ROC线准备
roc_obj_rf_train <- roc(prediction_train_rf$truth, 
                        prediction_train_rf$prob[,1], ci=T)
auc(roc_obj_rf_train);ci.auc(roc_obj_rf_train)

# 【与上面不一致】 不需要判断 prob 选哪个
# roc_obj_rf_train <- roc(prediction_test_rf$truth, 
#                  ifelse(
#                    prediction_test_rf$truth==1,
#                    prediction_test_rf$prob[,1],
#                    prediction_test_rf$prob[,2]
#                  ), ci=T)
# auc(roc_obj_rf_train);ci.auc(roc_obj_rf_train)
# test的AUC和CI,多ROC线准备
roc_obj_rf_test <- roc(prediction_test_rf$truth, 
                       prediction_test_rf$prob[,1], ci=T)
auc(roc_obj_rf_test);ci.auc(roc_obj_rf_test)
# 
pROC::plot.roc(roc_obj_rf_test)