cvms

Overview

Main functions

Evaluation utilities

Formula utilities

Plotting utilities

Custom functions

Other utilities

Datasets

Table of Contents

Important News

Installation

Vignettes

Examples

Attach packages

library(cvms)library(groupdata2)# fold() partition()library(knitr)# kable()library(dplyr)# %>% arrange()

Load data

data<-participant.scores

Fold data

# Set seed for reproducibilityset.seed(7)# Fold datadata<- fold(data=data,k=4,cat_col='diagnosis',id_col='participant') %>%   arrange(.folds)# Show first 15 rows of datadata %>% head(15) %>% kable()

Cross-validate a single model

Gaussian

CV1<- cross_validate(data=data,formulas="score ~ diagnosis",fold_cols='.folds',family='gaussian',REML=FALSE)# Show resultsCV1#> # A tibble: 1 × 21#>   Fixed  RMSE   MAE `NRMSE(IQR)`  RRSE   RAE RMSLE   AIC  AICc   BIC Predictions#>   <chr> <dbl> <dbl>        <dbl> <dbl> <dbl> <dbl> <dbl> <dbl> <dbl> <list>#> 1 diag…  16.4  13.8        0.937 0.900 0.932 0.474  195.  196.  198. <tibble>#> # ℹ 10 more variables: Results <list>, Coefficients <list>, Folds <int>,#> #   `Fold Columns` <int>, `Convergence Warnings` <int>,#> #   `Singular Fit Messages` <int>, `Other Warnings` <int>,#> #   `Warnings and Messages` <list>, Process <list>, Dependent <chr># Let's take a closer look at the different parts of the output# Metrics and formulasCV1 %>% select_metrics() %>% kable()

# Just the formulasCV1 %>% select_definitions() %>% kable()

# Nested predictions# Note that [[1]] picks predictions for the first rowCV1$Predictions[[1]] %>% head() %>% kable()

# Nested results from the different foldsCV1$Results[[1]] %>% kable()

# Nested model coefficients# Note that you have the full p-values,# but kable() only shows a certain number of digitsCV1$Coefficients[[1]] %>% kable()

# Additional information about the model# and the training processCV1 %>% select(14:19,21) %>% kable()

CV1$Process[[1]]#> ---#> Process Information#> ---#> Target column: target#> Prediction column: prediction#> Family / type: Gaussian#> Target summary: mean: 38.767, median: 35, range: [10, 81], SD: 19.294, IQR: 28#> Prediction summary: mean: 38.717, median: 33.667, range: [27.25, 53.333], SD: 10.386, IQR: 19.111#> Locale (LC_ALL):#>   en_US.UTF-8/en_US.UTF-8/en_US.UTF-8/C/en_US.UTF-8/en_US.UTF-8#> ---

Binomial

CV2<- cross_validate(data=data,formulas="diagnosis~score",fold_cols='.folds',family='binomial')# Show resultsCV2#> # A tibble: 1 × 28#>   Fixed `Balanced Accuracy`    F1 Sensitivity Specificity `Pos Pred Value`#>   <chr>               <dbl> <dbl>       <dbl>       <dbl>            <dbl>#> 1 score               0.736 0.821       0.889       0.583            0.762#> # ℹ 22 more variables: `Neg Pred Value` <dbl>, AUC <dbl>, `Lower CI` <dbl>,#> #   `Upper CI` <dbl>, Kappa <dbl>, MCC <dbl>, `Detection Rate` <dbl>,#> #   `Detection Prevalence` <dbl>, Prevalence <dbl>, Predictions <list>,#> #   ROC <list>, `Confusion Matrix` <list>, Results <list>, Coefficients <list>,#> #   Folds <int>, `Fold Columns` <int>, `Convergence Warnings` <int>,#> #   `Singular Fit Messages` <int>, `Other Warnings` <int>,#> #   `Warnings and Messages` <list>, Process <list>, Dependent <chr># Let's take a closer look at the different parts of the output# We won't repeat the parts too similar to those in Gaussian# MetricsCV2 %>% select(1:9) %>% kable(digits=5)

CV2 %>% select(10:15) %>% kable()

# Confusion matrixCV2$`Confusion Matrix`[[1]] %>% kable()

# Plot confusion matrixplot_confusion_matrix(CV2$`Confusion Matrix`[[1]],add_sums=TRUE)

Cross-validate multiple models

Create model formulas

model_formulas<- c("score ~ diagnosis","score ~ age")mixed_model_formulas<- c("score ~ diagnosis + (1|session)","score ~ age + (1|session)")

Cross-validate fixed effects models

CV3<- cross_validate(data=data,formulas=model_formulas,fold_cols='.folds',family='gaussian',REML=FALSE)# Show resultsCV3#> # A tibble: 2 × 21#>   Fixed  RMSE   MAE `NRMSE(IQR)`  RRSE   RAE RMSLE   AIC  AICc   BIC Predictions#>   <chr> <dbl> <dbl>        <dbl> <dbl> <dbl> <dbl> <dbl> <dbl> <dbl> <list>#> 1 diag…  16.4  13.8        0.937 0.900 0.932 0.474  195.  196.  198. <tibble>#> 2 age    22.4  18.9        1.35  1.23  1.29  0.618  201.  202.  204. <tibble>#> # ℹ 10 more variables: Results <list>, Coefficients <list>, Folds <int>,#> #   `Fold Columns` <int>, `Convergence Warnings` <int>,#> #   `Singular Fit Messages` <int>, `Other Warnings` <int>,#> #   `Warnings and Messages` <list>, Process <list>, Dependent <chr>

Cross-validate mixed effects models

CV4<- cross_validate(data=data,formulas=mixed_model_formulas,fold_cols='.folds',family='gaussian',REML=FALSE)# Show resultsCV4#> # A tibble: 2 × 22#>   Fixed  RMSE   MAE `NRMSE(IQR)`  RRSE   RAE RMSLE   AIC  AICc   BIC Predictions#>   <chr> <dbl> <dbl>        <dbl> <dbl> <dbl> <dbl> <dbl> <dbl> <dbl> <list>#> 1 diag…  7.95  6.41        0.438 0.432 0.428 0.226  176.  178.  180. <tibble>#> 2 age   17.5  16.2         1.08  0.953 1.11  0.480  194.  196.  198. <tibble>#> # ℹ 11 more variables: Results <list>, Coefficients <list>, Folds <int>,#> #   `Fold Columns` <int>, `Convergence Warnings` <int>,#> #   `Singular Fit Messages` <int>, `Other Warnings` <int>,#> #   `Warnings and Messages` <list>, Process <list>, Dependent <chr>,#> #   Random <chr>

Repeated cross-validation

# Set seed for reproducibilityset.seed(2)# Ungroup data# Ootherwise we would create folds within the existing foldsdata<-dplyr::ungroup(data)# Fold datadata<- fold(data=data,k=4,cat_col='diagnosis',id_col='participant',num_fold_cols=3,handle_existing_fold_cols="keep")# Show first 15 rows of datadata %>% head(10) %>% kable()

CV5<- cross_validate(data=data,formulas= c("diagnosis ~ score","diagnosis ~ score + age"),fold_cols= paste0(".folds_",1:4),family='binomial')# Show resultsCV5#> # A tibble: 2 × 28#>   Fixed     `Balanced Accuracy`    F1 Sensitivity Specificity `Pos Pred Value`#>   <chr>                   <dbl> <dbl>       <dbl>       <dbl>            <dbl>#> 1 score                   0.729 0.813       0.875       0.583            0.759#> 2 score+age               0.545 0.643       0.653       0.438            0.635#> # ℹ 22 more variables: `Neg Pred Value` <dbl>, AUC <dbl>, `Lower CI` <dbl>,#> #   `Upper CI` <dbl>, Kappa <dbl>, MCC <dbl>, `Detection Rate` <dbl>,#> #   `Detection Prevalence` <dbl>, Prevalence <dbl>, Predictions <list>,#> #   ROC <list>, `Confusion Matrix` <list>, Results <list>, Coefficients <list>,#> #   Folds <int>, `Fold Columns` <int>, `Convergence Warnings` <int>,#> #   `Singular Fit Messages` <int>, `Other Warnings` <int>,#> #   `Warnings and Messages` <list>, Process <list>, Dependent <chr># Subset of the results per fold for the first modelCV5$Results[[1]] %>% select(1:8) %>% kable()

Cross-validating custom model functions

SVM

# Create model function## train_data : tibble with the training data# formula : a formula object# hyperparameters : a named list of hyparameterssvm_model_fn<-function(train_data,formula,hyperparameters){# Note that `formula` must be passed first# when calling svm(), otherwise it failse1071::svm(formula=formula,data=train_data,kernel="linear",type="C-classification",probability=TRUE  )}

# Create predict function## test_data : tibble with the test data# model : fitted model object# formula : a formula object# hyperparameters : a named list of hyparameters# train_data : tibble with the training datasvm_predict_fn<-function(test_data,model,formula,hyperparameters,train_data){# Predict the test set with the modelpredictions<-stats::predict(object=model,newdata=test_data,allow.new.levels=TRUE,probability=TRUE  )# Extract the probabilities# Usually the predict function will just# output the probabilities directlyprobabilities<-dplyr::as_tibble(    attr(predictions,"probabilities")  )# Return second column# with probabilities of positive classprobabilities[[2]]}

# Cross-validate svm_model_fnCV6<- cross_validate_fn(data=data,model_fn=svm_model_fn,predict_fn=svm_predict_fn,formulas= c("diagnosis ~ score","diagnosis ~ age"),fold_cols='.folds_1',type='binomial')#> Will cross-validate 2 models. This requires fitting 8 model instances.CV6#> # A tibble: 2 × 27#>   Fixed `Balanced Accuracy`    F1 Sensitivity Specificity `Pos Pred Value`#>   <chr>               <dbl> <dbl>       <dbl>       <dbl>            <dbl>#> 1 score               0.653 0.780       0.889       0.417            0.696#> 2 age                 0.458 0.615       0.667       0.25             0.571#> # ℹ 21 more variables: `Neg Pred Value` <dbl>, AUC <dbl>, `Lower CI` <dbl>,#> #   `Upper CI` <dbl>, Kappa <dbl>, MCC <dbl>, `Detection Rate` <dbl>,#> #   `Detection Prevalence` <dbl>, Prevalence <dbl>, Predictions <list>,#> #   ROC <list>, `Confusion Matrix` <list>, Results <list>, Coefficients <list>,#> #   Folds <int>, `Fold Columns` <int>, `Convergence Warnings` <int>,#> #   `Other Warnings` <int>, `Warnings and Messages` <list>, Process <list>,#> #   Dependent <chr>

Naïve Bayes

# Create model function## train_data : tibble with the training data# formula : a formula object# hyperparameters : a named list of hyparametersnb_model_fn<-function(train_data,formula,hyperparameters){e1071::naiveBayes(formula=formula,data=train_data  )}

# Create predict function## test_data : tibble with the test data# model : fitted model object# formula : a formula object# hyperparameters : a named list of hyparameters# train_data : tibble with the training datanb_predict_fn<-function(test_data,model,formula,hyperparameters,train_data){stats::predict(object=model,newdata=test_data,type="raw",allow.new.levels=TRUE)[,2]}

CV7<- cross_validate_fn(data=data,model_fn=nb_model_fn,predict_fn=nb_predict_fn,formulas= c("diagnosis ~ score","diagnosis ~ age"),type='binomial',fold_cols='.folds_1')#> Will cross-validate 2 models. This requires fitting 8 model instances.CV7#> # A tibble: 2 × 27#>   Fixed `Balanced Accuracy`    F1 Sensitivity Specificity `Pos Pred Value`#>   <chr>               <dbl> <dbl>       <dbl>       <dbl>            <dbl>#> 1 score               0.736 0.821       0.889       0.583            0.762#> 2 age                 0.25  0.462       0.5         0                0.429#> # ℹ 21 more variables: `Neg Pred Value` <dbl>, AUC <dbl>, `Lower CI` <dbl>,#> #   `Upper CI` <dbl>, Kappa <dbl>, MCC <dbl>, `Detection Rate` <dbl>,#> #   `Detection Prevalence` <dbl>, Prevalence <dbl>, Predictions <list>,#> #   ROC <list>, `Confusion Matrix` <list>, Results <list>, Coefficients <list>,#> #   Folds <int>, `Fold Columns` <int>, `Convergence Warnings` <int>,#> #   `Other Warnings` <int>, `Warnings and Messages` <list>, Process <list>,#> #   Dependent <chr>

Extracting the most challenging observations

glm_predictions<-dplyr::bind_rows(CV5$Predictions,.id="Model")svm_predictions<-dplyr::bind_rows(CV6$Predictions,.id="Model")nb_predictions<-dplyr::bind_rows(CV7$Predictions,.id="Model")predictions<-dplyr::bind_rows(glm_predictions,svm_predictions,nb_predictions,.id="Architecture")predictions[["Target"]]<- as.character(predictions[["Target"]])predictions#> # A tibble: 360 × 8#>    Architecture Model `Fold Column`  Fold Observation Target Prediction#>    <chr>        <chr> <chr>         <int>       <int> <chr>       <dbl>#>  1 1            1     .folds_1          1          10 0           0.721#>  2 1            1     .folds_1          1          11 0           0.422#>  3 1            1     .folds_1          1          12 0           0.242#>  4 1            1     .folds_1          1          28 1           0.884#>  5 1            1     .folds_1          1          29 1           0.734#>  6 1            1     .folds_1          1          30 1           0.563#>  7 1            1     .folds_1          2           4 0           0.831#>  8 1            1     .folds_1          2           5 0           0.620#>  9 1            1     .folds_1          2           6 0           0.202#> 10 1            1     .folds_1          2          13 1           0.928#> # ℹ 350 more rows#> # ℹ 1 more variable: `Predicted Class` <chr>

challenging<- most_challenging(data=predictions,prediction_cols="Prediction",type="binomial",threshold=0.20,threshold_is="percentage")challenging#> # A tibble: 6 × 7#>   Observation Correct Incorrect Accuracy   MAE `Cross Entropy`  `<=`#>         <int>   <int>     <int>    <dbl> <dbl>           <dbl> <dbl>#> 1          21       1        11   0.0833 0.820            2.10 0.615#> 2           4       0        12   0      0.783            1.66 0.615#> 3          10       0        12   0      0.774            1.57 0.615#> 4          20       1        11   0.0833 0.742            1.50 0.615#> 5           1       1        11   0.0833 0.733            1.39 0.615#> 6           7       0        12   0      0.690            1.22 0.615

# Index with values 1:30data[["Observation"]]<- seq_len(nrow(data))# Add information to the challenging observationschallenging<-data %>%# Remove fold columns for claritydplyr::select(-c(.folds_1,.folds_2,.folds_3,.folds_4)) %>%# Add the scoresdplyr::right_join(challenging,by="Observation")challenging %>% kable()

Evaluating predictions

Gaussian evaluation

# Extract the predictions from the first cross-validationpredictions<-CV1$Predictions[[1]]predictions %>% head(6) %>% kable()

# Evaluate the predictions per foldpredictions %>%   group_by(Fold) %>%   evaluate(target_col="Target",prediction_cols="Prediction",type="gaussian"  )#> New names:#> New names:#> New names:#> New names:#> • `Fold` -> `Fold...1`#> • `Fold` -> `Fold...3`#> # A tibble: 4 × 9#>    Fold  RMSE   MAE `NRMSE(IQR)`  RRSE   RAE RMSLE Predictions      Process#>   <int> <dbl> <dbl>        <dbl> <dbl> <dbl> <dbl> <list>           <list>#> 1     1  12.6  10.7        0.679 0.783 0.785 0.356 <tibble [6 × 4]> <prcss_n_>#> 2     2  16.6  14.8        1.04  1.01  1.13  0.581 <tibble [9 × 4]> <prcss_n_>#> 3     3  16.0  12.9        1.25  0.795 0.864 0.477 <tibble [6 × 4]> <prcss_n_>#> 4     4  20.3  16.7        0.779 1.01  0.953 0.482 <tibble [9 × 4]> <prcss_n_>

Binomial evaluation

# Extract the predictions from the second cross-validationpredictions<-CV2$Predictions[[1]]predictions %>% head(6) %>% kable()

# Evaluate the predictions per foldpredictions %>%   group_by(Fold) %>%   evaluate(target_col="Target",prediction_cols="Prediction",type="binomial"  )#> New names:#> New names:#> New names:#> New names:#> • `Fold` -> `Fold...1`#> • `Fold` -> `Fold...3`#> # A tibble: 4 × 20#>    Fold `Balanced Accuracy` Accuracy    F1 Sensitivity Specificity#>   <int>               <dbl>    <dbl> <dbl>       <dbl>       <dbl>#> 1     1               0.833    0.833 0.857       1           0.667#> 2     2               0.667    0.778 0.857       1           0.333#> 3     3               0.833    0.833 0.857       1           0.667#> 4     4               0.667    0.667 0.727       0.667       0.667#> # ℹ 14 more variables: `Pos Pred Value` <dbl>, `Neg Pred Value` <dbl>,#> #   AUC <dbl>, `Lower CI` <dbl>, `Upper CI` <dbl>, Kappa <dbl>, MCC <dbl>,#> #   `Detection Rate` <dbl>, `Detection Prevalence` <dbl>, Prevalence <dbl>,#> #   Predictions <list>, ROC <named list>, `Confusion Matrix` <list>,#> #   Process <list>

Multinomial evaluation

# Create dataset for multinomial evaluationmulticlass_data<- multiclass_probability_tibble(num_classes=3,# Here, number of predictorsnum_observations=30,apply_softmax=TRUE,add_predicted_classes=TRUE,add_targets=TRUE)multiclass_data#> # A tibble: 30 × 5#>    class_1 class_2 class_3 `Predicted Class` Target#>      <dbl>   <dbl>   <dbl> <chr>             <chr>#>  1   0.200   0.490   0.309 class_2           class_2#>  2   0.256   0.255   0.489 class_3           class_2#>  3   0.255   0.423   0.322 class_2           class_2#>  4   0.391   0.316   0.293 class_1           class_2#>  5   0.314   0.364   0.321 class_2           class_1#>  6   0.258   0.449   0.293 class_2           class_1#>  7   0.406   0.173   0.421 class_3           class_3#>  8   0.317   0.273   0.410 class_3           class_1#>  9   0.351   0.227   0.422 class_3           class_3#> 10   0.373   0.395   0.233 class_2           class_2#> # ℹ 20 more rows# Evaluate probabilities# One prediction column *per class*ev<- evaluate(data=multiclass_data,target_col="Target",prediction_cols= paste0("class_",1:3),type="multinomial")ev#> # A tibble: 1 × 16#>   `Overall Accuracy` `Balanced Accuracy`    F1 Sensitivity Specificity#>                <dbl>               <dbl> <dbl>       <dbl>       <dbl>#> 1              0.533               0.646 0.516       0.530       0.762#> # ℹ 11 more variables: `Pos Pred Value` <dbl>, `Neg Pred Value` <dbl>,#> #   Kappa <dbl>, MCC <dbl>, `Detection Rate` <dbl>,#> #   `Detection Prevalence` <dbl>, Prevalence <dbl>, Predictions <list>,#> #   `Confusion Matrix` <list>, `Class Level Results` <list>, Process <list># The one-vs-all evaluationsev$`Class Level Results`[[1]]#> # A tibble: 3 × 13#>   Class   `Balanced Accuracy`    F1 Sensitivity Specificity `Pos Pred Value`#>   <chr>                 <dbl> <dbl>       <dbl>       <dbl>            <dbl>#> 1 class_1               0.659 0.526       0.556       0.762            0.5#> 2 class_2               0.633 0.593       0.533       0.733            0.667#> 3 class_3               0.646 0.429       0.5         0.792            0.375#> # ℹ 7 more variables: `Neg Pred Value` <dbl>, Kappa <dbl>,#> #   `Detection Rate` <dbl>, `Detection Prevalence` <dbl>, Prevalence <dbl>,#> #   Support <int>, `Confusion Matrix` <named list># Evaluate the predicted classes# One prediction column with the class namesevaluate(data=multiclass_data,target_col="Target",prediction_cols="Predicted Class",type="multinomial")#> # A tibble: 1 × 16#>   `Overall Accuracy` `Balanced Accuracy`    F1 Sensitivity Specificity#>                <dbl>               <dbl> <dbl>       <dbl>       <dbl>#> 1              0.533               0.646 0.516       0.530       0.762#> # ℹ 11 more variables: `Pos Pred Value` <dbl>, `Neg Pred Value` <dbl>,#> #   Kappa <dbl>, MCC <dbl>, `Detection Rate` <dbl>,#> #   `Detection Prevalence` <dbl>, Prevalence <dbl>, Predictions <list>,#> #   `Confusion Matrix` <list>, `Class Level Results` <list>, Process <list>

Baseline evaluations

# Set seed for reproducibilityset.seed(1)# Partition the datasetpartitions<-groupdata2::partition(participant.scores,p=0.7,cat_col='diagnosis',id_col='participant',list_out=TRUE)train_set<-partitions[[1]]test_set<-partitions[[2]]

Binomial baseline

# Perform binomial baseline evaluation# Note: It's worth enabling parallelization (see ?baseline examples)binomial_baseline<- baseline_binomial(test_data=test_set,dependent_col="diagnosis",n=100)binomial_baseline$summarized_metrics#> # A tibble: 10 × 16#>    Measure `Balanced Accuracy` Accuracy      F1 Sensitivity Specificity#>    <chr>                 <dbl>    <dbl>   <dbl>       <dbl>       <dbl>#>  1 Mean                  0.496    0.496   0.481       0.475       0.517#>  2 Median                0.5      0.5     0.5         0.5         0.5#>  3 SD                    0.130    0.130   0.144       0.178       0.181#>  4 IQR                   0.167    0.167   0.195       0.208       0.333#>  5 Max                   0.833    0.833   0.833       0.833       0.833#>  6 Min                   0.25     0.25    0.182       0           0.167#>  7 NAs                   0        0       1           0           0#>  8 INFs                  0        0       0           0           0#>  9 All_0                 0.5      0.5   NaN           0           1#> 10 All_1                 0.5      0.5     0.667       1           0#> # ℹ 10 more variables: `Pos Pred Value` <dbl>, `Neg Pred Value` <dbl>,#> #   AUC <dbl>, `Lower CI` <dbl>, `Upper CI` <dbl>, Kappa <dbl>, MCC <dbl>,#> #   `Detection Rate` <dbl>, `Detection Prevalence` <dbl>, Prevalence <dbl>

binomial_baseline$random_evaluations#> # A tibble: 100 × 20#>    `Balanced Accuracy` Accuracy    F1 Sensitivity Specificity `Pos Pred Value`#>                  <dbl>    <dbl> <dbl>       <dbl>       <dbl>            <dbl>#>  1               0.417    0.417 0.462       0.5         0.333            0.429#>  2               0.667    0.667 0.6         0.5         0.833            0.75#>  3               0.5      0.5   0.571       0.667       0.333            0.5#>  4               0.417    0.417 0.364       0.333       0.5              0.4#>  5               0.583    0.583 0.545       0.5         0.667            0.6#>  6               0.583    0.583 0.545       0.5         0.667            0.6#>  7               0.667    0.667 0.667       0.667       0.667            0.667#>  8               0.417    0.417 0.364       0.333       0.5              0.4#>  9               0.333    0.333 0.333       0.333       0.333            0.333#> 10               0.583    0.583 0.545       0.5         0.667            0.6#> # ℹ 90 more rows#> # ℹ 14 more variables: `Neg Pred Value` <dbl>, AUC <dbl>, `Lower CI` <dbl>,#> #   `Upper CI` <dbl>, Kappa <dbl>, MCC <dbl>, `Detection Rate` <dbl>,#> #   `Detection Prevalence` <dbl>, Prevalence <dbl>,#> #   Predictions <list<tibble[,4]>>, ROC <list>,#> #   `Confusion Matrix` <list<tibble[,6]>>, Process <list>, Dependent <chr>

# First, remove the NAs from the F1 columnrandom_evaluations<-binomial_baseline$random_evaluationsrandom_evaluations<-random_evaluations[!is.na(random_evaluations$F1),]# Create density plot for F1plot_metric_density(baseline=random_evaluations,metric="F1",xlim= c(0,1))

Multinomial baseline

multiclass_baseline<- baseline_multinomial(test_data=multiclass_data,dependent_col="Target",n=100)# Summarized metricsmulticlass_baseline$summarized_metrics#> # A tibble: 15 × 13#>    Measure     `Overall Accuracy` `Balanced Accuracy`       F1 Sensitivity#>    <chr>                    <dbl>               <dbl>    <dbl>       <dbl>#>  1 Mean                    0.330               0.497    0.324       0.329#>  2 Median                  0.333               0.496    0.325       0.330#>  3 SD                      0.0823              0.0662   0.0760      0.0904#>  4 IQR                     0.108               0.0897   0.0987      0.123#>  5 Max                     0.5                 0.664    0.499       0.556#>  6 Min                     0.133               0.352    0.131       0.137#>  7 NAs                     0                   0       10           0#>  8 INFs                    0                   0        0           0#>  9 CL_Max                 NA                   0.770    0.688       0.833#> 10 CL_Min                 NA                   0.286    0.0870      0#> 11 CL_NAs                 NA                   0       10           0#> 12 CL_INFs                NA                   0        0           0#> 13 All_class_1             0.3                 0.5    NaN           0.333#> 14 All_class_2             0.5                 0.5    NaN           0.333#> 15 All_class_3             0.2                 0.5    NaN           0.333#> # ℹ 8 more variables: Specificity <dbl>, `Pos Pred Value` <dbl>,#> #   `Neg Pred Value` <dbl>, Kappa <dbl>, MCC <dbl>, `Detection Rate` <dbl>,#> #   `Detection Prevalence` <dbl>, Prevalence <dbl>

# Summarized class level results for class 1multiclass_baseline$summarized_class_level_results %>%dplyr::filter(Class=="class_1") %>%tidyr::unnest(Results)#> # A tibble: 10 × 13#>    Class   Measure `Balanced Accuracy`      F1 Sensitivity Specificity#>    <chr>   <chr>                 <dbl>   <dbl>       <dbl>       <dbl>#>  1 class_1 Mean                 0.493    0.314       0.339       0.648#>  2 class_1 Median               0.472    0.3         0.333       0.667#>  3 class_1 SD                   0.0933   0.119       0.148       0.103#>  4 class_1 IQR                  0.127    0.159       0.222       0.143#>  5 class_1 Max                  0.770    0.667       0.778       0.905#>  6 class_1 Min                  0.286    0.105       0           0.286#>  7 class_1 NAs                  0        1           0           0#>  8 class_1 INFs                 0        0           0           0#>  9 class_1 All_0                0.5    NaN           0           1#> 10 class_1 All_1                0.5      0.462       1           0#> # ℹ 7 more variables: `Pos Pred Value` <dbl>, `Neg Pred Value` <dbl>,#> #   Kappa <dbl>, `Detection Rate` <dbl>, `Detection Prevalence` <dbl>,#> #   Prevalence <dbl>, Accuracy <dbl># Random evaluations# Note, that the class level results for each repetition# are available as wellmulticlass_baseline$random_evaluations#> # A tibble: 100 × 18#>    Repetition `Overall Accuracy` `Balanced Accuracy`      F1 Sensitivity#>         <int>              <dbl>               <dbl>   <dbl>       <dbl>#>  1          1              0.2                 0.401 NaN           0.207#>  2          2              0.233               0.427   0.239       0.252#>  3          3              0.433               0.564   0.410       0.415#>  4          4              0.367               0.529   0.352       0.356#>  5          5              0.167               0.394   0.161       0.189#>  6          6              0.333               0.496   0.314       0.319#>  7          7              0.4                 0.534   0.359       0.363#>  8          8              0.467               0.608   0.462       0.485#>  9          9              0.3                 0.476   0.286       0.296#> 10         10              0.267               0.430   0.261       0.259#> # ℹ 90 more rows#> # ℹ 13 more variables: Specificity <dbl>, `Pos Pred Value` <dbl>,#> #   `Neg Pred Value` <dbl>, Kappa <dbl>, MCC <dbl>, `Detection Rate` <dbl>,#> #   `Detection Prevalence` <dbl>, Prevalence <dbl>,#> #   Predictions <list<tibble[,4]>>, `Confusion Matrix` <list<tibble[,4]>>,#> #   `Class Level Results` <list<tibble[,16]>>, Process <list>, Dependent <chr>

Gaussian baseline

gaussian_baseline<- baseline_gaussian(test_data=test_set,train_data=train_set,dependent_col="score",n=100)gaussian_baseline$summarized_metrics#> # A tibble: 9 × 8#>   Measure    RMSE      MAE `NRMSE(IQR)`   RRSE      RAE  RMSLE `Training Rows`#>   <chr>     <dbl>    <dbl>        <dbl>  <dbl>    <dbl>  <dbl>           <dbl>#> 1 Mean     19.6   15.8           0.944  1.04   1.02     0.559             9.88#> 2 Median   19.2   15.5           0.925  1.01   1        0.548            10#> 3 SD        1.19   0.941         0.0575 0.0630 0.0607   0.0303            3.31#> 4 IQR       0.682  0.00321       0.0328 0.0360 0.000207 0.0189            6#> 5 Max      26.8   21.8           1.29   1.42   1.41     0.727            15#> 6 Min      18.9   15.5           0.912  1.00   1        0.541             5#> 7 NAs       0      0             0      0      0        0                 0#> 8 INFs      0      0             0      0      0        0                 0#> 9 All_rows 19.1   15.5           0.923  1.01   1        0.543            18

gaussian_baseline$random_evaluations#> # A tibble: 100 × 12#>     RMSE   MAE `NRMSE(IQR)`  RRSE   RAE RMSLE        Predictions    Coefficients#>    <dbl> <dbl>        <dbl> <dbl> <dbl> <dbl> <list<tibble[,3]>> <list<tibble[,>#>  1  19.1  15.5        0.921  1.01  1    0.544           [12 × 3]        [1 × 10]#>  2  19.2  15.5        0.926  1.02  1    0.543           [12 × 3]        [1 × 10]#>  3  19.0  15.5        0.917  1.01  1    0.568           [12 × 3]        [1 × 10]#>  4  19.0  15.5        0.916  1.00  1    0.566           [12 × 3]        [1 × 10]#>  5  19.2  15.5        0.927  1.02  1    0.542           [12 × 3]        [1 × 10]#>  6  19.5  15.5        0.937  1.03  1    0.541           [12 × 3]        [1 × 10]#>  7  20.4  15.9        0.983  1.08  1.02 0.546           [12 × 3]        [1 × 10]#>  8  18.9  15.5        0.912  1.00  1    0.558           [12 × 3]        [1 × 10]#>  9  19.5  15.5        0.939  1.03  1    0.541           [12 × 3]        [1 × 10]#> 10  18.9  15.5        0.912  1.00  1    0.558           [12 × 3]        [1 × 10]#> # ℹ 90 more rows#> # ℹ 4 more variables: Process <list>, `Training Rows` <int>, Dependent <chr>,#> #   Fixed <chr>

plot_metric_density(baseline=gaussian_baseline$random_evaluations,metric="RMSE")

Generate model formulas

combine_predictors(dependent="y",fixed_effects= c("a","b","c"),random_effects="(1|d)")#>  [1] "y ~ a + (1|d)"                     "y ~ b + (1|d)"#>  [3] "y ~ c + (1|d)"                     "y ~ a * b + (1|d)"#>  [5] "y ~ a * c + (1|d)"                 "y ~ a + b + (1|d)"#>  [7] "y ~ a + c + (1|d)"                 "y ~ b * c + (1|d)"#>  [9] "y ~ b + c + (1|d)"                 "y ~ a * b * c + (1|d)"#> [11] "y ~ a * b + c + (1|d)"             "y ~ a * c + b + (1|d)"#> [13] "y ~ a + b * c + (1|d)"             "y ~ a + b + c + (1|d)"#> [15] "y ~ a * b + a * c + (1|d)"         "y ~ a * b + b * c + (1|d)"#> [17] "y ~ a * c + b * c + (1|d)"         "y ~ a * b + a * c + b * c + (1|d)"

combine_predictors(dependent="y",fixed_effects=list("a",list("b","log_b")),random_effects="(1|d)")#> [1] "y ~ a + (1|d)"         "y ~ b + (1|d)"         "y ~ log_b + (1|d)"#> [4] "y ~ a * b + (1|d)"     "y ~ a * log_b + (1|d)" "y ~ a + b + (1|d)"#> [7] "y ~ a + log_b + (1|d)"