analyze_results.Rmd

---
title: "CFA Benchmark Results"
author: "Maximilian Ernst"
date: "2/15/2022"
output: html_document
---

```{r setup, include=FALSE}
knitr::opts_chunk$set(echo = TRUE)
```

```{r load packages, include = FALSE}
library(ggplot2)
library(readr)
library(stringr)
library(lubridate)
library(dplyr)
library(purrr)
library(tidyr)
```

```{r load data, echo = FALSE}
# list all result files

files <- list.files("results")

files_julia <- str_subset(files, "julia")
files_lavaan <- str_subset(files, "lavaan")

# extract dates
dates_julia <- str_remove_all(files_julia, "benchmarks_julia_|.csv")
dates_lavaan <- str_remove_all(files_lavaan, "benchmarks_lavaan_|.csv")

dates_julia <- lubridate::ymd_hm(dates_julia)
dates_lavaan <- lubridate::ymd_hm(dates_lavaan)

files_julia <- data.frame(filename = files_julia, date = dates_julia)
files_lavaan <- data.frame(filename = files_lavaan, date = dates_lavaan)

files_julia <- arrange(files_julia, desc(date))
files_lavaan <- arrange(files_lavaan, desc(date))

# extract only two recent files
dates_julia <- files_julia$date[1:2]
julia_latest <- dates_julia[1]
julia_previous <- dates_julia[2]
files_julia <- files_julia$filename[1:2]
data_julia <- map(
    files_julia, 
    ~read_delim(
        paste("results/", .x, sep = ""), 
        delim = ";",  
        locale = locale(decimal_mark = "."),
        show_col_types = FALSE)
        )

data_julia[[1]] <-
    mutate(data_julia[[1]],
        datetime = dates_julia[1],
        package = "julia")
data_julia[[2]] <- 
    mutate(data_julia[[2]],
    datetime = dates_julia[2],
    package = "julia")

data_julia <- bind_rows(data_julia)
```


# Check solutions

All models are correct: `r all(data_julia$correct)`

```{r prepare data, echo=FALSE}
data_julia <- mutate(
    data_julia,
    across(
        c(median_time, mean_time, sd_time), 
        function(x){x/1e9}))

date_lavaan <- files_lavaan$date[1]
file_lavaan <- files_lavaan$filename[1]
data_lavaan <- read_delim(
    paste("results/", file_lavaan, sep = ""),
    delim = ";",
    locale = locale(decimal_mark = ","),
    show_col_types = FALSE)
data_lavaan <-
    mutate(data_lavaan,
    datetime = date_lavaan,
    package = "lavaan"
    )
if (any(!is.na(data_lavaan$error))) {print(data_lavaan$error)}
if (any(!is.na(data_lavaan$warnings))) {print(data_lavaan$warnings)}
if (any(!is.na(data_lavaan$messages))) {print(data_lavaan$messages)}
data <- bind_rows(data_julia, data_lavaan)

data <- mutate(
    data,
    n_parameters = 2*(n_factors*n_items) + n_factors*(n_factors-1)/2,
    se = sd_time/sqrt(n_repetitions)
    )
```

# Julia Versions

```{r julia, echo=FALSE}
data %>%
    filter(package == "julia") %>%
    ggplot(aes(
        x = n_parameters, 
        y = median_time, 
        color = interaction(as.factor(datetime), package))) +
    geom_point() +
    geom_line() +
    geom_errorbar(aes(ymin = median_time - 2*se, ymax = median_time + 2*se)) +
    labs(color = "time:package") +
    theme_minimal()
```

```{r julia log, echo=FALSE}
data %>%
    filter(package == "julia") %>%
    ggplot(aes(
        x = n_parameters, 
        y = median_time, 
        color = interaction(as.factor(datetime), package))) +
    geom_point() +
    geom_line() +
    geom_errorbar(aes(ymin = median_time - 2*se, ymax = median_time + 2*se)) +
    labs(color="time:package") +
    theme_minimal() +
    scale_y_log10() +
    scale_x_log10()
```

```{r julia ratio, echo=FALSE}
data %>%
    filter(package == "julia") %>%
    mutate(datetime = ifelse(datetime == julia_latest, "latest", "previous")) %>%
    select(n_parameters, datetime, median_time) %>%
    pivot_wider(
        names_from = datetime,
        values_from = median_time) %>%
    mutate(ratio_latest_previous = latest/previous) %>%
    ggplot(aes(
        x = n_parameters, 
        y = ratio_latest_previous)) +
    geom_point() +
    geom_line() +
    theme_minimal() +
    theme(text = element_text(color = "white"))
```

# Comparison to Lavaan

```{r lavaan, echo=FALSE}
data %>%
    ggplot(aes(
        x = n_parameters, 
        y = median_time, 
        color = interaction(as.factor(datetime), package))) +
    geom_point() +
    geom_line() +
    geom_errorbar(aes(ymin = median_time - 2*se, ymax = median_time + 2*se)) +
    labs(color="time:package") +
    theme_minimal()
```

```{r lavaan log, echo=FALSE}
data %>%
    ggplot(aes(
        x = n_parameters, 
        y = median_time, 
        color = interaction(as.factor(datetime), package))) +
    geom_point() +
    geom_line() +
    geom_errorbar(aes(ymin = median_time - 2*se, ymax = median_time + 2*se)) +
    labs(color = "time:package") +
    theme_minimal() +
    scale_y_log10() +
    scale_x_log10()
```

```{r lavaan ratio, echo=FALSE}
data %>%
    filter((package == "lavaan"|datetime == julia_latest)) %>%
    select(n_parameters, package, median_time) %>%
    pivot_wider(
        names_from = package,
        values_from = median_time) %>%
    mutate(ratio_julia_lavaan = julia/lavaan) %>%
    ggplot(aes(
        x = n_parameters,
        y = ratio_julia_lavaan)) +
    geom_point(color = "#d11131") +
    geom_line(color = "#d11131") +
    theme_minimal() +
    scale_y_continuous(
        limits = c(0, 1),
        breaks = c(seq(0, 0.1, 0.02), seq(0.1, 1, 0.1))
    )
```