Merge pull request #684 from lorenzwalthert/issue-680

- Add one more benchmark plot on recording for cache only (#680).

Author: lorenzwalthert

.github/workflows/benchmarking.yaml

@@ -1,21 +1,33 @@
+name: Continuous Benchmarks
 on:
   push:
     branches:
     - master
   pull_request:
     branches:
     - master
-
-name: Continuous Benchmarks
-
 jobs:
   build:
-    runs-on: macOS-latest
+    runs-on: ${{ matrix.config.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        config:
+          - {os: ubuntu-18.04,   r: '4.0.0', rspm: 'https://packagemanager.rstudio.com/all/__linux__/bionic/291'}
+
+    env:
+      R_REMOTES_NO_ERRORS_FROM_WARNINGS: true
+      RSPM: ${{ matrix.config.rspm }}
+      GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }}
     steps:
       - name: Checkout repo
         with:
           fetch-depth: 0
         uses: actions/checkout@master
+      - name: Set up git user
+        run: |
+          git config --local user.name "GitHub Actions"
+          git config --local user.email "[email protected]"
       - name: Ensure base branch is fetched
         if: ${{ github.event_name == 'pull_request' }}
         run: git branch $GITHUB_BASE_REF remotes/origin/$GITHUB_BASE_REF; git branch
@@ -27,15 +39,22 @@ jobs:
           saveRDS(remotes::dev_package_deps(dependencies = TRUE), ".github/depends.Rds", version = 2)
           writeLines(sprintf("R-%i.%i", getRversion()$major, getRversion()$minor), ".github/R-version")
         shell: Rscript {0}
-
       - name: Cache R packages
         if: runner.os != 'Windows'
         uses: actions/cache@v1
         with:
           path: ${{ env.R_LIBS_USER }}
-          key: ${{ hashFiles('.github/R-version') }}-1-${{ hashFiles('.github/depends.Rds') }}
-          restore-keys: ${{ hashFiles('.github/R-version') }}-1-
-
+          key: ${{ runner.os }}-${{ hashFiles('.github/R-version') }}-3-${{ hashFiles('.github/depends.Rds') }}
+          restore-keys: ${{ runner.os }}-${{ hashFiles('.github/R-version') }}-3-
+      - name: Install system dependencies
+        if: runner.os == 'Linux'
+        env:
+          RHUB_PLATFORM: linux-x86_64-ubuntu-gcc
+        run: |
+          Rscript -e "remotes::install_github('r-hub/sysreqs')"
+          sysreqs=$(Rscript -e "cat(sysreqs::sysreq_commands('DESCRIPTION'))")
+          sudo -s eval "$sysreqs"
+          sudo apt-get install libcurl4-openssl-dev libgit2-dev
       - name: Install dependencies
         run: |
           Rscript -e "install.packages(c('gert', 'ggplot2', 'purrr'))" -e "remotes::install_deps(dependencies = TRUE); remotes::install_github('r-lib/bench')"
@@ -48,6 +67,10 @@ jobs:
           path: bench/sources/here
       - name: Fetch existing benchmarks
         run: Rscript -e 'rlang::with_handlers(bench::cb_fetch(), error = function(e) paste("Could not fetch benchmarks, skipping. The error was", conditionMessage(e)))'
+      - name: Prepare PR comment
+        run: |
+          mkdir -p bench/pr-comment
+          echo 'Here is how the current PR would change benchmark results:\n' > bench/pr-comment/info.txt
       - name: Run benchmarks
         run: Rscript -e 'bench::cb_run()'
       - name: Show benchmarks
@@ -59,3 +82,13 @@ jobs:
       - name: Push benchmarks
         if: ${{ github.event_name == 'push' }}
         run: Rscript -e "bench::cb_push()"
+      - uses: actions/upload-artifact@v1
+        with:
+          name: results
+          path: bench/pr-comment
+      - uses: actions/download-artifact@v1
+        with:
+          name: results
+      - name: comment PR
+        #uses: machine-learning-apps/pr-comment@master
+        run: cat bench/pr-comment/info.txt

bench/01-declarations.R

@@ -17,11 +17,17 @@ plot_against_base <- function(new_bm,
   if (any(commit_is_reference) && Sys.getenv("GITHUB_BASE_REF") != "") {
     # if a pull request
     reference <- bm[commit_is_reference, "benchmarks"][[1]][[1]]
-    if (nrow(reference) > 0 && "name" %in% names(reference)) {
-      reference <- reference %>%
-        dplyr::filter(.data$name %in% !!name)
+    # if benchmark exists in base branch
+    reference <- reference %>%
+      dplyr::filter(.data$name %in% !!name)
+    if (nrow(reference) > 0) {
+      # if benchmark exists in base branch
       reference$expression <- bench:::new_bench_expr(Sys.getenv("GITHUB_BASE_REF"))
       new_bm <- dplyr::bind_rows(reference, new_bm)
+      stopifnot(nrow(new_bm) == 2)
+      diff_in_percent <- round(100 * diff(new_bm$p50) / new_bm$p50[1])
+      pr_comment <- glue::glue("* {name}: {new_bm$p50[1]} -> {new_bm$p50[2]} ({diff_in_percent}%)\n")
+      cat(pr_comment, file = "pr-comment/info.txt", append = TRUE)
     }
   }
   new_bm$branch <- factor(new_bm$expression)

bench/02-basic.R

@@ -6,31 +6,69 @@ library(magrittr)
 path <- "sources/here"
 dir.create("plots")
 cache_clear(ask = FALSE)
-cache_activate()
-cache_info()
+
 
 marker <- purrr::partial(
   bench::mark,
-  min_iterations = 20,
+  min_iterations = 25,
   check = FALSE,
   filter_gc = FALSE,
   memory = TRUE # skip uncached first round
 )
 
+# basically applying cache only
+# No transformer is ran ever because all code is already compliant.
+cache_activate()
+cache_info()
+Sys.sleep(3) # make system more idle
 with_cache <- marker(
-  with_cache = {
+  cache_appyling = {
     style_pkg(path, filetype = c("R", "rmd"))
   }
 )
+
+# basically recording cache only
+# transformers are always ran on expressions that were not compliant with the
+# style guide (some but not all expressions are)
+Sys.sleep(3) # make system more idle
+with_cache <- marker(
+  cache_recording = {
+    cat(
+      bench::system_time(gert::git_reset_hard(repo = path))[["process"]],
+      sep = "\n",
+      file = "timing-reset",
+      append = TRUE
+    )
+    style_pkg(path, filetype = c("R", "rmd"))
+  }
+)
+
 cache_info()
+
+# cache turned off
+# recording and applying, transformers always ran on all expressions.
 gert::git_reset_hard(repo = path)
 cache_deactivate()
-
+time_for_git_reset <- as.numeric(readLines("timing-reset"))[-1]
+cat(
+  "Waiting ",
+  round(mean(time_for_git_reset), 3),
+  " seconds on average to simulate git reset. That way, `without_cache` and ",
+  "`cache_recording` are comparable. The 95% interval for reset are (",
+  round(quantile(time_for_git_reset, 0.025), 3), ", ",
+  round(quantile(time_for_git_reset, 0.975), 3), ").",
+  sep = ""
+)
+mean_time_for_git_reset <- mean(time_for_git_reset)
+Sys.sleep(3) # make system more idle
 without_cache <- marker(
   without_cache = {
+    Sys.sleep(mean_time_for_git_reset)
     style_pkg(path, filetype = c("R", "rmd"))
   }
 )
+
+# visualize results
 latest_bm <- bench::cb_read()$benchmarks[[1]]
 split(latest_bm, latest_bm$name) %>%
   purrr::imap(plot_against_base)