Merge pull request #698 from lorenzwalthert/touchstone

- Switch to lorenzwalthert/touchstone (#698).

Author: lorenzwalthert

.github/workflows/benchmarking.yaml

@@ -21,16 +21,18 @@ jobs:
       GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }}
     steps:
       - name: Checkout repo
+        uses: actions/checkout@v2
         with:
-          fetch-depth: 0
-        uses: actions/checkout@master
+           fetch-depth: 0
       - name: Set up git user
         run: |
           git config --local user.name "GitHub Actions"
           git config --local user.email "[email protected]"
       - name: Ensure base branch is fetched
         if: ${{ github.event_name == 'pull_request' }}
-        run: git branch $GITHUB_BASE_REF remotes/origin/$GITHUB_BASE_REF; git branch
+        run: |
+          git checkout -b $GITHUB_HEAD_REF
+          git branch $GITHUB_BASE_REF remotes/origin/$GITHUB_BASE_REF
       - name: Setup R
         uses: r-lib/actions/setup-r@master
       - name: Query dependencies
@@ -57,39 +59,33 @@ jobs:
           sudo apt-get install libcurl4-openssl-dev libgit2-dev
       - name: Install dependencies
         run: |
-          Rscript -e "install.packages(c('gert', 'ggplot2', 'purrr'))" -e "remotes::install_deps(dependencies = TRUE); remotes::install_github('r-lib/bench')"
+          Rscript -e "install.packages(c('ggplot2', 'purrr'))" -e "remotes::install_deps(dependencies = TRUE); remotes::install_github(c('r-lib/bench', 'lorenzwalthert/touchstone')); "
           R CMD INSTALL .
       - name: Checkout benchmarking repo
         uses: actions/checkout@v2
         with:
           repository: lorenzwalthert/here
           ref: ca9c8e69c727def88d8ba1c8b85b0e0bcea87b3f
-          path: bench/sources/here
-      - name: Fetch existing benchmarks
-        run: Rscript -e 'rlang::with_handlers(bench::cb_fetch(), error = function(e) paste("Could not fetch benchmarks, skipping. The error was", conditionMessage(e)))'
+          path: touchstone/sources/here
       - name: Prepare PR comment
         run: |
-          mkdir -p bench/pr-comment
-          echo "Here is how the current PR would change benchmark results when merged into $GITHUB_BASE_REF:" > bench/pr-comment/info.txt
-          echo "$GITHUB_EVENT_PULL_REQUEST_HEAD_SHA" >> bench/pr-comment/info.txt
+          mkdir -p touchstone/sources
+          mkdir -p touchstone/pr-comment
+          mkdir -p touchstone/plots
+          echo "Here is how the current PR would change benchmark results when merged into $GITHUB_BASE_REF:" > touchstone/pr-comment/info.txt
+          echo "$GITHUB_EVENT_PULL_REQUEST_HEAD_SHA" >> touchstone/pr-comment/info.txt
       - name: Run benchmarks
-        run: Rscript -e 'bench::cb_run()'
-      - name: Show benchmarks
-        run: git notes --ref benchmarks show
+        run: Rscript 'touchstone/script.R'
       - uses: actions/upload-artifact@v2
         with:
           name: visual-benchmarks
-          path: bench/plots/
-      - name: Push benchmarks
-        if: ${{ github.event_name == 'push' }}
-        run: Rscript -e "bench::cb_push()"
+          path: touchstone/plots/
       - uses: actions/upload-artifact@v1
         with:
           name: results
-          path: bench/pr-comment
+          path: touchstone/pr-comment
       - uses: actions/download-artifact@v1
         with:
           name: results
       - name: comment PR
-        #uses: machine-learning-apps/pr-comment@master
-        run: cat bench/pr-comment/info.txt
+        run: cat touchstone/pr-comment/info.txt

.gitignore

@@ -11,3 +11,5 @@ R/scratch
 revdep/
 !revdep/*.md
 !revdep/problems.md
+touchstone/*
+!touchstone/script.R

bench/01-declarations.R

@@ -0,0 +1,63 @@
+# touchstone:::touchstone_clear() # deletes itself and sources
+refs <- c(Sys.getenv("GITHUB_BASE_REF", "touchstone"), Sys.getenv("GITHUB_HEAD_REF", "touchstone"))
+
+timer <- purrr::partial(touchstone::benchmark_run_ref,
+  refs = refs, n = 10
+)
+
+styler::cache_clear()
+
+timer(
+  expr_before_benchmark = c("library(styler)", "cache_deactivate()"),
+  without_cache = 'style_pkg("touchstone/sources/here", filetype = c("R", "rmd"))'
+)
+
+styler::cache_clear()
+
+timer(
+  expr_before_benchmark = c("library(styler)", "cache_activate()"),
+  cache_applying = 'style_pkg("touchstone/sources/here", filetype = c("R", "rmd"))'
+)
+
+styler::cache_clear()
+
+timer(
+  expr_before_benchmark = c(
+    "library(styler)",
+    "cache_activate()"
+  ),
+  cache_recording = c(
+    "gert::git_reset_hard(repo = 'touchstone/sources/here')",
+    'style_pkg("touchstone/sources/here", filetype = c("R", "rmd"))'
+  )
+)
+
+styler::cache_clear()
+
+
+
+for (benchmark in touchstone::benchmark_ls()) {
+  timings <- touchstone::benchmark_read(benchmark, refs)
+
+  library(ggplot2)
+  library(magrittr)
+  timings %>%
+    ggplot(aes(x = elapsed, color = ref)) +
+    geom_density()
+  fs::path("touchstone/plots/", benchmark) %>%
+    fs::path_ext_set("png") %>%
+    ggsave()
+
+  tbl <- timings %>%
+    dplyr::group_by(.data$ref) %>%
+    dplyr::summarise(m = mean(.data$elapsed)) %>%
+    tibble::deframe()
+
+  diff_percent <- round(100 * (tbl[refs[2]] - tbl[refs[1]]) / tbl[refs[1]], 1)
+  cat(
+    glue::glue("{benchmark}: {round(tbl[refs[1]], 2)} -> {round(tbl[refs[2]], 2)} ({diff_percent}%)"),
+    fill = TRUE,
+    file = "touchstone/pr-comment/info.txt",
+    append = TRUE
+  )
+}