From e807b9bf94400b16132bcc28171df466c6893b7c Mon Sep 17 00:00:00 2001 From: Barret Schloerke Date: Fri, 9 Jul 2021 11:06:10 -0400 Subject: [PATCH 01/13] Fix bad whitespace --- DESCRIPTION | 1 - 1 file changed, 1 deletion(-) diff --git a/DESCRIPTION b/DESCRIPTION index 8b3f4bb9..e2724825 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -10,7 +10,6 @@ License: GPL-2 Encoding: UTF-8 LazyData: true SystemRequirements: Chrome or other Chromium-based browser. chromium: chromium (rpm) or chromium-browser (deb) - Imports: jsonlite, websocket (>= 1.2.0), From fdb993cf1e9957fc4a48d2e8f8fbc0bc8dc2bbef Mon Sep 17 00:00:00 2001 From: Barret Schloerke Date: Fri, 9 Jul 2021 11:06:22 -0400 Subject: [PATCH 02/13] namespace `globalVariables()` --- R/protocol.R | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/R/protocol.R b/R/protocol.R index ca67aee0..8dde6471 100644 --- a/R/protocol.R +++ b/R/protocol.R @@ -1,6 +1,6 @@ #' @import rlang -globalVariables(c("self", "private", "callback_", "error_", "timeout_", "wait_")) +utils::globalVariables(c("self", "private", "callback_", "error_", "timeout_", "wait_")) # Given a protocol spec (essentially, the Chrome Devtools Protocol JSON # converted to an R object), returns a list of domains of the Devtools From e4ca90a26f1cc5a6d830be2eeecc2ee33ea0de9e Mon Sep 17 00:00:00 2001 From: Barret Schloerke Date: Fri, 9 Jul 2021 11:07:09 -0400 Subject: [PATCH 03/13] use r cmd check w/ pak --- .Rbuildignore | 1 + .github/.gitignore | 1 + .github/workflows/R-CMD-check.yaml | 112 +++++++++++++++++++++++++++++ README.md | 6 +- 4 files changed, 119 insertions(+), 1 deletion(-) create mode 100644 .github/.gitignore create mode 100644 .github/workflows/R-CMD-check.yaml diff --git a/.Rbuildignore b/.Rbuildignore index 645cd633..ec2439c6 100644 --- a/.Rbuildignore +++ b/.Rbuildignore @@ -2,3 +2,4 @@ ^\.Rproj\.user$ ^temp$ ^chromote\.sublime-project$ +^\.github$ diff --git a/.github/.gitignore b/.github/.gitignore new file mode 100644 index 00000000..2d19fc76 --- /dev/null +++ b/.github/.gitignore @@ -0,0 +1 @@ +*.html diff --git a/.github/workflows/R-CMD-check.yaml b/.github/workflows/R-CMD-check.yaml new file mode 100644 index 00000000..78155482 --- /dev/null +++ b/.github/workflows/R-CMD-check.yaml @@ -0,0 +1,112 @@ +# NOTE: This workflow is overkill for most R packages +# check-standard.yaml is likely a better choice +# usethis::use_github_action("check-standard") will install it. +# +# For help debugging build failures open an issue on the RStudio community with the 'github-actions' tag. +# https://community.rstudio.com/new-topic?category=Package%20development&tags=github-actions +on: + push: + branches: + - main + - master + pull_request: + branches: + - main + - master + +name: R-CMD-check + +jobs: + R-CMD-check: + runs-on: ${{ matrix.config.os }} + + name: ${{ matrix.config.os }} (${{ matrix.config.r }}) + + strategy: + fail-fast: false + matrix: + config: + - {os: macOS-latest, r: 'release'} + - {os: windows-latest, r: 'release'} + - {os: windows-latest, r: '3.6', rspm: "https://packagemanager.rstudio.com/cran/latest"} + - {os: ubuntu-18.04, r: 'devel', rspm: "https://packagemanager.rstudio.com/cran/__linux__/bionic/latest", http-user-agent: "R/4.1.0 (ubuntu-18.04) R (4.1.0 x86_64-pc-linux-gnu x86_64 linux-gnu) on GitHub Actions" } + - {os: ubuntu-18.04, r: 'release', rspm: "https://packagemanager.rstudio.com/cran/__linux__/bionic/latest"} + - {os: ubuntu-18.04, r: 'oldrel', rspm: "https://packagemanager.rstudio.com/cran/__linux__/bionic/latest"} + - {os: ubuntu-18.04, r: '3.6', rspm: "https://packagemanager.rstudio.com/cran/__linux__/bionic/latest"} + - {os: ubuntu-18.04, r: '3.5', rspm: "https://packagemanager.rstudio.com/cran/__linux__/bionic/latest"} + - {os: ubuntu-18.04, r: '3.4', rspm: "https://packagemanager.rstudio.com/cran/__linux__/bionic/latest"} + + env: + RSPM: ${{ matrix.config.rspm }} + GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }} + + steps: + - uses: actions/checkout@v2 + + - uses: r-lib/actions/setup-r@v1 + id: install-r + with: + r-version: ${{ matrix.config.r }} + http-user-agent: ${{ matrix.config.http-user-agent }} + + - uses: r-lib/actions/setup-pandoc@v1 + + - name: Install pak and query dependencies + run: | + install.packages("pak", repos = "https://r-lib.github.io/p/pak/dev/") + saveRDS(pak::pkg_deps("local::.", dependencies = TRUE), ".github/r-depends.rds") + shell: Rscript {0} + + - name: Restore R package cache + uses: actions/cache@v2 + with: + path: | + ${{ env.R_LIBS_USER }}/* + !${{ env.R_LIBS_USER }}/pak + key: ${{ matrix.config.os }}-${{ steps.install-r.outputs.installed-r-version }}-1-${{ hashFiles('.github/r-depends.rds') }} + restore-keys: ${{ matrix.config.os }}-${{ steps.install-r.outputs.installed-r-version }}-1- + + - name: Install system dependencies + if: runner.os == 'Linux' + run: | + pak::local_system_requirements(execute = TRUE) + pak::pkg_system_requirements("rcmdcheck", execute = TRUE) + shell: Rscript {0} + + - name: Install dependencies + run: | + pak::local_install_dev_deps(upgrade = TRUE) + pak::pkg_install("rcmdcheck") + shell: Rscript {0} + + - name: Session info + run: | + options(width = 100) + pkgs <- installed.packages()[, "Package"] + sessioninfo::session_info(pkgs, include_base = TRUE) + shell: Rscript {0} + + - name: Check + env: + _R_CHECK_CRAN_INCOMING_: false + run: | + options(crayon.enabled = TRUE) + rcmdcheck::rcmdcheck(args = c("--no-manual", "--as-cran"), error_on = "warning", check_dir = "check") + shell: Rscript {0} + + - name: Show testthat output + if: always() + run: find check -name 'testthat.Rout*' -exec cat '{}' \; || true + shell: bash + + - name: Upload check results + if: failure() + uses: actions/upload-artifact@main + with: + name: ${{ matrix.config.os }}-r${{ matrix.config.r }}-results + path: check + + - name: Don't use tar from old Rtools to store the cache + if: ${{ runner.os == 'Windows' && startsWith(steps.install-r.outputs.installed-r-version, '3.6' ) }} + shell: bash + run: echo "C:/Program Files/Git/usr/bin" >> $GITHUB_PATH diff --git a/README.md b/README.md index acdc33bf..a0f359ee 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,11 @@ Chromote: Headless Chrome Remote Interface ========================================== -**Please note that Chromote is in development and the API is subject to change** + +[![R-CMD-check](https://github.com/rstudio/chromote/workflows/R-CMD-check/badge.svg)](https://github.com/rstudio/chromote/actions) + + +**!! Please note that Chromote is in development and the API is subject to change !!** Chromote is an R implementation of the [Chrome Devtools Protocol](https://chromedevtools.github.io/devtools-protocol/). It works with Chrome, Chromium, Opera, Vivaldi, and other browsers based on [Chromium](https://www.chromium.org/). By default it uses Google Chrome (which must already be installed on the system). To use a different browser, see [Specifying which browser to use](#specifying-which-browser-to-use). From a4a18b138e43f21249fb28b67ad98acdfb5cce31 Mon Sep 17 00:00:00 2001 From: Barret Schloerke Date: Fri, 9 Jul 2021 11:09:44 -0400 Subject: [PATCH 04/13] `usethis::use_github_links()` --- DESCRIPTION | 2 ++ 1 file changed, 2 insertions(+) diff --git a/DESCRIPTION b/DESCRIPTION index e2724825..0958d2d7 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -24,3 +24,5 @@ Suggests: showimage Roxygen: list(markdown = TRUE) RoxygenNote: 7.1.1 +URL: https://github.com/rstudio/chromote +BugReports: https://github.com/rstudio/chromote/issues From 94e268c1a5c78ee7ccdbff20fb6ea2f7c477c63d Mon Sep 17 00:00:00 2001 From: Barret Schloerke Date: Fri, 9 Jul 2021 11:10:48 -0400 Subject: [PATCH 05/13] init pkgdown website --- .Rbuildignore | 3 ++ .github/workflows/pkgdown-pak.yaml | 61 ++++++++++++++++++++++++++++++ .gitignore | 1 + pkgdown/_pkgdown.yml | 1 + 4 files changed, 66 insertions(+) create mode 100644 .github/workflows/pkgdown-pak.yaml create mode 100644 pkgdown/_pkgdown.yml diff --git a/.Rbuildignore b/.Rbuildignore index ec2439c6..bfc34423 100644 --- a/.Rbuildignore +++ b/.Rbuildignore @@ -3,3 +3,6 @@ ^temp$ ^chromote\.sublime-project$ ^\.github$ +^_pkgdown\.yml$ +^docs$ +^pkgdown$ diff --git a/.github/workflows/pkgdown-pak.yaml b/.github/workflows/pkgdown-pak.yaml new file mode 100644 index 00000000..723f7b6d --- /dev/null +++ b/.github/workflows/pkgdown-pak.yaml @@ -0,0 +1,61 @@ +on: + push: + branches: + - main + - master + tags: + -'*' + +name: pkgdown + +jobs: + pkgdown: + runs-on: ubuntu-18.04 + env: + RSPM: https://packagemanager.rstudio.com/cran/__linux__/bionic/latest + GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }} + + steps: + - uses: actions/checkout@v2 + + - uses: r-lib/actions/setup-r@v1 + id: install-r + + - uses: r-lib/actions/setup-pandoc@v1 + + - name: Install pak and query dependencies + run: | + install.packages("pak", repos = "https://r-lib.github.io/p/pak/dev/") + saveRDS(pak::pkg_deps("local::.", dependencies = TRUE), ".github/r-depends.rds") + shell: Rscript {0} + + - name: Restore R package cache + uses: actions/cache@v2 + with: + path: | + ${{ env.R_LIBS_USER }}/* + !${{ env.R_LIBS_USER }}/pak + key: ubuntu-18.04-${{ steps.install-r.outputs.installed-r-version }}-1-${{ hashFiles('.github/r-depends.rds') }} + restore-keys: ubuntu-18.04-${{ steps.install-r.outputs.installed-r-version }}-1- + + - name: Install system dependencies + if: runner.os == 'Linux' + run: | + pak::local_system_requirements(execute = TRUE) + pak::pkg_system_requirements("pkgdown", execute = TRUE) + shell: Rscript {0} + + - name: Install dependencies + run: | + pak::local_install_dev_deps(upgrade = TRUE, dependencies = c("all", "Config/Needs/website")) + pak::pkg_install("pkgdown") + shell: Rscript {0} + + - name: Install package + run: R CMD INSTALL . + + - name: Build and deploy pkgdown site + run: | + git config --local user.name "$GITHUB_ACTOR" + git config --local user.email "$GITHUB_ACTOR@users.noreply.github.com" + Rscript -e 'pkgdown::deploy_to_branch(new_process = FALSE)' diff --git a/.gitignore b/.gitignore index 20bbc7fb..1e8abaeb 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,4 @@ .RData .Rproj.user temp +docs diff --git a/pkgdown/_pkgdown.yml b/pkgdown/_pkgdown.yml new file mode 100644 index 00000000..37ec01b5 --- /dev/null +++ b/pkgdown/_pkgdown.yml @@ -0,0 +1 @@ +url: http://rstudio.github.io/chromote From 8095c2fe82c3364704d3445f83ba5a2df93f81af Mon Sep 17 00:00:00 2001 From: Barret Schloerke Date: Fri, 9 Jul 2021 11:13:14 -0400 Subject: [PATCH 06/13] add a cran badge; add lifecycle badge --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index a0f359ee..264b47aa 100644 --- a/README.md +++ b/README.md @@ -3,6 +3,8 @@ Chromote: Headless Chrome Remote Interface [![R-CMD-check](https://github.com/rstudio/chromote/workflows/R-CMD-check/badge.svg)](https://github.com/rstudio/chromote/actions) +[![CRAN status](https://www.r-pkg.org/badges/version/chromote)](https://CRAN.R-project.org/package=chromote) +[![Lifecycle: experimental](https://img.shields.io/badge/lifecycle-experimental-orange.svg)](https://lifecycle.r-lib.org/articles/stages.html#experimental) **!! Please note that Chromote is in development and the API is subject to change !!** From 98c1e11f6c497065e09457ce9ef5a4fb2cb63361 Mon Sep 17 00:00:00 2001 From: Barret Schloerke Date: Tue, 13 Jul 2021 17:20:50 -0400 Subject: [PATCH 07/13] Set up a README.Rmd file to leverage raw R chunk output --- .github/workflows/render-readme-pak.yaml | 64 ++ README.Rmd | 1025 +++++++++++++++++++ README.md | 1162 ++++++++++++++++------ 3 files changed, 1938 insertions(+), 313 deletions(-) create mode 100644 .github/workflows/render-readme-pak.yaml create mode 100644 README.Rmd diff --git a/.github/workflows/render-readme-pak.yaml b/.github/workflows/render-readme-pak.yaml new file mode 100644 index 00000000..ff11e8c0 --- /dev/null +++ b/.github/workflows/render-readme-pak.yaml @@ -0,0 +1,64 @@ +on: + push: + paths: + - 'README.Rmd' + +name: Render README.Rmd + +jobs: + render: + runs-on: ubuntu-18.04 + env: + RSPM: https://packagemanager.rstudio.com/cran/__linux__/bionic/latest + GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }} + + steps: + - uses: actions/checkout@v2 + + - uses: r-lib/actions/setup-r@v1 + id: install-r + + - uses: r-lib/actions/setup-pandoc@v1 + + - name: Install pak and query dependencies + run: | + install.packages("pak", repos = "https://r-lib.github.io/p/pak/dev/") + saveRDS(pak::pkg_deps("local::.", dependencies = TRUE), ".github/r-depends.rds") + shell: Rscript {0} + + - name: Restore R package cache + uses: actions/cache@v2 + with: + path: | + ${{ env.R_LIBS_USER }}/* + !${{ env.R_LIBS_USER }}/pak + key: ubuntu-18.04-${{ steps.install-r.outputs.installed-r-version }}-1-${{ hashFiles('.github/r-depends.rds') }} + restore-keys: ubuntu-18.04-${{ steps.install-r.outputs.installed-r-version }}-1- + + - name: Install system dependencies + if: runner.os == 'Linux' + run: | + pak::local_system_requirements(execute = TRUE) + pak::pkg_system_requirements("pkgdown", execute = TRUE) + shell: Rscript {0} + + - name: Install dependencies + run: | + pak::local_install_dev_deps(upgrade = TRUE, dependencies = c("all", "Config/Needs/website")) + pak::pkg_install(c("cran::pkgdown", "cran::rmarkdown")) + shell: Rscript {0} + + - name: Install package + run: R CMD INSTALL . + + - name: Render README + shell: Rscript {0} + run: | + rmarkdown::render("README.Rmd") + + - name: Commit results + run: | + git config --local user.email "actions@github.com" + git config --local user.name "GitHub Actions" + git commit README.md -m 'Re-build README.Rmd' || echo "No changes to commit" + git push origin || echo "No changes to commit" diff --git a/README.Rmd b/README.Rmd new file mode 100644 index 00000000..14ff71e0 --- /dev/null +++ b/README.Rmd @@ -0,0 +1,1025 @@ +--- +output: github_document +--- + + + +```{r, include = FALSE} +knitr::opts_chunk$set( + collapse = TRUE, + comment = "#>", + fig.path = "man/figures/README-", + out.width = "100%" +) +``` +Chromote: Headless Chrome Remote Interface +========================================== + + +[![R-CMD-check](https://github.com/rstudio/chromote/workflows/R-CMD-check/badge.svg)](https://github.com/rstudio/chromote/actions) +[![CRAN status](https://www.r-pkg.org/badges/version/chromote)](https://CRAN.R-project.org/package=chromote) +[![Lifecycle: experimental](https://img.shields.io/badge/lifecycle-experimental-orange.svg)](https://lifecycle.r-lib.org/articles/stages.html#experimental) + + +**!! Please note that Chromote is in development and the API is subject to change !!** + +Chromote is an R implementation of the [Chrome Devtools Protocol](https://chromedevtools.github.io/devtools-protocol/). It works with Chrome, Chromium, Opera, Vivaldi, and other browsers based on [Chromium](https://www.chromium.org/). By default it uses Google Chrome (which must already be installed on the system). To use a different browser, see [Specifying which browser to use](#specifying-which-browser-to-use). + +Chromote is not the only R package that implements the Chrome Devtools Protocol. Here are some others: + +* [crrri](https://github.com/RLesur/crrri) by Romain Lesur and Christophe Dervieux +* [decapitated](https://github.com/hrbrmstr/decapitated/) by Bob Rudis +* [chradle](https://github.com/milesmcbain/chradle) by Miles McBain + +The interface to Chromote is similar to [chrome-remote-interface](https://github.com/cyrus-and/chrome-remote-interface) for node.js. + +## Installation + +```R +remotes::install_github("rstudio/chromote") +``` + +## Basic usage + +This will start a headless browser and open an interactive viewer for it in a normal browser, so that you can see what the headless browser is doing. + +```{r init} +library(chromote) + +b <- ChromoteSession$new() + +# In a web browser, open a viewer for the headless browser. Works best with +# Chromium-based browsers. +b$view() +``` + +The browser can be given _commands_, as specified by the [Chrome Devtools Protocol](https://chromedevtools.github.io/devtools-protocol/). For example, `$Browser$getVersion()` (which corresponds to the [Browser.getVersion](https://chromedevtools.github.io/devtools-protocol/tot/Browser#method-getVersion) in the API docs) will query the browser for version information: + + +```{r getVersion} +b$Browser$getVersion() +``` + + +If you have the viewer open and run the following, you'll see the web page load in the viewer: + +```{r} +b$Page$navigate("https://www.r-project.org/") +``` + +In the official Chrome DevTools Protocol (CDP) documentation, this is the [`Page.navigate`](https://chromedevtools.github.io/devtools-protocol/tot/Page#method-navigate) command. + +In addition to full support of the CDP, ChromoteSession objects also some convenience methods, like `$screenshot()`. (See the Examples section below for more information about screenshots.) + +```{r screenshot} +# Saves to screenshot.png +b$screenshot() + +# Takes a screenshot of elements picked out by CSS selector +b$screenshot("sidebar.png", selector = ".sidebar") +``` + +![Sidebar Screenshot](sidebar.png) + +**Note:** All members of `Chromote` and `ChromoteSession` objects which start with a capital letter (like `b$Page`, `b$DOM`, and `b$Browser`) correspond to domains from the Chrome Devtools Protocol, and are documented in the [official CDP site](https://chromedevtools.github.io/devtools-protocol/). All members which start with a lower-case letter (like `b$screenshot` and `b$close`) are not part of the Chrome Devtools Protocol, and are specific to `Chromote` and `ChromoteSession`. + +Here is an example of how to use Chromote to find the position of a DOM element using [DOM.getBoxModel](https://chromedevtools.github.io/devtools-protocol/tot/DOM/#method-getBoxModel). + +```{r boxModel} +x <- b$DOM$getDocument() +x <- b$DOM$querySelector(x$root$nodeId, ".sidebar") +x <- b$DOM$getBoxModel(x$nodeId) +str(x) +``` + +Or you can do the same thing by chaining commands together with a magrittr pipe: + +```{r boxModelChain} +b$DOM$getDocument() %>% + { b$DOM$querySelector(.$root$nodeId, ".sidebar") } %>% + { b$DOM$getBoxModel(.$nodeId) } %>% + str() +``` + +### Creating new tabs and managing the process + +To create a new tab/window: + +```{r newSession} +b1 <- b$new_session() +``` + +Once it's created, you can perform operations with the new tab without affecting the first one. + +```{r newTabOperation} +b1$view() +b1$Page$navigate("https://github.com/rstudio/chromote") +``` + +To close a browser tab/window, you can run: + +```{r closeTab, eval = FALSE} +b1$close() +``` + +This is different from shutting down the browser process. If you call `b$close()`, the browser process will still be running, even if all tabs have been closed. If all tabs have been closed, you can still create a new tab by calling `b1$new_session()`. + +To shut down the process, call: + +```{r closeProcess, eval = FALSE} +b1$parent$stop() +``` + +`b1$parent` is a `Chromote` object (as opposed to `ChromoteSession`), which represents the browser as a whole. This is explained in [The Chromote object model](#the-chromote-object-model). + +### Commands and Events + +The Chrome Devtools Protocol has two types of methods: _commands_ and _events_. The methods used in the previous examples are commands. That is, they tell the browser to do something; the browser does it, and then sends back some data. + +Events are quite different from commands. When, for example, you run `b$Page$loadEventFired()`, it does not send a message to the browser. Rather, this method tells the R process to wait until it receives a `Page.loadEventFired` message from the browser. + +Here is an example of how that event can be used. Note that these two lines of code must be run together, without any delay at all (this can be enforced by wrapping both lines of code in `{ .... }`). + +```{r loadEventFired} +# Send a command to navigate to a page +b$Page$navigate("https://www.r-project.org") +# Wait for the Page.loadEventFired event +b$Page$loadEventFired() +``` + +After running these two lines, the R process will be blocked. While it's blocked, the browser will load the page, and then send a message to the R process saying that the `Page.loadEventFired` event has occurred. The message looks something like this: + +```JSON +{"method":"Page.loadEventFired","params":{"timestamp":699232.345338}} +``` + +After the R process receives this message, the function returns the value, which looks like this: + +``` +$timestamp +[1] 699232.3 +``` + +> **Technical note:** Chromote insulates the user from some of the details of how the CDP implements event notifications. Event notifications are not sent from the browser to the R process by default; you must first send a command to enable event notifications for a domain. For example `Page.enable` enables event notifications for the `Page` domain -- the browser will send messages for _all_ `Page` events. (See the Events section in [this page](https://chromedevtools.github.io/devtools-protocol/tot/Page)). These notifications will continue to be sent until the browser receives a `Page.disable` command. +> +> By default, Chromote hides this implementation detail. When you call `b$Page$loadEventFired()`, Chromote sends a `Page.enable` command automatically, and then waits until it receives the `Page.loadEventFired` event notification. Then it sends a `Page.disable` command. +> +> Note that in asynchronous mode, the behavior is slightly more sophisticated: it maintains a counter of how many outstanding events it is waiting for in a given domain. When that count goes from 0 to 1, it sends the `X.enable` command; when the count goes from 1 to 0, it sends the `X.disable` command. For more information, see the [Async events](#async-events) section. +> +> If you do not want automatic event enabling and disabling, then when creating the ChromoteSession object, use `ChromoteSession$new(auto_events = FALSE)`. + + +### The Chromote object model + +There are two R6 classes that are used to represent the Chrome browser. One is `Chromote`, and the other is `ChromoteSession`. A `Chromote` object represents the browser as a whole, and it can have multiple _targets_, which each represent a browser tab. In the Chrome Devtools Protocol, each target can have one or more debugging _sessions_ to control it. A `ChromoteSession` object represents a single _session_. + +When a `ChromoteSession` object is instantiated, a target is created, then a session is attached to that target, and the `ChromoteSession` object represents the session. (It is possible, though not very useful, to have multiple `ChromoteSession` objects connected to the same target, each with a different session.) + +A `Chromote` object can have any number of `ChromoteSession` objects as children. It is not necessary to create a `Chromote` object manually. You can simply call: + +```{r newSession2} +b <- ChromoteSession$new() +``` + +and it will automatically create a `Chromote` object if one has not already been created. The Chromote package will then designate that `Chromote` object as the _default_ Chromote object for the package, so that any future calls to `ChromoteSession$new()` will automatically use the same `Chromote`. This is so that it doesn't start a new browser for every `ChromoteSession` object that is created. + +In the Chrome Devtools Protocol, most commands can be sent to individual sessions using the `ChromoteSession` object, but there are some commands which can only be sent to the overall browser, using the `Chromote` object. + +To access the parent `Chromote` object from a `ChromoteSession`, you can simply use `$parent`: + +```{r chromoteObject} +b <- ChromoteSession$new() +m <- b$parent +``` + +With a `Chromote` object, you can get a list containing all the `ChromoteSession`s, with `$get_sessions()`: + +```{r sessions, include = FALSE} +m$get_sessions() +``` + +Normally, subsequent calls to `ChromoteSession$new()` will use the existing `Chromote` object. However, if you want to start a new browser process, you can manually create a `Chromote` object, then spawn a session from it; or you can pass the new `Chromote` object to ` ChromoteSession$new()`: + +```{r manualSession} +cm <- Chromote$new() +b1 <- cm$new_session() + +# Or: +b <- ChromoteSession$new(parent = Chromote$new()) +``` + +Note that if you use either of these methods, the new `Chromote` object will _not_ be set as the default that is used by future calls to `ChromoteSesssion$new()`. See [Specifying which browser to use](#specifying-which-browser-to-use) for information on setting the default `Chromote` object. + + +There are also the following classes which represent the browser at a lower level: + +* `Browser`: This represents an instance of a browser that supports the Chrome Devtools Protocol. It contains information about how to communicate with the Chrome browser. A `Chromote` object contains one of these. +* `Chrome`: This is a subclass of `Browser` that represents a local browser. It extends the `Browser` class with a `processx::process` object, which represents the browser's system process. +* `ChromeRemote`: This is a subclass of `Browser` that represents a browser running on a remote system. + + +### Debugging + +Calling `b$debug_messages(TRUE)` will enable the printing of all the JSON messages sent between R and Chrome. This can be very helpful for understanding how the Chrome Devtools Protocol works. + +```{r debugMessages} +b <- ChromoteSession$new() +b$parent$debug_messages(TRUE) +b$Page$navigate("https://www.r-project.org/") +``` + +### Synchronous vs. asynchronous usage + +By default, when you call methods from a `Chromote` or `ChromoteSession` object, it operates in _synchronous_ mode. For example, when you call a command function (like `b$Page$navigate()`), a command message is sent to the headless browser, the headless browser executes that command, and it sends a response message back. When the R process receives the response, it converts it from JSON to an R object and the function returns that value. During this time, the R process is blocked; no other R code can execute. + +The methods in Chromote/ChromoteSession objects can also be called in _asynchronous_ mode. In async mode, a command function fires off a message to the browser, and then the R process continues running other code; when the response comes back at some time in the future, the R process calls another function and passes the response value to it. + +There are two different ways of using async with Chromote. The first is with [promises](https://rstudio.github.io/promises/) (note that these are not the regular R-language promises; these are similar to JavaScript promises for async programming.) The second way is with callbacks: you call methods with a `callback_` argument. Although callbacks are initially easier to use than promises, once you start writing more complex code, managing callbacks becomes very difficult, especially when error handling is involved. For this reason, this document will focus mostly on promises instead of callback-style programming. + +When Chromote methods are called in synchronous mode, under the hood, they are implemented with asynchronous functions, and then waiting for the asynchronous functions to resolve. + +> **Technical note about the event loop**: When methods are called asynchronously, the R process will run callbacks and promises using an event loop provided by the [later](https://github.com/r-lib/later) package. This event loop is very similar to the one used in JavaScript, which is explained in depth by [this article](https://blog.sessionstack.com/how-javascript-works-event-loop-and-the-rise-of-async-programming-5-ways-to-better-coding-with-2f077c4438b5). One important difference between JavaScript's event loop and the one provided by **later**'s is that in JavaScript, the event loop only runs when the call stack is empty (essentially, when the JS runtime is idle); with **later** the event loop similarly runs when the call stack is empty (when the R console is idle), but it can also be run at any point by calling `later::run_now()`. +> +> There is another important difference between the JS event loop and the one used by Chromote: Chromote uses _private event loops_ provided by [later](https://github.com/r-lib/later). Running the private event loop with `run_now()` will not interfere with the global event loop. This is crucial for being able to run asynchronous code in a way that appears synchronous. + + +#### Why async? + +The synchronous API is easier to use than the asynchronous one. So why would you want to use the async API? Here are some reasons: + +* The async API allows you to send commands to the browser that may take some time for the browser to complete, and they will not block the R process from doing other work while the browser executes the command. +* The async API lets you send commands to multiple browser "tabs" and let them work in parallel. + +On the other hand, async programming can make it difficult to write code that proceeds in a straightforward, linear manner. Async programming may be difficult to use in, say, an analysis script. + +When using Chromote interactively at the R console, it's usually best to just call methods synchronously. This fits well with a iterative, interactive data analysis workflow. + +When you are _programming_ with Chromote instead of using it interactively, it is in many cases better to call the methods asynchronously, because it allows for better performance. In a later section, we'll see how to write asynchronous code with Chromote that can be run either synchronously or asynchronously. This provides the best of both worlds. + + +#### Async commands + +When a method is called in synchronous mode, it blocks until the browser sends back a response, and then it returns the value, converted from JSON to an R object. For example: + +```{r syncMode} +# Synchronous +str(b$Browser$getVersion()) +``` + +In async mode, there are two ways to use the value that the browser sends to the R process. One is to use the `callback_` argument with `wait_=FALSE`. The `wait_=FALSE` tells it to run the command in async mode; instead of returning the value from the browser, it returns a promise. For example: + +```{r asyncMode} +# Async with callback +b$Browser$getVersion(wait_ = FALSE, callback_ = str) +# After a very, very short pause, prints: +``` + +```{r asyncModeOutput, echo = FALSE, include = TRUE} +later::run_now() +later::run_now() +later::run_now() +later::run_now() +later::run_now() +later::run_now() +later::run_now() +``` + +Notice that the function returned ``, and then it printed out the data. We'll come back to the promise part. + +> **Technical note:** When you pass a function as `callback_`, that function is used as the first step in the promise chain that is returned. + +If you run the command in a code block (or a function), the entire code block will finish executing before the callback can be executed. For example: + +```{r codeBlock} +{ + b$Browser$getVersion(wait_ = FALSE, callback_ = str) + 1+1 +} +``` + +```{r codeBlockOutput, echo = FALSE, include = TRUE} +later::run_now() +later::run_now() +later::run_now() +later::run_now() +later::run_now() +later::run_now() +later::run_now() +``` + +In the code above, it executes the `1+1` and returns the value before the `str` callback can be executed on the message from the browser. + +If you want to store the value from the browser, you can write a callback that stores the value like so: + +```{r callbackValue} +# This will extract the product field +product <- NULL +b$Browser$getVersion(wait_ = FALSE, callback_ = function(msg) { + product <<- msg$product +}) +``` + +```{r callbackValueWait, echo = FALSE, include = TRUE} +later::run_now() +later::run_now() +later::run_now() +later::run_now() +later::run_now() +later::run_now() +later::run_now() +``` + +```{r callbackValueValue} +# Wait for a moment, then run: +product +``` + +But to get the value, you need to wait for the callback to execute before you can use the value. Waiting for a value is simple when running R interactively -- you can just add a `message("message arrived")` call in the callback and wait for it before running the next line of code -- but waiting for the value is not easy to do using ordinary straight-line coding. Fortunately, Chromote has a way to wait for async operations, which we'll see later. + +The other way of using the value is to use _promises_. If `wait_=FALSE` and no `callback_` is passed to the command, then it will simply return a promise. Promises have many advantages over plain old callbacks: they are easier to chain, and they provide better error-handling capabilities. You can _chain_ more steps to the promise: when the promise resolves -- that is, when the message is received from the browser -- it will run the next step in the promise chain. + +Here's an example that uses promises to print out the version information. Note that the surrounding curly braces are there to indicate that this whole thing must be run as a block without any idle time in between the function calls -- if you were to run the code in the R console line-by-line, the browser would send back the message and the promise would resolve before you called `p$then()`, which is where you tell the promise what to do with the return value. (The curly braces aren't strictly necessary -- you could run the code inside the braces in a single paste operation and have the same effect.) + +```{r callbackValueBlock} +{ + p <- b$Browser$getVersion(wait_ = FALSE) + p$then(function(value) { + print(value$product) + }) +} +# Wait for a moment, then prints: +``` + +```{r callbackValueBlockOutput, echo = FALSE, include = TRUE} +later::run_now() +later::run_now() +later::run_now() +later::run_now() +later::run_now() +later::run_now() +later::run_now() +``` + +Here are some progressively more concise ways of achieving the same thing. As you work with promises, you will see these various forms of promise chaining. For more information, see the [promises documentation](https://rstudio.github.io/promises/). + +```{r concise} +library(promises) + +# Regular function pipe to then() +b$Browser$getVersion(wait_ = FALSE) %>% then(function(value) { + print(value$product) +}) + +# Promise-pipe to anonymous function, which must be wrapped in parens +b$Browser$getVersion(wait_ = FALSE) %...>% (function(value) { + print(value$product) +}) + +# Promise-pipe to an expression (which gets converted to a function with the first argument `.`) +b$Browser$getVersion(wait_ = FALSE) %...>% { print(.$product) } + +# Promise-pipe to a named function, with parentheses +print_product <- function(msg) print(msg$product) +b$Browser$getVersion(wait_ = FALSE) %...>% print_product() + +# Promise-pipe to a named function, without parentheses +b$Browser$getVersion(wait_ = FALSE) %...>% print_product +``` + + +The earlier example where we found the dimensions of a DOM element using CSS selectors was done with the synchronous API and `%>%` pipes. The same can be done in async mode by switching from the regular pipe to the promise-pipe, and calling all the methods with `wait_=FALSE`: + +```{r promisePipe} +b$DOM$getDocument(wait_ = FALSE) %...>% + { b$DOM$querySelector(.$root$nodeId, ".sidebar", wait_ = FALSE) } %...>% + { b$DOM$getBoxModel(.$nodeId, wait_ = FALSE) } %...>% + str() + + +# Or, more verbosely: +b$DOM$getDocument(wait_ = FALSE)$ + then(function(value) { + b$DOM$querySelector(value$root$nodeId, ".sidebar", wait_ = FALSE) + })$ + then(function(value) { + b$DOM$getBoxModel(value$nodeId, wait_ = FALSE) + })$ + then(function(value) { + str(value) + }) +``` + +Each step in the promise chain uses the value from the previous step, via `.` or `value`. Note that not all asynchronous code works in such a linear, straightforward way. Sometimes it is necessary to save data from intermediate steps in a broader-scoped variable, if it is to be used in a later step in the promise chain. + + +#### Turning asynchronous code into synchronous code + +There may be times, especially when programming with Chromote, where you want to wait for a promise to resolve before continuing. To do this, you can use the Chromote or ChromoteSession's `wait_for()` method. + +```{r waitFor} +# A promise chain +p <- b$DOM$getDocument(wait_ = FALSE) %...>% + { b$DOM$querySelector(.$root$nodeId, ".sidebar", wait_ = FALSE) } %...>% + { b$DOM$getBoxModel(.$nodeId, wait_ = FALSE) } %...>% + str() + +b$wait_for(p) +``` + +This documentation will refer to this technique as _synchronizing_ asynchronous code. The way that `wait_for()` works is that it runs the Chromote object's private event loop until the promise has resolved. Because the event loop is _private_, running it will not interfere with the global event loop, which, for example, may used by Shiny to serve a web application. + +The `$wait_for()` method will return the value from the promise, so instead of putting the `str()` in the chain, you call `str()` on the value returned by `$wait_for()`: + +```{r waitForStr} +p <- b$DOM$getDocument(wait_ = FALSE) %...>% + { b$DOM$querySelector(.$root$nodeId, ".sidebar", wait_ = FALSE) } %...>% + { b$DOM$getBoxModel(.$nodeId, wait_ = FALSE) } + +x <- b$wait_for(p) +str(x) +``` + +There are some methods in Chromote and ChromoteSession objects which are written using asynchronous method calls, but conditionally use `wait_for()` so that they can be called either synchronously or asynchronously. The `$screenshot()` method works this way, for example. You can call `b$screenshot(wait_=TRUE)` (which is the default) for synchronous behavior, or `b$screenshot(wait_=FALSE)` for async behavior. + +If you want to write a function that can be called in either sync or async mode, you can use this basic structure: First, construct a promise chain by calling the CDP methods with `wait_=FALSE`. Then, at the end, if the user used `wait_=TRUE`, wait for the promise to resolve; otherwise, simply return the promise. + +```R +getBoxModel <- function(b, selector = "html", wait_ = TRUE) { + p <- b$DOM$getDocument(wait_ = FALSE) %...>% + { b$DOM$querySelector(.$root$nodeId, selector, wait_ = FALSE) } %...>% + { b$DOM$getBoxModel(.$nodeId, wait_ = FALSE) } + + if (wait_) { + b$wait_for(p) + } else { + p + } +} + +# Synchronous call +str(getBoxModel(b, ".sidebar")) + +# Asynchronous call +getBoxModel(b, ".sidebar", wait_ = FALSE) %...>% + str() +``` + +But, you might be wondering, if we want a synchronous API, why not simply write the synchronous code by calling the individual methods synchronously, and using a normal pipe to connect them, as in: + +```R +b$DOM$getDocument() %>% + { b$DOM$querySelector(.$root$nodeId, ".sidebar") } %>% + { b$DOM$getBoxModel(.$nodeId) } %>% + str() +``` + +There are two reasons for this. The first is that this would require a duplication of all the code for the sync and async code paths. Another reason is that the internal async code can be written to send multiple independent command chains to the ChromoteSession (or multiple ChromoteSessions), and they will be executed concurrently. If there are multiple promise chains, you can do something like the following to wait for all of them to resolve: + +```R +# Starting with promises p1, p2, and p3, create a promise that resolves only +# after they have all been resolved. +p <- promise_all(p1, p2, p3) +b$wait_for(p) +``` + + +## Async events + +In addition to _commands_ The Chrome Devtools Protocol also has _events_. These are messages that are sent from the browser to the R process when various browser events happen. + +As an example, it can be a bit tricky to find out when to take a screenshot. When you send the browser a command to navigate to a page, it sends a response immediately, but it may take several more seconds for it to actually finish loading that page. When it does, the `Page.loadEventFired` event will be fired. + +```{r asyncEvents} +b <- ChromoteSession$new() + +# Navigate and wait for Page.loadEventFired. +# Note: these lines are put in a single code block to ensure that there is no +# idle time in between. +{ + b$Page$navigate("https://www.r-project.org/") + str(b$Page$loadEventFired()) +} +``` + +With the synchronous API, the call to `b$Page$loadEventFired()` will block until Chromote receives a `Page.loadEventFired` message from the browser. However, with the async promise API, you would write it like this: + +```R +b$Page$navigate("https://www.r-project.org/", wait_ = FALSE) %...>% + { b$Page$loadEventFired(wait_ = FALSE) } %...>% + { str(.) } + +# Or, more verbosely: +b$Page$navigate("https://www.r-project.org/", wait_ = FALSE)$ + then(function(value) { + b$Page$loadEventFired(wait_ = FALSE) + })$ + then(function(value) { + str(value) + }) +``` + +There will be a short delay after running the code before the value is printed. + +If you want to schedule a chain of promises and then wait for them to resolve, you can once again use the `wait_for()` method. For example: + +```{r waitForAsyncEvents} +p <- b$Page$navigate("https://www.r-project.org/", wait_ = FALSE)$ + then(function(value) { + b$Page$loadEventFired(wait_ = FALSE) + }) + +# wait_for returns the last value in the chain, so we can call str() on it +str(b$wait_for(p)) +``` + +This particular example has a twist to it: After sending the `Page.navigate` command, the R process doesn't really need to wait for browser's response before it starts waiting for the `Page.loadEventFired` event. So instead of chaining, you could just do this: + +```{r waitForAsyncEventDirect} +p <- promise(function(resolve, reject) { + b$Page$navigate("https://www.r-project.org/", wait_ = FALSE) + resolve(b$Page$loadEventFired(wait_ = FALSE)) +}) + +str(b$wait_for(p)) +``` + +Essentially, the `Page.navigate` command gets sent off and we don't need to wait for the browser's reply. We can tell R to immediately start waiting for the `Page.loadEventFired` event. + +We can simplify it by not wrapping both method calls in a promise. We can just fire off the navigation command, and then directly use the promise that's returned by the event method: + +```{r waitForAsyncLoadEvent} +b$Page$navigate("https://www.r-project.org/", wait_ = FALSE) +p <- b$Page$loadEventFired(wait_ = FALSE) +str(b$wait_for(p)) +``` + +And we can make it yet simpler by firing off the navigation command and then calling `b$Page$loadEventFired()` in synchronous mode (with the default `wait_=TRUE`), which already calls `wait_for()`. + +```{r cleanAsyncEvents} +b$Page$navigate("https://www.r-project.org/", wait_ = FALSE) +x <- b$Page$loadEventFired() +str(x) +``` + +> **Technical note:** The Chrome Devtools Protocol itself does not automatically enable event notifications. Normally, you would have to call the `Page.enable` method to turn on event notifications for the Page domain. However, Chromote saves you from needing to do this step by keeping track of how many callbacks there are for each domain. When the number of event callbacks for a domain goes from 0 to 1, Chromote automatically calls `$enable()` for that domain, and when it goes from 1 to 0, it it calls `$disable()`. + + +In addition to async events with promises, they can also be used with regular callbacks. For example: + +```R +b$Page$loadEventFired(callback_ = str) +``` + +This tells Chromote to call `str()` (which prints to the console) on the message value every single time that a `Page.loadEventFired` event message is received. It will continue doing this indefinitely. (Calling an event method this way also increments the event callback counter.) + +When an event method is called with a callback, the return value is a function which will cancel the callback, so that it will no longer fire. (The canceller function also decrements the event callback counter. If you lose the canceller function, there is no way to decrement the callback counter back to 0.) + +```R +cancel_load_event_callback <- b$Page$loadEventFired(callback_ = str) + +# Each of these will cause the callback to fire. +n1 <- b$Page$navigate("https://www.r-project.org/") +n2 <- b$Page$navigate("https://cran.r-project.org/") + +cancel_load_event_callback() + +# No longer causes the callback to fire. +n3 <- b$Page$navigate("https://www.rstudio.com/") +``` + + +## Resource cleanup and garbage collection + +When Chromote starts a Chrome process, it calls `Chrome$new()`. This launches the Chrome process it using `processx::process()`, and enables a supervisor for the process. This means that if the R process stops, the supervisor will detect this and shut down any Chrome processes that were registered with the supervisor. This prevents the proliferation of Chrome processes that are no longer needed. + +The Chromote package will, by default, use a single Chrome process and a single `Chromote` object, and each time `ChromoteSession$new()` is called, it will spawn them from the `Chromote` object. See [The Chromote object model](#the-chromote-object-model) for more information. + + +## Specifying which browser to use + +Chromote will look in specific places for the Chrome web browser, depending on platform. This is done by the `chromote:::find_chrome()` function. + +If you wish to use a different browser from the default, you can set the `CHROMOTE_CHROME` environment variable, either with `Sys.setenv(CHROMOTE_CHROME="/path/to/browser")`. + +```R +Sys.setenv(CHROMOTE_CHROME = "/Applications/Chromium.app/Contents/MacOS/Chromium") + +b <- ChromoteSession$new() +b$view() +b$Page$navigate("https://www.whatismybrowser.com/") +``` + +Another way is create a `Chromote` object and explicitly specify the browser, then spawn `ChromoteSession`s from it. + +```R +m <- Chromote$new( + browser = Chrome$new(path = "/Applications/Chromium.app/Contents/MacOS/Chromium") +) + +# Spawn a ChromoteSession from the Chromote object +b <- m$new_session() +b$Page$navigate("https://www.whatismybrowser.com/") +``` + +Yet another way is to create a `Chromote` object with a specified browser, then set it as the default Chromote object. + +```R +m <- Chromote$new( + browser = Chrome$new(path = "/Applications/Chromium.app/Contents/MacOS/Chromium") +) + +# Set this Chromote object as the default. Then any +# ChromoteSession$new() will be spawned from it. +set_default_chromote_object(m) +b <- ChromoteSession$new() +b$view() +b$Page$navigate("https://www.whatismybrowser.com/") +``` + + +## Chrome on remote hosts + +Chromote can control a browser running on a remote host. To start the browser, open a terminal on the remote host and run one of the following, depending on your platform: + +**Warning: Depending on how the remote machine is configured, the Chrome debug server might be accessible to anyone on the Internet. Proceed with caution.** + +``` +# Mac +"/Applications/Google Chrome.app/Contents/MacOS/Google Chrome" --headless \ + --remote-debugging-address=0.0.0.0 --remote-debugging-port=9222 + +# Linux +google-chrome --headless --remote-debugging-address=0.0.0.0 --remote-debugging-port=9222 + +# Windows +"C:\Program Files (x86)\Google\Chrome\Application\chrome.exe" --headless \ + --remote-debugging-address=0.0.0.0 --remote-debugging-port=9222 +``` + +Then, in your local R session, create a Chromote object with the `host` and `port` (you will need to use the correct IP address). Once it's created, you can spawn a session off of it which you can control as normal: + +```R +r <- Chromote$new( + browser = ChromeRemote$new(host = "10.0.0.5", port = 9222) +) + +b <- r$new_session() + +b$Browser$getVersion() +b$view() +b$Page$navigate("https://www.whatismybrowser.com/") +b$Page$loadEventFired() +b$screenshot("browser.png") +b$screenshot("browser_string.png", selector = ".string-major") +``` + +When you use `$view()` on the remote browser, your local browser may block scripts for security reasons, which means that you won't be able to view the remote browser. If your local browser is Chrome, there will be a shield-shaped icon in the location bar that you can click in order to enable loading the scripts. (Note: Some browsers don't seem to work at all with the viewer.) + +**Technical note:** There seem to be some timing issues with remote browsers. In the example above, the browser may finish navigating to the web site before the R process receives the response message for `$navigate()`, and therefore before R starts waiting for `Page.loadEventFired`. In order to avoid these timing problems, it may be better to write code like this: + +```R +{ + b$Page$navigate("https://www.whatismybrowser.com/", wait_ = FALSE) + b$Page$loadEventFired() +} +b$screenshot("browser.png") +``` + +This tells it to fire off the `Page.navigate` command and _not_ wait for it, and then immediately start waiting for `Page.loadEventFired` event. + +### Attaching to existing tabs + +In the examples above, we connected to an existing browser, but created a new tab to attach to. It's also possible to attach to an existing browser *and* and existing tab. In Chrome debugging terminology a tab is called a "Target", and there is a command to retrieve the list of current Targets: + +```R +r$Target$getTargets() +``` + +Every target has a unique identifier string associated with it called the `targetId`; `"9DAE349A3A533718ED9E17441BA5159B"` is an example of one. + +Here we define a function that retrieves the ID of the first Target (tab) from a Chromote object: + +```R +first_id <- function(r) { + ts <- r$Target$getTargets()$targetInfos + stopifnot(length(ts) > 0) + r$Target$getTargets()$targetInfos[[1]]$targetId +} +``` + +The following code shows an alert box in the first tab, whatever it is: + +```R +rc <- ChromeRemote$new(host = "localhost", port = 9222) +r <- Chromote$new(browser = rc) +tid <- first_id(r) +b <- r$new_session(targetId = tid) +b$Runtime$evaluate('alert("this is the first tab")') +``` + +***** + +## Examples + +### Taking a screenshot of a web page + +Take a screenshot of the viewport and display it using the [showimage](https://github.com/r-lib/showimage#readme) package. This uses Chromote's `$screenshot()` method, which wraps up many calls to the Chrome Devtools Protocol. + +```R +b <- ChromoteSession$new() + +# ==== Synchronous version ==== +# Run the next two lines together, without any delay in between. +b$Page$navigate("https://www.r-project.org/") +b$Page$loadEventFired() + +b$screenshot(show = TRUE) # Saves to screenshot.png and displays in viewer + +# ==== Async version ==== +b$Page$navigate("https://www.r-project.org/", wait_ = FALSE) +b$Page$loadEventFired(wait_ = FALSE)$ + then(function(value) { + b$screenshot(show = TRUE) + }) +``` + +It is also possible to use selectors to specify what to screenshot, as well as the region ("content", "border", "padding", or "margin"). + +```R +# Using CSS selectors, choosing the region, and using scaling +b$screenshot("s1.png", selector = ".sidebar") +b$screenshot("s2.png", selector = ".sidebar", region = "margin") +b$screenshot("s3.png", selector = ".page", region = "margin", scale = 2) +``` + +If a vector is passed to `selector`, it will take a screenshot with a rectangle that encompasses all the DOM elements picked out by the selectors. Similarly, if a selector picks out multiple DOM elements, all of them will be in the screenshot region. + + +### Taking a screenshot of a web page after interacting with it + +Headless Chrome provides a remote debugging UI which you can use to interact with the web page. The ChromoteSession's `$view()` method opens a regular browser and navigates to the remote debugging UI. + +```R +b <- ChromoteSession$new() + +b$view() +b$Page$navigate("https://www.google.com") # Or just type the URL in the navigation bar +``` + +At this point, you can interact with the web page by typing in text and clicking on things. + +Then take a screenshot: + +```R +b$screenshot() +``` + +### Taking screenshots of web pages in parallel + +With async code, it's possible to navigate to and take screenshots of multiple websites in parallel. + +```R +library(promises) +library(chromote) +urls <- c( + "https://www.r-project.org/", + "https://github.com/", + "https://news.ycombinator.com/" +) + +screenshot_p <- function(url, filename = NULL) { + if (is.null(filename)) { + filename <- gsub("^.*://", "", url) + filename <- gsub("/", "_", filename) + filename <- gsub("\\.", "_", filename) + filename <- sub("_$", "", filename) + filename <- paste0(filename, ".png") + } + + b <- ChromoteSession$new() + b$Page$navigate(url, wait_ = FALSE) + b$Page$loadEventFired(wait_ = FALSE)$ + then(function(value) { + b$screenshot(filename, wait_ = FALSE) + })$ + then(function(value) { + message(filename) + })$ + finally(function() { + b$close() + }) +} + +# Screenshot multiple simultaneously +ps <- lapply(urls, screenshot_p) +pa <- promise_all(.list = ps)$then(function(value) { + message("Done!") +}) + +# Block the console until the screenshots finish (optional) +cm <- default_chromote_object() +cm$wait_for(pa) +#> www_r-project_org.png +#> github_com.png +#> news_ycombinator_com.png +#> Done! +``` + + +### Setting custom headers + +Currently setting custom headers requires a little extra work because it requires `Network.enable` be called before using it. In the future we'll streamline things so that it will happen automatically. + +```R +b <- ChromoteSession$new() +# Currently need to manually enable Network domain notifications. Calling +# b$Network$enable() would do it, but calling it directly will bypass the +# callback counting and the notifications could get automatically disabled by a +# different Network event. We'll enable notifications for the Network domain by +# listening for a particular event. We'll also store a callback that will +# decrement the callback counter, so that we can disable notifications ater. +disable_network_notifications <- b$Network$responseReceived(function (msg) NULL) +b$Network$setExtraHTTPHeaders(headers = list( + foo = "bar", + header1 = "value1" +)) + +# Visit a web page that prints out the request headers +b$Page$navigate("http://scooterlabs.com/echo") +b$screenshot(show = TRUE) + + +# Unset extra headers. Note that `list(a=1)[0]` creates an empty _named_ list; +# an empty unnamed list will cause an error because they're converted to JSON +# differently. A named list becomes "{}", but an unnamed list becomes "[]". +b$Network$setExtraHTTPHeaders(headers = list(a=1)[0]) + +# Request again +b$Page$navigate("http://scooterlabs.com/echo") +b$screenshot(show = TRUE) + + +# Disable extra headers entirely, by decrementing Network callback counter, +# which will disable Network notifications. +disable_network_notifications() +``` + +### Custom User-Agent + +Synchronous version: + +```R +# ==== Synchronous version ==== +b$Network$setUserAgentOverride(userAgent = "My fake browser") + +b$Page$navigate("http://scooterlabs.com/echo") +b$screenshot(show = TRUE) + + +# ==== Async version ==== +b$Network$setUserAgentOverride(userAgent = "My fake browser", wait_ = FALSE) +b$Page$navigate("http://scooterlabs.com/echo", wait_ = FALSE) +b$Page$loadEventFired(wait_ = FALSE)$ + then(function(value) { + b$screenshot(show = TRUE) + }) +``` + + +### Extracting text from a web page + +One way to extract text from a page is to tell the browser to run JavaScript code that does it: + +```R +# ==== Synchronous version ==== +b$Page$navigate("https://www.whatismybrowser.com/") + +# Run JavaScript to extract text from the page +x <- b$Runtime$evaluate('document.querySelector(".corset .string-major a").innerText') +x$result$value +#> [1] "Chrome 75 on macOS (Mojave)" + + +# ==== Async version ==== +b$Page$navigate("https://www.whatismybrowser.com/", wait_ = FALSE) +b$Page$loadEventFired(wait_ = FALSE)$ + then(function(value) { + b$Runtime$evaluate( + 'document.querySelector(".corset .string-major a").innerText' + ) + })$ + then(function(value) { + print(value$result$value) + }) +``` + +Another way is to use CDP commands to extract content from the DOM. This does not require executing JavaScript in the browser's context, but it is also not as flexible as JavaScript. + +```R +# ==== Synchronous version ==== +b$Page$navigate("https://www.whatismybrowser.com/") +x <- b$DOM$getDocument() +x <- b$DOM$querySelector(x$root$nodeId, ".corset .string-major a") +b$DOM$getOuterHTML(x$nodeId) +#> $outerHTML +#> [1] "Chrome 75 on macOS (Mojave)" + + +# ==== Async version ==== +b$Page$navigate("https://www.whatismybrowser.com/", wait_ = FALSE) +b$Page$loadEventFired(wait_ = FALSE)$ + then(function(value) { + b$DOM$getDocument() + })$ + then(function(value) { + b$DOM$querySelector(value$root$nodeId, ".corset .string-major a") + })$ + then(function(value) { + b$DOM$getOuterHTML(value$nodeId) + })$ + then(function(value) { + print(value) + }) +``` + +### Websites that require authentication + +For websites that require authentication, you can use Chromote to get screenshots by doing the following: + +1. Log in interactively and navigate to the page. +1. Capture cookies from the page and save them. +1. In a later R session, load the cookies. +1. Use the cookies in Chromote and navigate to the page. +1. Take a screenshot. + +There are two ways to capture the cookies. + + +**Method 1:** The first method uses the headless browser's viewer. This can be a bit inconvenient because it requires going through the entire login process, even if you have already logged in with a normal browser. + +First navigate to the page: + +```R +library(chromote) +b <- ChromoteSession$new() +b$view() +b$Page$navigate("https://beta.rstudioconnect.com/content/123456/") +``` + +Next, log in interactively via the viewer. Once that's done, use Chromote to capture the cookies. + +```R +cookies <- b$Network$getCookies() +str(cookies) +saveRDS(cookies, "cookies.rds") +``` + +After saving the cookies, you can restart R and navigate to the page, using the cookies. + +```R +library(chromote) +b <- ChromoteSession$new() +b$view() +cookies <- readRDS("cookies.rds") +b$Network$setCookies(cookies = cookies$cookies) +# Navigate to the app that requires a login +b$Page$navigate("https://beta.rstudioconnect.com/content/123456/") +b$screenshot() +``` + + +**Method 2:** The second method captures the cookies using a normal browser. This is can be more convenient because, if you are already logged in, you don't need to do it again. This requires a Chromium-based browser, and it requires running DevTools-in-DevTools on that browser. + +First, navigate to the page in your browser. Then press Cmd-Option-I (Mac) or Ctrl-Shift-I (Windows/Linux). The developer tools panel will open. Make sure to undock the developer tools so that they are in their own window. Then press Cmd-Option-I or Ctrl-Shift-I again. A second developer tools window will open. (See [this SO answer](https://stackoverflow.com/questions/12291138/how-do-you-inspect-the-web-inspector-in-chrome/12291163#12291163) for detailed instructions.) + +In the second developer tools window, run the following: + +```JS +var cookies = await Main.sendOverProtocol('Network.getCookies', {}) +JSON.stringify(cookies) +``` + +This will return a JSON string representing the cookies for that page. For example: + +```JSON +[{"cookies":[{"name":"AWSALB","value":"T3dNdcdnMasdf/cNn0j+JHMVkZ3RI8mitnAggd9AlPsaWJdsfoaje/OowIh0qe3dDPiHc0mSafe5jNH+1Aeinfalsd30AejBZDYwE","domain":"beta.rstudioconnect.com","path":"/","expires":1594632233.96943,"size":130,"httpOnly":false,"secure":false,"session":false}]}] +``` + +Copy that string to the clipboard. In your R session, you can paste it to this code, surrounded by single-quotes: + +```R +cookie_json <- '[{"cookies":[{"name":"AWSALB","value":"T3dNdcdnMasdf/cNn0j+JHMVkZ3RI8mitnAggd9AlPsaWJdsfoaje/OowIh0qe3dDPiHc0mSafe5jNH+1Aeinfalsd30AejBZDYwE","domain":"beta.rstudioconnect.com","path":"/","expires":1594632233.96943,"size":130,"httpOnly":false,"secure":false,"session":false}]}]' + +cookies <- jsonlite::fromJSON(cookie_json, simplifyVector = FALSE)[[1]] +``` + +Then you can use Chromote to navigate to the page and take a screenshot. + +```R +library(chromote) +b <- ChromoteSession$new() +b$view() +b$Network$setCookies(cookies = cookies$cookies) +b$Page$navigate("https://beta.rstudioconnect.com/content/123456/") +b$screenshot() +``` diff --git a/README.md b/README.md index 264b47aa..f4b467de 100644 --- a/README.md +++ b/README.md @@ -1,35 +1,53 @@ -Chromote: Headless Chrome Remote Interface -========================================== + + + +# Chromote: Headless Chrome Remote Interface + [![R-CMD-check](https://github.com/rstudio/chromote/workflows/R-CMD-check/badge.svg)](https://github.com/rstudio/chromote/actions) -[![CRAN status](https://www.r-pkg.org/badges/version/chromote)](https://CRAN.R-project.org/package=chromote) -[![Lifecycle: experimental](https://img.shields.io/badge/lifecycle-experimental-orange.svg)](https://lifecycle.r-lib.org/articles/stages.html#experimental) +[![CRAN +status](https://www.r-pkg.org/badges/version/chromote)](https://CRAN.R-project.org/package=chromote) +[![Lifecycle: +experimental](https://img.shields.io/badge/lifecycle-experimental-orange.svg)](https://lifecycle.r-lib.org/articles/stages.html#experimental) -**!! Please note that Chromote is in development and the API is subject to change !!** +**\!\! Please note that Chromote is in development and the API is +subject to change \!\!** -Chromote is an R implementation of the [Chrome Devtools Protocol](https://chromedevtools.github.io/devtools-protocol/). It works with Chrome, Chromium, Opera, Vivaldi, and other browsers based on [Chromium](https://www.chromium.org/). By default it uses Google Chrome (which must already be installed on the system). To use a different browser, see [Specifying which browser to use](#specifying-which-browser-to-use). +Chromote is an R implementation of the [Chrome Devtools +Protocol](https://chromedevtools.github.io/devtools-protocol/). It works +with Chrome, Chromium, Opera, Vivaldi, and other browsers based on +[Chromium](https://www.chromium.org/). By default it uses Google Chrome +(which must already be installed on the system). To use a different +browser, see [Specifying which browser to +use](#specifying-which-browser-to-use). -Chromote is not the only R package that implements the Chrome Devtools Protocol. Here are some others: +Chromote is not the only R package that implements the Chrome Devtools +Protocol. Here are some others: -* [crrri](https://github.com/RLesur/crrri) by Romain Lesur and Christophe Dervieux -* [decapitated](https://github.com/hrbrmstr/decapitated/) by Bob Rudis -* [chradle](https://github.com/milesmcbain/chradle) by Miles McBain + - [crrri](https://github.com/RLesur/crrri) by Romain Lesur and + Christophe Dervieux + - [decapitated](https://github.com/hrbrmstr/decapitated/) by Bob Rudis + - [chradle](https://github.com/milesmcbain/chradle) by Miles McBain -The interface to Chromote is similar to [chrome-remote-interface](https://github.com/cyrus-and/chrome-remote-interface) for node.js. +The interface to Chromote is similar to +[chrome-remote-interface](https://github.com/cyrus-and/chrome-remote-interface) +for node.js. ## Installation -```R +``` r remotes::install_github("rstudio/chromote") ``` ## Basic usage -This will start a headless browser and open an interactive viewer for it in a normal browser, so that you can see what the headless browser is doing. +This will start a headless browser and open an interactive viewer for it +in a normal browser, so that you can see what the headless browser is +doing. -```R +``` r library(chromote) b <- ChromoteSession$new() @@ -39,52 +57,77 @@ b <- ChromoteSession$new() b$view() ``` -The browser can be given _commands_, as specified by the [Chrome Devtools Protocol](https://chromedevtools.github.io/devtools-protocol/). For example, `$Browser$getVersion()` (which corresponds to the [Browser.getVersion](https://chromedevtools.github.io/devtools-protocol/tot/Browser#method-getVersion) in the API docs) will query the browser for version information: - +The browser can be given *commands*, as specified by the [Chrome +Devtools Protocol](https://chromedevtools.github.io/devtools-protocol/). +For example, `$Browser$getVersion()` (which corresponds to the +[Browser.getVersion](https://chromedevtools.github.io/devtools-protocol/tot/Browser#method-getVersion) +in the API docs) will query the browser for version information: -```R +``` r b$Browser$getVersion() #> $protocolVersion #> [1] "1.3" -#> +#> #> $product -#> [1] "HeadlessChrome/72.0.3626.109" -#> +#> [1] "HeadlessChrome/91.0.4472.114" +#> #> $revision -#> [1] "@fae8db7ab9280fa6704a59980263c804f809ebd5" -#> +#> [1] "@4bb19460e8d88c3446b360b0df8fd991fee49c0b" +#> #> $userAgent -#> [1] "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) -#> HeadlessChrome/72.0.3626.109 Safari/537.36" -#> +#> [1] "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) HeadlessChrome/91.0.4472.114 Safari/537.36" +#> #> $jsVersion -#> [1] "7.2.502.25" +#> [1] "9.1.269.36" ``` +If you have the viewer open and run the following, you’ll see the web +page load in the viewer: -If you have the viewer open and run the following, you'll see the web page load in the viewer: - -```R +``` r b$Page$navigate("https://www.r-project.org/") +#> $frameId +#> [1] "CC9ECA1E389ADB9A7B8DBEB03DFF9EF7" +#> +#> $loaderId +#> [1] "9E210CE4CF0832F80232331F062EA412" ``` -In the official Chrome DevTools Protocol (CDP) documentation, this is the [`Page.navigate`](https://chromedevtools.github.io/devtools-protocol/tot/Page#method-navigate) command. +In the official Chrome DevTools Protocol (CDP) documentation, this is +the +[`Page.navigate`](https://chromedevtools.github.io/devtools-protocol/tot/Page#method-navigate) +command. -In addition to full support of the CDP, ChromoteSession objects also some convenience methods, like `$screenshot()`. (See the Examples section below for more information about screenshots.) +In addition to full support of the CDP, ChromoteSession objects also +some convenience methods, like `$screenshot()`. (See the Examples +section below for more information about screenshots.) -```R +``` r # Saves to screenshot.png b$screenshot() +#> [1] "An error occurred: Error in onRejected(...): code: -32000\n message: Could not find node with given id\n" # Takes a screenshot of elements picked out by CSS selector -b$screenshot("sidebar.png", selector = ".sidebar")) +b$screenshot("sidebar.png", selector = ".sidebar") +#> [1] "sidebar.png" ``` -**Note:** All members of `Chromote` and `ChromoteSession` objects which start with a capital letter (like `b$Page`, `b$DOM`, and `b$Browser`) correspond to domains from the Chrome Devtools Protocol, and are documented in the [official CDP site](https://chromedevtools.github.io/devtools-protocol/). All members which start with a lower-case letter (like `b$screenshot` and `b$close`) are not part of the Chrome Devtools Protocol, and are specific to `Chromote` and `ChromoteSession`. +![Sidebar Screenshot](sidebar.png) + +**Note:** All members of `Chromote` and `ChromoteSession` objects which +start with a capital letter (like `b$Page`, `b$DOM`, and `b$Browser`) +correspond to domains from the Chrome Devtools Protocol, and are +documented in the [official CDP +site](https://chromedevtools.github.io/devtools-protocol/). All members +which start with a lower-case letter (like `b$screenshot` and `b$close`) +are not part of the Chrome Devtools Protocol, and are specific to +`Chromote` and `ChromoteSession`. -Here is an example of how to use Chromote to find the position of a DOM element using [DOM.getBoxModel](https://chromedevtools.github.io/devtools-protocol/tot/DOM/#method-getBoxModel). +Here is an example of how to use Chromote to find the position of a DOM +element using +[DOM.getBoxModel](https://chromedevtools.github.io/devtools-protocol/tot/DOM/#method-getBoxModel). -```R +``` r x <- b$DOM$getDocument() x <- b$DOM$querySelector(x$root$nodeId, ".sidebar") x <- b$DOM$getBoxModel(x$nodeId) @@ -92,138 +135,263 @@ str(x) #> List of 1 #> $ model:List of 6 #> ..$ content:List of 8 -#> .. ..$ : int 92 +#> .. ..$ : num 107 +#> .. ..$ : int 28 +#> .. ..$ : num 238 +#> .. ..$ : int 28 +#> .. ..$ : num 238 +#> .. ..$ : int 986 +#> .. ..$ : num 107 +#> .. ..$ : int 986 +#> ..$ padding:List of 8 +#> .. ..$ : num 91.8 #> .. ..$ : int 28 -#> .. ..$ : int 187 +#> .. ..$ : num 253 #> .. ..$ : int 28 -#> .. ..$ : int 187 -#> .. ..$ : int 1026 -#> .. ..$ : int 92 -#> .. ..$ : int 1026 -#> ... -#> ... +#> .. ..$ : num 253 +#> .. ..$ : int 986 +#> .. ..$ : num 91.8 +#> .. ..$ : int 986 +#> ..$ border :List of 8 +#> .. ..$ : num 91.8 +#> .. ..$ : int 28 +#> .. ..$ : num 253 +#> .. ..$ : int 28 +#> .. ..$ : num 253 +#> .. ..$ : int 986 +#> .. ..$ : num 91.8 +#> .. ..$ : int 986 #> ..$ margin :List of 8 -#> .. ..$ : num 14.5 +#> .. ..$ : int 11 #> .. ..$ : int 28 -#> .. ..$ : int 202 +#> .. ..$ : num 253 #> .. ..$ : int 28 -#> .. ..$ : int 202 -#> .. ..$ : int 1068 -#> .. ..$ : num 14.5 -#> .. ..$ : int 1068 -#> ..$ width : int 125 -#> ..$ height : int 998 +#> .. ..$ : num 253 +#> .. ..$ : int 1028 +#> .. ..$ : int 11 +#> .. ..$ : int 1028 +#> ..$ width : int 161 +#> ..$ height : int 958 ``` -Or you can do the same thing by chaining commands together with a magrittr pipe: +Or you can do the same thing by chaining commands together with a +magrittr pipe: -```R +``` r b$DOM$getDocument() %>% { b$DOM$querySelector(.$root$nodeId, ".sidebar") } %>% { b$DOM$getBoxModel(.$nodeId) } %>% str() +#> List of 1 +#> $ model:List of 6 +#> ..$ content:List of 8 +#> .. ..$ : num 107 +#> .. ..$ : int 28 +#> .. ..$ : num 238 +#> .. ..$ : int 28 +#> .. ..$ : num 238 +#> .. ..$ : int 986 +#> .. ..$ : num 107 +#> .. ..$ : int 986 +#> ..$ padding:List of 8 +#> .. ..$ : num 91.8 +#> .. ..$ : int 28 +#> .. ..$ : num 253 +#> .. ..$ : int 28 +#> .. ..$ : num 253 +#> .. ..$ : int 986 +#> .. ..$ : num 91.8 +#> .. ..$ : int 986 +#> ..$ border :List of 8 +#> .. ..$ : num 91.8 +#> .. ..$ : int 28 +#> .. ..$ : num 253 +#> .. ..$ : int 28 +#> .. ..$ : num 253 +#> .. ..$ : int 986 +#> .. ..$ : num 91.8 +#> .. ..$ : int 986 +#> ..$ margin :List of 8 +#> .. ..$ : int 11 +#> .. ..$ : int 28 +#> .. ..$ : num 253 +#> .. ..$ : int 28 +#> .. ..$ : num 253 +#> .. ..$ : int 1028 +#> .. ..$ : int 11 +#> .. ..$ : int 1028 +#> ..$ width : int 161 +#> ..$ height : int 958 ``` ### Creating new tabs and managing the process To create a new tab/window: -```R +``` r b1 <- b$new_session() ``` -Once it's created, you can perform operations with the new tab without affecting the first one. +Once it’s created, you can perform operations with the new tab without +affecting the first one. -```R +``` r b1$view() b1$Page$navigate("https://github.com/rstudio/chromote") +#> $frameId +#> [1] "2322142974B792020563906C9ECE6FB7" +#> +#> $loaderId +#> [1] "5C033DAC1481DDB6D7FDB28AEAD30B60" ``` To close a browser tab/window, you can run: -```R +``` r b1$close() ``` -This is different from shutting down the browser process. If you call `b$close()`, the browser process will still be running, even if all tabs have been closed. If all tabs have been closed, you can still create a new tab by calling `b1$new_session()`. +This is different from shutting down the browser process. If you call +`b$close()`, the browser process will still be running, even if all tabs +have been closed. If all tabs have been closed, you can still create a +new tab by calling `b1$new_session()`. To shut down the process, call: -```R +``` r b1$parent$stop() ``` -`b1$parent` is a `Chromote` object (as opposed to `ChromoteSession`), which represents the browser as a whole. This is explained in [The Chromote object model](#the-chromote-object-model). +`b1$parent` is a `Chromote` object (as opposed to `ChromoteSession`), +which represents the browser as a whole. This is explained in [The +Chromote object model](#the-chromote-object-model). ### Commands and Events -The Chrome Devtools Protocol has two types of methods: _commands_ and _events_. The methods used in the previous examples are commands. That is, they tell the browser to do something; the browser does it, and then sends back some data. +The Chrome Devtools Protocol has two types of methods: *commands* and +*events*. The methods used in the previous examples are commands. That +is, they tell the browser to do something; the browser does it, and then +sends back some data. -Events are quite different from commands. When, for example, you run `b$Page$loadEventFired()`, it does not send a message to the browser. Rather, this method tells the R process to wait until it receives a `Page.loadEventFired`message from the browser. +Events are quite different from commands. When, for example, you run +`b$Page$loadEventFired()`, it does not send a message to the browser. +Rather, this method tells the R process to wait until it receives a +`Page.loadEventFired` message from the browser. -Here is an example of how that event can be used. Note that these two lines of code must be run together, without any delay at all (this can be enforced by wrapping both lines of code in `{ .... }`). +Here is an example of how that event can be used. Note that these two +lines of code must be run together, without any delay at all (this can +be enforced by wrapping both lines of code in `{ .... }`). -```R +``` r # Send a command to navigate to a page b$Page$navigate("https://www.r-project.org") +#> $frameId +#> [1] "CC9ECA1E389ADB9A7B8DBEB03DFF9EF7" +#> +#> $loaderId +#> [1] "F5ECABCF3574197798EF2ADD936C0FCD" # Wait for the Page.loadEventFired event b$Page$loadEventFired() +#> $timestamp +#> [1] 930213.7 ``` -After running these two lines, the R process will be blocked. While it's blocked, the browser will load the page, and then send a message to the R process saying that the `Page.loadEventFired` event has occurred. The message looks something like this: +After running these two lines, the R process will be blocked. While it’s +blocked, the browser will load the page, and then send a message to the +R process saying that the `Page.loadEventFired` event has occurred. The +message looks something like this: -```JSON +``` json {"method":"Page.loadEventFired","params":{"timestamp":699232.345338}} ``` -After the R process receives this message, the function returns the value, which looks like this: - -``` -$timestamp -[1] 699232.3 -``` - -> **Technical note:** Chromote insulates the user from some of the details of how the CDP implements event notifications. Event notifications are not sent from the browser to the R process by default; you must first send a command to enable event notifications for a domain. For example `Page.enable` enables event notifications for the `Page` domain -- the browser will send messages for _all_ `Page` events. (See the Events section in [this page](https://chromedevtools.github.io/devtools-protocol/tot/Page)). These notifications will continue to be sent until the browser receives a `Page.disable` command. -> -> By default, Chromote hides this implementation detail. When you call `b$Page$loadEventFired()`, Chromote sends a `Page.enable` command automatically, and then waits until it receives the `Page.loadEventFired` event notification. Then it sends a `Page.disable` command. -> -> Note that in asynchronous mode, the behavior is slightly more sophisticated: it maintains a counter of how many outstanding events it is waiting for in a given domain. When that count goes from 0 to 1, it sends the `X.enable` command; when the count goes from 1 to 0, it sends the `X.disable` command. For more information, see the [Async events](#async-events) section. -> -> If you do not want automatic event enabling and disabling, then when creating the ChromoteSession object, use `ChromoteSession$new(auto_events = FALSE)`. - +After the R process receives this message, the function returns the +value, which looks like this: + + $timestamp + [1] 699232.3 + +> **Technical note:** Chromote insulates the user from some of the +> details of how the CDP implements event notifications. Event +> notifications are not sent from the browser to the R process by +> default; you must first send a command to enable event notifications +> for a domain. For example `Page.enable` enables event notifications +> for the `Page` domain – the browser will send messages for *all* +> `Page` events. (See the Events section in [this +> page](https://chromedevtools.github.io/devtools-protocol/tot/Page)). +> These notifications will continue to be sent until the browser +> receives a `Page.disable` command. +> +> By default, Chromote hides this implementation detail. When you call +> `b$Page$loadEventFired()`, Chromote sends a `Page.enable` command +> automatically, and then waits until it receives the +> `Page.loadEventFired` event notification. Then it sends a +> `Page.disable` command. +> +> Note that in asynchronous mode, the behavior is slightly more +> sophisticated: it maintains a counter of how many outstanding events +> it is waiting for in a given domain. When that count goes from 0 to 1, +> it sends the `X.enable` command; when the count goes from 1 to 0, it +> sends the `X.disable` command. For more information, see the [Async +> events](#async-events) section. +> +> If you do not want automatic event enabling and disabling, then when +> creating the ChromoteSession object, use +> `ChromoteSession$new(auto_events = FALSE)`. ### The Chromote object model -There are two R6 classes that are used to represent the Chrome browser. One is `Chromote`, and the other is `ChromoteSession`. A `Chromote` object represents the browser as a whole, and it can have multiple _targets_, which each represent a browser tab. In the Chrome Devtools Protocol, each target can have one or more debugging _sessions_ to control it. A `ChromoteSession` object represents a single _session_. +There are two R6 classes that are used to represent the Chrome browser. +One is `Chromote`, and the other is `ChromoteSession`. A `Chromote` +object represents the browser as a whole, and it can have multiple +*targets*, which each represent a browser tab. In the Chrome Devtools +Protocol, each target can have one or more debugging *sessions* to +control it. A `ChromoteSession` object represents a single *session*. -When a `ChromoteSession` object is instantiated, a target is created, then a session is attached to that target, and the `ChromoteSession` object represents the session. (It is possible, though not very useful, to have multiple `ChromoteSession` objects connected to the same target, each with a different session.) +When a `ChromoteSession` object is instantiated, a target is created, +then a session is attached to that target, and the `ChromoteSession` +object represents the session. (It is possible, though not very useful, +to have multiple `ChromoteSession` objects connected to the same target, +each with a different session.) -A `Chromote` object can have any number of `ChromoteSession` objects as children. It is not necessary to create a `Chromote` object manually. You can simply call: +A `Chromote` object can have any number of `ChromoteSession` objects as +children. It is not necessary to create a `Chromote` object manually. +You can simply call: -```R +``` r b <- ChromoteSession$new() ``` -and it will automatically create a `Chromote` object if one has not already been created. The Chromote package will then designate that `Chromote` object as the _default_ Chromote object for the package, so that any future calls to `ChromoteSession$new()` will automatically use the same `Chromote`. This is so that it doesn't start a new browser for every `ChromoteSession` object that is created. +and it will automatically create a `Chromote` object if one has not +already been created. The Chromote package will then designate that +`Chromote` object as the *default* Chromote object for the package, so +that any future calls to `ChromoteSession$new()` will automatically use +the same `Chromote`. This is so that it doesn’t start a new browser for +every `ChromoteSession` object that is created. -In the Chrome Devtools Protocol, most commands can be sent to individual sessions using the `ChromoteSession` object, but there are some commands which can only be sent to the overall browser, using the `Chromote` object. +In the Chrome Devtools Protocol, most commands can be sent to individual +sessions using the `ChromoteSession` object, but there are some commands +which can only be sent to the overall browser, using the `Chromote` +object. -To access the parent `Chromote` object from a `ChromoteSession`, you can simply use `$parent`: +To access the parent `Chromote` object from a `ChromoteSession`, you can +simply use `$parent`: -```R +``` r b <- ChromoteSession$new() m <- b$parent ``` -With a `Chromote` object, you can get a list containing all the `ChromoteSession`s, with `$get_sessions()`: - -```R -m$get_sessions() -``` +With a `Chromote` object, you can get a list containing all the +`ChromoteSession`s, with `$get_sessions()`: -Normally, subsequent calls to `ChromoteSession$new()` will use the existing `Chromote` object. However, if you want to start a new browser process, you can manually create a `Chromote` object, then spawn a session from it; or you can pass the new `Chromote` object to ` ChromoteSession$new()`: +Normally, subsequent calls to `ChromoteSession$new()` will use the +existing `Chromote` object. However, if you want to start a new browser +process, you can manually create a `Chromote` object, then spawn a +session from it; or you can pass the new `Chromote` object to +`ChromoteSession$new()`: -```R +``` r cm <- Chromote$new() b1 <- cm$new_session() @@ -231,177 +399,293 @@ b1 <- cm$new_session() b <- ChromoteSession$new(parent = Chromote$new()) ``` -Note that if you use either of these methods, the new `Chromote` object will _not_ be set as the default that is used by future calls to `ChromoteSesssion$new()`. See [Specifying which browser to use](#specifying-which-browser-to-use) for information on setting the default `Chromote` object. +Note that if you use either of these methods, the new `Chromote` object +will *not* be set as the default that is used by future calls to +`ChromoteSesssion$new()`. See [Specifying which browser to +use](#specifying-which-browser-to-use) for information on setting the +default `Chromote` object. +There are also the following classes which represent the browser at a +lower level: -There are also the following classes which represent the browser at a lower level: - -* `Browser`: This represents an instance of a browser that supports the Chrome Devtools Protocol. It contains information about how to communicate with the Chrome browser. A `Chromote` object contains one of these. -* `Chrome`: This is a subclass of `Browser` that represents a local browser. It extends the `Browser` class with a `processx::process` object, which represents the browser's system process. -* `ChromeRemote`: This is a subclass of `Browser` that represents a browser running on a remote system. - + - `Browser`: This represents an instance of a browser that supports + the Chrome Devtools Protocol. It contains information about how to + communicate with the Chrome browser. A `Chromote` object contains + one of these. + - `Chrome`: This is a subclass of `Browser` that represents a local + browser. It extends the `Browser` class with a `processx::process` + object, which represents the browser’s system process. + - `ChromeRemote`: This is a subclass of `Browser` that represents a + browser running on a remote system. ### Debugging -Calling `b$debug_messages(TRUE)` will enable the printing of all the JSON messages sent between R and Chrome. This can be very helpful for understanding how the Chrome Devtools Protocol works. +Calling `b$debug_messages(TRUE)` will enable the printing of all the +JSON messages sent between R and Chrome. This can be very helpful for +understanding how the Chrome Devtools Protocol works. -```R +``` r b <- ChromoteSession$new() b$parent$debug_messages(TRUE) b$Page$navigate("https://www.r-project.org/") -#> SEND {"method":"Page.navigate","params":{"url":"https://www.r-project.org/"}, -#> "id":6,"sessionId":"5E7BCE653B813DB05D1EFFB2FB2920EC"} -#> RECV {"id":6,"result":{"frameId":"7056B4136FE7D88AC335C649944F4122", -#> "loaderId":"DB7F918C0ED9A2921C06EE24DB4CFE9E"}, -#> "sessionId": "5E7BCE653B813DB05D1EFFB2FB2920EC"} -``` - -### Synchronous vs. asynchronous usage - -By default, when you call methods from a `Chromote` or `ChromoteSession` object, it operates in _synchronous_ mode. For example, when you call a command function (like `b$Page$navigate()`), a command message is sent to the headless browser, the headless browser executes that command, and it sends a response message back. When the R process receives the response, it converts it from JSON to an R object and the function returns that value. During this time, the R process is blocked; no other R code can execute. - -The methods in Chromote/ChromoteSession objects can also be called in _asynchronous_ mode. In async mode, a command function fires off a message to the browser, and then the R process continues running other code; when the response comes back at some time in the future, the R process calls another function and passes the response value to it. - -There are two different ways of using async with Chromote. The first is with [promises](https://rstudio.github.io/promises/) (note that these are not the regular R-language promises; these are similar to JavaScript promises for async programming.) The second way is with callbacks: you call methods with a `callback_` argument. Although callbacks are initially easier to use than promises, once you start writing more complex code, managing callbacks becomes very difficult, especially when error handling is involved. For this reason, this document will focus mostly on promises instead of callback-style programming. - -When Chromote methods are called in synchronous mode, under the hood, they are implemented with asynchronous functions, and then waiting for the asynchronous functions to resolve. - -> **Technical note about the event loop**: When methods are called asynchronously, the R process will run callbacks and promises using an event loop provided by the [later](https://github.com/r-lib/later) package. This event loop is very similar to the one used in JavaScript, which is explained in depth by [this article](https://blog.sessionstack.com/how-javascript-works-event-loop-and-the-rise-of-async-programming-5-ways-to-better-coding-with-2f077c4438b5). One important difference between JavaScript's event loop and the one provided by **later**'s is that in JavaScript, the event loop only runs when the call stack is empty (essentially, when the JS runtime is idle); with **later** the event loop similarly runs when the call stack is empty (when the R console is idle), but it can also be run at any point by calling `later::run_now()`. -> -> There is another important difference between the JS event loop and the one used by Chromote: Chromote uses _private event loops_ provided by [later](https://github.com/r-lib/later). Running the private event loop with `run_now()` will not interfere with the global event loop. This is crucial for being able to run asynchronous code in a way that appears synchronous. - +#> SEND {"method":"Page.navigate","params":{"url":"https://www.r-project.org/"},"id":53,"sessionId":"32B0A19254EB899A4EE36A988B580E45"} +#> $frameId +#> [1] "2A34B3A6869278EC3BFB87879CE67C1A" +#> +#> $loaderId +#> [1] "55074CFE2757CA6F941FAA4F9F875C58" +``` + +### Synchronous vs. asynchronous usage + +By default, when you call methods from a `Chromote` or `ChromoteSession` +object, it operates in *synchronous* mode. For example, when you call a +command function (like `b$Page$navigate()`), a command message is sent +to the headless browser, the headless browser executes that command, and +it sends a response message back. When the R process receives the +response, it converts it from JSON to an R object and the function +returns that value. During this time, the R process is blocked; no other +R code can execute. + +The methods in Chromote/ChromoteSession objects can also be called in +*asynchronous* mode. In async mode, a command function fires off a +message to the browser, and then the R process continues running other +code; when the response comes back at some time in the future, the R +process calls another function and passes the response value to it. + +There are two different ways of using async with Chromote. The first is +with [promises](https://rstudio.github.io/promises/) (note that these +are not the regular R-language promises; these are similar to JavaScript +promises for async programming.) The second way is with callbacks: you +call methods with a `callback_` argument. Although callbacks are +initially easier to use than promises, once you start writing more +complex code, managing callbacks becomes very difficult, especially when +error handling is involved. For this reason, this document will focus +mostly on promises instead of callback-style programming. + +When Chromote methods are called in synchronous mode, under the hood, +they are implemented with asynchronous functions, and then waiting for +the asynchronous functions to resolve. + +> **Technical note about the event loop**: When methods are called +> asynchronously, the R process will run callbacks and promises using an +> event loop provided by the [later](https://github.com/r-lib/later) +> package. This event loop is very similar to the one used in +> JavaScript, which is explained in depth by [this +> article](https://blog.sessionstack.com/how-javascript-works-event-loop-and-the-rise-of-async-programming-5-ways-to-better-coding-with-2f077c4438b5). +> One important difference between JavaScript’s event loop and the one +> provided by **later**’s is that in JavaScript, the event loop only +> runs when the call stack is empty (essentially, when the JS runtime is +> idle); with **later** the event loop similarly runs when the call +> stack is empty (when the R console is idle), but it can also be run at +> any point by calling `later::run_now()`. +> +> There is another important difference between the JS event loop and +> the one used by Chromote: Chromote uses *private event loops* provided +> by [later](https://github.com/r-lib/later). Running the private event +> loop with `run_now()` will not interfere with the global event loop. +> This is crucial for being able to run asynchronous code in a way that +> appears synchronous. #### Why async? -The synchronous API is easier to use than the asynchronous one. So why would you want to use the async API? Here are some reasons: +The synchronous API is easier to use than the asynchronous one. So why +would you want to use the async API? Here are some reasons: -* The async API allows you to send commands to the browser that may take some time for the browser to complete, and they will not block the R process from doing other work while the browser executes the command. -* The async API lets you send commands to multiple browser "tabs" and let them work in parallel. + - The async API allows you to send commands to the browser that may + take some time for the browser to complete, and they will not block + the R process from doing other work while the browser executes the + command. + - The async API lets you send commands to multiple browser “tabs” and + let them work in parallel. -On the other hand, async programming can make it difficult to write code that proceeds in a straightforward, linear manner. Async programming may be difficult to use in, say, an analysis script. +On the other hand, async programming can make it difficult to write code +that proceeds in a straightforward, linear manner. Async programming may +be difficult to use in, say, an analysis script. -When using Chromote interactively at the R console, it's usually best to just call methods synchronously. This fits well with a iterative, interactive data analysis workflow. - -When you are _programming_ with Chromote instead of using it interactively, it is in many cases better to call the methods asynchronously, because it allows for better performance. In a later section, we'll see how to write asynchronous code with Chromote that can be run either synchronously or asynchronously. This provides the best of both worlds. +When using Chromote interactively at the R console, it’s usually best to +just call methods synchronously. This fits well with a iterative, +interactive data analysis workflow. +When you are *programming* with Chromote instead of using it +interactively, it is in many cases better to call the methods +asynchronously, because it allows for better performance. In a later +section, we’ll see how to write asynchronous code with Chromote that can +be run either synchronously or asynchronously. This provides the best of +both worlds. #### Async commands -When a method is called in synchronous mode, it blocks until the browser sends back a response, and then it returns the value, converted from JSON to an R object. For example: +When a method is called in synchronous mode, it blocks until the browser +sends back a response, and then it returns the value, converted from +JSON to an R object. For example: -```R +``` r # Synchronous str(b$Browser$getVersion()) +#> SEND {"method":"Browser.getVersion","params":[],"id":54,"sessionId":"32B0A19254EB899A4EE36A988B580E45"} #> List of 5 #> $ protocolVersion: chr "1.3" -#> $ product : chr "HeadlessChrome/72.0.3626.109" -#> $ revision : chr "@fae8db7ab9280fa6704a59980263c804f809ebd5" -#> $ userAgent : chr "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like #> Gecko) HeadlessChrome/72.0.3626.109 Safari/537.36" -#> $ jsVersion : chr "7.2.502.25" +#> $ product : chr "HeadlessChrome/91.0.4472.114" +#> $ revision : chr "@4bb19460e8d88c3446b360b0df8fd991fee49c0b" +#> $ userAgent : chr "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) HeadlessChrome/91.0.4472"| __truncated__ +#> $ jsVersion : chr "9.1.269.36" ``` -In async mode, there are two ways to use the value that the browser sends to the R process. One is to use the `callback_` argument with `wait_=FALSE`. The `wait_=FALSE` tells it to run the command in async mode; instead of returning the value from the browser, it returns a promise. For example: +In async mode, there are two ways to use the value that the browser +sends to the R process. One is to use the `callback_` argument with +`wait_=FALSE`. The `wait_=FALSE` tells it to run the command in async +mode; instead of returning the value from the browser, it returns a +promise. For example: -```R +``` r # Async with callback b$Browser$getVersion(wait_ = FALSE, callback_ = str) +#> SEND {"method":"Browser.getVersion","params":[],"id":55,"sessionId":"32B0A19254EB899A4EE36A988B580E45"} #> # After a very, very short pause, prints: -#> List of 5 -#> $ protocolVersion: chr "1.3" -#> $ product : chr "HeadlessChrome/72.0.3626.109" -#> $ revision : chr "@fae8db7ab9280fa6704a59980263c804f809ebd5" -#> $ userAgent : chr "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like #> Gecko) HeadlessChrome/72.0.3626.109 Safari/537.36" -#> $ jsVersion : chr "7.2.502.25" ``` -Notice that the function returned ``, and then it printed out the data. We'll come back to the promise part. + #> List of 5 + #> $ protocolVersion: chr "1.3" + #> $ product : chr "HeadlessChrome/91.0.4472.114" + #> $ revision : chr "@4bb19460e8d88c3446b360b0df8fd991fee49c0b" + #> $ userAgent : chr "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) HeadlessChrome/91.0.4472"| __truncated__ + #> $ jsVersion : chr "9.1.269.36" + +Notice that the function returned ``, and then it +printed out the data. We’ll come back to the promise part. -> **Technical note:** When you pass a function as `callback_`, that function is used as the first step in the promise chain that is returned. +> **Technical note:** When you pass a function as `callback_`, that +> function is used as the first step in the promise chain that is +> returned. -If you run the command in a code block (or a function), the entire code block will finish executing before the callback can be executed. For example: +If you run the command in a code block (or a function), the entire code +block will finish executing before the callback can be executed. For +example: -```R +``` r { b$Browser$getVersion(wait_ = FALSE, callback_ = str) 1+1 } +#> SEND {"method":"Browser.getVersion","params":[],"id":56,"sessionId":"32B0A19254EB899A4EE36A988B580E45"} #> [1] 2 -#> List of 5 -#> $ protocolVersion: chr "1.3" -#> $ product : chr "HeadlessChrome/75.0.3770.142" -#> $ revision : chr "@8d93477bd18154c67a70a67019cb8032382c55dd" -#> $ userAgent : chr "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_0) AppleWebKit/537.36 (KHTML, like Gecko) HeadlessChrome/75.0.3770"| __truncated__ -#> $ jsVersion : chr "7.5.288.30" - ``` +``` -In the code above, it executes the `1+1` and returns the value before the `str` callback can be executed on the message from the browser. + #> List of 5 + #> $ protocolVersion: chr "1.3" + #> $ product : chr "HeadlessChrome/91.0.4472.114" + #> $ revision : chr "@4bb19460e8d88c3446b360b0df8fd991fee49c0b" + #> $ userAgent : chr "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) HeadlessChrome/91.0.4472"| __truncated__ + #> $ jsVersion : chr "9.1.269.36" -If you want to store the value from the browser, you can write a callback that stores the value like so: +In the code above, it executes the `1+1` and returns the value before +the `str` callback can be executed on the message from the browser. -```R +If you want to store the value from the browser, you can write a +callback that stores the value like so: + +``` r # This will extract the product field product <- NULL b$Browser$getVersion(wait_ = FALSE, callback_ = function(msg) { product <<- msg$product }) +#> SEND {"method":"Browser.getVersion","params":[],"id":57,"sessionId":"32B0A19254EB899A4EE36A988B580E45"} +#> +``` +``` r # Wait for a moment, then run: product -#> [1] "HeadlessChrome/75.0.3770.142" -``` - -But to get the value, you need to wait for the callback to execute before you can use the value. Waiting for a value is simple when running R interactively -- you can just add a `message("message arrived")` call in the callback and wait for it before running the next line of code -- but waiting for the value is not easy to do using ordinary straight-line coding. Fortunately, Chromote has a way to wait for async operations, which we'll see later. - -The other way of using the value is to use _promises_. If `wait_=FALSE` and no `callback_` is passed to the command, then it will simply return a promise. Promises have many advantages over plain old callbacks: they are easier to chain, and they provide better error-handling capabilities. You can _chain_ more steps to the promise: when the promise resolves -- that is, when the message is received from the browser -- it will run the next step in the promise chain. - -Here's an example that uses promises to print out the version information. Note that the surrounding curly braces are there to indicate that this whole thing must be run as a block without any idle time in between the function calls -- if you were to run the code in the R console line-by-line, the browser would send back the message and the promise would resolve before you called `p$then()`, which is where you tell the promise what to do with the return value. (The curly braces aren't strictly necessary -- you could run the code inside the braces in a single paste operation and have the same effect.) - -```R +#> [1] "HeadlessChrome/91.0.4472.114" +``` + +But to get the value, you need to wait for the callback to execute +before you can use the value. Waiting for a value is simple when running +R interactively – you can just add a `message("message arrived")` call +in the callback and wait for it before running the next line of code – +but waiting for the value is not easy to do using ordinary straight-line +coding. Fortunately, Chromote has a way to wait for async operations, +which we’ll see later. + +The other way of using the value is to use *promises*. If `wait_=FALSE` +and no `callback_` is passed to the command, then it will simply return +a promise. Promises have many advantages over plain old callbacks: they +are easier to chain, and they provide better error-handling +capabilities. You can *chain* more steps to the promise: when the +promise resolves – that is, when the message is received from the +browser – it will run the next step in the promise chain. + +Here’s an example that uses promises to print out the version +information. Note that the surrounding curly braces are there to +indicate that this whole thing must be run as a block without any idle +time in between the function calls – if you were to run the code in the +R console line-by-line, the browser would send back the message and the +promise would resolve before you called `p$then()`, which is where you +tell the promise what to do with the return value. (The curly braces +aren’t strictly necessary – you could run the code inside the braces in +a single paste operation and have the same effect.) + +``` r { p <- b$Browser$getVersion(wait_ = FALSE) p$then(function(value) { print(value$product) }) } -# Waits for a moment, then prints: -#> [1] "HeadlessChrome/75.0.3770.142" +#> SEND {"method":"Browser.getVersion","params":[],"id":58,"sessionId":"32B0A19254EB899A4EE36A988B580E45"} +# Wait for a moment, then prints: ``` -Here are some progressively more concise ways of achieving the same thing. As you work with promises, you will see these various forms of promise chaining. For more information, see the [promises documentation](https://rstudio.github.io/promises/). + #> [1] "HeadlessChrome/91.0.4472.114" + +Here are some progressively more concise ways of achieving the same +thing. As you work with promises, you will see these various forms of +promise chaining. For more information, see the [promises +documentation](https://rstudio.github.io/promises/). -```R +``` r library(promises) # Regular function pipe to then() b$Browser$getVersion(wait_ = FALSE) %>% then(function(value) { print(value$product) }) +#> SEND {"method":"Browser.getVersion","params":[],"id":59,"sessionId":"32B0A19254EB899A4EE36A988B580E45"} # Promise-pipe to anonymous function, which must be wrapped in parens b$Browser$getVersion(wait_ = FALSE) %...>% (function(value) { print(value$product) }) +#> SEND {"method":"Browser.getVersion","params":[],"id":60,"sessionId":"32B0A19254EB899A4EE36A988B580E45"} # Promise-pipe to an expression (which gets converted to a function with the first argument `.`) b$Browser$getVersion(wait_ = FALSE) %...>% { print(.$product) } +#> SEND {"method":"Browser.getVersion","params":[],"id":61,"sessionId":"32B0A19254EB899A4EE36A988B580E45"} # Promise-pipe to a named function, with parentheses print_product <- function(msg) print(msg$product) b$Browser$getVersion(wait_ = FALSE) %...>% print_product() +#> SEND {"method":"Browser.getVersion","params":[],"id":62,"sessionId":"32B0A19254EB899A4EE36A988B580E45"} # Promise-pipe to a named function, without parentheses b$Browser$getVersion(wait_ = FALSE) %...>% print_product +#> SEND {"method":"Browser.getVersion","params":[],"id":63,"sessionId":"32B0A19254EB899A4EE36A988B580E45"} ``` +The earlier example where we found the dimensions of a DOM element using +CSS selectors was done with the synchronous API and `%>%` pipes. The +same can be done in async mode by switching from the regular pipe to the +promise-pipe, and calling all the methods with `wait_=FALSE`: -The earlier example where we found the dimensions of a DOM element using CSS selectors was done with the synchronous API and `%>%` pipes. The same can be done in async mode by switching from the regular pipe to the promise-pipe, and calling all the methods with `wait_=FALSE`: - -```R +``` r b$DOM$getDocument(wait_ = FALSE) %...>% { b$DOM$querySelector(.$root$nodeId, ".sidebar", wait_ = FALSE) } %...>% { b$DOM$getBoxModel(.$nodeId, wait_ = FALSE) } %...>% str() +#> SEND {"method":"DOM.getDocument","params":{},"id":64,"sessionId":"32B0A19254EB899A4EE36A988B580E45"} # Or, more verbosely: @@ -415,43 +699,153 @@ b$DOM$getDocument(wait_ = FALSE)$ then(function(value) { str(value) }) +#> SEND {"method":"DOM.getDocument","params":{},"id":65,"sessionId":"32B0A19254EB899A4EE36A988B580E45"} ``` -Each step in the promise chain uses the value from the previous step, via `.` or `value`. Note that not all asynchronous code works in such a linear, straightforward way. Sometimes it is necessary to save data from intermediate steps in a broader-scoped variable, if it is to be used in a later step in the promise chain. - +Each step in the promise chain uses the value from the previous step, +via `.` or `value`. Note that not all asynchronous code works in such a +linear, straightforward way. Sometimes it is necessary to save data from +intermediate steps in a broader-scoped variable, if it is to be used in +a later step in the promise chain. #### Turning asynchronous code into synchronous code -There may be times, especially when programming with Chromote, where you want to wait for a promise to resolve before continuing. To do this, you can use the Chromote or ChromoteSession's `wait_for()` method. +There may be times, especially when programming with Chromote, where you +want to wait for a promise to resolve before continuing. To do this, you +can use the Chromote or ChromoteSession’s `wait_for()` method. -```R +``` r # A promise chain p <- b$DOM$getDocument(wait_ = FALSE) %...>% { b$DOM$querySelector(.$root$nodeId, ".sidebar", wait_ = FALSE) } %...>% { b$DOM$getBoxModel(.$nodeId, wait_ = FALSE) } %...>% str() +#> SEND {"method":"DOM.getDocument","params":{},"id":66,"sessionId":"32B0A19254EB899A4EE36A988B580E45"} b$wait_for(p) -``` - -This documentation will refer to this technique as _synchronizing_ asynchronous code. The way that `wait_for()` works is that it runs the Chromote object's private event loop until the promise has resolved. Because the event loop is _private_, running it will not interfere with the global event loop, which, for example, may used by Shiny to serve a web application. - -The `$wait_for()` method will return the value from the promise, so instead of putting the `str()` in the chain, you call `str()` on the value returned by `$wait_for()`: - -```R +#> [1] "HeadlessChrome/91.0.4472.114" +#> [1] "HeadlessChrome/91.0.4472.114" +#> [1] "HeadlessChrome/91.0.4472.114" +#> [1] "HeadlessChrome/91.0.4472.114" +#> [1] "HeadlessChrome/91.0.4472.114" +#> List of 1 +#> $ model:List of 6 +#> ..$ content:List of 8 +#> .. ..$ : num 107 +#> .. ..$ : int 28 +#> .. ..$ : num 238 +#> .. ..$ : int 28 +#> .. ..$ : num 238 +#> .. ..$ : int 986 +#> .. ..$ : num 107 +#> .. ..$ : int 986 +#> ..$ padding:List of 8 +#> .. ..$ : num 91.8 +#> .. ..$ : int 28 +#> .. ..$ : num 253 +#> .. ..$ : int 28 +#> .. ..$ : num 253 +#> .. ..$ : int 986 +#> .. ..$ : num 91.8 +#> .. ..$ : int 986 +#> ..$ border :List of 8 +#> .. ..$ : num 91.8 +#> .. ..$ : int 28 +#> .. ..$ : num 253 +#> .. ..$ : int 28 +#> .. ..$ : num 253 +#> .. ..$ : int 986 +#> .. ..$ : num 91.8 +#> .. ..$ : int 986 +#> ..$ margin :List of 8 +#> .. ..$ : int 11 +#> .. ..$ : int 28 +#> .. ..$ : num 253 +#> .. ..$ : int 28 +#> .. ..$ : num 253 +#> .. ..$ : int 1028 +#> .. ..$ : int 11 +#> .. ..$ : int 1028 +#> ..$ width : int 161 +#> ..$ height : int 958 +#> NULL +``` + +This documentation will refer to this technique as *synchronizing* +asynchronous code. The way that `wait_for()` works is that it runs the +Chromote object’s private event loop until the promise has resolved. +Because the event loop is *private*, running it will not interfere with +the global event loop, which, for example, may used by Shiny to serve a +web application. + +The `$wait_for()` method will return the value from the promise, so +instead of putting the `str()` in the chain, you call `str()` on the +value returned by `$wait_for()`: + +``` r p <- b$DOM$getDocument(wait_ = FALSE) %...>% { b$DOM$querySelector(.$root$nodeId, ".sidebar", wait_ = FALSE) } %...>% { b$DOM$getBoxModel(.$nodeId, wait_ = FALSE) } +#> SEND {"method":"DOM.getDocument","params":{},"id":71,"sessionId":"32B0A19254EB899A4EE36A988B580E45"} x <- b$wait_for(p) str(x) -``` - -There are some methods in Chromote and ChromoteSession objects which are written using asynchronous method calls, but conditionally use `wait_for()` so that they can be called either synchronously or asynchronously. The `$screenshot()` method works this way, for example. You can call `b$screenshot(wait_=TRUE)` (which is the default) for synchronous behavior, or `b$screenshot(wait_=FALSE)` for async behavior. - -If you want to write a function that can be called in either sync or async mode, you can use this basic structure: First, construct a promise chain by calling the CDP methods with `wait_=FALSE`. Then, at the end, if the user used `wait_=TRUE`, wait for the promise to resolve; otherwise, simply return the promise. - -```R +#> List of 1 +#> $ model:List of 6 +#> ..$ content:List of 8 +#> .. ..$ : num 107 +#> .. ..$ : int 28 +#> .. ..$ : num 238 +#> .. ..$ : int 28 +#> .. ..$ : num 238 +#> .. ..$ : int 986 +#> .. ..$ : num 107 +#> .. ..$ : int 986 +#> ..$ padding:List of 8 +#> .. ..$ : num 91.8 +#> .. ..$ : int 28 +#> .. ..$ : num 253 +#> .. ..$ : int 28 +#> .. ..$ : num 253 +#> .. ..$ : int 986 +#> .. ..$ : num 91.8 +#> .. ..$ : int 986 +#> ..$ border :List of 8 +#> .. ..$ : num 91.8 +#> .. ..$ : int 28 +#> .. ..$ : num 253 +#> .. ..$ : int 28 +#> .. ..$ : num 253 +#> .. ..$ : int 986 +#> .. ..$ : num 91.8 +#> .. ..$ : int 986 +#> ..$ margin :List of 8 +#> .. ..$ : int 11 +#> .. ..$ : int 28 +#> .. ..$ : num 253 +#> .. ..$ : int 28 +#> .. ..$ : num 253 +#> .. ..$ : int 1028 +#> .. ..$ : int 11 +#> .. ..$ : int 1028 +#> ..$ width : int 161 +#> ..$ height : int 958 +``` + +There are some methods in Chromote and ChromoteSession objects which are +written using asynchronous method calls, but conditionally use +`wait_for()` so that they can be called either synchronously or +asynchronously. The `$screenshot()` method works this way, for example. +You can call `b$screenshot(wait_=TRUE)` (which is the default) for +synchronous behavior, or `b$screenshot(wait_=FALSE)` for async behavior. + +If you want to write a function that can be called in either sync or +async mode, you can use this basic structure: First, construct a promise +chain by calling the CDP methods with `wait_=FALSE`. Then, at the end, +if the user used `wait_=TRUE`, wait for the promise to resolve; +otherwise, simply return the promise. + +``` r getBoxModel <- function(b, selector = "html", wait_ = TRUE) { p <- b$DOM$getDocument(wait_ = FALSE) %...>% { b$DOM$querySelector(.$root$nodeId, selector, wait_ = FALSE) } %...>% @@ -472,33 +866,47 @@ getBoxModel(b, ".sidebar", wait_ = FALSE) %...>% str() ``` -But, you might be wondering, if we want a synchronous API, why not simply write the synchronous code by calling the individual methods synchronously, and using a normal pipe to connect them, as in: +But, you might be wondering, if we want a synchronous API, why not +simply write the synchronous code by calling the individual methods +synchronously, and using a normal pipe to connect them, as in: -```R +``` r b$DOM$getDocument() %>% { b$DOM$querySelector(.$root$nodeId, ".sidebar") } %>% { b$DOM$getBoxModel(.$nodeId) } %>% str() ``` -There are two reasons for this. The first is that this would require a duplication of all the code for the sync and async code paths. Another reason is that the internal async code can be written to send multiple independent command chains to the ChromoteSession (or multiple ChromoteSessions), and they will be executed concurrently. If there are multiple promise chains, you can do something like the following to wait for all of them to resolve: +There are two reasons for this. The first is that this would require a +duplication of all the code for the sync and async code paths. Another +reason is that the internal async code can be written to send multiple +independent command chains to the ChromoteSession (or multiple +ChromoteSessions), and they will be executed concurrently. If there are +multiple promise chains, you can do something like the following to wait +for all of them to resolve: -```R +``` r # Starting with promises p1, p2, and p3, create a promise that resolves only # after they have all been resolved. p <- promise_all(p1, p2, p3) b$wait_for(p) ``` - ## Async events -In addition to _commands_ The Chrome Devtools Protocol also has _events_. These are messages that are sent from the browser to the R process when various browser events happen. +In addition to *commands* The Chrome Devtools Protocol also has +*events*. These are messages that are sent from the browser to the R +process when various browser events happen. -As an example, it can be a bit tricky to find out when to take a screenshot. When you send the browser a command to navigate to a page, it sends a response immediately, but it may take several more seconds for it to actually finish loading that page. When it does, the `Page.loadEventFired` event will be fired. +As an example, it can be a bit tricky to find out when to take a +screenshot. When you send the browser a command to navigate to a page, +it sends a response immediately, but it may take several more seconds +for it to actually finish loading that page. When it does, the +`Page.loadEventFired` event will be fired. -```R +``` r b <- ChromoteSession$new() +#> SEND {"method":"Target.createTarget","params":{"url":"about:blank","width":992,"height":744},"id":74} # Navigate and wait for Page.loadEventFired. # Note: these lines are put in a single code block to ensure that there is no @@ -507,11 +915,20 @@ b <- ChromoteSession$new() b$Page$navigate("https://www.r-project.org/") str(b$Page$loadEventFired()) } +#> SEND {"method":"Page.navigate","params":{"url":"https://www.r-project.org/"},"id":78,"sessionId":"9BD223769AF31F43D4B265773EB825B6"} +#> Callbacks for Page++: 1 +#> Enabling events for Page +#> SEND {"method":"Page.enable","params":[],"id":79,"sessionId":"9BD223769AF31F43D4B265773EB825B6"} +#> List of 1 +#> $ timestamp: num 930216 ``` -With the synchronous API, the call to `b$Page$loadEventFired()` will block until Chromote receives a `Page.loadEventFired` message from the browser. However, with the async promise API, you would write it like this: +With the synchronous API, the call to `b$Page$loadEventFired()` will +block until Chromote receives a `Page.loadEventFired` message from the +browser. However, with the async promise API, you would write it like +this: -```R +``` r b$Page$navigate("https://www.r-project.org/", wait_ = FALSE) %...>% { b$Page$loadEventFired(wait_ = FALSE) } %...>% { str(.) } @@ -526,90 +943,158 @@ b$Page$navigate("https://www.r-project.org/", wait_ = FALSE)$ }) ``` -There will be a short delay after running the code before the value is printed. +There will be a short delay after running the code before the value is +printed. -If you want to schedule a chain of promises and then wait for them to resolve, you can once again use the `wait_for()` method. For example: +If you want to schedule a chain of promises and then wait for them to +resolve, you can once again use the `wait_for()` method. For example: -```R +``` r p <- b$Page$navigate("https://www.r-project.org/", wait_ = FALSE)$ then(function(value) { b$Page$loadEventFired(wait_ = FALSE) }) +#> SEND {"method":"Page.navigate","params":{"url":"https://www.r-project.org/"},"id":81,"sessionId":"9BD223769AF31F43D4B265773EB825B6"} # wait_for returns the last value in the chain, so we can call str() on it str(b$wait_for(p)) +#> List of 1 +#> $ timestamp: num 930217 ``` -This particular example has a twist to it: After sending the `Page.navigate` command, the R process doesn't really need to wait for browser's response before it starts waiting for the `Page.loadEventFired` event. So instead of chaining, you could just do this: +This particular example has a twist to it: After sending the +`Page.navigate` command, the R process doesn’t really need to wait for +browser’s response before it starts waiting for the +`Page.loadEventFired` event. So instead of chaining, you could just do +this: -```R +``` r p <- promise(function(resolve, reject) { b$Page$navigate("https://www.r-project.org/", wait_ = FALSE) resolve(b$Page$loadEventFired(wait_ = FALSE)) }) +#> SEND {"method":"Page.navigate","params":{"url":"https://www.r-project.org/"},"id":84,"sessionId":"9BD223769AF31F43D4B265773EB825B6"} +#> Callbacks for Page++: 1 +#> Enabling events for Page +#> SEND {"method":"Page.enable","params":[],"id":85,"sessionId":"9BD223769AF31F43D4B265773EB825B6"} str(b$wait_for(p)) +#> List of 1 +#> $ timestamp: num 930217 ``` -Essentially, the `Page.navigate` command gets sent off and we don't need to wait for the browser's reply. We can tell R to immediately start waiting for the `Page.loadEventFired` event. +Essentially, the `Page.navigate` command gets sent off and we don’t need +to wait for the browser’s reply. We can tell R to immediately start +waiting for the `Page.loadEventFired` event. -We can simplify it by not wrapping both method calls in a promise. We can just fire off the navigation command, and then directly use the promise that's returned by the event method: +We can simplify it by not wrapping both method calls in a promise. We +can just fire off the navigation command, and then directly use the +promise that’s returned by the event method: -```R +``` r b$Page$navigate("https://www.r-project.org/", wait_ = FALSE) +#> SEND {"method":"Page.navigate","params":{"url":"https://www.r-project.org/"},"id":87,"sessionId":"9BD223769AF31F43D4B265773EB825B6"} +#> p <- b$Page$loadEventFired(wait_ = FALSE) +#> Callbacks for Page++: 1 +#> Enabling events for Page +#> SEND {"method":"Page.enable","params":[],"id":88,"sessionId":"9BD223769AF31F43D4B265773EB825B6"} str(b$wait_for(p)) +#> List of 1 +#> $ timestamp: num 930217 ``` -And we can make it yet simpler by firing off the navigation command and then calling `b$Page$loadEventFired()` in synchronous mode (with the default `wait_=TRUE`), which already calls `wait_for()`. +And we can make it yet simpler by firing off the navigation command and +then calling `b$Page$loadEventFired()` in synchronous mode (with the +default `wait_=TRUE`), which already calls `wait_for()`. -```R +``` r b$Page$navigate("https://www.r-project.org/", wait_ = FALSE) +#> SEND {"method":"Page.navigate","params":{"url":"https://www.r-project.org/"},"id":90,"sessionId":"9BD223769AF31F43D4B265773EB825B6"} +#> x <- b$Page$loadEventFired() +#> Callbacks for Page++: 1 +#> Enabling events for Page +#> SEND {"method":"Page.enable","params":[],"id":91,"sessionId":"9BD223769AF31F43D4B265773EB825B6"} str(x) +#> List of 1 +#> $ timestamp: num 930217 ``` -> **Technical note:** The Chrome Devtools Protocol itself does not automatically enable event notifications. Normally, you would have to call the `Page.enable` method to turn on event notifications for the Page domain. However, Chromote saves you from needing to do this step by keeping track of how many callbacks there are for each domain. When the number of event callbacks for a domain goes from 0 to 1, Chromote automatically calls `$enable()` for that domain, and when it goes from 1 to 0, it it calls `$disable()`. - +> **Technical note:** The Chrome Devtools Protocol itself does not +> automatically enable event notifications. Normally, you would have to +> call the `Page.enable` method to turn on event notifications for the +> Page domain. However, Chromote saves you from needing to do this step +> by keeping track of how many callbacks there are for each domain. When +> the number of event callbacks for a domain goes from 0 to 1, Chromote +> automatically calls `$enable()` for that domain, and when it goes from +> 1 to 0, it it calls `$disable()`. -In addition to async events with promises, they can also be used with regular callbacks. For example: +In addition to async events with promises, they can also be used with +regular callbacks. For example: -```R +``` r b$Page$loadEventFired(callback_ = str) ``` -This tells Chromote to call `str()` (which prints to the console) on the message value every single time that a `Page.loadEventFired` event message is received. It will continue doing this indefinitely. (Calling an event method this way also increments the event callback counter.) +This tells Chromote to call `str()` (which prints to the console) on the +message value every single time that a `Page.loadEventFired` event +message is received. It will continue doing this indefinitely. (Calling +an event method this way also increments the event callback counter.) -When an event method is called with a callback, the return value is a function which will cancel the callback, so that it will no longer fire. (The canceller function also decrements the event callback counter. If you lose the canceller function, there is no way to decrement the callback counter back to 0.) +When an event method is called with a callback, the return value is a +function which will cancel the callback, so that it will no longer fire. +(The canceller function also decrements the event callback counter. If +you lose the canceller function, there is no way to decrement the +callback counter back to 0.) -```R +``` r cancel_load_event_callback <- b$Page$loadEventFired(callback_ = str) +#> Callbacks for Page++: 1 +#> Enabling events for Page +#> SEND {"method":"Page.enable","params":[],"id":93,"sessionId":"9BD223769AF31F43D4B265773EB825B6"} # Each of these will cause the callback to fire. -b$Page$navigate("https://www.r-project.org/") -b$Page$navigate("https://cran.r-project.org/") +n1 <- b$Page$navigate("https://www.r-project.org/") +#> SEND {"method":"Page.navigate","params":{"url":"https://www.r-project.org/"},"id":94,"sessionId":"9BD223769AF31F43D4B265773EB825B6"} +n2 <- b$Page$navigate("https://cran.r-project.org/") +#> SEND {"method":"Page.navigate","params":{"url":"https://cran.r-project.org/"},"id":95,"sessionId":"9BD223769AF31F43D4B265773EB825B6"} cancel_load_event_callback() +#> Callbacks for Page--: 0 +#> Disabling events for Page +#> SEND {"method":"Page.disable","params":[],"id":96,"sessionId":"9BD223769AF31F43D4B265773EB825B6"} # No longer causes the callback to fire. -b$Page$navigate("https://www.rstudio.com/") +n3 <- b$Page$navigate("https://www.rstudio.com/") +#> SEND {"method":"Page.navigate","params":{"url":"https://www.rstudio.com/"},"id":97,"sessionId":"9BD223769AF31F43D4B265773EB825B6"} ``` - ## Resource cleanup and garbage collection -When Chromote starts a Chrome process, it calls `Chrome$new()`. This launches the Chrome process it using `processx::process()`, and enables a supervisor for the process. This means that if the R process stops, the supervisor will detect this and shut down any Chrome processes that were registered with the supervisor. This prevents the proliferation of Chrome processes that are no longer needed. - -The Chromote package will, by default, use a single Chrome process and a single `Chromote` object, and each time `ChromoteSession$new()` is called, it will spawn them from the `Chromote` object. See [The Chromote object model](#the-chromote-object-model) for more information. +When Chromote starts a Chrome process, it calls `Chrome$new()`. This +launches the Chrome process it using `processx::process()`, and enables +a supervisor for the process. This means that if the R process stops, +the supervisor will detect this and shut down any Chrome processes that +were registered with the supervisor. This prevents the proliferation of +Chrome processes that are no longer needed. +The Chromote package will, by default, use a single Chrome process and a +single `Chromote` object, and each time `ChromoteSession$new()` is +called, it will spawn them from the `Chromote` object. See [The Chromote +object model](#the-chromote-object-model) for more information. ## Specifying which browser to use -Chromote will look in specific places for the Chrome web browser, depending on platform. This is done by the `chromote:::find_chrome()` function. +Chromote will look in specific places for the Chrome web browser, +depending on platform. This is done by the `chromote:::find_chrome()` +function. -If you wish to use a different browser from the default, you can set the `CHROMOTE_CHROME` environment variable, either with `Sys.setenv(CHROMOTE_CHROME="/path/to/browser")`. +If you wish to use a different browser from the default, you can set the +`CHROMOTE_CHROME` environment variable, either with +`Sys.setenv(CHROMOTE_CHROME="/path/to/browser")`. -```R +``` r Sys.setenv(CHROMOTE_CHROME = "/Applications/Chromium.app/Contents/MacOS/Chromium") b <- ChromoteSession$new() @@ -617,9 +1102,10 @@ b$view() b$Page$navigate("https://www.whatismybrowser.com/") ``` -Another way is create a `Chromote` object and explicitly specify the browser, then spawn `ChromoteSession`s from it. +Another way is create a `Chromote` object and explicitly specify the +browser, then spawn `ChromoteSession`s from it. -```R +``` r m <- Chromote$new( browser = Chrome$new(path = "/Applications/Chromium.app/Contents/MacOS/Chromium") ) @@ -629,9 +1115,10 @@ b <- m$new_session() b$Page$navigate("https://www.whatismybrowser.com/") ``` -Yet another way is to create a `Chromote` object with a specified browser, then set it as the default Chromote object. +Yet another way is to create a `Chromote` object with a specified +browser, then set it as the default Chromote object. -```R +``` r m <- Chromote$new( browser = Chrome$new(path = "/Applications/Chromium.app/Contents/MacOS/Chromium") ) @@ -644,29 +1131,33 @@ b$view() b$Page$navigate("https://www.whatismybrowser.com/") ``` - ## Chrome on remote hosts -Chromote can control a browser running on a remote host. To start the browser, open a terminal on the remote host and run one of the following, depending on your platform: - -**Warning: Depending on how the remote machine is configured, the Chrome debug server might be accessible to anyone on the Internet. Proceed with caution.** - -``` -# Mac -"/Applications/Google Chrome.app/Contents/MacOS/Google Chrome" --headless \ - --remote-debugging-address=0.0.0.0 --remote-debugging-port=9222 - -# Linux -google-chrome --headless --remote-debugging-address=0.0.0.0 --remote-debugging-port=9222 - -# Windows -"C:\Program Files (x86)\Google\Chrome\Application\chrome.exe" --headless \ - --remote-debugging-address=0.0.0.0 --remote-debugging-port=9222 -``` - -Then, in your local R session, create a Chromote object with the `host` and `port` (you will need to use the correct IP address). Once it's created, you can spawn a session off of it which you can control as normal: - -```R +Chromote can control a browser running on a remote host. To start the +browser, open a terminal on the remote host and run one of the +following, depending on your platform: + +**Warning: Depending on how the remote machine is configured, the Chrome +debug server might be accessible to anyone on the Internet. Proceed with +caution.** + + # Mac + "/Applications/Google Chrome.app/Contents/MacOS/Google Chrome" --headless \ + --remote-debugging-address=0.0.0.0 --remote-debugging-port=9222 + + # Linux + google-chrome --headless --remote-debugging-address=0.0.0.0 --remote-debugging-port=9222 + + # Windows + "C:\Program Files (x86)\Google\Chrome\Application\chrome.exe" --headless \ + --remote-debugging-address=0.0.0.0 --remote-debugging-port=9222 + +Then, in your local R session, create a Chromote object with the `host` +and `port` (you will need to use the correct IP address). Once it’s +created, you can spawn a session off of it which you can control as +normal: + +``` r r <- Chromote$new( browser = ChromeRemote$new(host = "10.0.0.5", port = 9222) ) @@ -681,11 +1172,21 @@ b$screenshot("browser.png") b$screenshot("browser_string.png", selector = ".string-major") ``` -When you use `$view()` on the remote browser, your local browser may block scripts for security reasons, which means that you won't be able to view the remote browser. If your local browser is Chrome, there will be a shield-shaped icon in the location bar that you can click in order to enable loading the scripts. (Note: Some browsers don't seem to work at all with the viewer.) +When you use `$view()` on the remote browser, your local browser may +block scripts for security reasons, which means that you won’t be able +to view the remote browser. If your local browser is Chrome, there will +be a shield-shaped icon in the location bar that you can click in order +to enable loading the scripts. (Note: Some browsers don’t seem to work +at all with the viewer.) -**Technical note:** There seem to be some timing issues with remote browsers. In the example above, the browser may finish navigating to the web site before the R process receives the response message for `$navigate()`, and therefore before R starts waiting for `Page.loadEventFired`. In order to avoid these timing problems, it may be better to write code like this: +**Technical note:** There seem to be some timing issues with remote +browsers. In the example above, the browser may finish navigating to the +web site before the R process receives the response message for +`$navigate()`, and therefore before R starts waiting for +`Page.loadEventFired`. In order to avoid these timing problems, it may +be better to write code like this: -```R +``` r { b$Page$navigate("https://www.whatismybrowser.com/", wait_ = FALSE) b$Page$loadEventFired() @@ -693,21 +1194,29 @@ When you use `$view()` on the remote browser, your local browser may block scrip b$screenshot("browser.png") ``` -This tells it to fire off the `Page.navigate` command and _not_ wait for it, and then immediately start waiting for `Page.loadEventFired` event. +This tells it to fire off the `Page.navigate` command and *not* wait for +it, and then immediately start waiting for `Page.loadEventFired` event. ### Attaching to existing tabs -In the examples above, we connected to an existing browser, but created a new tab to attach to. It's also possible to attach to an existing browser *and* and existing tab. In Chrome debugging terminology a tab is called a "Target", and there is a command to retrieve the list of current Targets: +In the examples above, we connected to an existing browser, but created +a new tab to attach to. It’s also possible to attach to an existing +browser *and* and existing tab. In Chrome debugging terminology a tab is +called a “Target”, and there is a command to retrieve the list of +current Targets: -```R +``` r r$Target$getTargets() ``` -Every target has a unique identifier string associated with it called the `targetId`; `"9DAE349A3A533718ED9E17441BA5159B"` is an example of one. +Every target has a unique identifier string associated with it called +the `targetId`; `"9DAE349A3A533718ED9E17441BA5159B"` is an example of +one. -Here we define a function that retrieves the ID of the first Target (tab) from a Chromote object: +Here we define a function that retrieves the ID of the first Target +(tab) from a Chromote object: -```R +``` r first_id <- function(r) { ts <- r$Target$getTargets()$targetInfos stopifnot(length(ts) > 0) @@ -717,7 +1226,7 @@ first_id <- function(r) { The following code shows an alert box in the first tab, whatever it is: -```R +``` r rc <- ChromeRemote$new(host = "localhost", port = 9222) r <- Chromote$new(browser = rc) tid <- first_id(r) @@ -725,15 +1234,18 @@ b <- r$new_session(targetId = tid) b$Runtime$evaluate('alert("this is the first tab")') ``` -***** +----- ## Examples ### Taking a screenshot of a web page -Take a screenshot of the viewport and display it using the [showimage](https://github.com/r-lib/showimage#readme) package. This uses Chromote's `$screenshot()` method, which wraps up many calls to the Chrome Devtools Protocol. +Take a screenshot of the viewport and display it using the +[showimage](https://github.com/r-lib/showimage#readme) package. This +uses Chromote’s `$screenshot()` method, which wraps up many calls to the +Chrome Devtools Protocol. -```R +``` r b <- ChromoteSession$new() # ==== Synchronous version ==== @@ -751,42 +1263,49 @@ b$Page$loadEventFired(wait_ = FALSE)$ }) ``` -It is also possible to use selectors to specify what to screenshot, as well as the region ("content", "border", "padding", or "margin"). +It is also possible to use selectors to specify what to screenshot, as +well as the region (“content”, “border”, “padding”, or “margin”). -```R +``` r # Using CSS selectors, choosing the region, and using scaling b$screenshot("s1.png", selector = ".sidebar") b$screenshot("s2.png", selector = ".sidebar", region = "margin") b$screenshot("s3.png", selector = ".page", region = "margin", scale = 2) ``` -If a vector is passed to `selector`, it will take a screenshot with a rectangle that encompasses all the DOM elements picked out by the selectors. Similarly, if a selector picks out multiple DOM elements, all of them will be in the screenshot region. - +If a vector is passed to `selector`, it will take a screenshot with a +rectangle that encompasses all the DOM elements picked out by the +selectors. Similarly, if a selector picks out multiple DOM elements, all +of them will be in the screenshot region. ### Taking a screenshot of a web page after interacting with it -Headless Chrome provides a remote debugging UI which you can use to interact with the web page. The ChromoteSession's `$view()` method opens a regular browser and navigates to the remote debugging UI. +Headless Chrome provides a remote debugging UI which you can use to +interact with the web page. The ChromoteSession’s `$view()` method opens +a regular browser and navigates to the remote debugging UI. -```R +``` r b <- ChromoteSession$new() b$view() b$Page$navigate("https://www.google.com") # Or just type the URL in the navigation bar ``` -At this point, you can interact with the web page by typing in text and clicking on things. +At this point, you can interact with the web page by typing in text and +clicking on things. Then take a screenshot: -```R +``` r b$screenshot() ``` ### Taking screenshots of web pages in parallel -With async code, it's possible to navigate to and take screenshots of multiple websites in parallel. +With async code, it’s possible to navigate to and take screenshots of +multiple websites in parallel. -```R +``` r library(promises) library(chromote) urls <- c( @@ -833,12 +1352,13 @@ cm$wait_for(pa) #> Done! ``` - ### Setting custom headers -Currently setting custom headers requires a little extra work because it requires `Network.enable` be called before using it. In the future we'll streamline things so that it will happen automatically. +Currently setting custom headers requires a little extra work because it +requires `Network.enable` be called before using it. In the future we’ll +streamline things so that it will happen automatically. -```R +``` r b <- ChromoteSession$new() # Currently need to manually enable Network domain notifications. Calling # b$Network$enable() would do it, but calling it directly will bypass the @@ -876,7 +1396,7 @@ disable_network_notifications() Synchronous version: -```R +``` r # ==== Synchronous version ==== b$Network$setUserAgentOverride(userAgent = "My fake browser") @@ -893,12 +1413,12 @@ b$Page$loadEventFired(wait_ = FALSE)$ }) ``` - ### Extracting text from a web page -One way to extract text from a page is to tell the browser to run JavaScript code that does it: +One way to extract text from a page is to tell the browser to run +JavaScript code that does it: -```R +``` r # ==== Synchronous version ==== b$Page$navigate("https://www.whatismybrowser.com/") @@ -921,9 +1441,11 @@ b$Page$loadEventFired(wait_ = FALSE)$ }) ``` -Another way is to use CDP commands to extract content from the DOM. This does not require executing JavaScript in the browser's context, but it is also not as flexible as JavaScript. +Another way is to use CDP commands to extract content from the DOM. This +does not require executing JavaScript in the browser’s context, but it +is also not as flexible as JavaScript. -```R +``` r # ==== Synchronous version ==== b$Page$navigate("https://www.whatismybrowser.com/") x <- b$DOM$getDocument() @@ -952,39 +1474,43 @@ b$Page$loadEventFired(wait_ = FALSE)$ ### Websites that require authentication -For websites that require authentication, you can use Chromote to get screenshots by doing the following: +For websites that require authentication, you can use Chromote to get +screenshots by doing the following: -1. Log in interactively and navigate to the page. -1. Capture cookies from the page and save them. -1. In a later R session, load the cookies. -1. Use the cookies in Chromote and navigate to the page. -1. Take a screenshot. +1. Log in interactively and navigate to the page. +2. Capture cookies from the page and save them. +3. In a later R session, load the cookies. +4. Use the cookies in Chromote and navigate to the page. +5. Take a screenshot. There are two ways to capture the cookies. - -**Method 1:** The first method uses the headless browser's viewer. This can be a bit inconvenient because it requires going through the entire login process, even if you have already logged in with a normal browser. +**Method 1:** The first method uses the headless browser’s viewer. This +can be a bit inconvenient because it requires going through the entire +login process, even if you have already logged in with a normal browser. First navigate to the page: -```R +``` r library(chromote) b <- ChromoteSession$new() b$view() b$Page$navigate("https://beta.rstudioconnect.com/content/123456/") ``` -Next, log in interactively via the viewer. Once that's done, use Chromote to capture the cookies. +Next, log in interactively via the viewer. Once that’s done, use +Chromote to capture the cookies. -```R +``` r cookies <- b$Network$getCookies() str(cookies) saveRDS(cookies, "cookies.rds") ``` -After saving the cookies, you can restart R and navigate to the page, using the cookies. +After saving the cookies, you can restart R and navigate to the page, +using the cookies. -```R +``` r library(chromote) b <- ChromoteSession$new() b$view() @@ -995,27 +1521,37 @@ b$Page$navigate("https://beta.rstudioconnect.com/content/123456/") b$screenshot() ``` +**Method 2:** The second method captures the cookies using a normal +browser. This is can be more convenient because, if you are already +logged in, you don’t need to do it again. This requires a Chromium-based +browser, and it requires running DevTools-in-DevTools on that browser. -**Method 2:** The second method captures the cookies using a normal browser. This is can be more convenient because, if you are already logged in, you don't need to do it again. This requires a Chromium-based browser, and it requires running DevTools-in-DevTools on that browser. - -First, navigate to the page in your browser. Then press Cmd-Option-I (Mac) or Ctrl-Shift-I (Windows/Linux). The developer tools panel will open. Make sure to undock the developer tools so that they are in their own window. Then press Cmd-Option-I or Ctrl-Shift-I again. A second developer tools window will open. (See [this SO answer](https://stackoverflow.com/questions/12291138/how-do-you-inspect-the-web-inspector-in-chrome/12291163#12291163) for detailed instructions.) +First, navigate to the page in your browser. Then press Cmd-Option-I +(Mac) or Ctrl-Shift-I (Windows/Linux). The developer tools panel will +open. Make sure to undock the developer tools so that they are in their +own window. Then press Cmd-Option-I or Ctrl-Shift-I again. A second +developer tools window will open. (See [this SO +answer](https://stackoverflow.com/questions/12291138/how-do-you-inspect-the-web-inspector-in-chrome/12291163#12291163) +for detailed instructions.) In the second developer tools window, run the following: -```JS +``` js var cookies = await Main.sendOverProtocol('Network.getCookies', {}) JSON.stringify(cookies) ``` -This will return a JSON string representing the cookies for that page. For example: +This will return a JSON string representing the cookies for that page. +For example: -```JSON +``` json [{"cookies":[{"name":"AWSALB","value":"T3dNdcdnMasdf/cNn0j+JHMVkZ3RI8mitnAggd9AlPsaWJdsfoaje/OowIh0qe3dDPiHc0mSafe5jNH+1Aeinfalsd30AejBZDYwE","domain":"beta.rstudioconnect.com","path":"/","expires":1594632233.96943,"size":130,"httpOnly":false,"secure":false,"session":false}]}] ``` -Copy that string to the clipboard. In your R session, you can paste it to this code, surrounded by single-quotes: +Copy that string to the clipboard. In your R session, you can paste it +to this code, surrounded by single-quotes: -```R +``` r cookie_json <- '[{"cookies":[{"name":"AWSALB","value":"T3dNdcdnMasdf/cNn0j+JHMVkZ3RI8mitnAggd9AlPsaWJdsfoaje/OowIh0qe3dDPiHc0mSafe5jNH+1Aeinfalsd30AejBZDYwE","domain":"beta.rstudioconnect.com","path":"/","expires":1594632233.96943,"size":130,"httpOnly":false,"secure":false,"session":false}]}]' cookies <- jsonlite::fromJSON(cookie_json, simplifyVector = FALSE)[[1]] @@ -1023,7 +1559,7 @@ cookies <- jsonlite::fromJSON(cookie_json, simplifyVector = FALSE)[[1]] Then you can use Chromote to navigate to the page and take a screenshot. -```R +``` r library(chromote) b <- ChromoteSession$new() b$view() From 355eafb8a123fea20a42609e9e2ab8fad0945129 Mon Sep 17 00:00:00 2001 From: Barret Schloerke Date: Tue, 13 Jul 2021 17:21:13 -0400 Subject: [PATCH 08/13] ignore sidebar (and readme.rmd) --- .Rbuildignore | 2 ++ .gitignore | 1 + 2 files changed, 3 insertions(+) diff --git a/.Rbuildignore b/.Rbuildignore index bfc34423..6e43e6c8 100644 --- a/.Rbuildignore +++ b/.Rbuildignore @@ -6,3 +6,5 @@ ^_pkgdown\.yml$ ^docs$ ^pkgdown$ +^README\.Rmd$ +^sidebar.png$ diff --git a/.gitignore b/.gitignore index 1e8abaeb..18f5b1d2 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,4 @@ .Rproj.user temp docs +sidebar.png From da4f259743276c58e075ad2df5717ccdb2be0b39 Mon Sep 17 00:00:00 2001 From: Barret Schloerke Date: Tue, 13 Jul 2021 17:21:57 -0400 Subject: [PATCH 09/13] Update README.md --- README.md | 85 +++++++++++++++++++++++++------------------------------ 1 file changed, 38 insertions(+), 47 deletions(-) diff --git a/README.md b/README.md index f4b467de..c96ce259 100644 --- a/README.md +++ b/README.md @@ -87,10 +87,10 @@ page load in the viewer: ``` r b$Page$navigate("https://www.r-project.org/") #> $frameId -#> [1] "CC9ECA1E389ADB9A7B8DBEB03DFF9EF7" +#> [1] "E45F218CE72B1BAEFA376140F5158676" #> #> $loaderId -#> [1] "9E210CE4CF0832F80232331F062EA412" +#> [1] "AAE0287C68F36828B5810C103AB9D9EA" ``` In the official Chrome DevTools Protocol (CDP) documentation, this is @@ -239,10 +239,10 @@ affecting the first one. b1$view() b1$Page$navigate("https://github.com/rstudio/chromote") #> $frameId -#> [1] "2322142974B792020563906C9ECE6FB7" +#> [1] "F6265B867FDB447141D094636EDDCE1C" #> #> $loaderId -#> [1] "5C033DAC1481DDB6D7FDB28AEAD30B60" +#> [1] "E3B7DF6412CF2677A089E240E524409F" ``` To close a browser tab/window, you can run: @@ -286,14 +286,14 @@ be enforced by wrapping both lines of code in `{ .... }`). # Send a command to navigate to a page b$Page$navigate("https://www.r-project.org") #> $frameId -#> [1] "CC9ECA1E389ADB9A7B8DBEB03DFF9EF7" +#> [1] "E45F218CE72B1BAEFA376140F5158676" #> #> $loaderId -#> [1] "F5ECABCF3574197798EF2ADD936C0FCD" +#> [1] "62631131C8570E6AF7061AAD50833D18" # Wait for the Page.loadEventFired event b$Page$loadEventFired() #> $timestamp -#> [1] 930213.7 +#> [1] 930407.9 ``` After running these two lines, the R process will be blocked. While it’s @@ -428,12 +428,12 @@ understanding how the Chrome Devtools Protocol works. b <- ChromoteSession$new() b$parent$debug_messages(TRUE) b$Page$navigate("https://www.r-project.org/") -#> SEND {"method":"Page.navigate","params":{"url":"https://www.r-project.org/"},"id":53,"sessionId":"32B0A19254EB899A4EE36A988B580E45"} +#> SEND {"method":"Page.navigate","params":{"url":"https://www.r-project.org/"},"id":53,"sessionId":"AC60732C88EA310A3674B2EEE7454ED7"} #> $frameId -#> [1] "2A34B3A6869278EC3BFB87879CE67C1A" +#> [1] "E47F1F1D63B0951F4E97A98E04805D52" #> #> $loaderId -#> [1] "55074CFE2757CA6F941FAA4F9F875C58" +#> [1] "3A69CFD13600D4270F9F969F424D77A1" ``` ### Synchronous vs. asynchronous usage @@ -523,7 +523,7 @@ JSON to an R object. For example: ``` r # Synchronous str(b$Browser$getVersion()) -#> SEND {"method":"Browser.getVersion","params":[],"id":54,"sessionId":"32B0A19254EB899A4EE36A988B580E45"} +#> SEND {"method":"Browser.getVersion","params":[],"id":54,"sessionId":"AC60732C88EA310A3674B2EEE7454ED7"} #> List of 5 #> $ protocolVersion: chr "1.3" #> $ product : chr "HeadlessChrome/91.0.4472.114" @@ -541,7 +541,7 @@ promise. For example: ``` r # Async with callback b$Browser$getVersion(wait_ = FALSE, callback_ = str) -#> SEND {"method":"Browser.getVersion","params":[],"id":55,"sessionId":"32B0A19254EB899A4EE36A988B580E45"} +#> SEND {"method":"Browser.getVersion","params":[],"id":55,"sessionId":"AC60732C88EA310A3674B2EEE7454ED7"} #> # After a very, very short pause, prints: ``` @@ -569,7 +569,7 @@ example: b$Browser$getVersion(wait_ = FALSE, callback_ = str) 1+1 } -#> SEND {"method":"Browser.getVersion","params":[],"id":56,"sessionId":"32B0A19254EB899A4EE36A988B580E45"} +#> SEND {"method":"Browser.getVersion","params":[],"id":56,"sessionId":"AC60732C88EA310A3674B2EEE7454ED7"} #> [1] 2 ``` @@ -592,7 +592,7 @@ product <- NULL b$Browser$getVersion(wait_ = FALSE, callback_ = function(msg) { product <<- msg$product }) -#> SEND {"method":"Browser.getVersion","params":[],"id":57,"sessionId":"32B0A19254EB899A4EE36A988B580E45"} +#> SEND {"method":"Browser.getVersion","params":[],"id":57,"sessionId":"AC60732C88EA310A3674B2EEE7454ED7"} #> ``` @@ -635,7 +635,7 @@ a single paste operation and have the same effect.) print(value$product) }) } -#> SEND {"method":"Browser.getVersion","params":[],"id":58,"sessionId":"32B0A19254EB899A4EE36A988B580E45"} +#> SEND {"method":"Browser.getVersion","params":[],"id":58,"sessionId":"AC60732C88EA310A3674B2EEE7454ED7"} # Wait for a moment, then prints: ``` @@ -653,26 +653,26 @@ library(promises) b$Browser$getVersion(wait_ = FALSE) %>% then(function(value) { print(value$product) }) -#> SEND {"method":"Browser.getVersion","params":[],"id":59,"sessionId":"32B0A19254EB899A4EE36A988B580E45"} +#> SEND {"method":"Browser.getVersion","params":[],"id":59,"sessionId":"AC60732C88EA310A3674B2EEE7454ED7"} # Promise-pipe to anonymous function, which must be wrapped in parens b$Browser$getVersion(wait_ = FALSE) %...>% (function(value) { print(value$product) }) -#> SEND {"method":"Browser.getVersion","params":[],"id":60,"sessionId":"32B0A19254EB899A4EE36A988B580E45"} +#> SEND {"method":"Browser.getVersion","params":[],"id":60,"sessionId":"AC60732C88EA310A3674B2EEE7454ED7"} # Promise-pipe to an expression (which gets converted to a function with the first argument `.`) b$Browser$getVersion(wait_ = FALSE) %...>% { print(.$product) } -#> SEND {"method":"Browser.getVersion","params":[],"id":61,"sessionId":"32B0A19254EB899A4EE36A988B580E45"} +#> SEND {"method":"Browser.getVersion","params":[],"id":61,"sessionId":"AC60732C88EA310A3674B2EEE7454ED7"} # Promise-pipe to a named function, with parentheses print_product <- function(msg) print(msg$product) b$Browser$getVersion(wait_ = FALSE) %...>% print_product() -#> SEND {"method":"Browser.getVersion","params":[],"id":62,"sessionId":"32B0A19254EB899A4EE36A988B580E45"} +#> SEND {"method":"Browser.getVersion","params":[],"id":62,"sessionId":"AC60732C88EA310A3674B2EEE7454ED7"} # Promise-pipe to a named function, without parentheses b$Browser$getVersion(wait_ = FALSE) %...>% print_product -#> SEND {"method":"Browser.getVersion","params":[],"id":63,"sessionId":"32B0A19254EB899A4EE36A988B580E45"} +#> SEND {"method":"Browser.getVersion","params":[],"id":63,"sessionId":"AC60732C88EA310A3674B2EEE7454ED7"} ``` The earlier example where we found the dimensions of a DOM element using @@ -685,7 +685,7 @@ b$DOM$getDocument(wait_ = FALSE) %...>% { b$DOM$querySelector(.$root$nodeId, ".sidebar", wait_ = FALSE) } %...>% { b$DOM$getBoxModel(.$nodeId, wait_ = FALSE) } %...>% str() -#> SEND {"method":"DOM.getDocument","params":{},"id":64,"sessionId":"32B0A19254EB899A4EE36A988B580E45"} +#> SEND {"method":"DOM.getDocument","params":{},"id":64,"sessionId":"AC60732C88EA310A3674B2EEE7454ED7"} # Or, more verbosely: @@ -699,7 +699,7 @@ b$DOM$getDocument(wait_ = FALSE)$ then(function(value) { str(value) }) -#> SEND {"method":"DOM.getDocument","params":{},"id":65,"sessionId":"32B0A19254EB899A4EE36A988B580E45"} +#> SEND {"method":"DOM.getDocument","params":{},"id":65,"sessionId":"AC60732C88EA310A3674B2EEE7454ED7"} ``` Each step in the promise chain uses the value from the previous step, @@ -720,7 +720,7 @@ p <- b$DOM$getDocument(wait_ = FALSE) %...>% { b$DOM$querySelector(.$root$nodeId, ".sidebar", wait_ = FALSE) } %...>% { b$DOM$getBoxModel(.$nodeId, wait_ = FALSE) } %...>% str() -#> SEND {"method":"DOM.getDocument","params":{},"id":66,"sessionId":"32B0A19254EB899A4EE36A988B580E45"} +#> SEND {"method":"DOM.getDocument","params":{},"id":66,"sessionId":"AC60732C88EA310A3674B2EEE7454ED7"} b$wait_for(p) #> [1] "HeadlessChrome/91.0.4472.114" @@ -786,7 +786,7 @@ value returned by `$wait_for()`: p <- b$DOM$getDocument(wait_ = FALSE) %...>% { b$DOM$querySelector(.$root$nodeId, ".sidebar", wait_ = FALSE) } %...>% { b$DOM$getBoxModel(.$nodeId, wait_ = FALSE) } -#> SEND {"method":"DOM.getDocument","params":{},"id":71,"sessionId":"32B0A19254EB899A4EE36A988B580E45"} +#> SEND {"method":"DOM.getDocument","params":{},"id":71,"sessionId":"AC60732C88EA310A3674B2EEE7454ED7"} x <- b$wait_for(p) str(x) @@ -915,12 +915,12 @@ b <- ChromoteSession$new() b$Page$navigate("https://www.r-project.org/") str(b$Page$loadEventFired()) } -#> SEND {"method":"Page.navigate","params":{"url":"https://www.r-project.org/"},"id":78,"sessionId":"9BD223769AF31F43D4B265773EB825B6"} +#> SEND {"method":"Page.navigate","params":{"url":"https://www.r-project.org/"},"id":78,"sessionId":"2B74C343877CB2F93C9F7366679A7ED3"} #> Callbacks for Page++: 1 #> Enabling events for Page -#> SEND {"method":"Page.enable","params":[],"id":79,"sessionId":"9BD223769AF31F43D4B265773EB825B6"} +#> SEND {"method":"Page.enable","params":[],"id":79,"sessionId":"2B74C343877CB2F93C9F7366679A7ED3"} #> List of 1 -#> $ timestamp: num 930216 +#> $ timestamp: num 930411 ``` With the synchronous API, the call to `b$Page$loadEventFired()` will @@ -954,12 +954,12 @@ p <- b$Page$navigate("https://www.r-project.org/", wait_ = FALSE)$ then(function(value) { b$Page$loadEventFired(wait_ = FALSE) }) -#> SEND {"method":"Page.navigate","params":{"url":"https://www.r-project.org/"},"id":81,"sessionId":"9BD223769AF31F43D4B265773EB825B6"} +#> SEND {"method":"Page.navigate","params":{"url":"https://www.r-project.org/"},"id":81,"sessionId":"2B74C343877CB2F93C9F7366679A7ED3"} # wait_for returns the last value in the chain, so we can call str() on it str(b$wait_for(p)) #> List of 1 -#> $ timestamp: num 930217 +#> $ timestamp: num 930411 ``` This particular example has a twist to it: After sending the @@ -973,14 +973,14 @@ p <- promise(function(resolve, reject) { b$Page$navigate("https://www.r-project.org/", wait_ = FALSE) resolve(b$Page$loadEventFired(wait_ = FALSE)) }) -#> SEND {"method":"Page.navigate","params":{"url":"https://www.r-project.org/"},"id":84,"sessionId":"9BD223769AF31F43D4B265773EB825B6"} +#> SEND {"method":"Page.navigate","params":{"url":"https://www.r-project.org/"},"id":84,"sessionId":"2B74C343877CB2F93C9F7366679A7ED3"} #> Callbacks for Page++: 1 #> Enabling events for Page -#> SEND {"method":"Page.enable","params":[],"id":85,"sessionId":"9BD223769AF31F43D4B265773EB825B6"} +#> SEND {"method":"Page.enable","params":[],"id":85,"sessionId":"2B74C343877CB2F93C9F7366679A7ED3"} str(b$wait_for(p)) #> List of 1 -#> $ timestamp: num 930217 +#> $ timestamp: num 930411 ``` Essentially, the `Page.navigate` command gets sent off and we don’t need @@ -993,15 +993,15 @@ promise that’s returned by the event method: ``` r b$Page$navigate("https://www.r-project.org/", wait_ = FALSE) -#> SEND {"method":"Page.navigate","params":{"url":"https://www.r-project.org/"},"id":87,"sessionId":"9BD223769AF31F43D4B265773EB825B6"} +#> SEND {"method":"Page.navigate","params":{"url":"https://www.r-project.org/"},"id":87,"sessionId":"2B74C343877CB2F93C9F7366679A7ED3"} #> p <- b$Page$loadEventFired(wait_ = FALSE) #> Callbacks for Page++: 1 #> Enabling events for Page -#> SEND {"method":"Page.enable","params":[],"id":88,"sessionId":"9BD223769AF31F43D4B265773EB825B6"} +#> SEND {"method":"Page.enable","params":[],"id":88,"sessionId":"2B74C343877CB2F93C9F7366679A7ED3"} str(b$wait_for(p)) #> List of 1 -#> $ timestamp: num 930217 +#> $ timestamp: num 930411 ``` And we can make it yet simpler by firing off the navigation command and @@ -1010,15 +1010,15 @@ default `wait_=TRUE`), which already calls `wait_for()`. ``` r b$Page$navigate("https://www.r-project.org/", wait_ = FALSE) -#> SEND {"method":"Page.navigate","params":{"url":"https://www.r-project.org/"},"id":90,"sessionId":"9BD223769AF31F43D4B265773EB825B6"} +#> SEND {"method":"Page.navigate","params":{"url":"https://www.r-project.org/"},"id":90,"sessionId":"2B74C343877CB2F93C9F7366679A7ED3"} #> x <- b$Page$loadEventFired() #> Callbacks for Page++: 1 #> Enabling events for Page -#> SEND {"method":"Page.enable","params":[],"id":91,"sessionId":"9BD223769AF31F43D4B265773EB825B6"} +#> SEND {"method":"Page.enable","params":[],"id":91,"sessionId":"2B74C343877CB2F93C9F7366679A7ED3"} str(x) #> List of 1 -#> $ timestamp: num 930217 +#> $ timestamp: num 930411 ``` > **Technical note:** The Chrome Devtools Protocol itself does not @@ -1050,24 +1050,15 @@ callback counter back to 0.) ``` r cancel_load_event_callback <- b$Page$loadEventFired(callback_ = str) -#> Callbacks for Page++: 1 -#> Enabling events for Page -#> SEND {"method":"Page.enable","params":[],"id":93,"sessionId":"9BD223769AF31F43D4B265773EB825B6"} # Each of these will cause the callback to fire. n1 <- b$Page$navigate("https://www.r-project.org/") -#> SEND {"method":"Page.navigate","params":{"url":"https://www.r-project.org/"},"id":94,"sessionId":"9BD223769AF31F43D4B265773EB825B6"} n2 <- b$Page$navigate("https://cran.r-project.org/") -#> SEND {"method":"Page.navigate","params":{"url":"https://cran.r-project.org/"},"id":95,"sessionId":"9BD223769AF31F43D4B265773EB825B6"} cancel_load_event_callback() -#> Callbacks for Page--: 0 -#> Disabling events for Page -#> SEND {"method":"Page.disable","params":[],"id":96,"sessionId":"9BD223769AF31F43D4B265773EB825B6"} # No longer causes the callback to fire. n3 <- b$Page$navigate("https://www.rstudio.com/") -#> SEND {"method":"Page.navigate","params":{"url":"https://www.rstudio.com/"},"id":97,"sessionId":"9BD223769AF31F43D4B265773EB825B6"} ``` ## Resource cleanup and garbage collection From 44d3dc720056cfef66b841d9c3953a857686169c Mon Sep 17 00:00:00 2001 From: Barret Schloerke Date: Tue, 13 Jul 2021 17:29:42 -0400 Subject: [PATCH 10/13] support PR checking the pkgdown build (but not posting) --- .github/workflows/pkgdown-pak.yaml | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/.github/workflows/pkgdown-pak.yaml b/.github/workflows/pkgdown-pak.yaml index 723f7b6d..3f46b952 100644 --- a/.github/workflows/pkgdown-pak.yaml +++ b/.github/workflows/pkgdown-pak.yaml @@ -3,8 +3,11 @@ on: branches: - main - master + - rc-** tags: -'*' + pull_request: + branches: master name: pkgdown @@ -54,7 +57,20 @@ jobs: - name: Install package run: R CMD INSTALL . - - name: Build and deploy pkgdown site + - name: Build Site (PR) + if: github.event_name != 'push' + shell: Rscript {0} + run: | + pkgdown::build_site(new_process = FALSE) + # Must validate after. Otherwise files are saved and `pkgdown::build_site()` gets mad + - name: Validate all topics exist (PR) + if: github.event_name != 'push' + shell: Rscript {0} + run: | + pkgdown::build_reference_index() + stopifnot(length(warnings()) == 0) + + - name: Build and deploy pkgdown site (push) run: | git config --local user.name "$GITHUB_ACTOR" git config --local user.email "$GITHUB_ACTOR@users.noreply.github.com" From 628d3a9a9a28dc8b02a8031b548bb72e488ee903 Mon Sep 17 00:00:00 2001 From: Barret Schloerke Date: Tue, 13 Jul 2021 17:36:48 -0400 Subject: [PATCH 11/13] Only push pkgdown on push event --- .github/workflows/pkgdown-pak.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/pkgdown-pak.yaml b/.github/workflows/pkgdown-pak.yaml index 3f46b952..706c0c5e 100644 --- a/.github/workflows/pkgdown-pak.yaml +++ b/.github/workflows/pkgdown-pak.yaml @@ -71,6 +71,7 @@ jobs: stopifnot(length(warnings()) == 0) - name: Build and deploy pkgdown site (push) + if: github.event_name == 'push' run: | git config --local user.name "$GITHUB_ACTOR" git config --local user.email "$GITHUB_ACTOR@users.noreply.github.com" From 0049579aa7c9838b07bfdbc53ff6a0645023fb33 Mon Sep 17 00:00:00 2001 From: Barret Schloerke Date: Tue, 13 Jul 2021 17:37:05 -0400 Subject: [PATCH 12/13] use R version aliases --- .github/workflows/R-CMD-check.yaml | 84 ++++++++++++++++++++++++++---- 1 file changed, 75 insertions(+), 9 deletions(-) diff --git a/.github/workflows/R-CMD-check.yaml b/.github/workflows/R-CMD-check.yaml index 78155482..079c4d20 100644 --- a/.github/workflows/R-CMD-check.yaml +++ b/.github/workflows/R-CMD-check.yaml @@ -17,24 +17,90 @@ on: name: R-CMD-check jobs: + rversions: + name: R Versions + runs-on: ubuntu-latest + outputs: + devel: ${{ steps.devel.outputs.installed-r-version }} + release: ${{ steps.release.outputs.installed-r-version }} + oldrel1: ${{ steps.oldrel1.outputs.installed-r-version }} + oldrel2: ${{ steps.oldrel2.outputs.installed-r-version }} + oldrel3: ${{ steps.oldrel3.outputs.installed-r-version }} + oldrel4: ${{ steps.oldrel4.outputs.installed-r-version }} + steps: + - name: devel + uses: r-lib/actions/setup-r@master + id: devel + with: + r-version: devel + install-r: false # No need to install. Just need version + + - name: release + uses: r-lib/actions/setup-r@master + id: release + with: + r-version: release + install-r: false # No need to install. Just need version + + - name: oldrel/1 + uses: r-lib/actions/setup-r@master + id: oldrel1 + with: + r-version: oldrel/1 + install-r: false # No need to install. Just need version + + - name: oldrel/2 + uses: r-lib/actions/setup-r@master + id: oldrel2 + with: + r-version: oldrel/2 + install-r: false # No need to install. Just need version + + - name: oldrel/3 + uses: r-lib/actions/setup-r@master + id: oldrel3 + with: + r-version: oldrel/3 + install-r: false # No need to install. Just need version + + - name: oldrel/4 + uses: r-lib/actions/setup-r@master + id: oldrel4 + with: + r-version: oldrel/4 + install-r: false # No need to install. Just need version + + - name: Set Output + id: set_versions + run: | + echo "devel: ${{ steps.devel.outputs.installed-r-version }}" + echo "release: ${{ steps.release.outputs.installed-r-version }}" + echo "oldrel1: ${{ steps.oldrel1.outputs.installed-r-version }}" + echo "oldrel2: ${{ steps.oldrel2.outputs.installed-r-version }}" + echo "oldrel3: ${{ steps.oldrel3.outputs.installed-r-version }}" + echo "oldrel4: ${{ steps.oldrel4.outputs.installed-r-version }}" + + R-CMD-check: runs-on: ${{ matrix.config.os }} name: ${{ matrix.config.os }} (${{ matrix.config.r }}) + needs: + - rversions strategy: fail-fast: false matrix: config: - - {os: macOS-latest, r: 'release'} - - {os: windows-latest, r: 'release'} + - {os: macOS-latest, r: '${{ needs.rversions.outputs.release }}'} + - {os: windows-latest, r: '${{ needs.rversions.outputs.release }}'} - {os: windows-latest, r: '3.6', rspm: "https://packagemanager.rstudio.com/cran/latest"} - - {os: ubuntu-18.04, r: 'devel', rspm: "https://packagemanager.rstudio.com/cran/__linux__/bionic/latest", http-user-agent: "R/4.1.0 (ubuntu-18.04) R (4.1.0 x86_64-pc-linux-gnu x86_64 linux-gnu) on GitHub Actions" } - - {os: ubuntu-18.04, r: 'release', rspm: "https://packagemanager.rstudio.com/cran/__linux__/bionic/latest"} - - {os: ubuntu-18.04, r: 'oldrel', rspm: "https://packagemanager.rstudio.com/cran/__linux__/bionic/latest"} - - {os: ubuntu-18.04, r: '3.6', rspm: "https://packagemanager.rstudio.com/cran/__linux__/bionic/latest"} - - {os: ubuntu-18.04, r: '3.5', rspm: "https://packagemanager.rstudio.com/cran/__linux__/bionic/latest"} - - {os: ubuntu-18.04, r: '3.4', rspm: "https://packagemanager.rstudio.com/cran/__linux__/bionic/latest"} + - {os: ubuntu-18.04, r: 'devel', rspm: "https://packagemanager.rstudio.com/cran/__linux__/bionic/latest", http-user-agent: "release" } + - {os: ubuntu-18.04, r: '${{ needs.rversions.outputs.release }}', rspm: "https://packagemanager.rstudio.com/cran/__linux__/bionic/latest"} + - {os: ubuntu-18.04, r: '${{ needs.rversions.outputs.oldrel1 }}', rspm: "https://packagemanager.rstudio.com/cran/__linux__/bionic/latest"} + - {os: ubuntu-18.04, r: '${{ needs.rversions.outputs.oldrel2 }}', rspm: "https://packagemanager.rstudio.com/cran/__linux__/bionic/latest"} + - {os: ubuntu-18.04, r: '${{ needs.rversions.outputs.oldrel3 }}', rspm: "https://packagemanager.rstudio.com/cran/__linux__/bionic/latest"} + - {os: ubuntu-18.04, r: '${{ needs.rversions.outputs.oldrel4 }}', rspm: "https://packagemanager.rstudio.com/cran/__linux__/bionic/latest"} env: RSPM: ${{ matrix.config.rspm }} @@ -43,7 +109,7 @@ jobs: steps: - uses: actions/checkout@v2 - - uses: r-lib/actions/setup-r@v1 + - uses: r-lib/actions/setup-r@master id: install-r with: r-version: ${{ matrix.config.r }} From e9852f0703f5920505b82a62d59f41659a5d5e87 Mon Sep 17 00:00:00 2001 From: Barret Schloerke Date: Tue, 13 Jul 2021 17:42:44 -0400 Subject: [PATCH 13/13] `find_chrome: no visible global function definition for 'readRegistry'` --- R/chrome.R | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/R/chrome.R b/R/chrome.R index 17ad7001..ec224668 100644 --- a/R/chrome.R +++ b/R/chrome.R @@ -41,7 +41,7 @@ find_chrome <- function() { } else if (is_windows()) { tryCatch( { - path <- readRegistry("SOFTWARE\\Microsoft\\Windows\\CurrentVersion\\App Paths\\chrome.exe\\") + path <- utils::readRegistry("SOFTWARE\\Microsoft\\Windows\\CurrentVersion\\App Paths\\chrome.exe\\") path <- path[["(Default)"]] }, error = function(e) {