diff --git a/DESCRIPTION b/DESCRIPTION index 6c73d86c2..adafd649d 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -41,6 +41,8 @@ Suggests: curl (>= 0.9.5), diffviewer (>= 0.1.0), knitr, + otel (>= 0.0.0.9000), + otelsdk (>= 0.0.0.9000), rmarkdown, rstudioapi, S7, @@ -57,3 +59,6 @@ Config/testthat/start-first: watcher, parallel* Encoding: UTF-8 Roxygen: list(markdown = TRUE, r6 = FALSE) RoxygenNote: 7.3.2.9000 +Remotes: + r-lib/otel, + r-lib/otelsdk diff --git a/NAMESPACE b/NAMESPACE index 634b60815..67296d30a 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -40,6 +40,7 @@ export(ListReporter) export(LocationReporter) export(MinimalReporter) export(MultiReporter) +export(OpenTelemetryReporter) export(ParallelProgressReporter) export(ProgressReporter) export(RStudioReporter) diff --git a/R/reporter-otel.R b/R/reporter-otel.R new file mode 100644 index 000000000..263e432de --- /dev/null +++ b/R/reporter-otel.R @@ -0,0 +1,246 @@ +#' OpenTelemetry reporter: traces for test results +#' +#' A variant of the Check reporter that also emits OpenTelemetry traces for +#' tests. Span attributes are drawn from [the semantic conventions for +#' tests](https://opentelemetry.io/docs/specs/semconv/registry/attributes/test/). +#' +#' @export +#' @family reporters +OpenTelemetryReporter <- R6::R6Class( + "OpenTelemetryReporter", + inherit = CheckReporter, + public = list( + tracer = NULL, + sessions = NULL, + suite_spans = NULL, + test_spans = NULL, + current_file = NULL, + + #' @param pkg A path to an R package, by default the one in the current + #' directory. + #' @param tracer An \pkg{otel} tracer, or `NULL` to use the default tracer. + initialize = function(pkg = ".", tracer = NULL, ..., call = caller_env()) { + check_installed("otel", "for emitting Open Telemetry traces", call = call) + set_pkg_resource_attributes(pkg) + self$tracer <- tracer %||% otel::get_tracer("testthat") + self$sessions <- new_environment() + self$suite_spans <- new_environment() + self$test_spans <- new_environment() + super$initialize(...) + }, + + start_file = function(file) { + # Track the current file so we can set it as an attribute on spans. + self$current_file <- file.path("tests/testthat", file) + context_start_file(file) + }, + + end_file = function(file) { + self$current_file <- NULL + }, + + start_context = function(context) { + # In order to handle concurrency issues with parallel tests, we maintain + # an otel session for each context and switch in and out of it as needed. + session <- self$tracer$start_session() + env_poke(self$sessions, context, session) + on.exit(self$tracer$deactivate_session()) + + span <- self$tracer$start_span( + name = "test_suite", + attributes = compact(list( + "test.suite.name" = context, + "code.filepath" = self$current_file + )), + scope = NULL + ) + env_poke(self$suite_spans, context, span) + }, + + end_context = function(context) { + span <- env_get(self$suite_spans, context) + span$end() + env_unbind(self$suite_spans, context) + + # Clean up the session. + session <- env_get(self$sessions, context) + self$tracer$finish_session(session) + env_unbind(self$sessions, context) + }, + + start_test = function(context, test) { + if (is.null(context)) { + # It seems like this can happen when running tests with a filter. + context <- names(self$sessions)[1] + } + + # Ensure we start test spans (and any spans started by functions within + # that test) in the context's session. + session <- env_get(self$sessions, context) + self$tracer$activate_session(session) + + key <- paste(context, test, sep = "|") + parent <- env_get(self$suite_spans, context) + span <- self$tracer$start_span( + name = "test_case", + attributes = list("test.case.name" = test), + options = list(parent = parent), + scope = NULL + ) + env_poke(self$test_spans, key, span) + }, + + end_test = function(context, test) { + # Deactivate the context's session before starting the next test (which + # might have a different one). + on.exit(self$tracer$deactivate_session()) + + context <- context %||% names(self$sessions)[1] + key <- paste(context, test, sep = "|") + span <- env_get(self$test_spans, key) + if (span$is_recording() && !span$status_set) { + # If the span's status hasn't been set, we assume the test passed. + span$set_status("ok") + span$set_attribute("test.case.result.status", "pass") + } + span$end() + env_unbind(self$test_spans, key) + }, + + add_result = function(context, test, result) { + if (expectation_broken(result) || expectation_skip(result)) { + context <- context %||% names(self$sessions)[1] + key <- paste(context, test, sep = "|") + span <- env_get(self$test_spans, key) + if (!span$is_recording()) { + return(super$add_result(context, test, result)) + } + + # Extract source references, if possible. + filename <- NULL + line <- NULL + column <- NULL + if (inherits(result$srcref, "srcref")) { + filename <- attr(result$srcref, "srcfile")$filename + line <- result$srcref[1] + column <- result$srcref[2] + } + attributes <- compact(list( + "code.filepath" = file.path("tests/testthat", filename), + "code.lineno" = line, + "code.column" = column + )) + + if (expectation_broken(result)) { + # Record error or failure expectations as exceptions on the test span. + span$record_exception(result, attributes = attributes) + # Mark the span as having errored. This is also what + # pytest-opentelemetry does. + span$set_status("error") + span$set_attribute("test.case.result.status", "fail") + } else if (expectation_skip(result)) { + # Record a special "skipped" event for skip expectations. + span$add_event("test_skipped", attributes = attributes) + span$set_status("unset") + span$set_attribute("test.case.result.status", "skipped") + } + } + super$add_result(context, test, result) + } + ) +) + +set_pkg_resource_attributes <- function(pkg = ".") { + attributes <- get_pkg_resource_attributes(pkg) + if (is.null(attributes)) { + return() + } + set_resource_attributes(.attributes = attributes) +} + +get_pkg_resource_attributes <- function(pkg = ".") { + # Try to detect when we are testing a package. + if (!env_var_is_true("NOT_CRAN")) { + return(NULL) + } + # Use what we know about the package to set some resource attributes. + desc <- pkgload::pkg_desc(pkg) + attributes <- list( + "service.name" = desc$get_field("Package"), + "service.version" = desc$get_version(), + "vcs.repository.url.full" = get_repo_url(), + "vcs.repository.ref.revision" = get_git_revision() + ) + # Existing environment variables take precedence. + from_env <- get_resource_attributes() + utils::modifyList(attributes, from_env) +} + +get_repo_url <- function(pkg = ".") { + # Default to using the Github Actions context, if available. + if (nchar(repo <- Sys.getenv("GITHUB_REPOSITORY")) != 0) { + return(paste0(Sys.getenv("GITHUB_SERVER_URL"), "/", repo)) + } + # Otherwise check if the package has a GitHub URL in its DESCRIPTION file. + desc <- pkgload::pkg_desc(pkg) + github_urls <- startsWith(desc$get_urls(), "https://github.com") + if (any(github_urls)) { + return(desc$get_urls()[github_urls][1]) + } + NULL +} + +get_git_revision <- function() { + # Default to using the Github Actions context, if available. + if (nchar(revision <- Sys.getenv("GITHUB_SHA")) != 0) { + return(revision) + } + tryCatch( + system2( + "git", + c("rev-parse", "HEAD"), + stdout = TRUE, + stderr = TRUE + )[1], + error = function(e) NULL + ) +} + +get_resource_attributes <- function( + env = Sys.getenv("OTEL_RESOURCE_ATTRIBUTES") +) { + if (nchar(env) == 0) { + return(list()) + } + # Split the attributes by comma and then by equals sign. + attrs <- strsplit(env, ",", fixed = TRUE)[[1]] + split <- strsplit(attrs, "=", fixed = TRUE) + out <- structure( + vector("list", length(split)), + .Names = character(length(split)) + ) + for (i in seq_along(split)) { + x <- split[[i]] + if (length(x) != 2) { + cli::cli_abort( + "Invalid {.env OTEL_RESOURCE_ATTRIBUTES} format: {.str {env}}", + .internal = TRUE + ) + } + out[[i]] <- x[2] + names(out)[i] <- x[1] + } + out +} + +set_resource_attributes <- function(..., .attributes = list()) { + attrs <- utils::modifyList(list(...), .attributes) + # Special handling for service.name, which isn't picked up by the SDK unless + # it's set in the dedicated environment variable. + if (!is.null(attrs["service.name"])) { + Sys.setenv(OTEL_SERVICE_NAME = attrs[["service.name"]]) + } + attrs <- vapply(attrs, format, character(1L)) + formatted <- paste(names(attrs), attrs, sep = "=", collapse = ",") + Sys.setenv(OTEL_RESOURCE_ATTRIBUTES = formatted) +} diff --git a/man/CheckReporter.Rd b/man/CheckReporter.Rd index a32c58c3d..b6f07a81f 100644 --- a/man/CheckReporter.Rd +++ b/man/CheckReporter.Rd @@ -16,6 +16,7 @@ Other reporters: \code{\link{LocationReporter}}, \code{\link{MinimalReporter}}, \code{\link{MultiReporter}}, +\code{\link{OpenTelemetryReporter}}, \code{\link{ProgressReporter}}, \code{\link{RStudioReporter}}, \code{\link{Reporter}}, diff --git a/man/DebugReporter.Rd b/man/DebugReporter.Rd index 2ad10ba08..24756d500 100644 --- a/man/DebugReporter.Rd +++ b/man/DebugReporter.Rd @@ -16,6 +16,7 @@ Other reporters: \code{\link{LocationReporter}}, \code{\link{MinimalReporter}}, \code{\link{MultiReporter}}, +\code{\link{OpenTelemetryReporter}}, \code{\link{ProgressReporter}}, \code{\link{RStudioReporter}}, \code{\link{Reporter}}, diff --git a/man/FailReporter.Rd b/man/FailReporter.Rd index 1b9d0d2a7..2ea4c7f66 100644 --- a/man/FailReporter.Rd +++ b/man/FailReporter.Rd @@ -17,6 +17,7 @@ Other reporters: \code{\link{LocationReporter}}, \code{\link{MinimalReporter}}, \code{\link{MultiReporter}}, +\code{\link{OpenTelemetryReporter}}, \code{\link{ProgressReporter}}, \code{\link{RStudioReporter}}, \code{\link{Reporter}}, diff --git a/man/JunitReporter.Rd b/man/JunitReporter.Rd index c822771ac..7bd4c6391 100644 --- a/man/JunitReporter.Rd +++ b/man/JunitReporter.Rd @@ -31,6 +31,7 @@ Other reporters: \code{\link{LocationReporter}}, \code{\link{MinimalReporter}}, \code{\link{MultiReporter}}, +\code{\link{OpenTelemetryReporter}}, \code{\link{ProgressReporter}}, \code{\link{RStudioReporter}}, \code{\link{Reporter}}, diff --git a/man/ListReporter.Rd b/man/ListReporter.Rd index dbcb62f77..425727ed2 100644 --- a/man/ListReporter.Rd +++ b/man/ListReporter.Rd @@ -17,6 +17,7 @@ Other reporters: \code{\link{LocationReporter}}, \code{\link{MinimalReporter}}, \code{\link{MultiReporter}}, +\code{\link{OpenTelemetryReporter}}, \code{\link{ProgressReporter}}, \code{\link{RStudioReporter}}, \code{\link{Reporter}}, diff --git a/man/LocationReporter.Rd b/man/LocationReporter.Rd index f0b5d86ca..06d33b894 100644 --- a/man/LocationReporter.Rd +++ b/man/LocationReporter.Rd @@ -17,6 +17,7 @@ Other reporters: \code{\link{ListReporter}}, \code{\link{MinimalReporter}}, \code{\link{MultiReporter}}, +\code{\link{OpenTelemetryReporter}}, \code{\link{ProgressReporter}}, \code{\link{RStudioReporter}}, \code{\link{Reporter}}, diff --git a/man/MinimalReporter.Rd b/man/MinimalReporter.Rd index 5365d7e46..8ab05d065 100644 --- a/man/MinimalReporter.Rd +++ b/man/MinimalReporter.Rd @@ -18,6 +18,7 @@ Other reporters: \code{\link{ListReporter}}, \code{\link{LocationReporter}}, \code{\link{MultiReporter}}, +\code{\link{OpenTelemetryReporter}}, \code{\link{ProgressReporter}}, \code{\link{RStudioReporter}}, \code{\link{Reporter}}, diff --git a/man/MultiReporter.Rd b/man/MultiReporter.Rd index bb5d9cb34..7836f74e5 100644 --- a/man/MultiReporter.Rd +++ b/man/MultiReporter.Rd @@ -16,6 +16,7 @@ Other reporters: \code{\link{ListReporter}}, \code{\link{LocationReporter}}, \code{\link{MinimalReporter}}, +\code{\link{OpenTelemetryReporter}}, \code{\link{ProgressReporter}}, \code{\link{RStudioReporter}}, \code{\link{Reporter}}, diff --git a/man/OpenTelemetryReporter.Rd b/man/OpenTelemetryReporter.Rd new file mode 100644 index 000000000..4197973a3 --- /dev/null +++ b/man/OpenTelemetryReporter.Rd @@ -0,0 +1,35 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/reporter-otel.R +\name{OpenTelemetryReporter} +\alias{OpenTelemetryReporter} +\title{OpenTelemetry reporter: traces for test results} +\arguments{ +\item{pkg}{A path to an R package, by default the one in the current +directory.} + +\item{tracer}{An \pkg{otel} tracer, or \code{NULL} to use the default tracer.} +} +\description{ +A variant of the Check reporter that also emits OpenTelemetry traces for +tests. Span attributes are drawn from \href{https://opentelemetry.io/docs/specs/semconv/registry/attributes/test/}{the semantic conventions for tests}. +} +\seealso{ +Other reporters: +\code{\link{CheckReporter}}, +\code{\link{DebugReporter}}, +\code{\link{FailReporter}}, +\code{\link{JunitReporter}}, +\code{\link{ListReporter}}, +\code{\link{LocationReporter}}, +\code{\link{MinimalReporter}}, +\code{\link{MultiReporter}}, +\code{\link{ProgressReporter}}, +\code{\link{RStudioReporter}}, +\code{\link{Reporter}}, +\code{\link{SilentReporter}}, +\code{\link{StopReporter}}, +\code{\link{SummaryReporter}}, +\code{\link{TapReporter}}, +\code{\link{TeamcityReporter}} +} +\concept{reporters} diff --git a/man/ProgressReporter.Rd b/man/ProgressReporter.Rd index be63cf3b4..873cb8f10 100644 --- a/man/ProgressReporter.Rd +++ b/man/ProgressReporter.Rd @@ -28,6 +28,7 @@ Other reporters: \code{\link{LocationReporter}}, \code{\link{MinimalReporter}}, \code{\link{MultiReporter}}, +\code{\link{OpenTelemetryReporter}}, \code{\link{RStudioReporter}}, \code{\link{Reporter}}, \code{\link{SilentReporter}}, diff --git a/man/RStudioReporter.Rd b/man/RStudioReporter.Rd index a25c39f9d..8365152fe 100644 --- a/man/RStudioReporter.Rd +++ b/man/RStudioReporter.Rd @@ -17,6 +17,7 @@ Other reporters: \code{\link{LocationReporter}}, \code{\link{MinimalReporter}}, \code{\link{MultiReporter}}, +\code{\link{OpenTelemetryReporter}}, \code{\link{ProgressReporter}}, \code{\link{Reporter}}, \code{\link{SilentReporter}}, diff --git a/man/Reporter.Rd b/man/Reporter.Rd index eeb7a0ab2..d002d9c8b 100644 --- a/man/Reporter.Rd +++ b/man/Reporter.Rd @@ -33,6 +33,7 @@ Other reporters: \code{\link{LocationReporter}}, \code{\link{MinimalReporter}}, \code{\link{MultiReporter}}, +\code{\link{OpenTelemetryReporter}}, \code{\link{ProgressReporter}}, \code{\link{RStudioReporter}}, \code{\link{SilentReporter}}, diff --git a/man/SilentReporter.Rd b/man/SilentReporter.Rd index dde2f9641..56373c99d 100644 --- a/man/SilentReporter.Rd +++ b/man/SilentReporter.Rd @@ -19,6 +19,7 @@ Other reporters: \code{\link{LocationReporter}}, \code{\link{MinimalReporter}}, \code{\link{MultiReporter}}, +\code{\link{OpenTelemetryReporter}}, \code{\link{ProgressReporter}}, \code{\link{RStudioReporter}}, \code{\link{Reporter}}, diff --git a/man/StopReporter.Rd b/man/StopReporter.Rd index 53d5f41f5..ea9777ddc 100644 --- a/man/StopReporter.Rd +++ b/man/StopReporter.Rd @@ -23,6 +23,7 @@ Other reporters: \code{\link{LocationReporter}}, \code{\link{MinimalReporter}}, \code{\link{MultiReporter}}, +\code{\link{OpenTelemetryReporter}}, \code{\link{ProgressReporter}}, \code{\link{RStudioReporter}}, \code{\link{Reporter}}, diff --git a/man/SummaryReporter.Rd b/man/SummaryReporter.Rd index 149182127..8e71a0650 100644 --- a/man/SummaryReporter.Rd +++ b/man/SummaryReporter.Rd @@ -26,6 +26,7 @@ Other reporters: \code{\link{LocationReporter}}, \code{\link{MinimalReporter}}, \code{\link{MultiReporter}}, +\code{\link{OpenTelemetryReporter}}, \code{\link{ProgressReporter}}, \code{\link{RStudioReporter}}, \code{\link{Reporter}}, diff --git a/man/TapReporter.Rd b/man/TapReporter.Rd index 6c8a680e0..f8fc28df7 100644 --- a/man/TapReporter.Rd +++ b/man/TapReporter.Rd @@ -18,6 +18,7 @@ Other reporters: \code{\link{LocationReporter}}, \code{\link{MinimalReporter}}, \code{\link{MultiReporter}}, +\code{\link{OpenTelemetryReporter}}, \code{\link{ProgressReporter}}, \code{\link{RStudioReporter}}, \code{\link{Reporter}}, diff --git a/man/TeamcityReporter.Rd b/man/TeamcityReporter.Rd index f2b2b79de..3dd3b491d 100644 --- a/man/TeamcityReporter.Rd +++ b/man/TeamcityReporter.Rd @@ -18,6 +18,7 @@ Other reporters: \code{\link{LocationReporter}}, \code{\link{MinimalReporter}}, \code{\link{MultiReporter}}, +\code{\link{OpenTelemetryReporter}}, \code{\link{ProgressReporter}}, \code{\link{RStudioReporter}}, \code{\link{Reporter}}, diff --git a/tests/testthat/test-reporter-otel.R b/tests/testthat/test-reporter-otel.R new file mode 100644 index 000000000..9f07f3568 --- /dev/null +++ b/tests/testthat/test-reporter-otel.R @@ -0,0 +1,112 @@ +test_that("we skip setting attributes for non-package tests", { + withr::local_envvar(NOT_CRAN = NA) + expect_null(get_pkg_resource_attributes()) +}) + +test_that("we detect package resources attributes correctly", { + withr::local_envvar( + NOT_CRAN = "true", + OTEL_RESOURCE_ATTRIBUTES = NA, + GITHUB_REPOSITORY = NA + ) + + # Since we're in testthat, we should get attributes for the testthat package. + attrs <- get_pkg_resource_attributes(test_path("..")) + expect_type(attrs, "list") + expect_equal(attrs$service.name, "testthat") + expect_match(as.character(attrs$service.version), "^[0-9]+\\.[0-9]+\\.[0-9]+") + expect_equal( + attrs$vcs.repository.url.full, + "https://github.com/r-lib/testthat" + ) +}) + +test_that("package resources attributes respect environment variable overrides", { + withr::local_envvar( + NOT_CRAN = "true", + OTEL_RESOURCE_ATTRIBUTES = "service.name=override,service.version=1.2.3" + ) + + attrs <- get_pkg_resource_attributes(test_path("..")) + expect_equal(attrs$service.name, "override") + expect_equal(attrs$service.version, "1.2.3") + # Other attributes should still be present + expect_equal( + attrs$vcs.repository.url.full, + "https://github.com/r-lib/testthat" + ) +}) + +test_that("we detect resource attributes from GitHub Actions", { + withr::local_envvar( + GITHUB_REPOSITORY = "r-lib/testthat-otel", + GITHUB_SERVER_URL = "https://github.com", + GITHUB_SHA = "test" + ) + + expect_equal(get_repo_url(), "https://github.com/r-lib/testthat-otel") + expect_equal(get_git_revision(), "test") +}) + +test_that("the reporter generates the expected spans", { + skip_if_not_installed("otelsdk") + + withr::local_envvar(NOT_CRAN = "true") + + spans <- otelsdk::with_otel_record({ + tracer <- otel::get_tracer("test") + reporter <- OpenTelemetryReporter$new(tracer = tracer) + with_reporter(reporter, { + test_one_file(test_path("reporters/tests.R")) + }) + })[["traces"]] + + expect_length(spans, 14) + + # TODO: Ensure that the test_suite and test_case spans have the correct parent. + + # Ensure we generate "test_suite" spans for each context. + suite_spans <- Filter(function(x) x$name == "test_suite", spans) + expect_length(suite_spans, 6) + expect_equal( + sapply(suite_spans, function(x) x$attributes$test.suite.name), + c("reporters/tests", "Successes", "Failures", "Errors", "Skips", "Warnings") + ) + + # Ensure that the filepath is set correctly on the spans. + expect_equal( + sapply(suite_spans, function(x) x$attributes$code.filepath), + rep("tests/testthat/reporters/tests.R", 6) + ) + + # Ensure we generate "test_case" spans for each test. + test_spans <- Filter(function(x) x$name == "test_case", spans) + expect_length(test_spans, 8) + expect_equal( + sapply(test_spans, function(x) x$attributes$test.case.name), + c( + "Success", + "Failure:1", + "Failure:2a", + "Error:1", + "errors get tracebacks", + "explicit skips are reported", + "empty tests are implicitly skipped", + "warnings get backtraces" + ) + ) + + # Ensure that tests that failed or errored have the expected status. + failed_spans <- Filter(function(x) x$status == "error", spans) + expect_length(failed_spans, 4) + + # And that they record an exception with the correct file path. + expect_true(all(sapply( + failed_spans, + function(x) + length(x$events) > 0 && + x$events[[1]]$name == "exception" && + x$events[[1]]$attributes$code.filepath == + "tests/testthat/reporters/tests.R" + ))) +})