Provide access to TileDB shared libraries in downstream packages (#782)

New helper function to provide access to `libtiledb` used by tiledb-r for downstream packages. This builds off the prototype code @LTLA provided

New functions provided:
- `.pkg_config()`: provide compiler flags for linking to the version of `libtiledb` that tiledb-r either vendors or links to at the system level
- `.core_info()`: provides a vector with the `libtiledb` version and install type (eg. `"vendored"`, `"system"`)
- `.core_hash()`: provides the same information as `.core_info()`, but as an MD5 hash

Also includes a mini-test suite for CI that tests this functionality with both a system install and vendored install of `libtiledb`

[SC-59185](https://app.shortcut.com/tiledb-inc/story/59185/)

resolves #780

* Update windows install to include libtiledb source

* Clean up comments

* Better Windows support

* Add tests for .pkg_config()

* Add utility functions to check version of libtiledb a package was built
with vs version package is loaded with

* Rename `R/Init.R` to `R/zzz.R`

* Add quoting for Windows

* Update changelog
Bump develop version

Author: mojaveazure

.github/workflows/pkgconfig.yaml

@@ -0,0 +1,73 @@
+on:
+  push:
+  pull_request:
+
+name: pkgconfig
+
+permissions: read-all
+
+jobs:
+  pkgconfig:
+    runs-on: ${{ matrix.os }}
+    name: ${{ matrix.os }} (r-${{ matrix.r }}) (${{ matrix.vendored }} libtiledb)
+
+    strategy:
+      fail-fast: false
+      matrix:
+        os:
+          - ubuntu-latest
+        r:
+          - release
+          # - devel
+        vendored:
+          - 'vendored'
+          - 'system'
+        include:
+          - os: ubuntu-latest
+            use-public-rspm: true
+          # - os: ubuntu-latest
+          #   r: devel
+          #   http-user-agent: 'release'
+        exclude:
+          - os: macos-latest
+            r: devel
+
+    env:
+      GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }}
+      R_KEEP_PKG_SOURCE: yes
+      _R_CHECK_FORCE_SUGGESTS_: "FALSE"
+      _R_TILEDB_LIBTILEDB_: ${{ matrix.vendored }}
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: r-lib/actions/setup-pandoc@v2
+
+      - uses: r-lib/actions/setup-r@v2
+        with:
+          r-version: ${{ matrix.r }}
+          http-user-agent: ${{ matrix.http-user-agent }}
+          use-public-rspm: ${{ matrix.use-public-rspm }}
+
+      - name: Setup libtiledb
+        if: ${{ matrix.vendored == 'system' }}
+        run: |
+          VERSION=$(grep version tools/tiledbVersion.txt  | cut -f 2 -d ':' | tr -d '[:space:]')
+          SHA=$(grep sha tools/tiledbVersion.txt  | cut -f 2 -d ':' | tr -d '[:space:]')
+          URL="https://github.com/TileDB-Inc/TileDB/releases/download/${VERSION}/tiledb-linux-x86_64-${VERSION}-${SHA}.tar.gz"
+          mkdir -vp libtiledb
+          cd libtiledb
+          wget -O libtiledb.tar.gz ${URL}
+          tar -xvzf libtiledb.tar.gz
+          /usr/bin/sudo cp -Rv include/* /usr/local/include/
+          /usr/bin/sudo cp -Rv lib/* /usr/local/lib/
+          cd ..
+          rm -rfv libtiledb
+          sudo ldconfig
+
+      - uses: r-lib/actions/setup-r-dependencies@v2
+        with:
+          extra-packages: local::.
+
+      - name: Test tiledb::.pkg_config()
+        run: Rscript tests/pkgconfig-test.R

DESCRIPTION

@@ -1,6 +1,6 @@
 Package: tiledb
 Type: Package
-Version: 0.30.2.3
+Version: 0.30.2.4
 Title: Modern Database Engine for Complex Data Based on Multi-Dimensional Arrays
 Authors@R: c(
   person("TileDB, Inc.", role = c("aut", "cph")),
@@ -24,7 +24,13 @@ SystemRequirements: A C++17 compiler is required; on macOS compilation version 1
   build selected); on x86_64 and M1 platforms pre-built TileDB Embedded libraries
   are available at GitHub and are used if no TileDB installation is detected, and
   no other option to build or download was specified by the user.
-Imports: methods, Rcpp (>= 1.0.8), nanotime, spdl, nanoarrow
+Imports: 
+    methods,
+    Rcpp (>= 1.0.8),
+    nanotime,
+    spdl,
+    nanoarrow,
+    tools
 LinkingTo: Rcpp, RcppInt64, nanoarrow
 Suggests: tinytest, simplermarkdown, curl, bit64, Matrix, palmerpenguins, nycflights13, data.table, tibble, arrow
 VignetteBuilder: simplermarkdown

NAMESPACE

@@ -24,6 +24,9 @@ export("return_as<-")
 export("selected_points<-")
 export("selected_ranges<-")
 export("strings_as_factors<-")
+export(.core_hash)
+export(.core_info)
+export(.pkg_config)
 export(allows_dups)
 export(array_consolidate)
 export(array_vacuum)

NEWS.md

@@ -6,6 +6,7 @@
 * Support parentheses in query conditions
 * memory alloc: Accomodate zero buffer size estimate v2
 * Apply `styler::style_pkg()`
+* Expose include/linking flags for re-using `libtiledb` in downstream packages
 
 # tiledb 0.30.2
 

R/Version.R

@@ -41,3 +41,170 @@ tiledb_version <- function(compact = FALSE) {
     libtiledb_version()
   }
 }
+
+#' \code{libtiledb} Information
+#'
+#' Get version and install information of the core \code{libtiledb} install
+#'
+#' @section Checking the \code{libtiledb} information in downstream packages:
+#' These functions are designed to make it easy to test if the core
+#' \code{libtiledb} install has changed. This is accomplished by adding a
+#' build-time constant to cache the version of \code{libtiledb} was built with.
+#' For example, in \code{zzz.R}, put the following line to cache the
+#' \code{libtiledb} information during package build
+#' \preformatted{
+#' .built_with <- list(libtiledb = tiledb::.core_hash())
+#' }
+#' Then, in the \link[base:ns-hooks]{load hook}, add the following check
+#' \preformatted{
+#' .onLoad <- function(libname, pkgname) {
+#'   if (.built_with$libtiledb != tiledb::.core_hash()) {
+#'     warning("Core libtiledb has changed, please reinstall ", pkgname)
+#'   }
+#' }
+#' }
+#' This will throw a warning if \pkg{tiledb}, and therefore \code{libtiledb},
+#' has changed between downstream package install and load
+#'
+#' @return \code{.core_info()}: A named character vector with the following entries:
+#' \itemize{
+#'  \item \dQuote{\code{version}}: \code{libtiledb} version
+#'  \item \dQuote{\code{libtype}}: type of \code{libtiledb} install; will be one
+#'  of \dQuote{\code{vendored}}, \dQuote{\code{system}}, or \dQuote{\code{unknown}}
+#' }
+#'
+#' @keywords internal
+#'
+#' @export
+#'
+#' @examples
+#' .core_info()
+#'
+.core_info <- function() {
+  info <- c(
+    version = as.character(tiledb_version(TRUE)),
+    libtype = character(1L)
+  )
+  lib <- system.file(
+    "tiledb",
+    package = .pkgenv$pkgname,
+    lib.loc = .pkgenv$libname
+  )
+  if (nzchar(lib)) {
+    info['libtype'] <- 'vendored'
+    return(info)
+  }
+  if (nzchar(pkgconfig <- Sys.which("pkg-config"))) {
+    if (!system2(pkgconfig, args = c("--exists", "tiledb"))) {
+      info['libtype'] <- 'system'
+      return(info)
+    }
+  }
+  info['libtype'] <- 'unknown'
+  return(info)
+}
+
+#' @return \code{.core_hash()}: The \link[tools:md5sum]{MD5 hash} of the core info
+#'
+#' @rdname dot-core_info
+#'
+#' @export
+#'
+#' @examples
+#' .core_hash()
+#'
+.core_hash <- function() {
+  tmp <- tempfile()
+  on.exit(file.remove(tmp), add = TRUE, after = FALSE)
+  info <- .core_info()
+  writeLines(paste(names(info), info, sep = ':\t', collapse = '\n'), con = tmp)
+  return(unname(tools::md5sum(tmp)))
+}
+
+#' Compiler Arguments for Using \code{libtiledb}
+#'
+#' Get compiler flags for using the core \code{libtiledb} install
+#' used by \pkg{tiledb}
+#'
+#' @param opt A single character value with the package configuration variable
+#' to fetch; choose from
+#' \itemize{
+#'  \item \dQuote{\code{PKG_CXX_FLAGS}}: compiler flags for \code{libtiledb}
+#'  \item \dQuote{\code{PKG_CXX_LIBS}}: linking flags for \code{libtiledb}
+#' }
+#'
+#' @return A single string containing either the include directories or linking
+#' directories for \code{libtiledb}
+#'
+#' @keywords internal
+#'
+#' @export
+#'
+#' @examples
+#' .pkg_config()
+#' .pkg_config("PKG_CXX_LIBS")
+#'
+.pkg_config <- function(opt = c("PKG_CXX_FLAGS", "PKG_CXX_LIBS")) {
+  opt <- match.arg(opt)
+  lib <- system.file(
+    "tiledb",
+    package = .pkgenv$pkgname,
+    lib.loc = .pkgenv$libname
+  )
+  if (nzchar(lib)) {
+    pkgdir <- system.file(package = .pkgenv$pkgname, lib.loc = .pkgenv$libname)
+    return(switch(
+      EXPR = opt,
+      PKG_CXX_FLAGS = switch(
+        EXPR = .Platform$OS.type,
+        # Adapted from Makevars.win, which includes libdir/include/tiledb in
+        # addition to libdir/include and pkgdir/include
+        windows = sprintf(
+          "-I%s/include -I%s/include -I%s/include/tiledb",
+          shQuote(pkgdir, type = "cmd"),
+          shQuote(lib, type = "cmd"),
+          shQuote(lib, type = "cmd")
+        ),
+        sprintf("-I%s/include -I%s/include", pkgdir, lib)
+      ),
+      PKG_CXX_LIBS = switch(
+        EXPR = .Platform$OS.type,
+        # rwinlib-tiledb is structured slightly differently than libtiledb for
+        # Unix-alikes; R 4.2 and higher require ucrt
+        windows = {
+          arch <- .Platform$r_arch
+          libs <- as.vector(vapply(
+            c(pkgdir, lib),
+            FUN = \(x) c(
+              sprintf("%s/lib/%s", shQuote(x, type = "cmd"), arch),
+              ifelse(
+                test = getRversion() > '4.2.0',
+                yes = sprintf("%s/lib/%s-ucrt", shQuote(x, type = "cmd"), arch),
+                no = ""
+              )
+            ),
+            FUN.VALUE = character(2L),
+            USE.NAMES = FALSE
+          ))
+          paste('-ltiledb', paste0('-L', Filter(dir.exists, libs), collapse = ' '))
+        },
+        sprintf("-ltiledb -L%s/lib -L%s/lib", pkgdir, lib)
+      )
+    ))
+  }
+  if (nzchar(pkgconfig <- Sys.which("pkg-config"))) {
+    if (!system2(pkgconfig, args = c("--exists", "tiledb"))) {
+      flag <- switch(
+        EXPR = opt,
+        PKG_CXX_FLAGS = "--cflags",
+        PKG_CXX_LIBS = "--libs"
+      )
+      return(trimws(system2(pkgconfig, args = c(flag, "tiledb"), stdout = TRUE)))
+    }
+  }
+  return(switch(
+    EXPR = opt,
+    PKG_CXX_FLAGS = "",
+    PKG_CXX_LIBS = "-ltiledb"
+  ))
+}

R/Init.R R/zzz.RR/Init.R renamed to R/zzz.R

@@ -53,6 +53,10 @@
   ## set a preference for allocation size defaults
   .pkgenv[["allocation_size"]] <- load_allocation_size_preference()
 
+  # cache package name and path
+  .pkgenv[["pkgname"]] <- pkgname
+  .pkgenv[["libname"]] <- libname
+
   ## call setter for Rcpp plugin support
   .set_compile_link_options()
 
@@ -121,7 +125,6 @@ inlineCxxPlugin <- function(...) {
   }
 }
 
-
 #' @importFrom utils read.table
 .getLinuxFlavor <- function() {
   res <- NA_character_

TileDB-R.Rproj

@@ -12,6 +12,9 @@ Encoding: UTF-8
 RnwWeave: Sweave
 LaTeX: pdfLaTeX
 
+AutoAppendNewline: Yes
+StripTrailingWhitespace: Yes
+
 BuildType: Package
 PackageUseDevtools: Yes
 PackageInstallArgs: --no-multiarch --with-keep.source --install-tests

inst/pkgconfigtest/test_core_info.R

@@ -0,0 +1,61 @@
+
+if (!at_home()) {
+  exit_file("'.pkg_config()' tests run at home")
+}
+
+libtype <- Sys.getenv("_R_TILEDB_LIBTILEDB_")
+if (nzchar(libtype)) {
+  expect_true(
+    libtype %in% c("system", "vendored"),
+    info = "`Sys.getenv('_R_TILEDB_LIBTILEDB_')` must be 'system' or 'vendored'"
+  )
+}
+
+# Check .core_info()
+expect_inherits(
+  info <- .core_info(),
+  class = "character",
+  info = "'.core_info()' returns a character vector"
+)
+expect_length(info, length = 2L, info = "'.core_info()' returns a two-length vector")
+expect_identical(
+  names(info),
+  target = c("version", "libtype"),
+  info = "'.core_info()' returns a named vector with names of 'version' and 'libtype'"
+)
+expect_identical(
+  info[["version"]],
+  target = as.character(tiledb_version(compact = TRUE)),
+  info = "'.core_info()' returns the correct core version"
+)
+libtarget <- ifelse(nzchar(libtype), yes = libtype, no = "unknown")
+expect_identical(
+  info[["libtype"]],
+  target = libtarget,
+  info = sprintf("'.core_info()' returns the correct libtype ('%s')", libtarget)
+)
+
+# Check .core_hash()
+tmpfile <- tempfile(tmpdir = tempdir(check = TRUE))
+writeLines(
+  sprintf(
+    "version:\t%s\nlibtype:\t%s",
+    as.character(tiledb_version(compact = TRUE)),
+    ifelse(nzchar(libtype), yes = libtype, no = "unknown")
+  ),
+  con = tmpfile
+)
+target <- unname(tools::md5sum(tmpfile))
+hash <- .core_hash()
+expect_inherits(
+  hash,
+  class = "character",
+  info = "'.core_hash()' returns a character value"
+)
+expect_length(hash, length = 1L, info = "'.core_hash()' returns a single value")
+expect_null(names(hash), info = "'.core_hash()' returns an unnamed value")
+expect_identical(
+  hash,
+  target = target,
+  info = "'.core_hash()' returns the correct hash value"
+)