Merge branch 'master' of github.com:stemangiola/CuratedAtlasQueryR

Conflicts:
	DESCRIPTION

Author: multimeric

.github/workflows/check-bioc.yml

@@ -38,7 +38,8 @@ env:
   run_covr: 'false'
   run_pkgdown: 'true'
   has_RUnit: 'false'
-  cache-version: 'cache-v2'
+  cache-version: 'cache-v1'
+  run_docker: 'false'
 
 jobs:
   build-check:
@@ -48,12 +49,14 @@ jobs:
     ## Environment variables unique to this job.
 
     strategy:
-      fail-fast: true
+      fail-fast: false
       matrix:
         config:
-          - { os: ubuntu-latest, r: '4.2', bioc: '3.16', cont: "bioconductor/bioconductor_docker:RELEASE_3_16", rspm: "https://packagemanager.rstudio.com/cran/__linux__/focal/latest" }
-          - { os: macOS-latest, r: '4.2', bioc: '3.16'}
-          - { os: windows-latest, r: '4.2', bioc: '3.16'}
+          - { os: ubuntu-latest, r: '4.3', bioc: '3.19', cont: "bioconductor/bioconductor_docker:devel", rspm: "https://packagemanager.rstudio.com/cran/__linux__/jammy/latest" }
+          - { os: macOS-latest, r: '4.3', bioc: '3.18'}
+          - { os: windows-latest, r: '4.3', bioc: '3.18'}
+          ## Check https://github.com/r-lib/actions/tree/master/examples
+          ## for examples using the http-user-agent
     env:
       R_REMOTES_NO_ERRORS_FROM_WARNINGS: true
       RSPM: ${{ matrix.config.rspm }}
@@ -76,14 +79,15 @@ jobs:
       ## https://github.com/r-lib/actions/blob/master/examples/check-standard.yaml
       ## If they update their steps, we will also need to update ours.
       - name: Checkout Repository
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
 
       ## R is already included in the Bioconductor docker images
       - name: Setup R from r-lib
         if: runner.os != 'Linux'
         uses: r-lib/actions/setup-r@v2
         with:
           r-version: ${{ matrix.config.r }}
+          http-user-agent: ${{ matrix.config.http-user-agent }}
 
       ## pandoc is already included in the Bioconductor docker images
       - name: Setup pandoc from r-lib
@@ -96,26 +100,28 @@ jobs:
           saveRDS(remotes::dev_package_deps(dependencies = TRUE), ".github/depends.Rds", version = 2)
         shell: Rscript {0}
 
-      - name: Cache R packages
+      - name: Restore R package cache
         if: "!contains(github.event.head_commit.message, '/nocache') && runner.os != 'Linux'"
-        uses: actions/cache@v2
+        uses: actions/cache@v3
         with:
           path: ${{ env.R_LIBS_USER }}
-          key: "${{ env.cache-version }}-${{ runner.os }}-biocversion-RELEASE_${{ matrix.config.bioc }}r-{{ matrix.config.r }}"
+          key: ${{ env.cache-version }}-${{ runner.os }}-biocversion-devel-r-4.3-${{ hashFiles('.github/depends.Rds') }}
+          restore-keys: ${{ env.cache-version }}-${{ runner.os }}-biocversion-devel-r-4.3-
 
       - name: Cache R packages on Linux
         if: "!contains(github.event.head_commit.message, '/nocache') && runner.os == 'Linux' "
-        uses: actions/cache@v2
+        uses: actions/cache@v3
         with:
           path: /home/runner/work/_temp/Library
-          key: "${{ env.cache-version }}-${{ runner.os }}-biocversion-RELEASE_${{ matrix.config.bioc }}r-{{ matrix.config.r }}"
+          key: ${{ env.cache-version }}-${{ runner.os }}-biocversion-devel-r-4.3-${{ hashFiles('.github/depends.Rds') }}
+          restore-keys: ${{ env.cache-version }}-${{ runner.os }}-biocversion-devel-r-4.3-
 
-      - name: Install Linux system dependencies
-        if: runner.os == 'Linux'
-        run: |
-          sysreqs=$(Rscript -e 'cat("apt-get update -y && apt-get install -y", paste(gsub("apt-get install -y ", "", remotes::system_requirements("ubuntu", "20.04")), collapse = " "))')
-          echo $sysreqs
-          sudo -s eval "$sysreqs"
+      # - name: Install Linux system dependencies
+      #   if: runner.os == 'Linux'
+      #   run: |
+      #     sysreqs=$(Rscript -e 'cat("apt-get update -y && apt-get install -y", paste(gsub("apt-get install -y ", "", remotes::system_requirements("ubuntu", "20.04")), collapse = " "))')
+      #     echo $sysreqs
+      #     sudo -s eval "$sysreqs"
 
       - name: Install macOS system dependencies
         if: matrix.config.os == 'macOS-latest'
@@ -133,10 +139,9 @@ jobs:
 
           ## For installing usethis's dependency gert
           brew install libgit2
-          
-          ## To fix x11/cairo error with tidyHeatmap/Complexheatmap here https://github.com/stemangiola/tidybulk/runs/1388237421?check_suite_focus=true#step:14:2134
-          ## Suggested here https://stackoverflow.com/questions/63648591/how-to-install-x11-before-testing-with-github-actions-for-macos
-          brew install --cask xquartz 
+
+          ## Required for tcltk
+          brew install xquartz --cask
 
       - name: Install Windows system dependencies
         if: runner.os == 'Windows'
@@ -147,12 +152,12 @@ jobs:
       - name: Install BiocManager
         run: |
           message(paste('****', Sys.time(), 'installing BiocManager ****'))
-          remotes::install_github("Bioconductor/BiocManager")
+          remotes::install_cran("BiocManager")
         shell: Rscript {0}
 
       - name: Set BiocVersion
         run: |
-          BiocManager::install(version = "${{ matrix.config.bioc }}", ask = FALSE)
+          BiocManager::install(version = "${{ matrix.config.bioc }}", ask = FALSE, force = TRUE)
         shell: Rscript {0}
 
       - name: Install dependencies pass 1
@@ -164,6 +169,10 @@ jobs:
           ## https://github.com/r-lib/remotes/issues/296
           ## Ideally, all dependencies should get installed in the first pass.
 
+          ## For running the checks
+          message(paste('****', Sys.time(), 'installing rcmdcheck and BiocCheck ****'))
+          install.packages(c("rcmdcheck", "BiocCheck"), repos = BiocManager::repositories())
+
           ## Pass #1 at installing dependencies
           message(paste('****', Sys.time(), 'pass number 1 at installing dependencies: local dependencies ****'))
           remotes::install_local(dependencies = TRUE, repos = BiocManager::repositories(), build_vignettes = FALSE, upgrade = TRUE)
@@ -174,21 +183,7 @@ jobs:
         run: |
           ## Pass #2 at installing dependencies
           message(paste('****', Sys.time(), 'pass number 2 at installing dependencies: any remaining dependencies ****'))
-          remotes::install_local(dependencies = TRUE, repos = BiocManager::repositories(), build_vignettes = FALSE, upgrade = TRUE)
-          
-          ## For running the checks
-          message(paste('****', Sys.time(), 'installing rcmdcheck and BiocCheck ****'))
-          remotes::install_cran("rcmdcheck")
-          BiocManager::install("BiocCheck")
-        shell: Rscript {0}
-        
-      # This is a hack to fix this bug: https://stackoverflow.com/q/73700130/2148718
-      # Basically the Seurat and SeuratObject binary packages are compiled against an
-      # older version of Matrix, which causes an error we can fix by re-compiling from source
-      - name: Fix Matrix on MacOS.
-        if: matrix.config.os == 'macOS-latest'
-        run: |
-            install.packages(c("Seurat", "SeuratObject"), type="source")
+          remotes::install_local(dependencies = TRUE, repos = BiocManager::repositories(), build_vignettes = TRUE, upgrade = TRUE, force = TRUE)
         shell: Rscript {0}
 
       - name: Install BiocGenerics
@@ -199,15 +194,15 @@ jobs:
         shell: Rscript {0}
 
       - name: Install covr
-        if: github.ref == 'refs/heads/master' && env.run_covr == 'true' && runner.os == 'Linux'
+        if: github.ref == 'refs/heads/devel' && env.run_covr == 'true' && runner.os == 'Linux'
         run: |
           remotes::install_cran("covr")
         shell: Rscript {0}
 
       - name: Install pkgdown
-        if: github.ref == 'refs/heads/master' && env.run_pkgdown == 'true' && runner.os == 'Linux'
+        if: github.ref == 'refs/heads/devel' && env.run_pkgdown == 'true' && runner.os == 'Linux'
         run: |
-          remotes::install_cran("pkgdown")
+          remotes::install_github("r-lib/pkgdown")
         shell: Rscript {0}
 
       - name: Session info
@@ -220,10 +215,12 @@ jobs:
       - name: Run CMD check
         env:
           _R_CHECK_CRAN_INCOMING_: false
+          DISPLAY: 99.0
         run: |
+          options(crayon.enabled = TRUE)
           rcmdcheck::rcmdcheck(
-              args = c("--no-manual", "--timings"),
-              build_args = c("--no-manual", "--no-resave-data"),
+              args = c("--no-manual", "--no-vignettes", "--timings"),
+              build_args = c("--no-manual", "--keep-empty-dirs", "--no-resave-data"),
               error_on = "warning",
               check_dir = "check"
           )
@@ -241,45 +238,101 @@ jobs:
         shell: Rscript {0}
 
       - name: Run BiocCheck
+        env:
+          DISPLAY: 99.0
         run: |
           BiocCheck::BiocCheck(
               dir('check', 'tar.gz$', full.names = TRUE),
               `quit-with-status` = TRUE,
               `no-check-R-ver` = TRUE,
-              `no-check-bioc-help` = TRUE,
-              `new-package` = TRUE
+              `no-check-bioc-help` = TRUE
           )
         shell: Rscript {0}
 
       - name: Test coverage
-        if: github.ref == 'refs/heads/master' && env.run_covr == 'true' && runner.os == 'Linux'
+        if: github.ref == 'refs/heads/devel' && env.run_covr == 'true' && runner.os == 'Linux'
         run: |
           covr::codecov()
         shell: Rscript {0}
 
       - name: Install package
-        if: github.ref == 'refs/heads/master' && env.run_pkgdown == 'true' && runner.os == 'Linux'
+        if: github.ref == 'refs/heads/devel' && env.run_pkgdown == 'true' && runner.os == 'Linux'
         run: R CMD INSTALL .
 
-      - name: Deploy package
-        if: github.ref == 'refs/heads/master' && env.run_pkgdown == 'true' && runner.os == 'Linux'
-        run: |
-          ## Temporary workaround for https://github.com/actions/checkout/issues/766
-          git config --global --add safe.directory "$GITHUB_WORKSPACE"
-          
-          git config --local user.email "[email protected]"
-          git config --local user.name "GitHub Actions"
-          Rscript -e "pkgdown::deploy_to_branch(new_process = FALSE)"
-        shell: bash {0}
+      - name: Build pkgdown site
+        if: github.ref == 'refs/heads/devel' && env.run_pkgdown == 'true' && runner.os == 'Linux'
+        run: pkgdown::build_site_github_pages(new_process = FALSE, install = FALSE)
+        shell: Rscript {0}
         ## Note that you need to run pkgdown::deploy_to_branch(new_process = FALSE)
         ## at least one locally before this will work. This creates the gh-pages
         ## branch (erasing anything you haven't version controlled!) and
         ## makes the git history recognizable by pkgdown.
 
+      - name: Install deploy dependencies
+        if: github.ref == 'refs/heads/devel' && env.run_pkgdown == 'true' && runner.os == 'Linux'
+        run: |
+          apt-get update && apt-get -y install rsync
+
+      - name: Deploy pkgdown site to GitHub pages 🚀
+        if: github.ref == 'refs/heads/devel' && env.run_pkgdown == 'true' && runner.os == 'Linux'
+        uses: JamesIves/github-pages-deploy-action@releases/v4
+        with:
+          clean: false
+          branch: gh-pages
+          folder: docs
+
       - name: Upload check results
         if: failure()
-        uses: actions/upload-artifact@v2
+        uses: actions/upload-artifact@master
         with:
-          name: ${{ runner.os }}-biocversion-RELEASE_3_15-r-4.2-results
+          name: ${{ runner.os }}-biocversion-devel-r-4.3-results
           path: check
-          
+
+
+  ## Code adapted from
+  ## https://github.com/waldronlab/cBioPortalData/blob/e0440a4445f0cc731e426363a76faa22ee5e0f9d/.github/workflows/devel_check_dock.yml#L65-L92
+  docker-build-and-push:
+    runs-on: ubuntu-latest
+    needs: build-check
+    steps:
+      - name: Checkout Repository
+        if: "!contains(github.event.head_commit.message, '/nodocker') && env.run_docker == 'true' && github.ref == 'refs/heads/devel'"
+        uses: actions/checkout@v3
+
+      - name: Register repo name
+        if: "!contains(github.event.head_commit.message, '/nodocker') && env.run_docker == 'true' && github.ref == 'refs/heads/devel'"
+        id: reg_repo_name
+        run: |
+          echo CONT_IMG_NAME=$(echo ${{ github.event.repository.name }} | tr '[:upper:]' '[:lower:]') >> $GITHUB_ENV
+
+      - name: Set up QEMU
+        if: "!contains(github.event.head_commit.message, '/nodocker') && env.run_docker == 'true' && github.ref == 'refs/heads/devel'"
+        uses: docker/setup-qemu-action@v2
+
+      - name: Set up Docker Buildx
+        if: "!contains(github.event.head_commit.message, '/nodocker') && env.run_docker == 'true' && github.ref == 'refs/heads/devel'"
+        uses: docker/setup-buildx-action@v2
+
+      - name: Login to Docker Hub
+        if: "!contains(github.event.head_commit.message, '/nodocker') && env.run_docker == 'true' && github.ref == 'refs/heads/devel'"
+        uses: docker/login-action@v2
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+      ## Note that DOCKERHUB_TOKEN is really a token for your dockerhub
+      ## account, not your actual dockerhub account password. You can get it
+      ## from https://hub.docker.com/settings/security.
+      ## Check https://github.com/docker/build-push-action/tree/v4.0.0
+      ## for more details.
+      ## Alternatively, try checking
+      ## https://seandavi.github.io/BuildABiocWorkshop/articles/HOWTO_BUILD_WORKSHOP.html.
+
+      - name: Build and Push Docker
+        if: "!contains(github.event.head_commit.message, '/nodocker') && env.run_docker == 'true' && github.ref == 'refs/heads/devel' && success()"
+        uses: docker/build-push-action@v4
+        with:
+          context: .
+          push: true
+          tags: >
+            ${{ secrets.DOCKERHUB_USERNAME }}/${{ env.CONT_IMG_NAME }}:latest,
+            ${{ secrets.DOCKERHUB_USERNAME }}/${{ env.CONT_IMG_NAME }}:devel

NAMESPACE

@@ -30,6 +30,7 @@ importFrom(cli,cli_alert_success)
 importFrom(cli,cli_alert_warning)
 importFrom(cli,hash_sha256)
 importFrom(dbplyr,remote_con)
+importFrom(dbplyr,sql)
 importFrom(dplyr,as_tibble)
 importFrom(dplyr,collect)
 importFrom(dplyr,filter)

R/metadata.R

@@ -67,6 +67,7 @@ SAMPLE_DATABASE_URL <- single_line_str(
 #' @importFrom dplyr tbl
 #' @importFrom httr progress
 #' @importFrom cli cli_alert_info hash_sha256
+#' @importFrom glue glue
 #'
 #' @details
 #'
@@ -162,10 +163,10 @@ get_metadata <- function(
                 progress(type = "down", con = stderr())
             )
         }
-
+        
         table <- duckdb() |>
             dbConnect(drv = _, read_only = TRUE) |>
-            tbl(db_path)
+            read_parquet(db_path)
         cache$metadata_table[[hash]] <- table
         table
     }

R/unharmonised.R

@@ -64,7 +64,8 @@ get_unharmonised_dataset <- function(
             local_path,
             progress(type = "down", con = stderr())
         )
-    tbl(conn, local_path) |>
+    
+    read_parquet(conn, local_path) |>
         filter(.data$cell_ %in% cells)
 }
 

R/utils.R

@@ -80,3 +80,20 @@ sync_remote_file <- function(full_url, output_file, ...) {
     }
     invisible(NULL)
 }
+
+#' Returns a tibble from a parquet file path
+#' 
+#' Since dbplyr 2.4.0, raw file paths aren't handled very well
+#' See: https://github.com/duckdb/duckdb-r/issues/38
+#' Hence the need for this method
+#' @importFrom glue glue
+#' @importFrom dplyr tbl
+#' @importFrom dbplyr sql
+#' @return An SQL data frame
+#' @keywords internal
+read_parquet <- function(conn, path){
+    from_clause <- glue("FROM read_parquet('{path}')") |> sql()
+    tbl(conn, from_clause)
+}
+
+