Update GitLab CI rules (#7065)

* GLCI: update R versions and links

Since R-4.5.0 has been released, provide the links to up-to-date
versions of released, development, and old-release versions of R.

* GLCI: fail Mac checks if worse than NOTE

* glci: install gettext from mac.r-project.org

* test-lin-rel: run --as-cran checks

* mirror-packages: version-2 PACKAGES.rds

Apparently, unlike R-3.3, R-3.4 outright fails the call to
available.packages() when it sees PACKAGES.rds it cannot parse. Try to
make tools::write_PACKAGES() prepare a file compatible with R<3.5.

* test-lin-rel: try harder to ignore -Wvla NOTE

* glci: update failure rules

* glci: use R CMD check dependencies from containers

Not as easy for lin-dev checks, those install packages from source

Author: aitap

.gitlab-ci.yml

@@ -13,14 +13,14 @@ variables:
   TZ: "UTC"  ## to avoid 'Failed to create bus connection' from timedatectl via Sys.timezone() on Docker with R 3.4.
              ## Setting TZ for all GLCI jobs to isolate them from timezone. We could have a new GLCI job to test under
              ## a non-UTC timezone, although, that's what we do routinely in dev.
-  R_REL_VERSION:  "4.4" # only raise when RTOOLS for REL is available
-  R_REL_WIN_BIN:  "https://cloud.r-project.org/bin/windows/base/old/4.4.1/R-4.4.1-win.exe"
-  R_DEV_VERSION:  "4.5"
+  R_REL_VERSION:  "4.5" # only raise when RTOOLS for REL is available
+  R_REL_WIN_BIN:  "https://cloud.r-project.org/bin/windows/base/old/4.5.0/R-4.5.0-win.exe"
+  R_DEV_VERSION:  "4.6"
   R_DEV_WIN_BIN:  "https://cloud.r-project.org/bin/windows/base/R-devel-win.exe"
-  R_OLD_VERSION:  "4.3"
-  R_OLD_WIN_BIN:  "https://cloud.r-project.org/bin/windows/base/old/4.3.3/R-4.3.3-win.exe"  
-  R_REL_MAC_BIN:  "https://cloud.r-project.org/bin/macosx/big-sur-arm64/base/R-4.4.1-arm64.pkg"
-  R_OLD_MAC_BIN:  "https://cloud.r-project.org/bin/macosx/big-sur-arm64/base/R-4.3.3-arm64.pkg"
+  R_OLD_VERSION:  "4.4"
+  R_OLD_WIN_BIN:  "https://cloud.r-project.org/bin/windows/base/old/4.4.3/R-4.4.3-win.exe"
+  R_REL_MAC_BIN:  "https://cloud.r-project.org/bin/macosx/big-sur-arm64/base/R-4.5.0-arm64.pkg"
+  R_OLD_MAC_BIN:  "https://cloud.r-project.org/bin/macosx/big-sur-arm64/base/R-4.4.3-arm64.pkg"
 
 stages:
   - dependencies
@@ -48,6 +48,9 @@ mirror-packages:
   cache:
     paths:
       - bus/$CI_JOB_NAME/cran
+  variables:
+    # TODO(R-ancient>=3.5.0): remove this; let it save PACKAGES.rds in version-3 format
+    R_DEFAULT_SERIALIZE_VERSION: 2
   script:
     - echo 'source(".ci/ci.R")' >> .Rprofile
     - mkdir -p bus/$CI_JOB_NAME/cran/src/contrib
@@ -104,21 +107,23 @@ build:
 # force all suggests
 # flags: gcc -O3 -flto=auto -fno-common -Wunused-result
 # tests for compilation warnings
+# runs the --as-cran check, including the URL checks
 test-lin-rel:
   <<: *test-lin
   image: registry.gitlab.com/rdatatable/dockerfiles/r-data.table
   variables:
-    _R_CHECK_COMPILATION_FLAGS_KNOWN_: "-Wvla"
-    _R_CHECK_CRAN_INCOMING_: "FALSE"
-    _R_CHECK_CRAN_INCOMING_REMOTE_: "FALSE"
     _R_CHECK_FORCE_SUGGESTS_: "TRUE"
     OPENBLAS_MAIN_FREE: "1"
   script:
     - *install-deps
     - echo 'CFLAGS=-g -O3 -flto=auto -fno-common -fopenmp -Wall -Wvla -pedantic -fstack-protector-strong -D_FORTIFY_SOURCE=2' > ~/.R/Makevars
     - echo 'CXXFLAGS=-g -O3 -flto=auto -fno-common -fopenmp -Wall -Wvla -pedantic -fstack-protector-strong -D_FORTIFY_SOURCE=2' >> ~/.R/Makevars
-    - R CMD check $(ls -1t data.table_*.tar.gz | head -n 1)
-    - (! grep "warning:" data.table.Rcheck/00install.out)
+    - echo '_R_CHECK_COMPILATION_FLAGS_KNOWN_=-Wvla' >> ~/.Renviron
+    - |
+        res1=0; R CMD check --as-cran $(ls -1t data.table_*.tar.gz | head -n 1) || res1=$?
+        res2=0; grep -v "warning:" data.table.Rcheck/00install.out || res2=$?
+        res3=0; Rscript -e 'l=tail(readLines("data.table.Rcheck/00check.log"), 1L); if (!identical(l, "Status: 2 NOTEs")) stop("Last line of ", shQuote("00check.log"), " is not ", shQuote("Status: 2 NOTEs"), " (CRAN incoming feasibility, non-API) but ", shQuote(l))' || res3=$?
+        if [ $res1 -ne 0 ] || [ $res2 -ne 0 ] || [ $res3 -ne 0 ]; then exit 1; fi
 
 ## vanilla minimal
 # no zlib
@@ -151,9 +156,10 @@ test-lin-rel-cran:
     - *install-deps
     - echo 'CFLAGS=-g -O2 -fopenmp -Wall -pedantic -fstack-protector-strong -D_FORTIFY_SOURCE=2' > ~/.R/Makevars
     - echo 'CXXFLAGS=-g -O2 -fopenmp -Wall -pedantic -fstack-protector-strong -D_FORTIFY_SOURCE=2' >> ~/.R/Makevars
-    - R CMD check --as-cran $(ls -1t data.table_*.tar.gz | head -n 1)
-    - >-
-        Rscript -e 'l=tail(readLines("data.table.Rcheck/00check.log"), 1L); if (!identical(l, "Status: OK")) stop("Last line of ", shQuote("00check.log"), " is not ", shQuote("Status: OK"), " but ", shQuote(l)) else q("no")'
+    - |
+        res1=0; R CMD check --as-cran $(ls -1t data.table_*.tar.gz | head -n 1) || res1=$?
+        res2=0; Rscript -e 'l=tail(readLines("data.table.Rcheck/00check.log"), 1L); if (!identical(l, "Status: 1 NOTE")) stop("Last line of ", shQuote("00check.log"), " is not ", shQuote("Status: 1 NOTE"), " (non-API) but ", shQuote(l))' || res2=$?
+        if [ $res1 -ne 0 ] || [ $res2 -ne 0 ]; then exit 1; fi
 
 ## R-devel on Linux gcc strict
 # R built with --enable-strict-barrier --disable-long-double
@@ -175,7 +181,7 @@ test-lin-dev-gcc-strict-cran:
     - R CMD check --as-cran $(ls -1t data.table_*.tar.gz | head -n 1)
     - (! grep "warning:" data.table.Rcheck/00install.out)
     - >-
-        Rscript -e 'l=tail(readLines("data.table.Rcheck/00check.log"), 1L); notes<-"Status: 3 NOTEs"; if (!identical(l, notes)) stop("Last line of ", shQuote("00check.log"), " is not ", shQuote(notes), " (size of tarball, installed package size, non-API calls) but ", shQuote(l)) else q("no")'
+        Rscript -e 'l=tail(readLines("data.table.Rcheck/00check.log"), 1L); notes<-"Status: 3 NOTEs"; if (!identical(l, notes)) stop("Last line of ", shQuote("00check.log"), " is not ", shQuote(notes), " (size of tarball, non-API calls, V8 package) but ", shQuote(l)) else q("no")'
 
 ## R-devel on Linux clang
 # R compiled with clang, flags removed: -flto=auto -fopenmp
@@ -198,7 +204,7 @@ test-lin-dev-clang-cran:
     - R CMD check --as-cran $(ls -1t data.table_*.tar.gz | head -n 1)
     - (! grep "warning:" data.table.Rcheck/00install.out)
     - >-
-        Rscript -e 'l=tail(readLines("data.table.Rcheck/00check.log"), 1L); notes<-"Status: 2 NOTEs"; if (!identical(l, notes)) stop("Last line of ", shQuote("00check.log"), " is not ", shQuote(notes), " (size of tarball, non-API calls) but ", shQuote(l)) else q("no")'
+        Rscript -e 'l=tail(readLines("data.table.Rcheck/00check.log"), 1L); notes<-"Status: 3 NOTEs"; if (!identical(l, notes)) stop("Last line of ", shQuote("00check.log"), " is not ", shQuote(notes), " (size of tarball, non-API calls, V8 package) but ", shQuote(l)) else q("no")'
 
 # stated dependency on R
 test-lin-ancient-cran:
@@ -290,11 +296,14 @@ test-win-old:
   before_script:
     - curl -O $R_BIN
     - sudo installer -pkg "$(ls -1t R-*-arm64.pkg | head -n 1)" -target /
+    - sudo Rscript -e "source('https://mac.R-project.org/bin/install.R'); install.libs('gettext')"
     - *install-deps
     - cp $(ls -1t bus/build/data.table_*.tar.gz | head -n 1) .
   script:
     - R CMD check --no-manual $(ls -1t data.table_*.tar.gz | head -n 1)
     - R CMD INSTALL --build $(ls -1t data.table_*.tar.gz | head -n 1)
+    - >-
+        tail -n 1 data.table.Rcheck/00check.log | grep -q -e '^Status: [0-9]* NOTEs*$' -e '^Status: OK$'
   after_script:
     - mkdir -p bus/$CI_JOB_NAME
     - '[ -d data.table.Rcheck ] && mv data.table.Rcheck bus/$CI_JOB_NAME/'