Merge branch 'master' into fix_fwrite_length

Author: MichaelChirico

.Rbuildignore

@@ -27,6 +27,7 @@
 ^src/Makevars$
 ^CODEOWNERS$
 ^GOVERNANCE\.md$
+^Seal_of_Approval\.md$
 
 ^\.RData$
 ^\.Rhistory$

.ci/linters/po/msgfmt_linter.R

@@ -0,0 +1,42 @@
+# Use msgfmt to check for untranslated/fuzzy messages, and for whether
+#   the implied .mo compiled form matches that which is already checked in
+msgfmt_linter <- function(po_file) {
+  mo_tmp <- tempfile()
+  on.exit(unlink(mo_tmp))
+
+  res = system2("msgfmt", c("--statistics", po_file, "-o", mo_tmp), stdout=TRUE, stderr=TRUE)
+  if (any(grepl("untranslated message|fuzzy translation", res))) {
+    cat(sprintf("In %s, found incomplete translations:\n%s\n", po_file, paste(res, collapse="\n")))
+    stop("Please fix.")
+  }
+
+  mo_ref = sprintf(
+    "inst/%s/LC_MESSAGES/%sdata.table.mo",
+    gsub("^R-|[.]po$", "", po_file),
+    if (startsWith(basename(po_file), "R-")) "R-" else ""
+  )
+
+  if (!file.exists(mo_ref)) {
+    stop(po_file, " has not been compiled as ", mo_ref, ". Please fix.")
+  }
+  if (tools::md5sum(mo_ref) == tools::md5sum(mo_tmp)) return(invisible())
+
+  # NB: file.mtime() will probably be wrong, it will reflect the check-out time of the git repo.
+  last_edit_time = system2("git",
+    c("log", "-1", '--format="%ad"', "--date=format:'%Y-%m-%d %H:%M:%S'", "--", mo_ref),
+    stdout=TRUE
+  )
+  cat(sprintf(
+    ".mo compilation %s of .po translation %s appears out of date! It was last updated %s\n",
+    mo_ref, po_file, last_edit_time
+  ))
+
+  unmo_tmp = tempfile()
+  unmo_ref = tempfile()
+  on.exit(unlink(c(unmo_tmp, unmo_ref), add=TRUE))
+  system2("msgunfmt", c(mo_tmp, "-o", unmo_tmp))
+  system2("msgunfmt", c(mo_ref, "-o", unmo_ref))
+  cat("Here are the observed differences after converting back to .po:\n\n")
+  system2("diff", c(unmo_tmp, unmo_ref))
+  stop("Please fix.")
+}

.ci/linters/po/tools_check_linter.R

@@ -0,0 +1,7 @@
+tools_check_linter = function(po_file) {
+  res = tools::checkPoFile(po_file, strictPlural=TRUE)
+  if (NROW(res)) {
+    print(res)
+    stop("Fix the above .po file issues.")
+  }
+}

.ci/linters/po/utf8_linter.R

@@ -0,0 +1,4 @@
+utf8_linter <- function(po_file) {
+  if (!any(grepl("charset=UTF-8", readLines(po_file), fixed=TRUE)))
+    stop("In ", po_file, ", please use charset=UTF-8.")
+}

.dev/CRAN_Release.cmd

@@ -189,15 +189,15 @@ R CMD build .
 export GITHUB_PAT="f1c.. github personal access token ..7ad"
 # avoids many too-many-requests in --as-cran's ping-all-URLs step (20 mins) inside the `checking CRAN incoming feasibility...` step.
 # Many thanks to Dirk for the tipoff that setting this env variable solves the problem, #4832.
-R CMD check data.table_1.15.99.tar.gz --as-cran
-R CMD INSTALL data.table_1.15.99.tar.gz --html
+R CMD check data.table_1.16.99.tar.gz --as-cran
+R CMD INSTALL data.table_1.16.99.tar.gz --html
 
 # Test C locale doesn't break test suite (#2771)
 echo LC_ALL=C > ~/.Renviron
 R
 Sys.getlocale()=="C"
 q("no")
-R CMD check data.table_1.15.99.tar.gz
+R CMD check data.table_1.16.99.tar.gz
 rm ~/.Renviron
 
 # Test non-English does not break test.data.table() due to translation of messages; #3039, #630
@@ -214,9 +214,9 @@ q("no")
 
 # User supplied PKG_CFLAGS and PKG_LIBS passed through, #4664
 # Next line from https://mac.r-project.org/openmp/. Should see the arguments passed through and then fail with gcc on linux.
-PKG_CFLAGS='-Xclang -fopenmp' PKG_LIBS=-lomp R CMD INSTALL data.table_1.15.99.tar.gz
+PKG_CFLAGS='-Xclang -fopenmp' PKG_LIBS=-lomp R CMD INSTALL data.table_1.16.99.tar.gz
 # Next line should work on Linux, just using superfluous and duplicate but valid parameters here to see them retained and work 
-PKG_CFLAGS='-fopenmp' PKG_LIBS=-lz R CMD INSTALL data.table_1.15.99.tar.gz
+PKG_CFLAGS='-fopenmp' PKG_LIBS=-lz R CMD INSTALL data.table_1.16.99.tar.gz
 
 R
 remove.packages("xml2")    # we checked the URLs; don't need to do it again (many minutes)
@@ -260,7 +260,7 @@ alias R330=~/build/R-3.3.0/bin/R
 ### END ONE TIME BUILD
 
 cd ~/GitHub/data.table
-R330 CMD INSTALL ./data.table_1.15.99.tar.gz
+R330 CMD INSTALL ./data.table_1.16.99.tar.gz
 R330
 require(data.table)
 test.data.table(script="*.Rraw")
@@ -272,15 +272,15 @@ test.data.table(script="*.Rraw")
 vi ~/.R/Makevars
 # Make line SHLIB_OPENMP_CFLAGS= active to remove -fopenmp
 R CMD build .
-R CMD INSTALL data.table_1.15.99.tar.gz   # ensure that -fopenmp is missing and there are no warnings
+R CMD INSTALL data.table_1.16.99.tar.gz   # ensure that -fopenmp is missing and there are no warnings
 R
 require(data.table)   # observe startup message about no OpenMP detected
 test.data.table()
 q("no")
 vi ~/.R/Makevars
 # revert change above
 R CMD build .
-R CMD check data.table_1.15.99.tar.gz
+R CMD check data.table_1.16.99.tar.gz
 
 
 #####################################################
@@ -335,11 +335,11 @@ alias Rdevel-strict-gcc='~/build/R-devel-strict-gcc/bin/R --vanilla'
 alias Rdevel-strict-clang='~/build/R-devel-strict-clang/bin/R --vanilla'
 
 cd ~/GitHub/data.table
-Rdevel-strict-[gcc|clang] CMD INSTALL data.table_1.15.99.tar.gz
+Rdevel-strict-[gcc|clang] CMD INSTALL data.table_1.16.99.tar.gz
 # Check UBSAN and ASAN flags appear in compiler output above. Rdevel was compiled with them so they should be
 # passed through to here. However, our configure script seems to get in the way and gets them from {R_HOME}/bin/R
 # So I needed to edit my ~/.R/Makevars to get CFLAGS the way I needed.
-Rdevel-strict-[gcc|clang] CMD check data.table_1.15.99.tar.gz
+Rdevel-strict-[gcc|clang] CMD check data.table_1.16.99.tar.gz
 # Use the (failed) output to get the list of currently needed packages and install them
 Rdevel-strict-[gcc|clang]
 isTRUE(.Machine$sizeof.longdouble==0)  # check noLD is being tested
@@ -348,7 +348,7 @@ install.packages(c("bit64", "bit", "R.utils", "xts", "zoo", "yaml", "knitr", "ma
                  Ncpus=4)
 # Issue #5491 showed that CRAN is running UBSAN on .Rd examples which found an error so we now run full R CMD check
 q("no")
-Rdevel-strict-[gcc|clang] CMD check data.table_1.15.99.tar.gz
+Rdevel-strict-[gcc|clang] CMD check data.table_1.16.99.tar.gz
 # UBSAN errors occur on stderr and don't affect R CMD check result. Made many failed attempts to capture them. So grep for them. 
 find data.table.Rcheck -name "*Rout*" -exec grep -H "runtime error" {} \;
 
@@ -385,7 +385,7 @@ cd R-devel-valgrind
 make
 cd ~/GitHub/data.table
 vi ~/.R/Makevars  # make the -O2 -g line active, for info on source lines with any problems
-Rdevel-valgrind CMD INSTALL data.table_1.15.99.tar.gz
+Rdevel-valgrind CMD INSTALL data.table_1.16.99.tar.gz
 R_DONT_USE_TK=true Rdevel-valgrind -d "valgrind --tool=memcheck --leak-check=full --track-origins=yes --show-leak-kinds=definite,possible --gen-suppressions=all --suppressions=./.dev/valgrind.supp -s"
 # the default for --show-leak-kinds is 'definite,possible' which we're setting explicitly here as a reminder. CRAN uses the default too.
 #   including 'reachable' (as 'all' does) generates too much output from R itself about by-design permanent blocks
@@ -423,7 +423,7 @@ cd ~/build/rchk/trunk
 . ../scripts/config.inc
 . ../scripts/cmpconfig.inc
 vi ~/.R/Makevars   # set CFLAGS=-O0 -g so that rchk can provide source line numbers
-echo 'install.packages("~/GitHub/data.table/data.table_1.15.99.tar.gz",repos=NULL)' | ./bin/R --slave
+echo 'install.packages("~/GitHub/data.table/data.table_1.16.99.tar.gz",repos=NULL)' | ./bin/R --slave
 # objcopy warnings (if any) can be ignored: https://github.com/kalibera/rchk/issues/17#issuecomment-497312504
 . ../scripts/check_package.sh data.table
 cat packages/lib/data.table/libs/*check
@@ -589,7 +589,7 @@ du -k inst/tests                # 0.75MB after
 R CMD build .
 export GITHUB_PAT="f1c.. github personal access token ..7ad"
 Rdevel -q -e "packageVersion('xml2')"   # ensure installed
-Rdevel CMD check data.table_1.16.0.tar.gz --as-cran  # use latest Rdevel as it may have extra checks
+Rdevel CMD check data.table_1.17.0.tar.gz --as-cran  # use latest Rdevel as it may have extra checks
 bunzip2 inst/tests/*.Rraw.bz2  # decompress *.Rraw again so as not to commit compressed *.Rraw to git
 
 #
@@ -621,10 +621,10 @@ bunzip2 inst/tests/*.Rraw.bz2  # decompress *.Rraw again so as not to commit com
 # 3. Add new heading in NEWS for the next dev version. Add "(submitted to CRAN on <today>)" on the released heading.
 # 4. Bump minor version in dllVersion() in init.c
 # 5. Bump 3 minor version numbers in Makefile
-# 6. Search and replace this .dev/CRAN_Release.cmd to update 1.15.99 to 1.16.99 inc below, 1.16.0 to 1.17.0 above, 1.15.0 to 1.16.0 below
+# 6. Search and replace this .dev/CRAN_Release.cmd to update 1.16.99 to 1.16.99 inc below, 1.16.0 to 1.17.0 above, 1.15.0 to 1.16.0 below
 # 7. Another final gd to view all diffs using meld. (I have `alias gd='git difftool &> /dev/null'` and difftool meld: http://meldmerge.org/)
-# 8. Push to master with this consistent commit message: "1.16.0 on CRAN. Bump to 1.16.99"
-# 9. Take sha from the previous step and run `git tag 1.16.0 96c..sha..d77` then `git push origin 1.16.0` (not `git push --tags` according to https://stackoverflow.com/a/5195913/403310)
+# 8. Push to master with this consistent commit message: "1.17.0 on CRAN. Bump to 1.17.99"
+# 9. Take sha from the previous step and run `git tag 1.17.0 96c..sha..d77` then `git push origin 1.16.0` (not `git push --tags` according to https://stackoverflow.com/a/5195913/403310)
 ######
 
 ###### Branching policy for PATCH RELEASE

.dev/cc.R

@@ -70,20 +70,35 @@ cc = function(test=FALSE, clean=FALSE, debug=FALSE, omp=!debug, path=Sys.getenv(
   dll = grep("data_table.so", dll, fixed=TRUE, value=TRUE)
   sapply(dll, dyn.unload)
   gc()
+  
+  # hack for windows under mingw-like environment
+  # PROJ_PATH must be windows like i.e.: export PROJ_PATH=$(Rscript -e 'getwd()')
+  if (.Platform$OS.type == "windows") {
+    system = function(command, ...) shell(shQuote(command, "sh"), shell = "sh",...)
+    # when -std=c99, macro WIN32 is not defined (MinGW) so we define it
+    # otherwise will fail as sys/mman.h does not exists in windows (fread.c)
+    W32 = "-DWIN32"
+    dt_object = "data_table"
+  } else {
+    W32 = ""
+    dt_object = "data_table.so"
+  }
 
   old = getwd()
   on.exit(setwd(old))
   setwd(file.path(path,"src"))
   if (!quiet) cat(getwd(),"\n")
   if (clean) system("rm *.o *.so")
-  OMP = if (omp) "" else "no-"
+  OMP = if (omp) "openmp" else "no-openmp"
   if (debug) {
-    ret = system(ignore.stdout=quiet, sprintf("MAKEFLAGS='-j CC=%s PKG_CFLAGS=-f%sopenmp CFLAGS=-std=c99\\ -O0\\ -ggdb\\ -pedantic' R CMD SHLIB -d -o data_table.so *.c", CC, OMP))
+    cmd = sprintf(R"(MAKEFLAGS='-j CC=%s PKG_CFLAGS=-f% CFLAGS=-std=c99\ -O0\ -ggdb\ %s\ -pedantic' R CMD SHLIB -d -o data_table.so *.c)", CC, OMP, W32)
   } else {
-    ret = system(ignore.stdout=quiet, sprintf("MAKEFLAGS='-j CC=%s CFLAGS=-f%sopenmp\\ -std=c99\\ -O3\\ -pipe\\ -Wall\\ -pedantic\\ -Wstrict-prototypes\\ -isystem\\ /usr/share/R/include\\ -fno-common' R CMD SHLIB -o data_table.so *.c", CC, OMP))
+    cmd = sprintf(R"(MAKEFLAGS='-j CC=%s CFLAGS=-f%s\ -std=c99\ -O3\ -pipe\ -Wall\ -pedantic\ -Wstrict-prototypes\ -isystem\ /usr/share/R/include\ %s\ -fno-common' R CMD SHLIB -o data_table.so *.c)", CC, OMP, W32)
     # the -isystem suppresses strict-prototypes warnings from R's headers, #5477. Look at the output to see what -I is and pass the same path to -isystem.
     # TODO add -Wextra too?
   }
+  cat(sprintf("Running command:\n%s\n", cmd))
+  ret = system(cmd, ignore.stdout=quiet)
   if (ret) return()
   # clang -Weverything includes -pedantic and issues many more warnings than gcc
   # system("R CMD SHLIB -o data_table.so *.c")
@@ -93,7 +108,7 @@ cc = function(test=FALSE, clean=FALSE, debug=FALSE, omp=!debug, path=Sys.getenv(
       break
     }
   }
-  dyn.load("data_table.so")
+  dyn.load(dt_object)
   setwd(old)
   xx = getDLLRegisteredRoutines("data_table",TRUE)
   for (Call in xx$.Call)

.github/workflows/code-quality.yaml

@@ -56,21 +56,18 @@ jobs:
       - uses: r-lib/actions/setup-r@v2
       - name: Check translations
         run: |
-          setwd("po")
-          for (po_file in list.files(pattern = "[.]po$")) {
-            res = tools::checkPoFile(po_file, strictPlural=TRUE)
-            if (NROW(res)) { print(res); stop("Fix the above .po file issues.") }
-
-            if (!any(grepl("charset=UTF-8", readLines(po_file), fixed=TRUE)))
-              stop("In ", po_file, ", please use charset=UTF-8.")
-
-            res = system2("msgfmt", c("--statistics", po_file, "-o", tempfile()), stdout=TRUE, stderr=TRUE)
-            if (any(grepl("untranslated message|fuzzy translation", res))) {
-              cat(sprintf("In %s, found incomplete translations:\n%s\n", po_file, paste(res, collapse="\n")))
-              stop("Please fix.")
-            }
+          linter_env = new.env()
+          for (f in list.files('.ci/linters/po', full.names=TRUE)) sys.source(f, linter_env)
+          for (po_file in list.files(pattern = "[.]po$", full.names=TRUE)) {
+            # only pay attention to files edited in the current PR, otherwise we can get
+            #   a situation like after #6424 where some untranslated messages were added
+            #   as part of non-translation maintenance, but this GHA would go red repeatedly
+            #   until a translation is added or the blank/fuzzy translations removed. We'd
+            #   rather only have the failure on one PR, then ignore these files later.
+            diff_v_master = system2("git", c("diff", "master", po_file), stdout=TRUE)
+            if (!length(diff_v_master)) next
+            for (linter in ls(linter_env)) linter_env[[linter]](po_file)
           }
-          cat("All translation quality checks completed successfully!\n")
         shell: Rscript {0}
   lint-md:
     runs-on: ubuntu-latest

.gitlab-ci.yml

@@ -272,6 +272,7 @@ test-win-old:
   tags:
     - saas-macos-medium-m1
   before_script:
+    - if ! command -v R &> /dev/null || ! command -v Rscript &> /dev/null; then brew install r; fi
     - *install-deps
     - cp $(ls -1t bus/build/data.table_*.tar.gz | head -n 1) .
   after_script:
@@ -288,7 +289,7 @@ test-mac-rel:
   variables:
     R_VERSION: "$R_REL_VERSION"
   script:
-    - R CMD check $(ls -1t data.table_*.tar.gz | head -n 1)
+    - R CMD check --no-manual $(ls -1t data.table_*.tar.gz | head -n 1)
     - R CMD INSTALL --build $(ls -1t data.table_*.tar.gz | head -n 1)
 
 ## integrate artifacts