Merge branch 'master' into new-pt-po

Author: MichaelChirico

.ci/linters/c/cocci_linter.R

@@ -0,0 +1,20 @@
+cocci_linter = if (!nzchar(Sys.which("spatch"))) function(...) {} else function(c_obj) {
+  bad <- FALSE
+  for (spfile in list.files(".ci/linters/cocci", full.names = TRUE)) {
+    # Coccinelle parser gets confused sometimes, so ignore stderr and the exit code
+    out = suppressWarnings(system2(
+      "spatch",
+      shQuote(c(
+        "--sp-file", spfile, c_obj$path, "--recursive-includes",
+        "-I", R.home("include"), "-I", "src"
+      )),
+      stdout = TRUE, stderr = FALSE
+    ))
+    if (length(out) > 0) {
+      cat(sprintf("In file '%s', Coccinelle patch '%s' recommends the following changes:\n", c_obj$path, spfile))
+      writeLines(out)
+      bad <- TRUE
+    }
+  }
+  if (bad) stop("Please apply the changes above or fix the linter")
+}

.ci/linters/cocci/redundant_cast_to_same_pointer_type.cocci

@@ -0,0 +1,6 @@
+@@
+type T;
+T* E;
+@@
+- (T*)
+  E

.ci/linters/md/heading_id_linter.R

@@ -0,0 +1,26 @@
+any_mismatch = FALSE
+
+# ensure that ids are limited to alphanumerics and dashes
+# (in particular, dots and underscores break the links)
+check_header_ids = function(md) {
+  # A bit surprisingly, some headings don't start with a letter.
+  # We're interested in those that set an id to link to, i.e., end with {#id}.
+  heading_captures = regmatches(md, regexec("^#+ \\S.*[{]#([^}]*)[}]$", md))
+  lines_with_id = which(lengths(heading_captures) > 0)
+  ids = vapply(heading_captures[lines_with_id], `[`, '', 2)
+  # ids must start with a letter and consist of alphanumerics or dashes.
+  good_ids = grepl('^[A-Za-z][A-Za-z0-9-]*$', ids)
+  for (line in lines_with_id[!good_ids]) cat(sprintf(
+    "On line %d, bad heading id '%s':\n%s\n",
+    line, heading_captures[[line]][2], heading_captures[[line]][1]
+  ))
+  !all(good_ids)
+}
+
+any_error = FALSE
+for (vignette in list.files('vignettes', pattern = "[.]Rmd$", recursive = TRUE, full.name = TRUE)) {
+  cat(sprintf("Checking vignette file %s...\n", vignette))
+  rmd_lines = readLines(vignette)
+  any_error = check_header_ids(rmd_lines) || any_error
+}
+if (any_error) stop("Please fix the vignette issues above.")

.dev/CRAN_Release.cmd

@@ -3,17 +3,19 @@
 ###############################################
 
 # 1) Update messages for new release
-## (a) Update C template file: src/data.table.pot
-##     ideally, we are including _() wrapping in
-##     new PRs throughout dev cycle, and this step
-##     becomes about tying up loose ends
-## Check the output here for translatable messages
-xgettext -o /dev/stdout ./*.c \
-  --keyword=Rprintf --keyword=error --keyword=warning --keyword=STOP --keyword=DTWARN --keyword=Error --keyword=DTPRINT --keyword=snprintf:3
-
-## (b) Update R template file: src/R-data.table.pot
-##  NB: this relies on R >= 4.0 to remove a bug in update_pkg_po
-Rscript -e "tools::update_pkg_po('.')"
+dt_custom_translators = list(
+  R = 'catf:fmt|1',
+  # TODO(MichaelChirico/potools#318): restore snprintf:3 here too
+  src = c('STOP:1', 'DTWARN:1', 'DTPRINT:1')
+)
+message_db =
+  potools::get_message_data(custom_translation_functions = dt_custom_translators)
+potools::check_cracked_messages(message_db)
+potools::check_untranslated_cat(message_db)
+potools::check_untranslated_src(message_db)
+
+## (b) Update R template files (po/*.pot)
+potools::po_extract(custom_translation_functions = dt_custom_translators)
 
 # 2) Open a PR with the new templates & contact the translators
 #   * zh_CN: @hongyuanjia

.github/workflows/code-quality.yaml

@@ -35,6 +35,9 @@ jobs:
     steps:
       - uses: actions/checkout@v4
       - uses: r-lib/actions/setup-r@v2
+      - name: Install Coccinelle
+        # relying on the action above us to have updated the package cache
+        run: /usr/bin/sudo apt-get -y install coccinelle
       - name: Lint
         run: |
           linter_env = new.env()

CODEOWNERS

@@ -48,6 +48,8 @@
 /po/*zh_CN.po @Rdatatable/chinese
 /po/*pt_BR.po @Rdatatable/brazil
 /po/*es.po @Rdatatable/spanish
+/po/*ru.po @Rdatatable/russian
+/vignettes/ru/*.Rmd @Rdatatable/russian
 
 # printing
 /R/print.data.table.R @michaelchirico

DESCRIPTION

@@ -19,6 +19,7 @@ Authors@R: c(
   person("Michael","Chirico",      role="aut", comment = c(ORCID="0000-0003-0787-087X")),
   person("Toby","Hocking",         role="aut", comment = c(ORCID="0000-0002-3146-0865")),
   person("Benjamin","Schwendinger",role="aut", comment = c(ORCID="0000-0003-3315-8114")),
+  person("Ivan", "Krylov",         role="aut",          email="[email protected]",   comment = c(ORCID="0000-0002-0172-3812")),
   person("Pasha","Stetsenko",      role="ctb"),
   person("Tom","Short",            role="ctb"),
   person("Steve","Lianoglou",      role="ctb"),
@@ -89,7 +90,6 @@ Authors@R: c(
   person("Iago", "Giné-Vázquez",   role="ctb"),
   person("Anirban", "Chetia",      role="ctb"),
   person("Doris", "Amoakohene",    role="ctb"),
-  person("Ivan", "Krylov",         role="ctb"),
   person("Angel", "Feliz",         role="ctb"),
   person("Michael","Young",        role="ctb"),
   person("Mark", "Seeto",          role="ctb"),

NEWS.md

@@ -133,6 +133,8 @@ rowwiseDT(
 
 19. An integer overflow in `fread()` with lines longer than `2^(31/2)` bytes is prevented, [#6729](https://github.com/Rdatatable/data.table/issues/6729). The typical impact was no worse than a wrong initial allocation size, corrected later. Thanks to @TaikiSan21 for the report and @aitap for the fix.
 
+20. Fixed a memory issue causing segfaults in `forder`, [#6797](https://github.com/Rdatatable/data.table/issues/6797). Thanks @dkutner for the report and @MichaelChirico for the fix.
+
 ## NOTES
 
 1. There is a new vignette on joins! See `vignette("datatable-joins")`. Thanks to Angel Feliz for authoring it! Feedback welcome. This vignette has been highly requested since 2017: [#2181](https://github.com/Rdatatable/data.table/issues/2181).

R/between.R

@@ -75,7 +75,7 @@ inrange = function(x,lower,upper,incbounds=TRUE) {
   ops = if (incbounds) c(4L, 2L) else c(5L, 3L) # >=,<= and >,<
   verbose = isTRUE(getOption("datatable.verbose"))
   if (verbose) {last.started.at=proc.time();catf("forderv(query) took ... ");flush.console()}
-  if (verbose) {cat(timetaken(last.started.at),"\n"); flush.console()}
+  if (verbose) {cat(timetaken(last.started.at),"\n"); flush.console()} # notranslate
   ans = bmerge(
     shallow(subject), query,
     icols=1L:2L, xcols=c(1L, 1L),

R/bmerge.R

@@ -188,7 +188,7 @@ bmerge = function(i, x, icols, xcols, roll, rollends, nomatch, mult, ops, verbos
     if (verbose) {last.started.at=proc.time();catf("  forder took ... ");flush.console()}
     # TODO: could check/reuse secondary indices, but we need 'starts' attribute as well!
     xo = forderv(x, xcols, retGrp=TRUE)
-    if (verbose) {cat(timetaken(last.started.at),"\n"); flush.console()}
+    if (verbose) {cat(timetaken(last.started.at),"\n"); flush.console()} # notranslate
     xg = attr(xo, 'starts', exact=TRUE)
     resetcols = head(xcols, non_equi-1L)
     if (length(resetcols)) {