Merge branch 'master' into modular_gforce

Author: ben-schwen

.gitlab-ci.yml

@@ -337,13 +337,15 @@ test-win-old:
 ## R-release on MacOS
 test-mac-rel:
   <<: *test-mac
+  image: macos-14-xcode-15
   variables:
     R_VERSION: "$R_REL_VERSION"
     R_BIN: "$R_REL_MAC_BIN"
 
 ## R-oldrel on MacOS
 test-mac-old:
   <<: *test-mac
+  image: macos-14-xcode-15
   variables:
     R_VERSION: "$R_OLD_VERSION"
     R_BIN: "$R_OLD_MAC_BIN"

NEWS.md

@@ -353,6 +353,8 @@ See [#2611](https://github.com/Rdatatable/data.table/issues/2611) for details. T
 
 22. `setDTthreads(percent=)` and `setDTthreads(threads=)` now respect `OMP_NUM_THREADS` and `omp_get_max_threads()`, ensuring consistency with `setDTthreads()` (no arguments) when OpenMP environment variables are set, [#7165](https://github.com/Rdatatable/data.table/issues/7165). Previously, explicitly setting a thread count or percentage would ignore these OpenMP limits, potentially exceeding the user's intended thread cap. Thanks to @bastistician for the report and @ben-schwen for the fix.
 
+23. `fread()` auto-detects separators for single-column files consisting solely of quoted values (e.g. `"this_that"\n"2025-01-01 00:00:01"`), [#7366](https://github.com/Rdatatable/data.table/issues/7366). Thanks @arunsrinivasan for the report and @ben-schwen for the fix.
+
 ### NOTES
 
 1. The following in-progress deprecations have proceeded:

inst/tests/tests.Rraw

@@ -21194,28 +21194,31 @@ test(2344.03, setkey(d1[, .(V1, label = c("one", "zero", "one"), V2)][data.table
 DT = data.table(V1 = 1:2, V2 = 3:4, V3 = 5:6, key = c("V1", "V2", "V3"))
 test(2344.04, key(DT[, .(V4 = c("b", "a"), V2, V5 = c("y", "x"), V1)]), c("V1", "V2"))
 
+# fread with quotes and single column #7366
+test(2345, fread('"this_that"\n"2025-01-01 00:00:01"'), data.table(this_that = as.POSIXct("2025-01-01 00:00:01", tz="UTC")))
+
 # gforce should also work with Map in j #5336
 # conversions should not turn gforce off #2934
 # lapply gforce should also work without .SD #5032
 # support arithmetic in j with gforce #3815
 out = c("GForce FALSE", "GForce FALSE", "GForce TRUE")
 dt = data.table(a=1:4, b=1:2)
-test(2345.01,optimize=0:2, dt[, max(as.character(a)), by=b, verbose=TRUE], data.table(b=1:2, V1=c("3","4")), output=out)
-test(2345.02,optimize=0:2, dt[, max(as.numeric(a)), by=b, verbose=TRUE], data.table(b=1:2, V1=c(3,4)), output=out)
+test(2346.01,optimize=0:2, dt[, max(as.character(a)), by=b, verbose=TRUE], data.table(b=1:2, V1=c("3","4")), output=out)
+test(2346.02,optimize=0:2, dt[, max(as.numeric(a)), by=b, verbose=TRUE], data.table(b=1:2, V1=c(3,4)), output=out)
 dt = data.table(a=1:4, b=1:2)
-test(2345.11,optimize=0:2, dt[, Map(sum, .SD), b, verbose=TRUE], dt[, lapply(.SD, sum), b], output=out)
-test(2345.12,optimize=0:2, dt[, Map(sum, .SD, .SD), by=b, verbose=TRUE], output="GForce FALSE")
+test(2346.11,optimize=0:2, dt[, Map(sum, .SD), b, verbose=TRUE], dt[, lapply(.SD, sum), b], output=out)
+test(2346.12,optimize=0:2, dt[, Map(sum, .SD, .SD), by=b, verbose=TRUE], output="GForce FALSE")
 dt = data.table(a = NA_integer_, b = 1:2, c = c(TRUE, FALSE))
-test(2345.13,optimize=0:2, dt[, Map(weighted.mean, .SD, na.rm=c), b, .SDcols="a", verbose=TRUE], data.table(b=1:2, a=c(NaN, NA_real_)), output="GForce FALSE")
-test(2345.14,optimize=0:2, dt[,list(weighted.mean(a, na.rm=c)), b, verbose=TRUE], data.table(b=1:2, V1=c(NaN, NA_real_)), output="GForce FALSE")
+test(2346.13,optimize=0:2, dt[, Map(weighted.mean, .SD, na.rm=c), b, .SDcols="a", verbose=TRUE], data.table(b=1:2, a=c(NaN, NA_real_)), output="GForce FALSE")
+test(2346.14,optimize=0:2, dt[,list(weighted.mean(a, na.rm=c)), b, verbose=TRUE], data.table(b=1:2, V1=c(NaN, NA_real_)), output="GForce FALSE")
 dt = data.table(a=1:2, b=1, c=1:4)
-test(2345.21,optimize=0:2, dt[, lapply(list(b, c), sum), by=a, verbose=TRUE], output=out)
-test(2345.22,optimize=0:2, dt[, c(list(sum(b), sum(c))), by=a, verbose=TRUE], output=out)
-test(2345.23,optimize=0:2, names(dt[, lapply(list(b, c), sum), by=a]))
+test(2346.21,optimize=0:2, dt[, lapply(list(b, c), sum), by=a, verbose=TRUE], output=out)
+test(2346.22,optimize=0:2, dt[, c(list(sum(b), sum(c))), by=a, verbose=TRUE], output=out)
+test(2346.23,optimize=0:2, names(dt[, lapply(list(b, c), sum), by=a]))
 dt = data.table(a=1:4, b=1:2)
-test(2345.31,optimize=0:2, dt[, .(max(a)-min(a)), by=b, verbose=TRUE], output=out)
-test(2345.32,optimize=0:2, dt[, .((max(a) - min(a)) / (max(a) + min(a))), by=b, verbose=TRUE], data.table(b=1:2, V1=c(0.5, 1/3)), output=out)
-test(2345.33,optimize=0:2, dt[, sum(a) / .N, b, verbose=TRUE], output=out)
-test(2345.34,optimize=0:2, dt[, mean(a) * 2L + sum(a), b, verbose=TRUE], output=out)
-test(2345.35,optimize=0:2, dt[, list(range=max(a)-min(a), avg=mean(a)), by=b, verbose=TRUE], output=out)
-test(2345.36,optimize=0:2, dt[, .(max(a)-sqrt(min(a))), by=b, verbose=TRUE], output="GForce FALSE")
+test(2346.31,optimize=0:2, dt[, .(max(a)-min(a)), by=b, verbose=TRUE], output=out)
+test(2346.32,optimize=0:2, dt[, .((max(a) - min(a)) / (max(a) + min(a))), by=b, verbose=TRUE], data.table(b=1:2, V1=c(0.5, 1/3)), output=out)
+test(2346.33,optimize=0:2, dt[, sum(a) / .N, b, verbose=TRUE], output=out)
+test(2346.34,optimize=0:2, dt[, mean(a) * 2L + sum(a), b, verbose=TRUE], output=out)
+test(2346.35,optimize=0:2, dt[, list(range=max(a)-min(a), avg=mean(a)), by=b, verbose=TRUE], output=out)
+test(2346.36,optimize=0:2, dt[, .(max(a)-sqrt(min(a))), by=b, verbose=TRUE], output="GForce FALSE")

src/fread.c

@@ -1899,14 +1899,29 @@ int freadMain(freadMainArgs _args)
                 thisBlockStart = lineStart;
               }
             }
-            if ((thisBlockLines > topNumLines && lastncol > 1) ||  // more lines wins even with fewer fields, so long as number of fields >= 2
-                (thisBlockLines == topNumLines &&
-                 lastncol > topNumFields &&                      // when number of lines is tied, choose the sep which separates it into more columns
-                 (quoteRule < QUOTE_RULE_EMBEDDED_QUOTES_NOT_ESCAPED || quoteRule <= topQuoteRule) && // for test 1834 where every line contains a correctly quoted field contain sep
-                 (topNumFields <= 1 || sep != ' '))) {
+            bool blockHasQuote = false;
+            if (quote && lastncol == 1) {
+              for (const char *scan = thisBlockStart; scan < ch; scan++) {
+                if (*scan == quote) {
+                  blockHasQuote = true;
+                  break;
+                }
+              }
+            }
+            bool singleColumnCandidate = (lastncol == 1 && thisBlockLines >= 2 && blockHasQuote && quoteRule < QUOTE_RULE_IGNORE_QUOTES);
+            // more contiguous rows than the current best; only allow 1-column wins while we still have no multi-column pick
+            bool betterLines = thisBlockLines > topNumLines && (lastncol > 1 || (singleColumnCandidate && topNumFields <= 1));
+            // first multi-column candidate after only single-column options so far
+            bool promoteOverSingle = (topNumFields <= 1 && lastncol > topNumFields && thisBlockLines >= 2);
+            // more lines wins even with fewer fields, so long as number of fields >= 2
+            bool betterTie = (thisBlockLines == topNumLines &&
+                              lastncol > topNumFields &&                      // when number of lines is tied, choose the sep which separates it into more columns
+                              (quoteRule < QUOTE_RULE_EMBEDDED_QUOTES_NOT_ESCAPED || quoteRule <= topQuoteRule) && // for test 1834 where every line contains a correctly quoted field contain sep
+                              (topNumFields <= 1 || sep != ' '));
+            if (betterLines || promoteOverSingle || betterTie) {
               topNumLines = thisBlockLines;
               topNumFields = lastncol;
-              topSep = sep;
+              topSep = singleColumnCandidate ? 127 : sep;  // treat consistent single-column quoted blocks as single-column input (#7366)
               topQuoteRule = quoteRule;
               firstJumpEnd = ch;
               topStart = thisBlockStart;

src/idatetime.c

@@ -145,7 +145,7 @@ SEXP convertDate(SEXP x, SEXP type)
         SEXP ans = PROTECT(allocVector(INTSXP, n));
         int *ansp = INTEGER(ans);
 
-        SEXP opt = GetOption(install("datatable.week"), R_NilValue);
+        SEXP opt = GetOption1(install("datatable.week"));
         const char *mode = isString(opt) && length(opt) == 1 ? CHAR(STRING_ELT(opt, 0)) : "default";
 
         bool use_sequential = !strcmp(mode, "sequential");