-tune (FFT tuning) takes carry 32/64 into account

preda · preda · commit 49af97c9ffe7 · 2024-06-26T08:34:03.000+03:00
diff --git a/src/Args.cpp b/src/Args.cpp
@@ -272,7 +272,7 @@ void Args::parse(const string& line) {
       log(" FFT         | BPW   | Max exp (M)\n");
       for (const FFTShape& shape : FFTShape::multiSpec(s)) {
         for (u32 variant = 0; variant < FFTConfig::N_VARIANT; ++variant) {
-          FFTConfig fft{shape, variant};
+          FFTConfig fft{shape, variant, FFTConfig::CARRY_AUTO};
           log("%12s | %.2f | %5.1f\n", fft.spec().c_str(), fft.maxBpw(), fft.maxExp() / 1'000'000.0);
         }
       }
diff --git a/src/FFTConfig.cpp b/src/FFTConfig.cpp
@@ -108,7 +108,7 @@ FFTShape::FFTShape(const string& w, const string& m, const string& h) :
   FFTShape{parseInt(w), parseInt(m), parseInt(h)}
 {}
 
-double FFTShape::carryLimitBPW() const {
+double FFTShape::carry32BPW() const {
   // The formula below was validated empirically with -carryTune
 
   // We observe that FFT 6.5M (1024:13:256) has safe carry32 up to 18.3 BPW
@@ -119,7 +119,7 @@ double FFTShape::carryLimitBPW() const {
 }
 
 bool FFTShape::needsLargeCarry(u32 E) const {
-  return E / double(size()) > carryLimitBPW();
+  return E / double(size()) > carry32BPW();
 }
 
 FFTShape::FFTShape(u32 w, u32 m, u32 h) :
@@ -149,9 +149,9 @@ FFTConfig::FFTConfig(const string& spec) {
   assert(v.size() == 3 || v.size() == 4 || v.size() == 5);
 
   if (v.size() == 3) {
-    *this = {FFTShape{v[0], v[1], v[2]}, 3};
+    *this = {FFTShape{v[0], v[1], v[2]}, 3, CARRY_AUTO};
   } else if (v.size() == 4) {
-    *this = {FFTShape{v[0], v[1], v[2]}, parseInt(v[3])};
+    *this = {FFTShape{v[0], v[1], v[2]}, parseInt(v[3]), CARRY_AUTO};
   } else if (v.size() == 5) {
     int c = parseInt(v[4]);
     assert(c == 0 || c == 1);
@@ -161,7 +161,7 @@ FFTConfig::FFTConfig(const string& spec) {
   }
 }
 
-FFTConfig::FFTConfig(FFTShape shape, u32 variant, CARRY_KIND carry) :
+FFTConfig::FFTConfig(FFTShape shape, u32 variant, u32 carry) :
   shape{shape},
   variant{variant},
   carry{carry}
@@ -176,7 +176,7 @@ string FFTConfig::spec() const {
 
 double FFTConfig::maxBpw() const {
   double b = shape.bpw[variant];
-  return carry == CARRY_32 ? std::min(shape.carryLimitBPW(), b) : b;
+  return carry == CARRY_32 ? std::min(shape.carry32BPW(), b) : b;
 }
 
 FFTConfig FFTConfig::bestFit(const Args& args, u32 E, const string& spec) {
@@ -211,7 +211,7 @@ FFTConfig FFTConfig::bestFit(const Args& args, u32 E, const string& spec) {
   // Take the first FFT that can handle E
   for (const FFTShape& shape : FFTShape::allShapes()) {
     for (u32 v = 0; v < 4; ++v) {
-      if (FFTConfig fft{shape, v}; fft.maxExp() >= E) { return fft; }
+      if (FFTConfig fft{shape, v, CARRY_AUTO}; fft.maxExp() >= E) { return fft; }
     }
   }
 
diff --git a/src/FFTConfig.h b/src/FFTConfig.h
@@ -43,7 +43,7 @@ class FFTShape {
   double maxBpw() const { return *max_element(bpw.begin(), bpw.end()); }
   std::string spec() const { return numberK(width) + ':' + numberK(middle) + ':' + numberK(height); }
 
-  double carryLimitBPW() const;
+  double carry32BPW() const;
   bool needsLargeCarry(u32 E) const;
 };
 
@@ -56,10 +56,10 @@ struct FFTConfig {
 
   FFTShape shape{};
   u32 variant;
-  CARRY_KIND carry;
+  u32 carry;
 
   explicit FFTConfig(const string& spec);
-  FFTConfig(FFTShape shape, u32 variant, CARRY_KIND carry = CARRY_AUTO);
+  FFTConfig(FFTShape shape, u32 variant, u32 carry /* = CARRY_AUTO*/);
 
   std::string spec() const;
   u32 size() const { return shape.size(); }
diff --git a/src/TuneEntry.cpp b/src/TuneEntry.cpp
@@ -76,9 +76,9 @@ void TuneEntry::writeTuneFile(const vector<TuneEntry>& results) {
   CycleFile tune{"tune.txt"};
   for (const TuneEntry& r : results) {
     u32 maxExp = r.fft.maxExp();
-    assert(r.cost > prevCost && maxExp > prevMaxExp);
+    assert(r.cost >= prevCost && maxExp > prevMaxExp);
     prevCost = r.cost;
     prevMaxExp = maxExp;
-    tune->printf("%6.0f %12s # %u\n", r.cost, r.fft.spec().c_str(), maxExp);
+    tune->printf("%6.0f %14s # %u\n", r.cost, r.fft.spec().c_str(), maxExp);
   }
 }
diff --git a/src/tune.cpp b/src/tune.cpp
@@ -162,7 +162,7 @@ void Tune::ztune() {
     double bpw[4];
     double A[4];
     for (u32 variant = 0; variant < FFTConfig::N_VARIANT; ++variant) {
-      FFTConfig fft{shape, variant};
+      FFTConfig fft{shape, variant, FFTConfig::CARRY_AUTO};
       std::tie(bpw[variant], A[variant]) = maxBpw(fft);
     }
     string s = "\""s + shape.spec() + "\"";
@@ -178,13 +178,13 @@ void Tune::carryTune() {
   shared.args->flags["STATS"] = "1";
   u32 prevSize = 0;
   for (FFTShape shape : FFTShape::multiSpec(shared.args->fftSpec)) {
-    FFTConfig fft{shape, 3};
+    FFTConfig fft{shape, 3, FFTConfig::CARRY_AUTO};
     if (prevSize == fft.size()) { continue; }
     prevSize = fft.size();
 
     vector<double> zv;
     double m = 0;
-    const double mid = fft.shape.carryLimitBPW();
+    const double mid = fft.shape.carry32BPW();
     for (double bpw : {mid - 0.05, mid + 0.05}) {
       u32 exponent = primes.nearestPrime(fft.size() * bpw);
       auto [ok, carry] = Gpu::make(q, exponent, shared, fft, {}, false)->measureCarry();
@@ -223,7 +223,7 @@ void Tune::ctune() {
   }
 
   for (FFTShape shape : shapes) {
-    u32 exponent = primes.prevPrime(FFTConfig{shape, 0}.maxExp());
+    u32 exponent = primes.prevPrime(FFTConfig{shape, 0, FFTConfig::CARRY_AUTO}.maxExp());
     // log("tuning %10s with exponent %u\n", fft.shape.spec().c_str(), exponent);
 
     vector<int> bestPos(configsVect.size());
@@ -240,7 +240,7 @@ void Tune::ctune() {
         for (u32 k = i + 1; k < configsVect.size(); ++k) {
           add(c, configsVect[k][bestPos[k]]);
         }
-        auto cost = Gpu::make(q, exponent, shared, FFTConfig{shape, 0}, c, false)->timePRP();
+        auto cost = Gpu::make(q, exponent, shared, FFTConfig{shape, 0, FFTConfig::CARRY_AUTO}, c, false)->timePRP();
 
         bool isBest = (cost < best.cost);
         if (isBest) {
@@ -264,22 +264,29 @@ void Tune::tune() {
   vector<TuneEntry> results = TuneEntry::readTuneFile(*args);
 
   for (const FFTShape& shape : FFTShape::multiSpec(args->fftSpec)) {
-    double costZero{};
-    for (u32 variant = 0; variant < FFTConfig::N_VARIANT; ++variant) {
-      FFTConfig fft{shape, variant};
+    double minCost = -1;
 
-      if (variant > 0 && !TuneEntry{costZero, fft}.willUpdate(results)) {
-        log("skipped %s\n", fft.spec().c_str());
-        continue;
-      }
+    // Time an exponent that's good for all variants and carry-config.
+    u32 exponent = primes.prevPrime(FFTConfig{shape, 0, FFTConfig::CARRY_32}.maxExp());
+
+    for (u32 variant = 0; variant < FFTConfig::N_VARIANT; ++variant) {
+      // We need to test both carry-32 and carry-64 only when the carry cutoff BPW is within the range.
+      bool testBoth = shape.carry32BPW() < FFTConfig{shape, variant, FFTConfig::CARRY_64}.maxBpw();
+      auto carries = testBoth ? vector{FFTConfig::CARRY_32, FFTConfig::CARRY_64} : vector{FFTConfig::CARRY_AUTO};
+      for (auto carry : carries) {
+        FFTConfig fft{shape, variant, carry};
+
+        if (minCost > 0 && !TuneEntry{minCost, fft}.willUpdate(results)) {
+          log("skipped %s %9u\n", fft.spec().c_str(), fft.maxExp());
+          continue;
+        }
 
-      u32 maxExp = fft.maxExp();
-      u32 exponent = primes.prevPrime(maxExp);
-      double cost = Gpu::make(q, exponent, shared, fft, {}, false)->timePRP();
-      if (variant == 0) { costZero = cost; }
+        double cost = Gpu::make(q, exponent, shared, fft, {}, false)->timePRP();
+        if (minCost <= 0) { minCost = cost; }
 
-      bool isUseful = TuneEntry{cost, fft}.update(results);
-      log("%c %6.0f %12s %9u\n", isUseful ? '*' : ' ', cost, fft.spec().c_str(), exponent);
+        bool isUseful = TuneEntry{cost, fft}.update(results);
+        log("%c %6.0f %12s %9u\n", isUseful ? '*' : ' ', cost, fft.spec().c_str(), fft.maxExp());
+      }
     }
   }
 
diff --git a/tune.txt b/tune.txt
@@ -0,0 +1,150 @@
+    68    256:2:256:2 # 5151391
+    84    256:3:256:0 # 7613448
+    85    256:3:256:3 # 7625244
+   111    256:4:256:2 # 10140778
+   112    256:4:256:3 # 10153361
+   127    256:5:256:2 # 12587499
+   129    256:5:256:3 # 12627476
+   150    512:3:256:0 # 14983888
+   150    512:3:256:2 # 15019278
+   153    512:3:256:3 # 15023996
+   153    256:3:512:2 # 15032647
+   155    256:6:256:3 # 15042084
+   166    256:7:256:2 # 17427988
+   166    256:7:256:3 # 17483038
+   193    512:4:256:0 # 19791872
+   194    256:8:256:0 # 19828572
+   195    256:8:256:2 # 19918749
+   196    512:4:256:2 # 19938672
+   198    512:4:256:3 # 19960692
+   203    256:9:256:2 # 22238724
+   207    256:9:256:3 # 22375563
+   224    512:5:256:0 # 24537989
+   225    512:5:256:2 # 24718868
+   228    512:5:256:3 # 24806686
+   235   256:10:256:3 # 24936448
+   241   256:11:256:2 # 27125874
+   252   256:11:256:3 # 27383955
+   269    512:6:256:0 # 29500637
+   269    512:6:256:2 # 29547823
+   270     1K:3:256:0 # 29617029
+   272     1K:3:256:2 # 29646913
+   272     1K:3:256:3 # 29678370
+   279   256:13:256:2 # 31965839
+   293    512:7:256:0 # 34259599
+   293    512:7:256:2 # 34309144
+   295    512:7:256:1 # 34377039
+   296    512:7:256:3 # 34439430
+   322   256:15:256:0 # 36588748
+   322   256:15:256:2 # 36679188
+   331   256:15:256:3 # 36991795
+   345    512:8:256:0 # 38944112
+   350    512:8:256:2 # 39097204
+   361    512:9:256:0 # 43774377
+   363    512:9:256:2 # 43864031
+   368    512:9:256:1 # 43993792
+   369    512:9:256:3 # 44052774
+   396    256:9:512:3 # 44102320
+   414     1K:5:256:0 # 48625090
+   415     1K:5:256:1 # 48735191
+   419     1K:5:256:2 # 48958013
+   420     1K:5:256:3 # 49112678
+   434   512:11:256:0 # 53063712
+   435   512:11:256:2 # 53285748
+   442   512:11:256:1 # 53401092
+   448   512:11:256:3 # 53744238
+   492   512:12:256:0 # 58029244
+   494   512:12:256:2 # 58129907
+   502   512:12:256:1 # 58337525
+   503   512:13:256:0 # 62452662
+   507   512:13:256:2 # 62841159
+   528   512:13:256:3 # 63168315
+   557     1K:7:256:0 # 67704455
+   557     1K:7:256:1 # 67924656
+   564     1K:7:256:3 # 68009066
+   586   512:15:256:1 # 72418590
+   590   512:15:256:3 # 72831467
+   672    512:8:512:0 # 76319555
+   679    512:8:512:2 # 76822872
+   682     1K:8:256:0 # 77007421
+   691     1K:8:256:2 # 77468794
+   702     1K:9:256:0 # 86411575
+   706     1K:9:256:1 # 86940057
+   713     1K:9:256:3 # 87081615
+   810    1K:10:256:0 # 95635374
+   823    1K:10:256:2 # 96254033
+   836     1K:5:512:3 # 96322191
+   838  1K:10:256:3:0 # 96936949
+   846    1K:11:256:0 # 104899018
+   856    1K:11:256:2 # 105331556
+   862    1K:11:256:1 # 105723723
+   871  1K:11:256:3:0 # 106234141
+   927  1K:11:256:3:1 # 106467688
+   968    1K:12:256:0 # 114517082
+   987    1K:12:256:1 # 115209142
+   988    1K:13:256:0 # 123733016
+   999    1K:13:256:2 # 124305539
+  1032  1K:13:256:3:0 # 124728115
+  1097  1K:13:256:3:1 # 125293821
+  1125     1K:7:512:0 # 133309661
+  1129     1K:7:512:1 # 133647302
+  1148     1K:7:512:3 # 133911543
+  1160  1K:15:256:1:0 # 143105258
+  1228  1K:15:256:1:1 # 143287910
+  1243  1K:15:256:3:1 # 143712583
+  1372     1K:8:512:0 # 151120773
+  1387     1K:8:512:1 # 151749918
+  1390     1K:8:512:2 # 152236457
+  1406   1K:8:512:3:0 # 152255079
+  1433     1K:9:512:0 # 170124115
+  1437   1K:9:512:1:0 # 170485157
+  1521   1K:9:512:1:1 # 170982899
+  1544   1K:9:512:3:1 # 171426447
+  1644    1K:10:512:0 # 187789475
+  1664    1K:10:512:1 # 188512993
+  1676  1K:10:512:2:0 # 188631019
+  1740    1K:11:512:0 # 206153187
+  1763  1K:11:512:2:0 # 206701114
+  1872  1K:11:512:1:1 # 207594979
+  1902  1K:11:512:3:1 # 209094443
+  1990  1K:12:512:0:0 # 224702353
+  2032  1K:13:512:0:0 # 242640486
+  2150  1K:13:512:0:1 # 242722275
+  2196  1K:13:512:2:1 # 244439842
+  2273  1K:13:512:3:1 # 245639413
+  2306   4K:7:256:0:0 # 260520378
+  2387  1K:15:512:0:0 # 278346196
+  2503  1K:15:512:1:1 # 281290997
+  2559  1K:15:512:3:1 # 283052605
+  2726   4K:8:256:0:0 # 296121551
+  2913   4K:9:256:0:0 # 331533131
+  3290    1K:9:1K:1:1 # 337700192
+  3350    1K:9:1K:3:1 # 338625036
+  3352  4K:10:256:0:0 # 366776278
+  3569  4K:11:256:0:0 # 401867893
+  4018  4K:12:256:0:0 # 436821794
+  4178  4K:13:256:0:0 # 471649484
+  4728  4K:14:256:0:0 # 506360692
+  4879  4K:15:256:0:0 # 540963752
+  5514   1K:15:1K:1:1 # 555913052
+  5617   1K:15:1K:3:1 # 558272348
+  5683   4K:8:512:0:0 # 575465886
+  6303   4K:9:512:0:0 # 644191894
+  7048  4K:10:512:0:0 # 712581036
+  7710  4K:11:512:0:0 # 780667115
+  8473  4K:12:512:0:0 # 848477765
+  8847  4K:13:512:0:0 # 916035993
+ 10176  4K:14:512:0:0 # 983361256
+ 10555  4K:15:512:0:0 # 1050470224
+ 12494  4K:15:512:1:1 # 1083892039
+ 12506  4K:15:512:3:1 # 1093203394
+ 12683    4K:8:1K:0:0 # 1117377340
+ 13761    4K:9:1K:0:0 # 1250635053
+ 15480   4K:10:1K:0:0 # 1383219032
+ 16880   4K:11:1K:0:0 # 1515196887
+ 18631   4K:12:1K:0:0 # 1646623883
+ 20056   4K:13:1K:0:0 # 1777546035
+ 22309   4K:14:1K:0:0 # 1908002257
+ 23235   4K:15:1K:0:0 # 2038025889
+ 27010   4K:15:1K:1:1 # 2145386496
+ 27238   4K:15:1K:3:1 # 2159731015

Original file line number	Diff line number	Diff line change
`@@ -272,7 +272,7 @@ void Args::parse(const string& line) {`
`272`	`272`	`log(" FFT \| BPW \| Max exp (M)\n");`
`273`	`273`	`for (const FFTShape& shape : FFTShape::multiSpec(s)) {`
`274`	`274`	`for (u32 variant = 0; variant < FFTConfig::N_VARIANT; ++variant) {`
`275`		`- FFTConfig fft{shape, variant};`
	`275`	`+ FFTConfig fft{shape, variant, FFTConfig::CARRY_AUTO};`
`276`	`276`	`log("%12s \| %.2f \| %5.1f\n", fft.spec().c_str(), fft.maxBpw(), fft.maxExp() / 1'000'000.0);`
`277`	`277`	`}`
`278`	`278`	`}`
Original file line number	Diff line number	Diff line change
`@@ -76,9 +76,9 @@ void TuneEntry::writeTuneFile(const vector<TuneEntry>& results) {`
`76`	`76`	`CycleFile tune{"tune.txt"};`
`77`	`77`	`for (const TuneEntry& r : results) {`
`78`	`78`	`u32 maxExp = r.fft.maxExp();`
`79`		`- assert(r.cost > prevCost && maxExp > prevMaxExp);`
	`79`	`+ assert(r.cost >= prevCost && maxExp > prevMaxExp);`
`80`	`80`	`prevCost = r.cost;`
`81`	`81`	`prevMaxExp = maxExp;`
`82`		`- tune->printf("%6.0f %12s # %u\n", r.cost, r.fft.spec().c_str(), maxExp);`
	`82`	`+ tune->printf("%6.0f %14s # %u\n", r.cost, r.fft.spec().c_str(), maxExp);`
`83`	`83`	`}`
`84`	`84`	`}`