Merge #20: Make built field sizes configurable

1e3abc3 Remove a few unnecessary includes (Pieter Wuille)
2818397 Compile less when all of a module's fields are disabled (Pieter Wuille)
b6f01a8 Test configured field sizes in CI (Pieter Wuille)
a0c7977 Clean up CI configuration (Pieter Wuille)
dd914ce Permit disabling of certain field sizes (Pieter Wuille)

Pull request description:

  This adds a `--enable-fields=LIST` configure options, which takes a list of integers. The default is all supported field sizes (currently 2 through 64 inclusive). Binary sizes and compile time go down significantly when reducing the list.

ACKs for top commit:
  gmaxwell:
    ACK 1e3abc3   much better.

Tree-SHA512: 6f2ab388b20e5c094bdda860a2acbfe28ade69c001abb1680144bc8536b0012e1b3a75ee79dc86c0fd5d319ebf5e01e4e35d30e23c5c905c1d3ec131a8e88184

Author: sipa

.cirrus.yml

@@ -5,9 +5,12 @@ env:
   BENCH: yes
   TESTRUNS:
   EXEC_CMD:
+  ENABLE_FIELDS:
 
 cat_logs_snippet: &CAT_LOGS
   on_failure:
+    cat_test_log_script:
+      - cat test-suite.log || true
     cat_config_log_script:
       - cat config.log || true
     cat_test_env_script:
@@ -23,25 +26,50 @@ merge_base_script_snippet: &MERGE_BASE
     - git config --global user.name "ci"
     - git merge FETCH_HEAD  # Merge base to detect silent merge conflicts
 
+env_matrix_snippet: &ENV_MATRIX_VALGRIND
+  - env:
+      ENABLE_FIELDS: "7,32,58"
+  - env:
+      BUILD: distcheck
+  - env:
+      EXEC_CMD: valgrind --error-exitcode=42
+      TESTRUNS: 1
+      BUILD:
+
+env_matrix_snippet: &ENV_MATRIX_SAN
+  - env:
+      ENABLE_FIELDS: 28
+  - env:
+      BUILD: distcheck
+  - env:
+      CFLAGS:  "-fsanitize=undefined -fno-omit-frame-pointer"
+      LDFLAGS: "-fsanitize=undefined -fno-omit-frame-pointer"
+      UBSAN_OPTIONS: "print_stacktrace=1:halt_on_error=1"
+      BENCH: no
+
+env_matrix_snippet: &ENV_MATRIX_SAN_VALGRIND
+  - env:
+      ENABLE_FIELDS: "11,64,37"
+  - env:
+      BUILD: distcheck
+  - env:
+      EXEC_CMD: valgrind --error-exitcode=42
+      TESTRUNS: 1
+      BUILD:
+  - env:
+      CFLAGS:  "-fsanitize=undefined -fno-omit-frame-pointer"
+      LDFLAGS: "-fsanitize=undefined -fno-omit-frame-pointer"
+      UBSAN_OPTIONS: "print_stacktrace=1:halt_on_error=1"
+      BENCH: no
+
 task:
   name: "x86_64: Linux (Debian stable)"
   container:
     dockerfile: ci/linux-debian.Dockerfile
     memory: 2G
     cpu: 4
-  matrix: &ENV_MATRIX
-    - env:
-    - env:
-        BUILD: distcheck
-    - env:
-        CFLAGS:  "-fsanitize=undefined -fno-omit-frame-pointer"
-        LDFLAGS: "-fsanitize=undefined -fno-omit-frame-pointer"
-        UBSAN_OPTIONS: "print_stacktrace=1:halt_on_error=1"
-        BENCH: no
-    - env:
-        EXEC_CMD: valgrind --error-exitcode=42
-        TESTRUNS: 1
-        BUILD:
+  matrix:
+    << : *ENV_MATRIX_SAN_VALGRIND
   matrix:
     - env:
         CC: gcc
@@ -60,6 +88,8 @@ task:
     cpu: 4
   env:
     HOST: i686-linux-gnu
+  matrix:
+    << : *ENV_MATRIX_VALGRIND
   matrix:
     - env:
         CC: i686-linux-gnu-gcc
@@ -77,12 +107,7 @@ task:
     # Cirrus gives us a fixed number of 12 virtual CPUs.
     MAKEFLAGS: -j13
   matrix:
-    - env: {}
-    - env: {BUILD: distcheck}
-    - env:
-        CFLAGS:  "-fsanitize=undefined -fno-omit-frame-pointer"
-        LDFLAGS: "-fsanitize=undefined -fno-omit-frame-pointer"
-        UBSAN_OPTIONS: "print_stacktrace=1:halt_on_error=1"
+    << : *ENV_MATRIX_SAN
   matrix:
     - env:
         CC: gcc-9

ci/cirrus.sh

@@ -12,7 +12,11 @@ valgrind --version || true
 
 ./autogen.sh
 
-./configure --host="$HOST" --enable-benchmark="$BENCH"
+FIELDS=
+if [ -n "$ENABLE_FIELDS" ]; then
+    FIELDS="--enable-fields=$ENABLE_FIELDS"
+fi
+./configure --host="$HOST" --enable-benchmark="$BENCH" $FIELDS
 
 # We have set "-j<n>" in MAKEFLAGS.
 make

configure.ac

@@ -30,6 +30,29 @@ AC_ARG_ENABLE(benchmark,
     [use_benchmark=$enableval],
     [use_benchmark=no])
 
+m4_define([SUPPORTED_FIELDS], [2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64])
+
+AC_MSG_CHECKING([which field sizes to build])
+AC_ARG_ENABLE([fields], AS_HELP_STRING([--enable-fields=LIST], [Comma-separated list of field sizes to build. Default=all. Available sizes:] m4_translit(m4_defn([SUPPORTED_FIELDS]), [,], [ ])), [], [enable_fields=SUPPORTED_FIELDS])
+have_disabled_fields=no
+have_enabled_fields=no
+m4_foreach([FIELD], [SUPPORTED_FIELDS], [
+    case ",$enable_fields," in
+    *,FIELD,*)
+        have_enabled_fields=yes
+        ;;
+    *)
+        AC_DEFINE(DISABLE_FIELD_[]FIELD, [1],
+        [Define to 1 to remove support for field size] FIELD [.])
+        have_disabled_fields=yes
+        ;;
+    esac
+])
+AC_MSG_RESULT([$enable_fields])
+if test "x$have_enabled_fields" = xno; then
+    AC_MSG_ERROR([No field sizes are enabled.])
+fi
+
 AX_CHECK_COMPILE_FLAG([-Werror],[CXXFLAG_WERROR="-Werror"],[CXXFLAG_WERROR=""])
 
 AX_CXX_COMPILE_STDCXX([11], [noext], [mandatory], [nodefault])
@@ -116,3 +139,8 @@ echo "  CXX                     = $CXX"
 echo "  CXXFLAGS                = $CXXFLAGS"
 echo "  CPPFLAGS                = $CPPFLAGS"
 echo "  LDFLAGS                 = $LDFLAGS"
+if test "$have_disabled_fields" = "yes"; then
+echo
+echo "Only compiling in support for field sizes: $enable_fields"
+echo "WARNING: this means the library will lack support for other field sizes entirely"
+fi

doc/gen_params.sage

@@ -312,14 +312,18 @@ def print_result(bits, style):
         assert(False)
 
 for bits in range(2, 65):
+    print("#ifndef DISABLE_FIELD_%i" % bits)
     print("// %i bit field" % bits)
     print_result(bits, INT)
+    print("#endif")
     print("")
 
 for bits in range(2, 65):
+    print("#ifndef DISABLE_FIELD_%i" % bits)
     print("// %i bit field" % bits)
     print_result(bits, CLMUL)
     print_result(bits, CLMUL_TRI)
+    print("#endif")
     print("")
 
 for bits in range(2, 65):

src/fields/clmul_1byte.cpp

@@ -6,82 +6,113 @@
 
 /* This file was substantially auto-generated by doc/gen_params.sage. */
 
-#include <stdint.h>
+#if !defined(DISABLE_FIELD_2) || !defined(DISABLE_FIELD_3) || !defined(DISABLE_FIELD_4) || !defined(DISABLE_FIELD_5) || !defined(DISABLE_FIELD_6) || !defined(DISABLE_FIELD_7) || !defined(DISABLE_FIELD_8)
 
 #include "clmul_common_impl.h"
 
 #include "../int_utils.h"
 #include "../lintrans.h"
 #include "../sketch_impl.h"
+
+#endif
+
 #include "../sketch.h"
 
 namespace {
-
+#ifndef DISABLE_FIELD_2
 // 2 bit field
 typedef RecLinTrans<uint8_t, 2> StatTableTRI2;
 constexpr StatTableTRI2 SQR_TABLE_TRI2({0x1, 0x3});
 constexpr StatTableTRI2 QRT_TABLE_TRI2({0x2, 0});
 typedef FieldTri<uint8_t, 2, 1, StatTableTRI2, &SQR_TABLE_TRI2, nullptr, nullptr, nullptr, nullptr, &QRT_TABLE_TRI2, IdTrans, &ID_TRANS, &ID_TRANS> FieldTri2;
+#endif
 
+#ifndef DISABLE_FIELD_3
 // 3 bit field
 typedef RecLinTrans<uint8_t, 3> StatTableTRI3;
 constexpr StatTableTRI3 SQR_TABLE_TRI3({0x1, 0x4, 0x6});
 constexpr StatTableTRI3 QRT_TABLE_TRI3({0, 0x4, 0x6});
 typedef FieldTri<uint8_t, 3, 1, StatTableTRI3, &SQR_TABLE_TRI3, nullptr, nullptr, nullptr, nullptr, &QRT_TABLE_TRI3, IdTrans, &ID_TRANS, &ID_TRANS> FieldTri3;
+#endif
 
+#ifndef DISABLE_FIELD_4
 // 4 bit field
 typedef RecLinTrans<uint8_t, 4> StatTableTRI4;
 constexpr StatTableTRI4 SQR_TABLE_TRI4({0x1, 0x4, 0x3, 0xc});
 constexpr StatTableTRI4 QRT_TABLE_TRI4({0x6, 0xa, 0x8, 0});
 typedef FieldTri<uint8_t, 4, 1, StatTableTRI4, &SQR_TABLE_TRI4, nullptr, nullptr, nullptr, nullptr, &QRT_TABLE_TRI4, IdTrans, &ID_TRANS, &ID_TRANS> FieldTri4;
+#endif
 
+#ifndef DISABLE_FIELD_5
 // 5 bit field
 typedef RecLinTrans<uint8_t, 5> StatTable5;
 constexpr StatTable5 SQR_TABLE_5({0x1, 0x4, 0x10, 0xa, 0xd});
 constexpr StatTable5 SQR2_TABLE_5({0x1, 0x10, 0xd, 0xe, 0x1b});
 constexpr StatTable5 QRT_TABLE_5({0x14, 0x8, 0xa, 0, 0xe});
 typedef Field<uint8_t, 5, 5, StatTable5, &SQR_TABLE_5, &SQR2_TABLE_5, nullptr, nullptr, nullptr, &QRT_TABLE_5, IdTrans, &ID_TRANS, &ID_TRANS> Field5;
 typedef FieldTri<uint8_t, 5, 2, RecLinTrans<uint8_t, 5>, &SQR_TABLE_5, &SQR2_TABLE_5, nullptr, nullptr, nullptr, &QRT_TABLE_5, IdTrans, &ID_TRANS, &ID_TRANS> FieldTri5;
+#endif
 
+#ifndef DISABLE_FIELD_6
 // 6 bit field
 typedef RecLinTrans<uint8_t, 6> StatTableTRI6;
 constexpr StatTableTRI6 SQR_TABLE_TRI6({0x1, 0x4, 0x10, 0x3, 0xc, 0x30});
 constexpr StatTableTRI6 SQR2_TABLE_TRI6({0x1, 0x10, 0xc, 0x5, 0x13, 0x3c});
 constexpr StatTableTRI6 QRT_TABLE_TRI6({0x3a, 0x26, 0x24, 0x14, 0x20, 0});
 typedef FieldTri<uint8_t, 6, 1, StatTableTRI6, &SQR_TABLE_TRI6, &SQR2_TABLE_TRI6, nullptr, nullptr, nullptr, &QRT_TABLE_TRI6, IdTrans, &ID_TRANS, &ID_TRANS> FieldTri6;
+#endif
 
+#ifndef DISABLE_FIELD_7
 // 7 bit field
 typedef RecLinTrans<uint8_t, 4, 3> StatTableTRI7;
 constexpr StatTableTRI7 SQR_TABLE_TRI7({0x1, 0x4, 0x10, 0x40, 0x6, 0x18, 0x60});
 constexpr StatTableTRI7 SQR2_TABLE_TRI7({0x1, 0x10, 0x6, 0x60, 0x14, 0x46, 0x78});
 constexpr StatTableTRI7 QRT_TABLE_TRI7({0, 0x14, 0x16, 0x72, 0x12, 0x40, 0x7a});
 typedef FieldTri<uint8_t, 7, 1, StatTableTRI7, &SQR_TABLE_TRI7, &SQR2_TABLE_TRI7, nullptr, nullptr, nullptr, &QRT_TABLE_TRI7, IdTrans, &ID_TRANS, &ID_TRANS> FieldTri7;
+#endif
 
+#ifndef DISABLE_FIELD_8
 // 8 bit field
 typedef RecLinTrans<uint8_t, 4, 4> StatTable8;
 constexpr StatTable8 SQR_TABLE_8({0x1, 0x4, 0x10, 0x40, 0x1b, 0x6c, 0xab, 0x9a});
 constexpr StatTable8 SQR2_TABLE_8({0x1, 0x10, 0x1b, 0xab, 0x5e, 0x97, 0xb3, 0xc5});
 constexpr StatTable8 QRT_TABLE_8({0xbc, 0x2a, 0x28, 0x86, 0x2c, 0xde, 0x8e, 0});
 typedef Field<uint8_t, 8, 27, StatTable8, &SQR_TABLE_8, &SQR2_TABLE_8, nullptr, nullptr, nullptr, &QRT_TABLE_8, IdTrans, &ID_TRANS, &ID_TRANS> Field8;
-
+#endif
 }
 
 Sketch* ConstructClMul1Byte(int bits, int implementation) {
     switch (bits) {
+#ifndef DISABLE_FIELD_5
     case 5: return new SketchImpl<Field5>(implementation, 5);
+#endif
+#ifndef DISABLE_FIELD_8
     case 8: return new SketchImpl<Field8>(implementation, 8);
+#endif
     }
     return nullptr;
 }
 
 Sketch* ConstructClMulTri1Byte(int bits, int implementation) {
     switch (bits) {
+#ifndef DISABLE_FIELD_2
     case 2: return new SketchImpl<FieldTri2>(implementation, 2);
+#endif
+#ifndef DISABLE_FIELD_3
     case 3: return new SketchImpl<FieldTri3>(implementation, 3);
+#endif
+#ifndef DISABLE_FIELD_4
     case 4: return new SketchImpl<FieldTri4>(implementation, 4);
+#endif
+#ifndef DISABLE_FIELD_5
     case 5: return new SketchImpl<FieldTri5>(implementation, 5);
+#endif
+#ifndef DISABLE_FIELD_6
     case 6: return new SketchImpl<FieldTri6>(implementation, 6);
+#endif
+#ifndef DISABLE_FIELD_7
     case 7: return new SketchImpl<FieldTri7>(implementation, 7);
+#endif
     }
     return nullptr;
 }