Merge #20861: BIP 350: Implement Bech32m and use it for v1+ segwit addresses

0334602 naming nits (Fabian Jahr)
2e7c80f Add signet support to gen_key_io_test_vectors.py (Pieter Wuille)
fe5e495 Use Bech32m encoding for v1+ segwit addresses (Pieter Wuille)
25b1c6e Add Bech32m test vectors (Pieter Wuille)
da2bb69 Implement Bech32m encoding/decoding (Pieter Wuille)

Pull request description:

  This implements [BIP 350](https://github.com/bitcoin/bips/blob/master/bip-0350.mediawiki):
  * For segwit v1+ addresses, a new checksum algorithm called Bech32m is used.
  * Segwit v0 address keep using Bech32 as specified in [BIP 173](https://github.com/bitcoin/bips/blob/master/bip-0173.mediawiki).

ACKs for top commit:
  Sjors:
    utACK 0334602
  jnewbery:
    utACK 0334602
  achow101:
    ACK 0334602
  fjahr:
    re-ACK 0334602
  benthecarman:
    ACK 0334602

Tree-SHA512: 4424cfd44869d813d6152fb3ed867b204036736bc2344a039b93700b6f36a43e9110478f138eb81c97c77ab27ecb776dada5ba632cb5a3a9d244924d2540a557

Author: laanwj

contrib/testgen/README.md

@@ -4,5 +4,5 @@ Utilities to generate test vectors for the data-driven Bitcoin tests.
 
 Usage:
 
-    PYTHONPATH=../../test/functional/test_framework ./gen_key_io_test_vectors.py valid 50 > ../../src/test/data/key_io_valid.json
-    PYTHONPATH=../../test/functional/test_framework ./gen_key_io_test_vectors.py invalid 50 > ../../src/test/data/key_io_invalid.json
+    PYTHONPATH=../../test/functional/test_framework ./gen_key_io_test_vectors.py valid 70 > ../../src/test/data/key_io_valid.json
+    PYTHONPATH=../../test/functional/test_framework ./gen_key_io_test_vectors.py invalid 70 > ../../src/test/data/key_io_invalid.json

contrib/testgen/gen_key_io_test_vectors.py

@@ -3,19 +3,19 @@
 # Distributed under the MIT software license, see the accompanying
 # file COPYING or http://www.opensource.org/licenses/mit-license.php.
 '''
-Generate valid and invalid base58 address and private key test vectors.
+Generate valid and invalid base58/bech32(m) address and private key test vectors.
 
 Usage:
-    PYTHONPATH=../../test/functional/test_framework ./gen_key_io_test_vectors.py valid 50 > ../../src/test/data/key_io_valid.json
-    PYTHONPATH=../../test/functional/test_framework ./gen_key_io_test_vectors.py invalid 50 > ../../src/test/data/key_io_invalid.json
+    PYTHONPATH=../../test/functional/test_framework ./gen_key_io_test_vectors.py valid 70 > ../../src/test/data/key_io_valid.json
+    PYTHONPATH=../../test/functional/test_framework ./gen_key_io_test_vectors.py invalid 70 > ../../src/test/data/key_io_invalid.json
 '''
 # 2012 Wladimir J. van der Laan
 # Released under MIT License
 import os
 from itertools import islice
 from base58 import b58encode_chk, b58decode_chk, b58chars
 import random
-from segwit_addr import bech32_encode, decode_segwit_address, convertbits, CHARSET
+from segwit_addr import bech32_encode, decode_segwit_address, convertbits, CHARSET, Encoding
 
 # key types
 PUBKEY_ADDRESS = 0
@@ -32,6 +32,7 @@
 OP_0 = 0x00
 OP_1 = 0x51
 OP_2 = 0x52
+OP_3 = 0x53
 OP_16 = 0x60
 OP_DUP = 0x76
 OP_EQUAL = 0x87
@@ -44,6 +45,7 @@
 script_suffix = (OP_EQUAL,)
 p2wpkh_prefix = (OP_0, 20)
 p2wsh_prefix = (OP_0, 32)
+p2tr_prefix = (OP_1, 32)
 
 metadata_keys = ['isPrivkey', 'chain', 'isCompressed', 'tryCaseFlip']
 # templates for valid sequences
@@ -54,40 +56,58 @@
   ((SCRIPT_ADDRESS,),         20, (),   (False, 'main',    None,  None), script_prefix, script_suffix),
   ((PUBKEY_ADDRESS_TEST,),    20, (),   (False, 'test',    None,  None), pubkey_prefix, pubkey_suffix),
   ((SCRIPT_ADDRESS_TEST,),    20, (),   (False, 'test',    None,  None), script_prefix, script_suffix),
+  ((PUBKEY_ADDRESS_TEST,),    20, (),   (False, 'signet',  None,  None), pubkey_prefix, pubkey_suffix),
+  ((SCRIPT_ADDRESS_TEST,),    20, (),   (False, 'signet',  None,  None), script_prefix, script_suffix),
   ((PUBKEY_ADDRESS_REGTEST,), 20, (),   (False, 'regtest', None,  None), pubkey_prefix, pubkey_suffix),
   ((SCRIPT_ADDRESS_REGTEST,), 20, (),   (False, 'regtest', None,  None), script_prefix, script_suffix),
   ((PRIVKEY,),                32, (),   (True,  'main',    False, None), (),            ()),
   ((PRIVKEY,),                32, (1,), (True,  'main',    True,  None), (),            ()),
   ((PRIVKEY_TEST,),           32, (),   (True,  'test',    False, None), (),            ()),
   ((PRIVKEY_TEST,),           32, (1,), (True,  'test',    True,  None), (),            ()),
+  ((PRIVKEY_TEST,),           32, (),   (True,  'signet',  False, None), (),            ()),
+  ((PRIVKEY_TEST,),           32, (1,), (True,  'signet',  True,  None), (),            ()),
   ((PRIVKEY_REGTEST,),        32, (),   (True,  'regtest', False, None), (),            ()),
   ((PRIVKEY_REGTEST,),        32, (1,), (True,  'regtest', True,  None), (),            ())
 ]
 # templates for valid bech32 sequences
 bech32_templates = [
-  # hrp, version, witprog_size, metadata, output_prefix
-  ('bc',    0, 20, (False, 'main',    None, True), p2wpkh_prefix),
-  ('bc',    0, 32, (False, 'main',    None, True), p2wsh_prefix),
-  ('bc',    1,  2, (False, 'main',    None, True), (OP_1, 2)),
-  ('tb',    0, 20, (False, 'test',    None, True), p2wpkh_prefix),
-  ('tb',    0, 32, (False, 'test',    None, True), p2wsh_prefix),
-  ('tb',    2, 16, (False, 'test',    None, True), (OP_2, 16)),
-  ('bcrt',  0, 20, (False, 'regtest', None, True), p2wpkh_prefix),
-  ('bcrt',  0, 32, (False, 'regtest', None, True), p2wsh_prefix),
-  ('bcrt', 16, 40, (False, 'regtest', None, True), (OP_16, 40))
+  # hrp, version, witprog_size, metadata, encoding, output_prefix
+  ('bc',    0, 20, (False, 'main',    None, True), Encoding.BECH32,  p2wpkh_prefix),
+  ('bc',    0, 32, (False, 'main',    None, True), Encoding.BECH32,  p2wsh_prefix),
+  ('bc',    1, 32, (False, 'main',    None, True), Encoding.BECH32M, p2tr_prefix),
+  ('bc',    2,  2, (False, 'main',    None, True), Encoding.BECH32M, (OP_2, 2)),
+  ('tb',    0, 20, (False, 'test',    None, True), Encoding.BECH32,  p2wpkh_prefix),
+  ('tb',    0, 32, (False, 'test',    None, True), Encoding.BECH32,  p2wsh_prefix),
+  ('tb',    1, 32, (False, 'test',    None, True), Encoding.BECH32M, p2tr_prefix),
+  ('tb',    3, 16, (False, 'test',    None, True), Encoding.BECH32M, (OP_3, 16)),
+  ('tb',    0, 20, (False, 'signet',  None, True), Encoding.BECH32,  p2wpkh_prefix),
+  ('tb',    0, 32, (False, 'signet',  None, True), Encoding.BECH32,  p2wsh_prefix),
+  ('tb',    1, 32, (False, 'signet',  None, True), Encoding.BECH32M, p2tr_prefix),
+  ('tb',    3, 32, (False, 'signet',  None, True), Encoding.BECH32M, (OP_3, 32)),
+  ('bcrt',  0, 20, (False, 'regtest', None, True), Encoding.BECH32,  p2wpkh_prefix),
+  ('bcrt',  0, 32, (False, 'regtest', None, True), Encoding.BECH32,  p2wsh_prefix),
+  ('bcrt',  1, 32, (False, 'regtest', None, True), Encoding.BECH32M, p2tr_prefix),
+  ('bcrt', 16, 40, (False, 'regtest', None, True), Encoding.BECH32M, (OP_16, 40))
 ]
 # templates for invalid bech32 sequences
 bech32_ng_templates = [
-  # hrp, version, witprog_size, invalid_bech32, invalid_checksum, invalid_char
-  ('tc',    0, 20, False, False, False),
-  ('tb',   17, 32, False, False, False),
-  ('bcrt',  3,  1, False, False, False),
-  ('bc',   15, 41, False, False, False),
-  ('tb',    0, 16, False, False, False),
-  ('bcrt',  0, 32, True,  False, False),
-  ('bc',    0, 16, True,  False, False),
-  ('tb',    0, 32, False, True,  False),
-  ('bcrt',  0, 20, False, False, True)
+  # hrp, version, witprog_size, encoding, invalid_bech32, invalid_checksum, invalid_char
+  ('tc',    0, 20, Encoding.BECH32,  False, False, False),
+  ('bt',    1, 32, Encoding.BECH32M, False, False, False),
+  ('tb',   17, 32, Encoding.BECH32M, False, False, False),
+  ('bcrt',  3,  1, Encoding.BECH32M, False, False, False),
+  ('bc',   15, 41, Encoding.BECH32M, False, False, False),
+  ('tb',    0, 16, Encoding.BECH32,  False, False, False),
+  ('bcrt',  0, 32, Encoding.BECH32,  True,  False, False),
+  ('bc',    0, 16, Encoding.BECH32,  True,  False, False),
+  ('tb',    0, 32, Encoding.BECH32,  False, True,  False),
+  ('bcrt',  0, 20, Encoding.BECH32,  False, False, True),
+  ('bc',    0, 20, Encoding.BECH32M, False, False, False),
+  ('tb',    0, 32, Encoding.BECH32M, False, False, False),
+  ('bcrt',  0, 20, Encoding.BECH32M, False, False, False),
+  ('bc',    1, 32, Encoding.BECH32,  False, False, False),
+  ('tb',    2, 16, Encoding.BECH32,  False, False, False),
+  ('bcrt', 16, 20, Encoding.BECH32,  False, False, False),
 ]
 
 def is_valid(v):
@@ -127,8 +147,9 @@ def gen_valid_bech32_vector(template):
     hrp = template[0]
     witver = template[1]
     witprog = bytearray(os.urandom(template[2]))
-    dst_prefix = bytearray(template[4])
-    rv = bech32_encode(hrp, [witver] + convertbits(witprog, 8, 5))
+    encoding = template[4]
+    dst_prefix = bytearray(template[5])
+    rv = bech32_encode(hrp, [witver] + convertbits(witprog, 8, 5), encoding)
     return rv, dst_prefix + witprog
 
 def gen_valid_vectors():
@@ -186,22 +207,23 @@ def gen_invalid_bech32_vector(template):
     hrp = template[0]
     witver = template[1]
     witprog = bytearray(os.urandom(template[2]))
+    encoding = template[3]
 
     if no_data:
-        rv = bech32_encode(hrp, [])
+        rv = bech32_encode(hrp, [], encoding)
     else:
         data = [witver] + convertbits(witprog, 8, 5)
-        if template[3] and not no_data:
+        if template[4] and not no_data:
             if template[2] % 5 in {2, 4}:
                 data[-1] |= 1
             else:
                 data.append(0)
-        rv = bech32_encode(hrp, data)
+        rv = bech32_encode(hrp, data, encoding)
 
-    if template[4]:
+    if template[5]:
         i = len(rv) - random.randrange(1, 7)
         rv = rv[:i] + random.choice(CHARSET.replace(rv[i], '')) + rv[i + 1:]
-    if template[5]:
+    if template[6]:
         i = len(hrp) + 1 + random.randrange(0, len(rv) - len(hrp) - 4)
         rv = rv[:i] + rv[i:i + 4].upper() + rv[i + 4:]
 

doc/bips.md

@@ -1,4 +1,4 @@
-BIPs that are implemented by Bitcoin Core (up-to-date up to **v0.21.0**):
+BIPs that are implemented by Bitcoin Core (up-to-date up to **v22.0**):
 
 * [`BIP 9`](https://github.com/bitcoin/bips/blob/master/bip-0009.mediawiki): The changes allowing multiple soft-forks to be deployed in parallel have been implemented since **v0.12.1**  ([PR #7575](https://github.com/bitcoin/bitcoin/pull/7575))
 * [`BIP 11`](https://github.com/bitcoin/bips/blob/master/bip-0011.mediawiki): Multisig outputs are standard since **v0.6.0** ([PR #669](https://github.com/bitcoin/bitcoin/pull/669)).
@@ -50,3 +50,4 @@ BIPs that are implemented by Bitcoin Core (up-to-date up to **v0.21.0**):
 * [`BIP 325`](https://github.com/bitcoin/bips/blob/master/bip-0325.mediawiki): Signet test network is supported as of **v0.21.0** ([PR 18267](https://github.com/bitcoin/bitcoin/pull/18267)).
 * [`BIP 339`](https://github.com/bitcoin/bips/blob/master/bip-0339.mediawiki): Relay of transactions by wtxid is supported as of **v0.21.0** ([PR 18044](https://github.com/bitcoin/bitcoin/pull/18044)).
 * [`BIP 340`](https://github.com/bitcoin/bips/blob/master/bip-0340.mediawiki) [`341`](https://github.com/bitcoin/bips/blob/master/bip-0341.mediawiki) [`342`](https://github.com/bitcoin/bips/blob/master/bip-0342.mediawiki): Validation rules for Taproot (including Schnorr signatures and Tapscript leaves) are implemented as of **v0.21.0** ([PR 19953](https://github.com/bitcoin/bitcoin/pull/19953)), without mainnet activation.
+* [`BIP 350`](https://github.com/bitcoin/bips/blob/master/bip-0350.mediawiki): Addresses for native v1+ segregated Witness outputs use Bech32m instead of Bech32 as of **v22.0** ([PR 20861](https://github.com/bitcoin/bitcoin/pull/20861)).

doc/release-notes-20861.md

@@ -0,0 +1,13 @@
+Updated RPCs
+------------
+
+- Due to [BIP 350](https://github.com/bitcoin/bips/blob/master/bip-0350.mediawiki)
+  being implemented, behavior for all RPCs that accept addresses is changed when
+  a native witness version 1 (or higher) is passed. These now require a Bech32m
+  encoding instead of a Bech32 one, and Bech32m encoding will be used for such
+  addresses in RPC output as well. No version 1 addresses should be created
+  for mainnet until consensus rules are adopted that give them meaning
+  (e.g. through [BIP 341](https://github.com/bitcoin/bips/blob/master/bip-0341.mediawiki)).
+  Once that happens, Bech32m is expected to be used for them, so this shouldn't
+  affect any production systems, but may be observed on other networks where such
+  addresses already have meaning (like signet).

src/bech32.cpp

@@ -1,4 +1,4 @@
-// Copyright (c) 2017 Pieter Wuille
+// Copyright (c) 2017, 2021 Pieter Wuille
 // Distributed under the MIT software license, see the accompanying
 // file COPYING or http://www.opensource.org/licenses/mit-license.php.
 
@@ -7,15 +7,18 @@
 
 #include <assert.h>
 
+namespace bech32
+{
+
 namespace
 {
 
 typedef std::vector<uint8_t> data;
 
-/** The Bech32 character set for encoding. */
+/** The Bech32 and Bech32m character set for encoding. */
 const char* CHARSET = "qpzry9x8gf2tvdw0s3jn54khce6mua7l";
 
-/** The Bech32 character set for decoding. */
+/** The Bech32 and Bech32m character set for decoding. */
 const int8_t CHARSET_REV[128] = {
     -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
     -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
@@ -27,6 +30,12 @@ const int8_t CHARSET_REV[128] = {
      1,  0,  3, 16, 11, 28, 12, 14,  6,  4,  2, -1, -1, -1, -1, -1
 };
 
+/* Determine the final constant to use for the specified encoding. */
+uint32_t EncodingConstant(Encoding encoding) {
+    assert(encoding == Encoding::BECH32 || encoding == Encoding::BECH32M);
+    return encoding == Encoding::BECH32 ? 1 : 0x2bc830a3;
+}
+
 /** This function will compute what 6 5-bit values to XOR into the last 6 input values, in order to
  *  make the checksum 0. These 6 values are packed together in a single 30-bit integer. The higher
  *  bits correspond to earlier values. */
@@ -111,21 +120,24 @@ data ExpandHRP(const std::string& hrp)
 }
 
 /** Verify a checksum. */
-bool VerifyChecksum(const std::string& hrp, const data& values)
+Encoding VerifyChecksum(const std::string& hrp, const data& values)
 {
     // PolyMod computes what value to xor into the final values to make the checksum 0. However,
     // if we required that the checksum was 0, it would be the case that appending a 0 to a valid
     // list of values would result in a new valid list. For that reason, Bech32 requires the
-    // resulting checksum to be 1 instead.
-    return PolyMod(Cat(ExpandHRP(hrp), values)) == 1;
+    // resulting checksum to be 1 instead. In Bech32m, this constant was amended.
+    const uint32_t check = PolyMod(Cat(ExpandHRP(hrp), values));
+    if (check == EncodingConstant(Encoding::BECH32)) return Encoding::BECH32;
+    if (check == EncodingConstant(Encoding::BECH32M)) return Encoding::BECH32M;
+    return Encoding::INVALID;
 }
 
 /** Create a checksum. */
-data CreateChecksum(const std::string& hrp, const data& values)
+data CreateChecksum(Encoding encoding, const std::string& hrp, const data& values)
 {
     data enc = Cat(ExpandHRP(hrp), values);
     enc.resize(enc.size() + 6); // Append 6 zeroes
-    uint32_t mod = PolyMod(enc) ^ 1; // Determine what to XOR into those 6 zeroes.
+    uint32_t mod = PolyMod(enc) ^ EncodingConstant(encoding); // Determine what to XOR into those 6 zeroes.
     data ret(6);
     for (size_t i = 0; i < 6; ++i) {
         // Convert the 5-bit groups in mod to checksum values.
@@ -136,16 +148,13 @@ data CreateChecksum(const std::string& hrp, const data& values)
 
 } // namespace
 
-namespace bech32
-{
-
-/** Encode a Bech32 string. */
-std::string Encode(const std::string& hrp, const data& values) {
-    // First ensure that the HRP is all lowercase. BIP-173 requires an encoder
-    // to return a lowercase Bech32 string, but if given an uppercase HRP, the
+/** Encode a Bech32 or Bech32m string. */
+std::string Encode(Encoding encoding, const std::string& hrp, const data& values) {
+    // First ensure that the HRP is all lowercase. BIP-173 and BIP350 require an encoder
+    // to return a lowercase Bech32/Bech32m string, but if given an uppercase HRP, the
     // result will always be invalid.
     for (const char& c : hrp) assert(c < 'A' || c > 'Z');
-    data checksum = CreateChecksum(hrp, values);
+    data checksum = CreateChecksum(encoding, hrp, values);
     data combined = Cat(values, checksum);
     std::string ret = hrp + '1';
     ret.reserve(ret.size() + combined.size());
@@ -155,8 +164,8 @@ std::string Encode(const std::string& hrp, const data& values) {
     return ret;
 }
 
-/** Decode a Bech32 string. */
-std::pair<std::string, data> Decode(const std::string& str) {
+/** Decode a Bech32 or Bech32m string. */
+DecodeResult Decode(const std::string& str) {
     bool lower = false, upper = false;
     for (size_t i = 0; i < str.size(); ++i) {
         unsigned char c = str[i];
@@ -183,10 +192,9 @@ std::pair<std::string, data> Decode(const std::string& str) {
     for (size_t i = 0; i < pos; ++i) {
         hrp += LowerCase(str[i]);
     }
-    if (!VerifyChecksum(hrp, values)) {
-        return {};
-    }
-    return {hrp, data(values.begin(), values.end() - 6)};
+    Encoding result = VerifyChecksum(hrp, values);
+    if (result == Encoding::INVALID) return {};
+    return {result, std::move(hrp), data(values.begin(), values.end() - 6)};
 }
 
 } // namespace bech32

src/bech32.h

@@ -1,13 +1,14 @@
-// Copyright (c) 2017 Pieter Wuille
+// Copyright (c) 2017, 2021 Pieter Wuille
 // Distributed under the MIT software license, see the accompanying
 // file COPYING or http://www.opensource.org/licenses/mit-license.php.
 
-// Bech32 is a string encoding format used in newer address types.
-// The output consists of a human-readable part (alphanumeric), a
-// separator character (1), and a base32 data section, the last
-// 6 characters of which are a checksum.
+// Bech32 and Bech32m are string encoding formats used in newer
+// address types. The outputs consist of a human-readable part
+// (alphanumeric), a separator character (1), and a base32 data
+// section, the last 6 characters of which are a checksum. The
+// module is namespaced under bech32 for historical reasons.
 //
-// For more information, see BIP 173.
+// For more information, see BIP 173 and BIP 350.
 
 #ifndef BITCOIN_BECH32_H
 #define BITCOIN_BECH32_H
@@ -19,11 +20,29 @@
 namespace bech32
 {
 
-/** Encode a Bech32 string. If hrp contains uppercase characters, this will cause an assertion error. */
-std::string Encode(const std::string& hrp, const std::vector<uint8_t>& values);
+enum class Encoding {
+    INVALID, //!< Failed decoding
 
-/** Decode a Bech32 string. Returns (hrp, data). Empty hrp means failure. */
-std::pair<std::string, std::vector<uint8_t>> Decode(const std::string& str);
+    BECH32,  //!< Bech32 encoding as defined in BIP173
+    BECH32M, //!< Bech32m encoding as defined in BIP350
+};
+
+/** Encode a Bech32 or Bech32m string. If hrp contains uppercase characters, this will cause an
+ *  assertion error. Encoding must be one of BECH32 or BECH32M. */
+std::string Encode(Encoding encoding, const std::string& hrp, const std::vector<uint8_t>& values);
+
+struct DecodeResult
+{
+    Encoding encoding;         //!< What encoding was detected in the result; Encoding::INVALID if failed.
+    std::string hrp;           //!< The human readable part
+    std::vector<uint8_t> data; //!< The payload (excluding checksum)
+
+    DecodeResult() : encoding(Encoding::INVALID) {}
+    DecodeResult(Encoding enc, std::string&& h, std::vector<uint8_t>&& d) : encoding(enc), hrp(std::move(h)), data(std::move(d)) {}
+};
+
+/** Decode a Bech32 or Bech32m string. */
+DecodeResult Decode(const std::string& str);
 
 } // namespace bech32
 

src/bench/bech32.cpp

@@ -19,7 +19,7 @@ static void Bech32Encode(benchmark::Bench& bench)
     tmp.reserve(1 + 32 * 8 / 5);
     ConvertBits<8, 5, true>([&](unsigned char c) { tmp.push_back(c); }, v.begin(), v.end());
     bench.batch(v.size()).unit("byte").run([&] {
-        bech32::Encode("bc", tmp);
+        bech32::Encode(bech32::Encoding::BECH32, "bc", tmp);
     });
 }