multi: make forward-compatible by parsing unknown odd types

This commit makes it possible for the protocol to add new odd (optional)
types in the future without old clients breaking.
By parsing and keeping any left-over unknown odd types, then re-encoding
them when calculating the commitment leaf, old software can still
correctly use and validate an asset, even if it doesn't fully understand
all of its (optional) new fields.

Author: guggero

address/address.go

@@ -415,7 +415,9 @@ func (a *Tap) Decode(r io.Reader) error {
 		return err
 	}
 
-	return asset.TlvStrictDecodeP2P(stream, r, KnownAddressTypes)
+	// TODO(guggero): Store unknown types (next commits).
+	_, err = asset.TlvStrictDecodeP2P(stream, r, KnownAddressTypes)
+	return err
 }
 
 // EncodeAddress returns a bech32m string encoding of a Taproot Asset address.

asset/asset.go

@@ -1470,6 +1470,16 @@ type Asset struct {
 	// together across distinct asset IDs, allowing further issuance of the
 	// asset to be made possible.
 	GroupKey *GroupKey
+
+	// UnknownOddTypes is a map of unknown odd types that were encountered
+	// during decoding. This map is used to preserve unknown types that we
+	// don't know of yet, so we can still encode them back when serializing
+	// as a leaf to arrive at the same byte representation and with that
+	// same commitment root hash. This enables forward compatibility with
+	// future versions of the protocol as it allows new odd (optional) types
+	// to be added without breaking old clients that don't yet fully
+	// understand them.
+	UnknownOddTypes tlv.TypeMap
 }
 
 // IsUnknownVersion returns true if an asset has a version that is not
@@ -1924,7 +1934,9 @@ func (a *Asset) encodeRecords(encodeType EncodeType) []tlv.Record {
 	if a.GroupKey != nil {
 		records = append(records, NewLeafGroupKeyRecord(&a.GroupKey))
 	}
-	return records
+
+	// Add any unknown odd types that were encountered during decoding.
+	return CombineRecords(records, a.UnknownOddTypes)
 }
 
 // EncodeRecords determines the non-nil records to include when encoding an
@@ -2001,7 +2013,14 @@ func (a *Asset) Decode(r io.Reader) error {
 		return err
 	}
 
-	return TlvStrictDecode(stream, r, KnownAssetLeafTypes)
+	unknownOddTypes, err := TlvStrictDecode(stream, r, KnownAssetLeafTypes)
+	if err != nil {
+		return err
+	}
+
+	a.UnknownOddTypes = unknownOddTypes
+
+	return nil
 }
 
 // Leaf returns the asset encoded as a MS-SMT leaf node.

asset/asset_test.go

@@ -16,6 +16,7 @@ import (
 	"github.com/lightninglabs/taproot-assets/mssmt"
 	"github.com/lightningnetwork/lnd/input"
 	"github.com/lightningnetwork/lnd/keychain"
+	"github.com/lightningnetwork/lnd/tlv"
 	"github.com/stretchr/testify/require"
 )
 
@@ -496,6 +497,18 @@ func TestAssetEncoding(t *testing.T) {
 		ScriptKey: NewScriptKey(pubKey),
 	})
 
+	assertAssetEncoding("minimal asset with unknown odd type", &Asset{
+		Genesis: Genesis{
+			MetaHash: [MetaHashLen]byte{},
+		},
+		ScriptKey: NewScriptKey(pubKey),
+		UnknownOddTypes: tlv.TypeMap{
+			test.TestVectorAllowedUnknownType: []byte(
+				"the great unknown",
+			),
+		},
+	})
+
 	// Write test vectors to file. This is a no-op if the "gen_test_vectors"
 	// build tag is not set.
 	test.WriteTestVectors(t, generatedTestVectorName, testVectors)
@@ -1073,6 +1086,14 @@ func runBIPTestVector(t *testing.T, testVectors *TestVectors) {
 			// yet. If the following check fails, you need to update
 			// the KnownAssetLeafTypes set.
 			for _, record := range a.encodeRecords(EncodeNormal) {
+				// Test vectors may contain this one type to
+				// demonstrate that it is not rejected.
+				if record.Type() ==
+					test.TestVectorAllowedUnknownType {
+
+					continue
+				}
+
 				require.Contains(
 					tt, KnownAssetLeafTypes, record.Type(),
 				)

asset/mock.go

@@ -17,6 +17,7 @@ import (
 	"github.com/lightninglabs/taproot-assets/mssmt"
 	"github.com/lightningnetwork/lnd/input"
 	"github.com/lightningnetwork/lnd/keychain"
+	"github.com/lightningnetwork/lnd/tlv"
 	"github.com/stretchr/testify/require"
 )
 
@@ -552,6 +553,7 @@ func NewTestFromAsset(t testing.TB, a *Asset) *TestAsset {
 		RelativeLockTime:    a.RelativeLockTime,
 		ScriptVersion:       uint16(a.ScriptVersion),
 		ScriptKey:           test.HexPubKey(a.ScriptKey.PubKey),
+		UnknownOddTypes:     a.UnknownOddTypes,
 	}
 
 	for _, w := range a.PrevWitnesses {
@@ -588,6 +590,7 @@ type TestAsset struct {
 	ScriptVersion       uint16          `json:"script_version"`
 	ScriptKey           string          `json:"script_key"`
 	GroupKey            *TestGroupKey   `json:"group_key"`
+	UnknownOddTypes     tlv.TypeMap     `json:"unknown_odd_types"`
 }
 
 func (ta *TestAsset) ToAsset(t testing.TB) *Asset {
@@ -635,6 +638,7 @@ func (ta *TestAsset) ToAsset(t testing.TB) *Asset {
 		ScriptKey: ScriptKey{
 			PubKey: test.ParsePubKey(t, ta.ScriptKey),
 		},
+		UnknownOddTypes: ta.UnknownOddTypes,
 	}
 
 	for _, tw := range ta.PrevWitnesses {

asset/records.go

@@ -84,32 +84,75 @@ func AssertNoUnknownEvenTypes(parsedTypes tlv.TypeMap,
 	return nil
 }
 
+// FilterUnknownTypes filters out all types that are unknown from the given
+// parsed types. The known types are specified as a set.
+func FilterUnknownTypes(parsedTypes tlv.TypeMap,
+	knownTypes fn.Set[tlv.Type]) tlv.TypeMap {
+
+	result := make(tlv.TypeMap, len(parsedTypes))
+	for t, v := range parsedTypes {
+		if !knownTypes.Contains(t) {
+			result[t] = v
+		}
+	}
+
+	// Avoid failures due to comparisons with nil vs. empty map.
+	if len(result) == 0 {
+		return nil
+	}
+
+	return result
+}
+
 // TlvStrictDecode attempts to decode the passed buffer into the TLV stream. It
 // takes the set of known types for a given stream, and returns an error if the
 // buffer includes any unknown even types.
 func TlvStrictDecode(stream *tlv.Stream, r io.Reader,
-	knownTypes fn.Set[tlv.Type]) error {
+	knownTypes fn.Set[tlv.Type]) (tlv.TypeMap, error) {
 
 	parsedTypes, err := stream.DecodeWithParsedTypes(r)
 	if err != nil {
-		return err
+		return nil, err
 	}
 
-	return AssertNoUnknownEvenTypes(parsedTypes, knownTypes)
+	err = AssertNoUnknownEvenTypes(parsedTypes, knownTypes)
+	if err != nil {
+		return nil, err
+	}
+
+	return FilterUnknownTypes(parsedTypes, knownTypes), nil
 }
 
 // TlvStrictDecodeP2P is identical to TlvStrictDecode except that the record
 // size is capped at 65535. This should only be called from a p2p setting where
 // untrusted input is being deserialized.
 func TlvStrictDecodeP2P(stream *tlv.Stream, r io.Reader,
-	knownTypes fn.Set[tlv.Type]) error {
+	knownTypes fn.Set[tlv.Type]) (tlv.TypeMap, error) {
 
 	parsedTypes, err := stream.DecodeWithParsedTypesP2P(r)
 	if err != nil {
-		return err
+		return nil, err
+	}
+
+	err = AssertNoUnknownEvenTypes(parsedTypes, knownTypes)
+	if err != nil {
+		return nil, err
+	}
+
+	return FilterUnknownTypes(parsedTypes, knownTypes), nil
+}
+
+// CombineRecords returns a new slice of records that combines the given records
+// with the unparsed types converted to static records.
+func CombineRecords(records []tlv.Record, unparsed tlv.TypeMap) []tlv.Record {
+	stubRecords := make([]tlv.Record, 0, len(unparsed))
+	for k, v := range unparsed {
+		stubRecords = append(stubRecords, tlv.MakeStaticRecord(
+			k, nil, uint64(len(v)), tlv.StubEncoder(v), nil,
+		))
 	}
 
-	return AssertNoUnknownEvenTypes(parsedTypes, knownTypes)
+	return append(records, stubRecords...)
 }
 
 // WitnessTlvType represents the different TLV types for Asset Witness TLV

asset/records_test.go

@@ -1,9 +1,12 @@
 package asset
 
 import (
+	"bytes"
 	"testing"
 
 	"github.com/lightninglabs/taproot-assets/fn"
+	"github.com/lightninglabs/taproot-assets/internal/test"
+	"github.com/lightningnetwork/lnd/keychain"
 	"github.com/lightningnetwork/lnd/tlv"
 	"github.com/stretchr/testify/require"
 )
@@ -60,3 +63,109 @@ func TestTlvStrictDecode(t *testing.T) {
 		))
 	}
 }
+
+// TestFilterUnknownTypes tests that the filtering of unknown TLV records works
+// as expected.
+func TestFilterUnknownTypes(t *testing.T) {
+	t.Parallel()
+
+	testCases := []struct {
+		parsedTypes tlv.TypeMap
+		knownTypes  fn.Set[tlv.Type]
+		result      tlv.TypeMap
+	}{
+		// No unknown types.
+		{
+			parsedTypes: tlv.TypeMap{
+				0: []byte{},
+				2: []byte{},
+			},
+			knownTypes: fn.NewSet[tlv.Type](0, 2),
+			result:     nil,
+		},
+
+		// Unknown type, but odd.
+		{
+			parsedTypes: tlv.TypeMap{
+				0: []byte{},
+				2: []byte{},
+				3: []byte{},
+			},
+			knownTypes: fn.NewSet[tlv.Type](0, 2),
+			result: tlv.TypeMap{
+				3: []byte{},
+			},
+		},
+
+		// Multiple unknown types, both odd and even.
+		{
+			parsedTypes: tlv.TypeMap{
+				0: []byte{},
+				2: []byte{},
+				3: []byte{},
+				4: []byte{},
+			},
+			knownTypes: fn.NewSet[tlv.Type](0, 2),
+			result: tlv.TypeMap{
+				3: []byte{},
+				4: []byte{},
+			},
+		},
+	}
+
+	for _, testCase := range testCases {
+		require.Equal(t, testCase.result, FilterUnknownTypes(
+			testCase.parsedTypes, testCase.knownTypes,
+		))
+	}
+}
+
+// TestAssetUnknownOddType tests that an unknown odd type is allowed in an asset
+// and that we can still arrive at the correct leaf hash with it.
+func TestAssetUnknownOddType(t *testing.T) {
+	knownAsset := RandAsset(t, Normal)
+	knownAssetLeaf, err := knownAsset.Leaf()
+	require.NoError(t, err)
+
+	test.RunUnknownOddTypeTest(
+		t, knownAsset, &ErrUnknownType{},
+		func(buf *bytes.Buffer, asset *Asset) error {
+			return asset.Encode(buf)
+		},
+		func(buf *bytes.Buffer) (*Asset, error) {
+			var asset Asset
+			return &asset, asset.Decode(buf)
+		},
+		func(parsedAsset *Asset, unknownTypes tlv.TypeMap) {
+			// The unknown types should be reported correctly.
+			require.Equal(
+				t, unknownTypes, parsedAsset.UnknownOddTypes,
+			)
+
+			// The leaf should've changed, to make sure the unknown
+			// value was taken into account when creating the
+			// serialized leaf.
+			parsedAssetLeaf, err := parsedAsset.Leaf()
+			require.NoError(t, err)
+
+			require.Equal(
+				t, knownAssetLeaf.NodeSum(),
+				parsedAssetLeaf.NodeSum(),
+			)
+			require.NotEqual(
+				t, knownAssetLeaf.NodeHash(),
+				parsedAssetLeaf.NodeHash(),
+			)
+
+			parsedAsset.UnknownOddTypes = nil
+
+			// The group key's raw key and witness aren't
+			// serialized, so we need to clear them out before
+			// comparing.
+			knownAsset.GroupKey.RawKey = keychain.KeyDescriptor{}
+			knownAsset.GroupKey.Witness = nil
+
+			require.Equal(t, knownAsset, parsedAsset)
+		},
+	)
+}

asset/testdata/asset_tlv_encoding_generated.json

@@ -53,7 +53,8 @@
                 "script_key": "02a0afeb165f0ec36880b68e0baabd9ad9c62fd1a69aa998bc30e9a346202e078f",
                 "group_key": {
                   "group_key": "03a0afeb165f0ec36880b68e0baabd9ad9c62fd1a69aa998bc30e9a346202e078f"
-                }
+                },
+                "unknown_odd_types": null
               }
             }
           }
@@ -63,7 +64,8 @@
         "script_key": "02a0afeb165f0ec36880b68e0baabd9ad9c62fd1a69aa998bc30e9a346202e078f",
         "group_key": {
           "group_key": "03a0afeb165f0ec36880b68e0baabd9ad9c62fd1a69aa998bc30e9a346202e078f"
-        }
+        },
+        "unknown_odd_types": null
       },
       "expected": "000101024f010101010101010101010101010101010101010101010101010101010101010100000001056173736574010203000000000000000000000000000000000000000000000000000000000000000001010401010601010703fd05390901060bfd020501fd02010165010101010101010101010101010101010101010101010101010101010101010100000001010101010101010101010101010101010101010101010101010101010101010103a0afeb165f0ec36880b68e0baabd9ad9c62fd1a69aa998bc30e9a346202e078f05fd01964a000197efb64d447880bacc7070f428a1310b2592d155b752da382934d4bd0fbb419a000000000000000affffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff7ffd0148000101024f010101010101010101010101010101010101010101010101010101010101010100000001056173736574010203000000000000000000000000000000000000000000000000000000000000000001010401010601010703fd05390901060b70016e0165020202020202020202020202020202020202020202020202020202020202020200000002020202020202020202020202020202020202020202020202020202020202020203a0afeb165f0ec36880b68e0baabd9ad9c62fd1a69aa998bc30e9a346202e078f030502010201020d28010101010101010101010101010101010101010101010101010101010101010100000000000005390e020001102102a0afeb165f0ec36880b68e0baabd9ad9c62fd1a69aa998bc30e9a346202e078f112103a0afeb165f0ec36880b68e0baabd9ad9c62fd1a69aa998bc30e9a346202e078f0e020001102102a0afeb165f0ec36880b68e0baabd9ad9c62fd1a69aa998bc30e9a346202e078f112103a0afeb165f0ec36880b68e0baabd9ad9c62fd1a69aa998bc30e9a346202e078f",
       "comment": "random split asset with root asset"
@@ -110,7 +112,8 @@
         "split_commitment_root": null,
         "script_version": 2,
         "script_key": "02a0afeb165f0ec36880b68e0baabd9ad9c62fd1a69aa998bc30e9a346202e078f",
-        "group_key": null
+        "group_key": null,
+        "unknown_odd_types": null
       },
       "expected": "000102024f020202020202020202020202020202020202020202020202020202020202020200000002056173736574010203000000000000000000000000000000000000000000000000000000000000000002020401020601020703fd05390901060bd9030067016500000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000006e0165020202020202020202020202020202020202020202020202020202020202020200000002020202020202020202020202020202020202020202020202020202020202020203a0afeb165f0ec36880b68e0baabd9ad9c62fd1a69aa998bc30e9a346202e078f030502010201020e020002102102a0afeb165f0ec36880b68e0baabd9ad9c62fd1a69aa998bc30e9a346202e078f",
       "comment": "random asset with multiple previous witnesses"
@@ -130,10 +133,34 @@
         "split_commitment_root": null,
         "script_version": 0,
         "script_key": "02a0afeb165f0ec36880b68e0baabd9ad9c62fd1a69aa998bc30e9a346202e078f",
-        "group_key": null
+        "group_key": null,
+        "unknown_odd_types": null
       },
       "expected": "000100024a00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000401000601000e020000102102a0afeb165f0ec36880b68e0baabd9ad9c62fd1a69aa998bc30e9a346202e078f",
       "comment": "minimal asset"
+    },
+    {
+      "asset": {
+        "version": 0,
+        "genesis_first_prev_out": "0000000000000000000000000000000000000000000000000000000000000000:0",
+        "genesis_tag": "",
+        "genesis_meta_hash": "0000000000000000000000000000000000000000000000000000000000000000",
+        "genesis_output_index": 0,
+        "genesis_type": 0,
+        "amount": 0,
+        "lock_time": 0,
+        "relative_lock_time": 0,
+        "prev_witnesses": null,
+        "split_commitment_root": null,
+        "script_version": 0,
+        "script_key": "02a0afeb165f0ec36880b68e0baabd9ad9c62fd1a69aa998bc30e9a346202e078f",
+        "group_key": null,
+        "unknown_odd_types": {
+          "31337": "dGhlIGdyZWF0IHVua25vd24="
+        }
+      },
+      "expected": "000100024a00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000401000601000e020000102102a0afeb165f0ec36880b68e0baabd9ad9c62fd1a69aa998bc30e9a346202e078ffd7a691174686520677265617420756e6b6e6f776e",
+      "comment": "minimal asset with unknown odd type"
     }
   ],
   "error_test_cases": null