feat(abstract-utxo): add fixedScriptWallet to decodeTransactionAsPsbt

Support wasm-utxo based fixed script wallet PSBTs in the
getMusig2Nonces method by implementing Musig2Participant for both PSBT
types.

Issue: BTC-2806

Co-authored-by: llm-git <[email protected]>

Author: OttoAllmendinger

modules/abstract-utxo/src/abstractUtxoCoin.ts

@@ -42,6 +42,7 @@ import {
   isValidPrv,
   isValidXprv,
 } from '@bitgo/sdk-core';
+import { fixedScriptWallet } from '@bitgo/wasm-utxo';
 
 import {
   backupKeyRecovery,
@@ -68,6 +69,7 @@ import {
   verifyTransaction,
 } from './transaction';
 import type { TransactionExplanation } from './transaction/fixedScript/explainTransaction';
+import { Musig2Participant } from './transaction/fixedScript/musig2';
 import {
   AggregateValidationError,
   ErrorMissingOutputs,
@@ -77,7 +79,7 @@ import { assertDescriptorWalletAddress, getDescriptorMapFromWallet, isDescriptor
 import { getChainFromNetwork, getFamilyFromNetwork, getFullNameFromNetwork } from './names';
 import { assertFixedScriptWalletAddress } from './address/fixedScript';
 import { ParsedTransaction } from './transaction/types';
-import { decodePsbtWith, stringToBufferTryFormats } from './transaction/decode';
+import { decodePsbtWith, encodeTransaction, stringToBufferTryFormats } from './transaction/decode';
 import { toBip32Triple, UtxoKeychain } from './keychains';
 import { verifyKeySignature, verifyUserPublicKey } from './verifyKey';
 import { getPolicyForEnv } from './descriptor/validatePolicy';
@@ -360,7 +362,10 @@ export interface SignPsbtResponse {
   psbt: string;
 }
 
-export abstract class AbstractUtxoCoin extends BaseCoin {
+export abstract class AbstractUtxoCoin
+  extends BaseCoin
+  implements Musig2Participant<utxolib.bitgo.UtxoPsbt>, Musig2Participant<fixedScriptWallet.BitGoPsbt>
+{
   public altScriptHash?: number;
   public supportAltScriptDestination?: boolean;
   public readonly amountType: 'number' | 'bigint';
@@ -509,7 +514,7 @@ export abstract class AbstractUtxoCoin extends BaseCoin {
     if (_.isUndefined(prebuild.blockHeight)) {
       prebuild.blockHeight = (await this.getLatestBlockHeight()) as number;
     }
-    return _.extend({}, prebuild, { txHex: tx.toHex() });
+    return _.extend({}, prebuild, { txHex: encodeTransaction(tx).toString('hex') });
   }
 
   /**
@@ -541,12 +546,12 @@ export abstract class AbstractUtxoCoin extends BaseCoin {
     }
   }
 
-  decodeTransactionAsPsbt(input: Buffer | string): utxolib.bitgo.UtxoPsbt {
+  decodeTransactionAsPsbt(input: Buffer | string): utxolib.bitgo.UtxoPsbt | fixedScriptWallet.BitGoPsbt {
     const decoded = this.decodeTransaction(input);
-    if (!(decoded instanceof utxolib.bitgo.UtxoPsbt)) {
-      throw new Error('expected psbt but got transaction');
+    if (decoded instanceof fixedScriptWallet.BitGoPsbt || decoded instanceof utxolib.bitgo.UtxoPsbt) {
+      return decoded;
     }
-    return decoded;
+    throw new Error('expected psbt but got transaction');
   }
 
   decodeTransactionFromPrebuild<TNumber extends number | bigint>(prebuild: {
@@ -720,16 +725,29 @@ export abstract class AbstractUtxoCoin extends BaseCoin {
 
   /**
    * @returns input psbt added with deterministic MuSig2 nonce for bitgo key for each MuSig2 inputs.
-   * @param psbtHex all MuSig2 inputs should contain user MuSig2 nonce
+   * @param psbt all MuSig2 inputs should contain user MuSig2 nonce
    * @param walletId
    */
-  async getMusig2Nonces(psbt: utxolib.bitgo.UtxoPsbt, walletId: string): Promise<utxolib.bitgo.UtxoPsbt> {
-    const params: SignPsbtRequest = { psbt: psbt.toHex() };
+  async getMusig2Nonces(psbt: utxolib.bitgo.UtxoPsbt, walletId: string): Promise<utxolib.bitgo.UtxoPsbt>;
+  async getMusig2Nonces(psbt: fixedScriptWallet.BitGoPsbt, walletId: string): Promise<fixedScriptWallet.BitGoPsbt>;
+  async getMusig2Nonces<T extends utxolib.bitgo.UtxoPsbt | fixedScriptWallet.BitGoPsbt>(
+    psbt: T,
+    walletId: string
+  ): Promise<T>;
+  async getMusig2Nonces<T extends utxolib.bitgo.UtxoPsbt | fixedScriptWallet.BitGoPsbt>(
+    psbt: T,
+    walletId: string
+  ): Promise<T> {
+    const buffer = encodeTransaction(psbt);
     const response = await this.bitgo
       .post(this.url('/wallet/' + walletId + '/tx/signpsbt'))
-      .send(params)
+      .send({ psbt: buffer.toString('hex') })
       .result();
-    return this.decodeTransactionAsPsbt(response.psbt);
+    if (psbt instanceof utxolib.bitgo.UtxoPsbt) {
+      return decodePsbtWith(response.psbt, this.network, 'utxolib') as T;
+    } else {
+      return decodePsbtWith(response.psbt, this.network, 'wasm-utxo') as T;
+    }
   }
 
   /**
@@ -739,7 +757,8 @@ export abstract class AbstractUtxoCoin extends BaseCoin {
    * @param walletId
    */
   async signPsbt(psbtHex: string, walletId: string): Promise<SignPsbtResponse> {
-    return { psbt: (await this.getMusig2Nonces(this.decodeTransactionAsPsbt(psbtHex), walletId)).toHex() };
+    const psbt = await this.getMusig2Nonces(this.decodeTransactionAsPsbt(psbtHex), walletId);
+    return { psbt: encodeTransaction(psbt).toString('hex') };
   }
 
   /**
@@ -749,11 +768,10 @@ export abstract class AbstractUtxoCoin extends BaseCoin {
   async signPsbtFromOVC(ovcJson: Record<string, unknown>): Promise<Record<string, unknown>> {
     assert(ovcJson['psbtHex'], 'ovcJson must contain psbtHex');
     assert(ovcJson['walletId'], 'ovcJson must contain walletId');
-    const psbt = await this.getMusig2Nonces(
-      this.decodeTransactionAsPsbt(ovcJson['psbtHex'] as string),
-      ovcJson['walletId'] as string
-    );
-    return _.extend(ovcJson, { txHex: psbt.toHex() });
+    const hex = ovcJson['psbtHex'] as string;
+    const walletId = ovcJson['walletId'] as string;
+    const psbt = await this.getMusig2Nonces(this.decodeTransactionAsPsbt(hex), walletId);
+    return _.extend(ovcJson, { txHex: encodeTransaction(psbt).toString('hex') });
   }
 
   /**