feat(abstract-utxo): enable signing psbt with wasm implementation

Update signTransaction to support wasm PSBT signing. Return Buffer instead of
Uint8Array for extracted transactions. Fix type signatures and parameter
structure in signPsbtWithMusig2ParticipantWasm to match other implementations.

Issue: BTC-2806

Co-authored-by: llm-git <[email protected]>

Author: OttoAllmendinger

modules/abstract-utxo/src/transaction/fixedScript/signPsbtWasm.ts

@@ -20,7 +20,7 @@ const PSBT_CACHE_WASM = new Map<string, fixedScriptWallet.BitGoPsbt>();
 
 function hasKeyPathSpendInput(
   tx: fixedScriptWallet.BitGoPsbt,
-  rootWalletKeys: fixedScriptWallet.IWalletKeys,
+  rootWalletKeys: fixedScriptWallet.RootWalletKeys,
   replayProtection: ReplayProtectionKeys
 ): boolean {
   const parsed = tx.parseTransactionWithWalletKeys(rootWalletKeys, replayProtection);
@@ -36,10 +36,10 @@ function hasKeyPathSpendInput(
 export function signAndVerifyPsbtWasm(
   tx: fixedScriptWallet.BitGoPsbt,
   signerKeychain: BIP32Interface,
-  rootWalletKeys: fixedScriptWallet.IWalletKeys,
+  rootWalletKeys: fixedScriptWallet.RootWalletKeys,
   replayProtection: ReplayProtectionKeys,
   { isLastSignature }: { isLastSignature: boolean }
-): fixedScriptWallet.BitGoPsbt | Uint8Array {
+): fixedScriptWallet.BitGoPsbt | Buffer {
   const wasmSigner = toWasmBIP32(signerKeychain);
   const parsed = tx.parseTransactionWithWalletKeys(rootWalletKeys, replayProtection);
 
@@ -85,7 +85,7 @@ export function signAndVerifyPsbtWasm(
 
   if (isLastSignature) {
     tx.finalizeAllInputs();
-    return tx.extractTransaction();
+    return Buffer.from(tx.extractTransaction());
   }
 
   return tx;
@@ -100,17 +100,17 @@ export async function signPsbtWithMusig2ParticipantWasm(
   coin: Musig2Participant<fixedScriptWallet.BitGoPsbt>,
   tx: fixedScriptWallet.BitGoPsbt,
   signerKeychain: BIP32Interface | undefined,
-  rootWalletKeys: fixedScriptWallet.IWalletKeys,
-  replayProtection: ReplayProtectionKeys,
+  rootWalletKeys: fixedScriptWallet.RootWalletKeys,
   params: {
+    replayProtection: ReplayProtectionKeys;
     isLastSignature: boolean;
     signingStep: 'signerNonce' | 'cosignerNonce' | 'signerSignature' | undefined;
     walletId: string | undefined;
   }
-): Promise<fixedScriptWallet.BitGoPsbt | Uint8Array> {
+): Promise<fixedScriptWallet.BitGoPsbt | Buffer> {
   const wasmSigner = signerKeychain ? toWasmBIP32(signerKeychain) : undefined;
 
-  if (hasKeyPathSpendInput(tx, rootWalletKeys, replayProtection)) {
+  if (hasKeyPathSpendInput(tx, rootWalletKeys, params.replayProtection)) {
     // We can only be the first signature on a transaction with taproot key path spend inputs because
     // we require the secret nonce in the cache of the first signer, which is impossible to retrieve if
     // deserialized from a hex.
@@ -162,7 +162,7 @@ export async function signPsbtWithMusig2ParticipantWasm(
   }
 
   assert(signerKeychain);
-  return signAndVerifyPsbtWasm(tx, signerKeychain, rootWalletKeys, replayProtection, {
+  return signAndVerifyPsbtWasm(tx, signerKeychain, rootWalletKeys, params.replayProtection, {
     isLastSignature: params.isLastSignature,
   });
 }

modules/abstract-utxo/src/transaction/fixedScript/signTransaction.ts

@@ -1,17 +1,23 @@
+import assert from 'assert';
+
+import { isTriple } from '@bitgo/sdk-core';
 import _ from 'lodash';
 import { BIP32Interface } from '@bitgo/secp256k1';
 import { bitgo } from '@bitgo/utxo-lib';
 import * as utxolib from '@bitgo/utxo-lib';
-
-import { DecodedTransaction } from '../types';
+import { fixedScriptWallet } from '@bitgo/wasm-utxo';
 
 import { Musig2Participant } from './musig2';
 import { signLegacyTransaction } from './signLegacyTransaction';
 import { signPsbtWithMusig2Participant } from './signPsbt';
+import { signPsbtWithMusig2ParticipantWasm } from './signPsbtWasm';
+import { getReplayProtectionPubkeys } from './replayProtection';
 
-export async function signTransaction(
-  coin: Musig2Participant<utxolib.bitgo.UtxoPsbt>,
-  tx: DecodedTransaction<bigint | number>,
+export async function signTransaction<
+  T extends utxolib.bitgo.UtxoPsbt | utxolib.bitgo.UtxoTransaction<bigint | number> | fixedScriptWallet.BitGoPsbt
+>(
+  coin: Musig2Participant<utxolib.bitgo.UtxoPsbt> | Musig2Participant<fixedScriptWallet.BitGoPsbt>,
+  tx: T,
   signerKeychain: BIP32Interface | undefined,
   network: utxolib.Network,
   params: {
@@ -24,19 +30,39 @@ export async function signTransaction(
     pubs: string[] | undefined;
     cosignerPub: string | undefined;
   }
-): Promise<utxolib.bitgo.UtxoPsbt | utxolib.bitgo.UtxoTransaction<bigint | number>> {
+): Promise<
+  utxolib.bitgo.UtxoPsbt | utxolib.bitgo.UtxoTransaction<bigint | number> | fixedScriptWallet.BitGoPsbt | Buffer
+> {
   let isLastSignature = false;
   if (_.isBoolean(params.isLastSignature)) {
     // if build is called instead of buildIncomplete, no signature placeholders are left in the sig script
     isLastSignature = params.isLastSignature;
   }
 
   if (tx instanceof bitgo.UtxoPsbt) {
-    return signPsbtWithMusig2Participant(coin, tx, signerKeychain, {
+    return signPsbtWithMusig2Participant(coin as Musig2Participant<utxolib.bitgo.UtxoPsbt>, tx, signerKeychain, {
       isLastSignature,
       signingStep: params.signingStep,
       walletId: params.walletId,
     });
+  } else if (tx instanceof fixedScriptWallet.BitGoPsbt) {
+    assert(params.pubs, 'pubs are required for fixed script signing');
+    assert(isTriple(params.pubs), 'pubs must be a triple');
+    const rootWalletKeys = fixedScriptWallet.RootWalletKeys.fromXpubs(params.pubs);
+    return signPsbtWithMusig2ParticipantWasm(
+      coin as Musig2Participant<fixedScriptWallet.BitGoPsbt>,
+      tx,
+      signerKeychain,
+      rootWalletKeys,
+      {
+        replayProtection: {
+          publicKeys: getReplayProtectionPubkeys(network),
+        },
+        isLastSignature,
+        signingStep: params.signingStep,
+        walletId: params.walletId,
+      }
+    );
   }
 
   return signLegacyTransaction(tx, signerKeychain, {

modules/abstract-utxo/src/transaction/signTransaction.ts

@@ -10,6 +10,7 @@ import { fetchKeychains, toBip32Triple } from '../keychains';
 
 import * as fixedScript from './fixedScript';
 import * as descriptor from './descriptor';
+import { encodeTransaction } from './decode';
 
 const debug = buildDebug('bitgo:abstract-utxo:transaction:signTransaction');
 
@@ -72,6 +73,7 @@ export async function signTransaction<TNumber extends number | bigint>(
       pubs: params.pubs,
       cosignerPub: params.cosignerPub,
     });
-    return { txHex: signedTx.toBuffer().toString('hex') };
+    const buffer = Buffer.isBuffer(signedTx) ? signedTx : encodeTransaction(signedTx);
+    return { txHex: buffer.toString('hex') };
   }
 }

modules/abstract-utxo/test/unit/transaction/fixedScript/signPsbt.ts

@@ -130,9 +130,9 @@ function describeSignPsbtWithMusig2Participant(
           getMockCoinWasm(acidTest.rootWalletKeys, acidTest.network),
           psbt,
           acidTest.rootWalletKeys.user,
-          wasmWalletKeys,
-          replayProtection,
+          fixedScriptWallet.RootWalletKeys.from(acidTest.rootWalletKeys),
           {
+            replayProtection,
             isLastSignature: false,
             signingStep: undefined,
             walletId: 'test-wallet-id',