refactor(abstract-utxo): update getMusig2Nonces to accept UtxoPsbt

Update the getMusig2Nonces method to accept a UtxoPsbt object directly
instead of a hex string. This improves type safety and reduces unnecessary
encoding/decoding. Also add a helper method decodeTransactionAsPsbt to
validate and cast decoded transactions to UtxoPsbt.

Update dependent functions to use the new method signature and refactor
to avoid unnecessary conversion steps.

Issue: BTC-2806

Co-authored-by: llm-git <[email protected]>

Author: OttoAllmendinger

modules/abstract-utxo/src/abstractUtxoCoin.ts

@@ -547,6 +547,14 @@ export abstract class AbstractUtxoCoin extends BaseCoin {
     }
   }
 
+  decodeTransactionAsPsbt(input: Buffer | string): utxolib.bitgo.UtxoPsbt {
+    const decoded = this.decodeTransaction(input);
+    if (!(decoded instanceof utxolib.bitgo.UtxoPsbt)) {
+      throw new Error('expected psbt but got transaction');
+    }
+    return decoded;
+  }
+
   decodeTransactionFromPrebuild<TNumber extends number | bigint>(prebuild: {
     txHex?: string;
     txBase64?: string;
@@ -711,12 +719,13 @@ export abstract class AbstractUtxoCoin extends BaseCoin {
    * @param psbtHex all MuSig2 inputs should contain user MuSig2 nonce
    * @param walletId
    */
-  async getMusig2Nonces(psbtHex: string, walletId: string): Promise<SignPsbtResponse> {
-    const params: SignPsbtRequest = { psbt: psbtHex };
-    return await this.bitgo
+  async getMusig2Nonces(psbt: utxolib.bitgo.UtxoPsbt, walletId: string): Promise<utxolib.bitgo.UtxoPsbt> {
+    const params: SignPsbtRequest = { psbt: psbt.toHex() };
+    const response = await this.bitgo
       .post(this.url('/wallet/' + walletId + '/tx/signpsbt'))
       .send(params)
       .result();
+    return this.decodeTransactionAsPsbt(response.psbt);
   }
 
   /**
@@ -726,7 +735,7 @@ export abstract class AbstractUtxoCoin extends BaseCoin {
    * @param walletId
    */
   async signPsbt(psbtHex: string, walletId: string): Promise<SignPsbtResponse> {
-    return this.getMusig2Nonces(psbtHex, walletId);
+    return { psbt: (await this.getMusig2Nonces(this.decodeTransactionAsPsbt(psbtHex), walletId)).toHex() };
   }
 
   /**
@@ -736,9 +745,11 @@ export abstract class AbstractUtxoCoin extends BaseCoin {
   async signPsbtFromOVC(ovcJson: Record<string, unknown>): Promise<Record<string, unknown>> {
     assert(ovcJson['psbtHex'], 'ovcJson must contain psbtHex');
     assert(ovcJson['walletId'], 'ovcJson must contain walletId');
-    const psbt = (await this.getMusig2Nonces(ovcJson['psbtHex'] as string, ovcJson['walletId'] as string)).psbt;
-    assert(psbt, 'psbt not found');
-    return _.extend(ovcJson, { txHex: psbt });
+    const psbt = await this.getMusig2Nonces(
+      this.decodeTransactionAsPsbt(ovcJson['psbtHex'] as string),
+      ovcJson['walletId'] as string
+    );
+    return _.extend(ovcJson, { txHex: psbt.toHex() });
   }
 
   /**

modules/abstract-utxo/src/transaction/fixedScript/signTransaction.ts

@@ -13,7 +13,7 @@ import { signAndVerifyPsbt, signAndVerifyWalletTransaction } from './sign';
 type RootWalletKeys = bitgo.RootWalletKeys;
 
 export interface Musig2Participant {
-  getMusig2Nonces(psbtHex: string, walletId: string): Promise<{ psbt: string }>;
+  getMusig2Nonces(psbt: utxolib.bitgo.UtxoPsbt, walletId: string): Promise<utxolib.bitgo.UtxoPsbt>;
 }
 
 /**
@@ -66,7 +66,7 @@ export async function signTransaction<TNumber extends number | bigint>(
         return { txHex: tx.toHex() };
       case 'cosignerNonce':
         assert(params.walletId, 'walletId is required for MuSig2 bitgo nonce');
-        return { txHex: (await coin.getMusig2Nonces(tx.toHex(), params.walletId)).psbt };
+        return { txHex: (await coin.getMusig2Nonces(tx, params.walletId)).toHex() };
       case 'signerSignature':
         const txId = tx.getUnsignedTx().getId();
         const psbt = PSBT_CACHE.get(txId);
@@ -83,8 +83,8 @@ export async function signTransaction<TNumber extends number | bigint>(
         assert(params.walletId, 'walletId is required for MuSig2 bitgo nonce');
         assert(signerKeychain);
         tx.setAllInputsMusig2NonceHD(signerKeychain);
-        const response = await coin.getMusig2Nonces(tx.toHex(), params.walletId);
-        tx.combine(bitgo.createPsbtFromHex(response.psbt, network));
+        const response = await coin.getMusig2Nonces(tx, params.walletId);
+        tx = tx.combine(response);
         break;
     }
   } else {