feat(abstract-utxo): default to psbt-lite for testnet

Change the default transaction format on testnets from 'psbt' to
'psbt-lite', which uses a more efficient serialization that omits full
prevTx data. Add documentation for transaction format options.

Issue: BTC-2732

Co-authored-by: llm-git <[email protected]>

Author: OttoAllmendinger

modules/abstract-utxo/src/abstractUtxoCoin.ts

@@ -84,7 +84,18 @@ import { isDescriptorWalletData } from './descriptor/descriptorWallet';
 
 import ScriptType2Of3 = utxolib.bitgo.outputScripts.ScriptType2Of3;
 
-export type TxFormat = 'legacy' | 'psbt';
+export type TxFormat =
+  // This is a legacy transaction format based around the bitcoinjs-lib serialization of unsigned transactions
+  // does not include prevOut data and is a bit painful to work with
+  // going to be deprecated in favor of psbt
+  // @deprecated
+  | 'legacy'
+  // This is the standard psbt format, including the full prevTx data for legacy transactions.
+  // This will remain supported but is not the default, since the data sizes can become prohibitively large.
+  | 'psbt'
+  // This is a nonstandard psbt version where legacy inputs are serialized as if they were segwit inputs.
+  // While this prevents us to fully verify the transaction fee, we have other checks in place to ensure the fee is within bounds.
+  | 'psbt-lite';
 
 type UtxoCustomSigningFunction<TNumber extends number | bigint> = {
   (params: {
@@ -972,7 +983,7 @@ export abstract class AbstractUtxoCoin extends BaseCoin {
     }
 
     if (isTestnet(this.network)) {
-      return 'psbt';
+      return 'psbt-lite';
     }
 
     const walletFlagMusigKp = wallet.flag('musigKp') === 'true';

modules/abstract-utxo/test/unit/txFormat.ts

@@ -103,11 +103,11 @@ function runTest(params: {
 
 describe('txFormat', function () {
   describe('getDefaultTxFormat', function () {
-    // All testnet wallets default to PSBT
+    // All testnet wallets default to PSBT-lite
     runTest({
-      description: 'should always return psbt for testnet',
+      description: 'should always return psbt-lite for testnet',
       coinFilter: (coin) => utxolib.isTestnet(coin.network),
-      expectedTxFormat: 'psbt',
+      expectedTxFormat: 'psbt-lite',
     });
 
     // DistributedCustody wallets default to PSBT (mainnet only, testnet already covered)
@@ -161,5 +161,11 @@ describe('txFormat', function () {
       expectedTxFormat: 'psbt',
       requestedTxFormat: 'psbt',
     });
+
+    runTest({
+      description: 'should respect explicitly requested psbt-lite format',
+      expectedTxFormat: 'psbt-lite',
+      requestedTxFormat: 'psbt-lite',
+    });
   });
 });