feat(ts/idl): Update IDL fetching for the new PMP metadata

cli/src/metadata.rs

@@ -41,9 +41,14 @@ impl IdlCommand {
 
     pub fn status(self) -> io::Result<ExitStatus> {
         let mut command = Command::new("npx");
+        // Force on first-time install
+        command.arg("--yes");
+        // Use pinned version
         command.arg(format!(
-            "@solana-program/program-metadata@{PMP_CLIENT_VERSION}"
+            "--package=@solana-program/program-metadata@{PMP_CLIENT_VERSION}"
         ));
+        command.arg("--");
+        command.arg("program-metadata");
         command.args(["--rpc", &self.rpc_url]);
         command.args(self.subcommand.args());
         command.status()

tests/anchor-cli-idl/.gitignore

@@ -0,0 +1 @@
+testLargeIdl.json

tests/anchor-cli-idl/test.sh

@@ -2,6 +2,10 @@
 
 set -euo pipefail
 
+# FIXME: For some reason, in CI, executing with NPX results in `sh: program-metadata: not found`
+# Installing this globally fixes this in CI, but this should be investigated and fixed properly
+# Implementing IDL fetching via Rust client will make this redundant
+npm install --global @solana-program/program-metadata@0.5.1
 DEPLOYER_KEYPAIR="keypairs/deployer-keypair.json"
 PROGRAM_ONE="2uA3amp95zsEHUpo8qnLMhcFAUsiKVEcKHXS1JetFjU5"
 

tests/anchor-cli-idl/tests/idl.ts

@@ -15,16 +15,8 @@ describe("Test CLI IDL commands", () => {
   const programOne = anchor.workspace.IdlCommandsOne as Program<IdlCommandsOne>;
   const programTwo = anchor.workspace.IdlCommandsTwo as Program<IdlCommandsTwo>;
 
-  // FIXME: Once the TS client is updated to use PMP, this should use the TS API and not CLI
   const fetchIdl = async (programId) => {
-    let idl;
-    try {
-      idl = execSync(`anchor idl fetch ${programId}`).toString();
-    } catch {
-      // CLI errors on failure, TS API returns null
-      return null;
-    }
-    return JSON.parse(idl);
+    return await anchor.Program.fetchIdl(programId, provider);
   };
 
   it("Can initialize IDL account", async () => {

ts/packages/anchor/package.json

@@ -33,9 +33,11 @@
     "test": "jest tests --detectOpenHandles"
   },
   "dependencies": {
-    "@anchor-lang/errors": "^0.32.1",
     "@anchor-lang/borsh": "^0.32.1",
+    "@anchor-lang/errors": "^0.32.1",
     "@noble/hashes": "^1.3.1",
+    "@solana/kit": "^6.0.0",
+    "@solana-program/program-metadata": "0.5.1",
     "@solana/web3.js": "^1.69.0",
     "bn.js": "^5.1.2",
     "bs58": "^4.0.1",
@@ -75,7 +77,7 @@
     "ts-node": "^9.0.0",
     "tslib": "^2.3.1",
     "typedoc": "^0.22.10",
-    "typescript": "^5.5.4"
+    "typescript": "^5.9.3"
   },
   "files": [
     "dist",

ts/packages/anchor/src/coder/borsh/accounts.ts

@@ -5,6 +5,14 @@ import { Idl, IdlDiscriminator } from "../../idl.js";
 import { IdlCoder } from "./idl.js";
 import { AccountsCoder } from "../index.js";
 
+function bytesEqual(a: Uint8Array, b: Uint8Array): boolean {
+  if (a.length !== b.length) return false;
+  for (let i = 0; i < a.length; i += 1) {
+    if (a[i] !== b[i]) return false;
+  }
+  return true;
+}
+
 /**
  * Encodes and decodes account objects.
  */
@@ -56,22 +64,28 @@ export class BorshAccountsCoder<A extends string = string>
     const len = layout.layout.encode(account, buffer);
     const accountData = buffer.slice(0, len);
     const discriminator = this.accountDiscriminator(accountName);
-    return Buffer.concat([discriminator, accountData]);
+    return Buffer.from([...discriminator, ...accountData]);
   }
 
   public decode<T = any>(accountName: A, data: Buffer): T {
     // Assert the account discriminator is correct.
     const discriminator = this.accountDiscriminator(accountName);
-    if (discriminator.compare(data.slice(0, discriminator.length))) {
+    const givenDisc = Uint8Array.from(data.subarray(0, discriminator.length));
+    if (!bytesEqual(Uint8Array.from(discriminator), givenDisc)) {
       throw new Error("Invalid account discriminator");
     }
     return this.decodeUnchecked(accountName, data);
   }
 
   public decodeAny<T = any>(data: Buffer): T {
     for (const [name, layout] of this.accountLayouts) {
-      const givenDisc = data.subarray(0, layout.discriminator.length);
-      const matches = givenDisc.equals(Buffer.from(layout.discriminator));
+      const givenDisc = Uint8Array.from(
+        data.subarray(0, layout.discriminator.length)
+      );
+      const matches = bytesEqual(
+        Uint8Array.from(layout.discriminator),
+        givenDisc
+      );
       if (matches) return this.decodeUnchecked(name, data);
     }
 
@@ -91,11 +105,12 @@ export class BorshAccountsCoder<A extends string = string>
 
   public memcmp(accountName: A, appendData?: Buffer): any {
     const discriminator = this.accountDiscriminator(accountName);
+    const bytes = appendData
+      ? Uint8Array.from([...discriminator, ...appendData])
+      : Uint8Array.from(discriminator);
     return {
       offset: 0,
-      bytes: bs58.encode(
-        appendData ? Buffer.concat([discriminator, appendData]) : discriminator
-      ),
+      bytes: bs58.encode(Buffer.from(bytes)),
     };
   }
 

ts/packages/anchor/src/coder/borsh/event.ts

@@ -5,6 +5,14 @@ import { Idl, IdlDiscriminator } from "../../idl.js";
 import { IdlCoder } from "./idl.js";
 import { EventCoder } from "../index.js";
 
+function bytesEqual(a: Uint8Array, b: Uint8Array): boolean {
+  if (a.length !== b.length) return false;
+  for (let i = 0; i < a.length; i += 1) {
+    if (a[i] !== b[i]) return false;
+  }
+  return true;
+}
+
 export class BorshEventCoder implements EventCoder {
   /**
    * Maps account type identifier to a layout.
@@ -54,8 +62,13 @@ export class BorshEventCoder implements EventCoder {
     }
 
     for (const [name, layout] of this.layouts) {
-      const givenDisc = logArr.subarray(0, layout.discriminator.length);
-      const matches = givenDisc.equals(Buffer.from(layout.discriminator));
+      const givenDisc = Uint8Array.from(
+        logArr.subarray(0, layout.discriminator.length)
+      );
+      const matches = bytesEqual(
+        Uint8Array.from(layout.discriminator),
+        givenDisc
+      );
       if (matches) {
         return {
           name,

ts/packages/anchor/src/coder/borsh/instruction.ts

@@ -18,6 +18,14 @@ import {
 import { IdlCoder } from "./idl.js";
 import { InstructionCoder } from "../index.js";
 
+function bytesEqual(a: Uint8Array, b: Uint8Array): boolean {
+  if (a.length !== b.length) return false;
+  for (let i = 0; i < a.length; i += 1) {
+    if (a[i] !== b[i]) return false;
+  }
+  return true;
+}
+
 /**
  * Encodes and decodes program instructions.
  */
@@ -53,7 +61,7 @@ export class BorshInstructionCoder implements InstructionCoder {
     const len = encoder.layout.encode(ix, buffer);
     const data = buffer.slice(0, len);
 
-    return Buffer.concat([Buffer.from(encoder.discriminator), data]);
+    return Buffer.from([...Buffer.from(encoder.discriminator), ...data]);
   }
 
   /**
@@ -64,12 +72,20 @@ export class BorshInstructionCoder implements InstructionCoder {
     encoding: "hex" | "base58" = "hex"
   ): Instruction | null {
     if (typeof ix === "string") {
-      ix = encoding === "hex" ? Buffer.from(ix, "hex") : bs58.decode(ix);
+      ix =
+        encoding === "hex"
+          ? Buffer.from(ix, "hex")
+          : Buffer.from(bs58.decode(ix) as Uint8Array);
     }
 
     for (const [name, layout] of this.ixLayouts) {
-      const givenDisc = ix.subarray(0, layout.discriminator.length);
-      const matches = givenDisc.equals(Buffer.from(layout.discriminator));
+      const givenDisc = Uint8Array.from(
+        ix.subarray(0, layout.discriminator.length)
+      );
+      const matches = bytesEqual(
+        Uint8Array.from(layout.discriminator),
+        givenDisc
+      );
       if (matches) {
         return {
           name,

ts/packages/anchor/src/idl.ts

@@ -1,7 +1,6 @@
 import camelCase from "camelcase";
 import { Buffer } from "buffer";
 import { PublicKey } from "@solana/web3.js";
-import * as borsh from "@anchor-lang/borsh";
 
 export type Idl = {
   address: string;
@@ -271,38 +270,12 @@ export function isCompositeAccounts(
   return "accounts" in accountItem;
 }
 
-// Deterministic IDL address as a function of the program id.
-export async function idlAddress(programId: PublicKey): Promise<PublicKey> {
-  const base = (await PublicKey.findProgramAddress([], programId))[0];
-  return await PublicKey.createWithSeed(base, seed(), programId);
-}
-
-// Seed for generating the idlAddress.
-export function seed(): string {
-  return "anchor:idl";
-}
-
 // The on-chain account of the IDL.
 export interface IdlProgramAccount {
   authority: PublicKey;
   data: Buffer;
 }
 
-const IDL_ACCOUNT_LAYOUT: borsh.Layout<IdlProgramAccount> = borsh.struct([
-  borsh.publicKey("authority"),
-  borsh.vecU8("data"),
-]);
-
-export function decodeIdlAccount(data: Buffer): IdlProgramAccount {
-  return IDL_ACCOUNT_LAYOUT.decode(data);
-}
-
-export function encodeIdlAccount(acc: IdlProgramAccount): Buffer {
-  const buffer = Buffer.alloc(1000); // TODO: use a tighter buffer.
-  const len = IDL_ACCOUNT_LAYOUT.encode(acc, buffer);
-  return buffer.slice(0, len);
-}
-
 /**
  * Convert the given IDL to camelCase.
  *

ts/packages/anchor/src/nodewallet.ts

@@ -1,4 +1,3 @@
-import { Buffer } from "buffer";
 import {
   Keypair,
   PublicKey,
@@ -24,7 +23,7 @@ export default class NodeWallet implements Wallet {
     }
 
     const payer = Keypair.fromSecretKey(
-      Buffer.from(
+      Uint8Array.from(
         JSON.parse(
           require("fs").readFileSync(process.env.ANCHOR_WALLET, {
             encoding: "utf-8",

ts/packages/anchor/src/program/index.ts

@@ -1,15 +1,16 @@
 import { Commitment, PublicKey } from "@solana/web3.js";
-import { inflate } from "pako";
-import { BorshCoder, Coder } from "../coder/index.js";
 import {
-  Idl,
-  IdlInstruction,
-  convertIdlToCamelCase,
-  decodeIdlAccount,
-  idlAddress,
-} from "../idl.js";
+  DataSource,
+  fetchMaybeMetadataFromSeeds,
+  Format,
+  unpackAndFetchData,
+  unpackDirectData,
+} from "@solana-program/program-metadata";
+import { address } from "@solana/addresses";
+import { createSolanaRpc } from "@solana/kit";
+import { BorshCoder, Coder } from "../coder/index.js";
+import { Idl, IdlInstruction, convertIdlToCamelCase } from "../idl.js";
 import Provider, { getProvider } from "../provider.js";
-import { utf8 } from "../utils/bytes/index.js";
 import { CustomAccountResolver } from "./accounts-resolver.js";
 import { Address, translateAddress } from "./common.js";
 import { EventManager } from "./event.js";
@@ -345,21 +346,32 @@ export class Program<IDL extends Idl = Idl> {
    * @param provider  The network and wallet context.
    */
   public static async fetchIdl<IDL extends Idl = Idl>(
-    address: Address,
+    programAddress: Address,
     provider?: Provider
   ): Promise<IDL | null> {
     provider = provider ?? getProvider();
-    const programId = translateAddress(address);
-
-    const idlAddr = await idlAddress(programId);
-    const accountInfo = await provider.connection.getAccountInfo(idlAddr);
-    if (!accountInfo) {
+    const rpc = createSolanaRpc(provider.connection.rpcEndpoint);
+    const program = address(programAddress.toString());
+    const metadataAccount = await fetchMaybeMetadataFromSeeds(rpc, {
+      program,
+      authority: null,
+      seed: "idl",
+    });
+    if (!metadataAccount.exists) {
       return null;
     }
-    // Chop off account discriminator.
-    let idlAccount = decodeIdlAccount(accountInfo.data.slice(8));
-    const inflatedIdl = inflate(idlAccount.data);
-    return JSON.parse(utf8.decode(inflatedIdl));
+    if (metadataAccount.data.dataSource !== DataSource.Direct) {
+      throw new Error(
+        `IDL has source '${metadataAccount.data.dataSource.toString()}', only directly embedded data is supported`
+      );
+    }
+    const content = unpackDirectData(metadataAccount.data);
+    if (metadataAccount.data.format !== Format.Json) {
+      throw new Error(
+        `IDL has data format '${metadataAccount.data.format.toString()}', only JSON IDLs are supported`
+      );
+    }
+    return JSON.parse(content);
   }
 
   /**

ts/packages/anchor/src/program/token-account-layout.ts

@@ -103,7 +103,7 @@ class u64 extends BN {
     }
 
     const zeroPad = Buffer.alloc(8);
-    b.copy(zeroPad);
+    zeroPad.set(Uint8Array.from(b));
     return zeroPad;
   }
 

ts/packages/anchor/src/utils/pubkey.ts

@@ -1,4 +1,3 @@
-import { Buffer } from "buffer";
 import { PublicKey } from "@solana/web3.js";
 import { sha256 } from "@noble/hashes/sha256";
 
@@ -8,10 +7,14 @@ export function createWithSeedSync(
   seed: string,
   programId: PublicKey
 ): PublicKey {
-  const buffer = Buffer.concat([
-    fromPublicKey.toBuffer(),
-    Buffer.from(seed),
-    programId.toBuffer(),
-  ]);
-  return new PublicKey(sha256(buffer));
+  const fromKey = fromPublicKey.toBytes();
+  const seedBytes = new TextEncoder().encode(seed);
+  const program = programId.toBytes();
+  const data = new Uint8Array(
+    fromKey.length + seedBytes.length + program.length
+  );
+  data.set(fromKey, 0);
+  data.set(seedBytes, fromKey.length);
+  data.set(program, fromKey.length + seedBytes.length);
+  return new PublicKey(sha256(data));
 }

ts/packages/anchor/tsconfig.base.json

@@ -3,6 +3,7 @@
     "./src/**/*"
   ],
   "compilerOptions": {
+    "skipLibCheck": true,
     "sourceMap": true,
     "declaration": true,
     "declarationMap": true,

ts/packages/spl-associated-token-account/package.json

@@ -41,6 +41,6 @@
     "rollup": "=2.70.1",
     "rollup-plugin-terser": "=7.0.2",
     "tslib": "=2.3.1",
-    "typescript": "=4.6.2"
+    "typescript": "5.9.3"
   }
 }