Add metadataVer cli arg for polkadot-types-internal-metadata, and ensure backwards compatibility with v14 (#6063)

* Ensure ws util has added options

* Ensure backwards compatibility

Author: TarikGul

packages/typegen/src/metadataMd.ts

@@ -31,8 +31,9 @@ import polkadotRpc from '@polkadot/types-support/metadata/v15/polkadot-rpc';
 import polkadotVer from '@polkadot/types-support/metadata/v15/polkadot-ver';
 import substrateMeta from '@polkadot/types-support/metadata/v15/substrate-hex';
 import { isHex, stringCamelCase, stringLowerFirst } from '@polkadot/util';
+import { blake2AsHex } from '@polkadot/util-crypto';
 
-import { assertFile, getMetadataViaWs, getRpcMethodsViaWs } from './util/index.js';
+import { assertFile, getMetadataViaWs, getRpcMethodsViaWs, getRuntimeVersionViaWs } from './util/index.js';
 
 interface SectionItem {
   link?: string;
@@ -318,6 +319,63 @@ function addRuntime (_runtimeDesc: string, registry: Registry): string {
   });
 }
 
+/** @internal */
+function addLegacyRuntime (_runtimeDesc: string, _registry: Registry, apis?: ApiDef[]) {
+  return renderPage({
+    description: 'The following section contains known runtime calls that may be available on specific runtimes (depending on configuration and available pallets). These call directly into the WASM runtime for queries and operations.',
+    sections: Object
+      .keys(definitions)
+      .filter((key) => Object.keys(definitions[key as 'babe'].runtime || {}).length !== 0)
+      .sort()
+      .reduce((all: Section[], _sectionName): Section[] => {
+        Object
+          .entries(definitions[_sectionName as 'babe'].runtime || {})
+          .forEach(([apiName, versions]) => {
+            versions
+              .sort((a, b) => b.version - a.version)
+              .forEach(({ methods, version }, index) => {
+                if (apis) {
+                  // if we are passing the api hashes and we cannot find this one, skip it
+                  const apiHash = blake2AsHex(apiName, 64);
+                  const api = apis.find(([hash]) => hash === apiHash);
+
+                  if (!api || api[1] !== version) {
+                    return;
+                  }
+                } else if (index) {
+                  // we only want the highest version
+                  return;
+                }
+
+                const container: Section = { items: [], name: apiName };
+
+                all.push(container);
+
+                Object
+                  .entries(methods)
+                  .sort(([a], [b]) => a.localeCompare(b))
+                  .forEach(([methodName, { description, params, type }]): void => {
+                    const args = params.map(({ name, type }): string => {
+                      // eslint-disable-next-line @typescript-eslint/restrict-plus-operands
+                      return name + ': `' + type + '`';
+                    }).join(', ');
+
+                    container.items.push({
+                      interface: '`' + `api.call.${stringCamelCase(apiName)}.${stringCamelCase(methodName)}` + '`',
+                      name: `${stringCamelCase(methodName)}(${args}): ${'`' + type + '`'}`,
+                      runtime: '`' + `${apiName}_${methodName}` + '`',
+                      summary: description
+                    });
+                  });
+              });
+          });
+
+        return all;
+      }, []).sort(sortByName),
+    title: 'Runtime'
+  });
+}
+
 /** @internal */
 function addConstants (runtimeDesc: string, { lookup, pallets }: MetadataLatest): string {
   return renderPage({
@@ -504,28 +562,40 @@ function writeFile (name: string, ...chunks: any[]): void {
   writeStream.end();
 }
 
-interface ArgV { chain?: string; endpoint?: string; }
+interface ArgV { chain?: string; endpoint?: string; metadataVer?: number; }
 
 async function mainPromise (): Promise<void> {
-  const { chain, endpoint } = yargs(hideBin(process.argv)).strict().options({
+  const { chain, endpoint, metadataVer } = yargs(hideBin(process.argv)).strict().options({
     chain: {
       description: 'The chain name to use for the output (defaults to "Substrate")',
       type: 'string'
     },
     endpoint: {
       description: 'The endpoint to connect to (e.g. wss://kusama-rpc.polkadot.io) or relative path to a file containing the JSON output of an RPC state_getMetadata call',
       type: 'string'
+    },
+    metadataVer: {
+      description: 'The metadata version to use for generating type information. This will use state_call::Metadata_metadata_at_version to query metadata',
+      type: 'number'
     }
   }).argv as ArgV;
 
+  /**
+   * This is unique to when the endpoint arg is used. Since the endpoint requires us to query the chain, it may query the chains
+   * rpc state_getMetadata method to get metadata, but this restricts us to v14 only. Therefore we must also check if the `metadataVer` is passed
+   * in as well. These checks will help us decide if we are using v14 or newer.
+   */
+  const useV14Metadata = endpoint && ((metadataVer && metadataVer < 15) || !metadataVer);
   const chainName = chain || 'Substrate';
   let metaHex: HexString;
   let rpcMethods: string[] | undefined;
+  let runtimeApis: ApiDef[] | undefined;
 
   if (endpoint) {
     if (endpoint.startsWith('wss://') || endpoint.startsWith('ws://')) {
-      metaHex = await getMetadataViaWs(endpoint);
+      metaHex = await getMetadataViaWs(endpoint, metadataVer);
       rpcMethods = await getRpcMethodsViaWs(endpoint);
+      runtimeApis = await getRuntimeVersionViaWs(endpoint);
     } else {
       metaHex = (
         JSON.parse(
@@ -544,9 +614,16 @@ async function mainPromise (): Promise<void> {
     metaHex = substrateMeta;
   }
 
+  let metadata: Metadata;
   const registry = new TypeRegistry();
-  const opaqueMetadata = registry.createType('Option<OpaqueMetadata>', registry.createType('Raw', metaHex).toU8a()).unwrap();
-  const metadata = new Metadata(registry, opaqueMetadata.toHex());
+
+  if (useV14Metadata) {
+    metadata = new Metadata(registry, metaHex);
+  } else {
+    const opaqueMetadata = registry.createType('Option<OpaqueMetadata>', registry.createType('Raw', metaHex).toU8a()).unwrap();
+
+    metadata = new Metadata(registry, opaqueMetadata.toHex());
+  }
 
   registry.setMetadata(metadata);
 
@@ -556,7 +633,9 @@ async function mainPromise (): Promise<void> {
 
   writeFile(`${docRoot}/rpc.md`, addRpc(runtimeDesc, rpcMethods));
 
-  writeFile(`${docRoot}/runtime.md`, addRuntime(runtimeDesc, registry));
+  useV14Metadata
+    ? writeFile(`${docRoot}/runtime.md`, addLegacyRuntime(runtimeDesc, registry, runtimeApis))
+    : writeFile(`${docRoot}/runtime.md`, addRuntime(runtimeDesc, registry));
 
   writeFile(`${docRoot}/constants.md`, addConstants(runtimeDesc, latest));
   writeFile(`${docRoot}/storage.md`, addStorage(runtimeDesc, latest));

packages/typegen/src/util/wsMeta.ts

@@ -4,10 +4,11 @@
 import type { HexString } from '@polkadot/util/types';
 
 import { promiseTracker } from '@polkadot/api/promise/decorateMethod';
-import { stringify } from '@polkadot/util';
+import { TypeRegistry } from '@polkadot/types';
+import { stringify, u8aToHex } from '@polkadot/util';
 import { WebSocket } from '@polkadot/x-ws';
 
-async function getWsData <T> (endpoint: string, method: 'rpc_methods' | 'state_getMetadata' | 'state_getRuntimeVersion'): Promise<T> {
+async function getWsData <T> (endpoint: string, method: 'rpc_methods' | 'state_call' | 'state_getMetadata' | 'state_getRuntimeVersion', params?: string[]): Promise<T> {
   return new Promise((resolve, reject): void => {
     const tracker = promiseTracker<T>(resolve, reject);
 
@@ -26,7 +27,9 @@ async function getWsData <T> (endpoint: string, method: 'rpc_methods' | 'state_g
 
       websocket.onopen = (): void => {
         console.log('connected');
-        websocket.send(`{"id":"1","jsonrpc":"2.0","method":"${method}","params":[]}`);
+        params
+          ? websocket.send(`{"id":"1","jsonrpc":"2.0","method":"${method}","params":[${params.map((param) => `"${param}"`).join(',')}]}`)
+          : websocket.send(`{"id":"1","jsonrpc":"2.0","method":"${method}","params":[]}`);
       };
 
       websocket.onmessage = (message: { data: string }): void => {
@@ -44,8 +47,14 @@ async function getWsData <T> (endpoint: string, method: 'rpc_methods' | 'state_g
   });
 }
 
-export async function getMetadataViaWs (endpoint: string): Promise<HexString> {
-  return getWsData<HexString>(endpoint, 'state_getMetadata');
+export async function getMetadataViaWs (endpoint: string, metadataVer?: number): Promise<HexString> {
+  const registry = new TypeRegistry();
+
+  if (metadataVer) {
+    return await getWsData<HexString>(endpoint, 'state_call', ['Metadata_metadata_at_version', u8aToHex(registry.createType('u32', metadataVer).toU8a())]);
+  } else {
+    return await getWsData<HexString>(endpoint, 'state_getMetadata');
+  }
 }
 
 export async function getRpcMethodsViaWs (endpoint: string): Promise<string[]> {