Add .keys on storage entries (#2032)

* Add .keys on storage entries

* Pass isOptional to storage type unwrap

* Linked maps TS types

Author: jacogr

CHANGELOG.md

@@ -2,13 +2,15 @@
 
 - **Breaking change** `api.rpc.state.queryStorage(...)` now properly decodes the `Vec<StorageChangeSet>`. this means that results will come back as `[Hash, Codec[]][]` when using this RPC.
 - `StorageKey` now have an `.args` property, decoded from meta in the cases twox64concat is used on maps
-- Fix storage `.entries` type conversions to return exact types (not `Option` in some cases)
+- Fix `api.query.*.*.entries` type conversions to return exact types (not `Option` in some cases)
+- Add `api.query.*.*.keys` to retrieve only the storage keys, similar to `.entries`
 - Full linked map retrievals will now use direct getStorage queries for faster operation
 - Underlying rpc-core interfaces now unwraps `Error("...")` when found in responses
 - Added `derive.eras*` interfaces for queries to new Substrate staking interfaces
 - Update `derive.account` to cater for new indices module storage (detected, fallbacks)
 - Adjust derive queries for session move away from module prefix (DoubleMap -> Map), detected on use
 - Add runtime validation for map arguments to `api.query.*`
+- TypeScript interfaces for linked maps now correctly generates as `[Type, Linkage<Next>]`
 
 # 1.5.1 Mar 06, 2020
 

packages/api-derive/src/staking/erasExposure.ts

@@ -13,12 +13,7 @@ import { StorageKey } from '@polkadot/types';
 
 import { memo } from '../util';
 
-interface Mapped {
-  nominators: DeriveEraNominatorExposure;
-  validators: DeriveEraValidatorExposure;
-}
-
-function mapStakers (stakers: [StorageKey, Exposure][]): Mapped {
+function mapStakers (era: EraIndex, stakers: [StorageKey, Exposure][]): DeriveEraExposure {
   const nominators: DeriveEraNominatorExposure = {};
   const validators: DeriveEraValidatorExposure = {};
 
@@ -35,7 +30,7 @@ function mapStakers (stakers: [StorageKey, Exposure][]): Mapped {
     });
   });
 
-  return { nominators, validators };
+  return { era, nominators, validators };
 }
 
 export function erasExposure (api: ApiInterfaceRx): (withActive?: boolean | BN | number) => Observable<DeriveEraExposure[]> {
@@ -56,10 +51,9 @@ export function erasExposure (api: ApiInterfaceRx): (withActive?: boolean | BN |
         ])
       ),
       map(([eras, erasStakers]): DeriveEraExposure[] =>
-        eras.map((era, index): DeriveEraExposure => ({
-          era,
-          ...mapStakers(erasStakers[index])
-        }))
+        eras.map((era, index): DeriveEraExposure =>
+          mapStakers(era, erasStakers[index])
+        )
       )
     )
   );

packages/api-derive/src/staking/validators.ts

@@ -3,7 +3,7 @@
 // of the Apache-2.0 license. See the LICENSE file for details.
 
 import { ApiInterfaceRx } from '@polkadot/api/types';
-import { AccountId, Exposure } from '@polkadot/types/interfaces';
+import { AccountId } from '@polkadot/types/interfaces';
 import { DeriveStakingValidators } from '../types';
 
 import { Observable, combineLatest, of } from 'rxjs';
@@ -16,11 +16,11 @@ function queryNextStakers (api: ApiInterfaceRx): Observable<AccountId[]> {
   // only populate for next era in the last session, so track both here - entries are not
   // subscriptions, so we need a trigger - currentIndex acts as that trigger to refresh
   return api.derive.session.indexes().pipe(
-    switchMap(({ activeEra }): Observable<[StorageKey, Exposure][]> =>
-      api.query.staking.erasStakers.entries(activeEra.addn(1)).pipe(take(1))
+    switchMap(({ activeEra }): Observable<StorageKey[]> =>
+      api.query.staking.erasStakers.keys(activeEra.addn(1)).pipe(take(1))
     ),
-    map((entries): AccountId[] =>
-      entries.map(([key]): AccountId => key.args[1] as AccountId)
+    map((keys): AccountId[] =>
+      keys.map((key): AccountId => key.args[1] as AccountId)
     )
   );
 }

packages/api/src/augment/query.ts

@@ -2,7 +2,7 @@
 /* eslint-disable @typescript-eslint/no-empty-interface */
 
 import { AnyNumber, ITuple } from '@polkadot/types/types';
-import { Option, U8aFixed, Vec } from '@polkadot/types/codec';
+import { Linkage, Option, U8aFixed, Vec } from '@polkadot/types/codec';
 import { Bytes, Data, bool, u32, u64 } from '@polkadot/types/primitive';
 import { UncleEntryItem } from '@polkadot/types/interfaces/authorship';
 import { BabeAuthorityWeight, MaybeVrf } from '@polkadot/types/interfaces/babe';
@@ -255,11 +255,11 @@ declare module '@polkadot/api/types/storage' {
       /**
        * The map from (wannabe) validator stash key to the preferences of that validator.
        **/
-      validators: AugmentedQuery<ApiType, (arg: AccountId | string | Uint8Array) => Observable<ValidatorPrefs>> & QueryableStorageEntry<ApiType>;
+      validators: AugmentedQuery<ApiType, (arg: AccountId | string | Uint8Array) => Observable<ITuple<[ValidatorPrefs, Linkage<AccountId>]>>> & QueryableStorageEntry<ApiType>;
       /**
        * The map from nominator stash key to the set of stash keys of all validators to nominate.
        **/
-      nominators: AugmentedQuery<ApiType, (arg: AccountId | string | Uint8Array) => Observable<Option<Nominations>>> & QueryableStorageEntry<ApiType>;
+      nominators: AugmentedQuery<ApiType, (arg: AccountId | string | Uint8Array) => Observable<Option<ITuple<[Nominations, Linkage<AccountId>]>>>> & QueryableStorageEntry<ApiType>;
       /**
        * The current era index.
        * This is the latest planned era, depending on how session module queues the validator
@@ -459,7 +459,7 @@ declare module '@polkadot/api/types/storage' {
       /**
        * Get the account (and lock periods) to which another account is delegating vote.
        **/
-      delegations: AugmentedQuery<ApiType, (arg: AccountId | string | Uint8Array) => Observable<ITuple<[AccountId, Conviction]>>> & QueryableStorageEntry<ApiType>;
+      delegations: AugmentedQuery<ApiType, (arg: AccountId | string | Uint8Array) => Observable<ITuple<[ITuple<[AccountId, Conviction]>, Linkage<AccountId>]>>> & QueryableStorageEntry<ApiType>;
       /**
        * Accounts for which there are locks in action which may be removed at some point in the
        * future. The value is the block number at which the lock expires and may be removed.
@@ -560,7 +560,7 @@ declare module '@polkadot/api/types/storage' {
       /**
        * Votes of a particular voter, with the round index of the votes.
        **/
-      votesOf: AugmentedQuery<ApiType, (arg: AccountId | string | Uint8Array) => Observable<Vec<AccountId>>> & QueryableStorageEntry<ApiType>;
+      votesOf: AugmentedQuery<ApiType, (arg: AccountId | string | Uint8Array) => Observable<ITuple<[Vec<AccountId>, Linkage<AccountId>]>>> & QueryableStorageEntry<ApiType>;
       /**
        * Locked stake of a voter.
        **/

packages/api/src/base/Decorate.ts

@@ -326,6 +326,9 @@ export default abstract class Decorate<ApiType extends ApiTypes> extends Events
     decorated.keyPrefix = (): string =>
       u8aToHex(creator.keyPrefix);
 
+    decorated.keys = decorateMethod(memo((doubleMapArg?: Arg): Observable<StorageKey[]> =>
+      this.retrieveMapKeys(creator, doubleMapArg)));
+
     if (this.hasSubscriptions) {
       // When using double map storage function, user need to pass double map key as an array
       decorated.multi = decorateMethod((args: (Arg | Arg[])[]): Observable<Codec[]> =>
@@ -343,11 +346,7 @@ export default abstract class Decorate<ApiType extends ApiTypes> extends Events
   }
 
   private decorateStorageRange<ApiType extends ApiTypes> (decorated: QueryableStorageEntry<ApiType>, args: [Arg?, Arg?], range: [Hash, Hash?]): Observable<[Hash, Codec][]> {
-    let outputType: any = unwrapStorageType(decorated.creator.meta.type);
-
-    if (decorated.creator.meta.modifier.isOptional) {
-      outputType = `Option<${outputType}>`;
-    }
+    const outputType = unwrapStorageType(decorated.creator.meta.type, decorated.creator.meta.modifier.isOptional);
 
     return this._rpcCore.state
       .queryStorage<[Option<Raw>]>([decorated.key(...args)], ...range)
@@ -431,34 +430,36 @@ export default abstract class Decorate<ApiType extends ApiTypes> extends Events
     );
   }
 
-  private retrieveMapEntries ({ iterKey, meta }: StorageEntry, arg?: Arg): Observable<[StorageKey, Codec][]> {
-    assert(iterKey && (meta.type.isMap || meta.type.isDoubleMap), 'entries can only be retrieved on maps, linked maps and double maps');
+  private retrieveMapKeys ({ iterKey, meta }: StorageEntry, arg?: Arg): Observable<StorageKey[]> {
+    assert(iterKey && (meta.type.isMap || meta.type.isDoubleMap), 'keys can only be retrieved on maps, linked maps and double maps');
 
-    let outputType: any = unwrapStorageType(meta.type);
-
-    if (meta.modifier.isOptional) {
-      outputType = `Option<${outputType}>`;
-    }
+    const startKey = this.createType('Raw', u8aConcat(
+      iterKey,
+      meta.type.isDoubleMap && !isUndefined(arg) && !isNull(arg)
+        ? getHasher(meta.type.asDoubleMap.hasher)(
+          this.createType(meta.type.asDoubleMap.key1.toString() as 'Raw', arg).toU8a()
+        )
+        : new Uint8Array([])
+    ));
 
-    return this._rpcCore.state
-      // TODO This should really be some sort of subscription, so we can get stuff as
-      // it changes (as of now it is a one-shot query). Not available on Substrate.
-      .getKeys(
-        this.createType('Raw', u8aConcat(
-          iterKey,
-          meta.type.isDoubleMap && !isUndefined(arg) && !isNull(arg)
-            ? getHasher(meta.type.asDoubleMap.hasher)(
-              this.createType(meta.type.asDoubleMap.key1.toString() as 'Raw', arg).toU8a()
-            )
-            : new Uint8Array([])
-        ))
+    return this._rpcCore.state.getKeys(startKey).pipe(
+      map((keys): StorageKey[] =>
+        keys.map((key) => key.decodeArgsFromMeta(meta))
       )
+    );
+  }
+
+  private retrieveMapEntries (entry: StorageEntry, arg?: Arg): Observable<[StorageKey, Codec][]> {
+    const outputType = unwrapStorageType(entry.meta.type, entry.meta.modifier.isOptional);
+
+    return this.retrieveMapKeys(entry, arg)
       .pipe(
         switchMap((keys): Observable<[StorageKey[], Option<Raw>[]]> =>
           combineLatest([
-            of(keys.map((key) => key.decodeArgsFromMeta(meta))),
-            // since we have a default constructed key, we have Option<Raw> as the result
-            // TODO Don't keep the subscription here active, retrieve and get out of Dodge
+            of(keys),
+            // Since we have a default constructed key, we have Option<Raw> as the result
+            // Don't keep the subscription here active, retrieve and get out of Dodge
+            // TODO Once we have a multiple key RPC available, use that via fallback
             this._rpcCore.state.subscribeStorage<Option<Raw>[]>(keys).pipe(take(1))
           ])
         ),

packages/api/src/types/storage.ts

@@ -32,6 +32,7 @@ export interface StorageEntryBase<ApiType extends ApiTypes, F extends AnyFunctio
   hash: (...args: Parameters<F>) => PromiseOrObs<ApiType, Hash>;
   key: (...args: Parameters<F>) => string;
   keyPrefix: () => string;
+  keys: (arg?: any) => PromiseOrObs<ApiType, StorageKey[]>;
   range: <T extends Codec | any = ObsInnerType<ReturnType<F>>>([from, to]: [Hash | Uint8Array | string, Hash | Uint8Array | string | undefined] | [Hash | Uint8Array | string], ...args: Parameters<F>) => PromiseOrObs<ApiType, [Hash, T][]>;
   size: (...args: Parameters<F>) => PromiseOrObs<ApiType, u64>;
   multi: ApiType extends 'rxjs' ? StorageEntryObservableMulti : StorageEntryPromiseMulti;

packages/typegen/src/generate/query.ts

@@ -8,30 +8,22 @@ import * as defaultDefs from '@polkadot/types/interfaces/definitions';
 
 import staticData from '@polkadot/metadata/Metadata/static';
 import Metadata from '@polkadot/metadata/Metadata';
+import { unwrapStorageType } from '@polkadot/types/primitive/StorageKey';
 import { TypeRegistry } from '@polkadot/types/create';
 import { stringLowerFirst } from '@polkadot/util';
 
 import { FOOTER, HEADER, TypeImports, createDocComments, createImportCode, createImports, formatType, getSimilarTypes, indent, registerDefinitions, setImports, writeFile } from '../util';
 
-// If the StorageEntry returns T, output `Option<T>` if the modifier is optional
-/** @internal */
-function addModifier (storageEntry: StorageEntryMetadataLatest, returnType: string): string {
-  if (storageEntry.modifier.isOptional) {
-    return `Option<${returnType}>`;
-  }
-
-  return returnType;
-}
-
 // From a storage entry metadata, we return [args, returnType]
 /** @internal */
 function entrySignature (allDefs: object, registry: Registry, storageEntry: StorageEntryMetadataLatest, imports: TypeImports): [string, string] {
   const format = (type: string): string => formatType(allDefs, type, imports);
+  const outputType = unwrapStorageType(storageEntry.type, storageEntry.modifier.isOptional);
 
   if (storageEntry.type.isPlain) {
     setImports(allDefs, imports, [storageEntry.type.asPlain.toString()]);
 
-    return ['', formatType(allDefs, addModifier(storageEntry, storageEntry.type.asPlain.toString()), imports)];
+    return ['', formatType(allDefs, outputType, imports)];
   } else if (storageEntry.type.isMap) {
     // Find similar types of the `key` type
     const similarTypes = getSimilarTypes(allDefs, registry, storageEntry.type.asMap.key.toString(), imports);
@@ -43,7 +35,7 @@ function entrySignature (allDefs: object, registry: Registry, storageEntry: Stor
 
     return [
       `arg: ${similarTypes.map(format).join(' | ')}`,
-      formatType(allDefs, addModifier(storageEntry, storageEntry.type.asMap.value.toString()), imports)
+      formatType(allDefs, outputType, imports)
     ];
   } else if (storageEntry.type.isDoubleMap) {
     // Find similar types of `key1` and `key2` types
@@ -61,7 +53,7 @@ function entrySignature (allDefs: object, registry: Registry, storageEntry: Stor
 
     return [
       `key1: ${key1Types}, key2: ${key2Types}`,
-      formatType(allDefs, addModifier(storageEntry, storageEntry.type.asDoubleMap.value.toString()), imports)
+      formatType(allDefs, outputType, imports)
     ];
   }
 

packages/typegen/src/metadataMd.ts

@@ -3,7 +3,6 @@
 // of the Apache-2.0 license. See the LICENSE file for details.
 
 import { MetadataLatest } from '@polkadot/types/interfaces/metadata';
-import { InterfaceTypes } from '@polkadot/types/types';
 
 import fs from 'fs';
 import interfaces from '@polkadot/jsonrpc';
@@ -181,14 +180,10 @@ function addStorage (metadata: MetadataLatest): string {
                 ? ('`' + func.type.asDoubleMap.key1.toString() + ', ' + func.type.asDoubleMap.key2.toString() + '`')
                 : '';
             const methodName = stringLowerFirst(func.name.toString());
-            let result = unwrapStorageType(func.type);
-
-            if (func.modifier.isOptional) {
-              result = `Option<${result}>` as keyof InterfaceTypes;
-            }
+            const outputType = unwrapStorageType(func.type, func.modifier.isOptional);
 
             return {
-              name: `${methodName}(${arg}): ` + '`' + result + '`',
+              name: `${methodName}(${arg}): ` + '`' + outputType + '`',
               interface: '`' + `api.query.${sectionName}.${methodName}` + '`',
               ...(func.documentation.length && { summary: func.documentation })
             };

packages/typegen/src/util/formatting.ts

@@ -105,6 +105,14 @@ function formatHashMap (key: string, val: string): string {
   return `HashMap<${key}, ${val}>`;
 }
 
+/**
+ * Given the inner `T`, return a `Vec<T>` string
+ */
+/** @internal */
+function formatLinkage (inner: string): string {
+  return paramsNotation('Linkage', inner);
+}
+
 /**
  * Given the inner `O` & `E`, return a `Result<O, E>`  string
  */
@@ -221,6 +229,13 @@ export function formatType (definitions: object, type: string | TypeDef, imports
 
       return formatHashMap(formatType(definitions, keyDef.type, imports), formatType(definitions, valDef.type, imports));
     }
+    case TypeDefInfo.Linkage: {
+      const type = (typeDef.sub as TypeDef).type;
+
+      setImports(definitions, imports, ['Linkage']);
+
+      return formatLinkage(formatType(definitions, type, imports));
+    }
     case TypeDefInfo.Result: {
       setImports(definitions, imports, ['Result']);
 

packages/types/src/codec/index.ts

@@ -14,6 +14,7 @@ export { default as Compact } from './Compact';
 // export { default as Date } from './Date';
 export { default as Enum } from './Enum';
 export { default as HashMap } from './HashMap';
+export { default as Linkage } from './Linkage';
 export { default as Option } from './Option';
 export { default as Result } from './Result';
 export { default as Set } from './Set';