refactor(data-access): thegraph-data-access package (#931)

Author: benjlevesque

.circleci/config.yml

@@ -95,6 +95,10 @@ jobs:
           name: Generate Payment Detection queries
           command: yarn workspace @requestnetwork/payment-detection run codegen
 
+      - run:
+          name: Build all packages (tsc)
+          command: yarn build:tsc
+
       - persist_to_workspace:
           root: *working_directory
           paths: .

README.md

@@ -18,25 +18,26 @@ Join the [Request Hub][requesthub-slack-url] to get in touch with us.
 
 ### Published Packages
 
-| Package                                                                | Version                                                                                                                                           | Description                                                         |
-| ---------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------- |
-| [`@requestnetwork/advanced-logic`](/packages/advanced-logic)           | [![npm](https://img.shields.io/npm/v/@requestnetwork/advanced-logic.svg)](https://www.npmjs.com/package/@requestnetwork/advanced-logic)           | Extensions to the protocol                                          |
-| [`@requestnetwork/request-client.js`](/packages/request-client.js)     | [![npm](https://img.shields.io/npm/v/@requestnetwork/request-client.js.svg)](https://www.npmjs.com/package/@requestnetwork/request-client.js)     | Library to use Request nodes as servers                             |
-| [`@requestnetwork/data-access`](/packages/data-access)                 | [![npm](https://img.shields.io/npm/v/@requestnetwork/data-access.svg)](https://www.npmjs.com/package/@requestnetwork/data-access)                 | Indexing an batching of transactions                                |
-| [`@requestnetwork/data-format`](/packages/data-format)                 | [![npm](https://img.shields.io/npm/v/@requestnetwork/data-format.svg)](https://www.npmjs.com/package/@requestnetwork/data-format)                 | Standards for data stored on Request, like invoices format          |
-| [`@requestnetwork/epk-signature`](/packages/epk-signature)             | [![npm](https://img.shields.io/npm/v/@requestnetwork/epk-signature.svg)](https://www.npmjs.com/package/@requestnetwork/epk-signature)             | Sign requests using Ethereum private keys                           |
-| [`@requestnetwork/ethereum-storage`](/packages/ethereum-storage)       | [![npm](https://img.shields.io/npm/v/@requestnetwork/ethereum-storage.svg)](https://www.npmjs.com/package/@requestnetwork/ethereum-storage)       | Storage of Request data on Ethereum and IPFS                        |
-| [`@requestnetwork/epk-decryption`](/packages/epk-decryption)           | [![npm](https://img.shields.io/npm/v/@requestnetwork/epk-decryption.svg)](https://www.npmjs.com/package/@requestnetwork/epk-decryption)           | Decrypt encrypted requests using Ethereum private keys              |
-| [`@requestnetwork/payment-detection`](/packages/payment-detection)     | [![npm](https://img.shields.io/npm/v/@requestnetwork/payment-detection.svg)](https://www.npmjs.com/package/@requestnetwork/payment-detection)     | Client-side payment detection, to compute the balance.              |
-| [`@requestnetwork/payment-processor`](/packages/payment-processor)     | [![npm](https://img.shields.io/npm/v/@requestnetwork/payment-processor.svg)](https://www.npmjs.com/package/@requestnetwork/payment-processor)     | Pay a request using a web3 wallet                                   |
-| [`@requestnetwork/request-logic`](/packages/request-logic)             | [![npm](https://img.shields.io/npm/v/@requestnetwork/request-logic.svg)](https://www.npmjs.com/package/@requestnetwork/request-logic)             | The Request business logic: properties and actions of requests      |
-| [`@requestnetwork/request-node`](/packages/request-node)               | [![npm](https://img.shields.io/npm/v/@requestnetwork/request-node.svg)](https://www.npmjs.com/package/@requestnetwork/request-node)               | Web server that allows easy access to Request system                |
-| [`@requestnetwork/transaction-manager`](/packages/transaction-manager) | [![npm](https://img.shields.io/npm/v/@requestnetwork/transaction-manager.svg)](https://www.npmjs.com/package/@requestnetwork/transaction-manager) | Creates transactions to be sent to Data Access, managing encryption |
-| [`@requestnetwork/types`](/packages/types)                             | [![npm](https://img.shields.io/npm/v/@requestnetwork/types.svg)](https://www.npmjs.com/package/@requestnetwork/types)                             | Typescript types shared across @requestnetwork packages             |
-| [`@requestnetwork/utils`](/packages/utils)                             | [![npm](https://img.shields.io/npm/v/@requestnetwork/utils.svg)](https://www.npmjs.com/package/@requestnetwork/utils)                             | Collection of tools shared between the @requestnetwork packages     |
-| [`@requestnetwork/web3-signature`](/packages/web3-signature)           | [![npm](https://img.shields.io/npm/v/@requestnetwork/web3-signature.svg)](https://www.npmjs.com/package/@requestnetwork/web3-signature)           | Sign requests using web3 tools (like Metamask)                      |
-| [`@requestnetwork/multi-format`](/packages/multi-format)               | [![npm](https://img.shields.io/npm/v/@requestnetwork/multi-format.svg)](https://www.npmjs.com/package/@requestnetwork/multi-format)               | Serialize and deserialize object in the Request Network protocol    |
-| [`@requestnetwork/smart-contracts`](/packages/smart-contracts)         | [![npm](https://img.shields.io/npm/v/@requestnetwork/smart-contracts.svg)](https://www.npmjs.com/package/@requestnetwork/smart-contracts)         | Sources and artifacts of the smart contracts                        |
+| Package                                                                  | Version                                                                                                                                             | Description                                                         |
+| ------------------------------------------------------------------------ | --------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------- |
+| [`@requestnetwork/advanced-logic`](/packages/advanced-logic)             | [![npm](https://img.shields.io/npm/v/@requestnetwork/advanced-logic.svg)](https://www.npmjs.com/package/@requestnetwork/advanced-logic)             | Extensions to the protocol                                          |
+| [`@requestnetwork/request-client.js`](/packages/request-client.js)       | [![npm](https://img.shields.io/npm/v/@requestnetwork/request-client.js.svg)](https://www.npmjs.com/package/@requestnetwork/request-client.js)       | Library to use Request nodes as servers                             |
+| [`@requestnetwork/data-access`](/packages/data-access)                   | [![npm](https://img.shields.io/npm/v/@requestnetwork/data-access.svg)](https://www.npmjs.com/package/@requestnetwork/data-access)                   | Indexing an batching of transactions                                |
+| [`@requestnetwork/data-format`](/packages/data-format)                   | [![npm](https://img.shields.io/npm/v/@requestnetwork/data-format.svg)](https://www.npmjs.com/package/@requestnetwork/data-format)                   | Standards for data stored on Request, like invoices format          |
+| [`@requestnetwork/epk-signature`](/packages/epk-signature)               | [![npm](https://img.shields.io/npm/v/@requestnetwork/epk-signature.svg)](https://www.npmjs.com/package/@requestnetwork/epk-signature)               | Sign requests using Ethereum private keys                           |
+| [`@requestnetwork/ethereum-storage`](/packages/ethereum-storage)         | [![npm](https://img.shields.io/npm/v/@requestnetwork/ethereum-storage.svg)](https://www.npmjs.com/package/@requestnetwork/ethereum-storage)         | Storage of Request data on Ethereum and IPFS, with custom indexing  |
+| [`@requestnetwork/epk-decryption`](/packages/epk-decryption)             | [![npm](https://img.shields.io/npm/v/@requestnetwork/epk-decryption.svg)](https://www.npmjs.com/package/@requestnetwork/epk-decryption)             | Decrypt encrypted requests using Ethereum private keys              |
+| [`@requestnetwork/payment-detection`](/packages/payment-detection)       | [![npm](https://img.shields.io/npm/v/@requestnetwork/payment-detection.svg)](https://www.npmjs.com/package/@requestnetwork/payment-detection)       | Client-side payment detection, to compute the balance.              |
+| [`@requestnetwork/payment-processor`](/packages/payment-processor)       | [![npm](https://img.shields.io/npm/v/@requestnetwork/payment-processor.svg)](https://www.npmjs.com/package/@requestnetwork/payment-processor)       | Pay a request using a web3 wallet                                   |
+| [`@requestnetwork/request-logic`](/packages/request-logic)               | [![npm](https://img.shields.io/npm/v/@requestnetwork/request-logic.svg)](https://www.npmjs.com/package/@requestnetwork/request-logic)               | The Request business logic: properties and actions of requests      |
+| [`@requestnetwork/request-node`](/packages/request-node)                 | [![npm](https://img.shields.io/npm/v/@requestnetwork/request-node.svg)](https://www.npmjs.com/package/@requestnetwork/request-node)                 | Web server that allows easy access to Request system                |
+| [`@requestnetwork/transaction-manager`](/packages/transaction-manager)   | [![npm](https://img.shields.io/npm/v/@requestnetwork/transaction-manager.svg)](https://www.npmjs.com/package/@requestnetwork/transaction-manager)   | Creates transactions to be sent to Data Access, managing encryption |
+| [`@requestnetwork/types`](/packages/types)                               | [![npm](https://img.shields.io/npm/v/@requestnetwork/types.svg)](https://www.npmjs.com/package/@requestnetwork/types)                               | Typescript types shared across @requestnetwork packages             |
+| [`@requestnetwork/utils`](/packages/utils)                               | [![npm](https://img.shields.io/npm/v/@requestnetwork/utils.svg)](https://www.npmjs.com/package/@requestnetwork/utils)                               | Collection of tools shared between the @requestnetwork packages     |
+| [`@requestnetwork/web3-signature`](/packages/web3-signature)             | [![npm](https://img.shields.io/npm/v/@requestnetwork/web3-signature.svg)](https://www.npmjs.com/package/@requestnetwork/web3-signature)             | Sign requests using web3 tools (like Metamask)                      |
+| [`@requestnetwork/multi-format`](/packages/multi-format)                 | [![npm](https://img.shields.io/npm/v/@requestnetwork/multi-format.svg)](https://www.npmjs.com/package/@requestnetwork/multi-format)                 | Serialize and deserialize object in the Request Network protocol    |
+| [`@requestnetwork/thegraph-data-access`](/packages/thegraph-data-access) | [![npm](https://img.shields.io/npm/v/@requestnetwork/thegraph-data-access.svg)](https://www.npmjs.com/package/@requestnetwork/thegraph-data-access) | Storage of Request data on Ethereum and IPFS, indexed by TheGraph   |
+| [`@requestnetwork/smart-contracts`](/packages/smart-contracts)           | [![npm](https://img.shields.io/npm/v/@requestnetwork/smart-contracts.svg)](https://www.npmjs.com/package/@requestnetwork/smart-contracts)           | Sources and artifacts of the smart contracts                        |
 
 ### Private Packages
 

package.json

@@ -16,7 +16,7 @@
   "scripts": {
     "build": "lerna run build",
     "clean": "lerna run clean",
-    "build:tsc": "tsc -b packages/advanced-logic/tsconfig.build.json packages/currency/tsconfig.build.json packages/data-access/tsconfig.build.json packages/data-format/tsconfig.build.json packages/epk-decryption/tsconfig.build.json packages/epk-signature/tsconfig.build.json packages/ethereum-storage/tsconfig.build.json packages/integration-test/tsconfig.build.json packages/multi-format/tsconfig.build.json packages/payment-detection/tsconfig.build.json packages/prototype-estimator/tsconfig.build.json packages/request-client.js/tsconfig.build.json packages/request-logic/tsconfig.build.json packages/request-node/tsconfig.build.json packages/smart-contracts/tsconfig.build.json packages/toolbox/tsconfig.build.json packages/transaction-manager/tsconfig.build.json packages/types/tsconfig.build.json packages/usage-examples/tsconfig.build.json packages/utils/tsconfig.build.json packages/web3-signature/tsconfig.build.json",
+    "build:tsc": "tsc -b packages/**/tsconfig.build.json",
     "lint": "eslint . --fix --quiet",
     "lint:check": "eslint . --quiet",
     "lint-staged": "lint-staged",

packages/request-node/src/thegraph/CombinedDataAccess.ts renamed to packages/data-access/src/combined-data-access.ts

@@ -1,3 +1,4 @@
 export { default as DataAccess } from './data-access';
 export { default as TransactionIndex } from './transaction-index';
 export { default as Block } from './block';
+export { CombinedDataAccess } from './combined-data-access';

packages/data-access/src/index.ts

@@ -45,13 +45,15 @@
     "@requestnetwork/utils": "0.35.0",
     "axios": "0.27.2",
     "bluebird": "3.7.2",
+    "eip1559-fee-suggestions-ethers": "1.3.3",
     "ethers": "5.5.1",
     "form-data": "3.0.0",
     "ipfs-unixfs": "6.0.7",
     "keyv": "4.0.3",
     "qs": "6.10.3",
     "shelljs": "0.8.5",
     "tslib": "2.3.1",
+    "typed-emitter": "1.4.0",
     "web3-eth": "1.3.6",
     "web3-utils": "1.3.6",
     "yargs": "16.2.0"

packages/ethereum-storage/package.json

@@ -6,29 +6,28 @@ import { LogTypes, StorageTypes } from '@requestnetwork/types';
 import { requestHashSubmitterArtifact } from '@requestnetwork/smart-contracts';
 import { RequestOpenHashSubmitter } from '@requestnetwork/smart-contracts/types';
 import { suggestFees } from 'eip1559-fee-suggestions-ethers';
-import { GasPriceDefiner } from '@requestnetwork/ethereum-storage';
 
-type TheGraphStorageProps = {
+type StorageProps = {
   network: string;
   signer: Signer;
   ipfsStorage: StorageTypes.IIpfsStorage;
   logger?: LogTypes.ILogger;
 };
 
-export type TheGraphStorageEventEmitter = TypedEmitter<{
+export type StorageEventEmitter = TypedEmitter<{
   confirmed: (receipt: ContractReceipt) => void;
   error: (error: unknown) => void;
 }>;
 
-export class TheGraphStorage {
+export class EthereumStorageEthers implements StorageTypes.IStorageWrite {
   private readonly logger: LogTypes.ILogger;
   private readonly ipfsStorage: StorageTypes.IIpfsStorage;
   private readonly hashSubmitter: RequestOpenHashSubmitter;
   private readonly network: string;
   private readonly provider: providers.JsonRpcProvider;
   private enableEip1559 = true;
 
-  constructor({ network, signer, ipfsStorage, logger }: TheGraphStorageProps) {
+  constructor({ network, signer, ipfsStorage, logger }: StorageProps) {
     this.logger = logger || new Utils.SimpleLogger();
     this.ipfsStorage = ipfsStorage;
     this.network = network;
@@ -49,7 +48,7 @@ export class TheGraphStorage {
       );
       this.enableEip1559 = false;
     }
-    this.logger.debug('TheGraph storage initialized');
+    this.logger.debug(`${EthereumStorageEthers.name} storage initialized`);
   }
 
   async append(content: string): Promise<StorageTypes.IAppendResult> {
@@ -65,21 +64,14 @@ export class TheGraphStorage {
       const maxFeePerGas = maxPriorityFeePerGas.add(suggestedFee.baseFeeSuggestion);
       overrides.maxPriorityFeePerGas = maxPriorityFeePerGas;
       overrides.maxFeePerGas = maxFeePerGas;
-    } else {
-      // retro-compatibility for networks where the eth_feeHistory RPC method is not available (pre EIP-1559)
-      const gasPriceDefiner = new GasPriceDefiner();
-      overrides.gasPrice = await gasPriceDefiner.getGasPrice(
-        StorageTypes.GasPriceType.FAST,
-        this.network,
-      );
     }
     const tx = await this.hashSubmitter.submitHash(
       ipfsHash,
       utils.hexZeroPad(utils.hexlify(ipfsSize), 32),
       overrides,
     );
 
-    const eventEmitter = new EventEmitter() as TheGraphStorageEventEmitter;
+    const eventEmitter = new EventEmitter() as StorageEventEmitter;
     const result: StorageTypes.IEntry = {
       id: ipfsHash,
       content,

packages/request-node/src/thegraph/TheGraphStorage.ts renamed to packages/ethereum-storage/src/ethereum-storage-ethers.ts

@@ -1,3 +1,4 @@
 export { EthereumStorage } from './ethereum-storage';
+export { EthereumStorageEthers } from './ethereum-storage-ethers';
 export { GasPriceDefiner } from './gas-price-definer';
 export { IpfsStorage } from './ipfs-storage';

packages/ethereum-storage/src/index.ts

@@ -45,13 +45,13 @@
     "@requestnetwork/data-access": "0.26.0",
     "@requestnetwork/ethereum-storage": "0.26.0",
     "@requestnetwork/smart-contracts": "0.28.0",
+    "@requestnetwork/thegraph-data-access": "0.35.0",
     "@requestnetwork/types": "0.35.0",
     "@requestnetwork/utils": "0.35.0",
     "@truffle/hdwallet-provider": "1.2.3",
     "chalk": "4.1.0",
     "cors": "2.8.5",
     "dotenv": "8.2.0",
-    "eip1559-fee-suggestions-ethers": "1.3.3",
     "ethers": "5.5.1",
     "express": "4.17.1",
     "graphql": "15.5.0",
@@ -81,7 +81,6 @@
     "ts-jest": "26.3.0",
     "ts-node": "9.0.0",
     "ts-node-dev": "1.0.0-pre.62",
-    "typed-emitter": "1.4.0",
     "typescript": "4.4.4"
   },
   "gitHead": "6155223cfce769e48ccae480c510b35b4f54b4d0"

packages/request-node/package.json

@@ -5,7 +5,7 @@ import * as config from './config';
 import { Logger } from './logger';
 import { RequestNode } from './requestNode';
 import withShutdown from 'http-shutdown';
-import { TheGraphRequestNode } from './thegraph';
+import { TheGraphRequestNode } from './thegraph-node';
 
 // Initialize the node logger
 const { logLevel, logMode } = config.getLogConfig();