Eth 138 migrate streams to ocr (#190)

* added bulk add stream function and test to migrate streams from core-api

* added first draft of script, parsing tsv file with all streams, checking validity of ids

* further into migration script, nonce is issue

* migrate script is working with manual nonces

* adding an array of permissions per stream into registry ([][] not supported by thegraph, need to remove the method from abi)

* migrate script does not spam mempool and saves progress to disk

* added stream data clean script

* refactores migration script to use new cleanded up data

* little linting

* detect stuck transactions, resend them with higher gasPrice working

* renaming of permission attributes cherrypicked from master

* reset progress file

* fixed possible timing async issue

* created new package with migration, that can connect to the sqldb and uses the contracts sources from the other package to talk to the chain

* added graphql client

* graphql query not working completely yet

* refactoring migration logic

* further writing logic for migration

* continue implementation and refactor of migration script

* refactor migration algorithm

* seems to work as intended

* code cleanup, adding better log outputs

* resolve dependency cycle, some renaming

* small fix; gasprice streategy; permissions are migrated in chunks of 20

* fix pagination

* add correct metadata from the database

* script is writing back migrated_at timestamps to the DB

* looping migration

* some cleanup

* some cleanup, loop-running without self-referencing setTimeouts

* remove setup of migrator role

* production endpoint settings

* moved configs to .env file/env variables

* replacement logic for transactions, where they are replaced with higher gas ones

* debug code cleanup

* increase timeout on stuck transaction to 30 seconds

* remove one debug output

* add one debug output

* change of gas increase, some improvements in error handling, env variables, log outputs

* fixed bug with number addition not working in classes in typescript

* better error handling

* fix: bugfix with permissions, where all their streams are filtered out in the graphQL query

* fix linting

* fix: fact that migrationscript has no tests should not throw error

* cleanup

* fix: lint

* doc: add comments

* refactor: mostly renaming and adding comments

while reviewing the PR

* fix: rpc URL

* build: remove IDE file

* Revert "refactor: mostly renaming and adding comments"

This reverts commit fc0475c.

Co-authored-by: Juuso Takalainen <[email protected]>

Author: samt1803

.gitignore

@@ -5,6 +5,8 @@
 /packages/smartcontracts/artifacts/
 /packages/smartcontracts/typechain/
 /packages/smartcontracts/.openzeppelin/
+/packages/brubeck-migration-script/artifacts
+/packages/brubeck-migration-script/typechain
 /bin/
 .DS_Store
 .idea

.vscode/launch.json

@@ -46,6 +46,16 @@
             },
             "cwd": "${workspaceFolder}/packages/smartcontracts"
         },
+        {
+            "command": "npm run migration-run",
+            "name": "Run migration",
+            "request": "launch",
+            "type": "node-terminal",
+            "env": {
+                "DEBUG": "*,-mocha:*"
+            },
+            "cwd": "${workspaceFolder}"
+        },
         {
             "type": "node",
             "request": "launch",

.vscode/settings.json

@@ -16,7 +16,9 @@
         "graph2_copyInfo": "./scripts/copyAbis.sh && ./scripts/generateYaml.sh",
         "graph3_build": "npm run doAll --workspace=streamregistry-thegraph-subgraph",
         "graph": "npm run graph1_build && npm run graph2_copyInfo && npm run graph3_build",
-        "e2etest": "npm run e2etest --workspace=smartcontracts"
+        "e2etest": "npm run e2etest --workspace=smartcontracts",
+        "migration-build": "npm run build -w=brubeck-migration-script",
+        "migration-run": "npm run migration -w=brubeck-migration-script"
     },
     "devDependencies": {
         "@types/chai": "4.3.0",

package.json

@@ -0,0 +1,14 @@
+DB_HOST="localhost"
+DB_USER="root"
+DB_PASSWORD="password"
+DB_NAME="core_test"
+
+PAUSE_BETWEEN_MIGRATIONS_MS="5000"
+THE_GRAPH_URL="http://10.200.10.1:8000/subgraphs/name/streamr-dev/network-contracts"
+CHAIN_NODE_URL="http://localhost:8546"
+ADMIN_PRIVATEKEY="0x5e98cce00cff5dea6b454889f359a4ec06b9fa6b88e9d69b86de8e1c81887da0"
+MIGRATOR_PRIVATEKEY="0x000000000000000000000000000000000000000000000000000000000000000c"
+STREAMREGISTRY_ADDRESS="0x6cCdd5d866ea766f6DF5965aA98DeCCD629ff222"
+
+GASPRICE_INCREASE_PERCENT="10"
+GASPRICE_INCREASE_TIMEOUT_MS="40000"

packages/brubeck-migration-script/.env

@@ -0,0 +1,12 @@
+DB_HOST="localhost"
+DB_USER="root"
+DB_PASSWORD=""
+DB_NAME="streamr_prod"
+
+PAUSE_BETWEEN_MIGRATIONS_MS="30000"
+THE_GRAPH_URL="https://api.thegraph.com/subgraphs/name/streamr-dev/streams"
+CHAIN_NODE_URL="https://polygon-rpc.com/"
+STREAMREGISTRY_ADDRESS="0x0D483E10612F327FC11965Fc82E90dC19b141641"
+
+GASPRICE_INCREASE_PERCENT="10"
+GASPRICE_INCREASE_TIMEOUT_MS="40000"

packages/brubeck-migration-script/.env_prod

@@ -0,0 +1,84 @@
+// import { task } from 'hardhat/config'
+// import '@nomiclabs/hardhat-waffle'
+import 'hardhat-typechain'
+// import { HardhatUserConfig } from 'hardhat/types'
+// import 'hardhat-deploy'
+import 'hardhat-deploy-ethers'
+require('@openzeppelin/hardhat-upgrades')
+
+// require('solidity-coverage')
+require('hardhat-dependency-compiler')
+
+// This is a sample Hardhat task. To learn how to create your own go to
+// https://hardhat.org/guides/create-task.html
+// task('accounts', 'Prints the list of accounts', async (args, hre) => {
+//     const accounts = await hre.ethers.getSigners()
+//     // eslint-disable-next-line no-restricted-syntax
+//     for (const account of accounts) {
+//         // eslint-disable-next-line no-console
+//         console.log(account.address)
+//     }
+// })
+
+// TODO: add this to the hardhat-dependency-compiler repo as a pull request or whatever
+// declare module 'hardhat/types/config' {
+//     interface HardhatUserConfig {
+//       dependencyCompiler?: any;
+//     }
+// }
+
+// You need to export an object to set up your config
+// Go to https://hardhat.org/config/ to learn more
+
+module.exports = {
+
+    defaultNetwork: 'hardhat',
+    networks: {
+        hardhat: {
+            gas: 12000000,
+            blockGasLimit: 0x1fffffffffffff,
+            allowUnlimitedContractSize: true
+        },
+        localsidechain: {
+            chainId: 8997,
+            url: 'http://10.200.10.1:8546',
+            accounts: ['0x5e98cce00cff5dea6b454889f359a4ec06b9fa6b88e9d69b86de8e1c81887da0']
+        },
+        polygonTestMumbai1: {
+            chainId: 80001,
+            url: 'https://rpc-mumbai.maticvigil.com',
+            // accounts: ['0x4059de411f15511a85ce332e7a428f36492ab4e87c7830099dadbf130f1896ae']
+        },
+        polygonTestMumbai2: {
+            chainId: 80001,
+            url: 'https://matic-mumbai.chainstacklabs.com/',
+            // accounts: ['0x4059de411f15511a85ce332e7a428f36492ab4e87c7830099dadbf130f1896ae']
+        },
+        polygonMainnet: {
+            chainId: 137,
+            // url: 'https://polygon-rpc.com',
+            url: 'https://wild-dark-thunder.matic.quiknode.pro/08b0fa6254499defc975c381ee21777cb197fac5/',
+        }
+    },
+    dependencyCompiler: {
+        paths: [
+            'smartcontracts/contracts/StreamRegistry/StreamRegistryV3.sol'
+        ],
+    },
+    solidity: {
+        compilers: [
+            {
+                version: '0.8.9',
+                settings: {
+                    optimizer: {
+                        enabled: true,
+                        runs: 100,
+                    },
+                },
+            }],
+    },
+    typechain: {
+        outDir: './typechain',
+        target: 'ethers-v5',
+    }
+}