Merge branch 'staging' into chart-resources

Author: ironoa

.circleci/config.yml

@@ -171,44 +171,12 @@ jobs:
     resource_class: xlarge
     steps:
       - checkout
-      - setup_remote_docker
+      - setup_remote_docker:
+          docker_layer_caching: true
       - run:
           command: |
             /scripts/build-image.sh web3f/otv-backend . --build-arg PACKAGE=core
 
-  buildGatewayImage:
-    docker:
-      - image: web3f/ci-commons:v3.2.3
-    resource_class: xlarge
-    steps:
-      - checkout
-      - setup_remote_docker
-      - run:
-          command: |
-            /scripts/build-image.sh web3f/otv-gateway . --build-arg PACKAGE=gateway
-
-  buildTelemetryImage:
-    docker:
-      - image: web3f/ci-commons:v3.2.3
-    resource_class: xlarge
-    steps:
-      - checkout
-      - setup_remote_docker
-      - run:
-          command: |
-            /scripts/build-image.sh web3f/otv-telemetry . --build-arg PACKAGE=telemetry
-
-  buildWorkerImage:
-    docker:
-      - image: web3f/ci-commons:v3.2.3
-    resource_class: xlarge
-    steps:
-      - checkout
-      - setup_remote_docker
-      - run:
-          command: |
-            /scripts/build-image.sh web3f/otv-worker . --build-arg PACKAGE=worker
-
   integrationTests:
     docker:
       - image: web3f/ci-commons:v3.2.3
@@ -344,21 +312,6 @@ workflows:
           filters:
             tags:
               only: /.*/
-      - buildGatewayImage:
-          context: dockerhub-bot
-          filters:
-            tags:
-              only: /.*/
-      - buildTelemetryImage:
-          context: dockerhub-bot
-          filters:
-            tags:
-              only: /.*/
-      - buildWorkerImage:
-          context: dockerhub-bot
-          filters:
-            tags:
-              only: /.*/
       - integrationTests:
           filters:
             tags:

.dockerignore

@@ -1,22 +1,13 @@
-alice
-bob
-.dockerignore
-node_modules
-scripts
-test
-zero
-*.db
-charts
-helmfile.d
-build
-.circleci
-.editorconfig
-.git
-.github
-.gitignore
-.idea
-combined.log
-docker-compose.yml
-node_modules/
-
-*/**/node_modules/
+*
+!docs
+!packages
+!package.json
+!tsconfig.json
+!turbo.json
+!.yarnrc.yml
+!yarn.lock
+!.yarn/patches
+!.yarn/releases
+!.yarn/plugins
+!.yarn/sdks
+!.yarn/versions

Dockerfile

@@ -3,12 +3,15 @@ ARG MATRIX_TOKEN
 ARG PACKAGE
 ENV PACKAGE ${PACKAGE}
 WORKDIR /code
+
+COPY docs docs
+RUN cd docs && npm install && npm run build
+
 COPY . .
-RUN echo "building ${PACKAGE}... "; \
-    yarn install; \
-    echo "yarn install done. Building...." ; \
-    yarn build; \
-    echo "building ${PACKAGE} done."; \
-    apt-get update && \
-    apt-get clean
-CMD yarn run start:js:${PACKAGE}
+
+RUN --mount=type=cache,target=/code/.yarn/cache \
+    --mount=type=cache,target=/turbo_cache \
+    yarn install --immutable && \
+    yarn turbo --cache-dir /turbo_cache
+
+CMD yarn run start:js:${PACKAGE}

Dockerfile-dev

@@ -5,12 +5,12 @@ ARG PACKAGE
 ENV PACKAGE ${PACKAGE}
 COPY . /code
 WORKDIR /code
-RUN echo "building ${PACKAGE}... "; \
-    yarn install; \
-    echo "yarn install done. Building...." ; \
-    yarn build; \
-    echo "building ${PACKAGE} done."; \
+RUN echo "building ${PACKAGE}... " && \
+    yarn install && \
+    echo "yarn install done. Building...." && \
+    yarn build && \
+    echo "building ${PACKAGE} done." && \
     apt-get update && \
     apt-get install -y libssl-dev && \
     apt-get clean
-CMD yarn run start:dev:${PACKAGE}
+CMD yarn run start:dev:${PACKAGE}

apps/1kv-backend-staging/templates/kusama-otv-backend.yaml

@@ -51,22 +51,24 @@ spec:
                     "wss://rpc.dotters.network/kusama",
                     "wss://ksm-rpc.stakeworld.io"
                   ],
+                  "apiPeopleEndpoints": ["wss://kusama-people-rpc.polkadot.io"],
                   "candidatesUrl": "https://raw.githubusercontent.com/w3f/1k-validators-be/master/candidates/kusama.json"
                 },
                 "constraints": {
                   "skipConnectionTime": true,
-                  "skipIdentity": true,
-                  "skipClientUpgrade": false,
+                  "skipIdentity": false,
                   "skipUnclaimed": true,
+                  "clientUpgrade": {
+                    "skip": false
+                  },
                   "minSelfStake": 10000000000000,
                   "commission": 150000000,
                   "unclaimedEraThreshold": 4,
                   "sanctionedGeoArea": {
                     "skip": false,
                     "sanctionedCountries": ["RU", "IR", "CU", "KP", "SY"],
                     "sanctionedRegions": ["Crimea", "Autonomous Republic of Crimea", "Republic of Crimea", "Luhansk", "Luhanska Oblast", "Luhanska", "Luganskaya Oblast’", "Luganskaya", "Donetsk", "Donetska Oblast", "Donetskaya Oblast’", "Donetska", "Donetskaya", "Sevastopol City", "Sevastopol", "Gorod Sevastopol"]
-                  },
-                  "forceClientVersion": "v1.10.0"
+                  }
                 },
                 "cron": {
                   "monitor": "0 */15 * * * *",

apps/1kv-backend-staging/templates/polkadot-otv-backend.yaml

@@ -55,17 +55,18 @@ spec:
                 "constraints": {
                   "skipConnectionTime": false,
                   "skipIdentity": false,
-                  "skipClientUpgrade": false,
                   "skipUnclaimed": true,
+                  "clientUpgrade": {
+                    "skip": false
+                  },
                   "minSelfStake": 50000000000000,
                   "commission": 50000000,
                   "unclaimedEraThreshold": 1,
                   "sanctionedGeoArea": {
                     "skip": false,
                     "sanctionedCountries": ["RU", "IR", "CU", "KP", "SY"],
                     "sanctionedRegions": ["Crimea", "Autonomous Republic of Crimea", "Republic of Crimea", "Luhansk", "Luhanska Oblast", "Luhanska", "Luganskaya Oblast’", "Luganskaya", "Donetsk", "Donetska Oblast", "Donetskaya Oblast’", "Donetska", "Donetskaya", "Sevastopol City", "Sevastopol", "Gorod Sevastopol"]
-                  },
-                  "forceClientVersion": "v1.10.0"
+                  }
                 },
                 "cron": {
                   "monitor": "0 */15 * * * *",

apps/1kv-backend/templates/kusama-otv-backend.yaml

@@ -17,7 +17,7 @@ spec:
   source:
     repoURL: https://w3f.github.io/helm-charts/
     chart: otv-backend
-    targetRevision: v3.2.2
+    targetRevision: v3.3.0
     plugin:
       env:
         - name: HELM_VALUES
@@ -48,23 +48,25 @@ spec:
                     "wss://rpc.dotters.network/kusama",
                     "wss://ksm-rpc.stakeworld.io"
                   ],
+                  "apiPeopleEndpoints": ["wss://kusama-people-rpc.polkadot.io"],
                   "candidatesUrl": "https://raw.githubusercontent.com/w3f/1k-validators-be/master/candidates/kusama.json"
                 },
                 "constraints": {
                   "skipConnectionTime": false,
-                  "skipIdentity": true,
+                  "skipIdentity": false,
                   "skipStakedDestination": true,
-                  "skipClientUpgrade": false,
                   "skipUnclaimed": true,
+                  "clientUpgrade": {
+                    "skip": false
+                  },
                   "minSelfStake": 10000000000000,
                   "commission": 150000000,
                   "unclaimedEraThreshold": 4,
                   "sanctionedGeoArea": {
                     "skip": false,
                     "sanctionedCountries": ["RU", "IR", "CU", "KP", "SY"],
                     "sanctionedRegions": ["Crimea", "Autonomous Republic of Crimea", "Republic of Crimea", "Luhansk", "Luhanska Oblast", "Luhanska", "Luganskaya Oblast’", "Luganskaya", "Donetsk", "Donetska Oblast", "Donetskaya Oblast’", "Donetska", "Donetskaya", "Sevastopol City", "Sevastopol", "Gorod Sevastopol"]
-                  },
-                  "forceClientVersion": "v1.10.0"
+                  }
                 },
                 "cron": {
                   "monitor": "0 */15 * * * *",

apps/1kv-backend/templates/polkadot-otv-backend.yaml

@@ -17,7 +17,7 @@ spec:
   source:
     repoURL: https://w3f.github.io/helm-charts/
     chart: otv-backend
-    targetRevision: v3.2.2
+    targetRevision: v3.3.0
     plugin:
       env:
         - name: HELM_VALUES
@@ -53,17 +53,18 @@ spec:
                   "skipConnectionTime": false,
                   "skipIdentity": false,
                   "skipStakedDestination": true,
-                  "skipClientUpgrade": false,
                   "skipUnclaimed": true,
+                  "clientUpgrade": {
+                    "skip": false
+                  },
                   "minSelfStake": 50000000000000,
                   "commission": 50000000,
                   "unclaimedEraThreshold": 1,
                   "sanctionedGeoArea": {
                     "skip": false,
                     "sanctionedCountries": ["RU", "IR", "CU", "KP", "SY"],
                     "sanctionedRegions": ["Crimea", "Autonomous Republic of Crimea", "Republic of Crimea", "Luhansk", "Luhanska Oblast", "Luhanska", "Luganskaya Oblast’", "Luganskaya", "Donetsk", "Donetska Oblast", "Donetskaya Oblast’", "Donetska", "Donetskaya", "Sevastopol City", "Sevastopol", "Gorod Sevastopol"]
-                  },
-                  "forceClientVersion": "v1.10.0"
+                  }
                 },
                 "cron": {
                   "monitor": "0 */15 * * * *",

charts/otv-backend/Chart.yaml

@@ -1,5 +1,5 @@
 description: 1K Validators Backend
 name: otv-backend
-version: v3.2.3
-appVersion: v3.2.2
+version: v3.3.0
+appVersion: v3.3.0
 apiVersion: v2

docs/docs/backend/config.md

@@ -42,7 +42,8 @@ An example config may look something like:
 
 - `dryRun`: Boolean (true/false). If set to true, the Nominator accounts that are added will calculate what a nomination would look like (how many and which validators), but not craft or submit any transactions. No nominations will be done when this flag is set. In the `nominate` function of the `Nominator` class, the `dryRun` flag is checked, and if it is set to true, the function will return after logging the validators it would nominate without doing anything. This flag is optional and set to `false` by default.
 - `networkPrefix`: Integer. Defines the network prefix. For Kusama, this is `2`, and for Polkadot, this is `0`. It can be set to `3` for running a local test network, although this isn't used much anymore. **This flag is required for `core` and `worker` services.**
-- `apiEndpoints`: Array of strings. Lists the RPC endpoints for the chain. When given a list of multiple, it will pick one at random to create a websocket connection to - this single connection is used throughout the entire service for any queries or submitting transactions. **This is required for `core` and `worker` services.
+- `apiEndpoints`: Array of strings. Lists the RPC endpoints for the chain. When given a list of multiple, it will pick one at random to create a websocket connection to - this single connection is used throughout the entire service for any queries or submitting transactions. **This is required for `core` and `worker` services.**
+- `apiPeopleEndpoints`: Optional array of strings. Lists the RPC endpoints for People parachain, if it's enabled on the network. 
 - `bootstrap`: Boolean. An **optional** flag that can be set to `true` to enable the bootstrap process. This can be used when running a instance of the backend and would query the main Kusama or Polkadot instances at the api endpoints specified below to populate the db with non-deterministic values like `rank` or `discoveredAt`. _This isn't currently used anywhere yet_
 - `kusamaBootstrapEndpoint`: String. URL for the Kusama bootstrap endpoint. **optional**. _This isn't currently used anywhere yet_. 
 - `polkadotBootstrapEndpoint`: String. URL for the Polkadot bootstrap endpoint. **optional**. _This isn't currently used anywhere yet_.
@@ -56,9 +57,10 @@ An example config may look something like:
   "constraints": {
     "skipConnectionTime": true,
     "skipIdentity": false,
-    "skipClientUpgrade": false,
     "skipUnclaimed": false,
-    "forceClientVersion": "v0.9.30",
+    "clientUpgrade": {
+      "skip": false
+    },
     "minSelfStake": 10000000000000,
     "commission": 150000000,
     "unclaimedEraThreshold": 4
@@ -69,9 +71,9 @@ The `constraints` section defines validity constraint parameters for validators,
 
 - `skipConnectionTime`: Boolean. Skips checking the 7 day required connection time if set to true. __optional__, defaults to `false`.
 - `skipIdentity`: Boolean. Skips the check for a verified identity. __optional__, defaults to `false`.
-- `skipClientUpgrade`: Boolean. Skips client version upgrade check. __optional__, defaults to `false`.
 - `skipUnclaimed`: Boolean. Skips the check for unclaimed rewards. __optional__, defaults to `false`.
-- `forceClientVersion`: String. Specific client version to be enforced. __optional__, if this is set, it will allow versions higher than what is specified. 
+- `clientUpgrade.skip`: Boolean. Skips client version upgrade check. __optional__, defaults to `false`.
+- `clientUpgrade.forceVersion`: String. Specific client version to be enforced. __optional__, if this is set, it will allow versions >= than what is specified. 
 - `minSelfStake`: Integer. Minimum self-stake required. **required**. This number needs to be specified in `Plancks` (1 DOT = 10^10 Plancks, 1 KSM = 10^12 Plancks).
 - `commission`: Integer. Max commission rate. **required**. This number needs to be specified in chain units that have 6 decimal places -  for example `150000000`  corresponds to 15% commission. 
 - `unclaimedEraThreshold`: Integer. Threshold for unclaimed eras. **required**. A validator having pending rewards for past eras longer than this threshold will be deemed invalid. This gets skipped if `skipUnclaimed` is set to `true`. This number is speciefied as number of eras, so `4` for example means validators are invalid if they have pending rewards older than 4 eras ago.
@@ -459,9 +461,10 @@ An example `core` config run as microservices may look something like:
   "constraints": {
     "skipConnectionTime": true,
     "skipIdentity": false,
-    "skipClientUpgrade": false,
     "skipUnclaimed": false,
-    "forceClientVersion": "v0.9.30",
+    "clientUpgrade": {
+      "skip": false
+    },
     "minSelfStake": 10000000000000,
     "commission": 150000000,
     "unclaimedEraThreshold": 4
@@ -645,4 +648,4 @@ An example Worker config run as microservices may look something like:
   }
 }
 
-```
+```