Merge pull request #2930 from w3f/staging

ironoa · web-flow · commit f3e30e0bd66d · 2024-05-15T14:50:58.000+02:00
Staging
diff --git a/apps/1kv-backend/templates/kusama-otv-backend.yaml b/apps/1kv-backend/templates/kusama-otv-backend.yaml
@@ -17,7 +17,7 @@ spec:
   source:
     repoURL: https://w3f.github.io/helm-charts/
     chart: otv-backend
-    targetRevision: v3.2.1
+    targetRevision: v3.2.2
     plugin:
       env:
         - name: HELM_VALUES
diff --git a/apps/1kv-backend/templates/polkadot-otv-backend.yaml b/apps/1kv-backend/templates/polkadot-otv-backend.yaml
@@ -17,7 +17,7 @@ spec:
   source:
     repoURL: https://w3f.github.io/helm-charts/
     chart: otv-backend
-    targetRevision: v3.2.1
+    targetRevision: v3.2.2
     plugin:
       env:
         - name: HELM_VALUES
diff --git a/charts/otv-backend/Chart.yaml b/charts/otv-backend/Chart.yaml
@@ -1,5 +1,5 @@
 description: 1K Validators Backend
 name: otv-backend
-version: v3.2.1
-appVersion: v3.2.1
+version: v3.2.2
+appVersion: v3.2.2
 apiVersion: v2
diff --git a/packages/common/package.json b/packages/common/package.json
@@ -1,6 +1,6 @@
 {
   "name": "@1kv/common",
-  "version": "3.2.1",
+  "version": "3.2.2",
   "description": "Services for running the Thousand Validator Program.",
   "main": "build/index.js",
   "types": "build/index.d.ts",
diff --git a/packages/common/src/chaindata/queries/Era.ts b/packages/common/src/chaindata/queries/Era.ts
@@ -46,7 +46,10 @@ export const getTotalEraPoints = async (
     if (!chainType) {
       return {} as EraPointsInfo;
     }
-    const [blockHash, err] = await chaindata.findEraBlockHash(era, chainType);
+    const [blockHash, err] = await chaindata.findEraBlockHash(
+      era + 1,
+      chainType,
+    );
 
     if (blockHash) {
       const apiAt = await chaindata?.api?.at(blockHash);
@@ -204,12 +207,12 @@ export const findEraBlockHash = async (
         return ["", "API at block hash is null"];
       }
 
-      const testEra = await apiAt.query.staking.currentEra();
+      const testEra = await apiAt.query.staking.activeEra();
       if (testEra && testEra.isEmpty) {
         logger.info(`Test era is empty: ${JSON.stringify(testEra)}`);
         return ["", "Test era is none"];
       }
-      const testIndex = testEra.unwrap().toNumber();
+      const testIndex = testEra.unwrap().index.toNumber();
       if (era == testIndex) {
         return [blockHash.toString(), null];
       }
diff --git a/packages/common/src/constants.ts b/packages/common/src/constants.ts
@@ -207,3 +207,5 @@ export const USE_PROVIDER = true;
 export const USE_NOMINATIONS = true;
 export const USE_RPC = true;
 export const USE_CLIENT = true;
+
+export const ERAPOINTS_JOB_MAX_ERAS = 84;
diff --git a/packages/common/src/constraints/CheckCandidates.ts b/packages/common/src/constraints/CheckCandidates.ts
@@ -1,4 +1,4 @@
-import { logger } from "../index";
+import { ChainData, logger } from "../index";
 import { allCandidates, Candidate, setLastValid, setValid } from "../db";
 import { constraintsLabel, OTV } from "./constraints";
 import {
@@ -22,6 +22,7 @@ import { percentage, timeRemaining } from "../utils/util";
 export const checkCandidate = async (
   constraints: OTV,
   candidate: Candidate,
+  validators: string[],
 ): Promise<boolean> => {
   try {
     let valid = false;
@@ -33,8 +34,8 @@ export const checkCandidate = async (
 
     const validateValid = await checkValidateIntention(
       constraints.config,
-      constraints.chaindata,
       candidate,
+      validators,
     );
     if (!validateValid) {
       logger.info(
@@ -62,7 +63,10 @@ export const checkCandidate = async (
       );
     }
 
-    const identityValid = await checkIdentity(constraints.chaindata, candidate);
+    const identityValid =
+      constraints.config?.constraints?.skipIdentity == true
+        ? true
+        : (await checkIdentity(constraints.chaindata, candidate)) || false;
     if (!identityValid) {
       logger.info(`${candidate.name} identity not valid`, constraintsLabel);
     }
@@ -166,14 +170,16 @@ export const checkCandidate = async (
 
 export const checkAllCandidates = async (
   constraints: OTV,
+  chaindata: ChainData,
 ): Promise<boolean> => {
   try {
     const candidates = await allCandidates();
+    const validators = await chaindata.getValidators();
     logger.info(`checking ${candidates.length} candidates`, constraintsLabel);
     for (const [index, candidate] of candidates.entries()) {
       const start = Date.now();
 
-      const isValid = await constraints.checkCandidate(candidate);
+      const isValid = await constraints.checkCandidate(candidate, validators);
       const end = Date.now();
       const time = `(${end - start}ms)`;
       const remaining = timeRemaining(
diff --git a/packages/common/src/constraints/ValidityChecks.ts b/packages/common/src/constraints/ValidityChecks.ts
@@ -44,11 +44,10 @@ export const checkOnline = async (candidate: Candidate): Promise<boolean> => {
 // Check the validate intention for a single validator
 export const checkValidateIntention = async (
   config: Config.ConfigSchema,
-  chaindata: ChainData,
   candidate: Candidate,
+  validators: string[],
 ): Promise<boolean> => {
   try {
-    const validators = await chaindata.getValidators();
     if (
       !validators?.length ||
       validators.includes(Util.formatAddress(candidate?.stash, config))
@@ -65,28 +64,6 @@ export const checkValidateIntention = async (
   }
 };
 
-// checks the validate intention for all validators
-export const checkAllValidateIntentions = async (
-  config: Config.ConfigSchema,
-  chaindata: ChainData,
-  candidates: Candidate[],
-): Promise<boolean> => {
-  try {
-    const validators = await chaindata.getValidators();
-    for (const candidate of candidates) {
-      if (!validators.includes(Util.formatAddress(candidate.stash, config))) {
-        await setValidateIntentionValidity(candidate, false);
-      } else {
-        await setValidateIntentionValidity(candidate, true);
-      }
-    }
-    return true;
-  } catch (e) {
-    logger.error(`Error checking validate intentions: ${e}`, constraintsLabel);
-    throw new Error("could not make validity check");
-  }
-};
-
 // checks that the validator is on the latest client version
 export const checkLatestClientVersion = async (
   config: Config.ConfigSchema,
diff --git a/packages/common/src/constraints/constraints.ts b/packages/common/src/constraints/constraints.ts
@@ -100,12 +100,15 @@ export class OTV implements Constraints {
 
   // Checks the validity of all candidates
   async checkAllCandidates(): Promise<boolean> {
-    return await checkAllCandidates(this);
+    return await checkAllCandidates(this, this.chaindata);
   }
 
   // Check the candidate and set any invalidity fields
-  async checkCandidate(candidate: Candidate): Promise<boolean> {
-    return await checkCandidate(this, candidate);
+  async checkCandidate(
+    candidate: Candidate,
+    validators: string[],
+  ): Promise<boolean> {
+    return await checkCandidate(this, candidate, validators);
   }
 
   async scoreAllCandidates(): Promise<boolean> {
diff --git a/packages/common/src/constraints/score.ts b/packages/common/src/constraints/score.ts
@@ -99,7 +99,7 @@ export const getStats = (arr: number[]): Stats => {
   const arrQ75 = arr.length !== 0 ? q75(arr) : 0;
   const arrQ90 = arr.length !== 0 ? q90(arr) : 0;
   const arrMean = arr.length !== 0 ? mean(arr) : 0;
-  const arrStd = arr.length !== 0 ? std(arr) : 0;
+  const arrStd = arr.length > 1 ? std(arr) : 0;
 
   return {
     values: arr,
diff --git a/packages/common/src/db/models.ts b/packages/common/src/db/models.ts
@@ -143,8 +143,8 @@ export interface ValidatorSet {
 }
 
 export const ValidatorSetSchema = new Schema({
-  session: Number,
-  era: Number,
+  session: { type: Number, index: true },
+  era: { type: Number, index: true },
   validators: [String],
 });
 
diff --git a/packages/common/src/scorekeeper/Round.ts b/packages/common/src/scorekeeper/Round.ts
@@ -76,10 +76,11 @@ export const startRound = async (
 
   // Get all Candidates and set validity
   const allCandidates = await queries.allCandidates();
+  const validators = await chaindata.getValidators();
 
   // Set Validity
   for (const [index, candidate] of allCandidates.entries()) {
-    const isValid = await constraints.checkCandidate(candidate);
+    const isValid = await constraints.checkCandidate(candidate, validators);
 
     const progress = Math.floor((index / allCandidates.length) * 100);
     jobStatusEmitter.emit("jobProgress", {
diff --git a/packages/common/src/scorekeeper/jobs/specificJobs/ConstraintsJob.ts b/packages/common/src/scorekeeper/jobs/specificJobs/ConstraintsJob.ts
@@ -35,6 +35,7 @@ export const validityJob = async (
   try {
     const { constraints } = metadata;
     const candidates = await allCandidates();
+    const validators = await metadata.chaindata.getValidators();
     logger.info(`Checking ${candidates.length} candidates`, constraintsLabel);
 
     // Calculate total number of candidates
@@ -43,7 +44,7 @@ export const validityJob = async (
     for (const [index, candidate] of candidates.entries()) {
       const start = Date.now();
 
-      const isValid = await constraints.checkCandidate(candidate);
+      const isValid = await constraints.checkCandidate(candidate, validators);
       const end = Date.now();
       const time = `(${end - start}ms)`;
       const remaining = timeRemaining(index + 1, totalCandidates, end - start);
@@ -82,57 +83,6 @@ export const validityJobWithTiming = withExecutionTimeLogging(
   "Validity Job Done",
 );
 
-export const candidateValidityJob = async (
-  constraints: Constraints.OTV,
-  candidateAddress: string,
-) => {
-  try {
-    const start = Date.now();
-
-    const candidate = await queries.getCandidateByStash(candidateAddress);
-    if (candidate) {
-      await constraints.checkCandidate(candidate);
-
-      const end = Date.now();
-      const executionTime = (end - start) / 1000;
-
-      logger.info(
-        `validity for ${candidate.name} Done. (${executionTime}s)`,
-        constraintsLabel,
-      );
-    }
-  } catch (e) {
-    logger.error(`Error running validity job: ${e}`, constraintsLabel);
-  }
-};
-
-export const individualScoreJob = async (
-  constraints: Constraints.OTV,
-  candidateAddress: string,
-) => {
-  try {
-    const start = Date.now();
-    const candidate = await queries.getCandidateByStash(candidateAddress);
-    if (candidate) {
-      let scoreMetadata = await queries.getLatestValidatorScoreMetadata();
-      if (!scoreMetadata) {
-        logger.warn(
-          `no score metadata, cannot score candidates`,
-          constraintsLabel,
-        );
-        await constraints.setScoreMetadata();
-        scoreMetadata = await queries.getLatestValidatorScoreMetadata();
-      }
-      await constraints.scoreCandidate(candidate, scoreMetadata);
-
-      const end = Date.now();
-      const executionTime = (end - start) / 1000;
-    }
-  } catch (e) {
-    logger.error(`Error running individual score job: ${e}`, constraintsLabel);
-  }
-};
-
 export const scoreJob = async (
   metadata: JobRunnerMetadata,
 ): Promise<boolean> => {
diff --git a/packages/common/src/scorekeeper/jobs/specificJobs/EraPointsJob.ts b/packages/common/src/scorekeeper/jobs/specificJobs/EraPointsJob.ts
@@ -1,4 +1,4 @@
-import { ChainData, logger, queries } from "../../../index";
+import { ChainData, Constants, logger, queries } from "../../../index";
 import { Job, JobConfig, JobRunnerMetadata, JobStatus } from "../JobsClass";
 import { jobStatusEmitter } from "../../../Events";
 import { withExecutionTimeLogging } from "../../../utils";
@@ -63,16 +63,15 @@ export const eraPointsJob = async (
     //    - if a record doesn't exist, create it
     const [activeEra, err] = await chaindata.getActiveEraIndex();
 
-    // Calculate total number of eras to process
-    const totalEras = activeEra;
-    let processedEras = 0;
-
-    for (let i = activeEra - 1; i >= 0; i--) {
+    for (
+      let i = activeEra - 1, processedEras = 1;
+      i >= activeEra - Constants.ERAPOINTS_JOB_MAX_ERAS;
+      i--, processedEras++
+    ) {
       await individualEraPointsJob(chaindata, i);
 
       // Calculate progress percentage
-      processedEras++;
-      const progress = (processedEras / totalEras) * 100;
+      const progress = (processedEras / Constants.ERAPOINTS_JOB_MAX_ERAS) * 100;
 
       // Emit progress update with active era as iteration
       jobStatusEmitter.emit("jobProgress", {
@@ -83,7 +82,7 @@ export const eraPointsJob = async (
       });
 
       logger.info(
-        `Processed Era Points for Era: ${i} (${activeEra - i}/${activeEra})`,
+        `Processed Era Points for Era: ${i} (${activeEra - i}/${Constants.ERAPOINTS_JOB_MAX_ERAS})`,
         erapointsLabel,
       );
     }
diff --git a/packages/core/package.json b/packages/core/package.json
@@ -1,6 +1,6 @@
 {
   "name": "@1kv/core",
-  "version": "3.2.1",
+  "version": "3.2.2",
   "description": "Services for running the Thousand Validator Program.",
   "main": "index.js",
   "scripts": {
diff --git a/packages/gateway/package.json b/packages/gateway/package.json
@@ -1,6 +1,6 @@
 {
   "name": "@1kv/gateway",
-  "version": "3.2.1",
+  "version": "3.2.2",
   "description": "Services for running the Thousand Validator Program.",
   "main": "build/index.js",
   "types": "build/index.d.ts",
diff --git a/packages/scorekeeper-status-ui/package.json b/packages/scorekeeper-status-ui/package.json
@@ -1,7 +1,7 @@
 {
   "name": "@1kv/scorekeeper-status-ui",
   "private": true,
-  "version": "3.2.1",
+  "version": "3.2.2",
   "type": "module",
   "scripts": {
     "dev": "vite",
diff --git a/packages/telemetry/package.json b/packages/telemetry/package.json
@@ -1,6 +1,6 @@
 {
   "name": "@1kv/telemetry",
-  "version": "3.2.1",
+  "version": "3.2.2",
   "description": "Services for running the Thousand Validator Program.",
   "main": "build/index.js",
   "types": "build/index.d.ts",
diff --git a/packages/worker/package.json b/packages/worker/package.json
@@ -1,6 +1,6 @@
 {
   "name": "@1kv/worker",
-  "version": "3.2.1",
+  "version": "3.2.2",
   "description": "Services for running the Thousand Validator Program.",
   "main": "build/index.js",
   "types": "build/index.d.ts",

Original file line number	Diff line number	Diff line change
`@@ -1,6 +1,6 @@`
`1`	`1`	`{`
`2`	`2`	`"name": "@1kv/common",`
`3`		`- "version": "3.2.1",`
	`3`	`+ "version": "3.2.2",`
`4`	`4`	`"description": "Services for running the Thousand Validator Program.",`
`5`	`5`	`"main": "build/index.js",`
`6`	`6`	`"types": "build/index.d.ts",`
Original file line number	Diff line number	Diff line change
`@@ -46,7 +46,10 @@ export const getTotalEraPoints = async (`
`46`	`46`	`if (!chainType) {`
`47`	`47`	`return {} as EraPointsInfo;`
`48`	`48`	`}`
`49`		`- const [blockHash, err] = await chaindata.findEraBlockHash(era, chainType);`
	`49`	`+ const [blockHash, err] = await chaindata.findEraBlockHash(`
	`50`	`+ era + 1,`
	`51`	`+ chainType,`
	`52`	`+ );`
`50`	`53`
`51`	`54`	`if (blockHash) {`
`52`	`55`	`const apiAt = await chaindata?.api?.at(blockHash);`
`@@ -204,12 +207,12 @@ export const findEraBlockHash = async (`
`204`	`207`	`return ["", "API at block hash is null"];`
`205`	`208`	`}`
`206`	`209`
`207`		`- const testEra = await apiAt.query.staking.currentEra();`
	`210`	`+ const testEra = await apiAt.query.staking.activeEra();`
`208`	`211`	`if (testEra && testEra.isEmpty) {`
`209`	`212`	logger.info(`Test era is empty: ${JSON.stringify(testEra)}`);
`210`	`213`	`return ["", "Test era is none"];`
`211`	`214`	`}`
`212`		`- const testIndex = testEra.unwrap().toNumber();`
	`215`	`+ const testIndex = testEra.unwrap().index.toNumber();`
`213`	`216`	`if (era == testIndex) {`
`214`	`217`	`return [blockHash.toString(), null];`
`215`	`218`	`}`