Merge pull request #2857 from w3f/fix-ip-location-dedup2

Fix ip location dedup2

Author: ironoa

apps/1kv-backend/templates/kusama-otv-backend.yaml

@@ -17,7 +17,7 @@ spec:
   source:
     repoURL: https://w3f.github.io/helm-charts/
     chart: otv-backend
-    targetRevision: v3.1.13
+    targetRevision: v3.1.14
     plugin:
       env:
         - name: HELM_VALUES

apps/1kv-backend/templates/polkadot-otv-backend.yaml

@@ -17,7 +17,7 @@ spec:
   source:
     repoURL: https://w3f.github.io/helm-charts/
     chart: otv-backend
-    targetRevision: v3.1.13
+    targetRevision: v3.1.14
     plugin:
       env:
         - name: HELM_VALUES

charts/otv-backend/Chart.yaml

@@ -1,5 +1,5 @@
 description: 1K Validators Backend
 name: otv-backend
-version: v3.1.13
-appVersion: v3.1.13
+version: v3.1.14
+appVersion: v3.1.14
 apiVersion: v2

packages/common/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@1kv/common",
-  "version": "3.1.13",
+  "version": "3.1.14",
   "description": "Services for running the Thousand Validator Program.",
   "main": "build/index.js",
   "types": "build/index.d.ts",

packages/common/src/constraints/ScoreCandidates.ts

@@ -96,8 +96,6 @@ export const scoreCandidate = async (
 
     const latestCandidateLocation = await queries.getCandidateLocation(
       candidate.slotId,
-      candidate.name,
-      candidate.stash,
     );
 
     const provider = latestCandidateLocation?.provider || "No Provider";

packages/common/src/constraints/ValidityChecks.ts

@@ -361,11 +361,7 @@ export const checkProvider = async (
   candidate: Candidate,
 ): Promise<boolean> => {
   try {
-    const location = await queries.getCandidateLocation(
-      candidate.slotId,
-      candidate.name,
-      candidate.stash,
-    );
+    const location = await queries.getCandidateLocation(candidate.slotId);
     if (location && location.provider) {
       const bannedProviders = config.telemetry?.blacklistedProviders;
       if (bannedProviders?.includes(location.provider)) {

packages/common/src/constraints/Values.ts

@@ -117,11 +117,7 @@ export const getLocationValues = async (
   const locationMap = new Map<string, number>();
   const locationArr: { name: string; numberOfNodes: number }[] = [];
   for (const candidate of validCandidates) {
-    const candidateLocation = await getCandidateLocation(
-      candidate.slotId,
-      candidate.name,
-      candidate.stash,
-    );
+    const candidateLocation = await getCandidateLocation(candidate.slotId);
     const location = candidateLocation?.city || NoLocation.NoLocation;
 
     const locationCount = locationMap.get(location);
@@ -155,11 +151,7 @@ export const getRegionValues = async (
   const regionMap = new Map<string, number>();
   const regionArr: { name: string; numberOfNodes: number }[] = [];
   for (const candidate of validCandidates) {
-    const candidateLocation = await getCandidateLocation(
-      candidate.slotId,
-      candidate.name,
-      candidate.stash,
-    );
+    const candidateLocation = await getCandidateLocation(candidate.slotId);
     const region =
       candidateLocation && candidateLocation.region
         ? candidateLocation.region
@@ -196,11 +188,7 @@ export const getCountryValues = async (
   const countryMap = new Map<string, number>();
   const countryArr: { name: string; numberOfNodes: number }[] = [];
   for (const candidate of validCandidates) {
-    const candidateLocation = await getCandidateLocation(
-      candidate.slotId,
-      candidate.name,
-      candidate.stash,
-    );
+    const candidateLocation = await getCandidateLocation(candidate.slotId);
     const country =
       candidateLocation && candidateLocation.country
         ? candidateLocation.country
@@ -235,11 +223,7 @@ export const getProviderValues = async (
   const providerMap = new Map<string, number>();
   const providerArr: { name: string; numberOfNodes: number }[] = [];
   for (const candidate of validCandidates) {
-    const candidateLocation = await getCandidateLocation(
-      candidate.slotId,
-      candidate.name,
-      candidate.stash,
-    );
+    const candidateLocation = await getCandidateLocation(candidate.slotId);
     const provider =
       candidateLocation && candidateLocation.provider
         ? candidateLocation.provider

packages/common/src/db/queries/Location.ts

@@ -36,10 +36,8 @@ export const getLocation = async (
 
 export const getCandidateLocation = async (
   slotId: number,
-  stash?: string,
-  name?: string,
 ): Promise<Location | null> => {
-  const query = [{ slotId: slotId }, { stash: stash }, { name: name }];
+  const query = [{ slotId: slotId }];
 
   return LocationModel.findOne({ $or: query })
     .sort({ updated: -1 })
@@ -69,7 +67,7 @@ export const setLocation = async (
     }
     // Try and find an existing record
     const query = {
-      $or: [{ addr }, { name }, { slotId }],
+      $or: [{ slotId }],
     };
     const data = await LocationModel.findOne(query).lean<Location>();
 
@@ -79,6 +77,7 @@ export const setLocation = async (
     }
 
     // Create a new Location record if there is no existing record, or there is a different ip address
+    // TODO: check if we need multiple addresses to be stored
     if (!data || data?.addr != addr) {
       const location = new LocationModel({
         slotId: slotId,
@@ -103,7 +102,7 @@ export const setLocation = async (
       await location.save();
     } else {
       await LocationModel.findOneAndUpdate(
-        { addr, name },
+        { slotId },
         {
           slotId: slotId,
           address: stash,

packages/common/src/scorekeeper/jobs/specificJobs/LocationStatsJob.ts

@@ -44,10 +44,7 @@ export const locationStatsJob = async (metadata: JobRunnerMetadata) => {
 
     // Add all candidate entries to the list of nodes
     for (const [index, candidate] of candidates.entries()) {
-      const location = await queries.getCandidateLocation(
-        candidate.slotId,
-        candidate.name,
-      );
+      const location = await queries.getCandidateLocation(candidate.slotId);
       if (
         location?.city != "None" &&
         location?.region != "None" &&

packages/common/src/utils/Location.ts

@@ -127,7 +127,6 @@ export const fetchAndSetCandidateLocation = async (
       iit && iit.iit ? iit.iit : null,
     );
     await updateIITRequestCount();
-    const candidate = await getCandidateByName(telemetryNodeDetails.name);
     const slotId = candidate?.slotId;
     const stash = candidate?.stash;
     if (slotId == undefined || !stash) {