fix(api): unlist removed validator profiles

Author: onmax

server/api/[version]/status.get.ts

@@ -13,6 +13,7 @@ import { getRpcUrl } from '~~/server/utils/rpc'
  *   - The addresses of the active validators in the current epoch
  *   - The addresses of inactive validators
  *   - The addresses of untracked validators (new validators)
+ *   - The addresses of active validators with removed profile metadata (`unlistedActiveValidators`)
  *   - The addresses of all validators regardless of their status
  *
  *   Blockchain information:

server/db/migrations/0004_add_validator_is_listed.sql

@@ -0,0 +1 @@
+ALTER TABLE `validators` ADD `is_listed` integer;

server/db/schema.ts

@@ -16,6 +16,7 @@ export const validators = sqliteTable('validators', {
   accentColor: text('accent_color').notNull(),
   website: text('website'),
   contact: text('contact', { mode: 'json' }),
+  isListed: integer('is_listed', { mode: 'boolean' }),
 }, table => [
   uniqueIndex('validators_address_unique').on(table.address),
   check(

server/utils/types.ts

@@ -12,6 +12,7 @@ export interface SnapshotEpochValidators {
   electedValidators: (ElectedValidator | UnelectedValidator)[]
   unelectedValidators: (ElectedValidator | UnelectedValidator)[]
   deletedValidators: string[]
+  unlistedActiveValidators: string[]
 
   /**
    * Validators that are not tracked by the database. The untracked validators are

server/utils/validator-listing.ts

@@ -0,0 +1,38 @@
+const UNKNOWN_VALIDATOR_NAME = 'Unknown validator'
+
+interface ValidatorListState {
+  isListed: boolean | null
+  name: string
+}
+
+interface ValidatorAddress {
+  address: string
+}
+
+interface StoredValidatorAddressState extends ValidatorAddress {
+  isListed: boolean | null
+}
+
+export function isKnownValidatorProfile({ isListed, name }: ValidatorListState) {
+  if (typeof isListed === 'boolean')
+    return isListed
+  return name.toLowerCase() !== UNKNOWN_VALIDATOR_NAME.toLowerCase()
+}
+
+export function getUnlistedAddresses(storedAddresses: string[], bundledAddresses: Set<string>) {
+  return storedAddresses.filter(address => !bundledAddresses.has(address))
+}
+
+export function getUnlistedActiveValidatorAddresses(
+  epochValidators: ValidatorAddress[],
+  storedValidators: StoredValidatorAddressState[],
+) {
+  const unlistedAddresses = new Set(
+    storedValidators
+      .filter(v => v.isListed === false)
+      .map(v => v.address),
+  )
+  return epochValidators
+    .filter(v => unlistedAddresses.has(v.address))
+    .map(v => v.address)
+}

server/utils/validators-bundle.ts

@@ -2,7 +2,8 @@ import type { Result } from 'nimiq-validator-trustscore/types'
 import type { ValidatorJSON } from './schemas'
 import { bundledValidatorsByNetwork } from '../generated/validators-bundle.generated'
 import { validatorSchema } from './schemas'
-import { storeValidator } from './validators'
+import { getUnlistedAddresses } from './validator-listing'
+import { getStoredValidatorsAddress, markValidatorsAsUnlisted, storeValidator } from './validators'
 
 interface ImportValidatorsBundledOptions {
   shouldStore?: boolean
@@ -28,10 +29,16 @@ export async function importValidatorsBundled(nimiqNetwork?: string, options: Im
   if (!shouldStore)
     return [true, undefined, validators]
 
-  const results = await Promise.allSettled(validators.map(v => storeValidator(v.address, v, { upsert: true })))
+  const bundledAddresses = new Set(validators.map(v => v.address))
+  const storedAddresses = await getStoredValidatorsAddress()
+  const unlistedAddresses = getUnlistedAddresses(storedAddresses, bundledAddresses)
+
+  const results = await Promise.allSettled(validators.map(v => storeValidator(v.address, v, { upsert: true, isListed: true })))
   const failures = results.filter(r => r.status === 'rejected')
   if (failures.length > 0)
     return [false, `Errors importing validators: ${failures.map((f: any) => f.reason).join(', ')}`, undefined]
 
+  await markValidatorsAsUnlisted(unlistedAddresses)
+
   return [true, undefined, validators]
 }

server/utils/validators.ts

@@ -10,9 +10,12 @@ import { fetchSnapshotEpoch } from '~~/packages/nimiq-validator-trustscore/src/f
 import { tables, useDrizzle } from './drizzle'
 import { handleValidatorLogo } from './logo'
 import { defaultValidatorJSON } from './schemas'
+import { PayoutType } from './types'
+import { getUnlistedActiveValidatorAddresses, isKnownValidatorProfile } from './validator-listing'
 
 export const getStoredValidatorsId = () => useDrizzle().select({ id: tables.validators.id }).from(tables.validators).execute().then(r => r.map(v => v.id))
 export const getStoredValidatorsAddress = () => useDrizzle().select({ address: tables.validators.address }).from(tables.validators).execute().then(r => r.map(v => v.address))
+export const getStoredValidatorsListState = () => useDrizzle().select({ address: tables.validators.address, isListed: tables.validators.isListed }).from(tables.validators).execute()
 
 const validators = new Map<string, number>()
 
@@ -22,6 +25,12 @@ interface StoreValidatorOptions {
    * @default false
    */
   upsert?: boolean
+
+  /**
+   * Controls if the validator should appear in `only-known=true`.
+   * @default false
+   */
+  isListed?: boolean
 }
 
 export async function storeValidator(address: string, rest: ValidatorJSON = defaultValidatorJSON, options: StoreValidatorOptions = {}): Promise<number | undefined> {
@@ -34,7 +43,7 @@ export async function storeValidator(address: string, rest: ValidatorJSON = defa
     return
   }
 
-  const { upsert = false } = options
+  const { upsert = false, isListed = false } = options
 
   // If the validator is cached and upsert is not true, return it
   if (!upsert && validators.has(address)) {
@@ -63,14 +72,14 @@ export async function storeValidator(address: string, rest: ValidatorJSON = defa
     if (validatorId) {
       await useDrizzle()
         .update(tables.validators)
-        .set({ ...rest, ...brandingParameters })
+        .set({ ...rest, ...brandingParameters, isListed })
         .where(eq(tables.validators.id, validatorId))
         .execute()
     }
     else {
       validatorId = await useDrizzle()
         .insert(tables.validators)
-        .values({ ...rest, address, ...brandingParameters })
+        .values({ ...rest, address, ...brandingParameters, isListed })
         .returning()
         .get()
         .then(r => r.id)
@@ -84,6 +93,28 @@ export async function storeValidator(address: string, rest: ValidatorJSON = defa
   return validatorId
 }
 
+export async function markValidatorsAsUnlisted(addresses: string[]) {
+  await Promise.all(addresses.map(async (address) => {
+    const brandingParameters = await handleValidatorLogo(address, defaultValidatorJSON)
+    await useDrizzle()
+      .update(tables.validators)
+      .set({
+        name: 'Unknown validator',
+        description: null,
+        fee: null,
+        payoutType: PayoutType.None,
+        payoutSchedule: '',
+        isMaintainedByNimiq: false,
+        website: null,
+        contact: null,
+        isListed: false,
+        ...brandingParameters,
+      })
+      .where(eq(tables.validators.address, address))
+      .execute()
+  }))
+}
+
 export type FetchValidatorsOptions = MainQuerySchema & { epochNumber: number }
 
 export async function fetchValidators(_event: H3Event, params: FetchValidatorsOptions): Result<FetchedValidator[]> {
@@ -98,16 +129,15 @@ export async function fetchValidators(_event: H3Event, params: FetchValidatorsOp
   const filters: SQLWrapper[] = []
   if (payoutType)
     filters.push(eq(tables.validators.payoutType, payoutType))
-  if (onlyKnown)
-    filters.push(sql`lower(${tables.validators.name}) NOT LIKE lower('%Unknown validator%')`)
 
   try {
     const validatorsQuery = useDrizzle().select().from(tables.validators)
     const dbValidators = filters.length > 0
       ? await validatorsQuery.where(and(...filters)).execute()
       : await validatorsQuery.execute()
 
-    const validatorIds = dbValidators.map(v => v.id)
+    const visibleValidators = onlyKnown ? dbValidators.filter(isKnownValidatorProfile) : dbValidators
+    const validatorIds = visibleValidators.map(v => v.id)
     if (validatorIds.length === 0)
       return [true, undefined, []]
 
@@ -172,7 +202,7 @@ export async function fetchValidators(_event: H3Event, params: FetchValidatorsOp
     const scoresByValidatorId = new Map(scoresRows.map(row => [row.validatorId, row]))
     const activityByValidatorId = new Map(activityRows.map(row => [row.validatorId, row]))
 
-    const validators = dbValidators.map((validator) => {
+    const validators = visibleValidators.map((validator) => {
       const { logo, contact, hasDefaultLogo, ...rest } = validator
       const scoreRow = scoresByValidatorId.get(validator.id)
       const activityRow = activityByValidatorId.get(validator.id)
@@ -301,17 +331,20 @@ export async function categorizeValidatorsSnapshotEpoch(): Result<SnapshotEpochV
   if (!epochOk)
     return [false, error, undefined]
 
-  const dbAddresses = await getStoredValidatorsAddress()
+  const storedValidators = await getStoredValidatorsListState()
+  const dbAddresses = storedValidators.map(v => v.address)
   const electedValidators = epoch.validators.filter(v => v.elected) as ElectedValidator[]
   const unelectedValidators = epoch.validators.filter(v => !v.elected) as UnelectedValidator[]
   const untrackedValidators = electedValidators.filter(v => !dbAddresses.includes(v.address)) as (ElectedValidator & UnelectedValidator)[]
   const deletedValidators = dbAddresses.filter(dbAddress => !epoch.validators.map(v => v.address).includes(dbAddress))
+  const unlistedActiveValidators = getUnlistedActiveValidatorAddresses(epoch.validators, storedValidators)
 
   return [true, undefined, {
     epochNumber: epoch.epochNumber,
     electedValidators,
     unelectedValidators,
     untrackedValidators,
     deletedValidators,
+    unlistedActiveValidators,
   }]
 }