fix(api): unlist removed validator profiles (#139)

Author: onmax

server/api/[version]/status.get.ts

@@ -13,6 +13,7 @@ import { getRpcUrl } from '~~/server/utils/rpc'
  *   - The addresses of the active validators in the current epoch
  *   - The addresses of inactive validators
  *   - The addresses of untracked validators (new validators)
+ *   - The addresses of active validators with removed profile metadata (`unlistedActiveValidators`)
  *   - The addresses of all validators regardless of their status
  *
  *   Blockchain information:

server/db/migrations/0003_add_cron_runs.sql

@@ -1,4 +1,4 @@
-CREATE TABLE `cron_runs` (
+CREATE TABLE IF NOT EXISTS `cron_runs` (
 	`id` integer PRIMARY KEY AUTOINCREMENT NOT NULL,
 	`cron` text NOT NULL,
 	`network` text NOT NULL,
@@ -10,5 +10,4 @@ CREATE TABLE `cron_runs` (
 	`meta` text
 );
 --> statement-breakpoint
-CREATE INDEX `idx_cron_runs_started_at` ON `cron_runs` (`started_at`);
-
+CREATE INDEX IF NOT EXISTS `idx_cron_runs_started_at` ON `cron_runs` (`started_at`);

server/db/migrations/0004_add_validator_is_listed.sql

@@ -0,0 +1,4 @@
+-- The `is_listed` column was added manually in remote D1 databases.
+-- Keep this as a no-op migration so deploy-time migrations don't fail
+-- with "duplicate column name: is_listed".
+SELECT 1;

server/db/schema.ts

@@ -16,6 +16,7 @@ export const validators = sqliteTable('validators', {
   accentColor: text('accent_color').notNull(),
   website: text('website'),
   contact: text('contact', { mode: 'json' }),
+  isListed: integer('is_listed', { mode: 'boolean' }),
 }, table => [
   uniqueIndex('validators_address_unique').on(table.address),
   check(

server/utils/types.ts

@@ -12,6 +12,7 @@ export interface SnapshotEpochValidators {
   electedValidators: (ElectedValidator | UnelectedValidator)[]
   unelectedValidators: (ElectedValidator | UnelectedValidator)[]
   deletedValidators: string[]
+  unlistedActiveValidators: string[]
 
   /**
    * Validators that are not tracked by the database. The untracked validators are

server/utils/validator-listing.ts

@@ -0,0 +1,38 @@
+const UNKNOWN_VALIDATOR_NAME = 'Unknown validator'
+
+interface ValidatorListState {
+  isListed: boolean | null
+  name: string
+}
+
+interface ValidatorAddress {
+  address: string
+}
+
+interface StoredValidatorAddressState extends ValidatorAddress {
+  isListed: boolean | null
+}
+
+export function isKnownValidatorProfile({ isListed, name }: ValidatorListState) {
+  if (typeof isListed === 'boolean')
+    return isListed
+  return name.toLowerCase() !== UNKNOWN_VALIDATOR_NAME.toLowerCase()
+}
+
+export function getUnlistedAddresses(storedAddresses: string[], bundledAddresses: Set<string>) {
+  return storedAddresses.filter(address => !bundledAddresses.has(address))
+}
+
+export function getUnlistedActiveValidatorAddresses(
+  epochValidators: ValidatorAddress[],
+  storedValidators: StoredValidatorAddressState[],
+) {
+  const unlistedAddresses = new Set(
+    storedValidators
+      .filter(v => v.isListed === false)
+      .map(v => v.address),
+  )
+  return epochValidators
+    .filter(v => unlistedAddresses.has(v.address))
+    .map(v => v.address)
+}

server/utils/validators-bundle.ts

@@ -1,8 +1,8 @@
 import type { Result } from 'nimiq-validator-trustscore/types'
 import type { ValidatorJSON } from './schemas'
-import { bundledValidatorsByNetwork } from '../generated/validators-bundle.generated'
 import { validatorSchema } from './schemas'
-import { storeValidator } from './validators'
+import { getUnlistedAddresses } from './validator-listing'
+import { getStoredValidatorsAddress, markValidatorsAsUnlisted, storeValidator } from './validators'
 
 interface ImportValidatorsBundledOptions {
   shouldStore?: boolean
@@ -13,25 +13,37 @@ export async function importValidatorsBundled(nimiqNetwork?: string, options: Im
     return [false, 'Nimiq network is required', undefined]
 
   const { shouldStore = true } = options
-  const bundledValidators = bundledValidatorsByNetwork[nimiqNetwork as keyof typeof bundledValidatorsByNetwork]
-  if (!bundledValidators || bundledValidators.length === 0)
-    return [false, `No bundled validators found for network: ${nimiqNetwork}`, undefined]
+  const storage = useStorage('assets:public')
+  const keys = await storage.getKeys(`validators/${nimiqNetwork}`)
 
   const validators: ValidatorJSON[] = []
-  for (const data of bundledValidators) {
+  for (const key of keys) {
+    if (!key.endsWith('.json') || key.endsWith('.example.json'))
+      continue
+
+    const data = await storage.getItem(key)
     const parsed = validatorSchema.safeParse(data)
     if (!parsed.success)
-      return [false, `Invalid bundled validator data: ${parsed.error}`, undefined]
+      return [false, `Invalid validator data at ${key}: ${parsed.error}`, undefined]
     validators.push(parsed.data)
   }
 
   if (!shouldStore)
     return [true, undefined, validators]
 
-  const results = await Promise.allSettled(validators.map(v => storeValidator(v.address, v, { upsert: true })))
+  if (validators.length === 0)
+    return [false, `No bundled validators found for network: ${nimiqNetwork}`, undefined]
+
+  const bundledAddresses = new Set(validators.map(v => v.address))
+  const storedAddresses = await getStoredValidatorsAddress()
+  const unlistedAddresses = getUnlistedAddresses(storedAddresses, bundledAddresses)
+
+  const results = await Promise.allSettled(validators.map(v => storeValidator(v.address, v, { upsert: true, isListed: true })))
   const failures = results.filter(r => r.status === 'rejected')
   if (failures.length > 0)
     return [false, `Errors importing validators: ${failures.map((f: any) => f.reason).join(', ')}`, undefined]
 
+  await markValidatorsAsUnlisted(unlistedAddresses)
+
   return [true, undefined, validators]
 }