Replace script config for fixing mismatched metadata (#2285)

* Add replaceMetadata method to customReplaceMethods

* Add excludeContract option to ReplaceConfig and implement contract skipping logic

* Add error handling for on-chain creation bytecode fetching in replaceContract

* Add configuration for replacing metadata in sourcify_matches table via massive replace script

Author: manuelwedler

services/database/.gitignore

@@ -1,2 +1,3 @@
 migrations-temp/
-massive-replace-script/CURRENT_VERIFIED_CONTRACT
+massive-replace-script/CURRENT_VERIFIED_CONTRACT
+massive-replace-script/FAILED_CONTRACTS

services/database/massive-replace-script/config-replace-metadata.js

@@ -0,0 +1,100 @@
+// Configuration for replacing metadata in sourcify_matches table
+// This configuration targets contracts with partial runtime matches where metadata needs to be updated
+// because it does not match the sources in the database.
+// Only processes contracts where source hashes don't match the existing metadata
+
+const { id: keccak256str } = require("ethers");
+
+module.exports = {
+  query: async (sourcePool, sourcifySchema, currentVerifiedContract, n) => {
+    return await sourcePool.query(
+      `
+      SELECT 
+          cd.chain_id,
+          cd.address,
+          sm.id as verified_contract_id,
+          json_build_object(
+            'language', INITCAP(cc.language), 
+            'sources', json_object_agg(compiled_contracts_sources.path, json_build_object('content', sources.content)),
+            'settings', cc.compiler_settings
+          ) as std_json_input,
+          cc.version as compiler_version,
+          cc.fully_qualified_name,
+          sm.metadata
+      FROM ${sourcifySchema}.sourcify_matches sm
+      JOIN ${sourcifySchema}.verified_contracts vc ON sm.verified_contract_id = vc.id
+      JOIN ${sourcifySchema}.contract_deployments cd ON vc.deployment_id = cd.id
+      JOIN ${sourcifySchema}.compiled_contracts cc ON vc.compilation_id = cc.id
+      JOIN ${sourcifySchema}.compiled_contracts_sources ON compiled_contracts_sources.compilation_id = cc.id
+      LEFT JOIN ${sourcifySchema}.sources ON sources.source_hash = compiled_contracts_sources.source_hash
+      WHERE sm.created_at < '2024-08-29 08:58:57 +0200'
+          AND sm.runtime_match ='partial'
+          AND sm.id >= $1
+      GROUP BY sm.id, vc.id, cc.id, cd.id
+      ORDER BY sm.id ASC
+      LIMIT $2
+    `,
+      [currentVerifiedContract, n],
+    );
+  },
+  buildRequestBody: (contract) => {
+    return {
+      chainId: contract.chain_id.toString(),
+      address: `0x${contract.address.toString("hex")}`,
+      forceCompilation: true,
+      jsonInput: contract.std_json_input,
+      compilerVersion: contract.compiler_version,
+      compilationTarget: contract.fully_qualified_name,
+      forceRPCRequest: false,
+      customReplaceMethod: "replace-metadata",
+    };
+  },
+  excludeContract: (contract) => {
+    const address = `0x${contract.address.toString("hex")}`;
+    const sources = contract.std_json_input.sources;
+    const currentMetadata = contract.metadata;
+
+    if (!sources || !currentMetadata) {
+      console.log(
+        `Contract address=${address}, chain_id=${contract.chain_id}: Missing sources or metadata -> skipping`,
+      );
+      return true; // Exclude if no sources or metadata
+    }
+
+    if (
+      Object.keys(sources).length !==
+      Object.keys(currentMetadata.sources).length
+    ) {
+      console.log(
+        `Contract address=${address}, chain_id=${contract.chain_id}: wrong sources length: ${Object.keys(sources).length} (std json) vs ${Object.keys(currentMetadata.sources).length} (metadata)`,
+      );
+      return false; // something is wrong -> replace metadata
+    }
+
+    for (const [sourcePath, sourceMetadata] of Object.entries(
+      currentMetadata.sources,
+    )) {
+      const expectedHash = sourceMetadata.keccak256;
+
+      if (!sources[sourcePath]) {
+        console.log(
+          `Contract address=${address}, chain_id=${contract.chain_id}: Metadata source ${sourcePath} not in sources`,
+        );
+        return false; // something is wrong -> replace metadata
+      }
+
+      const contentHash = keccak256str(sources[sourcePath].content);
+
+      if (contentHash !== expectedHash) {
+        console.log(
+          `Contract address=${address}, chain_id=${contract.chain_id}: ContentHash does not match metadata hash for source ${sourcePath}: ${contentHash} (std json) vs ${expectedHash} (metadata)`,
+        );
+        return false; // something is wrong -> replace metadata
+      }
+    }
+
+    return true; // All sources match the metadata, exclude this contract
+  },
+  description:
+    "Replaces metadata in sourcify_matches table for contracts where source content hashes don't match the existing metadata hashes.",
+};

services/database/massive-replace-script/massive-replace-script.ts

@@ -13,6 +13,7 @@ interface ReplaceConfig {
     n: number,
   ) => Promise<pg.QueryResult>;
   buildRequestBody: (contract: any) => any;
+  excludeContract?: (contract: any) => boolean;
   description?: string;
 }
 
@@ -136,6 +137,13 @@ async function processContract(
   config: ReplaceConfig,
 ): Promise<void> {
   const address = `0x${contract.address.toString("hex")}`;
+  if (config.excludeContract && config.excludeContract(contract)) {
+    console.log(
+      `Skipping contract: chainId=${contract.chain_id}, address=${address}, verifiedContractId=${contract.verified_contract_id}`,
+    );
+    return;
+  }
+
   try {
     console.log(
       `Processing contract: chainId=${contract.chain_id}, address=${address}, verifiedContractId=${contract.verified_contract_id}`,

services/server/src/server/apiv1/verification/private/stateless/customReplaceMethods.ts

@@ -127,6 +127,34 @@ export const replaceCreationInformation: CustomReplaceMethod = async (
   });
 };
 
+export const replaceMetadata: CustomReplaceMethod = async (
+  sourcifyDatabaseService: SourcifyDatabaseService,
+  verification: VerificationExport,
+) => {
+  const existingSourcifyMatch =
+    await sourcifyDatabaseService.database.getSourcifyMatchByChainAddressWithProperties(
+      verification.chainId,
+      bytesFromString(verification.address),
+      ["id"],
+    );
+  if (existingSourcifyMatch.rows.length === 0) {
+    throw new Error(
+      `No existing verified contract found for address ${verification.address} on chain ${verification.chainId}`,
+    );
+  }
+
+  const matchId = existingSourcifyMatch.rows[0].id;
+
+  await sourcifyDatabaseService.database.pool.query(
+    `UPDATE sourcify_matches 
+       SET 
+         metadata = $2
+       WHERE id = $1`,
+    [matchId, verification.compilation.metadata],
+  );
+};
+
 export const REPLACE_METHODS: Record<string, CustomReplaceMethod> = {
   "replace-creation-information": replaceCreationInformation,
+  "replace-metadata": replaceMetadata,
 };

services/server/src/server/apiv1/verification/private/stateless/private.stateless.handlers.ts

@@ -315,14 +315,22 @@ export async function replaceContract(
       throw error;
     }
 
+    let rpcFailedFetchingCreationBytecode = false;
+    try {
+      rpcFailedFetchingCreationBytecode =
+        verification.onchainCreationBytecode === undefined;
+    } catch (error) {
+      // verification.onchainCreationBytecode throws if not available
+      rpcFailedFetchingCreationBytecode = true;
+    }
+
     res.send({
       replaced: true,
       address: address,
       chainId: chainId,
       transactionHash: transactionHash,
       newStatus: verificationStatus,
-      rpcFailedFetchingCreationBytecode:
-        verification.onchainCreationBytecode === undefined,
+      rpcFailedFetchingCreationBytecode,
     });
   } catch (error: any) {
     throw new InternalServerError(error.message);

services/server/src/server/apiv1/verification/private/stateless/private.stateless.paths.yaml

@@ -201,6 +201,7 @@ paths:
                       creation match status while preserving the runtime verification data. This method is 
                       useful for correcting incomplete or missing creation data on contracts that were 
                       previously verified only against runtime bytecode.
+                    - "replace-metadata": Updates only the metadata of an existing verified contract.
                   example: "replace-creation-information"
       responses:
         "200":