Move Etherscan utils to lib-sourcify (#2297)

* move etherscan utils from server to lib-sourcify

* enhance error handling for Etherscan and update related tests

* move stringToBase64 utility function from etherscan-util to etherscan.session.handlers

* refactor: reorganize Etherscan utilities and error handling for improved clarity and maintainability

* streamline Etherscan error handling by removing unnecessary parameters and enhancing error message generation

* simplify Etherscan error handling by removing URL parameters and enhancing error messages

* update dependencies and enhance test setup with chai-http

* test failing tests on circleci

* fix linter

* try disabling parallel mode on circleci

* disable concurrency in circleci test

* test fix circleci tests

* enhance error handling in fetchFromEtherscan and update test to focus on this function

* fix circleci tests by forcing the mocked api key to ''

* allow empty userApiKey in mockEtherscanApi

* Use etherscanResponseMock from server in lib-sourcify's tests

* remove AGENTS.md file

* small fixes

* add documentation and refactor code in server

* Rename `url` to `maskedUrl` following 0a417e6

* fix PR comments

* improve types in etherscan-util.ts

* fix conflicts on package-lock.json

---------

Co-authored-by: Kaan Uzdoğan <[email protected]>

Author: marcocastignoli

.circleci/continue_config.yml

@@ -13,10 +13,10 @@ commands:
   gcp-oidc-generate-cred-config-file:
     description: "Authenticate with GCP using a CircleCI OIDC token."
     parameters:
-      gcp_cred_config_file_path: 
+      gcp_cred_config_file_path:
         type: string
         default: /home/circleci/gcp_cred_config.json
-      oidc_token_file_path: 
+      oidc_token_file_path:
         type: string
         default: /home/circleci/oidc_token.json
     steps:
@@ -35,7 +35,7 @@ commands:
   gcp-oidc-authenticate:
     description: "Authenticate with GCP using a GCP credentials file."
     parameters:
-      gcp_cred_config_file_path: 
+      gcp_cred_config_file_path:
         type: string
         default: /home/circleci/gcp_cred_config.json
     steps:
@@ -248,7 +248,7 @@ jobs:
       - gcp-cli/install
       - gcp-oidc-generate-cred-config-file
       - gcp-oidc-authenticate
-      - run: 
+      - run:
           name: Deploy to GCP
           no_output_timeout: 10m
           command: |

.circleci/new_branch.yml

@@ -7,7 +7,7 @@ parameters:
   run-validate-database-schema:
     type: boolean
     default: false
-  
+
 workflows:
   # Always run
   node-build:
@@ -139,7 +139,7 @@ jobs:
             npm run migrate:up
             # Force dump in case problems were ignored
             npx dbmate dump
-            
+
             # Check if the schema file has changed
             if git diff --exit-code sourcify-database.sql; then
               echo "✅ Schema validation passed - sourcify-database.sql is in sync with migrations"
@@ -175,4 +175,4 @@ jobs:
             done;
 
             # Check if migrations can be run again
-            npm run migrate:up
+            npm run migrate:up

package-lock.json

@@ -26,9 +26,9 @@ import {
   ISolidityCompiler,
   SolidityJsonInput,
   SolidityOutput,
-} from "@ethereum-sourcify/lib-sourcify";
-import { useSolidityCompiler } from "@ethereum-sourcify/compilers";
-import * as fs from "fs";
+} from '@ethereum-sourcify/lib-sourcify';
+import { useSolidityCompiler } from '@ethereum-sourcify/compilers';
+import * as fs from 'fs';
 
 // Step 1: Setup your compiler
 class Solc implements ISolidityCompiler {
@@ -43,26 +43,26 @@ class Solc implements ISolidityCompiler {
   async compile(
     version: string,
     solcJsonInput: SolidityJsonInput,
-    forceEmscripten: boolean = false
+    forceEmscripten: boolean = false,
   ): Promise<SolidityOutput> {
     return await useSolidityCompiler(
       this.solcRepoPath, // useSolidityCompiler will automatically download and store solc here
       this.solJsonRepoPath, // useSolidityCompiler will automatically download and store solcjs here
       version,
       solcJsonInput,
-      forceEmscripten
+      forceEmscripten,
     );
   }
 }
 
-const solc = new Solc("/path/to/solc", "/path/to/solcjs");
+const solc = new Solc('/path/to/solc', '/path/to/solcjs');
 
 // Step 2: Prepare your standard JSON input
 const jsonInput = {
-  language: "Solidity",
+  language: 'Solidity',
   sources: {
-    "Contract.sol": {
-      content: "contract MyContract { function foo() public {} }",
+    'Contract.sol': {
+      content: 'contract MyContract { function foo() public {} }',
     },
   },
   settings: {
@@ -71,35 +71,35 @@ const jsonInput = {
       runs: 200,
     },
     outputSelection: {
-      "*": [],
+      '*': [],
     },
   },
 } as SolidityJsonInput;
 
 // Step 3: Create a compilation
 const compilation = new SolidityCompilation(
   solc,
-  "0.8.20", // compiler version
+  '0.8.20', // compiler version
   jsonInput,
   {
-    path: "Contract.sol",
-    name: "MyContract", // The name of your contract
-  }
+    path: 'Contract.sol',
+    name: 'MyContract', // The name of your contract
+  },
 );
 
 // Step 4: Set up a SourcifyChain instance
 const myChain = new SourcifyChain({
-  name: "My EVM Chain",
+  name: 'My EVM Chain',
   chainId: 1337,
-  rpc: ["http://localhost:8545"],
+  rpc: ['http://localhost:8545'],
   supported: true,
 });
 
 // Step 5: Verify the contract
 const verification = new Verification(
   compilation,
   myChain,
-  "0xc0ffee254729296a45a3885639AC7E10F9d54979"
+  '0xc0ffee254729296a45a3885639AC7E10F9d54979',
 );
 
 await verification.verify();
@@ -115,13 +115,16 @@ This example shows the complete verification flow for a Solidity contract using
 For browser usage, we recommend using [web-solc](https://github.com/gnidan/web-solc) instead of `@ethereum-sourcify/compilers`. Here's an example of how to implement the `ISolidityCompiler` interface with `web-solc`:
 
 ```typescript
-import { ISolidityCompiler, SolidityJsonInput } from "@ethereum-sourcify/lib-sourcify";
-import { fetchSolc } from "web-solc";
+import {
+  ISolidityCompiler,
+  SolidityJsonInput,
+} from '@ethereum-sourcify/lib-sourcify';
+import { fetchSolc } from 'web-solc';
 
 class Solc implements ISolidityCompiler {
   async compile(
     version: string,
-    solcJsonInput: SolidityJsonInput
+    solcJsonInput: SolidityJsonInput,
   ): Promise<any> {
     const { compile } = await fetchSolc(version);
     return await compile(solcJsonInput);
@@ -136,12 +139,10 @@ const solc = new Solc();
 lib-sourcify v2 consists of several key components:
 
 - **Validation**: Based on a Solidity metadata.json file that describes a contract build, checks if all source files are present and valid. Fetches missing sources from IPFS if necessary.
-
   - `SolidityMetadataContract`: Represents a Solidity contract with its metadata and source files
   - `processFiles.ts`: Utility functions for creating `SolidityMetadataContract` instances from files
 
 - **Compilation**: Handles contract compilation
-
   - `AbstractCompilation`: Base class for compilation
   - `SolidityCompilation`: Handles Solidity compilation
   - `VyperCompilation`: Handles Vyper compilation
@@ -177,7 +178,7 @@ import { SolidityMetadataContract } from '@ethereum-sourcify/lib-sourcify';
 // Set a custom IPFS gateway
 SolidityMetadataContract.setGlobalIpfsGateway({
   url: 'https://my-ipfs-gateway.com/ipfs/',
-  headers: { 'Authorization': 'Bearer my-token' }
+  headers: { Authorization: 'Bearer my-token' },
 });
 
 // Get the current IPFS gateway configuration
@@ -189,7 +190,11 @@ const gateway = SolidityMetadataContract.getGlobalIpfsGateway();
 You can set the global log level and provide a custom logger:
 
 ```typescript
-import { setLibSourcifyLoggerLevel, setLibSourcifyLogger, getLibSourcifyLoggerLevel } from '@ethereum-sourcify/lib-sourcify';
+import {
+  setLibSourcifyLoggerLevel,
+  setLibSourcifyLogger,
+  getLibSourcifyLoggerLevel,
+} from '@ethereum-sourcify/lib-sourcify';
 
 // Set the log level
 setLibSourcifyLoggerLevel(5); // 0=errors, 1=warnings, 2=info, 5=debug, 6=silly
@@ -205,7 +210,7 @@ setLibSourcifyLogger({
   },
   log(level, msg) {
     // Custom logging implementation
-  }
+  },
 });
 ```
 
@@ -222,28 +227,31 @@ import {
   SolidityOutput,
   ISolidityCompiler,
   SolidityJsonInput,
-} from "@ethereum-sourcify/lib-sourcify";
-import { useSolidityCompiler } from "@ethereum-sourcify/compilers";
+} from '@ethereum-sourcify/lib-sourcify';
+import { useSolidityCompiler } from '@ethereum-sourcify/compilers';
 
 class Solc implements ISolidityCompiler {
-  constructor(private solcRepoPath: string, private solJsonRepoPath: string) {}
+  constructor(
+    private solcRepoPath: string,
+    private solJsonRepoPath: string,
+  ) {}
 
   async compile(
     version: string,
     solcJsonInput: SolidityJsonInput,
-    forceEmscripten: boolean = false
+    forceEmscripten: boolean = false,
   ): Promise<SolidityOutput> {
     return await useSolidityCompiler(
       this.solcRepoPath,
       this.solJsonRepoPath,
       version,
       solcJsonInput,
-      forceEmscripten
+      forceEmscripten,
     );
   }
 }
 
-const solc = new Solc("/path/to/solc/repo", "/path/to/solcjs/repo");
+const solc = new Solc('/path/to/solc/repo', '/path/to/solcjs/repo');
 ```
 
 ### Vyper Compiler Example
@@ -359,7 +367,10 @@ const compilation = await metadataContract.createCompilation(solidityCompiler);
 For file-based validation:
 
 ```typescript
-import { createMetadataContractsFromFiles, PathBuffer } from '@ethereum-sourcify/lib-sourcify';
+import {
+  createMetadataContractsFromFiles,
+  PathBuffer,
+} from '@ethereum-sourcify/lib-sourcify';
 
 const pathBuffers: PathBuffer[] = [];
 pathBuffers.push({
@@ -488,14 +499,45 @@ setLibSourcifyLogger({
 });
 ```
 
+## EtherscanUtils
+
+EtherscanUtils provides utilities for importing and processing verified contracts from Etherscan APIs.
+
+### Fetching Contract Data from Etherscan
+
+```typescript
+import { EtherscanUtils } from '@ethereum-sourcify/lib-sourcify';
+
+// Fetch contract source code and metadata from Etherscan
+const etherscanResult = await EtherscanUtils.fetchFromEtherscan(
+  1, // chainId
+  '0x6B175474E89094C44Da98b954EedeAC495271d0F', // contract address
+  'YOUR_API_KEY', // API key
+);
+```
+
+### Creating Compilations from Etherscan Results
+
+```typescript
+// Create a compilation object directly from Etherscan data
+const compilation = await EtherscanUtils.getCompilationFromEtherscanResult(
+  etherscanResult,
+  solidityCompiler, // ISolidityCompiler instance
+  vyperCompiler, // IVyperCompiler instance (for Vyper contracts)
+);
+
+// Use the compilation for verification
+const verification = new Verification(compilation, chain, contractAddress);
+await verification.verify();
+```
+
 ## Migration Guide from v1 to v2
 
 Version 2 of lib-sourcify brings a significant redesign of the library's architecture. Here's how to migrate from v1 to v2:
 
 ### Key Changes
 
 1. **Class Structure Changes**:
-
    - `AbstractCheckedContract` is replaced by language-specific compilation classes
    - `SolidityCheckedContract` functionality is now split between `SolidityMetadataContract` and `SolidityCompilation`
    - `VyperCheckedContract` is now represented by `VyperCompilation`

packages/lib-sourcify/README.md

@@ -44,6 +44,7 @@
     "@ethersproject/bignumber": "5.8.0",
     "@ethersproject/bytes": "5.8.0",
     "@fairdatasociety/bmt-js": "2.1.0",
+    "@solidity-parser/parser": "0.20.2",
     "bs58": "5.0.0",
     "ethers": "6.15.0",
     "jszip": "3.10.1",
@@ -62,6 +63,7 @@
     "c8": "10.1.3",
     "chai": "4.5.0",
     "chai-as-promised": "7.1.2",
+    "chai-http": "4.4.0",
     "cspell": "8.19.4",
     "cz-conventional-changelog": "3.3.0",
     "gh-pages": "6.3.0",

packages/lib-sourcify/package.json

@@ -30,7 +30,7 @@ import {
   getVariationsByContentHash,
 } from './variationsUtils';
 import { logDebug } from '../logger';
-import { splitFullyQualifiedName } from '../utils';
+import { splitFullyQualifiedName } from '../utils/utils';
 
 export class SolidityMetadataContract {
   metadata: Metadata;

packages/lib-sourcify/src/Validation/SolidityMetadataContract.ts

@@ -16,6 +16,13 @@ export * from './Compilation/CompilationTypes';
 export * from './Verification/Verification';
 export * from './Verification/VerificationTypes';
 
+// Etherscan utils exports
+export * from './utils/etherscan/EtherscanTypes';
+import * as etherscanUtils from './utils/etherscan/etherscan-util';
+export const EtherscanUtils = {
+  ...etherscanUtils,
+};
+
 // Validation exports
 export * from './Validation/SolidityMetadataContract';
 export * from './Validation/ValidationTypes';
@@ -32,7 +39,7 @@ export * from './SourcifyChain/SourcifyChainTypes';
 export * from './SourcifyLibError';
 
 // Utils exports
-export * from './utils';
+export * from './utils/utils';
 
 // Export all compilers types
 export * from '@ethereum-sourcify/compilers-types';