Add Clickhouse Client

fees/ethereum-wip/index.ts

@@ -0,0 +1,89 @@
+import { Row } from "@clickhouse/client"
+import { FetchOptions, ProtocolType, Adapter } from "../../adapters/types";
+import { METRIC } from "../../helpers/metrics";
+import { queryClickhouse } from "../../helpers/clickhouse";
+import { CHAIN } from "../../helpers/chains";
+
+type FeesRow = Row & {
+  total_fees_wei: string;
+  base_burn_wei: string;
+};
+
+export const SQL_BLOCK_RANGE = `
+  SELECT
+    CAST(
+      sum(toDecimal256(effective_gas_price, 0) * toDecimal256(gas_used, 0))
+      AS String
+    ) AS total_fees_wei,
+    CAST(
+      sum(toDecimal256(base_fee_per_gas, 0) * toDecimal256(gas_used, 0))
+      AS String
+    ) AS base_burn_wei
+  FROM evm_indexer.transactions
+  WHERE
+    chain = {chain:UInt64}
+    AND block_number >= {fromBlock:UInt32}
+    AND block_number <  {toBlock:UInt32}
+`;
+
+export const fetch = async (options: FetchOptions) => {
+  const chainId = options.api.chainId
+  const fromBlock = await options.getFromBlock()
+  const _toBlock = await options.getToBlock()
+  const safeBlock = _toBlock - 150
+
+  if (safeBlock <= fromBlock) {
+    const dailyFees = options.createBalances();
+    const dailyRevenue = options.createBalances();
+    return { dailyFees, dailyRevenue, dailyHoldersRevenue: dailyRevenue };
+  }
+
+  const rows = await queryClickhouse<FeesRow>(SQL_BLOCK_RANGE, {
+    chain: chainId,
+    fromBlock,
+    toBlock: safeBlock,
+  });
+
+  const totalFeesWei = BigInt(rows?.[0]?.total_fees_wei ?? "0");
+  const baseFeesWei  = BigInt(rows?.[0]?.base_burn_wei  ?? "0");
+  const priorityWei  = totalFeesWei - baseFeesWei;
+
+  const dailyFees = options.createBalances();
+  const dailyRevenue = options.createBalances();
+
+  dailyFees.addGasToken(baseFeesWei, METRIC.TRANSACTION_BASE_FEES);
+  dailyFees.addGasToken(priorityWei, METRIC.TRANSACTION_PRIORITY_FEES);
+  dailyRevenue.addGasToken(baseFeesWei, METRIC.TRANSACTION_BASE_FEES);
+
+  return { dailyFees, dailyRevenue, dailyHoldersRevenue: dailyRevenue };
+}
+
+const adapter: Adapter = {
+  version: 2,
+  adapter: {
+    [CHAIN.ETHEREUM]: {
+      fetch,
+      start: '2015-07-30',
+    },
+  },
+  protocolType: ProtocolType.CHAIN,
+  methodology: {
+    Fees: 'Total ETH gas fees (including base fees and priority fees) paid by users',
+    Revenue: 'Amount of ETH base fees that were burned',
+    HoldersRevenue: 'Amount of ETH base fees that were burned',
+  },
+  breakdownMethodology: {
+    Fees: {
+      [METRIC.TRANSACTION_BASE_FEES]: 'Total ETH base fees paid by users',
+      [METRIC.TRANSACTION_PRIORITY_FEES]: 'Total ETH priority fees paid by users',
+    },
+    Revenue: {
+      [METRIC.TRANSACTION_BASE_FEES]: 'Total ETH base fees will be burned',
+    },
+    HoldersRevenue: {
+      [METRIC.TRANSACTION_BASE_FEES]: 'Total ETH base fees will be burned',
+    },
+  }
+}
+
+export default adapter;

helpers/clickhouse.ts

@@ -0,0 +1,65 @@
+import { ClickHouseClient, createClient, Row } from "@clickhouse/client";
+import { getEnv } from "./env";
+
+let client: ClickHouseClient | null = null;
+let connectionPromise: Promise<ClickHouseClient> | null = null;
+
+const requiredVars = ["CLICKHOUSE_HOST","CLICKHOUSE_USERNAME","CLICKHOUSE_PASSWORD","CLICKHOUSE_PORT", "CLICKHOUSE_DATABASE"];
+
+export async function connectClickhouse(): Promise<ClickHouseClient> {
+  if (client) return client;
+  if (connectionPromise) return connectionPromise;
+
+  connectionPromise = (async () => {
+    const missing = requiredVars.filter(v => !getEnv(v));
+    if (missing.length) throw new Error(`Missing required environment variables: ${missing.join(", ")}`);
+
+    const url = `http://${getEnv("CLICKHOUSE_HOST")}:${getEnv("CLICKHOUSE_PORT")}`;
+    const database = getEnv("CLICKHOUSE_DATABASE");
+
+    const _client = createClient({
+      url,
+      username: getEnv("CLICKHOUSE_USERNAME")!,
+      password: getEnv("CLICKHOUSE_PASSWORD")!,
+      database,
+      keep_alive: { enabled: true, idle_socket_ttl: 180000 },
+      compression: { response: true, request: false },
+      max_open_connections: 10,
+      request_timeout: 180000,
+    });
+
+    await _client.ping();
+    client = _client;
+    return _client;
+  })();
+
+  return connectionPromise;
+}
+
+export async function queryClickhouse<T extends Row>(
+  sql: string,
+  params?: Record<string, unknown>,
+): Promise<T[]> {
+  const c = await connectClickhouse();
+  try {
+    const rs = await c.query({
+      query: sql,
+      query_params: params,
+      format: "JSONEachRow",
+    });
+    return await rs.json<T>();
+  } catch (error: any) {
+    if (error?.code === "ECONNRESET" || error?.code === "ECONNREFUSED") {
+      await disconnectClickhouse();
+    }
+    throw error;
+  }
+}
+
+export async function disconnectClickhouse() {
+  if (client) {
+    await client.close();
+    client = null;
+  }
+  connectionPromise = null;
+}

helpers/env.ts

@@ -69,6 +69,11 @@ export const ENV_KEYS = new Set([
   'DUNE_BULK_MODE',
   'DUNE_BULK_MODE_BATCH_TIME',
   'LLAMA_HL_INDEXER',
+  'CLICKHOUSE_HOST',
+  'CLICKHOUSE_PORT',
+  'CLICKHOUSE_USERNAME',
+  'CLICKHOUSE_PASSWORD',
+  'CLICKHOUSE_DATABASE'
 ])
 
 // This is done to support both ZEROx_API_KEY and ZEROX_API_KEY

package.json

@@ -23,6 +23,7 @@
   },
   "homepage": "https://github.com/DefiLlama/adapters#readme",
   "dependencies": {
+    "@clickhouse/client": "^1.12.1",
     "@defillama/sdk": "^5.0.185",
     "@supercharge/promise-pool": "^3.1.0",
     "@types/async-retry": "^1.4.8",