Skip to content

Commit 9dcee8b

Browse files
ravenac95ravenac95
authored
Fix contracts v0 bugs (#3008)
* Address issues with proper start times for incremental models * more fixes for start times * smaller size batch * fix contracts_v0 * smaller batch size * more partitioning * test 60 * Fixes for intermediate * fix handling of null or empty string in deployers * set correct start date * fix handling of null or empty string in deployers * wip * more wip * Fix start time * Change to deployment_date * Cast deployment_date to date
1 parent c09a42e commit 9dcee8b

19 files changed

+313
-218
lines changed

‎ops/tf-modules/warehouse-cluster/main.tf‎

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -7,10 +7,10 @@ locals {
77
node_pools = concat([
88
{
99
name = "${var.cluster_name}-default-node-pool"
10-
machine_type = "e2-standard-4"
10+
machine_type = "e2-standard-2"
1111
node_locations = join(",", var.default_node_pool_cluster_zones)
1212
min_count = 0
13-
max_count = 4
13+
max_count = 3
1414
local_ssd_count = 0
1515
spot = false
1616
disk_size_gb = 50
@@ -371,8 +371,6 @@ module "gke" {
371371
horizontal_pod_autoscaling = true
372372
filestore_csi_driver = false
373373
deletion_protection = false
374-
monitoring_enable_managed_prometheus = true
375-
monitoring_enabled_components = ["SYSTEM_COMPONENTS", "APISERVER", "SCHEDULER", "CONTROLLER_MANAGER"]
376374

377375
node_pools = local.node_pools
378376

‎warehouse/metrics_mesh/macros/onchain/deployers.py‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -37,7 +37,12 @@ def transactions_with_receipts_deployers(
3737
*additional_column_defs,
3838
)
3939
.from_(transactions_table.as_("transactions"))
40-
.where(exp.Is(this=to_address, expression=exp.Null()))
40+
.where(
41+
exp.Or(
42+
this=exp.Is(this=to_address, expression=exp.Null()),
43+
expression=exp.EQ(this=to_address, expression=exp.Literal.string("")),
44+
)
45+
)
4146
.where(
4247
exp.EQ(
4348
this=receipt_status, expression=exp.Literal(this="1", is_string=False)

‎warehouse/metrics_mesh/macros/onchain/factory_deployments.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -64,6 +64,7 @@ def factory_deployments(
6464
transactions_originating_contract.as_("originating_contract"),
6565
traces_factory_address.as_("factory_address"),
6666
traces_contract_address.as_("contract_address"),
67+
traces_trace_type.as_("create_type"),
6768
*additional_column_defs,
6869
)
6970
.with_("transactions_cte", as_=transactions_cte)

‎warehouse/metrics_mesh/models/intermediate/blockchain_artifacts/int_contracts_deployment.sql‎

Lines changed: 65 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,65 @@
1+
MODEL (
2+
name metrics.int_contracts_deployment,
3+
kind INCREMENTAL_BY_TIME_RANGE (
4+
time_column deployment_timestamp,
5+
batch_size 90,
6+
batch_concurrency 1,
7+
--forward_only true,
8+
--on_destructive_change warn
9+
),
10+
start '2021-10-01',
11+
partitioned_by (DAY("deployment_timestamp"), "chain"),
12+
);
13+
14+
-- The intent is to get the _first_ factory deployments as some contracts
15+
-- deployed via deterministic deployers allows for multiple calls to the
16+
-- create2 function.
17+
18+
19+
20+
with ordered_deployments_in_period as (
21+
select
22+
factories.block_timestamp as deployment_timestamp,
23+
factories.chain as chain,
24+
factories.transaction_hash as transaction_hash,
25+
case
26+
when proxies.address is not null then proxies.address
27+
else factories.originating_address
28+
end as originating_address,
29+
factories.originating_contract as originating_contract,
30+
factories.contract_address as contract_address,
31+
factories.factory_address as factory_address,
32+
factories.create_type,
33+
row_number() over (
34+
partition by factories.contract_address, factories.chain
35+
order by block_timestamp asc
36+
) as creation_order,
37+
case
38+
when proxies.address is not null then true
39+
else false
40+
end as is_proxy
41+
from metrics.int_factories as factories
42+
left join metrics.int_proxies as proxies
43+
on
44+
factories.originating_contract = proxies.address
45+
and factories.chain = proxies.chain
46+
where contract_address is not null
47+
and block_timestamp between @start_dt and @end_dt
48+
-- ignore anything that already has already been processed
49+
and contract_address not in (
50+
select contract_address
51+
from @this_model
52+
where block_timestamp < @start_dt
53+
)
54+
)
55+
select
56+
deployment_timestamp::TIMESTAMP,
57+
chain::VARCHAR,
58+
transaction_hash::VARCHAR,
59+
originating_address::VARCHAR,
60+
contract_address::VARCHAR,
61+
factory_address::VARCHAR,
62+
create_type::VARCHAR,
63+
is_proxy::BOOLEAN
64+
from ordered_deployments_in_period
65+
where creation_order = 1

‎warehouse/metrics_mesh/models/intermediate/blockchain_artifacts/int_contracts_overview.sql‎

Lines changed: 5 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -3,28 +3,17 @@ MODEL (
33
kind FULL,
44
);
55

6-
@DEF(start, current_date() - INTERVAL 180 DAY);
7-
@DEF(end, current_date());
8-
96
select
10-
MIN(derived_contracts.deployment_date) as deployment_date,
7+
derived_contracts.deployment_timestamp as deployment_timestamp,
118
derived_contracts.contract_address,
129
derived_contracts.chain as contract_namespace,
1310
derived_contracts.originating_address as originating_address,
1411
derived_contracts.factory_address,
1512
derived_contracts.root_deployer_address as root_deployer_address,
16-
SUM(transactions_weekly.tx_count) as sort_weight
13+
sort_weights.sort_weight as sort_weight
1714
from metrics.int_derived_contracts as derived_contracts
18-
inner join metrics.int_derived_contracts_transactions_weekly as transactions_weekly
15+
inner join metrics.int_derived_contracts_sort_weights as sort_weights
1916
on
20-
derived_contracts.contract_address = transactions_weekly.contract_address
21-
and derived_contracts.chain = transactions_weekly.chain
22-
where derived_contracts.deployment_date between @start and @end
23-
group by
24-
derived_contracts.contract_address,
25-
derived_contracts.chain,
26-
derived_contracts.factory_address,
27-
derived_contracts.root_deployer_address,
28-
derived_contracts.originating_address
29-
17+
derived_contracts.contract_address = sort_weights.contract_address
18+
and derived_contracts.chain = sort_weights.chain
3019

‎warehouse/metrics_mesh/models/intermediate/blockchain_artifacts/int_contracts_root_deployers.sql‎

Lines changed: 88 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,88 @@
1+
-- Root deployers is an interesting problem. When a contract gets created we
2+
-- don't know if it's a factory based on our current processing. We can only
3+
-- know that a contract is a factory if it has deployed other contracts. This
4+
-- model is an attempt to identify the root deployer of a contract. The root
5+
-- deployer is discovered by looking backwards through contract creators. To
6+
-- prevent our warehouse from storing far too much data we only look back 365
7+
-- days and incrementally update this model. If the contract is used as a
8+
-- factory within that time and was also deployed within that time a row will be
9+
-- created in this model.
10+
MODEL (
11+
name metrics.int_contracts_root_deployers,
12+
kind INCREMENTAL_BY_TIME_RANGE (
13+
time_column deployment_timestamp,
14+
batch_size 90,
15+
batch_concurrency 1,
16+
lookback 30
17+
--forward_only true
18+
),
19+
start '2021-10-01',
20+
partitioned_by (DAY("deployment_timestamp"), "chain")
21+
);
22+
23+
with existing_contracts as (
24+
select *
25+
from @this_model
26+
where deployment_timestamp < @start_dt
27+
), new_contracts as (
28+
select
29+
deployment_timestamp,
30+
chain,
31+
transaction_hash,
32+
originating_address,
33+
contract_address,
34+
-- if the originating address is the same as the factory address then
35+
-- this was a create_type of create and deployed directly by an EOA
36+
case
37+
when originating_address = factory_address then null
38+
else factory_address
39+
end as factory_address,
40+
create_type,
41+
case
42+
-- for create deployed factories we can just use the originating address
43+
when originating_address = factory_address then originating_address
44+
when is_proxy then originating_address
45+
else null
46+
end as root_deployer_address,
47+
0 as depth
48+
from metrics.int_contracts_deployment
49+
where deployment_timestamp between @start_dt and @end_dt
50+
), all_contracts as (
51+
select * from existing_contracts
52+
union all
53+
select * from new_contracts
54+
), new_resolved as (
55+
select
56+
new.deployment_timestamp,
57+
new.chain,
58+
new.transaction_hash,
59+
new.originating_address,
60+
new.contract_address,
61+
new.factory_address,
62+
new.create_type,
63+
case
64+
when new.root_deployer_address is not null then new.root_deployer_address
65+
else all.originating_address
66+
end as root_deployer_address,
67+
case
68+
when all.depth is not null then all.depth + 1
69+
else new.depth
70+
end as depth
71+
from new_contracts as new
72+
left join all_contracts as all
73+
on
74+
all.chain = new.chain
75+
and all.contract_address = new.factory_address
76+
)
77+
select
78+
deployment_timestamp::TIMESTAMP,
79+
chain::VARCHAR,
80+
transaction_hash::VARCHAR,
81+
originating_address::VARCHAR,
82+
contract_address::VARCHAR,
83+
factory_address::VARCHAR,
84+
create_type::VARCHAR,
85+
root_deployer_address::VARCHAR,
86+
depth::INT
87+
from new_resolved
88+

‎warehouse/metrics_mesh/models/intermediate/blockchain_artifacts/int_contracts_transactions_weekly.sql‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
-- Part of the int_derived_contracts_transactions_weekly model. This is an
2+
-- intermediate model that looks at the "universe" of blockchain traces and
3+
-- attempts to give a summary of the activity on any given contract on a weekly
4+
-- bucket. This is only used for the contracts overview model so it doesn't have
5+
-- all of history
6+
7+
MODEL (
8+
name metrics.int_contracts_transactions_weekly,
9+
kind INCREMENTAL_BY_TIME_RANGE (
10+
time_column week,
11+
batch_size 180,
12+
batch_concurrency 3,
13+
--forward_only true,
14+
--on_destructive_change warn
15+
),
16+
cron '@weekly',
17+
partitioned_by (DAY(week), chain),
18+
-- We only need to start a year before from the time we start using this
19+
-- model. We are using this as a way to categorize model ordering.
20+
start '2024-08-01'
21+
);
22+
23+
-- Find all transactions involving the contracts from `derived_contracts` and
24+
-- aggregate their tx_count on a weekly basis
25+
select
26+
date_trunc('week', traces.dt) as week,
27+
upper(traces.chain) as chain,
28+
lower(traces.to_address) as contract_address,
29+
count(*) as tx_count
30+
from @oso_source('bigquery.optimism_superchain_raw_onchain_data.traces') as traces
31+
where
32+
traces.network = 'mainnet'
33+
and "status" = 1
34+
and dt between @start_date and @end_date + INTERVAL 6 DAY
35+
group by 1, 2, 3

‎warehouse/metrics_mesh/models/intermediate/blockchain_artifacts/int_deployers.sql‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ MODEL (
55
batch_size 365,
66
batch_concurrency 1
77
),
8+
start '2021-10-01',
89
partitioned_by (DAY("block_timestamp"), "chain"),
910
);
1011

‎warehouse/metrics_mesh/models/intermediate/blockchain_artifacts/int_derived_contracts.sql‎

Lines changed: 10 additions & 105 deletions
Original file line numberDiff line numberDiff line change
@@ -1,109 +1,14 @@
11
MODEL (
22
name metrics.int_derived_contracts,
3-
kind INCREMENTAL_BY_TIME_RANGE (
4-
time_column deployment_date,
5-
batch_size 365,
6-
batch_concurrency 1,
7-
--forward_only true,
8-
--on_destructive_change allow
9-
),
10-
partitioned_by (DAY("deployment_date"), "chain"),
11-
cron '@daily',
3+
kind VIEW
124
);
135

14-
with contracts_deployed_no_factory as (
15-
select
16-
block_timestamp as deployment_date,
17-
chain,
18-
deployer_address as originating_address,
19-
contract_address,
20-
'' as factory_address,
21-
deployer_address as root_deployer_address
22-
from metrics.int_deployers
23-
where contract_address is not null
24-
and block_timestamp between @start_dt and @end_dt
25-
),
26-
27-
contracts_deployed_via_factory as (
28-
-- This gets all of the contracts deployed by any factory.
29-
-- Deployer Address is the EOA address that started the transaction
30-
select
31-
block_timestamp as deployment_date,
32-
factories.chain,
33-
originating_address as originating_address,
34-
contract_address as contract_address,
35-
factories.factory_address as factory_address,
36-
factory_root_deployers.root_deployer_address as root_deployer_address
37-
from metrics.int_factories as factories
38-
inner join metrics.int_factory_root_deployers as factory_root_deployers
39-
on
40-
factory_root_deployers.factory_address = factories.factory_address
41-
and factories.chain = factory_root_deployers.chain
42-
where contract_address is not null
43-
and block_timestamp between @start_dt and @end_dt
44-
),
45-
46-
contracts_deployed_by_safe_or_known_proxy as (
47-
-- This gets all of the contracts deployed by a safe or other known proxy
48-
-- Deployer address is a proxy (safe or other known proxy) that deployed the contract
49-
select
50-
factories.block_timestamp as deployment_date,
51-
factories.chain,
52-
proxies.address as originating_address,
53-
factories.contract_address as contract_address,
54-
factories.factory_address as factory_address,
55-
factory_root_deployers.root_deployer_address as root_deployer_address
56-
from metrics.int_factories as factories
57-
inner join metrics.int_proxies as proxies
58-
on
59-
factories.originating_contract = proxies.address
60-
and factories.chain = proxies.chain
61-
inner join metrics.int_factory_root_deployers as factory_root_deployers
62-
on
63-
factory_root_deployers.factory_address = factories.factory_address
64-
and factories.chain = factory_root_deployers.chain
65-
where contract_address is not null
66-
and factories.block_timestamp between @start_dt and @end_dt
67-
),
68-
69-
derived_contracts as (
70-
select
71-
deployment_date,
72-
chain,
73-
originating_address,
74-
contract_address,
75-
factory_address,
76-
root_deployer_address
77-
from contracts_deployed_no_factory
78-
79-
union all
80-
81-
select
82-
deployment_date,
83-
chain,
84-
originating_address,
85-
contract_address,
86-
factory_address,
87-
root_deployer_address
88-
from contracts_deployed_via_factory
89-
90-
union all
91-
92-
select
93-
deployment_date,
94-
chain,
95-
originating_address,
96-
contract_address,
97-
factory_address,
98-
root_deployer_address
99-
from contracts_deployed_by_safe_or_known_proxy
100-
)
101-
102-
select distinct
103-
deployment_date,
104-
chain,
105-
originating_address,
106-
contract_address,
107-
factory_address,
108-
root_deployer_address
109-
from derived_contracts
6+
select
7+
deployment_timestamp::TIMESTAMP,
8+
chain::VARCHAR,
9+
originating_address::VARCHAR,
10+
contract_address::VARCHAR,
11+
factory_address::VARCHAR,
12+
create_type::VARCHAR,
13+
root_deployer_address::VARCHAR,
14+
from metrics.int_contracts_root_deployers

0 commit comments

Comments
 (0)