Skip to content

Commit 618b670

Browse files
ryschengryscheng
authored
op-atlas as a project/artifact source (#3073)
* feat: Adding op-atlas to the directory models * Adding op-atlas projects into int_projects * Creating a separate model for int_artifacts_by_project_in_op_atlas * feat: url_parts macro * feat: adding staging models for op-atlas defillama, websites, and farcaster * complete op-atlas directory * fix NULL typing
1 parent 4290851 commit 618b670

21 files changed

+582
-36
lines changed

‎warehouse/metrics_mesh/macros/onchain/chain_id_to_chain_name.py‎

Lines changed: 74 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,74 @@
1+
from sqlglot import expressions as exp
2+
from sqlmesh import macro
3+
from sqlmesh.core.macros import MacroEvaluator
4+
5+
6+
@macro()
7+
def chain_id_to_chain_name(
8+
evaluator: MacroEvaluator,
9+
chain_id: exp.Expression,
10+
):
11+
"""
12+
Macro to translate chain_id to chain_name
13+
Note: we need to keep this synced with
14+
https://github.com/voteagora/op-atlas/blob/main/app/src/lib/oso.ts#L6
15+
https://github.com/opensource-observer/oss-directory/blob/main/src/resources/schema/blockchain-address.json
16+
"""
17+
chain_name = exp.Case(
18+
ifs=[
19+
exp.If(
20+
this=exp.EQ(
21+
this=chain_id,
22+
expression=exp.Literal.number(1),
23+
),
24+
true=exp.Literal.string("mainnet"),
25+
),
26+
],
27+
default=exp.cast(chain_id, "string"),
28+
).when(
29+
exp.EQ(this=chain_id, expression=exp.Literal.number(10)),
30+
exp.Literal.string("optimism"),
31+
).when(
32+
exp.EQ(this=chain_id, expression=exp.Literal.number(8453)),
33+
exp.Literal.string("base"),
34+
).when(
35+
exp.EQ(this=chain_id, expression=exp.Literal.number(34443)),
36+
exp.Literal.string("mode"),
37+
).when(
38+
exp.EQ(this=chain_id, expression=exp.Literal.number(480)),
39+
exp.Literal.string("worldchain"),
40+
).when(
41+
exp.EQ(this=chain_id, expression=exp.Literal.number(8008)),
42+
exp.Literal.string("polynomial"),
43+
).when(
44+
exp.EQ(this=chain_id, expression=exp.Literal.number(60808)),
45+
exp.Literal.string("bob"),
46+
).when(
47+
exp.EQ(this=chain_id, expression=exp.Literal.number(57073)),
48+
exp.Literal.string("ink"),
49+
).when(
50+
exp.EQ(this=chain_id, expression=exp.Literal.number(1135)),
51+
exp.Literal.string("lisk"),
52+
).when(
53+
exp.EQ(this=chain_id, expression=exp.Literal.number(1750)),
54+
exp.Literal.string("metal"),
55+
).when(
56+
exp.EQ(this=chain_id, expression=exp.Literal.number(185)),
57+
exp.Literal.string("mint"),
58+
).when(
59+
exp.EQ(this=chain_id, expression=exp.Literal.number(6805)),
60+
exp.Literal.string("race"),
61+
).when(
62+
exp.EQ(this=chain_id, expression=exp.Literal.number(360)),
63+
exp.Literal.string("shape"),
64+
).when(
65+
exp.EQ(this=chain_id, expression=exp.Literal.number(1868)),
66+
exp.Literal.string("soneium"),
67+
).when(
68+
exp.EQ(this=chain_id, expression=exp.Literal.number(1923)),
69+
exp.Literal.string("swell"),
70+
).when(
71+
exp.EQ(this=chain_id, expression=exp.Literal.number(7777777)),
72+
exp.Literal.string("zora"),
73+
)
74+
return exp.Upper(this=chain_name)

‎warehouse/metrics_mesh/macros/url_parts.py‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
from sqlglot import expressions as exp
2+
from sqlmesh import macro
3+
from sqlmesh.core.macros import MacroEvaluator
4+
5+
6+
@macro()
7+
def url_parts(
8+
evaluator: MacroEvaluator,
9+
url: exp.ExpOrStr,
10+
index: int
11+
) -> exp.Expression:
12+
"""
13+
This will parse a URI/URL, split on '/' and return the part
14+
- 1 is usually the domain
15+
- 2 is usually the first part of the path
16+
return the index-th part.
17+
"""
18+
19+
without_protocol = exp.SplitPart(
20+
this=url,
21+
delimiter=exp.Literal.string("://"),
22+
part_index=exp.Literal.number(2)
23+
)
24+
25+
part = exp.SplitPart(
26+
this=without_protocol,
27+
delimiter=exp.Literal.string("/"),
28+
part_index=exp.Literal.number(index)
29+
)
30+
31+
return part

‎warehouse/metrics_mesh/models/intermediate/blockchain_artifacts/int_contracts_by_project.sql‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@ with contracts_in_ossd as (
88
project_id,
99
artifact_source as chain,
1010
artifact_name as contract_address
11-
from metrics.int_artifacts_in_ossd_by_project
11+
from metrics.int_artifacts_by_project_in_ossd
1212
where artifact_type = 'CONTRACT'
1313
),
1414

‎warehouse/metrics_mesh/models/intermediate/code_artifacts/int_repositories.sql‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,6 @@ select
2222
repos.language,
2323
repos.created_at,
2424
repos.updated_at
25-
from metrics.int_artifacts_in_ossd_by_project as artifacts
25+
from metrics.int_artifacts_by_project_in_ossd as artifacts
2626
inner join metrics.stg_ossd__current_repositories as repos
2727
on artifacts.artifact_source_id = CAST(repos.id as STRING)

‎warehouse/metrics_mesh/models/intermediate/directory/int_all_artifacts.sql‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@ with ossd_artifacts as (
2222
artifact_namespace,
2323
artifact_name,
2424
artifact_url
25-
from metrics.int_artifacts_in_ossd_by_project
25+
from metrics.int_artifacts_by_project_in_ossd
2626
where artifact_type not in ('DEPLOYER', 'CONTRACT')
2727
),
2828

‎warehouse/metrics_mesh/models/intermediate/directory/int_artifacts_by_project_in_op_atlas.sql‎

Lines changed: 114 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,114 @@
1+
MODEL (
2+
name metrics.int_artifacts_by_project_in_op_atlas,
3+
kind FULL,
4+
dialect trino
5+
);
6+
7+
with all_websites as (
8+
select
9+
project_id,
10+
artifact_source_id,
11+
artifact_source,
12+
artifact_namespace,
13+
artifact_name,
14+
artifact_url,
15+
artifact_type
16+
from metrics.stg_op_atlas_project_website as sites
17+
),
18+
19+
all_farcaster as (
20+
select
21+
project_id,
22+
artifact_source_id,
23+
artifact_source,
24+
artifact_namespace,
25+
artifact_name,
26+
artifact_url,
27+
artifact_type
28+
from metrics.stg_op_atlas_project_farcaster as farcaster
29+
),
30+
31+
all_twitter as (
32+
select
33+
project_id,
34+
artifact_source_id,
35+
artifact_source,
36+
artifact_namespace,
37+
artifact_name,
38+
artifact_url,
39+
artifact_type
40+
from metrics.stg_op_atlas_project_twitter as twitter
41+
),
42+
43+
all_repository as (
44+
select
45+
project_id,
46+
artifact_source_id,
47+
artifact_source,
48+
artifact_namespace,
49+
artifact_name,
50+
artifact_url,
51+
artifact_type
52+
from metrics.stg_op_atlas_project_repository
53+
),
54+
55+
all_contracts as (
56+
select
57+
project_id,
58+
artifact_source_id,
59+
artifact_source,
60+
artifact_namespace,
61+
artifact_name,
62+
artifact_url,
63+
artifact_type
64+
from metrics.stg_op_atlas_project_contract
65+
),
66+
67+
all_defillama as (
68+
select
69+
project_id,
70+
artifact_source_id,
71+
artifact_source,
72+
artifact_namespace,
73+
artifact_name,
74+
artifact_url,
75+
artifact_type
76+
from metrics.stg_op_atlas_project_defillama
77+
),
78+
79+
all_artifacts as (
80+
select * from all_websites
81+
union all
82+
select * from all_farcaster
83+
union all
84+
select * from all_twitter
85+
union all
86+
select * from all_repository
87+
union all
88+
select * from all_contracts
89+
union all
90+
select * from all_defillama
91+
),
92+
93+
all_normalized_artifacts as (
94+
select distinct
95+
project_id,
96+
LOWER(artifact_source_id) as artifact_source_id,
97+
UPPER(artifact_source) as artifact_source,
98+
LOWER(artifact_namespace) as artifact_namespace,
99+
LOWER(artifact_name) as artifact_name,
100+
LOWER(artifact_url) as artifact_url,
101+
UPPER(artifact_type) as artifact_type
102+
from all_artifacts
103+
)
104+
105+
select
106+
project_id,
107+
@oso_id(artifact_source, artifact_source_id) as artifact_id,
108+
artifact_source_id,
109+
artifact_source,
110+
artifact_namespace,
111+
artifact_name,
112+
artifact_url,
113+
artifact_type
114+
from all_normalized_artifacts

warehouse/metrics_mesh/models/intermediate/directory/int_artifacts_in_ossd_by_project.sql renamed to warehouse/metrics_mesh/models/intermediate/directory/int_artifacts_by_project_in_ossd.sql

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
MODEL (
2-
name metrics.int_artifacts_in_ossd_by_project,
2+
name metrics.int_artifacts_by_project_in_ossd,
33
kind FULL,
44
dialect trino
55
);

‎warehouse/metrics_mesh/models/intermediate/directory/int_deployers_by_project.sql‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@ known_deployers as (
1414
project_id,
1515
artifact_source,
1616
artifact_name
17-
from metrics.int_artifacts_in_ossd_by_project
17+
from metrics.int_artifacts_by_project_in_ossd
1818
where artifact_type = 'DEPLOYER'
1919
),
2020

‎warehouse/metrics_mesh/models/intermediate/directory/int_projects.sql‎

Lines changed: 28 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -4,14 +4,31 @@ MODEL (
44
kind FULL
55
);
66

7-
select
8-
project_id,
9-
project_source,
10-
project_namespace,
11-
project_name,
12-
display_name,
13-
description,
14-
ARRAY_LENGTH(github) as github_artifact_count,
15-
ARRAY_LENGTH(blockchain) as blockchain_artifact_count,
16-
ARRAY_LENGTH(npm) as npm_artifact_count
17-
from metrics.stg_ossd__current_projects
7+
with ossd_projects as (
8+
select
9+
project_id,
10+
project_source,
11+
project_namespace,
12+
project_name,
13+
display_name,
14+
description,
15+
--ARRAY_LENGTH(github) as github_artifact_count,
16+
--ARRAY_LENGTH(blockchain) as blockchain_artifact_count,
17+
--ARRAY_LENGTH(npm) as npm_artifact_count
18+
from metrics.stg_ossd__current_projects
19+
),
20+
21+
op_atlas_projects as (
22+
select
23+
project_id,
24+
project_source,
25+
project_namespace,
26+
project_name,
27+
display_name,
28+
description
29+
from metrics.stg_op_atlas_project
30+
)
31+
32+
select * from ossd_projects
33+
union all
34+
select * from op_atlas_projects

‎warehouse/metrics_mesh/models/staging/op_atlas/stg_op_atlas_project.sql‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ MODEL (
66

77
select
88
@oso_id('OP_ATLAS', id)::TEXT as project_id,
9+
id::TEXT as project_source_id,
910
'OP_ATLAS' as project_source,
1011
null::TEXT as project_namespace,
1112
id::TEXT as project_name,

0 commit comments

Comments
 (0)