Skip to content

Commit b17e6eb

Browse files
kenyiukenyiu
authored
feat(ens): add GraphQL queries for text changes and domains for ENS (#4830)
* feat(ens): add GraphQL queries for text changes and domains for ENS * fix: update resolver key in parameters for text_changeds_for_domain function * fix: update endpoint to use a constant and improve security comments * fix: enhance logging for GraphQL field expansion and endpoint retrieval * fix: refactor text_changeds function and improve pagination configuration * feat: add model and tests for ENS domains with GitHub handles * fix: refactor SQL model for ENS domains with GitHub handles and remove obsolete tests * fix: remove commented-out pagination limits in text_changeds and domains functions * feat: enhance get_endpoint function with masked option and update GraphQLResourceConfig for masked endpoint logging * feat: update get_endpoint to accept API key and enhance error message sanitization in GraphQL functions * fix: rename ENS_API_KEY to DAGSTER__ENS__API_KEY in .env.example for consistency * feat: add ENSDomains and ENSTextChangeds schemas with seed configurations * doc: add masked_endpoint to GraphQLResourceConfig doc --------- Co-authored-by: ken <[email protected]>
1 parent 6c695f6 commit b17e6eb

File tree

7 files changed

+302
-17
lines changed

7 files changed

+302
-17
lines changed

‎.env.example‎

Lines changed: 7 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -2,8 +2,8 @@
22
## This .env file is mostly used for Python data ops
33

44
## Dagster Setup
5-
# You may want to change the location of dagster home if you want it to survive resets
6-
DAGSTER_HOME=/tmp/dagster-home
5+
# You may want to change the location of dagster home if you want it to survive resets
6+
DAGSTER_HOME=/tmp/dagster-home
77

88
## sqlmesh
99
SQLMESH_DUCKDB_LOCAL_PATH=/tmp/oso.duckdb
@@ -14,7 +14,7 @@ SQLMESH_DUCKDB_LOCAL_PATH=/tmp/oso.duckdb
1414
# the official oso gcp account uncommenting these will likely not work. The GCP
1515
# secrets prefix should likely match the dagster deployment's search prefix in flux
1616
DAGSTER_USE_LOCAL_SECRETS=True
17-
#DAGSTER_GCP_SECRETS_PREFIX=dagster
17+
#DAGSTER_GCP_SECRETS_PREFIX=dagster
1818

1919
# OSO's python libraries are configured to use json logging by default but this
2020
# can be annoying when viewing things locally. This will configure logs to be
@@ -88,4 +88,7 @@ AGENT_VECTOR_STORE__DIR=
8888
#AGENT_VECTOR_STORE__GCS_BUCKET=
8989
#AGENT_VECTOR_STORE__PROJECT_ID=
9090
#AGENT_VECTOR_STORE__INDEX_ID=
91-
#AGENT_VECTOR_STORE__ENDPOINT_ID=
91+
#AGENT_VECTOR_STORE__ENDPOINT_ID=
92+
93+
# ENS
94+
DAGSTER__ENS__API_KEY=

‎apps/docs/docs/contribute-data/graphql-api.md‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,7 @@ from ..config import DagsterConfig
4141
config = GraphQLResourceConfig(
4242
name="transactions",
4343
endpoint="https://api.opencollective.com/graphql/v2",
44+
masked_endpoint="***",
4445
target_type="Query",
4546
target_query="transactions",
4647
max_depth=2, # Limit the introspection depth
@@ -91,6 +92,7 @@ In this configuration, we define the following fields:
9192

9293
- **name**: A unique identifier for the dagster asset.
9394
- **endpoint**: The URL of the GraphQL API.
95+
- **masked_endpoint**: The masked URL of the GraphQL API. If exists, it will be used for logging instead of the real endpoint.
9496
- **target_type**: The GraphQL type containing the target query (usually
9597
"Query").
9698
- **target_query**: The name of the query to execute.
@@ -121,6 +123,7 @@ def fetch_account_details(context: AssetExecutionContext, global_config: Dagster
121123
account_config = GraphQLResourceConfig(
122124
name=f"account_{transaction_data['fromAccount']['id']}",
123125
endpoint="https://api.opencollective.com/graphql/v2",
126+
masked_endpoint="***",
124127
target_type="Query",
125128
target_query="account",
126129
parameters={
@@ -147,6 +150,7 @@ def transactions_with_accounts(context: AssetExecutionContext, global_config: Da
147150
config = GraphQLResourceConfig(
148151
name="transactions_with_accounts",
149152
endpoint="https://api.opencollective.com/graphql/v2",
153+
masked_endpoint="***",
150154
target_type="Query",
151155
target_query="transactions",
152156
parameters={

‎warehouse/metrics_tools/seed/schemas/ens/domains.py‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
from metrics_tools.seed.types import Column, SeedConfig
2+
from pydantic import BaseModel
3+
4+
5+
class ENSDomains(BaseModel):
6+
"""ENS domains"""
7+
8+
id: str = Column("VARCHAR", description="Domain ID")
9+
name: str = Column("VARCHAR", description="Domain name")
10+
resolver: str = Column("VARCHAR", description="Resolver JSON")
11+
12+
13+
seed = SeedConfig(
14+
catalog="bigquery",
15+
schema="ens",
16+
table="domains",
17+
base=ENSDomains,
18+
rows=[
19+
ENSDomains(
20+
id="0x0000000000000000000000000000000000000001",
21+
name="test.eth",
22+
resolver='{"id": "0xresolver1"}',
23+
),
24+
ENSDomains(
25+
id="0x0000000000000000000000000000000000000002",
26+
name="another.eth",
27+
resolver='{"id": "0xresolver2"}',
28+
),
29+
],
30+
)

‎warehouse/metrics_tools/seed/schemas/ens/text_changeds.py‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
from metrics_tools.seed.types import Column, SeedConfig
2+
from pydantic import BaseModel
3+
4+
5+
class ENSTextChangeds(BaseModel):
6+
"""ENS text_changeds"""
7+
8+
resolver: str = Column("VARCHAR", description="Resolver JSON")
9+
key: str = Column("VARCHAR", description="Key")
10+
value: str = Column("VARCHAR", description="Value")
11+
12+
13+
seed = SeedConfig(
14+
catalog="bigquery",
15+
schema="ens",
16+
table="text_changeds",
17+
base=ENSTextChangeds,
18+
rows=[
19+
ENSTextChangeds(
20+
resolver='{"id": "0xresolver1"}',
21+
key="com.github",
22+
value="testuser",
23+
),
24+
ENSTextChangeds(
25+
resolver='{"id": "0xresolver2"}',
26+
key="com.github",
27+
value="anotheruser",
28+
),
29+
ENSTextChangeds(
30+
resolver='{"id": "0xresolver1"}',
31+
key="com.twitter",
32+
value="testuser",
33+
),
34+
],
35+
)

‎warehouse/oso_dagster/assets/default/ens.py‎

Lines changed: 129 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,129 @@
1+
from typing import Optional
2+
3+
from dagster import AssetExecutionContext
4+
from oso_dagster.config import DagsterConfig
5+
from oso_dagster.factories.dlt import dlt_factory
6+
from oso_dagster.factories.graphql import (
7+
GraphQLResourceConfig,
8+
PaginationConfig,
9+
PaginationType,
10+
RetryConfig,
11+
graphql_factory,
12+
)
13+
from oso_dagster.utils.secrets import secret_ref_arg
14+
15+
16+
def get_endpoint(ens_api_key: Optional[str] = None, masked: bool = False) -> str:
17+
if masked:
18+
return "https://gateway.thegraph.com/api/***/subgraphs/id/5XqPmWe6gjyrJtFn9cLy237i4cWw2j9HcUJEXsP5qGtH"
19+
20+
return f"https://gateway.thegraph.com/api/{ens_api_key}/subgraphs/id/5XqPmWe6gjyrJtFn9cLy237i4cWw2j9HcUJEXsP5qGtH"
21+
22+
23+
@dlt_factory(
24+
key_prefix="ens",
25+
)
26+
def text_changeds(
27+
context: AssetExecutionContext,
28+
global_config: DagsterConfig,
29+
ens_api_key: str = secret_ref_arg(group_name="ens", key="api_key"),
30+
):
31+
config = GraphQLResourceConfig(
32+
name="text_changeds",
33+
endpoint=get_endpoint(ens_api_key=ens_api_key),
34+
masked_endpoint=get_endpoint(masked=True),
35+
target_type="Query",
36+
target_query="textChangeds",
37+
max_depth=1,
38+
exclude=[
39+
# "id",
40+
"resolver",
41+
# "blockNumber",
42+
# "transactionID",
43+
# "key",
44+
# "value",
45+
],
46+
transform_fn=lambda result: result["textChangeds"],
47+
pagination=PaginationConfig(
48+
type=PaginationType.OFFSET,
49+
page_size=500,
50+
offset_field="skip",
51+
limit_field="first",
52+
rate_limit_seconds=2.0,
53+
),
54+
retry=RetryConfig(
55+
max_retries=10,
56+
initial_delay=1.0,
57+
max_delay=5.0,
58+
backoff_multiplier=1.5,
59+
jitter=True,
60+
reduce_page_size=True,
61+
min_page_size=1,
62+
page_size_reduction_factor=0.6,
63+
),
64+
)
65+
66+
yield graphql_factory(
67+
config, global_config, context, max_table_nesting=0, write_disposition="replace"
68+
)
69+
70+
71+
@dlt_factory(
72+
key_prefix="ens",
73+
)
74+
def domains(
75+
context: AssetExecutionContext,
76+
global_config: DagsterConfig,
77+
ens_api_key: str = secret_ref_arg(group_name="ens", key="api_key"),
78+
):
79+
config = GraphQLResourceConfig(
80+
name="domains",
81+
endpoint=get_endpoint(ens_api_key=ens_api_key),
82+
masked_endpoint=get_endpoint(masked=True),
83+
target_type="Query",
84+
target_query="domains",
85+
max_depth=2,
86+
transform_fn=lambda result: result["domains"],
87+
pagination=PaginationConfig(
88+
type=PaginationType.OFFSET,
89+
page_size=200,
90+
offset_field="skip",
91+
limit_field="first",
92+
rate_limit_seconds=2.0,
93+
),
94+
exclude=[
95+
# "id",
96+
# "name",
97+
"labelhash",
98+
"labelName",
99+
"parent",
100+
# "subdomains",
101+
# "subdomainCount",
102+
"resolvedAddress",
103+
# "resolver",
104+
"ttl",
105+
"isMigrated",
106+
"createdAt",
107+
# "owner",
108+
# "registrant",
109+
"wrappedOwner",
110+
# "expiryDate",
111+
# "registration",
112+
"wrappedDomain",
113+
"events",
114+
],
115+
retry=RetryConfig(
116+
max_retries=10,
117+
initial_delay=1.0,
118+
max_delay=5.0,
119+
backoff_multiplier=1.5,
120+
jitter=True,
121+
reduce_page_size=True,
122+
min_page_size=1,
123+
page_size_reduction_factor=0.6,
124+
),
125+
)
126+
127+
yield graphql_factory(
128+
config, global_config, context, max_table_nesting=0, write_disposition="replace"
129+
)

0 commit comments

Comments
 (0)