Skip to content

Commit 92b1cea

Browse files
authored
chore: disable open_collective/accounts asset temporarily (#6080)
1 parent c9e8e6c commit 92b1cea

File tree

1 file changed

+71
-78
lines changed

1 file changed

+71
-78
lines changed

warehouse/oso_dagster/oso_dagster/assets/default/open_collective.py

Lines changed: 71 additions & 78 deletions
Original file line numberDiff line numberDiff line change
@@ -15,19 +15,11 @@
1515
from gql.transport.requests import RequestsHTTPTransport
1616
from oso_dagster.config import DagsterConfig
1717
from oso_dagster.factories import dlt_factory, pydantic_to_dlt_nullable_columns
18-
from oso_dagster.factories.graphql import (
19-
GraphQLResourceConfig,
20-
PaginationConfig,
21-
PaginationType,
22-
RetryConfig,
23-
graphql_factory,
24-
)
2518
from oso_dagster.utils.common import (
2619
QueryArguments,
2720
QueryConfig,
2821
QueryRetriesExceeded,
2922
query_with_retry,
30-
stringify_large_integers,
3123
)
3224
from oso_dagster.utils.secrets import secret_ref_arg
3325
from pydantic import UUID4, BaseModel, Field
@@ -731,73 +723,74 @@ def deposits(
731723
yield resource
732724

733725

734-
@dlt_factory(
735-
key_prefix="open_collective",
736-
)
737-
def accounts(
738-
context: AssetExecutionContext,
739-
global_config: DagsterConfig,
740-
personal_token: str = secret_ref_arg(
741-
group_name="open_collective", key="personal_token"
742-
),
743-
):
744-
config = GraphQLResourceConfig(
745-
name="accounts",
746-
endpoint="https://api.opencollective.com/graphql/v2",
747-
target_type="Query",
748-
target_query="accounts",
749-
transform_fn=lambda result: stringify_large_integers(
750-
result["accounts"]["nodes"]
751-
),
752-
headers={
753-
"Personal-Token": personal_token,
754-
},
755-
exclude=[
756-
"nodes.activitySubscriptions",
757-
"nodes.feed",
758-
"nodes.parentAccount.activitySubscriptions",
759-
"nodes.parentAccount.feed",
760-
"nodes.stats.managedAmount",
761-
"nodes.stats.activitySubscriptions",
762-
"nodes.stats.contributionsAmountTimeSeries",
763-
"nodes.stats.expensesTagsTimeSeries",
764-
"nodes.stats.totalNetAmountReceivedTimeSeries",
765-
],
766-
pagination=PaginationConfig(
767-
type=PaginationType.OFFSET,
768-
page_size=200,
769-
offset_field="offset",
770-
limit_field="limit",
771-
total_count_path="totalCount",
772-
rate_limit_seconds=1.0,
773-
),
774-
max_depth=3,
775-
retry=RetryConfig(
776-
max_retries=10,
777-
initial_delay=1.0,
778-
max_delay=5.0,
779-
backoff_multiplier=1.5,
780-
jitter=True,
781-
reduce_page_size=True,
782-
min_page_size=10,
783-
page_size_reduction_factor=0.6,
784-
continue_on_failure=True,
785-
),
786-
)
787-
788-
resource = graphql_factory(
789-
config,
790-
global_config,
791-
context,
792-
max_table_nesting=0,
793-
write_disposition="merge",
794-
merge_key="id",
795-
)
796-
797-
if global_config.gcp_bigquery_enabled:
798-
bigquery_adapter(
799-
resource,
800-
partition="created_at",
801-
)
802-
803-
yield resource
726+
# TODO(jabolo): Enable back on #6062
727+
# @dlt_factory(
728+
# key_prefix="open_collective",
729+
# )
730+
# def accounts(
731+
# context: AssetExecutionContext,
732+
# global_config: DagsterConfig,
733+
# personal_token: str = secret_ref_arg(
734+
# group_name="open_collective", key="personal_token"
735+
# ),
736+
# ):
737+
# config = GraphQLResourceConfig(
738+
# name="accounts",
739+
# endpoint="https://api.opencollective.com/graphql/v2",
740+
# target_type="Query",
741+
# target_query="accounts",
742+
# transform_fn=lambda result: stringify_large_integers(
743+
# result["accounts"]["nodes"]
744+
# ),
745+
# headers={
746+
# "Personal-Token": personal_token,
747+
# },
748+
# exclude=[
749+
# "nodes.activitySubscriptions",
750+
# "nodes.feed",
751+
# "nodes.parentAccount.activitySubscriptions",
752+
# "nodes.parentAccount.feed",
753+
# "nodes.stats.managedAmount",
754+
# "nodes.stats.activitySubscriptions",
755+
# "nodes.stats.contributionsAmountTimeSeries",
756+
# "nodes.stats.expensesTagsTimeSeries",
757+
# "nodes.stats.totalNetAmountReceivedTimeSeries",
758+
# ],
759+
# pagination=PaginationConfig(
760+
# type=PaginationType.OFFSET,
761+
# page_size=200,
762+
# offset_field="offset",
763+
# limit_field="limit",
764+
# total_count_path="totalCount",
765+
# rate_limit_seconds=1.0,
766+
# ),
767+
# max_depth=3,
768+
# retry=RetryConfig(
769+
# max_retries=10,
770+
# initial_delay=1.0,
771+
# max_delay=5.0,
772+
# backoff_multiplier=1.5,
773+
# jitter=True,
774+
# reduce_page_size=True,
775+
# min_page_size=10,
776+
# page_size_reduction_factor=0.6,
777+
# continue_on_failure=True,
778+
# ),
779+
# )
780+
781+
# resource = graphql_factory(
782+
# config,
783+
# global_config,
784+
# context,
785+
# max_table_nesting=0,
786+
# write_disposition="merge",
787+
# merge_key="id",
788+
# )
789+
790+
# if global_config.gcp_bigquery_enabled:
791+
# bigquery_adapter(
792+
# resource,
793+
# partition="created_at",
794+
# )
795+
796+
# yield resource

0 commit comments

Comments
 (0)