|
15 | 15 | from gql.transport.requests import RequestsHTTPTransport |
16 | 16 | from oso_dagster.config import DagsterConfig |
17 | 17 | from oso_dagster.factories import dlt_factory, pydantic_to_dlt_nullable_columns |
18 | | -from oso_dagster.factories.graphql import ( |
19 | | - GraphQLResourceConfig, |
20 | | - PaginationConfig, |
21 | | - PaginationType, |
22 | | - RetryConfig, |
23 | | - graphql_factory, |
24 | | -) |
25 | 18 | from oso_dagster.utils.common import ( |
26 | 19 | QueryArguments, |
27 | 20 | QueryConfig, |
28 | 21 | QueryRetriesExceeded, |
29 | 22 | query_with_retry, |
30 | | - stringify_large_integers, |
31 | 23 | ) |
32 | 24 | from oso_dagster.utils.secrets import secret_ref_arg |
33 | 25 | from pydantic import UUID4, BaseModel, Field |
@@ -731,73 +723,74 @@ def deposits( |
731 | 723 | yield resource |
732 | 724 |
|
733 | 725 |
|
734 | | -@dlt_factory( |
735 | | - key_prefix="open_collective", |
736 | | -) |
737 | | -def accounts( |
738 | | - context: AssetExecutionContext, |
739 | | - global_config: DagsterConfig, |
740 | | - personal_token: str = secret_ref_arg( |
741 | | - group_name="open_collective", key="personal_token" |
742 | | - ), |
743 | | -): |
744 | | - config = GraphQLResourceConfig( |
745 | | - name="accounts", |
746 | | - endpoint="https://api.opencollective.com/graphql/v2", |
747 | | - target_type="Query", |
748 | | - target_query="accounts", |
749 | | - transform_fn=lambda result: stringify_large_integers( |
750 | | - result["accounts"]["nodes"] |
751 | | - ), |
752 | | - headers={ |
753 | | - "Personal-Token": personal_token, |
754 | | - }, |
755 | | - exclude=[ |
756 | | - "nodes.activitySubscriptions", |
757 | | - "nodes.feed", |
758 | | - "nodes.parentAccount.activitySubscriptions", |
759 | | - "nodes.parentAccount.feed", |
760 | | - "nodes.stats.managedAmount", |
761 | | - "nodes.stats.activitySubscriptions", |
762 | | - "nodes.stats.contributionsAmountTimeSeries", |
763 | | - "nodes.stats.expensesTagsTimeSeries", |
764 | | - "nodes.stats.totalNetAmountReceivedTimeSeries", |
765 | | - ], |
766 | | - pagination=PaginationConfig( |
767 | | - type=PaginationType.OFFSET, |
768 | | - page_size=200, |
769 | | - offset_field="offset", |
770 | | - limit_field="limit", |
771 | | - total_count_path="totalCount", |
772 | | - rate_limit_seconds=1.0, |
773 | | - ), |
774 | | - max_depth=3, |
775 | | - retry=RetryConfig( |
776 | | - max_retries=10, |
777 | | - initial_delay=1.0, |
778 | | - max_delay=5.0, |
779 | | - backoff_multiplier=1.5, |
780 | | - jitter=True, |
781 | | - reduce_page_size=True, |
782 | | - min_page_size=10, |
783 | | - page_size_reduction_factor=0.6, |
784 | | - continue_on_failure=True, |
785 | | - ), |
786 | | - ) |
787 | | - |
788 | | - resource = graphql_factory( |
789 | | - config, |
790 | | - global_config, |
791 | | - context, |
792 | | - max_table_nesting=0, |
793 | | - write_disposition="merge", |
794 | | - merge_key="id", |
795 | | - ) |
796 | | - |
797 | | - if global_config.gcp_bigquery_enabled: |
798 | | - bigquery_adapter( |
799 | | - resource, |
800 | | - partition="created_at", |
801 | | - ) |
802 | | - |
803 | | - yield resource |
| 726 | +# TODO(jabolo): Enable back on #6062 |
| 727 | +# @dlt_factory( |
| 728 | +# key_prefix="open_collective", |
| 729 | +# ) |
| 730 | +# def accounts( |
| 731 | +# context: AssetExecutionContext, |
| 732 | +# global_config: DagsterConfig, |
| 733 | +# personal_token: str = secret_ref_arg( |
| 734 | +# group_name="open_collective", key="personal_token" |
| 735 | +# ), |
| 736 | +# ): |
| 737 | +# config = GraphQLResourceConfig( |
| 738 | +# name="accounts", |
| 739 | +# endpoint="https://api.opencollective.com/graphql/v2", |
| 740 | +# target_type="Query", |
| 741 | +# target_query="accounts", |
| 742 | +# transform_fn=lambda result: stringify_large_integers( |
| 743 | +# result["accounts"]["nodes"] |
| 744 | +# ), |
| 745 | +# headers={ |
| 746 | +# "Personal-Token": personal_token, |
| 747 | +# }, |
| 748 | +# exclude=[ |
| 749 | +# "nodes.activitySubscriptions", |
| 750 | +# "nodes.feed", |
| 751 | +# "nodes.parentAccount.activitySubscriptions", |
| 752 | +# "nodes.parentAccount.feed", |
| 753 | +# "nodes.stats.managedAmount", |
| 754 | +# "nodes.stats.activitySubscriptions", |
| 755 | +# "nodes.stats.contributionsAmountTimeSeries", |
| 756 | +# "nodes.stats.expensesTagsTimeSeries", |
| 757 | +# "nodes.stats.totalNetAmountReceivedTimeSeries", |
| 758 | +# ], |
| 759 | +# pagination=PaginationConfig( |
| 760 | +# type=PaginationType.OFFSET, |
| 761 | +# page_size=200, |
| 762 | +# offset_field="offset", |
| 763 | +# limit_field="limit", |
| 764 | +# total_count_path="totalCount", |
| 765 | +# rate_limit_seconds=1.0, |
| 766 | +# ), |
| 767 | +# max_depth=3, |
| 768 | +# retry=RetryConfig( |
| 769 | +# max_retries=10, |
| 770 | +# initial_delay=1.0, |
| 771 | +# max_delay=5.0, |
| 772 | +# backoff_multiplier=1.5, |
| 773 | +# jitter=True, |
| 774 | +# reduce_page_size=True, |
| 775 | +# min_page_size=10, |
| 776 | +# page_size_reduction_factor=0.6, |
| 777 | +# continue_on_failure=True, |
| 778 | +# ), |
| 779 | +# ) |
| 780 | + |
| 781 | +# resource = graphql_factory( |
| 782 | +# config, |
| 783 | +# global_config, |
| 784 | +# context, |
| 785 | +# max_table_nesting=0, |
| 786 | +# write_disposition="merge", |
| 787 | +# merge_key="id", |
| 788 | +# ) |
| 789 | + |
| 790 | +# if global_config.gcp_bigquery_enabled: |
| 791 | +# bigquery_adapter( |
| 792 | +# resource, |
| 793 | +# partition="created_at", |
| 794 | +# ) |
| 795 | + |
| 796 | +# yield resource |
0 commit comments