Skip to content

Commit 789cbca

Browse files
authored
Merge branch 'main' into executor-fixes
2 parents 1ab5102 + 6794987 commit 789cbca

30 files changed

+1625
-616
lines changed

.github/workflows/auto_merge_notification.yml

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@ jobs:
1414
steps:
1515
- name: Add Auto-Merge Notice
1616
if: github.event.action == 'auto_merge_enabled'
17-
uses: bcgov/action-pr-description-add@main
17+
uses: bcgov/action-pr-description-add@v2.0.0
1818
with:
1919
add_markdown: |
2020
> [!IMPORTANT]
@@ -24,6 +24,8 @@ jobs:
2424
2525
- name: Remove Auto-Merge Notice
2626
if: github.event.action == 'auto_merge_disabled'
27-
uses: bcgov/action-pr-description-add@main
27+
uses: bcgov/action-pr-description-add@v2.0.0
2828
with:
29-
add_markdown: "" # Empty string to remove the notice
29+
add_markdown: |
30+
> [!NOTE]
31+
> **Auto-merge may have been disabled. Please check the PR status to confirm.**

.github/workflows/python_lint.yml

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -43,7 +43,7 @@ jobs:
4343
- name: Set up Poetry
4444
uses: Gr1N/setup-poetry@v9
4545
with:
46-
poetry-version: "1.7.1"
46+
poetry-version: "2.0.1"
4747
- name: Set up Python
4848
uses: actions/setup-python@v5
4949
with:
@@ -66,7 +66,7 @@ jobs:
6666
- name: Set up Poetry
6767
uses: Gr1N/setup-poetry@v9
6868
with:
69-
poetry-version: "1.7.1"
69+
poetry-version: "2.0.1"
7070
- name: Set up Python
7171
uses: actions/setup-python@v5
7272
with:

.github/workflows/python_pytest.yml

Lines changed: 23 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -72,6 +72,16 @@ jobs:
7272
name: pytest-fast-test-logs
7373
path: /tmp/airbyte/logs/
7474

75+
- name: Publish Test Results
76+
uses: EnricoMi/publish-unit-test-result-action@v2
77+
if: always() && !cancelled()
78+
continue-on-error: true
79+
with:
80+
check_name: "PyTest Results (Fast Tests Only, No Creds)"
81+
large_files: true
82+
files: |
83+
build/test-results/**/*.xml
84+
7585
pytest-no-creds:
7686
name: Pytest (No Creds)
7787
runs-on: ubuntu-latest
@@ -82,7 +92,7 @@ jobs:
8292
- name: Set up Poetry
8393
uses: Gr1N/setup-poetry@v9
8494
with:
85-
poetry-version: "1.7.1"
95+
poetry-version: "2.0.1"
8696
- name: Set up Python
8797
uses: actions/setup-python@v5
8898
with:
@@ -157,7 +167,7 @@ jobs:
157167
- name: Set up Poetry
158168
uses: Gr1N/setup-poetry@v9
159169
with:
160-
poetry-version: "1.7.1"
170+
poetry-version: "2.0.1"
161171
- name: Set up Python
162172
uses: actions/setup-python@v5
163173
with:
@@ -200,6 +210,16 @@ jobs:
200210
name: py${{ matrix.python-version }}-${{ matrix.os }}-test-logs
201211
path: /tmp/airbyte/logs/
202212

213+
- name: Publish Test Results
214+
uses: EnricoMi/publish-unit-test-result-action@v2
215+
if: always() && !cancelled() && matrix.python-version == '3.10' && matrix.os == 'Ubuntu'
216+
continue-on-error: true
217+
with:
218+
check_name: "PyTest Results (Full)"
219+
large_files: true
220+
files: |
221+
build/test-results/**/*.xml
222+
203223
dependency-analysis:
204224
name: Dependency Analysis with Deptry
205225
runs-on: ubuntu-latest
@@ -213,7 +233,7 @@ jobs:
213233
- name: Set up Poetry
214234
uses: Gr1N/setup-poetry@v9
215235
with:
216-
poetry-version: "1.7.1"
236+
poetry-version: "2.0.1"
217237
- name: Install dependencies
218238
run: poetry install
219239

.github/workflows/semantic_pr_check.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -50,7 +50,7 @@ jobs:
5050

5151
- name: Check for "do not merge" in PR title
5252
if: ${{ github.event.pull_request.draft == false }}
53-
uses: actions/github-script@v6
53+
uses: actions/github-script@v7
5454
with:
5555
script: |
5656
const title = context.payload.pull_request.title.toLowerCase();

.github/workflows/test-pr-command.yml

Lines changed: 75 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -15,9 +15,17 @@ on:
1515
env:
1616
AIRBYTE_ANALYTICS_ID: ${{ vars.AIRBYTE_ANALYTICS_ID }}
1717

18+
permissions:
19+
contents: read
20+
pull-requests: write
21+
issues: write
22+
checks: write
23+
actions: write
24+
statuses: write
25+
1826
jobs:
1927
start-workflow:
20-
name: Append 'Starting' Comment
28+
name: Start Workflow
2129
runs-on: ubuntu-latest
2230
steps:
2331

@@ -36,6 +44,20 @@ jobs:
3644
3745
[1]: ${{ steps.vars.outputs.run-url }}
3846
47+
- name: Get PR Info
48+
id: pr-info
49+
uses: cloudposse-github-actions/get-pr@v2.0.0
50+
with:
51+
id: ${{ inputs.pr }}
52+
53+
outputs:
54+
source-repo: ${{ fromJSON(steps.pr-info.outputs.json).head.repo.full_name }}
55+
source-branch: ${{ fromJSON(steps.pr-info.outputs.json).head.ref }}
56+
commit-sha: ${{ fromJSON(steps.pr-info.outputs.json).head.sha }}
57+
pr-number: ${{ steps.pr-info.outputs.number }}
58+
pr-title: ${{ steps.pr-info.outputs.title }}
59+
job-run-url: ${{ steps.vars.outputs.run-url }}
60+
3961
# This is copied from the `python_pytest.yml` file.
4062
# Only the first two steps of the job are different, and they check out the PR's branch.
4163
pytest-on-demand:
@@ -59,16 +81,35 @@ jobs:
5981
PYTHONIOENCODING: utf-8
6082
steps:
6183

62-
# Custom steps to fetch the PR and checkout the code:
84+
- name: Authenticate as GitHub App
85+
uses: actions/create-github-app-token@v2
86+
id: get-app-token
87+
with:
88+
owner: "airbytehq"
89+
repositories: "PyAirbyte"
90+
app-id: ${{ secrets.OCTAVIA_BOT_APP_ID }}
91+
private-key: ${{ secrets.OCTAVIA_BOT_PRIVATE_KEY }}
6392

6493
- name: Checkout PR
6594
uses: actions/checkout@v4
6695
with:
6796
token: ${{ secrets.GITHUB_TOKEN }}
68-
- name: Checkout PR (${{ github.event.inputs.pr }})
69-
uses: dawidd6/action-checkout-pr@v1
97+
ref: ${{ needs.start-workflow.outputs.commit-sha }}
98+
99+
# Post "In Progress" status to the PR.
100+
# This is required because otherwise slash commands won't automatically
101+
# update the PR checks status.
102+
- name: Create "Pending" Check Status
103+
if: always() && !cancelled()
104+
uses: LouisBrunner/checks-action@v2.0.0
70105
with:
71-
pr: ${{ github.event.inputs.pr }}
106+
name: "Pytest (All, Python ${{ matrix.python-version }}, ${{ matrix.os }})"
107+
repo: "airbytehq/PyAirbyte" # Post to the main repo, not the fork
108+
sha: ${{ needs.start-workflow.outputs.commit-sha }}
109+
status: in_progress
110+
action_url: ${{ needs.start-workflow.outputs.job-run-url }}
111+
details_url: ${{ needs.start-workflow.outputs.job-run-url }}
112+
token: ${{ steps.get-app-token.outputs.token }}
72113

73114
# Same as the `python_pytest.yml` file:
74115

@@ -88,10 +129,38 @@ jobs:
88129
timeout-minutes: 60
89130
env:
90131
GCP_GSM_CREDENTIALS: ${{ secrets.GCP_GSM_CREDENTIALS }}
132+
# We exclude super-slow and flaky tests in order to expedite PR reviews
133+
# and to reduce chance of unrelated noise in the review process.
91134
run: >
92135
poetry run pytest
93136
--verbose
94-
-m "not super_slow"
137+
-m "not super_slow and not flaky"
138+
139+
- name: Publish Test Results
140+
uses: EnricoMi/publish-unit-test-result-action@v2
141+
if: always() && !cancelled()
142+
continue-on-error: true
143+
with:
144+
check_name: "PyTest Results (On-Demand)"
145+
large_files: true
146+
files: |
147+
build/test-results/**/*.xml
148+
149+
# Post completion status to the PR.
150+
# This is required because otherwise slash commands won't automatically
151+
# update the PR checks status.
152+
- name: Create Final Check Status
153+
if: always()
154+
uses: LouisBrunner/checks-action@v2.0.0
155+
with:
156+
name: "Pytest (All, Python ${{ matrix.python-version }}, ${{ matrix.os }})"
157+
repo: "airbytehq/PyAirbyte" # Post to the main repo, not the fork
158+
sha: ${{ needs.start-workflow.outputs.commit-sha }}
159+
status: completed
160+
conclusion: ${{ job.status }}
161+
action_url: ${{ needs.start-workflow.outputs.job-run-url }}
162+
details_url: ${{ needs.start-workflow.outputs.job-run-url }}
163+
token: ${{ steps.get-app-token.outputs.token }}
95164

96165
log-success-comment:
97166
name: Append 'Success' Comment

.ruff.toml

Lines changed: 151 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,151 @@
1+
# Ruff configuration file
2+
# For rules reference, see https://docs.astral.sh/ruff/rules/
3+
target-version = "py310"
4+
preview = true
5+
line-length = 100
6+
7+
select = [
8+
# For rules reference, see https://docs.astral.sh/ruff/rules/
9+
"A", # flake8-builtins
10+
"ANN", # flake8-annotations
11+
"ARG", # flake8-unused-arguments
12+
"ASYNC", # flake8-async
13+
"B", # flake8-bugbear
14+
"FBT", # flake8-boolean-trap
15+
"BLE", # Blind except
16+
"C4", # flake8-comprehensions
17+
"C90", # mccabe (complexity)
18+
"COM", # flake8-commas
19+
"CPY", # missing copyright notice
20+
"D", # pydocstyle (Docstring conventions)
21+
"DTZ", # flake8-datetimez
22+
"E", # pycodestyle (errors)
23+
"ERA", # flake8-eradicate (commented out code)
24+
"EXE", # flake8-executable
25+
"F", # Pyflakes
26+
"FA", # flake8-future-annotations
27+
"FIX", # flake8-fixme
28+
"FLY", # flynt
29+
"FURB", # Refurb
30+
"I", # isort
31+
"ICN", # flake8-import-conventions
32+
"INP", # flake8-no-pep420
33+
"INT", # flake8-gettext
34+
"ISC", # flake8-implicit-str-concat
35+
"ICN", # flake8-import-conventions
36+
"LOG", # flake8-logging
37+
"N", # pep8-naming
38+
"PD", # pandas-vet
39+
"PERF", # Perflint
40+
"PIE", # flake8-pie
41+
"PGH", # pygrep-hooks
42+
"PL", # Pylint
43+
"PT", # flake8-pytest-style
44+
"PTH", # flake8-use-pathlib
45+
"PYI", # flake8-pyi
46+
"Q", # flake8-quotes
47+
"RET", # flake8-return
48+
"RSE", # flake8-raise
49+
"RUF", # Ruff-specific rules
50+
"SIM", # flake8-simplify
51+
"SLF", # flake8-self
52+
"SLOT", # flake8-slots
53+
"T10", # debugger calls
54+
# "T20", # flake8-print # TODO: Re-enable once we have logging
55+
"TCH", # flake8-type-checking
56+
"TD", # flake8-todos
57+
"TID", # flake8-tidy-imports
58+
"TRY", # tryceratops
59+
"TRY002", # Disallow raising vanilla Exception. Create or use a custom exception instead.
60+
"UP", # pyupgrade
61+
"W", # pycodestyle (warnings)
62+
"YTT", # flake8-2020
63+
]
64+
65+
[lint]
66+
ignore = [
67+
# For rules reference, see https://docs.astral.sh/ruff/rules/
68+
69+
# These we don't agree with or don't want to prioritize to enforce:
70+
"ANN003", # kwargs missing type annotations
71+
"COM812", # Because it conflicts with ruff auto-format
72+
"EM", # flake8-errmsgs (may reconsider later)
73+
"DJ", # Django linting
74+
"G", # flake8-logging-format
75+
"ISC001", # Conflicts with ruff auto-format
76+
"NPY", # NumPy-specific rules
77+
"PIE790", # Allow unnecssary 'pass' (sometimes useful for readability)
78+
"PERF203", # exception handling in loop
79+
"PLR6301", # Allow class methods that don't use 'self' (otherwise noisy)
80+
"RUF022", # Allow unsorted __all__ (sometimes useful for grouping by type with pdoc)
81+
"S", # flake8-bandit (noisy, security related)
82+
"SIM910", # Allow "None" as second argument to Dict.get(). "Explicit is better than implicit."
83+
"TD002", # Require author for TODOs
84+
"ASYNC1", # flake8-trio (opinionated, noisy)
85+
"INP001", # Dir 'examples' is part of an implicit namespace package. Add an __init__.py.
86+
87+
# TODO: Consider re-enabling these before release:
88+
"A003", # Class attribute 'type' is shadowing a Python builtin
89+
"BLE001", # Do not catch blind exception: Exception
90+
"ERA001", # Remove commented-out code
91+
"FIX002", # Allow "TODO:" until release (then switch to requiring links via TDO003)
92+
"PLW0603", # Using the global statement to update _cache is discouraged
93+
"PLW0108", # Lambda may be unnecessary; consider inlining inner function
94+
"TRY003", # Allow exceptions to receive strings in constructors.
95+
# "TD003", # Require links for TODOs (now enabled)
96+
"UP038", # Allow tuples instead of "|" syntax in `isinstance()` checks ("|" is sometimes slower)
97+
]
98+
fixable = ["ALL"]
99+
unfixable = [
100+
"ERA001", # Commented-out code (avoid silent loss of code)
101+
"T201", # print() calls (avoid silent loss of code / log messages)
102+
]
103+
dummy-variable-rgx = "^(_+|(_+[a-zA-Z0-9_]*[a-zA-Z0-9]+?))$"
104+
105+
[lint.pylint]
106+
max-args = 8 # Relaxed from default of 5
107+
max-branches = 15 # Relaxed from default of 12
108+
109+
[lint.flake8-import-conventions.aliases]
110+
airbyte = "ab"
111+
"airbyte.exceptions" = "exc"
112+
113+
[lint.isort]
114+
force-sort-within-sections = false
115+
lines-after-imports = 2
116+
known-first-party = [
117+
"airbyte_cdk",
118+
"airbyte_protocol",
119+
"airbyte_protocol_dataclasses",
120+
]
121+
known-local-folder = ["airbyte"]
122+
required-imports = ["from __future__ import annotations"]
123+
known-third-party = []
124+
section-order = [
125+
"future",
126+
"standard-library",
127+
"third-party",
128+
"first-party",
129+
"local-folder",
130+
]
131+
132+
[lint.mccabe]
133+
max-complexity = 24
134+
135+
[lint.pycodestyle]
136+
ignore-overlong-task-comments = true
137+
138+
[lint.pydocstyle]
139+
convention = "google"
140+
141+
[lint.flake8-annotations]
142+
allow-star-arg-any = false
143+
ignore-fully-untyped = false
144+
145+
[format]
146+
quote-style = "double"
147+
indent-style = "space"
148+
skip-magic-trailing-comma = false
149+
line-ending = "auto"
150+
preview = false
151+
docstring-code-format = true

airbyte/_connector_base.py

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@
66
import abc
77
import json
88
from pathlib import Path
9+
from time import sleep
910
from typing import TYPE_CHECKING, Any, Literal
1011

1112
import jsonschema
@@ -324,6 +325,8 @@ def check(self) -> None:
324325
name=self.name,
325326
state=EventState.FAILED,
326327
)
328+
# Give logs a chance to be flushed.
329+
sleep(1)
327330
raise exc.AirbyteConnectorCheckFailedError(
328331
connector_name=self.name,
329332
help_url=self.docs_url,

0 commit comments

Comments
 (0)