Skip to content

Commit df6dd69

Browse files
brianjlaibrianjlai
committed
Merge branch 'main' into brian/property_chunking_only_fetch_enabled_fields
2 parents 4b872d6 + 20ae208 commit df6dd69

File tree

6 files changed

+357
-15
lines changed

6 files changed

+357
-15
lines changed

‎airbyte_cdk/sources/file_based/exceptions.py‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -157,3 +157,7 @@ class CustomFileBasedException(AirbyteTracedException):
157157

158158
class FileSizeLimitError(CustomFileBasedException):
159159
pass
160+
161+
162+
class EmptyFileSchemaInferenceError(AirbyteTracedException):
163+
pass

‎airbyte_cdk/sources/file_based/file_types/csv_parser.py‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,11 @@
2222
InferenceType,
2323
)
2424
from airbyte_cdk.sources.file_based.config.file_based_stream_config import FileBasedStreamConfig
25-
from airbyte_cdk.sources.file_based.exceptions import FileBasedSourceError, RecordParseError
25+
from airbyte_cdk.sources.file_based.exceptions import (
26+
EmptyFileSchemaInferenceError,
27+
FileBasedSourceError,
28+
RecordParseError,
29+
)
2630
from airbyte_cdk.sources.file_based.file_based_stream_reader import (
2731
AbstractFileBasedStreamReader,
2832
FileReadMode,
@@ -203,7 +207,7 @@ async def infer_schema(
203207
break
204208

205209
if not type_inferrer_by_field:
206-
raise AirbyteTracedException(
210+
raise EmptyFileSchemaInferenceError(
207211
message=f"Could not infer schema as there are no rows in {file.uri}. If having an empty CSV file is expected, ignore this. "
208212
f"Else, please contact Airbyte.",
209213
failure_type=FailureType.config_error,

‎airbyte_cdk/sources/file_based/stream/default_file_based_stream.py‎

Lines changed: 32 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -9,13 +9,26 @@
99
from copy import deepcopy
1010
from functools import cache
1111
from os import path
12-
from typing import Any, Dict, Iterable, List, Mapping, MutableMapping, Optional, Set, Tuple, Union
12+
from typing import (
13+
Any,
14+
Dict,
15+
Iterable,
16+
List,
17+
Mapping,
18+
MutableMapping,
19+
NoReturn,
20+
Optional,
21+
Set,
22+
Tuple,
23+
Union,
24+
)
1325

1426
from airbyte_cdk.models import AirbyteLogMessage, AirbyteMessage, AirbyteStream, FailureType, Level
1527
from airbyte_cdk.models import Type as MessageType
1628
from airbyte_cdk.sources.file_based.config.file_based_stream_config import PrimaryKeyType
1729
from airbyte_cdk.sources.file_based.exceptions import (
1830
DuplicatedFilesError,
31+
EmptyFileSchemaInferenceError,
1932
FileBasedSourceError,
2033
InvalidSchemaError,
2134
MissingSchemaError,
@@ -230,7 +243,7 @@ def cursor_field(self) -> Union[str, List[str]]:
230243
return self.ab_last_mod_col
231244

232245
@cache
233-
def get_json_schema(self) -> JsonSchema:
246+
def get_json_schema(self) -> JsonSchema: # type: ignore
234247
if self.use_file_transfer:
235248
return file_transfer_schema
236249
extra_fields = {
@@ -246,12 +259,12 @@ def get_json_schema(self) -> JsonSchema:
246259
exception=AirbyteTracedException(exception=config_exception),
247260
failure_type=FailureType.config_error,
248261
)
262+
except EmptyFileSchemaInferenceError as exc:
263+
self._raise_schema_inference_error(exc)
249264
except AirbyteTracedException as ate:
250265
raise ate
251266
except Exception as exc:
252-
raise SchemaInferenceError(
253-
FileBasedSourceError.SCHEMA_INFERENCE_ERROR, stream=self.name
254-
) from exc
267+
self._raise_schema_inference_error(exc)
255268
else:
256269
return {"type": "object", "properties": {**extra_fields, **schema["properties"]}}
257270

@@ -380,17 +393,24 @@ async def _infer_schema(self, files: List[RemoteFile]) -> Mapping[str, Any]:
380393

381394
return base_schema
382395

383-
async def _infer_file_schema(self, file: RemoteFile) -> SchemaType:
396+
async def _infer_file_schema(self, file: RemoteFile) -> SchemaType: # type: ignore
384397
try:
385398
return await self.get_parser().infer_schema(
386399
self.config, file, self.stream_reader, self.logger
387400
)
401+
except EmptyFileSchemaInferenceError as exc:
402+
self._raise_schema_inference_error(exc, file)
388403
except AirbyteTracedException as ate:
389404
raise ate
390405
except Exception as exc:
391-
raise SchemaInferenceError(
392-
FileBasedSourceError.SCHEMA_INFERENCE_ERROR,
393-
file=file.uri,
394-
format=str(self.config.format),
395-
stream=self.name,
396-
) from exc
406+
self._raise_schema_inference_error(exc, file)
407+
408+
def _raise_schema_inference_error(
409+
self, exc: Exception, file: Optional[RemoteFile] = None
410+
) -> NoReturn:
411+
raise SchemaInferenceError(
412+
FileBasedSourceError.SCHEMA_INFERENCE_ERROR,
413+
file=file.uri if file else None,
414+
format=str(self.config.format) if self.config.format else None,
415+
stream=self.name,
416+
) from exc

‎docs/generate.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -65,7 +65,7 @@ def run() -> None:
6565
shutil.rmtree("docs/generated")
6666

6767
pdoc.render.configure(
68-
template_directory="docs",
68+
template_directory=pathlib.Path("docs/templates"),
6969
show_source=True,
7070
search=True,
7171
logo="https://docs.airbyte.com/img/logo-dark.png",

‎docs/templates/custom.css‎

Lines changed: 268 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,268 @@
1+
/*
2+
* Python CDK Documentation - Custom Styling
3+
*
4+
* This file applies Airbyte branding and design elements to the Python CDK API documentation.
5+
*
6+
* METHODOLOGY:
7+
* - Colors and design tokens are sourced from the main Airbyte documentation site
8+
* - Primary source: https://github.com/airbytehq/airbyte/blob/main/docusaurus/src/css/custom.css
9+
* - The Airbyte design system uses a purple primary color (#615eff) and Inter font family
10+
* - This CSS overrides pdoc's default theme variables to match Airbyte branding
11+
*
12+
* UPDATING STYLES:
13+
* When Airbyte brand styles are updated, check the following upstream sources:
14+
* 1. Main docs CSS: https://github.com/airbytehq/airbyte/blob/main/docusaurus/src/css/custom.css
15+
* 2. Docusaurus config: https://github.com/airbytehq/airbyte/blob/main/docusaurus/docusaurus.config.js
16+
* 3. Live site: https://docs.airbyte.com (inspect styles in browser dev tools)
17+
*
18+
* Key variables to sync:
19+
* - --ifm-color-primary and variants (primary brand color)
20+
* - --ifm-font-family-base (typography)
21+
* - Color palette variables (grey, green, blue scales)
22+
*
23+
* PDOC INTEGRATION:
24+
* pdoc uses specific CSS variables that we override here:
25+
* - --link: Link color (set to Airbyte primary)
26+
* - --link-hover: Link hover color (set to Airbyte primary-dark)
27+
* - --code: Code background color
28+
* - --accent: Accent/highlight color
29+
* - --text: Main text color
30+
*/
31+
32+
@import url("https://fonts.googleapis.com/css2?family=Inter:wght@200;300;400;500;600;700&display=swap");
33+
34+
:root {
35+
/* Airbyte Primary Colors - from docusaurus/src/css/custom.css */
36+
--airbyte-primary: #615eff;
37+
--airbyte-primary-dark: #3f3bff;
38+
--airbyte-primary-darker: #2e2aff;
39+
--airbyte-primary-darkest: #0500f4;
40+
--airbyte-primary-light: #8381ff;
41+
--airbyte-primary-lighter: #9492ff;
42+
--airbyte-primary-lightest: #c8c7ff;
43+
44+
/* Airbyte Color Palette - from docusaurus/src/css/custom.css */
45+
--color-white: hsl(0, 0%, 100%);
46+
--color-grey-40: hsl(240, 25%, 98%);
47+
--color-grey-100: hsl(240, 12%, 92%);
48+
--color-grey-400: hsl(239, 10%, 59%);
49+
--color-grey-500: hsl(240, 10%, 49%);
50+
--color-grey-900: hsl(240, 19%, 18%);
51+
--color-green-40: hsl(185, 76%, 97%);
52+
--color-green-50: hsl(184, 67%, 92%);
53+
--color-green-600: hsl(185, 100%, 35%);
54+
--color-green-800: hsl(184, 100%, 26%);
55+
--color-blue-30: hsl(240, 100%, 98%);
56+
--color-dark-blue-40: hsl(230, 23%, 95%);
57+
--color-dark-blue-700: hsl(236, 43%, 31%);
58+
}
59+
60+
/* Override pdoc's default theme variables - must be in .pdoc selector for specificity */
61+
.pdoc {
62+
--link: #615eff;
63+
--link-hover: #3f3bff;
64+
--code: hsl(240, 100%, 98%);
65+
--accent: hsl(240, 25%, 98%);
66+
--text: hsl(240, 19%, 18%);
67+
--name: #615eff;
68+
--def: #3f3bff;
69+
}
70+
71+
/* Dark mode colors - from docusaurus/src/css/custom.css */
72+
@media (prefers-color-scheme: dark) {
73+
:root {
74+
--airbyte-primary: #7976ff;
75+
--airbyte-primary-dark: #3f3bff;
76+
--airbyte-primary-darker: #2e2aff;
77+
--airbyte-primary-light: #8381ff;
78+
--airbyte-primary-lighter: #9492ff;
79+
--airbyte-primary-lightest: #c8c7ff;
80+
81+
--color-grey-40: hsl(240, 14%, 14%);
82+
--color-grey-100: hsl(240, 15%, 15%);
83+
--color-grey-400: hsl(240, 13%, 72%);
84+
--color-grey-500: hsl(240, 10%, 70%);
85+
--color-grey-900: hsl(240, 10%, 90%);
86+
--color-green-50: hsl(184, 100%, 12%);
87+
--color-blue-30: hsl(252, 25%, 18%);
88+
--color-dark-blue-700: hsl(240, 100%, 98%);
89+
--color-white: hsl(0, 0%, 16%);
90+
}
91+
92+
/* Override pdoc's default theme variables for dark mode - must be in .pdoc selector */
93+
.pdoc {
94+
--link: #9492ff;
95+
--link-hover: #c8c7ff;
96+
--code: hsl(252, 25%, 18%);
97+
--accent: hsl(240, 14%, 14%);
98+
--text: hsl(240, 10%, 90%);
99+
--name: #9492ff;
100+
--def: #8381ff;
101+
}
102+
}
103+
104+
/* Apply Inter font family - Airbyte's brand font */
105+
body {
106+
font-family: "Inter", -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif;
107+
}
108+
109+
/* Logo styling - swap to light logo in dark mode for better visibility */
110+
@media (prefers-color-scheme: dark) {
111+
.logo {
112+
/* Replace the dark logo with the light logo in dark mode */
113+
content: url("https://docs.airbyte.com/img/logo-light.png");
114+
}
115+
}
116+
117+
/* Enhanced heading styles */
118+
h1, h2, h3, h4, h5, h6 {
119+
font-weight: 600;
120+
}
121+
122+
/* Enhanced code block styling */
123+
pre code {
124+
border-radius: 8px;
125+
}
126+
127+
/* Style tables with Airbyte theme */
128+
table {
129+
border-spacing: 0;
130+
border-collapse: separate;
131+
border-radius: 10px;
132+
overflow: hidden;
133+
}
134+
135+
table th {
136+
background-color: var(--airbyte-primary);
137+
color: var(--color-white);
138+
font-weight: 600;
139+
padding: 12px;
140+
}
141+
142+
table th:first-child {
143+
border-top-left-radius: 10px;
144+
}
145+
146+
table th:last-child {
147+
border-top-right-radius: 10px;
148+
}
149+
150+
table td {
151+
padding: 10px 12px;
152+
border-bottom: 1px solid var(--color-grey-100);
153+
}
154+
155+
table tr:hover {
156+
background-color: var(--color-grey-40);
157+
transition: background-color 0.2s ease;
158+
}
159+
160+
table tr:last-child td:first-child {
161+
border-bottom-left-radius: 10px;
162+
}
163+
164+
table tr:last-child td:last-child {
165+
border-bottom-right-radius: 10px;
166+
}
167+
168+
/* Style navigation with Airbyte colors - use !important to override pdoc's default nav styles */
169+
nav a {
170+
color: var(--link) !important;
171+
}
172+
173+
nav a:hover {
174+
color: var(--link-hover) !important;
175+
}
176+
177+
/* Style badges and labels */
178+
.badge {
179+
background-color: var(--color-green-40);
180+
border: 1px solid var(--color-green-800);
181+
color: var(--color-green-800);
182+
font-weight: 500;
183+
border-radius: 4px;
184+
padding: 2px 8px;
185+
font-size: 0.85em;
186+
}
187+
188+
/* Add rounded corners to images */
189+
img {
190+
max-width: 100%;
191+
border-radius: 10px;
192+
}
193+
194+
/* Style the header/logo area */
195+
header {
196+
border-bottom: 2px solid var(--airbyte-primary-lightest);
197+
}
198+
199+
/* Style buttons and interactive elements */
200+
button, .button {
201+
background-color: var(--airbyte-primary);
202+
color: var(--color-white);
203+
border: none;
204+
border-radius: 6px;
205+
padding: 8px 16px;
206+
font-weight: 500;
207+
cursor: pointer;
208+
transition: background-color 0.2s ease;
209+
}
210+
211+
button:hover, .button:hover {
212+
background-color: var(--airbyte-primary-dark);
213+
}
214+
215+
/* Improve code syntax highlighting */
216+
.pdoc-code {
217+
border-radius: 8px;
218+
border: 1px solid var(--color-grey-100);
219+
}
220+
221+
/* Style docstring sections */
222+
.docstring {
223+
line-height: 1.6;
224+
}
225+
226+
/* Add subtle shadow to main content area */
227+
main {
228+
box-shadow: 0 2px 8px rgba(0, 0, 0, 0.05);
229+
}
230+
231+
/* Style search box if present */
232+
input[type="search"], input[type="text"] {
233+
border: 1px solid var(--color-grey-100);
234+
border-radius: 6px;
235+
padding: 8px 12px;
236+
font-family: inherit;
237+
}
238+
239+
input[type="search"]:focus, input[type="text"]:focus {
240+
outline: none;
241+
border-color: var(--airbyte-primary);
242+
box-shadow: 0 0 0 3px var(--airbyte-primary-lightest);
243+
}
244+
245+
/* Improve spacing and readability */
246+
.pdoc-module-list {
247+
line-height: 1.8;
248+
}
249+
250+
/* Style admonitions/notes */
251+
.admonition {
252+
border-left: 4px solid var(--airbyte-primary);
253+
background-color: var(--color-blue-30);
254+
padding: 12px 16px;
255+
border-radius: 4px;
256+
margin: 16px 0;
257+
}
258+
259+
/* Responsive improvements */
260+
@media (max-width: 768px) {
261+
body {
262+
font-size: 14px;
263+
}
264+
265+
table {
266+
font-size: 0.9em;
267+
}
268+
}

0 commit comments

Comments
 (0)