@@ -50,7 +50,7 @@ async def download_metadata_schemas(temp_dir: Path) -> list[str]:
5050 }
5151 if token :
5252 headers ["Authorization" ] = f"Bearer { token } "
53-
53+
5454 async with httpx .AsyncClient (headers = headers , timeout = 30.0 ) as client :
5555 try :
5656 response = await client .get (METADATA_SCHEMAS_GITHUB_URL )
@@ -64,21 +64,21 @@ async def download_metadata_schemas(temp_dir: Path) -> list[str]:
6464 )
6565 raise
6666 raise
67-
67+
6868 yaml_files = []
6969 for file_info in files_info :
7070 if file_info ["name" ].endswith (".yaml" ):
7171 file_name = file_info ["name" ]
7272 file_url = f"{ METADATA_SCHEMAS_RAW_URL_BASE } /{ file_name } "
73-
73+
7474 print (f"Downloading { file_name } ..." , file = sys .stderr )
7575 file_response = await client .get (file_url )
7676 file_response .raise_for_status ()
77-
77+
7878 file_path = temp_dir / file_name
7979 file_path .write_text (file_response .text )
8080 yaml_files .append (Path (file_name ).stem )
81-
81+
8282 return yaml_files
8383
8484
@@ -173,11 +173,9 @@ async def post_process_metadata_models(codegen_container: dagger.Container):
173173 original_content = await codegen_container .file (
174174 f"/generated/{ generated_file } "
175175 ).contents ()
176-
177- post_processed_content = original_content .replace (
178- "from pydantic" , "from pydantic.v1"
179- )
180-
176+
177+ post_processed_content = original_content .replace ("from pydantic" , "from pydantic.v1" )
178+
181179 codegen_container = codegen_container .with_new_file (
182180 f"/generated_post_processed/{ generated_file } " , contents = post_processed_content
183181 )
@@ -194,7 +192,7 @@ async def generate_models_from_schemas(
194192) -> None :
195193 """Generate Pydantic models from YAML schemas using datamodel-codegen."""
196194 init_module_content = generate_init_module_content (yaml_files )
197-
195+
198196 codegen_container = (
199197 dagger_client .container ()
200198 .from_ (PYTHON_IMAGE )
@@ -205,7 +203,7 @@ async def generate_models_from_schemas(
205203 )
206204 .with_new_file ("/generated/__init__.py" , contents = init_module_content )
207205 )
208-
206+
209207 for yaml_file in yaml_files :
210208 codegen_container = codegen_container .with_exec (
211209 [
@@ -226,7 +224,7 @@ async def generate_models_from_schemas(
226224 ],
227225 use_entrypoint = True ,
228226 )
229-
227+
230228 if post_process :
231229 codegen_container = await post_process_codegen (codegen_container )
232230 await codegen_container .directory ("/generated_post_processed" ).export (output_dir_path )
@@ -240,35 +238,38 @@ async def generate_models_from_schemas(
240238def consolidate_yaml_schemas_to_json (yaml_dir_path : Path , output_json_path : str ) -> None :
241239 """Consolidate all YAML schemas into a single JSON schema file."""
242240 schemas = {}
243-
241+
244242 for yaml_file in yaml_dir_path .glob ("*.yaml" ):
245243 schema_name = yaml_file .stem
246- with yaml_file .open ('r' ) as f :
244+ with yaml_file .open ("r" ) as f :
247245 schema_content = yaml .safe_load (f )
248246 schemas [schema_name ] = schema_content
249-
247+
250248 # Find the main schema (ConnectorMetadataDefinitionV0)
251249 main_schema = schemas .get ("ConnectorMetadataDefinitionV0" )
252-
250+
253251 if main_schema :
254252 # Create a consolidated schema with definitions
255253 consolidated = {
256254 "$schema" : main_schema .get ("$schema" , "http://json-schema.org/draft-07/schema#" ),
257255 "title" : "Connector Metadata Schema" ,
258256 "description" : "Consolidated JSON schema for Airbyte connector metadata validation" ,
259257 ** main_schema ,
260- "definitions" : {}
258+ "definitions" : {},
261259 }
262-
260+
263261 # Add all other schemas as definitions
264262 for schema_name , schema_content in schemas .items ():
265263 if schema_name != "ConnectorMetadataDefinitionV0" :
266264 consolidated ["definitions" ][schema_name ] = schema_content
267-
265+
268266 Path (output_json_path ).write_text (json .dumps (consolidated , indent = 2 ))
269267 print (f"Generated consolidated JSON schema: { output_json_path } " , file = sys .stderr )
270268 else :
271- print ("Warning: ConnectorMetadataDefinitionV0 not found, generating simple consolidation" , file = sys .stderr )
269+ print (
270+ "Warning: ConnectorMetadataDefinitionV0 not found, generating simple consolidation" ,
271+ file = sys .stderr ,
272+ )
272273 Path (output_json_path ).write_text (json .dumps (schemas , indent = 2 ))
273274
274275
@@ -287,7 +288,7 @@ async def generate_metadata_models_single_file(
287288 "/yaml" , dagger_client .host ().directory (yaml_dir_path , include = ["*.yaml" ])
288289 )
289290 )
290-
291+
291292 codegen_container = codegen_container .with_exec (
292293 [
293294 "datamodel-codegen" ,
@@ -307,16 +308,14 @@ async def generate_metadata_models_single_file(
307308 ],
308309 use_entrypoint = True ,
309310 )
310-
311+
311312 original_content = await codegen_container .file ("/generated/models.py" ).contents ()
312- post_processed_content = original_content .replace (
313- "from pydantic" , "from pydantic.v1"
314- )
315-
313+ post_processed_content = original_content .replace ("from pydantic" , "from pydantic.v1" )
314+
316315 codegen_container = codegen_container .with_new_file (
317316 "/generated/models_processed.py" , contents = post_processed_content
318317 )
319-
318+
320319 await codegen_container .file ("/generated/models_processed.py" ).export (output_file_path )
321320
322321
@@ -331,27 +330,27 @@ async def main():
331330 yaml_files = declarative_yaml_files ,
332331 post_process = True ,
333332 )
334-
333+
335334 print ("\n Generating metadata models..." , file = sys .stderr )
336335 with tempfile .TemporaryDirectory () as temp_dir :
337336 temp_path = Path (temp_dir )
338337 await download_metadata_schemas (temp_path )
339-
338+
340339 output_dir = Path (LOCAL_METADATA_OUTPUT_DIR_PATH )
341340 output_dir .mkdir (parents = True , exist_ok = True )
342-
341+
343342 print ("Generating single Python file with all models..." , file = sys .stderr )
344343 output_file = str (output_dir / "models.py" )
345344 await generate_metadata_models_single_file (
346345 dagger_client = dagger_client ,
347346 yaml_dir_path = str (temp_path ),
348347 output_file_path = output_file ,
349348 )
350-
349+
351350 print ("Generating consolidated JSON schema..." , file = sys .stderr )
352351 json_schema_file = str (output_dir / "metadata_schema.json" )
353352 consolidate_yaml_schemas_to_json (temp_path , json_schema_file )
354-
353+
355354 print ("\n Model generation complete!" , file = sys .stderr )
356355
357356
0 commit comments