[DOP-22735] All Spark urls should be singular

Author: dolfinus

data_rentgen/consumer/extractors/run.py

@@ -84,14 +84,19 @@ def enrich_run_identifiers(run: RunDTO, event: OpenLineageRunEvent) -> RunDTO:
     return run
 
 
-def enrich_run_logs(run: RunDTO, event: OpenLineageRunEvent) -> RunDTO:
+def enrich_run_logs(run: RunDTO, event: OpenLineageRunEvent) -> RunDTO:  # noqa: WPS231
     spark_application_details = event.run.facets.spark_applicationDetails
     if spark_application_details:
         if spark_application_details.proxyUrl:
-            run.running_log_url = spark_application_details.proxyUrl.split(",")[0]
+            run.running_log_url = spark_application_details.proxyUrl
         else:
             run.running_log_url = spark_application_details.uiWebUrl
         run.persistent_log_url = spark_application_details.historyUrl
+
+        if run.running_log_url:
+            run.running_log_url = run.running_log_url.split(",")[0]
+        if run.persistent_log_url:
+            run.persistent_log_url = run.persistent_log_url.split(",")[0]
         return run
 
     airflow_dag_run_facet = event.run.facets.airflowDagRun

tests/test_consumer/test_extractors/test_extractors_run.py

@@ -72,8 +72,8 @@ def test_extractors_extract_run_spark_app_yarn():
                     driverHost="localhost",
                     userName="myuser",
                     uiWebUrl="http://localhost:4040",
-                    proxyUrl="http://yarn-proxy:8088/proxy/application_1234_5678,http://yarn-proxy:8088/proxy/application_8765_4321",
-                    historyUrl="http://history-server:18081/history/application_1234_5678/",
+                    proxyUrl="http://yarn-proxy:8088/proxy/application_1234_5678,http://yarn-proxy:18088/proxy/application_1234_5678",
+                    historyUrl="http://history-server:18080/history/application_1234_5678,http://history-server:18081/history/application_1234_5678",
                 ),
             ),
         ),
@@ -92,7 +92,7 @@ def test_extractors_extract_run_spark_app_yarn():
         ended_at=None,
         external_id="application_1234_5678",
         attempt=None,
-        persistent_log_url="http://history-server:18081/history/application_1234_5678/",
+        persistent_log_url="http://history-server:18080/history/application_1234_5678",
         running_log_url="http://yarn-proxy:8088/proxy/application_1234_5678",
     )
 
@@ -124,7 +124,7 @@ def test_extractors_extract_run_spark_app_local():
                     deployMode=OpenLineageSparkDeployMode.CLIENT,
                     driverHost="localhost",
                     userName="myuser",
-                    uiWebUrl="http://localhost:4040",
+                    uiWebUrl="http://localhost:4040,http://localhost:4041",
                 ),
             ),
         ),