Make log streaming more reliable

.github/workflows/main.yaml

@@ -45,5 +45,8 @@ jobs:
 
   test-submit:
     uses: ./.github/workflows/test-examples.yml
+    with:
+      CHIPFLOW_BACKEND_VERSION: 'branch-opentracing-batch'
+      CHIPFLOW_API_ORIGIN: "https://build-staging.chipflow.org"
     secrets:
-      CHIPFLOW_API_KEY: ${{ secrets.CHIPFLOW_API_KEY}}
+      CHIPFLOW_API_KEY: ${{ secrets.CHIPFLOW_API_KEY_STAGING }}

.github/workflows/test-examples.yml

@@ -2,6 +2,15 @@ name: Reusable workflow to test everything in chipflow-examples works
 
 on:
   workflow_call:
+    inputs:
+      CHIPFLOW_BACKEND_VERSION:
+        required: false
+        type: string
+        default: 'branch-main'
+      CHIPFLOW_API_ORIGIN:
+        required: false
+        type: string
+        default: 'https://build.chipflow.org'
     secrets:
       CHIPFLOW_API_KEY:
         required: true
@@ -66,3 +75,6 @@ jobs:
           pdm run chipflow silicon submit --wait $DRY
         env:
           CHIPFLOW_API_KEY: ${{ secrets.CHIPFLOW_API_KEY}}
+          CHIPFLOW_API_ORIGIN: ${{ inputs.CHIPFLOW_API_ORIGIN }}
+          CHIPFLOW_BACKEND_VERSION: ${{ inputs.CHIPFLOW_BACKEND_VERSION }}
+

chipflow_lib/steps/silicon.py

@@ -21,7 +21,6 @@
 
 logger = logging.getLogger(__name__)
 
-
 class SiliconTop(StepBase, Elaboratable):
     def __init__(self, config={}):
         self._config = config
@@ -191,68 +190,7 @@ def submit(self, rtlil_path, *, dry_run=False, wait=False):
             resp_data = resp.text
 
         # Handle response based on status code
-        if resp.status_code == 200:
-            logger.info(f"Submitted design: {resp_data}")
-            build_url = f"{chipflow_api_origin}/build/{resp_data['build_id']}"
-            build_status_url = f"{chipflow_api_origin}/build/{resp_data['build_id']}/status"
-            log_stream_url = f"{chipflow_api_origin}/build/{resp_data['build_id']}/logs?follow=true"
-
-            print(f"Design submitted successfully! Build URL: {build_url}")
-
-            # Poll the status API until the build is completed or failed
-            stream_event_counter = 0
-            fail_counter = 0
-            if wait:
-                while True:
-                    logger.info("Polling build status...")
-                    status_resp = requests.get(
-                        build_status_url,
-                        auth=(None, chipflow_api_key)
-                    )
-                    if status_resp.status_code != 200:
-                        fail_counter += 1
-                        logger.error(f"Failed to fetch build status {fail_counter} times: {status_resp.text}")
-                        if fail_counter > 5:
-                            logger.error(f"Failed to fetch build status {fail_counter} times. Exiting.")
-                            raise ChipFlowError("Error while checking build status.")
-
-                    status_data = status_resp.json()
-                    build_status = status_data.get("status")
-                    logger.info(f"Build status: {build_status}")
-
-                    if build_status == "completed":
-                        print("Build completed successfully!")
-                        exit(0)
-                    elif build_status == "failed":
-                        print("Build failed.")
-                        exit(1)
-                    elif build_status == "running":
-                        print("Build running.")
-                        # Wait before polling again
-                        # time.sleep(10)
-                        # Attempt to stream logs rather than time.sleep
-                        try:
-                            if stream_event_counter > 1:
-                                logger.warning("Log streaming may have been interrupted. Some logs may be missing.")
-                                logger.warning(f"Check {build_url}")
-                            stream_event_counter += 1
-                            with requests.get(
-                                log_stream_url,
-                                auth=(None, chipflow_api_key),
-                                stream=True
-                            ) as log_resp:
-                                if log_resp.status_code == 200:
-                                    for line in log_resp.iter_lines():
-                                        if line:
-                                            print(line.decode("utf-8"))  # Print logs in real-time
-                                            sys.stdout.flush()
-                                else:
-                                    logger.warning(f"Failed to stream logs: {log_resp.text}")
-                        except requests.RequestException as e:
-                            logger.error(f"Error while streaming logs: {e}")
-                            pass
-                    time.sleep(10)  # Wait before polling again
-        else:
+        if resp.status_code != 200:
             # Log detailed information about the failed request
             logger.error(f"Request failed with status code {resp.status_code}")
             logger.error(f"Request URL: {resp.request.url}")
@@ -268,3 +206,114 @@ def submit(self, rtlil_path, *, dry_run=False, wait=False):
             logger.error(f"Response body: {resp_data}")
 
             raise ChipFlowError(f"Failed to submit design: {resp_data}")
+
+        logger.info(f"Submitted design: {resp_data}")
+        build_url = f"{chipflow_api_origin}/build/{resp_data['build_id']}"
+        build_status_url = f"{chipflow_api_origin}/build/{resp_data['build_id']}/status"
+        log_stream_url = f"{chipflow_api_origin}/build/{resp_data['build_id']}/logs?follow=true"
+
+        print(f"Design submitted successfully! Build URL: {build_url}")
+
+        # Poll the status API until the build is completed or failed
+        timeout = 10.0
+        fail_count = 0
+
+        def poll_build_status():
+            nonlocal timeout
+            nonlocal fail_count
+            logger.info("Polling build status...")
+            with requests.get(
+                build_status_url,
+                auth=(None, chipflow_api_key),
+                timeout=timeout
+                ) as status_resp:
+                if status_resp.status_code != 200:
+                    fail_count += 1
+                    logger.error(f"Failed to fetch build status {fail_count} times: {status_resp.text}")
+                status_data = status_resp.json()
+                return status_data.get("status")
+
+
+        def stream_until_fail_or_done():
+            nonlocal timeout
+            nonlocal fail_count
+            print_log_warning = False
+            while fail_count < (2*60//timeout):
+                try:
+                   if fail_count > 1:
+                       print_log_warning = True
+                   with requests.get(
+                        log_stream_url,
+                        auth=(None, chipflow_api_key),
+                        stream=True, timeout=timeout
+                    ) as log_resp:
+                        if log_resp.status_code == 200:
+                            for line in log_resp.iter_lines():
+                                if line:
+                                    print(line.decode("utf-8"))  # Print logs in real-time
+                                    sys.stdout.flush()
+                        else:
+                            logger.warning(f"Failed to stream logs: {log_resp.text}")
+                            fail_count += 1
+                except requests.Timeout:
+                    fail_count +=1
+                    continue  #go round again
+                except requests.RequestException as e:
+                    if type(e) is requests.exceptions.ConnectionError and e.response is None:
+                        fail_count +=1
+                        logger.warn(f"Issue while streaming logs: {type(e)}:{e} response={e.response}. Trying again.")
+                        continue  #try again
+                    logger.error(f"Error while streaming logs: {type(e)}:{e} response={e.response}")
+                    return False
+                if print_log_warning:
+                    logger.warning("Log streaming may have been interrupted. Some logs may be missing.")
+                    logger.warning(f"Check {build_url}")
+
+                return True
+
+
+        if not wait:
+            exit(0)
+
+        fail_count = 0
+        status = "waiting"
+        while True:
+            logger.info("Polling build status...")
+            try:
+                status = poll_build_status()
+            except requests.Timeout:
+                 continue  #go round again
+            except requests.RequestException as e:
+                if type(e) is requests.exceptions.ConnectionError and e.response is None:
+                    fail_count +=1
+                    logger.warn(f"Issue while polling build: {type(e)}:{e} response={e.response}. Trying again.")
+                    continue  #try again
+                logger.error(f"Network error while polling build: {type(e)}:{e} response={e.response}")
+                exit(1)
+            except Exception as e:
+                logger.error(f"Unexpected error while polling build: {type(e)}:{e}")
+
+            match status:
+                case "completed":
+                    print("Build completed successfully!")
+                    exit(0)
+                case "failed":
+                    print("Build failed.")
+                    exit(1)
+                case "unknown":
+                    continue  # poll again
+                case "running":
+                    print("Build running.")
+
+            if not stream_until_fail_or_done():
+                fail_count += 1
+                logger.warn("Issue while streaming logs. Trying again.")
+
+            if fail_count > 5:
+                logger.error(f"Failed to fetch build status {fail_count} times. Exiting.")
+                raise ChipFlowError("Error while checking build status.")
+            return "unknown"
+
+            # Wait before polling again
+            time.sleep(0.5)  # Wait before polling again
+