From 8064021062a433de9cd833ede9aa1962f9fa8cef Mon Sep 17 00:00:00 2001 From: Andy Doan Date: Wed, 14 May 2025 14:55:53 -0500 Subject: [PATCH] ci: Move to new artifact upload action Signed-off-by: Andy Doan --- .github/workflows/build-daily.yml | 1 - .github/workflows/build-on-pr.yml | 1 - .github/workflows/build-on-push.yml | 1 - .github/workflows/debos.yml | 26 ++---- .github/workflows/linux.yml | 1 - .github/workflows/publish_artifacts.py | 111 ------------------------- .github/workflows/u-boot.yml | 1 - 7 files changed, 8 insertions(+), 134 deletions(-) delete mode 100755 .github/workflows/publish_artifacts.py diff --git a/.github/workflows/build-daily.yml b/.github/workflows/build-daily.yml index acd67dc6..a79209cc 100644 --- a/.github/workflows/build-daily.yml +++ b/.github/workflows/build-daily.yml @@ -9,7 +9,6 @@ on: permissions: contents: read - security-events: read # This is required to handle authentication to our artifact publishing API jobs: build-daily: diff --git a/.github/workflows/build-on-pr.yml b/.github/workflows/build-on-pr.yml index 5112e040..a3ec4636 100644 --- a/.github/workflows/build-on-pr.yml +++ b/.github/workflows/build-on-pr.yml @@ -5,7 +5,6 @@ on: permissions: contents: read - security-events: read # This is required to handle authentication to our artifact publishing API jobs: build-pr: diff --git a/.github/workflows/build-on-push.yml b/.github/workflows/build-on-push.yml index f48b6705..beb50e03 100644 --- a/.github/workflows/build-on-push.yml +++ b/.github/workflows/build-on-push.yml @@ -6,7 +6,6 @@ on: permissions: contents: read - security-events: read # This is required to handle authentication to our artifact publishing API jobs: build-daily: diff --git a/.github/workflows/debos.yml b/.github/workflows/debos.yml index c26f39ed..7adb0c36 100644 --- a/.github/workflows/debos.yml +++ b/.github/workflows/debos.yml @@ -11,13 +11,6 @@ on: # permissions to none permissions: contents: read - security-events: read # This is required to handle authentication to our artifact publishing API - -env: - # github runs are only unique per repository and may also be re-run; create a - # build id for the current run - BUILD_ID: ${{ github.repository }}-${{ github.run_id }}-${{ github.run_attempt }} - FILESERVER_URL: https://quic-yocto-fileserver-1029608027416.us-central1.run.app # cancel in progress builds for this workflow triggered by the same ref concurrency: @@ -93,16 +86,11 @@ jobs: debos -t u_boot_rb1:rb1-boot.img \ debos-recipes/qualcomm-linux-debian-flash.yaml - - name: Upload artifacts to fileserver space for builds - id: upload_artifacts - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + - name: Stage build artifacts for publishing run: | set -ux - # python3-requests is used by publish_aritfacts.py - apt -y install python3-requests # create a directory for the current run - export BUILD_DIR="/tmp/${BUILD_ID}" + BUILD_DIR="./uploads" mkdir -vp "${BUILD_DIR}" # copy output files cp -av rootfs.tar.gz "${BUILD_DIR}" @@ -118,7 +106,9 @@ jobs: disk-sdcard.img1 \ disk-sdcard.img2 \ flash_rb1* - # instruct fileserver to publish this directory - export URL="${FILESERVER_URL}/${BUILD_ID}/" - .github/workflows/publish_artifacts.py - echo Image available at: ${URL} + + - name: Upload private artifacts + uses: qualcomm-linux/upload-private-artifact-action@v1 + id: upload_artifacts + with: + path: ./uploads diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index 5f830151..d533fb38 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -11,7 +11,6 @@ on: # permissions to none permissions: contents: read - security-events: read # This is required to handle authentication to our artifact publishing API env: # where results will be posted/hosted diff --git a/.github/workflows/publish_artifacts.py b/.github/workflows/publish_artifacts.py deleted file mode 100755 index 2e5554a5..00000000 --- a/.github/workflows/publish_artifacts.py +++ /dev/null @@ -1,111 +0,0 @@ -#!/usr/bin/env python3 -# Copyright (c) 2025 Qualcomm Innovation Center, Inc. All rights reserved. -# SPDX-License-Identifier: BSD-3-Clause - -from multiprocessing import Pool -import os -import sys -from time import sleep -from typing import List - -import requests - -gh_token = os.environ["GITHUB_TOKEN"] -num_threads_str = os.environ.get("UPLOAD_THREADS", "5") - - -def upload_file(args): - """ - Uploads a file to our file upload service. The service is a GCP CloudRun - project that returns signed URLs to Google Storage objects we can upload to. - """ - try: - url, base, name = args - - headers = { - "Authentication": f"Bearer {gh_token}", - } - - # Obtain the signed-url for GCS using Fibonacci backoff/retries - for x in (1, 2, 3, 5, 0): - r = requests.put(url, headers=headers, allow_redirects=False) - if not r.ok: - correlation_id = r.headers.get("X-Correlation-ID", "?") - if not x: - return ( - name, - f"Unable to get signed url HTTP_{r.status_code}. Correlation ID: {correlation_id} - {r.text}", - ) - else: - print( - f"Error getting signed URL for {name}: Correlation ID: {correlation_id} HTTP_{r.status_code} - {r.text}", - flush=True, - ) - print(f"Retrying in {x} seconds", flush=True) - sleep(x) - - # Upload the file to the signed URL with backoff/retry logic - url = r.headers["location"] - path = os.path.join(base, name) - for x in (1, 2, 3, 0): - r = requests.put( - url, - data=open(path, "rb"), - headers={"Content-type": "application/octet-stream"}, - ) - if not r.ok: - if not x: - return ( - name, - f"Unable to upload content HTTP_{r.status_code} - {r.text}", - ) - else: - print( - f"Unable to upload content for {name}: HTTP_{r.status_code} - {r.text}" - ) - print(f"Retrying in {x} seconds") - sleep(x) - - return name, None - except Exception as e: - return name, str(e) - - -def get_files_to_publish(path: str) -> List[str]: - paths = [] - for root, dirs, files in os.walk(path): - for file in files: - paths.append(os.path.join(root, file)[len(path) :]) - return paths - - -def main(num_threads: int, artifacts_dir: str, base_url: str): - paths = get_files_to_publish(artifacts_dir) - print(f"= Found {len(paths)} files to publish", flush=True) - - failed = False - work = [(f"{base_url}{x}", artifacts_dir, x) for x in paths] - with Pool(num_threads) as p: - results = p.imap_unordered(upload_file, work) - for i, res in enumerate(results): - name, err = res - print(f"= {i+1} of {len(work)} - {name}", flush=True) - if err: - print(f"|-> ERROR: {err}", flush=True) - failed = True - - if failed: - sys.exit(1) - - -if __name__ == "__main__": - BUILD_DIR = os.environ["BUILD_DIR"] - if BUILD_DIR[-1] != "/": - BUILD_DIR = BUILD_DIR + "/" - - URL = os.environ["URL"] - if URL[-1] != "/": - URL = URL + "/" - - num_threads = int(num_threads_str) - main(num_threads, BUILD_DIR, URL) diff --git a/.github/workflows/u-boot.yml b/.github/workflows/u-boot.yml index c3b3102a..cd605ec9 100644 --- a/.github/workflows/u-boot.yml +++ b/.github/workflows/u-boot.yml @@ -8,7 +8,6 @@ on: # permissions to none permissions: contents: read - security-events: read # This is required to handle authentication to our artifact publishing API env: # where results will be posted/hosted