diff --git a/fluent-package/manage-fluent-repositories.sh b/fluent-package/manage-fluent-repositories.sh index 7c10398f1..07298b930 100755 --- a/fluent-package/manage-fluent-repositories.sh +++ b/fluent-package/manage-fluent-repositories.sh @@ -197,78 +197,79 @@ EOF https://api.github.com/repos/fluent/fluent-package-builder/pulls/$PULL_NUMBER | jq --raw-output '.head | .ref + " " + .sha') head_branch=$(echo $response | cut -d' ' -f1) head_sha=$(echo $response | cut -d' ' -f2) - rm -f dl.list - for d in `seq 1 5`; do + rm -f dl.*.list + for d in `seq 1 20`; do curl --silent --location \ -H "Accept: application/vnd.github+json" \ -H "Authorization: Bearer $GITHUB_ACCESS_TOKEN" \ -H "X-GitHub-Api-Version: 2022-11-28" \ "https://api.github.com/repos/fluent/fluent-package-builder/actions/artifacts?per_page=100&page=$d" | \ - jq --raw-output '.artifacts[] | select(.workflow_run.head_branch == "'$head_branch'" and .workflow_run.head_sha == "'$head_sha'") | .name + " " + (.size_in_bytes|tostring) + " " + .archive_download_url' > dl.list - filesize=$(stat -c "%s" dl.list) + jq --raw-output '.artifacts[] | select(.workflow_run.head_branch == "'$head_branch'" and .workflow_run.head_sha == "'$head_sha'") | .name + " " + (.size_in_bytes|tostring) + " " + .archive_download_url' > dl.$d.list + filesize=$(stat -c "%s" dl.$d.list) + if [ $filesize -eq 0 ]; then echo "Failed to fetch artifacts list" - rm -f dl.list + rm -f dl.$d.list continue - else - break fi - done - while read line - do - package=$(echo $line | cut -d' ' -f1) - download_url=$(echo $line | cut -d' ' -f3) - echo "Downloading $package.zip from $download_url" - case $package in - *debian*|*ubuntu*) - mkdir -p apt/repositories - (cd apt/repositories && - rm -f $package.zip && - curl --silent --location --output $package.zip \ - -H "Authorization: Bearer $GITHUB_ACCESS_TOKEN" $download_url ) & - ;; - *centos*|*rockylinux*|*almalinux*|*amazonlinux*) - mkdir -p yum/repositories - (cd yum/repositories && - rm -f $package.zip && - curl --silent --location --output $package.zip \ - -H "Authorization: Bearer $GITHUB_ACCESS_TOKEN" $download_url ) & - ;; - *) - curl --silent --location --output $package.zip \ - -H "Authorization: Bearer $GITHUB_ACCESS_TOKEN" $download_url & - ;; - esac - done < dl.list - wait + while read line + do + package=$(echo $line | cut -d' ' -f1) + download_url=$(echo $line | cut -d' ' -f3) + echo "Downloading $package.zip from $download_url" + case $package in + *debian*|*ubuntu*) + mkdir -p apt/repositories + (cd apt/repositories && + rm -f $package.zip && + curl --silent --location --output $package.zip \ + -H "Authorization: Bearer $GITHUB_ACCESS_TOKEN" $download_url ) & + ;; + *centos*|*rockylinux*|*almalinux*|*amazonlinux*) + mkdir -p yum/repositories + (cd yum/repositories && + rm -f $package.zip && + curl --silent --location --output $package.zip \ + -H "Authorization: Bearer $GITHUB_ACCESS_TOKEN" $download_url ) & + ;; + *) + curl --silent --location --output $package.zip \ + -H "Authorization: Bearer $GITHUB_ACCESS_TOKEN" $download_url & + ;; + esac + done < dl.$d.list + wait - verified=1 - while read line - do - package=$(echo $line | cut -d' ' -f1) - download_size=$(echo $line | cut -d' ' -f2) - case $package in - *debian*|*ubuntu*) - actual_size=$(stat --format="%s" apt/repositories/$package.zip) - ;; - *rockylinux*|*almalinux*|*amazonlinux*) - actual_size=$(stat --format="%s" yum/repositories/$package.zip) - ;; - *) - actual_size=$(stat --format="%s" $package.zip) - ;; - esac - if [ $download_size = "$actual_size" ]; then - echo -e "[\e[32m\e[40mOK\e[0m] Verify apt/repositories/$package.zip" - else - echo -e "[\e[31m\e[40mNG\e[0m] Verify apt/repositories/$package.zip (expected: $download_size actual: $actual_size)" - verified=0 + verified=1 + while read line + do + package=$(echo $line | cut -d' ' -f1) + download_size=$(echo $line | cut -d' ' -f2) + case $package in + *debian*|*ubuntu*) + actual_size=$(stat --format="%s" apt/repositories/$package.zip) + ;; + *rockylinux*|*almalinux*|*amazonlinux*) + actual_size=$(stat --format="%s" yum/repositories/$package.zip) + ;; + *) + actual_size=$(stat --format="%s" $package.zip) + ;; + esac + if [ $download_size = "$actual_size" ]; then + echo -e "[\e[32m\e[40mOK\e[0m] Verify apt/repositories/$package.zip" + else + echo -e "[\e[31m\e[40mNG\e[0m] Verify apt/repositories/$package.zip (expected: $download_size actual: $actual_size)" + verified=0 + fi + done < dl.$d.list + if [ $verified -eq 0 ]; then + echo "Downloaded artifacts were corrupted! Check the downloaded artifacts." + exit 1 fi - done < dl.list - if [ $verified -eq 0 ]; then - echo "Downloaded artifacts were corrupted! Check the downloaded artifacts." - exit 1 - fi + done + mkdir -p apt/repositories + mkdir -p yum/repositories (cd apt/repositories && find . -name '*.zip' -exec unzip -u -o {} \;) (cd yum/repositories && find . -name '*.zip' -exec unzip -u -o {} \;) ;;