Package built artifacts in a archive before making a gh release

Author: Alenar

.github/workflows/ci.yml

@@ -296,32 +296,66 @@ jobs:
       - run-test-lab
       - check
     steps:
-      - name: Download built artifacts (${{ runner.os }}-${{ runner.arch }})
+      - name: Checkout sources
+        uses: actions/checkout@v3
+      
+      - name: Prepare packaging
+        run: mkdir package
+
+      - name: Get short SHA
+        id: slug
+        run: echo "sha8=$(echo ${{ github.sha }} | cut -c1-7)" >> $GITHUB_OUTPUT
+      
+      - name: Download built artifacts (Linux-X64)
         uses: actions/download-artifact@v3
         with:
-          name: mithril-distribution-${{ runner.os }}-${{ runner.arch }}
-          path: ./build
+          name: mithril-distribution-Linux-X64
+          path: ./package-Linux-X64
 
       - name: Download built artifacts (macOS-X64)
         uses: actions/download-artifact@v3
         with:
           name: mithril-distribution-macOS-X64
-          path: ./build
+          path: ./package-macOS-X64
 
       - name: Download built artifacts (Windows-X64)
         uses: actions/download-artifact@v3
         with:
           name: mithril-distribution-Windows-X64
-          path: ./build
+          path: ./package-Windows-X64
 
+      - name: Package distribution (Linux-X64)
+        run: |
+          python3 ./.github/workflows/scripts/package-distribution.py \
+          --input package-Linux-X64/ \
+          --dest package/ \
+          --version "unstable-${{ steps.slug.outputs.sha8 }}" \
+          --target "linux-x64"
+
+      - name: Package distribution (macOS-X64)
+        run: |
+          python3 ./.github/workflows/scripts/package-distribution.py \
+          --input package-macOS-X64/ \
+          --dest package/ \
+          --version "unstable-${{ steps.slug.outputs.sha8 }}" \
+          --target "macos-x64"
+
+      - name: Package distribution (Windows-X64)
+        run: |
+          python3 ./.github/workflows/scripts/package-distribution.py \
+          --input package-Windows-X64/ \
+          --dest package/ \
+          --version "unstable-${{ steps.slug.outputs.sha8 }}" \
+          --target "windows-x64"
+      
       - name: Update unstable release
         uses: marvinpinto/action-automatic-releases@latest
         with:
           repo_token: ${{ secrets.GITHUB_TOKEN }}
           automatic_release_tag: unstable
           prerelease: true
           title: Unstable Development Builds
-          files: build/*
+          files: package/*
 
   deploy-testing:
     runs-on: ubuntu-22.04

.github/workflows/pre-release.yml

@@ -10,36 +10,66 @@ jobs:
   create-pre-release:
     runs-on: ubuntu-22.04
     steps:
+      - name: Checkout sources
+        uses: actions/checkout@v3
+      
+      - name: Prepare packaging
+        run: mkdir package
+
       - name: Download built artifacts (Linux-x64)
         uses: dawidd6/action-download-artifact@v2
         with:
           name: mithril-distribution-Linux-x64
-          path: ./build
+          path: ./package-Linux-X64
           commit: ${{ github.sha }}
           workflow: ci.yml
           workflow_conclusion: success
-      
-      - name: Download built artifacts (Windows-x64)
+
+      - name: Download built artifacts (macOS-x64)
         uses: dawidd6/action-download-artifact@v2
         with:
-          name: mithril-distribution-Windows-x64
-          path: ./build
+          name: mithril-distribution-macOS-x64
+          path: ./package-macOS-X64
           commit: ${{ github.sha }}
           workflow: ci.yml
           workflow_conclusion: success
 
-      - name: Download built artifacts (macOS-x64)
+      - name: Download built artifacts (Windows-x64)
         uses: dawidd6/action-download-artifact@v2
         with:
-          name: mithril-distribution-macOS-x64
-          path: ./build
+          name: mithril-distribution-Windows-x64
+          path: ./package-Windows-X64
           commit: ${{ github.sha }}
           workflow: ci.yml
           workflow_conclusion: success
+
+      - name: Package distribution (Linux-X64)
+        run: |
+          python3 ./.github/workflows/scripts/package-distribution.py \
+          --input package-Linux-X64/ \
+          --dest package/ \
+          --version "${{ github.ref_name }}" \
+          --target "linux-x64"
+
+      - name: Package distribution (macOS-X64)
+        run: |
+          python3 ./.github/workflows/scripts/package-distribution.py \
+          --input package-macOS-X64/ \
+          --dest package/ \
+          --version "${{ github.ref_name }}" \
+          --target "macos-x64"
+
+      - name: Package distribution (Windows-X64)
+        run: |
+          python3 ./.github/workflows/scripts/package-distribution.py \
+          --input package-Windows-X64/ \
+          --dest package/ \
+          --version "${{ github.ref_name }}" \
+          --target "windows-x64"
       
       - name: Append VERSION file
         run: |
-          echo ${{ github.ref_name }} >> ./build/VERSION
+          echo ${{ github.ref_name }} >> ./package/VERSION
       
       - name: Create pre-release ${{ github.ref_name }}
         uses: marvinpinto/action-automatic-releases@latest
@@ -48,7 +78,7 @@ jobs:
           automatic_release_tag: ${{ github.ref_name }}
           prerelease: true
           title: Mithril v${{ github.ref_name }}
-          files: build/**/*
+          files: package/*
 
   build-push-docker:
     runs-on: ubuntu-22.04

.github/workflows/scripts/package-distribution.py

@@ -0,0 +1,91 @@
+import hashlib
+import argparse
+import os
+import platform
+import shutil
+
+
+def sha256sum(path: str):
+    h = hashlib.sha256()
+    paths = []
+
+    # Note: this won't works if the given path is a directory with a subdirectory
+    if os.path.isdir(path):
+        for file in os.listdir(path):
+            paths.append(os.path.join(path, file))
+    else:
+        paths.append(path)
+
+    for path in paths:
+        with open(path, "rb") as f:
+            data = f.read()
+            h.update(data)
+    return h.hexdigest()
+
+
+def dir_path(path):
+    if os.path.isdir(path):
+        return path
+    else:
+        raise NotADirectoryError(path)
+
+
+def build_archive(input_dir, destination_dir, archive_basename):
+    input_files = os.listdir(args.input)
+    print(f"packing mithril distribution {args.version}-{args.target} with files: {input_files}")
+
+    if platform.system() == "Windows":
+        import zipfile
+
+        archive_name = os.path.join(destination_dir, f"{archive_basename}.zip")
+        with zipfile.ZipFile(archive_name, mode="x") as archive:
+            for filename in input_files:
+                archive.write(os.path.join(input_dir, filename), arcname=filename)
+    else:
+        import tarfile
+
+        archive_name = os.path.join(destination_dir, f"{archive_basename}.tar.gz")
+        with tarfile.open(archive_name, "x:gz") as archive:
+            for filename in input_files:
+                archive.add(os.path.join(input_dir, filename), arcname=filename)
+
+    print(f"successfully packed mithril distribution into: {archive_name}")
+    return archive_name
+
+
+def check_archive(archive, original_input_dir):
+    print(f"checking archive ...")
+    test_dir = "./unpack-test"
+    shutil.unpack_archive(archive, test_dir)
+    original_checksum = sha256sum(original_input_dir)
+    archive_content_checksum = sha256sum(test_dir)
+    if original_checksum != archive_content_checksum:
+        print(
+            f"mithril distribution checksum mismatch: before {original_checksum} != after {archive_content_checksum}"
+        )
+        exit(1)
+    print("OK ! Checksum of the archive files matches original files")
+
+    shutil.rmtree(test_dir)
+
+
+def compute_sha256_checksum(archive):
+    print(f"computing archive checksum...")
+    archive_checksum = sha256sum(archive)
+    checksum_filename = f"{archive}.sha256"
+    with open(checksum_filename, "x") as f:
+        f.write(archive_checksum)
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--input", type=dir_path, help="input folder which content will be archived", required=True)
+    parser.add_argument("--dest", type=dir_path, help="destination folder for the archive, default to current folder",
+                        default="./")
+    parser.add_argument("--version", help="version of the distribution to package", required=True)
+    parser.add_argument("--target", help="target os & architecture of the package", required=True)
+    args = parser.parse_args()
+
+    archive_path = build_archive(args.input, args.dest, archive_basename=f"mithril-{args.version}-{args.target}")
+    check_archive(archive_path, args.input)
+    compute_sha256_checksum(archive_path)