TUN-5943: Add RPM support

This PR extends release_pkgs.py to now also support uploading rpm based
assets to R2. The packages are not signed yet  and will be done in a
subsequent PR.

This PR
- Packs the .rpm assets into relevant directories
- Calls createrepo on them to make them yum repo ready
- Uploads them to R2

Author: sudarshan-reddy

Makefile

@@ -261,6 +261,10 @@ github-release: cloudflared
 github-release-built-pkgs:
 	python3 github_release.py --path $(PWD)/built_artifacts --release-version $(VERSION)
 
+.PHONY: release-pkgs-linux
+release-pkgs-linux:
+	python3 ./release_pkgs.py
+
 .PHONY: github-message
 github-message:
 	python3 github_message.py --release-version $(VERSION)

cfsetup.yaml

@@ -40,14 +40,19 @@ stretch: &stretch
       - libffi-dev
       - python3-setuptools
       - python3-pip
+      - reprepro
+      - createrepo
     pre-cache: &github_release_pkgs_pre_cache
       - wget https://github.com/sudarshan-reddy/msitools/releases/download/v0.101b/wixl -P /usr/local/bin
       - chmod a+x /usr/local/bin/wixl
       - pip3 install pynacl==1.4.0
       - pip3 install pygithub==1.55
+      - pip3 install boto3==1.22.9
     post-cache:
       # build all packages (except macos and FIPS) and move them to /cfsetup/built_artifacts
       - ./build-packages.sh
+      # publish packages to linux repos
+      # TODO TUN-6180: Uncomment this - make release-pkgs-linux
       # release the packages built and moved to /cfsetup/built_artifacts
       - make github-release-built-pkgs
   # handle FIPS separately so that we built with gofips compiler
@@ -255,4 +260,4 @@ centos-7:
       - export PATH=$PATH:/usr/local/go/bin
       - export GOOS=linux
       - export GOARCH=amd64
-      - make publish-rpm
+      - make publish-rpm

release_pkgs.py

@@ -12,6 +12,7 @@
 import os
 import argparse
 import logging
+import shutil
 from hashlib import sha256
 
 import boto3
@@ -100,13 +101,38 @@ def create_distribution_conf(self,
     """
     def create_deb_pkgs(self, release, deb_file):
         self._clean_build_resources()
-        subprocess.call(("reprepro", "includedeb", release, deb_file))
+        subprocess.call(("reprepro", "includedeb", release, deb_file), timeout=120)
 
     """
         This is mostly useful to clear previously built db, dist and pool resources.
     """
     def _clean_build_resources(self):
-        subprocess.call(("reprepro", "clearvanished"))
+        subprocess.call(("reprepro", "clearvanished"), timeout=120)
+
+    # TODO https://jira.cfops.it/browse/TUN-6209 : Sign these packages.
+    def create_rpm_pkgs(self, artifacts_path):
+        self._setup_rpm_pkg_directories(artifacts_path)
+        subprocess.call(("createrepo", "./rpm"), timeout=120)
+
+    """
+        sets up the RPM directories in the following format:
+        - rpm 
+           - aarch64
+           - x86_64
+           - 386
+
+        this assumes the assets are in the format <prefix>-<aarch64/x86_64/386>.rpm
+    """
+    def _setup_rpm_pkg_directories(self, artifacts_path, archs=["aarch64", "x86_64", "386"]):
+        for arch in archs:
+            for root, _ , files in os.walk(artifacts_path):
+                for file in files:
+                    if file.endswith(f"{arch}.rpm"):
+                        new_dir = f"./rpm/{arch}"
+                        os.makedirs(new_dir, exist_ok=True)
+                        old_path = os.path.join(root, file)
+                        new_path = os.path.join(new_dir, file)
+                        shutil.copyfile(old_path, new_path)
 
 """
     Walks through a directory and uploads it's assets to R2.
@@ -167,6 +193,17 @@ def create_deb_packaging(pkg_creator, pkg_uploader, releases, gpg_key_id, binary
     upload_from_directories(pkg_uploader, "dists", release_version, binary_name)
     upload_from_directories(pkg_uploader, "pool", release_version, binary_name)
 
+def create_rpm_packaging(pkg_creator, pkg_uploader, artifacts_path, release_version, binary_name):
+    print(f"creating rpm pkgs...")
+    pkg_creator.create_rpm_pkgs(artifacts_path)
+
+    print("uploading latest to r2...")
+    upload_from_directories(pkg_uploader, "rpm", None, binary_name)
+
+    print(f"uploading versioned release {release_version} to r2...")
+    upload_from_directories(pkg_uploader, "rpm", release_version, binary_name)
+
+
 def parse_args():
     parser = argparse.ArgumentParser(
         description="Creates linux releases and uploads them in a packaged format"
@@ -223,3 +260,5 @@ def parse_args():
     pkg_uploader = PkgUploader(args.account, args.bucket, args.id, args.secret)
     create_deb_packaging(pkg_creator, pkg_uploader, args.deb_based_releases, args.gpg_key_id, args.binary, 
             args.archs, "main", args.release_tag)
+    
+    create_rpm_packaging(pkg_creator, pkg_uploader, "./built_artifacts", args.release_tag, args.binary )