TUN-6395: Fix writing RPM repo data

Author: ipostelnik

release_pkgs.py

@@ -8,23 +8,24 @@
        them to be in an uploadable state.
     2. Upload these packages to a storage in a format that apt and yum expect.
 """
-from subprocess import Popen, PIPE
-import os
 import argparse
 import base64
-from pathlib import Path
 import logging
+import os
 import shutil
 from hashlib import sha256
+from pathlib import Path
+from subprocess import Popen, PIPE
 
-import gnupg
 import boto3
+import gnupg
 from botocore.client import Config
 from botocore.exceptions import ClientError
 
 # The front facing R2 URL to access assets from.
 R2_ASSET_URL = 'https://demo-r2-worker.cloudflare-tunnel.workers.dev/'
 
+
 class PkgUploader:
     def __init__(self, account_id, bucket_name, client_id, client_secret):
         self.account_id = account_id
@@ -35,14 +36,14 @@ def __init__(self, account_id, bucket_name, client_id, client_secret):
     def upload_pkg_to_r2(self, filename, upload_file_path):
         endpoint_url = f"https://{self.account_id}.r2.cloudflarestorage.com"
         token_secret_hash = sha256(self.client_secret.encode()).hexdigest()
-         
+
         config = Config(
-            region_name = 'auto',
+            region_name='auto',
             s3={
                 "addressing_style": "path",
             }
         )
-         
+
         r2 = boto3.client(
             "s3",
             endpoint_url=endpoint_url,
@@ -57,7 +58,7 @@ def upload_pkg_to_r2(self, filename, upload_file_path):
         except ClientError as e:
             raise e
 
-                  
+
 class PkgCreator:
     """
         The distribution conf is what dictates to reprepro, the debian packaging building
@@ -72,15 +73,16 @@ class PkgCreator:
         description - (String)
         gpg_key_id - gpg key id of what you want to use to sign the packages.(String) 
     """
-    def create_distribution_conf(self, 
-            file_path,
-            origin,
-            label,
-            releases,
-            archs,
-            components, 
-            description,
-            gpg_key_id ):
+
+    def create_distribution_conf(self,
+                                 file_path,
+                                 origin,
+                                 label,
+                                 releases,
+                                 archs,
+                                 components,
+                                 description,
+                                 gpg_key_id):
         with open(file_path, "w+") as distributions_file:
             for release in releases:
                 distributions_file.write(f"Origin: {origin}\n")
@@ -102,6 +104,7 @@ def create_distribution_conf(self,
         db and pool contain information and metadata about builds. We can ignore these.
         dist: contains all the pkgs and signed releases that are necessary for an apt download.
     """
+
     def create_deb_pkgs(self, release, deb_file):
         print(f"creating deb pkgs: {release} : {deb_file}")
         p = Popen(["reprepro", "includedeb", release, deb_file], stdout=PIPE, stderr=PIPE)
@@ -130,8 +133,9 @@ def create_rpm_pkgs(self, artifacts_path, gpg_key_name):
         gpgcheck=1
         gpgkey=https://pkg.cloudflare.com/cloudflare-main.gpg
     """
+
     def create_repo_file(self, file_path, binary_name, baseurl, gpgkey_url):
-        with open(file_path, "w+") as repo_file:
+        with open(os.path.join(file_path, binary_name + '.repo'), "w+") as repo_file:
             repo_file.write(f"[{binary_name}-stable]")
             repo_file.write(f"{binary_name}-stable")
             repo_file.write(f"baseurl={baseurl}/rpm")
@@ -141,19 +145,19 @@ def create_repo_file(self, file_path, binary_name, baseurl, gpgkey_url):
             repo_file.write(f"gpgkey={gpgkey_url}")
 
     def _sign_rpms(self, file_path):
-        p = Popen(["rpm" , "--define", f"_gpg_name {gpg_key_name}", "--addsign", file_path], stdout=PIPE, stderr=PIPE)
+        p = Popen(["rpm", "--define", f"_gpg_name {gpg_key_name}", "--addsign", file_path], stdout=PIPE, stderr=PIPE)
         out, err = p.communicate()
         if p.returncode != 0:
             print(f"rpm sign result result => {out}, {err}")
             raise
 
     def _sign_repomd(self):
-        p = Popen(["gpg", "--batch", "--detach-sign", "--armor", "./rpm/repodata/repomd.xml"], stdout=PIPE, stderr=PIPE) 
+        p = Popen(["gpg", "--batch", "--detach-sign", "--armor", "./rpm/repodata/repomd.xml"], stdout=PIPE, stderr=PIPE)
         out, err = p.communicate()
         if p.returncode != 0:
             print(f"sign repomd result => {out}, {err}")
             raise
-        
+
     """
         sets up and signs the RPM directories in the following format:
         - rpm 
@@ -163,9 +167,10 @@ def _sign_repomd(self):
 
         this assumes the assets are in the format <prefix>-<aarch64/x86_64/386>.rpm
     """
+
     def _setup_rpm_pkg_directories(self, artifacts_path, gpg_key_name, archs=["aarch64", "x86_64", "386"]):
         for arch in archs:
-            for root, _ , files in os.walk(artifacts_path):
+            for root, _, files in os.walk(artifacts_path):
                 for file in files:
                     if file.endswith(f"{arch}.rpm"):
                         new_dir = f"./rpm/{arch}"
@@ -174,11 +179,12 @@ def _setup_rpm_pkg_directories(self, artifacts_path, gpg_key_name, archs=["aarch
                         new_path = os.path.join(new_dir, file)
                         shutil.copyfile(old_path, new_path)
                         self._sign_rpms(new_path)
-    
+
     """
         imports gpg keys into the system so reprepro and createrepo can use it to sign packages.
         it returns the GPG ID after a successful import
     """
+
     def import_gpg_keys(self, private_key, public_key):
         gpg = gnupg.GPG()
         private_key = base64.b64decode(private_key)
@@ -192,12 +198,13 @@ def import_gpg_keys(self, private_key, public_key):
         basically rpm --import <key_file>
         This enables us to sign rpms.
     """
+
     def import_rpm_key(self, public_key):
         file_name = "pb.key"
         with open(file_name, "wb") as f:
             public_key = base64.b64decode(public_key)
             f.write(public_key)
-        
+
         p = Popen(["rpm", "--import", file_name], stdout=PIPE, stderr=PIPE)
         out, err = p.communicate()
         if p.returncode != 0:
@@ -212,18 +219,21 @@ def import_rpm_key(self, public_key):
               and the release will be uploaded to the default path. 
     binary: name of the binary to upload
 """
+
+
 def upload_from_directories(pkg_uploader, directory, release, binary):
-     for root, _ , files in os.walk(directory):
+    for root, _, files in os.walk(directory):
         for file in files:
             upload_file_name = os.path.join(binary, root, file)
             if release:
                 upload_file_name = os.path.join(release, upload_file_name)
-            filename = os.path.join(root,file)
-            try: 
+            filename = os.path.join(root, file)
+            try:
                 pkg_uploader.upload_pkg_to_r2(filename, upload_file_name)
             except ClientError as e:
                 logging.error(e)
-                return 
+                return
+
 
 """ 
     1. looks into a built_artifacts folder for cloudflared debs
@@ -237,19 +247,22 @@ def upload_from_directories(pkg_uploader, directory, release, binary):
 
     release_version: is the cloudflared release version. Only provide this if you want a permanent backup.
 """
-def create_deb_packaging(pkg_creator, pkg_uploader, releases, gpg_key_id, binary_name, archs, package_component, release_version):
+
+
+def create_deb_packaging(pkg_creator, pkg_uploader, releases, gpg_key_id, binary_name, archs, package_component,
+                         release_version):
     # set configuration for package creation.
     print(f"initialising configuration for {binary_name} , {archs}")
     Path("./conf").mkdir(parents=True, exist_ok=True)
     pkg_creator.create_distribution_conf(
-    "./conf/distributions",
-    binary_name,
-    binary_name,
-    releases,
-    archs,
-    package_component,
-    f"apt repository for {binary_name}",
-    gpg_key_id)
+        "./conf/distributions",
+        binary_name,
+        binary_name,
+        releases,
+        archs,
+        package_component,
+        f"apt repository for {binary_name}",
+        gpg_key_id)
 
     # create deb pkgs
     for release in releases:
@@ -266,6 +279,7 @@ def create_deb_packaging(pkg_creator, pkg_uploader, releases, gpg_key_id, binary
         upload_from_directories(pkg_uploader, "dists", release_version, binary_name)
         upload_from_directories(pkg_uploader, "pool", release_version, binary_name)
 
+
 def create_rpm_packaging(
         pkg_creator,
         pkg_uploader,
@@ -294,60 +308,61 @@ def parse_args():
     )
 
     parser.add_argument(
-            "--bucket", default=os.environ.get("R2_BUCKET"), help="R2 Bucket name"
+        "--bucket", default=os.environ.get("R2_BUCKET"), help="R2 Bucket name"
     )
     parser.add_argument(
-            "--id", default=os.environ.get("R2_CLIENT_ID"), help="R2 Client ID"
+        "--id", default=os.environ.get("R2_CLIENT_ID"), help="R2 Client ID"
     )
     parser.add_argument(
-            "--secret", default=os.environ.get("R2_CLIENT_SECRET"), help="R2 Client Secret"
+        "--secret", default=os.environ.get("R2_CLIENT_SECRET"), help="R2 Client Secret"
     )
     parser.add_argument(
-            "--account", default=os.environ.get("R2_ACCOUNT_ID"), help="R2 Account Tag"
+        "--account", default=os.environ.get("R2_ACCOUNT_ID"), help="R2 Account Tag"
     )
     parser.add_argument(
-            "--release-tag", default=os.environ.get("RELEASE_VERSION"), help="Release version you want your pkgs to be\
+        "--release-tag", default=os.environ.get("RELEASE_VERSION"), help="Release version you want your pkgs to be\
             prefixed with. Leave empty if you don't want tagged release versions backed up to R2."
     )
 
     parser.add_argument(
-           "--binary", default=os.environ.get("BINARY_NAME"), help="The name of the binary the packages are for"
+        "--binary", default=os.environ.get("BINARY_NAME"), help="The name of the binary the packages are for"
     )
 
     parser.add_argument(
-            "--gpg-private-key", default=os.environ.get("LINUX_SIGNING_PRIVATE_KEY"), help="GPG private key to sign the\
+        "--gpg-private-key", default=os.environ.get("LINUX_SIGNING_PRIVATE_KEY"), help="GPG private key to sign the\
             packages"
     )
 
     parser.add_argument(
-            "--gpg-public-key", default=os.environ.get("LINUX_SIGNING_PUBLIC_KEY"), help="GPG public key used for\
+        "--gpg-public-key", default=os.environ.get("LINUX_SIGNING_PUBLIC_KEY"), help="GPG public key used for\
             signing packages"
     )
 
     parser.add_argument(
-            "--gpg-public-key-url", default=os.environ.get("GPG_PUBLIC_KEY_URL"), help="GPG public key url that\
+        "--gpg-public-key-url", default=os.environ.get("GPG_PUBLIC_KEY_URL"), help="GPG public key url that\
             downloaders can use to verify signing"
     )
 
     parser.add_argument(
-            "--pkg-upload-url", default=os.environ.get("PKG_URL"), help="URL to be used by downloaders"
+        "--pkg-upload-url", default=os.environ.get("PKG_URL"), help="URL to be used by downloaders"
     )
 
     parser.add_argument(
-            "--deb-based-releases", default=["bookworm", "bullseye", "buster", "jammy", "impish", "focal", "bionic",
-            "xenial", "trusty"],
-            help="list of debian based releases that need to be packaged for"
+        "--deb-based-releases", default=["bookworm", "bullseye", "buster", "jammy", "impish", "focal", "bionic",
+                                         "xenial", "trusty"],
+        help="list of debian based releases that need to be packaged for"
     )
 
     parser.add_argument(
-            "--archs", default=["amd64", "386", "arm64", "arm"], help="list of architectures we want to package for. Note that\
+        "--archs", default=["amd64", "386", "arm64", "arm"], help="list of architectures we want to package for. Note that\
             it is the caller's responsiblity to ensure that these debs are already present in a directory. This script\
             will not build binaries or create their debs."
-            )
+    )
     args = parser.parse_args()
 
     return args
 
+
 if __name__ == "__main__":
     try:
         args = parse_args()
@@ -362,15 +377,15 @@ def parse_args():
     pkg_uploader = PkgUploader(args.account, args.bucket, args.id, args.secret)
     print(f"signing with gpg_key: {gpg_key_id}")
     create_deb_packaging(pkg_creator, pkg_uploader, args.deb_based_releases, gpg_key_id, args.binary, args.archs,
-            "main", args.release_tag)
-    
+                         "main", args.release_tag)
+
     create_rpm_packaging(
-            pkg_creator,
-            pkg_uploader,
-            "./built_artifacts",
-            args.release_tag,
-            args.binary,
-            gpg_key_name,
-            args.gpg_public_key_url,
-            args.pkg_upload_url,
-            )
+        pkg_creator,
+        pkg_uploader,
+        "./built_artifacts",
+        args.release_tag,
+        args.binary,
+        gpg_key_name,
+        args.gpg_public_key_url,
+        args.pkg_upload_url,
+    )