Merge pull request #47 from craigcomstock/ENT-8978

ENT-8978: Added support for aarch64/arm64 packages

Author: olehermanse

cf_remote/commands.py

@@ -283,6 +283,7 @@ def _iterate_over_packages(tags=None, version=None, edition=None, download=False
     if version:
         release = releases.pick_version(version)
     print("Using {}:".format(release))
+    log.debug("Looking for a release based on host tags: {}".format(tags))
     artifacts = release.find(tags)
 
     if len(artifacts) == 0:

cf_remote/packages.py

@@ -86,7 +86,7 @@ def add_tags_from_filename(self, filename):
             part = part.strip()
             if part == "x86":
                 continue
-            if part == "amd64" or part == "x86_64":
+            if part == "amd64" or part == "x86_64" or part == "arm64":
                 self.add_tag("64")
             try:
                 _ = int(part)
@@ -97,13 +97,46 @@ def add_tags_from_filename(self, filename):
         else:
             self.add_tag("client")
             self.add_tag("agent")
+
+        # sorry, debian packages work great on ubuntu
+        # if we don't have a specific ubuntu package, note which debian is equivalent
+        # only add "ubuntu" tag if we know we have an equivalent debian :)
+        if "debian11" in self.tags:
+            self.add_tag("ubuntu20")
+            self.add_tag("ubuntu")
+        if "debian10" in self.tags:
+            self.add_tag("ubuntu19")
+            self.add_tag("ubuntu18")
+            self.add_tag("ubuntu")
+        if "debian9" in self.tags:
+            self.add_tag("ubuntu17")
+            self.add_tag("ubuntu16")
+            self.add_tag("ubuntu")
+        if "debian8" in self.tags:
+            self.add_tag("ubuntu15")
+            self.add_tag("ubuntu14")
+            self.add_tag("ubuntu")
+        if "debian7" in self.tags:
+            self.add_tag("ubuntu13")
+            self.add_tag("ubuntu12")
+            self.add_tag("ubuntu11")
+            self.add_tag("ubuntu")
+
         parts = filename.split(".")
         if "x86_64" in parts:
             self.add_tag("x86_64")
             self.add_tag("64")
         if "amd64" in parts:
             self.add_tag("amd64")
             self.add_tag("64")
+        if "arm64" in parts:
+            self.add_tag("aarch64")
+            self.add_tag("64")
+        log.debug(
+            "After looking at filename {}, tags for this package are {}".format(
+                filename, self.tags
+            )
+        )
 
     def __str__(self):
         return self.filename + " ({})".format(" ".join(self.tags))
@@ -115,7 +148,7 @@ def __repr__(self):
 def filter_artifacts(artifacts, tags, extension):
     if extension:
         artifacts = [a for a in artifacts if a.extension == extension]
-    log.debug("Looking for tags: {}".format(tags))
+    log.debug("Looking for host tags: {}".format(tags))
     log.debug("In artifacts: {}".format(artifacts))
     for tag in tags or []:
         tag = canonify(tag)
@@ -124,6 +157,9 @@ def filter_artifacts(artifacts, tags, extension):
         # since we are overwriting artifacts
         if len(new_artifacts) > 0:
             artifacts = new_artifacts
+        log.debug(
+            "Artifacts filtered on canonified tag {} are {}".format(tag, new_artifacts)
+        )
 
     log.debug("Found artifacts: {}".format(artifacts))
     return artifacts
@@ -135,12 +171,13 @@ def __init__(self, data):
         self.version = data["version"]
         self.url = data["URL"]
         self.lts = data["lts_branch"] if "lts_branch" in data else None
-        self.extended_data = None
+        self.extended_data = data["data"] if "data" in data else None
         self.artifacts = None
         self.default = False
 
     def init_download(self):
-        self.extended_data = get_json(self.url)
+        if self.extended_data is None:
+            self.extended_data = get_json(self.url)
         artifacts = self.extended_data["artifacts"]
         self.artifacts = []
         for header in artifacts:
@@ -150,7 +187,7 @@ def init_download(self):
                 self.artifacts.append(artifact)
 
     def find(self, tags, extension=None):
-        if not self.extended_data:
+        if self.extended_data is None:
             self.init_download()
         return filter_artifacts(self.artifacts, tags, extension)
 

cf_remote/remote.py

@@ -316,6 +316,7 @@ def _package_from_list(tags, extension, packages):
 
 
 def _package_from_releases(tags, extension, version, edition, remote_download):
+    log.debug("Looking for a package from releases based on host tags: {}".format(tags))
     releases = Releases(edition)
     release = releases.default
     if version:
@@ -360,7 +361,9 @@ def get_package_from_host_info(
     if edition == "enterprise":
         tags.append("hub" if hub else "agent")
 
-    tags.append("64" if arch in ("x86_64", "amd64") else arch)
+    tags.append("64" if arch in ("x86_64", "amd64", "aarch64") else arch)
+    if arch == "aarch64":
+        tags.append("arm64")
     if arch in ("i386", "i486", "i586", "i686"):
         tags.append("32")
 
@@ -375,7 +378,7 @@ def get_package_from_host_info(
     if package_tags is not None:
         tags.extend(tag for tag in package_tags if tag != "msi")
 
-    if packages is None:  # No commandd line argument given
+    if packages is None:  # No command line argument given
         package = _package_from_releases(
             tags, extension, version, edition, remote_download
         )

cf_remote/utils.py

@@ -89,10 +89,14 @@ def get_package_name(url):
     return url.rsplit("/", 1)[-1]
 
 
+def parse_json(text):
+    return json.loads(text, object_pairs_hook=OrderedDict)
+
+
 def read_json(path):
     try:
         with open(path, "r") as f:
-            return json.loads(f.read(), object_pairs_hook=OrderedDict)
+            return parse_json(f.read())
     except FileNotFoundError:
         return None
 

cf_remote/web.py

@@ -2,15 +2,15 @@
 import fcntl
 import urllib.request
 import requests
-from cf_remote.utils import write_json, mkdir
+from cf_remote.utils import write_json, mkdir, parse_json
 from cf_remote import log
 from cf_remote.paths import cf_remote_dir, cf_remote_packages_dir
 
 
 def get_json(url):
     r = requests.get(url)
     assert r.status_code >= 200 and r.status_code < 300
-    data = r.json()
+    data = parse_json(r.text)
 
     filename = os.path.basename(url)
     dir = cf_remote_dir("json")

run_tests.sh

@@ -0,0 +1,11 @@
+set -e
+set -x
+black --check cf_remote/*.py || true
+black --check tests/*.py || true
+if [ -z "$1" ]; then
+  python3 -m coverage run -m pytest
+  python3 -m coverage html
+else
+  # for debugging, getting all output, and running a single test method:
+  python3 -m pytest -k "$1" --show-capture=all -vv
+fi