Merge pull request #3722 from tannewt/remove_file_urls

Update build board info.

Author: jepler

tools/build_board_info.py

@@ -19,20 +19,24 @@
 sys.path.append("adabot")
 import adabot.github_requests as github
 
-SUPPORTED_PORTS = ["atmel-samd", "cxd56", "esp32s2", "litex", "mimxrt10xx", "nrf", "stm"]
-
-BIN = ('bin',)
-UF2 = ('uf2',)
-BIN_UF2 = ('bin', 'uf2')
-HEX = ('hex',)
-HEX_UF2 = ('hex', 'uf2')
-SPK = ('spk',)
-DFU = ('dfu',)
-BIN_DFU = ('bin', 'dfu')
-
-# Example:
-# https://downloads.circuitpython.org/bin/trinket_m0/en_US/adafruit-circuitpython-trinket_m0-en_US-5.0.0-rc.0.uf2
-DOWNLOAD_BASE_URL = "https://downloads.circuitpython.org/bin"
+SUPPORTED_PORTS = [
+    "atmel-samd",
+    "cxd56",
+    "esp32s2",
+    "litex",
+    "mimxrt10xx",
+    "nrf",
+    "stm",
+]
+
+BIN = ("bin",)
+UF2 = ("uf2",)
+BIN_UF2 = ("bin", "uf2")
+HEX = ("hex",)
+HEX_UF2 = ("hex", "uf2")
+SPK = ("spk",)
+DFU = ("dfu",)
+BIN_DFU = ("bin", "dfu")
 
 # Default extensions
 extension_by_port = {
@@ -42,7 +46,7 @@
     "cxd56": SPK,
     "mimxrt10xx": HEX_UF2,
     "litex": DFU,
-    "esp32s2": BIN_UF2
+    "esp32s2": BIN_UF2,
 }
 
 # Per board overrides
@@ -57,33 +61,36 @@
     "feather_m0_rfm69": BIN_UF2,
     "feather_m0_rfm9x": BIN_UF2,
     "uchip": BIN_UF2,
-
     # nRF52840 dev kits that may not have UF2 bootloaders,
     "makerdiary_nrf52840_mdk": HEX,
     "makerdiary_nrf52840_mdk_usb_dongle": HEX_UF2,
     "pca10056": BIN_UF2,
     "pca10059": BIN_UF2,
     "electronut_labs_blip": HEX,
-
     # stm32
-    "meowbit_v121": UF2
+    "meowbit_v121": UF2,
 }
 
 aliases_by_board = {
-    "circuitplayground_express": ["circuitplayground_express_4h", "circuitplayground_express_digikey_pycon2019"],
+    "circuitplayground_express": [
+        "circuitplayground_express_4h",
+        "circuitplayground_express_digikey_pycon2019",
+    ],
     "pybadge": ["edgebadge"],
     "pyportal": ["pyportal_pynt"],
     "gemma_m0": ["gemma_m0_pycon2018"],
-    "pewpew10": ["pewpew13"]
+    "pewpew10": ["pewpew13"],
 }
 
+
 def get_languages():
     languages = []
     for f in os.scandir("../locale"):
         if f.name.endswith(".po"):
             languages.append(f.name[:-3])
     return languages
 
+
 def get_board_mapping():
     boards = {}
     for port in SUPPORTED_PORTS:
@@ -95,23 +102,30 @@ def get_board_mapping():
                 extensions = extension_by_port[port]
                 extensions = extension_by_board.get(board_path.name, extensions)
                 aliases = aliases_by_board.get(board_path.name, [])
-                boards[board_id] = {"port": port,
-                                    "extensions": extensions,
-                                    "download_count": 0,
-                                    "aliases": aliases}
+                boards[board_id] = {
+                    "port": port,
+                    "extensions": extensions,
+                    "download_count": 0,
+                    "aliases": aliases,
+                }
                 for alias in aliases:
-                    boards[alias] = {"port": port,
-                                     "extensions": extensions,
-                                     "download_count": 0,
-                                     "alias": True,
-                                     "aliases": []}
+                    boards[alias] = {
+                        "port": port,
+                        "extensions": extensions,
+                        "download_count": 0,
+                        "alias": True,
+                        "aliases": [],
+                    }
     return boards
 
+
 def get_version_info():
     version = None
     sha = git("rev-parse", "--short", "HEAD").stdout.decode("utf-8")
     try:
-        version = git("describe", "--tags", "--exact-match").stdout.decode("utf-8").strip()
+        version = (
+            git("describe", "--tags", "--exact-match").stdout.decode("utf-8").strip()
+        )
     except sh.ErrorReturnCode_128:
         # No exact match
         pass
@@ -120,18 +134,21 @@ def get_version_info():
         sha = os.environ["GITHUB_SHA"]
 
     if not version:
-        version="{}-{}".format(date.today().strftime("%Y%m%d"), sha[:7])
+        version = "{}-{}".format(date.today().strftime("%Y%m%d"), sha[:7])
 
     return sha, version
 
+
 def get_current_info():
     response = github.get("/repos/adafruit/circuitpython-org/git/refs/heads/master")
     if not response.ok:
         print(response.text)
         raise RuntimeError("cannot get master sha")
     commit_sha = response.json()["object"]["sha"]
 
-    response = github.get("/repos/adafruit/circuitpython-org/contents/_data/files.json?ref=" + commit_sha)
+    response = github.get(
+        "/repos/adafruit/circuitpython-org/contents/_data/files.json?ref=" + commit_sha
+    )
     if not response.ok:
         print(response.text)
         raise RuntimeError("cannot get previous files.json")
@@ -145,6 +162,7 @@ def get_current_info():
         current_info[info["id"]] = info
     return git_info, current_info
 
+
 def create_pr(changes, updated, git_info, user):
     commit_sha, original_blob_sha = git_info
     branch_name = "new_release_" + changes["new_release"]
@@ -158,7 +176,7 @@ def create_pr(changes, updated, git_info, user):
         updated_list.append(info)
 
     updated = json.dumps(updated_list, sort_keys=True, indent=1).encode("utf-8") + b"\n"
-    #print(updated.decode("utf-8"))
+    # print(updated.decode("utf-8"))
     pr_title = "Automated website update for release {}".format(changes["new_release"])
     boards = ""
     if changes["new_boards"]:
@@ -167,16 +185,13 @@ def create_pr(changes, updated, git_info, user):
     if changes["new_languages"]:
         languages = "New languages:\n* " + "\n* ".join(changes["new_languages"])
     message = "Automated website update for release {} by Blinka.\n\n{}\n\n{}\n".format(
-        changes["new_release"],
-        boards,
-        languages
+        changes["new_release"], boards, languages
     )
 
-    create_branch = {
-        "ref": "refs/heads/" + branch_name,
-        "sha": commit_sha
-    }
-    response = github.post("/repos/{}/circuitpython-org/git/refs".format(user), json=create_branch)
+    create_branch = {"ref": "refs/heads/" + branch_name, "sha": commit_sha}
+    response = github.post(
+        "/repos/{}/circuitpython-org/git/refs".format(user), json=create_branch
+    )
     if not response.ok and response.json()["message"] != "Reference already exists":
         print("unable to create branch")
         print(response.text)
@@ -186,10 +201,13 @@ def create_pr(changes, updated, git_info, user):
         "message": message,
         "content": base64.b64encode(updated).decode("utf-8"),
         "sha": original_blob_sha,
-        "branch": branch_name
+        "branch": branch_name,
     }
 
-    response = github.put("/repos/{}/circuitpython-org/contents/_data/files.json".format(user), json=update_file)
+    response = github.put(
+        "/repos/{}/circuitpython-org/contents/_data/files.json".format(user),
+        json=update_file,
+    )
     if not response.ok:
         print("unable to post new file")
         print(response.text)
@@ -199,7 +217,7 @@ def create_pr(changes, updated, git_info, user):
         "head": user + ":" + branch_name,
         "base": "master",
         "body": message,
-        "maintainer_can_modify": True
+        "maintainer_can_modify": True,
     }
     response = github.post("/repos/adafruit/circuitpython-org/pulls", json=pr_info)
     if not response.ok:
@@ -220,17 +238,14 @@ def print_active_user():
         print("Not logged in")
     return None
 
+
 def generate_download_info():
     boards = {}
     errors = []
 
     new_tag = os.environ["RELEASE_TAG"]
 
-    changes = {
-        "new_release": new_tag,
-        "new_boards": [],
-        "new_languages": []
-    }
+    changes = {"new_release": new_tag, "new_boards": [], "new_languages": []}
 
     user = print_active_user()
 
@@ -254,8 +269,9 @@ def generate_download_info():
         info = current_info[board]
         for version in info["versions"]:
             previous_releases.add(version["version"])
-            previous_languages.update(version["files"].keys())
-            if version["stable"] == new_stable:
+            if version["stable"] == new_stable or (
+                new_stable and version["version"].startswith(this_version)
+            ):
                 info["versions"].remove(version)
 
     board_mapping = get_board_mapping()
@@ -272,29 +288,15 @@ def generate_download_info():
                     alias_info = board_mapping[alias]
                     if alias not in current_info:
                         changes["new_boards"].append(alias)
-                        current_info[alias] = {"downloads": 0,
-                                                "versions": []}
+                        current_info[alias] = {"downloads": 0, "versions": []}
 
                     new_version = {
                         "stable": new_stable,
                         "version": new_tag,
                         "modules": support_matrix.get(alias, "[]"),
-                        "files": {},
                         "languages": languages,
-                        "extensions": board_info["extensions"]
+                        "extensions": board_info["extensions"],
                     }
-                    for language in languages:
-                        files = []
-                        new_version["files"][language] = files
-                        for extension in board_info["extensions"]:
-                            files.append(
-                                "{base_url}/{alias}/{language}/adafruit-circuitpython-{alias}-{language}-{tag}.{extension}"
-                                .format(
-                                    base_url=DOWNLOAD_BASE_URL,
-                                    tag=new_tag,
-                                    alias=alias,
-                                    language=language,
-                                    extension=extension))
                     current_info[alias]["downloads"] = alias_info["download_count"]
                     current_info[alias]["versions"].append(new_version)
 
@@ -305,6 +307,7 @@ def generate_download_info():
     else:
         print("No new release to update")
 
+
 if __name__ == "__main__":
     if "RELEASE_TAG" in os.environ and os.environ["RELEASE_TAG"]:
         generate_download_info()