asu: allow the usage of branches.yml

It's easier to write and understand. It's now possible to define a file
called `branches.yml` containing whatever branch information you'd like.

It's loaded from the config option BRANCHES_PATH if defined, if not the
internal branches.yml file is used.

To stay compatible with existing setups changes only happen if BRANCHES
wasn't previously defined, like it would be the case if still added in
the config.py

Signed-off-by: Paul Spooren <mail@aparcar.org>

Author: aparcar

asu/asu.py

@@ -3,9 +3,11 @@
 
 import connexion
 from flask import Flask, render_template, send_from_directory
+from pkg_resources import resource_filename
 from prometheus_client import CollectorRegistry, make_wsgi_app
 from redis import Redis
 from werkzeug.middleware.dispatcher import DispatcherMiddleware
+from yaml import safe_load
 
 import asu.common
 from asu import __version__
@@ -34,7 +36,6 @@ def create_app(test_config: dict = None) -> Flask:
         TESTING=False,
         DEBUG=False,
         UPSTREAM_URL="https://downloads.openwrt.org",
-        BRANCHES={},
         ALLOW_DEFAULTS=False,
         ASYNC_QUEUE=True,
     )

asu/branches.yml

@@ -0,0 +1,71 @@
+branches:
+  '21.02':
+    enabled: true
+    extra_keys: []
+    extra_repos: {}
+    git_branch: openwrt-21.02
+    name: '21.02'
+    path: releases/{version}
+    path_packages: releases/packages-{branch}
+    pubkey: RWQviwuY4IMGvwLfs6842A0m4EZU1IjczTxKMSk3BQP8DAQLHBwdQiaU
+    release_date: '2021-08-04'
+    repos:
+    - base
+    - packages
+    - luci
+    - routing
+    - telephony
+    snapshot: false
+    updates: features
+    versions:
+    - 21.02.3
+    - 21.02-SNAPSHOT
+
+  '22.03':
+    enabled: true
+    extra_keys: []
+    extra_repos: {}
+    git_branch: openwrt-22.03
+    name: '22.03'
+    path: releases/{version}
+    path_packages: releases/packages-{branch}
+    pubkey: RWRNAX5vHtXWFmt+n5di7XX8rTu0w+c8X7Ihv4oCyD6tzsUwmH0A6kO0
+    release_date: '2022-09-06'
+    repos:
+    - base
+    - packages
+    - luci
+    - routing
+    - telephony
+    snapshot: false
+    updates: features
+    versions:
+    - 22.03.0
+    - 22.03.0-rc6
+    - 22.03-SNAPSHOT
+    package_changes:
+      kmod-nft-nat6:
+      firewall: firewall4
+    
+  SNAPSHOT:
+    enabled: true
+    extra_keys: []
+    extra_repos: {}
+    git_branch: master
+    name: SNAPSHOT
+    path: snapshots
+    path_packages: snapshots/packages
+    pubkey: RWS1BD5w+adc3j2Hqg9+b66CvLR7NlHbsj7wjNVj0XGt/othDgIAOJS+
+    repos:
+    - base
+    - packages
+    - luci
+    - routing
+    - telephony
+    snapshot: true
+    updates: dev
+    versions:
+    - SNAPSHOT
+    package_changes:
+      kmod-nft-nat6:
+      firewall: firewall4

asu/build.py

@@ -9,12 +9,8 @@
 import requests
 from rq import get_current_job
 
-from .common import (
-    fingerprint_pubkey_usign,
-    get_file_hash,
-    get_packages_hash,
-    verify_usign,
-)
+from .common import (fingerprint_pubkey_usign, get_file_hash,
+                     get_packages_hash, verify_usign)
 
 log = logging.getLogger("rq.worker")
 log.setLevel(logging.DEBUG)
@@ -399,7 +395,9 @@ def download_file(filename: str, dest: str = None):
             if (cache / target_subtarget).exists():
                 rmtree(cache / target_subtarget)
                 for suffix in [".stamp", ".sha256sums", ".sha256sums.sig"]:
-                    (cache / target_subtarget).with_suffix(suffix).unlink(missing_ok=True)
+                    (cache / target_subtarget).with_suffix(suffix).unlink(
+                        missing_ok=True
+                    )
         else:
             log.debug("Keeping ImageBuilder for %s", target_subtarget)
 

misc/config.py

@@ -17,56 +17,3 @@
 JSON_PATH = Path.cwd() / "public/json/v1/"
 
 MAPPING_ABI = {"libubus20191227": "libubus"}
-
-# supported versions
-BRANCHES = {
-    "SNAPSHOT": {
-        "name": "SNAPSHOT",
-        "updates": "dev",
-        "enabled": True,
-        "snapshot": True,
-        "versions": ["SNAPSHOT"],
-        "git_branch": "master",
-        "path": "snapshots",
-        "path_packages": "snapshots/packages",
-        "pubkey": "RWS1BD5w+adc3j2Hqg9+b66CvLR7NlHbsj7wjNVj0XGt/othDgIAOJS+",
-        "repos": ["base", "packages", "luci", "routing", "telephony"],
-        "extra_repos": {},
-        "extra_keys": [],
-    },
-    "21.02": {
-        "name": "21.02",
-        "updates": "features",
-        "release_date": "2021-08-04",
-        "enabled": True,
-        "snapshot": False,
-        "versions": [
-            "21.02.0",
-            "21.02-SNAPSHOT",
-        ],
-        "git_branch": "openwrt-21.02",
-        "path": "releases/{version}",
-        "path_packages": "releases/packages-{branch}",
-        "pubkey": "RWQviwuY4IMGvwLfs6842A0m4EZU1IjczTxKMSk3BQP8DAQLHBwdQiaU",
-        "repos": ["base", "packages", "luci", "routing", "telephony"],
-        "extra_repos": {},
-        "extra_keys": [],
-    },
-    "19.07": {
-        "name": "19.07",
-        "updates": "security",
-        "release_date": "2021-08-07",
-        "enabled": False,
-        "versions": [
-            "19.07.8",
-            "19.07-SNAPSHOT",
-        ],
-        "git_branch": "openwrt-19.07",
-        "path": "releases/{version}",
-        "path_packages": "releases/packages-{branch}",
-        "pubkey": "RWT5S53W/rrJY9BiIod3JF04AZ/eU1xDpVOb+rjZzAQBEcoETGx8BXEK",
-        "repos": ["base", "packages", "luci", "routing", "telephony"],
-        "extra_repos": {},
-        "extra_keys": [],
-    },
-}

pyproject.toml

@@ -5,6 +5,7 @@ description = "An image on demand server for OpenWrt based distributions"
 authors = ["Paul Spooren <mail@aparcar.org>"]
 license = "GPL-2.0"
 readme = "README.md"
+include = ["asu/branches.yml"]
 
 [tool.poetry.dependencies]
 python = "^3.7"