feat: move local manifest gen to py

* multi-line remotes/projects now work
* based on
  https://github.com/LineageOS/android_vendor_lineage/blob/ca60a89015cd910dd0d1dd77888e40a738ffc414/build/tools/roomservice.py
  and https://github.com/lineageos4microg/docker-lineage-cicd/blob/6c800342fa2ef4bb1cc8135323cee6cb811d69fa/src/build_manifest.py

Co-authored-by: Nicola Corna <[email protected]>
Co-authored-by: Jyotiraditya Panda <[email protected]>
Co-authored-by: tieu1991 <[email protected]>

Author: 4 people

TODO.md

@@ -1,4 +1,2 @@
 ## ToDo
-- multi-line remotes/repos in local manifests do not work for more than one manifest
 - automatic repo pulling for officially supported lineage devices
-- move xml handling to python

py-utils/xml_manifest_gen.py

@@ -0,0 +1,113 @@
+#!/usr/bin/python3
+
+from xml.etree import ElementTree
+import sys
+from urllib.request import urlopen, Request
+
+
+def is_in_manifest(manifest: str, project_path: str = "", project_remote: str = "") -> bool:
+    for manifest_project in manifest.findall("project"):
+        if project_path == manifest_project.get("path"):
+            return True
+
+    for manifest_project in manifest.findall("remote"):
+        if project_remote == manifest_project.get("name"):
+            return True
+
+    return False
+
+
+def add_project_to_manifest(manifest: str, project_name: str, project_path: str, project_remote: str = "",
+                            project_revision: str = "") -> str:
+    if is_in_manifest(manifest=manifest, project_path=project_path):
+        return manifest
+
+    element = ElementTree.Element(
+        "project",
+        attrib={
+            "name": project_name,
+            "path": project_path,
+        },
+    )
+
+    if project_remote:
+        element.attrib["remote"] = project_remote
+
+    if project_revision:
+        element.attrib["revision"] = project_revision
+
+    manifest.append(element)
+    return manifest
+
+
+def add_remote_to_manifest(manifest: str, remote_name: str, remote_fetch: str, remote_revision: str = "") -> str:
+    if is_in_manifest(manifest=manifest, project_remote=remote_name):
+        return manifest
+
+    element = ElementTree.Element(
+        "remote",
+        attrib={
+            "name": remote_name,
+            "fetch": remote_fetch,
+        },
+    )
+
+    if remote_revision:
+        element.attrib["revision"] = remote_revision
+
+    manifest.append(element)
+    return manifest
+
+def generate_manifest(local_manifest: str, remote_manifest: str) -> str:
+    for projects in remote_manifest.findall("remote"):
+        if projects.get("revision") == "":
+            revision = ""
+        else:
+            revision = projects.get("revision")
+
+        local_manifest = add_remote_to_manifest(
+            manifest=local_manifest,
+            remote_name=projects.get("name"),
+            remote_fetch=projects.get("fetch"),
+            remote_revision=revision
+        )
+
+    for projects in remote_manifest.findall("project"):
+        if projects.get("remote") == "":
+            remote = ""
+        else:
+            remote = projects.get("remote")
+
+        if projects.get("revision") == "":
+            revision = ""
+        else:
+            revision = projects.get("revision")
+
+        local_manifest = add_project_to_manifest(
+            manifest=local_manifest,
+            project_name=projects.get("name"),
+            project_path=projects.get("path"),
+            project_remote=remote,
+            project_revision=revision
+        )
+
+    ElementTree.indent(local_manifest)
+    return local_manifest
+
+def main() -> None:
+    local_manifest = ElementTree.Element("manifest")
+
+    for urls in sys.argv[1].split(","):
+        request = Request(urls, headers={"User-Agent": "Mozilla/5.0"})
+        source_manifest = urlopen(request).read()
+        remote_manifest = ElementTree.fromstring(source_manifest)
+
+        local_manifest = generate_manifest(local_manifest, remote_manifest)
+
+    print('<?xml version="1.0" encoding="UTF-8"?>')
+    print(ElementTree.tostring(local_manifest).decode())
+
+
+
+if __name__ == '__main__':
+    main()

scripts/sync.sh

@@ -16,13 +16,8 @@ _sync() {
   cd "${ROM_DIR}" || exit
   # Remove local manifests
   find "${ROM_DIR}"/.repo/local_manifests/ -type f -exec rm {} \;
-  if [[ -n "${LOCAL_MANIFEST}" ]]; then
-    if grep -q ',' <<< "${LOCAL_MANIFEST}"; then
-      _merge_local_manifests
-    else
-      curl_cmd "${LOCAL_MANIFEST}" --output "${ROM_DIR}"/.repo/local_manifests/manifest.xml
-    fi
-  fi
+  # Merge local manifests into one to avoid conflicts with duplicate dependencies
+  xml_manifest_gen.py "${LOCAL_MANIFEST}" > "${ROM_DIR}"/.repo/local_manifests/manifest.xml
   local threads
   threads=$(nproc)
   repo forall -c "rm .git/*.lock" || true
@@ -34,21 +29,6 @@ _sync() {
   unset ROM_MANIFEST LOCAL_MANIFEST CLONE_REPOS
 }
 
-# Merge local manifests into one
-# to avoid conflicts with duplicate dependencies
-_merge_local_manifests() {
-  echo -e '<?xml version="1.0" encoding="UTF-8"?>\n<manifest>' > "${ROM_DIR}"/.repo/local_manifests/manifest.xml
-  IFS=',' read -r -a "LOCAL_MANIFEST" <<< "${LOCAL_MANIFEST}"
-  for url in "${LOCAL_MANIFEST[@]}"; do
-    # Remove heading and end
-    curl_cmd "${url}" | sed '/<?xml version="1.0" encoding="UTF-8"?>/d; /<manifest>/d; /<\/manifest>/d; /<!--/d; /-->/d; /^$/d' >> "${ROM_DIR}"/.repo/local_manifests/.merge.txt
-  done
-  # Remove duplicated entries
-  sort < "${ROM_DIR}"/.repo/local_manifests/.merge.txt | uniq >> "${ROM_DIR}"/.repo/local_manifests/manifest.xml
-  find "${ROM_DIR}"/.repo/local_manifests/ -type f ! -name "*.xml" -exec rm -r {} \; || true
-  echo '</manifest>' >> "${ROM_DIR}"/.repo/local_manifests/manifest.xml
-}
-
 # Clone a repo
 _clone() {
   full_repo_name="${1}"