FxCI: Fix dmg+pkg+msi enterprise repacks signature/notarization

Author: Alexandre Lissy

python/mozbuild/mozbuild/repackaging/utils.py

@@ -2,6 +2,7 @@
 # License, v. 2.0. If a copy of the MPL was not distributed with this file,
 # You can obtain one at http://mozilla.org/MPL/2.0/.
 
+import configparser
 import json
 import os
 import pathlib
@@ -165,14 +166,44 @@ def inject_desktop_entry_file(
 
 def inject_distribution_folder(source_dir, source_type, app_name):
     distribution_ini_path = mozpath.join(source_dir, source_type, "distribution.ini")
+    distribution_ini_dir = mozpath.join(source_dir, app_name.lower(), "distribution")
+    distribution_ini_target = mozpath.join(distribution_ini_dir, "distribution.ini")
 
     os.makedirs(
         mozpath.join(source_dir, app_name.lower(), "distribution"), exist_ok=True
     )
-    shutil.move(
-        distribution_ini_path,
-        mozpath.join(source_dir, app_name.lower(), "distribution"),
-    )
+
+    if not os.path.exists(distribution_ini_target):
+        shutil.move(
+            distribution_ini_path,
+            distribution_ini_dir,
+        )
+    else:
+        print(f"{distribution_ini_target} exists, merging")
+        existing_ini = configparser.ConfigParser()
+        existing_ini.read(distribution_ini_target)
+
+        debian_ini = configparser.ConfigParser()
+        debian_ini.read(distribution_ini_path)
+
+        for section in debian_ini.sections():
+            if "global" in section.lower():
+                continue
+
+            if "preferences" in section.lower():
+                if not section in existing_ini.sections():
+                    print(f"{distribution_ini_target} misses {section}")
+                    existing_ini[section] = {}
+
+                for pref in debian_ini[section]:
+                    if pref in existing_ini[section]:
+                        print(f"{distribution_ini_target} defines {pref}, skipping")
+                    else:
+                        print(f"{distribution_ini_target} adding {pref}")
+                        existing_ini[section][pref] = debian_ini[section][pref]
+
+        with open(distribution_ini_target, "w") as distribution_ini_modified:
+            existing_ini.write(distribution_ini_modified, space_around_delimiters=False)
 
 
 def inject_prefs_file(source_dir, app_name, template_dir):

taskcluster/docs/kinds.rst

@@ -978,3 +978,11 @@ Generates customized versions of releases for enterprises.
 enterprise-repack-repackage
 ---------------------------
 Repackage customized versions of releases for enterprises.
+
+enterprise-repack-mac-notarization
+----------------------------------
+Mac notarization of customized versions of releases for enterprises.
+
+enterprise-repack-mac-signing
+----------------------------------
+Mac signature of customized versions of releases for enterprises.

taskcluster/gecko_taskgraph/decision.py

@@ -119,6 +119,21 @@
         "target_tasks_method": "enterprise_firefox_with_tests_tasks",
         "release_type": "nightly-enterprise",
         "release_product": "firefox-enterprise",
+        "release_partners": ["sample"],
+        "release_partner_config": {
+            "enterprise-repack-repackage": {
+                "sample": {
+                    "gcpEU": {
+                        "locales": ["en-US"],
+                        "platforms": [
+                            "linux64-enterprise-shippable",
+                            "macosx64-enterprise-shippable",
+                            "win64-enterprise-shippable",
+                        ],
+                    },
+                },
+            },
+        },
     },
     # the default parameters are used for projects that do not match above.
     "default": {

taskcluster/gecko_taskgraph/transforms/partner_signing.py

@@ -42,6 +42,7 @@ def define_upstream_artifacts(config, jobs):
             copy_attributes_from_dependent_job(dep_job)
         )
 
+        print("define_upstream_artifacts", config.kind, job["label"])
         repack_ids = job["extra"]["repack_ids"]
         artifacts_specifications = generate_specifications_of_artifacts_to_sign(
             config,

taskcluster/gecko_taskgraph/transforms/repackage.py

@@ -7,7 +7,7 @@
 
 from taskgraph.transforms.base import TransformSequence
 from taskgraph.util.copy import deepcopy
-from taskgraph.util.dependencies import get_primary_dependency
+from taskgraph.util.dependencies import get_primary_dependency, get_dependencies
 from taskgraph.util.schema import Schema, optionally_keyed_by, resolve_keyed_by
 from taskgraph.util.taskcluster import get_artifact_prefix
 from voluptuous import Any, Extra, Optional, Required
@@ -416,6 +416,7 @@ def handle_keyed_by(config, jobs):
         "worker.max-run-time",
         "flatpak.name",
         "flatpak.branch",
+        "treeherder",
     ]
     for job in jobs:
         job = deepcopy(job)  # don't overwrite dict values here
@@ -487,14 +488,30 @@ def make_job_description(config, jobs):
             if "repack" in dep_job.label:
                 variant = f"{variant}-ent"
 
-            treeherder["symbol"] = f"Rpk({platform_simple}{variant})"
+            repack_id = dep_job.task.get("extra").get("repack_id")
+            if repack_id:
+                variant = f"-{repack_id}"
 
-            if "enterprise-repack" in dep_job.label:
+            treeherder["symbol"] = f"Rpk({platform_simple}{variant})"
+            
+            if "enterprise-repack" in dep_job.kind:
                 job["label"] = job["label"].replace(
                     "repackage", "repackage-enterprise-repack"
                 )
 
         dep_th_platform = dep_job.task.get("extra", {}).get("treeherder-platform")
+        # TODO: Hack because we loose the platform that was from enterprise-repack
+        if not dep_th_platform and "enterprise-repack-repackage" in dep_job.kind:
+            build_platform = attributes.get("build_platform")
+            if "linux64" in build_platform:
+                dep_th_platform = "linux64/opt"
+            elif "macosx64" in build_platform:
+                dep_th_platform = "osx-cross/opt"
+            elif "win64" in build_platform:
+                dep_th_platform = "windows2012-64/opt"
+            else:
+                raise ValueError(f"Unsupported {build_platform}")
+
         treeherder.setdefault("platform", dep_th_platform)
         treeherder.setdefault("tier", 1)
         treeherder.setdefault("kind", "build")
@@ -505,6 +522,8 @@ def make_job_description(config, jobs):
         for dependency in dependencies.keys():
             if "repackage-signing" in dependency:
                 repackage_signing_task = dependency
+            elif "enterprise-repack-repackage" in dependency:
+                repackage_signing_task = dependency
             elif "signing" in dependency or "notarization" in dependency:
                 signing_task = dependency
             elif "shippable-l10n" in dependency:
@@ -811,6 +830,8 @@ def _generate_download_config(
 
     if "enterprise-repack" in task.label:
         enterprise_repacks = task.attributes.get("enterprise_repacks", [])
+        if not enterprise_repacks:
+            enterprise_repacks = [ task.task.get("extra").get("repack_id") ]
         # TODO: Only one repack, gcpEU for now
         locale_path = f"{enterprise_repacks[0]}/"
 

taskcluster/gecko_taskgraph/transforms/repackage_partner.py

@@ -206,13 +206,15 @@ def make_job_description(config, jobs):
             )
         )
 
+        scopes = [] if "enterprise-repack" in job["label"] else ["queue:get-artifact:releng/partner/*"]
+
         task = {
             "label": job["label"],
             "description": description,
             "worker-type": worker_type,
             "dependencies": dependencies,
             "attributes": attributes,
-            "scopes": ["queue:get-artifact:releng/partner/*"],
+            "scopes": scopes,
             "run-on-projects": dep_job.attributes.get("run_on_projects"),
             "routes": job.get("routes", []),
             "extra": job.get("extra", {}),
@@ -228,6 +230,17 @@ def make_job_description(config, jobs):
             ),
         }
 
+        if "enterprise-repack-repackage" in job["label"]:
+            # TODO:
+            # Exception: repackage-enterprise-repack-msi-win64-enterprise-shippable/opt (tier 1) cannot depend on enterprise-repack-repackage-win64-enterprise-shippable/opt (tier unknown)
+            task["extra"]["treeherder"] = {
+                "platform": "linux64/opt",
+                "tier": 1,
+                "kind": "build",
+                "collection": {"opt": True},
+                "symbol": "Ent-Rpk-Rpk",
+            }
+
         # we may have reduced the priority for partner jobs, otherwise task.py will set it
         if job.get("priority"):
             task["priority"] = job["priority"]

taskcluster/gecko_taskgraph/util/partners.py

@@ -500,6 +500,7 @@ def get_partner_url_config(parameters, graph_config):
 
 def get_repack_ids_by_platform(config, build_platform):
     partner_config = get_partner_config_by_kind(config, config.kind)
+    print("get_repack_ids_by_platform", config.kind)
     combinations = []
     for partner, subconfigs in partner_config.items():
         for sub_config_name, sub_config in subconfigs.items():

taskcluster/kinds/enterprise-repack-mac-notarization/kind.yml

@@ -0,0 +1,29 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+---
+loader: taskgraph.loader.transform:loader
+
+transforms:
+    - taskgraph.transforms.from_deps
+    - gecko_taskgraph.transforms.mac_notarization
+    - gecko_taskgraph.transforms.chunk_partners
+    - gecko_taskgraph.transforms.partner_signing
+    - gecko_taskgraph.transforms.signing
+    - gecko_taskgraph.transforms.task
+
+kind-dependencies:
+    - enterprise-repack-mac-signing
+
+only-for-build-platforms:
+    - macosx64-shippable/opt
+    - macosx64-enterprise-shippable/opt
+
+tasks:
+    enterprise-repack-mac-notarization:
+        from-deps:
+            group-by: partner-repack-ids
+            copy-attributes: true
+        shipping-product: firefox
+        # TODO: shipping-phase: promote
+        copy-repack-ids: true

taskcluster/kinds/enterprise-repack-mac-signing/kind.yml

@@ -0,0 +1,28 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+---
+loader: taskgraph.loader.transform:loader
+
+transforms:
+    - taskgraph.transforms.from_deps
+    - gecko_taskgraph.transforms.chunk_partners
+    - gecko_taskgraph.transforms.name_sanity
+    - gecko_taskgraph.transforms.partner_signing
+    - gecko_taskgraph.transforms.signing
+    - gecko_taskgraph.transforms.hardened_signing
+    - gecko_taskgraph.transforms.task
+
+kind-dependencies:
+    - enterprise-repack
+
+only-for-build-platforms:
+    - macosx64-enterprise-shippable/opt
+
+tasks:
+    enterprise-repack-mac-signing:
+        from-deps:
+            group-by: partner-repack-ids
+        shipping-product: firefox
+        # TODO: shipping-phase: promote
+        repacks-per-chunk: 5

taskcluster/kinds/enterprise-repack-repackage/kind.yml

@@ -13,20 +13,13 @@ transforms:
     - gecko_taskgraph.transforms.job
     - gecko_taskgraph.transforms.task
 
-# Mac signing/notarization being listed here and not executed will mask
-# this task
 kind-dependencies:
     - enterprise-repack
     - toolchain
     - enterprise-repack-mac-signing
     - enterprise-repack-mac-notarization
 
-task-defaults:
-    run-on-repo-type: [git]
-    run-on-projects: ["enterprise-firefox"]
-
 only-for-build-platforms:
-    - linux64-enterprise-shippable/opt
     - macosx64-enterprise-shippable/opt
     - win64-enterprise-shippable/opt
 
@@ -47,23 +40,14 @@ tasks:
                     macosx64-.*:
                         - repackage/base.py
                         - repackage/osx_partner.py
-                    win32-.*:
-                        - repackage/base.py
-                        - repackage/win32_sfx_stub.py
-                        - repackage/win32_partner.py
                     win64-(?!aarch64).*:
                         - repackage/base.py
                         - repackage/win32_sfx_stub.py
                         - repackage/win64_partner.py
-                    win64-aarch64-.*:
-                        - repackage/base.py
-                        - repackage/win64-aarch64_sfx_stub.py
-                        - repackage/win64_partner.py
                     linux64.*:
                         - repackage/base.py
         package-formats:
             by-build-platform:
-                macosx64\b.*: [dmg]
+                macosx64\b.*: [dmg, pkg]
                 win32\b.*: [installer]
                 win64\b.*: [installer]
-                linux64\b.*: [tar]