Add an option to create a source package

The new action 'package-src' can be used to create a source package
consisting of:
- The LLVM Embedded Toolchain for Arm repository
- Patched LLVM and newlib repositories
- Version information

The build scripts cannot make use of the bundled LLVM and newlib
repositories, this functionality will be added in a follow-up patch.

Author: miyuki

scripts/build.py

@@ -164,6 +164,7 @@ def parse_args_to_config() -> Config:
                              '  configure - write target configuration files\n'
                              '  package - create binary package\n'
                              '  all - perform all of the above\n'
+                             '  package-src - create source package\n'
                              '  test - run tests\n'
                              'Default: all')
     args = parser.parse_args()
@@ -353,12 +354,17 @@ def main() -> int:
         build_all(cfg)
         run_tests(cfg)
 
-        def do_package():
+        def do_source_package():
             version.poplulate_commits(cfg.repos_dir)
-            package.write_version_file(cfg, version)
-            package.copy_samples(cfg)
-            package.package_toolchain(cfg)
-        run_or_skip(cfg, Action.PACKAGE, do_package, 'packaging')
+            package.create_source_package(cfg, version)
+        run_or_skip(cfg, Action.PACKAGE_SRC, do_source_package,
+                    'source package')
+
+        def do_binary_package():
+            version.poplulate_commits(cfg.repos_dir)
+            package.create_binary_package(cfg, version)
+        run_or_skip(cfg, Action.PACKAGE, do_binary_package, 'binary package')
+
     except util.ToolchainBuildError:
         # By this time the error must have already been logged
         return 1

scripts/config.py

@@ -112,8 +112,9 @@ class Action(enum.Enum):
     CONFIGURE = 'configure'
     PACKAGE = 'package'
     ALL = 'all'
-    # The 'test' phase is not part of 'all'
+    # The 'test' and 'package-src' phases are not part of 'all'
     TEST = 'test'
+    PACKAGE_SRC = 'package-src'
 
 
 @enum.unique
@@ -256,30 +257,19 @@ def _default_target(self):
             return 'aarch64-none-elf'
         return None
 
-    def _fill_args(self, args: argparse.Namespace):
-        if 'all' in args.variants:
-            variant_names = LIBRARY_SPECS.keys()
-        else:
-            variant_names = set(args.variants)
-        self.variants = [LIBRARY_SPECS[v] for v in sorted(variant_names)]
-
+    def _configure_actions(self, args: argparse.Namespace):
         if not args.actions or Action.ALL.value in args.actions:
+            # Actions that are not part of the "ALL" action:
+            exclude_from_all = [Action.ALL, Action.TEST, Action.PACKAGE_SRC]
             self.actions = set(action for action in Action
-                               if action not in (Action.ALL, Action.TEST))
-            if Action.TEST.value in args.actions:
-                self.actions.add(Action.TEST)
+                               if action not in exclude_from_all)
+            for action in [Action.TEST, Action.PACKAGE_SRC]:
+                if action.value in args.actions:
+                    self.actions.add(action)
         else:
             self.actions = set(Action(act_str) for act_str in args.actions)
 
-        self.default_target = self._default_target()
-
-        rev = args.revision
-        self.revision = rev
-        self.source_dir = os.path.abspath(args.source_dir)
-        self.repos_dir = _assign_dir(args.repositories_dir, 'repos', rev)
-        self.build_dir = _assign_dir(args.build_dir, 'build', rev)
-        self.install_dir = _assign_dir(args.install_dir, 'install', rev)
-        self.package_dir = os.path.abspath(args.package_dir)
+    def _configure_toolchains(self, args: argparse.Namespace):
         # According to
         # https://docs.python.org/3.6/library/enum.html#using-a-custom-new:
         # "The __new__() method, if defined, is used during creation of the
@@ -296,15 +286,33 @@ def _fill_args(self, args: argparse.Namespace):
         native_toolchain_dir = os.path.abspath(args.native_toolchain_dir)
         self.native_toolchain = Toolchain(native_toolchain_dir,
                                           native_toolchain_kind)
+
+        self.is_using_mingw = self.host_toolchain.kind == ToolchainKind.MINGW
+        self.is_windows = self.is_using_mingw
+        self.is_cross_compiling = (os.name == 'posix' and self.is_windows)
+
+    def _fill_args(self, args: argparse.Namespace):
+        if 'all' in args.variants:
+            variant_names = LIBRARY_SPECS.keys()
+        else:
+            variant_names = set(args.variants)
+        self.variants = [LIBRARY_SPECS[v] for v in sorted(variant_names)]
+
+        self.default_target = self._default_target()
+
+        rev = args.revision
+        self.revision = rev
+        self.source_dir = os.path.abspath(args.source_dir)
+        self.repos_dir = _assign_dir(args.repositories_dir, 'repos', rev)
+        self.build_dir = _assign_dir(args.build_dir, 'build', rev)
+        self.install_dir = _assign_dir(args.install_dir, 'install', rev)
+        self.package_dir = os.path.abspath(args.package_dir)
         self.checkout_mode = CheckoutMode(args.checkout_mode)
         self.build_mode = BuildMode(args.build_mode)
 
-        is_using_mingw = self.host_toolchain.kind == ToolchainKind.MINGW
-        is_windows = is_using_mingw
-
         copy_runtime = CopyRuntime(args.copy_runtime_dlls)
         self.ask_copy_runtime_dlls = True
-        if is_using_mingw:
+        if self.is_using_mingw:
             self.ask_copy_runtime_dlls = (copy_runtime == CopyRuntime.ASK)
             if not self.ask_copy_runtime_dlls:
                 self._copy_runtime_dlls = (copy_runtime == CopyRuntime.YES)
@@ -316,7 +324,7 @@ def _fill_args(self, args: argparse.Namespace):
             self._copy_runtime_dlls = False
 
         if args.package_format is None:
-            self.package_format = (PackageFormat.ZIP if is_windows else
+            self.package_format = (PackageFormat.ZIP if self.is_windows else
                                    PackageFormat.TGZ)
         else:
             self.package_format = PackageFormat(args.package_format)
@@ -344,9 +352,6 @@ def _fill_inferred(self):
         join = os.path.join
         self.llvm_repo_dir = join(self.repos_dir, 'llvm.git')
         self.newlib_repo_dir = join(self.repos_dir, 'newlib.git')
-        is_using_mingw = self.host_toolchain.kind == ToolchainKind.MINGW
-        self.is_cross_compiling = (os.name == 'posix' and is_using_mingw)
-        self.is_windows = is_using_mingw
         self.cmake_generator = 'Ninja' if self.use_ninja else 'Unix Makefiles'
         self.release_mode = self.revision != 'HEAD'
         if self.release_mode:
@@ -358,10 +363,14 @@ def _fill_inferred(self):
             self.version_string = now.strftime('%Y-%m-%d-%H:%M:%S')
         self.skip_reconfigure = self.build_mode == BuildMode.INCREMENTAL
         product_name = 'LLVMEmbeddedToolchainForArm'
-        self.package_base_name = product_name + version_suffix
+        self.bin_package_base_name = product_name + version_suffix
+        self.src_package_base_name = product_name + version_suffix + '-src'
         self.target_llvm_dir = join(
             self.install_dir,
             '{}-{}'.format(product_name, self.revision))
+        self.install_src_subdir = join(
+            self.install_dir,
+            '{}-{}-src'.format(product_name, self.revision))
         if self.is_cross_compiling:
             self.native_llvm_build_dir = join(self.build_dir, 'native-llvm')
             self.native_llvm_dir = join(self.install_dir, 'native-llvm')
@@ -377,5 +386,7 @@ def _fill_inferred(self):
 
     def __init__(self, args: argparse.Namespace):
         self._copy_runtime_dlls = None
+        self._configure_actions(args)
+        self._configure_toolchains(args)
         self._fill_args(args)
         self._fill_inferred()

scripts/package.py

@@ -26,9 +26,10 @@
 import util
 
 
-def write_version_file(cfg: config.Config, version: repos.LLVMBMTC) -> None:
+def _write_version_file(cfg: config.Config, version: repos.LLVMBMTC,
+                        target_dir: str) -> None:
     """Create VERSION.txt in the install directory."""
-    dest = os.path.join(cfg.target_llvm_dir, 'VERSION.txt')
+    dest = os.path.join(target_dir, 'VERSION.txt')
     if cfg.verbose:
         logging.info('Writing "%s" to %s', cfg.version_string, dest)
     lines = [cfg.version_string, '', 'Components:']
@@ -40,7 +41,7 @@ def write_version_file(cfg: config.Config, version: repos.LLVMBMTC) -> None:
     util.write_lines(lines, dest)
 
 
-def copy_samples(cfg: config.Config) -> None:
+def _copy_samples(cfg: config.Config) -> None:
     """Copy code samples, filter out files that are not usable on the target
        platform."""
     src = os.path.join(cfg.source_dir, 'samples')
@@ -76,7 +77,7 @@ def _get_excluded_symlinks(cfg: config.Config) -> FrozenSet[str]:
     return frozenset(excludes)
 
 
-def _create_tarball(cfg: config.Config, dest_pkg: str) -> None:
+def _create_tarball(cfg: config.Config, src_dir: str, dest_pkg: str) -> None:
     """Create a tarball package."""
     def reset_uid(tarinfo: tarfile.TarInfo) -> tarfile.TarInfo:
         tarinfo.uid = tarinfo.gid = 0
@@ -88,7 +89,7 @@ def reset_uid(tarinfo: tarfile.TarInfo) -> tarfile.TarInfo:
         exclude_set = _get_excluded_symlinks(cfg)
         dereference = True
     with tarfile.open(dest_pkg, 'w:gz') as dest:
-        for root, _, files in os.walk(cfg.target_llvm_dir):
+        for root, _, files in os.walk(src_dir):
             for fname in sorted(files):
                 if fname in exclude_set:
                     continue
@@ -102,11 +103,11 @@ def reset_uid(tarinfo: tarfile.TarInfo) -> tarfile.TarInfo:
                          filter=reset_uid)
 
 
-def _create_zip(cfg: config.Config, dest_pkg: str) -> None:
+def _create_zip(cfg: config.Config, src_dir: str, dest_pkg: str) -> None:
     """Create a zip package."""
     exclude_set = _get_excluded_symlinks(cfg)
     with zipfile.ZipFile(dest_pkg, 'w', zipfile.ZIP_DEFLATED) as dest:
-        for root, _, files in os.walk(cfg.target_llvm_dir):
+        for root, _, files in os.walk(src_dir):
             for fname in files:
                 if fname in exclude_set:
                     continue
@@ -117,20 +118,39 @@ def _create_zip(cfg: config.Config, dest_pkg: str) -> None:
                 dest.write(abs_path, arc_name)
 
 
-def package_toolchain(cfg: config.Config) -> None:
-    """Create a package with a newly built toolchain."""
+def _create_archive(cfg: config.Config, pkg_src_dir, pkg_dest_name) -> None:
+    """Create a package from a given directory."""
     if not os.path.exists(cfg.package_dir):
         os.makedirs(cfg.package_dir)
     format_mapping = {
         PackageFormat.TGZ: ('.tar.gz', _create_tarball),
         PackageFormat.ZIP: ('.zip', _create_zip),
     }
     package_ext, package_fn = format_mapping[cfg.package_format]
-    dest_pkg = os.path.join(cfg.package_dir,
-                            cfg.package_base_name + package_ext)
+    dest_pkg = os.path.join(cfg.package_dir, pkg_dest_name + package_ext)
     logging.info('Creating package %s', dest_pkg)
     try:
-        package_fn(cfg, dest_pkg)
+        package_fn(cfg, pkg_src_dir, dest_pkg)
     except RuntimeError as ex:
         logging.error('Failed to create %s', dest_pkg)
         raise util.ToolchainBuildError from ex
+
+
+def create_binary_package(cfg: config.Config, version: repos.LLVMBMTC) -> None:
+    """Create a binary package with a newly built toolchain."""
+    _write_version_file(cfg, version, cfg.target_llvm_dir)
+    _copy_samples(cfg)
+    _create_archive(cfg, cfg.target_llvm_dir, cfg.bin_package_base_name)
+
+
+def create_source_package(cfg: config.Config, version: repos.LLVMBMTC) -> None:
+    """Create a source package with a newly built toolchain."""
+    if os.path.exists(cfg.install_src_subdir):
+        logging.info('Removing existing directory %s', cfg.install_src_subdir)
+        shutil.rmtree(cfg.install_src_subdir)
+    os.makedirs(cfg.install_src_subdir)
+    repos.export_repository(cfg.source_dir, cfg.install_src_subdir, False)
+    repos.export_toolchain_repositories(cfg.repos_dir, version,
+                                        cfg.install_src_subdir)
+    _write_version_file(cfg, version, cfg.install_src_subdir)
+    _create_archive(cfg, cfg.install_src_subdir, cfg.src_package_base_name)

scripts/repos.py

@@ -20,6 +20,7 @@
 import argparse
 import logging
 import os
+import shutil
 import sys
 from typing import List, Mapping, Optional, Any
 
@@ -143,6 +144,9 @@ def __repr__(self):
 
     def poplulate_commits(self, repos_dir: str) -> None:
         """Populate commit hashes from checked out repositoties."""
+        # Check if commit hashes have already been set
+        if all(module.status is not None for module in self.modules.values()):
+            return
         status = get_repositories_status(repos_dir)
         for name, module in self.modules.items():
             if name not in status:
@@ -285,6 +289,34 @@ def clone_repositories(checkout_path: str, tc_version: LLVMBMTC,
     patch_repositories(checkout_path, tc_version, patches)
 
 
+def export_repository(src_repo_path: str, dest_repo_path: str,
+                      copy_untracked: bool = False) -> None:
+    """Copy the files of a checked out repository to a new directory. The copy
+       will include unstaged changes and optionally untracked files if
+       copy_untracked is True."""
+    logging.info('Exporting "%s" to "%s"', src_repo_path, dest_repo_path)
+    git_cmd = git.cmd.Git(src_repo_path)
+    args = ['--cached']
+    if copy_untracked:
+        args.append('--other')
+    for fname in git_cmd.ls_files(*args).splitlines():
+        src_path = os.path.join(src_repo_path, fname)
+        dest_path = os.path.join(dest_repo_path, fname)
+        dest_dir = os.path.dirname(dest_path)
+        if not os.path.exists(dest_dir):
+            os.makedirs(dest_dir)
+        shutil.copy2(src_path, dest_path, follow_symlinks=False)
+
+
+def export_toolchain_repositories(checkout_path: str, tc_version: LLVMBMTC,
+                                  dest_path: str) -> None:
+    """Export all checked out repositories."""
+    for repo_dir in tc_version.modules.keys():
+        export_repository(os.path.join(checkout_path, repo_dir),
+                          os.path.join(dest_path, repo_dir),
+                          True)
+
+
 def freeze_repositories(checkout_path: str, version: str) -> None:
     """Print a YAML compatible output of the repositories state."""
     statuses = get_repositories_status(checkout_path)