Implement building from source package

Currently the build scripts require the LLVM and newlib sources to
be git repositories (e.g. the scripts query git commit hashes). This
patch adds a new mode for building the toolchain from a source package
in which the scripts, LLVM and newlib are bundled together.

The file versions.yml is now used for distinguishing between the two
modes. When the top-level entry 'SourceType' is set to
'standalone-build-scripts' the scripts will assume that LLVM and
newlib need to be checked out from git. Otherwise, if the value is set
to 'source-package' the scripts will assume that LLVM and newlib
sources are located in the same directory as the versions.yml file.

When building a source package the scripts generate versions.yml and
set SourceType to 'source-package'. Also, when build from a source
package the file VERSION.txt is copied from the source package rather
than generated on the fly.

The following command-line options are incompatible with the
'source-package' mode:
- prepare
- package-src
- --repositories-dir
- -r/--revision
If any of these options is specified when build from a source package
the scripts will ignore the option and print a warning.

Author: miyuki

scripts/build.py

@@ -47,8 +47,9 @@ def parse_args_to_config() -> Config:
     )
     parser.add_argument('-v', '--verbose', help='log more information',
                         action='store_true')
-    parser.add_argument('-r', '--revision', metavar='R', default='13.0.0',
-                        help='revision to build (default: 13.0.0)')
+    parser.add_argument('-r', '--revision', metavar='R',
+                        help='revision to build '
+                             '(default: {})'.format(config.DEFAULT_REVISION))
     variant_names = sorted(config.LIBRARY_SPECS.keys())
     parser.add_argument('--variants', metavar='VAR', nargs='+',
                         choices=variant_names + ['all'], default=['all'],
@@ -176,6 +177,7 @@ def prepare_repositories(cfg: config.Config,
     """Prepare source repositories according to the selected --checkout-mode
        option and the current state of repositories.
     """
+    assert not cfg.is_source_package
     patches_dir = os.path.join(cfg.source_dir, 'patches')
 
     # Determine which git action to perform
@@ -330,13 +332,15 @@ def ask_about_runtime_dlls(cfg: Config) -> Optional[bool]:
 def main() -> int:
     util.configure_logging()
     cfg = parse_args_to_config()
-    versions = repos.get_all_versions(os.path.join(cfg.source_dir,
-                                                   'versions.yml'))
-    if cfg.revision not in versions:
-        logging.error('Invalid revision %s', cfg.revision)
-        return 1
-
-    version = versions[cfg.revision]
+    if cfg.is_source_package:
+        version = None
+    else:
+        versions = repos.get_all_versions(os.path.join(cfg.source_dir,
+                                                       'versions.yml'))
+        if cfg.revision not in versions:
+            logging.error('Invalid revision %s', cfg.revision)
+            return 1
+        version = versions[cfg.revision]
 
     try:
         if (cfg.host_toolchain.kind == config.ToolchainKind.MINGW
@@ -348,8 +352,11 @@ def main() -> int:
 
         if not cfg.skip_checks:
             check.check_prerequisites(cfg)
-        run_or_skip(cfg, Action.PREPARE,
-                    lambda: prepare_repositories(cfg, version),
+
+        def do_prepare_repositories():
+            assert version is not None
+            prepare_repositories(cfg, version)
+        run_or_skip(cfg, Action.PREPARE, do_prepare_repositories,
                     'source code checkout')
         build_all(cfg)
         run_tests(cfg)
@@ -361,7 +368,8 @@ def do_source_package():
                     'source package')
 
         def do_binary_package():
-            version.poplulate_commits(cfg.repos_dir)
+            if version is not None:
+                version.poplulate_commits(cfg.repos_dir)
             package.create_binary_package(cfg, version)
         run_or_skip(cfg, Action.PACKAGE, do_binary_package, 'binary package')
 

scripts/config.py

@@ -21,6 +21,20 @@
 from typing import Optional, Set, TYPE_CHECKING
 
 import execution
+import util
+
+DEFAULT_REVISION = '13.0.0'
+
+
+@enum.unique
+class SourceType(enum.Enum):
+    """Enumeration for the SourceType value in versions.yml:
+       STANDALONE - the source contains just the build scripts, LLVM and newlib
+                    need to be checked out separately
+       SOURCE_PACKAGE - the source contains LLVM and newlib bundled with the
+                        build scripts"""
+    STANDALONE = 'standalone-build-scripts'
+    SOURCE_PACKAGE = 'source-package'
 
 
 @enum.unique
@@ -235,6 +249,11 @@ def _assign_dir(arg, default, rev):
     return os.path.abspath(res)
 
 
+def _warn_source_package_unsupported(feature: str) -> None:
+    logging.warning('%s is not supported when building from '
+                    'source package, ignoring', feature)
+
+
 class Config:  # pylint: disable=too-many-instance-attributes
     # pylint: disable=too-few-public-methods
     """Configuration for the whole build process"""
@@ -257,10 +276,20 @@ def _default_target(self):
             return 'aarch64-none-elf'
         return None
 
+    def _configure_source_type(self, args: argparse.Namespace):
+        self.source_dir = os.path.abspath(args.source_dir)
+        source_yaml = util.read_yaml(os.path.join(self.source_dir,
+                                                  'versions.yml'))
+        self.source_type = SourceType(source_yaml['SourceType'])
+        if self.source_type == SourceType.SOURCE_PACKAGE:
+            self.revision = source_yaml['Revision']
+
     def _configure_actions(self, args: argparse.Namespace):
         if not args.actions or Action.ALL.value in args.actions:
             # Actions that are not part of the "ALL" action:
             exclude_from_all = [Action.ALL, Action.TEST, Action.PACKAGE_SRC]
+            if self.is_source_package:
+                exclude_from_all.append(Action.PREPARE)
             self.actions = set(action for action in Action
                                if action not in exclude_from_all)
             for action in [Action.TEST, Action.PACKAGE_SRC]:
@@ -269,6 +298,13 @@ def _configure_actions(self, args: argparse.Namespace):
         else:
             self.actions = set(Action(act_str) for act_str in args.actions)
 
+        if self.is_source_package:
+            for action in [Action.PREPARE, Action.PACKAGE_SRC]:
+                if action.value in args.actions:
+                    _warn_source_package_unsupported(
+                        'action "{}"'.format(action.value))
+                    self.actions.remove(action)
+
     def _configure_toolchains(self, args: argparse.Namespace):
         # According to
         # https://docs.python.org/3.6/library/enum.html#using-a-custom-new:
@@ -292,18 +328,27 @@ def _configure_toolchains(self, args: argparse.Namespace):
         self.is_cross_compiling = (os.name == 'posix' and self.is_windows)
 
     def _fill_args(self, args: argparse.Namespace):
-        if 'all' in args.variants:
-            variant_names = LIBRARY_SPECS.keys()
-        else:
-            variant_names = set(args.variants)
+        variant_names = (LIBRARY_SPECS.keys() if 'all' in args.variants
+                         else set(args.variants))
         self.variants = [LIBRARY_SPECS[v] for v in sorted(variant_names)]
 
         self.default_target = self._default_target()
 
-        rev = args.revision
-        self.revision = rev
-        self.source_dir = os.path.abspath(args.source_dir)
-        self.repos_dir = _assign_dir(args.repositories_dir, 'repos', rev)
+        if self.is_source_package:
+            # For SOURCE_PACKAGE self.revision is set in _configure_source_type
+            assert self.revision is not None
+            rev = self.revision
+            self.repos_dir = self.source_dir
+            if args.repositories_dir is not None:
+                _warn_source_package_unsupported('--repositories-dir')
+            if args.revision is not None:
+                _warn_source_package_unsupported('--revision')
+        else:
+            rev = (args.revision if args.revision is not None
+                   else DEFAULT_REVISION)
+            self.revision = rev
+            self.repos_dir = _assign_dir(args.repositories_dir, 'repos', rev)
+
         self.build_dir = _assign_dir(args.build_dir, 'build', rev)
         self.install_dir = _assign_dir(args.install_dir, 'install', rev)
         self.package_dir = os.path.abspath(args.package_dir)
@@ -346,6 +391,14 @@ def copy_runtime_dlls(self) -> bool:
     def copy_runtime_dlls(self, value: bool) -> None:
         self._copy_runtime_dlls = value
 
+    @property
+    def is_source_package(self) -> bool:
+        """True iff building from a bundled source (i.e. build scripts, LLVM
+           and newlib) rather than a repository containing just the build
+           scripts."""
+        assert self.source_type is not None
+        return self.source_type == SourceType.SOURCE_PACKAGE
+
     def _fill_inferred(self):
         """Fill in additional fields that can be inferred from the
            configuration, but are still useful for convenience."""
@@ -386,6 +439,7 @@ def _fill_inferred(self):
 
     def __init__(self, args: argparse.Namespace):
         self._copy_runtime_dlls = None
+        self._configure_source_type(args)
         self._configure_actions(args)
         self._configure_toolchains(args)
         self._fill_args(args)

scripts/package.py

@@ -16,7 +16,7 @@
 import logging
 import os
 import shutil
-from typing import FrozenSet
+from typing import FrozenSet, Optional
 import tarfile
 import zipfile
 
@@ -30,9 +30,10 @@ def _write_version_file(cfg: config.Config, version: repos.LLVMBMTC,
                         target_dir: str) -> None:
     """Create VERSION.txt in the install directory."""
     dest = os.path.join(target_dir, 'VERSION.txt')
+    toolchain_ver = 'LLVM Embedded Toolchain for Arm ' + cfg.version_string
     if cfg.verbose:
-        logging.info('Writing "%s" to %s', cfg.version_string, dest)
-    lines = [cfg.version_string, '', 'Components:']
+        logging.info('Writing "%s" to %s', toolchain_ver, dest)
+    lines = [toolchain_ver, '', 'Components:']
     for name in sorted(version.modules.keys()):
         comp_info = version.modules[name].checkout_info
         lines.append('* {}'.format(comp_info))
@@ -41,6 +42,17 @@ def _write_version_file(cfg: config.Config, version: repos.LLVMBMTC,
     util.write_lines(lines, dest)
 
 
+def _write_versions_yml(cfg: config.Config, dest_dir: str) -> None:
+    """Create versions.yml for a source package."""
+    dest_file = os.path.join(dest_dir, 'versions.yml')
+    logging.info('Creating %s', dest_file)
+    with open(dest_file, 'wt') as out_f:
+        source_type = config.SourceType.SOURCE_PACKAGE.value
+        out_f.write('---\n'
+                    'SourceType: "{}"\n'
+                    'Revision: "{}"\n'.format(source_type, cfg.revision))
+
+
 def _copy_samples(cfg: config.Config) -> None:
     """Copy code samples, filter out files that are not usable on the target
        platform."""
@@ -136,9 +148,18 @@ def _create_archive(cfg: config.Config, pkg_src_dir, pkg_dest_name) -> None:
         raise util.ToolchainBuildError from ex
 
 
-def create_binary_package(cfg: config.Config, version: repos.LLVMBMTC) -> None:
+def create_binary_package(cfg: config.Config,
+                          version: Optional[repos.LLVMBMTC]) -> None:
     """Create a binary package with a newly built toolchain."""
-    _write_version_file(cfg, version, cfg.target_llvm_dir)
+    if cfg.is_source_package:
+        ver_file_src = os.path.join(cfg.source_dir, 'VERSION.txt')
+        ver_file_dest = os.path.join(cfg.target_llvm_dir, 'VERSION.txt')
+        logging.info('Copying version file from %s to %s', ver_file_src,
+                     ver_file_dest)
+        shutil.copy2(ver_file_src, ver_file_dest)
+    else:
+        assert version is not None
+        _write_version_file(cfg, version, cfg.target_llvm_dir)
     _copy_samples(cfg)
     _create_archive(cfg, cfg.target_llvm_dir, cfg.bin_package_base_name)
 
@@ -153,4 +174,5 @@ def create_source_package(cfg: config.Config, version: repos.LLVMBMTC) -> None:
     repos.export_toolchain_repositories(cfg.repos_dir, version,
                                         cfg.install_src_subdir)
     _write_version_file(cfg, version, cfg.install_src_subdir)
+    _write_versions_yml(cfg, cfg.install_src_subdir)
     _create_archive(cfg, cfg.install_src_subdir, cfg.src_package_base_name)

scripts/repos.py

@@ -25,7 +25,6 @@
 from typing import List, Mapping, Optional, Any
 
 import git  # type: ignore
-import yaml
 
 import util
 
@@ -158,17 +157,13 @@ def poplulate_commits(self, repos_dir: str) -> None:
 def get_all_versions(filename: str) -> Mapping[str, LLVMBMTC]:
     """Build the database containing all releases from a YAML file."""
     versions = {}
-    with open(filename, 'r') as stream:
-        try:
-            yml = yaml.load(stream, Loader=yaml.FullLoader)
-            for value in yml:
-                toolchain = LLVMBMTC(value)
-                if toolchain.revision in versions:
-                    die('toolchain revision {} previously defined'.format(
-                        toolchain.revision))
-                versions[toolchain.revision] = toolchain
-        except yaml.YAMLError as ex:
-            logging.error(ex)
+    yml = util.read_yaml(filename)
+    for value in yml['Revisions']:
+        toolchain = LLVMBMTC(value)
+        if toolchain.revision in versions:
+            die('toolchain revision {} previously defined'.format(
+                toolchain.revision))
+        versions[toolchain.revision] = toolchain
 
     return versions
 

scripts/util.py

@@ -16,6 +16,8 @@
 import logging
 from typing import Any, Sequence, List
 
+import yaml
+
 
 class ToolchainBuildError(RuntimeError):
     """A single error for all failures related to toolchain build: this
@@ -39,3 +41,13 @@ def configure_logging() -> None:
     """Set logging format and level threshold for the default logger."""
     log_format = '%(levelname)s: %(message)s'
     logging.basicConfig(format=log_format, level=logging.INFO)
+
+
+def read_yaml(path: str) -> Any:
+    """Read a YAML file and return its contents as a Python dict or list."""
+    with open(path, 'rt') as in_f:
+        try:
+            return yaml.load(in_f, Loader=yaml.FullLoader)
+        except yaml.YAMLError as ex:
+            logging.error(ex)
+            raise