Add support for sapling

Sapling is a somewhat novel SCM from Facebook / Meta.

It is fully compatible with Git and provides a more powerful
yet userfriendly frontend, modeled after mercurial.

This adds Sapling support in alibuild, including:

* Support for alidist sapling checkouts
* Support for development packages which were cloned via sapling

Author: ktf

.github/workflows/pr-check.yml

@@ -82,7 +82,7 @@ jobs:
 
       - name: Install test dependencies
         run: |
-          brew install modules alisw/system-deps/o2-full-deps
+          brew install modules alisw/system-deps/o2-full-deps sapling
           python3 -m pip install --upgrade tox tox-gh-actions coverage
 
       - name: Run tests

alibuild_helpers/build.py

@@ -14,6 +14,7 @@
 from alibuild_helpers.utilities import yamlDump
 from alibuild_helpers.utilities import resolve_tag, resolve_version
 from alibuild_helpers.git import git, clone_speedup_options, Git
+from alibuild_helpers.sl import sapling, Sapling
 from alibuild_helpers.sync import (NoRemoteSync, HttpRemoteSync, S3RemoteSync,
                                    Boto3RemoteSync, RsyncRemoteSync)
 import yaml
@@ -62,6 +63,10 @@ def update_git_repos(args, specs, buildOrder, develPkgs):
 
     def update_repo(package, git_prompt):
         specs[package]["scm"] = Git()
+        if package in develPkgs:
+          localCheckout = os.path.join(os.getcwd(), specs[package]["package"])
+          if exists("%s/.sl" % localCheckout):
+            specs[package]["scm"] = Sapling()
         updateReferenceRepoSpec(args.referenceSources, package, specs[package],
                                 fetch=args.fetchRepos,
                                 usePartialClone=not args.docker,
@@ -363,10 +368,19 @@ def doBuild(args, parser):
   if not exists(specDir):
     makedirs(specDir)
 
-  # By default Git is our SCM, so we find the commit hash of alidist
-  # using it.
-  scm = Git()
-  os.environ["ALIBUILD_ALIDIST_HASH"] = scm.checkedOutCommitName(directory=args.configDir)
+  # If the alidist workdir contains a .sl directory, we use Saplign as SCM
+  # otherwise we default to git (without checking for the actual presence of 
+  # .git). 
+  # We do it this way, because we have one test which uses an embedded folder 
+  # in the alibuild source, which therefore does not contain a .git directory
+  # and falls back to the alibuild git commit.
+  scm = exists("%s/.sl" % args.configDir) and Sapling() or Git()
+  try:
+    checkedOutCommitName = scm.checkedOutCommitName(directory=args.configDir)
+  except:
+    error("Cannot find SCM directory in %s.", args.configDir)
+    return 1
+  os.environ["ALIBUILD_ALIDIST_HASH"] = checkedOutCommitName
 
   debug("Building for architecture %s", args.architecture)
   debug("Number of parallel builds: %d", args.jobs)

alibuild_helpers/sl.py

@@ -0,0 +1,53 @@
+from shlex import quote  # Python 3.3+
+from alibuild_helpers.cmd import getstatusoutput
+from alibuild_helpers.log import debug
+from alibuild_helpers.scm import SCM
+
+SL_COMMAND_TIMEOUT_SEC = 120
+"""How many seconds to let any sl command execute before being terminated."""
+
+# Sapling is a novel SCM by Meta (i.e. Facebook) that is fully compatible with
+# git, but has a different command line interface. Among the reasons why it's
+# worth suporting it is the ability to handle unnamed branches, the ability to
+# absorb changes to the correct commit without having to explicitly rebase and
+# the integration with github to allow for pull requests to be created from the
+# command line from each commit of a branch.
+class Sapling(SCM):
+  name = "Sapling"
+  def checkedOutCommitName(self, directory):
+    return sapling(("whereami", ), directory)
+  def branchOrRef(self, directory):
+    # Format is <hash>[+] <branch>
+    identity = sapling(("identify", ), directory)
+    return identity.split(" ")[-1]
+  def exec(self, *args, **kwargs):
+    return sapling(*args, **kwargs)
+  def parseRefs(self, output):
+    return {
+      sl_ref: sl_hash for sl_ref, sep, sl_hash
+      in (line.partition("\t") for line in output.splitlines()) if sep
+    }
+  def listRefsCmd(self):
+    return ["bookmark", "--list", "--remote", "-R"]
+  def diffCmd(self, directory):
+    return "cd %s && sl diff && sl status" % directory
+  def checkUntracked(self, line):
+    return line.startswith("? ")
+
+def sapling(args, directory=".", check=True, prompt=True):
+  debug("Executing sl %s (in directory %s)", " ".join(args), directory)
+  # We can't use git --git-dir=%s/.git or git -C %s here as the former requires
+  # that the directory we're inspecting to be the root of a git directory, not
+  # just contained in one (and that breaks CI tests), and the latter isn't
+  # supported by the git version we have on slc6.
+  # Silence cd as shell configuration can cause the new directory to be echoed.
+  err, output = getstatusoutput("""\
+  set -e +x
+  sl -R {directory} {args}
+  """.format(
+    directory=quote(directory),
+    args=" ".join(map(quote, args)),
+  ), timeout=SL_COMMAND_TIMEOUT_SEC)
+  if check and err != 0:
+    raise RuntimeError("Error {} from sl {}: {}".format(err, " ".join(args), output))
+  return output if check else (err, output)

tests/test_build.py

@@ -181,6 +181,7 @@ def dummy_exists(x):
     return {
         "/alidist": True,
         "/alidist/.git": True,
+        "/alidist/.sl": False,
         "/sw": True,
         "/sw/SPECS": False,
         "/sw/MIRROR/root": True,

tox.ini

@@ -40,6 +40,8 @@ allowlist_externals =
     git
     test
     touch
+    sl
+    rm
 deps =
     py27: mock
     coverage
@@ -109,6 +111,14 @@ commands =
     touch zlib/foo
     coverage run --source={toxinidir} -a {toxinidir}/aliBuild -a {env:ARCHITECTURE} --no-system --disable GCC-Toolchain build zlib
     coverage run --source={envsitepackagesdir} -a -m unittest discover {toxinidir}/tests
+    # On Darwin we also test sapling support
+    darwin: sl clone https://github.com/alisw/alidist alidist-sapling
+    darwin: coverage run --source={toxinidir} -a {toxinidir}/aliBuild -a {env:ARCHITECTURE} -c alidist-sapling --no-system --disable GCC-Toolchain build zlib
+    darwin: rm -fr zlib
+    darwin: sl clone https://github.com/alisw/zlib
+    darwin: coverage run --source={toxinidir} -a {toxinidir}/aliBuild -a {env:ARCHITECTURE} --no-system --disable GCC-Toolchain build zlib
+    touch zlib/foo
+    darwin: coverage run --source={toxinidir} -a {toxinidir}/aliBuild -a {env:ARCHITECTURE} --no-system --disable GCC-Toolchain build zlib
 
 [coverage:run]
 branch = True