services/nomad/build: add buildbot nomad job

Author: classabbyamp

services/nomad/build/buildbot.cfg

@@ -0,0 +1,368 @@
+# vim: set ft=python:
+
+# {{ $allocID := env "NOMAD_ALLOC_ID" }}
+
+import configparser
+import json
+import shlex
+from pathlib import Path
+
+from twisted.internet import defer
+
+from buildbot.process.results import SUCCESS, SKIPPED
+from buildbot.plugins import util, secrets, reporters, worker, schedulers
+from buildbot.plugins import steps
+
+ini = configparser.ConfigParser()
+ini.read('/local/config.ini')
+
+with open(ini['buildbot'].get('workers', '/local/workers.json')) as f:
+    js = json.load(f)
+    workers = js.get("workers", [])
+    builders = js.get("builders", [])
+
+netauth = util.BuildbotNetAuth(conf=Path("/etc/netauth/config.toml"))
+
+authz = util.Authz(
+    allowRules=[
+        util.AnyEndpointMatcher(role="ops", defaultDeny=False),
+        util.AnyControlEndpointMatcher(role="ops"),
+    ],
+    roleMatchers=[
+        util.RolesFromGroups(groupPrefix="build-"),
+    ]
+)
+
+c = BuildmasterConfig = {
+    'buildbotNetUsageData': None,
+    'protocols': {'pb': {'port': ini['buildbot'].getint('worker-port', 9989)}},
+    'secretsProviders': [secrets.SecretInAFile(dirname="/secrets/buildbot")],
+    'workers': [],
+    'change_source': [],
+    'collapseRequests': True,
+    'schedulers': [],
+    'builders': [],
+    'services': [reporters.Prometheus(port=9100)],
+    'title': ini['buildbot'].get('title', 'Void Linux'),
+    'titleURL': ini['buildbot'].get('title-url', 'https://voidlinux.org/'),
+    'buildbotURL': ini['buildbot'].get('url', 'http://localhost:8010/'),
+    'www': {
+        'port': ini['buildbot'].getint('www-port', 8010),
+        'plugins': {'waterfall_view': {}, 'console_view': {}, 'grid_view': {}, 'void_view': {}},
+        'change_hook_dialects': {
+            'github': {
+                'secret': util.Secret('github-webhook'),
+                'strict': True,
+            },
+        },
+        'avatar_methods': [
+            netauth, util.AvatarGitHub(), util.AvatarGravatar(),
+        ],
+        'authz': authz,
+        'auth': netauth,
+        'ui_default_config': {
+            'Links.build_link_template': "%(build_number)",
+            'Waterfall.number_background_waterfall': True,
+            'Waterfall.show_builders_without_builds': True,
+            'Builders.show_workers_name': True,
+            'Grid.fullChanges': True,
+        },
+    },
+    'db': {
+        'db_url': ini['buildbot'].get('db-url', 'sqlite:////db/state.sqlite'),
+    },
+}
+
+if 'irc' in ini:
+    c['services'].append(reporters.IRC(
+        host=ini['irc']['host'],
+        port=ini['irc'].getint('port', 6697),
+        nick=ini['irc']['nick'],
+        password=util.Secret('irc-password'),
+        channels=[ini['irc']['channel']],
+        authz={'!': ini['irc'].get('authz-users', '').split(' ')},
+        notify_events=ini['irc'].get(
+            'notify-events', 'failure exception cancelled worker'
+        ).split(),
+        noticeOnChannel=ini['irc'].getboolean('notice', True),
+        # useRevisions and showBlameList are not implemented and will cause
+        # the irc notifier to error out, do NOT enable them
+        useRevisions=ini['irc'].getboolean('use-revisions', False),
+        showBlameList=ini['irc'].getboolean('show-blame', False),
+        useSSL=ini['irc'].getboolean('use-ssl', True),
+        useColors=ini['irc'].getboolean('use-colors', True),
+    ))
+
+# ###### WORKERS
+
+for w in workers:
+    name = 'worker-' + w['name']
+    passwd = util.Secret('worker-password')
+    max_builds = w.get('max-builds', 1)
+    c['workers'].append(worker.Worker(name, passwd, max_builds=max_builds))
+
+
+# ###### SCHEDULERS
+
+builder_names = []
+for b in builders:
+    builder_names.append(b['name'])
+
+c['schedulers'].append(schedulers.SingleBranchScheduler(
+    name="all",
+    change_filter=util.ChangeFilter(branch='master'),
+    treeStableTimer=None,
+    builderNames=builder_names))
+
+c['schedulers'].append(schedulers.ForceScheduler(
+    name="force",
+    builderNames=builder_names))
+
+
+# ###### BUILDERS
+
+distdir = 'void-packages'
+bulkdir = 'xbps-bulk'
+hostdir = '/hostdir'
+buildroot = 'buildroot'
+builddir = lambda: util.Interpolate('builddir-%(prop:buildnumber)s')
+do_sync = lambda: util.Interpolate('%(prop:sync)s') == "True"
+hide_skipped = lambda results, _: results == SKIPPED
+
+
+factory = util.BuildFactory()
+
+
+@util.renderer
+def make_xbps_src_cmd(props, cmd):
+    command = [
+        f'{distdir}/xbps-src',
+        '-H', hostdir,
+        '-m', buildroot,
+        '-A', props.getProperty('host'),
+    ]
+
+    if cmd == 'binary-bootstrap':
+        command += shlex.split(str(props.getProperty('bootstrap_args')))
+    elif props.getProperty('cross') == 'True':
+        command += ['-a', props.getProperty('target')]
+
+    command += [cmd]
+
+    return command
+
+
+@util.renderer
+def make_xbps_bulk_cmd(props):
+    command = [
+        f'../{bulkdir}/configure',
+        '-h', hostdir,
+        '-d', f'../{distdir}',
+        '-m', f'../{buildroot}',
+        '-t',
+    ]
+    if props.getProperty('cross') == 'True':
+        command += ['-a', props.getProperty('target')]
+    else:
+        command += ['-a', 'native-', props.getProperty('host')]
+
+    return command
+
+
+class ShellCommandWithChanges(steps.ShellCommand):
+    @defer.inlineCallbacks
+    def run(self):
+        cmd = yield self.makeRemoteShellCommand()
+        pkgs = [f.split("/")[-2] for f in self.build.allFiles()
+            if f.startswith("srcpkgs/") and f.endswith("/template")]
+        cmd.command += pkgs
+        yield self.runCommand(cmd)
+        return cmd.results()
+
+
+@util.renderer
+def build_packages(props):
+    # if a better solver is written
+    # cmds = []
+    # for p in str(props.getProperty('packages')).strip().split():
+    #     cmds.append(util.ShellArg(
+    #         command=['make', f'built/{p}'],
+    #         logname=f'pkg:{p}',
+    #         haltOnFailure=True,
+    #     ))
+    cmds = [util.ShellArg(
+        command=['make', 'all'],
+        logname='build',
+        haltOnFailure=True,
+    )]
+    if cmds:
+        cmds.append(util.ShellArg(
+            command=['make', 'clean'],
+            logname='cleanup',
+            haltOnFailure=True,
+        ))
+    return cmds
+
+
+@util.renderer
+def make_prune_cmd(props):
+    return ['bash', '-c',
+            util.Interpolate(f"""
+export XBPS_TARGET_ARCH="%(prop:target)s"
+for repo in / /debug /nonfree /bootstrap; do
+    xbps-rindex -r "{hostdir}/binpkgs/$repo"
+done
+if [ "$XBPS_TARGET_ARCH" = i686 ]; then
+    for repo in /multilib /multilib/nonfree /multilib/bootstrap; do
+        XBPS_TARGET_ARCH=x86_64 xbps-rindex -r "{hostdir}/binpkgs/$repo"
+    done
+fi
+""")]
+
+@util.renderer
+def make_rsync_cmd(props):
+    return ['bash', '-c',
+            util.Interpolate("""
+rsync -vurk --delete-after --delay-updates \
+--filter='+ */ + %(prop:target)s-repodata + %(prop:target)s-stagedata + *.%(prop:target)s.xbps + otime - .* - *' \
+--password-file=/secrets/rsync/password /hostdir/binpkgs/ \
+{{ range nomadService 1 $allocID "build-rsyncd" -}}
+rsync://buildsync-%(prop:worker)s@{{ .Address }}:{{ .Port }}/%(prop:worker)s
+{{ end -}}""")]
+
+
+factory.addStep(steps.Git(
+    repourl='https://github.com/void-linux/void-packages.git',
+    mode='incremental',
+    workdir=distdir,
+    progress=True,
+    alwaysUseLatest=True,
+    name='update_void_packages',
+    description='updating void-packages from git',
+    descriptionDone='void-packages updated',
+    haltOnFailure=True,
+    logEnviron=False,
+))
+
+factory.addStep(steps.Git(
+    repourl='https://github.com/void-linux/xbps-bulk.git',
+    mode='incremental',
+    workdir=bulkdir,
+    progress=True,
+    alwaysUseLatest=True,
+    name='update_xbps_bulk',
+    description='updating xbps-bulk from git',
+    descriptionDone='xbps-bulk updated',
+    haltOnFailure=True,
+    logEnviron=False,
+))
+
+factory.addStep(steps.ShellCommand(
+    command=make_xbps_src_cmd.withArgs('binary-bootstrap'),
+    name='bootstrap',
+    description='running xbps-src binary-bootstrap',
+    descriptionDone='xbps-src binary-bootstrap done',
+    haltOnFailure=True,
+    logEnviron=False,
+    usePTY=True,
+    workdir='.',
+))
+
+factory.addStep(steps.ShellCommand(
+    command=make_xbps_src_cmd.withArgs('bootstrap-update'),
+    name='bootstrap_update',
+    description='updating xbps-src bootstrap packages',
+    descriptionDone='xbps-src bootstrap-update done',
+    haltOnFailure=True,
+    logEnviron=False,
+    usePTY=True,
+    workdir='.',
+))
+
+factory.addStep(ShellCommandWithChanges(
+    command=make_xbps_bulk_cmd,
+    name='find_packages',
+    description='finding packages to build',
+    descriptionDone='found packages',
+    haltOnFailure=True,
+    logEnviron=False,
+    usePTY=True,
+    workdir=builddir(),
+))
+
+factory.addStep(steps.SetPropertyFromCommand(
+    command=['make', 'print_pkgs'],
+    property='packages',
+    name='get_packages',
+    description='collecting packages to build',
+    descriptionDone='collected packages',
+    haltOnFailure=True,
+    logEnviron=False,
+    workdir=builddir(),
+))
+
+factory.addStep(steps.ShellSequence(
+    commands=build_packages,
+    name='build_packages',
+    description='building packages',
+    descriptionDone='built packages',
+    haltOnFailure=True,
+    logEnviron=False,
+    usePTY=True,
+    workdir=builddir(),
+    timeout=14400,
+))
+
+factory.addStep(steps.ShellCommand(
+    command=make_prune_cmd,
+    name='prune_packages',
+    description='removing obsolete packages',
+    descriptionDone='removed obsolete packages',
+    haltOnFailure=True,
+    logEnviron=False,
+    usePTY=True,
+    workdir='.',
+    timeout=14400,
+))
+
+factory.addStep(steps.ShellCommand(
+    command=make_rsync_cmd,
+    name='sync_packages',
+    description='syncing packages to the shadow repository',
+    descriptionDone='synced packages to the shadow repository',
+    alwaysRun=True,
+    logEnviron=False,
+    usePTY=True,
+    workdir='.',
+    doStepIf=do_sync(),
+    hideStepIf=hide_skipped,
+    timeout=14400,
+    decodeRC={
+        0: SUCCESS,
+        23: SUCCESS,
+        24: SUCCESS,
+    },
+))
+
+for b in builders:
+    workernames = ["worker-" + b['worker']]
+
+    name = b['name']
+    hostarch = b['host']
+    targetarch = b.get('target', hostarch)
+    props = {
+        'name': name,
+        'host': hostarch,
+        'target': targetarch,
+        'cross': str(hostarch != targetarch),
+        'worker': b['worker'],
+        'sync': str(b['sync']),
+        'bootstrap_args': b.get('bootstrap_args', '-N'),
+    }
+
+    c['builders'].append(util.BuilderConfig(
+        name=name,
+        workernames=workernames,
+        factory=factory,
+        properties=props,
+    ))