Merge pull request #38 from rancher-sandbox/provision

Implement system and user provisioning scripts

Author: AkihiroSuda

examples/k3s.yaml

@@ -0,0 +1,42 @@
+# Deploy kubernetes via k3s (which installs a bundled containerd).
+#
+# It can be accessed from the host by exporting the kubeconfig file;
+# the ports are already forwarded automatically by lima:
+#
+# $ export KUBECONFIG=$PWD/kubeconfig.yaml
+# $ limactl shell k3s sudo cat /etc/rancher/k3s/k3s.yaml >$KUBECONFIG
+# $ kubectl get no
+# NAME       STATUS   ROLES                  AGE   VERSION
+# lima-k3s   Ready    control-plane,master   69s   v1.21.1+k3s1
+
+images:
+- location: "~/Downloads/hirsute-server-cloudimg-amd64.img"
+  arch: "x86_64"
+
+mounts: []
+
+ssh:
+  localPort: 60022
+
+containerd:
+  system: false
+  user: false
+
+provision:
+- mode: system
+  script: |
+    #!/bin/sh
+    curl -sfL https://get.k3s.io | sh -
+
+probes:
+- script: |
+    #!/bin/bash
+    set -eux -o pipefail
+    if ! timeout 30s bash -c "until test -f /etc/rancher/k3s/k3s.yaml; do sleep 3; done"; then
+            echo >&2 "k3s is not running yet"
+            exit 1
+    fi
+  hint: |
+    The k3s kubeconfig file has not yet been created.
+    Run "lima bash sudo journalctl -u k3s" to check the log.
+    If that is still empty, check the bottom of the log at "/var/log/cloud-init-output.log".

pkg/cidata/cidata.go

@@ -29,9 +29,11 @@ func GenerateISO9660(isoPath, name string, y *limayaml.LimaYAML) error {
 		return err
 	}
 	args := TemplateArgs{
-		Name: name,
-		User: u.Username,
-		UID:  uid,
+		Name:       name,
+		User:       u.Username,
+		UID:        uid,
+		Provision:  y.Provision,
+		Containerd: Containerd{System: *y.Containerd.System, User: *y.Containerd.User},
 	}
 	for _, f := range sshutil.DefaultPubKeys() {
 		args.SSHPubKeys = append(args.SSHPubKeys, f.Content)

pkg/cidata/template.go

@@ -4,6 +4,8 @@ import (
 	_ "embed"
 	"path/filepath"
 
+	"github.com/AkihiroSuda/lima/pkg/limayaml"
+
 	"github.com/AkihiroSuda/lima/pkg/templateutil"
 	"github.com/containerd/containerd/identifiers"
 	"github.com/pkg/errors"
@@ -16,12 +18,18 @@ var (
 	metaDataTemplate string
 )
 
+type Containerd struct {
+	System bool
+	User   bool
+}
 type TemplateArgs struct {
 	Name       string // instance name
 	User       string // user name
 	UID        int
 	SSHPubKeys []string
 	Mounts     []string // abs path, accessible by the User
+	Provision  []limayaml.Provision
+	Containerd Containerd
 }
 
 func ValidateTemplateArgs(args TemplateArgs) error {

pkg/cidata/user-data.TEMPLATE

@@ -13,15 +13,16 @@ users:
     sudo: ALL=(ALL) NOPASSWD:ALL
     lock_passwd: true
     ssh-authorized-keys:
-{{range $val := .SSHPubKeys}}
+    {{- range $val := .SSHPubKeys}}
       - {{$val}}
-{{end}}
+    {{- end}}
 
 write_files:
  - content: |
       #!/bin/bash
       set -eux -o pipefail
 
+      {{- if .Containerd.User}}
       # Enable rootless containers
       if [ ! -e "/etc/systemd/system/[email protected]/lima.conf" ]; then
         mkdir -p "/etc/systemd/system/[email protected]"
@@ -47,12 +48,13 @@ write_files:
         grep -qw "{{.User}}" $f || echo "{{.User}}:100000:65536" >> $f
       done
       loginctl enable-linger "{{.User}}"
+      {{- end}}
 
       # Create mount points
-      {{range $val := .Mounts}}
+      {{- range $val := .Mounts}}
       mkdir -p "{{$val}}"
       chown "{{$.User}}" "{{$val}}" || true
-      {{end}}
+      {{- end}}
 
       # Install or update the guestagent binary
       mkdir -p -m 600 /mnt/lima-cidata
@@ -67,23 +69,50 @@ write_files:
    # We do not use per-once.
    path: /var/lib/cloud/scripts/per-boot/00-base.boot.sh
    permissions: '0755'
+ {{- if or .Mounts .Containerd.User}}
  - content: |
       #!/bin/bash
       set -eux -o pipefail
       # Install minimum dependencies
       if command -v apt-get 2>&1 >/dev/null; then
         export DEBIAN_FRONTEND=noninteractive
         apt-get update
-        apt-get install -y sshfs uidmap
+        {{- if .Mounts}}
+        apt-get install -y sshfs
+        {{- end }}
+        {{- if .Containerd.User}}
+        apt-get install -y uidmap
+        {{- end }}
       elif command -v dnf 2>&1 >/dev/null; then
-        dnf install -y fuse-sshfs shadow-utils
+        : {{/* make sure the "elif" block is never empty */}}
+        {{- if .Mounts}}
+        dnf install -y fuse-sshfs
+        {{- end}}
+        {{- if .Containerd.User}}
+        dnf install -y shadow-utils
+        {{- end}}
       fi
    owner: root:root
    path: /var/lib/cloud/scripts/per-boot/10-install-packages.boot.sh
    permissions: '0755'
+{{- end}}
+{{- if or .Containerd.System .Containerd.User}}
  - content: |
       #!/bin/bash
       set -eux -o pipefail
+      if [ ! -x /usr/local/bin/nerdctl ]; then
+        version="0.8.3"
+        goarch="amd64"
+        if [ "$(uname -m )" = "aarch64 "]; then
+          goarch="arm64"
+        fi
+        curl -fsSL https://github.com/containerd/nerdctl/releases/download/v${version}/nerdctl-full-${version}-linux-${goarch}.tar.gz | tar Cxz /usr/local
+      fi
+      {{- if .Containerd.System}}
+      systemctl enable containerd
+      systemctl start containerd
+      {{- end}}
+      {{- if .Containerd.User}}
       modprobe tap || true
       if [ ! -e "/home/{{.User}}.linux/.config/containerd/config.toml" ]; then
         mkdir -p "/home/{{.User}}.linux/.config/containerd"
@@ -95,18 +124,37 @@ write_files:
       EOF
         chown -R "{{.User}}" "/home/{{.User}}.linux/.config"
       fi
-      if [ ! -x /usr/local/bin/nerdctl ]; then
-        version="0.8.3"
-        goarch="amd64"
-        if [ "$(uname -m )" = "aarch64 "]; then
-          goarch="arm64"
-        fi
-        curl -fsSL https://github.com/containerd/nerdctl/releases/download/v${version}/nerdctl-full-${version}-linux-${goarch}.tar.gz | tar Cxz /usr/local
+      if [ ! -e "/home/{{.User}}}}.linux/.config/systemd/user/containerd.service" ]; then
         until [ -e "/run/user/{{.UID}}/systemd/private" ]; do sleep 3; done
         sudo -iu "{{.User}}" "XDG_RUNTIME_DIR=/run/user/{{.UID}}" containerd-rootless-setuptool.sh install
         sudo -iu "{{.User}}" "XDG_RUNTIME_DIR=/run/user/{{.UID}}" containerd-rootless-setuptool.sh install-buildkit
         sudo -iu "{{.User}}" "XDG_RUNTIME_DIR=/run/user/{{.UID}}" containerd-rootless-setuptool.sh install-stargz
       fi
+      {{- end}}
    owner: root:root
    path: /var/lib/cloud/scripts/per-boot/20-install-containerd.boot.sh
    permissions: '0755'
+{{- end}}
+{{- if .Provision}}
+ - content: |
+      #!/bin/bash
+      set -eu -o pipefail
+      {{- range $i, $val := .Provision}}
+      {{- $script := printf "/var/lib/lima-guestagent/provision-%02d-%s" $i $val.Mode}}
+      {{- if eq $val.Mode "system"}}
+      {{$script}}
+      {{- else}}
+      until [ -e "/run/user/{{.UID}}/systemd/private" ]; do sleep 3; done
+      sudo -iu "{{.User}}" "XDG_RUNTIME_DIR=/run/user/{{.UID}}" {{$script}}
+      {{- end}}
+      {{- end}}
+   owner: root:root
+   path: /var/lib/cloud/scripts/per-boot/30-execute-provision-scripts.boot.sh
+   permissions: '0755'
+{{- end}}
+{{- range $i, $val := .Provision}}
+ - content: {{printf "%q" $val.Script}}
+   owner: root:root
+   path: {{printf "/var/lib/lima-guestagent/provision-%02d-%s" $i $val.Mode}}
+   permissions: '0755'
+{{- end}}

pkg/hostagent/hostagent.go

@@ -54,7 +54,7 @@ func (a *HostAgent) Run(ctx context.Context) error {
 		return nil
 	})
 	var mErr error
-	if err := a.waitForRequirements(ctx, "essential", essentialRequirements); err != nil {
+	if err := a.waitForRequirements(ctx, "essential", a.essentialRequirements()); err != nil {
 		mErr = multierror.Append(mErr, err)
 	}
 	mounts, err := a.setupMounts(ctx)
@@ -71,7 +71,7 @@ func (a *HostAgent) Run(ctx context.Context) error {
 		return unmountMErr
 	})
 	go a.watchGuestAgentEvents(ctx)
-	if err := a.waitForRequirements(ctx, "optional", optionalRequirements); err != nil {
+	if err := a.waitForRequirements(ctx, "optional", a.optionalRequirements()); err != nil {
 		mErr = multierror.Append(mErr, err)
 	}
 	return mErr

pkg/hostagent/requirements.go

@@ -2,6 +2,7 @@ package hostagent
 
 import (
 	"context"
+	"github.com/AkihiroSuda/lima/pkg/limayaml"
 	"time"
 
 	"github.com/AkihiroSuda/sshocker/pkg/ssh"
@@ -54,8 +55,9 @@ type requirement struct {
 	debugHint   string
 }
 
-var essentialRequirements = []requirement{
-	{
+func (a *HostAgent) essentialRequirements() []requirement {
+	req := make([]requirement, 0)
+	req = append(req, requirement{
 		description: "ssh",
 		script: `#!/bin/bash
 true
@@ -64,23 +66,25 @@ true
 Make sure that the YAML field "ssh.localPort" is not used by other processes on the host.
 If any private key under ~/.ssh is protected with a passphrase, you need to have ssh-agent to be running.
 `,
-	},
-	{
-		description: "sshfs binary to be installed",
-		script: `#!/bin/bash
+	})
+	if len(a.y.Mounts) > 0 {
+		req = append(req, requirement{
+			description: "sshfs binary to be installed",
+			script: `#!/bin/bash
 set -eux -o pipefail
 if ! timeout 30s bash -c "until command -v sshfs; do sleep 3; done"; then
 	echo >&2 "sshfs is not installed yet"
 	exit 1
 fi
 `,
-		debugHint: `The sshfs binary was not installed in the guest.
+			debugHint: `The sshfs binary was not installed in the guest.
 Make sure that you are using an officially supported image.
 Also see "/var/log/cloud-init-output.log" in the guest.
 A possible workaround is to run "apt-get install sshfs" in the guest.
 `,
-	},
-	{
+		})
+	}
+	req = append(req, requirement{
 		description: "the guest agent to be running",
 		script: `#!/bin/bash
 set -eux -o pipefail
@@ -95,22 +99,36 @@ Make sure that you are using an officially supported image.
 Also see "/var/log/cloud-init-output.log" in the guest.
 A possible workaround is to run "lima-guestagent install-systemd" in the guest.
 `,
-	},
+	})
+	return req
 }
 
-var optionalRequirements = []requirement{
-	{
-		description: "containerd binaries to be installed",
-		script: `#!/bin/bash
+func (a *HostAgent) optionalRequirements() []requirement {
+	req := make([]requirement, 0)
+	if *a.y.Containerd.System || *a.y.Containerd.User {
+		req = append(req, requirement{
+			description: "containerd binaries to be installed",
+			script: `#!/bin/bash
 set -eux -o pipefail
 if ! timeout 30s bash -c "until command -v nerdctl; do sleep 3; done"; then
 	echo >&2 "nerdctl is not installed yet"
 	exit 1
 fi
 `,
-		debugHint: `The nerdctl binary was not installed in the guest.
+			debugHint: `The nerdctl binary was not installed in the guest.
 Make sure that you are using an officially supported image.
 Also see "/var/log/cloud-init-output.log" in the guest.
 `,
-	},
+		})
+	}
+	for _, probe := range a.y.Probes {
+		if probe.Mode == limayaml.ProbeModeReadiness {
+			req = append(req, requirement{
+				description: probe.Description,
+				script:      probe.Script,
+				debugHint:   probe.Hint,
+			})
+		}
+	}
+	return req
 }

pkg/limayaml/default.TEMPLATE.yaml

@@ -58,17 +58,44 @@ video:
   # Default: "none"
   display: "none"
 
-#UNIMPLEMENTED| provision:
-#UNIMPLEMENTED|   # `system` is executed with the root privilege
-#UNIMPLEMENTED|   system: |
-#UNIMPLEMENTED|     #!/bin/bash
-#UNIMPLEMENTED|     set -eux -o pipefail
-#UNIMPLEMENTED|     export DEBIAN_FRONTEND=noninteractive
-#UNIMPLEMENTED|     apt-get install -y vim
-#UNIMPLEMENTED|   # `user` is executed without the root privilege
-#UNIMPLEMENTED|   user: |
-#UNIMPLEMENTED|     #!/bin/bash
-#UNIMPLEMENTED|     set -eux -o pipefail
-#UNIMPLEMENTED|     cat <<EOF > ~/.vimrc
-#UNIMPLEMENTED|     set number
-#UNIMPLEMENTED|     EOF
+containerd:
+  # Enable system-wide containerd (systemctl enable containerd)
+  # Default: false
+  system: false
+  # Enable user-scoped containerd (systemctl --user enable containerd)
+  # Default: true
+  user: true
+
+# Provisioning scripts need to be idempotent because they might be called
+# multiple times, e.g. when the host VM is being restarted.
+# provision:
+#   # `system` is executed with the root privilege
+#   - mode: system
+#     script: |
+#       #!/bin/bash
+#       set -eux -o pipefail
+#       export DEBIAN_FRONTEND=noninteractive
+#       apt-get install -y vim
+#   # `user` is executed without the root privilege
+#   - mode: user
+#     script: |
+#       #!/bin/bash
+#       set -eux -o pipefail
+#       cat <<EOF > ~/.vimrc
+#       set number
+#       EOF
+
+# probes:
+#  # Only `readiness` probes are supported right now.
+#  - mode: readiness
+#    description: vim to be installed
+#    script: |
+#       #!/bin/bash
+#       set -eux -o pipefail
+#       if ! timeout 30s bash -c "until command -v vim; do sleep 3; done"; then
+#         echo >&2 "vim is not installed yet"
+#         exit 1
+#       fi
+#    hint: |
+#      vim was not installed in the guest. Make sure the package system is working correctly.
+#      Also see "/var/log/cloud-init-output.log" in the guest.