osguard-ci: Add Code Integrity variant of OS Guard (microsoft#14505)

Add new image configuration definition for OS Guard that enables code
integrity enhancements.

To enable code integrity checking for containers, this image activates
the containerd erofs-snapshotter with an updated
/etc/containerd/config.toml configuration, and also configures cni
appropriately for pod networking.

Additionally this image enables SELinux in enforcing mode for another
important security layer.

Finally, update the OS Guard generation script to handle generating
OS Guard image configurations using different delta files, and simplify
the process of adding new delta configurations by creating the GEN_JOBS
array, where each entry follows the schema:

   <base-template>|<delta-template>|<output>

Also update the test function to check all entries of GEN_JOBS for diffs.

Signed-off-by: Chris Co <[email protected]>

Author: christopherco

toolkit/imageconfigs/files/osguard-ci/config.toml

@@ -0,0 +1,58 @@
+version = 2
+# Explicitly defining no adjustment to Linux OOM Killer
+oom_score = 0
+[plugins."io.containerd.grpc.v1.cri"]
+  # Enable SELinux labeling support for pods and containers
+  enable_selinux = true
+  # Use same infra container image as AKS does for pod sandboxes
+  sandbox_image = "mcr.microsoft.com/oss/kubernetes/pause:3.6"
+  [plugins."io.containerd.grpc.v1.cri".containerd]
+    # Set default snapshotter to erofs-snapshotter
+    snapshotter = "erofs"
+    # Allow snapshot annotations
+    disable_snapshot_annotations = false
+    # Explicitly define using runc for runtime
+    default_runtime_name = "runc"
+    [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc]
+      # Set default snapshotter to erofs for this runtime 
+      snapshotter = "erofs"
+      # Explicitly define using runc v2 shim
+      runtime_type = "io.containerd.runc.v2"
+    # Section is configured by AKS but not strictly required in general
+    # [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options]
+      # BinaryName = "/usr/bin/runc"
+    [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.untrusted]
+      # Explicitly define using runc v2 shim for runtime named "untrusted"
+      runtime_type = "io.containerd.runc.v2"
+    # Section is configured in AKS but not strictly required in general
+    # [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.untrusted.options]
+      # BinaryName = "/usr/bin/runc"
+  # Section is configured in AKS but not strictly required in general
+  # [plugins."io.containerd.grpc.v1.cri".registry]
+    # config_path = "/etc/containerd/certs.d"
+  # Section is configured in AKS but not strictly required in general
+  # [plugins."io.containerd.grpc.v1.cri".registry.headers]
+    # X-Meta-Source-Client = ["azure/aks"]
+  [plugins."io.containerd.grpc.v1.cri".cni]
+    # Set default locations for cni binary and config files
+    bin_dir = "/usr/libexec/cni"
+    conf_dir = "/etc/cni/net.d"
+    conf_template = ""
+# Section is configured in AKS but not strictly required in general
+# [metrics]
+  # address = "0.0.0.0:10257"
+
+[plugins."io.containerd.snapshotter.v1.erofs"]
+  # Optional: Additional mount options for overlayfs
+  ovl_mount_options = []
+  # Enable dm-verity integrity verification in erofs layers
+  enable_dmverity = true
+
+[plugins."io.containerd.service.v1.diff-service"]
+  default = ["erofs"]
+
+[plugins."io.containerd.differ.v1.erofs"]
+  # Using well-known UUID for reproducibility of erofs container layers
+  mkfs_options = ["--sort=none", "-T 0", "--mkfs-time", "-Uc1b9d5a2-f162-11cf-9ece-0020afc76f16"]
+  # Enable use of tar index to more efficiently handle OCI image layers
+  enable_tar_index = true

toolkit/imageconfigs/osguard-ci-amd64.yaml

@@ -0,0 +1,209 @@
+# This file was automatically generated by merge_yaml.py
+# Sources: base=templates/osguard-base.yaml delta=templates/osguard-ci-delta.yaml
+
+storage:
+  bootType: efi
+  disks:
+  - partitionTableType: gpt
+    maxSize: 40G
+    partitions:
+    - id: esp
+      type: esp
+      label: esp
+      size: 512M
+    - id: boot-a
+      type: linux-generic
+      label: boot-a
+      size: 100M
+    - id: usr-a
+      type: linux-generic
+      size: 1G
+    - id: usr-hash-a
+      type: usr-verity
+      size: 128M
+    - id: root-a
+      type: root
+      label: root-a
+      size: 12G
+  verity:
+  - id: usrverity
+    name: usr
+    dataDeviceId: usr-a
+    hashDeviceId: usr-hash-a
+    dataDeviceMountIdType: uuid
+    hashDeviceMountIdType: uuid
+    hashSignaturePath: /boot/usr.hash.sig
+  filesystems:
+  - deviceId: esp
+    type: fat32
+    mountPoint:
+      idType: part-label
+      path: /boot/efi
+      options: nodev,noexec,umask=0077
+  - deviceId: boot-a
+    type: ext4
+    mountPoint:
+      idType: uuid
+      path: /boot
+      options: nodev,noexec,nosuid
+  - deviceId: usrverity
+    type: ext4
+    mountPoint:
+      path: /usr
+      options: nodev,ro
+  - deviceId: root-a
+    type: ext4
+    mountPoint:
+      path: /
+      options: nodev,nosuid,x-systemd.growfs,x-initrd.mount
+os:
+  bootloader:
+    resetType: hard-reset
+  hostname: azure-linux-os-guard
+  selinux:
+    mode: enforcing
+  uki:
+    kernels: auto
+  kernelCommandLine:
+    extraCommandLine:
+    - console=tty0
+    - console=tty1
+    - console=ttyS0
+    - rd.luks=0
+    - rd.hostonly=0
+    - fips=1
+    - net.ifnames=1
+    - dm_verity.require_signatures=1
+  packages:
+    remove:
+    - dracut-hostonly
+    - grub2-efi-binary
+    - kernel
+    install:
+    - syslog
+    - WALinuxAgent
+    - device-mapper
+    - kernel-ipe
+    - cni
+    - containerd2
+    - cri-tools
+    - systemd-boot
+    - dracut-hyperv
+    - hyperv-daemons
+    - cloud-init
+    - checkpolicy
+    - libselinux
+    - policycoreutils-python-utils
+    - secilc
+    - selinux-policy
+    - selinux-policy-ci
+    - selinux-policy-modules
+    - setools-console
+    - systemd-ukify
+    - systemd-boot
+    - efibootmgr
+    - lvm2
+    - veritysetup
+    - selinux-policy
+    - selinux-policy-modules
+    - gptfdisk
+    - curl
+    - bind-utils
+    - tar
+    - wget
+    - blobfuse2
+    - ca-certificates
+    - chrony
+    - cifs-utils
+    - cloud-init-azure-kvp
+    - conntrack-tools
+    - cracklib
+    - ebtables
+    - ethtool
+    - fuse
+    - inotify-tools
+    - iotop
+    - iproute
+    - ipset
+    - iptables
+    - iscsi-initiator-utils
+    - jq
+    - logrotate
+    - lsof
+    - netplan
+    - nftables
+    - nmap-ncat
+    - nfs-utils
+    - oras
+    - pam
+    - psmisc
+    - rsyslog
+    - socat
+    - sysstat
+    - traceroute
+    - util-linux
+    - xz
+    - zip
+    - erofs-utils
+  additionalDirs:
+  - source: files/osguard/repart.d
+    destination: /etc/repart.d
+    childFilePermissions: 644
+  additionalFiles:
+  - source: files/osguard/selinux-ci-uki.semanage
+    destination: /etc/selinux/targeted/selinux-ci.semanage
+  - source: files/osguard/cloud.cfg
+    destination: /etc/cloud/cloud.cfg
+    permissions: '644'
+  - source: files/osguard/10-repart.conf
+    destination: /etc/dracut.conf.d/10-repart.conf
+    permissions: '644'
+  - source: files/osguard/chrony.conf
+    destination: /etc/chrony.conf
+    permissions: '644'
+  - source: files/osguard/resolv-uplink-override.service
+    destination: /etc/systemd/system/resolv-uplink-override.service
+    permissions: '600'
+  - source: files/osguard-ci/config.toml
+    destination: /etc/containerd/config.toml
+    permissions: '644'
+  services:
+    disable:
+    - sshd
+    enable:
+    - systemd-networkd
+    - systemd-resolved
+  modules:
+  - name: iptable_nat
+    loadMode: always
+  - name: erofs
+    loadMode: always
+scripts:
+  postCustomization:
+  - path: scripts/common/performance-tuning.sh
+  - path: scripts/common/azlinuxagentconfig.sh
+  - path: scripts/common/selinux-ci-config.py
+    interpreter: /usr/bin/python3
+  - path: scripts/common/cleanup-machineid.sh
+  - path: scripts/common/prepare_trusted_cni_plugins.sh
+  - path: scripts/common/move-iptables-scripts-to-usr.sh
+  - path: scripts/common/tmp-no-exec.sh
+  - path: scripts/common/remove-getty-import-credential.sh
+  - path: scripts/osguard/create-empty-certs-dir.sh
+  - path: scripts/set_os_release_variant_entries.sh
+    arguments:
+    - --variant-id
+    - osguard-ci
+    - --variant
+    - OS Guard Code Integrity Image
+output:
+  artifacts:
+    items:
+    - verity-hash
+    - ukis
+    path: ./output
+  image:
+    format: vhdx
+previewFeatures:
+- output-artifacts
+- uki

toolkit/imageconfigs/templates/osguard-ci-delta.yaml

@@ -0,0 +1,48 @@
+# The OS Guard Code Integrity image variant extends code integrity features
+# beyond the host binaries, to include container images and their layers. This
+# is achieved through configuring containerd to use erofs-snapshotter with
+# dm-verity support and requiring dm-verity signature verification. On container
+# execution, IPE will only allow the execution of containers that are dm-verity 
+# verified.
+#
+# This file defines the delta (differences) to apply to the osguard-base.yaml
+# template in order to generate the OS Guard Code Integrity (CI) image variant.
+# This template is merged with the base template by the
+# generate-osguard-imageconfigs.sh script to produce osguard-ci-amd64.yaml
+# Only settings that differ from the base should be included here.
+os:
+  # Ensure SELinux is in Enforcing Mode for OS Guard Code Integrity image.
+  selinux:
+    mode: enforcing
+  kernelCommandLine:
+    extraCommandLine:
+      # Enforce signatures for all dm-verity volumes on the system. This
+      # verification is needed in conjunction with our dm-verity-enabled
+      # erofs-snapshotter to ensure erofs container layers, which are
+      # dm-verity volumes, are signed by a trusted entity
+      - dm_verity.require_signatures=1
+  packages:
+    install:
+    # For containerd erofs-snapshotter to function, supply its userland
+    # utilities
+    - erofs-utils
+  modules:
+  # Ensure the erofs kernel module is always loaded so containerd
+  # erofs-snapshotter can use it.
+  - name: erofs
+    loadMode: always
+  additionalFiles:
+  # Place custom containerd config that configures erofs-snapshotter as the
+  # default snapshotter when setting up container images
+  - source: files/osguard-ci/config.toml
+    destination: /etc/containerd/config.toml
+    permissions: "644"
+scripts:
+  postCustomization:
+  # Tag this image variant with its specific variant-id
+  - path: scripts/set_os_release_variant_entries.sh
+    arguments:
+    - --variant-id
+    - osguard-ci
+    - --variant
+    - OS Guard Code Integrity Image