diff --git a/mantle/cmd/kola/options.go b/mantle/cmd/kola/options.go index a63877e52c..59e145bc77 100644 --- a/mantle/cmd/kola/options.go +++ b/mantle/cmd/kola/options.go @@ -162,6 +162,10 @@ func init() { sv(&kola.QEMUOptions.SecureExecutionHostKey, "qemu-secex-hostkey", "", "Path to Secure Execution HKD certificate") // s390x CEX-specific options bv(&kola.QEMUOptions.Cex, "qemu-cex", false, "Attach CEX device to guest") + + // kola run iso.* options + bv(&kola.QEMUOptions.InstInsecure, "inst-insecure", false, "Do not verify signature on metal image") + ssv(&kola.QEMUOptions.PxeKernelArgs, "pxe-kargs", nil, "Additional kernel arguments for PXE") } // Sync up the command line options if there is dependency diff --git a/mantle/cmd/kola/testiso.go b/mantle/cmd/kola/testiso.go deleted file mode 100644 index 199091e26b..0000000000 --- a/mantle/cmd/kola/testiso.go +++ /dev/null @@ -1,1172 +0,0 @@ -// Copyright 2020 Red Hat, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// TODO: -// - Support testing the "just run Live" case - maybe try to figure out -// how to have main `kola` tests apply? -// - Test `coreos-install iso embed` path - -package main - -import ( - "bufio" - "context" - _ "embed" - "fmt" - "io" - "os" - "path/filepath" - "strconv" - "strings" - "time" - - "github.com/coreos/coreos-assembler/mantle/harness" - "github.com/coreos/coreos-assembler/mantle/harness/reporters" - "github.com/coreos/coreos-assembler/mantle/harness/testresult" - "github.com/coreos/coreos-assembler/mantle/platform/conf" - "github.com/coreos/coreos-assembler/mantle/util" - coreosarch "github.com/coreos/stream-metadata-go/arch" - "github.com/pkg/errors" - - "github.com/spf13/cobra" - - "github.com/coreos/coreos-assembler/mantle/kola" - "github.com/coreos/coreos-assembler/mantle/platform" -) - -var ( - cmdTestIso = &cobra.Command{ - RunE: runTestIso, - PreRunE: preRun, - Use: "testiso [glob pattern...]", - Short: "Test a CoreOS PXE boot or ISO install path", - - SilenceUsage: true, - } - - instInsecure bool - - pxeKernelArgs []string - - console bool - - addNmKeyfile bool - enable4k bool - enableMultipath bool - enableUefi bool - enableUefiSecure bool - isOffline bool - isISOFromRAM bool - - // These tests only run on RHCOS - tests_RHCOS_uefi = []string{ - "iso-fips.uefi", - } - - // The iso-as-disk tests are only supported in x86_64 because other - // architectures don't have the required hybrid partition table. - tests_x86_64 = []string{ - "iso-as-disk.bios", - "iso-as-disk.uefi", - "iso-as-disk.uefi-secure", - "iso-as-disk.4k.uefi", - "iso-install.bios", - "iso-live-login.bios", - "iso-live-login.uefi", - "iso-live-login.uefi-secure", - "iso-live-login.4k.uefi", - "iso-offline-install.bios", - "iso-offline-install.mpath.bios", - "iso-offline-install-fromram.4k.uefi", - "iso-offline-install-iscsi.ibft.uefi", - "iso-offline-install-iscsi.ibft-with-mpath.bios", - "iso-offline-install-iscsi.manual.bios", - "miniso-install.bios", - "miniso-install.nm.bios", - "miniso-install.4k.uefi", - "miniso-install.4k.nm.uefi", - "pxe-offline-install.rootfs-appended.bios", - "pxe-offline-install.4k.uefi", - "pxe-online-install.bios", - "pxe-online-install.4k.uefi", - } - tests_s390x = []string{ - "iso-live-login.s390fw", - "iso-offline-install.s390fw", - "iso-offline-install.mpath.s390fw", - "iso-offline-install.4k.s390fw", - "pxe-online-install.rootfs-appended.s390fw", - "pxe-offline-install.s390fw", - "miniso-install.s390fw", - "miniso-install.nm.s390fw", - "miniso-install.4k.nm.s390fw", - // FIXME https://github.com/coreos/fedora-coreos-tracker/issues/1657 - //"iso-offline-install-iscsi.ibft.s390fw, - //"iso-offline-install-iscsi.ibft-with-mpath.s390fw", - //"iso-offline-install-iscsi.manual.s390fw", - } - tests_ppc64le = []string{ - "iso-live-login.ppcfw", - "iso-offline-install.ppcfw", - "iso-offline-install.mpath.ppcfw", - "iso-offline-install-fromram.4k.ppcfw", - "miniso-install.ppcfw", - "miniso-install.nm.ppcfw", - "miniso-install.4k.ppcfw", - "miniso-install.4k.nm.ppcfw", - "pxe-online-install.rootfs-appended.ppcfw", - "pxe-offline-install.4k.ppcfw", - // FIXME https://github.com/coreos/fedora-coreos-tracker/issues/1657 - //"iso-offline-install-iscsi.ibft.ppcfw", - //"iso-offline-install-iscsi.ibft-with-mpath.ppcfw", - //"iso-offline-install-iscsi.manual.ppcfw", - } - tests_aarch64 = []string{ - "iso-live-login.uefi", - "iso-live-login.4k.uefi", - "iso-offline-install.uefi", - "iso-offline-install.mpath.uefi", - "iso-offline-install-fromram.4k.uefi", - "miniso-install.uefi", - "miniso-install.nm.uefi", - "miniso-install.4k.uefi", - "miniso-install.4k.nm.uefi", - "pxe-offline-install.uefi", - "pxe-offline-install.rootfs-appended.4k.uefi", - "pxe-online-install.uefi", - "pxe-online-install.4k.uefi", - // FIXME https://github.com/coreos/fedora-coreos-tracker/issues/1657 - //"iso-offline-install-iscsi.ibft.uefi", - //"iso-offline-install-iscsi.ibft-with-mpath.uefi", - //"iso-offline-install-iscsi.manual.uefi", - } -) - -const ( - installTimeoutMins = 12 - // https://github.com/coreos/fedora-coreos-config/pull/2544 - liveISOFromRAMKarg = "coreos.liveiso.fromram" -) - -var liveOKSignal = "live-test-OK" -var liveSignalOKUnit = fmt.Sprintf(`[Unit] -Description=TestISO Signal Live ISO Completion -Requires=dev-virtio\\x2dports-testisocompletion.device -OnFailure=emergency.target -OnFailureJobMode=isolate -Before=coreos-installer.service -[Service] -Type=oneshot -RemainAfterExit=yes -ExecStart=/bin/sh -c '/usr/bin/echo %s >/dev/virtio-ports/testisocompletion' -[Install] -# for install tests -RequiredBy=coreos-installer.target -# for iso-as-disk -RequiredBy=multi-user.target -`, liveOKSignal) - -var downloadCheck = `[Unit] -Description=TestISO Verify CoreOS Installer Download -After=coreos-installer.service -Before=coreos-installer.target -[Service] -Type=oneshot -StandardOutput=kmsg+console -StandardError=kmsg+console -ExecStart=/bin/sh -c "journalctl -t coreos-installer-service | /usr/bin/awk '/[Dd]ownload/ {exit 1}'" -ExecStart=/bin/sh -c "/usr/bin/udevadm settle" -ExecStart=/bin/sh -c "/usr/bin/mount /dev/disk/by-label/root /mnt" -ExecStart=/bin/sh -c "/usr/bin/jq -er '.[\"build\"]? + .[\"version\"]? == \"%s\"' /mnt/.coreos-aleph-version.json" -[Install] -RequiredBy=coreos-installer.target -` - -var signalCompleteString = "coreos-installer-test-OK" -var signalCompletionUnit = fmt.Sprintf(`[Unit] -Description=TestISO Signal Completion -Requires=dev-virtio\\x2dports-testisocompletion.device -OnFailure=emergency.target -OnFailureJobMode=isolate -[Service] -Type=oneshot -RemainAfterExit=yes -ExecStart=/bin/sh -c '/usr/bin/echo %s >/dev/virtio-ports/testisocompletion && systemctl poweroff' -[Install] -RequiredBy=multi-user.target -`, signalCompleteString) - -var signalEmergencyString = "coreos-installer-test-entered-emergency-target" -var signalFailureUnit = fmt.Sprintf(`[Unit] -Description=TestISO Signal Failure -Requires=dev-virtio\\x2dports-testisocompletion.device -DefaultDependencies=false -[Service] -Type=oneshot -RemainAfterExit=yes -ExecStart=/bin/sh -c '/usr/bin/echo %s >/dev/virtio-ports/testisocompletion && systemctl poweroff' -[Install] -RequiredBy=emergency.target -`, signalEmergencyString) - -var checkNoIgnition = `[Unit] -Description=TestISO Verify No Ignition Config -OnFailure=emergency.target -OnFailureJobMode=isolate -Before=coreos-test-installer.service -After=coreos-ignition-firstboot-complete.service -RequiresMountsFor=/boot -[Service] -Type=oneshot -RemainAfterExit=yes -ExecStart=/bin/sh -c '[ ! -e /boot/ignition ]' -[Install] -RequiredBy=multi-user.target` - -var multipathedRoot = `[Unit] -Description=TestISO Verify Multipathed Root -OnFailure=emergency.target -OnFailureJobMode=isolate -Before=coreos-test-installer.service -[Service] -Type=oneshot -RemainAfterExit=yes -ExecStart=/bin/bash -c 'lsblk -pno NAME "/dev/mapper/$(multipath -l -v 1)" | grep -qw "$(findmnt -nvr /sysroot -o SOURCE)"' -[Install] -RequiredBy=multi-user.target` - -// This test is broken. Please fix! -// https://github.com/coreos/coreos-assembler/issues/3554 -var verifyNoEFIBootEntry = `[Unit] -Description=TestISO Verify No EFI Boot Entry -OnFailure=emergency.target -OnFailureJobMode=isolate -ConditionPathExists=/sys/firmware/efi -Before=live-signal-ok.service -[Service] -Type=oneshot -RemainAfterExit=yes -ExecStart=/bin/sh -c '! efibootmgr -v | grep -E "(HD|CDROM)\("' -[Install] -# for install tests -RequiredBy=coreos-installer.target -# for iso-as-disk -RequiredBy=multi-user.target` - -// Verify that the volume ID is the OS name. See also -// https://github.com/openshift/assisted-image-service/pull/477. -// This is the same as the LABEL of the block device for ISO9660. See -// https://github.com/util-linux/util-linux/blob/643bdae8e38055e36acf2963c3416de206081507/libblkid/src/superblocks/iso9660.c#L366-L377 -var verifyIsoVolumeId = `[Unit] -Description=Verify ISO Volume ID -OnFailure=emergency.target -OnFailureJobMode=isolate -# only if we're actually mounting the ISO -ConditionPathIsMountPoint=/run/media/iso -[Service] -Type=oneshot -RemainAfterExit=yes -# the backing device name is arch-dependent, but we know it's mounted on /run/media/iso -ExecStart=bash -c "[[ $(findmnt -no LABEL /run/media/iso) == %s-* ]]" -[Install] -RequiredBy=coreos-installer.target` - -// Unit to check that /run/media/iso is not mounted when -// coreos.liveiso.fromram kernel argument is passed -var isoNotMountedUnit = `[Unit] -Description=Verify ISO is not mounted when coreos.liveiso.fromram -OnFailure=emergency.target -OnFailureJobMode=isolate -ConditionKernelCommandLine=coreos.liveiso.fromram -[Service] -Type=oneshot -StandardOutput=kmsg+console -StandardError=kmsg+console -RemainAfterExit=yes -# Would like to use SuccessExitStatus but it doesn't support what -# we want: https://github.com/systemd/systemd/issues/10297#issuecomment-1672002635 -ExecStart=bash -c "if mountpoint /run/media/iso 2>/dev/null; then exit 1; fi" -[Install] -RequiredBy=coreos-installer.target` - -var nmConnectionId = "CoreOS DHCP" -var nmConnectionFile = "coreos-dhcp.nmconnection" -var nmConnection = fmt.Sprintf(`[connection] -id=%s -type=ethernet -# add wait-device-timeout here so we make sure NetworkManager-wait-online.service will -# wait for a device to be present before exiting. See -# https://github.com/coreos/fedora-coreos-tracker/issues/1275#issuecomment-1231605438 -wait-device-timeout=20000 - -[ipv4] -method=auto -`, nmConnectionId) - -var nmstateConfigFile = "/etc/nmstate/br-ex.yml" -var nmstateConfig = `interfaces: - - name: br-ex - type: linux-bridge - state: up - ipv4: - enabled: false - ipv6: - enabled: false - bridge: - port: [] -` - -// This is used to verify *both* the live and the target system in the `--add-nm-keyfile` path. -var verifyNmKeyfile = fmt.Sprintf(`[Unit] -Description=TestISO Verify NM Keyfile Propagation -OnFailure=emergency.target -OnFailureJobMode=isolate -Wants=network-online.target -After=network-online.target -Before=live-signal-ok.service -Before=coreos-test-installer.service -[Service] -Type=oneshot -RemainAfterExit=yes -ExecStart=/usr/bin/journalctl -u nm-initrd --no-pager --grep "policy: set '%[1]s' (.*) as default .* routing and DNS" -ExecStart=/usr/bin/journalctl -u NetworkManager --no-pager --grep "policy: set '%[1]s' (.*) as default .* routing and DNS" -ExecStart=/usr/bin/grep "%[1]s" /etc/NetworkManager/system-connections/%[2]s -# Also verify nmstate config -ExecStart=/usr/bin/nmcli c show br-ex -[Install] -# for live system -RequiredBy=coreos-installer.target -# for target system -RequiredBy=multi-user.target`, nmConnectionId, nmConnectionFile) - -//go:embed resources/iscsi_butane_setup.yaml -var iscsi_butane_config string - -func init() { - cmdTestIso.Flags().BoolVarP(&instInsecure, "inst-insecure", "S", false, "Do not verify signature on metal image") - cmdTestIso.Flags().BoolVar(&console, "console", false, "Connect qemu console to terminal, turn off automatic initramfs failure checking") - cmdTestIso.Flags().StringSliceVar(&pxeKernelArgs, "pxe-kargs", nil, "Additional kernel arguments for PXE") - - root.AddCommand(cmdTestIso) -} - -func liveArtifactExistsInBuild() error { - - if kola.CosaBuild.Meta.BuildArtifacts.LiveIso == nil || kola.CosaBuild.Meta.BuildArtifacts.LiveKernel == nil { - return fmt.Errorf("build %s is missing live artifacts", kola.CosaBuild.Meta.Name) - } - return nil -} - -func getAllTests(build *util.LocalBuild) []string { - arch := coreosarch.CurrentRpmArch() - var tests []string - switch arch { - case "x86_64": - tests = tests_x86_64 - case "ppc64le": - tests = tests_ppc64le - case "s390x": - tests = tests_s390x - case "aarch64": - tests = tests_aarch64 - } - if kola.CosaBuild.Meta.Name == "rhcos" && arch != "s390x" && arch != "ppc64le" { - tests = append(tests, tests_RHCOS_uefi...) - } - return tests -} - -func newBaseQemuBuilder(outdir string) (*platform.QemuBuilder, error) { - builder := platform.NewMetalQemuBuilderDefault() - if enableUefiSecure { - builder.Firmware = "uefi-secure" - } else if enableUefi { - builder.Firmware = "uefi" - } - - if err := os.MkdirAll(outdir, 0755); err != nil { - return nil, err - } - - builder.InheritConsole = console - if !console { - builder.ConsoleFile = filepath.Join(outdir, "console.txt") - } - - if kola.QEMUOptions.Memory != "" { - parsedMem, err := strconv.ParseInt(kola.QEMUOptions.Memory, 10, 32) - if err != nil { - return nil, err - } - builder.MemoryMiB = int(parsedMem) - } - - return builder, nil -} - -func newQemuBuilder(outdir string) (*platform.QemuBuilder, *conf.Conf, error) { - builder, err := newBaseQemuBuilder(outdir) - if err != nil { - return nil, nil, err - } - - config, err := conf.EmptyIgnition().Render(conf.FailWarnings) - if err != nil { - return nil, nil, err - } - - err = forwardJournal(outdir, builder, config) - if err != nil { - return nil, nil, err - } - - return builder, config, nil -} - -func forwardJournal(outdir string, builder *platform.QemuBuilder, config *conf.Conf) error { - journalPipe, err := builder.VirtioJournal(config, "") - if err != nil { - return err - } - journalOut, err := os.OpenFile(filepath.Join(outdir, "journal.txt"), os.O_WRONLY|os.O_CREATE, 0644) - if err != nil { - return err - } - - go func() { - _, err := io.Copy(journalOut, journalPipe) - if err != nil && err != io.EOF { - panic(err) - } - }() - - return nil -} - -func newQemuBuilderWithDisk(outdir string) (*platform.QemuBuilder, *conf.Conf, error) { - builder, config, err := newQemuBuilder(outdir) - - if err != nil { - return nil, nil, err - } - - sectorSize := 0 - if enable4k { - sectorSize = 4096 - } - - disk := platform.Disk{ - Size: "12G", // Arbitrary - SectorSize: sectorSize, - MultiPathDisk: enableMultipath, - } - - //TBD: see if we can remove this and just use AddDisk and inject bootindex during startup - if coreosarch.CurrentRpmArch() == "s390x" || coreosarch.CurrentRpmArch() == "aarch64" { - // s390x and aarch64 need to use bootindex as they don't support boot once - if err := builder.AddDisk(&disk); err != nil { - return nil, nil, err - } - } else { - if err := builder.AddPrimaryDisk(&disk); err != nil { - return nil, nil, err - } - } - - return builder, config, nil -} - -// See similar semantics in the `filterTests` of `kola.go`. -func filterTests(tests []string, patterns []string) ([]string, error) { - r := []string{} - for _, test := range tests { - if matches, err := kola.MatchesPatterns(test, patterns); err != nil { - return nil, err - } else if matches { - r = append(r, test) - } - } - return r, nil -} - -func runTestIso(cmd *cobra.Command, args []string) (err error) { - if kola.CosaBuild == nil { - return fmt.Errorf("Must provide --build") - } - tests := getAllTests(kola.CosaBuild) - if len(args) != 0 { - if tests, err = filterTests(tests, args); err != nil { - return err - } else if len(tests) == 0 { - return harness.SuiteEmpty - } - } - - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - // Call `ParseDenyListYaml` to populate the `kola.DenylistedTests` var - err = kola.ParseDenyListYaml("qemu") - if err != nil { - plog.Fatal(err) - } - - finalTests := []string{} - for _, test := range tests { - if !kola.HasString(test, kola.DenylistedTests) { - matchTest, err := kola.MatchesPatterns(test, kola.DenylistedTests) - if err != nil { - return err - - } - if !matchTest { - finalTests = append(finalTests, test) - } - } - } - - // note this reassigns a *global* - outputDir, err = kola.SetupOutputDir(outputDir, "testiso") - if err != nil { - return err - } - - // see similar code in suite.go - reportDir := filepath.Join(outputDir, "reports") - if err := os.Mkdir(reportDir, 0777); err != nil { - return err - } - - reporter := reporters.NewJSONReporter("report.json", "testiso", "") - defer func() { - if reportErr := reporter.Output(reportDir); reportErr != nil && err != nil { - err = reportErr - } - }() - - baseInst := platform.Install{ - CosaBuild: kola.CosaBuild, - NmKeyfiles: make(map[string]string), - } - - if instInsecure { - baseInst.Insecure = true - fmt.Printf("Ignoring verification of signature on metal image\n") - } - - // Ignore signing verification by default when running with development build - // https://github.com/coreos/fedora-coreos-tracker/issues/908 - if !baseInst.Insecure && strings.Contains(kola.CosaBuild.Meta.BuildID, ".dev.") { - baseInst.Insecure = true - fmt.Printf("Detected development build; disabling signature verification\n") - } - - var duration time.Duration - - atLeastOneFailed := false - for _, test := range finalTests { - - // All of these tests require buildextend-live to have been run - err = liveArtifactExistsInBuild() - if err != nil { - return err - } - - addNmKeyfile = false - enable4k = false - enableMultipath = false - enableUefi = false - enableUefiSecure = false - isOffline = false - inst := baseInst // Pretend this is Rust and I wrote .copy() - - fmt.Printf("Running test: %s\n", test) - components := strings.Split(test, ".") - - inst.PxeAppendRootfs = kola.HasString("rootfs-appended", components) - - if kola.HasString("4k", components) { - enable4k = true - inst.Native4k = true - } - if kola.HasString("nm", components) { - addNmKeyfile = true - } - if kola.HasString("mpath", components) { - enableMultipath = true - inst.MultiPathDisk = true - } - if kola.HasString("uefi-secure", components) { - enableUefiSecure = true - } else if kola.HasString("uefi", components) { - enableUefi = true - } - // For offline it is a part of the first component. i.e. for - // iso-offline-install.bios we need to search for 'offline' in - // iso-offline-install, which is currently in components[0]. - if kola.HasString("offline", strings.Split(components[0], "-")) { - isOffline = true - } - // For fromram it is a part of the first component. i.e. for - // iso-offline-install-fromram.uefi we need to search for 'fromram' in - // iso-offline-install-fromram, which is currently in components[0]. - if kola.HasString("fromram", strings.Split(components[0], "-")) { - isISOFromRAM = true - } - - switch components[0] { - case "pxe-offline-install", "pxe-online-install": - duration, err = testPXE(ctx, inst, filepath.Join(outputDir, test)) - case "iso-as-disk": - duration, err = testAsDisk(ctx, filepath.Join(outputDir, test)) - case "iso-live-login": - duration, err = testLiveLogin(ctx, filepath.Join(outputDir, test)) - case "iso-fips": - duration, err = testLiveFIPS(ctx, filepath.Join(outputDir, test)) - case "iso-install", "iso-offline-install", "iso-offline-install-fromram": - duration, err = testLiveIso(ctx, inst, filepath.Join(outputDir, test), false) - case "miniso-install": - duration, err = testLiveIso(ctx, inst, filepath.Join(outputDir, test), true) - case "iso-offline-install-iscsi": - var butane_config string - switch components[1] { - case "ibft": - butane_config = strings.ReplaceAll(iscsi_butane_config, "COREOS_INSTALLER_KARGS", "--append-karg rd.iscsi.firmware=1") - case "manual": - butane_config = strings.ReplaceAll(iscsi_butane_config, "COREOS_INSTALLER_KARGS", "--append-karg netroot=iscsi:10.0.2.15::::iqn.2024-05.com.coreos:0") - case "ibft-with-mpath": - butane_config = strings.ReplaceAll(iscsi_butane_config, "COREOS_INSTALLER_KARGS", "--append-karg rd.iscsi.firmware=1 --append-karg rd.multipath=default --append-karg root=/dev/disk/by-label/dm-mpath-root --append-karg rw") - default: - plog.Fatalf("Unknown test name:%s", test) - } - duration, err = testLiveInstalliscsi(ctx, inst, filepath.Join(outputDir, test), butane_config) - default: - plog.Fatalf("Unknown test name:%s", test) - } - - result := testresult.Pass - output := []byte{} - if err != nil { - result = testresult.Fail - output = []byte(err.Error()) - } - reporter.ReportTest(test, []string{}, result, duration, output) - if printResult(test, duration, err) { - atLeastOneFailed = true - } - } - - reporter.SetResult(testresult.Pass) - if atLeastOneFailed { - reporter.SetResult(testresult.Fail) - return harness.SuiteFailed - } - - return nil -} - -func awaitCompletion(ctx context.Context, inst *platform.QemuInstance, outdir string, qchan *os.File, booterrchan chan error, expected []string) (time.Duration, error) { - start := time.Now() - errchan := make(chan error) - go func() { - timeout := (time.Duration(installTimeoutMins*(100+kola.Options.ExtendTimeoutPercent)) * time.Minute) / 100 - time.Sleep(timeout) - errchan <- fmt.Errorf("timed out after %v", timeout) - }() - if !console { - go func() { - errBuf, err := inst.WaitIgnitionError(ctx) - if err == nil { - if errBuf != "" { - plog.Info("entered emergency.target in initramfs") - path := filepath.Join(outdir, "ignition-virtio-dump.txt") - if err := os.WriteFile(path, []byte(errBuf), 0644); err != nil { - plog.Errorf("Failed to write journal: %v", err) - } - err = platform.ErrInitramfsEmergency - } - } - if err != nil { - errchan <- err - } - }() - } - go func() { - err := inst.Wait() - // only one Wait() gets process data, so also manually check for signal - plog.Debugf("qemu exited err=%v", err) - if err == nil && inst.Signaled() { - err = errors.New("process killed") - } - if err != nil { - errchan <- errors.Wrapf(err, "QEMU unexpectedly exited while awaiting completion") - } - time.Sleep(1 * time.Minute) - errchan <- fmt.Errorf("QEMU exited; timed out waiting for completion") - }() - go func() { - r := bufio.NewReader(qchan) - for _, exp := range expected { - l, err := r.ReadString('\n') - if err != nil { - if err == io.EOF { - // this may be from QEMU getting killed or exiting; wait a bit - // to give a chance for .Wait() above to feed the channel with a - // better error - time.Sleep(1 * time.Second) - errchan <- fmt.Errorf("Got EOF from completion channel, %s expected", exp) - } else { - errchan <- errors.Wrapf(err, "reading from completion channel") - } - return - } - line := strings.TrimSpace(l) - if line != exp { - errchan <- fmt.Errorf("Unexpected string from completion channel: %s expected: %s", line, exp) - return - } - plog.Debugf("Matched expected message %s", exp) - } - plog.Debugf("Matched all expected messages") - // OK! - errchan <- nil - }() - go func() { - //check for error when switching boot order - if booterrchan != nil { - if err := <-booterrchan; err != nil { - errchan <- err - } - } - }() - err := <-errchan - elapsed := time.Since(start) - if err == nil { - // No error so far, check the console and journal files - consoleFile := filepath.Join(outdir, "console.txt") - journalFile := filepath.Join(outdir, "journal.txt") - files := []string{consoleFile, journalFile} - for _, file := range files { - fileName := filepath.Base(file) - // Check if the file exists - _, err := os.Stat(file) - if os.IsNotExist(err) { - fmt.Printf("The file: %v does not exist\n", fileName) - continue - } else if err != nil { - fmt.Println(err) - return elapsed, err - } - // Read the contents of the file - fileContent, err := os.ReadFile(file) - if err != nil { - fmt.Println(err) - return elapsed, err - } - // Check for badness with CheckConsole - warnOnly, badlines := kola.CheckConsole([]byte(fileContent), nil) - if len(badlines) > 0 { - for _, badline := range badlines { - if warnOnly { - plog.Errorf("bad log line detected: %v", badline) - } else { - plog.Warningf("bad log line detected: %v", badline) - } - } - if !warnOnly { - err = fmt.Errorf("errors found in log files") - return elapsed, err - } - } - } - } - return elapsed, err -} - -func printResult(test string, duration time.Duration, err error) bool { - result := "PASS" - if err != nil { - result = "FAIL" - } - fmt.Printf("%s: %s (%s)\n", result, test, duration.Round(time.Millisecond).String()) - if err != nil { - fmt.Printf(" %s\n", err) - return true - } - return false -} - -func testPXE(ctx context.Context, inst platform.Install, outdir string) (time.Duration, error) { - if addNmKeyfile { - return 0, errors.New("--add-nm-keyfile not yet supported for PXE") - } - tmpd, err := os.MkdirTemp("", "kola-testiso") - if err != nil { - return 0, errors.Wrapf(err, "creating tempdir") - } - defer os.RemoveAll(tmpd) - - sshPubKeyBuf, _, err := util.CreateSSHAuthorizedKey(tmpd) - if err != nil { - return 0, errors.Wrapf(err, "creating SSH AuthorizedKey") - } - - builder, virtioJournalConfig, err := newQemuBuilderWithDisk(outdir) - if err != nil { - return 0, errors.Wrapf(err, "creating QemuBuilder") - } - - // increase the memory for pxe tests with appended rootfs in the initrd - // we were bumping up into the 4GiB limit in RHCOS/c9s - // pxe-offline-install.rootfs-appended.bios tests - if inst.PxeAppendRootfs && builder.MemoryMiB < 5120 { - builder.MemoryMiB = 5120 - } - - inst.Builder = builder - completionChannel, err := inst.Builder.VirtioChannelRead("testisocompletion") - if err != nil { - return 0, errors.Wrapf(err, "setting up virtio-serial channel") - } - - var keys []string - keys = append(keys, strings.TrimSpace(string(sshPubKeyBuf))) - virtioJournalConfig.AddAuthorizedKeys("core", keys) - - liveConfig := *virtioJournalConfig - liveConfig.AddSystemdUnit("live-signal-ok.service", liveSignalOKUnit, conf.Enable) - liveConfig.AddSystemdUnit("coreos-test-entered-emergency-target.service", signalFailureUnit, conf.Enable) - - if isOffline { - contents := fmt.Sprintf(downloadCheck, kola.CosaBuild.Meta.OstreeVersion) - liveConfig.AddSystemdUnit("coreos-installer-offline-check.service", contents, conf.Enable) - } - - targetConfig := *virtioJournalConfig - targetConfig.AddSystemdUnit("coreos-test-installer.service", signalCompletionUnit, conf.Enable) - targetConfig.AddSystemdUnit("coreos-test-entered-emergency-target.service", signalFailureUnit, conf.Enable) - targetConfig.AddSystemdUnit("coreos-test-installer-no-ignition.service", checkNoIgnition, conf.Enable) - - mach, err := inst.PXE(pxeKernelArgs, liveConfig, targetConfig, isOffline) - if err != nil { - return 0, errors.Wrapf(err, "running PXE") - } - defer func() { - if err := mach.Destroy(); err != nil { - plog.Errorf("Failed to destroy PXE: %v", err) - } - }() - - return awaitCompletion(ctx, mach.QemuInst, outdir, completionChannel, mach.BootStartedErrorChannel, []string{liveOKSignal, signalCompleteString}) -} - -func testLiveIso(ctx context.Context, inst platform.Install, outdir string, minimal bool) (time.Duration, error) { - tmpd, err := os.MkdirTemp("", "kola-testiso") - if err != nil { - return 0, err - } - defer os.RemoveAll(tmpd) - - sshPubKeyBuf, _, err := util.CreateSSHAuthorizedKey(tmpd) - if err != nil { - return 0, err - } - - builder, virtioJournalConfig, err := newQemuBuilderWithDisk(outdir) - if err != nil { - return 0, err - } - inst.Builder = builder - completionChannel, err := inst.Builder.VirtioChannelRead("testisocompletion") - if err != nil { - return 0, err - } - - var isoKernelArgs []string - var keys []string - keys = append(keys, strings.TrimSpace(string(sshPubKeyBuf))) - virtioJournalConfig.AddAuthorizedKeys("core", keys) - - liveConfig := *virtioJournalConfig - liveConfig.AddSystemdUnit("live-signal-ok.service", liveSignalOKUnit, conf.Enable) - liveConfig.AddSystemdUnit("verify-no-efi-boot-entry.service", verifyNoEFIBootEntry, conf.Enable) - liveConfig.AddSystemdUnit("iso-not-mounted-when-fromram.service", isoNotMountedUnit, conf.Enable) - liveConfig.AddSystemdUnit("coreos-test-entered-emergency-target.service", signalFailureUnit, conf.Enable) - volumeIdUnitContents := fmt.Sprintf(verifyIsoVolumeId, kola.CosaBuild.Meta.Name) - liveConfig.AddSystemdUnit("verify-iso-volume-id.service", volumeIdUnitContents, conf.Enable) - - targetConfig := *virtioJournalConfig - targetConfig.AddSystemdUnit("coreos-test-installer.service", signalCompletionUnit, conf.Enable) - targetConfig.AddSystemdUnit("coreos-test-entered-emergency-target.service", signalFailureUnit, conf.Enable) - targetConfig.AddSystemdUnit("coreos-test-installer-no-ignition.service", checkNoIgnition, conf.Enable) - if inst.MultiPathDisk { - targetConfig.AddSystemdUnit("coreos-test-installer-multipathed.service", multipathedRoot, conf.Enable) - } - - if addNmKeyfile { - liveConfig.AddSystemdUnit("coreos-test-nm-keyfile.service", verifyNmKeyfile, conf.Enable) - targetConfig.AddSystemdUnit("coreos-test-nm-keyfile.service", verifyNmKeyfile, conf.Enable) - // NM keyfile via `iso network embed` - inst.NmKeyfiles[nmConnectionFile] = nmConnection - // nmstate config via live Ignition config, propagated via - // --copy-network, which is enabled by inst.NmKeyfiles - liveConfig.AddFile(nmstateConfigFile, nmstateConfig, 0644) - } - - if isISOFromRAM { - isoKernelArgs = append(isoKernelArgs, liveISOFromRAMKarg) - } - - mach, err := inst.InstallViaISOEmbed(isoKernelArgs, liveConfig, targetConfig, outdir, isOffline, minimal) - if err != nil { - return 0, errors.Wrapf(err, "running iso install") - } - defer func() { - if err := mach.Destroy(); err != nil { - plog.Errorf("Failed to destroy iso: %v", err) - } - }() - - return awaitCompletion(ctx, mach.QemuInst, outdir, completionChannel, mach.BootStartedErrorChannel, []string{liveOKSignal, signalCompleteString}) -} - -// testLiveFIPS verifies that adding fips=1 to the ISO results in a FIPS mode system -func testLiveFIPS(ctx context.Context, outdir string) (time.Duration, error) { - tmpd, err := os.MkdirTemp("", "kola-testiso") - if err != nil { - return 0, err - } - defer os.RemoveAll(tmpd) - - builddir := kola.CosaBuild.Dir - isopath := filepath.Join(builddir, kola.CosaBuild.Meta.BuildArtifacts.LiveIso.Path) - builder, config, err := newQemuBuilder(outdir) - if err != nil { - return 0, err - } - defer builder.Close() - if err := builder.AddIso(isopath, "", false); err != nil { - return 0, err - } - - // This is the core change under test - adding the `fips=1` kernel argument via - // coreos-installer iso kargs modify should enter fips mode. - // Removing this line should cause this test to fail. - builder.AppendKernelArgs = "fips=1" - - completionChannel, err := builder.VirtioChannelRead("testisocompletion") - if err != nil { - return 0, err - } - - config.AddSystemdUnit("fips-verify.service", ` -[Unit] -OnFailure=emergency.target -OnFailureJobMode=isolate -Before=fips-signal-ok.service - -[Service] -Type=oneshot -RemainAfterExit=yes -ExecStart=grep 1 /proc/sys/crypto/fips_enabled -ExecStart=grep FIPS etc/crypto-policies/config - -[Install] -RequiredBy=fips-signal-ok.service -`, conf.Enable) - config.AddSystemdUnit("fips-signal-ok.service", liveSignalOKUnit, conf.Enable) - config.AddSystemdUnit("fips-emergency-target.service", signalFailureUnit, conf.Enable) - - // Just for reliability, we'll run fully offline - builder.Append("-net", "none") - - builder.SetConfig(config) - mach, err := builder.Exec() - if err != nil { - return 0, errors.Wrapf(err, "running iso") - } - defer mach.Destroy() - - return awaitCompletion(ctx, mach, outdir, completionChannel, nil, []string{liveOKSignal}) -} - -func testLiveLogin(ctx context.Context, outdir string) (time.Duration, error) { - builddir := kola.CosaBuild.Dir - isopath := filepath.Join(builddir, kola.CosaBuild.Meta.BuildArtifacts.LiveIso.Path) - builder, err := newBaseQemuBuilder(outdir) - if err != nil { - return 0, err - } - defer builder.Close() - // Drop the bootindex bit (applicable to all arches except s390x and ppc64le); we want it to be the default - if err := builder.AddIso(isopath, "", false); err != nil { - return 0, err - } - - completionChannel, err := builder.VirtioChannelRead("coreos.liveiso-success") - if err != nil { - return 0, err - } - - // No network device to test https://github.com/coreos/fedora-coreos-config/pull/326 - builder.Append("-net", "none") - - mach, err := builder.Exec() - if err != nil { - return 0, errors.Wrapf(err, "running iso") - } - defer mach.Destroy() - - return awaitCompletion(ctx, mach, outdir, completionChannel, nil, []string{"coreos-liveiso-success"}) -} - -func testAsDisk(ctx context.Context, outdir string) (time.Duration, error) { - builddir := kola.CosaBuild.Dir - isopath := filepath.Join(builddir, kola.CosaBuild.Meta.BuildArtifacts.LiveIso.Path) - builder, config, err := newQemuBuilder(outdir) - if err != nil { - return 0, err - } - defer builder.Close() - // Drop the bootindex bit (applicable to all arches except s390x and ppc64le); we want it to be the default - if err := builder.AddIso(isopath, "", true); err != nil { - return 0, err - } - - completionChannel, err := builder.VirtioChannelRead("testisocompletion") - if err != nil { - return 0, err - } - - config.AddSystemdUnit("live-signal-ok.service", liveSignalOKUnit, conf.Enable) - config.AddSystemdUnit("verify-no-efi-boot-entry.service", verifyNoEFIBootEntry, conf.Enable) - builder.SetConfig(config) - - mach, err := builder.Exec() - if err != nil { - return 0, errors.Wrapf(err, "running iso") - } - defer mach.Destroy() - - return awaitCompletion(ctx, mach, outdir, completionChannel, nil, []string{liveOKSignal}) -} - -// iscsi_butane_setup.yaml contains the full butane config but here is an overview of the setup -// 1 - Boot a live ISO with two extra 10G disks with labels "target" and "var" -// - Format and mount `virtio-var` to /var -// -// 2 - target.container -> start an iscsi target, using quay.io/coreos-assembler/targetcli -// 3 - setup-targetcli.service calls /usr/local/bin/targetcli_script: -// - instructs targetcli to serve /dev/disk/by-id/virtio-target as an iscsi target -// - disables authentication -// - verifies the iscsi service is active and reachable -// -// 4 - install-coreos-to-iscsi-target.service calls /usr/local/bin/install-coreos-iscsi: -// - mount iscsi target -// - run coreos-installer on the mounted block device -// - unmount iscsi -// -// 5 - coreos-iscsi-vm.container -> start a coreos-assembler conainer: -// - launch kola qemuexec instructing it to boot from an iPXE script -// wich in turns mount the iscsi target and load kernel -// - note the virtserial port device: we pass through the serial port -// that was created by kola for test completion -// -// 6 - /var/nested-ign.json contains an ignition config: -// - when the system is booted, write a success string to /dev/virtio-ports/testisocompletion -// - as this serial device is mapped to the host serial device, the test concludes -func testLiveInstalliscsi(ctx context.Context, inst platform.Install, outdir string, butane string) (time.Duration, error) { - - builddir := kola.CosaBuild.Dir - isopath := filepath.Join(builddir, kola.CosaBuild.Meta.BuildArtifacts.LiveIso.Path) - builder, err := newBaseQemuBuilder(outdir) - if err != nil { - return 0, err - } - defer builder.Close() - if err := builder.AddIso(isopath, "", false); err != nil { - return 0, err - } - - completionChannel, err := builder.VirtioChannelRead("testisocompletion") - if err != nil { - return 0, err - } - - // Create a serial channel to read the logs from the nested VM - nestedVmLogsChannel, err := builder.VirtioChannelRead("nestedvmlogs") - if err != nil { - return 0, err - } - - // Create a file to write the contents of the serial channel into - nestedVMConsole, err := os.OpenFile(filepath.Join(outdir, "nested_vm_console.txt"), os.O_WRONLY|os.O_CREATE, 0644) - if err != nil { - return 0, err - } - - go func() { - _, err := io.Copy(nestedVMConsole, nestedVmLogsChannel) - if err != nil && err != io.EOF { - panic(err) - } - }() - - // empty disk to use as an iscsi target to install coreOS on and subseqently boot - // Also add a 10G disk that we will mount on /var, to increase space available when pulling containers - err = builder.AddDisksFromSpecs([]string{"10G:serial=target", "10G:serial=var"}) - if err != nil { - return 0, err - } - - // We need more memory to start another VM within ! - builder.MemoryMiB = 2048 - - var iscsiTargetConfig = conf.Butane(butane) - - config, err := iscsiTargetConfig.Render(conf.FailWarnings) - if err != nil { - return 0, err - } - err = forwardJournal(outdir, builder, config) - if err != nil { - return 0, err - } - - // Add a failure target to stop the test if something go wrong rather than waiting for the 10min timeout - config.AddSystemdUnit("coreos-test-entered-emergency-target.service", signalFailureUnit, conf.Enable) - - // enable network - builder.EnableUsermodeNetworking([]platform.HostForwardPort{}, "") - - // keep auto-login enabled for easier debug when running console - config.AddAutoLogin() - - builder.SetConfig(config) - - // Bind mount in the COSA rootfs into the VM so we can use it as a - // read-only rootfs for quickly starting the container to kola - // qemuexec the nested VM for the test. See resources/iscsi_butane_setup.yaml - builder.MountHost("/", "/var/cosaroot", true) - config.MountHost("/var/cosaroot", true) - - mach, err := builder.Exec() - if err != nil { - return 0, errors.Wrapf(err, "running iso") - } - defer mach.Destroy() - - return awaitCompletion(ctx, mach, outdir, completionChannel, nil, []string{"iscsi-boot-ok"}) -} diff --git a/mantle/kola/registry/registry.go b/mantle/kola/registry/registry.go index c22c474e86..94a1ab66c3 100644 --- a/mantle/kola/registry/registry.go +++ b/mantle/kola/registry/registry.go @@ -7,6 +7,7 @@ import ( _ "github.com/coreos/coreos-assembler/mantle/kola/tests/etcd" _ "github.com/coreos/coreos-assembler/mantle/kola/tests/fips" _ "github.com/coreos/coreos-assembler/mantle/kola/tests/ignition" + _ "github.com/coreos/coreos-assembler/mantle/kola/tests/iso" _ "github.com/coreos/coreos-assembler/mantle/kola/tests/metadata" _ "github.com/coreos/coreos-assembler/mantle/kola/tests/misc" _ "github.com/coreos/coreos-assembler/mantle/kola/tests/ostree" diff --git a/mantle/kola/tests/iso/common.go b/mantle/kola/tests/iso/common.go new file mode 100644 index 0000000000..09123e0c35 --- /dev/null +++ b/mantle/kola/tests/iso/common.go @@ -0,0 +1,364 @@ +package iso + +import ( + "bufio" + "fmt" + "io" + "os" + "path/filepath" + "strings" + "time" + + "github.com/coreos/coreos-assembler/mantle/kola" + "github.com/coreos/coreos-assembler/mantle/kola/cluster" + "github.com/coreos/coreos-assembler/mantle/kola/register" + "github.com/pkg/errors" +) + +const ( + installTimeoutMins = 12 + + // defaultQemuHostIPv4 is documented in `man qemu-kvm`, under the `-netdev` option + defaultQemuHostIPv4 = "10.0.2.2" +) + +// This object gets serialized to YAML and fed to coreos-installer: +// https://coreos.github.io/coreos-installer/customizing-install/#config-file-format +type CoreosInstallerConfig struct { + ImageURL string `yaml:"image-url,omitempty"` + IgnitionFile string `yaml:"ignition-file,omitempty"` + Insecure bool `yaml:"insecure,omitempty"` + AppendKargs []string `yaml:"append-karg,omitempty"` + CopyNetwork bool `yaml:"copy-network,omitempty"` + DestDevice string `yaml:"dest-device,omitempty"` + Console []string `yaml:"console,omitempty"` +} + +type IsoTestOpts struct { + // Flags().BoolVarP(&instInsecure, "inst-insecure", "S", false, "Do not verify signature on metal image") + instInsecure bool + addNmKeyfile bool + enable4k bool + enableMultipath bool + isOffline bool + isISOFromRAM bool + isMiniso bool + enableUefi bool + enableUefiSecure bool + enableIbft bool + manual bool + pxeAppendRootfs bool +} + +func (o *IsoTestOpts) SetInsecureOnDevBuild() { + // Ignore signing verification by default when running with development build + // https://github.com/coreos/fedora-coreos-tracker/issues/908 + if kola.QEMUOptions.InstInsecure || strings.Contains(kola.CosaBuild.Meta.BuildID, ".dev.") { + o.instInsecure = true + //fmt.Printf("Detected development build; disabling signature verification\n") + } +} + +func isoTest(name string, run func(c cluster.TestCluster), arch []string) *register.Test { + return ®ister.Test{ + Run: run, + ClusterSize: 0, + Name: "iso." + name, + Timeout: installTimeoutMins * time.Minute, + Platforms: []string{"qemu"}, + Architectures: arch, + } +} + +func checkTestOutput(output *os.File, expected []string) error { + reader := bufio.NewReader(output) + for _, exp := range expected { + line, err := reader.ReadString('\n') + if err != nil { + if err == io.EOF { + // this may be from QEMU getting killed or exiting; wait a bit + // to give a chance for .Wait() above to feed the channel with a + // better error + time.Sleep(1 * time.Second) + return fmt.Errorf("got EOF from completion channel, %s expected", exp) + } else { + return errors.Wrapf(err, "reading from completion channel") + } + } + line = strings.TrimSpace(line) + if line != exp { + return fmt.Errorf("unexpected string from completion channel: %q, expected: %q", line, exp) + } + } + return nil +} + +func ensureLiveArtifactsExist() error { + if kola.CosaBuild.Meta.BuildArtifacts.LiveIso == nil { + return errors.Errorf("Build %s is missing live-iso artifacts\n", kola.CosaBuild.Meta.Name) + } + if kola.CosaBuild.Meta.BuildArtifacts.LiveRootfs == nil || kola.CosaBuild.Meta.BuildArtifacts.LiveKernel == nil || kola.CosaBuild.Meta.BuildArtifacts.LiveInitramfs == nil { + return errors.Errorf("Build %s is missing live artifacts\n", kola.CosaBuild.Meta.Name) + } + if kola.CosaBuild.Meta.BuildArtifacts.Metal == nil || kola.CosaBuild.Meta.BuildArtifacts.Metal4KNative == nil { + return errors.Errorf("Build %s is missing live metal artifacts\n", kola.CosaBuild.Meta.Name) + } + return nil +} + +// Sometimes the logs that stream from various virtio streams can be +// incomplete because they depend on services inside the guest. +// When you are debugging earlyboot/initramfs issues this can be +// problematic. Let's add a hook here to enable more debugging. +func renderCosaTestIsoDebugKargs() []string { + if _, ok := os.LookupEnv("COSA_TESTISO_DEBUG"); ok { + return []string{"systemd.log_color=0", "systemd.log_level=debug", + "systemd.journald.forward_to_console=1", + "systemd.journald.max_level_console=debug"} + } else { + return []string{} + } +} + +func absSymlink(src, dest string) error { + src, err := filepath.Abs(src) + if err != nil { + return err + } + return os.Symlink(src, dest) +} + +// setupMetalImage creates a symlink to the metal image. +func setupMetalImage(builddir, metalimg, destdir string) (string, error) { + if err := absSymlink(filepath.Join(builddir, metalimg), filepath.Join(destdir, metalimg)); err != nil { + return "", err + } + return metalimg, nil +} + +func cat(outfile string, infiles ...string) error { + out, err := os.OpenFile(outfile, os.O_WRONLY|os.O_CREATE, 0644) + if err != nil { + return err + } + defer out.Close() + for _, infile := range infiles { + in, err := os.Open(infile) + if err != nil { + return err + } + defer in.Close() + _, err = io.Copy(out, in) + if err != nil { + return err + } + } + return nil +} + +var liveOKSignal = "live-test-OK" +var liveSignalOKUnit = fmt.Sprintf(` +[Unit] +Description=TestISO Signal Live ISO Completion +Requires=dev-virtio\\x2dports-testisocompletion.device +OnFailure=emergency.target +OnFailureJobMode=isolate +Before=coreos-installer.service +[Service] +Type=oneshot +RemainAfterExit=yes +ExecStart=/bin/sh -c '/usr/bin/echo %s >/dev/virtio-ports/testisocompletion' +[Install] +# for install tests +RequiredBy=coreos-installer.target +# for iso-as-disk +RequiredBy=multi-user.target`, liveOKSignal) + +var signalCompleteString = "coreos-installer-test-OK" +var signalCompletionUnit = fmt.Sprintf(` +[Unit] +Description=TestISO Signal Completion +Requires=dev-virtio\\x2dports-testisocompletion.device +OnFailure=emergency.target +OnFailureJobMode=isolate +[Service] +Type=oneshot +RemainAfterExit=yes +ExecStart=/bin/sh -c '/usr/bin/echo %s >/dev/virtio-ports/testisocompletion && systemctl poweroff' +[Install] +RequiredBy=multi-user.target`, signalCompleteString) + +var signalEmergencyString = "coreos-installer-test-entered-emergency-target" +var signalFailureUnit = fmt.Sprintf(` +[Unit] +Description=TestISO Signal Failure +Requires=dev-virtio\\x2dports-testisocompletion.device +DefaultDependencies=false +[Service] +Type=oneshot +RemainAfterExit=yes +ExecStart=/bin/sh -c '/usr/bin/echo %s >/dev/virtio-ports/testisocompletion && systemctl poweroff' +[Install] +RequiredBy=emergency.target`, signalEmergencyString) + +var multipathedRoot = `[Unit] +Description=TestISO Verify Multipathed Root +OnFailure=emergency.target +OnFailureJobMode=isolate +Before=coreos-test-installer.service +[Service] +Type=oneshot +RemainAfterExit=yes +ExecStart=/bin/bash -c 'lsblk -pno NAME "/dev/mapper/$(multipath -l -v 1)" | grep -qw "$(findmnt -nvr /sysroot -o SOURCE)"' +[Install] +RequiredBy=multi-user.target` + +var checkNoIgnition = ` +[Unit] +Description=TestISO Verify No Ignition Config +OnFailure=emergency.target +OnFailureJobMode=isolate +Before=coreos-test-installer.service +After=coreos-ignition-firstboot-complete.service +RequiresMountsFor=/boot +[Service] +Type=oneshot +RemainAfterExit=yes +ExecStart=/bin/sh -c '[ ! -e /boot/ignition ]' +[Install] +RequiredBy=multi-user.target` + +// This test is broken. Please fix! +// https://github.com/coreos/coreos-assembler/issues/3554 +var verifyNoEFIBootEntry = ` +[Unit] +Description=TestISO Verify No EFI Boot Entry +OnFailure=emergency.target +OnFailureJobMode=isolate +ConditionPathExists=/sys/firmware/efi +Before=live-signal-ok.service +[Service] +Type=oneshot +RemainAfterExit=yes +ExecStart=/bin/sh -c '! efibootmgr -v | grep -E "(HD|CDROM)\("' +[Install] +# for install tests +RequiredBy=coreos-installer.target +# for iso-as-disk +RequiredBy=multi-user.target` + +// Verify that the volume ID is the OS name. See also +// https://github.com/openshift/assisted-image-service/pull/477. +// This is the same as the LABEL of the block device for ISO9660. See +// https://github.com/util-linux/util-linux/blob/643bdae8e38055e36acf2963c3416de206081507/libblkid/src/superblocks/iso9660.c#L366-L377 +var verifyIsoVolumeId = ` +[Unit] +Description=Verify ISO Volume ID +OnFailure=emergency.target +OnFailureJobMode=isolate +# only if we're actually mounting the ISO +ConditionPathIsMountPoint=/run/media/iso +[Service] +Type=oneshot +RemainAfterExit=yes +# the backing device name is arch-dependent, but we know it's mounted on /run/media/iso +ExecStart=bash -c "[[ $(findmnt -no LABEL /run/media/iso) == %s-* ]]" +[Install] +RequiredBy=coreos-installer.target` + +// Unit to check that /run/media/iso is not mounted when +// coreos.liveiso.fromram kernel argument is passed +var isoNotMountedUnit = ` +[Unit] +Description=Verify ISO is not mounted when coreos.liveiso.fromram +OnFailure=emergency.target +OnFailureJobMode=isolate +ConditionKernelCommandLine=coreos.liveiso.fromram +[Service] +Type=oneshot +StandardOutput=kmsg+console +StandardError=kmsg+console +RemainAfterExit=yes +# Would like to use SuccessExitStatus but it doesn't support what +# we want: https://github.com/systemd/systemd/issues/10297#issuecomment-1672002635 +ExecStart=bash -c "if mountpoint /run/media/iso 2>/dev/null; then exit 1; fi" +[Install] +RequiredBy=coreos-installer.target` + +var nmConnectionId = "CoreOS DHCP" +var nmConnectionFile = "coreos-dhcp.nmconnection" +var nmConnection = fmt.Sprintf(`[connection] +id=%s +type=ethernet +# add wait-device-timeout here so we make sure NetworkManager-wait-online.service will +# wait for a device to be present before exiting. See +# https://github.com/coreos/fedora-coreos-tracker/issues/1275#issuecomment-1231605438 +wait-device-timeout=20000 + +[ipv4] +method=auto +`, nmConnectionId) + +var nmstateConfigFile = "/etc/nmstate/br-ex.yml" +var nmstateConfig = `interfaces: + - name: br-ex + type: linux-bridge + state: up + ipv4: + enabled: false + ipv6: + enabled: false + bridge: + port: [] +` + +// This is used to verify *both* the live and the target system in the `--add-nm-keyfile` path. +var verifyNmKeyfile = fmt.Sprintf(`[Unit] +Description=TestISO Verify NM Keyfile Propagation +OnFailure=emergency.target +OnFailureJobMode=isolate +Wants=network-online.target +After=network-online.target +Before=live-signal-ok.service +Before=coreos-test-installer.service +[Service] +Type=oneshot +RemainAfterExit=yes +ExecStart=/usr/bin/journalctl -u nm-initrd --no-pager --grep "policy: set '%[1]s' (.*) as default .* routing and DNS" +ExecStart=/usr/bin/journalctl -u NetworkManager --no-pager --grep "policy: set '%[1]s' (.*) as default .* routing and DNS" +ExecStart=/usr/bin/grep "%[1]s" /etc/NetworkManager/system-connections/%[2]s +# Also verify nmstate config +ExecStart=/usr/bin/nmcli c show br-ex +[Install] +# for live system +RequiredBy=coreos-installer.target +# for target system +RequiredBy=multi-user.target`, nmConnectionId, nmConnectionFile) + +var bootStartedSignal = "boot-started-OK" +var bootStartedUnit = fmt.Sprintf(`[Unit] +Description=TestISO Boot Started +Requires=dev-virtio\\x2dports-bootstarted.device +OnFailure=emergency.target +OnFailureJobMode=isolate +[Service] +Type=oneshot +RemainAfterExit=yes +ExecStart=/bin/sh -c '/usr/bin/echo %s >/dev/virtio-ports/bootstarted' +[Install] +RequiredBy=coreos-installer.target`, bootStartedSignal) + +var coreosInstallerMultipathUnit = `[Unit] +Description=TestISO Enable Multipath +Before=multipathd.service +DefaultDependencies=no +[Service] +Type=oneshot +RemainAfterExit=yes +ExecStart=/usr/sbin/mpathconf --enable +[Install] +WantedBy=coreos-installer.target` + +var waitForMpathTargetConf = `[Unit] +Requires=dev-mapper-mpatha.device +After=dev-mapper-mpatha.device` diff --git a/mantle/cmd/kola/resources/iscsi_butane_setup.yaml b/mantle/kola/tests/iso/iscsi_butane_setup.yaml similarity index 100% rename from mantle/cmd/kola/resources/iscsi_butane_setup.yaml rename to mantle/kola/tests/iso/iscsi_butane_setup.yaml diff --git a/mantle/kola/tests/iso/live-as-disk.go b/mantle/kola/tests/iso/live-as-disk.go new file mode 100644 index 0000000000..cc8a838c28 --- /dev/null +++ b/mantle/kola/tests/iso/live-as-disk.go @@ -0,0 +1,101 @@ +package iso + +import ( + "fmt" + "path/filepath" + + "github.com/coreos/coreos-assembler/mantle/kola" + "github.com/coreos/coreos-assembler/mantle/kola/cluster" + "github.com/coreos/coreos-assembler/mantle/kola/register" + "github.com/coreos/coreos-assembler/mantle/platform" + "github.com/coreos/coreos-assembler/mantle/platform/conf" + "github.com/coreos/coreos-assembler/mantle/platform/machine/qemu" + "github.com/pkg/errors" +) + +func init() { + // The iso-as-disk tests are only supported in x86_64 because other + // architectures don't have the required hybrid partition table. + register.RegisterTest(isoTest("as-disk", isoAsDisk, []string{"x86_64"})) + register.RegisterTest(isoTest("as-disk.uefi", isoAsDiskUefi, []string{"x86_64"})) + register.RegisterTest(isoTest("as-disk.uefi-secure", isoAsDiskUefiSecure, []string{"x86_64"})) +} + +func isoAsDisk(c cluster.TestCluster) { + opts := IsoTestOpts{} + opts.SetInsecureOnDevBuild() + isoTestAsDisk(c, opts) +} + +func isoAsDiskUefi(c cluster.TestCluster) { + opts := IsoTestOpts{ + enableUefi: true, + } + opts.SetInsecureOnDevBuild() + isoTestAsDisk(c, opts) +} +func isoAsDiskUefiSecure(c cluster.TestCluster) { + opts := IsoTestOpts{ + enableUefiSecure: true, + } + opts.SetInsecureOnDevBuild() + isoTestAsDisk(c, opts) +} + +func isoTestAsDisk(c cluster.TestCluster, opts IsoTestOpts) { + if err := ensureLiveArtifactsExist(); err != nil { + fmt.Println(err) + return + } + + qc, ok := c.Cluster.(*qemu.Cluster) + if !ok { + c.Fatalf("Unsupported cluster type") + } + + config, err := conf.EmptyIgnition().Render(conf.FailWarnings) + if err != nil { + c.Fatal(err) + } + config.AddSystemdUnit("live-signal-ok.service", liveSignalOKUnit, conf.Enable) + config.AddSystemdUnit("verify-no-efi-boot-entry.service", verifyNoEFIBootEntry, conf.Enable) + + overrideFW := func(builder *platform.QemuBuilder) error { + switch { + case opts.enableUefiSecure: + builder.Firmware = "uefi-secure" + case opts.enableUefi: + builder.Firmware = "uefi" + } + return nil + } + + errchan := make(chan error) + setupDisks := func(_ platform.QemuMachineOptions, builder *platform.QemuBuilder) error { + output, err := builder.VirtioChannelRead("testisocompletion") + if err != nil { + return errors.Wrap(err, "setting up virtio-serial channel") + } + + // Read line in a goroutine and send errors to channel + go func() { + errchan <- checkTestOutput(output, []string{liveOKSignal}) + }() + + isopath := filepath.Join(kola.CosaBuild.Dir, kola.CosaBuild.Meta.BuildArtifacts.LiveIso.Path) + return builder.AddIso(isopath, "", true) + } + + extra := platform.QemuMachineOptions{} + extra.SkipStartMachine = true + callbacks := qemu.BuilderCallbacks{SetupDisks: setupDisks, OverrideDefaults: overrideFW} + _, err = qc.NewMachineWithQemuOptionsAndBuilderCallbacks(config, extra, callbacks) + if err != nil { + c.Fatalf("Unable to create test machine: %v", err) + } + + err = <-errchan + if err != nil { + c.Fatal(err) + } +} diff --git a/mantle/kola/tests/iso/live-fips.go b/mantle/kola/tests/iso/live-fips.go new file mode 100644 index 0000000000..4a848fee83 --- /dev/null +++ b/mantle/kola/tests/iso/live-fips.go @@ -0,0 +1,98 @@ +package iso + +import ( + "fmt" + "path/filepath" + + "github.com/coreos/coreos-assembler/mantle/kola" + "github.com/coreos/coreos-assembler/mantle/kola/cluster" + "github.com/coreos/coreos-assembler/mantle/kola/register" + "github.com/coreos/coreos-assembler/mantle/platform" + "github.com/coreos/coreos-assembler/mantle/platform/conf" + "github.com/coreos/coreos-assembler/mantle/platform/machine/qemu" + "github.com/pkg/errors" +) + +func init() { + register.RegisterTest(®ister.Test{ + Run: testLiveFIPS, + ClusterSize: 0, + Name: "iso.fips.uefi", + Description: "verifies that adding fips=1 to the ISO results in a FIPS mode system", + Distros: []string{"rhcos"}, + Platforms: []string{"qemu"}, + Architectures: []string{"x86_64", "aarch64"}, + }) +} + +var fipsVerify = `[Unit] +OnFailure=emergency.target +OnFailureJobMode=isolate +Before=fips-signal-ok.service + +[Service] +Type=oneshot +RemainAfterExit=yes +ExecStart=grep 1 /proc/sys/crypto/fips_enabled +ExecStart=grep FIPS etc/crypto-policies/config + +[Install] +RequiredBy=fips-signal-ok.service` + +func testLiveFIPS(c cluster.TestCluster) { + if err := ensureLiveArtifactsExist(); err != nil { + fmt.Println(err) + return + } + + qc, ok := c.Cluster.(*qemu.Cluster) + if !ok { + c.Fatalf("Unsupported cluster type") + } + + config, err := conf.EmptyIgnition().Render(conf.FailWarnings) + if err != nil { + c.Fatal(err) + } + config.AddSystemdUnit("fips-verify.service", fipsVerify, conf.Enable) + config.AddSystemdUnit("fips-signal-ok.service", liveSignalOKUnit, conf.Enable) + config.AddSystemdUnit("fips-emergency-target.service", signalFailureUnit, conf.Enable) + + overrideFW := func(builder *platform.QemuBuilder) error { + builder.Firmware = "uefi" + // This is the core change under test - adding the `fips=1` kernel argument via + // coreos-installer iso kargs modify should enter fips mode. + // Removing this line should cause this test to fail. + builder.AppendKernelArgs = "fips=1" + return nil + } + + errchan := make(chan error) + setupDisks := func(_ platform.QemuMachineOptions, builder *platform.QemuBuilder) error { + output, err := builder.VirtioChannelRead("testisocompletion") + if err != nil { + return errors.Wrap(err, "setting up virtio-serial channel") + } + + // Read line in a goroutine and send errors to channel + go func() { + errchan <- checkTestOutput(output, []string{liveOKSignal}) + }() + + isopath := filepath.Join(kola.CosaBuild.Dir, kola.CosaBuild.Meta.BuildArtifacts.LiveIso.Path) + return builder.AddIso(isopath, "", false) + } + + extra := platform.QemuMachineOptions{} + extra.SkipStartMachine = true + callbacks := qemu.BuilderCallbacks{SetupDisks: setupDisks, OverrideDefaults: overrideFW} + _, err = qc.NewMachineWithQemuOptionsAndBuilderCallbacks(config, extra, callbacks) + if err != nil { + c.Fatalf("Unable to create test machine: %v", err) + } + + err = <-errchan + if err != nil { + c.Fatal(err) + } +} diff --git a/mantle/kola/tests/iso/live-iscsi.go b/mantle/kola/tests/iso/live-iscsi.go new file mode 100644 index 0000000000..9c504c8a6d --- /dev/null +++ b/mantle/kola/tests/iso/live-iscsi.go @@ -0,0 +1,190 @@ +package iso + +import ( + _ "embed" + "fmt" + "io" + "os" + "path/filepath" + "strings" + + "github.com/coreos/coreos-assembler/mantle/kola" + "github.com/coreos/coreos-assembler/mantle/kola/cluster" + "github.com/coreos/coreos-assembler/mantle/kola/register" + "github.com/coreos/coreos-assembler/mantle/platform" + "github.com/coreos/coreos-assembler/mantle/platform/conf" + "github.com/coreos/coreos-assembler/mantle/platform/machine/qemu" + "github.com/pkg/errors" +) + +// Don't restrict networking for iSCSI tests; otherwise they fail. +func withInternet(t *register.Test) *register.Test { + t.Tags = append(t.Tags, "needs-internet") + return t +} + +func init() { + // Those currently work only on x86, see: https://github.com/coreos/fedora-coreos-tracker/issues/1657 + register.RegisterTest(withInternet(isoTest("offline-install-iscsi.ibft.uefi", isoOfflineInstallIscsiIbftUefi, []string{"x86_64"}))) + register.RegisterTest(withInternet(isoTest("offline-install-iscsi.ibft-with-mpath", isoOfflineInstallIscsiIbftMpath, []string{"x86_64"}))) + register.RegisterTest(withInternet(isoTest("offline-install-iscsi.manual", isoOfflineInstallIscsiManual, []string{"x86_64"}))) +} + +func isoOfflineInstallIscsiIbftUefi(c cluster.TestCluster) { + opts := IsoTestOpts{ + enableUefi: true, + isOffline: true, + enableIbft: true, + } + opts.SetInsecureOnDevBuild() + isoInstalliScsi(c, opts) +} + +func isoOfflineInstallIscsiIbftMpath(c cluster.TestCluster) { + opts := IsoTestOpts{ + enableUefi: true, + isOffline: true, + enableMultipath: true, + enableIbft: true, + } + opts.SetInsecureOnDevBuild() + isoInstalliScsi(c, opts) +} + +func isoOfflineInstallIscsiManual(c cluster.TestCluster) { + opts := IsoTestOpts{ + isOffline: true, + manual: true, + enableIbft: true, + } + opts.SetInsecureOnDevBuild() + isoInstalliScsi(c, opts) +} + +//go:embed iscsi_butane_setup.yaml +var iscsi_butane_config string + +// iscsi_butane_setup.yaml contains the full butane config but here is an overview of the setup +// 1 - Boot a live ISO with two extra 10G disks with labels "target" and "var" +// - Format and mount `virtio-var` to /var +// +// 2 - target.container -> start an iscsi target, using quay.io/coreos-assembler/targetcli +// 3 - setup-targetcli.service calls /usr/local/bin/targetcli_script: +// - instructs targetcli to serve /dev/disk/by-id/virtio-target as an iscsi target +// - disables authentication +// - verifies the iscsi service is active and reachable +// +// 4 - install-coreos-to-iscsi-target.service calls /usr/local/bin/install-coreos-iscsi: +// - mount iscsi target +// - run coreos-installer on the mounted block device +// - unmount iscsi +// +// 5 - coreos-iscsi-vm.container -> start a coreos-assembler conainer: +// - launch kola qemuexec instructing it to boot from an iPXE script +// wich in turns mount the iscsi target and load kernel +// - note the virtserial port device: we pass through the serial port +// that was created by kola for test completion +// +// 6 - /var/nested-ign.json contains an ignition config: +// - when the system is booted, write a success string to /dev/virtio-ports/testisocompletion +// - as this serial device is mapped to the host serial device, the test concludes +func isoInstalliScsi(c cluster.TestCluster, opts IsoTestOpts) { + if err := ensureLiveArtifactsExist(); err != nil { + fmt.Println(err) + return + } + + qc, ok := c.Cluster.(*qemu.Cluster) + if !ok { + c.Fatalf("Unsupported cluster type") + } + + // Prepare config + var butane string + if opts.enableIbft && opts.enableMultipath { + butane = strings.ReplaceAll(iscsi_butane_config, "COREOS_INSTALLER_KARGS", "--append-karg rd.iscsi.firmware=1 --append-karg rd.multipath=default --append-karg root=/dev/disk/by-label/dm-mpath-root --append-karg rw") + } else if opts.enableIbft { + butane = strings.ReplaceAll(iscsi_butane_config, "COREOS_INSTALLER_KARGS", "--append-karg rd.iscsi.firmware=1") + } else if opts.manual { + butane = strings.ReplaceAll(iscsi_butane_config, "COREOS_INSTALLER_KARGS", "--append-karg netroot=iscsi:10.0.2.15::::iqn.2024-05.com.coreos:0") + } + var iscsiTargetConfig = conf.Butane(butane) + config, err := iscsiTargetConfig.Render(conf.FailWarnings) + if err != nil { + c.Fatal(err) + } + + // Add a failure target to stop the test if something go wrong rather than waiting for the 10min timeout + config.AddSystemdUnit("coreos-test-entered-emergency-target.service", signalFailureUnit, conf.Enable) + config.MountHost("/var/cosaroot", true) + + overrideFW := func(builder *platform.QemuBuilder) error { + if opts.enableUefi { + builder.Firmware = "uefi" + } + // We need more memory to start another VM within ! + builder.MemoryMiB = 2048 + return nil + } + + errchan := make(chan error) + setupDisks := func(_ platform.QemuMachineOptions, builder *platform.QemuBuilder) error { + isopath := filepath.Join(kola.CosaBuild.Dir, kola.CosaBuild.Meta.BuildArtifacts.LiveIso.Path) + if err := builder.AddIso(isopath, "", false); err != nil { + return err + } + + output, err := builder.VirtioChannelRead("testisocompletion") + if err != nil { + return errors.Wrap(err, "setting up virtio-serial channel") + } + + // Create a serial channel to read the logs from the nested VM + nestedVmLogsChannel, err := builder.VirtioChannelRead("nestedvmlogs") + if err != nil { + return err + } + // Create a file to write the contents of the serial channel into + path := filepath.Join(filepath.Dir(builder.ConsoleFile), "nested_vm_console.txt") + nestedVMConsole, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE, 0644) + if err != nil { + return err + } + go func() { + _, err := io.Copy(nestedVMConsole, nestedVmLogsChannel) + if err != nil && err != io.EOF { + fmt.Printf("error copying nested VM logs: %v\n", err) + } + }() + + // empty disk to use as an iscsi target to install coreOS on and subseqently boot + // Also add a 10G disk that we will mount on /var, to increase space available when pulling containers + if err := builder.AddDisksFromSpecs([]string{"10G:serial=target", "10G:serial=var"}); err != nil { + return err + } + // Bind mount in the COSA rootfs into the VM so we can use it as a + // read-only rootfs for quickly starting the container to kola + // qemuexec the nested VM for the test. See resources/iscsi_butane_setup.yaml + builder.MountHost("/", "/var/cosaroot", true) + + // Read line in a goroutine and send errors to channel + go func() { + errchan <- checkTestOutput(output, []string{"iscsi-boot-ok"}) + }() + + return nil + } + + extra := platform.QemuMachineOptions{} + extra.SkipStartMachine = true + callbacks := qemu.BuilderCallbacks{SetupDisks: setupDisks, OverrideDefaults: overrideFW} + _, err = qc.NewMachineWithQemuOptionsAndBuilderCallbacks(config, extra, callbacks) + if err != nil { + c.Fatalf("Unable to create test machine: %v", err) + } + + err = <-errchan + if err != nil { + c.Fatal(err) + } +} diff --git a/mantle/kola/tests/iso/live-iso.go b/mantle/kola/tests/iso/live-iso.go new file mode 100644 index 0000000000..dfdd5fd334 --- /dev/null +++ b/mantle/kola/tests/iso/live-iso.go @@ -0,0 +1,507 @@ +package iso + +import ( + "context" + _ "embed" + "fmt" + "log" + "net" + "net/http" + "os" + "os/exec" + "path/filepath" + "strings" + + "github.com/coreos/coreos-assembler/mantle/kola" + "github.com/coreos/coreos-assembler/mantle/kola/cluster" + "github.com/coreos/coreos-assembler/mantle/kola/register" + "github.com/coreos/coreos-assembler/mantle/platform" + "github.com/coreos/coreos-assembler/mantle/platform/conf" + "github.com/coreos/coreos-assembler/mantle/platform/machine/qemu" + coreosarch "github.com/coreos/stream-metadata-go/arch" + "github.com/pkg/errors" + "gopkg.in/yaml.v2" +) + +func init() { + register.RegisterTest(withInternet(isoTest("install", isoInstall, []string{"x86_64"}))) + + register.RegisterTest(isoTest("offline-install", isoOfflineInstall, []string{"x86_64", "s390x", "ppc64le"})) + register.RegisterTest(isoTest("offline-install.uefi", isoOfflineInstallUefi, []string{"aarch64"})) + register.RegisterTest(isoTest("offline-install.4k", isoOfflineInstall4k, []string{"s390x"})) + register.RegisterTest(isoTest("offline-install.mpath", isoOfflineInstallMpath, []string{"x86_64", "s390x", "ppc64le"})) + register.RegisterTest(isoTest("offline-install.mpath.uefi", isoOfflineInstallMpathUefi, []string{"aarch64"})) + register.RegisterTest(isoTest("offline-install-fromram.4k", isoOfflineInstallFromRam4k, []string{"ppc64le"})) + register.RegisterTest(isoTest("offline-install-fromram.4k.uefi", isoOfflineInstallFromRam4kUefi, []string{"x86_64", "aarch64"})) + + register.RegisterTest(withInternet(isoTest("miniso-install", isoMinisoInstall, []string{"x86_64", "s390x", "ppc64le"}))) + register.RegisterTest(withInternet(isoTest("miniso-install.uefi", isoMinisoInstallUefi, []string{"aarch64"}))) + register.RegisterTest(withInternet(isoTest("miniso-install.4k", isoMinisoInstall4k, []string{"ppc64le"}))) + register.RegisterTest(withInternet(isoTest("miniso-install.4k.uefi", isoMinisoInstall4kUefi, []string{"x86_64", "aarch64"}))) + + register.RegisterTest(withInternet(isoTest("miniso-install.nm", isoMinisoInstallNm, []string{"x86_64", "s390x", "ppc64le"}))) + register.RegisterTest(withInternet(isoTest("miniso-install.nm.uefi", isoMinisoInstallNmUefi, []string{"aarch64"}))) + register.RegisterTest(withInternet(isoTest("miniso-install.4k.nm", isoMinisoInstall4kNm, []string{"ppc64le", "s390x"}))) + register.RegisterTest(withInternet(isoTest("miniso-install.4k.nm.uefi", isoMinisoInstall4kNmUefi, []string{"x86_64", "aarch64"}))) +} + +func isoMinisoInstall(c cluster.TestCluster) { + opts := IsoTestOpts{ + isMiniso: true, + } + opts.SetInsecureOnDevBuild() + isoLiveInstall(c, opts) +} + +func isoMinisoInstallUefi(c cluster.TestCluster) { + opts := IsoTestOpts{ + isMiniso: true, + enableUefi: true, + } + opts.SetInsecureOnDevBuild() + isoLiveInstall(c, opts) +} + +func isoMinisoInstall4k(c cluster.TestCluster) { + opts := IsoTestOpts{ + enable4k: true, + isMiniso: true, + } + opts.SetInsecureOnDevBuild() + isoLiveInstall(c, opts) +} + +func isoMinisoInstall4kUefi(c cluster.TestCluster) { + opts := IsoTestOpts{ + enable4k: true, + enableUefi: true, + isMiniso: true, + } + opts.SetInsecureOnDevBuild() + isoLiveInstall(c, opts) +} + +func isoMinisoInstallNm(c cluster.TestCluster) { + opts := IsoTestOpts{ + addNmKeyfile: true, + isMiniso: true, + } + opts.SetInsecureOnDevBuild() + isoLiveInstall(c, opts) +} + +func isoMinisoInstallNmUefi(c cluster.TestCluster) { + opts := IsoTestOpts{ + addNmKeyfile: true, + isMiniso: true, + enableUefi: true, + } + opts.SetInsecureOnDevBuild() + isoLiveInstall(c, opts) +} + +func isoMinisoInstall4kNm(c cluster.TestCluster) { + opts := IsoTestOpts{ + addNmKeyfile: true, + enable4k: true, + isMiniso: true, + } + opts.SetInsecureOnDevBuild() + isoLiveInstall(c, opts) +} + +func isoMinisoInstall4kNmUefi(c cluster.TestCluster) { + opts := IsoTestOpts{ + addNmKeyfile: true, + enable4k: true, + enableUefi: true, + isMiniso: true, + } + opts.SetInsecureOnDevBuild() + isoLiveInstall(c, opts) +} + +func isoInstall(c cluster.TestCluster) { + opts := IsoTestOpts{} + opts.SetInsecureOnDevBuild() + isoLiveInstall(c, opts) +} + +func isoOfflineInstall(c cluster.TestCluster) { + opts := IsoTestOpts{ + isOffline: true, + } + opts.SetInsecureOnDevBuild() + isoLiveInstall(c, opts) +} + +func isoOfflineInstallUefi(c cluster.TestCluster) { + opts := IsoTestOpts{ + enableUefi: true, + isOffline: true, + } + opts.SetInsecureOnDevBuild() + isoLiveInstall(c, opts) +} + +func isoOfflineInstall4k(c cluster.TestCluster) { + opts := IsoTestOpts{ + enable4k: true, + isOffline: true, + } + opts.SetInsecureOnDevBuild() + isoLiveInstall(c, opts) +} + +func isoOfflineInstallMpath(c cluster.TestCluster) { + opts := IsoTestOpts{ + enableMultipath: true, + isOffline: true, + } + opts.SetInsecureOnDevBuild() + isoLiveInstall(c, opts) +} + +func isoOfflineInstallMpathUefi(c cluster.TestCluster) { + opts := IsoTestOpts{ + enableMultipath: true, + enableUefi: true, + isOffline: true, + } + opts.SetInsecureOnDevBuild() + isoLiveInstall(c, opts) +} + +func isoOfflineInstallFromRam4k(c cluster.TestCluster) { + opts := IsoTestOpts{ + enable4k: true, + isOffline: true, + isISOFromRAM: true, + } + opts.SetInsecureOnDevBuild() + isoLiveInstall(c, opts) +} + +func isoOfflineInstallFromRam4kUefi(c cluster.TestCluster) { + opts := IsoTestOpts{ + enable4k: true, + enableUefi: true, + isOffline: true, + isISOFromRAM: true, + } + opts.SetInsecureOnDevBuild() + isoLiveInstall(c, opts) +} + +func isoLiveInstall(c cluster.TestCluster, opts IsoTestOpts) { + if err := ensureLiveArtifactsExist(); err != nil { + fmt.Println(err) + return + } + if opts.isMiniso && opts.isOffline { // ideally this'd be one enum parameter + c.Fatal("Can't run minimal install offline") + } + if opts.isOffline && opts.addNmKeyfile { + c.Fatal("Cannot use `--add-nm-keyfile` with offline mode") + } + + tempdir, err := os.MkdirTemp("/var/tmp", "iso") + if err != nil { + c.Fatal(err) + } + defer func() { + os.RemoveAll(tempdir) + }() + + if err := isoRunTest(c, opts, tempdir); err != nil { + c.Fatal(err) + } +} + +func isoRunTest(c cluster.TestCluster, opts IsoTestOpts, tempdir string) error { + qc, ok := c.Cluster.(*qemu.Cluster) + if !ok { + return errors.Errorf("Unsupported cluster type") + } + if opts.enable4k { + qc.EnforceNative4k() + } + if opts.enableMultipath { + qc.EnforceMultipath() + } + + targetConfig, err := conf.EmptyIgnition().Render(conf.FailWarnings) + if err != nil { + return err + } + targetConfig.AddSystemdUnit("coreos-test-installer.service", signalCompletionUnit, conf.Enable) + targetConfig.AddSystemdUnit("coreos-test-entered-emergency-target.service", signalFailureUnit, conf.Enable) + targetConfig.AddSystemdUnit("coreos-test-installer-no-ignition.service", checkNoIgnition, conf.Enable) + if opts.enableMultipath { + targetConfig.AddSystemdUnit("coreos-test-installer-multipathed.service", multipathedRoot, conf.Enable) + } + if opts.addNmKeyfile { + targetConfig.AddSystemdUnit("coreos-test-nm-keyfile.service", verifyNmKeyfile, conf.Enable) + } + + isopath := filepath.Join(kola.CosaBuild.Dir, kola.CosaBuild.Meta.BuildArtifacts.LiveIso.Path) + + installerConfig := CoreosInstallerConfig{ + IgnitionFile: "/var/opt/pointer.ign", + DestDevice: "/dev/vda", + AppendKargs: renderCosaTestIsoDebugKargs(), + Insecure: opts.instInsecure, + CopyNetwork: opts.addNmKeyfile, // force networking on in the initrd to verify the keyfile was used + } + + var serializedTargetConfig string + if opts.isOffline { + // note we leave ImageURL empty here; offline installs should now be the + // default! + + // we want to test that a full offline install works; that includes the + // final installed host booting offline + serializedTargetConfig = targetConfig.String() + } else { + listener, err := net.Listen("tcp", ":0") + if err != nil { + return err + } + port := listener.Addr().(*net.TCPAddr).Port + baseurl := fmt.Sprintf("http://%s:%d", defaultQemuHostIPv4, port) + + // This is subtle but: for the minimal case, while we need networking to fetch the + // rootfs, the primary install flow will still rely on osmet. So let's keep ImageURL + // empty to exercise that path. In the future, this could be a separate scenario + // (likely we should drop the "offline" naming and have a "remote" tag on the + // opposite scenarios instead which fetch the metal image, so then we'd have + // "[min]iso-install" and "[min]iso-remote-install"). + if opts.isMiniso { + isopath, err = createMiniso(tempdir, isopath, baseurl) + if err != nil { + return err + } + } else { + var metalimg string + if opts.enable4k { + metalimg = kola.CosaBuild.Meta.BuildArtifacts.Metal4KNative.Path + } else { + metalimg = kola.CosaBuild.Meta.BuildArtifacts.Metal.Path + } + metalname, err := setupMetalImage(kola.CosaBuild.Dir, metalimg, tempdir) + if err != nil { + return err + } + installerConfig.ImageURL = fmt.Sprintf("%s/%s", baseurl, metalname) + } + + if opts.addNmKeyfile { + nmKeyfiles := make(map[string]string) + nmKeyfiles[nmConnectionFile] = nmConnection + if err := embedNmkeyfiles(tempdir, nmKeyfiles, isopath); err != nil { + return err + } + } + + // In this case; the target config is jut a tiny wrapper that wants to + // fetch our hosted target.ign config + // TODO also use https://github.com/coreos/coreos-installer/issues/118#issuecomment-585572952 + // when it arrives + if err := targetConfig.WriteFile(filepath.Join(tempdir, "target.ign")); err != nil { + return err + } + targetConfig, err = conf.EmptyIgnition().Render(conf.FailWarnings) + if err != nil { + return err + } + targetConfig.AddConfigSource(baseurl + "/target.ign") + serializedTargetConfig = targetConfig.String() + + ctx := c.Context() + mux := http.NewServeMux() + mux.Handle("/", http.FileServer(http.Dir(tempdir))) + srv := &http.Server{Handler: mux} + + go func() { + if err := srv.Serve(listener); err != nil && !errors.Is(err, http.ErrServerClosed) { + log.Printf("http serve: %v", err) + } + }() + + // stop server when ctx is canceled + go func() { + <-ctx.Done() + _ = srv.Shutdown(context.Background()) + }() + } + + // XXX: https://github.com/coreos/coreos-installer/issues/1171 + if coreosarch.CurrentRpmArch() != "s390x" { + installerConfig.Console = []string{platform.ConsoleKernelArgument[coreosarch.CurrentRpmArch()]} + } + if opts.enableMultipath { + // we only have one multipath device so it has to be that + installerConfig.DestDevice = "/dev/mapper/mpatha" + installerConfig.AppendKargs = append(installerConfig.AppendKargs, "rd.multipath=default", "root=/dev/disk/by-label/dm-mpath-root", "rw") + } + + installerConfigData, err := yaml.Marshal(installerConfig) + if err != nil { + return err + } + mode := 0644 + + liveConfig, err := conf.EmptyIgnition().Render(conf.FailWarnings) + if err != nil { + return err + } + liveConfig.AddSystemdUnit("live-signal-ok.service", liveSignalOKUnit, conf.Enable) + liveConfig.AddSystemdUnit("verify-no-efi-boot-entry.service", verifyNoEFIBootEntry, conf.Enable) + liveConfig.AddSystemdUnit("iso-not-mounted-when-fromram.service", isoNotMountedUnit, conf.Enable) + liveConfig.AddSystemdUnit("coreos-test-entered-emergency-target.service", signalFailureUnit, conf.Enable) + volumeIdUnitContents := fmt.Sprintf(verifyIsoVolumeId, kola.CosaBuild.Meta.Name) + liveConfig.AddSystemdUnit("verify-iso-volume-id.service", volumeIdUnitContents, conf.Enable) + liveConfig.AddSystemdUnit("boot-started.service", bootStartedUnit, conf.Enable) + liveConfig.AddFile(installerConfig.IgnitionFile, serializedTargetConfig, mode) + liveConfig.AddFile("/etc/coreos/installer.d/mantle.yaml", string(installerConfigData), mode) + liveConfig.AddAutoLogin() + if opts.enableMultipath { + liveConfig.AddSystemdUnit("coreos-installer-multipath.service", coreosInstallerMultipathUnit, conf.Enable) + liveConfig.AddSystemdUnitDropin("coreos-installer.service", "wait-for-mpath-target.conf", waitForMpathTargetConf) + } + if opts.addNmKeyfile { + liveConfig.AddSystemdUnit("coreos-test-nm-keyfile.service", verifyNmKeyfile, conf.Enable) + // nmstate config via live Ignition config, propagated via + // --copy-network, which is enabled by inst.NmKeyfiles + liveConfig.AddFile(nmstateConfigFile, nmstateConfig, 0644) + } + + overrideFW := func(builder *platform.QemuBuilder) error { + builder.MemoryMiB = 4096 + if opts.enableUefi { + builder.Firmware = "uefi" + } + kargs := renderCosaTestIsoDebugKargs() + if opts.isISOFromRAM { + // https://github.com/coreos/fedora-coreos-config/pull/2544 + kargs = append(kargs, "coreos.liveiso.fromram") + } + if opts.addNmKeyfile { + kargs = append(kargs, "rd.neednet=1") + } + builder.AppendKernelArgs = strings.Join(kargs, " ") + return nil + } + + setupNet := func(o platform.QemuMachineOptions, builder *platform.QemuBuilder) error { + if !opts.isOffline { + // also save pointer config into the output dir for debugging + path := filepath.Join(qc.RuntimeConf().OutputDir, builder.UUID, "config-target-pointer.ign") + if err := targetConfig.WriteFile(path); err != nil { + return err + } + return qc.SetupDefaultNetwork(o, builder) + } + return nil + } + + var isoCompletionOutput *os.File + var bootStartedOutput *os.File + setupDisks := func(_ platform.QemuMachineOptions, builder *platform.QemuBuilder) error { + sectorSize := 0 + if opts.enable4k { + sectorSize = 4096 + } + disk := platform.Disk{ + Size: "12G", // Arbitrary + SectorSize: sectorSize, + MultiPathDisk: opts.enableMultipath, + } + //TBD: see if we can remove this and just use AddDisk and inject bootindex during startup + if coreosarch.CurrentRpmArch() == "s390x" || coreosarch.CurrentRpmArch() == "aarch64" { + // s390x and aarch64 need to use bootindex as they don't support boot once + if err := builder.AddDisk(&disk); err != nil { + return err + } + } else { + if err := builder.AddPrimaryDisk(&disk); err != nil { + return err + } + } + isoCompletionOutput, err = builder.VirtioChannelRead("testisocompletion") + if err != nil { + return errors.Wrap(err, "setting up testisocompletion virtio-serial channel") + } + bootStartedOutput, err = builder.VirtioChannelRead("bootstarted") + if err != nil { + return errors.Wrap(err, "setting up bootstarted virtio-serial channel") + } + return builder.AddIso(isopath, "bootindex=3", false) + } + + extra := platform.QemuMachineOptions{} + extra.SkipStartMachine = true + callbacks := qemu.BuilderCallbacks{SetupDisks: setupDisks, SetupNetwork: setupNet, OverrideDefaults: overrideFW} + qm, err := qc.NewMachineWithQemuOptionsAndBuilderCallbacks(liveConfig, extra, callbacks) + if err != nil { + return errors.Wrap(err, "unable to create test machine") + } + + errchan := make(chan error) + go func() { + errchan <- checkTestOutput(isoCompletionOutput, []string{liveOKSignal, signalCompleteString}) + }() + + //check for error when switching boot order + go func() { + if err := checkTestOutput(bootStartedOutput, []string{bootStartedSignal}); err != nil { + errchan <- err + return + } + if err := qc.Instance(qm).SwitchBootOrder(); err != nil { + errchan <- errors.Wrapf(err, "switching boot order failed") + return + } + }() + + err = <-errchan + return err +} + +func createMiniso(tempd string, isopath string, url string) (string, error) { + minisopath := filepath.Join(tempd, "minimal.iso") + // This is obviously also available in the build dir, but to be realistic, + // let's take it from --rootfs-output + rootfs_path := filepath.Join(tempd, "rootfs.img") + // Ideally we'd use the coreos-installer of the target build here, because it's part + // of the test workflow, but that's complex... Sadly, probably easiest is to spin up + // a VM just to get the minimal ISO. + cmd := exec.Command("coreos-installer", "iso", "extract", "minimal-iso", isopath, + minisopath, "--output-rootfs", rootfs_path, "--rootfs-url", url+"/rootfs.img") + cmd.Stderr = os.Stderr + if err := cmd.Run(); err != nil { + return "", errors.Wrapf(err, "running coreos-installer iso extract minimal") + } + return minisopath, nil +} + +func embedNmkeyfiles(tempd string, nmKeyfiles map[string]string, isopath string) error { + var keyfileArgs []string + for nmName, nmContents := range nmKeyfiles { + path := filepath.Join(tempd, nmName) + if err := os.WriteFile(path, []byte(nmContents), 0600); err != nil { + return err + } + keyfileArgs = append(keyfileArgs, "--keyfile", path) + } + if len(keyfileArgs) > 0 { + args := []string{"iso", "network", "embed", isopath} + args = append(args, keyfileArgs...) + cmd := exec.Command("coreos-installer", args...) + cmd.Stderr = os.Stderr + if err := cmd.Run(); err != nil { + return errors.Wrapf(err, "running coreos-installer iso network embed") + } + } + return nil +} diff --git a/mantle/kola/tests/iso/live-login.go b/mantle/kola/tests/iso/live-login.go new file mode 100644 index 0000000000..6338773906 --- /dev/null +++ b/mantle/kola/tests/iso/live-login.go @@ -0,0 +1,84 @@ +package iso + +import ( + "fmt" + "path/filepath" + + "github.com/coreos/coreos-assembler/mantle/kola" + "github.com/coreos/coreos-assembler/mantle/platform" + "github.com/pkg/errors" + + "github.com/coreos/coreos-assembler/mantle/kola/cluster" + "github.com/coreos/coreos-assembler/mantle/kola/register" + "github.com/coreos/coreos-assembler/mantle/platform/conf" + "github.com/coreos/coreos-assembler/mantle/platform/machine/qemu" +) + +func init() { + register.RegisterTest(isoTest("live-login", isoLiveLogin, []string{})) + register.RegisterTest(isoTest("live-login.uefi", isoLiveLoginUefi, []string{"x86_64", "aarch64"})) + register.RegisterTest(isoTest("live-login.uefi-secure", isoLiveLoginUefiSecure, []string{"x86_64", "aarch64"})) +} + +func testLiveLogin(c cluster.TestCluster, enableUefi bool, enableUefiSecure bool) { + if err := ensureLiveArtifactsExist(); err != nil { + fmt.Println(err) + return + } + butane := conf.Butane(` +variant: fcos +version: 1.1.0`) + + errchan := make(chan error) + overrideFW := func(builder *platform.QemuBuilder) error { + switch { + case enableUefiSecure: + builder.Firmware = "uefi-secure" + case enableUefi: + builder.Firmware = "uefi" + } + return nil + } + setupDisks := func(_ platform.QemuMachineOptions, builder *platform.QemuBuilder) error { + // https://github.com/coreos/fedora-coreos-config/blob/testing-devel/overlay.d/05core/usr/lib/systemd/system/coreos-liveiso-success.service + output, err := builder.VirtioChannelRead("coreos.liveiso-success") + if err != nil { + return errors.Wrap(err, "setting up virtio-serial channel") + } + + // Read line in a goroutine and send errors to channel + go func() { + errchan <- checkTestOutput(output, []string{"coreos-liveiso-success"}) + }() + + isopath := filepath.Join(kola.CosaBuild.Dir, kola.CosaBuild.Meta.BuildArtifacts.LiveIso.Path) + return builder.AddIso(isopath, "", false) + } + + switch pc := c.Cluster.(type) { + case *qemu.Cluster: + callbacks := qemu.BuilderCallbacks{SetupDisks: setupDisks, OverrideDefaults: overrideFW} + _, err := pc.NewMachineWithQemuOptionsAndBuilderCallbacks(butane, platform.QemuMachineOptions{}, callbacks) + if err != nil { + c.Fatalf("Unable to create test machine: %v", err) + } + default: + c.Fatalf("Unsupported cluster type") + } + + err := <-errchan + if err != nil { + c.Fatal(err) + } +} + +func isoLiveLogin(c cluster.TestCluster) { + testLiveLogin(c, false, false) +} + +func isoLiveLoginUefi(c cluster.TestCluster) { + testLiveLogin(c, true, false) +} +func isoLiveLoginUefiSecure(c cluster.TestCluster) { + testLiveLogin(c, false, true) +} diff --git a/mantle/kola/tests/iso/live-pxe.go b/mantle/kola/tests/iso/live-pxe.go new file mode 100644 index 0000000000..6b5ffca4f3 --- /dev/null +++ b/mantle/kola/tests/iso/live-pxe.go @@ -0,0 +1,555 @@ +package iso + +import ( + "context" + "fmt" + "log" + "net" + "net/http" + "os" + "os/exec" + "path/filepath" + "strings" + + "github.com/coreos/coreos-assembler/mantle/kola" + "github.com/coreos/coreos-assembler/mantle/kola/cluster" + "github.com/coreos/coreos-assembler/mantle/kola/register" + "github.com/coreos/coreos-assembler/mantle/platform" + "github.com/coreos/coreos-assembler/mantle/platform/conf" + "github.com/coreos/coreos-assembler/mantle/platform/machine/qemu" + coreosarch "github.com/coreos/stream-metadata-go/arch" + "github.com/pkg/errors" + "gopkg.in/yaml.v2" +) + +func init() { + register.RegisterTest(isoTest("pxe-online-install", isoPxeOnlineInstall, []string{"x86_64"})) + register.RegisterTest(isoTest("pxe-online-install.uefi", isoPxeOnlineInstallUefi, []string{"aarch64"})) + register.RegisterTest(isoTest("pxe-online-install.4k.uefi", isoPxeOnlineInstall4kUefi, []string{"x86_64", "aarch64"})) + register.RegisterTest(isoTest("pxe-online-install.rootfs-appended", isoPxeOnlineInstallRootfsAppended, []string{"ppc64le", "s390x"})) + register.RegisterTest(isoTest("pxe-offline-install", isoPxeOfflineInstall, []string{"s390x"})) + register.RegisterTest(isoTest("pxe-offline-install.uefi", isoPxeOfflineInstallUefi, []string{"aarch64"})) + register.RegisterTest(isoTest("pxe-offline-install.4k", isoPxeOfflineInstall4k, []string{"ppc64le"})) + register.RegisterTest(isoTest("pxe-offline-install.4k.uefi", isoPxeOfflineInstall4kUefi, []string{"x86_64", "aarch64"})) + register.RegisterTest(isoTest("pxe-offline-install.rootfs-appended", isoPxeOfflineInstallRootfsAppended, []string{"x86_64"})) + register.RegisterTest(isoTest("pxe-offline-install.rootfs-appended.4k.uefi", isoPxeOfflineInstallRootfsAppended4kUefi, []string{"aarch64"})) +} + +func isoPxeOnlineInstall(c cluster.TestCluster) { + opts := IsoTestOpts{} + opts.SetInsecureOnDevBuild() + testPXE(c, opts) +} + +func isoPxeOnlineInstallUefi(c cluster.TestCluster) { + opts := IsoTestOpts{ + enableUefi: true, + } + opts.SetInsecureOnDevBuild() + testPXE(c, opts) +} + +func isoPxeOnlineInstall4kUefi(c cluster.TestCluster) { + opts := IsoTestOpts{ + enable4k: true, + enableUefi: true, + } + opts.SetInsecureOnDevBuild() + testPXE(c, opts) +} + +func isoPxeOnlineInstallRootfsAppended(c cluster.TestCluster) { + opts := IsoTestOpts{ + pxeAppendRootfs: true, + } + opts.SetInsecureOnDevBuild() + testPXE(c, opts) +} + +func isoPxeOfflineInstall(c cluster.TestCluster) { + opts := IsoTestOpts{ + isOffline: true, + } + opts.SetInsecureOnDevBuild() + testPXE(c, opts) +} + +func isoPxeOfflineInstallUefi(c cluster.TestCluster) { + opts := IsoTestOpts{ + isOffline: true, + enableUefi: true, + } + opts.SetInsecureOnDevBuild() + testPXE(c, opts) +} + +func isoPxeOfflineInstall4k(c cluster.TestCluster) { + opts := IsoTestOpts{ + isOffline: true, + enable4k: true, + } + opts.SetInsecureOnDevBuild() + testPXE(c, opts) +} + +func isoPxeOfflineInstall4kUefi(c cluster.TestCluster) { + opts := IsoTestOpts{ + isOffline: true, + enable4k: true, + enableUefi: true, + } + opts.SetInsecureOnDevBuild() + testPXE(c, opts) +} + +func isoPxeOfflineInstallRootfsAppended(c cluster.TestCluster) { + opts := IsoTestOpts{ + isOffline: true, + pxeAppendRootfs: true, + } + opts.SetInsecureOnDevBuild() + testPXE(c, opts) +} + +func isoPxeOfflineInstallRootfsAppended4kUefi(c cluster.TestCluster) { + opts := IsoTestOpts{ + isOffline: true, + pxeAppendRootfs: true, + enable4k: true, + enableUefi: true, + } + opts.SetInsecureOnDevBuild() + testPXE(c, opts) +} + +var downloadCheck = `[Unit] +Description=TestISO Verify CoreOS Installer Download +After=coreos-installer.service +Before=coreos-installer.target +[Service] +Type=oneshot +StandardOutput=kmsg+console +StandardError=kmsg+console +ExecStart=/bin/sh -c "journalctl -t coreos-installer-service | /usr/bin/awk '/[Dd]ownload/ {exit 1}'" +ExecStart=/bin/sh -c "/usr/bin/udevadm settle" +ExecStart=/bin/sh -c "/usr/bin/mount /dev/disk/by-label/root /mnt" +ExecStart=/bin/sh -c "/usr/bin/jq -er '.[\"build\"]? + .[\"version\"]? == \"%s\"' /mnt/.coreos-aleph-version.json" +[Install] +RequiredBy=coreos-installer.target +` + +func testPXE(c cluster.TestCluster, opts IsoTestOpts) { + if err := ensureLiveArtifactsExist(); err != nil { + fmt.Println(err) + return + } + if opts.addNmKeyfile { + c.Fatal("--add-nm-keyfile not yet supported for PXE") + } + + qc, ok := c.Cluster.(*qemu.Cluster) + if !ok { + c.Fatalf("Unsupported cluster type") + } + if opts.enable4k { + qc.EnforceNative4k() + } + + installerConfig := CoreosInstallerConfig{ + Console: []string{platform.ConsoleKernelArgument[coreosarch.CurrentRpmArch()]}, + AppendKargs: renderCosaTestIsoDebugKargs(), + Insecure: opts.instInsecure, + } + + installerConfigData, err := yaml.Marshal(installerConfig) + if err != nil { + c.Fatal(err) + } + mode := 0644 + + liveConfig, err := conf.EmptyIgnition().Render(conf.FailWarnings) + if err != nil { + c.Fatal(err) + } + liveConfig.AddSystemdUnit("live-signal-ok.service", liveSignalOKUnit, conf.Enable) + liveConfig.AddSystemdUnit("coreos-test-entered-emergency-target.service", signalFailureUnit, conf.Enable) + if opts.isOffline { + contents := fmt.Sprintf(downloadCheck, kola.CosaBuild.Meta.OstreeVersion) + liveConfig.AddSystemdUnit("coreos-installer-offline-check.service", contents, conf.Enable) + } + // XXX: https://github.com/coreos/coreos-installer/issues/1171 + if coreosarch.CurrentRpmArch() != "s390x" { + liveConfig.AddFile("/etc/coreos/installer.d/mantle.yaml", string(installerConfigData), mode) + } + liveConfig.AddAutoLogin() + liveConfig.AddSystemdUnit("boot-started.service", bootStartedUnit, conf.Enable) + + targetConfig, err := conf.EmptyIgnition().Render(conf.FailWarnings) + if err != nil { + c.Fatal(err) + } + targetConfig.AddSystemdUnit("coreos-test-installer.service", signalCompletionUnit, conf.Enable) + targetConfig.AddSystemdUnit("coreos-test-entered-emergency-target.service", signalFailureUnit, conf.Enable) + targetConfig.AddSystemdUnit("coreos-test-installer-no-ignition.service", checkNoIgnition, conf.Enable) + + tempdir, err := os.MkdirTemp("/var/tmp", "mantle-pxe") + if err != nil { + c.Fatal(err) + } + defer func() { + os.RemoveAll(tempdir) + }() + + pxe, err := createPXE(c.Context(), tempdir, opts) + if err != nil { + c.Fatal(errors.Wrapf(err, "setting up install")) + } + + overrideFW := func(builder *platform.QemuBuilder) error { + if opts.enableUefi { + builder.Firmware = "uefi" + } + // increase the memory for pxe tests with appended rootfs in the initrd + // we were bumping up into the 4GiB limit in RHCOS/c9s + builder.MemoryMiB = 4096 + if opts.pxeAppendRootfs { + builder.MemoryMiB = 5120 + } + + if err := absSymlink(builder.ConfigFile, filepath.Join(pxe.tftpdir, "pxe-live.ign")); err != nil { + return err + } + + targetpath := filepath.Join(filepath.Dir(builder.ConfigFile), "pxe-target.ign") + if err := targetConfig.WriteFile(targetpath); err != nil { + return err + } + if err := absSymlink(targetpath, filepath.Join(pxe.tftpdir, "pxe-target.ign")); err != nil { + return err + } + // don't attach config to VM + builder.ConfigFile = "" + return nil + } + + setupNet := func(_ platform.QemuMachineOptions, builder *platform.QemuBuilder) error { + netdev := fmt.Sprintf("%s,netdev=mynet0,mac=52:54:00:12:34:56", pxe.networkdevice) + if pxe.bootindex == "" { + builder.Append("-boot", "once=n") + } else { + netdev += fmt.Sprintf(",bootindex=%s", pxe.bootindex) + } + builder.Append("-device", netdev) + usernetdev := fmt.Sprintf("user,id=mynet0,tftp=%s,bootfile=%s", pxe.tftpdir, pxe.bootfile) + if pxe.tftpipaddr != "10.0.2.2" { + usernetdev += ",net=192.168.76.0/24,dhcpstart=192.168.76.9" + } + builder.Append("-netdev", usernetdev) + return nil + } + + var isoCompletionOutput *os.File + var bootStartedOutput *os.File + setupDisks := func(_ platform.QemuMachineOptions, builder *platform.QemuBuilder) error { + sectorSize := 0 + if opts.enable4k { + sectorSize = 4096 + } + disk := platform.Disk{ + Size: "12G", // Arbitrary + SectorSize: sectorSize, + MultiPathDisk: opts.enableMultipath, + } + //TBD: see if we can remove this and just use AddDisk and inject bootindex during startup + if coreosarch.CurrentRpmArch() == "s390x" || coreosarch.CurrentRpmArch() == "aarch64" { + // s390x and aarch64 need to use bootindex as they don't support boot once + if err := builder.AddDisk(&disk); err != nil { + return err + } + } else { + if err := builder.AddPrimaryDisk(&disk); err != nil { + return err + } + } + isoCompletionOutput, err = builder.VirtioChannelRead("testisocompletion") + if err != nil { + return errors.Wrap(err, "setting up testisocompletion virtio-serial channel") + } + bootStartedOutput, err = builder.VirtioChannelRead("bootstarted") + if err != nil { + return errors.Wrap(err, "setting up bootstarted virtio-serial channel") + } + return nil + } + + extra := platform.QemuMachineOptions{} + extra.SkipStartMachine = true + callbacks := qemu.BuilderCallbacks{SetupDisks: setupDisks, SetupNetwork: setupNet, OverrideDefaults: overrideFW} + qm, err := qc.NewMachineWithQemuOptionsAndBuilderCallbacks(liveConfig, extra, callbacks) + if err != nil { + c.Fatal(errors.Wrap(err, "unable to create test machine")) + } + + errchan := make(chan error) + go func() { + errchan <- checkTestOutput(isoCompletionOutput, []string{liveOKSignal, signalCompleteString}) + }() + + //check for error when switching boot order + go func() { + if err := checkTestOutput(bootStartedOutput, []string{bootStartedSignal}); err != nil { + errchan <- err + return + } + if err := qc.Instance(qm).SwitchBootOrder(); err != nil { + errchan <- errors.Wrapf(err, "switching boot order failed") + return + } + }() + + err = <-errchan + if err != nil { + c.Fatal(err) + } +} + +type PXE struct { + tftpdir string + tftpipaddr string + boottype string + networkdevice string + bootindex string + pxeimagepath string + bootfile string +} + +func createPXE(ctx context.Context, tempdir string, opts IsoTestOpts) (*PXE, error) { + kernel := kola.CosaBuild.Meta.BuildArtifacts.LiveKernel.Path + initramfs := kola.CosaBuild.Meta.BuildArtifacts.LiveInitramfs.Path + rootfs := kola.CosaBuild.Meta.BuildArtifacts.LiveRootfs.Path + builddir := kola.CosaBuild.Dir + + tftpdir := filepath.Join(tempdir, "tftp") + if err := os.Mkdir(tftpdir, 0777); err != nil { + return nil, err + } + + for _, name := range []string{kernel, initramfs, rootfs} { + if err := absSymlink(filepath.Join(builddir, name), filepath.Join(tftpdir, name)); err != nil { + return nil, err + } + } + + if opts.pxeAppendRootfs { + // replace the initramfs symlink with a concatenation of + // the initramfs and rootfs + initrd := filepath.Join(tftpdir, initramfs) + if err := os.Remove(initrd); err != nil { + return nil, err + } + if err := cat(initrd, filepath.Join(builddir, initramfs), filepath.Join(builddir, rootfs)); err != nil { + return nil, err + } + } + + var metalimg string + if opts.enable4k { + metalimg = kola.CosaBuild.Meta.BuildArtifacts.Metal4KNative.Path + } else { + metalimg = kola.CosaBuild.Meta.BuildArtifacts.Metal.Path + } + metalname, err := setupMetalImage(builddir, metalimg, tftpdir) + if err != nil { + return nil, errors.Wrapf(err, "setting up metal image") + } + + pxe := &PXE{ + tftpdir: tftpdir, + } + if err := pxe.setupArchDefaults(opts); err != nil { + return nil, err + } + + listener, err := net.Listen("tcp", ":0") + if err != nil { + return nil, err + } + port := listener.Addr().(*net.TCPAddr).Port + baseurl := fmt.Sprintf("http://%s:%d", pxe.tftpipaddr, port) + + kargs := renderCosaTestIsoDebugKargs() + kargs = append(kargs, renderBaseKargs()...) + kargs = append(kargs, kola.QEMUOptions.PxeKernelArgs...) + kargs = append(kargs, fmt.Sprintf("ignition.config.url=%s/pxe-live.ign", baseurl)) + kargs = append(kargs, renderInstallKargs(baseurl, metalname, opts)...) + if rootfs != "" && !opts.pxeAppendRootfs { + kargs = append(kargs, fmt.Sprintf("coreos.live.rootfs_url=%s/%s", baseurl, rootfs)) + } + kargsStr := strings.Join(kargs, " ") + + switch pxe.boottype { + case "pxe": + if err := pxe.configBootPxe(kargsStr); err != nil { + return nil, err + } + case "grub": + if err := pxe.configBootGrub(kargsStr); err != nil { + return nil, err + } + default: + return nil, errors.Errorf("Unhandled boottype %s", pxe.boottype) + } + + mux := http.NewServeMux() + mux.Handle("/", http.FileServer(http.Dir(tftpdir))) + srv := &http.Server{Handler: mux} + + go func() { + if err := srv.Serve(listener); err != nil && !errors.Is(err, http.ErrServerClosed) { + log.Printf("http serve: %v", err) + } + }() + + // stop server when ctx is canceled + go func() { + <-ctx.Done() + _ = srv.Shutdown(context.Background()) + }() + + return pxe, nil +} + +func (pxe *PXE) setupArchDefaults(opts IsoTestOpts) error { + pxe.tftpipaddr = "192.168.76.2" + switch coreosarch.CurrentRpmArch() { + case "x86_64": + pxe.networkdevice = "e1000" + if opts.enableUefi { + pxe.boottype = "grub" + pxe.bootfile = "/boot/grub2/grubx64.efi" + pxe.pxeimagepath = "/boot/efi/EFI/fedora/grubx64.efi" + // Choose bootindex=2. First boot the hard drive won't + // have an OS and will fall through to bootindex 2 (net) + pxe.bootindex = "2" + } else { + pxe.boottype = "pxe" + pxe.pxeimagepath = "/usr/share/syslinux/" + } + case "aarch64": + pxe.boottype = "grub" + pxe.networkdevice = "virtio-net-pci" + pxe.bootfile = "/boot/grub2/grubaa64.efi" + pxe.pxeimagepath = "/boot/efi/EFI/fedora/grubaa64.efi" + pxe.bootindex = "1" + case "ppc64le": + pxe.boottype = "grub" + pxe.networkdevice = "virtio-net-pci" + pxe.bootfile = "/boot/grub2/powerpc-ieee1275/core.elf" + case "s390x": + pxe.boottype = "pxe" + pxe.networkdevice = "virtio-net-ccw" + pxe.bootindex = "1" + pxe.tftpipaddr = "10.0.2.2" + default: + return fmt.Errorf("unsupported arch %s", coreosarch.CurrentRpmArch()) + } + return nil +} + +func (pxe *PXE) configBootPxe(kargs string) error { + kernel := kola.CosaBuild.Meta.BuildArtifacts.LiveKernel.Path + initramfs := kola.CosaBuild.Meta.BuildArtifacts.LiveInitramfs.Path + + pxeconfigdir := filepath.Join(pxe.tftpdir, "pxelinux.cfg") + if err := os.Mkdir(pxeconfigdir, 0777); err != nil { + return errors.Wrapf(err, "creating dir %s", pxeconfigdir) + } + pxeimages := []string{"pxelinux.0", "ldlinux.c32"} + pxeconfig := []byte(fmt.Sprintf(` +DEFAULT pxeboot +TIMEOUT 20 +PROMPT 0 +LABEL pxeboot + KERNEL %s + APPEND initrd=%s %s +`, kernel, initramfs, kargs)) + if coreosarch.CurrentRpmArch() == "s390x" { + pxeconfig = []byte(kargs) + } + pxeconfig_path := filepath.Join(pxeconfigdir, "default") + if err := os.WriteFile(pxeconfig_path, pxeconfig, 0777); err != nil { + return errors.Wrapf(err, "writing file %s", pxeconfig_path) + } + + // this is only for s390x where the pxe image has to be created; + // s390 doesn't seem to have a pre-created pxe image although have to check on this + if pxe.pxeimagepath == "" { + kernelpath := filepath.Join(pxe.tftpdir, kernel) + initrdpath := filepath.Join(pxe.tftpdir, initramfs) + err := exec.Command("/usr/bin/mk-s390image", kernelpath, "-r", initrdpath, + "-p", filepath.Join(pxeconfigdir, "default"), filepath.Join(pxe.tftpdir, pxeimages[0])).Run() + if err != nil { + return errors.Wrap(err, "running mk-s390image") + } + } else { + for _, img := range pxeimages { + srcpath := filepath.Join("/usr/share/syslinux", img) + cp_cmd := exec.Command("/usr/lib/coreos-assembler/cp-reflink", srcpath, pxe.tftpdir) + cp_cmd.Stderr = os.Stderr + if err := cp_cmd.Run(); err != nil { + return errors.Wrapf(err, "running cp-reflink %s %s", srcpath, pxe.tftpdir) + } + } + } + pxe.bootfile = "/" + pxeimages[0] + return nil +} + +func (pxe *PXE) configBootGrub(kargs string) error { + kernel := kola.CosaBuild.Meta.BuildArtifacts.LiveKernel.Path + initramfs := kola.CosaBuild.Meta.BuildArtifacts.LiveInitramfs.Path + + grub2_mknetdir_cmd := exec.Command("grub2-mknetdir", "--net-directory="+pxe.tftpdir) + grub2_mknetdir_cmd.Stderr = os.Stderr + if err := grub2_mknetdir_cmd.Run(); err != nil { + return errors.Wrap(err, "running grub2-mknetdir") + } + if pxe.pxeimagepath != "" { + dstpath := filepath.Join(pxe.tftpdir, "boot/grub2") + cp_cmd := exec.Command("/usr/lib/coreos-assembler/cp-reflink", pxe.pxeimagepath, dstpath) + cp_cmd.Stderr = os.Stderr + if err := cp_cmd.Run(); err != nil { + return errors.Wrapf(err, "running cp-reflink %s %s", pxe.pxeimagepath, dstpath) + } + } + if err := os.WriteFile(filepath.Join(pxe.tftpdir, "boot/grub2/grub.cfg"), []byte(fmt.Sprintf(` +default=0 +timeout=1 +menuentry "CoreOS (BIOS/UEFI)" { + echo "Loading kernel" + linux /%s %s + echo "Loading initrd" + initrd %s +}`, kernel, kargs, initramfs)), 0777); err != nil { + return errors.Wrap(err, "writing grub.cfg") + } + return nil +} + +func renderBaseKargs() []string { + baseKargs := []string{"rd.neednet=1", "ip=dhcp", "ignition.firstboot", "ignition.platform.id=metal"} + return append(baseKargs, fmt.Sprintf("console=%s", platform.ConsoleKernelArgument[coreosarch.CurrentRpmArch()])) +} + +func renderInstallKargs(baseurl string, metalname string, opts IsoTestOpts) []string { + args := []string{"coreos.inst.install_dev=/dev/vda", + fmt.Sprintf("coreos.inst.ignition_url=%s/pxe-target.ign", baseurl)} + if !opts.isOffline { + args = append(args, fmt.Sprintf("coreos.inst.image_url=%s/%s", baseurl, metalname)) + } + // FIXME - ship signatures by default too + if opts.instInsecure { + args = append(args, "coreos.inst.insecure") + } + return args +} diff --git a/mantle/platform/machine/qemu/cluster.go b/mantle/platform/machine/qemu/cluster.go index a6b51e9219..cab3e1ddc9 100644 --- a/mantle/platform/machine/qemu/cluster.go +++ b/mantle/platform/machine/qemu/cluster.go @@ -44,6 +44,21 @@ type Cluster struct { tearingDown bool } +type BuilderCallbacks struct { + BuilderInit func(options platform.QemuMachineOptions, builder *platform.QemuBuilder) error + SetupDisks func(options platform.QemuMachineOptions, builder *platform.QemuBuilder) error + SetupNetwork func(options platform.QemuMachineOptions, builder *platform.QemuBuilder) error + OverrideDefaults func(builder *platform.QemuBuilder) error +} + +func (qc *Cluster) EnforceNative4k() { + qc.flight.opts.Native4k = true +} + +func (qc *Cluster) EnforceMultipath() { + qc.flight.opts.MultiPathDisk = true +} + func (qc *Cluster) NewMachine(userdata *conf.UserData) (platform.Machine, error) { return qc.NewMachineWithOptions(userdata, platform.MachineOptions{}) } @@ -59,6 +74,25 @@ func (qc *Cluster) NewMachineWithOptions(userdata *conf.UserData, options platfo } func (qc *Cluster) NewMachineWithQemuOptions(userdata *conf.UserData, options platform.QemuMachineOptions) (platform.Machine, error) { + return qc.NewMachineWithQemuOptionsAndBuilderCallbacks(userdata, options, BuilderCallbacks{ + BuilderInit: qc.InitDefaultBuilder, + SetupDisks: qc.SetupDefaultDisks, + SetupNetwork: qc.SetupDefaultNetwork, + OverrideDefaults: nil, + }) +} + +func (qc *Cluster) NewMachineWithQemuOptionsAndBuilderCallbacks(userdata any, options platform.QemuMachineOptions, callbacks BuilderCallbacks) (platform.Machine, error) { + if callbacks.BuilderInit == nil { + callbacks.BuilderInit = qc.InitDefaultBuilder + } + if callbacks.SetupDisks == nil { + callbacks.SetupDisks = qc.SetupDefaultDisks + } + if callbacks.SetupNetwork == nil { + callbacks.SetupNetwork = qc.SetupDefaultNetwork + } + id := uuid.New() dir := filepath.Join(qc.RuntimeConf().OutputDir, id) @@ -66,16 +100,10 @@ func (qc *Cluster) NewMachineWithQemuOptions(userdata *conf.UserData, options pl return nil, err } - // hacky solution for cloud config ip substitution - // NOTE: escaping is not supported - qc.mu.Lock() - - conf, err := qc.RenderUserData(userdata, map[string]string{}) + conf, confPath, err := qc.ProcessIgnitionConfig(userdata, dir) if err != nil { - qc.mu.Unlock() return nil, err } - qc.mu.Unlock() journal, err := platform.NewJournal(dir) if err != nil { @@ -90,53 +118,166 @@ func (qc *Cluster) NewMachineWithQemuOptions(userdata *conf.UserData, options pl } builder := platform.NewQemuBuilder() - if options.DisablePDeathSig { - builder.Pdeathsig = false + defer builder.Close() + if err := callbacks.BuilderInit(options, builder); err != nil { + return nil, err + } + builder.UUID = qm.id + builder.ConsoleFile = qm.consolePath + builder.ConfigFile = confPath + // This one doesn't support configuring the path because we can't + // reliably change the Ignition config here... + for _, path := range qc.flight.opts.BindRO { + destpathrel := strings.TrimLeft(path, "/") + builder.MountHost(path, "/kola/host/"+destpathrel, true) } + if err := callbacks.SetupDisks(options, builder); err != nil { + return nil, err + } + if err := callbacks.SetupNetwork(options, builder); err != nil { + return nil, err + } + if callbacks.OverrideDefaults != nil { + if err := callbacks.OverrideDefaults(builder); err != nil { + return nil, err + } + } + // S390x specific stuff if qc.flight.opts.SecureExecution { if err := builder.SetSecureExecution(qc.flight.opts.SecureExecutionIgnitionPubKey, qc.flight.opts.SecureExecutionHostKey, conf); err != nil { return nil, err } } + if qc.flight.opts.Cex || options.Cex { + if err := builder.AddCexDevice(); err != nil { + return nil, err + } + } + + inst, err := builder.Exec() + if err != nil { + return nil, err + } + qm.inst = inst + + if builder.UsermodeNetworking { + err = util.Retry(6, 5*time.Second, func() error { + var err error + qm.ip, err = inst.SSHAddress() + if err != nil { + return err + } + return nil + }) + if err != nil { + return nil, err + } + } + // Run StartMachine, which blocks on the machine being booted up enough + // for SSH access, but only if the caller didn't tell us not to. + if !options.SkipStartMachine { + if err := platform.StartMachine(qm, qm.journal); err != nil { + qm.Destroy() + return nil, err + } + } + + qc.AddMach(qm) + + // In this flow, nothing actually Wait()s for the QEMU process. Let's do it here + // and print something if it exited unexpectedly. Ideally in the future, this + // interface allows the test harness to provide e.g. a channel we can signal on so + // it knows to stop the test once QEMU dies. + go func() { + err := inst.Wait() + if err != nil && !qc.tearingDown { + plog.Errorf("QEMU process finished abnormally: %v", err) + } + }() + return qm, nil +} +func (qc *Cluster) Instance(m platform.Machine) *platform.QemuInstance { + switch pm := m.(type) { + case *machine: + return pm.inst + default: + return nil + } +} + +func (qc *Cluster) Destroy() { + qc.tearingDown = true + qc.BaseCluster.Destroy() + qc.flight.DelCluster(qc) +} + +func (qc *Cluster) ProcessIgnitionConfig(cfg any, dir string) (*conf.Conf, string, error) { + var config *conf.Conf var confPath string + var err error + switch p := cfg.(type) { + case *conf.UserData: + config, err = qc.RenderUserDataForCloudIpSubstitution(p) + if err != nil { + return nil, "", err + } + case *conf.Conf: + config = p + default: + return nil, "", fmt.Errorf("unknown config pointer type: %T", p) + } + if config != nil { + confPath, err = qc.WriteIgnitionConfigToDir(config, dir) + if err != nil { + return nil, "", err + } + } + return config, confPath, nil +} + +// hacky solution for cloud config ip substitution +// NOTE: escaping is not supported +func (qc *Cluster) RenderUserDataForCloudIpSubstitution(userdata *conf.UserData) (conf *conf.Conf, err error) { + qc.mu.Lock() + conf, err = qc.RenderUserData(userdata, map[string]string{}) + if err != nil { + qc.mu.Unlock() + return nil, err + } + qc.mu.Unlock() + return conf, nil +} + +func (qc *Cluster) WriteIgnitionConfigToDir(conf *conf.Conf, dir string) (confPath string, err error) { if conf.IsIgnition() { confPath = filepath.Join(dir, "ignition.json") - if err := conf.WriteFile(confPath); err != nil { - return nil, err + if err = conf.WriteFile(confPath); err != nil { + return confPath, err } - } else if conf.IsEmpty() { - } else { - return nil, fmt.Errorf("qemu only supports Ignition or empty configs") + } else if !conf.IsEmpty() { + return confPath, fmt.Errorf("qemu only supports Ignition or empty configs") } + return confPath, nil +} - builder.ConfigFile = confPath - defer builder.Close() - builder.UUID = qm.id +func (qc *Cluster) InitDefaultBuilder(options platform.QemuMachineOptions, builder *platform.QemuBuilder) error { + if options.DisablePDeathSig { + builder.Pdeathsig = false + } if qc.flight.opts.Arch != "" { if err := builder.SetArchitecture(qc.flight.opts.Arch); err != nil { - return nil, err + return err } } if qc.flight.opts.Firmware != "" { builder.Firmware = qc.flight.opts.Firmware } - builder.Swtpm = qc.flight.opts.Swtpm - builder.Hostname = fmt.Sprintf("qemu%d", qc.BaseCluster.AllocateMachineSerial()) - builder.ConsoleFile = qm.consolePath - - // This one doesn't support configuring the path because we can't - // reliably change the Ignition config here... - for _, path := range qc.flight.opts.BindRO { - destpathrel := strings.TrimLeft(path, "/") - builder.MountHost(path, "/kola/host/"+destpathrel, true) - } - if qc.flight.opts.Memory != "" { memory, err := strconv.ParseInt(qc.flight.opts.Memory, 10, 32) if err != nil { - return nil, errors.Wrapf(err, "parsing memory option") + return errors.Wrapf(err, "parsing memory option") } builder.MemoryMiB = int(memory) } else if options.MinMemory != 0 { @@ -144,22 +285,30 @@ func (qc *Cluster) NewMachineWithQemuOptions(userdata *conf.UserData, options pl } else if qc.flight.opts.SecureExecution { builder.MemoryMiB = 4096 // SE needs at least 4GB } + builder.Swtpm = qc.flight.opts.Swtpm + builder.Hostname = fmt.Sprintf("qemu%d", qc.BaseCluster.AllocateMachineSerial()) + if options.Firmware != "" { + builder.Firmware = options.Firmware + } + if options.AppendKernelArgs != "" { + builder.AppendKernelArgs = options.AppendKernelArgs + } + if options.AppendFirstbootKernelArgs != "" { + builder.AppendFirstbootKernelArgs = options.AppendFirstbootKernelArgs + } + return nil +} + +func (qc *Cluster) SetupDefaultDisks(options platform.QemuMachineOptions, builder *platform.QemuBuilder) error { var primaryDisk platform.Disk if options.PrimaryDisk != "" { - var diskp *platform.Disk - if diskp, err = platform.ParseDisk(options.PrimaryDisk, true); err != nil { - return nil, errors.Wrapf(err, "parsing primary disk spec '%s'", options.PrimaryDisk) + diskp, err := platform.ParseDisk(options.PrimaryDisk, true) + if err != nil { + return errors.Wrapf(err, "parsing primary disk spec '%s'", options.PrimaryDisk) } primaryDisk = *diskp } - - if qc.flight.opts.Cex || options.Cex { - if err := builder.AddCexDevice(); err != nil { - return nil, err - } - } - if qc.flight.opts.Nvme || options.Nvme { primaryDisk.Channel = "nvme" } @@ -181,14 +330,16 @@ func (qc *Cluster) NewMachineWithQemuOptions(userdata *conf.UserData, options pl if options.OverrideBackingFile != "" { primaryDisk.BackingFile = options.OverrideBackingFile } - - if err = builder.AddBootDisk(&primaryDisk); err != nil { - return nil, err + if err := builder.AddBootDisk(&primaryDisk); err != nil { + return err } - if err = builder.AddDisksFromSpecs(options.AdditionalDisks); err != nil { - return nil, err + if err := builder.AddDisksFromSpecs(options.AdditionalDisks); err != nil { + return err } + return nil +} +func (qc *Cluster) SetupDefaultNetwork(options platform.QemuMachineOptions, builder *platform.QemuBuilder) error { if len(options.HostForwardPorts) > 0 { builder.EnableUsermodeNetworking(options.HostForwardPorts, "") } else { @@ -200,64 +351,8 @@ func (qc *Cluster) NewMachineWithQemuOptions(userdata *conf.UserData, options pl if options.AdditionalNics > 0 { builder.AddAdditionalNics(options.AdditionalNics) } - if options.AppendKernelArgs != "" { - builder.AppendKernelArgs = options.AppendKernelArgs - } - if options.AppendFirstbootKernelArgs != "" { - builder.AppendFirstbootKernelArgs = options.AppendFirstbootKernelArgs - } if !qc.RuntimeConf().InternetAccess { builder.RestrictNetworking = true } - if options.Firmware != "" { - builder.Firmware = options.Firmware - } - - inst, err := builder.Exec() - if err != nil { - return nil, err - } - qm.inst = inst - - err = util.Retry(6, 5*time.Second, func() error { - var err error - qm.ip, err = inst.SSHAddress() - if err != nil { - return err - } - return nil - }) - if err != nil { - return nil, err - } - - // Run StartMachine, which blocks on the machine being booted up enough - // for SSH access, but only if the caller didn't tell us not to. - if !options.SkipStartMachine { - if err := platform.StartMachine(qm, qm.journal); err != nil { - qm.Destroy() - return nil, err - } - } - - qc.AddMach(qm) - - // In this flow, nothing actually Wait()s for the QEMU process. Let's do it here - // and print something if it exited unexpectedly. Ideally in the future, this - // interface allows the test harness to provide e.g. a channel we can signal on so - // it knows to stop the test once QEMU dies. - go func() { - err := inst.Wait() - if err != nil && !qc.tearingDown { - plog.Errorf("QEMU process finished abnormally: %v", err) - } - }() - - return qm, nil -} - -func (qc *Cluster) Destroy() { - qc.tearingDown = true - qc.BaseCluster.Destroy() - qc.flight.DelCluster(qc) + return nil } diff --git a/mantle/platform/machine/qemu/flight.go b/mantle/platform/machine/qemu/flight.go index 4a008b1a0a..bc3bd90a54 100644 --- a/mantle/platform/machine/qemu/flight.go +++ b/mantle/platform/machine/qemu/flight.go @@ -55,6 +55,11 @@ type Options struct { SecureExecutionIgnitionPubKey string SecureExecutionHostKey string + // kola run iso.* options + // Do not verify signature on metal image + InstInsecure bool + PxeKernelArgs []string + // Option to create IBM cex based luks encryption Cex bool diff --git a/mantle/platform/machine/qemu/machine.go b/mantle/platform/machine/qemu/machine.go index 9ecfc62651..92488314fc 100644 --- a/mantle/platform/machine/qemu/machine.go +++ b/mantle/platform/machine/qemu/machine.go @@ -92,17 +92,27 @@ func (m *machine) WaitForSoftReboot(timeout time.Duration, oldSoftRebootsCount s } func (m *machine) Destroy() { - m.inst.Destroy() + if m.inst != nil { + m.inst.Destroy() + m.inst = nil + } - m.journal.Destroy() + if m.journal != nil { + m.journal.Destroy() + m.journal = nil + } - if buf, err := os.ReadFile(m.consolePath); err == nil { - m.console = string(buf) - } else { - plog.Errorf("Error reading console for instance %v: %v", m.ID(), err) + if m.consolePath != "" { + if buf, err := os.ReadFile(m.consolePath); err == nil { + m.console = string(buf) + } else { + plog.Errorf("Error reading console for instance %v: %v", m.ID(), err) + } } - m.qc.DelMach(m) + if m.qc != nil { + m.qc.DelMach(m) + } } func (m *machine) ConsoleOutput() string { diff --git a/mantle/platform/metal.go b/mantle/platform/metal.go deleted file mode 100644 index 02a9da88c6..0000000000 --- a/mantle/platform/metal.go +++ /dev/null @@ -1,861 +0,0 @@ -// Copyright 2020 Red Hat -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package platform - -import ( - "bufio" - "fmt" - "io" - "net" - "net/http" - "os" - "path/filepath" - "strings" - "time" - - coreosarch "github.com/coreos/stream-metadata-go/arch" - "github.com/pkg/errors" - "gopkg.in/yaml.v2" - - "github.com/coreos/coreos-assembler/mantle/platform/conf" - "github.com/coreos/coreos-assembler/mantle/system/exec" - "github.com/coreos/coreos-assembler/mantle/util" -) - -const ( - // defaultQemuHostIPv4 is documented in `man qemu-kvm`, under the `-netdev` option - defaultQemuHostIPv4 = "10.0.2.2" - - bootStartedSignal = "boot-started-OK" -) - -// TODO derive this from docs, or perhaps include kargs in cosa metadata? -var baseKargs = []string{"rd.neednet=1", "ip=dhcp", "ignition.firstboot", "ignition.platform.id=metal"} - -var ( - // TODO expose this as an API that can be used by cosa too - consoleKernelArgument = map[string]string{ - "x86_64": "ttyS0,115200n8", - "ppc64le": "hvc0", - "aarch64": "ttyAMA0", - "s390x": "ttysclp0", - } - - bootStartedUnit = fmt.Sprintf(`[Unit] - Description=TestISO Boot Started - Requires=dev-virtio\\x2dports-bootstarted.device - OnFailure=emergency.target - OnFailureJobMode=isolate - [Service] - Type=oneshot - RemainAfterExit=yes - ExecStart=/bin/sh -c '/usr/bin/echo %s >/dev/virtio-ports/bootstarted' - [Install] - RequiredBy=coreos-installer.target - `, bootStartedSignal) -) - -// NewMetalQemuBuilderDefault returns a QEMU builder instance with some -// defaults set up for bare metal. -func NewMetalQemuBuilderDefault() *QemuBuilder { - builder := NewQemuBuilder() - // https://github.com/coreos/fedora-coreos-tracker/issues/388 - // https://github.com/coreos/fedora-coreos-docs/pull/46 - builder.MemoryMiB = 4096 - return builder -} - -type Install struct { - CosaBuild *util.LocalBuild - Builder *QemuBuilder - Insecure bool - Native4k bool - MultiPathDisk bool - PxeAppendRootfs bool - NmKeyfiles map[string]string - - // These are set by the install path - kargs []string - ignition conf.Conf - liveIgnition conf.Conf -} - -type InstalledMachine struct { - Tempdir string - QemuInst *QemuInstance - BootStartedErrorChannel chan error -} - -// Check that artifact has been built and locally exists -func (inst *Install) checkArtifactsExist(artifacts []string) error { - version := inst.CosaBuild.Meta.OstreeVersion - for _, name := range artifacts { - artifact, err := inst.CosaBuild.Meta.GetArtifact(name) - if err != nil { - return fmt.Errorf("Missing artifact %s for %s build: %s", name, version, err) - } - path := filepath.Join(inst.CosaBuild.Dir, artifact.Path) - if _, err := os.Stat(path); err != nil { - if os.IsNotExist(err) { - return fmt.Errorf("Missing local file for artifact %s for build %s", name, version) - } - } - } - return nil -} - -func (inst *Install) PXE(kargs []string, liveIgnition, ignition conf.Conf, offline bool) (*InstalledMachine, error) { - artifacts := []string{"live-kernel", "live-rootfs"} - if err := inst.checkArtifactsExist(artifacts); err != nil { - return nil, err - } - - installerConfig := installerConfig{ - Console: []string{consoleKernelArgument[coreosarch.CurrentRpmArch()]}, - AppendKargs: renderCosaTestIsoDebugKargs(), - } - installerConfigData, err := yaml.Marshal(installerConfig) - if err != nil { - return nil, err - } - mode := 0644 - - // XXX: https://github.com/coreos/coreos-installer/issues/1171 - if coreosarch.CurrentRpmArch() != "s390x" { - liveIgnition.AddFile("/etc/coreos/installer.d/mantle.yaml", string(installerConfigData), mode) - } - - inst.kargs = append(renderCosaTestIsoDebugKargs(), kargs...) - inst.ignition = ignition - inst.liveIgnition = liveIgnition - - mach, err := inst.runPXE(&kernelSetup{ - kernel: inst.CosaBuild.Meta.BuildArtifacts.LiveKernel.Path, - initramfs: inst.CosaBuild.Meta.BuildArtifacts.LiveInitramfs.Path, - rootfs: inst.CosaBuild.Meta.BuildArtifacts.LiveRootfs.Path, - }, offline) - if err != nil { - return nil, errors.Wrapf(err, "testing live installer") - } - - return mach, nil -} - -func (inst *InstalledMachine) Destroy() error { - if inst.QemuInst != nil { - inst.QemuInst.Destroy() - inst.QemuInst = nil - } - if inst.Tempdir != "" { - return os.RemoveAll(inst.Tempdir) - } - return nil -} - -type kernelSetup struct { - kernel, initramfs, rootfs string -} - -type pxeSetup struct { - tftpipaddr string - boottype string - networkdevice string - bootindex string - pxeimagepath string - - // bootfile is initialized later - bootfile string -} - -type installerRun struct { - inst *Install - builder *QemuBuilder - - builddir string - tempdir string - tftpdir string - - metalimg string - metalname string - - baseurl string - - kern kernelSetup - pxe pxeSetup -} - -func absSymlink(src, dest string) error { - src, err := filepath.Abs(src) - if err != nil { - return err - } - return os.Symlink(src, dest) -} - -// setupMetalImage creates a symlink to the metal image. -func setupMetalImage(builddir, metalimg, destdir string) (string, error) { - if err := absSymlink(filepath.Join(builddir, metalimg), filepath.Join(destdir, metalimg)); err != nil { - return "", err - } - return metalimg, nil -} - -func (inst *Install) setup(kern *kernelSetup) (*installerRun, error) { - var artifacts []string - if inst.Native4k { - artifacts = append(artifacts, "metal4k") - } else { - artifacts = append(artifacts, "metal") - } - if err := inst.checkArtifactsExist(artifacts); err != nil { - return nil, err - } - - builder := inst.Builder - - tempdir, err := os.MkdirTemp("/var/tmp", "mantle-pxe") - if err != nil { - return nil, err - } - cleanupTempdir := true - defer func() { - if cleanupTempdir { - os.RemoveAll(tempdir) - } - }() - - tftpdir := filepath.Join(tempdir, "tftp") - if err := os.Mkdir(tftpdir, 0777); err != nil { - return nil, err - } - - builddir := inst.CosaBuild.Dir - if err := inst.ignition.WriteFile(filepath.Join(tftpdir, "config.ign")); err != nil { - return nil, err - } - // This code will ensure to add an SSH key to `pxe-live.ign` config. - inst.liveIgnition.AddAutoLogin() - inst.liveIgnition.AddSystemdUnit("boot-started.service", bootStartedUnit, conf.Enable) - if err := inst.liveIgnition.WriteFile(filepath.Join(tftpdir, "pxe-live.ign")); err != nil { - return nil, err - } - - for _, name := range []string{kern.kernel, kern.initramfs, kern.rootfs} { - if err := absSymlink(filepath.Join(builddir, name), filepath.Join(tftpdir, name)); err != nil { - return nil, err - } - } - if inst.PxeAppendRootfs { - // replace the initramfs symlink with a concatenation of - // the initramfs and rootfs - initrd := filepath.Join(tftpdir, kern.initramfs) - if err := os.Remove(initrd); err != nil { - return nil, err - } - if err := cat(initrd, filepath.Join(builddir, kern.initramfs), filepath.Join(builddir, kern.rootfs)); err != nil { - return nil, err - } - } - - var metalimg string - if inst.Native4k { - metalimg = inst.CosaBuild.Meta.BuildArtifacts.Metal4KNative.Path - } else { - metalimg = inst.CosaBuild.Meta.BuildArtifacts.Metal.Path - } - metalname, err := setupMetalImage(builddir, metalimg, tftpdir) - if err != nil { - return nil, errors.Wrapf(err, "setting up metal image") - } - - pxe := pxeSetup{} - pxe.tftpipaddr = "192.168.76.2" - switch coreosarch.CurrentRpmArch() { - case "x86_64": - pxe.networkdevice = "e1000" - if builder.Firmware == "uefi" { - pxe.boottype = "grub" - pxe.bootfile = "/boot/grub2/grubx64.efi" - pxe.pxeimagepath = "/boot/efi/EFI/fedora/grubx64.efi" - // Choose bootindex=2. First boot the hard drive won't - // have an OS and will fall through to bootindex 2 (net) - pxe.bootindex = "2" - } else { - pxe.boottype = "pxe" - pxe.pxeimagepath = "/usr/share/syslinux/" - } - case "aarch64": - pxe.boottype = "grub" - pxe.networkdevice = "virtio-net-pci" - pxe.bootfile = "/boot/grub2/grubaa64.efi" - pxe.pxeimagepath = "/boot/efi/EFI/fedora/grubaa64.efi" - pxe.bootindex = "1" - case "ppc64le": - pxe.boottype = "grub" - pxe.networkdevice = "virtio-net-pci" - pxe.bootfile = "/boot/grub2/powerpc-ieee1275/core.elf" - case "s390x": - pxe.boottype = "pxe" - pxe.networkdevice = "virtio-net-ccw" - pxe.tftpipaddr = "10.0.2.2" - pxe.bootindex = "1" - default: - return nil, fmt.Errorf("Unsupported arch %s", coreosarch.CurrentRpmArch()) - } - - mux := http.NewServeMux() - mux.Handle("/", http.FileServer(http.Dir(tftpdir))) - listener, err := net.Listen("tcp", ":0") - if err != nil { - return nil, err - } - port := listener.Addr().(*net.TCPAddr).Port - //nolint // Yeah this leaks - go func() { - http.Serve(listener, mux) - }() - baseurl := fmt.Sprintf("http://%s:%d", pxe.tftpipaddr, port) - - cleanupTempdir = false // Transfer ownership - return &installerRun{ - inst: inst, - - builder: builder, - tempdir: tempdir, - tftpdir: tftpdir, - builddir: builddir, - - metalimg: metalimg, - metalname: metalname, - - baseurl: baseurl, - - pxe: pxe, - kern: *kern, - }, nil -} - -func renderBaseKargs() []string { - return append(baseKargs, fmt.Sprintf("console=%s", consoleKernelArgument[coreosarch.CurrentRpmArch()])) -} - -func renderInstallKargs(t *installerRun, offline bool) []string { - args := []string{"coreos.inst.install_dev=/dev/vda", - fmt.Sprintf("coreos.inst.ignition_url=%s/config.ign", t.baseurl)} - if !offline { - args = append(args, fmt.Sprintf("coreos.inst.image_url=%s/%s", t.baseurl, t.metalname)) - } - // FIXME - ship signatures by default too - if t.inst.Insecure { - args = append(args, "coreos.inst.insecure") - } - return args -} - -// Sometimes the logs that stream from various virtio streams can be -// incomplete because they depend on services inside the guest. -// When you are debugging earlyboot/initramfs issues this can be -// problematic. Let's add a hook here to enable more debugging. -func renderCosaTestIsoDebugKargs() []string { - if _, ok := os.LookupEnv("COSA_TESTISO_DEBUG"); ok { - return []string{"systemd.log_color=0", "systemd.log_level=debug", - "systemd.journald.forward_to_console=1", - "systemd.journald.max_level_console=debug"} - } else { - return []string{} - } -} - -func (t *installerRun) destroy() error { - t.builder.Close() - if t.tempdir != "" { - return os.RemoveAll(t.tempdir) - } - return nil -} - -func (t *installerRun) completePxeSetup(kargs []string) error { - if t.kern.rootfs != "" && !t.inst.PxeAppendRootfs { - kargs = append(kargs, fmt.Sprintf("coreos.live.rootfs_url=%s/%s", t.baseurl, t.kern.rootfs)) - } - kargsStr := strings.Join(kargs, " ") - - switch t.pxe.boottype { - case "pxe": - pxeconfigdir := filepath.Join(t.tftpdir, "pxelinux.cfg") - if err := os.Mkdir(pxeconfigdir, 0777); err != nil { - return errors.Wrapf(err, "creating dir %s", pxeconfigdir) - } - pxeimages := []string{"pxelinux.0", "ldlinux.c32"} - pxeconfig := []byte(fmt.Sprintf(` - DEFAULT pxeboot - TIMEOUT 20 - PROMPT 0 - LABEL pxeboot - KERNEL %s - APPEND initrd=%s %s - `, t.kern.kernel, t.kern.initramfs, kargsStr)) - if coreosarch.CurrentRpmArch() == "s390x" { - pxeconfig = []byte(kargsStr) - } - pxeconfig_path := filepath.Join(pxeconfigdir, "default") - if err := os.WriteFile(pxeconfig_path, pxeconfig, 0777); err != nil { - return errors.Wrapf(err, "writing file %s", pxeconfig_path) - } - - // this is only for s390x where the pxe image has to be created; - // s390 doesn't seem to have a pre-created pxe image although have to check on this - if t.pxe.pxeimagepath == "" { - kernelpath := filepath.Join(t.tftpdir, t.kern.kernel) - initrdpath := filepath.Join(t.tftpdir, t.kern.initramfs) - err := exec.Command("/usr/bin/mk-s390image", kernelpath, "-r", initrdpath, - "-p", filepath.Join(pxeconfigdir, "default"), filepath.Join(t.tftpdir, pxeimages[0])).Run() - if err != nil { - return errors.Wrap(err, "running mk-s390image") - } - } else { - for _, img := range pxeimages { - srcpath := filepath.Join("/usr/share/syslinux", img) - cp_cmd := exec.Command("/usr/lib/coreos-assembler/cp-reflink", srcpath, t.tftpdir) - cp_cmd.Stderr = os.Stderr - if err := cp_cmd.Run(); err != nil { - return errors.Wrapf(err, "running cp-reflink %s %s", srcpath, t.tftpdir) - } - } - } - t.pxe.bootfile = "/" + pxeimages[0] - case "grub": - grub2_mknetdir_cmd := exec.Command("grub2-mknetdir", "--net-directory="+t.tftpdir) - grub2_mknetdir_cmd.Stderr = os.Stderr - if err := grub2_mknetdir_cmd.Run(); err != nil { - return errors.Wrap(err, "running grub2-mknetdir") - } - if t.pxe.pxeimagepath != "" { - dstpath := filepath.Join(t.tftpdir, "boot/grub2") - cp_cmd := exec.Command("/usr/lib/coreos-assembler/cp-reflink", t.pxe.pxeimagepath, dstpath) - cp_cmd.Stderr = os.Stderr - if err := cp_cmd.Run(); err != nil { - return errors.Wrapf(err, "running cp-reflink %s %s", t.pxe.pxeimagepath, dstpath) - } - } - if err := os.WriteFile(filepath.Join(t.tftpdir, "boot/grub2/grub.cfg"), []byte(fmt.Sprintf(` - default=0 - timeout=1 - menuentry "CoreOS (BIOS/UEFI)" { - echo "Loading kernel" - linux /%s %s - echo "Loading initrd" - initrd %s - } - `, t.kern.kernel, kargsStr, t.kern.initramfs)), 0777); err != nil { - return errors.Wrap(err, "writing grub.cfg") - } - default: - panic("Unhandled boottype " + t.pxe.boottype) - } - - return nil -} - -func switchBootOrderSignal(qinst *QemuInstance, bootstartedchan *os.File, booterrchan *chan error) { - *booterrchan = make(chan error) - go func() { - err := qinst.Wait() - // only one Wait() gets process data, so also manually check for signal - if err == nil && qinst.Signaled() { - err = errors.New("process killed") - } - if err != nil { - *booterrchan <- errors.Wrapf(err, "QEMU unexpectedly exited while waiting for %s", bootStartedSignal) - } - }() - go func() { - r := bufio.NewReader(bootstartedchan) - l, err := r.ReadString('\n') - if err != nil { - if err == io.EOF { - // this may be from QEMU getting killed or exiting; wait a bit - // to give a chance for .Wait() above to feed the channel with a - // better error - time.Sleep(1 * time.Second) - *booterrchan <- fmt.Errorf("Got EOF from boot started channel, %s expected", bootStartedSignal) - } else { - *booterrchan <- errors.Wrapf(err, "reading from boot started channel") - } - return - } - line := strings.TrimSpace(l) - // switch the boot order here, we are well into the installation process - only for aarch64 and s390x - if line == bootStartedSignal { - if err := qinst.SwitchBootOrder(); err != nil { - *booterrchan <- errors.Wrapf(err, "switching boot order failed") - return - } - } - // OK! - *booterrchan <- nil - }() -} - -func cat(outfile string, infiles ...string) error { - out, err := os.OpenFile(outfile, os.O_WRONLY|os.O_CREATE, 0644) - if err != nil { - return err - } - defer out.Close() - for _, infile := range infiles { - in, err := os.Open(infile) - if err != nil { - return err - } - defer in.Close() - _, err = io.Copy(out, in) - if err != nil { - return err - } - } - return nil -} - -func (t *installerRun) run() (*QemuInstance, error) { - builder := t.builder - netdev := fmt.Sprintf("%s,netdev=mynet0,mac=52:54:00:12:34:56", t.pxe.networkdevice) - if t.pxe.bootindex == "" { - builder.Append("-boot", "once=n") - } else { - netdev += fmt.Sprintf(",bootindex=%s", t.pxe.bootindex) - } - builder.Append("-device", netdev) - usernetdev := fmt.Sprintf("user,id=mynet0,tftp=%s,bootfile=%s", t.tftpdir, t.pxe.bootfile) - if t.pxe.tftpipaddr != "10.0.2.2" { - usernetdev += ",net=192.168.76.0/24,dhcpstart=192.168.76.9" - } - builder.Append("-netdev", usernetdev) - - inst, err := builder.Exec() - if err != nil { - return nil, err - } - return inst, nil -} - -func (inst *Install) runPXE(kern *kernelSetup, offline bool) (*InstalledMachine, error) { - t, err := inst.setup(kern) - if err != nil { - return nil, errors.Wrapf(err, "setting up install") - } - defer func() { - err = t.destroy() - }() - - bootStartedChan, err := inst.Builder.VirtioChannelRead("bootstarted") - if err != nil { - return nil, errors.Wrapf(err, "setting up bootstarted virtio-serial channel") - } - - kargs := renderBaseKargs() - kargs = append(kargs, inst.kargs...) - kargs = append(kargs, fmt.Sprintf("ignition.config.url=%s/pxe-live.ign", t.baseurl)) - - kargs = append(kargs, renderInstallKargs(t, offline)...) - if err := t.completePxeSetup(kargs); err != nil { - return nil, errors.Wrapf(err, "completing PXE setup") - } - qinst, err := t.run() - if err != nil { - return nil, errors.Wrapf(err, "running PXE install") - } - tempdir := t.tempdir - t.tempdir = "" // Transfer ownership - instmachine := InstalledMachine{ - QemuInst: qinst, - Tempdir: tempdir, - } - switchBootOrderSignal(qinst, bootStartedChan, &instmachine.BootStartedErrorChannel) - return &instmachine, nil -} - -// This object gets serialized to YAML and fed to coreos-installer: -// https://coreos.github.io/coreos-installer/customizing-install/#config-file-format -type installerConfig struct { - ImageURL string `yaml:"image-url,omitempty"` - IgnitionFile string `yaml:"ignition-file,omitempty"` - Insecure bool `yaml:",omitempty"` - AppendKargs []string `yaml:"append-karg,omitempty"` - CopyNetwork bool `yaml:"copy-network,omitempty"` - DestDevice string `yaml:"dest-device,omitempty"` - Console []string `yaml:"console,omitempty"` -} - -func (inst *Install) InstallViaISOEmbed(kargs []string, liveIgnition, targetIgnition conf.Conf, outdir string, offline, minimal bool) (*InstalledMachine, error) { - artifacts := []string{"live-iso"} - if !offline { - if inst.Native4k { - artifacts = append(artifacts, "metal4k") - } else { - artifacts = append(artifacts, "metal") - } - } - if err := inst.checkArtifactsExist(artifacts); err != nil { - return nil, err - } - if minimal && offline { // ideally this'd be one enum parameter - panic("Can't run minimal install offline") - } - if offline && len(inst.NmKeyfiles) > 0 { - return nil, fmt.Errorf("Cannot use `--add-nm-keyfile` with offline mode") - } - - installerConfig := installerConfig{ - IgnitionFile: "/var/opt/pointer.ign", - DestDevice: "/dev/vda", - AppendKargs: renderCosaTestIsoDebugKargs(), - } - - // XXX: https://github.com/coreos/coreos-installer/issues/1171 - if coreosarch.CurrentRpmArch() != "s390x" { - installerConfig.Console = []string{consoleKernelArgument[coreosarch.CurrentRpmArch()]} - } - - if inst.MultiPathDisk { - // we only have one multipath device so it has to be that - installerConfig.DestDevice = "/dev/mapper/mpatha" - installerConfig.AppendKargs = append(installerConfig.AppendKargs, "rd.multipath=default", "root=/dev/disk/by-label/dm-mpath-root", "rw") - } - - inst.kargs = append(renderCosaTestIsoDebugKargs(), kargs...) - inst.ignition = targetIgnition - inst.liveIgnition = liveIgnition - - tempdir, err := os.MkdirTemp("/var/tmp", "mantle-metal") - if err != nil { - return nil, err - } - cleanupTempdir := true - defer func() { - if cleanupTempdir { - os.RemoveAll(tempdir) - } - }() - - if err := inst.ignition.WriteFile(filepath.Join(tempdir, "target.ign")); err != nil { - return nil, err - } - // and write it once more in the output dir for debugging - if err := inst.ignition.WriteFile(filepath.Join(outdir, "config-target.ign")); err != nil { - return nil, err - } - - builddir := inst.CosaBuild.Dir - srcisopath := filepath.Join(builddir, inst.CosaBuild.Meta.BuildArtifacts.LiveIso.Path) - - // Copy the ISO to a new location for modification. - // This is a bit awkward; we copy here, but QemuBuilder will also copy - // again (in `setupIso()`). I didn't want to lower the NM keyfile stuff - // into QemuBuilder. And plus, both tempdirs should be in /var/tmp so - // the `cp --reflink=auto` that QemuBuilder does should just reflink. - newIso := filepath.Join(tempdir, "install.iso") - cmd := exec.Command("cp", "--reflink=auto", srcisopath, newIso) - cmd.Stderr = os.Stderr - if err := cmd.Run(); err != nil { - return nil, errors.Wrapf(err, "copying iso") - } - // Make it writable so we can modify it - if err := os.Chmod(newIso, 0644); err != nil { - return nil, errors.Wrapf(err, "setting permissions on iso") - } - srcisopath = newIso - - var serializedTargetConfig string - if offline { - // note we leave ImageURL empty here; offline installs should now be the - // default! - - // we want to test that a full offline install works; that includes the - // final installed host booting offline - serializedTargetConfig = inst.ignition.String() - } else { - var metalimg string - if inst.Native4k { - metalimg = inst.CosaBuild.Meta.BuildArtifacts.Metal4KNative.Path - } else { - metalimg = inst.CosaBuild.Meta.BuildArtifacts.Metal.Path - } - metalname, err := setupMetalImage(builddir, metalimg, tempdir) - if err != nil { - return nil, errors.Wrapf(err, "setting up metal image") - } - - mux := http.NewServeMux() - mux.Handle("/", http.FileServer(http.Dir(tempdir))) - listener, err := net.Listen("tcp", ":0") - if err != nil { - return nil, err - } - port := listener.Addr().(*net.TCPAddr).Port - //nolint // Yeah this leaks - go func() { - http.Serve(listener, mux) - }() - baseurl := fmt.Sprintf("http://%s:%d", defaultQemuHostIPv4, port) - - // This is subtle but: for the minimal case, while we need networking to fetch the - // rootfs, the primary install flow will still rely on osmet. So let's keep ImageURL - // empty to exercise that path. In the future, this could be a separate scenario - // (likely we should drop the "offline" naming and have a "remote" tag on the - // opposite scenarios instead which fetch the metal image, so then we'd have - // "[min]iso-install" and "[min]iso-remote-install"). - if !minimal { - installerConfig.ImageURL = fmt.Sprintf("%s/%s", baseurl, metalname) - } - - if minimal { - minisopath := filepath.Join(tempdir, "minimal.iso") - // This is obviously also available in the build dir, but to be realistic, - // let's take it from --rootfs-output - rootfs_path := filepath.Join(tempdir, "rootfs.img") - // Ideally we'd use the coreos-installer of the target build here, because it's part - // of the test workflow, but that's complex... Sadly, probably easiest is to spin up - // a VM just to get the minimal ISO. - cmd := exec.Command("coreos-installer", "iso", "extract", "minimal-iso", srcisopath, - minisopath, "--output-rootfs", rootfs_path, "--rootfs-url", baseurl+"/rootfs.img") - cmd.Stderr = os.Stderr - if err := cmd.Run(); err != nil { - return nil, errors.Wrapf(err, "running coreos-installer iso extract minimal") - } - srcisopath = minisopath - } - - // In this case; the target config is jut a tiny wrapper that wants to - // fetch our hosted target.ign config - - // TODO also use https://github.com/coreos/coreos-installer/issues/118#issuecomment-585572952 - // when it arrives - targetConfig, err := conf.EmptyIgnition().Render(conf.FailWarnings) - if err != nil { - return nil, err - } - targetConfig.AddConfigSource(baseurl + "/target.ign") - serializedTargetConfig = targetConfig.String() - - // also save pointer config into the output dir for debugging - if err := targetConfig.WriteFile(filepath.Join(outdir, "config-target-pointer.ign")); err != nil { - return nil, err - } - } - - var keyfileArgs []string - for nmName, nmContents := range inst.NmKeyfiles { - path := filepath.Join(tempdir, nmName) - if err := os.WriteFile(path, []byte(nmContents), 0600); err != nil { - return nil, err - } - keyfileArgs = append(keyfileArgs, "--keyfile", path) - } - if len(keyfileArgs) > 0 { - - args := []string{"iso", "network", "embed", srcisopath} - args = append(args, keyfileArgs...) - cmd = exec.Command("coreos-installer", args...) - cmd.Stderr = os.Stderr - if err := cmd.Run(); err != nil { - return nil, errors.Wrapf(err, "running coreos-installer iso network embed") - } - - installerConfig.CopyNetwork = true - - // force networking on in the initrd to verify the keyfile was used - inst.kargs = append(inst.kargs, "rd.neednet=1") - } - - if len(inst.kargs) > 0 { - args := []string{"iso", "kargs", "modify", srcisopath} - for _, karg := range inst.kargs { - args = append(args, "--append", karg) - } - cmd = exec.Command("coreos-installer", args...) - cmd.Stderr = os.Stderr - if err := cmd.Run(); err != nil { - return nil, errors.Wrapf(err, "running coreos-installer iso kargs") - } - } - - if inst.Insecure { - installerConfig.Insecure = true - } - - installerConfigData, err := yaml.Marshal(installerConfig) - if err != nil { - return nil, err - } - mode := 0644 - - inst.liveIgnition.AddSystemdUnit("boot-started.service", bootStartedUnit, conf.Enable) - inst.liveIgnition.AddFile(installerConfig.IgnitionFile, serializedTargetConfig, mode) - inst.liveIgnition.AddFile("/etc/coreos/installer.d/mantle.yaml", string(installerConfigData), mode) - inst.liveIgnition.AddAutoLogin() - - if inst.MultiPathDisk { - inst.liveIgnition.AddSystemdUnit("coreos-installer-multipath.service", `[Unit] -Description=TestISO Enable Multipath -Before=multipathd.service -DefaultDependencies=no -[Service] -Type=oneshot -RemainAfterExit=yes -ExecStart=/usr/sbin/mpathconf --enable -[Install] -WantedBy=coreos-installer.target`, conf.Enable) - inst.liveIgnition.AddSystemdUnitDropin("coreos-installer.service", "wait-for-mpath-target.conf", `[Unit] -Requires=dev-mapper-mpatha.device -After=dev-mapper-mpatha.device`) - } - - qemubuilder := inst.Builder - bootStartedChan, err := qemubuilder.VirtioChannelRead("bootstarted") - if err != nil { - return nil, err - } - - qemubuilder.SetConfig(&inst.liveIgnition) - - // also save live config into the output dir for debugging - liveConfigPath := filepath.Join(outdir, "config-live.ign") - if err := inst.liveIgnition.WriteFile(liveConfigPath); err != nil { - return nil, err - } - - if err := qemubuilder.AddIso(srcisopath, "bootindex=3", false); err != nil { - return nil, err - } - - // With the recent change to use qemu -nodefaults (bc68d7c) we need to - // request network. Otherwise we get no network devices. - if !offline { - qemubuilder.UsermodeNetworking = true - } - - qinst, err := qemubuilder.Exec() - if err != nil { - return nil, err - } - cleanupTempdir = false // Transfer ownership - instmachine := InstalledMachine{ - QemuInst: qinst, - Tempdir: tempdir, - } - switchBootOrderSignal(qinst, bootStartedChan, &instmachine.BootStartedErrorChannel) - return &instmachine, nil -} diff --git a/mantle/platform/qemu.go b/mantle/platform/qemu.go index c236601f30..07a15bd249 100644 --- a/mantle/platform/qemu.go +++ b/mantle/platform/qemu.go @@ -57,6 +57,13 @@ import ( var ( // ErrInitramfsEmergency is the marker error returned upon node blocking in emergency mode in initramfs. ErrInitramfsEmergency = errors.New("entered emergency.target in initramfs") + + ConsoleKernelArgument = map[string]string{ + "x86_64": "ttyS0,115200n8", + "ppc64le": "hvc0", + "aarch64": "ttyAMA0", + "s390x": "ttysclp0", + } ) // HostForwardPort contains details about port-forwarding for the VM. @@ -1531,7 +1538,7 @@ func (builder *QemuBuilder) setupIso() error { if kargsSupported, err := coreosInstallerSupportsISOKargs(); err != nil { return err } else if kargsSupported { - allargs := fmt.Sprintf("console=%s %s", consoleKernelArgument[coreosarch.CurrentRpmArch()], builder.AppendKernelArgs) + allargs := fmt.Sprintf("console=%s %s", ConsoleKernelArgument[coreosarch.CurrentRpmArch()], builder.AppendKernelArgs) instCmdKargs := exec.Command("coreos-installer", "iso", "kargs", "modify", "--append", allargs, isoEmbeddedPath) var stderrb bytes.Buffer instCmdKargs.Stderr = &stderrb