Merge pull request #8441 from patrickdillon/capi-etcd-kas-logs

OCPBUGS-35182: write etcd and kube-apiserver logs

Author: openshift-merge-bot[bot]

cmd/openshift-install/gather.go

@@ -24,11 +24,11 @@ import (
 	"github.com/openshift/installer/pkg/asset/installconfig"
 	assetstore "github.com/openshift/installer/pkg/asset/store"
 	"github.com/openshift/installer/pkg/asset/tls"
+	"github.com/openshift/installer/pkg/clusterapi"
 	serialgather "github.com/openshift/installer/pkg/gather"
 	"github.com/openshift/installer/pkg/gather/service"
 	"github.com/openshift/installer/pkg/gather/ssh"
 	"github.com/openshift/installer/pkg/infrastructure"
-	"github.com/openshift/installer/pkg/infrastructure/clusterapi"
 	infra "github.com/openshift/installer/pkg/infrastructure/platform"
 
 	_ "github.com/openshift/installer/pkg/gather/aws"
@@ -144,7 +144,7 @@ func gatherBootstrap(bootstrap string, port int, masters []string, directory str
 	gatherID := time.Now().Format("20060102150405")
 	archives := map[string]string{}
 
-	if capiManifestsBundlePath, err := gatherCAPIManifests(directory, gatherID); err != nil {
+	if capiManifestsBundlePath, err := gatherCAPIArtifacts(directory, gatherID); err != nil {
 		// Do not fail the whole gather if we can't find capi manifests (we can be running terraform)
 		logrus.Infof("Failed to gather Cluster API manifests: %s", err.Error())
 	} else {
@@ -247,36 +247,44 @@ func logClusterOperatorConditions(ctx context.Context, config *rest.Config) erro
 	return nil
 }
 
-func gatherCAPIManifests(directory, gatherID string) (string, error) {
-	logrus.Infoln("Pulling Cluster API manifests")
+func gatherCAPIArtifacts(directory, gatherID string) (string, error) {
+	logrus.Infoln("Pulling Cluster API artifacts")
 	dir, err := filepath.Abs(directory)
 	if err != nil {
 		return "", fmt.Errorf("failed to get absolute path for %s: %w", directory, err)
 	}
 
-	capiDir := filepath.Join(dir, clusterapi.CAPIArtifactsDir)
+	capiDir := filepath.Join(dir, clusterapi.ArtifactsDir)
 	if _, err := os.Stat(capiDir); err != nil {
 		if errors.Is(err, fs.ErrNotExist) {
 			return "", fmt.Errorf("either Cluster API manifests not generated or terraform provision")
 		}
 		return "", fmt.Errorf("failed to stat Cluster API output directory: %w", err)
 	}
 
-	bundleDir := filepath.Join(dir, fmt.Sprintf("capi-manifests-bundle-%s", gatherID))
-	// Symlink the hidden directory so the manifests are not hidden in the archive
+	bundleDir := filepath.Join(dir, fmt.Sprintf("capi-artifacts-bundle-%s", gatherID))
+	// Symlink the hidden directory so the artifacts are not hidden in the archive
 	if err := os.Symlink(capiDir, bundleDir); err != nil {
-		return "", fmt.Errorf("failed to copy Cluster API manifests: %w", err)
+		return "", fmt.Errorf("failed to copy Cluster API artifacts: %w", err)
 	}
 	defer os.Remove(bundleDir)
 
-	capiManifests, err := filepath.Glob(filepath.Join(bundleDir, "*.yaml"))
+	var capiArtifacts []string
+	manifests, err := filepath.Glob(filepath.Join(bundleDir, "*.yaml"))
 	if err != nil {
 		return "", fmt.Errorf("failed to gather Cluster API manifests: %w", err)
 	}
+	capiArtifacts = append(capiArtifacts, manifests...)
 
-	capiManifestsBundlePath := fmt.Sprintf("%s.tar.gz", bundleDir)
-	if err := serialgather.CreateArchive(capiManifests, capiManifestsBundlePath); err != nil {
+	logs, err := filepath.Glob(filepath.Join(bundleDir, "*.log"))
+	if err != nil {
+		return "", fmt.Errorf("failed to gather Cluster API control plane logs: %w", err)
+	}
+	capiArtifacts = append(capiArtifacts, logs...)
+
+	capiArtifactsBundlePath := fmt.Sprintf("%s.tar.gz", bundleDir)
+	if err := serialgather.CreateArchive(capiArtifacts, capiArtifactsBundlePath); err != nil {
 		return "", fmt.Errorf("failed to create clusterapi bundle file: %w", err)
 	}
-	return capiManifestsBundlePath, nil
+	return capiArtifactsBundlePath, nil
 }

pkg/clusterapi/localcontrolplane.go

@@ -55,6 +55,8 @@ type localControlPlane struct {
 	Cfg            *rest.Config
 	BinDir         string
 	KubeconfigPath string
+	EtcdLog        *os.File
+	APIServerLog   *os.File
 }
 
 // Run launches the local control plane.
@@ -68,21 +70,38 @@ func (c *localControlPlane) Run(ctx context.Context) error {
 		return fmt.Errorf("failed to unpack envtest binaries: %w", err)
 	}
 
+	// Write etcd & kube-apiserver output to log files.
+	var err error
+	if err := os.MkdirAll(filepath.Join(command.RootOpts.Dir, ArtifactsDir), 0750); err != nil {
+		return fmt.Errorf("error creating artifacts dir: %w", err)
+	}
+	if c.EtcdLog, err = os.Create(filepath.Join(command.RootOpts.Dir, ArtifactsDir, "etcd.log")); err != nil {
+		return fmt.Errorf("failed to create etcd log file: %w", err)
+	}
+	if c.APIServerLog, err = os.Create(filepath.Join(command.RootOpts.Dir, ArtifactsDir, "kube-apiserver.log")); err != nil {
+		return fmt.Errorf("failed to create kube-apiserver log file: %w", err)
+	}
+
 	log.SetLogger(klog.NewKlogr())
 	logrus.Info("Started local control plane with envtest")
 	c.Env = &envtest.Environment{
 		Scheme:                   Scheme,
-		AttachControlPlaneOutput: false,
+		AttachControlPlaneOutput: true,
 		BinaryAssetsDirectory:    c.BinDir,
 		ControlPlaneStartTimeout: 10 * time.Second,
 		ControlPlaneStopTimeout:  10 * time.Second,
 		ControlPlane: envtest.ControlPlane{
 			Etcd: &envtest.Etcd{
 				DataDir: c.BinDir,
+				Out:     c.EtcdLog,
+				Err:     c.EtcdLog,
+			},
+			APIServer: &envtest.APIServer{
+				Out: c.APIServerLog,
+				Err: c.APIServerLog,
 			},
 		},
 	}
-	var err error
 	c.Cfg, err = c.Env.Start()
 	if err != nil {
 		return err

pkg/clusterapi/system.go

@@ -47,6 +47,10 @@ const (
 	SystemStateRunning SystemState = "running"
 	// SystemStateStopped indicates the system is stopped.
 	SystemStateStopped SystemState = "stopped"
+
+	// ArtifactsDir is the directory where output (manifests, kubeconfig, etc.)
+	// related to CAPI-based installs are stored.
+	ArtifactsDir = ".clusterapi_output"
 )
 
 // Interface is the interface for the cluster-api system.
@@ -365,6 +369,10 @@ func (c *system) Teardown() {
 	// Clean up the binary directory.
 	defer os.RemoveAll(c.lcp.BinDir)
 
+	// Clean up log file handles.
+	defer c.lcp.EtcdLog.Close()
+	defer c.lcp.APIServerLog.Close()
+
 	// Proceed to shutdown.
 	c.teardownOnce.Do(func() {
 		c.cancel()

pkg/infrastructure/clusterapi/clusterapi.go

@@ -53,9 +53,6 @@ const (
 	ignitionStage            = "Bootstrap Ignition Provisioning"
 	machineStage             = "Machine Provisioning"
 	postProvisionStage       = "Infrastructure Post-provisioning"
-
-	// CAPIArtifactsDir is the directory where the manifests generated by CAPI are stored.
-	CAPIArtifactsDir = ".clusterapi_output"
 )
 
 // InfraProvider implements common Cluster API logic and
@@ -463,7 +460,7 @@ func extractIPAddress(manifestPath string) (string, error) {
 
 // ExtractHostAddresses extracts the IPs of the bootstrap and control plane machines.
 func (i *InfraProvider) ExtractHostAddresses(dir string, config *types.InstallConfig, ha *infrastructure.HostAddresses) error {
-	manifestsDir := filepath.Join(dir, CAPIArtifactsDir)
+	manifestsDir := filepath.Join(dir, clusterapi.ArtifactsDir)
 	logrus.Debugf("Looking for machine manifests in %s", manifestsDir)
 
 	bootstrapFiles, err := filepath.Glob(filepath.Join(manifestsDir, "Machine\\-openshift\\-cluster\\-api\\-guests\\-*\\-bootstrap.yaml"))
@@ -544,7 +541,7 @@ func (i *InfraProvider) collectManifests(ctx context.Context, cl client.Client)
 			errorList = append(errorList, fmt.Errorf("failed to get GVK for manifest %s: %w", m.GetName(), err))
 			continue
 		}
-		fileName := filepath.Join(CAPIArtifactsDir, fmt.Sprintf("%s-%s-%s.yaml", gvk.Kind, m.GetNamespace(), m.GetName()))
+		fileName := filepath.Join(clusterapi.ArtifactsDir, fmt.Sprintf("%s-%s-%s.yaml", gvk.Kind, m.GetNamespace(), m.GetName()))
 		objData, err := yaml.Marshal(m)
 		if err != nil {
 			errorList = append(errorList, fmt.Errorf("failed to marshal manifest %s: %w", fileName, err))