diff --git a/api/v1alpha3/azurestackhciloadbalancer_types.go b/api/v1alpha3/azurestackhciloadbalancer_types.go index 394861c6..7bc5fbd4 100644 --- a/api/v1alpha3/azurestackhciloadbalancer_types.go +++ b/api/v1alpha3/azurestackhciloadbalancer_types.go @@ -32,7 +32,6 @@ type AzureStackHCILoadBalancerSpec struct { SSHPublicKey string `json:"sshPublicKey"` Image Image `json:"image"` VMSize string `json:"vmSize"` - // Number of desired loadbalancer machines. Defaults to 1. // This is a pointer to distinguish between explicit zero and not specified. // +optional diff --git a/api/v1alpha4/azurestackhciloadbalancer_types.go b/api/v1alpha4/azurestackhciloadbalancer_types.go index eef6b313..cd0e3925 100644 --- a/api/v1alpha4/azurestackhciloadbalancer_types.go +++ b/api/v1alpha4/azurestackhciloadbalancer_types.go @@ -32,7 +32,6 @@ type AzureStackHCILoadBalancerSpec struct { SSHPublicKey string `json:"sshPublicKey"` Image Image `json:"image"` VMSize string `json:"vmSize"` - // Number of desired loadbalancer machines. Defaults to 1. // This is a pointer to distinguish between explicit zero and not specified. // +optional diff --git a/api/v1beta1/azurestackhciloadbalancer_types.go b/api/v1beta1/azurestackhciloadbalancer_types.go index 0923eb25..f5ef7034 100644 --- a/api/v1beta1/azurestackhciloadbalancer_types.go +++ b/api/v1beta1/azurestackhciloadbalancer_types.go @@ -32,7 +32,8 @@ type AzureStackHCILoadBalancerSpec struct { SSHPublicKey string `json:"sshPublicKey"` Image Image `json:"image"` VMSize string `json:"vmSize"` - + // +optional + HostType HostType `json:"hostType,omitempty"` // +optional StorageContainer string `json:"storageContainer"` diff --git a/api/v1beta1/azurestackhcimachine_types.go b/api/v1beta1/azurestackhcimachine_types.go index f7864bb6..8931ee19 100644 --- a/api/v1beta1/azurestackhcimachine_types.go +++ b/api/v1beta1/azurestackhcimachine_types.go @@ -59,6 +59,8 @@ type AzureStackHCIMachineSpec struct { // +optional NetworkInterfaces NetworkInterfaces `json:"networkInterfaces,omitempty"` + // +optional + HostType HostType `json:"hostType,omitempty"` } // AzureStackHCIMachineStatus defines the observed state of AzureStackHCIMachine diff --git a/api/v1beta1/azurestackhcivirtualmachine_types.go b/api/v1beta1/azurestackhcivirtualmachine_types.go index 52b1b910..7e674b37 100644 --- a/api/v1beta1/azurestackhcivirtualmachine_types.go +++ b/api/v1beta1/azurestackhcivirtualmachine_types.go @@ -55,6 +55,8 @@ type AzureStackHCIVirtualMachineSpec struct { // +optional NetworkInterfaces NetworkInterfaces `json:"networkInterfaces,omitempty"` + // +optional + HostType HostType `json:"hostType,omitempty"` } // AzureStackHCIVirtualMachineStatus defines the observed state of AzureStackHCIVirtualMachine diff --git a/api/v1beta1/types.go b/api/v1beta1/types.go index 12f24543..52e2c334 100644 --- a/api/v1beta1/types.go +++ b/api/v1beta1/types.go @@ -265,3 +265,15 @@ const ( AzureOperationIDAnnotationKey = "management.azure.com/operationId" AzureCorrelationIDAnnotationKey = "management.azure.com/correlationId" ) + +// HostType specifies what type of machine a node should be deployed on. +type HostType string + +const ( + // HostTypeVM specifies that the node should be deployed on a virtual machine. + // Default value. + HostTypeVM = HostType("vm") + + // HostTypeBareMetal specifies that the node should be deployed on a bare metal machine. + HostTypeBareMetal = HostType("baremetal") +) diff --git a/cloud/converters/vm.go b/cloud/converters/vm.go index 78be96ad..85499709 100644 --- a/cloud/converters/vm.go +++ b/cloud/converters/vm.go @@ -32,3 +32,13 @@ func SDKToVM(v compute.VirtualMachine) (*infrav1.VM, error) { } return vm, nil } + +// BareMetalMachineConvertToCAPH converts an SDK BareMetalMachine to the provider VM type. +func BareMetalMachineConvertToCAPH(v compute.BareMetalMachine) (*infrav1.VM, error) { + vm := &infrav1.VM{ + ID: to.String(v.ID), + Name: to.String(v.Name), + State: infrav1.VMStateSucceeded, // Hard-coded for now until we expose provisioning state + } + return vm, nil +} diff --git a/cloud/services/virtualmachines/service.go b/cloud/services/virtualmachines/service.go index 991624ee..c06d05a9 100644 --- a/cloud/services/virtualmachines/service.go +++ b/cloud/services/virtualmachines/service.go @@ -20,6 +20,7 @@ package virtualmachines import ( azurestackhci "github.com/microsoft/cluster-api-provider-azurestackhci/cloud" "github.com/microsoft/cluster-api-provider-azurestackhci/cloud/scope" + "github.com/microsoft/moc-sdk-for-go/services/compute/baremetalmachine" "github.com/microsoft/moc-sdk-for-go/services/compute/virtualmachine" "github.com/microsoft/moc/pkg/auth" ) @@ -28,8 +29,9 @@ var _ azurestackhci.Service = (*Service)(nil) // Service provides operations on virtual machines. type Service struct { - Client virtualmachine.VirtualMachineClient - Scope scope.ScopeInterface + Client virtualmachine.VirtualMachineClient + BareMetalClient baremetalmachine.BareMetalMachineClient + Scope scope.ScopeInterface } // getVirtualMachinesClient creates a new virtual machines client. @@ -38,10 +40,16 @@ func getVirtualMachinesClient(cloudAgentFqdn string, authorizer auth.Authorizer) return *vmClient } +func getBareMetalMachinesClient(cloudAgentFqdn string, authorizer auth.Authorizer) baremetalmachine.BareMetalMachineClient { + bareMetalClient, _ := baremetalmachine.NewBareMetalMachineClient(cloudAgentFqdn, authorizer) + return *bareMetalClient +} + // NewService creates a new virtual machines service. func NewService(scope scope.ScopeInterface) *Service { return &Service{ - Client: getVirtualMachinesClient(scope.GetCloudAgentFqdn(), scope.GetAuthorizer()), - Scope: scope, + Client: getVirtualMachinesClient(scope.GetCloudAgentFqdn(), scope.GetAuthorizer()), + BareMetalClient: getBareMetalMachinesClient(scope.GetCloudAgentFqdn(), scope.GetAuthorizer()), + Scope: scope, } } diff --git a/cloud/services/virtualmachines/virtualmachines.go b/cloud/services/virtualmachines/virtualmachines.go index 54749303..00a5a13f 100644 --- a/cloud/services/virtualmachines/virtualmachines.go +++ b/cloud/services/virtualmachines/virtualmachines.go @@ -35,6 +35,7 @@ import ( "github.com/microsoft/moc-sdk-for-go/services/network" "github.com/pkg/errors" "golang.org/x/crypto/ssh" + "k8s.io/klog" ) const ( @@ -54,24 +55,43 @@ type Spec struct { CustomData string VMType compute.VMType StorageContainer string + HostType infrav1.HostType } // Get provides information about a virtual machine. func (s *Service) Get(ctx context.Context, spec interface{}) (interface{}, error) { + var err error vmSpec, ok := spec.(*Spec) if !ok { - return compute.VirtualMachine{}, errors.New("invalid vm specification") + return nil, errors.New("invalid vm specification") } - vm, err := s.Client.Get(ctx, s.Scope.GetResourceGroup(), vmSpec.Name) - if err != nil { - return nil, err - } - if vm == nil || len(*vm) == 0 { - return nil, errors.Wrapf(err, "vm %s not found", vmSpec.Name) - } + switch vmSpec.HostType { + case infrav1.HostTypeBareMetal: + var baremetalmachine *[]compute.BareMetalMachine + + baremetalmachine, err = s.BareMetalClient.Get(ctx, s.Scope.GetResourceGroup(), vmSpec.Name) + if err != nil { + return nil, err + } + + if baremetalmachine == nil || len(*baremetalmachine) == 0 { + return nil, errors.Errorf("bare-metal machine %s not found", vmSpec.Name) + } - return converters.SDKToVM((*vm)[0]) + return converters.BareMetalMachineConvertToCAPH((*baremetalmachine)[0]) + + default: + vm, err := s.Client.Get(ctx, s.Scope.GetResourceGroup(), vmSpec.Name) + if err != nil { + return nil, err + } + if vm == nil || len(*vm) == 0 { + return nil, errors.Errorf("vm %s not found", vmSpec.Name) + } + + return converters.SDKToVM((*vm)[0]) + } } // Reconcile gets/creates/updates a virtual machine. @@ -88,16 +108,22 @@ func (s *Service) Reconcile(ctx context.Context, spec interface{}) error { } logger := s.Scope.GetLogger() - logger.Info("getting nic", "nic", vmSpec.NICName) - nicInterface, err := networkinterfaces.NewService(s.Scope).Get(ctx, &networkinterfaces.Spec{Name: vmSpec.NICName}) - if err != nil { - return err - } - nic, ok := nicInterface.(network.Interface) - if !ok { - return errors.New("error getting network interface") + nicStr := "" + + // ignore for baremetal + if vmSpec.HostType != infrav1.HostTypeBareMetal { + logger.Info("getting nic", "nic", vmSpec.NICName) + nicInterface, err := networkinterfaces.NewService(s.Scope).Get(ctx, &networkinterfaces.Spec{Name: vmSpec.NICName}) + if err != nil { + return err + } + nic, ok := nicInterface.(network.Interface) + if !ok { + return errors.New("error getting network interface") + } + logger.Info("got nic", "nic", vmSpec.NICName) + nicStr = *nic.Name } - logger.Info("got nic", "nic", vmSpec.NICName) logger.Info("creating vm", "Name", vmSpec.Name, @@ -156,7 +182,7 @@ func (s *Service) Reconcile(ctx context.Context, spec interface{}) error { NetworkProfile: &compute.NetworkProfile{ NetworkInterfaces: &[]compute.NetworkInterfaceReference{ { - ID: nic.Name, + ID: &nicStr, }, }, }, @@ -181,21 +207,68 @@ func (s *Service) Reconcile(ctx context.Context, spec interface{}) error { } } - _, err = s.Client.CreateOrUpdate( - ctx, - s.Scope.GetResourceGroup(), - vmSpec.Name, - &virtualMachine) - telemetry.WriteMocOperationLog(logger, telemetry.CreateOrUpdate, s.Scope.GetCustomResourceTypeWithName(), telemetry.VirtualMachine, - telemetry.GenerateMocResourceName(s.Scope.GetResourceGroup(), vmSpec.Name), nil, err) - if err != nil { - return errors.Wrapf(err, "cannot create vm") + switch vmSpec.HostType { + case infrav1.HostTypeBareMetal: + _, err := s.createOrUpdateBareMetal( + ctx, + &virtualMachine) + if err != nil { + return errors.Wrapf(err, "cannot create bare-metal machine") + } + + default: + _, err = s.Client.CreateOrUpdate( + ctx, + s.Scope.GetResourceGroup(), + vmSpec.Name, + &virtualMachine) + telemetry.WriteMocOperationLog(logger, telemetry.CreateOrUpdate, s.Scope.GetCustomResourceTypeWithName(), telemetry.VirtualMachine, + telemetry.GenerateMocResourceName(s.Scope.GetResourceGroup(), vmSpec.Name), nil, err) + if err != nil { + return errors.Wrapf(err, "cannot create vm") + } } logger.Info("successfully created vm", "name", vmSpec.Name) return err } +func (s *Service) createOrUpdateBareMetal(ctx context.Context, virtualMachine *compute.VirtualMachine) (*compute.BareMetalMachine, error) { + // Create a new baremetal machine + bareMetalMachine := &compute.BareMetalMachine{ + Name: virtualMachine.Name, + } + + bareMetalMachine.BareMetalMachineProperties = &compute.BareMetalMachineProperties{ + StorageProfile: &compute.BareMetalMachineStorageProfile{ + ImageReference: &compute.BareMetalMachineImageReference{ + ID: virtualMachine.VirtualMachineProperties.StorageProfile.ImageReference.ID, + Name: virtualMachine.VirtualMachineProperties.StorageProfile.ImageReference.Name, + }, + }, + OsProfile: &compute.BareMetalMachineOSProfile{ + ComputerName: virtualMachine.VirtualMachineProperties.OsProfile.ComputerName, + AdminUsername: virtualMachine.VirtualMachineProperties.OsProfile.AdminUsername, + AdminPassword: virtualMachine.VirtualMachineProperties.OsProfile.AdminPassword, + CustomData: virtualMachine.VirtualMachineProperties.OsProfile.CustomData, + LinuxConfiguration: virtualMachine.VirtualMachineProperties.OsProfile.LinuxConfiguration, + }, + SecurityProfile: virtualMachine.VirtualMachineProperties.SecurityProfile, + ProvisioningState: virtualMachine.VirtualMachineProperties.ProvisioningState, + Statuses: virtualMachine.VirtualMachineProperties.Statuses, + } + + // Try to apply the update. + _, err := s.BareMetalClient.CreateOrUpdate(ctx, s.Scope.GetResourceGroup(), *bareMetalMachine.Name, bareMetalMachine) + + if err != nil { + return nil, errors.Wrap(err, "Failed to create baremetal machine.") + } + + // klog.V(2).Infof("Successfully created baremetal machine %s ", bareMetalMachine.Name) + return bareMetalMachine, nil +} + // Delete deletes the virtual machine with the provided name. func (s *Service) Delete(ctx context.Context, spec interface{}) error { telemetry.WriteMocInfoLog(ctx, s.Scope) @@ -204,19 +277,27 @@ func (s *Service) Delete(ctx context.Context, spec interface{}) error { return errors.New("invalid vm Specification") } logger := s.Scope.GetLogger() - logger.Info("deleting vm", "vm", vmSpec.Name) - err := s.Client.Delete(ctx, s.Scope.GetResourceGroup(), vmSpec.Name) - telemetry.WriteMocOperationLog(s.Scope.GetLogger(), telemetry.Delete, s.Scope.GetCustomResourceTypeWithName(), telemetry.VirtualMachine, - telemetry.GenerateMocResourceName(s.Scope.GetResourceGroup(), vmSpec.Name), nil, err) + + var err error + switch vmSpec.HostType { + case infrav1.HostTypeBareMetal: + klog.V(2).Infof("deleting bare-metal machine %s ", vmSpec.Name) + err = s.BareMetalClient.Delete(ctx, s.Scope.GetResourceGroup(), vmSpec.Name) + default: + logger.Info("deleting vm", "vm", vmSpec.Name) + err = s.Client.Delete(ctx, s.Scope.GetResourceGroup(), vmSpec.Name) + telemetry.WriteMocOperationLog(s.Scope.GetLogger(), telemetry.Delete, s.Scope.GetCustomResourceTypeWithName(), telemetry.VirtualMachine, + telemetry.GenerateMocResourceName(s.Scope.GetResourceGroup(), vmSpec.Name), nil, err) + } if err != nil && azurestackhci.ResourceNotFound(err) { // already deleted return nil } if err != nil { - return errors.Wrapf(err, "failed to delete vm %s in resource group %s", vmSpec.Name, s.Scope.GetResourceGroup()) + return errors.Wrapf(err, "failed to delete %s %s in resource group %s", vmSpec.HostType, vmSpec.Name, s.Scope.GetResourceGroup()) } - logger.Info("successfully deleted vm", "vm", vmSpec.Name) + klog.V(2).Infof("successfully deleted %s %s ", vmSpec.HostType, vmSpec.Name) return err } diff --git a/config/crd/bases/infrastructure.cluster.x-k8s.io_azurestackhciclusters.yaml b/config/crd/bases/infrastructure.cluster.x-k8s.io_azurestackhciclusters.yaml index 3e1aadde..94178197 100644 --- a/config/crd/bases/infrastructure.cluster.x-k8s.io_azurestackhciclusters.yaml +++ b/config/crd/bases/infrastructure.cluster.x-k8s.io_azurestackhciclusters.yaml @@ -46,6 +46,10 @@ spec: description: AzureStackHCILoadBalancer is used to declare the AzureStackHCILoadBalancerSpec if a LoadBalancer is desired for the AzureStackHCICluster. properties: + hostType: + description: HostType specifies what type of machine a node should + be deployed on. + type: string image: description: 'Image defines information about the image to use for VM creation. There are three ways to specify an image: by diff --git a/config/crd/bases/infrastructure.cluster.x-k8s.io_azurestackhciloadbalancers.yaml b/config/crd/bases/infrastructure.cluster.x-k8s.io_azurestackhciloadbalancers.yaml index a8e921df..3482420b 100644 --- a/config/crd/bases/infrastructure.cluster.x-k8s.io_azurestackhciloadbalancers.yaml +++ b/config/crd/bases/infrastructure.cluster.x-k8s.io_azurestackhciloadbalancers.yaml @@ -67,6 +67,10 @@ spec: type: object spec: properties: + hostType: + description: HostType specifies what type of machine a node should + be deployed on. + type: string image: description: 'Image defines information about the image to use for VM creation. There are three ways to specify an image: by ID, by diff --git a/config/crd/bases/infrastructure.cluster.x-k8s.io_azurestackhcimachines.yaml b/config/crd/bases/infrastructure.cluster.x-k8s.io_azurestackhcimachines.yaml index 2f0d0337..43fb8180 100644 --- a/config/crd/bases/infrastructure.cluster.x-k8s.io_azurestackhcimachines.yaml +++ b/config/crd/bases/infrastructure.cluster.x-k8s.io_azurestackhcimachines.yaml @@ -52,6 +52,10 @@ spec: id: type: string type: object + hostType: + description: HostType specifies what type of machine a node should + be deployed on. + type: string image: description: 'Image defines information about the image to use for VM creation. There are three ways to specify an image: by ID, by diff --git a/config/crd/bases/infrastructure.cluster.x-k8s.io_azurestackhcimachinetemplates.yaml b/config/crd/bases/infrastructure.cluster.x-k8s.io_azurestackhcimachinetemplates.yaml index e0b37465..833c48d9 100644 --- a/config/crd/bases/infrastructure.cluster.x-k8s.io_azurestackhcimachinetemplates.yaml +++ b/config/crd/bases/infrastructure.cluster.x-k8s.io_azurestackhcimachinetemplates.yaml @@ -61,6 +61,10 @@ spec: id: type: string type: object + hostType: + description: HostType specifies what type of machine a node + should be deployed on. + type: string image: description: 'Image defines information about the image to use for VM creation. There are three ways to specify an diff --git a/config/crd/bases/infrastructure.cluster.x-k8s.io_azurestackhcivirtualmachines.yaml b/config/crd/bases/infrastructure.cluster.x-k8s.io_azurestackhcivirtualmachines.yaml index 50676783..c5cac300 100644 --- a/config/crd/bases/infrastructure.cluster.x-k8s.io_azurestackhcivirtualmachines.yaml +++ b/config/crd/bases/infrastructure.cluster.x-k8s.io_azurestackhcivirtualmachines.yaml @@ -57,6 +57,10 @@ spec: type: string clusterName: type: string + hostType: + description: HostType specifies what type of machine a node should + be deployed on. + type: string identity: description: VMIdentity defines the identity of the virtual machine, if configured. diff --git a/controllers/azurestackhcimachine_controller.go b/controllers/azurestackhcimachine_controller.go index 621c0ddc..e0065bc7 100644 --- a/controllers/azurestackhcimachine_controller.go +++ b/controllers/azurestackhcimachine_controller.go @@ -30,7 +30,7 @@ import ( "github.com/microsoft/cluster-api-provider-azurestackhci/cloud/scope" "github.com/microsoft/cluster-api-provider-azurestackhci/cloud/telemetry" infrav1util "github.com/microsoft/cluster-api-provider-azurestackhci/pkg/util" - + "github.com/microsoft/moc/pkg/providerid" "github.com/pkg/errors" corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" @@ -232,7 +232,7 @@ func (r *AzureStackHCIMachineReconciler) reconcileNormal(machineScope *scope.Mac } // Make sure Spec.ProviderID is always set. - machineScope.SetProviderID(fmt.Sprintf("moc://%s", vm.Name)) + machineScope.SetProviderID(providerid.FormatProviderID(providerid.HostType(vm.Spec.HostType), vm.Name)) // TODO(vincepri): Remove this annotation when clusterctl is no longer relevant. machineScope.SetAnnotation("cluster-api-provider-azurestackhci", "true") @@ -313,7 +313,7 @@ func (r *AzureStackHCIMachineReconciler) reconcileVirtualMachineNormal(machineSc return errors.Wrap(err, "failed to get VM image") } image.DeepCopyInto(&vm.Spec.Image) - + vm.Spec.HostType = machineScope.AzureStackHCIMachine.Spec.HostType vm.Spec.VMSize = machineScope.AzureStackHCIMachine.Spec.VMSize machineScope.AzureStackHCIMachine.Spec.AvailabilityZone.DeepCopyInto(&vm.Spec.AvailabilityZone) machineScope.AzureStackHCIMachine.Spec.OSDisk.DeepCopyInto(&vm.Spec.OSDisk) diff --git a/controllers/azurestackhcivirtualmachine_reconciler.go b/controllers/azurestackhcivirtualmachine_reconciler.go index 3ef45a0f..c3d1125f 100644 --- a/controllers/azurestackhcivirtualmachine_reconciler.go +++ b/controllers/azurestackhcivirtualmachine_reconciler.go @@ -87,7 +87,8 @@ func (s *azureStackHCIVirtualMachineService) Create() (*infrav1.VM, error) { // Delete reconciles all the services in pre determined order func (s *azureStackHCIVirtualMachineService) Delete() error { vmSpec := &virtualmachines.Spec{ - Name: s.vmScope.Name(), + Name: s.vmScope.Name(), + HostType: s.vmScope.AzureStackHCIVirtualMachine.Spec.HostType, } err := s.virtualMachinesSvc.Delete(s.vmScope.Context, vmSpec) @@ -120,7 +121,8 @@ func (s *azureStackHCIVirtualMachineService) Delete() error { func (s *azureStackHCIVirtualMachineService) VMIfExists() (*infrav1.VM, error) { vmSpec := &virtualmachines.Spec{ - Name: s.vmScope.Name(), + Name: s.vmScope.Name(), + HostType: s.vmScope.AzureStackHCIVirtualMachine.Spec.HostType, } vmInterface, err := s.virtualMachinesSvc.Get(s.vmScope.Context, vmSpec) @@ -163,6 +165,11 @@ func (s *azureStackHCIVirtualMachineService) reconcileDisk(disk infrav1.OSDisk) } func (s *azureStackHCIVirtualMachineService) reconcileNetworkInterface(nicName string, ipconfigs networkinterfaces.IPConfigurations) error { + // ignore nic creation for baremetal host + if s.vmScope.AzureStackHCIVirtualMachine.Spec.HostType == infrav1.HostTypeBareMetal { + return nil + } + networkInterfaceSpec := &networkinterfaces.Spec{ Name: nicName, VnetName: s.vmScope.VnetName(), @@ -196,7 +203,8 @@ func (s *azureStackHCIVirtualMachineService) createVirtualMachine(nicName string } vmSpec := &virtualmachines.Spec{ - Name: s.vmScope.Name(), + Name: s.vmScope.Name(), + HostType: s.vmScope.AzureStackHCIVirtualMachine.Spec.HostType, } vmInterface, err := s.virtualMachinesSvc.Get(s.vmScope.Context, vmSpec) @@ -239,6 +247,7 @@ func (s *azureStackHCIVirtualMachineService) createVirtualMachine(nicName string Zone: vmZone, VMType: vmType, StorageContainer: s.vmScope.StorageContainer(), + HostType: s.vmScope.AzureStackHCIVirtualMachine.Spec.HostType, } err = s.virtualMachinesSvc.Reconcile(s.vmScope.Context, vmSpec) diff --git a/go.mod b/go.mod index 22728f1a..54eb73c4 100644 --- a/go.mod +++ b/go.mod @@ -18,6 +18,7 @@ require ( k8s.io/api v0.27.7 k8s.io/apimachinery v0.27.7 k8s.io/client-go v0.27.7 + k8s.io/klog v1.0.0 k8s.io/klog/v2 v2.90.1 k8s.io/utils v0.0.0-20230209194617-a36077c30491 sigs.k8s.io/cluster-api v1.5.3 @@ -100,7 +101,6 @@ require ( gopkg.in/yaml.v3 v3.0.1 // indirect k8s.io/apiextensions-apiserver v0.27.7 // indirect k8s.io/component-base v0.27.7 // indirect - k8s.io/klog v1.0.0 // indirect k8s.io/kube-openapi v0.0.0-20230501164219-8b0f38b5fd1f // indirect sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect