Skip to content

Commit 5f27bec

Browse files
authored
enable tlsMinVersion and tlsCipherSuites configuration in jobset configuration (#1127)
1 parent 4ac567e commit 5f27bec

File tree

16 files changed

+1038
-12
lines changed

16 files changed

+1038
-12
lines changed

Makefile

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -58,6 +58,8 @@ JOBSET_CHART_DIR := charts/jobset
5858
E2E_TARGET ?= ./test/e2e/...
5959
E2E_KIND_VERSION ?= kindest/node:v1.34.0
6060
USE_EXISTING_CLUSTER ?= false
61+
# E2E config folder to use (e.g., "default", "certmanager", etc.)
62+
E2E_TARGET_FOLDER ?= default
6163

6264
# For local testing, we should allow user to use different kind cluster name
6365
# Default will delete default kind cluster
@@ -408,7 +410,7 @@ test-integration: manifests fmt vet envtest ginkgo ## Run tests.
408410

409411
.PHONY: test-e2e-kind
410412
test-e2e-kind: manifests kustomize fmt vet envtest ginkgo kind-image-build
411-
E2E_KIND_VERSION=$(E2E_KIND_VERSION) KIND_CLUSTER_NAME=$(KIND_CLUSTER_NAME) USE_EXISTING_CLUSTER=$(USE_EXISTING_CLUSTER) ARTIFACTS=$(ARTIFACTS) IMAGE_TAG=$(IMAGE_TAG) ./hack/e2e-test.sh
413+
E2E_KIND_VERSION=$(E2E_KIND_VERSION) KIND_CLUSTER_NAME=$(KIND_CLUSTER_NAME) USE_EXISTING_CLUSTER=$(USE_EXISTING_CLUSTER) ARTIFACTS=$(ARTIFACTS) IMAGE_TAG=$(IMAGE_TAG) E2E_TARGET_FOLDER=$(E2E_TARGET_FOLDER) ./hack/e2e-test.sh
412414

413415
.PHONY: prometheus
414416
prometheus:

api/config/v1alpha1/configuration_types.go

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -58,6 +58,11 @@ type ControllerManager struct {
5858
// Health contains the controller health configuration
5959
// +optional
6060
Health ControllerHealth `json:"health,omitempty"`
61+
62+
// TLS contains TLS security settings for all JobSet API servers
63+
// (webhooks and metrics).
64+
// +optional
65+
TLS *TLSOptions `json:"tls,omitempty"`
6166
}
6267

6368
// ControllerWebhook defines the webhook server for the controller.
@@ -135,3 +140,22 @@ type ClientConnection struct {
135140
// Burst allows extra queries to accumulate when a client is exceeding its rate.
136141
Burst *int32 `json:"burst,omitempty"`
137142
}
143+
144+
// TLSOptions defines TLS security settings for JobSet servers
145+
type TLSOptions struct {
146+
// minVersion is the minimum TLS version supported.
147+
// Values are from tls package constants (https://golang.org/pkg/crypto/tls/#pkg-constants).
148+
// Valid values are: "VersionTLS12", "VersionTLS13"
149+
// If unset, the server defaults to TLS 1.2.
150+
// +optional
151+
MinVersion string `json:"minVersion,omitempty"`
152+
153+
// cipherSuites is the list of allowed cipher suites for the server.
154+
// Values are from tls package constants (https://golang.org/pkg/crypto/tls/#pkg-constants).
155+
// This setting only applies to TLS versions up to 1.2; TLS 1.3 cipher suites are
156+
// hardcoded by Go and are not configurable. Any attempt to configure TLS 1.3
157+
// cipher suites will be rejected by validation.
158+
// The default is to leave this unset and inherit golang's default cipher suites.
159+
// +optional
160+
CipherSuites []string `json:"cipherSuites,omitempty"`
161+
}

api/config/v1alpha1/zz_generated.deepcopy.go

Lines changed: 25 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

hack/e2e-test.sh

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,8 @@ export KIND=$PWD/bin/kind
55
# For testing against existing clusters, one needs to set this environment variable
66
export NAMESPACE="jobset-system"
77
export JOBSET_E2E_TESTS_DUMP_NAMESPACE=true
8+
# E2E config folder to use (defaults to "default")
9+
export E2E_TARGET_FOLDER="${E2E_TARGET_FOLDER:-default}"
810

911
# Use a temporary KUBECONFIG so that the script does not mess with the current user's kubeconfig.
1012
KUBECONFIG=""
@@ -43,8 +45,8 @@ function kind_load {
4345
function jobset_deploy {
4446
echo "cd config/components/manager && $KUSTOMIZE edit set image controller=$IMAGE_TAG"
4547
(cd config/components/manager && $KUSTOMIZE edit set image controller=$IMAGE_TAG)
46-
echo "kubectl apply --server-side -k test/e2e/config"
47-
kubectl apply --server-side -k test/e2e/config
48+
echo "kubectl apply --server-side -k test/e2e/config/$E2E_TARGET_FOLDER"
49+
kubectl apply --server-side -k test/e2e/config/$E2E_TARGET_FOLDER
4850
}
4951
trap cleanup EXIT
5052
startup

main.go

Lines changed: 17 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -42,8 +42,10 @@ import (
4242
jobset "sigs.k8s.io/jobset/api/jobset/v1alpha2"
4343
"sigs.k8s.io/jobset/pkg/config"
4444
"sigs.k8s.io/jobset/pkg/controllers"
45+
"sigs.k8s.io/jobset/pkg/features"
4546
"sigs.k8s.io/jobset/pkg/metrics"
4647
"sigs.k8s.io/jobset/pkg/util/cert"
48+
"sigs.k8s.io/jobset/pkg/util/tlsconfig"
4749
"sigs.k8s.io/jobset/pkg/util/useragent"
4850
"sigs.k8s.io/jobset/pkg/version"
4951
"sigs.k8s.io/jobset/pkg/webhooks"
@@ -156,6 +158,20 @@ func main() {
156158
setupLog.Info("disabling http/2")
157159
c.NextProtos = []string{"http/1.1"}
158160
}
161+
162+
// Parse TLS options from configuration if feature gate is enabled
163+
tlsOpts := []func(*tls.Config){disableHTTP2}
164+
if features.Enabled(features.TLSOptions) && cfg.TLS != nil {
165+
parsedTLS, err := tlsconfig.ParseTLSOptions(cfg.TLS)
166+
if err != nil {
167+
setupLog.Error(err, "Unable to parse TLS options from configuration")
168+
os.Exit(1)
169+
}
170+
if parsedTLS != nil {
171+
tlsOpts = append(tlsOpts, tlsconfig.BuildTLSOptions(parsedTLS)...)
172+
}
173+
}
174+
159175
// Metrics endpoint is enabled in 'config/default/kustomization.yaml'. The Metrics options configure the server.
160176
// More info:
161177
// - https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.19.1/pkg/metrics/server
@@ -164,7 +180,7 @@ func main() {
164180
BindAddress: metricsAddr,
165181
SecureServing: true,
166182
FilterProvider: filters.WithAuthenticationAndAuthorization,
167-
TLSOpts: []func(*tls.Config){disableHTTP2},
183+
TLSOpts: tlsOpts,
168184
CertDir: options.Metrics.CertDir,
169185
CertName: options.Metrics.CertName,
170186
KeyName: options.Metrics.KeyName,

pkg/config/config.go

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,8 @@ import (
1414
"sigs.k8s.io/controller-runtime/pkg/webhook"
1515

1616
configapi "sigs.k8s.io/jobset/api/config/v1alpha1"
17+
"sigs.k8s.io/jobset/pkg/features"
18+
"sigs.k8s.io/jobset/pkg/util/tlsconfig"
1719
)
1820

1921
func fromFile(path string, scheme *runtime.Scheme, cfg *configapi.Configuration) error {
@@ -71,6 +73,17 @@ func addTo(o *ctrl.Options, cfg *configapi.Configuration) {
7173
if cfg.Webhook.CertDir != "" {
7274
wo.CertDir = cfg.Webhook.CertDir
7375
}
76+
77+
// Apply TLS configuration if feature gate is enabled
78+
if features.Enabled(features.TLSOptions) && cfg.TLS != nil {
79+
tlsOpts, err := tlsconfig.ParseTLSOptions(cfg.TLS)
80+
if err != nil {
81+
ctrl.Log.Error(err, "failed to parse TLS options, webhook server will start without custom TLS configuration")
82+
} else if tlsOpts != nil {
83+
wo.TLSOpts = append(wo.TLSOpts, tlsconfig.BuildTLSOptions(tlsOpts)...)
84+
}
85+
}
86+
7487
o.WebhookServer = webhook.NewServer(wo)
7588
}
7689
}

pkg/config/config_test.go

Lines changed: 143 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@ limitations under the License.
1717
package config
1818

1919
import (
20+
"crypto/tls"
2021
"errors"
2122
"io/fs"
2223
"net"
@@ -37,6 +38,7 @@ import (
3738
"sigs.k8s.io/controller-runtime/pkg/webhook"
3839

3940
configapi "sigs.k8s.io/jobset/api/config/v1alpha1"
41+
"sigs.k8s.io/jobset/pkg/features"
4042
)
4143

4244
func TestLoad(t *testing.T) {
@@ -596,3 +598,144 @@ func TestEncode(t *testing.T) {
596598
})
597599
}
598600
}
601+
602+
func TestTLSConfiguration(t *testing.T) {
603+
testScheme := runtime.NewScheme()
604+
err := configapi.AddToScheme(testScheme)
605+
if err != nil {
606+
t.Fatal(err)
607+
}
608+
609+
tmpDir := t.TempDir()
610+
611+
// Config with TLS settings
612+
tlsConfig := filepath.Join(tmpDir, "tls-config.yaml")
613+
if err := os.WriteFile(tlsConfig, []byte(`
614+
apiVersion: config.jobset.x-k8s.io/v1alpha1
615+
kind: Configuration
616+
health:
617+
healthProbeBindAddress: :8081
618+
metrics:
619+
bindAddress: :8443
620+
leaderElection:
621+
leaderElect: true
622+
resourceName: 6d4f6a47.jobset.x-k8s.io
623+
webhook:
624+
port: 9443
625+
tls:
626+
minVersion: VersionTLS12
627+
cipherSuites:
628+
- TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256
629+
- TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384
630+
`), os.FileMode(0600)); err != nil {
631+
t.Fatal(err)
632+
}
633+
634+
// Config with TLS 1.3 (no cipher suites)
635+
tls13Config := filepath.Join(tmpDir, "tls13-config.yaml")
636+
if err := os.WriteFile(tls13Config, []byte(`
637+
apiVersion: config.jobset.x-k8s.io/v1alpha1
638+
kind: Configuration
639+
health:
640+
healthProbeBindAddress: :8081
641+
metrics:
642+
bindAddress: :8443
643+
leaderElection:
644+
leaderElect: true
645+
resourceName: 6d4f6a47.jobset.x-k8s.io
646+
webhook:
647+
port: 9443
648+
tls:
649+
minVersion: VersionTLS13
650+
`), os.FileMode(0600)); err != nil {
651+
t.Fatal(err)
652+
}
653+
654+
testcases := []struct {
655+
name string
656+
configFile string
657+
featureGateEnabled bool
658+
wantTLSOptsApplied bool
659+
wantMinVersion uint16
660+
wantCipherSuiteSet bool
661+
}{
662+
{
663+
name: "TLS config with feature gate enabled",
664+
configFile: tlsConfig,
665+
featureGateEnabled: true,
666+
wantTLSOptsApplied: true,
667+
wantMinVersion: tls.VersionTLS12,
668+
wantCipherSuiteSet: true,
669+
},
670+
{
671+
name: "TLS config with feature gate disabled",
672+
configFile: tlsConfig,
673+
featureGateEnabled: false,
674+
wantTLSOptsApplied: false,
675+
},
676+
{
677+
name: "TLS 1.3 config with feature gate enabled",
678+
configFile: tls13Config,
679+
featureGateEnabled: true,
680+
wantTLSOptsApplied: true,
681+
wantMinVersion: tls.VersionTLS13,
682+
wantCipherSuiteSet: false,
683+
},
684+
}
685+
686+
for _, tc := range testcases {
687+
t.Run(tc.name, func(t *testing.T) {
688+
features.SetFeatureGateDuringTest(t, features.TLSOptions, tc.featureGateEnabled)
689+
690+
options, cfg, err := Load(testScheme, tc.configFile)
691+
if err != nil {
692+
t.Fatalf("Unexpected error: %s", err)
693+
}
694+
695+
// Verify TLS config is in the parsed configuration
696+
if cfg.TLS == nil {
697+
t.Errorf("Expected TLS configuration to be present in config")
698+
return
699+
}
700+
701+
// Check webhook server TLS options
702+
webhookServer := options.WebhookServer
703+
if webhookServer == nil {
704+
t.Errorf("Expected webhook server to be set")
705+
return
706+
}
707+
708+
defaultServer, ok := webhookServer.(*webhook.DefaultServer)
709+
if !ok {
710+
t.Errorf("Expected webhook server to be DefaultServer type")
711+
return
712+
}
713+
714+
// Verify TLSOpts is set correctly based on feature gate
715+
if tc.wantTLSOptsApplied {
716+
if len(defaultServer.Options.TLSOpts) == 0 {
717+
t.Errorf("Expected TLSOpts to be set when feature gate is enabled")
718+
} else {
719+
// Verify the TLS options are correctly applied by invoking them
720+
tlsConfig := &tls.Config{}
721+
for _, opt := range defaultServer.Options.TLSOpts {
722+
opt(tlsConfig)
723+
}
724+
if tlsConfig.MinVersion != tc.wantMinVersion {
725+
t.Errorf("MinVersion = %v, want %v", tlsConfig.MinVersion, tc.wantMinVersion)
726+
}
727+
if tc.wantCipherSuiteSet && len(tlsConfig.CipherSuites) == 0 {
728+
t.Errorf("Expected cipher suites to be set")
729+
}
730+
if !tc.wantCipherSuiteSet && len(tlsConfig.CipherSuites) != 0 {
731+
t.Errorf("Expected cipher suites to not be set for TLS 1.3")
732+
}
733+
}
734+
} else {
735+
if len(defaultServer.Options.TLSOpts) != 0 {
736+
t.Errorf("Expected TLSOpts to be empty when feature gate is disabled, got %d options", len(defaultServer.Options.TLSOpts))
737+
}
738+
}
739+
})
740+
}
741+
}

pkg/config/validation.go

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,15 +8,18 @@ import (
88
"k8s.io/utils/ptr"
99

1010
configapi "sigs.k8s.io/jobset/api/config/v1alpha1"
11+
"sigs.k8s.io/jobset/pkg/util/tlsconfig"
1112
)
1213

1314
var (
1415
internalCertManagementPath = field.NewPath("internalCertManagement")
16+
tlsPath = field.NewPath("tls")
1517
)
1618

1719
func validate(c *configapi.Configuration) field.ErrorList {
1820
var allErrs field.ErrorList
1921
allErrs = append(allErrs, validateInternalCertManagement(c)...)
22+
allErrs = append(allErrs, validateTLS(c)...)
2023
return allErrs
2124
}
2225

@@ -37,3 +40,27 @@ func validateInternalCertManagement(c *configapi.Configuration) field.ErrorList
3740
}
3841
return allErrs
3942
}
43+
44+
func validateTLS(c *configapi.Configuration) field.ErrorList {
45+
var allErrs field.ErrorList
46+
if c.TLS == nil {
47+
return allErrs
48+
}
49+
50+
// Validate using ParseTLSOptions first - this provides clearer error messages
51+
// for invalid input (including TLS 1.0/1.1 rejection and invalid cipher suites)
52+
_, err := tlsconfig.ParseTLSOptions(c.TLS)
53+
if err != nil {
54+
allErrs = append(allErrs, field.Invalid(tlsPath.Root(), c.TLS, err.Error()))
55+
return allErrs
56+
}
57+
58+
// TLS 1.3 cipher suites are not configurable in Go's crypto/tls package.
59+
// When TLS 1.3 is set as the minimum version, cipher suites must not be specified.
60+
if c.TLS.MinVersion == "VersionTLS13" && len(c.TLS.CipherSuites) > 0 {
61+
allErrs = append(allErrs, field.Invalid(tlsPath.Child("cipherSuites"),
62+
c.TLS.CipherSuites, "may not be specified when `minVersion` is 'VersionTLS13'"))
63+
}
64+
65+
return allErrs
66+
}

0 commit comments

Comments
 (0)