Skip to content

Commit 15403da

Browse files
pdabelf5pdabelf5
authored
Update secrets when mgmt configmap changes (#6947)
1 parent 552992d commit 15403da

File tree

3 files changed

+166
-13
lines changed

3 files changed

+166
-13
lines changed

‎internal/k8s/controller.go‎

Lines changed: 53 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -883,6 +883,7 @@ func (lbc *LoadBalancerController) updateAllConfigs() {
883883
var isNGINXConfigValid bool
884884
var mgmtConfigHasWarnings bool
885885
var mgmtErr error
886+
var reloadNginx bool
886887

887888
if lbc.configMap != nil {
888889
cfgParams, isNGINXConfigValid = configs.ParseConfigMap(ctx, lbc.configMap, lbc.isNginxPlus, lbc.appProtectEnabled, lbc.appProtectDosEnabled, lbc.configuration.isTLSPassthroughEnabled, lbc.recorder)
@@ -892,6 +893,15 @@ func (lbc *LoadBalancerController) updateAllConfigs() {
892893
if mgmtErr != nil {
893894
nl.Errorf(lbc.Logger, "configmap %s/%s: %v", lbc.mgmtConfigMap.GetNamespace(), lbc.mgmtConfigMap.GetName(), mgmtErr)
894895
}
896+
// update special license secret in mgmtConfigParams
897+
if mgmtCfgParams.Secrets.License != "" {
898+
secret, err := lbc.client.CoreV1().Secrets(lbc.mgmtConfigMap.GetNamespace()).Get(context.TODO(), mgmtCfgParams.Secrets.License, meta_v1.GetOptions{})
899+
if err != nil {
900+
nl.Errorf(lbc.Logger, "secret %s/%s: %v", lbc.mgmtConfigMap.GetNamespace(), mgmtCfgParams.Secrets.License, err)
901+
}
902+
lbc.specialSecrets.licenseSecret = fmt.Sprintf("%s/%s", secret.Namespace, secret.Name)
903+
lbc.handleSpecialSecretUpdate(secret, reloadNginx)
904+
}
895905
// update special CA secret in mgmtConfigParams
896906
if mgmtCfgParams.Secrets.TrustedCert != "" {
897907
secret, err := lbc.client.CoreV1().Secrets(lbc.mgmtConfigMap.GetNamespace()).Get(context.TODO(), mgmtCfgParams.Secrets.TrustedCert, meta_v1.GetOptions{})
@@ -901,6 +911,17 @@ func (lbc *LoadBalancerController) updateAllConfigs() {
901911
if _, hasCRL := secret.Data[configs.CACrlKey]; hasCRL {
902912
mgmtCfgParams.Secrets.TrustedCRL = secret.Name
903913
}
914+
lbc.specialSecrets.trustedCertSecret = fmt.Sprintf("%s/%s", secret.Namespace, secret.Name)
915+
lbc.handleSpecialSecretUpdate(secret, reloadNginx)
916+
}
917+
// update special ClientAuth secret in mgmtConfigParams
918+
if mgmtCfgParams.Secrets.ClientAuth != "" {
919+
secret, err := lbc.client.CoreV1().Secrets(lbc.mgmtConfigMap.GetNamespace()).Get(context.TODO(), mgmtCfgParams.Secrets.ClientAuth, meta_v1.GetOptions{})
920+
if err != nil {
921+
nl.Errorf(lbc.Logger, "secret %s/%s: %v", lbc.mgmtConfigMap.GetNamespace(), mgmtCfgParams.Secrets.ClientAuth, err)
922+
}
923+
lbc.specialSecrets.clientAuthSecret = fmt.Sprintf("%s/%s", secret.Namespace, secret.Name)
924+
lbc.handleSpecialSecretUpdate(secret, reloadNginx)
904925
}
905926
}
906927

@@ -1769,7 +1790,8 @@ func (lbc *LoadBalancerController) syncSecret(task task) {
17691790
lbc.secretStore.AddOrUpdateSecret(secret)
17701791

17711792
if lbc.isSpecialSecret(key) {
1772-
lbc.handleSpecialSecretUpdate(secret)
1793+
reloadNginx := true
1794+
lbc.handleSpecialSecretUpdate(secret, reloadNginx)
17731795
// we don't return here in case the special secret is also used in resources.
17741796
}
17751797

@@ -1828,25 +1850,22 @@ func (lbc *LoadBalancerController) handleSecretUpdate(secret *api_v1.Secret, res
18281850
warnings, addOrUpdateErr = lbc.configurator.AddOrUpdateResources(resourceExes, !lbc.configurator.DynamicSSLReloadEnabled())
18291851
if addOrUpdateErr != nil {
18301852
nl.Errorf(lbc.Logger, "Error when updating Secret %v: %v", secretNsName, addOrUpdateErr)
1831-
lbc.recorder.Eventf(secret, api_v1.EventTypeWarning, "UpdatedWithError", "%v was updated, but not applied: %v", secretNsName, addOrUpdateErr)
1853+
lbc.recorder.Eventf(lbc.metadata.pod, api_v1.EventTypeWarning, "UpdatedWithError", "%v was updated, but not applied: %v", secretNsName, addOrUpdateErr)
18321854
}
18331855

18341856
lbc.updateResourcesStatusAndEvents(resources, warnings, addOrUpdateErr)
18351857
}
18361858

1837-
func (lbc *LoadBalancerController) validationTLSSpecialSecret(secret *api_v1.Secret, secretName string, secretList *[]string) {
1838-
secretNsName := generateSecretNSName(secret)
1839-
1859+
func (lbc *LoadBalancerController) validationTLSSpecialSecret(secret *api_v1.Secret, secretName string, secretList *[]string) error {
18401860
err := secrets.ValidateTLSSecret(secret)
18411861
if err != nil {
1842-
nl.Errorf(lbc.Logger, "Couldn't validate the special Secret %v: %v", secretNsName, err)
1843-
lbc.recorder.Eventf(secret, api_v1.EventTypeWarning, "Rejected", "the special Secret %v was rejected, using the previous version: %v", secretNsName, err)
1844-
return
1862+
return err
18451863
}
18461864
*secretList = append(*secretList, secretName)
1865+
return nil
18471866
}
18481867

1849-
func (lbc *LoadBalancerController) handleSpecialSecretUpdate(secret *api_v1.Secret) {
1868+
func (lbc *LoadBalancerController) handleSpecialSecretUpdate(secret *api_v1.Secret, reload bool) {
18501869
var specialTLSSecretsToUpdate []string
18511870
secretNsName := generateSecretNSName(secret)
18521871

@@ -1860,6 +1879,12 @@ func (lbc *LoadBalancerController) handleSpecialSecretUpdate(secret *api_v1.Secr
18601879
return
18611880
}
18621881

1882+
// When the MGMT Configmap updates, we don't need to reload here, we are reloading in updateAllConfigs().
1883+
if !reload {
1884+
lbc.recorder.Eventf(lbc.metadata.pod, api_v1.EventTypeNormal, "SecretUpdated", "the special Secret %v was updated", secretNsName)
1885+
return
1886+
}
1887+
18631888
// reload nginx when the TLS special secrets are updated
18641889
switch secretNsName {
18651890
case lbc.specialSecrets.licenseSecret:
@@ -1881,7 +1906,7 @@ func (lbc *LoadBalancerController) handleSpecialSecretUpdate(secret *api_v1.Secr
18811906
}
18821907
}
18831908

1884-
lbc.recorder.Eventf(secret, api_v1.EventTypeNormal, "Updated", "the special Secret %v was updated", secretNsName)
1909+
lbc.recorder.Eventf(lbc.metadata.pod, api_v1.EventTypeNormal, "SecretUpdated", "the special Secret %v was updated", secretNsName)
18851910
}
18861911

18871912
// writeSpecialSecrets generates content and writes the secret to disk
@@ -1904,10 +1929,20 @@ func (lbc *LoadBalancerController) writeSpecialSecrets(secret *api_v1.Secret, se
19041929

19051930
func (lbc *LoadBalancerController) specialSecretValidation(secretNsName string, secret *api_v1.Secret, specialTLSSecretsToUpdate *[]string) bool {
19061931
if secretNsName == lbc.specialSecrets.defaultServerSecret {
1907-
lbc.validationTLSSpecialSecret(secret, configs.DefaultServerSecretFileName, specialTLSSecretsToUpdate)
1932+
err := lbc.validationTLSSpecialSecret(secret, configs.DefaultServerSecretFileName, specialTLSSecretsToUpdate)
1933+
if err != nil {
1934+
nl.Errorf(lbc.Logger, "Couldn't validate the special Secret %v: %v", secretNsName, err)
1935+
lbc.recorder.Eventf(lbc.metadata.pod, api_v1.EventTypeWarning, "Rejected", "the special Secret %v was rejected, using the previous version: %v", secretNsName, err)
1936+
return false
1937+
}
19081938
}
19091939
if secretNsName == lbc.specialSecrets.wildcardTLSSecret {
1910-
lbc.validationTLSSpecialSecret(secret, configs.WildcardSecretFileName, specialTLSSecretsToUpdate)
1940+
err := lbc.validationTLSSpecialSecret(secret, configs.WildcardSecretFileName, specialTLSSecretsToUpdate)
1941+
if err != nil {
1942+
nl.Errorf(lbc.Logger, "Couldn't validate the special Secret %v: %v", secretNsName, err)
1943+
lbc.recorder.Eventf(lbc.metadata.pod, api_v1.EventTypeWarning, "Rejected", "the special Secret %v was rejected, using the previous version: %v", secretNsName, err)
1944+
return false
1945+
}
19111946
}
19121947
if secretNsName == lbc.specialSecrets.licenseSecret {
19131948
err := secrets.ValidateLicenseSecret(secret)
@@ -1926,7 +1961,12 @@ func (lbc *LoadBalancerController) specialSecretValidation(secretNsName string,
19261961
}
19271962
}
19281963
if secretNsName == lbc.specialSecrets.clientAuthSecret {
1929-
lbc.validationTLSSpecialSecret(secret, configs.ClientAuthCertSecretFileName, specialTLSSecretsToUpdate)
1964+
err := lbc.validationTLSSpecialSecret(secret, configs.ClientAuthCertSecretFileName, specialTLSSecretsToUpdate)
1965+
if err != nil {
1966+
nl.Errorf(lbc.Logger, "Couldn't validate the special Secret %v: %v", secretNsName, err)
1967+
lbc.recorder.Eventf(lbc.metadata.pod, api_v1.EventTypeWarning, "Rejected", "the special Secret %v was rejected, using the previous version: %v", secretNsName, err)
1968+
return false
1969+
}
19301970
}
19311971
return true
19321972
}

‎tests/data/mgmt-configmap-keys/plus-token-name-keys.yaml‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
apiVersion: v1
2+
kind: ConfigMap
3+
metadata:
4+
name: nginx-config-mgmt
5+
namespace: nginx-ingress
6+
data:
7+
license-token-secret-name: "license-token-changed"

‎tests/suite/test_mgmt_configmap_keys.py‎

Lines changed: 106 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,106 @@
1+
import pytest
2+
from settings import TEST_DATA
3+
from suite.utils.resources_utils import (
4+
create_license,
5+
ensure_connection_to_public_endpoint,
6+
get_events_for_object,
7+
get_first_pod_name,
8+
get_reload_count,
9+
is_secret_present,
10+
replace_configmap_from_yaml,
11+
wait_before_test,
12+
)
13+
14+
15+
def assert_event(event_list, event_type, reason, message_substring):
16+
"""
17+
Assert that an event with specific type, reason, and message substring exists.
18+
19+
:param event_list: List of events
20+
:param event_type: 'Normal' or 'Warning'
21+
:param reason: Event reason
22+
:param message_substring: Substring expected in the event message
23+
"""
24+
for event in event_list:
25+
if event.type == event_type and event.reason == reason and message_substring in event.message:
26+
return
27+
assert (
28+
False
29+
), f"Expected event with type '{event_type}', reason '{reason}', and message containing '{message_substring}' not found."
30+
31+
32+
@pytest.mark.skip_for_nginx_oss
33+
@pytest.mark.ingresses
34+
@pytest.mark.smoke
35+
class TestMGMTConfigMap:
36+
@pytest.mark.parametrize(
37+
"ingress_controller",
38+
[
39+
pytest.param(
40+
{"extra_args": ["-enable-prometheus-metrics"]},
41+
)
42+
],
43+
indirect=["ingress_controller"],
44+
)
45+
def test_mgmt_configmap_events(
46+
self,
47+
cli_arguments,
48+
kube_apis,
49+
ingress_controller_prerequisites,
50+
ingress_controller,
51+
ingress_controller_endpoint,
52+
):
53+
ensure_connection_to_public_endpoint(
54+
ingress_controller_endpoint.public_ip,
55+
ingress_controller_endpoint.port,
56+
ingress_controller_endpoint.port_ssl,
57+
)
58+
ic_pod_name = get_first_pod_name(kube_apis.v1, ingress_controller_prerequisites.namespace)
59+
metrics_url = (
60+
f"http://{ingress_controller_endpoint.public_ip}:{ingress_controller_endpoint.metrics_port}/metrics"
61+
)
62+
63+
print("Step 1: get reload count")
64+
reload_count = get_reload_count(metrics_url)
65+
66+
wait_before_test(1)
67+
print(f"Step 1a: initial reload count is {reload_count}")
68+
69+
print("Step 2: create duplicate existing secret with new name")
70+
license_name = create_license(
71+
kube_apis.v1,
72+
ingress_controller_prerequisites.namespace,
73+
cli_arguments["plus-jwt"],
74+
license_token_name="license-token-changed",
75+
)
76+
assert is_secret_present(kube_apis.v1, license_name, ingress_controller_prerequisites.namespace)
77+
78+
print("Step 3: update the ConfigMap/license-token-secret-name to the new secret")
79+
replace_configmap_from_yaml(
80+
kube_apis.v1,
81+
"nginx-config-mgmt",
82+
ingress_controller_prerequisites.namespace,
83+
f"{TEST_DATA}/mgmt-configmap-keys/plus-token-name-keys.yaml",
84+
)
85+
86+
wait_before_test()
87+
88+
print("Step 4: check reload count has incremented")
89+
new_reload_count = get_reload_count(metrics_url)
90+
print(f"Step 4a: new reload count is {new_reload_count}")
91+
assert new_reload_count > reload_count
92+
93+
print("Step 5: check pod for SecretUpdated event")
94+
events = get_events_for_object(
95+
kube_apis.v1,
96+
ingress_controller_prerequisites.namespace,
97+
ic_pod_name,
98+
)
99+
100+
# Assert that the 'SecretUpdated' event is present
101+
assert_event(
102+
events,
103+
"Normal",
104+
"SecretUpdated",
105+
f"the special Secret {ingress_controller_prerequisites.namespace}/{license_name} was updated",
106+
)

0 commit comments

Comments
 (0)