Skip to content

Commit 6e8a218

Browse files
author
nwickramasin
committed
e2e tests for opaAuthorizeRequest filter
1 parent 07d21ac commit 6e8a218

File tree

6 files changed

+159
-2
lines changed

6 files changed

+159
-2
lines changed

cluster/node-pools/master-default/userdata.yaml

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -357,7 +357,7 @@ write_files:
357357
- mountPath: /etc/kubernetes/k8s-authnz-webhook-kubeconfig
358358
name: k8s-authnz-webhook-kubeconfig
359359
readOnly: true
360-
- image: 926694233939.dkr.ecr.eu-central-1.amazonaws.com/production_namespace/foundation/platform-iam-tokeninfo:master-130
360+
- image: 926694233939.dkr.ecr.eu-central-1.amazonaws.com/production_namespace/foundation/platform-iam-tokeninfo:master-131
361361
name: tokeninfo
362362
ports:
363363
- containerPort: 9021
@@ -388,7 +388,7 @@ write_files:
388388
value: {{ .Cluster.ConfigItems.apiserver_business_partner_ids }}
389389
{{ if ne .Cluster.Environment "production" }}
390390
- name: tokeninfo-sandbox
391-
image: 926694233939.dkr.ecr.eu-central-1.amazonaws.com/production_namespace/foundation/platform-iam-tokeninfo:master-130
391+
image: 926694233939.dkr.ecr.eu-central-1.amazonaws.com/production_namespace/foundation/platform-iam-tokeninfo:master-131
392392
ports:
393393
- containerPort: 9022
394394
lifecycle:

delivery.yaml

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -80,6 +80,8 @@ pipeline:
8080
value: "teapot-kubernetes-e2e-results"
8181
- name: ETCD_ENDPOINTS
8282
value: "https://etcd-server.etcd.teapot-e2e.zalan.do:2479"
83+
- name: SKIPPER_OPA_ENABLED
84+
value: "true"
8385
- name: HOSTED_ZONE
8486
valueFrom:
8587
configMapKeyRef:
@@ -155,6 +157,26 @@ pipeline:
155157
secretKeyRef:
156158
name: kubernetes-e2e-config-secret
157159
key: "OKTA_AUTH_ISSUER_URL"
160+
- name: STYRA_TOKEN
161+
valueFrom:
162+
secretKeyRef:
163+
name: kubernetes-e2e-config-secret
164+
key: "STYRA_TOKEN"
165+
- name: SKIPPER_OPA_BUCKET_ARN
166+
valueFrom:
167+
secretKeyRef:
168+
name: kubernetes-e2e-config-secret
169+
key: "SKIPPER_OPA_BUCKET_ARN"
170+
- name: SKIPPER_OPA_OBSERVABILITY_URL
171+
valueFrom:
172+
secretKeyRef:
173+
name: kubernetes-e2e-config-secret
174+
key: "SKIPPER_OPA_OBSERVABILITY_URL"
175+
- name: SKIPPER_OPA_BUNDLES_URL
176+
valueFrom:
177+
secretKeyRef:
178+
name: kubernetes-e2e-config-secret
179+
key: "SKIPPER_OPA_BUNDLES_URL"
158180
- name: CLUSTER_ADMIN_TOKEN
159181
valueFrom:
160182
secretKeyRef:

test/e2e/apply/secret.yaml

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,3 +19,7 @@ data:
1919
ETCD_SCALYR_KEY: "deployment-secret:2:stups-test:AQICAHjXIrc66g/+P4X1Gl4MKcInWmwpFxivAqFGMI0fr9DvCwETRzvm1hGplyUn23FEXUVtAAAAnjCBmwYJKoZIhvcNAQcGoIGNMIGKAgEAMIGEBgkqhkiG9w0BBwEwHgYJYIZIAWUDBAEuMBEEDOfPJJJy60sDkZEIHgIBEIBXiANNciEqpcuZ3hFPCt6NkFtk0WBTSasDQHHbyuR8O+n5iM9k8/nUTLUrFlhba8blArq/ALE8vuKNdlS17q6PxGlvwJFFXQn/McohMpdyfnfQYKW8MPCu"
2020
OKTA_AUTH_ISSUER_URL: "deployment-secret:2:stups-test:AQICAHjXIrc66g/+P4X1Gl4MKcInWmwpFxivAqFGMI0fr9DvCwGmCMhSN2Er1sw2ofYnI44EAAAApDCBoQYJKoZIhvcNAQcGoIGTMIGQAgEAMIGKBgkqhkiG9w0BBwEwHgYJYIZIAWUDBAEuMBEEDO2IC+r/zcUzXoQEHAIBEIBdrFchwu9i7LpMbyDbslu/lBxvfyh+nCGK33jtcxT3RdxuTXWuSJhkX+gU4cgFXAI5LLnXh4M20jHUEEPU78MJWR47HLTPGPJcKQj5fOpPqpD3duuKIrZDRm5ba6AN"
2121
SESSION_MANAGER_DESTINATION_ARN: "deployment-secret:2:stups-test:AQICAHjXIrc66g/+P4X1Gl4MKcInWmwpFxivAqFGMI0fr9DvCwF7fOZ9i6BDvWdNEddR7LZOAAAArjCBqwYJKoZIhvcNAQcGoIGdMIGaAgEAMIGUBgkqhkiG9w0BBwEwHgYJYIZIAWUDBAEuMBEEDBJwU/Zns+mzOBgczQIBEIBn/86xpnVO2Apr5nG3waPEAGCFYDWdOXcaS7pFKdNIhpXaADtODQtEd874HcE0W2I3bjKr3d3ghJFdN8r0BZiSmTbgc0fn+5ZiBTyGBfzWP4BCzxjRMvURl/7MX8ygwL78hpSxyRypAQ=="
22+
STYRA_TOKEN: "deployment-secret:2:stups-test:AQICAHjXIrc66g/+P4X1Gl4MKcInWmwpFxivAqFGMI0fr9DvCwEECuXXi+W3FFt7qLjWk/S6AAAAwDCBvQYJKoZIhvcNAQcGoIGvMIGsAgEAMIGmBgkqhkiG9w0BBwEwHgYJYIZIAWUDBAEuMBEEDGAuwqmeDmRyjVy91gIBEIB5u6jiCoj1vIwZJ/dJtdI/8cxG9y6RGjopd20Sh1+5TCoHKzPfyV97Whl6YFLRke6ixO+UBnA4KeNh5A/ykQ7yUIvg5b9WDH5tV8Gb+vWyvsd4sdULVfioeTS67e6S0ApSMd/CHCfZdsTwTi1iZ2spSkS0YWolGyY+9A=="
23+
SKIPPER_OPA_BUCKET_ARN: "deployment-secret:2:stups-test:AQICAHjXIrc66g/+P4X1Gl4MKcInWmwpFxivAqFGMI0fr9DvCwGZdCVDLsCdProfzvZU7UAwAAAAlzCBlAYJKoZIhvcNAQcGoIGGMIGDAgEAMH4GCSqGSIb3DQEHATAeBglghkgBZQMEAS4wEQQMehOf7Uu444SWS6kbAgEQgFFPMaa0flwHLpxrkYjJMK4jXc0q4kX+KGrB5GFjKuUgOUPmQ+ME/aQduxwl2+xUilrKP50/NLXgMNHjeeHuZfoyiSgpGFBM4z8L0N6ggf2uE5U="
24+
SKIPPER_OPA_OBSERVABILITY_URL: "deployment-secret:2:stups-test:AQICAHjXIrc66g/+P4X1Gl4MKcInWmwpFxivAqFGMI0fr9DvCwHl773AuNEvIpzaM6ycpDNSAAAAqzCBqAYJKoZIhvcNAQcGoIGaMIGXAgEAMIGRBgkqhkiG9w0BBwEwHgYJYIZIAWUDBAEuMBEEDGld6jpQ38gOzVcn0gIBEIBkTHbv3adeEfRntVTUQyyQkIhUnc0QXKtmtJEdvBoRzWiJIBKQUQuM1VBV0re3HkO8HSY59nkwyHEncBMkHJoI9rC2LJuWU20oCjPw9lbweih+6Sxo+nqkDrQd+mHp+uA9Om3KqA=="
25+
SKIPPER_OPA_BUNDLES_URL: "deployment-secret:2:stups-test:AQICAHjXIrc66g/+P4X1Gl4MKcInWmwpFxivAqFGMI0fr9DvCwFnhaIRP4+3Y69xp1ycTI7qAAAAsTCBrgYJKoZIhvcNAQcGoIGgMIGdAgEAMIGXBgkqhkiG9w0BBwEwHgYJYIZIAWUDBAEuMBEEDF9gAl70l2g2kwfnJgIBEIBqP/DgIhIu5x5XNR1Ubqinz6r4ttQoHty8nXd6mxie2r6NxHskNOqkiSactUKhNIhboNlNsO4p4rKEkhglTeFZlEQvgEYNioWPw39xqICnUDPVr+Kp0Yrs/bzPLPV9wOlB917UiT7WJNybPg=="

test/e2e/cluster_config.sh

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,11 @@ clusters:
4747
karpenter_pools_enabled: "true"
4848
okta_auth_client_id: "kubernetes.cluster.teapot-e2e"
4949
teapot_admission_controller_validate_pod_images_soft_fail_namespaces: "^kube-system$"
50+
skipper_open_policy_agent_enabled: "${SKIPPER_OPA_ENABLED}"
51+
skipper_open_policy_agent_styra_token: "${STYRA_TOKEN}"
52+
skipper_open_policy_agent_bucket_arn: "${SKIPPER_OPA_BUCKET_ARN}"
53+
skipper_open_policy_agent_observability_url: "${SKIPPER_OPA_OBSERVABILITY_URL}"
54+
skipper_open_policy_agent_bundles_url: "${SKIPPER_OPA_BUNDLES_URL}"
5055
criticality_level: 1
5156
environment: e2e
5257
id: ${CLUSTER_ID}

test/e2e/ingress.go

Lines changed: 126 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1027,3 +1027,129 @@ var _______ = describe("Ingress tests simple NLB", func() {
10271027
Expect(resp.Header.Get("Request-Host")).To(Equal(hostName))
10281028
})
10291029
})
1030+
1031+
var ________ = describe("Ingress tests for OPA filters", func() {
1032+
f := framework.NewDefaultFramework("skipper-ingress-with-opa")
1033+
f.NamespacePodSecurityEnforceLevel = admissionapi.LevelBaseline
1034+
var (
1035+
cs kubernetes.Interface
1036+
jig *ingress.TestJig
1037+
)
1038+
1039+
It("Should activate OPA filter without issue [Ingress] [Zalando]", func() {
1040+
jig = ingress.NewIngressTestJig(f.ClientSet)
1041+
cs = f.ClientSet
1042+
serviceName := "styra-smoketest"
1043+
ns := f.Namespace.Name
1044+
hostName := fmt.Sprintf("%s-%d.%s", serviceName, time.Now().UTC().Unix(), E2EHostedZone())
1045+
labels := map[string]string{
1046+
"app": serviceName,
1047+
}
1048+
port := 8080
1049+
replicas := int32(3)
1050+
targetPort := 9090
1051+
backendContent := "mytest"
1052+
route := fmt.Sprintf(`* -> inlineContent("%s") -> <shunt>`, backendContent)
1053+
waitTime := 10 * time.Minute
1054+
1055+
// CREATE setup
1056+
// backend deployment
1057+
By("Creating a deployment with " + serviceName + " in namespace " + ns)
1058+
depl := createSkipperBackendDeployment(serviceName, ns, route, labels, int32(targetPort), replicas)
1059+
_, err := cs.AppsV1().Deployments(ns).Create(context.TODO(), depl, metav1.CreateOptions{})
1060+
framework.ExpectNoError(err)
1061+
1062+
By("Creating service " + serviceName + " in namespace " + ns)
1063+
service := createServiceTypeClusterIP(serviceName, labels, port, targetPort)
1064+
_, err = cs.CoreV1().Services(ns).Create(context.TODO(), service, metav1.CreateOptions{})
1065+
framework.ExpectNoError(err)
1066+
1067+
ing := createIngress(serviceName, hostName, ns, "/", netv1.PathTypeImplementationSpecific, labels, nil, port)
1068+
ingressCreate, err := cs.NetworkingV1().Ingresses(ns).Create(context.TODO(), ing, metav1.CreateOptions{})
1069+
framework.ExpectNoError(err)
1070+
1071+
addr, err := jig.WaitForIngressAddress(context.TODO(), cs, ns, ingressCreate.Name, waitTime)
1072+
framework.ExpectNoError(err)
1073+
1074+
_, err = cs.NetworkingV1().Ingresses(ns).Get(context.TODO(), ing.Name, metav1.GetOptions{ResourceVersion: "0"})
1075+
framework.ExpectNoError(err)
1076+
1077+
// skipper http -> https redirect
1078+
By("Waiting for skipper route to default redirect from http to https, to see that our ingress-controller and skipper works")
1079+
err = waitForResponse(addr, "http", waitTime, isRedirect, true)
1080+
framework.ExpectNoError(err)
1081+
1082+
// LB ready
1083+
By("Waiting for ALB to create endpoint " + addr + " and skipper route, to see that our ingress-controller and skipper works")
1084+
err = waitForResponse(addr, "https", waitTime, isNotFound, true)
1085+
framework.ExpectNoError(err)
1086+
1087+
// DNS ready
1088+
By("Waiting for DNS to see that external-dns and skipper route to service and pod works")
1089+
err = waitForResponse(hostName, "https", waitTime, isSuccess, false)
1090+
framework.ExpectNoError(err)
1091+
1092+
// Test that we get content from the default ingress
1093+
By("By checking the content of the reply we see that the ingress stack works")
1094+
rt, quit := createHTTPRoundTripper()
1095+
defer func() {
1096+
quit <- struct{}{}
1097+
}()
1098+
url := "https://" + hostName + "/"
1099+
req, err := http.NewRequest("GET", url, nil)
1100+
framework.ExpectNoError(err)
1101+
resp, err := rt.RoundTrip(req)
1102+
framework.ExpectNoError(err)
1103+
s, err := getBody(resp)
1104+
framework.ExpectNoError(err)
1105+
Expect(s).To(Equal(backendContent))
1106+
1107+
// Start actual ingress tests
1108+
// Test ingress Filters: opaAuthorizeRequest
1109+
1110+
/**
1111+
## The Rule looks like below.
1112+
## Reference https://github.bus.zalan.do/corporate-iam/styra-smoketest-policies/blob/main/bundle/policy/ingress/rules.rego
1113+
default allow := false
1114+
1115+
allow if {
1116+
input.attributes.request.http.method == "GET"
1117+
auth_header_val := input.attributes.request.http.headers.authorization
1118+
startswith(auth_header_val, "Basic ")
1119+
token := substring(auth_header_val, count("Basic "), -1)
1120+
token == "valid_token"
1121+
}
1122+
*/
1123+
path := "/"
1124+
opaPolicyName := "styra-smoketest"
1125+
updatedIng := updateIngress(ingressCreate.ObjectMeta.Name,
1126+
ingressCreate.ObjectMeta.Namespace,
1127+
hostName,
1128+
serviceName,
1129+
path,
1130+
netv1.PathTypeImplementationSpecific,
1131+
ingressCreate.ObjectMeta.Labels,
1132+
map[string]string{
1133+
"zalando.org/skipper-filter": fmt.Sprintf(`opaAuthorizeRequest("%s")`, opaPolicyName),
1134+
},
1135+
port,
1136+
)
1137+
ingressUpdate, err := cs.NetworkingV1().Ingresses(ingressCreate.ObjectMeta.Namespace).Update(context.TODO(), updatedIng, metav1.UpdateOptions{})
1138+
framework.ExpectNoError(err)
1139+
time.Sleep(20 * time.Second) // wait for routing change propagation
1140+
1141+
By(fmt.Sprintf("Calling ingress %s/%s we wait to get a 403 with opaAuthorizeRequest %s policy", ingressUpdate.Namespace, ingressUpdate.Name, opaPolicyName))
1142+
resp, err = getAndWaitResponse(rt, req, 10*time.Second, http.StatusForbidden)
1143+
framework.ExpectNoError(err)
1144+
Expect(resp.StatusCode).To(Equal(http.StatusForbidden))
1145+
1146+
By(fmt.Sprintf("Calling ingress %s/%s we wait to get a 200 with opaAuthorizeRequest %s policy", ingressUpdate.Namespace, ingressUpdate.Name, opaPolicyName))
1147+
req.Header.Set("Authorization", "Basic valid_token") //Authorized request
1148+
resp, err = getAndWaitResponse(rt, req, 10*time.Second, http.StatusOK)
1149+
framework.ExpectNoError(err)
1150+
Expect(resp.StatusCode).To(Equal(http.StatusOK))
1151+
s, err = getBody(resp)
1152+
framework.ExpectNoError(err)
1153+
Expect(s).To(Equal(backendContent))
1154+
})
1155+
})

0 commit comments

Comments
 (0)