Skip to content

Commit e516156

Browse files
author
nwickramasin
committed
Introduce separate AaaS tests
1 parent 9157fdf commit e516156

File tree

4 files changed

+160
-128
lines changed

4 files changed

+160
-128
lines changed

‎test/e2e/aaas.go

Lines changed: 158 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,158 @@
1+
package e2e
2+
3+
import (
4+
"context"
5+
"fmt"
6+
7+
. "github.com/onsi/ginkgo/v2"
8+
. "github.com/onsi/gomega"
9+
10+
netv1 "k8s.io/api/networking/v1"
11+
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
12+
"k8s.io/client-go/kubernetes"
13+
"k8s.io/kubernetes/test/e2e/framework"
14+
"k8s.io/kubernetes/test/e2e/framework/ingress"
15+
admissionapi "k8s.io/pod-security-admission/api"
16+
"net/http"
17+
"time"
18+
)
19+
20+
var _ = Describe("Ingress tests for OPA filters", func() {
21+
f := framework.NewDefaultFramework("skipper-ingress-with-opa")
22+
f.NamespacePodSecurityEnforceLevel = admissionapi.LevelBaseline
23+
24+
var (
25+
cs kubernetes.Interface
26+
jig *ingress.TestJig
27+
ingressCreate *netv1.Ingress
28+
ns string
29+
hostName string
30+
port int
31+
serviceName = "aaas-infrastructure-tests"
32+
opaPolicyName = "aaas-infrastructure-tests"
33+
)
34+
35+
BeforeEach(func() {
36+
jig = ingress.NewIngressTestJig(f.ClientSet)
37+
cs = f.ClientSet
38+
ns = f.Namespace.Name
39+
hostName = fmt.Sprintf("%s-%d.%s", serviceName, time.Now().UTC().Unix(), E2EHostedZone())
40+
port = 8080
41+
42+
ingressCreate = createIngressWithInfo(serviceName, hostName, ns, port, cs, jig)
43+
})
44+
45+
It("opaAuthorizeRequest filter should pass through authorized requests [Ingress] [Opa]", func() {
46+
// Authorization done with the rule
47+
// https://github.bus.zalan.do/corporate-iam/aaas-infrastructure-tests-policies/blob/main/bundle/policy/ingress/rules.rego
48+
49+
authorizationEnforcedPath := "/auth"
50+
path := "/"
51+
ingressRoute := fmt.Sprintf(`authorize: PathRegexp("%s") -> modPath("%s", "%s") -> opaAuthorizeRequest("%s") -> inlineContent("Got response") -> <shunt>;`, authorizationEnforcedPath, authorizationEnforcedPath, path, opaPolicyName)
52+
ingressUpdate := updateIngressAndWait(serviceName, hostName, path, ingressRoute, port, ingressCreate, cs)
53+
54+
By(fmt.Sprintf("Calling ingress %s/%s we wait to get a 403 with opaAuthorizeRequest %s policy", ingressUpdate.Namespace, ingressUpdate.Name, opaPolicyName))
55+
rt, quit := createHTTPRoundTripper()
56+
defer func() {
57+
quit <- struct{}{}
58+
}()
59+
60+
url := "https://" + hostName + authorizationEnforcedPath
61+
req, err := http.NewRequest("GET", url, nil)
62+
resp, err := getAndWaitResponse(rt, req, 10*time.Second, http.StatusForbidden)
63+
framework.ExpectNoError(err)
64+
Expect(resp.StatusCode).To(Equal(http.StatusForbidden))
65+
66+
By(fmt.Sprintf("Calling ingress %s/%s we wait to get a 200 with opaAuthorizeRequest %s policy", ingressUpdate.Namespace, ingressUpdate.Name, opaPolicyName))
67+
req.Header.Set("Authorization", "Basic valid_token") // Authorized request
68+
resp, err = getAndWaitResponse(rt, req, 10*time.Second, http.StatusOK)
69+
framework.ExpectNoError(err)
70+
Expect(resp.StatusCode).To(Equal(http.StatusOK))
71+
s, err := getBody(resp)
72+
framework.ExpectNoError(err)
73+
Expect(s).To(Equal("Got response"))
74+
})
75+
76+
It("opaServeResponse filter should return body [Ingress] [Opa]", func() {
77+
// Authorization done with the rule
78+
// https://github.bus.zalan.do/corporate-iam/aaas-infrastructure-tests-policies/blob/main/bundle/policy/ingress/permissions.rego
79+
80+
serveResponsePath := "/permissions"
81+
expectedPermissionsContent := "\"permissions\":{\"permission1\":{},\"permission2\":{}}"
82+
path := serveResponsePath
83+
ingressRoute := fmt.Sprintf(`authorize: PathRegexp("%s") -> opaServeResponse("%s") -> <shunt>;`, serveResponsePath, opaPolicyName)
84+
ingressUpdate := updateIngressAndWait(serviceName, hostName, path, ingressRoute, port, ingressCreate, cs)
85+
86+
By(fmt.Sprintf("Calling ingress %s/%s we wait to get a 403 with opaServeResponse %s policy", ingressUpdate.Namespace, ingressUpdate.Name, opaPolicyName))
87+
rt, quit := createHTTPRoundTripper()
88+
defer func() {
89+
quit <- struct{}{}
90+
}()
91+
92+
url := "https://" + hostName + serveResponsePath
93+
req, err := http.NewRequest("GET", url, nil)
94+
resp, err := getAndWaitResponse(rt, req, 10*time.Second, http.StatusForbidden)
95+
framework.ExpectNoError(err)
96+
Expect(resp.StatusCode).To(Equal(http.StatusForbidden))
97+
98+
By(fmt.Sprintf("Calling ingress %s/%s we wait to get a 200 with opaServeResponse %s policy", ingressUpdate.Namespace, ingressUpdate.Name, opaPolicyName))
99+
req.Header.Set("Authorization", "Basic permissions_token") // Authorized request
100+
resp, err = getAndWaitResponse(rt, req, 10*time.Second, http.StatusOK)
101+
framework.ExpectNoError(err)
102+
Expect(resp.StatusCode).To(Equal(http.StatusOK))
103+
s, err := getBody(resp)
104+
framework.ExpectNoError(err)
105+
Expect(s).To(ContainSubstring(expectedPermissionsContent))
106+
})
107+
108+
})
109+
110+
func createIngressWithInfo(serviceName, hostName, ns string, port int, cs kubernetes.Interface, jig *ingress.TestJig) *netv1.Ingress {
111+
labels := map[string]string{"app": serviceName}
112+
waitTime := 10 * time.Minute
113+
114+
// Create initial ingress
115+
ing := createIngress(serviceName, hostName, ns, "/", netv1.PathTypeImplementationSpecific, labels, nil, port)
116+
var err error
117+
ingressCreate, err := cs.NetworkingV1().Ingresses(ns).Create(context.TODO(), ing, metav1.CreateOptions{})
118+
framework.ExpectNoError(err)
119+
120+
// Wait for ingress address
121+
addr, err := jig.WaitForIngressAddress(context.TODO(), cs, ns, ingressCreate.Name, waitTime)
122+
framework.ExpectNoError(err)
123+
124+
// Ensure ingress exists
125+
_, err = cs.NetworkingV1().Ingresses(ns).Get(context.TODO(), ing.Name, metav1.GetOptions{ResourceVersion: "0"})
126+
framework.ExpectNoError(err)
127+
128+
// Verify skipper routing
129+
By("Waiting for skipper route to default redirect from http to https")
130+
err = waitForResponse(addr, "http", waitTime, isRedirect, true)
131+
framework.ExpectNoError(err)
132+
133+
By("Waiting for ALB to create endpoint and skipper route")
134+
err = waitForResponse(addr, "https", waitTime, isNotFound, true)
135+
framework.ExpectNoError(err)
136+
137+
return ingressCreate
138+
}
139+
140+
func updateIngressAndWait(serviceName, hostName, path, ingressRoute string, port int, ingressCreate *netv1.Ingress, cs kubernetes.Interface) *netv1.Ingress {
141+
updatedIng := updateIngress(ingressCreate.ObjectMeta.Name,
142+
ingressCreate.ObjectMeta.Namespace,
143+
hostName,
144+
serviceName,
145+
path,
146+
netv1.PathTypeImplementationSpecific,
147+
ingressCreate.ObjectMeta.Labels,
148+
map[string]string{
149+
"zalando.org/skipper-routes": ingressRoute,
150+
},
151+
port,
152+
)
153+
ingressUpdate, err := cs.NetworkingV1().Ingresses(ingressCreate.ObjectMeta.Namespace).Update(context.TODO(), updatedIng, metav1.UpdateOptions{})
154+
framework.ExpectNoError(err)
155+
time.Sleep(2 * time.Minute) // wait for routing change propagation
156+
157+
return ingressUpdate
158+
}

‎test/e2e/apply/secret.yaml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@ data:
1919
ETCD_SCALYR_KEY: "deployment-secret:2:stups-test:AQICAHjXIrc66g/+P4X1Gl4MKcInWmwpFxivAqFGMI0fr9DvCwETRzvm1hGplyUn23FEXUVtAAAAnjCBmwYJKoZIhvcNAQcGoIGNMIGKAgEAMIGEBgkqhkiG9w0BBwEwHgYJYIZIAWUDBAEuMBEEDOfPJJJy60sDkZEIHgIBEIBXiANNciEqpcuZ3hFPCt6NkFtk0WBTSasDQHHbyuR8O+n5iM9k8/nUTLUrFlhba8blArq/ALE8vuKNdlS17q6PxGlvwJFFXQn/McohMpdyfnfQYKW8MPCu"
2020
OKTA_AUTH_ISSUER_URL: "deployment-secret:2:stups-test:AQICAHjXIrc66g/+P4X1Gl4MKcInWmwpFxivAqFGMI0fr9DvCwGmCMhSN2Er1sw2ofYnI44EAAAApDCBoQYJKoZIhvcNAQcGoIGTMIGQAgEAMIGKBgkqhkiG9w0BBwEwHgYJYIZIAWUDBAEuMBEEDO2IC+r/zcUzXoQEHAIBEIBdrFchwu9i7LpMbyDbslu/lBxvfyh+nCGK33jtcxT3RdxuTXWuSJhkX+gU4cgFXAI5LLnXh4M20jHUEEPU78MJWR47HLTPGPJcKQj5fOpPqpD3duuKIrZDRm5ba6AN"
2121
SESSION_MANAGER_DESTINATION_ARN: "deployment-secret:2:stups-test:AQICAHjXIrc66g/+P4X1Gl4MKcInWmwpFxivAqFGMI0fr9DvCwF7fOZ9i6BDvWdNEddR7LZOAAAArjCBqwYJKoZIhvcNAQcGoIGdMIGaAgEAMIGUBgkqhkiG9w0BBwEwHgYJYIZIAWUDBAEuMBEEDBJwU/Zns+mzOBgczQIBEIBn/86xpnVO2Apr5nG3waPEAGCFYDWdOXcaS7pFKdNIhpXaADtODQtEd874HcE0W2I3bjKr3d3ghJFdN8r0BZiSmTbgc0fn+5ZiBTyGBfzWP4BCzxjRMvURl/7MX8ygwL78hpSxyRypAQ=="
22-
STYRA_TOKEN: "deployment-secret:2:stups-test:AQICAHjXIrc66g/+P4X1Gl4MKcInWmwpFxivAqFGMI0fr9DvCwEECuXXi+W3FFt7qLjWk/S6AAAAwDCBvQYJKoZIhvcNAQcGoIGvMIGsAgEAMIGmBgkqhkiG9w0BBwEwHgYJYIZIAWUDBAEuMBEEDGAuwqmeDmRyjVy91gIBEIB5u6jiCoj1vIwZJ/dJtdI/8cxG9y6RGjopd20Sh1+5TCoHKzPfyV97Whl6YFLRke6ixO+UBnA4KeNh5A/ykQ7yUIvg5b9WDH5tV8Gb+vWyvsd4sdULVfioeTS67e6S0ApSMd/CHCfZdsTwTi1iZ2spSkS0YWolGyY+9A=="
22+
STYRA_TOKEN: "deployment-secret:2:stups-test:AQICAHjXIrc66g/+P4X1Gl4MKcInWmwpFxivAqFGMI0fr9DvCwGba1bnMYs8qNGHMd7a66H8AAAAwjCBvwYJKoZIhvcNAQcGoIGxMIGuAgEAMIGoBgkqhkiG9w0BBwEwHgYJYIZIAWUDBAEuMBEEDE/go6xBMbsPDNgTsQIBEIB7E+VUre5YvyOJ8Apbi1SXMqNsr1rMp38is8iM5nP3LXHknTKIBZMgTeRilreIGQ25+UjWjBrTRUWmmD5jq6oN/d8t+0AEcLOpGZUplZSV+za31/3pAOPCFSqhUgzlUiI6LWW1kNKtmWwNh9aDEu4geiOAJSxJTAot6H77"
2323
SKIPPER_OPA_BUCKET_ARN: "deployment-secret:2:stups-test:AQICAHjXIrc66g/+P4X1Gl4MKcInWmwpFxivAqFGMI0fr9DvCwGZdCVDLsCdProfzvZU7UAwAAAAlzCBlAYJKoZIhvcNAQcGoIGGMIGDAgEAMH4GCSqGSIb3DQEHATAeBglghkgBZQMEAS4wEQQMehOf7Uu444SWS6kbAgEQgFFPMaa0flwHLpxrkYjJMK4jXc0q4kX+KGrB5GFjKuUgOUPmQ+ME/aQduxwl2+xUilrKP50/NLXgMNHjeeHuZfoyiSgpGFBM4z8L0N6ggf2uE5U="
2424
SKIPPER_OPA_OBSERVABILITY_URL: "deployment-secret:2:stups-test:AQICAHjXIrc66g/+P4X1Gl4MKcInWmwpFxivAqFGMI0fr9DvCwHl773AuNEvIpzaM6ycpDNSAAAAqzCBqAYJKoZIhvcNAQcGoIGaMIGXAgEAMIGRBgkqhkiG9w0BBwEwHgYJYIZIAWUDBAEuMBEEDGld6jpQ38gOzVcn0gIBEIBkTHbv3adeEfRntVTUQyyQkIhUnc0QXKtmtJEdvBoRzWiJIBKQUQuM1VBV0re3HkO8HSY59nkwyHEncBMkHJoI9rC2LJuWU20oCjPw9lbweih+6Sxo+nqkDrQd+mHp+uA9Om3KqA=="
2525
SKIPPER_OPA_BUNDLES_URL: "deployment-secret:2:stups-test:AQICAHjXIrc66g/+P4X1Gl4MKcInWmwpFxivAqFGMI0fr9DvCwFnhaIRP4+3Y69xp1ycTI7qAAAAsTCBrgYJKoZIhvcNAQcGoIGgMIGdAgEAMIGXBgkqhkiG9w0BBwEwHgYJYIZIAWUDBAEuMBEEDF9gAl70l2g2kwfnJgIBEIBqP/DgIhIu5x5XNR1Ubqinz6r4ttQoHty8nXd6mxie2r6NxHskNOqkiSactUKhNIhboNlNsO4p4rKEkhglTeFZlEQvgEYNioWPw39xqICnUDPVr+Kp0Yrs/bzPLPV9wOlB917UiT7WJNybPg=="

‎test/e2e/ingress.go

Lines changed: 0 additions & 126 deletions
Original file line numberDiff line numberDiff line change
@@ -1027,129 +1027,3 @@ var _______ = describe("Ingress tests simple NLB", func() {
10271027
Expect(resp.Header.Get("Request-Host")).To(Equal(hostName))
10281028
})
10291029
})
1030-
1031-
var ________ = describe("Ingress tests for OPA filters", func() {
1032-
f := framework.NewDefaultFramework("skipper-ingress-with-opa")
1033-
f.NamespacePodSecurityEnforceLevel = admissionapi.LevelBaseline
1034-
var (
1035-
cs kubernetes.Interface
1036-
jig *ingress.TestJig
1037-
)
1038-
1039-
It("Should activate OPA filter without issue [Ingress] [Zalando]", func() {
1040-
jig = ingress.NewIngressTestJig(f.ClientSet)
1041-
cs = f.ClientSet
1042-
serviceName := "styra-smoketest"
1043-
ns := f.Namespace.Name
1044-
hostName := fmt.Sprintf("%s-%d.%s", serviceName, time.Now().UTC().Unix(), E2EHostedZone())
1045-
labels := map[string]string{
1046-
"app": serviceName,
1047-
}
1048-
port := 8080
1049-
replicas := int32(3)
1050-
targetPort := 9090
1051-
backendContent := "mytest"
1052-
route := fmt.Sprintf(`* -> inlineContent("%s") -> <shunt>`, backendContent)
1053-
waitTime := 10 * time.Minute
1054-
1055-
// CREATE setup
1056-
// backend deployment
1057-
By("Creating a deployment with " + serviceName + " in namespace " + ns)
1058-
depl := createSkipperBackendDeployment(serviceName, ns, route, labels, int32(targetPort), replicas)
1059-
_, err := cs.AppsV1().Deployments(ns).Create(context.TODO(), depl, metav1.CreateOptions{})
1060-
framework.ExpectNoError(err)
1061-
1062-
By("Creating service " + serviceName + " in namespace " + ns)
1063-
service := createServiceTypeClusterIP(serviceName, labels, port, targetPort)
1064-
_, err = cs.CoreV1().Services(ns).Create(context.TODO(), service, metav1.CreateOptions{})
1065-
framework.ExpectNoError(err)
1066-
1067-
ing := createIngress(serviceName, hostName, ns, "/", netv1.PathTypeImplementationSpecific, labels, nil, port)
1068-
ingressCreate, err := cs.NetworkingV1().Ingresses(ns).Create(context.TODO(), ing, metav1.CreateOptions{})
1069-
framework.ExpectNoError(err)
1070-
1071-
addr, err := jig.WaitForIngressAddress(context.TODO(), cs, ns, ingressCreate.Name, waitTime)
1072-
framework.ExpectNoError(err)
1073-
1074-
_, err = cs.NetworkingV1().Ingresses(ns).Get(context.TODO(), ing.Name, metav1.GetOptions{ResourceVersion: "0"})
1075-
framework.ExpectNoError(err)
1076-
1077-
// skipper http -> https redirect
1078-
By("Waiting for skipper route to default redirect from http to https, to see that our ingress-controller and skipper works")
1079-
err = waitForResponse(addr, "http", waitTime, isRedirect, true)
1080-
framework.ExpectNoError(err)
1081-
1082-
// LB ready
1083-
By("Waiting for ALB to create endpoint " + addr + " and skipper route, to see that our ingress-controller and skipper works")
1084-
err = waitForResponse(addr, "https", waitTime, isNotFound, true)
1085-
framework.ExpectNoError(err)
1086-
1087-
// DNS ready
1088-
By("Waiting for DNS to see that external-dns and skipper route to service and pod works")
1089-
err = waitForResponse(hostName, "https", waitTime, isSuccess, false)
1090-
framework.ExpectNoError(err)
1091-
1092-
// Test that we get content from the default ingress
1093-
By("By checking the content of the reply we see that the ingress stack works")
1094-
rt, quit := createHTTPRoundTripper()
1095-
defer func() {
1096-
quit <- struct{}{}
1097-
}()
1098-
url := "https://" + hostName + "/"
1099-
req, err := http.NewRequest("GET", url, nil)
1100-
framework.ExpectNoError(err)
1101-
resp, err := rt.RoundTrip(req)
1102-
framework.ExpectNoError(err)
1103-
s, err := getBody(resp)
1104-
framework.ExpectNoError(err)
1105-
Expect(s).To(Equal(backendContent))
1106-
1107-
// Start actual ingress tests
1108-
// Test ingress Filters: opaAuthorizeRequest
1109-
1110-
/**
1111-
## The Rule looks like below.
1112-
## Reference https://github.bus.zalan.do/corporate-iam/styra-smoketest-policies/blob/main/bundle/policy/ingress/rules.rego
1113-
default allow := false
1114-
1115-
allow if {
1116-
input.attributes.request.http.method == "GET"
1117-
auth_header_val := input.attributes.request.http.headers.authorization
1118-
startswith(auth_header_val, "Basic ")
1119-
token := substring(auth_header_val, count("Basic "), -1)
1120-
token == "valid_token"
1121-
}
1122-
*/
1123-
path := "/"
1124-
opaPolicyName := "styra-smoketest"
1125-
updatedIng := updateIngress(ingressCreate.ObjectMeta.Name,
1126-
ingressCreate.ObjectMeta.Namespace,
1127-
hostName,
1128-
serviceName,
1129-
path,
1130-
netv1.PathTypeImplementationSpecific,
1131-
ingressCreate.ObjectMeta.Labels,
1132-
map[string]string{
1133-
"zalando.org/skipper-filter": fmt.Sprintf(`opaAuthorizeRequest("%s")`, opaPolicyName),
1134-
},
1135-
port,
1136-
)
1137-
ingressUpdate, err := cs.NetworkingV1().Ingresses(ingressCreate.ObjectMeta.Namespace).Update(context.TODO(), updatedIng, metav1.UpdateOptions{})
1138-
framework.ExpectNoError(err)
1139-
time.Sleep(20 * time.Second) // wait for routing change propagation
1140-
1141-
By(fmt.Sprintf("Calling ingress %s/%s we wait to get a 403 with opaAuthorizeRequest %s policy", ingressUpdate.Namespace, ingressUpdate.Name, opaPolicyName))
1142-
resp, err = getAndWaitResponse(rt, req, 10*time.Second, http.StatusForbidden)
1143-
framework.ExpectNoError(err)
1144-
Expect(resp.StatusCode).To(Equal(http.StatusForbidden))
1145-
1146-
By(fmt.Sprintf("Calling ingress %s/%s we wait to get a 200 with opaAuthorizeRequest %s policy", ingressUpdate.Namespace, ingressUpdate.Name, opaPolicyName))
1147-
req.Header.Set("Authorization", "Basic valid_token") //Authorized request
1148-
resp, err = getAndWaitResponse(rt, req, 10*time.Second, http.StatusOK)
1149-
framework.ExpectNoError(err)
1150-
Expect(resp.StatusCode).To(Equal(http.StatusOK))
1151-
s, err = getBody(resp)
1152-
framework.ExpectNoError(err)
1153-
Expect(s).To(Equal(backendContent))
1154-
})
1155-
})

‎test/e2e/run_e2e.sh

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -183,7 +183,7 @@ if [ "$e2e" = true ]; then
183183

184184
mkdir -p junit_reports
185185
ginkgo -procs=25 -flake-attempts=2 \
186-
-focus="(\[Conformance\]|\[StatefulSetBasic\]|\[Feature:StatefulSet\]\s\[Slow\].*mysql|\[Zalando\])" \
186+
-focus="(\[Conformance\]|\[StatefulSetBasic\]|\[Feature:StatefulSet\]\s\[Slow\].*mysql|\[Zalando\]|\[Opa\])" \
187187
-skip="(\[Serial\]|validates.that.there.is.no.conflict.between.pods.with.same.hostPort.but.different.hostIP.and.protocol|Should.create.gradual.traffic.routes)" \
188188
"e2e.test" -- \
189189
-delete-namespace-on-failure=false \

0 commit comments

Comments
 (0)