From adc616711e22bb9b51707b369e058d250c078a1f Mon Sep 17 00:00:00 2001 From: Ashish Tiwari Date: Wed, 30 Jul 2025 17:55:41 +0530 Subject: [PATCH 1/5] chore: add test cases for external service --- test/e2e/crds/v2/route.go | 298 +++++++++++++++++++++++++++++++++++++- 1 file changed, 294 insertions(+), 4 deletions(-) diff --git a/test/e2e/crds/v2/route.go b/test/e2e/crds/v2/route.go index 78d14636ce..47c94f511f 100644 --- a/test/e2e/crds/v2/route.go +++ b/test/e2e/crds/v2/route.go @@ -35,7 +35,7 @@ import ( "github.com/apache/apisix-ingress-controller/test/e2e/scaffold" ) -var _ = Describe("Test ApisixRoute", Label("apisix.apache.org", "v2", "apisixroute"), func() { +var _ = FDescribe("Test ApisixRoute", Label("apisix.apache.org", "v2", "apisixroute"), func() { var ( s = scaffold.NewScaffold(&scaffold.Options{ ControllerName: "apisix.apache.org/apisix-ingress-controller", @@ -56,7 +56,7 @@ var _ = Describe("Test ApisixRoute", Label("apisix.apache.org", "v2", "apisixrou time.Sleep(5 * time.Second) }) - Context("Test ApisixRoute", func() { + PContext("Test ApisixRoute", func() { It("Basic tests", func() { const apisixRouteSpec = ` @@ -441,7 +441,7 @@ spec: }) }) - Context("Test ApisixRoute reference ApisixUpstream", func() { + PContext("Test ApisixRoute reference ApisixUpstream", func() { It("Test reference ApisixUpstream", func() { const apisixRouteSpec = ` apiVersion: apisix.apache.org/v2 @@ -651,7 +651,7 @@ spec: }) }) - Context("Test ApisixRoute sync during startup", func() { + PContext("Test ApisixRoute sync during startup", func() { const route = ` apiVersion: apisix.apache.org/v2 kind: ApisixRoute @@ -757,4 +757,294 @@ spec: }) }) }) + + Context("Test ApisixRoute with External Services", func() { + const ( + externalServiceName = "ext-httpbin" + upstreamName = "httpbin-upstream" + routeName = "httpbin-route" + ) + + createExternalService := func(externalName string) { + By(fmt.Sprintf("create ExternalName service: %s -> %s", externalServiceName, externalName)) + svcSpec := fmt.Sprintf(` +apiVersion: v1 +kind: Service +metadata: + name: %s +spec: + type: ExternalName + externalName: %s +`, externalServiceName, externalName) + err := s.CreateResourceFromString(svcSpec) + Expect(err).ShouldNot(HaveOccurred(), "creating ExternalName service") + } + + createApisixUpstream := func(externalType apiv2.ApisixUpstreamExternalType, name string) { + By(fmt.Sprintf("create ApisixUpstream: type=%s, name=%s", externalType, name)) + upstreamSpec := fmt.Sprintf(` +apiVersion: apisix.apache.org/v2 +kind: ApisixUpstream +metadata: + name: %s +spec: + externalNodes: + - type: %s + name: %s +`, upstreamName, externalType, name) + var upstream apiv2.ApisixUpstream + applier.MustApplyAPIv2( + types.NamespacedName{Namespace: s.Namespace(), Name: upstreamName}, + &upstream, + upstreamSpec, + ) + } + + createApisixRoute := func() { + By("create ApisixRoute referencing ApisixUpstream") + routeSpec := fmt.Sprintf(` +apiVersion: apisix.apache.org/v2 +kind: ApisixRoute +metadata: + name: %s +spec: + ingressClassName: apisix + http: + - name: rule1 + match: + hosts: + - httpbin.org + paths: + - /ip + upstreams: + - name: %s +`, routeName, upstreamName) + var route apiv2.ApisixRoute + applier.MustApplyAPIv2( + types.NamespacedName{Namespace: s.Namespace(), Name: routeName}, + &route, + routeSpec, + ) + } + + createApisixRouteWithHostRewrite := func(host string) { + By("create ApisixRoute with host rewrite") + routeSpec := fmt.Sprintf(` +apiVersion: apisix.apache.org/v2 +kind: ApisixRoute +metadata: + name: %s +spec: + ingressClassName: apisix + http: + - name: rule1 + match: + hosts: + - httpbin.org + paths: + - /ip + upstreams: + - name: %s + plugins: + - name: proxy-rewrite + enable: true + config: + host: %s +`, routeName, upstreamName, host) + var route apiv2.ApisixRoute + applier.MustApplyAPIv2( + types.NamespacedName{Namespace: s.Namespace(), Name: routeName}, + &route, + routeSpec, + ) + } + + verifyAccess := func() { + By("verify access to external service") + request := func() int { + return s.NewAPISIXClient().GET("/ip"). + WithHost("httpbin.org"). + Expect().Raw().StatusCode + } + Eventually(request).WithTimeout(30 * time.Second).ProbeEvery(2 * time.Second). + Should(Equal(http.StatusOK)) + } + + It("access third-party service directly", func() { + createApisixUpstream(apiv2.ExternalTypeDomain, "httpbin.org") + createApisixRoute() + verifyAccess() + }) + + It("access third-party service with host rewrite", func() { + createApisixUpstream(apiv2.ExternalTypeDomain, "httpbin.org") + createApisixRouteWithHostRewrite("httpbin.org") + verifyAccess() + }) + + It("access external domain via ExternalName service", func() { + createExternalService("httpbin.org") + createApisixUpstream(apiv2.ExternalTypeService, externalServiceName) + createApisixRoute() + verifyAccess() + }) + + It("access in-cluster service via ExternalName", func() { + By("create temporary httpbin service") + + By("get FQDN of temporary service") + fqdn := fmt.Sprintf("%s.%s.svc.cluster.local", "httpbin-service-e2e-test", s.Namespace()) + + By("setup external service and route") + createExternalService(fqdn) + createApisixUpstream(apiv2.ExternalTypeService, externalServiceName) + createApisixRoute() + verifyAccess() + }) + + Context("complex scenarios", func() { + It("multiple external services in one upstream", func() { + By("create ApisixUpstream with multiple external nodes") + upstreamSpec := ` +apiVersion: apisix.apache.org/v2 +kind: ApisixUpstream +metadata: + name: httpbin-upstream +spec: + externalNodes: + - type: Domain + name: httpbin.org + - type: Domain + name: postman-echo.com +` + var upstream apiv2.ApisixUpstream + applier.MustApplyAPIv2( + types.NamespacedName{Namespace: s.Namespace(), Name: upstreamName}, + &upstream, + upstreamSpec, + ) + + createApisixRouteWithHostRewrite("postman-echo.com") + + By("verify access to multiple services") + request := func() int { + return s.NewAPISIXClient().GET("/get"). + WithHost("httpbin.org"). + Expect().Raw().StatusCode + } + Eventually(request).WithTimeout(30 * time.Second).ProbeEvery(2 * time.Second). + Should(Equal(http.StatusOK)) + }) + + It("mix of backends and upstreams", func() { + By("create in-cluster httpbin service") + By("create ApisixUpstream for external service") + createApisixUpstream(apiv2.ExternalTypeDomain, "postman-echo.com") + + By("create ApisixRoute with both backends and upstreams") + routeSpec := fmt.Sprintf(` +apiVersion: apisix.apache.org/v2 +kind: ApisixRoute +metadata: + name: %s +spec: + ingressClassName: apisix + http: + - name: rule1 + match: + hosts: + - httpbin.org + paths: + - /get + backends: + - serviceName: %s + servicePort: 80 + resolveGranularity: service + upstreams: + - name: %s + plugins: + - name: proxy-rewrite + enable: true + config: + host: postman-echo.com +`, routeName, "httpbin-service-e2e-test", upstreamName) + var route apiv2.ApisixRoute + applier.MustApplyAPIv2( + types.NamespacedName{Namespace: s.Namespace(), Name: routeName}, + &route, + routeSpec, + ) + + By("verify traffic split between backends and upstreams") + hasEtag := false // postman-echo.com + hasNoEtag := false // httpbin.org + for i := 0; i < 20; i++ { + headers := s.NewAPISIXClient().GET("/get"). + WithHost("httpbin.org"). + Expect().Status(http.StatusOK). + Headers().Raw() + if _, ok := headers["Etag"]; ok { + hasEtag = true + } else { + hasNoEtag = true + } + if hasEtag && hasNoEtag { + break + } + } + Expect(hasEtag).To(BeTrue(), "should get responses from postman-echo.com") + Expect(hasNoEtag).To(BeTrue(), "should get responses from in-cluster httpbin") + }) + }) + + // Context("update scenarios", func() { + // It("create ApisixUpstream after route", func() { + // createApisixRoute() + + // By("verify no upstream initially") + // time.Sleep(5 * time.Second) + // ups, err := s.ListApisixUpstreams() + // Expect(err).ShouldNot(HaveOccurred()) + // Expect(ups).Should(BeEmpty(), "upstream count before creation") + + // createApisixUpstream(apiv2.ExternalTypeDomain, "httpbin.org") + // verifyAccess() + // }) + + // It("create ExternalName service after upstream", func() { + // createApisixUpstream(apiv2.ExternalTypeService, externalServiceName) + // createApisixRoute() + + // By("verify no upstream initially") + // time.Sleep(5 * time.Second) + // ups, err := s.ListApisixUpstreams() + // Expect(err).ShouldNot(HaveOccurred()) + // Expect(ups).Should(BeEmpty(), "upstream count before service creation") + + // createExternalService("httpbin.org") + // verifyAccess() + // }) + + // It("update ApisixUpstream to point to different service", func() { + // createExternalService("unknown.org") + // createApisixUpstream(apiv2.ExternalTypeService, externalServiceName) + // createApisixRoute() + + // By("update ApisixUpstream to point to valid service") + // createApisixUpstream(apiv2.ExternalTypeService, externalServiceName) + // createExternalService("httpbin.org") + // verifyAccess() + // }) + + // It("update ExternalName service", func() { + // createExternalService("unknown.org") + // createApisixUpstream(apiv2.ExternalTypeService, externalServiceName) + // createApisixRoute() + + // By("update ExternalName service to valid domain") + // createExternalService("httpbin.org") + // verifyAccess() + // }) + // }) + }) }) From 7a41b275ad8aed3263490dde09e4a44d94d4e98b Mon Sep 17 00:00:00 2001 From: Ashish Tiwari Date: Wed, 30 Jul 2025 19:34:51 +0530 Subject: [PATCH 2/5] fix --- test/e2e/crds/v2/route.go | 135 +++++++++++++++----------------------- 1 file changed, 53 insertions(+), 82 deletions(-) diff --git a/test/e2e/crds/v2/route.go b/test/e2e/crds/v2/route.go index 47c94f511f..3fc3509d99 100644 --- a/test/e2e/crds/v2/route.go +++ b/test/e2e/crds/v2/route.go @@ -25,8 +25,10 @@ import ( "net/http" "time" + "github.com/onsi/ginkgo/v2" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + "github.com/stretchr/testify/assert" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/wait" @@ -35,7 +37,7 @@ import ( "github.com/apache/apisix-ingress-controller/test/e2e/scaffold" ) -var _ = FDescribe("Test ApisixRoute", Label("apisix.apache.org", "v2", "apisixroute"), func() { +var _ = Describe("Test ApisixRoute", Label("apisix.apache.org", "v2", "apisixroute"), func() { var ( s = scaffold.NewScaffold(&scaffold.Options{ ControllerName: "apisix.apache.org/apisix-ingress-controller", @@ -56,7 +58,7 @@ var _ = FDescribe("Test ApisixRoute", Label("apisix.apache.org", "v2", "apisixro time.Sleep(5 * time.Second) }) - PContext("Test ApisixRoute", func() { + Context("Test ApisixRoute", func() { It("Basic tests", func() { const apisixRouteSpec = ` @@ -441,7 +443,7 @@ spec: }) }) - PContext("Test ApisixRoute reference ApisixUpstream", func() { + Context("Test ApisixRoute reference ApisixUpstream", func() { It("Test reference ApisixUpstream", func() { const apisixRouteSpec = ` apiVersion: apisix.apache.org/v2 @@ -651,7 +653,7 @@ spec: }) }) - PContext("Test ApisixRoute sync during startup", func() { + Context("Test ApisixRoute sync during startup", func() { const route = ` apiVersion: apisix.apache.org/v2 kind: ApisixRoute @@ -924,23 +926,47 @@ spec: upstreamSpec, ) - createApisixRouteWithHostRewrite("postman-echo.com") + createApisixRoute() By("verify access to multiple services") - request := func() int { - return s.NewAPISIXClient().GET("/get"). - WithHost("httpbin.org"). - Expect().Raw().StatusCode + time.Sleep(7 * time.Second) + hasEtag := false // postman-echo.com + hasNoEtag := false // httpbin.org + for range 20 { + headers := s.NewAPISIXClient().GET("/ip"). + WithHeader("Host", "httpbin.org"). + WithHeader("X-Foo", "bar"). + Expect(). + Headers().Raw() + if _, ok := headers["Etag"]; ok { + hasEtag = true + } else { + hasNoEtag = true + } + if hasEtag && hasNoEtag { + break + } } - Eventually(request).WithTimeout(30 * time.Second).ProbeEvery(2 * time.Second). - Should(Equal(http.StatusOK)) + assert.True(ginkgo.GinkgoT(), hasEtag && hasNoEtag, "both httpbin and postman should be accessed at least once") }) - It("mix of backends and upstreams", func() { - By("create in-cluster httpbin service") - By("create ApisixUpstream for external service") - createApisixUpstream(apiv2.ExternalTypeDomain, "postman-echo.com") - + It("should be able to use backends and upstreams together", func() { + upstreamSpec := ` +apiVersion: apisix.apache.org/v2 +kind: ApisixUpstream +metadata: + name: httpbin-upstream +spec: + externalNodes: + - type: Domain + name: postman-echo.com +` + var upstream apiv2.ApisixUpstream + applier.MustApplyAPIv2( + types.NamespacedName{Namespace: s.Namespace(), Name: upstreamName}, + &upstream, + upstreamSpec, + ) By("create ApisixRoute with both backends and upstreams") routeSpec := fmt.Sprintf(` apiVersion: apisix.apache.org/v2 @@ -955,33 +981,29 @@ spec: hosts: - httpbin.org paths: - - /get + - /ip backends: - - serviceName: %s + - serviceName: httpbin-service-e2e-test servicePort: 80 resolveGranularity: service upstreams: - name: %s - plugins: - - name: proxy-rewrite - enable: true - config: - host: postman-echo.com -`, routeName, "httpbin-service-e2e-test", upstreamName) +`, routeName, upstreamName) var route apiv2.ApisixRoute applier.MustApplyAPIv2( types.NamespacedName{Namespace: s.Namespace(), Name: routeName}, &route, routeSpec, ) - - By("verify traffic split between backends and upstreams") + By("verify access to multiple services") + time.Sleep(7 * time.Second) hasEtag := false // postman-echo.com hasNoEtag := false // httpbin.org - for i := 0; i < 20; i++ { - headers := s.NewAPISIXClient().GET("/get"). - WithHost("httpbin.org"). - Expect().Status(http.StatusOK). + for range 20 { + headers := s.NewAPISIXClient().GET("/ip"). + WithHeader("Host", "httpbin.org"). + WithHeader("X-Foo", "bar"). + Expect(). Headers().Raw() if _, ok := headers["Etag"]; ok { hasEtag = true @@ -992,59 +1014,8 @@ spec: break } } - Expect(hasEtag).To(BeTrue(), "should get responses from postman-echo.com") - Expect(hasNoEtag).To(BeTrue(), "should get responses from in-cluster httpbin") + assert.True(ginkgo.GinkgoT(), hasEtag && hasNoEtag, "both httpbin and postman should be accessed at least once") }) }) - - // Context("update scenarios", func() { - // It("create ApisixUpstream after route", func() { - // createApisixRoute() - - // By("verify no upstream initially") - // time.Sleep(5 * time.Second) - // ups, err := s.ListApisixUpstreams() - // Expect(err).ShouldNot(HaveOccurred()) - // Expect(ups).Should(BeEmpty(), "upstream count before creation") - - // createApisixUpstream(apiv2.ExternalTypeDomain, "httpbin.org") - // verifyAccess() - // }) - - // It("create ExternalName service after upstream", func() { - // createApisixUpstream(apiv2.ExternalTypeService, externalServiceName) - // createApisixRoute() - - // By("verify no upstream initially") - // time.Sleep(5 * time.Second) - // ups, err := s.ListApisixUpstreams() - // Expect(err).ShouldNot(HaveOccurred()) - // Expect(ups).Should(BeEmpty(), "upstream count before service creation") - - // createExternalService("httpbin.org") - // verifyAccess() - // }) - - // It("update ApisixUpstream to point to different service", func() { - // createExternalService("unknown.org") - // createApisixUpstream(apiv2.ExternalTypeService, externalServiceName) - // createApisixRoute() - - // By("update ApisixUpstream to point to valid service") - // createApisixUpstream(apiv2.ExternalTypeService, externalServiceName) - // createExternalService("httpbin.org") - // verifyAccess() - // }) - - // It("update ExternalName service", func() { - // createExternalService("unknown.org") - // createApisixUpstream(apiv2.ExternalTypeService, externalServiceName) - // createApisixRoute() - - // By("update ExternalName service to valid domain") - // createExternalService("httpbin.org") - // verifyAccess() - // }) - // }) }) }) From e32b08f72b9d154ca16db248e4213e0605598eda Mon Sep 17 00:00:00 2001 From: Ashish Tiwari Date: Wed, 30 Jul 2025 19:38:29 +0530 Subject: [PATCH 3/5] readd --- test/e2e/crds/v2/route.go | 150 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 150 insertions(+) diff --git a/test/e2e/crds/v2/route.go b/test/e2e/crds/v2/route.go index 1c839fa63f..fbabc68ea3 100644 --- a/test/e2e/crds/v2/route.go +++ b/test/e2e/crds/v2/route.go @@ -21,6 +21,7 @@ import ( "context" "fmt" "io" + "math" "net" "net/http" "net/url" @@ -655,6 +656,155 @@ spec: }) }) + Context("Test ApisixRoute Traffic Split", func() { + It("2:1 traffic split test", func() { + const apisixRouteSpec = ` +apiVersion: apisix.apache.org/v2 +kind: ApisixRoute +metadata: + name: default +spec: + ingressClassName: apisix + http: + - name: rule1 + match: + hosts: + - httpbin.org + paths: + - /get + backends: + - serviceName: httpbin-service-e2e-test + servicePort: 80 + weight: 10 + - serviceName: %s + servicePort: 9180 + weight: 5 +` + By("apply ApisixRoute with traffic split") + applier.MustApplyAPIv2(types.NamespacedName{Namespace: s.Namespace(), Name: "default"}, new(apiv2.ApisixRoute), + fmt.Sprintf(apisixRouteSpec, s.Deployer.GetAdminServiceName())) + verifyRequest := func() int { + return s.NewAPISIXClient().GET("/get").WithHost("httpbin.org").Expect().Raw().StatusCode + } + By("send requests to verify traffic split") + var ( + successCount int + failCount int + ) + + s.RequestAssert(&scaffold.RequestAssert{ + Method: "GET", + Path: "/get", + Host: "httpbin.org", + Check: scaffold.WithExpectedStatus(http.StatusOK), + Timeout: 10 * time.Second, + }) + for range 90 { + code := verifyRequest() + if code == http.StatusOK { + successCount++ + } else { + failCount++ + } + } + + By("verify traffic distribution ratio") + ratio := float64(successCount) / float64(failCount) + expectedRatio := 10.0 / 5.0 // 2:1 ratio + deviation := math.Abs(ratio - expectedRatio) + Expect(deviation).Should(BeNumerically("<", 0.5), + "traffic distribution deviation too large (got %.2f, expected %.2f)", ratio, expectedRatio) + }) + + It("zero-weight test", func() { + const apisixRouteSpec = ` +apiVersion: apisix.apache.org/v2 +kind: ApisixRoute +metadata: + name: default +spec: + ingressClassName: apisix + http: + - name: rule1 + match: + hosts: + - httpbin.org + paths: + - /get + backends: + - serviceName: httpbin-service-e2e-test + servicePort: 80 + weight: 10 + - serviceName: %s + servicePort: 9180 + weight: 0 +` + By("apply ApisixRoute with zero-weight backend") + applier.MustApplyAPIv2(types.NamespacedName{Namespace: s.Namespace(), Name: "default"}, new(apiv2.ApisixRoute), + fmt.Sprintf(apisixRouteSpec, s.Deployer.GetAdminServiceName())) + verifyRequest := func() int { + return s.NewAPISIXClient().GET("/get").WithHost("httpbin.org").Expect().Raw().StatusCode + } + + By("wait for route to be ready") + s.RequestAssert(&scaffold.RequestAssert{ + Method: "GET", + Path: "/get", + Host: "httpbin.org", + Check: scaffold.WithExpectedStatus(http.StatusOK), + Timeout: 10 * time.Second, + }) + By("send requests to verify zero-weight behavior") + for range 30 { + code := verifyRequest() + Expect(code).Should(Equal(200)) + } + }) + It("valid backend is set even if other backend is invalid", func() { + const apisixRouteSpec = ` +apiVersion: apisix.apache.org/v2 +kind: ApisixRoute +metadata: + name: default +spec: + ingressClassName: apisix + http: + - name: rule1 + match: + hosts: + - httpbin.org + paths: + - /get + backends: + - serviceName: httpbin-service-e2e-test + servicePort: 80 + weight: 10 + - serviceName: invalid-service + servicePort: 9180 + weight: 5 +` + By("apply ApisixRoute with traffic split") + applier.MustApplyAPIv2(types.NamespacedName{Namespace: s.Namespace(), Name: "default"}, new(apiv2.ApisixRoute), apisixRouteSpec) + verifyRequest := func() int { + return s.NewAPISIXClient().GET("/get").WithHost("httpbin.org").Expect().Raw().StatusCode + } + + By("wait for route to be ready") + s.RequestAssert(&scaffold.RequestAssert{ + Method: "GET", + Path: "/get", + Host: "httpbin.org", + Check: scaffold.WithExpectedStatus(http.StatusOK), + Timeout: 10 * time.Second, + }) + By("send requests to verify all requests routed to valid upstream") + for range 30 { + code := verifyRequest() + Expect(code).Should(Equal(200)) + } + }) + }) + Context("Test ApisixRoute sync during startup", func() { const route = ` apiVersion: apisix.apache.org/v2 From d6d5e851777f23c7b7494e7f15349084cd24a0c9 Mon Sep 17 00:00:00 2001 From: Ashish Tiwari Date: Wed, 30 Jul 2025 19:44:42 +0530 Subject: [PATCH 4/5] fix lint --- test/e2e/crds/v2/route.go | 1 - 1 file changed, 1 deletion(-) diff --git a/test/e2e/crds/v2/route.go b/test/e2e/crds/v2/route.go index fbabc68ea3..964d8f8b0b 100644 --- a/test/e2e/crds/v2/route.go +++ b/test/e2e/crds/v2/route.go @@ -28,7 +28,6 @@ import ( "time" "github.com/gorilla/websocket" - "github.com/onsi/ginkgo/v2" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" "github.com/stretchr/testify/assert" From 7720b0344d1d7fac247e125b567c448a9cfa3dbf Mon Sep 17 00:00:00 2001 From: Ashish Tiwari Date: Wed, 30 Jul 2025 19:45:26 +0530 Subject: [PATCH 5/5] fix --- test/e2e/crds/v2/route.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/e2e/crds/v2/route.go b/test/e2e/crds/v2/route.go index 964d8f8b0b..fdc1cd5ad3 100644 --- a/test/e2e/crds/v2/route.go +++ b/test/e2e/crds/v2/route.go @@ -1231,7 +1231,7 @@ spec: break } } - assert.True(ginkgo.GinkgoT(), hasEtag && hasNoEtag, "both httpbin and postman should be accessed at least once") + assert.True(GinkgoT(), hasEtag && hasNoEtag, "both httpbin and postman should be accessed at least once") }) It("should be able to use backends and upstreams together", func() { @@ -1298,7 +1298,7 @@ spec: break } } - assert.True(ginkgo.GinkgoT(), hasEtag && hasNoEtag, "both httpbin and postman should be accessed at least once") + assert.True(GinkgoT(), hasEtag && hasNoEtag, "both httpbin and postman should be accessed at least once") }) }) })