Merge pull request #250 from TykTechnologies/release-5.3.0

Release 5.3.0

Author: davegarvey

.github/workflows/tyk-demo-tests.yml

@@ -11,7 +11,7 @@ jobs:
       - run: echo "🐧 This job is now running on a ${{ runner.os }} server hosted by GitHub!"
       - run: echo "🔎 The name of your branch is ${{ github.ref }} and your repository is ${{ github.repository }}."
       - name: Check Out Repository Code
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
       - run: echo "💡 The ${{ github.repository }} repository has been cloned to the runner."
       - name: Create Environment File
         run: |
@@ -22,15 +22,15 @@ jobs:
       - name: Bootstrap and Test All Eligible Deployments
         run: sudo ./scripts/test-all.sh
       - name: Store Logs
-        if: success() || failure()
-        uses: actions/upload-artifact@v3
+        if: always()
+        uses: actions/upload-artifact@v4
         with:
           name: logs
           path: logs/
       - name: Show Bootstrap Log
-        if: success() || failure()
+        if: always()
         run: cat logs/bootstrap.log
       - name: Show Test Log
-        if: success() || failure()
+        if: always()
         run: cat logs/test.log
       - run: echo "🍏 This job's status is ${{ job.status }}."

CONTRIBUTING.md

@@ -137,7 +137,7 @@ These utility scripts are available in the `scripts` directory:
 * `add-gateway.sh`: Creates a new Tyk Gateway container, using the same configuration as the base Tyk deployment Gateway
 * `common.sh`: Contains functions useful for bootstrap scripts
 * `export.sh`: Uses the Dashboard API to export API and Policy definitions, overwriting data used to bootstrap the base Tyk deployment
-* `test.sh`: Uses a Newman container to run the Postman collection tests for only the **Tyk** deployment
+* `test.sh`: Uses a Newman container to run the Postman collection tests for all deployment that are currently bootstrapped
 * `test-all.sh`: As `test.sh`, but runs for all deployments
 * `update-hosts.sh`: Adds the necessary hosts to the `/etc/hosts` file
 
@@ -299,6 +299,14 @@ Once the tests are finished, the `delete` function can be called to remove the k
 tyk.dashboardApi.tools.apiKey.delete(pm);
 ```
 
+## Test Automation
+
+Deployments are included in test scripts (`scripts/test.sh` and `scripts/test-all.sh`) if all of the following criteria are met:
+
+- A Postman collection is found in the deployment
+- The collection does *not* contain a variable `test-runner-ignore` with the value `true`
+- The collection contains tests
+
 ## Testing Responses
 
 The Tyk Demo Postman collection contains many requests, each of which demonstrate a particular piece of functionality. Testing the responses generated by these requests provides validation that the desired result was achieved.

deployments/analytics-kibana/bootstrap.sh

@@ -6,6 +6,8 @@ log_start_deployment
 bootstrap_progress
 
 kibana_base_url="http://localhost:5601"
+gateway_base_url="http://$(jq -r '.host_config.override_hostname' deployments/tyk/volumes/tyk-dashboard/tyk_analytics.conf)"
+gateway_api_credentials=$(cat deployments/tyk/volumes/tyk-gateway/tyk.conf | jq -r .secret)
 
 log_message "Waiting for kibana to return desired response"
 wait_for_response "$kibana_base_url/app/kibana" "200"
@@ -45,6 +47,12 @@ fi
 log_ok
 bootstrap_progress
 
+log_message "Waiting for API availability"
+# this api id is for the 'basic open api' called by the next section
+wait_for_api_loaded "727dad853a8a45f64ab981154d1ffdad" "$gateway_base_url" "$gateway_api_credentials"
+log_ok
+bootstrap_progress        
+
 log_message "Sending a test request to provide Kibana with data, as Tyk bootstrap requests will not have been picked up by the Pump from this deployment"
 log_http_result "$(curl -s localhost:8080/basic-open-api/get -o /dev/null -w "%{http_code}" 2>> logs/bootstrap.log)"
 bootstrap_progress

deployments/analytics-splunk/bootstrap.sh

@@ -7,6 +7,8 @@ bootstrap_progress
 
 splunk_base_url="http://localhost:8000"
 splunk_base_mgmt_url="https://localhost:8089"
+gateway_base_url="http://$(jq -r '.host_config.override_hostname' deployments/tyk/volumes/tyk-dashboard/tyk_analytics.conf)"
+gateway_api_credentials=$(cat deployments/tyk/volumes/tyk-gateway/tyk.conf | jq -r .secret)
 
 log_message "Waiting for splunk to return desired response"
 wait_for_response "$splunk_base_url/en-GB/account/login" "200"
@@ -52,6 +54,14 @@ fi
 log_ok
 bootstrap_progress
 
+log_message "Waiting for API availability"
+# this api id is for the 'basic open api' called by the next section
+wait_for_api_loaded "727dad853a8a45f64ab981154d1ffdad" "$gateway_base_url" "$gateway_api_credentials"
+# this api id is for the 'httpbin acme' API called by the deployment tests
+wait_for_api_loaded "93fd5c15961041115974216e7a3e7175" "$gateway_base_url" "$gateway_api_credentials"
+log_ok
+bootstrap_progress        
+
 log_message "Sending a test request to provide Splunk with data"
 # since request sent in base bootstrap process will not have been picked up by elasticsearch-enabled pump
 log_http_result "$(curl -s localhost:8080/basic-open-api/get -o /dev/null -w "%{http_code}" 2>> logs/bootstrap.log)"

deployments/federation/bootstrap.sh

@@ -1,12 +1,13 @@
 #!/bin/bash
 
 source scripts/common.sh
-deployment="GraphQL Federations"
+deployment="GraphQL Federation"
 log_start_deployment
 
 # Setup variables
 dashboard_base_url="http://tyk-dashboard.localhost:$(jq -r '.listen_port' deployments/tyk/volumes/tyk-dashboard/tyk_analytics.conf)"
 gateway_base_url="http://$(jq -r '.host_config.override_hostname' deployments/tyk/volumes/tyk-dashboard/tyk_analytics.conf)"
+gateway_api_credentials=$(cat deployments/tyk/volumes/tyk-gateway/tyk.conf | jq -r .secret)
 dashboard_admin_api_credentials=$(cat deployments/tyk/volumes/tyk-dashboard/tyk_analytics.conf | jq -r .admin_secret)
 dashboard_user_api_key=$(get_context_data "1" "dashboard-user" "1" "api-key")
 
@@ -23,6 +24,17 @@ bootstrap_progress
 create_api "deployments/federation/data/apis-supergraph.json" "$dashboard_admin_api_credentials" "$dashboard_user_api_key"
 bootstrap_progress
 
+log_message "Waiting for API availability"
+for file in deployments/federation/data/*; do
+  if [[ -f $file ]]; then
+    target_api_id=$(cat $file | jq '.api_definition.api_id' --raw-output)
+    wait_for_api_loaded "$target_api_id" "$gateway_base_url" "$gateway_api_credentials"
+    bootstrap_progress        
+  fi
+done
+bootstrap_progress        
+log_ok
+
 log_end_deployment
 
 echo -e "\033[2K

deployments/keycloak-dcr/README.md

@@ -13,7 +13,7 @@ Run `keycloak-dcr` as one of your deployment configs, ie
 ./up.sh keycloak-dcr
 ```
 
-Visit `http://localhost:8180/auth/admin/` to access the keycloak admin console, and login with the credentials `admin/admin`
+Visit `http://keycloak:8180/` to access the keycloak admin console, and login with the credentials `admin/admin`
 
 Under the Client Registration tab you should find that an `initial access` token has been automagically created during the deployment bootstrap.
 

deployments/keycloak-dcr/bootstrap.sh

@@ -10,14 +10,13 @@ dashboard_admin_api_credentials=$(cat deployments/tyk/volumes/tyk-dashboard/tyk_
 dashboard_user_api_key=$(get_context_data "1" "dashboard-user" "1" "api-key")
 gateway_base_url="http://tyk-gateway.localhost:8080"
 gateway_api_credentials=$(cat deployments/tyk/volumes/tyk-gateway/tyk.conf | jq -r .secret)
-keycloak_base_url="http://keycloak:8180/auth"
-keycloak_admin_url="http://keycloak:8180/auth/admin"
+keycloak_base_url="http://keycloak:8180"
 
 log_message "Waiting for Dashboard API to be ready"
 wait_for_response "$dashboard_base_url/admin/organisations" "200" "admin-auth: $dashboard_admin_api_credentials"
 
 log_message "Waiting for Keycloak to respond ok"
-wait_for_response "$keycloak_base_url/" "200"
+wait_for_response "$keycloak_base_url/health/ready" "200"
 
 
 log_message "Obtaining keycloak user access token"
@@ -34,7 +33,7 @@ bootstrap_progress
 
 
 log_message "Creating a new initial access token"
-api_response="$(curl $keycloak_admin_url/realms/master/clients-initial-access -s \
+api_response="$(curl $keycloak_base_url/admin/realms/master/clients-initial-access -s \
   -H "Content-Type: application/json" \
   -H "Authorization: Bearer $access_token" \
   -d '{"count": 5}')"
@@ -107,7 +106,7 @@ bootstrap_progress
 log_end_deployment
 
 echo -e "\033[2K
-▼ IdP
+▼ Dynamic Client Registration
   ▽ Keycloak
-            Browser URL : $keycloak_admin_url
-                  Login : admin/admin"
+            Browser URL : $keycloak_base_url
+      Username/Password : admin/admin"

deployments/keycloak-dcr/data/tyk-dashboard/apis.json

@@ -177,7 +177,7 @@
             "use_go_plugin_auth": false,
             "enable_coprocess_auth": false,
             "jwt_signing_method": "rsa",
-            "jwt_source": "aHR0cDovL2tleWNsb2FrOjgxODAvYXV0aC9yZWFsbXMvbWFzdGVyL3Byb3RvY29sL29wZW5pZC1jb25uZWN0L2NlcnRz",
+            "jwt_source": "aHR0cDovL2tleWNsb2FrOjgxODAvcmVhbG1zL21hc3Rlci9wcm90b2NvbC9vcGVuaWQtY29ubmVjdC9jZXJ0cw==",
             "jwt_identity_base_field": "sub",
             "jwt_client_base_field": "",
             "jwt_policy_field_name": "pol",
@@ -258,7 +258,7 @@
             "proxy": {
                 "preserve_host_header": false,
                 "listen_path": "/keycloak-dcr-api/",
-                "target_url": "http://httpbin.org",
+                "target_url": "http://httpbin",
                 "disable_strip_slash": true,
                 "strip_listen_path": true,
                 "enable_load_balancing": false,

deployments/keycloak-dcr/data/tyk-dashboard/catalog.json

@@ -54,7 +54,7 @@
         "dcr_options": {
             "idp_host": "http://keycloak:8180/",
             "access_token": "ACCESS_TOKEN",
-            "registration_endpoint": "http://keycloak:8180/auth/realms/master/clients-registrations/openid-connect",
+            "registration_endpoint": "http://keycloak:8180/realms/master/clients-registrations/openid-connect",
             "provider": "keycloak",
             "grant_types": [
                 "client_credentials"

deployments/keycloak-dcr/docker-compose.yml

@@ -1,5 +1,5 @@
 ---
-version: '3.3'
+version: "3.3"
 services:
   keycloak-db:
     image: postgres:9.6
@@ -10,11 +10,15 @@ services:
     networks:
       - tyk
     ports:
-      - "25432:5432"
+      - 25432:5432
     volumes:
       - keycloak-data:/var/lib/postresql/data
     healthcheck:
-      test: [ "CMD", "pg_isready", "-U", "keycloak" ]
+      test:
+        - CMD
+        - pg_isready
+        - -U
+        - keycloak
       interval: 30s
       timeout: 30s
       retries: 3
@@ -23,23 +27,25 @@ services:
       restart_policy:
         condition: on-failure
   keycloak:
-    image: jboss/keycloak:11.0.2
+    image: keycloak/keycloak:24.0.1
     depends_on:
       - keycloak-db
-    command: [ "-Djboss.http.port=8180" ]
     networks:
       - tyk
     ports:
-      - "8180:8180"
+      - 8180:8180
+    command:
+      - start-dev
     environment:
-      DB_VENDOR: POSTGRES
-      DB_ADDR: keycloak-db
-      DB_PORT: 5432
-      DB_DATABASE: keycloak
-      DB_USER: keycloak
-      DB_PASSWORD: keycloak
-      KEYCLOAK_USER: admin
-      KEYCLOAK_PASSWORD: admin
-
+      KC_DB: postgres
+      KC_DB_URL_HOST: keycloak-db
+      KC_DB_URL_PORT: 5432
+      KC_DB_URL_DATABASE: keycloak
+      KC_DB_USERNAME: keycloak
+      KC_DB_PASSWORD: keycloak
+      KEYCLOAK_ADMIN: admin
+      KEYCLOAK_ADMIN_PASSWORD: admin
+      KC_HEALTH_ENABLED: true
+      KC_HTTP_PORT: 8180
 volumes:
-  keycloak-data:
+  keycloak-data: null