Add spring component

Add improve a few thigns around capturing logs in ci jobs
among others things

Author: MarcialRosales

.github/workflows/test-authnz.yaml

@@ -74,20 +74,22 @@ jobs:
     - name: Run Suites
       id: tests
       run: |        
-        IMAGE_TAG=$(find PACKAGES/rabbitmq-server-generic-unix-*.tar.xz | awk -F 'PACKAGES/rabbitmq-server-generic-unix-|.tar.xz' '{print $2}')
-        CONF_DIR_PREFIX="$(mktemp -d)" RABBITMQ_DOCKER_IMAGE=pivotalrabbitmq/rabbitmq:$IMAGE_TAG \
-         ${SELENIUM_DIR}/run-suites.sh full-suite-authnz-messaging        
-        echo "SELENIUM_ARTIFACTS=$CONF_DIR_PREFIX" >> "$GITHUB_OUTPUT"
-        
+        export IMAGE_TAG=$(find PACKAGES/rabbitmq-server-generic-unix-*.tar.xz | awk -F 'PACKAGES/rabbitmq-server-generic-unix-|.tar.xz' '{print $2}')
+        export CONF_DIR_PREFIX="$(mktemp -d)" 
+        export RABBITMQ_DOCKER_IMAGE=pivotalrabbitmq/rabbitmq:$IMAGE_TAG
+        echo "Running selenium tests with "
+        echo " - CONF_DIR_PREFIX: ${CONF_DIR_PREFIX}"
+        echo " - IMAGE_TAG: ${IMAGE_TAG}"
+        echo " - RABBITMQ_DOCKER_IMAGE: ${RABBITMQ_DOCKER_IMAGE}"        
+        echo "SELENIUM_ARTIFACTS=${CONF_DIR_PREFIX}" >> $GITHUB_ENV        
+        ${SELENIUM_DIR}/run-suites.sh full-suite-authnz-messaging 
+                
     - name: Upload Test Artifacts
-      if: always()
+      if: ${{ failure() && steps.tests.outcome == 'failure' }}
       uses: actions/[email protected]
-      env: 
-          SELENIUM_ARTIFACTS: ${{ steps.tests.outputs.SELENIUM_ARTIFACTS }}
       with:
         name: test-artifacts-${{ matrix.browser }}-${{ matrix.erlang_version }}        
-        path: |
-           $SELENIUM_ARTIFACTS/*
+        path: ${{ env.SELENIUM_ARTIFACTS }}/*
 
   summary-selenium:
     needs:

.github/workflows/test-management-ui-for-pr.yaml

@@ -62,17 +62,19 @@ jobs:
     - name: Run short UI suites on a standalone rabbitmq server
       id: tests
       run: |
-        IMAGE_TAG=$(find PACKAGES/rabbitmq-server-generic-unix-*.tar.xz | awk -F 'PACKAGES/rabbitmq-server-generic-unix-|.tar.xz' '{print $2}')
-        CONF_DIR_PREFIX="$(mktemp -d)" RABBITMQ_DOCKER_IMAGE=pivotalrabbitmq/rabbitmq:$IMAGE_TAG \
-          ${SELENIUM_DIR}/run-suites.sh short-suite-management-ui        
-        echo "SELENIUM_ARTIFACTS=$CONF_DIR_PREFIX" >> "$GITHUB_OUTPUT"
-        
+        export IMAGE_TAG=$(find PACKAGES/rabbitmq-server-generic-unix-*.tar.xz | awk -F 'PACKAGES/rabbitmq-server-generic-unix-|.tar.xz' '{print $2}')
+        export CONF_DIR_PREFIX="$(mktemp -d)" 
+        export RABBITMQ_DOCKER_IMAGE=pivotalrabbitmq/rabbitmq:$IMAGE_TAG
+        echo "Running selenium tests with "
+        echo " - CONF_DIR_PREFIX: ${CONF_DIR_PREFIX}"
+        echo " - IMAGE_TAG: ${IMAGE_TAG}"
+        echo " - RABBITMQ_DOCKER_IMAGE: ${RABBITMQ_DOCKER_IMAGE}"       
+        echo "SELENIUM_ARTIFACTS=${CONF_DIR_PREFIX}" >> $GITHUB_ENV
+        ${SELENIUM_DIR}/run-suites.sh short-suite-management-ui
+      
     - name: Upload Test Artifacts
-      if: ${{ failure() && steps.tests.outcome == 'failed' }}
+      if: ${{ failure() && steps.tests.outcome == 'failure' }}
       uses: actions/upload-artifact@v4
-      env: 
-          SELENIUM_ARTIFACTS: ${{ steps.tests.outputs.SELENIUM_ARTIFACTS }}
       with:
         name: test-artifacts-${{ matrix.browser }}-${{ matrix.erlang_version }}
-        path: |
-          $SELENIUM_ARTIFACTS/*
+        path: ${{ env.SELENIUM_ARTIFACTS }}/*

.github/workflows/test-management-ui.yaml

@@ -66,17 +66,19 @@ jobs:
     - name: Run full UI suite on a 3-node rabbitmq cluster
       id: tests
       run: |
-        IMAGE_TAG=$(find PACKAGES/rabbitmq-server-generic-unix-*.tar.xz | awk -F 'PACKAGES/rabbitmq-server-generic-unix-|.tar.xz' '{print $2}')
-        CONF_DIR_PREFIX="$(mktemp -d)" RABBITMQ_DOCKER_IMAGE=pivotalrabbitmq/rabbitmq:$IMAGE_TAG \
-          ${SELENIUM_DIR}/run-suites.sh full-suite-management-ui
-        echo "SELENIUM_ARTIFACTS=$CONF_DIR_PREFIX" >> "$GITHUB_OUTPUT"
+        export IMAGE_TAG=$(find PACKAGES/rabbitmq-server-generic-unix-*.tar.xz | awk -F 'PACKAGES/rabbitmq-server-generic-unix-|.tar.xz' '{print $2}')
+        export CONF_DIR_PREFIX="$(mktemp -d)"
+        export RABBITMQ_DOCKER_IMAGE=pivotalrabbitmq/rabbitmq:$IMAGE_TAG
+        echo "Running selenium tests with "
+        echo " - CONF_DIR_PREFIX: ${CONF_DIR_PREFIX}"
+        echo " - IMAGE_TAG: ${IMAGE_TAG}"
+        echo " - RABBITMQ_DOCKER_IMAGE: ${RABBITMQ_DOCKER_IMAGE}"       
+        echo "SELENIUM_ARTIFACTS=${CONF_DIR_PREFIX}" >> $GITHUB_ENV
+        ${SELENIUM_DIR}/run-suites.sh full-suite-management-ui        
         
     - name: Upload Test Artifacts
-      if: ${{ failure() && steps.tests.outcome == 'failed' }}
+      if: ${{ failure() && steps.tests.outcome == 'failure' }}
       uses: actions/[email protected]
-      env: 
-          SELENIUM_ARTIFACTS: ${{ steps.run-suites.outputs.SELENIUM_ARTIFACTS }}
       with:
         name: test-artifacts-${{ matrix.browser }}-${{ matrix.erlang_version }}
-        path: |
-          $SELENIUM_ARTIFACTS/*
+        path: ${{ env.SELENIUM_ARTIFACTS }}/*

selenium/bin/components/fakeportal

@@ -32,6 +32,8 @@ init_fakeportal() {
   print "> CLIENT_ID: ${CLIENT_ID}"
   print "> CLIENT_SECRET: ${CLIENT_SECRET}"
   print "> RABBITMQ_URL: ${RABBITMQ_URL}"
+  print "> IDP_TOKEN_ENDPOINT: ${IDP_TOKEN_ENDPOINT}"
+  print "> IDP: ${IDP}"
 }
 start_fakeportal() {
   begin "Starting fakeportal ..."
@@ -48,11 +50,11 @@ start_fakeportal() {
     --env PORT=3000 \
     --env RABBITMQ_URL="${RABBITMQ_URL_FOR_FAKEPORTAL}" \
     --env PROXIED_RABBITMQ_URL="${RABBITMQ_URL}" \
-    --env UAA_URL="${UAA_URL_FOR_FAKEPORTAL}" \
+    --env IDP_TOKEN_ENDPOINT="${IDP_TOKEN_ENDPOINT}" \
     --env CLIENT_ID="${CLIENT_ID}" \
     --env CLIENT_SECRET="${CLIENT_SECRET}" \
-    --env NODE_EXTRA_CA_CERTS=/etc/uaa/ca_uaa_certificate.pem \
-    -v ${TEST_CONFIG_DIR}/uaa:/etc/uaa \
+    --env NODE_EXTRA_CA_CERTS=/etc/${IDP}/ca_${IDP}_certificate.pem \
+    -v ${TEST_CONFIG_DIR}/${IDP}:/etc/${IDP} \
     -v ${FAKEPORTAL_DIR}:/code/fakeportal \
     mocha-test:${mocha_test_tag} run fakeportal
 

selenium/bin/components/spring

@@ -0,0 +1,49 @@
+#!/usr/bin/env bash
+
+SPRING_DOCKER_IMAGE=${SPRING_DOCKER_IMAGE:-pivotalrabbitmq/spring-authorization-server:0.0.10}
+
+ensure_spring() {
+  if docker ps | grep spring &> /dev/null; then
+     print "spring already running ..."
+ else
+   start_spring
+ fi
+}
+init_spring() {
+  SPRING_CONFIG_DIR=${TEST_CONFIG_PATH}/spring
+  SPRING_URL=${SPRING_URL:-$OAUTH_PROVIDER_URL}
+
+  print "> SPRING_CONFIG_DIR: ${SPRING_CONFIG_DIR}"
+  print "> SPRING_URL: ${SPRING_URL}"
+  print "> SPRING_DOCKER_IMAGE: ${SPRING_DOCKER_IMAGE}"
+
+  generate-ca-server-client-kpi spring $SPRING_CONFIG_DIR
+  generate-server-keystore-if-required spring $SPRING_CONFIG_DIR
+}
+start_spring() {
+  begin "Starting spring ..."
+
+  init_spring
+  kill_container_if_exist spring
+
+  MOUNT_SPRING_CONF_DIR=$CONF_DIR/spring
+
+  mkdir -p $MOUNT_SPRING_CONF_DIR
+  ${BIN_DIR}/gen-spring-yml ${SPRING_CONFIG_DIR} $ENV_FILE $MOUNT_SPRING_CONF_DIR/application.yml
+  print "> EFFECTIVE SPRING_CONFIG_FILE: $MOUNT_SPRING_CONF_DIR/application.yml"
+  cp ${SPRING_CONFIG_DIR}/*.pem $MOUNT_SPRING_CONF_DIR
+  cp ${SPRING_CONFIG_DIR}/*.jks $MOUNT_SPRING_CONF_DIR
+
+  docker run \
+		--detach \
+		--name spring \
+    --net ${DOCKER_NETWORK} \
+		--publish 8080:8080 \
+    --publish 8443:8443 \
+    -v ${MOUNT_SPRING_CONF_DIR}:/config \
+		${SPRING_DOCKER_IMAGE} 
+
+  wait_for_oidc_endpoint spring $SPRING_URL $MOUNT_SPRING_CONF_DIR/ca_spring_certificate.pem
+  end "spring is ready"
+
+}

selenium/bin/gen-spring-yml

@@ -0,0 +1,20 @@
+#!/usr/bin/env bash
+SCRIPT="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+
+#set -x
+
+SPRING_PATH=${1:?First parameter is the directory env and config files are relative to}
+ENV_FILE=${2:?Second parameter is a comma-separated list of .env file which has exported template variables}
+FINAL_CONFIG_FILE=${3:?Forth parameter is the name of the final config file. It is relative to where this script is run from}
+
+source $ENV_FILE
+
+parentdir="$(dirname "$FINAL_CONFIG_FILE")"
+mkdir -p $parentdir
+
+echo "" > $FINAL_CONFIG_FILE
+
+for f in $($SCRIPT/find-template-files "${PROFILES}" $SPRING_PATH "application" "yml")
+do
+  envsubst < $f  >> $FINAL_CONFIG_FILE
+done

selenium/bin/suite_template

@@ -154,16 +154,17 @@ build_mocha_image() {
 }
 
 kill_container_if_exist() {
-  if docker stop $1 &> /dev/null; then
-     docker rm $1 &> /dev/null
-  fi
+  docker kill $1 &> /dev/null 
+  docker rm $1 &> /dev/null
 }
 wait_for_message() {
-  attemps_left=10
+  delay=${3:-5}
+  attemps_left=${4:-10}
+  
   while ! docker logs $1 2>&1 | grep -q "$2";
   do
-      sleep 5
-      print "Waiting 5sec for $1 to start ($attemps_left attempts left )..."
+      sleep $delay
+      print "Waiting $delay sec for $1 to start ($attemps_left attempts left )..."
       ((attemps_left--))
       if [[ "$attemps_left" -lt 1 ]]; then
         print "Timed out waiting"
@@ -185,17 +186,20 @@ wait_for_oidc_endpoint() {
 wait_for_oidc_endpoint_local() {
   NAME=$1
   BASE_URL=$2
-  CURL_ARGS="-k --tlsv1.2 -L --fail "
-  DELAY_BETWEEN_ATTEMPTS=5
+  CURL_ARGS="--tlsv1.2 -L --fail "
+  DELAY_BETWEEN_ATTEMPTS=10
   if [[ $# -eq 3 ]]; then
     CURL_ARGS="$CURL_ARGS --cacert $3"
     DELAY_BETWEEN_ATTEMPTS=10
+  else 
+    CURL_ARGS="$CURL_ARGS -k " 
   fi
   max_retry=15
   counter=0
   print "Waiting for OIDC discovery endpoint $NAME ... (BASE_URL: $BASE_URL)"
   until (curl $CURL_ARGS ${BASE_URL}/.well-known/openid-configuration >/dev/null 2>&1)
   do
+    echo "Failed $?"
     sleep $DELAY_BETWEEN_ATTEMPTS
     [[ counter -eq $max_retry ]] && print "Failed!" && exit 1
     print "Trying again. Try #$counter"
@@ -208,7 +212,7 @@ wait_for_oidc_endpoint_docker() {
   BASE_URL=$2
   CURL_ARGS="-k --tlsv1.2 -L --fail "
   DOCKER_ARGS="--rm --net ${DOCKER_NETWORK} "
-  DELAY_BETWEEN_ATTEMPTS=5
+  DELAY_BETWEEN_ATTEMPTS=10
   if [[ $# -gt 2  ]]; then
     DOCKER_ARGS="$DOCKER_ARGS -v $3:/tmp/ca_certificate.pem"
     CURL_ARGS="$CURL_ARGS --cacert /tmp/ca_certificate.pem"
@@ -469,7 +473,7 @@ do_generate-ca-server-client-kpi() {
   cd $ROOT/tls-gen/basic
   cp openssl.cnf openssl.cnf.bak
   if [ -f "$FOLDER/openssl.cnf.in" ]; then     
-    cp $FOLDER/openssl.cnf.in >> openssl.cnf
+    cat $FOLDER/openssl.cnf.in >> openssl.cnf
   fi 
   if [[ ! -z "${DEBUG}" ]]; then
     print "Used this openssl.conf"

selenium/fakeportal/app.js

@@ -7,7 +7,7 @@ const rabbitmq_url = process.env.RABBITMQ_URL;
 const proxied_rabbitmq_url = process.env.PROXIED_RABBITMQ_URL;
 const client_id = process.env.CLIENT_ID;
 const client_secret = process.env.CLIENT_SECRET;
-const uaa_url = process.env.UAA_URL;
+const idp_token_endpoint = process.env.IDP_TOKEN_ENDPOINT;
 const port = process.env.PORT || 3000;
 
 app.engine('.html', require('ejs').__express);
@@ -28,7 +28,10 @@ app.get('/favicon.ico', (req, res) => res.status(204));
 
 
 app.listen(port);
-console.log('Express started on port ' + port);
+console.log('Express started on port ' + port + " using ") 
+console.log(" -  idp_token_endpoint: " + idp_token_endpoint)
+console.log(" -  rabbitmq_url: " + rabbitmq_url)
+console.log(" -  proxied_rabbitmq_url: " + proxied_rabbitmq_url)
 
 function default_if_blank(value, defaultValue) {
   if (typeof value === "undefined" || value === null || value == "") {
@@ -40,14 +43,13 @@ function default_if_blank(value, defaultValue) {
 
 function access_token(id, secret) {
   const req = new XMLHttpRequest();
-  const url = uaa_url + '/oauth/token';
+  const url = idp_token_endpoint
   const params = 'client_id=' + id +
     '&client_secret=' + secret +
     '&grant_type=client_credentials' +
-    '&token_format=jwt' +
     '&response_type=token';
 
-  console.debug("Sending " + url + " with params "+  params);
+  console.debug("Sending " + url + " with params " +  params);
 
   req.open('POST', url, false);
   req.setRequestHeader('Content-Type', 'application/x-www-form-urlencoded');