Moving pipelines to shepherd (#869)

Author: Albertoimpl

.editorconfig

@@ -1,15 +1,25 @@
 [*]
-charset=utf-8
-end_of_line=lf
-insert_final_newline=true
-indent_style=tab
-indent_size=4
+charset = utf-8
+end_of_line = lf
+insert_final_newline = true
+indent_style = tab
+indent_size = 4
 
-[*.{yml,yaml,sh}]
-indent_style=space
-indent_size=2
+[{spec,*.yml,*.yaml,Kilnfile,Kilnfile.lock}]
+indent_style = space
+indent_size = 2
+ij_yaml_keep_indents_on_empty_lines = false
+ij_yaml_keep_line_breaks = true
+
+[*.sh]
+indent_size = 2
+indent_style = space
+
+[{.asciidoctorconfig,*.ad,*.adoc,*.asciidoc}]
+ij_asciidoc_formatting_enabled = true
+ij_asciidoc_one_sentence_per_line = true
 
 [*.java]
 ij_java_class_count_to_use_import_on_demand = 100
-ij_java_imports_layout = jakarta.**,|,java.**,|,*,|,org.springframework.**,|,$*
+ij_java_imports_layout = jakarta.**, |, java.**, |, *, |, org.springframework.**, |, $*
 ij_java_names_count_to_use_import_on_demand = 100

ci/pipeline.yml

@@ -98,22 +98,24 @@ jobs:
           - get: git-repo
             passed: [build]
             trigger: true
-          - put: toolsmith-env
-            inputs: []
+          - put: cf-environment
             params:
-              action: claim
+              action: create
+              duration: 6h
+              resource: cf-environment
+            timeout: 6h
       - task: acceptance-tests
         file: git-repo/ci/tasks/acceptance-tests.yml
         vars:
           ci-image-tag: ((ci-image-tag))
     on_success:
       in_parallel:
         - *slack-success-notification
-        - put: toolsmith-env
+        - put: cf-environment
           params:
-            action: unclaim
-            env_file: toolsmith-env/metadata
-          inputs: [toolsmith-env]
+            action: release
+            resource: cf-environment
+
     on_failure:
       *slack-failure-notification
     on_error:
@@ -244,11 +246,11 @@ resource_types:
       repository: ((dockerhub-mirror-registry))/cfcommunity/slack-notification-resource
       tag: latest
 
-  - name: pcf-pool
+  - name: shepherd
     type: registry-image
     source:
-      repository: ((dockerhub-mirror-registry))/cftoolsmiths/toolsmiths-envs-resource
-      tag: latest
+      repository: ((shepherd-image-repository))/shepherd-268822/shepherd2/concourse-resource
+      tag: v1
 
 resources:
   - name: nightly
@@ -323,13 +325,17 @@ resources:
     source:
       url: ((scs-slack-webhook))
 
-  - name: toolsmith-env
-    type: pcf-pool
-    icon: pool
+  - name: cf-environment
+    type: shepherd
     source:
-      api_token: ((toolsmith-api-token))
-      hostname: environments.toolsmiths.cf-app.com
-      pool_name: cf-deployment
+      url: https://v2.shepherd.run
+      service-account-key: ((shepherd-service-account-key))
+      lease:
+        namespace: spring-cloud-app-broker
+        pool:
+          namespace: official
+          name: cfd
+      compatibility-mode: environments-app
 
 groups:
   - name: "build"

ci/pr-pipeline.yml

@@ -35,12 +35,14 @@ jobs:
     plan:
       - in_parallel:
           - get: pull-request
-            passed: [build]
+            passed: [ build ]
             trigger: true
-          - put: toolsmith-env
-            inputs: []
+          - put: cf-environment
             params:
-              action: claim
+              action: create
+              duration: 6h
+              resource: cf-environment
+            timeout: 6h
       - task: acceptance-tests
         file: pull-request/ci/tasks/acceptance-tests.yml
         vars:
@@ -50,11 +52,10 @@ jobs:
     on_success:
       in_parallel:
         - *slack-success-notification
-        - put: toolsmith-env
+        - put: cf-environment
           params:
-            action: unclaim
-            env_file: toolsmith-env/metadata
-          inputs: [ toolsmith-env ]
+            action: release
+            resource: cf-environment
     on_failure: *slack-failure-notification
     on_error: *slack-failure-notification
 
@@ -71,11 +72,11 @@ resource_types:
       repository: ((dockerhub-mirror-registry))/cfcommunity/slack-notification-resource
       tag: latest
 
-  - name: pcf-pool
+  - name: shepherd
     type: registry-image
     source:
-      repository: ((dockerhub-mirror-registry))/cftoolsmiths/toolsmiths-envs-resource
-      tag: latest
+      repository: ((shepherd-image-repository))/shepherd-268822/shepherd2/concourse-resource
+      tag: v1
 
 resources:
   - name: pull-request
@@ -95,10 +96,14 @@ resources:
     source:
       url: ((scs-slack-webhook))
 
-  - name: toolsmith-env
-    type: pcf-pool
-    icon: pool
+  - name: cf-environment
+    type: shepherd
     source:
-      api_token: ((toolsmith-api-token))
-      hostname: environments.toolsmiths.cf-app.com
-      pool_name: cf-deployment
+      url: https://v2.shepherd.run
+      service-account-key: ((shepherd-service-account-key))
+      lease:
+        namespace: spring-cloud-app-broker
+        pool:
+          namespace: official
+          name: cfd
+      compatibility-mode: environments-app

ci/scripts/acceptance-tests.sh

@@ -2,37 +2,38 @@
 
 set -euo pipefail
 
-readonly TOOLSMITH_ENV_INPUT="${TOOLSMITH_ENV_INPUT:?must be set}"
-readonly DEFAULT_ORG="${DEFAULT_ORG:?must be set}"
-readonly DEFAULT_SPACE="${DEFAULT_SPACE:?must be set}"
-readonly SKIP_SSL_VALIDATION="${SKIP_SSL_VALIDATION:?must be set}"
-readonly ONLY_SHOW_STANDARD_STREAMS_ON_TEST_FAILURE="${ONLY_SHOW_STANDARD_STREAMS_ON_TEST_FAILURE:-true}"
-readonly DEPLOYMENT_DIRECTORY="$(mktemp -d)"
-
-declare API_HOST
-readonly API_PORT=443
-readonly USERNAME="admin"
-declare PASSWORD
+readonly METADATA_FILE="${METADATA_FILE:?must be set}"
+
+declare CF_API_HOST
+readonly CF_USERNAME="admin"
+declare CF_PASSWORD
+
 readonly CLIENT_ID="admin"
 declare CLIENT_SECRET
 
+readonly DEFAULT_ORG="test"
+readonly DEFAULT_SPACE="development"
+
+DEPLOYMENT_DIRECTORY="$(mktemp -d)"
+readonly DEPLOYMENT_DIRECTORY
+
 discover_environment() {
   local env_name
-  env_name=$(cat "$TOOLSMITH_ENV_INPUT/name")
+  env_name=$(jq -r .name <"${METADATA_FILE}")
 
-  eval "$(bbl print-env --metadata-file "$TOOLSMITH_ENV_INPUT/metadata")"
+  eval "$(bbl print-env --metadata-file "${METADATA_FILE}")"
 
-  API_HOST="$(jq -r .cf.api_url <"$TOOLSMITH_ENV_INPUT/metadata")"
-  PASSWORD="$(credhub get -n "/bosh-${env_name}/cf/cf_admin_password" -q)"
+  CF_API_HOST="$(jq -r .cf.api_url <"${METADATA_FILE}")"
+  CF_PASSWORD="$(credhub get -n "/bosh-${env_name}/cf/cf_admin_password" -q)"
   CLIENT_SECRET="$(credhub get -n "/bosh-${env_name}/cf/uaa_admin_client_secret" -q)"
 }
 
 prepare_cf_deployment() {
-  pushd "$DEPLOYMENT_DIRECTORY" > /dev/null
+  pushd "$DEPLOYMENT_DIRECTORY" >/dev/null
 
-  bosh --deployment cf manifest > manifest.yml
+  bosh --deployment cf manifest >manifest.yml
 
-  cat << EOF > ops.yml
+  cat <<EOF >ops.yml
 - type: replace
   path: /instance_groups/name=diego-cell/vm_type
   value: large
@@ -46,18 +47,13 @@ EOF
 
   bosh --non-interactive --deployment cf deploy --ops-file ops.yml manifest.yml
 
-  popd > /dev/null
+  popd >/dev/null
 }
 
 prepare_cf() {
   local -r test_instances_org="$DEFAULT_ORG-instances"
 
-  local skip_ssl_validation=""
-  if [ "$SKIP_SSL_VALIDATION" = "true" ]; then
-    skip_ssl_validation="--skip-ssl-validation"
-  fi
-
-  cf login -a "$API_HOST" -u "$USERNAME" -p "$PASSWORD" -o system "$skip_ssl_validation"
+  cf login -a "$CF_API_HOST" -u "$CF_USERNAME" -p "$CF_PASSWORD" -o system --skip-ssl-validation
 
   cf create-org "$DEFAULT_ORG"
   cf create-space "$DEFAULT_SPACE" -o "$DEFAULT_ORG"
@@ -67,27 +63,26 @@ prepare_cf() {
 }
 
 run_tests() {
-  export SPRING_CLOUD_APPBROKER_ACCEPTANCETEST_CLOUDFOUNDRY_API_HOST="${API_HOST}"
-  export SPRING_CLOUD_APPBROKER_ACCEPTANCETEST_CLOUDFOUNDRY_API_PORT="${API_PORT}"
-  export SPRING_CLOUD_APPBROKER_ACCEPTANCETEST_CLOUDFOUNDRY_USERNAME="${USERNAME}"
-  export SPRING_CLOUD_APPBROKER_ACCEPTANCETEST_CLOUDFOUNDRY_PASSWORD="${PASSWORD}"
+  export SPRING_CLOUD_APPBROKER_ACCEPTANCETEST_CLOUDFOUNDRY_API_HOST="${CF_API_HOST}"
+  export SPRING_CLOUD_APPBROKER_ACCEPTANCETEST_CLOUDFOUNDRY_API_PORT=443
+  export SPRING_CLOUD_APPBROKER_ACCEPTANCETEST_CLOUDFOUNDRY_USERNAME="${CF_USERNAME}"
+  export SPRING_CLOUD_APPBROKER_ACCEPTANCETEST_CLOUDFOUNDRY_PASSWORD="${CF_PASSWORD}"
   export SPRING_CLOUD_APPBROKER_ACCEPTANCETEST_CLOUDFOUNDRY_CLIENT_ID="${CLIENT_ID}"
   export SPRING_CLOUD_APPBROKER_ACCEPTANCETEST_CLOUDFOUNDRY_CLIENT_SECRET="${CLIENT_SECRET}"
   export SPRING_CLOUD_APPBROKER_ACCEPTANCETEST_CLOUDFOUNDRY_DEFAULT_ORG="${DEFAULT_ORG}"
   export SPRING_CLOUD_APPBROKER_ACCEPTANCETEST_CLOUDFOUNDRY_DEFAULT_SPACE="${DEFAULT_SPACE}"
-  export SPRING_CLOUD_APPBROKER_ACCEPTANCETEST_CLOUDFOUNDRY_SKIP_SSL_VALIDATION="${SKIP_SSL_VALIDATION}"
+  export SPRING_CLOUD_APPBROKER_ACCEPTANCETEST_CLOUDFOUNDRY_SKIP_SSL_VALIDATION=true
   export TESTS_BROKERAPPPATH=build/libs/spring-cloud-app-broker-acceptance-tests.jar
   ./gradlew -PacceptanceTests \
-    -PonlyShowStandardStreamsOnTestFailure="${ONLY_SHOW_STANDARD_STREAMS_ON_TEST_FAILURE}" \
+    -PonlyShowStandardStreamsOnTestFailure=true \
     :spring-cloud-app-broker-acceptance-tests:test
 }
 
 main() {
   discover_environment
 
-  echo "Running tests against $API_HOST"
+  echo "Running tests against ${CF_API_HOST}"
   echo
-
   prepare_cf_deployment
   prepare_cf
 

ci/tasks/acceptance-tests.yml

@@ -9,13 +9,9 @@ image_resource:
     tag: ((ci-image-tag))
 inputs:
   - name: git-repo
-  - name: toolsmith-env
+  - name: cf-environment
 run:
   path: ci/scripts/acceptance-tests.sh
   dir: git-repo
 params:
-  TOOLSMITH_ENV_INPUT: ../toolsmith-env
-  DEFAULT_ORG: ((acceptance-test-default-org))
-  DEFAULT_SPACE: ((acceptance-test-default-space))
-  SKIP_SSL_VALIDATION: ((acceptance-test-skip-ssl-validation))
-  ONLY_SHOW_STANDARD_STREAMS_ON_TEST_FAILURE: true
+  METADATA_FILE: ../cf-environment/metadata