(feature): Enable ScalingEngine in mtar deployment

ci/infrastructure/scripts/deploy-multiapps-controller.sh

@@ -22,9 +22,11 @@ function deploy_multiapps_controller() {
 
   mv multiapps-controller-web-war/*.war .
   pushd multiapps-controller-web-manifest
+
   cf push -f ./*.yml "${app_name}"
   # scale up to be able to handle huge (>1GB) .MTARs
   cf scale -m 4G -k 2G deploy-service -f
+
   popd
 }
 

ci/operations/scale_out_cf_for_app-autoscaler.yaml

@@ -13,3 +13,13 @@
   path: /instance_groups/name=diego-cell/vm_extensions
   value:
   - 500GB_ephemeral_disk
+
+
+- type: replace
+  path: /instance_groups/name=cc-worker/jobs/name=cloud_controller_worker/properties/cc/maximum_app_disk_in_mb?
+  value: 4096
+
+- type: replace
+  path: /instance_groups/name=api/jobs/name=cloud_controller_ng/properties/cc/maximum_app_disk_in_mb?
+  value: 4096
+

operations/configure-cf-services.yml

@@ -69,4 +69,3 @@
       component: autoscaler_cf_eventgenerator
     uris:
       - ((deployment_name))-cf-eventgenerator.((system_domain))
-

operations/use-cf-services.yml

@@ -63,6 +63,10 @@
 - type: remove
   path: /instance_groups/name=operator
 
+## ScalingEngine - remove scalingengine
+- type: remove
+  path: /instance_groups/name=scalingengine
+
 ## SCHEDULER - remove scheduler
 - type: remove
   path: /instance_groups/name=scheduler

packages/scalingengine/spec

@@ -9,6 +9,7 @@ files:
 - autoscaler/vendor/*
 - autoscaler/scalingengine/db/*
 - autoscaler/cf/* # gosub
+- autoscaler/configutil/* # gosub
 - autoscaler/db/* # gosub
 - autoscaler/db/sqldb/* # gosub
 - autoscaler/healthendpoint/* # gosub
@@ -35,6 +36,7 @@ files:
 - autoscaler/vendor/filippo.io/edwards25519/field/* # gosub
 - autoscaler/vendor/github.com/beorn7/perks/quantile/* # gosub
 - autoscaler/vendor/github.com/cespare/xxhash/v2/* # gosub
+- autoscaler/vendor/github.com/cloud-gov/go-cfenv/* # gosub
 - autoscaler/vendor/github.com/dlclark/regexp2/* # gosub
 - autoscaler/vendor/github.com/dlclark/regexp2/syntax/* # gosub
 - autoscaler/vendor/github.com/fatih/color/* # gosub
@@ -72,6 +74,7 @@ files:
 - autoscaler/vendor/github.com/jmoiron/sqlx/reflectx/* # gosub
 - autoscaler/vendor/github.com/mattn/go-colorable/* # gosub
 - autoscaler/vendor/github.com/mattn/go-isatty/* # gosub
+- autoscaler/vendor/github.com/mitchellh/mapstructure/* # gosub
 - autoscaler/vendor/github.com/munnerz/goautoneg/* # gosub
 - autoscaler/vendor/github.com/ogen-go/ogen/conv/* # gosub
 - autoscaler/vendor/github.com/ogen-go/ogen/http/* # gosub

scripts/initialise_db.sh

@@ -1,6 +1,8 @@
 #!/usr/bin/env bash
 
 set -euo pipefail
+
+set -x
 echo "Running $0"
 
 DB_HOST="${DB_HOST:-localhost}"
@@ -66,6 +68,7 @@ trap 'rm -f "${LOG_FILE}" || echo "no log file"' EXIT
 echo "# Applying liquibase change sets to: '${URL}'"
 for file in ${files}; do
   echo "  - applying: '$file'" | tee -a "${LOG_FILE}"
+  # shellcheck disable=SC2086
   java -cp "${class_path}" liquibase.integration.commandline.Main \
     --url="${URL}" \
     --driver="${DRIVER}" \

src/autoscaler/.gitignore

@@ -3,4 +3,5 @@ fakes/*
 mta_archives/
 api/default_catalog.json
 mta.yaml
+*.mta
 Makefile_*

src/autoscaler/Makefile

@@ -16,7 +16,8 @@ PACKAGE_DIRS = $(shell go list './...' | grep --invert-match --regexp='/vendor/'
 								 | grep --invert-match --regexp='e2e')
 
 
-MODULES ?= dbtasks,apiserver,eventgenerator,metricsforwarder,operator,scheduler
+MODULES ?= dbtasks,apiserver,eventgenerator,metricsforwarder,operator,scheduler,scalingengine
+
 db_type ?= postgres
 DB_HOST ?= localhost
 DBURL := $(shell case "${db_type}" in\
@@ -33,7 +34,7 @@ binaries=$(shell find . -name "main.go" -exec dirname {} \; |  cut -d/ -f2 | sor
 test_dirs=$(shell find . -name "*_test.go" -exec dirname {} \; |  cut -d/ -f2 | sort | uniq)
 export GO111MODULE=on
 
-.PHONY: clean-dbtasks package-dbtasks vendor-changelogs clean-scheduler package-scheduler
+.PHONY: clean-dbtasks package-dbtasks vendor-changelogs clean-scheduler package-scheduler clean mta-deploy mta-undeploy mta-build mta-logs
 
 GINKGO_OPTS = -r --race --require-suite --randomize-all --cover ${OPTS}
 
@@ -91,7 +92,6 @@ go-mod-tidy: ./go.mod ./go.sum ${go_deps_without_generated_sources}
 	go mod tidy
 
 
-
 go-vendoring-folder := ./vendor
 go-vendored-files = $(shell find '${go-vendoring-folder}' -type f -name '*.go' 2> '/dev/null')
 ## This does not work: go-vendored-files = $(wildcard ${go-vendoring-folder}/**/*.go)
@@ -176,14 +176,13 @@ clean-scheduler:
 build-scheduler:
 	pushd scheduler; mvn package -Dmaven.test.skip=true; popd
 
-
 vendor-changelogs:
 	cp $(MAKEFILE_DIR)/api/db/* $(MAKEFILE_DIR)/dbtasks/src/main/resources/.
 	cp $(MAKEFILE_DIR)/eventgenerator/db/* $(MAKEFILE_DIR)/dbtasks/src/main/resources/.
 	cp $(MAKEFILE_DIR)/operator/db/* $(MAKEFILE_DIR)/dbtasks/src/main/resources/.
+	cp $(MAKEFILE_DIR)/scalingengine/db/* $(MAKEFILE_DIR)/dbtasks/src/main/resources/.
 	cp $(MAKEFILE_DIR)/scheduler/db/* $(MAKEFILE_DIR)/dbtasks/src/main/resources/.
 
-.PHONY: clean
 clean:
 	@echo "# cleaning autoscaler"
 	@go clean -cache -testcache
@@ -193,13 +192,11 @@ clean:
 	@rm --force --recursive "${openapi-generated-clients-and-servers-api-dir}"
 	@rm --force --recursive "${openapi-generated-clients-and-servers-scalingengine-dir}"
 
-.PHONY: mta-deploy
 mta-deploy: mta-build build-extension-file
 	$(MAKE) -f metricsforwarder/Makefile set-security-group
 	@echo "Deploying with extension file: $(EXTENSION_FILE)"
 	@cf deploy $(DEST)/$(MTAR_FILENAME) --version-rule ALL -f --delete-services -e $(EXTENSION_FILE) -m $(MODULES)
 
-.PHONY: mta-deploy
 mta-undeploy:
 	@cf undeploy com.github.cloudfoundry.app-autoscaler-release -f
 
@@ -212,7 +209,6 @@ mta-logs:
 	cf dmol --mta com.github.cloudfoundry.app-autoscaler-release --last 1
 	vim mta-*
 
-.PHONY: mta-build
 mta-build: mta-build-clean
 	@echo "building mtar file for version: $(VERSION)"
 	cp mta.tpl.yaml mta.yaml
@@ -225,7 +221,6 @@ mta-build: mta-build-clean
 mta-build-clean:
 	rm -rf mta_archives
 
-
 .PHONY: cf-login
 cf-login:
 	@echo '⚠️ Please note that this login only works for cf and concourse,' \

src/autoscaler/api/broker/broker_test.go

@@ -4,9 +4,10 @@ import (
 	"encoding/json"
 	"os"
 
+	"code.cloudfoundry.org/app-autoscaler/src/autoscaler/fakes"
+
 	"code.cloudfoundry.org/app-autoscaler/src/autoscaler/api/broker"
 	"code.cloudfoundry.org/app-autoscaler/src/autoscaler/db"
-	"code.cloudfoundry.org/app-autoscaler/src/autoscaler/fakes"
 	"code.cloudfoundry.org/app-autoscaler/src/autoscaler/models"
 
 	"context"

src/autoscaler/api/config/config.go

@@ -13,7 +13,6 @@ import (
 	"golang.org/x/crypto/bcrypt"
 
 	"github.com/xeipuuv/gojsonschema"
-	"gopkg.in/yaml.v3"
 
 	"code.cloudfoundry.org/app-autoscaler/src/autoscaler/db"
 	"code.cloudfoundry.org/app-autoscaler/src/autoscaler/helpers"
@@ -124,6 +123,10 @@ type Config struct {
 	DefaultCustomMetricsCredentialType string                        `yaml:"default_credential_type" json:"default_credential_type"`
 }
 
+func (c *Config) SetLoggingLevel() {
+	c.Logging.Level = strings.ToLower(c.Logging.Level)
+}
+
 type PlanCheckConfig struct {
 	PlanDefinitions map[string]PlanDefinition `yaml:"plan_definitions" json:"plan_definitions"`
 }
@@ -163,15 +166,8 @@ func defaultConfig() Config {
 		},
 	}
 }
-func loadPublicApiServerConfig(conf *Config, vcapReader configutil.VCAPConfigurationReader) error {
-	data, err := vcapReader.GetServiceCredentialContent("apiserver-config", "apiserver-config")
-	if err != nil {
-		return fmt.Errorf("%w: %v", ErrPublicApiServerConfigNotFound, err)
-	}
-	return yaml.Unmarshal(data, conf)
-}
 
-func loadVcapConfig(conf *Config, vcapReader configutil.VCAPConfigurationReader) error {
+func LoadVcapConfig(conf *Config, vcapReader configutil.VCAPConfigurationReader) error {
 	if !vcapReader.IsRunningOnCF() {
 		return nil
 	}
@@ -185,7 +181,7 @@ func loadVcapConfig(conf *Config, vcapReader configutil.VCAPConfigurationReader)
 	conf.CFServer.Port = vcapReader.GetPort()
 	conf.Server.Port = 0
 
-	if err := loadPublicApiServerConfig(conf, vcapReader); err != nil {
+	if err := configutil.LoadConfig(&conf, vcapReader, "apiserver-config"); err != nil {
 		return err
 	}
 
@@ -221,18 +217,7 @@ func configureCatalog(conf *Config, vcapReader configutil.VCAPConfigurationReade
 }
 
 func LoadConfig(filepath string, vcapReader configutil.VCAPConfigurationReader) (*Config, error) {
-	conf := defaultConfig()
-
-	if err := helpers.LoadYamlFile(filepath, &conf); err != nil {
-		return nil, err
-	}
-
-	if err := loadVcapConfig(&conf, vcapReader); err != nil {
-		return nil, err
-	}
-
-	conf.Logging.Level = strings.ToLower(conf.Logging.Level)
-	return &conf, nil
+	return configutil.GenericLoadConfig(filepath, vcapReader, defaultConfig, configutil.VCAPConfigurableFunc[Config](LoadVcapConfig))
 }
 
 func FromJSON(data []byte) (*Config, error) {

src/autoscaler/build-extension-file.sh

@@ -5,6 +5,7 @@ set -e
 
 script_dir=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
 # shellcheck source=../../ci/autoscaler/scripts/vars.source.sh
+# shellcheck disable=SC1091
 source "${script_dir}/../../ci/autoscaler/scripts/vars.source.sh"
 
 if [ -z "$1" ]; then
@@ -22,7 +23,6 @@ fi
 export SYSTEM_DOMAIN="autoscaler.app-runtime-interfaces.ci.cloudfoundry.org"
 export POSTGRES_EXTERNAL_PORT="${PR_NUMBER:-5432}"
 
-
 export CPU_LOWER_THRESHOLD="${CPU_LOWER_THRESHOLD:-"100"}"
 
 cat << EOF > /tmp/extension-file-secrets.yml.tpl
@@ -36,7 +36,7 @@ eventgenerator_health_password: ((/bosh-autoscaler/${DEPLOYMENT_NAME}/autoscaler
 eventgenerator_log_cache_uaa_client_id: eventgenerator_log_cache
 eventgenerator_log_cache_uaa_client_secret: ((/bosh-autoscaler/cf/uaa_clients_eventgenerator_log_cache_secret))
 
-
+scalingengine_health_password: ((/bosh-autoscaler/${DEPLOYMENT_NAME}/autoscaler_scalingengine_health_password))
 
 policy_db_password: ((/bosh-autoscaler/${DEPLOYMENT_NAME}/database_password))
 policy_db_server_ca: ((/bosh-autoscaler/${DEPLOYMENT_NAME}/postgres_server.ca))
@@ -51,15 +51,9 @@ EOF
 
 credhub interpolate -f "/tmp/extension-file-secrets.yml.tpl" > /tmp/mtar-secrets.yml
 
-export METRICSFORWARDER_UAA_SKIP_SSL_VALIDATION="$(yq ".metricsforwarder_uaa_skip_ssl_validation" /tmp/mtar-secrets.yml)"
-export METRICSFORWARDER_APPNAME="${METRICSFORWARDER_APPNAME:-"${DEPLOYMENT_NAME}-metricsforwarder"}"
-export METRICSFORWARDER_HEALTH_PASSWORD="$(yq ".metricsforwarder_health_password" /tmp/mtar-secrets.yml)"
-export METRICSFORWARDER_HOST="${METRICSFORWARDER_HOST:-"${DEPLOYMENT_NAME}-metricsforwarder"}"
-export METRICSFORWARDER_MTLS_HOST="${METRICSFORWARDER_MTLS_HOST:-"${DEPLOYMENT_NAME}-metricsforwarder-mtls"}"
-export METRICSFORWARDER_INSTANCES="${METRICSFORWARDER_INSTANCES:-2}"
-
 export APISERVER_HOST="${APISERVER_HOST:-"${DEPLOYMENT_NAME}"}"
 export APISERVER_INSTANCES="${APISERVER_INSTANCES:-2}"
+export SERVICEBROKER_HOST="${SERVICEBROKER_HOST:-"${DEPLOYMENT_NAME}servicebroker"}"
 
 export EVENTGENERATOR_HEALTH_PASSWORD="$(yq ".eventgenerator_health_password" /tmp/mtar-secrets.yml)"
 export EVENTGENERATOR_LOG_CACHE_UAA_CLIENT_ID="$(yq ".eventgenerator_log_cache_uaa_client_id" /tmp/mtar-secrets.yml)"
@@ -68,14 +62,25 @@ export EVENTGENERATOR_CF_HOST="${EVENTGENERATOR_CF_HOST:-"${DEPLOYMENT_NAME}-cf-
 export EVENTGENERATOR_HOST="${EVENTGENERATOR_HOST:-"${DEPLOYMENT_NAME}-eventgenerator"}"
 export EVENTGENERATOR_INSTANCES="${EVENTGENERATOR_INSTANCES:-2}"
 
+export METRICSFORWARDER_UAA_SKIP_SSL_VALIDATION="$(yq ".metricsforwarder_uaa_skip_ssl_validation" /tmp/mtar-secrets.yml)"
+export METRICSFORWARDER_APPNAME="${METRICSFORWARDER_APPNAME:-"${DEPLOYMENT_NAME}-metricsforwarder"}"
+export METRICSFORWARDER_HEALTH_PASSWORD="$(yq ".metricsforwarder_health_password" /tmp/mtar-secrets.yml)"
+export METRICSFORWARDER_HOST="${METRICSFORWARDER_HOST:-"${DEPLOYMENT_NAME}-metricsforwarder"}"
+export METRICSFORWARDER_MTLS_HOST="${METRICSFORWARDER_MTLS_HOST:-"${DEPLOYMENT_NAME}-metricsforwarder-mtls"}"
+export METRICSFORWARDER_INSTANCES="${METRICSFORWARDER_INSTANCES:-2}"
+
+export SCALINGENGINE_CF_CLIENT_ID="autoscaler_client_id"
+export SCALINGENGINE_CF_CLIENT_SECRET="autoscaler_client_secret"
+export SCALINGENGINE_HEALTH_PASSWORD="$(yq ".scalingengine_health_password" /tmp/mtar-secrets.yml)"
+export SCALINGENGINE_CF_HOST="${SCALINGENGINE_CF_HOST:-"${DEPLOYMENT_NAME}-cf-scalingengine"}"
+export SCALINGENGINE_HOST="${SCALINGENGINE_HOST:-"${DEPLOYMENT_NAME}-scalingengine"}"
+export SCALINGENGINE_INSTANCES="${SCALINGENGINE_INSTANCES:-2}"
 
 export SCHEDULER_HOST="${SCHEDULER_HOST:-"${DEPLOYMENT_NAME}-scheduler"}"
 export SCHEDULER_CF_HOST="${SCHEDULER_CF_HOST:-"${DEPLOYMENT_NAME}-cf-scheduler"}"
 export SCHEDULER_INSTANCES="${SCHEDULER_INSTANCES:-2}"
 
-export SERVICEBROKER_HOST="${SERVICEBROKER_HOST:-"${DEPLOYMENT_NAME}servicebroker"}"
-
-export SCALINGENGINE_HOST="${SCALINGENGINE_HOST:-"${DEPLOYMENT_NAME}-cf-scalingengine"}"
+export SCHEDULER_CF_HOST="${SCHEDULER_CF_HOST:-"${DEPLOYMENT_NAME}-cf-scheduler"}"
 
 export OPERATOR_CF_CLIENT_ID="autoscaler_client_id"
 export OPERATOR_CF_CLIENT_SECRET="autoscaler_client_secret"
@@ -130,6 +135,15 @@ modules:
       routes:
       - route: ${EVENTGENERATOR_CF_HOST}.\${default-domain}
       - route: ${EVENTGENERATOR_HOST}.\${default-domain}
+  - name: scalingengine
+    requires:
+    - name: scalingengine-config
+    - name: database
+    parameters:
+      instances: ${SCALINGENGINE_INSTANCES}
+      routes:
+      - route: ${SCALINGENGINE_CF_HOST}.\${default-domain}
+      - route: ${SCALINGENGINE_HOST}.\${default-domain}
   - name: metricsforwarder
     requires:
     - name: metricsforwarder-config
@@ -162,7 +176,7 @@ resources:
 - name: metricsforwarder-config
   parameters:
     config:
-      metricsforwarder:
+      metricsforwarder-config:
         health:
           basic_auth:
             password: "${METRICSFORWARDER_HEALTH_PASSWORD}"
@@ -185,7 +199,7 @@ resources:
           basic_auth:
             password: "${EVENTGENERATOR_HEALTH_PASSWORD}"
         scalingEngine:
-          scaling_engine_url: https://${SCALINGENGINE_HOST}.\${default-domain}
+          scaling_engine_url: https://${SCALINGENGINE_CF_HOST}.\${default-domain}
 
 - name: apiserver-config
   parameters:
@@ -205,7 +219,7 @@ resources:
           metrics_forwarder_url: https://${METRICSFORWARDER_HOST}.\${default-domain}
           metrics_forwarder_mtls_url: https://${METRICSFORWARDER_MTLS_HOST}.\${default-domain}
         scaling_engine:
-          scaling_engine_url: https://${SCALINGENGINE_HOST}.\${default-domain}
+          scaling_engine_url: https://${SCALINGENGINE_CF_HOST}.\${default-domain}
         event_generator:
           event_generator_url: https://${EVENTGENERATOR_CF_HOST}.\${default-domain}
         broker_credentials:
@@ -236,9 +250,20 @@ resources:
           client_id: ${OPERATOR_CF_CLIENT_ID}
           secret: ${OPERATOR_CF_CLIENT_SECRET}
         scaling_engine:
-          scaling_engine_url: https://${SCALINGENGINE_HOST}.\${default-domain}
+          scaling_engine_url: https://${SCALINGENGINE_CF_HOST}.\${default-domain}
         scheduler:
           scheduler_url: https://${SCHEDULER_CF_HOST}.\${default-domain}
+- name: scalingengine-config
+  parameters:
+    config:
+      scalingengine-config:
+        health:
+          basic_auth:
+            password: "${SCALINGENGINE_HEALTH_PASSWORD}"
+        cf:
+          api:  https://api.\${default-domain}
+          client_id: ${SCALINGENGINE_CF_CLIENT_ID}
+          secret: ${SCALINGENGINE_CF_CLIENT_SECRET}
 
 - name: database
   parameters:
@@ -247,12 +272,14 @@ resources:
       client_cert: "${POLICY_DB_CLIENT_CERT//$'\n'/\\n}"
       client_key: "${POLICY_DB_CLIENT_KEY//$'\n'/\\n}"
       server_ca: "${POLICY_DB_SERVER_CA//$'\n'/\\n}"
+
 - name: syslog-client
   parameters:
     config:
       client_cert: "${SYSLOG_CLIENT_CERT//$'\n'/\\n}"
       client_key: "${SYSLOG_CLIENT_KEY//$'\n'/\\n}"
       server_ca: "${SYSLOG_CLIENT_CA//$'\n'/\\n}"
+
 - name: broker-catalog
   parameters:
     config: