Merge pull request #350 from cloudfoundry/scale-workers

Scale workers and fix concourse setup

Author: johha

flake.nix

@@ -24,16 +24,7 @@
       forAllSystems = nixpkgsLib.genAttrs supportedSystems;
 
       # Nixpkgs instantiated for supported system types.
-      nixpkgsFor = forAllSystems (system: import nixpkgs-repo {
-        inherit system;
-        config = {
-          ## Prefer allowing specific packages when unfree over general permission.
-          # allowUnfree = true;
-          allowUnfreePredicate = pkg: builtins.elem (nixpkgsLib.getName pkg) [
-             "terraform"
-           ];
-        };
-      });
+      nixpkgsFor = forAllSystems (system: import nixpkgs-repo { inherit system; });
     in {
       packages = forAllSystems (system:
         let
@@ -55,7 +46,7 @@
               kapp
               kubectl
               kubernetes-helm
-              terraform
+              opentofu
               terragrunt
               vendir
               ytt

terraform-modules/concourse/app/concourse.tf

@@ -40,6 +40,11 @@ data "helm_template" "concourse" {
     name = "web.replicas"
     value = var.gke_default_pool_node_count
   }
+
+  set {
+    name = "concourse.worker.runtime"
+    value = "containerd"
+  }
 }
 
 data "carvel_ytt" "concourse_app" {

terraform-modules/concourse/backend/files/vendir.lock.yml

@@ -3,7 +3,7 @@ directories:
 - contents:
   - githubRelease:
       tag: v0.10.3
-      url: https://api.github.com/repos/vmware-tanzu/carvel-secretgen-controller/releases/71524101
+      url: https://api.github.com/repos/carvel-dev/secretgen-controller/releases/71524101
     path: .
   path: config/carvel-secretgen-controller
 kind: LockConfig

terragrunt/concourse-wg-ci/app/.terraform.lock.hcl

@@ -21,7 +21,7 @@ concourse_github_mainTeam: "cloudfoundry:wg-app-runtime-interfaces-autoscaler-ap
 concourse_github_mainTeamUser: ""
 
 # Concourse helm chart
-concourse_helm_version: "17.3.1"
+concourse_helm_version: "17.2.0"
 
 # Module sources for the stack - git or local
 # ie.
@@ -46,7 +46,7 @@ sql_instance_backup_location: eu
 sql_instance_disk_size: 38
 
 # Other GKE vars
-gke_controlplane_version: 1.23.8-gke.1900
+gke_controlplane_version: 1.27.8-gke.1067004
 gke_cluster_ipv4_cidr: 10.104.0.0/14
 gke_services_ipv4_cidr_block: 10.108.0.0/20
 gke_master_ipv4_cidr_block: 172.16.0.32/28
@@ -58,10 +58,10 @@ gke_default_pool_ssd_count: 0
 
 # typical config for concourse worker is n4-standard-4 and ssd_count: 1
 # note: economy e2-standard machine can't use local ssd drives
-gke_workers_pool_machine_type: e2-standard-4
-gke_workers_pool_node_count: 2
+gke_workers_pool_machine_type: n2-standard-4
+gke_workers_pool_node_count: 4
 gke_workers_pool_autoscaling_max: 4
-gke_workers_pool_ssd_count: 0
+gke_workers_pool_ssd_count: 1
 
 # low number of preallocated ports will impact networking for heavy concourse pipelines
 gke_cloud_nat_min_ports_per_vm: 16384