From 6bc13defc843fce2fbc110feb8af1b55f5bd449a Mon Sep 17 00:00:00 2001
From: Adam Lewis <23342526+Adam-D-Lewis@users.noreply.github.com>
Date: Mon, 20 Jan 2025 17:58:59 -0600
Subject: [PATCH 01/38] add jhub apps service account with admin permissions

---
 .../kubernetes/services/jupyterhub/main.tf    | 21 +++++++++++++++++++
 .../services/jupyterhub/versions.tf           |  9 ++++++++
 2 files changed, 30 insertions(+)
 create mode 100644 src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/versions.tf

diff --git a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/main.tf b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/main.tf
index 9a0675fc85..9c6de3935a 100644
--- a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/main.tf
+++ b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/main.tf
@@ -23,6 +23,7 @@ locals {
   userscheduler_nodeselector_value = var.general-node-group.value
 }
 
+
 resource "kubernetes_secret" "jhub_apps_secrets" {
   metadata {
     name      = local.jhub_apps_secrets_name
@@ -36,6 +37,26 @@ resource "kubernetes_secret" "jhub_apps_secrets" {
   type = "Opaque"
 }
 
+resource "keycloak_user" "jhub_apps_service_account" {
+  realm_id = var.realm_id
+  username = "jhub-apps-sa"
+  # email    = "jhub-apps-sa@${var.external-url}"
+  enabled = true # not sure if they need to be enabled,  TODO: check
+}
+
+data "keycloak_group" "admin_group" {
+  realm_id = var.realm_id
+  name     = "admin"
+}
+
+
+resource "keycloak_user_groups" "jhub_apps_service_account_groups" {
+  realm_id   = var.realm_id
+  user_id    = keycloak_user.jhub_apps_service_account.id
+  group_ids  = [data.keycloak_group.admin_group.id]
+  exhaustive = true # remove all other groups
+}
+
 locals {
   jupyterhub_env_vars = [
     {
diff --git a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/versions.tf b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/versions.tf
new file mode 100644
index 0000000000..0ddb981e5e
--- /dev/null
+++ b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/versions.tf
@@ -0,0 +1,9 @@
+terraform {
+  required_providers {
+    keycloak = {
+      source  = "mrparkers/keycloak"
+      version = "3.7.0"
+    }
+  }
+  required_version = ">= 1.0"
+}

From 234baa25fc7b5d17c6d0dec2e9767d2252fcc830 Mon Sep 17 00:00:00 2001
From: Adam Lewis <23342526+Adam-D-Lewis@users.noreply.github.com>
Date: Tue, 21 Jan 2025 12:15:28 -0600
Subject: [PATCH 02/38] reduce permissions

---
 .../kubernetes/services/jupyterhub/main.tf    | 20 +++++++++----------
 .../services/keycloak-client/outputs.tf       |  7 +++++++
 2 files changed, 16 insertions(+), 11 deletions(-)

diff --git a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/main.tf b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/main.tf
index 9c6de3935a..92f841650e 100644
--- a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/main.tf
+++ b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/main.tf
@@ -37,24 +37,22 @@ resource "kubernetes_secret" "jhub_apps_secrets" {
   type = "Opaque"
 }
 
+
 resource "keycloak_user" "jhub_apps_service_account" {
   realm_id = var.realm_id
   username = "jhub-apps-sa"
   # email    = "jhub-apps-sa@${var.external-url}"
-  enabled = true # not sure if they need to be enabled,  TODO: check
-}
-
-data "keycloak_group" "admin_group" {
-  realm_id = var.realm_id
-  name     = "admin"
+  enabled = false # not sure if they need to be enabled,  TODO: check
 }
 
 
-resource "keycloak_user_groups" "jhub_apps_service_account_groups" {
-  realm_id   = var.realm_id
-  user_id    = keycloak_user.jhub_apps_service_account.id
-  group_ids  = [data.keycloak_group.admin_group.id]
-  exhaustive = true # remove all other groups
+resource "keycloak_user_roles" "allow_app_sharing_role" {
+  realm_id = var.realm_id
+  user_id  = keycloak_user.jhub_apps_service_account.id
+  role_ids = [
+    module.jupyterhub-openid-client.client_role_ids["allow-app-sharing-role"]
+  ]
+  exhaustive = true
 }
 
 locals {
diff --git a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/keycloak-client/outputs.tf b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/keycloak-client/outputs.tf
index 6077c22b0e..8f87eaf108 100644
--- a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/keycloak-client/outputs.tf
+++ b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/keycloak-client/outputs.tf
@@ -12,3 +12,10 @@ output "config" {
     callback_urls      = var.callback-url-paths
   }
 }
+
+output "client_role_ids" {
+  description = "Map of role names to their IDs"
+  value = {
+    for role_key, role in keycloak_role.default_client_roles : role_key => role.id
+  }
+}

From d6092713015fac94f75bc6615ff09730da31e8b6 Mon Sep 17 00:00:00 2001
From: Adam Lewis <23342526+Adam-D-Lewis@users.noreply.github.com>
Date: Tue, 21 Jan 2025 12:20:39 -0600
Subject: [PATCH 03/38] cleanup

---
 .../modules/kubernetes/services/jupyterhub/main.tf     | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/main.tf b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/main.tf
index 92f841650e..a1ac4f4417 100644
--- a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/main.tf
+++ b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/main.tf
@@ -23,7 +23,6 @@ locals {
   userscheduler_nodeselector_value = var.general-node-group.value
 }
 
-
 resource "kubernetes_secret" "jhub_apps_secrets" {
   metadata {
     name      = local.jhub_apps_secrets_name
@@ -39,14 +38,15 @@ resource "kubernetes_secret" "jhub_apps_secrets" {
 
 
 resource "keycloak_user" "jhub_apps_service_account" {
+  count    = var.jhub-apps-enabled ? 1 : 0
   realm_id = var.realm_id
-  username = "jhub-apps-sa"
-  # email    = "jhub-apps-sa@${var.external-url}"
-  enabled = false # not sure if they need to be enabled,  TODO: check
+  username = "service-account-jhub-apps"
+  enabled  = false
 }
 
 
-resource "keycloak_user_roles" "allow_app_sharing_role" {
+resource "keycloak_user_roles" "jhub_apps_sa_allow_app_sharing_role" {
+  count    = var.jhub-apps-enabled ? 1 : 0
   realm_id = var.realm_id
   user_id  = keycloak_user.jhub_apps_service_account.id
   role_ids = [

From 01d1d5d71f0e74f2bd824c357fb16fcd91d3e994 Mon Sep 17 00:00:00 2001
From: Adam Lewis <23342526+Adam-D-Lewis@users.noreply.github.com>
Date: Tue, 21 Jan 2025 16:40:15 -0600
Subject: [PATCH 04/38] consolidate calls

---
 .../kubernetes/services/conda-store/server.tf |  6 +-
 .../kubernetes/services/jupyterhub/main.tf    | 40 +++++-----
 .../services/jupyterhub/versions.tf           | 18 ++---
 .../services/keycloak-client/main.tf          | 76 +++++++++++++++----
 .../services/keycloak-client/outputs.tf       | 12 +--
 .../services/keycloak-client/variables.tf     | 18 ++++-
 6 files changed, 115 insertions(+), 55 deletions(-)

diff --git a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/conda-store/server.tf b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/conda-store/server.tf
index 8a29bc2d41..5931b258b3 100644
--- a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/conda-store/server.tf
+++ b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/conda-store/server.tf
@@ -70,9 +70,9 @@ module "conda-store-openid-client" {
     "https://${var.external-url}/conda-store/oauth_callback"
   ]
   service-accounts-enabled = true
-  service-account-roles = [
-    "view-realm", "view-users", "view-clients"
-  ]
+  service-account-roles = {
+    "realm-management" : ["view-realm", "view-users", "view-clients"]
+  }
 }
 
 
diff --git a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/main.tf b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/main.tf
index a1ac4f4417..3e35f30c71 100644
--- a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/main.tf
+++ b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/main.tf
@@ -37,23 +37,23 @@ resource "kubernetes_secret" "jhub_apps_secrets" {
 }
 
 
-resource "keycloak_user" "jhub_apps_service_account" {
-  count    = var.jhub-apps-enabled ? 1 : 0
-  realm_id = var.realm_id
-  username = "service-account-jhub-apps"
-  enabled  = false
-}
-
-
-resource "keycloak_user_roles" "jhub_apps_sa_allow_app_sharing_role" {
-  count    = var.jhub-apps-enabled ? 1 : 0
-  realm_id = var.realm_id
-  user_id  = keycloak_user.jhub_apps_service_account.id
-  role_ids = [
-    module.jupyterhub-openid-client.client_role_ids["allow-app-sharing-role"]
-  ]
-  exhaustive = true
-}
+# resource "keycloak_user" "jhub_apps_service_account" {
+#   count    = var.jhub-apps-enabled ? 1 : 0
+#   realm_id = var.realm_id
+#   username = "service-account-jhub-apps"
+#   enabled  = true
+# }
+
+
+# resource "keycloak_user_roles" "jhub_apps_sa_allow_app_sharing_role" {
+#   count    = var.jhub-apps-enabled ? 1 : 0
+#   realm_id = var.realm_id
+#   user_id  = keycloak_user.jhub_apps_service_account[0].id
+#   role_ids = [
+#     module.jupyterhub-openid-client.client_role_ids["allow-app-sharing-role"]
+#   ]
+#   exhaustive = true
+# }
 
 locals {
   jupyterhub_env_vars = [
@@ -363,9 +363,9 @@ module "jupyterhub-openid-client" {
   ]
   jupyterlab_profiles_mapper = true
   service-accounts-enabled   = true
-  service-account-roles = [
-    "view-realm", "view-users", "view-clients"
-  ]
+  service-account-roles = {
+    "realm-management" : ["view-realm", "view-users", "view-clients"],
+  "jupyterhub" = ["allow-app-sharing-role"] }
 }
 
 
diff --git a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/versions.tf b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/versions.tf
index 0ddb981e5e..b66ec63ebf 100644
--- a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/versions.tf
+++ b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/versions.tf
@@ -1,9 +1,9 @@
-terraform {
-  required_providers {
-    keycloak = {
-      source  = "mrparkers/keycloak"
-      version = "3.7.0"
-    }
-  }
-  required_version = ">= 1.0"
-}
+# terraform {
+#   required_providers {
+#     keycloak = {
+#       source  = "mrparkers/keycloak"
+#       version = "3.7.0"
+#     }
+#   }
+#   required_version = ">= 1.0"
+# }
diff --git a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/keycloak-client/main.tf b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/keycloak-client/main.tf
index e23aeb13c8..20338001f4 100644
--- a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/keycloak-client/main.tf
+++ b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/keycloak-client/main.tf
@@ -67,28 +67,76 @@ data "keycloak_realm" "master" {
   realm = "nebari"
 }
 
-data "keycloak_openid_client" "realm_management" {
-  realm_id  = var.realm_id
-  client_id = "realm-management"
-}
+# data "keycloak_openid_client" "realm_management" {
+#   realm_id  = var.realm_id
+#   client_id = "realm-management"
+# }
 
-data "keycloak_role" "main-service" {
-  for_each = toset(var.service-account-roles)
+# data "keycloak_role" "main-service" {
+#   for_each = toset(var.service-account-roles)
 
-  realm_id  = data.keycloak_realm.master.id
-  client_id = data.keycloak_openid_client.realm_management.id
-  name      = each.key
-}
+#   realm_id  = data.keycloak_realm.master.id
+#   client_id = data.keycloak_openid_client.realm_management.id
+#   name      = each.key
+# }
+
+
+# Get client data for each service account client
+data "keycloak_openid_client" "service_clients" {
+  for_each = var.service-account-roles
 
-resource "keycloak_openid_client_service_account_role" "main" {
-  for_each = toset(var.service-account-roles)
+  realm_id   = var.realm_id
+  client_id  = each.key
+  depends_on = [keycloak_openid_client.main]
+}
+
+# Get role data for each client's roles
+data "keycloak_role" "client_roles" {
+  for_each = {
+    for pair in flatten([
+      for client, roles in var.service-account-roles : [
+        for role in roles : {
+          key    = "${client}-${role}"
+          client = client
+          role   = role
+        }
+      ]
+    ]) : pair.key => pair
+  }
+
+  realm_id  = var.realm_id
+  client_id = data.keycloak_openid_client.service_clients[each.value.client].id
+  name      = each.value.role
+}
+
+resource "keycloak_openid_client_service_account_role" "client_roles" {
+  for_each = {
+    for pair in flatten([
+      for client, roles in var.service-account-roles : [
+        for role in roles : {
+          key    = "${client}-${role}"
+          client = client
+          role   = role
+        }
+      ]
+    ]) : pair.key => pair
+  }
 
   realm_id                = var.realm_id
   service_account_user_id = keycloak_openid_client.main.service_account_user_id
-  client_id               = data.keycloak_openid_client.realm_management.id
-  role                    = data.keycloak_role.main-service[each.key].name
+  client_id               = data.keycloak_openid_client.service_clients[each.value.client].id
+  role                    = data.keycloak_role.client_roles[each.key].name
 }
 
+# resource "keycloak_openid_client_service_account_role" "main" {
+#   for_each = toset(var.service-account-roles)
+
+#   realm_id                = var.realm_id
+#   service_account_user_id = keycloak_openid_client.main.service_account_user_id
+#   client_id               = data.keycloak_openid_client.realm_management.id
+#   role                    = data.keycloak_role.main-service[each.key].name
+# }
+
 
 resource "keycloak_role" "main" {
   for_each = toset(flatten(values(var.role_mapping)))
diff --git a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/keycloak-client/outputs.tf b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/keycloak-client/outputs.tf
index 8f87eaf108..617eab7c3a 100644
--- a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/keycloak-client/outputs.tf
+++ b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/keycloak-client/outputs.tf
@@ -13,9 +13,9 @@ output "config" {
   }
 }
 
-output "client_role_ids" {
-  description = "Map of role names to their IDs"
-  value = {
-    for role_key, role in keycloak_role.default_client_roles : role_key => role.id
-  }
-}
+# output "client_role_ids" {
+#   description = "Map of role names to their IDs"
+#   value = {
+#     for role_key, role in keycloak_role.default_client_roles : role_key => role.id
+#   }
+# }
diff --git a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/keycloak-client/variables.tf b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/keycloak-client/variables.tf
index 7626cc2b93..7ab2a08dbf 100644
--- a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/keycloak-client/variables.tf
+++ b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/keycloak-client/variables.tf
@@ -22,10 +22,22 @@ variable "service-accounts-enabled" {
   default     = false
 }
 
+# variable "service-account-roles" {
+#   description = "Realm roles to be granted to the service account. Requires setting service-accounts-enabled to true."
+#   type        = list(string)
+#   default     = []
+# }
+
 variable "service-account-roles" {
-  description = "Roles to be granted to the service account. Requires setting service-accounts-enabled to true."
-  type        = list(string)
-  default     = []
+  description = <<-EOT
+  List of client roles to be granted to the service account client. Requires setting service-accounts-enabled to true.
+
+  e.g. {
+    \"my-client\": [\"my-role\"],
+  }
+  EOT
+  type        = map(list(string))
+  default     = {}
 }
 
 

From 1bfe644737c168890f54988debc2dbfbb0506b37 Mon Sep 17 00:00:00 2001
From: Adam Lewis <23342526+Adam-D-Lewis@users.noreply.github.com>
Date: Tue, 21 Jan 2025 17:39:20 -0600
Subject: [PATCH 05/38] revert to non service account user for jhub apps
 startup apps

---
 .../kubernetes/services/jupyterhub/main.tf    | 36 +++++++++----------
 .../services/jupyterhub/versions.tf           | 18 +++++-----
 .../services/keycloak-client/outputs.tf       | 12 +++----
 3 files changed, 33 insertions(+), 33 deletions(-)

diff --git a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/main.tf b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/main.tf
index 3e35f30c71..f8c93c2870 100644
--- a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/main.tf
+++ b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/main.tf
@@ -37,23 +37,23 @@ resource "kubernetes_secret" "jhub_apps_secrets" {
 }
 
 
-# resource "keycloak_user" "jhub_apps_service_account" {
-#   count    = var.jhub-apps-enabled ? 1 : 0
-#   realm_id = var.realm_id
-#   username = "service-account-jhub-apps"
-#   enabled  = true
-# }
-
-
-# resource "keycloak_user_roles" "jhub_apps_sa_allow_app_sharing_role" {
-#   count    = var.jhub-apps-enabled ? 1 : 0
-#   realm_id = var.realm_id
-#   user_id  = keycloak_user.jhub_apps_service_account[0].id
-#   role_ids = [
-#     module.jupyterhub-openid-client.client_role_ids["allow-app-sharing-role"]
-#   ]
-#   exhaustive = true
-# }
+resource "keycloak_user" "jhub_apps_service_account" {
+  count    = var.jhub-apps-enabled ? 1 : 0
+  realm_id = var.realm_id
+  username = "service-account-jhub-apps"
+  enabled  = true
+}
+
+
+resource "keycloak_user_roles" "jhub_apps_sa_allow_app_sharing_role" {
+  count    = var.jhub-apps-enabled ? 1 : 0
+  realm_id = var.realm_id
+  user_id  = keycloak_user.jhub_apps_service_account[0].id
+  role_ids = [
+    module.jupyterhub-openid-client.client_role_ids["allow-app-sharing-role"]
+  ]
+  exhaustive = true
+}
 
 locals {
   jupyterhub_env_vars = [
@@ -365,7 +365,7 @@ module "jupyterhub-openid-client" {
   service-accounts-enabled   = true
   service-account-roles = {
     "realm-management" : ["view-realm", "view-users", "view-clients"],
-  "jupyterhub" = ["allow-app-sharing-role"] }
+  }
 }
 
 
diff --git a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/versions.tf b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/versions.tf
index b66ec63ebf..0ddb981e5e 100644
--- a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/versions.tf
+++ b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/versions.tf
@@ -1,9 +1,9 @@
-# terraform {
-#   required_providers {
-#     keycloak = {
-#       source  = "mrparkers/keycloak"
-#       version = "3.7.0"
-#     }
-#   }
-#   required_version = ">= 1.0"
-# }
+terraform {
+  required_providers {
+    keycloak = {
+      source  = "mrparkers/keycloak"
+      version = "3.7.0"
+    }
+  }
+  required_version = ">= 1.0"
+}
diff --git a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/keycloak-client/outputs.tf b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/keycloak-client/outputs.tf
index 617eab7c3a..8f87eaf108 100644
--- a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/keycloak-client/outputs.tf
+++ b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/keycloak-client/outputs.tf
@@ -13,9 +13,9 @@ output "config" {
   }
 }
 
-# output "client_role_ids" {
-#   description = "Map of role names to their IDs"
-#   value = {
-#     for role_key, role in keycloak_role.default_client_roles : role_key => role.id
-#   }
-# }
+output "client_role_ids" {
+  description = "Map of role names to their IDs"
+  value = {
+    for role_key, role in keycloak_role.default_client_roles : role_key => role.id
+  }
+}

From a4943bbaa5d0676372e95334da805f19c59c223b Mon Sep 17 00:00:00 2001
From: Adam Lewis <23342526+Adam-D-Lewis@users.noreply.github.com>
Date: Tue, 21 Jan 2025 17:42:36 -0600
Subject: [PATCH 06/38] cleanup

---
 .../kubernetes/services/jupyterhub/main.tf    |  2 +-
 .../services/keycloak-client/main.tf          | 23 -------------------
 2 files changed, 1 insertion(+), 24 deletions(-)

diff --git a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/main.tf b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/main.tf
index f8c93c2870..519d250ed0 100644
--- a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/main.tf
+++ b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/main.tf
@@ -40,7 +40,7 @@ resource "kubernetes_secret" "jhub_apps_secrets" {
 resource "keycloak_user" "jhub_apps_service_account" {
   count    = var.jhub-apps-enabled ? 1 : 0
   realm_id = var.realm_id
-  username = "service-account-jhub-apps"
+  username = "jhub-apps-sa"
   enabled  = true
 }
 
diff --git a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/keycloak-client/main.tf b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/keycloak-client/main.tf
index 20338001f4..20a37e580c 100644
--- a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/keycloak-client/main.tf
+++ b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/keycloak-client/main.tf
@@ -67,19 +67,6 @@ data "keycloak_realm" "master" {
   realm = "nebari"
 }
 
-# data "keycloak_openid_client" "realm_management" {
-#   realm_id  = var.realm_id
-#   client_id = "realm-management"
-# }
-
-# data "keycloak_role" "main-service" {
-#   for_each = toset(var.service-account-roles)
-
-#   realm_id  = data.keycloak_realm.master.id
-#   client_id = data.keycloak_openid_client.realm_management.id
-#   name      = each.key
-# }
-
 
 # Get client data for each service account client
 data "keycloak_openid_client" "service_clients" {
@@ -128,16 +115,6 @@ resource "keycloak_openid_client_service_account_role" "client_roles" {
   role                    = data.keycloak_role.client_roles[each.key].name
 }
 
-# resource "keycloak_openid_client_service_account_role" "main" {
-#   for_each = toset(var.service-account-roles)
-
-#   realm_id                = var.realm_id
-#   service_account_user_id = keycloak_openid_client.main.service_account_user_id
-#   client_id               = data.keycloak_openid_client.realm_management.id
-#   role                    = data.keycloak_role.main-service[each.key].name
-# }
-
-
 resource "keycloak_role" "main" {
   for_each = toset(flatten(values(var.role_mapping)))
 

From 5f9834ac92e9d49263dcc440967ce3c903aa7387 Mon Sep 17 00:00:00 2001
From: Adam Lewis <23342526+Adam-D-Lewis@users.noreply.github.com>
Date: Mon, 27 Jan 2025 17:51:55 -0600
Subject: [PATCH 07/38] hacky, but works

---
 .../jupyterhub/files/jupyterhub/02-spawner.py |  7 +++++
 .../files/jupyterhub/03-profiles.py           |  3 ++
 .../jupyterhub/files/jupyterhub/04-auth.py    | 31 +++++++++++++++++--
 .../kubernetes/services/jupyterhub/main.tf    |  3 +-
 4 files changed, 40 insertions(+), 4 deletions(-)

diff --git a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/files/jupyterhub/02-spawner.py b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/files/jupyterhub/02-spawner.py
index 09bb649c01..ff7ec79745 100644
--- a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/files/jupyterhub/02-spawner.py
+++ b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/files/jupyterhub/02-spawner.py
@@ -1,5 +1,6 @@
 import inspect
 import json
+import logging
 
 import kubernetes.client.models
 from tornado import gen
@@ -14,6 +15,12 @@
 @gen.coroutine
 def get_username_hook(spawner):
     auth_state = yield spawner.user.get_auth_state()
+    if auth_state is not None:
+        logging.warning(f"======auth_state: {auth_state}")
+    else:
+        logging.warning("======auth_state is None")
+        if spawner.user.name == "service-account-jupyterhub":
+            auth_state = yield spawner.authenticator.authenticate_service_account()
     username = auth_state["oauth_user"]["preferred_username"]
 
     spawner.environment.update(
diff --git a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/files/jupyterhub/03-profiles.py b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/files/jupyterhub/03-profiles.py
index b298ae5ae1..b032694480 100644
--- a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/files/jupyterhub/03-profiles.py
+++ b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/files/jupyterhub/03-profiles.py
@@ -554,6 +554,9 @@ def render_profiles(spawner):
     # userinfo request to have the groups in the key
     # "auth_state.oauth_user.groups"
     auth_state = yield spawner.user.get_auth_state()
+    if not auth_state:
+        if spawner.user.name == "service-account-jupyterhub":
+            auth_state = yield spawner.authenticator.authenticate_service_account()
 
     username = auth_state["oauth_user"]["preferred_username"]
 
diff --git a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/files/jupyterhub/04-auth.py b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/files/jupyterhub/04-auth.py
index 2694b2a34e..4f52e69463 100644
--- a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/files/jupyterhub/04-auth.py
+++ b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/files/jupyterhub/04-auth.py
@@ -30,6 +30,27 @@ class KeyCloakOAuthenticator(GenericOAuthenticator):
 
     reset_managed_roles_on_startup = Bool(True)
 
+    async def authenticate_service_account(self):
+        token_info = await self._get_token_info()
+
+        # Get user info using the access token
+        user_info = await self.token_to_user(token_info)
+
+        # Get/set username
+        username = self.user_info_to_username(user_info)
+        username = self.normalize_username(username)
+
+        # Build auth model similar to OAuth flow
+        auth_model = {
+            "name": username,
+            "admin": True if username in self.admin_users else None,
+            "auth_state": self.build_auth_state_dict(token_info, user_info),
+        }
+
+        auth_model = await self.update_auth_model(auth_model)
+
+        return auth_model["auth_state"]
+
     async def update_auth_model(self, auth_model):
         """Updates and returns the auth_model dict.
         This function is called every time a user authenticates with JupyterHub, as in
@@ -307,7 +328,7 @@ def _get_user_roles(self, user_info):
             )
             return set()
 
-    async def _get_token(self) -> str:
+    async def _get_token_info(self) -> str:
         http = self.http_client
 
         body = urllib.parse.urlencode(
@@ -322,8 +343,12 @@ async def _get_token(self) -> str:
             method="POST",
             body=body,
         )
-        data = json.loads(response.body)
-        return data["access_token"]  # type: ignore[no-any-return]
+        token_info = json.loads(response.body)
+        return token_info
+
+    async def _get_token(self) -> str:
+        token_info = await self._get_token_info()
+        return token_info["access_token"]  # type: ignore[no-any-return]
 
     async def _fetch_api(self, endpoint: str, token: str):
         response = await self.http_client.fetch(
diff --git a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/main.tf b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/main.tf
index 519d250ed0..9469ee38e3 100644
--- a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/main.tf
+++ b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/main.tf
@@ -52,7 +52,8 @@ resource "keycloak_user_roles" "jhub_apps_sa_allow_app_sharing_role" {
   role_ids = [
     module.jupyterhub-openid-client.client_role_ids["allow-app-sharing-role"]
   ]
-  exhaustive = true
+  # include default roles as well
+  exhaustive = false
 }
 
 locals {

From 7e6204aa7fcf5cbb07c3ec91d26d8f63f5fa5c20 Mon Sep 17 00:00:00 2001
From: Adam Lewis <23342526+Adam-D-Lewis@users.noreply.github.com>
Date: Mon, 27 Jan 2025 17:55:17 -0600
Subject: [PATCH 08/38] add role to service account + cleanup

---
 .../template/modules/kubernetes/services/jupyterhub/main.tf  | 1 +
 .../modules/kubernetes/services/keycloak-client/variables.tf | 5 -----
 2 files changed, 1 insertion(+), 5 deletions(-)

diff --git a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/main.tf b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/main.tf
index 9469ee38e3..1a0884c60e 100644
--- a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/main.tf
+++ b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/main.tf
@@ -366,6 +366,7 @@ module "jupyterhub-openid-client" {
   service-accounts-enabled   = true
   service-account-roles = {
     "realm-management" : ["view-realm", "view-users", "view-clients"],
+    "jupyterhub" : ["allow-app-sharing-role"]
   }
 }
 
diff --git a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/keycloak-client/variables.tf b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/keycloak-client/variables.tf
index 7ab2a08dbf..22ba732411 100644
--- a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/keycloak-client/variables.tf
+++ b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/keycloak-client/variables.tf
@@ -22,11 +22,6 @@ variable "service-accounts-enabled" {
   default     = false
 }
 
-# variable "service-account-roles" {
-#   description = "Realm roles to be granted to the service account. Requires setting service-accounts-enabled to true."
-#   type        = list(string)
-#   default     = []
-# }
 
 variable "service-account-roles" {
   description = <<-EOT

From 2a3e49b68131914b069ca8bdc08f184bdeca6e76 Mon Sep 17 00:00:00 2001
From: Adam Lewis <23342526+Adam-D-Lewis@users.noreply.github.com>
Date: Mon, 27 Jan 2025 18:36:33 -0600
Subject: [PATCH 09/38] try to set service account auth state, but I don't
 think it's working

---
 .../services/jupyterhub/files/jupyterhub/02-spawner.py     | 3 +++
 .../services/jupyterhub/files/jupyterhub/04-auth.py        | 7 +++++++
 2 files changed, 10 insertions(+)

diff --git a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/files/jupyterhub/02-spawner.py b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/files/jupyterhub/02-spawner.py
index ff7ec79745..f5c504d584 100644
--- a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/files/jupyterhub/02-spawner.py
+++ b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/files/jupyterhub/02-spawner.py
@@ -20,7 +20,10 @@ def get_username_hook(spawner):
     else:
         logging.warning("======auth_state is None")
         if spawner.user.name == "service-account-jupyterhub":
+            logging.warning(f"========type(spawner.user): {type(spawner.user)}")
+            spawner.authenticator.set_service_account_auth_state(spawner.user)
             auth_state = yield spawner.authenticator.authenticate_service_account()
+
     username = auth_state["oauth_user"]["preferred_username"]
 
     spawner.environment.update(
diff --git a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/files/jupyterhub/04-auth.py b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/files/jupyterhub/04-auth.py
index 4f52e69463..410efeb2b1 100644
--- a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/files/jupyterhub/04-auth.py
+++ b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/files/jupyterhub/04-auth.py
@@ -30,6 +30,13 @@ class KeyCloakOAuthenticator(GenericOAuthenticator):
 
     reset_managed_roles_on_startup = Bool(True)
 
+    async def set_service_account_auth_state(self, user):
+
+        # get user from spawner.user, I think
+        service_account_auth_state = await self.authenticate_service_account()
+
+        await user.save_auth_state(service_account_auth_state)
+
     async def authenticate_service_account(self):
         token_info = await self._get_token_info()
 

From 110b0ee837f036a79b0a73437d8f8fb895f69ec3 Mon Sep 17 00:00:00 2001
From: Adam Lewis <23342526+Adam-D-Lewis@users.noreply.github.com>
Date: Tue, 28 Jan 2025 12:57:18 -0600
Subject: [PATCH 10/38] fix bug and set auth state for service account

---
 .../services/jupyterhub/files/jupyterhub/02-spawner.py      | 2 +-
 .../services/jupyterhub/files/jupyterhub/04-auth.py         | 6 +++++-
 2 files changed, 6 insertions(+), 2 deletions(-)

diff --git a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/files/jupyterhub/02-spawner.py b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/files/jupyterhub/02-spawner.py
index f5c504d584..e93979845d 100644
--- a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/files/jupyterhub/02-spawner.py
+++ b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/files/jupyterhub/02-spawner.py
@@ -21,7 +21,7 @@ def get_username_hook(spawner):
         logging.warning("======auth_state is None")
         if spawner.user.name == "service-account-jupyterhub":
             logging.warning(f"========type(spawner.user): {type(spawner.user)}")
-            spawner.authenticator.set_service_account_auth_state(spawner.user)
+            yield spawner.authenticator.set_service_account_auth_state(spawner.user)
             auth_state = yield spawner.authenticator.authenticate_service_account()
 
     username = auth_state["oauth_user"]["preferred_username"]
diff --git a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/files/jupyterhub/04-auth.py b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/files/jupyterhub/04-auth.py
index 410efeb2b1..17ab5fbd89 100644
--- a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/files/jupyterhub/04-auth.py
+++ b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/files/jupyterhub/04-auth.py
@@ -1,5 +1,6 @@
 import asyncio
 import json
+import logging
 import os
 import time
 import urllib
@@ -34,8 +35,11 @@ async def set_service_account_auth_state(self, user):
 
         # get user from spawner.user, I think
         service_account_auth_state = await self.authenticate_service_account()
-
         await user.save_auth_state(service_account_auth_state)
+        auth_state = await user.get_auth_state()
+        logging.warning(
+            f"======auth_state after save_auth_state/get_auth_state: {auth_state}"
+        )
 
     async def authenticate_service_account(self):
         token_info = await self._get_token_info()

From a0f4efe80eb8e044cd34dfa30eef8a1f6fca2811 Mon Sep 17 00:00:00 2001
From: Adam Lewis <23342526+Adam-D-Lewis@users.noreply.github.com>
Date: Tue, 28 Jan 2025 14:57:58 -0600
Subject: [PATCH 11/38] cleanup

---
 .../jupyterhub/files/jupyterhub/02-spawner.py | 25 ++++++++-----------
 .../jupyterhub/files/jupyterhub/04-auth.py    |  7 +-----
 .../services/keycloak-client/variables.tf     |  2 +-
 3 files changed, 12 insertions(+), 22 deletions(-)

diff --git a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/files/jupyterhub/02-spawner.py b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/files/jupyterhub/02-spawner.py
index e93979845d..b2c38da9de 100644
--- a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/files/jupyterhub/02-spawner.py
+++ b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/files/jupyterhub/02-spawner.py
@@ -1,9 +1,7 @@
 import inspect
 import json
-import logging
 
 import kubernetes.client.models
-from tornado import gen
 
 kubernetes.client.models.V1EndpointPort = (
     kubernetes.client.models.CoreV1EndpointPort
@@ -12,18 +10,8 @@
 from kubespawner import KubeSpawner  # noqa: E402
 
 
-@gen.coroutine
-def get_username_hook(spawner):
-    auth_state = yield spawner.user.get_auth_state()
-    if auth_state is not None:
-        logging.warning(f"======auth_state: {auth_state}")
-    else:
-        logging.warning("======auth_state is None")
-        if spawner.user.name == "service-account-jupyterhub":
-            logging.warning(f"========type(spawner.user): {type(spawner.user)}")
-            yield spawner.authenticator.set_service_account_auth_state(spawner.user)
-            auth_state = yield spawner.authenticator.authenticate_service_account()
-
+async def get_username_hook(spawner):
+    auth_state = await spawner.user.get_auth_state()
     username = auth_state["oauth_user"]["preferred_username"]
 
     spawner.environment.update(
@@ -33,6 +21,13 @@ def get_username_hook(spawner):
     )
 
 
+async def pre_spawn_hook(spawner):
+    # if we are starting a service account pod, set/update auth_state
+    if spawner.user.name == "service-account-jupyterhub":
+        await spawner.authenticator.set_service_account_auth_state(spawner.user)
+    await get_username_hook(spawner)
+
+
 def get_conda_store_environments(user_info: dict):
     import urllib3
     import yarl
@@ -54,7 +49,7 @@ def get_conda_store_environments(user_info: dict):
     return [f"{env['namespace']['name']}-{env['name']}" for env in j.get("data", [])]
 
 
-c.Spawner.pre_spawn_hook = get_username_hook
+c.Spawner.pre_spawn_hook = pre_spawn_hook
 
 c.JupyterHub.allow_named_servers = False
 c.JupyterHub.spawner_class = KubeSpawner
diff --git a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/files/jupyterhub/04-auth.py b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/files/jupyterhub/04-auth.py
index 17ab5fbd89..b02224c3c0 100644
--- a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/files/jupyterhub/04-auth.py
+++ b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/files/jupyterhub/04-auth.py
@@ -32,14 +32,9 @@ class KeyCloakOAuthenticator(GenericOAuthenticator):
     reset_managed_roles_on_startup = Bool(True)
 
     async def set_service_account_auth_state(self, user):
-
-        # get user from spawner.user, I think
         service_account_auth_state = await self.authenticate_service_account()
         await user.save_auth_state(service_account_auth_state)
-        auth_state = await user.get_auth_state()
-        logging.warning(
-            f"======auth_state after save_auth_state/get_auth_state: {auth_state}"
-        )
+        logging.info(f'Auth state set for service account "{user.name}"')
 
     async def authenticate_service_account(self):
         token_info = await self._get_token_info()
diff --git a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/keycloak-client/variables.tf b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/keycloak-client/variables.tf
index 22ba732411..171e87b1c6 100644
--- a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/keycloak-client/variables.tf
+++ b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/keycloak-client/variables.tf
@@ -28,7 +28,7 @@ variable "service-account-roles" {
   List of client roles to be granted to the service account client. Requires setting service-accounts-enabled to true.
 
   e.g. {
-    \"my-client\": [\"my-role\"],
+    "my-client": ["my-role"],
   }
   EOT
   type        = map(list(string))

From f180f07bdd0f0b95e4271bed2da1369b0ea3f792 Mon Sep 17 00:00:00 2001
From: Adam Lewis <23342526+Adam-D-Lewis@users.noreply.github.com>
Date: Tue, 28 Jan 2025 15:01:26 -0600
Subject: [PATCH 12/38] cleanup

---
 .../jupyterhub/files/jupyterhub/03-profiles.py         | 10 ++--------
 1 file changed, 2 insertions(+), 8 deletions(-)

diff --git a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/files/jupyterhub/03-profiles.py b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/files/jupyterhub/03-profiles.py
index b032694480..9ddc0133dc 100644
--- a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/files/jupyterhub/03-profiles.py
+++ b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/files/jupyterhub/03-profiles.py
@@ -5,7 +5,6 @@
 from pathlib import Path
 
 import z2jh
-from tornado import gen
 
 
 def base_profile_home_mounts(username):
@@ -547,17 +546,12 @@ def preserve_envvars(spawner):
     return profile
 
 
-@gen.coroutine
-def render_profiles(spawner):
+async def render_profiles(spawner):
     # jupyterhub does not yet manage groups but it will soon
     # so for now we rely on auth_state from the keycloak
     # userinfo request to have the groups in the key
     # "auth_state.oauth_user.groups"
-    auth_state = yield spawner.user.get_auth_state()
-    if not auth_state:
-        if spawner.user.name == "service-account-jupyterhub":
-            auth_state = yield spawner.authenticator.authenticate_service_account()
-
+    auth_state = await spawner.user.get_auth_state()
     username = auth_state["oauth_user"]["preferred_username"]
 
     # only return the lowest level group name

From 6406e82de1e3a536823169d1f2a0e55cb34e5177 Mon Sep 17 00:00:00 2001
From: Adam Lewis <23342526+Adam-D-Lewis@users.noreply.github.com>
Date: Tue, 28 Jan 2025 15:29:01 -0600
Subject: [PATCH 13/38] cleanup

---
 .../jupyterhub/files/jupyterhub/02-spawner.py   |  2 +-
 .../jupyterhub/files/jupyterhub/04-auth.py      | 17 ++++++++++++-----
 2 files changed, 13 insertions(+), 6 deletions(-)

diff --git a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/files/jupyterhub/02-spawner.py b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/files/jupyterhub/02-spawner.py
index b2c38da9de..b50a7fe47b 100644
--- a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/files/jupyterhub/02-spawner.py
+++ b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/files/jupyterhub/02-spawner.py
@@ -23,7 +23,7 @@ async def get_username_hook(spawner):
 
 async def pre_spawn_hook(spawner):
     # if we are starting a service account pod, set/update auth_state
-    if spawner.user.name == "service-account-jupyterhub":
+    if spawner.user.name == spawner.authenticator.JHUB_SERVICE_ACCOUNT_NAME:
         await spawner.authenticator.set_service_account_auth_state(spawner.user)
     await get_username_hook(spawner)
 
diff --git a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/files/jupyterhub/04-auth.py b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/files/jupyterhub/04-auth.py
index b02224c3c0..636d306477 100644
--- a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/files/jupyterhub/04-auth.py
+++ b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/files/jupyterhub/04-auth.py
@@ -19,6 +19,8 @@ class KeyCloakOAuthenticator(GenericOAuthenticator):
     feature added in JupyterHub 5.0 (https://github.com/jupyterhub/jupyterhub/pull/4748).
     """
 
+    JHUB_SERVICE_ACCOUNT_NAME = "service-account-jupyterhub"
+
     claim_roles_key = Union(
         [Unicode(os.environ.get("OAUTH2_ROLES_KEY", "groups")), Callable()],
         config=True,
@@ -31,10 +33,15 @@ class KeyCloakOAuthenticator(GenericOAuthenticator):
 
     reset_managed_roles_on_startup = Bool(True)
 
-    async def set_service_account_auth_state(self, user):
-        service_account_auth_state = await self.authenticate_service_account()
-        await user.save_auth_state(service_account_auth_state)
-        logging.info(f'Auth state set for service account "{user.name}"')
+    async def set_jhub_service_account_auth_state(self, user):
+        auth_model = await self.authenticate_service_account()
+        if user.name != self.JHUB_SERVICE_ACCOUNT_NAME:
+            raise ValueError(
+                'User name "{user.name}" does not match service account name "{self.JHUB_SERVICE_ACCOUNT_NAME}"'
+            )
+
+        await user.save_auth_state(auth_model["auth_state"])
+        logging.info(f'Auth state set for service account: "{user.name}"')
 
     async def authenticate_service_account(self):
         token_info = await self._get_token_info()
@@ -55,7 +62,7 @@ async def authenticate_service_account(self):
 
         auth_model = await self.update_auth_model(auth_model)
 
-        return auth_model["auth_state"]
+        return auth_model
 
     async def update_auth_model(self, auth_model):
         """Updates and returns the auth_model dict.

From 325a601ed4751546f4a2724f8c479496d7c2fde6 Mon Sep 17 00:00:00 2001
From: Adam Lewis <23342526+Adam-D-Lewis@users.noreply.github.com>
Date: Tue, 28 Jan 2025 16:11:00 -0600
Subject: [PATCH 14/38] make service account name a variable

---
 .../services/jupyterhub/files/jupyterhub/04-auth.py       | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/files/jupyterhub/04-auth.py b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/files/jupyterhub/04-auth.py
index 636d306477..4916235404 100644
--- a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/files/jupyterhub/04-auth.py
+++ b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/files/jupyterhub/04-auth.py
@@ -9,7 +9,7 @@
 from jupyterhub import scopes
 from jupyterhub.traitlets import Callable
 from oauthenticator.generic import GenericOAuthenticator
-from traitlets import Bool, Unicode, Union
+from traitlets import Bool, Unicode, Union, default
 
 
 class KeyCloakOAuthenticator(GenericOAuthenticator):
@@ -19,7 +19,11 @@ class KeyCloakOAuthenticator(GenericOAuthenticator):
     feature added in JupyterHub 5.0 (https://github.com/jupyterhub/jupyterhub/pull/4748).
     """
 
-    JHUB_SERVICE_ACCOUNT_NAME = "service-account-jupyterhub"
+    JHUB_SERVICE_ACCOUNT_NAME = Unicode()
+
+    @default("JHUB_SERVICE_ACCOUNT_NAME")
+    def _default_jhub_service_account_name(self):
+        return f"service-account-{self.client_id}"
 
     claim_roles_key = Union(
         [Unicode(os.environ.get("OAUTH2_ROLES_KEY", "groups")), Callable()],

From 64d3e0b7e52e34ea58cb7dadfbf866185a5c5b36 Mon Sep 17 00:00:00 2001
From: Adam Lewis <23342526+Adam-D-Lewis@users.noreply.github.com>
Date: Tue, 28 Jan 2025 16:11:17 -0600
Subject: [PATCH 15/38] rename id to uuid for clarity

---
 .../jupyterhub/files/jupyterhub/04-auth.py       | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/files/jupyterhub/04-auth.py b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/files/jupyterhub/04-auth.py
index 4916235404..d98fc6061e 100644
--- a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/files/jupyterhub/04-auth.py
+++ b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/files/jupyterhub/04-auth.py
@@ -84,15 +84,15 @@ async def update_auth_model(self, auth_model):
         user_id = auth_model["auth_state"]["oauth_user"]["sub"]
         token = await self._get_token()
 
-        jupyterhub_client_id = await self._get_jupyterhub_client_id(token=token)
+        jupyterhub_client_uuid = await self._get_jupyterhub_client_uuid(token=token)
         user_info = auth_model["auth_state"][self.user_auth_state_key]
         user_roles_from_claims = self._get_user_roles(user_info=user_info)
         keycloak_api_call_start = time.time()
         user_roles = await self._get_client_roles_for_user(
-            user_id=user_id, client_id=jupyterhub_client_id, token=token
+            user_id=user_id, client_id=jupyterhub_client_uuid, token=token
         )
         user_roles_rich = await self._get_roles_with_attributes(
-            roles=user_roles, client_id=jupyterhub_client_id, token=token
+            roles=user_roles, client_id=jupyterhub_client_uuid, token=token
         )
 
         # Include which groups have permission to mount shared directories (user by
@@ -101,7 +101,7 @@ async def update_auth_model(self, auth_model):
             await self.get_client_groups_with_mount_permissions(
                 user_groups=auth_model["auth_state"]["oauth_user"]["groups"],
                 user_roles=user_roles_rich,
-                client_id=jupyterhub_client_id,
+                client_id=jupyterhub_client_uuid,
                 token=token,
             )
         )
@@ -152,7 +152,7 @@ async def _get_jupyterhub_client_roles(self, jupyterhub_client_id, token):
         )
         return client_roles_rich
 
-    async def _get_jupyterhub_client_id(self, token):
+    async def _get_jupyterhub_client_uuid(self, token):
         # Get the clients list to find the "id" of "jupyterhub" client.
         clients_data = await self._fetch_api(endpoint="clients/", token=token)
         jupyterhub_clients = [
@@ -169,9 +169,9 @@ async def load_managed_roles(self):
                 "Managed roles can only be loaded when `manage_roles` is True"
             )
         token = await self._get_token()
-        jupyterhub_client_id = await self._get_jupyterhub_client_id(token=token)
+        jupyterhub_client_uuid = await self._get_jupyterhub_client_uuid(token=token)
         client_roles_rich = await self._get_jupyterhub_client_roles(
-            jupyterhub_client_id=jupyterhub_client_id, token=token
+            jupyterhub_client_id=jupyterhub_client_uuid, token=token
         )
 
         # Includes roles like "default-roles-nebari", "offline_access", "uma_authorization"
@@ -206,7 +206,7 @@ async def load_managed_roles(self):
                 await self._get_users_and_groups_for_role(
                     role_name,
                     token=token,
-                    client_id=jupyterhub_client_id,
+                    client_id=jupyterhub_client_uuid,
                 )
             )
 

From cb775e0c3687b0bc9f7ebbad58c70b06c281b1db Mon Sep 17 00:00:00 2001
From: Adam Lewis <23342526+Adam-D-Lewis@users.noreply.github.com>
Date: Tue, 28 Jan 2025 16:12:56 -0600
Subject: [PATCH 16/38] remove unneeded code

---
 .../kubernetes/services/jupyterhub/main.tf    | 19 -------------------
 1 file changed, 19 deletions(-)

diff --git a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/main.tf b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/main.tf
index 1a0884c60e..300ca47119 100644
--- a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/main.tf
+++ b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/main.tf
@@ -37,25 +37,6 @@ resource "kubernetes_secret" "jhub_apps_secrets" {
 }
 
 
-resource "keycloak_user" "jhub_apps_service_account" {
-  count    = var.jhub-apps-enabled ? 1 : 0
-  realm_id = var.realm_id
-  username = "jhub-apps-sa"
-  enabled  = true
-}
-
-
-resource "keycloak_user_roles" "jhub_apps_sa_allow_app_sharing_role" {
-  count    = var.jhub-apps-enabled ? 1 : 0
-  realm_id = var.realm_id
-  user_id  = keycloak_user.jhub_apps_service_account[0].id
-  role_ids = [
-    module.jupyterhub-openid-client.client_role_ids["allow-app-sharing-role"]
-  ]
-  # include default roles as well
-  exhaustive = false
-}
-
 locals {
   jupyterhub_env_vars = [
     {

From 59078cc26bbe0224adbe9648f0e27832b4b40871 Mon Sep 17 00:00:00 2001
From: Adam Lewis <23342526+Adam-D-Lewis@users.noreply.github.com>
Date: Tue, 28 Jan 2025 16:13:21 -0600
Subject: [PATCH 17/38] fix

---
 .../template/modules/kubernetes/services/jupyterhub/main.tf      | 1 -
 1 file changed, 1 deletion(-)

diff --git a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/main.tf b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/main.tf
index 300ca47119..54ca96b655 100644
--- a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/main.tf
+++ b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/main.tf
@@ -36,7 +36,6 @@ resource "kubernetes_secret" "jhub_apps_secrets" {
   type = "Opaque"
 }
 
-
 locals {
   jupyterhub_env_vars = [
     {

From f799f3ee9d56bfdb97e9b033c5d7127e74ac1a09 Mon Sep 17 00:00:00 2001
From: Adam Lewis <23342526+Adam-D-Lewis@users.noreply.github.com>
Date: Tue, 28 Jan 2025 16:27:40 -0600
Subject: [PATCH 18/38] cleanup

---
 .../services/jupyterhub/files/jupyterhub/02-spawner.py      | 2 +-
 .../services/jupyterhub/files/jupyterhub/03-profiles.py     | 6 ++++--
 .../services/jupyterhub/files/jupyterhub/04-auth.py         | 3 +--
 3 files changed, 6 insertions(+), 5 deletions(-)

diff --git a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/files/jupyterhub/02-spawner.py b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/files/jupyterhub/02-spawner.py
index b50a7fe47b..e92469cbcd 100644
--- a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/files/jupyterhub/02-spawner.py
+++ b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/files/jupyterhub/02-spawner.py
@@ -24,7 +24,7 @@ async def get_username_hook(spawner):
 async def pre_spawn_hook(spawner):
     # if we are starting a service account pod, set/update auth_state
     if spawner.user.name == spawner.authenticator.JHUB_SERVICE_ACCOUNT_NAME:
-        await spawner.authenticator.set_service_account_auth_state(spawner.user)
+        await spawner.authenticator.set_jhub_service_account_auth_state(spawner.user)
     await get_username_hook(spawner)
 
 
diff --git a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/files/jupyterhub/03-profiles.py b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/files/jupyterhub/03-profiles.py
index 9ddc0133dc..5741ab89f9 100644
--- a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/files/jupyterhub/03-profiles.py
+++ b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/files/jupyterhub/03-profiles.py
@@ -5,6 +5,7 @@
 from pathlib import Path
 
 import z2jh
+from tornado import gen
 
 
 def base_profile_home_mounts(username):
@@ -546,12 +547,13 @@ def preserve_envvars(spawner):
     return profile
 
 
-async def render_profiles(spawner):
+@gen.coroutine
+def render_profiles(spawner):
     # jupyterhub does not yet manage groups but it will soon
     # so for now we rely on auth_state from the keycloak
     # userinfo request to have the groups in the key
     # "auth_state.oauth_user.groups"
-    auth_state = await spawner.user.get_auth_state()
+    auth_state = yield spawner.user.get_auth_state()
     username = auth_state["oauth_user"]["preferred_username"]
 
     # only return the lowest level group name
diff --git a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/files/jupyterhub/04-auth.py b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/files/jupyterhub/04-auth.py
index d98fc6061e..a400884f26 100644
--- a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/files/jupyterhub/04-auth.py
+++ b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/files/jupyterhub/04-auth.py
@@ -38,12 +38,11 @@ def _default_jhub_service_account_name(self):
     reset_managed_roles_on_startup = Bool(True)
 
     async def set_jhub_service_account_auth_state(self, user):
-        auth_model = await self.authenticate_service_account()
         if user.name != self.JHUB_SERVICE_ACCOUNT_NAME:
             raise ValueError(
                 'User name "{user.name}" does not match service account name "{self.JHUB_SERVICE_ACCOUNT_NAME}"'
             )
-
+        auth_model = await self.authenticate_service_account()
         await user.save_auth_state(auth_model["auth_state"])
         logging.info(f'Auth state set for service account: "{user.name}"')
 

From 21d08800fb85e88c38d9f1090b938cc4c626b878 Mon Sep 17 00:00:00 2001
From: Adam Lewis <23342526+Adam-D-Lewis@users.noreply.github.com>
Date: Tue, 28 Jan 2025 16:30:47 -0600
Subject: [PATCH 19/38] clarify docstring

---
 .../modules/kubernetes/services/keycloak-client/variables.tf    | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/keycloak-client/variables.tf b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/keycloak-client/variables.tf
index 171e87b1c6..80167fdffa 100644
--- a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/keycloak-client/variables.tf
+++ b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/keycloak-client/variables.tf
@@ -25,7 +25,7 @@ variable "service-accounts-enabled" {
 
 variable "service-account-roles" {
   description = <<-EOT
-  List of client roles to be granted to the service account client. Requires setting service-accounts-enabled to true.
+  Map of client to client-roles to be granted to the service account client. Requires setting service-accounts-enabled to true.
 
   e.g. {
     "my-client": ["my-role"],

From 0be385113d1c05c1e5c7b89c4980a1640a1f8f43 Mon Sep 17 00:00:00 2001
From: Adam Lewis <23342526+Adam-D-Lewis@users.noreply.github.com>
Date: Tue, 28 Jan 2025 16:31:50 -0600
Subject: [PATCH 20/38] clarify docstring

---
 .../modules/kubernetes/services/keycloak-client/variables.tf    | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/keycloak-client/variables.tf b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/keycloak-client/variables.tf
index 80167fdffa..08ce465dff 100644
--- a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/keycloak-client/variables.tf
+++ b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/keycloak-client/variables.tf
@@ -28,7 +28,7 @@ variable "service-account-roles" {
   Map of client to client-roles to be granted to the service account client. Requires setting service-accounts-enabled to true.
 
   e.g. {
-    "my-client": ["my-role"],
+    "my-client": ["my-client-role"],
   }
   EOT
   type        = map(list(string))

From 2fb4fa8089d4e55bf3edb944592da12cc503b58c Mon Sep 17 00:00:00 2001
From: Adam Lewis <23342526+Adam-D-Lewis@users.noreply.github.com>
Date: Wed, 29 Jan 2025 13:31:44 -0600
Subject: [PATCH 21/38] fix buffer full deadlock

---
 src/_nebari/utils.py | 88 +++++++++++++++++++++++++++++++++++---------
 1 file changed, 70 insertions(+), 18 deletions(-)

diff --git a/src/_nebari/utils.py b/src/_nebari/utils.py
index 48b8a91e9b..ef7153dca1 100644
--- a/src/_nebari/utils.py
+++ b/src/_nebari/utils.py
@@ -5,6 +5,7 @@
 import os
 import re
 import secrets
+import selectors
 import signal
 import string
 import subprocess
@@ -46,6 +47,67 @@ def change_directory(directory):
     os.chdir(current_directory)
 
 
+def strip_ansi_errors(line):
+    """Strips red ANSI escape code from a string."""
+    ansi_escape = re.compile(r"\x1b\[31m")
+    stripped_line = ansi_escape.sub("", line.decode("utf-8"))
+    return stripped_line.encode("utf-8")
+
+
+def process_streams(
+    process, line_prefix, strip_errors, print_stdout=True, print_stderr=True
+):
+    sel = selectors.DefaultSelector()
+    sel.register(process.stdout, selectors.EVENT_READ, data="stdout")
+    if process.stderr and process.stderr != process.stdout:
+        sel.register(process.stderr, selectors.EVENT_READ, data="stderr")
+
+    outputs = {"stdout": [], "stderr": []}
+    partial = {"stdout": b"", "stderr": b""}
+
+    try:
+        while True:
+            events = sel.select(timeout=0.1)
+            if not events and process.poll() is not None:
+                break
+
+            for key, _ in events:
+                data = key.fileobj.read1(8192)
+                if not data:
+                    sel.unregister(key.fileobj)
+                    continue
+
+                stream_name = key.data
+                chunk = partial[stream_name] + data
+                lines = chunk.split(b"\n")
+                partial[stream_name] = lines[-1]
+
+                for line in lines[:-1]:
+                    line_w_newline = line + b"\n"
+                    if strip_errors:
+                        line_w_newline = strip_ansi_errors(line_w_newline)
+
+                    # Handle stdout
+                    if stream_name == "stdout":
+                        if print_stdout:
+                            sys.stdout.buffer.write(line_prefix + line_w_newline)
+                            sys.stdout.flush()
+                        else:
+                            outputs["stdout"].append(line_w_newline)
+
+                    # Handle stderr
+                    if stream_name == "stderr":
+                        if print_stderr:
+                            sys.stderr.buffer.write(line_prefix + line_w_newline)
+                            sys.stderr.flush()
+                        else:
+                            outputs["stderr"].append(line_w_newline)
+    finally:
+        sel.close()
+
+    return outputs["stdout"], outputs["stderr"]
+
+
 def run_subprocess_cmd(processargs, prefix=b"", capture_output=False, **kwargs):
     """Runs subprocess command with realtime stdout logging with optional line prefix."""
     if prefix:
@@ -71,6 +133,7 @@ def run_subprocess_cmd(processargs, prefix=b"", capture_output=False, **kwargs):
         stderr=stderr_stream,
         preexec_fn=os.setsid,
     )
+
     # Set timeout thread
     timeout_timer = None
     if timeout > 0:
@@ -84,25 +147,14 @@ def kill_process():
         timeout_timer = threading.Timer(timeout, kill_process)
         timeout_timer.start()
 
-    print_stream = process.stderr if capture_output else process.stdout
-    for line in iter(lambda: print_stream.readline(), b""):
-        full_line = line_prefix + line
-        if strip_errors:
-            full_line = full_line.decode("utf-8")
-            full_line = re.sub(
-                r"\x1b\[31m", "", full_line
-            )  # Remove red ANSI escape code
-            full_line = full_line.encode("utf-8")
-
-        sys.stdout.buffer.write(full_line)
-        sys.stdout.flush()
-    print_stream.close()
-
-    output = []
     if capture_output:
-        for line in iter(lambda: process.stdout.readline(), b""):
-            output.append(line)
-        process.stdout.close()
+        output, _ = process_streams(
+            process, line_prefix, strip_errors, print_stdout=False, print_stderr=True
+        )
+    else:
+        process_streams(
+            process, line_prefix, strip_errors, print_stdout=True, print_stderr=True
+        )
 
     if timeout_timer is not None:
         timeout_timer.cancel()

From 8cb0e6321bb3e690432e120081485ffb2f18d18d Mon Sep 17 00:00:00 2001
From: Adam Lewis <23342526+Adam-D-Lewis@users.noreply.github.com>
Date: Wed, 29 Jan 2025 16:10:01 -0600
Subject: [PATCH 22/38] ensure binary raw string

---
 src/_nebari/utils.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/_nebari/utils.py b/src/_nebari/utils.py
index ef7153dca1..78b01a20f4 100644
--- a/src/_nebari/utils.py
+++ b/src/_nebari/utils.py
@@ -49,7 +49,7 @@ def change_directory(directory):
 
 def strip_ansi_errors(line):
     """Strips red ANSI escape code from a string."""
-    ansi_escape = re.compile(r"\x1b\[31m")
+    ansi_escape = re.compile(rb"\x1b\[31m")
     stripped_line = ansi_escape.sub("", line.decode("utf-8"))
     return stripped_line.encode("utf-8")
 

From 556661ff98b340438a1e175702aac4e226f15b48 Mon Sep 17 00:00:00 2001
From: Adam Lewis <23342526+Adam-D-Lewis@users.noreply.github.com>
Date: Wed, 29 Jan 2025 16:18:27 -0600
Subject: [PATCH 23/38] strip all ansi formatting sequences

---
 src/_nebari/utils.py | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/src/_nebari/utils.py b/src/_nebari/utils.py
index 78b01a20f4..61bb329672 100644
--- a/src/_nebari/utils.py
+++ b/src/_nebari/utils.py
@@ -48,10 +48,9 @@ def change_directory(directory):
 
 
 def strip_ansi_errors(line):
-    """Strips red ANSI escape code from a string."""
-    ansi_escape = re.compile(rb"\x1b\[31m")
-    stripped_line = ansi_escape.sub("", line.decode("utf-8"))
-    return stripped_line.encode("utf-8")
+    """Strips ANSI escape codes from a string."""
+    ansi_escape = re.compile(rb"\x1b\[[0-9;]*[mK]")
+    return ansi_escape.sub(b"", line)
 
 
 def process_streams(

From 7e5c2b08c879c31d61dd7859ce688f10492a14a6 Mon Sep 17 00:00:00 2001
From: Adam Lewis <23342526+Adam-D-Lewis@users.noreply.github.com>
Date: Wed, 29 Jan 2025 16:58:21 -0600
Subject: [PATCH 24/38] Revert "strip all ansi formatting sequences"

This reverts commit 556661ff98b340438a1e175702aac4e226f15b48.
---
 src/_nebari/utils.py | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/src/_nebari/utils.py b/src/_nebari/utils.py
index 61bb329672..78b01a20f4 100644
--- a/src/_nebari/utils.py
+++ b/src/_nebari/utils.py
@@ -48,9 +48,10 @@ def change_directory(directory):
 
 
 def strip_ansi_errors(line):
-    """Strips ANSI escape codes from a string."""
-    ansi_escape = re.compile(rb"\x1b\[[0-9;]*[mK]")
-    return ansi_escape.sub(b"", line)
+    """Strips red ANSI escape code from a string."""
+    ansi_escape = re.compile(rb"\x1b\[31m")
+    stripped_line = ansi_escape.sub("", line.decode("utf-8"))
+    return stripped_line.encode("utf-8")
 
 
 def process_streams(

From 37bd6362427d8b55939536f8e1c52bde72e34fae Mon Sep 17 00:00:00 2001
From: Adam Lewis <23342526+Adam-D-Lewis@users.noreply.github.com>
Date: Wed, 29 Jan 2025 16:58:35 -0600
Subject: [PATCH 25/38] Revert "ensure binary raw string"

This reverts commit 8cb0e6321bb3e690432e120081485ffb2f18d18d.
---
 src/_nebari/utils.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/_nebari/utils.py b/src/_nebari/utils.py
index 78b01a20f4..ef7153dca1 100644
--- a/src/_nebari/utils.py
+++ b/src/_nebari/utils.py
@@ -49,7 +49,7 @@ def change_directory(directory):
 
 def strip_ansi_errors(line):
     """Strips red ANSI escape code from a string."""
-    ansi_escape = re.compile(rb"\x1b\[31m")
+    ansi_escape = re.compile(r"\x1b\[31m")
     stripped_line = ansi_escape.sub("", line.decode("utf-8"))
     return stripped_line.encode("utf-8")
 

From b6e75dea09ee46cbc12a39308dd216fea4b8fcd7 Mon Sep 17 00:00:00 2001
From: Adam Lewis <23342526+Adam-D-Lewis@users.noreply.github.com>
Date: Wed, 29 Jan 2025 16:58:39 -0600
Subject: [PATCH 26/38] Revert "fix buffer full deadlock"

This reverts commit 2fb4fa8089d4e55bf3edb944592da12cc503b58c.
---
 src/_nebari/utils.py | 88 +++++++++-----------------------------------
 1 file changed, 18 insertions(+), 70 deletions(-)

diff --git a/src/_nebari/utils.py b/src/_nebari/utils.py
index ef7153dca1..48b8a91e9b 100644
--- a/src/_nebari/utils.py
+++ b/src/_nebari/utils.py
@@ -5,7 +5,6 @@
 import os
 import re
 import secrets
-import selectors
 import signal
 import string
 import subprocess
@@ -47,67 +46,6 @@ def change_directory(directory):
     os.chdir(current_directory)
 
 
-def strip_ansi_errors(line):
-    """Strips red ANSI escape code from a string."""
-    ansi_escape = re.compile(r"\x1b\[31m")
-    stripped_line = ansi_escape.sub("", line.decode("utf-8"))
-    return stripped_line.encode("utf-8")
-
-
-def process_streams(
-    process, line_prefix, strip_errors, print_stdout=True, print_stderr=True
-):
-    sel = selectors.DefaultSelector()
-    sel.register(process.stdout, selectors.EVENT_READ, data="stdout")
-    if process.stderr and process.stderr != process.stdout:
-        sel.register(process.stderr, selectors.EVENT_READ, data="stderr")
-
-    outputs = {"stdout": [], "stderr": []}
-    partial = {"stdout": b"", "stderr": b""}
-
-    try:
-        while True:
-            events = sel.select(timeout=0.1)
-            if not events and process.poll() is not None:
-                break
-
-            for key, _ in events:
-                data = key.fileobj.read1(8192)
-                if not data:
-                    sel.unregister(key.fileobj)
-                    continue
-
-                stream_name = key.data
-                chunk = partial[stream_name] + data
-                lines = chunk.split(b"\n")
-                partial[stream_name] = lines[-1]
-
-                for line in lines[:-1]:
-                    line_w_newline = line + b"\n"
-                    if strip_errors:
-                        line_w_newline = strip_ansi_errors(line_w_newline)
-
-                    # Handle stdout
-                    if stream_name == "stdout":
-                        if print_stdout:
-                            sys.stdout.buffer.write(line_prefix + line_w_newline)
-                            sys.stdout.flush()
-                        else:
-                            outputs["stdout"].append(line_w_newline)
-
-                    # Handle stderr
-                    if stream_name == "stderr":
-                        if print_stderr:
-                            sys.stderr.buffer.write(line_prefix + line_w_newline)
-                            sys.stderr.flush()
-                        else:
-                            outputs["stderr"].append(line_w_newline)
-    finally:
-        sel.close()
-
-    return outputs["stdout"], outputs["stderr"]
-
-
 def run_subprocess_cmd(processargs, prefix=b"", capture_output=False, **kwargs):
     """Runs subprocess command with realtime stdout logging with optional line prefix."""
     if prefix:
@@ -133,7 +71,6 @@ def run_subprocess_cmd(processargs, prefix=b"", capture_output=False, **kwargs):
         stderr=stderr_stream,
         preexec_fn=os.setsid,
     )
-
     # Set timeout thread
     timeout_timer = None
     if timeout > 0:
@@ -147,14 +84,25 @@ def kill_process():
         timeout_timer = threading.Timer(timeout, kill_process)
         timeout_timer.start()
 
+    print_stream = process.stderr if capture_output else process.stdout
+    for line in iter(lambda: print_stream.readline(), b""):
+        full_line = line_prefix + line
+        if strip_errors:
+            full_line = full_line.decode("utf-8")
+            full_line = re.sub(
+                r"\x1b\[31m", "", full_line
+            )  # Remove red ANSI escape code
+            full_line = full_line.encode("utf-8")
+
+        sys.stdout.buffer.write(full_line)
+        sys.stdout.flush()
+    print_stream.close()
+
+    output = []
     if capture_output:
-        output, _ = process_streams(
-            process, line_prefix, strip_errors, print_stdout=False, print_stderr=True
-        )
-    else:
-        process_streams(
-            process, line_prefix, strip_errors, print_stdout=True, print_stderr=True
-        )
+        for line in iter(lambda: process.stdout.readline(), b""):
+            output.append(line)
+        process.stdout.close()
 
     if timeout_timer is not None:
         timeout_timer.cancel()

From 1fce666afc0e2a2118f8040614f332860ce518a5 Mon Sep 17 00:00:00 2001
From: Adam Lewis <23342526+Adam-D-Lewis@users.noreply.github.com>
Date: Mon, 3 Feb 2025 10:10:45 -0600
Subject: [PATCH 27/38] fix fstring

---
 .../kubernetes/services/jupyterhub/files/jupyterhub/04-auth.py  | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/files/jupyterhub/04-auth.py b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/files/jupyterhub/04-auth.py
index a400884f26..18c3629d2f 100644
--- a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/files/jupyterhub/04-auth.py
+++ b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/files/jupyterhub/04-auth.py
@@ -40,7 +40,7 @@ def _default_jhub_service_account_name(self):
     async def set_jhub_service_account_auth_state(self, user):
         if user.name != self.JHUB_SERVICE_ACCOUNT_NAME:
             raise ValueError(
-                'User name "{user.name}" does not match service account name "{self.JHUB_SERVICE_ACCOUNT_NAME}"'
+                f'User name "{user.name}" does not match service account name "{self.JHUB_SERVICE_ACCOUNT_NAME}"'
             )
         auth_model = await self.authenticate_service_account()
         await user.save_auth_state(auth_model["auth_state"])

From 865c8d6497ed85e373c4f7c5ef77b61120df9a47 Mon Sep 17 00:00:00 2001
From: Adam Lewis <23342526+Adam-D-Lewis@users.noreply.github.com>
Date: Mon, 3 Feb 2025 10:14:46 -0600
Subject: [PATCH 28/38] add comment with jupyter/oauth code we are mimicking

---
 .../kubernetes/services/jupyterhub/files/jupyterhub/04-auth.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/files/jupyterhub/04-auth.py b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/files/jupyterhub/04-auth.py
index 18c3629d2f..1ea87fb144 100644
--- a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/files/jupyterhub/04-auth.py
+++ b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/files/jupyterhub/04-auth.py
@@ -47,6 +47,9 @@ async def set_jhub_service_account_auth_state(self, user):
         logging.info(f'Auth state set for service account: "{user.name}"')
 
     async def authenticate_service_account(self):
+        # We mimic what OAuthenticator currently does in `authenticate` method, but the logic may change in the future
+        # Currently, the logic is based on https://github.com/jupyterhub/oauthenticator/blob/d31bb193e84e7cda58b16f2f5d385c9b8affda4f/oauthenticator/oauth2.py#L1436
+
         token_info = await self._get_token_info()
 
         # Get user info using the access token

From fad0155aabb3e5ca8abe2fa6bed9b260a0b453ab Mon Sep 17 00:00:00 2001
From: Adam Lewis <23342526+Adam-D-Lewis@users.noreply.github.com>
Date: Mon, 3 Feb 2025 10:26:40 -0600
Subject: [PATCH 29/38] add keycloak service account name format comment

---
 .../kubernetes/services/jupyterhub/files/jupyterhub/04-auth.py   | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/files/jupyterhub/04-auth.py b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/files/jupyterhub/04-auth.py
index 1ea87fb144..0cc61035cd 100644
--- a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/files/jupyterhub/04-auth.py
+++ b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/files/jupyterhub/04-auth.py
@@ -21,6 +21,7 @@ class KeyCloakOAuthenticator(GenericOAuthenticator):
 
     JHUB_SERVICE_ACCOUNT_NAME = Unicode()
 
+    # Keycloak currently dictates service account name format as `service-account-<client_id>`  See https://github.com/keycloak/keycloak/blob/5e6bb9f7bd2c83febd12668f2605aa8ecbdcf130/docs/documentation/server_admin/topics/admin-cli.adoc?plain=1#L1008 for more info.
     @default("JHUB_SERVICE_ACCOUNT_NAME")
     def _default_jhub_service_account_name(self):
         return f"service-account-{self.client_id}"

From 80456c597142dd239bc15547805757423a46eac9 Mon Sep 17 00:00:00 2001
From: Adam Lewis <23342526+Adam-D-Lewis@users.noreply.github.com>
Date: Mon, 10 Feb 2025 12:05:52 -0600
Subject: [PATCH 30/38] test that jupyterhub service account gets needed roles

---
 tests/tests_deployment/test_jupyterhub_api.py | 56 ++++++++++++-------
 1 file changed, 37 insertions(+), 19 deletions(-)

diff --git a/tests/tests_deployment/test_jupyterhub_api.py b/tests/tests_deployment/test_jupyterhub_api.py
index aaeaf535ac..6e0b2ec72b 100644
--- a/tests/tests_deployment/test_jupyterhub_api.py
+++ b/tests/tests_deployment/test_jupyterhub_api.py
@@ -1,3 +1,5 @@
+from typing import Set
+
 import pytest
 import requests
 
@@ -14,30 +16,46 @@
 from tests.tests_deployment.utils import get_refresh_jupyterhub_token
 
 
+@pytest.mark.parametrize(
+    "username,expected_roles",
+    [
+        (
+            constants.KEYCLOAK_USERNAME,
+            {
+                "user",
+                "manage-account",
+                "jupyterhub_developer",
+                "argo-developer",
+                "dask_gateway_developer",
+                "grafana_viewer",
+                "conda_store_developer",
+                "argo-viewer",
+                "grafana_developer",
+                "manage-account-links",
+                "view-profile",
+                "allow-read-access-to-services-role",
+                "allow-group-directory-creation-role",
+            },
+        ),
+        (
+            "service-account-jupyterhub",
+            {"allow-app-sharing-role", "default-roles-nebari", "user"},
+        ),
+    ],
+    ids=["admin_user", "analyst_user"],
+)
 @pytest.mark.filterwarnings("ignore::urllib3.exceptions.InsecureRequestWarning")
-def test_jupyterhub_loads_roles_from_keycloak(jupyterhub_access_token):
+def test_jupyterhub_loads_roles_from_keycloak(
+    jupyterhub_access_token: str, username: str, expected_roles: Set[str]
+):
+    """Test that JupyterHub correctly loads roles from Keycloak for different users"""
     response = requests.get(
-        url=f"https://{constants.NEBARI_HOSTNAME}/hub/api/users/{constants.KEYCLOAK_USERNAME}",
+        url=f"https://{constants.NEBARI_HOSTNAME}/hub/api/users/{username}",
         headers={"Authorization": f"Bearer {jupyterhub_access_token}"},
         verify=False,
     )
-    user = response.json()
-    assert set(user["roles"]) == {
-        "user",
-        "manage-account",
-        "jupyterhub_developer",
-        "argo-developer",
-        "dask_gateway_developer",
-        "grafana_viewer",
-        "conda_store_developer",
-        "argo-viewer",
-        "grafana_developer",
-        "manage-account-links",
-        "view-profile",
-        # default roles
-        "allow-read-access-to-services-role",
-        "allow-group-directory-creation-role",
-    }
+    actual_roles = set(response.json()["roles"])
+    assert actual_roles == expected_roles
 
 
 @token_parameterized(note="get-default-scopes")

From 627c4aa6a7f31455dfd3f4557cda54a1702c1c8b Mon Sep 17 00:00:00 2001
From: Adam Lewis <23342526+Adam-D-Lewis@users.noreply.github.com>
Date: Mon, 10 Feb 2025 12:32:07 -0600
Subject: [PATCH 31/38] add a startup app to ci deployment

---
 .github/actions/init-local/action.yml | 31 +++++++++++++++++++++++++++
 1 file changed, 31 insertions(+)

diff --git a/.github/actions/init-local/action.yml b/.github/actions/init-local/action.yml
index 306876973a..af8b057fe0 100644
--- a/.github/actions/init-local/action.yml
+++ b/.github/actions/init-local/action.yml
@@ -75,6 +75,37 @@ runs:
                 size: 1Gi
         EOM
 
+        # Add a jhub startup app for testing
+        cat >> '${{ steps.metadata.outputs.config }}' <<- EOM
+        jhub_apps:
+          enabled: true
+          overrides: {
+            startup_apps: [
+              {
+                "username": "service-account-jupyterhub",
+                "servername": "my-startup-server",
+                "user_options": {
+                    "display_name": "My Startup Server-",
+                    "description": "description",
+                    "thumbnail": "data:image/jpeg;base64,iVBORw0KGgoAAAANSUhEUgAAAAoAAAAKCAYAAACNMs+9AAAAFUlEQVR42mNkYPhfz0AEYBxVSF+FAP5FDvcfRYWgAAAAAElFTkSuQmCC",
+                    "filepath": "panel_basic.py",
+                    "framework": "panel",
+                    "public": False,
+                    "keep_alive": False,
+                    "env": {"MY_ENV_VAR": "MY_VALUE"},
+                    "repository": {"url": "https://github.com/nebari-dev/jhub-apps-from-git-repo-example.git"},
+                    "conda_env": "global-mypanel",
+                    "profile": "small-instance",
+                    "share_with": {
+                      "users": [],
+                      "groups": ["/admin"]
+                    },
+                },
+              },
+            ]
+          }
+        EOM
+
     - shell: bash
       run: |
         # Display Nebari config

From 6de7c1d30bc1e53c51d3d75b8a76d4ecfc3a7a2c Mon Sep 17 00:00:00 2001
From: Adam Lewis <23342526+Adam-D-Lewis@users.noreply.github.com>
Date: Mon, 10 Feb 2025 12:41:43 -0600
Subject: [PATCH 32/38] assert startup server is created

---
 tests/tests_deployment/test_jupyterhub_api.py | 13 +++++++++++++
 1 file changed, 13 insertions(+)

diff --git a/tests/tests_deployment/test_jupyterhub_api.py b/tests/tests_deployment/test_jupyterhub_api.py
index 6e0b2ec72b..1aad531e05 100644
--- a/tests/tests_deployment/test_jupyterhub_api.py
+++ b/tests/tests_deployment/test_jupyterhub_api.py
@@ -170,3 +170,16 @@ def test_jupyterhub_loads_groups_from_keycloak(jupyterhub_access_token):
     )
     user = response.json()
     assert set(user["groups"]) == {"/analyst", "/developer", "/users"}
+
+
+@pytest.mark.filterwarnings("ignore::urllib3.exceptions.InsecureRequestWarning")
+def test_startup_apps_created(jupyterhub_access_token):
+    username = "service-account-jupyterhub"
+    response = requests.get(
+        f"https://{constants.NEBARI_HOSTNAME}/hub/api/users/{username}",
+        params={"include_stopped_servers": True},
+        headers={"Authorization": f"Bearer {jupyterhub_access_token}"},
+        verify=False,
+    )
+    user = response.json()
+    assert "my-startup-server" in user["servers"]

From 48eae290ce90d780d2cfa645f135c979f3d44a9f Mon Sep 17 00:00:00 2001
From: Adam Lewis <23342526+Adam-D-Lewis@users.noreply.github.com>
Date: Mon, 10 Feb 2025 16:32:15 -0600
Subject: [PATCH 33/38] fix test_startup_apps_created test

---
 tests/tests_deployment/test_jupyterhub_api.py | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/tests/tests_deployment/test_jupyterhub_api.py b/tests/tests_deployment/test_jupyterhub_api.py
index 1aad531e05..5438b5d322 100644
--- a/tests/tests_deployment/test_jupyterhub_api.py
+++ b/tests/tests_deployment/test_jupyterhub_api.py
@@ -174,12 +174,14 @@ def test_jupyterhub_loads_groups_from_keycloak(jupyterhub_access_token):
 
 @pytest.mark.filterwarnings("ignore::urllib3.exceptions.InsecureRequestWarning")
 def test_startup_apps_created(jupyterhub_access_token):
-    username = "service-account-jupyterhub"
     response = requests.get(
-        f"https://{constants.NEBARI_HOSTNAME}/hub/api/users/{username}",
+        f"https://{constants.NEBARI_HOSTNAME}/hub/api/users/{constants.KEYCLOAK_USERNAME}/shared",
         params={"include_stopped_servers": True},
         headers={"Authorization": f"Bearer {jupyterhub_access_token}"},
         verify=False,
     )
-    user = response.json()
-    assert "my-startup-server" in user["servers"]
+    shared_servers = response.json()
+    breakpoint()
+    assert "my-startup-server" in {
+        item["server"]["name"] for item in shared_servers["items"]
+    }

From fbaec09a7656c36e928a1283a6a85333fac24f87 Mon Sep 17 00:00:00 2001
From: Adam Lewis <23342526+Adam-D-Lewis@users.noreply.github.com>
Date: Mon, 10 Feb 2025 16:32:53 -0600
Subject: [PATCH 34/38] remove breakpoint

---
 tests/tests_deployment/test_jupyterhub_api.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/tests/tests_deployment/test_jupyterhub_api.py b/tests/tests_deployment/test_jupyterhub_api.py
index 5438b5d322..914e93504e 100644
--- a/tests/tests_deployment/test_jupyterhub_api.py
+++ b/tests/tests_deployment/test_jupyterhub_api.py
@@ -181,7 +181,6 @@ def test_startup_apps_created(jupyterhub_access_token):
         verify=False,
     )
     shared_servers = response.json()
-    breakpoint()
     assert "my-startup-server" in {
         item["server"]["name"] for item in shared_servers["items"]
     }

From 708f753d556d30365945c3fed46160e71c83c80b Mon Sep 17 00:00:00 2001
From: Adam Lewis <23342526+Adam-D-Lewis@users.noreply.github.com>
Date: Mon, 10 Feb 2025 17:16:26 -0600
Subject: [PATCH 35/38] refactor keycloak command cli

---
 src/_nebari/keycloak.py             | 14 ++++++--------
 src/_nebari/subcommands/keycloak.py | 19 +++++++++++++------
 2 files changed, 19 insertions(+), 14 deletions(-)

diff --git a/src/_nebari/keycloak.py b/src/_nebari/keycloak.py
index 6bfea9b8b3..0873a667f2 100644
--- a/src/_nebari/keycloak.py
+++ b/src/_nebari/keycloak.py
@@ -13,7 +13,7 @@
 logger = logging.getLogger(__name__)
 
 
-def do_keycloak(config: schema.Main, *args):
+def do_keycloak(config: schema.Main, command, **kwargs):
     # suppress insecure warnings
     import urllib3
 
@@ -21,19 +21,17 @@ def do_keycloak(config: schema.Main, *args):
 
     keycloak_admin = get_keycloak_admin_from_config(config)
 
-    if args[0] == "adduser":
-        if len(args) < 2:
+    if command == "adduser":
+        if "password" not in kwargs:
             raise ValueError(
                 "keycloak command 'adduser' requires `username [password]`"
             )
 
-        username = args[1]
-        password = args[2] if len(args) >= 3 else None
-        create_user(keycloak_admin, username, password, domain=config.domain)
-    elif args[0] == "listusers":
+        create_user(keycloak_admin, **kwargs, domain=config.domain)
+    elif command == "listusers":
         list_users(keycloak_admin)
     else:
-        raise ValueError(f"unknown keycloak command {args[0]}")
+        raise ValueError(f"unknown keycloak command {command}")
 
 
 def create_user(
diff --git a/src/_nebari/subcommands/keycloak.py b/src/_nebari/subcommands/keycloak.py
index 8f57d34175..d59b08232f 100644
--- a/src/_nebari/subcommands/keycloak.py
+++ b/src/_nebari/subcommands/keycloak.py
@@ -1,11 +1,14 @@
 import json
 import pathlib
-from typing import Tuple
+from typing import List, Tuple
 
 import typer
 
 from _nebari.config import read_configuration
-from _nebari.keycloak import do_keycloak, export_keycloak_users
+from _nebari.keycloak import (
+    do_keycloak,
+    export_keycloak_users,
+)
 from nebari.hookspecs import hookimpl
 
 
@@ -30,6 +33,9 @@ def add_user(
         add_users: Tuple[str, str] = typer.Option(
             ..., "--user", help="Provide both: <username> <password>"
         ),
+        groups: List[str] = typer.Option(
+            None, "-g", "--groups", help="Provide existing groups to add user to"
+        ),
         config_filename: pathlib.Path = typer.Option(
             ...,
             "-c",
@@ -40,10 +46,12 @@ def add_user(
         """Add a user to Keycloak. User will be automatically added to the [italic]analyst[/italic] group."""
         from nebari.plugins import nebari_plugin_manager
 
-        args = ["adduser", add_users[0], add_users[1]]
+        kwargs = {"username": add_users[0], "password": add_users[1], "groups": groups}
+
         config_schema = nebari_plugin_manager.config_schema
         config = read_configuration(config_filename, config_schema)
-        do_keycloak(config, *args)
+
+        do_keycloak(config, command="adduser", **kwargs)
 
     @app_keycloak.command(name="listusers")
     def list_users(
@@ -57,10 +65,9 @@ def list_users(
         """List the users in Keycloak."""
         from nebari.plugins import nebari_plugin_manager
 
-        args = ["listusers"]
         config_schema = nebari_plugin_manager.config_schema
         config = read_configuration(config_filename, config_schema)
-        do_keycloak(config, *args)
+        do_keycloak(config, command="listusers")
 
     @app_keycloak.command(name="export-users")
     def export_users(

From e7da0aa0b0ac12d2ced31a5a0bd434bedbd40586 Mon Sep 17 00:00:00 2001
From: Adam Lewis <23342526+Adam-D-Lewis@users.noreply.github.com>
Date: Mon, 10 Feb 2025 17:16:54 -0600
Subject: [PATCH 36/38] make test-user an admin

---
 .github/workflows/test_local_integration.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/test_local_integration.yaml b/.github/workflows/test_local_integration.yaml
index 7d79efc428..56d01cc715 100644
--- a/.github/workflows/test_local_integration.yaml
+++ b/.github/workflows/test_local_integration.yaml
@@ -99,7 +99,7 @@ jobs:
       - name: Create example-user
         working-directory: ${{ steps.init.outputs.directory }}
         run: |
-          nebari keycloak adduser --user "${TEST_USERNAME}" "${TEST_PASSWORD}" --config ${{ steps.init.outputs.config }}
+          nebari keycloak adduser --user "${TEST_USERNAME}" "${TEST_PASSWORD}" --groups developer --groups admin --config ${{ steps.init.outputs.config }}
           nebari keycloak listusers --config ${{ steps.init.outputs.config }}
 
       - name: Await Workloads

From de43a81612cac5477cce3e3c822301a86b8ba698 Mon Sep 17 00:00:00 2001
From: Adam Lewis <23342526+Adam-D-Lewis@users.noreply.github.com>
Date: Mon, 10 Feb 2025 17:17:13 -0600
Subject: [PATCH 37/38] fix test ids

---
 tests/tests_deployment/test_jupyterhub_api.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/tests_deployment/test_jupyterhub_api.py b/tests/tests_deployment/test_jupyterhub_api.py
index 914e93504e..d1a2576326 100644
--- a/tests/tests_deployment/test_jupyterhub_api.py
+++ b/tests/tests_deployment/test_jupyterhub_api.py
@@ -42,7 +42,7 @@
             {"allow-app-sharing-role", "default-roles-nebari", "user"},
         ),
     ],
-    ids=["admin_user", "analyst_user"],
+    ids=["test-user", "jupyterhub-service-account"],
 )
 @pytest.mark.filterwarnings("ignore::urllib3.exceptions.InsecureRequestWarning")
 def test_jupyterhub_loads_roles_from_keycloak(

From 9810fdbe2ef356663ab17281d63696517e08e99c Mon Sep 17 00:00:00 2001
From: Adam Lewis <23342526+Adam-D-Lewis@users.noreply.github.com>
Date: Mon, 10 Feb 2025 17:20:55 -0600
Subject: [PATCH 38/38] update tests since test-user is now an admin

---
 tests/tests_deployment/test_jupyterhub_api.py | 12 +++++++++++-
 1 file changed, 11 insertions(+), 1 deletion(-)

diff --git a/tests/tests_deployment/test_jupyterhub_api.py b/tests/tests_deployment/test_jupyterhub_api.py
index d1a2576326..33330c593d 100644
--- a/tests/tests_deployment/test_jupyterhub_api.py
+++ b/tests/tests_deployment/test_jupyterhub_api.py
@@ -35,6 +35,16 @@
                 "view-profile",
                 "allow-read-access-to-services-role",
                 "allow-group-directory-creation-role",
+                # admin roles
+                "admin",
+                "grafana_admin",
+                "conda_store_admin",
+                "argo-admin",
+                "manage-users",
+                "query-groups",
+                "query-users",
+                "jupyterhub_admin",
+                "dask_gateway_admin",
             },
         ),
         (
@@ -169,7 +179,7 @@ def test_jupyterhub_loads_groups_from_keycloak(jupyterhub_access_token):
         verify=False,
     )
     user = response.json()
-    assert set(user["groups"]) == {"/analyst", "/developer", "/users"}
+    assert set(user["groups"]) == {"/analyst", "/developer", "/admin", "/users"}
 
 
 @pytest.mark.filterwarnings("ignore::urllib3.exceptions.InsecureRequestWarning")