diff --git a/demos/argo-cd-git-ops/manifests/airflow/airflow.yaml b/demos/argo-cd-git-ops/manifests/airflow/airflow.yaml index 2453a9d2..616b9da1 100644 --- a/demos/argo-cd-git-ops/manifests/airflow/airflow.yaml +++ b/demos/argo-cd-git-ops/manifests/airflow/airflow.yaml @@ -42,7 +42,7 @@ spec: mountPath: /stackable/minio-tls webservers: roleConfig: - listenerClass: external-unstable + listenerClass: external-stable envOverrides: &envOverrides AIRFLOW_CONN_KUBERNETES_IN_CLUSTER: "kubernetes://?__extra__=%7B%22extra__kubernetes__in_cluster%22%3A+true%2C+%22extra__kubernetes__kube_config%22%3A+%22%22%2C+%22extra__kubernetes__kube_config_path%22%3A+%22%22%2C+%22extra__kubernetes__namespace%22%3A+%22%22%7D" # Via sealed secrets and pod overrides, just kept for reference here diff --git a/stacks/airflow/airflow.yaml b/stacks/airflow/airflow.yaml index 01094310..334eb07f 100644 --- a/stacks/airflow/airflow.yaml +++ b/stacks/airflow/airflow.yaml @@ -28,7 +28,7 @@ spec: subPath: pyspark_pi.yaml webservers: roleConfig: - listenerClass: external-unstable + listenerClass: external-stable config: resources: cpu: diff --git a/stacks/data-lakehouse-iceberg-trino-spark/kafka.yaml b/stacks/data-lakehouse-iceberg-trino-spark/kafka.yaml index ce2af1aa..d35b6877 100644 --- a/stacks/data-lakehouse-iceberg-trino-spark/kafka.yaml +++ b/stacks/data-lakehouse-iceberg-trino-spark/kafka.yaml @@ -12,6 +12,8 @@ spec: - authenticationClass: kafka-client-tls brokers: config: + bootstrapListenerClass: external-stable + brokerListenerClass: external-unstable resources: storage: logDirs: diff --git a/stacks/data-lakehouse-iceberg-trino-spark/nifi.yaml b/stacks/data-lakehouse-iceberg-trino-spark/nifi.yaml index 6c067d79..7d71f397 100644 --- a/stacks/data-lakehouse-iceberg-trino-spark/nifi.yaml +++ b/stacks/data-lakehouse-iceberg-trino-spark/nifi.yaml @@ -14,7 +14,7 @@ spec: autoGenerate: true nodes: roleConfig: - listenerClass: external-unstable + listenerClass: external-stable config: resources: cpu: diff --git a/stacks/data-lakehouse-iceberg-trino-spark/trino.yaml b/stacks/data-lakehouse-iceberg-trino-spark/trino.yaml index 666706ca..5b668896 100644 --- a/stacks/data-lakehouse-iceberg-trino-spark/trino.yaml +++ b/stacks/data-lakehouse-iceberg-trino-spark/trino.yaml @@ -18,7 +18,7 @@ spec: package: trino coordinators: roleConfig: - listenerClass: external-unstable + listenerClass: external-stable config: queryMaxMemory: 10TB resources: diff --git a/stacks/dual-hive-hdfs-s3/hdfs.yaml b/stacks/dual-hive-hdfs-s3/hdfs.yaml index cb689eba..0d35de36 100644 --- a/stacks/dual-hive-hdfs-s3/hdfs.yaml +++ b/stacks/dual-hive-hdfs-s3/hdfs.yaml @@ -27,14 +27,17 @@ spec: image: productVersion: 3.4.2 clusterConfig: - listenerClass: external-unstable dfsReplication: 1 zookeeperConfigMapName: hdfs-znode nameNodes: + config: + listenerClass: external-stable roleGroups: default: replicas: 2 dataNodes: + config: + listenerClass: external-unstable roleGroups: default: replicas: 1 diff --git a/stacks/dual-hive-hdfs-s3/trino.yaml b/stacks/dual-hive-hdfs-s3/trino.yaml index faa05991..e7fb4ae8 100644 --- a/stacks/dual-hive-hdfs-s3/trino.yaml +++ b/stacks/dual-hive-hdfs-s3/trino.yaml @@ -70,7 +70,7 @@ spec: catalogLabelSelector: matchLabels: trino: trino - listenerClass: external-unstable + listenerClass: external-stable coordinators: roleGroups: default: diff --git a/stacks/end-to-end-security/hdfs.yaml b/stacks/end-to-end-security/hdfs.yaml index 741611fc..1de2086b 100644 --- a/stacks/end-to-end-security/hdfs.yaml +++ b/stacks/end-to-end-security/hdfs.yaml @@ -18,6 +18,7 @@ spec: package: hdfs nameNodes: config: + listenerClass: external-stable logging: containers: hdfs: @@ -39,6 +40,8 @@ spec: default: replicas: 2 dataNodes: + config: + listenerClass: external-unstable roleGroups: default: replicas: 1 diff --git a/stacks/end-to-end-security/superset.yaml b/stacks/end-to-end-security/superset.yaml index ff580e01..f283f2c7 100644 --- a/stacks/end-to-end-security/superset.yaml +++ b/stacks/end-to-end-security/superset.yaml @@ -16,7 +16,7 @@ spec: userRegistrationRole: Gamma_extended nodes: roleConfig: - listenerClass: external-unstable + listenerClass: external-stable roleGroups: default: replicas: 1 diff --git a/stacks/end-to-end-security/trino.yaml b/stacks/end-to-end-security/trino.yaml index 658c4231..56302ca6 100644 --- a/stacks/end-to-end-security/trino.yaml +++ b/stacks/end-to-end-security/trino.yaml @@ -23,7 +23,7 @@ spec: package: trino coordinators: roleConfig: - listenerClass: external-unstable + listenerClass: external-stable podOverrides: &podOverrides spec: containers: diff --git a/stacks/hdfs-hbase/hbase.yaml b/stacks/hdfs-hbase/hbase.yaml index 268fe0d0..2e06284d 100644 --- a/stacks/hdfs-hbase/hbase.yaml +++ b/stacks/hdfs-hbase/hbase.yaml @@ -26,7 +26,7 @@ spec: replicas: 2 restServers: config: - listenerClass: external-unstable + listenerClass: external-stable roleGroups: default: replicas: 1 diff --git a/stacks/jupyterhub-pyspark-hdfs/spark_connect.yaml b/stacks/jupyterhub-pyspark-hdfs/spark_connect.yaml index 3bdefa71..f7804e58 100644 --- a/stacks/jupyterhub-pyspark-hdfs/spark_connect.yaml +++ b/stacks/jupyterhub-pyspark-hdfs/spark_connect.yaml @@ -54,7 +54,7 @@ spec: configMap: name: hdfs roleConfig: - listenerClass: external-unstable + listenerClass: external-stable config: resources: memory: diff --git a/stacks/keycloak-opa-poc/druid.yaml b/stacks/keycloak-opa-poc/druid.yaml deleted file mode 100644 index 3e547bf7..00000000 --- a/stacks/keycloak-opa-poc/druid.yaml +++ /dev/null @@ -1,121 +0,0 @@ ---- -apiVersion: druid.stackable.tech/v1alpha1 -kind: DruidCluster -metadata: - name: druid -spec: - image: - productVersion: 34.0.0 - clusterConfig: - listenerClass: external-unstable - deepStorage: - hdfs: - configMapName: hdfs - directory: /data - metadataStorageDatabase: - dbType: postgresql - connString: jdbc:postgresql://postgresql-druid/druid - host: postgresql-druid - port: 5432 - user: druid - password: druid - zookeeperConfigMapName: druid-znode - authorization: - opa: - configMapName: opa - package: druid - brokers: - roleGroups: - default: - replicas: 1 - podOverrides: &pod-overrides - spec: - containers: - - name: druid - env: - - name: KEYCLOAK_DISCOVERY_URL - valueFrom: - configMapKeyRef: - name: keycloak - key: KEYCLOAK_DISCOVERY_URL - - name: DRUID_CLIENT_SECRET - valueFrom: - secretKeyRef: - name: keycloak-client-secrets - key: druid - - name: DRUID_COOKIE_PASSPHRASE - valueFrom: - secretKeyRef: - name: keycloak-client-secrets - key: druidCookiePassphrase - - name: DRUID_SYSTEM_USER_PASSWORD - valueFrom: - secretKeyRef: - name: keycloak-client-secrets - key: druidSystemUserPassword - configOverrides: - runtime.properties: &runtime-properties - druid.extensions.loadList: >- - ["postgresql-metadata-storage", - "simple-client-sslcontext", - "druid-kafka-indexing-service", - "druid-datasketches", - "prometheus-emitter", - "druid-basic-security", - "druid-opa-authorizer", - "druid-hdfs-storage", - "druid-pac4j"] - - # basic authenticator needed for internal authentication among Druid processes - # Trying to use the pac4j authenticator in the escalator below leads to 302 errors, - # it seems like the Druid processes cannot handle the OIDC authentication flow. - druid.auth.authenticator.MyBasicMetadataAuthenticator.type: basic - druid.auth.authenticator.MyBasicMetadataAuthenticator.initialInternalClientPassword: '${env:DRUID_SYSTEM_USER_PASSWORD}' # Default password for internal 'druid_system' user - druid.auth.authenticator.MyBasicMetadataAuthenticator.skipOnFailure: "true" # for any non system user, skip to the pac4j authenticator - druid.auth.authenticator.MyBasicMetadataAuthenticator.authorizerName: OpaAuthorizer - - # pac4j authenticator - druid.auth.authenticator.pac4j.type: pac4j - druid.auth.authenticator.pac4j.authorizerName: OpaAuthorizer - # pac4j common config - druid.auth.pac4j.cookiePassphrase: '${env:DRUID_COOKIE_PASSPHRASE}' - # OIDC common config - druid.auth.pac4j.oidc.clientID: druid - druid.auth.pac4j.oidc.clientSecret: '{"type":"environment","variable":"DRUID_CLIENT_SECRET"}' - druid.auth.pac4j.oidc.discoveryURI: '${env:KEYCLOAK_DISCOVERY_URL}' - # druid.auth.pac4j.oidc.oidcClaim: preferred_username # setting doesn't work, but should? - - druid.auth.authenticatorChain: '["MyBasicMetadataAuthenticator","pac4j"]' - - druid.escalator.type: basic - druid.escalator.internalClientUsername: druid_system - druid.escalator.internalClientPassword: '{"type":"environment","variable":"DRUID_SYSTEM_USER_PASSWORD"}' - druid.escalator.authorizerName: OpaAuthorizer - coordinators: - roleGroups: - default: - replicas: 1 - podOverrides: *pod-overrides - configOverrides: - runtime.properties: *runtime-properties - historicals: - roleGroups: - default: - replicas: 1 - podOverrides: *pod-overrides - configOverrides: - runtime.properties: *runtime-properties - middleManagers: - roleGroups: - default: - replicas: 1 - podOverrides: *pod-overrides - configOverrides: - runtime.properties: *runtime-properties - routers: - roleGroups: - default: - replicas: 1 - podOverrides: *pod-overrides - configOverrides: - runtime.properties: *runtime-properties diff --git a/stacks/keycloak-opa-poc/hdfs.yaml b/stacks/keycloak-opa-poc/hdfs.yaml deleted file mode 100644 index 3f7b1839..00000000 --- a/stacks/keycloak-opa-poc/hdfs.yaml +++ /dev/null @@ -1,23 +0,0 @@ ---- -apiVersion: hdfs.stackable.tech/v1alpha1 -kind: HdfsCluster -metadata: - name: hdfs -spec: - image: - productVersion: 3.4.2 - clusterConfig: - dfsReplication: 1 - zookeeperConfigMapName: hdfs-znode - nameNodes: - roleGroups: - default: - replicas: 2 - dataNodes: - roleGroups: - default: - replicas: 1 - journalNodes: - roleGroups: - default: - replicas: 1 diff --git a/stacks/keycloak-opa-poc/keycloak.yaml b/stacks/keycloak-opa-poc/keycloak.yaml deleted file mode 100644 index a5d64637..00000000 --- a/stacks/keycloak-opa-poc/keycloak.yaml +++ /dev/null @@ -1,89 +0,0 @@ ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: keycloak - labels: - app: keycloak -spec: - replicas: 1 - selector: - matchLabels: - app: keycloak - template: - metadata: - labels: - app: keycloak - spec: - containers: - - name: keycloak - image: quay.io/keycloak/keycloak:23.0.0 - # Keycloak is running in development mode: https://www.keycloak.org/server/configuration#_starting_keycloak - # production mode disables HTTP and requires a TLS configuration, which is currently very difficult to configure - # given that we're running on a NodePort - args: ["start-dev"] - env: - - name: KEYCLOAK_ADMIN - value: admin - - name: KEYCLOAK_ADMIN_PASSWORD - valueFrom: - secretKeyRef: - name: keycloak-admin-credentials - key: admin - ports: - - name: http - containerPort: 8080 - readinessProbe: - httpGet: - path: /realms/master - port: 8080 ---- -apiVersion: v1 -kind: Service -metadata: - name: keycloak - labels: - app: keycloak -spec: - type: NodePort - selector: - app: keycloak - ports: - - name: http - port: 8080 - targetPort: 8080 ---- -apiVersion: v1 -kind: Secret -metadata: - name: keycloak-admin-credentials -stringData: - admin: "{{ keycloakAdminPassword }}" ---- -# This job creates a ConfigMap with connection info for Keycloak -apiVersion: batch/v1 -kind: Job -metadata: - name: propagate-keycloak-address -spec: - template: - spec: - containers: - - name: propagate-keycloak-address - image: oci.stackable.tech/sdp/testing-tools:0.2.0-stackable0.0.0-dev - command: - - bash - - -x - - -euo - - pipefail - - -c - - | - echo "Determining Keycloak public reachable address" - KEYCLOAK_ADDRESS=$(kubectl get svc keycloak -o json | jq -r --argfile endpoints <(kubectl get endpoints keycloak -o json) --argfile nodes <(kubectl get nodes -o json) '($nodes.items[] | select(.metadata.name == $endpoints.subsets[].addresses[].nodeName) | .status.addresses | map(select(.type == "ExternalIP" or .type == "InternalIP")) | min_by(.type) | .address | tostring) + ":" + (.spec.ports[] | select(.name == "http") | .nodePort | tostring)') - echo "Found Keycloak running at $KEYCLOAK_ADDRESS" - - echo "Writing Keycloak address to ConfigMap keycloak" - kubectl create configmap keycloak --from-literal="KEYCLOAK=$KEYCLOAK_ADDRESS" --from-literal="KEYCLOAK_DISCOVERY_URL=http://$KEYCLOAK_ADDRESS/realms/master/.well-known/openid-configuration" -o yaml --dry-run | kubectl apply -f - - serviceAccountName: demo-serviceaccount - restartPolicy: OnFailure - backoffLimit: 20 # give some time for the Keycloak to be available diff --git a/stacks/keycloak-opa-poc/opa.yaml b/stacks/keycloak-opa-poc/opa.yaml deleted file mode 100644 index 189930b4..00000000 --- a/stacks/keycloak-opa-poc/opa.yaml +++ /dev/null @@ -1,11 +0,0 @@ ---- -apiVersion: opa.stackable.tech/v1alpha1 -kind: OpaCluster -metadata: - name: opa -spec: - image: - productVersion: 1.8.0 - servers: - roleGroups: - default: {} diff --git a/stacks/keycloak-opa-poc/policies.yaml b/stacks/keycloak-opa-poc/policies.yaml deleted file mode 100644 index a43a3803..00000000 --- a/stacks/keycloak-opa-poc/policies.yaml +++ /dev/null @@ -1,59 +0,0 @@ ---- -apiVersion: v1 -kind: ConfigMap -metadata: - name: opa-bundle-trino - labels: - opa.stackable.tech/bundle: "true" -data: - trino.rego: | - package trino - - default allow = false - - allow if { - input.context.identity.user in ["alice", "admin"] - } - - allow if { - input.action.operation == "ImpersonateUser" - input.action.resource.user.name == input.context.identity.user - } ---- -apiVersion: v1 -kind: ConfigMap -metadata: - name: opa-bundle-druid - labels: - opa.stackable.tech/bundle: "true" -data: - druid.rego: | - package druid - import data.bundles.opagroups.admins - - default allow = false - - allow if { - input.user in admins - } - - allow if { - input.user == "druid_system" - } -# A CM like this is created by the setup keycloak Job -# It is used for Druid roles, as we currently need to write them based on the user uuids. -# --- -# apiVersion: v1 -# kind: ConfigMap -# metadata: -# name: opagroups -# labels: -# opa.stackable.tech/bundle: "true" -# data: -# data.json: | -# { -# "admins": [ -# "57d3b407-ecc0-4cc1-aaaf-45a63f43b96b", -# "170b4130-ca4d-417b-b229-f2917d5ab3d1" -# ] -# } diff --git a/stacks/keycloak-opa-poc/serviceaccount.yaml b/stacks/keycloak-opa-poc/serviceaccount.yaml deleted file mode 100644 index b29d205b..00000000 --- a/stacks/keycloak-opa-poc/serviceaccount.yaml +++ /dev/null @@ -1,45 +0,0 @@ ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: demo-serviceaccount - namespace: default ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: demo-clusterrolebinding -subjects: - - kind: ServiceAccount - name: demo-serviceaccount - namespace: default -roleRef: - kind: ClusterRole - name: demo-clusterrole - apiGroup: rbac.authorization.k8s.io ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: demo-clusterrole -rules: - - apiGroups: - - "" - resources: - - nodes - - services - - endpoints - verbs: - - get - - list - - watch - - apiGroups: - - "" - resources: - - configmaps - verbs: - - get - - list - - watch - - create - - patch diff --git a/stacks/keycloak-opa-poc/setup-keycloak.yaml b/stacks/keycloak-opa-poc/setup-keycloak.yaml deleted file mode 100644 index fca8cdd5..00000000 --- a/stacks/keycloak-opa-poc/setup-keycloak.yaml +++ /dev/null @@ -1,135 +0,0 @@ ---- -apiVersion: v1 -kind: Secret -metadata: - name: keycloak-client-secrets -stringData: - superset: "{{ keycloakSupersetClientSecret }}" - trino: "{{ keycloakTrinoClientSecret }}" - druid: "{{ keycloakDruidClientSecret }}" - druidCookiePassphrase: "{{ keycloakDruidCookiePassphrase }}" - druidSystemUserPassword: "{{ druidSystemUserPassword }}" ---- -apiVersion: v1 -kind: Secret -metadata: - name: keycloak-users -stringData: - alice: "{{ alicePassword }}" - bob: "{{ bobPassword }}" ---- -# This job creates users and clients in Keycloak. -# It also creates a ConfigMap with user IDs used by OPA -apiVersion: batch/v1 -kind: Job -metadata: - name: setup-keycloak -spec: - template: - spec: - containers: - - name: setup-keycloak - image: oci.stackable.tech/sdp/testing-tools:0.2.0-stackable0.0.0-dev - env: - - name: KEYCLOAK_ADMIN_PASSWORD - valueFrom: - secretKeyRef: - name: keycloak-admin-credentials - key: admin - - name: ALICE_PASSWORD - valueFrom: - secretKeyRef: - name: keycloak-users - key: alice - - name: BOB_PASSWORD - valueFrom: - secretKeyRef: - name: keycloak-users - key: bob - - name: SUPERSET_CLIENT_SECRET - valueFrom: - secretKeyRef: - name: keycloak-client-secrets - key: superset - - name: TRINO_CLIENT_SECRET - valueFrom: - secretKeyRef: - name: keycloak-client-secrets - key: trino - - name: DRUID_CLIENT_SECRET - valueFrom: - secretKeyRef: - name: keycloak-client-secrets - key: druid - command: - - bash - - -x - - -euo - - pipefail - - -c - - | - echo "Download keycloak" - curl -LO https://github.com/keycloak/keycloak/releases/download/23.0.0/keycloak-23.0.0.zip - unzip -o keycloak-23.0.0.zip - cd keycloak-23.0.0/bin - ./kcadm.sh config credentials --config kcadm.conf --server http://keycloak:8080/ --realm master --user admin --password "$KEYCLOAK_ADMIN_PASSWORD" - ./kcadm.sh create users --config kcadm.conf -s username=alice -s enabled=true || true - ./kcadm.sh create users --config kcadm.conf -s username=bob -s enabled=true || true - ./kcadm.sh set-password --config kcadm.conf --username alice --new-password "$ALICE_PASSWORD" - ./kcadm.sh set-password --config kcadm.conf --username bob --new-password "$BOB_PASSWORD" - ./kcadm.sh create clients --config kcadm.conf -f - << EOF || true - { - "clientId": "superset", - "enabled": true, - "clientAuthenticatorType": "client-secret", - "secret": "$SUPERSET_CLIENT_SECRET", - "redirectUris": [ - "*" - ], - "webOrigins": [ - "*" - ], - "standardFlowEnabled": true, - "protocol": "openid-connect" - } - EOF - ./kcadm.sh create clients --config kcadm.conf -f - << EOF || true - { - "clientId": "trino", - "enabled": true, - "clientAuthenticatorType": "client-secret", - "secret": "$TRINO_CLIENT_SECRET", - "redirectUris": [ - "*" - ], - "webOrigins": [ - "*" - ], - "standardFlowEnabled": true, - "protocol": "openid-connect" - } - EOF - ./kcadm.sh create clients --config kcadm.conf -f - << EOF || true - { - "clientId": "druid", - "enabled": true, - "clientAuthenticatorType": "client-secret", - "secret": "$DRUID_CLIENT_SECRET", - "redirectUris": [ - "*" - ], - "webOrigins": [ - "*" - ], - "standardFlowEnabled": true, - "protocol": "openid-connect" - } - EOF - ADMIN_ID=$(./kcadm.sh get users --config kcadm.conf -r master -q username=admin | sed -ne 's/^ "id" : \(.*\),$/\1/p') - ALICE_ID=$(./kcadm.sh get users --config kcadm.conf -r master -q username=alice | sed -ne 's/^ "id" : \(.*\),$/\1/p') - echo "Writing Keycloak address to ConfigMap keycloak" - kubectl create configmap opagroups --from-literal="data.json={\"admins\": [$ADMIN_ID, $ALICE_ID]}" -o yaml --dry-run | kubectl apply -f - - kubectl label configmap opagroups opa.stackable.tech/bundle=true - serviceAccountName: demo-serviceaccount - restartPolicy: OnFailure - backoffLimit: 20 # give some time for the Keycloak to be available diff --git a/stacks/keycloak-opa-poc/superset.yaml b/stacks/keycloak-opa-poc/superset.yaml deleted file mode 100644 index 62ec24d1..00000000 --- a/stacks/keycloak-opa-poc/superset.yaml +++ /dev/null @@ -1,68 +0,0 @@ ---- -apiVersion: v1 -kind: Secret -metadata: - name: superset-credentials -type: Opaque -stringData: - adminUser.username: admin - adminUser.firstname: Superset - adminUser.lastname: Admin - adminUser.email: admin@superset.com - adminUser.password: will-not-be-used # as we use auth/oidc - connections.secretKey: {{ supersetSecretKey }} - connections.sqlalchemyDatabaseUri: postgresql://superset:superset@postgresql-superset/superset ---- -apiVersion: superset.stackable.tech/v1alpha1 -kind: SupersetCluster -metadata: - name: superset -spec: - image: - productVersion: 4.1.4 - clusterConfig: - credentialsSecret: superset-credentials - nodes: - roleGroups: - default: - replicas: 1 - config: - listenerClass: external-unstable - configOverrides: - superset_config.py: - AUTH_TYPE: AUTH_OAUTH - AUTH_USER_REGISTRATION: "true" - AUTH_USER_REGISTRATION_ROLE: Gamma - OAUTH_PROVIDERS: |- - [ - { 'name': 'keycloak', - 'icon': 'fa-key', - 'token_key': 'access_token', - 'remote_app': { - 'client_id': 'superset', - 'client_secret': f'{os.environ.get("SUPERSET_CLIENT_SECRET")}', - 'api_base_url': f'http://{os.environ.get("KEYCLOAK_ADDRESS")}/realms/master/protocol/openid-connect', - 'client_kwargs': { - 'scope': 'email profile openid' - }, - 'access_token_url': f'http://{os.environ.get("KEYCLOAK_ADDRESS")}/realms/master/protocol/openid-connect/token', - 'authorize_url': f'http://{os.environ.get("KEYCLOAK_ADDRESS")}/realms/master/protocol/openid-connect/auth', - 'request_token_url': None, - }, - } - ] - podOverrides: - spec: - containers: - - name: superset - env: - - name: KEYCLOAK_ADDRESS - valueFrom: - configMapKeyRef: - name: keycloak - key: KEYCLOAK - - name: SUPERSET_CLIENT_SECRET - valueFrom: - secretKeyRef: - name: keycloak-client-secrets - key: superset diff --git a/stacks/keycloak-opa-poc/trino.yaml b/stacks/keycloak-opa-poc/trino.yaml deleted file mode 100644 index 98233382..00000000 --- a/stacks/keycloak-opa-poc/trino.yaml +++ /dev/null @@ -1,49 +0,0 @@ ---- -apiVersion: trino.stackable.tech/v1alpha1 -kind: TrinoCluster -metadata: - name: trino -spec: - image: - productVersion: "477" - clusterConfig: - listenerClass: external-unstable - tls: - serverSecretClass: tls - catalogLabelSelector: - matchLabels: - trino: trino - authorization: - opa: - configMapName: opa - package: trino - coordinators: - podOverrides: - spec: - containers: - - name: trino - env: - - name: KEYCLOAK_ADDRESS - valueFrom: - configMapKeyRef: - name: keycloak - key: KEYCLOAK - - name: TRINO_CLIENT_SECRET - valueFrom: - secretKeyRef: - name: keycloak-client-secrets - key: trino - configOverrides: - config.properties: - http-server.authentication.type: oauth2 - http-server.authentication.oauth2.client-id: trino - http-server.authentication.oauth2.client-secret: ${ENV:TRINO_CLIENT_SECRET} - http-server.authentication.oauth2.issuer: http://${ENV:KEYCLOAK_ADDRESS}/realms/master - http-server.authentication.oauth2.principal-field: preferred_username - roleGroups: - default: - replicas: 1 - workers: - roleGroups: - default: - replicas: 1 diff --git a/stacks/keycloak-opa-poc/zookeeper.yaml b/stacks/keycloak-opa-poc/zookeeper.yaml deleted file mode 100644 index 63497156..00000000 --- a/stacks/keycloak-opa-poc/zookeeper.yaml +++ /dev/null @@ -1,28 +0,0 @@ ---- -apiVersion: zookeeper.stackable.tech/v1alpha1 -kind: ZookeeperCluster -metadata: - name: zk -spec: - image: - productVersion: 3.9.4 - servers: - roleGroups: - default: - replicas: 1 ---- -apiVersion: zookeeper.stackable.tech/v1alpha1 -kind: ZookeeperZnode -metadata: - name: druid-znode -spec: - clusterRef: - name: zk ---- -apiVersion: zookeeper.stackable.tech/v1alpha1 -kind: ZookeeperZnode -metadata: - name: hdfs-znode -spec: - clusterRef: - name: zk diff --git a/stacks/nifi-kafka-druid-superset-s3/druid.yaml b/stacks/nifi-kafka-druid-superset-s3/druid.yaml index 8a81837d..80b345d7 100644 --- a/stacks/nifi-kafka-druid-superset-s3/druid.yaml +++ b/stacks/nifi-kafka-druid-superset-s3/druid.yaml @@ -29,13 +29,13 @@ spec: baseKey: data brokers: roleConfig: - listenerClass: external-unstable + listenerClass: external-stable roleGroups: default: replicas: 1 coordinators: roleConfig: - listenerClass: external-unstable + listenerClass: external-stable roleGroups: default: replicas: 1 @@ -67,7 +67,7 @@ spec: limit: 16Gi routers: roleConfig: - listenerClass: external-unstable + listenerClass: external-stable roleGroups: default: replicas: 1 diff --git a/stacks/nifi-kafka-druid-superset-s3/kafka.yaml b/stacks/nifi-kafka-druid-superset-s3/kafka.yaml index 3ce5fe7c..c67447f0 100644 --- a/stacks/nifi-kafka-druid-superset-s3/kafka.yaml +++ b/stacks/nifi-kafka-druid-superset-s3/kafka.yaml @@ -21,7 +21,7 @@ spec: brokers: config: bootstrapListenerClass: external-stable - brokerListenerClass: external-stable + brokerListenerClass: external-unstable resources: storage: logDirs: diff --git a/stacks/nifi-kafka-druid-superset-s3/superset.yaml b/stacks/nifi-kafka-druid-superset-s3/superset.yaml index 15774c02..87176201 100644 --- a/stacks/nifi-kafka-druid-superset-s3/superset.yaml +++ b/stacks/nifi-kafka-druid-superset-s3/superset.yaml @@ -11,7 +11,7 @@ spec: mapboxSecret: superset-mapbox-api-key nodes: roleConfig: - listenerClass: external-unstable + listenerClass: external-stable roleGroups: default: replicas: 1 diff --git a/stacks/signal-processing/nifi.yaml b/stacks/signal-processing/nifi.yaml index 0206a478..fc3fd8ee 100644 --- a/stacks/signal-processing/nifi.yaml +++ b/stacks/signal-processing/nifi.yaml @@ -36,7 +36,7 @@ spec: stateRepo: capacity: "1Gi" roleConfig: - listenerClass: external-unstable + listenerClass: external-stable roleGroups: default: replicas: 1 diff --git a/stacks/stacks-v2.yaml b/stacks/stacks-v2.yaml index 8909ceef..42790cee 100644 --- a/stacks/stacks-v2.yaml +++ b/stacks/stacks-v2.yaml @@ -509,74 +509,6 @@ stacks: pvc: 0Gi # TODO: Parameterize parameters: [] - keycloak-opa-poc: - description: >- - A Superset, Trino, Druid, Keycloak and OPA instance. - Superset, Trino and Druid have single sign-on with Keycloak enabled. - Trino and Druid have OPA authorization enabled. - 3 users are created in Keycloak: admin:adminadmin, alice:alicealice, bob:bobbob. admin and alice are admins with - full authorization in Druid and Trino, bob is not authorized. - This is a proof-of-concept and the mechanisms used here are subject to change. - stackableRelease: dev - stackableOperators: - - commons - - listener - - secret - - trino - - superset - - zookeeper - - druid - - hdfs - - opa - labels: - - authentication - - sso - manifests: - - helmChart: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/_templates/postgresql-superset.yaml - - helmChart: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/_templates/postgresql-druid.yaml - - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/keycloak-opa-poc/serviceaccount.yaml - - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/keycloak-opa-poc/keycloak.yaml - - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/keycloak-opa-poc/setup-keycloak.yaml - - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/keycloak-opa-poc/opa.yaml - - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/keycloak-opa-poc/policies.yaml - - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/keycloak-opa-poc/zookeeper.yaml - - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/keycloak-opa-poc/hdfs.yaml - - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/keycloak-opa-poc/druid.yaml - - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/keycloak-opa-poc/trino.yaml - - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/keycloak-opa-poc/superset.yaml - supportedNamespaces: ["default"] # ClusterRoleBinding needs explicit namespace - resourceRequests: - cpu: 7850m - memory: 23142Mi - pvc: 34Gi - parameters: - - name: alicePassword - description: Password of the user alice, which can log into Trino and Superset - default: alicealice - - name: bobPassword - description: Password of the user bob, which can log into Trino and Superset - default: bobbob - - name: keycloakAdminPassword - description: Password of the Keycloak admin user - default: adminadmin - - name: keycloakSupersetClientSecret - description: Secret ID of the Keycloak Superset client that is used by Superset to connect to Keycloak to authenticate users - default: supersetsuperset - - name: keycloakTrinoClientSecret - description: Secret ID of the Keycloak Trino client that is used by Trino to connect to Keycloak to authenticate users - default: trinotrino - - name: keycloakDruidClientSecret - description: Secret ID of the Keycloak Druid client that is used by Druid to connect to Keycloak to authenticate users - default: druiddruid - - name: keycloakDruidCookiePassphrase - description: Passphrase for encrypting the cookies used to manage authentication session with browser. - default: druiddruidcookiepassphrase - - name: druidSystemUserPassword - description: Password for the Druid user druid_system - default: druidsystemuserpassword - - name: supersetSecretKey - description: Superset's secret key used to generate e.g. user session tokens - default: supersetSecretKey end-to-end-security: description: >- A stack used to demonstrate an end-to-end security concept. diff --git a/stacks/trino-iceberg/trino.yaml b/stacks/trino-iceberg/trino.yaml index 6cc7bbdb..56060f75 100644 --- a/stacks/trino-iceberg/trino.yaml +++ b/stacks/trino-iceberg/trino.yaml @@ -29,7 +29,7 @@ spec: default: replicas: 1 roleConfig: - listenerClass: external-unstable + listenerClass: external-stable workers: config: resources: diff --git a/stacks/trino-superset-s3/superset.yaml b/stacks/trino-superset-s3/superset.yaml index d0dcfe6f..c189949e 100644 --- a/stacks/trino-superset-s3/superset.yaml +++ b/stacks/trino-superset-s3/superset.yaml @@ -11,7 +11,7 @@ spec: mapboxSecret: superset-mapbox-api-key nodes: roleConfig: - listenerClass: external-unstable + listenerClass: external-stable roleGroups: default: replicas: 1 diff --git a/stacks/trino-superset-s3/trino.yaml b/stacks/trino-superset-s3/trino.yaml index 5b563da1..ba13f3e9 100644 --- a/stacks/trino-superset-s3/trino.yaml +++ b/stacks/trino-superset-s3/trino.yaml @@ -18,7 +18,7 @@ spec: package: trino coordinators: roleConfig: - listenerClass: external-unstable + listenerClass: external-stable roleGroups: default: replicas: 1