From 2e82b8a9e1f512dd2b73f5ac5966201419721317 Mon Sep 17 00:00:00 2001 From: Tyler Graff <34993299+tylergraff@users.noreply.github.com> Date: Tue, 17 Sep 2024 14:27:22 -0400 Subject: [PATCH 01/16] Create enhanced-security.md --- docs/docs/references/enhanced-security.md | 617 ++++++++++++++++++++++ 1 file changed, 617 insertions(+) create mode 100644 docs/docs/references/enhanced-security.md diff --git a/docs/docs/references/enhanced-security.md b/docs/docs/references/enhanced-security.md new file mode 100644 index 00000000..03c8c702 --- /dev/null +++ b/docs/docs/references/enhanced-security.md @@ -0,0 +1,617 @@ + +The security of AWS Nebari deployments can be enhanced through the following deployment configuration options: + +- Explicit definition of container sources +This option allows for the use of security-hardened or otherwise customized container images in place of the containers used by default. +Containers can be sourced from internally-hosted private repositories to further enhance security. +``` +security: + keycloak: + overrides: + image: + # Keycloak image repository + #repository: quay.io/keycloak/keycloak # default + repository: example-change-me.dkr.ecr.us-gov-east-1.amazonaws.com/quay.io/keycloak/keycloak + # Overrides the Keycloak image tag whose default is the chart version + #tag: "15.0.2" # default + tag: "" + + # This container is used at deploy-time to download keycloak-metrics-spi + extraInitContainers: | + - command: + - sh + - -c + - | # --no-check-certificate is used below due to an in-line SSL content inspection re-encrypting the connection + wget --no-check-certificate https://github.com/aerogear/keycloak-metrics-spi/releases/download/2.5.3/keycloak-metrics-spi-2.5.3.jar -P /data/ && + export SHA256SUM=9b3f52f842a66dadf5ff3cc3a729b8e49042d32f84510a5d73d41a2e39f29a96 && + if ! (echo "$SHA256SUM /data/keycloak-metrics-spi-2.5.3.jar" | sha256sum -c) + then + echo "Error: Checksum not verified" && exit 1 + else + chown 1000:1000 /data/keycloak-metrics-spi-2.5.3.jar && + chmod 777 /data/keycloak-metrics-spi-2.5.3.jar + fi + image: example-change-me.dkr.ecr.us-gov-east-1.amazonaws.com/alpine:latest + name: initialize-spi-metrics-jar + securityContext: + runAsUser: 0 + volumeMounts: + - name: metrics-plugin + mountPath: /data + + pgchecker: + image: + # Docker image used to check Postgresql readiness at startup + # repository: docker.io/busybox # default + repository: example-change-me.dkr.ecr.us-gov-east-1.amazonaws.com/docker.io/busybox + # Image tag for the pgchecker image + tag: 1.32 + + postgresql: + ## https://github.com/bitnami/charts/blob/main/bitnami/postgresql/values.yaml + image: + #registry: docker.io # default + registry: example-change-me.dkr.ecr.us-gov-east-1.amazonaws.com + #repository: bitnami/postgresql + repository: docker.io/bitnami/postgresql + tag: 11.11.0-debian-10-r31 + digest: "" + +default_images: + #jupyterhub: quay.io/nebari/nebari-jupyterhub:2024.5.1 # default + jupyterhub: example-change-me.dkr.ecr.us-gov-east-1.amazonaws.com/quay.io/nebari/nebari-jupyterhub:2024.5.1 + #jupyterlab: quay.io/nebari/nebari-jupyterlab:2024.5.1 # default + jupyterlab: example-change-me.dkr.ecr.us-gov-east-1.amazonaws.com/quay.io/nebari/nebari-jupyterlab:2024.5.1 + #dask_worker: quay.io/nebari/nebari-dask-worker:2024.5.1 # default + dask_worker: example-change-me.dkr.ecr.us-gov-east-1.amazonaws.com/quay.io/nebari/nebari-dask-worker:2024.5.1 + +cluster_autoscaler: + ## https://staging.artifacthub.io/packages/helm/cluster-autoscaler/cluster-autoscaler/9.19.0 + overrides: + image: + #repository: k8s.gcr.io/autoscaling/cluster-autoscaler # default + repository: 096534317619.dkr.ecr.us-gov-east-1.amazonaws.com/k8s.gcr.io/autoscaling/cluster-autoscaler + # image.tag + tag: v1.23.0 + +ingress: + traefik-image: + image: exampe-change-me.dkr.ecr.us-gov-east-1.amazonaws.com/traefik + tag: 2.9.1 + +conda_store: + image: example-change-me.dkr.ecr.us-gov-east-1.amazonaws.com/quansight/conda-store-server + image_tag: 2024.3.1 + +conda_store: + nfs_server_image: 096534317619.dkr.ecr.us-gov-east-1.amazonaws.com/gcr.io/google_containers/volume-nfs + nfs_server_image_tag: "0.8" + overrides: + ## https://github.com/MetroStar/nebari/blob/eks-private-cluster-dev/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/minio/values.yaml + ## https://github.com/bitnami/charts/blob/main/bitnami/minio/values.yaml + minio: + image: + #registry: docker.io + registry: 096534317619.dkr.ecr.us-gov-east-1.amazonaws.com + #repository: bitnami/minio + repository: docker.io/bitnami/minio + tag: 2021.4.22-debian-10-r0 + ## https://github.com/MetroStar/nebari/blob/eks-private-cluster-dev/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/postgresql/values.yaml + ## https://github.com/bitnami/charts/blob/master/bitnami/postgresql/values.yaml + postgresql: + image: + #registry: docker.io + registry: 096534317619.dkr.ecr.us-gov-east-1.amazonaws.com + #repository: bitnami/postgresql + repository: docker.io/bitnami/postgresql + tag: 11.14.0-debian-10-r17 + digest: "" + ## https://github.com/MetroStar/nebari/blob/eks-private-cluster-dev/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/redis/values.yaml + ## https://github.com/bitnami/charts/blob/master/bitnami/redis/values.yaml + redis: + image: + #registry: docker.io + registry: 096534317619.dkr.ecr.us-gov-east-1.amazonaws.com + #repository: bitnami/redis + repository: docker.io/bitnami/redis + tag: 7.0.4-debian-11-r4 + digest: "" +argo_workflows: + #enabled: true + #https://github.com/MetroStar/nebari/blob/eks-private-cluster/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/argo-workflows/values.yaml + #https://github.com/argoproj/argo-helm/blob/argo-workflows-0.22.9/charts/argo-workflows/values.yaml + overrides: + controller: + image: + #registry: quay.io + registry: exampe-change-me.dkr.ecr.us-gov-east-1.amazonaws.com + #repository: argoproj/workflow-controller + repository: quay.io/argoproj/workflow-controller + # -- Image tag for the workflow controller. Defaults to `.Values.images.tag`. + tag: "" + #tag: "v3.4.4" + server: + image: + #registry: quay.io + registry: example-change-me.dkr.ecr.us-gov-east-1.amazonaws.com + #repository: argoproj/argocli + repository: quay.io/argoproj/argocli + #tag: "v3.4.4" + tag: "" + nebari_workflow_controller: + enabled: true + image: example-change-me.dkr.ecr.us-gov-east-1.amazonaws.com/quay.io/nebari/nebari-workflow-controller + image_tag: 2024.5.1 +``` + +- Definition of an ssh key that can access EKS hosts. +EKS hosts by default cannot be accessed via ssh. This configuration item allows ssh access into EKS hosts, which could be useful for troubleshooting or external monitoring and auditing purposes. +``` +amazon_web_services: + ec2_keypair_name: example_keypair_name +``` + +- Installation of custom SSL certificate(s) into EKS hosts. +This can be used to install non-public SSL certificates used by (e.g.) in-line SSL content inspection engines to re-encrypt traffic. +``` + extra_ssl_certificates: | + -----BEGIN CERTIFICATE----- + MIIF......ABCD + -----END CERTIFICATE----- + -----BEGIN CERTIFICATE----- + MIIF......EF01 + -----END CERTIFICATE----- +``` + +- Private EKS endpoint configuration. +This mirrors the corresponding AWS console option, which routes all EKS traffic within the VPC. +``` + eks_endpoint_private_access: true + eks_endpoint_public_access: false +``` + +- Deploy into existing subnets: +This configuration option instructs Nebari to be deployed into existing subnets, rather than creating its own subnets. +``` +existing_subnet_ids: + - subnet-0af78088e8661c36e + - subnet-05123c80a59276d4c + existing_security_group_id: sg-0ffadcc2aaa2a013a +ingress: + terraform_overrides: + # https://kubernetes-sigs.github.io/aws-load-balancer-controller/v2.8/guide/service/annotations/ + load-balancer-annotations: + service.beta.kubernetes.io/aws-load-balancer-internal: "true" + service.beta.kubernetes.io/aws-load-balancer-subnets: "subnet-0af78088e8661c36e,subnet-05123c80a59276d4c" +``` + +``` +monitoring: + enabled: true + overrides: + # https://github.com/MetroStar/nebari/blob/eks-private-cluster/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/monitoring/values.yaml + # https://github.com/prometheus-community/helm-charts/blob/main/charts/kube-prometheus-stack/values.yaml + prometheus: + ## Configuration for alertmanager + ## ref: https://prometheus.io/docs/alerting/alertmanager/ + alertmanager: + ## Settings affecting alertmanagerSpec + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#alertmanagerspec + alertmanagerSpec: + ## Image of Alertmanager + image: + #registry: quay.io + #repository: prometheus/alertmanager + registry: 096534317619.dkr.ecr.us-gov-east-1.amazonaws.com + repository: quay.io/prometheus/alertmanager + tag: v0.27.0 + sha: "" + ## Using default values from https://github.com/grafana/helm-charts/blob/main/charts/grafana/values.yaml + grafana: + image: + # -- The Docker registry + #registry: docker.io + registry: 096534317619.dkr.ecr.us-gov-east-1.amazonaws.com + # -- Docker image repository + #repository: grafana/grafana + repository: docker.io/grafana/grafana + # Overrides the Grafana image tag whose default is the chart appVersion + tag: "" + #tag: "10.4.1" + sha: "" + pullPolicy: IfNotPresent + ## Sidecars that collect the configmaps with specified label and stores the included files them into the respective folders + ## Requires at least Grafana 5 to work and can't be used together with parameters dashboardProviders, datasources and dashboards + sidecar: + image: + # -- The Docker registry + #registry: quay.io + registry: 096534317619.dkr.ecr.us-gov-east-1.amazonaws.com + #repository: kiwigrid/k8s-sidecar + repository: quay.io/kiwigrid/k8s-sidecar + tag: 1.26.1 + sha: "" + ## Manages Prometheus and Alertmanager components + prometheusOperator: + enabled: true + ## Prometheus-operator image + image: + #registry: quay.io + #repository: prometheus-operator/prometheus-operator + registry: 096534317619.dkr.ecr.us-gov-east-1.amazonaws.com + repository: quay.io/prometheus-operator/prometheus-operator + # if not set appVersion field from Chart.yaml is used + tag: "" + #tag: "v0.73.2" + sha: "" + pullPolicy: IfNotPresent + ## Prometheus-config-reloader + prometheusConfigReloader: + image: + #registry: quay.io + #repository: prometheus-operator/prometheus-config-reloader + registry: 096534317619.dkr.ecr.us-gov-east-1.amazonaws.com + repository: quay.io/prometheus-operator/prometheus-config-reloader + # if not set appVersion field from Chart.yaml is used + #tag: "v0.73.2" + tag: "" + sha: "" + ## Configuration for kube-state-metrics subchart + ## https://github.com/prometheus-community/helm-charts/blob/main/charts/kube-state-metrics/values.yaml + kube-state-metrics: + image: + #registry: registry.k8s.io + registry: 096534317619.dkr.ecr.us-gov-east-1.amazonaws.com + #repository: kube-state-metrics/kube-state-metrics + repository: registry.k8s.io/kube-state-metrics/kube-state-metrics + # If unset use v + .Charts.appVersion + tag: "" + #tag: "v2.12.0" + sha: "" + pullPolicy: IfNotPresent + ## Configuration for prometheus-node-exporter subchart + ## https://github.com/prometheus-community/helm-charts/blob/main/charts/prometheus-node-exporter/values.yaml + prometheus-node-exporter: + # Default values for prometheus-node-exporter. + image: + #registry: quay.io + registry: 096534317619.dkr.ecr.us-gov-east-1.amazonaws.com + #repository: prometheus/node-exporter + repository: quay.io/prometheus/node-exporter + # Overrides the image tag whose default is {{ printf "v%s" .Chart.AppVersion }} + tag: "" + #tag: "v1.8.0" + pullPolicy: IfNotPresent + digest: "" + ## Deploy a Prometheus instance + prometheus: + enabled: true + ## Settings affecting prometheusSpec + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#prometheusspec + prometheusSpec: + ## Image of Prometheus. + image: + #registry: quay.io + #repository: prometheus/prometheus + registry: 096534317619.dkr.ecr.us-gov-east-1.amazonaws.com + repository: quay.io/prometheus/prometheus + tag: v2.51.2 + #tag: v2.53.1 + sha: "" + loki: + loki: + storage: + type: s3 + commonConfig: + replication_factor: 1 + # Not required as it is inside cluster and not exposed to the public network + auth_enabled: false + # The Compactor deduplicates index entries and also apply granular retention. + compactor: + # is the directory where marked chunks and temporary tables will be saved. + working_directory: /var/loki/compactor/data/retention + # minio s3 + shared_store: s3 + # how often compaction will happen + compaction_interval: 1h + # should delete old logs after retention delete delay + # ideally we would want to do storage based retention, but this is not + # currently implemented in loki, that's why we're doing time based retention. + retention_enabled: true + # is the delay after which the Compactor will delete marked chunks. + retention_delete_delay: 1h + # specifies the maximum quantity of goroutine workers instantiated to delete chunks. + retention_delete_worker_count: 150 + limits_config: + # The minimum retention period is 24h. + # This is reasonable in most cases, but if people would like to retain logs for longer + # then they can override this variable from nebari-config.yaml + retention_period: 60d + schema_config: + configs: + # list of period_configs + # The date of the first day that index buckets should be created. + - from: "2024-03-01" + index: + period: 24h + prefix: loki_index_ + object_store: s3 + schema: v11 + store: boltdb-shipper + storage_config: + boltdb_shipper: + # Directory where ingesters would write index files which would then be + # uploaded by shipper to configured storage + active_index_directory: /var/loki/compactor/data/index + # Cache location for restoring index files from storage for queries + cache_location: /var/loki/compactor/data/boltdb-cache + # Shared store for keeping index files + shared_store: s3 + image: + # -- The Docker registry + #registry: docker.io + registry: 096534317619.dkr.ecr.us-gov-east-1.amazonaws.com + # -- Docker image repository + #repository: grafana/loki + repository: docker.io/grafana/loki + # -- Overrides the image tag whose default is the chart's appVersion + # TODO: needed for 3rd target backend functionality + # revert to null or latest once this behavior is relased + tag: null + #tag: "2.9.4" + ## -- Overrides the image tag with an image digest + #digest: null + # Configuration for the write pod(s) + write: + # -- Number of replicas for the write + # Keeping cost of running Nebari in mind + # We don't need so many replicas, if people need it + # they can always override from nebari-config.yaml + replicas: 1 + read: + # -- Number of replicas for the read + replicas: 1 + backend: + # -- Number of replicas for the backend + replicas: 1 + minio: + # We are deploying minio from bitnami chart separately + enabled: false + monitoring: + selfMonitoring: + grafanaAgent: + installOperator: false + lokiCanary: + image: + # -- The Docker registry + #registry: docker.io + registry: 096534317619.dkr.ecr.us-gov-east-1.amazonaws.com + # -- Docker image repository + #repository: grafana/loki-canary + repository: docker.io/grafana/loki-canary + # -- Overrides the image tag whose default is the chart's appVersion + tag: null + #tag: "2.9.4" + ## -- Overrides the image tag with an image digest + #digest: null + ## -- Docker image pull policy + #pullPolicy: IfNotPresent + # Configuration for the gateway + gateway: + image: + # -- The Docker registry for the gateway image + #registry: docker.io + registry: 096534317619.dkr.ecr.us-gov-east-1.amazonaws.com + # -- The gateway image repository + #repository: nginxinc/nginx-unprivileged + repository: docker.io/nginxinc/nginx-unprivileged + # -- The gateway image tag + tag: 1.24-alpine + ## -- Overrides the gateway image tag with an image digest + #digest: null + ## -- The gateway image pull policy + #pullPolicy: IfNotPresent + sidecar: + image: + # -- The Docker registry and image for the k8s sidecar + #repository: kiwigrid/k8s-sidecar + repository: 096534317619.dkr.ecr.us-gov-east-1.amazonaws.com/kiwigrid/k8s-sidecar + # -- Docker image tag + tag: 1.24.3 + ## -- Docker image sha. If empty, no sha will be used + #sha: "" + ## -- Docker image pull policy + #pullPolicy: IfNotPresent + # https://github.com/grafana/helm-charts/blob/3831194ba2abd2a0ca7a14ca00e578f8e9d2abc6/charts/promtail/values.yaml + promtail: + image: + # -- The Docker registry + #registry: docker.io + registry: 096534317619.dkr.ecr.us-gov-east-1.amazonaws.com + # -- Docker image repository + #repository: grafana/promtail + repository: docker.io/grafana/promtail + # -- Overrides the image tag whose default is the chart's appVersion + tag: null + #tag: 2.9.3 + ## -- Docker image pull policy + #pullPolicy: IfNotPresent + # https://github.com/bitnami/charts/blob/440ec159c26e4ff0748b9e9866b345d98220c40a/bitnami/minio/values.yaml + minio: + image: + #registry: docker.io + registry: 096534317619.dkr.ecr.us-gov-east-1.amazonaws.com + #repository: bitnami/minio + repository: docker.io/bitnami/minio + tag: 2021.4.22-debian-10-r0 + ### Specify a imagePullPolicy + ### Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent' + ### ref: http://kubernetes.io/docs/user-guide/images/#pre-pulling-images + ### + #pullPolicy: IfNotPresent + ### Optionally specify an array of imagePullSecrets. + ### Secrets must be manually created in the namespace. + ### ref: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/ + ### e.g: + ### pullSecrets: + ### - myRegistryKeySecretName + ### + #pullSecrets: [] + ### Set to true if you would like to see extra information on logs + ### + #debug: false +jupyterhub: + #volume_mount_init_image: "busybox:1.31" + volume_mount_init_image: 096534317619.dkr.ecr.us-gov-east-1.amazonaws.com/busybox:1.31 + # https://github.com/MetroStar/nebari/blob/eks-private-cluster/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/values.yaml + # https://github.com/jupyterhub/zero-to-jupyterhub-k8s/blob/main/jupyterhub/values.yaml + overrides: + hub: + db: + type: sqlite-pvc + pvc: + storage: 1Gi + baseUrl: "/" + networkPolicy: + ingress: + - ports: + - port: 10202 + from: + - podSelector: + matchLabels: + hub.jupyter.org/network-access-hub: "true" + service: + extraPorts: + - port: 10202 + targetPort: 10202 + name: jhub-apps + proxy: + secretToken: "" + service: + type: ClusterIP + chp: + image: + #name: quay.io/jupyterhub/configurable-http-proxy + name: 096534317619.dkr.ecr.us-gov-east-1.amazonaws.com/quay.io/jupyterhub/configurable-http-proxy + #tag: 4.6.2 # https://github.com/jupyterhub/configurable-http-proxy/releases + tag: 4.6.1 + #pullPolicy: + #pullSecrets: [] + networkPolicy: + egressAllowRules: + cloudMetadataServer: false + dnsPortsPrivateIPs: false + nonPrivateIPs: false + privateIPs: false + egress: + - ports: + - port: 53 + protocol: UDP + - port: 53 + protocol: TCP + - port: 10202 + protocol: TCP + - to: + - ipBlock: + cidr: 0.0.0.0/0 + scheduling: + userScheduler: + enabled: true + image: + # IMPORTANT: Bumping the minor version of this binary should go hand in + # hand with an inspection of the user-scheduelrs RBAC resources + # that we have forked. + #name: registry.k8s.io/kube-scheduler + name: 096534317619.dkr.ecr.us-gov-east-1.amazonaws.com/registry.k8s.io/kube-scheduler + #tag: "v1.28.11" # ref: https://github.com/kubernetes/website/blob/main/content/en/releases/patch-releases.md + tag: "v1.28.10" + #pullPolicy: + #pullSecrets: [] + podPriority: + enabled: true + userPlaceholder: + enabled: false + replicas: 1 + #image: + # name: registry.k8s.io/pause + # # tag is automatically bumped to new patch versions by the + # # watch-dependencies.yaml workflow. + # # If you update this, also update prePuller.pause.image.tag + # tag: "3.10" + imagePullSecrets: + - extcrcreds + singleuser: + defaultUrl: "/lab" + startTimeout: 600 # 10 minutes + profileList: [] + storage: + type: static + extraVolumeMounts: + - mountPath: "/home/shared" + name: home + subPath: "home/shared" + networkTools: + image: + #name: quay.io/jupyterhub/k8s-network-tools + name: 096534317619.dkr.ecr.us-gov-east-1.amazonaws.com/quay.io/jupyterhub/k8s-network-tools + #tag: "set-by-chartpress" + tag: 4.0.0-0.dev.git.6548.h9b2dfe22 + cpu: + limit: 1 + guarantee: 1 + memory: + limit: "1G" + guarantee: "1G" + networkPolicy: + enabled: false + # prePuller relates to the hook|continuous-image-puller DaemonsSets + prePuller: + pause: + image: + #name: registry.k8s.io/pause + name: 096534317619.dkr.ecr.us-gov-east-1.amazonaws.com/registry.k8s.io/pause + # tag is automatically bumped to new patch versions by the + # watch-dependencies.yaml workflow. + # If you update this, also update scheduling.userPlaceholder.image.tag + tag: "3.10" + # cull relates to the jupyterhub-idle-culler service, responsible for evicting + # inactive singleuser pods. + # + # The configuration below, except for enabled, corresponds to command-line flags + # for jupyterhub-idle-culler as documented here: + # https://github.com/jupyterhub/jupyterhub-idle-culler#as-a-standalone-script + # + cull: + enabled: true + users: false + removeNamedServers: false + timeout: 1800 + every: 600 + concurrency: 10 + maxAge: 0 +jupyterhub_ssh: + jupyterhub_ssh_image: + name: 096534317619.dkr.ecr.us-gov-east-1.amazonaws.com/quay.io/jupyterhub-ssh/ssh + tag: 0.0.1-0.dev.git.136.ha610981 + jupyterhub_sftp_image: + name: 096534317619.dkr.ecr.us-gov-east-1.amazonaws.com/quay.io/jupyterhub-ssh/sftp + tag: 0.0.1-0.dev.git.142.h402a3d6 +dask_gateway: + dask_gateway_image: + #name: ghcr.io/dask/dask-gateway-server + name: 096534317619.dkr.ecr.us-gov-east-1.amazonaws.com/ghcr.io/dask/dask-gateway-server + tag: "2022.4.0" + dask_controller_image: + #name: ghcr.io/dask/dask-gateway-server + name: 096534317619.dkr.ecr.us-gov-east-1.amazonaws.com/ghcr.io/dask/dask-gateway-server + tag: "2022.4.0" +forward_auth: + traefik_forwardauth_image: + #name: maxisme/traefik-forward-auth + name: 096534317619.dkr.ecr.us-gov-east-1.amazonaws.com/maxisme/traefik-forward-auth + tag: "sha-a98e568" +certificate: + #type: existing + #secret_name: nebari-custom-secret + type: self-signed +``` From 04fc5c5bd9bdc1b84fb9fe25de2e52d075baa8a6 Mon Sep 17 00:00:00 2001 From: Tyler Graff <34993299+tylergraff@users.noreply.github.com> Date: Wed, 23 Oct 2024 09:32:54 -0400 Subject: [PATCH 02/16] Create container-sources-2024-09-1.md --- .../references/container-sources-2024-09-1.md | 183 ++++++++++++++++++ 1 file changed, 183 insertions(+) create mode 100644 docs/docs/references/container-sources-2024-09-1.md diff --git a/docs/docs/references/container-sources-2024-09-1.md b/docs/docs/references/container-sources-2024-09-1.md new file mode 100644 index 00000000..ebd3e7dd --- /dev/null +++ b/docs/docs/references/container-sources-2024-09-1.md @@ -0,0 +1,183 @@ +## Deploying and Running Nebari from a Private Container Repository + +Nebari deploys and runs FOSS components as containers running in Kubernetes. +By default, Nebari sources each container from the container's respective public repository, typically `docker.io` or `quay.io`. +This introduces supply-chain concerns for security-focused customers. + + +One solution to these supply-chain concerns is to deploy Nebari from private locally-mirrored containers: +- Create a controlled private container repository (e.g. ECR or GitLab Container Repo) +- Mirror all containers used by Nebari into this private container repository +- Use the `overrides` mechanism in `nebari-config.yaml` to specify the mirrored container sources + +Deploying Nebari in this fashion eliminates significant supply chain surface-area, but requires identifying all containers used by Nebari. + +The following configuration enumerates all container images used by Nebari 2024-9-1 and demonstrates how to source them from a private repo denoted by the string `[LOCAL_ECR].dkr.ecr.us-gov-east-1.amazonaws.com`. +The commented-out `registry` and `repository` keys document the original public source repository from which the container images are to be mirrored. + +``` +monitoring: + overrides: + prometheus: + alertmanager: + alertmanagerSpec: + ## Image of Alertmanager + image: + #registry: quay.io + #repository: prometheus/alertmanager + registry: [LOCAL_ECR].dkr.ecr.us-gov-east-1.amazonaws.com + repository: quay.io/prometheus/alertmanager + tag: v0.27.0 + sha: "" + grafana: + image: + #registry: docker.io + registry: [LOCAL_ECR].dkr.ecr.us-gov-east-1.amazonaws.com + #repository: grafana/grafana + repository: docker.io/grafana/grafana + tag: "" + sha: "" + pullPolicy: IfNotPresent + sidecar: + image: + #registry: quay.io + registry: [LOCAL_ECR].dkr.ecr.us-gov-east-1.amazonaws.com + #repository: kiwigrid/k8s-sidecar + repository: quay.io/kiwigrid/k8s-sidecar + tag: 1.26.1 + sha: "" + prometheusOperator: + image: + #registry: quay.io + registry: [LOCAL_ECR].dkr.ecr.us-gov-east-1.amazonaws.com + #repository: prometheus-operator/prometheus-operator + repository: quay.io/prometheus-operator/prometheus-operator + tag: "" + sha: "" + prometheusConfigReloader: + image: + #registry: quay.io + registry: [LOCAL_ECR].dkr.ecr.us-gov-east-1.amazonaws.com + #repository: prometheus-operator/prometheus-config-reloader + repository: quay.io/prometheus-operator/prometheus-config-reloader + tag: "" + sha: "" + kube-state-metrics: + image: + #registry: registry.k8s.io + registry: [LOCAL_ECR].dkr.ecr.us-gov-east-1.amazonaws.com + #repository: kube-state-metrics/kube-state-metrics + repository: registry.k8s.io/kube-state-metrics/kube-state-metrics + tag: "" + sha: "" + pullPolicy: IfNotPresent + prometheus-node-exporter: + image: + #registry: quay.io + registry: [LOCAL_ECR].dkr.ecr.us-gov-east-1.amazonaws.com + #repository: prometheus/node-exporter + repository: quay.io/prometheus/node-exporter + tag: "" + pullPolicy: IfNotPresent + digest: "" + prometheus: + prometheusSpec: + image: + #registry: quay.io + registry: [LOCAL_ECR].dkr.ecr.us-gov-east-1.amazonaws.com + #repository: prometheus/prometheus + repository: quay.io/prometheus/prometheus + tag: v2.51.2 + sha: "" + loki: + loki: + image: + #registry: docker.io + registry: [LOCAL_ECR].dkr.ecr.us-gov-east-1.amazonaws.com + #repository: grafana/loki + repository: docker.io/grafana/loki + tag: null + lokiCanary: + image: + #registry: docker.io + registry: [LOCAL_ECR].dkr.ecr.us-gov-east-1.amazonaws.com + #repository: grafana/loki-canary + repository: docker.io/grafana/loki-canary + tag: null + gateway: + image: + #registry: docker.io + registry: [LOCAL_ECR].dkr.ecr.us-gov-east-1.amazonaws.com + #repository: nginxinc/nginx-unprivileged + repository: docker.io/nginxinc/nginx-unprivileged + tag: 1.24-alpine + sidecar: + image: + #repository: kiwigrid/k8s-sidecar + repository: [LOCAL_ECR].dkr.ecr.us-gov-east-1.amazonaws.com/kiwigrid/k8s-sidecar + tag: 1.24.3 + promtail: + image: + #registry: docker.io + registry: [LOCAL_ECR].dkr.ecr.us-gov-east-1.amazonaws.com + #repository: grafana/promtail + repository: docker.io/grafana/promtail + tag: null + # https://github.com/bitnami/charts/blob/440ec159c26e4ff0748b9e9866b345d98220c40a/bitnami/minio/values.yaml + minio: + image: + #registry: docker.io + registry: [LOCAL_ECR].dkr.ecr.us-gov-east-1.amazonaws.com + #repository: bitnami/minio + repository: docker.io/bitnami/minio + tag: 2021.4.22-debian-10-r0 +jupyterhub: + #volume_mount_init_image: "busybox:1.31" + volume_mount_init_image: [LOCAL_ECR].dkr.ecr.us-gov-east-1.amazonaws.com/busybox:1.31 + proxy: + chp: + image: + #name: quay.io/jupyterhub/configurable-http-proxy + name: [LOCAL_ECR].dkr.ecr.us-gov-east-1.amazonaws.com/quay.io/jupyterhub/configurable-http-proxy + tag: 4.6.1 + scheduling: + userScheduler: + enabled: true + image: + #name: registry.k8s.io/kube-scheduler + name: [LOCAL_ECR].dkr.ecr.us-gov-east-1.amazonaws.com/registry.k8s.io/kube-scheduler + tag: "v1.28.10" + singleuser: + networkTools: + image: + #name: quay.io/jupyterhub/k8s-network-tools + name: [LOCAL_ECR].dkr.ecr.us-gov-east-1.amazonaws.com/quay.io/jupyterhub/k8s-network-tools + tag: 4.0.0-0.dev.git.6548.h9b2dfe22 + prePuller: + pause: + image: + #name: registry.k8s.io/pause + name: [LOCAL_ECR].dkr.ecr.us-gov-east-1.amazonaws.com/registry.k8s.io/pause + tag: "3.10" +jupyterhub_ssh: + jupyterhub_ssh_image: + name: [LOCAL_ECR].dkr.ecr.us-gov-east-1.amazonaws.com/quay.io/jupyterhub-ssh/ssh + tag: 0.0.1-0.dev.git.136.ha610981 + jupyterhub_sftp_image: + name: [LOCAL_ECR].dkr.ecr.us-gov-east-1.amazonaws.com/quay.io/jupyterhub-ssh/sftp + tag: 0.0.1-0.dev.git.142.h402a3d6 +dask_gateway: + dask_gateway_image: + #name: ghcr.io/dask/dask-gateway-server + name: [LOCAL_ECR].dkr.ecr.us-gov-east-1.amazonaws.com/ghcr.io/dask/dask-gateway-server + tag: "2022.4.0" + dask_controller_image: + #name: ghcr.io/dask/dask-gateway-server + name: [LOCAL_ECR].dkr.ecr.us-gov-east-1.amazonaws.com/ghcr.io/dask/dask-gateway-server + tag: "2022.4.0" +forward_auth: + traefik_forwardauth_image: + #name: maxisme/traefik-forward-auth + name: [LOCAL_ECR].dkr.ecr.us-gov-east-1.amazonaws.com/maxisme/traefik-forward-auth + tag: "sha-a98e568" +``` From 0c35f6c480447b52f218d2f9b6c3840ad1319be5 Mon Sep 17 00:00:00 2001 From: Tyler Graff <34993299+tylergraff@users.noreply.github.com> Date: Wed, 23 Oct 2024 09:47:32 -0400 Subject: [PATCH 03/16] add more containers --- .../references/container-sources-2024-09-1.md | 120 +++++++++++++++++- 1 file changed, 118 insertions(+), 2 deletions(-) diff --git a/docs/docs/references/container-sources-2024-09-1.md b/docs/docs/references/container-sources-2024-09-1.md index ebd3e7dd..2ede2d1e 100644 --- a/docs/docs/references/container-sources-2024-09-1.md +++ b/docs/docs/references/container-sources-2024-09-1.md @@ -13,9 +13,123 @@ One solution to these supply-chain concerns is to deploy Nebari from private loc Deploying Nebari in this fashion eliminates significant supply chain surface-area, but requires identifying all containers used by Nebari. The following configuration enumerates all container images used by Nebari 2024-9-1 and demonstrates how to source them from a private repo denoted by the string `[LOCAL_ECR].dkr.ecr.us-gov-east-1.amazonaws.com`. -The commented-out `registry` and `repository` keys document the original public source repository from which the container images are to be mirrored. +The commented-out elements document the original public sources from which the container images are to be mirrored. ``` +default_images: + #jupyterhub: quay.io/nebari/nebari-jupyterhub:2024.5.1 + jupyterhub: [LOCAL_ECR].dkr.ecr.us-gov-east-1.amazonaws.com/quay.io/nebari/nebari-jupyterhub:2024.5.1 + #jupyterlab: quay.io/nebari/nebari-jupyterlab:2024.5.1 + jupyterlab: [LOCAL_ECR].dkr.ecr.us-gov-east-1.amazonaws.com/quay.io/nebari/nebari-jupyterlab:2024.5.1 + #dask_worker: quay.io/nebari/nebari-dask-worker:2024.5.1 + dask_worker: [LOCAL_ECR].dkr.ecr.us-gov-east-1.amazonaws.com/quay.io/nebari/nebari-dask-worker:2024.5.1 + +security: + keycloak: + overrides: + image: + # Keycloak image repository + #repository: quay.io/keycloak/keycloak # default + repository: [LOCAL_ECR].dkr.ecr.us-gov-east-1.amazonaws.com/quay.io/keycloak/keycloak + # Overrides the Keycloak image tag whose default is the chart version + #tag: "15.0.2" # default + tag: "" + + # This container is used at deploy-time to download keycloak-metrics-spi + extraInitContainers: | + - command: + - sh + - -c + - | wget --no-check-certificate https://github.com/aerogear/keycloak-metrics-spi/releases/download/2.5.3/keycloak-metrics-spi-2.5.3.jar -P /data/ && + export SHA256SUM=9b3f52f842a66dadf5ff3cc3a729b8e49042d32f84510a5d73d41a2e39f29a96 && + if ! (echo "$SHA256SUM /data/keycloak-metrics-spi-2.5.3.jar" | sha256sum -c) + then + echo "Error: Checksum not verified" && exit 1 + else + chown 1000:1000 /data/keycloak-metrics-spi-2.5.3.jar && + chmod 777 /data/keycloak-metrics-spi-2.5.3.jar + fi + image: [LOCAL_ECR].dkr.ecr.us-gov-east-1.amazonaws.com/alpine:latest + name: initialize-spi-metrics-jar + pgchecker: + image: + # repository: docker.io/busybox + repository: [LOCAL_ECR].dkr.ecr.us-gov-east-1.amazonaws.com/docker.io/busybox + tag: 1.32 + postgresql: + image: + #registry: docker.io + registry: [LOCAL_ECR].dkr.ecr.us-gov-east-1.amazonaws.com + #repository: bitnami/postgresql + repository: docker.io/bitnami/postgresql + tag: 11.11.0-debian-10-r31 + digest: "" + +cluster_autoscaler: + overrides: + image: + #repository: k8s.gcr.io/autoscaling/cluster-autoscaler + repository: [LOCAL_ECR].dkr.ecr.us-gov-east-1.amazonaws.com/k8s.gcr.io/autoscaling/cluster-autoscaler + tag: v1.23.0 + +ingress: + traefik-image: + image: [LOCAL_ECR].dkr.ecr.us-gov-east-1.amazonaws.com/traefik + tag: 2.9.1 + +conda_store: + image: [LOCAL_ECR].dkr.ecr.us-gov-east-1.amazonaws.com/quansight/conda-store-server + image_tag: 2024.3.1 + +conda_store: + nfs_server_image: [LOCAL_ECR].dkr.ecr.us-gov-east-1.amazonaws.com/gcr.io/google_containers/volume-nfs + nfs_server_image_tag: "0.8" + overrides: + minio: + image: + #registry: docker.io + registry: [LOCAL_ECR].dkr.ecr.us-gov-east-1.amazonaws.com + #repository: bitnami/minio + repository: docker.io/bitnami/minio + tag: 2021.4.22-debian-10-r0 + postgresql: + image: + #registry: docker.io + registry: [LOCAL_ECR].dkr.ecr.us-gov-east-1.amazonaws.com + #repository: bitnami/postgresql + repository: docker.io/bitnami/postgresql + tag: 11.14.0-debian-10-r17 + digest: "" + redis: + image: + #registry: docker.io + registry: [LOCAL_ECR].dkr.ecr.us-gov-east-1.amazonaws.com + #repository: bitnami/redis + repository: docker.io/bitnami/redis + tag: 7.0.4-debian-11-r4 + digest: "" + +argo_workflows: + overrides: + controller: + image: + #registry: quay.io + registry: [LOCAL_ECR].dkr.ecr.us-gov-east-1.amazonaws.com + #repository: argoproj/workflow-controller + repository: quay.io/argoproj/workflow-controller + tag: "" + server: + image: + #registry: quay.io + registry: [LOCAL_ECR].dkr.ecr.us-gov-east-1.amazonaws.com + #repository: argoproj/argocli + repository: quay.io/argoproj/argocli + tag: "v3.4.4" + nebari_workflow_controller: + enabled: true + image: [LOCAL_ECR].dkr.ecr.us-gov-east-1.amazonaws.com/quay.io/nebari/nebari-workflow-controller + image_tag: 2024.5.1 + monitoring: overrides: prometheus: @@ -123,7 +237,6 @@ monitoring: #repository: grafana/promtail repository: docker.io/grafana/promtail tag: null - # https://github.com/bitnami/charts/blob/440ec159c26e4ff0748b9e9866b345d98220c40a/bitnami/minio/values.yaml minio: image: #registry: docker.io @@ -131,6 +244,7 @@ monitoring: #repository: bitnami/minio repository: docker.io/bitnami/minio tag: 2021.4.22-debian-10-r0 + jupyterhub: #volume_mount_init_image: "busybox:1.31" volume_mount_init_image: [LOCAL_ECR].dkr.ecr.us-gov-east-1.amazonaws.com/busybox:1.31 @@ -166,6 +280,7 @@ jupyterhub_ssh: jupyterhub_sftp_image: name: [LOCAL_ECR].dkr.ecr.us-gov-east-1.amazonaws.com/quay.io/jupyterhub-ssh/sftp tag: 0.0.1-0.dev.git.142.h402a3d6 + dask_gateway: dask_gateway_image: #name: ghcr.io/dask/dask-gateway-server @@ -175,6 +290,7 @@ dask_gateway: #name: ghcr.io/dask/dask-gateway-server name: [LOCAL_ECR].dkr.ecr.us-gov-east-1.amazonaws.com/ghcr.io/dask/dask-gateway-server tag: "2022.4.0" + forward_auth: traefik_forwardauth_image: #name: maxisme/traefik-forward-auth From 150541e663bfbc3d00a91961156ed5953ac786f4 Mon Sep 17 00:00:00 2001 From: Tyler Graff <34993299+tylergraff@users.noreply.github.com> Date: Wed, 23 Oct 2024 10:02:34 -0400 Subject: [PATCH 04/16] Update enhanced-security.md --- docs/docs/references/enhanced-security.md | 606 +--------------------- 1 file changed, 23 insertions(+), 583 deletions(-) diff --git a/docs/docs/references/enhanced-security.md b/docs/docs/references/enhanced-security.md index 03c8c702..3a060951 100644 --- a/docs/docs/references/enhanced-security.md +++ b/docs/docs/references/enhanced-security.md @@ -1,158 +1,21 @@ +## Nebari Security Considerations -The security of AWS Nebari deployments can be enhanced through the following deployment configuration options: +The security of *AWS Nebari* deployments can be enhanced through the following deployment configuration options in `nebari-config.yaml`: -- Explicit definition of container sources -This option allows for the use of security-hardened or otherwise customized container images in place of the containers used by default. -Containers can be sourced from internally-hosted private repositories to further enhance security. -``` -security: - keycloak: - overrides: - image: - # Keycloak image repository - #repository: quay.io/keycloak/keycloak # default - repository: example-change-me.dkr.ecr.us-gov-east-1.amazonaws.com/quay.io/keycloak/keycloak - # Overrides the Keycloak image tag whose default is the chart version - #tag: "15.0.2" # default - tag: "" - - # This container is used at deploy-time to download keycloak-metrics-spi - extraInitContainers: | - - command: - - sh - - -c - - | # --no-check-certificate is used below due to an in-line SSL content inspection re-encrypting the connection - wget --no-check-certificate https://github.com/aerogear/keycloak-metrics-spi/releases/download/2.5.3/keycloak-metrics-spi-2.5.3.jar -P /data/ && - export SHA256SUM=9b3f52f842a66dadf5ff3cc3a729b8e49042d32f84510a5d73d41a2e39f29a96 && - if ! (echo "$SHA256SUM /data/keycloak-metrics-spi-2.5.3.jar" | sha256sum -c) - then - echo "Error: Checksum not verified" && exit 1 - else - chown 1000:1000 /data/keycloak-metrics-spi-2.5.3.jar && - chmod 777 /data/keycloak-metrics-spi-2.5.3.jar - fi - image: example-change-me.dkr.ecr.us-gov-east-1.amazonaws.com/alpine:latest - name: initialize-spi-metrics-jar - securityContext: - runAsUser: 0 - volumeMounts: - - name: metrics-plugin - mountPath: /data - - pgchecker: - image: - # Docker image used to check Postgresql readiness at startup - # repository: docker.io/busybox # default - repository: example-change-me.dkr.ecr.us-gov-east-1.amazonaws.com/docker.io/busybox - # Image tag for the pgchecker image - tag: 1.32 - - postgresql: - ## https://github.com/bitnami/charts/blob/main/bitnami/postgresql/values.yaml - image: - #registry: docker.io # default - registry: example-change-me.dkr.ecr.us-gov-east-1.amazonaws.com - #repository: bitnami/postgresql - repository: docker.io/bitnami/postgresql - tag: 11.11.0-debian-10-r31 - digest: "" - -default_images: - #jupyterhub: quay.io/nebari/nebari-jupyterhub:2024.5.1 # default - jupyterhub: example-change-me.dkr.ecr.us-gov-east-1.amazonaws.com/quay.io/nebari/nebari-jupyterhub:2024.5.1 - #jupyterlab: quay.io/nebari/nebari-jupyterlab:2024.5.1 # default - jupyterlab: example-change-me.dkr.ecr.us-gov-east-1.amazonaws.com/quay.io/nebari/nebari-jupyterlab:2024.5.1 - #dask_worker: quay.io/nebari/nebari-dask-worker:2024.5.1 # default - dask_worker: example-change-me.dkr.ecr.us-gov-east-1.amazonaws.com/quay.io/nebari/nebari-dask-worker:2024.5.1 +- **Explicit definition of container sources** +This option allows for the use of locally mirrored, security-hardened, or otherwise customized container images in place of the containers used by default. +See: [container-sources](container-sources-2024-09-1.md) -cluster_autoscaler: - ## https://staging.artifacthub.io/packages/helm/cluster-autoscaler/cluster-autoscaler/9.19.0 - overrides: - image: - #repository: k8s.gcr.io/autoscaling/cluster-autoscaler # default - repository: 096534317619.dkr.ecr.us-gov-east-1.amazonaws.com/k8s.gcr.io/autoscaling/cluster-autoscaler - # image.tag - tag: v1.23.0 - -ingress: - traefik-image: - image: exampe-change-me.dkr.ecr.us-gov-east-1.amazonaws.com/traefik - tag: 2.9.1 - -conda_store: - image: example-change-me.dkr.ecr.us-gov-east-1.amazonaws.com/quansight/conda-store-server - image_tag: 2024.3.1 - -conda_store: - nfs_server_image: 096534317619.dkr.ecr.us-gov-east-1.amazonaws.com/gcr.io/google_containers/volume-nfs - nfs_server_image_tag: "0.8" - overrides: - ## https://github.com/MetroStar/nebari/blob/eks-private-cluster-dev/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/minio/values.yaml - ## https://github.com/bitnami/charts/blob/main/bitnami/minio/values.yaml - minio: - image: - #registry: docker.io - registry: 096534317619.dkr.ecr.us-gov-east-1.amazonaws.com - #repository: bitnami/minio - repository: docker.io/bitnami/minio - tag: 2021.4.22-debian-10-r0 - ## https://github.com/MetroStar/nebari/blob/eks-private-cluster-dev/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/postgresql/values.yaml - ## https://github.com/bitnami/charts/blob/master/bitnami/postgresql/values.yaml - postgresql: - image: - #registry: docker.io - registry: 096534317619.dkr.ecr.us-gov-east-1.amazonaws.com - #repository: bitnami/postgresql - repository: docker.io/bitnami/postgresql - tag: 11.14.0-debian-10-r17 - digest: "" - ## https://github.com/MetroStar/nebari/blob/eks-private-cluster-dev/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/redis/values.yaml - ## https://github.com/bitnami/charts/blob/master/bitnami/redis/values.yaml - redis: - image: - #registry: docker.io - registry: 096534317619.dkr.ecr.us-gov-east-1.amazonaws.com - #repository: bitnami/redis - repository: docker.io/bitnami/redis - tag: 7.0.4-debian-11-r4 - digest: "" -argo_workflows: - #enabled: true - #https://github.com/MetroStar/nebari/blob/eks-private-cluster/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/argo-workflows/values.yaml - #https://github.com/argoproj/argo-helm/blob/argo-workflows-0.22.9/charts/argo-workflows/values.yaml - overrides: - controller: - image: - #registry: quay.io - registry: exampe-change-me.dkr.ecr.us-gov-east-1.amazonaws.com - #repository: argoproj/workflow-controller - repository: quay.io/argoproj/workflow-controller - # -- Image tag for the workflow controller. Defaults to `.Values.images.tag`. - tag: "" - #tag: "v3.4.4" - server: - image: - #registry: quay.io - registry: example-change-me.dkr.ecr.us-gov-east-1.amazonaws.com - #repository: argoproj/argocli - repository: quay.io/argoproj/argocli - #tag: "v3.4.4" - tag: "" - nebari_workflow_controller: - enabled: true - image: example-change-me.dkr.ecr.us-gov-east-1.amazonaws.com/quay.io/nebari/nebari-workflow-controller - image_tag: 2024.5.1 -``` -- Definition of an ssh key that can access EKS hosts. -EKS hosts by default cannot be accessed via ssh. This configuration item allows ssh access into EKS hosts, which could be useful for troubleshooting or external monitoring and auditing purposes. +- **Definition of an ssh key that can access EKS hosts** +EKS hosts by default cannot be accessed via ssh. This configuration item allows ssh access into EKS hosts, which can be useful for troubleshooting or external monitoring and auditing purposes. ``` amazon_web_services: - ec2_keypair_name: example_keypair_name + ec2_keypair_name: [example_keypair_name] # Name, not ARN ``` -- Installation of custom SSL certificate(s) into EKS hosts. -This can be used to install non-public SSL certificates used by (e.g.) in-line SSL content inspection engines to re-encrypt traffic. +- **Installation of custom SSL certificate(s) into EKS hosts** +Install private certificates used by (e.g.) in-line content inspection engines which re-encrypt traffic. ``` extra_ssl_certificates: | -----BEGIN CERTIFICATE----- @@ -163,455 +26,32 @@ This can be used to install non-public SSL certificates used by (e.g.) in-line S -----END CERTIFICATE----- ``` -- Private EKS endpoint configuration. -This mirrors the corresponding AWS console option, which routes all EKS traffic within the VPC. +- **Private EKS endpoint configuration** +Mirrors the corresponding AWS console option, which routes all EKS traffic within the VPC. ``` eks_endpoint_private_access: true eks_endpoint_public_access: false ``` -- Deploy into existing subnets: -This configuration option instructs Nebari to be deployed into existing subnets, rather than creating its own subnets. +- **Deploy into existing subnets** +Instructs Nebari to be deployed into existing subnets, rather than creating its own new subnets. ``` existing_subnet_ids: - - subnet-0af78088e8661c36e - - subnet-05123c80a59276d4c - existing_security_group_id: sg-0ffadcc2aaa2a013a + - subnet-0123456789abcdef + - subnet-abcdef0123456789 + existing_security_group_id: sg-0123456789abcdef ingress: terraform_overrides: - # https://kubernetes-sigs.github.io/aws-load-balancer-controller/v2.8/guide/service/annotations/ load-balancer-annotations: service.beta.kubernetes.io/aws-load-balancer-internal: "true" - service.beta.kubernetes.io/aws-load-balancer-subnets: "subnet-0af78088e8661c36e,subnet-05123c80a59276d4c" + # Ensure the subnet IDs are also set below + service.beta.kubernetes.io/aws-load-balancer-subnets: "subnet-0123456789abcdef,subnet-abcdef0123456789" ``` +- **Use existing SSL certificate** +Instructs Nebari to use the SSL certificate specified by `[k8s-custom-secret-name]` ``` -monitoring: - enabled: true - overrides: - # https://github.com/MetroStar/nebari/blob/eks-private-cluster/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/monitoring/values.yaml - # https://github.com/prometheus-community/helm-charts/blob/main/charts/kube-prometheus-stack/values.yaml - prometheus: - ## Configuration for alertmanager - ## ref: https://prometheus.io/docs/alerting/alertmanager/ - alertmanager: - ## Settings affecting alertmanagerSpec - ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#alertmanagerspec - alertmanagerSpec: - ## Image of Alertmanager - image: - #registry: quay.io - #repository: prometheus/alertmanager - registry: 096534317619.dkr.ecr.us-gov-east-1.amazonaws.com - repository: quay.io/prometheus/alertmanager - tag: v0.27.0 - sha: "" - ## Using default values from https://github.com/grafana/helm-charts/blob/main/charts/grafana/values.yaml - grafana: - image: - # -- The Docker registry - #registry: docker.io - registry: 096534317619.dkr.ecr.us-gov-east-1.amazonaws.com - # -- Docker image repository - #repository: grafana/grafana - repository: docker.io/grafana/grafana - # Overrides the Grafana image tag whose default is the chart appVersion - tag: "" - #tag: "10.4.1" - sha: "" - pullPolicy: IfNotPresent - ## Sidecars that collect the configmaps with specified label and stores the included files them into the respective folders - ## Requires at least Grafana 5 to work and can't be used together with parameters dashboardProviders, datasources and dashboards - sidecar: - image: - # -- The Docker registry - #registry: quay.io - registry: 096534317619.dkr.ecr.us-gov-east-1.amazonaws.com - #repository: kiwigrid/k8s-sidecar - repository: quay.io/kiwigrid/k8s-sidecar - tag: 1.26.1 - sha: "" - ## Manages Prometheus and Alertmanager components - prometheusOperator: - enabled: true - ## Prometheus-operator image - image: - #registry: quay.io - #repository: prometheus-operator/prometheus-operator - registry: 096534317619.dkr.ecr.us-gov-east-1.amazonaws.com - repository: quay.io/prometheus-operator/prometheus-operator - # if not set appVersion field from Chart.yaml is used - tag: "" - #tag: "v0.73.2" - sha: "" - pullPolicy: IfNotPresent - ## Prometheus-config-reloader - prometheusConfigReloader: - image: - #registry: quay.io - #repository: prometheus-operator/prometheus-config-reloader - registry: 096534317619.dkr.ecr.us-gov-east-1.amazonaws.com - repository: quay.io/prometheus-operator/prometheus-config-reloader - # if not set appVersion field from Chart.yaml is used - #tag: "v0.73.2" - tag: "" - sha: "" - ## Configuration for kube-state-metrics subchart - ## https://github.com/prometheus-community/helm-charts/blob/main/charts/kube-state-metrics/values.yaml - kube-state-metrics: - image: - #registry: registry.k8s.io - registry: 096534317619.dkr.ecr.us-gov-east-1.amazonaws.com - #repository: kube-state-metrics/kube-state-metrics - repository: registry.k8s.io/kube-state-metrics/kube-state-metrics - # If unset use v + .Charts.appVersion - tag: "" - #tag: "v2.12.0" - sha: "" - pullPolicy: IfNotPresent - ## Configuration for prometheus-node-exporter subchart - ## https://github.com/prometheus-community/helm-charts/blob/main/charts/prometheus-node-exporter/values.yaml - prometheus-node-exporter: - # Default values for prometheus-node-exporter. - image: - #registry: quay.io - registry: 096534317619.dkr.ecr.us-gov-east-1.amazonaws.com - #repository: prometheus/node-exporter - repository: quay.io/prometheus/node-exporter - # Overrides the image tag whose default is {{ printf "v%s" .Chart.AppVersion }} - tag: "" - #tag: "v1.8.0" - pullPolicy: IfNotPresent - digest: "" - ## Deploy a Prometheus instance - prometheus: - enabled: true - ## Settings affecting prometheusSpec - ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#prometheusspec - prometheusSpec: - ## Image of Prometheus. - image: - #registry: quay.io - #repository: prometheus/prometheus - registry: 096534317619.dkr.ecr.us-gov-east-1.amazonaws.com - repository: quay.io/prometheus/prometheus - tag: v2.51.2 - #tag: v2.53.1 - sha: "" - loki: - loki: - storage: - type: s3 - commonConfig: - replication_factor: 1 - # Not required as it is inside cluster and not exposed to the public network - auth_enabled: false - # The Compactor deduplicates index entries and also apply granular retention. - compactor: - # is the directory where marked chunks and temporary tables will be saved. - working_directory: /var/loki/compactor/data/retention - # minio s3 - shared_store: s3 - # how often compaction will happen - compaction_interval: 1h - # should delete old logs after retention delete delay - # ideally we would want to do storage based retention, but this is not - # currently implemented in loki, that's why we're doing time based retention. - retention_enabled: true - # is the delay after which the Compactor will delete marked chunks. - retention_delete_delay: 1h - # specifies the maximum quantity of goroutine workers instantiated to delete chunks. - retention_delete_worker_count: 150 - limits_config: - # The minimum retention period is 24h. - # This is reasonable in most cases, but if people would like to retain logs for longer - # then they can override this variable from nebari-config.yaml - retention_period: 60d - schema_config: - configs: - # list of period_configs - # The date of the first day that index buckets should be created. - - from: "2024-03-01" - index: - period: 24h - prefix: loki_index_ - object_store: s3 - schema: v11 - store: boltdb-shipper - storage_config: - boltdb_shipper: - # Directory where ingesters would write index files which would then be - # uploaded by shipper to configured storage - active_index_directory: /var/loki/compactor/data/index - # Cache location for restoring index files from storage for queries - cache_location: /var/loki/compactor/data/boltdb-cache - # Shared store for keeping index files - shared_store: s3 - image: - # -- The Docker registry - #registry: docker.io - registry: 096534317619.dkr.ecr.us-gov-east-1.amazonaws.com - # -- Docker image repository - #repository: grafana/loki - repository: docker.io/grafana/loki - # -- Overrides the image tag whose default is the chart's appVersion - # TODO: needed for 3rd target backend functionality - # revert to null or latest once this behavior is relased - tag: null - #tag: "2.9.4" - ## -- Overrides the image tag with an image digest - #digest: null - # Configuration for the write pod(s) - write: - # -- Number of replicas for the write - # Keeping cost of running Nebari in mind - # We don't need so many replicas, if people need it - # they can always override from nebari-config.yaml - replicas: 1 - read: - # -- Number of replicas for the read - replicas: 1 - backend: - # -- Number of replicas for the backend - replicas: 1 - minio: - # We are deploying minio from bitnami chart separately - enabled: false - monitoring: - selfMonitoring: - grafanaAgent: - installOperator: false - lokiCanary: - image: - # -- The Docker registry - #registry: docker.io - registry: 096534317619.dkr.ecr.us-gov-east-1.amazonaws.com - # -- Docker image repository - #repository: grafana/loki-canary - repository: docker.io/grafana/loki-canary - # -- Overrides the image tag whose default is the chart's appVersion - tag: null - #tag: "2.9.4" - ## -- Overrides the image tag with an image digest - #digest: null - ## -- Docker image pull policy - #pullPolicy: IfNotPresent - # Configuration for the gateway - gateway: - image: - # -- The Docker registry for the gateway image - #registry: docker.io - registry: 096534317619.dkr.ecr.us-gov-east-1.amazonaws.com - # -- The gateway image repository - #repository: nginxinc/nginx-unprivileged - repository: docker.io/nginxinc/nginx-unprivileged - # -- The gateway image tag - tag: 1.24-alpine - ## -- Overrides the gateway image tag with an image digest - #digest: null - ## -- The gateway image pull policy - #pullPolicy: IfNotPresent - sidecar: - image: - # -- The Docker registry and image for the k8s sidecar - #repository: kiwigrid/k8s-sidecar - repository: 096534317619.dkr.ecr.us-gov-east-1.amazonaws.com/kiwigrid/k8s-sidecar - # -- Docker image tag - tag: 1.24.3 - ## -- Docker image sha. If empty, no sha will be used - #sha: "" - ## -- Docker image pull policy - #pullPolicy: IfNotPresent - # https://github.com/grafana/helm-charts/blob/3831194ba2abd2a0ca7a14ca00e578f8e9d2abc6/charts/promtail/values.yaml - promtail: - image: - # -- The Docker registry - #registry: docker.io - registry: 096534317619.dkr.ecr.us-gov-east-1.amazonaws.com - # -- Docker image repository - #repository: grafana/promtail - repository: docker.io/grafana/promtail - # -- Overrides the image tag whose default is the chart's appVersion - tag: null - #tag: 2.9.3 - ## -- Docker image pull policy - #pullPolicy: IfNotPresent - # https://github.com/bitnami/charts/blob/440ec159c26e4ff0748b9e9866b345d98220c40a/bitnami/minio/values.yaml - minio: - image: - #registry: docker.io - registry: 096534317619.dkr.ecr.us-gov-east-1.amazonaws.com - #repository: bitnami/minio - repository: docker.io/bitnami/minio - tag: 2021.4.22-debian-10-r0 - ### Specify a imagePullPolicy - ### Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent' - ### ref: http://kubernetes.io/docs/user-guide/images/#pre-pulling-images - ### - #pullPolicy: IfNotPresent - ### Optionally specify an array of imagePullSecrets. - ### Secrets must be manually created in the namespace. - ### ref: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/ - ### e.g: - ### pullSecrets: - ### - myRegistryKeySecretName - ### - #pullSecrets: [] - ### Set to true if you would like to see extra information on logs - ### - #debug: false -jupyterhub: - #volume_mount_init_image: "busybox:1.31" - volume_mount_init_image: 096534317619.dkr.ecr.us-gov-east-1.amazonaws.com/busybox:1.31 - # https://github.com/MetroStar/nebari/blob/eks-private-cluster/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/values.yaml - # https://github.com/jupyterhub/zero-to-jupyterhub-k8s/blob/main/jupyterhub/values.yaml - overrides: - hub: - db: - type: sqlite-pvc - pvc: - storage: 1Gi - baseUrl: "/" - networkPolicy: - ingress: - - ports: - - port: 10202 - from: - - podSelector: - matchLabels: - hub.jupyter.org/network-access-hub: "true" - service: - extraPorts: - - port: 10202 - targetPort: 10202 - name: jhub-apps - proxy: - secretToken: "" - service: - type: ClusterIP - chp: - image: - #name: quay.io/jupyterhub/configurable-http-proxy - name: 096534317619.dkr.ecr.us-gov-east-1.amazonaws.com/quay.io/jupyterhub/configurable-http-proxy - #tag: 4.6.2 # https://github.com/jupyterhub/configurable-http-proxy/releases - tag: 4.6.1 - #pullPolicy: - #pullSecrets: [] - networkPolicy: - egressAllowRules: - cloudMetadataServer: false - dnsPortsPrivateIPs: false - nonPrivateIPs: false - privateIPs: false - egress: - - ports: - - port: 53 - protocol: UDP - - port: 53 - protocol: TCP - - port: 10202 - protocol: TCP - - to: - - ipBlock: - cidr: 0.0.0.0/0 - scheduling: - userScheduler: - enabled: true - image: - # IMPORTANT: Bumping the minor version of this binary should go hand in - # hand with an inspection of the user-scheduelrs RBAC resources - # that we have forked. - #name: registry.k8s.io/kube-scheduler - name: 096534317619.dkr.ecr.us-gov-east-1.amazonaws.com/registry.k8s.io/kube-scheduler - #tag: "v1.28.11" # ref: https://github.com/kubernetes/website/blob/main/content/en/releases/patch-releases.md - tag: "v1.28.10" - #pullPolicy: - #pullSecrets: [] - podPriority: - enabled: true - userPlaceholder: - enabled: false - replicas: 1 - #image: - # name: registry.k8s.io/pause - # # tag is automatically bumped to new patch versions by the - # # watch-dependencies.yaml workflow. - # # If you update this, also update prePuller.pause.image.tag - # tag: "3.10" - imagePullSecrets: - - extcrcreds - singleuser: - defaultUrl: "/lab" - startTimeout: 600 # 10 minutes - profileList: [] - storage: - type: static - extraVolumeMounts: - - mountPath: "/home/shared" - name: home - subPath: "home/shared" - networkTools: - image: - #name: quay.io/jupyterhub/k8s-network-tools - name: 096534317619.dkr.ecr.us-gov-east-1.amazonaws.com/quay.io/jupyterhub/k8s-network-tools - #tag: "set-by-chartpress" - tag: 4.0.0-0.dev.git.6548.h9b2dfe22 - cpu: - limit: 1 - guarantee: 1 - memory: - limit: "1G" - guarantee: "1G" - networkPolicy: - enabled: false - # prePuller relates to the hook|continuous-image-puller DaemonsSets - prePuller: - pause: - image: - #name: registry.k8s.io/pause - name: 096534317619.dkr.ecr.us-gov-east-1.amazonaws.com/registry.k8s.io/pause - # tag is automatically bumped to new patch versions by the - # watch-dependencies.yaml workflow. - # If you update this, also update scheduling.userPlaceholder.image.tag - tag: "3.10" - # cull relates to the jupyterhub-idle-culler service, responsible for evicting - # inactive singleuser pods. - # - # The configuration below, except for enabled, corresponds to command-line flags - # for jupyterhub-idle-culler as documented here: - # https://github.com/jupyterhub/jupyterhub-idle-culler#as-a-standalone-script - # - cull: - enabled: true - users: false - removeNamedServers: false - timeout: 1800 - every: 600 - concurrency: 10 - maxAge: 0 -jupyterhub_ssh: - jupyterhub_ssh_image: - name: 096534317619.dkr.ecr.us-gov-east-1.amazonaws.com/quay.io/jupyterhub-ssh/ssh - tag: 0.0.1-0.dev.git.136.ha610981 - jupyterhub_sftp_image: - name: 096534317619.dkr.ecr.us-gov-east-1.amazonaws.com/quay.io/jupyterhub-ssh/sftp - tag: 0.0.1-0.dev.git.142.h402a3d6 -dask_gateway: - dask_gateway_image: - #name: ghcr.io/dask/dask-gateway-server - name: 096534317619.dkr.ecr.us-gov-east-1.amazonaws.com/ghcr.io/dask/dask-gateway-server - tag: "2022.4.0" - dask_controller_image: - #name: ghcr.io/dask/dask-gateway-server - name: 096534317619.dkr.ecr.us-gov-east-1.amazonaws.com/ghcr.io/dask/dask-gateway-server - tag: "2022.4.0" -forward_auth: - traefik_forwardauth_image: - #name: maxisme/traefik-forward-auth - name: 096534317619.dkr.ecr.us-gov-east-1.amazonaws.com/maxisme/traefik-forward-auth - tag: "sha-a98e568" certificate: - #type: existing - #secret_name: nebari-custom-secret - type: self-signed + type: existing + secret_name: [k8s-custom-secret-name] ``` From 1e19131e3c45a791148bd6412ef56b728218b737 Mon Sep 17 00:00:00 2001 From: Tyler Graff <34993299+tylergraff@users.noreply.github.com> Date: Thu, 24 Oct 2024 10:27:06 -0400 Subject: [PATCH 05/16] Update and rename container-sources-2024-09-1.md to container-sources.md --- ...rces-2024-09-1.md => container-sources.md} | 86 +++++++++---------- 1 file changed, 43 insertions(+), 43 deletions(-) rename docs/docs/references/{container-sources-2024-09-1.md => container-sources.md} (69%) diff --git a/docs/docs/references/container-sources-2024-09-1.md b/docs/docs/references/container-sources.md similarity index 69% rename from docs/docs/references/container-sources-2024-09-1.md rename to docs/docs/references/container-sources.md index 2ede2d1e..e687be5e 100644 --- a/docs/docs/references/container-sources-2024-09-1.md +++ b/docs/docs/references/container-sources.md @@ -1,5 +1,4 @@ ## Deploying and Running Nebari from a Private Container Repository - Nebari deploys and runs FOSS components as containers running in Kubernetes. By default, Nebari sources each container from the container's respective public repository, typically `docker.io` or `quay.io`. This introduces supply-chain concerns for security-focused customers. @@ -12,17 +11,18 @@ One solution to these supply-chain concerns is to deploy Nebari from private loc Deploying Nebari in this fashion eliminates significant supply chain surface-area, but requires identifying all containers used by Nebari. -The following configuration enumerates all container images used by Nebari 2024-9-1 and demonstrates how to source them from a private repo denoted by the string `[LOCAL_ECR].dkr.ecr.us-gov-east-1.amazonaws.com`. +The following configuration enumerates all container images used by Nebari 2024-9-1 and demonstrates how to source them from a private repo denoted by the string `[LOCAL_REPO]`. The commented-out elements document the original public sources from which the container images are to be mirrored. +### Nebari 2024-9-1 Containers ``` default_images: #jupyterhub: quay.io/nebari/nebari-jupyterhub:2024.5.1 - jupyterhub: [LOCAL_ECR].dkr.ecr.us-gov-east-1.amazonaws.com/quay.io/nebari/nebari-jupyterhub:2024.5.1 + jupyterhub: [LOCAL_REPO]/quay.io/nebari/nebari-jupyterhub:2024.5.1 #jupyterlab: quay.io/nebari/nebari-jupyterlab:2024.5.1 - jupyterlab: [LOCAL_ECR].dkr.ecr.us-gov-east-1.amazonaws.com/quay.io/nebari/nebari-jupyterlab:2024.5.1 + jupyterlab: [LOCAL_REPO]/quay.io/nebari/nebari-jupyterlab:2024.5.1 #dask_worker: quay.io/nebari/nebari-dask-worker:2024.5.1 - dask_worker: [LOCAL_ECR].dkr.ecr.us-gov-east-1.amazonaws.com/quay.io/nebari/nebari-dask-worker:2024.5.1 + dask_worker: [LOCAL_REPO]/quay.io/nebari/nebari-dask-worker:2024.5.1 security: keycloak: @@ -30,7 +30,7 @@ security: image: # Keycloak image repository #repository: quay.io/keycloak/keycloak # default - repository: [LOCAL_ECR].dkr.ecr.us-gov-east-1.amazonaws.com/quay.io/keycloak/keycloak + repository: [LOCAL_REPO]/quay.io/keycloak/keycloak # Overrides the Keycloak image tag whose default is the chart version #tag: "15.0.2" # default tag: "" @@ -49,17 +49,17 @@ security: chown 1000:1000 /data/keycloak-metrics-spi-2.5.3.jar && chmod 777 /data/keycloak-metrics-spi-2.5.3.jar fi - image: [LOCAL_ECR].dkr.ecr.us-gov-east-1.amazonaws.com/alpine:latest + image: [LOCAL_REPO]/alpine:latest name: initialize-spi-metrics-jar pgchecker: image: # repository: docker.io/busybox - repository: [LOCAL_ECR].dkr.ecr.us-gov-east-1.amazonaws.com/docker.io/busybox + repository: [LOCAL_REPO]/docker.io/busybox tag: 1.32 postgresql: image: #registry: docker.io - registry: [LOCAL_ECR].dkr.ecr.us-gov-east-1.amazonaws.com + registry: [LOCAL_REPO] #repository: bitnami/postgresql repository: docker.io/bitnami/postgresql tag: 11.11.0-debian-10-r31 @@ -69,33 +69,33 @@ cluster_autoscaler: overrides: image: #repository: k8s.gcr.io/autoscaling/cluster-autoscaler - repository: [LOCAL_ECR].dkr.ecr.us-gov-east-1.amazonaws.com/k8s.gcr.io/autoscaling/cluster-autoscaler + repository: [LOCAL_REPO]/k8s.gcr.io/autoscaling/cluster-autoscaler tag: v1.23.0 ingress: traefik-image: - image: [LOCAL_ECR].dkr.ecr.us-gov-east-1.amazonaws.com/traefik + image: [LOCAL_REPO]/traefik tag: 2.9.1 conda_store: - image: [LOCAL_ECR].dkr.ecr.us-gov-east-1.amazonaws.com/quansight/conda-store-server + image: [LOCAL_REPO]/quansight/conda-store-server image_tag: 2024.3.1 conda_store: - nfs_server_image: [LOCAL_ECR].dkr.ecr.us-gov-east-1.amazonaws.com/gcr.io/google_containers/volume-nfs + nfs_server_image: [LOCAL_REPO]/gcr.io/google_containers/volume-nfs nfs_server_image_tag: "0.8" overrides: minio: image: #registry: docker.io - registry: [LOCAL_ECR].dkr.ecr.us-gov-east-1.amazonaws.com + registry: [LOCAL_REPO] #repository: bitnami/minio repository: docker.io/bitnami/minio tag: 2021.4.22-debian-10-r0 postgresql: image: #registry: docker.io - registry: [LOCAL_ECR].dkr.ecr.us-gov-east-1.amazonaws.com + registry: [LOCAL_REPO] #repository: bitnami/postgresql repository: docker.io/bitnami/postgresql tag: 11.14.0-debian-10-r17 @@ -103,7 +103,7 @@ conda_store: redis: image: #registry: docker.io - registry: [LOCAL_ECR].dkr.ecr.us-gov-east-1.amazonaws.com + registry: [LOCAL_REPO] #repository: bitnami/redis repository: docker.io/bitnami/redis tag: 7.0.4-debian-11-r4 @@ -114,20 +114,20 @@ argo_workflows: controller: image: #registry: quay.io - registry: [LOCAL_ECR].dkr.ecr.us-gov-east-1.amazonaws.com + registry: [LOCAL_REPO] #repository: argoproj/workflow-controller repository: quay.io/argoproj/workflow-controller tag: "" server: image: #registry: quay.io - registry: [LOCAL_ECR].dkr.ecr.us-gov-east-1.amazonaws.com + registry: [LOCAL_REPO] #repository: argoproj/argocli repository: quay.io/argoproj/argocli tag: "v3.4.4" nebari_workflow_controller: enabled: true - image: [LOCAL_ECR].dkr.ecr.us-gov-east-1.amazonaws.com/quay.io/nebari/nebari-workflow-controller + image: [LOCAL_REPO]/quay.io/nebari/nebari-workflow-controller image_tag: 2024.5.1 monitoring: @@ -139,14 +139,14 @@ monitoring: image: #registry: quay.io #repository: prometheus/alertmanager - registry: [LOCAL_ECR].dkr.ecr.us-gov-east-1.amazonaws.com + registry: [LOCAL_REPO] repository: quay.io/prometheus/alertmanager tag: v0.27.0 sha: "" grafana: image: #registry: docker.io - registry: [LOCAL_ECR].dkr.ecr.us-gov-east-1.amazonaws.com + registry: [LOCAL_REPO] #repository: grafana/grafana repository: docker.io/grafana/grafana tag: "" @@ -155,7 +155,7 @@ monitoring: sidecar: image: #registry: quay.io - registry: [LOCAL_ECR].dkr.ecr.us-gov-east-1.amazonaws.com + registry: [LOCAL_REPO] #repository: kiwigrid/k8s-sidecar repository: quay.io/kiwigrid/k8s-sidecar tag: 1.26.1 @@ -163,7 +163,7 @@ monitoring: prometheusOperator: image: #registry: quay.io - registry: [LOCAL_ECR].dkr.ecr.us-gov-east-1.amazonaws.com + registry: [LOCAL_REPO] #repository: prometheus-operator/prometheus-operator repository: quay.io/prometheus-operator/prometheus-operator tag: "" @@ -171,7 +171,7 @@ monitoring: prometheusConfigReloader: image: #registry: quay.io - registry: [LOCAL_ECR].dkr.ecr.us-gov-east-1.amazonaws.com + registry: [LOCAL_REPO] #repository: prometheus-operator/prometheus-config-reloader repository: quay.io/prometheus-operator/prometheus-config-reloader tag: "" @@ -179,7 +179,7 @@ monitoring: kube-state-metrics: image: #registry: registry.k8s.io - registry: [LOCAL_ECR].dkr.ecr.us-gov-east-1.amazonaws.com + registry: [LOCAL_REPO] #repository: kube-state-metrics/kube-state-metrics repository: registry.k8s.io/kube-state-metrics/kube-state-metrics tag: "" @@ -188,7 +188,7 @@ monitoring: prometheus-node-exporter: image: #registry: quay.io - registry: [LOCAL_ECR].dkr.ecr.us-gov-east-1.amazonaws.com + registry: [LOCAL_REPO] #repository: prometheus/node-exporter repository: quay.io/prometheus/node-exporter tag: "" @@ -198,7 +198,7 @@ monitoring: prometheusSpec: image: #registry: quay.io - registry: [LOCAL_ECR].dkr.ecr.us-gov-east-1.amazonaws.com + registry: [LOCAL_REPO] #repository: prometheus/prometheus repository: quay.io/prometheus/prometheus tag: v2.51.2 @@ -207,93 +207,93 @@ monitoring: loki: image: #registry: docker.io - registry: [LOCAL_ECR].dkr.ecr.us-gov-east-1.amazonaws.com + registry: [LOCAL_REPO] #repository: grafana/loki repository: docker.io/grafana/loki tag: null lokiCanary: image: #registry: docker.io - registry: [LOCAL_ECR].dkr.ecr.us-gov-east-1.amazonaws.com + registry: [LOCAL_REPO] #repository: grafana/loki-canary repository: docker.io/grafana/loki-canary tag: null gateway: image: #registry: docker.io - registry: [LOCAL_ECR].dkr.ecr.us-gov-east-1.amazonaws.com + registry: [LOCAL_REPO] #repository: nginxinc/nginx-unprivileged repository: docker.io/nginxinc/nginx-unprivileged tag: 1.24-alpine sidecar: image: #repository: kiwigrid/k8s-sidecar - repository: [LOCAL_ECR].dkr.ecr.us-gov-east-1.amazonaws.com/kiwigrid/k8s-sidecar + repository: [LOCAL_REPO]/kiwigrid/k8s-sidecar tag: 1.24.3 promtail: image: #registry: docker.io - registry: [LOCAL_ECR].dkr.ecr.us-gov-east-1.amazonaws.com + registry: [LOCAL_REPO] #repository: grafana/promtail repository: docker.io/grafana/promtail tag: null minio: image: #registry: docker.io - registry: [LOCAL_ECR].dkr.ecr.us-gov-east-1.amazonaws.com + registry: [LOCAL_REPO] #repository: bitnami/minio repository: docker.io/bitnami/minio tag: 2021.4.22-debian-10-r0 jupyterhub: #volume_mount_init_image: "busybox:1.31" - volume_mount_init_image: [LOCAL_ECR].dkr.ecr.us-gov-east-1.amazonaws.com/busybox:1.31 + volume_mount_init_image: [LOCAL_REPO]/busybox:1.31 proxy: chp: image: #name: quay.io/jupyterhub/configurable-http-proxy - name: [LOCAL_ECR].dkr.ecr.us-gov-east-1.amazonaws.com/quay.io/jupyterhub/configurable-http-proxy + name: [LOCAL_REPO]/quay.io/jupyterhub/configurable-http-proxy tag: 4.6.1 scheduling: userScheduler: enabled: true image: #name: registry.k8s.io/kube-scheduler - name: [LOCAL_ECR].dkr.ecr.us-gov-east-1.amazonaws.com/registry.k8s.io/kube-scheduler + name: [LOCAL_REPO]/registry.k8s.io/kube-scheduler tag: "v1.28.10" singleuser: networkTools: image: #name: quay.io/jupyterhub/k8s-network-tools - name: [LOCAL_ECR].dkr.ecr.us-gov-east-1.amazonaws.com/quay.io/jupyterhub/k8s-network-tools + name: [LOCAL_REPO]/quay.io/jupyterhub/k8s-network-tools tag: 4.0.0-0.dev.git.6548.h9b2dfe22 prePuller: pause: image: #name: registry.k8s.io/pause - name: [LOCAL_ECR].dkr.ecr.us-gov-east-1.amazonaws.com/registry.k8s.io/pause + name: [LOCAL_REPO]/registry.k8s.io/pause tag: "3.10" jupyterhub_ssh: jupyterhub_ssh_image: - name: [LOCAL_ECR].dkr.ecr.us-gov-east-1.amazonaws.com/quay.io/jupyterhub-ssh/ssh + name: [LOCAL_REPO]/quay.io/jupyterhub-ssh/ssh tag: 0.0.1-0.dev.git.136.ha610981 jupyterhub_sftp_image: - name: [LOCAL_ECR].dkr.ecr.us-gov-east-1.amazonaws.com/quay.io/jupyterhub-ssh/sftp + name: [LOCAL_REPO]/quay.io/jupyterhub-ssh/sftp tag: 0.0.1-0.dev.git.142.h402a3d6 dask_gateway: dask_gateway_image: #name: ghcr.io/dask/dask-gateway-server - name: [LOCAL_ECR].dkr.ecr.us-gov-east-1.amazonaws.com/ghcr.io/dask/dask-gateway-server + name: [LOCAL_REPO]/ghcr.io/dask/dask-gateway-server tag: "2022.4.0" dask_controller_image: #name: ghcr.io/dask/dask-gateway-server - name: [LOCAL_ECR].dkr.ecr.us-gov-east-1.amazonaws.com/ghcr.io/dask/dask-gateway-server + name: [LOCAL_REPO]/ghcr.io/dask/dask-gateway-server tag: "2022.4.0" forward_auth: traefik_forwardauth_image: #name: maxisme/traefik-forward-auth - name: [LOCAL_ECR].dkr.ecr.us-gov-east-1.amazonaws.com/maxisme/traefik-forward-auth + name: [LOCAL_REPO]/maxisme/traefik-forward-auth tag: "sha-a98e568" ``` From 331ec518e538c1ca35910301ae2ef5aadd1371ad Mon Sep 17 00:00:00 2001 From: Tyler Graff <34993299+tylergraff@users.noreply.github.com> Date: Thu, 24 Oct 2024 10:30:46 -0400 Subject: [PATCH 06/16] Update index.mdx Add security & container guides --- docs/docs/references/index.mdx | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/docs/references/index.mdx b/docs/docs/references/index.mdx index 572b3f24..cd346ffb 100644 --- a/docs/docs/references/index.mdx +++ b/docs/docs/references/index.mdx @@ -11,6 +11,8 @@ import {useCurrentSidebarCategory} from '@docusaurus/theme-common'; /> -Nitty-gritty technical descriptions of how Nebari works. +Technical descriptions of how Nebari works. +- [Enhanced Security](enhanced-security.md) - Nebari security configuration guide +- [Local Container Repo](container-sources.md) - Deploying Nebari from a Local Container Repo From 460716f06b8c71f90952415c8cc1f13e2da06a87 Mon Sep 17 00:00:00 2001 From: Tyler Graff Date: Fri, 25 Oct 2024 15:27:08 -0400 Subject: [PATCH 07/16] yarn run format:fix --- docs/docs/references/container-sources.md | 10 ++++---- docs/docs/references/enhanced-security.md | 28 +++++++++++++---------- 2 files changed, 22 insertions(+), 16 deletions(-) diff --git a/docs/docs/references/container-sources.md b/docs/docs/references/container-sources.md index e687be5e..dd99f20b 100644 --- a/docs/docs/references/container-sources.md +++ b/docs/docs/references/container-sources.md @@ -1,20 +1,22 @@ ## Deploying and Running Nebari from a Private Container Repository -Nebari deploys and runs FOSS components as containers running in Kubernetes. -By default, Nebari sources each container from the container's respective public repository, typically `docker.io` or `quay.io`. + +Nebari deploys and runs FOSS components as containers running in Kubernetes. +By default, Nebari sources each container from the container's respective public repository, typically `docker.io` or `quay.io`. This introduces supply-chain concerns for security-focused customers. +One solution to these supply-chain concerns is to deploy Nebari from private locally-mirrored containers: -One solution to these supply-chain concerns is to deploy Nebari from private locally-mirrored containers: - Create a controlled private container repository (e.g. ECR or GitLab Container Repo) - Mirror all containers used by Nebari into this private container repository - Use the `overrides` mechanism in `nebari-config.yaml` to specify the mirrored container sources -Deploying Nebari in this fashion eliminates significant supply chain surface-area, but requires identifying all containers used by Nebari. +Deploying Nebari in this fashion eliminates significant supply chain surface-area, but requires identifying all containers used by Nebari. The following configuration enumerates all container images used by Nebari 2024-9-1 and demonstrates how to source them from a private repo denoted by the string `[LOCAL_REPO]`. The commented-out elements document the original public sources from which the container images are to be mirrored. ### Nebari 2024-9-1 Containers + ``` default_images: #jupyterhub: quay.io/nebari/nebari-jupyterhub:2024.5.1 diff --git a/docs/docs/references/enhanced-security.md b/docs/docs/references/enhanced-security.md index 3a060951..f3849d77 100644 --- a/docs/docs/references/enhanced-security.md +++ b/docs/docs/references/enhanced-security.md @@ -1,22 +1,23 @@ ## Nebari Security Considerations -The security of *AWS Nebari* deployments can be enhanced through the following deployment configuration options in `nebari-config.yaml`: +The security of _AWS Nebari_ deployments can be enhanced through the following deployment configuration options in `nebari-config.yaml`: - **Explicit definition of container sources** -This option allows for the use of locally mirrored, security-hardened, or otherwise customized container images in place of the containers used by default. -See: [container-sources](container-sources-2024-09-1.md) - + This option allows for the use of locally mirrored, security-hardened, or otherwise customized container images in place of the containers used by default. + See: [container-sources](container-sources-2024-09-1.md) - **Definition of an ssh key that can access EKS hosts** -EKS hosts by default cannot be accessed via ssh. This configuration item allows ssh access into EKS hosts, which can be useful for troubleshooting or external monitoring and auditing purposes. + EKS hosts by default cannot be accessed via ssh. This configuration item allows ssh access into EKS hosts, which can be useful for troubleshooting or external monitoring and auditing purposes. + ``` amazon_web_services: ec2_keypair_name: [example_keypair_name] # Name, not ARN ``` - + - **Installation of custom SSL certificate(s) into EKS hosts** -Install private certificates used by (e.g.) in-line content inspection engines which re-encrypt traffic. -``` + Install private certificates used by (e.g.) in-line content inspection engines which re-encrypt traffic. + +``` extra_ssl_certificates: | -----BEGIN CERTIFICATE----- MIIF......ABCD @@ -27,14 +28,16 @@ Install private certificates used by (e.g.) in-line content inspection engines w ``` - **Private EKS endpoint configuration** -Mirrors the corresponding AWS console option, which routes all EKS traffic within the VPC. + Mirrors the corresponding AWS console option, which routes all EKS traffic within the VPC. + ``` eks_endpoint_private_access: true eks_endpoint_public_access: false ``` - **Deploy into existing subnets** -Instructs Nebari to be deployed into existing subnets, rather than creating its own new subnets. + Instructs Nebari to be deployed into existing subnets, rather than creating its own new subnets. + ``` existing_subnet_ids: - subnet-0123456789abcdef @@ -48,8 +51,9 @@ ingress: service.beta.kubernetes.io/aws-load-balancer-subnets: "subnet-0123456789abcdef,subnet-abcdef0123456789" ``` -- **Use existing SSL certificate** -Instructs Nebari to use the SSL certificate specified by `[k8s-custom-secret-name]` +- **Use existing SSL certificate** + Instructs Nebari to use the SSL certificate specified by `[k8s-custom-secret-name]` + ``` certificate: type: existing From aa7ee5de2d1e18ae04b05f08a3430b229127e48e Mon Sep 17 00:00:00 2001 From: Tyler Graff Date: Fri, 25 Oct 2024 15:30:47 -0400 Subject: [PATCH 08/16] fix broken link --- docs/docs/references/enhanced-security.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/docs/references/enhanced-security.md b/docs/docs/references/enhanced-security.md index f3849d77..36e15eec 100644 --- a/docs/docs/references/enhanced-security.md +++ b/docs/docs/references/enhanced-security.md @@ -4,7 +4,7 @@ The security of _AWS Nebari_ deployments can be enhanced through the following d - **Explicit definition of container sources** This option allows for the use of locally mirrored, security-hardened, or otherwise customized container images in place of the containers used by default. - See: [container-sources](container-sources-2024-09-1.md) + See: [container-sources](container-sources.md) - **Definition of an ssh key that can access EKS hosts** EKS hosts by default cannot be accessed via ssh. This configuration item allows ssh access into EKS hosts, which can be useful for troubleshooting or external monitoring and auditing purposes. From d4d3f9a76b30fb1a19a9b3ef8d808b6eecdac14e Mon Sep 17 00:00:00 2001 From: Tyler Graff <34993299+tylergraff@users.noreply.github.com> Date: Tue, 5 Nov 2024 09:18:14 -0500 Subject: [PATCH 09/16] update to wholesale repo specification --- docs/docs/references/container-sources.md | 397 +++++++--------------- 1 file changed, 115 insertions(+), 282 deletions(-) diff --git a/docs/docs/references/container-sources.md b/docs/docs/references/container-sources.md index dd99f20b..499f7674 100644 --- a/docs/docs/references/container-sources.md +++ b/docs/docs/references/container-sources.md @@ -6,296 +6,129 @@ This introduces supply-chain concerns for security-focused customers. One solution to these supply-chain concerns is to deploy Nebari from private locally-mirrored containers: -- Create a controlled private container repository (e.g. ECR or GitLab Container Repo) +- Create a controlled private container repository (e.g. ECR) - Mirror all containers used by Nebari into this private container repository -- Use the `overrides` mechanism in `nebari-config.yaml` to specify the mirrored container sources +- Use the `pre_bootstrap_command` mechanism in `nebari-config.yaml` to specify the mirrored container repo Deploying Nebari in this fashion eliminates significant supply chain surface-area, but requires identifying all containers used by Nebari. -The following configuration enumerates all container images used by Nebari 2024-9-1 and demonstrates how to source them from a private repo denoted by the string `[LOCAL_REPO]`. -The commented-out elements document the original public sources from which the container images are to be mirrored. +The following configurations demonstrate how to specify a private repo denoted by the string `[PRIVATE_REPO]`. -### Nebari 2024-9-1 Containers +### Set ECR as default container registry mirror ``` -default_images: - #jupyterhub: quay.io/nebari/nebari-jupyterhub:2024.5.1 - jupyterhub: [LOCAL_REPO]/quay.io/nebari/nebari-jupyterhub:2024.5.1 - #jupyterlab: quay.io/nebari/nebari-jupyterlab:2024.5.1 - jupyterlab: [LOCAL_REPO]/quay.io/nebari/nebari-jupyterlab:2024.5.1 - #dask_worker: quay.io/nebari/nebari-dask-worker:2024.5.1 - dask_worker: [LOCAL_REPO]/quay.io/nebari/nebari-dask-worker:2024.5.1 +amazon_web_services: + node_groups: + general: + instance: m5.2xlarge + launch_template: + pre_bootstrap_command: | + #!/bin/bash + # Verify that IP forwarding is enabled for worker nodes, as is required for containerd + if [[ $(sysctl net.ipv4.ip_forward | grep "net.ipv4.ip_forward = 1") ]]; then echo "net.ipv4.ip_forward is on"; else sysctl -w net.ipv4.ip_forward=1; fi + # Set ECR as default container registry mirror + mkdir -p /etc/containerd/certs.d/_default + ECR_TOKEN="$(aws ecr get-login-password --region us-east-1)" + BASIC_AUTH="$(echo -n "AWS:$ECR_TOKEN" | base64 -w 0)" + cat <<-EOT > /etc/containerd/certs.d/_default/hosts.toml + [host."https://[PRIVATE_REPO].dkr.ecr.us-east-1.amazonaws.com"] + capabilities = ["pull", "resolve"] + [host."https://[PRIVATE_REPO].dkr.ecr.us-east-1.amazonaws.com".header] + authorization = "Basic $BASIC_AUTH" + EOT -security: - keycloak: - overrides: - image: - # Keycloak image repository - #repository: quay.io/keycloak/keycloak # default - repository: [LOCAL_REPO]/quay.io/keycloak/keycloak - # Overrides the Keycloak image tag whose default is the chart version - #tag: "15.0.2" # default - tag: "" - - # This container is used at deploy-time to download keycloak-metrics-spi - extraInitContainers: | - - command: - - sh - - -c - - | wget --no-check-certificate https://github.com/aerogear/keycloak-metrics-spi/releases/download/2.5.3/keycloak-metrics-spi-2.5.3.jar -P /data/ && - export SHA256SUM=9b3f52f842a66dadf5ff3cc3a729b8e49042d32f84510a5d73d41a2e39f29a96 && - if ! (echo "$SHA256SUM /data/keycloak-metrics-spi-2.5.3.jar" | sha256sum -c) - then - echo "Error: Checksum not verified" && exit 1 - else - chown 1000:1000 /data/keycloak-metrics-spi-2.5.3.jar && - chmod 777 /data/keycloak-metrics-spi-2.5.3.jar - fi - image: [LOCAL_REPO]/alpine:latest - name: initialize-spi-metrics-jar - pgchecker: - image: - # repository: docker.io/busybox - repository: [LOCAL_REPO]/docker.io/busybox - tag: 1.32 - postgresql: - image: - #registry: docker.io - registry: [LOCAL_REPO] - #repository: bitnami/postgresql - repository: docker.io/bitnami/postgresql - tag: 11.11.0-debian-10-r31 - digest: "" - -cluster_autoscaler: - overrides: - image: - #repository: k8s.gcr.io/autoscaling/cluster-autoscaler - repository: [LOCAL_REPO]/k8s.gcr.io/autoscaling/cluster-autoscaler - tag: v1.23.0 - -ingress: - traefik-image: - image: [LOCAL_REPO]/traefik - tag: 2.9.1 - -conda_store: - image: [LOCAL_REPO]/quansight/conda-store-server - image_tag: 2024.3.1 - -conda_store: - nfs_server_image: [LOCAL_REPO]/gcr.io/google_containers/volume-nfs - nfs_server_image_tag: "0.8" - overrides: - minio: - image: - #registry: docker.io - registry: [LOCAL_REPO] - #repository: bitnami/minio - repository: docker.io/bitnami/minio - tag: 2021.4.22-debian-10-r0 - postgresql: - image: - #registry: docker.io - registry: [LOCAL_REPO] - #repository: bitnami/postgresql - repository: docker.io/bitnami/postgresql - tag: 11.14.0-debian-10-r17 - digest: "" - redis: - image: - #registry: docker.io - registry: [LOCAL_REPO] - #repository: bitnami/redis - repository: docker.io/bitnami/redis - tag: 7.0.4-debian-11-r4 - digest: "" - -argo_workflows: - overrides: - controller: - image: - #registry: quay.io - registry: [LOCAL_REPO] - #repository: argoproj/workflow-controller - repository: quay.io/argoproj/workflow-controller - tag: "" - server: - image: - #registry: quay.io - registry: [LOCAL_REPO] - #repository: argoproj/argocli - repository: quay.io/argoproj/argocli - tag: "v3.4.4" - nebari_workflow_controller: - enabled: true - image: [LOCAL_REPO]/quay.io/nebari/nebari-workflow-controller - image_tag: 2024.5.1 - -monitoring: - overrides: - prometheus: - alertmanager: - alertmanagerSpec: - ## Image of Alertmanager - image: - #registry: quay.io - #repository: prometheus/alertmanager - registry: [LOCAL_REPO] - repository: quay.io/prometheus/alertmanager - tag: v0.27.0 - sha: "" - grafana: - image: - #registry: docker.io - registry: [LOCAL_REPO] - #repository: grafana/grafana - repository: docker.io/grafana/grafana - tag: "" - sha: "" - pullPolicy: IfNotPresent - sidecar: - image: - #registry: quay.io - registry: [LOCAL_REPO] - #repository: kiwigrid/k8s-sidecar - repository: quay.io/kiwigrid/k8s-sidecar - tag: 1.26.1 - sha: "" - prometheusOperator: - image: - #registry: quay.io - registry: [LOCAL_REPO] - #repository: prometheus-operator/prometheus-operator - repository: quay.io/prometheus-operator/prometheus-operator - tag: "" - sha: "" - prometheusConfigReloader: - image: - #registry: quay.io - registry: [LOCAL_REPO] - #repository: prometheus-operator/prometheus-config-reloader - repository: quay.io/prometheus-operator/prometheus-config-reloader - tag: "" - sha: "" - kube-state-metrics: - image: - #registry: registry.k8s.io - registry: [LOCAL_REPO] - #repository: kube-state-metrics/kube-state-metrics - repository: registry.k8s.io/kube-state-metrics/kube-state-metrics - tag: "" - sha: "" - pullPolicy: IfNotPresent - prometheus-node-exporter: - image: - #registry: quay.io - registry: [LOCAL_REPO] - #repository: prometheus/node-exporter - repository: quay.io/prometheus/node-exporter - tag: "" - pullPolicy: IfNotPresent - digest: "" - prometheus: - prometheusSpec: - image: - #registry: quay.io - registry: [LOCAL_REPO] - #repository: prometheus/prometheus - repository: quay.io/prometheus/prometheus - tag: v2.51.2 - sha: "" - loki: - loki: - image: - #registry: docker.io - registry: [LOCAL_REPO] - #repository: grafana/loki - repository: docker.io/grafana/loki - tag: null - lokiCanary: - image: - #registry: docker.io - registry: [LOCAL_REPO] - #repository: grafana/loki-canary - repository: docker.io/grafana/loki-canary - tag: null - gateway: - image: - #registry: docker.io - registry: [LOCAL_REPO] - #repository: nginxinc/nginx-unprivileged - repository: docker.io/nginxinc/nginx-unprivileged - tag: 1.24-alpine - sidecar: - image: - #repository: kiwigrid/k8s-sidecar - repository: [LOCAL_REPO]/kiwigrid/k8s-sidecar - tag: 1.24.3 - promtail: - image: - #registry: docker.io - registry: [LOCAL_REPO] - #repository: grafana/promtail - repository: docker.io/grafana/promtail - tag: null - minio: - image: - #registry: docker.io - registry: [LOCAL_REPO] - #repository: bitnami/minio - repository: docker.io/bitnami/minio - tag: 2021.4.22-debian-10-r0 - -jupyterhub: - #volume_mount_init_image: "busybox:1.31" - volume_mount_init_image: [LOCAL_REPO]/busybox:1.31 - proxy: - chp: - image: - #name: quay.io/jupyterhub/configurable-http-proxy - name: [LOCAL_REPO]/quay.io/jupyterhub/configurable-http-proxy - tag: 4.6.1 - scheduling: - userScheduler: - enabled: true - image: - #name: registry.k8s.io/kube-scheduler - name: [LOCAL_REPO]/registry.k8s.io/kube-scheduler - tag: "v1.28.10" - singleuser: - networkTools: - image: - #name: quay.io/jupyterhub/k8s-network-tools - name: [LOCAL_REPO]/quay.io/jupyterhub/k8s-network-tools - tag: 4.0.0-0.dev.git.6548.h9b2dfe22 - prePuller: - pause: - image: - #name: registry.k8s.io/pause - name: [LOCAL_REPO]/registry.k8s.io/pause - tag: "3.10" -jupyterhub_ssh: - jupyterhub_ssh_image: - name: [LOCAL_REPO]/quay.io/jupyterhub-ssh/ssh - tag: 0.0.1-0.dev.git.136.ha610981 - jupyterhub_sftp_image: - name: [LOCAL_REPO]/quay.io/jupyterhub-ssh/sftp - tag: 0.0.1-0.dev.git.142.h402a3d6 +``` -dask_gateway: - dask_gateway_image: - #name: ghcr.io/dask/dask-gateway-server - name: [LOCAL_REPO]/ghcr.io/dask/dask-gateway-server - tag: "2022.4.0" - dask_controller_image: - #name: ghcr.io/dask/dask-gateway-server - name: [LOCAL_REPO]/ghcr.io/dask/dask-gateway-server - tag: "2022.4.0" +### Set GitLab CR as default container registry mirror +``` +# Set GitLab CR as default container registry mirror in hosts.toml; +# must have override_path set if project/group names don't match upstream container +amazon_web_services: + node_groups: + general: + instance: m5.2xlarge + launch_template: + pre_bootstrap_command: | + #!/bin/bash + # Verify that IP forwarding is enabled for worker nodes, as is required for containerd + if [[ $(sysctl net.ipv4.ip_forward | grep "net.ipv4.ip_forward = 1") ]]; then echo "net.ipv4.ip_forward is on"; else sysctl -w net.ipv4.ip_forward=1; fi + # Set default container registry mirror in hosts.toml; must have override_path set if project/group names don't match upstream container + CONTAINER_REGISTRY_URL="[PRIVATE_REPO]" + CONTAINER_REGISTRY_USERNAME="[username]" + CONTAINER_REGISTRY_TOKEN="[token]" + CONTAINER_REGISTRY_GROUP=as-nebari + CONTAINER_REGISTRY_PROJECT=nebari-test + mkdir -p /etc/containerd/certs.d/_default + cat <<-EOT > /etc/containerd/certs.d/_default/hosts.toml + [host."https://$CONTAINER_REGISTRY_URL/v2/$CONTAINER_REGISTRY_GROUP/$CONTAINER_REGISTRY_PROJECT"] + override_path = true + capabilities = ["pull", "resolve"] + EOT + + # Set containerd registry config auth in config.d .toml import dir + mkdir -p /etc/containerd/config.d + cat < /etc/containerd/certs.d/_default/hosts.toml + [host."https://$CONTAINER_REGISTRY_URL/v2/$CONTAINER_REGISTRY_GROUP/$CONTAINER_REGISTRY_PROJECT"] + override_path = true + capabilities = ["pull", "resolve"] + client = ["/etc/containerd/certs.d/$CONTAINER_REGISTRY_URL/client.pem"] + EOT + + # Set containerd registry config auth in config.d .toml import dir + mkdir -p /etc/containerd/config.d + cat <> /etc/containerd/certs.d/$CONTAINER_REGISTRY_URL/client.pem + -----BEGIN CERTIFICATE----- + XzxzxzxzxxzxzxzxzxzxzxzxxzxzxzxzxzxzxzxxzxzxzxzxzxzxzxzxzxxzxzZx + ZxyzxzxzxxzxzxzxzxzxzxzxxzxzxzxzxzxzxzxxzxzxzxzxzxzxzxzxzxxzxzXz + -----END CERTIFICATE----- + -----BEGIN PRIVATE KEY----- + XzxzxzxzxxzxzxzxzxzxzxzxxzxzxzxzxzxzxzxxzxzxzxzxzxzxzxzxzxxzxzZx + ZxyzxzxzxxzxzxzxzxzxzxzxxzxzxzxzxzxzxzxxzxzxzxzxzxzxzxzxzxxzxzXz + -----END PRIVATE KEY----- + EOT ``` From 92a4cb2241b7df73d310584a98e148f898022537 Mon Sep 17 00:00:00 2001 From: Tyler Graff <34993299+tylergraff@users.noreply.github.com> Date: Tue, 5 Nov 2024 09:22:26 -0500 Subject: [PATCH 10/16] update to as-merged features --- docs/docs/references/enhanced-security.md | 26 +++++++++++++++-------- 1 file changed, 17 insertions(+), 9 deletions(-) diff --git a/docs/docs/references/enhanced-security.md b/docs/docs/references/enhanced-security.md index 36e15eec..9d9be31f 100644 --- a/docs/docs/references/enhanced-security.md +++ b/docs/docs/references/enhanced-security.md @@ -18,21 +18,29 @@ amazon_web_services: Install private certificates used by (e.g.) in-line content inspection engines which re-encrypt traffic. ``` - extra_ssl_certificates: | - -----BEGIN CERTIFICATE----- - MIIF......ABCD - -----END CERTIFICATE----- - -----BEGIN CERTIFICATE----- - MIIF......EF01 - -----END CERTIFICATE----- +# Add client certificate to CA trust on node +amazon_web_services: + node_groups: + general: + instance: m5.2xlarge + launch_template: + pre_bootstrap_command: | + #!/bin/bash + cat <<-EOT >> /etc/pki/ca-trust/source/anchors/client.pem + -----BEGIN CERTIFICATE----- + XzxzxzxzxxzxzxzxzxzxzxzxxzxzxzxzxzxzxzxxzxzxzxzxzxzxzxzxzxxzxzZx + ZxyzxzxzxxzxzxzxzxzxzxzxxzxzxzxzxzxzxzxxzxzxzxzxzxzxzxzxzxxzxzXz + -----END CERTIFICATE----- + EOT + sudo update-ca-trust extract ``` - **Private EKS endpoint configuration** Mirrors the corresponding AWS console option, which routes all EKS traffic within the VPC. ``` - eks_endpoint_private_access: true - eks_endpoint_public_access: false + amazon_web_services: + eks_endpoint_access: private # valid values: [public, private, public_and_private] ``` - **Deploy into existing subnets** From be8f1530e0511bb5ec8654a7b2af3082ca2f8afd Mon Sep 17 00:00:00 2001 From: Tyler Graff Date: Tue, 5 Nov 2024 10:06:46 -0500 Subject: [PATCH 11/16] yarn format:fix --- docs/docs/references/container-sources.md | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/docs/docs/references/container-sources.md b/docs/docs/references/container-sources.md index 499f7674..831af427 100644 --- a/docs/docs/references/container-sources.md +++ b/docs/docs/references/container-sources.md @@ -12,7 +12,7 @@ One solution to these supply-chain concerns is to deploy Nebari from private loc Deploying Nebari in this fashion eliminates significant supply chain surface-area, but requires identifying all containers used by Nebari. -The following configurations demonstrate how to specify a private repo denoted by the string `[PRIVATE_REPO]`. +The following configurations demonstrate how to specify a private repo denoted by the string `[PRIVATE_REPO]`. ### Set ECR as default container registry mirror @@ -40,8 +40,9 @@ amazon_web_services: ``` ### Set GitLab CR as default container registry mirror + ``` -# Set GitLab CR as default container registry mirror in hosts.toml; +# Set GitLab CR as default container registry mirror in hosts.toml; # must have override_path set if project/group names don't match upstream container amazon_web_services: node_groups: @@ -79,7 +80,8 @@ amazon_web_services: EOT ``` -### Set GitLab CR as default container registry mirror, with custom Client SSL/TLS Certs +### Set GitLab CR as default container registry mirror, with custom Client SSL/TLS Certs + ``` # must have override_path set if project/group names don't match upstream container # Also add/set GitLab Client SSL/TLS Certificate for Containerd From 8dddb65becc95805d708ce70de3a1c7210ee5a0e Mon Sep 17 00:00:00 2001 From: Tyler Graff Date: Tue, 5 Nov 2024 10:10:33 -0500 Subject: [PATCH 12/16] yarn format:fix for files peripheral to my changes --- docs/docs/how-tos/nebari-gcp.md | 1 + docs/nebari-slurm/configuration.md | 3 +-- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/docs/how-tos/nebari-gcp.md b/docs/docs/how-tos/nebari-gcp.md index 90ef2a08..54aa950f 100644 --- a/docs/docs/how-tos/nebari-gcp.md +++ b/docs/docs/how-tos/nebari-gcp.md @@ -66,6 +66,7 @@ management. If it's your first time creating a service account, please follow [these detailed instructions](https://cloud.google.com/iam/docs/creating-managing-service-accounts) to create a Google Service Account with the following roles attached: + - [`roles/editor`](https://cloud.google.com/iam/docs/understanding-roles#editor) - [`roles/resourcemanager.projectIamAdmin`](https://cloud.google.com/iam/docs/understanding-roles#resourcemanager.projectIamAdmin) - [`roles/container.admin`](https://cloud.google.com/iam/docs/understanding-roles#container.admin) diff --git a/docs/nebari-slurm/configuration.md b/docs/nebari-slurm/configuration.md index 88874808..ba02db6d 100644 --- a/docs/nebari-slurm/configuration.md +++ b/docs/nebari-slurm/configuration.md @@ -186,8 +186,7 @@ _Note_: All slurm related configuration needs to be passed down as a string. ### Services Additional services can be added to the `jupyterhub_services` -variable. Currently this is only `: -`. You must keep the `dask_gateway` section. +variable. Currently this is only `: `. You must keep the `dask_gateway` section. ```yaml jupyterhub_services: From ec5e35440bafcf358450b6142f323d264da27eb6 Mon Sep 17 00:00:00 2001 From: Tyler Graff <34993299+tylergraff@users.noreply.github.com> Date: Tue, 5 Nov 2024 10:31:23 -0500 Subject: [PATCH 13/16] remove unmerged feature --- docs/docs/references/enhanced-security.md | 8 -------- 1 file changed, 8 deletions(-) diff --git a/docs/docs/references/enhanced-security.md b/docs/docs/references/enhanced-security.md index 9d9be31f..21cf1bc6 100644 --- a/docs/docs/references/enhanced-security.md +++ b/docs/docs/references/enhanced-security.md @@ -6,14 +6,6 @@ The security of _AWS Nebari_ deployments can be enhanced through the following d This option allows for the use of locally mirrored, security-hardened, or otherwise customized container images in place of the containers used by default. See: [container-sources](container-sources.md) -- **Definition of an ssh key that can access EKS hosts** - EKS hosts by default cannot be accessed via ssh. This configuration item allows ssh access into EKS hosts, which can be useful for troubleshooting or external monitoring and auditing purposes. - -``` -amazon_web_services: - ec2_keypair_name: [example_keypair_name] # Name, not ARN -``` - - **Installation of custom SSL certificate(s) into EKS hosts** Install private certificates used by (e.g.) in-line content inspection engines which re-encrypt traffic. From ed6190b458ee984094e411c5225fb3d9bd695bec Mon Sep 17 00:00:00 2001 From: Tyler Graff <34993299+tylergraff@users.noreply.github.com> Date: Tue, 5 Nov 2024 11:09:02 -0500 Subject: [PATCH 14/16] Clarify load balancer subnet notes --- docs/docs/references/enhanced-security.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/docs/references/enhanced-security.md b/docs/docs/references/enhanced-security.md index 21cf1bc6..991f7a54 100644 --- a/docs/docs/references/enhanced-security.md +++ b/docs/docs/references/enhanced-security.md @@ -37,6 +37,7 @@ amazon_web_services: - **Deploy into existing subnets** Instructs Nebari to be deployed into existing subnets, rather than creating its own new subnets. + An advantage of deploying to existing subnets is the ability to use private subnets. Note that the **ingress load-balancer-annotation** must be set appropriately based on the type (private or public) of subnet. ``` existing_subnet_ids: From 7e92f173c43aa8401274ec3d32ea94b3135ead4a Mon Sep 17 00:00:00 2001 From: Tyler Graff <34993299+tylergraff@users.noreply.github.com> Date: Tue, 5 Nov 2024 11:31:57 -0500 Subject: [PATCH 15/16] Note calling out image repository token auth considerations --- docs/docs/references/container-sources.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/docs/references/container-sources.md b/docs/docs/references/container-sources.md index 831af427..4e3e4960 100644 --- a/docs/docs/references/container-sources.md +++ b/docs/docs/references/container-sources.md @@ -12,7 +12,9 @@ One solution to these supply-chain concerns is to deploy Nebari from private loc Deploying Nebari in this fashion eliminates significant supply chain surface-area, but requires identifying all containers used by Nebari. -The following configurations demonstrate how to specify a private repo denoted by the string `[PRIVATE_REPO]`. +The following configurations demonstrate how to specify a private repo denoted by the string `[PRIVATE_REPO]`. + +**Note:** Authorization tokens are used in the examples below. It is important for administrators to understand the expiration policy of these tokens, because the Nebari k8s cluster may in some cases need to **use these tokens to pull container images at any time during run-time operation**. ### Set ECR as default container registry mirror From 528123a256a843afdfc9ff908be22c9372b1be59 Mon Sep 17 00:00:00 2001 From: Tyler Graff Date: Tue, 5 Nov 2024 11:38:24 -0500 Subject: [PATCH 16/16] format:fix --- docs/docs/references/container-sources.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/docs/references/container-sources.md b/docs/docs/references/container-sources.md index 4e3e4960..8041be50 100644 --- a/docs/docs/references/container-sources.md +++ b/docs/docs/references/container-sources.md @@ -12,7 +12,7 @@ One solution to these supply-chain concerns is to deploy Nebari from private loc Deploying Nebari in this fashion eliminates significant supply chain surface-area, but requires identifying all containers used by Nebari. -The following configurations demonstrate how to specify a private repo denoted by the string `[PRIVATE_REPO]`. +The following configurations demonstrate how to specify a private repo denoted by the string `[PRIVATE_REPO]`. **Note:** Authorization tokens are used in the examples below. It is important for administrators to understand the expiration policy of these tokens, because the Nebari k8s cluster may in some cases need to **use these tokens to pull container images at any time during run-time operation**.