Merge branch 'main' into upgrade_azurerm_provider

Author: dcmcand

.github/workflows/test-provider.yaml

@@ -29,6 +29,12 @@ on:
         required: true
         type: string
 
+env:
+  ARM_CLIENT_ID: ${{ secrets.ARM_CLIENT_ID }}
+  ARM_TENANT_ID: ${{ secrets.ARM_TENANT_ID }}
+  ARM_SUBSCRIPTION_ID: ${{ secrets.ARM_SUBSCRIPTION_ID }}
+  PROJECT_ID: ${{ secrets.GCP_PROJECT_ID }}
+
 jobs:
   test-render-providers:
     # Prevents the execution of this test under the following conditions:
@@ -72,32 +78,14 @@ jobs:
         with:
           python-version: "3.11"
 
-      - name: Retrieve secret from Vault
-        uses: hashicorp/vault-action@v3.0.0
-        with:
-          method: jwt
-          url: "https://quansight-vault-public-vault-b2379fa7.d415e30e.z1.hashicorp.cloud:8200"
-          namespace: "admin/quansight"
-          role: "repository-nebari-dev-nebari-role"
-          secrets: |
-            kv/data/repository/nebari-dev/nebari/amazon_web_services/nebari-dev-ci role_name | AWS_ROLE_ARN;
-            kv/data/repository/nebari-dev/nebari/google_cloud_platform/nebari-dev-ci/github-nebari-dev-repo-ci project_id | PROJECT_ID;
-            kv/data/repository/nebari-dev/nebari/google_cloud_platform/nebari-dev-ci/github-nebari-dev-repo-ci workload_identity_provider | GCP_WORKFLOW_PROVIDER;
-            kv/data/repository/nebari-dev/nebari/google_cloud_platform/nebari-dev-ci/github-nebari-dev-repo-ci service_account_name | GCP_SERVICE_ACCOUNT;
-            kv/data/repository/nebari-dev/nebari/azure/nebari-dev-ci/github-nebari-dev-repo-ci client_id | ARM_CLIENT_ID;
-            kv/data/repository/nebari-dev/nebari/azure/nebari-dev-ci/github-nebari-dev-repo-ci tenant_id | ARM_TENANT_ID;
-            kv/data/repository/nebari-dev/nebari/azure/nebari-dev-ci/github-nebari-dev-repo-ci subscription_id | ARM_SUBSCRIPTION_ID;
-            kv/data/repository/nebari-dev/nebari/shared_secrets SPACES_ACCESS_KEY_ID | SPACES_ACCESS_KEY_ID;
-            kv/data/repository/nebari-dev/nebari/shared_secrets SPACES_SECRET_ACCESS_KEY | SPACES_SECRET_ACCESS_KEY;
-
       - name: 'Authenticate to GCP'
         if: ${{ matrix.provider == 'gcp' }}
         uses: 'google-github-actions/auth@v1'
         with:
           token_format: access_token
           create_credentials_file: 'true'
-          workload_identity_provider: ${{ env.GCP_WORKFLOW_PROVIDER }}
-          service_account: ${{ env.GCP_SERVICE_ACCOUNT }}
+          workload_identity_provider: ${{ secrets.GCP_WORKFLOW_PROVIDER }}
+          service_account: ${{ secrets.GCP_SERVICE_ACCOUNT }}
 
       - name: Set required environment variables
         if: ${{ matrix.provider == 'gcp' }}
@@ -108,17 +96,17 @@ jobs:
         if: ${{ matrix.provider == 'aws' }}
         uses: aws-actions/configure-aws-credentials@v1
         with:
-          role-to-assume: ${{ env.AWS_ROLE_ARN }}
+          role-to-assume: ${{ secrets.AWS_ROLE_ARN }}
           role-session-name: github-action
           aws-region: us-east-1
 
       - name: 'Azure login'
         if: ${{ matrix.provider == 'azure' }}
         uses: azure/login@v1
         with:
-          client-id: ${{ env.ARM_CLIENT_ID }}
-          tenant-id: ${{ env.ARM_TENANT_ID }}
-          subscription-id: ${{ env.ARM_SUBSCRIPTION_ID }}
+          client-id: ${{ secrets.ARM_CLIENT_ID }}
+          tenant-id: ${{ secrets.ARM_TENANT_ID }}
+          subscription-id: ${{ secrets.ARM_SUBSCRIPTION_ID }}
 
       - name: Install Nebari
         run: |

.github/workflows/test_aws_integration.yaml

@@ -26,7 +26,7 @@ on:
 env:
   AWS_DEFAULT_REGION: "us-west-2"
   NEBARI_IMAGE_TAG: ${{ github.event.inputs.image-tag || 'main' }}
-  TF_LOG: ${{ github.event.inputs.tf-log-level || 'info' }}∏
+  TF_LOG: ${{ github.event.inputs.tf-log-level || 'info' }}
 
 jobs:
   test-aws-integration:
@@ -51,21 +51,10 @@ jobs:
           pip install .[dev]
           playwright install
 
-      - name: Retrieve secret from Vault
-        uses: hashicorp/vault-action@v3.0.0
-        with:
-          method: jwt
-          url: "https://quansight-vault-public-vault-b2379fa7.d415e30e.z1.hashicorp.cloud:8200"
-          namespace: "admin/quansight"
-          role: "repository-nebari-dev-nebari-role"
-          secrets: |
-            kv/data/repository/nebari-dev/nebari/amazon_web_services/nebari-dev-ci role_name | AWS_ROLE_ARN;
-            kv/data/repository/nebari-dev/nebari/cloudflare/internal-devops@quansight.com/nebari-dev-ci token | CLOUDFLARE_TOKEN;
-
       - name: Authenticate to AWS
         uses: aws-actions/configure-aws-credentials@v1
         with:
-          role-to-assume: ${{ env.AWS_ROLE_ARN }}
+          role-to-assume: ${{ secrets.AWS_ROLE_ARN }}
           role-session-name: github-action
           aws-region: ${{ env.AWS_DEFAULT_REGION }}
 
@@ -77,3 +66,4 @@ jobs:
           NEBARI_SECRET__default_images__jupyterhub: "quay.io/nebari/nebari-jupyterhub:${{ env.NEBARI_IMAGE_TAG }}"
           NEBARI_SECRET__default_images__jupyterlab: "quay.io/nebari/nebari-jupyterlab:${{ env.NEBARI_IMAGE_TAG }}"
           NEBARI_SECRET__default_images__dask_worker: "quay.io/nebari/nebari-dask-worker:${{ env.NEBARI_IMAGE_TAG }}"
+          CLOUDFLARE_TOKEN: ${{ secrets.CLOUDFLARE_TOKEN }}

.github/workflows/test_azure_integration.yaml

@@ -50,25 +50,12 @@ jobs:
           conda install --quiet --yes conda-build
           playwright install
 
-      - name: Retrieve secret from Vault
-        uses: hashicorp/vault-action@v3.0.0
-        with:
-          method: jwt
-          url: "https://quansight-vault-public-vault-b2379fa7.d415e30e.z1.hashicorp.cloud:8200"
-          namespace: "admin/quansight"
-          role: "repository-nebari-dev-nebari-role"
-          secrets: |
-            kv/data/repository/nebari-dev/nebari/azure/nebari-dev-ci/github-nebari-dev-repo-ci client_id | ARM_CLIENT_ID;
-            kv/data/repository/nebari-dev/nebari/azure/nebari-dev-ci/github-nebari-dev-repo-ci tenant_id | ARM_TENANT_ID;
-            kv/data/repository/nebari-dev/nebari/azure/nebari-dev-ci/github-nebari-dev-repo-ci subscription_id | ARM_SUBSCRIPTION_ID;
-            kv/data/repository/nebari-dev/nebari/cloudflare/internal-devops@quansight.com/nebari-dev-ci token | CLOUDFLARE_TOKEN;
-
       - name: 'Azure login'
         uses: azure/login@v2
         with:
-          client-id: ${{ env.ARM_CLIENT_ID }}
-          tenant-id: ${{ env.ARM_TENANT_ID }}
-          subscription-id: ${{ env.ARM_SUBSCRIPTION_ID }}
+          client-id: ${{ secrets.ARM_CLIENT_ID }}
+          tenant-id: ${{ secrets.ARM_TENANT_ID }}
+          subscription-id: ${{ secrets.ARM_SUBSCRIPTION_ID }}
 
       - name: Integration Tests
         run: |
@@ -78,8 +65,8 @@ jobs:
           NEBARI_SECRET__default_images__jupyterhub: "quay.io/nebari/nebari-jupyterhub:${{ env.NEBARI_IMAGE_TAG }}"
           NEBARI_SECRET__default_images__jupyterlab: "quay.io/nebari/nebari-jupyterlab:${{ env.NEBARI_IMAGE_TAG }}"
           NEBARI_SECRET__default_images__dask_worker: "quay.io/nebari/nebari-dask-worker:${{ env.NEBARI_IMAGE_TAG }}"
-          ARM_CLIENT_ID: ${{ env.ARM_CLIENT_ID }}
-          ARM_TENANT_ID: ${{ env.ARM_TENANT_ID }}
-          ARM_SUBSCRIPTION_ID: ${{ env.ARM_SUBSCRIPTION_ID }}
+          ARM_CLIENT_ID: ${{ secrets.ARM_CLIENT_ID }}
+          ARM_TENANT_ID: ${{ secrets.ARM_TENANT_ID }}
+          ARM_SUBSCRIPTION_ID: ${{ secrets.ARM_SUBSCRIPTION_ID }}
           ARM_USE_OIDC: "true"
-          CLOUDFLARE_TOKEN: ${{ env.CLOUDFLARE_TOKEN }}
+          CLOUDFLARE_TOKEN: ${{ secrets.CLOUDFLARE_TOKEN }}

.github/workflows/test_gcp_integration.yaml

@@ -50,24 +50,11 @@ jobs:
           pip install .[dev]
           playwright install
 
-      - name: Retrieve secret from Vault
-        uses: hashicorp/vault-action@v3.0.0
-        with:
-          method: jwt
-          url: "https://quansight-vault-public-vault-b2379fa7.d415e30e.z1.hashicorp.cloud:8200"
-          namespace: "admin/quansight"
-          role: "repository-nebari-dev-nebari-role"
-          secrets: |
-            kv/data/repository/nebari-dev/nebari/google_cloud_platform/nebari-dev-ci/github-nebari-dev-repo-ci project_id | PROJECT_ID;
-            kv/data/repository/nebari-dev/nebari/google_cloud_platform/nebari-dev-ci/github-nebari-dev-repo-ci workload_identity_provider | GCP_WORKFLOW_PROVIDER;
-            kv/data/repository/nebari-dev/nebari/google_cloud_platform/nebari-dev-ci/github-nebari-dev-repo-ci service_account_name | GCP_SERVICE_ACCOUNT;
-            kv/data/repository/nebari-dev/nebari/cloudflare/internal-devops@quansight.com/nebari-dev-ci token | CLOUDFLARE_TOKEN;
-
       - name: 'Authenticate to GCP'
         uses: 'google-github-actions/auth@v1'
         with:
-            workload_identity_provider: ${{ env.GCP_WORKFLOW_PROVIDER }}
-            service_account: ${{ env.GCP_SERVICE_ACCOUNT }}
+            workload_identity_provider: ${{ secrets.GCP_WORKFLOW_PROVIDER }}
+            service_account: ${{ secrets.GCP_SERVICE_ACCOUNT }}
 
       - name: Set required environment variables
         run: |
@@ -81,3 +68,5 @@ jobs:
           NEBARI_SECRET__default_images__jupyterhub: "quay.io/nebari/nebari-jupyterhub:${{ env.NEBARI_IMAGE_TAG }}"
           NEBARI_SECRET__default_images__jupyterlab: "quay.io/nebari/nebari-jupyterlab:${{ env.NEBARI_IMAGE_TAG }}"
           NEBARI_SECRET__default_images__dask_worker: "quay.io/nebari/nebari-dask-worker:${{ env.NEBARI_IMAGE_TAG }}"
+          PROJECT_ID: ${{ secrets.GCP_PROJECT_ID }}
+          CLOUDFLARE_TOKEN: ${{ secrets.CLOUDFLARE_TOKEN }}

.github/workflows/test_local_integration.yaml

@@ -73,6 +73,11 @@ jobs:
           python-version: "3.11"
           miniconda-version: "latest"
 
+      - name: Install JQ
+        run: |
+          sudo apt-get update
+          sudo apt-get install jq -y
+
       - name: Install Nebari and playwright
         run: |
           pip install .[dev]
@@ -97,6 +102,14 @@ jobs:
           nebari keycloak adduser --user "${TEST_USERNAME}" "${TEST_PASSWORD}" --config ${{ steps.init.outputs.config }}
           nebari keycloak listusers --config ${{ steps.init.outputs.config }}
 
+      - name: Await Workloads
+        uses: jupyterhub/action-k8s-await-workloads@v3
+        with:
+          workloads: "" # all
+          namespace: "dev"
+          timeout: 300
+          max-restarts: 3
+
       ### DEPLOYMENT TESTS
       - name: Deployment Pytests
         env:

.pre-commit-config.yaml

@@ -37,7 +37,7 @@ repos:
         exclude: "^src/_nebari/template/"
 
   - repo: https://github.com/crate-ci/typos
-    rev: v1.27.0
+    rev: dictgen-v0.3.1
     hooks:
       - id: typos
 
@@ -61,7 +61,7 @@ repos:
         args: ["--line-length=88", "--exclude=/src/_nebari/template/"]
 
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.7.2
+    rev: v0.8.6
     hooks:
       - id: ruff
         args: ["--fix"]
@@ -77,7 +77,7 @@ repos:
 
   # terraform
   - repo: https://github.com/antonbabenko/pre-commit-terraform
-    rev: v1.96.2
+    rev: v1.96.3
     hooks:
       - id: terraform_fmt
         args:

RELEASE.md

@@ -9,6 +9,106 @@ This file is copied to nebari-dev/nebari-docs using a GitHub Action. -->
 
 ---
 
+## Release 2025.1.1 - January 27, 2025
+
+> NOTE: In this release, we have updated our maximum supported Kubernetes version from
+> 1.29 to 1.31. we strongly recommend updating the Kubernetes version
+> specified in your nebari-config YAML file and redeploying to apply the changes.
+>
+> Remember that Kubernetes minor versions must be upgraded incrementally (1.29 → 1.30 →
+> 1.31).
+
+## What's Changed
+* fix bug to allow --import-plugin to work by @Adam-D-Lewis in https://github.com/nebari-dev/nebari/pull/2864
+* Add azure kubernetes policy add-on by @viniciusdc in https://github.com/nebari-dev/nebari/pull/2888
+* Yaml config sets by @Adam-D-Lewis in https://github.com/nebari-dev/nebari/pull/2876
+* Add ability to list user installed plugins from the CLI by @soapy1 in https://github.com/nebari-dev/nebari/pull/2891
+* [AUTO] Update CLI doc by @nebari-sensei in https://github.com/nebari-dev/nebari/pull/2841
+* [pre-commit.ci] pre-commit autoupdate by @pre-commit-ci in https://github.com/nebari-dev/nebari/pull/2896
+* [ENH] - Include "--attempt-fixes" flag from Nebari upgrade CLI in upgrade steps logic by @smokestacklightnin in https://github.com/nebari-dev/nebari/pull/2839
+* add authorized ip range variable for azure by @dcmcand in https://github.com/nebari-dev/nebari/pull/2880
+* Upgrade conda-store to 2024.11.2 by @marcelovilla in https://github.com/nebari-dev/nebari/pull/2815
+* Handle default value for azure addon policy by @viniciusdc in https://github.com/nebari-dev/nebari/pull/2905
+* Update conda-store-ui tests for updated page verbiage by @soapy1 in https://github.com/nebari-dev/nebari/pull/2911
+* Remove unintended character at the end of the TF_LOG variable by @marcelovilla in https://github.com/nebari-dev/nebari/pull/2912
+* Update k8s max version by @dcmcand in https://github.com/nebari-dev/nebari/pull/2903
+* [ENH] - Use GitHub secrets instead of Vault by @smokestacklightnin in https://github.com/nebari-dev/nebari/pull/2889
+* adds info command text display & change the order of command display by @kernel-loophole in https://github.com/nebari-dev/nebari/pull/2916
+* [AUTO] Update CLI doc by @nebari-sensei in https://github.com/nebari-dev/nebari/pull/2921
+
+## New Contributors
+* @soapy1 made their first contribution in https://github.com/nebari-dev/nebari/pull/2891
+* @smokestacklightnin made their first contribution in https://github.com/nebari-dev/nebari/pull/2839
+* @kernel-loophole made their first contribution in https://github.com/nebari-dev/nebari/pull/2916
+
+**Full Changelog**: https://github.com/nebari-dev/nebari/compare/2024.12.1...2025.1.1
+
+## Release 2024.12.1 - December 13, 2024
+
+> NOTE: Support for DigitalOcean has been removed in this release. If you plan to deploy Nebari on DigitalOcean, you first need to independently create a Kubernetes cluster and then use the `existing` deployment option.
+
+### What's Changed
+- Precommit typos by @blakerosenthal in https://github.com/nebari-dev/nebari/pull/2731
+- fix typo in KubernetesCredentials by @blakerosenthal in https://github.com/nebari-dev/nebari/pull/2729
+- handle branch rename from develop to main in github actions by @Adam-D-Lewis in https://github.com/nebari-dev/nebari/pull/2748
+- remove do integration test by @Adam-D-Lewis in https://github.com/nebari-dev/nebari/pull/2765
+- Remove old develop branch references after default branch renaming by @marcelovilla in https://github.com/nebari-dev/nebari/pull/2769
+- fix CICD issue with pre-commit action by @Adam-D-Lewis in https://github.com/nebari-dev/nebari/pull/2775
+- fix CHECK_URL in kuberhealthy checks to respect namespaces by @dcmcand in https://github.com/nebari-dev/nebari/pull/2779
+- remove duplicate GCPPrivateClusterConfig class by @dcmcand in https://github.com/nebari-dev/nebari/pull/2786
+- Fix hub variable for jupyterhub_dashboard by @kenafoster in https://github.com/nebari-dev/nebari/pull/2721
+- Fix Pytest Tests failing on PRs updating src by @joneszc in https://github.com/nebari-dev/nebari/pull/2790
+- Add ability to add overrides to jhub-apps config by @aktech in https://github.com/nebari-dev/nebari/pull/2754
+- Remove leftover develop reference by @marcelovilla in https://github.com/nebari-dev/nebari/pull/2792
+- fix bug where check_immutable_fields throws error with old version of Nebari by @Adam-D-Lewis in https://github.com/nebari-dev/nebari/pull/2796
+- Fix immutable field validation error when a sub-schema is not Pydantic by @kenafoster in https://github.com/nebari-dev/nebari/pull/2797
+- Address issue with AWS instance type schema by @viniciusdc in https://github.com/nebari-dev/nebari/pull/2787
+- add broken note by @Adam-D-Lewis in https://github.com/nebari-dev/nebari/pull/2802
+- Fix release notes formatting to restore docs syncing functionality by @marcelovilla in https://github.com/nebari-dev/nebari/pull/2809
+- Refactor role creation for upgrade command path by @viniciusdc in https://github.com/nebari-dev/nebari/pull/2795
+- add test workflow for upgrade by @pmeier in https://github.com/nebari-dev/nebari/pull/2780
+- Add config option to enable the encryption of AWS EKS secrets by @joneszc in https://github.com/nebari-dev/nebari/pull/2788
+- remove digital ocean tests by @dcmcand in https://github.com/nebari-dev/nebari/pull/2813
+- Python3 13 upgrade dependencies by @dcmcand in https://github.com/nebari-dev/nebari/pull/2823
+- Test support for Python 3.13 in CI by @aktech in https://github.com/nebari-dev/nebari/pull/2774
+- remove unmaintained nix files by @Adam-D-Lewis in https://github.com/nebari-dev/nebari/pull/2831
+- allow passing X.XX or X.XX.XX as k8s versions by @dcmcand in https://github.com/nebari-dev/nebari/pull/2840
+- Remove explicit branch inputs from cloud integration test workflows in GHA by @marcelovilla in https://github.com/nebari-dev/nebari/pull/2837
+- Allow overriding of keycloak root credentials for `2024.11.1` upgrade path by @viniciusdc in https://github.com/nebari-dev/nebari/pull/2843
+- Added security group rule descriptions by @jcbolling in https://github.com/nebari-dev/nebari/pull/2850
+- Set `launch_template.ami_id` attrs to private by @viniciusdc in https://github.com/nebari-dev/nebari/pull/2842
+- attempt to address paramiko connection errors by @viniciusdc in https://github.com/nebari-dev/nebari/pull/2811
+- specify terraform registry for providers not in opentofu registry by @dcmcand in https://github.com/nebari-dev/nebari/pull/2852
+- Disable AWS `launch_template` from nebari-config schema by @viniciusdc in https://github.com/nebari-dev/nebari/pull/2856
+- Remove Digital Ocean references by @marcelovilla in https://github.com/nebari-dev/nebari/pull/2838
+- Use tofu binary instead of terraform one by @marcelovilla in https://github.com/nebari-dev/nebari/pull/2773
+- Add 2024.11.1 release notes and bump version by @viniciusdc in https://github.com/nebari-dev/nebari/pull/2859
+- Disable `jupyterlab-jhub-apps` extension when jhub-apps is disabled by @krassowski in https://github.com/nebari-dev/nebari/pull/2804
+- Validate instance types for GCP by @blakerosenthal in https://github.com/nebari-dev/nebari/pull/2730
+- update gcp instance validation by @dcmcand in https://github.com/nebari-dev/nebari/pull/2875
+
+### New Contributors
+- @jcbolling made their first contribution in https://github.com/nebari-dev/nebari/pull/2850
+
+**Full Changelog**: https://github.com/nebari-dev/nebari/compare/2024.11.1...2024.12.1
+
+## Release 2024.11.1 - November 21, 2024 (Hotfix Release)
+
+> NOTE: This hotfix addresses several major bugs identified in the 2024.9.1 release. For a detailed overview, please refer to the related discussion at #2798. Users should upgrade directly from 2024.7.1 to 2024.11.1.
+
+## What's Changed
+
+- fix `CHECK_URL` in kuberhealthy checks to respect namespaces by @dcmcand in https://github.com/nebari-dev/nebari/pull/2779
+- fix bug where `check_immutable_fields` throws error with old version of Nebari by @Adam-D-Lewis in https://github.com/nebari-dev/nebari/pull/2796
+- Fix immutable field validation error when a sub-schema is not Pydantic by @kenafoster in https://github.com/nebari-dev/nebari/pull/2797
+- Address issue with AWS instance type schema by @viniciusdc in https://github.com/nebari-dev/nebari/pull/2787
+- Add broken note by @Adam-D-Lewis in https://github.com/nebari-dev/nebari/pull/2802
+- Refactor role creation for upgrade command path by @viniciusdc in https://github.com/nebari-dev/nebari/pull/2795
+- Allow overriding of keycloak root credentials for 2024.11.1 upgrade path #2843
+- Disable AWS `launch_template` from nebari-config schema #2856
+
+**Full Changelog**: https://github.com/nebari-dev/nebari/compare/2024.9.1...2024.11.1
+
 ## Release 2024.9.1 - September 27, 2024 (Broken Release)
 
 > WARNING: This release was later found to have unresolved issues described further in [issue 2798](https://github.com/nebari-dev/nebari/issues/2798). We have marked this release as broken on conda-forge and yanked it on PyPI. One of the bugs prevents any upgrade from 2024.9.1 to 2024.11.1. Users should skip this release entirely and upgrade directly from 2024.7.1 to 2024.11.1.