diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 95dbd3f..c8f6d46 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -12,12 +12,10 @@ jobs: fail-fast: false matrix: os: [ubuntu-latest, macos-latest] - python-version: ["3.8", "3.11"] + python-version: ["3.9", "3.11", "3.12"] include: - os: ubuntu-latest - python-version: "3.9" - - os: ubuntu-latest - python-version: "pypy-3.8" + python-version: "pypy-3.9" - os: macos-latest python-version: "3.10" steps: @@ -26,15 +24,17 @@ jobs: with: clean: true - uses: jupyterlab/maintainer-tools/.github/actions/base-setup@v1 + - uses: actions/setup-java@v4 + with: + distribution: temurin + java-version: 17 + - uses: sbt/setup-sbt@v1 - name: Display dependency info run: | python --version pip --version - conda --version - - name: Add SBT launcher - run: | - mkdir -p $HOME/.sbt/launchers/1.3.12 - curl -L -o $HOME/.sbt/launchers/1.3.12/sbt-launch.jar https://repo1.maven.org/maven2/org/scala-sbt/sbt-launch/1.3.12/sbt-launch.jar + java --version + sbt --version - name: Install Python dependencies run: | pip install ".[dev]" @@ -61,6 +61,7 @@ jobs: steps: - name: Checkout uses: actions/checkout@v4 + - uses: sbt/setup-sbt@v1 - name: Base Setup uses: jupyterlab/maintainer-tools/.github/actions/base-setup@v1 - name: Check Release @@ -76,7 +77,7 @@ jobs: - uses: jupyterlab/maintainer-tools/.github/actions/base-setup@v1 - uses: jupyterlab/maintainer-tools/.github/actions/check-links@v1 with: - ignore_links: "http://my-gateway-server.com:8888" + ignore_links: "http://my-gateway-server.com:8888 https://www.gnu.org/software/make/" ignore_glob: "gateway_provisioners/app-support/README.md" build_docs: diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 5ec921c..862a5f1 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -27,7 +27,7 @@ repos: - id: check-github-workflows - repo: https://github.com/executablebooks/mdformat - rev: 0.7.17 + rev: 0.7.22 hooks: - id: mdformat additional_dependencies: diff --git a/CHANGELOG.md b/CHANGELOG.md index c57c515..fefc97c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,11 +20,11 @@ - Update dependabot config [#130](https://github.com/jupyter-server/gateway_provisioners/pull/130) ([@blink1073](https://github.com/blink1073)) - Update Release Workflows [#129](https://github.com/jupyter-server/gateway_provisioners/pull/129) ([@blink1073](https://github.com/blink1073)) -- Bump black\[jupyter\] from 23.9.1 to 23.11.0 [#119](https://github.com/jupyter-server/gateway_provisioners/pull/119) ([@dependabot](https://github.com/dependabot)) +- Bump black[jupyter] from 23.9.1 to 23.11.0 [#119](https://github.com/jupyter-server/gateway_provisioners/pull/119) ([@dependabot](https://github.com/dependabot)) - Bump actions/checkout from 3 to 4 [#106](https://github.com/jupyter-server/gateway_provisioners/pull/106) ([@dependabot](https://github.com/dependabot)) -- Bump black\[jupyter\] from 23.7.0 to 23.9.1 [#105](https://github.com/jupyter-server/gateway_provisioners/pull/105) ([@dependabot](https://github.com/dependabot)) +- Bump black[jupyter] from 23.7.0 to 23.9.1 [#105](https://github.com/jupyter-server/gateway_provisioners/pull/105) ([@dependabot](https://github.com/dependabot)) - Adopt sp-repo-review [#104](https://github.com/jupyter-server/gateway_provisioners/pull/104) ([@blink1073](https://github.com/blink1073)) -- Bump black\[jupyter\] from 23.3.0 to 23.7.0 [#99](https://github.com/jupyter-server/gateway_provisioners/pull/99) ([@dependabot](https://github.com/dependabot)) +- Bump black[jupyter] from 23.3.0 to 23.7.0 [#99](https://github.com/jupyter-server/gateway_provisioners/pull/99) ([@dependabot](https://github.com/dependabot)) - Update mistune requirement from \<3.0.0 to \<4.0.0 [#94](https://github.com/jupyter-server/gateway_provisioners/pull/94) ([@dependabot](https://github.com/dependabot)) - Bump ruff from 0.0.269 to 0.0.270 [#92](https://github.com/jupyter-server/gateway_provisioners/pull/92) ([@dependabot](https://github.com/dependabot)) - Bump ruff from 0.0.267 to 0.0.269 [#91](https://github.com/jupyter-server/gateway_provisioners/pull/91) ([@dependabot](https://github.com/dependabot)) @@ -63,11 +63,11 @@ - Bump ruff from 0.0.260 to 0.0.261 [#79](https://github.com/jupyter-server/gateway_provisioners/pull/79) ([@dependabot](https://github.com/dependabot)) - Bump ruff from 0.0.259 to 0.0.260 [#77](https://github.com/jupyter-server/gateway_provisioners/pull/77) ([@dependabot](https://github.com/dependabot)) -- Bump black\[jupyter\] from 23.1.0 to 23.3.0 [#76](https://github.com/jupyter-server/gateway_provisioners/pull/76) ([@dependabot](https://github.com/dependabot)) +- Bump black[jupyter] from 23.1.0 to 23.3.0 [#76](https://github.com/jupyter-server/gateway_provisioners/pull/76) ([@dependabot](https://github.com/dependabot)) - Bump ruff from 0.0.257 to 0.0.259 [#75](https://github.com/jupyter-server/gateway_provisioners/pull/75) ([@dependabot](https://github.com/dependabot)) - Bump ruff from 0.0.254 to 0.0.257 [#73](https://github.com/jupyter-server/gateway_provisioners/pull/73) ([@dependabot](https://github.com/dependabot)) - Bump ruff from 0.0.252 to 0.0.254 [#69](https://github.com/jupyter-server/gateway_provisioners/pull/69) ([@dependabot](https://github.com/dependabot)) -- Fix relative link formatting in documentation to be consistent [#68](https://github.com/jupyter-server/gateway_provisioners/pull/68) ([@kiersten-stokes](https://github.com/kiersten-stokes)) +- Fix relative link formatting in documentation to be consistent [#68](https://github.com/jupyter-server/gateway_provisioners/pull/68) ([@kiersten-stokes](https://github.com/kiersten-stokes)) - Bump ruff from 0.0.249 to 0.0.252 [#65](https://github.com/jupyter-server/gateway_provisioners/pull/65) ([@dependabot](https://github.com/dependabot)) - Use releaser workflows [#64](https://github.com/jupyter-server/gateway_provisioners/pull/64) ([@blink1073](https://github.com/blink1073)) - Create hatch build env with make-related scripts [#63](https://github.com/jupyter-server/gateway_provisioners/pull/63) ([@kevin-bates](https://github.com/kevin-bates)) @@ -79,7 +79,7 @@ - Add link to SparkOperatorProvisioner class definition [#81](https://github.com/jupyter-server/gateway_provisioners/pull/81) ([@kevin-bates](https://github.com/kevin-bates)) - Fix minor issues in Developer and Contributor docs [#74](https://github.com/jupyter-server/gateway_provisioners/pull/74) ([@kiersten-stokes](https://github.com/kiersten-stokes)) - Fix grammar NITs in Operator's Guide of docs [#72](https://github.com/jupyter-server/gateway_provisioners/pull/72) ([@kiersten-stokes](https://github.com/kiersten-stokes)) -- Fix relative link formatting in documentation to be consistent [#68](https://github.com/jupyter-server/gateway_provisioners/pull/68) ([@kiersten-stokes](https://github.com/kiersten-stokes)) +- Fix relative link formatting in documentation to be consistent [#68](https://github.com/jupyter-server/gateway_provisioners/pull/68) ([@kiersten-stokes](https://github.com/kiersten-stokes)) - Fix minor errors in Users subsection of documentation [#67](https://github.com/jupyter-server/gateway_provisioners/pull/67) ([@kiersten-stokes](https://github.com/kiersten-stokes)) - Add application support information for deploying JKG and Lab [#66](https://github.com/jupyter-server/gateway_provisioners/pull/66) ([@kevin-bates](https://github.com/kevin-bates)) - Replace references to gateway-experiments with jupyter-server [#62](https://github.com/jupyter-server/gateway_provisioners/pull/62) ([@kevin-bates](https://github.com/kevin-bates)) diff --git a/docs/source/contributors/devinstall.md b/docs/source/contributors/devinstall.md index 46638fb..992dfdb 100644 --- a/docs/source/contributors/devinstall.md +++ b/docs/source/contributors/devinstall.md @@ -23,7 +23,7 @@ will not occur! Always use `make dist` to build the distribution. ### `sbt` Our Scala launcher is built using `sbt` -([Scala Build Tool](https://www.scala-sbt.org/index.html)). Please check +([Scala Build Tool](https://www.scala-sbt.org/index.html)). Please check [here](https://www.scala-sbt.org/1.x/docs/Setup.html) for installation instructions for your platform. ## Clone the repo diff --git a/docs/source/contributors/system-architecture.md b/docs/source/contributors/system-architecture.md index e155f42..0a75a29 100644 --- a/docs/source/contributors/system-architecture.md +++ b/docs/source/contributors/system-architecture.md @@ -51,8 +51,8 @@ step from its implementation. ### Gateway Provisioner Class Hierarchy -The following block diagram depicts the current class hierarchy for the Gateway Provisioners. The blocks with an -`ABC` badge and dashed border indicate abstract base classes. Those light blue blocks come from `jupyter_client`, +The following block diagram depicts the current class hierarchy for the Gateway Provisioners. The blocks with an +`ABC` badge and dashed border indicate abstract base classes. Those light blue blocks come from `jupyter_client`, while the others reside in Gateway Provisioners. ```{blockdiag} @@ -157,8 +157,8 @@ kernel provisioner configuration to override the global value, enabling finer-gr `ContainerProvisionerBase` is an abstract base class that derives from `RemoteProvisionerBase`. It implements all the methods inherited from `RemoteProvsionerBase` interacting with the container API and requiring method implementations -to perform the platform's integration. Subclasses -of `ContainerProvisionerBase` must also implement `get_initial_states()`, `get_error_states()`, `get_container_status()`, +to perform the platform's integration. Subclasses +of `ContainerProvisionerBase` must also implement `get_initial_states()`, `get_error_states()`, `get_container_status()`, and `terminate_container_resources()`: ```python @@ -204,9 +204,9 @@ manages kernels via a custom resource definition (CRD). For example, `SparkAppli many components of a Spark-on-Kubernetes application. `CustomResourceProvisioner` could be considered a _virtual abstract base class_ that provides the necessary method overrides of -`KubernetesProvisioner` to manage the lifecycle of CRDs. If you are going to extend `CustomResourceProvisioner`, +`KubernetesProvisioner` to manage the lifecycle of CRDs. If you are going to extend `CustomResourceProvisioner`, all that should be necessary is to override these custom resource related attributes (i.e. `group`, `version`, `plural` and -`object_kind`) that define the CRD attributes and its implementation should cover the rest. Note that `object_kind` is +`object_kind`) that define the CRD attributes and its implementation should cover the rest. Note that `object_kind` is an internal attribute that Gateway Provisioners uses, while the other attributes are associated with the Kubernetes CRD object definition. @@ -218,7 +218,7 @@ to function. In addition, the class itself doesn't define any abstract methods #### `SparkOperatorProvisioner` -A great example of a `CustomResourceProvisioner` is `SparkOperatorProvisioner`. As described in the previous section, +A great example of a `CustomResourceProvisioner` is `SparkOperatorProvisioner`. As described in the previous section, it's implementation consists of overrides of attributes `group` (e.g, `"sparkoperator.k8s.io"`), `version` (i.e., `"v1beta2"`), `plural` (i.e., `"sparkapplications"`) and `object_kind` (i.e., `"SparkApplication"`). @@ -231,7 +231,7 @@ Operators Guide for details. Gateway Provisioners provides an implementation of a kernel provisioner that communicates with the Docker Swarm resource manager via the Docker API. When used, the kernels are launched as swarm services and can reside anywhere in the -managed cluster. The core of a Docker Swarm service is a container, so `DockerSwarmProvisioner` derives from +managed cluster. The core of a Docker Swarm service is a container, so `DockerSwarmProvisioner` derives from `ContainerProvisionerBase`. To leverage kernels configured in this manner, the host application can be deployed either as a Docker Swarm _service_ or a traditional Docker container. diff --git a/docs/source/developers/custom-images.md b/docs/source/developers/custom-images.md index ca21fd6..628e669 100644 --- a/docs/source/developers/custom-images.md +++ b/docs/source/developers/custom-images.md @@ -108,8 +108,8 @@ the appropriate directory in place. For the purposes of this discussion, we'll a directory, `/usr/local/share/jupyter/kernels`, is externally mounted. Depending on the environment, Kubernetes or Docker, you can use with `jupyter-k8s-spec` or `jupyter-docker-spec`, -respectively. Invoke the appropriate script by adding the `--image-name` parameter identifying the name of your -custom kernel image. For example, if your custom image is named `acme/data-sci-py:2.0` and you are targeting +respectively. Invoke the appropriate script by adding the `--image-name` parameter identifying the name of your +custom kernel image. For example, if your custom image is named `acme/data-sci-py:2.0` and you are targeting Kubernetes, issue: ```dockerfile diff --git a/docs/source/operators/config-file.md b/docs/source/operators/config-file.md index a6e4c8a..5b2888a 100644 --- a/docs/source/operators/config-file.md +++ b/docs/source/operators/config-file.md @@ -21,7 +21,7 @@ c.DistributedProvisioner.remote_hosts = ["localhost"] ## Provisioner-specific Configuration Options -A complete set of configuration options available for each Gateway Provisioner follows. Where applicable, the +A complete set of configuration options available for each Gateway Provisioner follows. Where applicable, the configurable option's default value is also provided. ### `KubernetesProvisioner` diff --git a/docs/source/operators/deploy-distributed.md b/docs/source/operators/deploy-distributed.md index 3e48307..fbae02b 100644 --- a/docs/source/operators/deploy-distributed.md +++ b/docs/source/operators/deploy-distributed.md @@ -18,7 +18,7 @@ Steps required to complete deployment on a distributed cluster are: ## Prerequisites -The distributed capabilities of the `DistributedProvisioner` utilize SSH. As a result, you must ensure appropriate +The distributed capabilities of the `DistributedProvisioner` utilize SSH. As a result, you must ensure appropriate password-less functionality is in place. If you want to use Spark in "client mode", you'll want to ensure the `SPARK_HOME` environment variable is properly @@ -92,9 +92,9 @@ where each provides the following function: its display name (`display_name`) and language (`language`), as well as its kernel provisioner's configuration (`metadata.kernel_provisioner`) - which, in this case, will reflect the `DistributedProvisioner`. -- `logo-64x64.png` - the icon resource corresponding to this kernel specification. Icon resource files must be start +- `logo-64x64.png` - the icon resource corresponding to this kernel specification. Icon resource files must be start with the `logo-` prefix to be included in the kernel specification. -- `scripts/launch_ipykernel.py` - the "launcher" for the IPyKernel kernel (or subclasses thereof). This file is typically +- `scripts/launch_ipykernel.py` - the "launcher" for the IPyKernel kernel (or subclasses thereof). This file is typically implemented in the language of the kernel and is responsible for creating the local connection information, asynchronously starting a SparkContext (if asked), spawning a listener process to receive interrupts and shutdown requests, and starting the IPyKernel itself. @@ -300,7 +300,7 @@ To see all available configurables, use `--help-all`. ## Specifying a load-balancing algorithm The `DistributedProvisioner` provides two ways to configure how kernels are distributed across -the configured set of hosts: round-robin or least-connection. This configurable option is a _host application_ +the configured set of hosts: round-robin or least-connection. This configurable option is a _host application_ setting and is not available to be overridden on a per-kernel basis. ### Round-robin @@ -362,7 +362,7 @@ YARN client mode kernel specifications can be considered _distributed mode kerne happen to use `spark-submit` from different nodes in the cluster but use the `DistributedProvisioner` to manage their lifecycle. -These kernel specifications are generated using the `'--spark'` command line option as noted above. When provided, +These kernel specifications are generated using the `'--spark'` command line option as noted above. When provided, a kernel specification similar to the following is produced: ```json diff --git a/docs/source/operators/deploy-docker.md b/docs/source/operators/deploy-docker.md index 40779df..719230d 100644 --- a/docs/source/operators/deploy-docker.md +++ b/docs/source/operators/deploy-docker.md @@ -16,10 +16,10 @@ for examples of how to configure and deploy such applications. ## Generating Kernel Specifications -Kernelspec generation for Docker and Docker Swarm deployments is performed using the `jupyter-docker-spec` command. Because +Kernelspec generation for Docker and Docker Swarm deployments is performed using the `jupyter-docker-spec` command. Because the host application will also reside within a docker image, the commands are usually placed into a Dockerfile -that _extends_ an existing image. However, some may choose to `docker exec` into a running container, perform and test -the necessary configuration, then use `docker commit` to generate a new image. That said, the following will assume a +that _extends_ an existing image. However, some may choose to `docker exec` into a running container, perform and test +the necessary configuration, then use `docker commit` to generate a new image. That said, the following will assume a Dockerfile approach. ```{attention} @@ -57,10 +57,10 @@ where each provides the following function: its display name (`display_name`) and language (`language`), as well as its kernel provisioner's configuration (`metadata.kernel_provisioner`) - which, in this case, will reflect the `DockerProvisioner`. -- `logo-64x64.png` - the icon resource corresponding to this kernel specification. Icon resource files must be start +- `logo-64x64.png` - the icon resource corresponding to this kernel specification. Icon resource files must be start with the `logo-` prefix to be included in the kernel specification. - `scripts/launch_docker.py` - the "launcher" for the kernel image identified by the - `metadata.kernel_provisioner.config.image_name` entry. This file can be modified to include instructions for + `metadata.kernel_provisioner.config.image_name` entry. This file can be modified to include instructions for volume mounts, etc., and is compatible with both Docker and Docker Swarm - performing the applicable instructions for each environment. @@ -71,7 +71,7 @@ others. ### Generating Multiple Specifications -Its common practice to support multiple languages or use different images for kernels of the same language. For each +Its common practice to support multiple languages or use different images for kernels of the same language. For each of those differences, a separate installation command should be provided: ```dockerfile @@ -102,9 +102,9 @@ Items worth noting: ## Other Configuration Items There are some environment variables that can be set in the host application's environment that affect how Gateway -Provisioners operate within a Docker and Docker Swarm environment. For example, `GP_MIRROR_WORKING_DIRS` can be set +Provisioners operate within a Docker and Docker Swarm environment. For example, `GP_MIRROR_WORKING_DIRS` can be set to `True`, instructing Gateway Provisioners to set the launched container's working directory to the value of -`KERNEL_WORKING_DIR`. When this environment variable is enabled, it usually implies that volume mounts are in play +`KERNEL_WORKING_DIR`. When this environment variable is enabled, it usually implies that volume mounts are in play such that the per-user volumes are then available to the launched container. Other [environment variables](config-add-env.md#additional-environment-variables) applicable to Docker/Docker Swarm diff --git a/docs/source/operators/deploy-kubernetes.md b/docs/source/operators/deploy-kubernetes.md index d72bdbc..76dfc55 100644 --- a/docs/source/operators/deploy-kubernetes.md +++ b/docs/source/operators/deploy-kubernetes.md @@ -13,10 +13,10 @@ for examples of how to configure and deploy such applications. ## Generating Kernel Specifications -Kernelspec generation for Kubernetes deployments is performed using the `jupyter-k8s-spec` command. Because +Kernelspec generation for Kubernetes deployments is performed using the `jupyter-k8s-spec` command. Because the host application will also reside within a docker image, the commands are usually placed into a Dockerfile -that _extends_ an existing image. However, some may choose to `docker exec` into a running container, perform and test -the necessary configuration, then use `docker commit` to generate a new image. That said, the following will assume a +that _extends_ an existing image. However, some may choose to `docker exec` into a running container, perform and test +the necessary configuration, then use `docker commit` to generate a new image. That said, the following will assume a Dockerfile approach. To generate a default kernel specification (where Python is the default kernel) enter: @@ -50,17 +50,17 @@ where each provides the following function: its display name (`display_name`) and language (`language`), as well as its kernel provisioner's configuration (`metadata.kernel_provisioner`) - which, in this case, will reflect the `KubernetesProvisioner`. -- `logo-64x64.png` - the icon resource corresponding to this kernel specification. Icon resource files must be start +- `logo-64x64.png` - the icon resource corresponding to this kernel specification. Icon resource files must be start with the `logo-` prefix to be included in the kernel specification. -- `scripts/launch_kubernetes.py` - the "launcher" for the kernel pod. This script processes its sibling Jinja +- `scripts/launch_kubernetes.py` - the "launcher" for the kernel pod. This script processes its sibling Jinja templates by applying appropriate substitutes and creating each of the Kubernetes resources as described in the template. - `scripts/kernel-pod.yaml.j2` - the Jinja template describing the to-be-launched kernel pod corresponding to the - kernel image identified by the `metadata.kernel_provisioner.config.image_name` entry. This file can be modified to + kernel image identified by the `metadata.kernel_provisioner.config.image_name` entry. This file can be modified to include instructions for volume mounts, etc., for establishing the pod's configuration. -- `bin/run.sh` - This file will be present only when `--spark` is specified. The first entry in the `kernel.json`'s +- `bin/run.sh` - This file will be present only when `--spark` is specified. The first entry in the `kernel.json`'s `argv` stanza will be a reference to `bin/run.sh`. This script sets up and invokes the `spark-submit` command that - is responsible for interacting with the Spark-on-Kubernetes resource manager. With the introduction of Spark Pod - Templates, we can leverage the same templating behavior in Spark-based environments. As a result, both the driver + is responsible for interacting with the Spark-on-Kubernetes resource manager. With the introduction of Spark Pod + Templates, we can leverage the same templating behavior in Spark-based environments. As a result, both the driver and executor pods will have similar configurations. ```{seealso} @@ -71,7 +71,7 @@ others. ### Deploying Custom Resource Definitions Gateway Provisioners currently supports one form of Custom Resource Definitions (CRDs) via the -[`SparkOperatorProvisioner`](../contributors/system-architecture.md#sparkoperatorprovisioner). To generate a kernel +[`SparkOperatorProvisioner`](../contributors/system-architecture.md#sparkoperatorprovisioner). To generate a kernel specification to use `SparkOperatorProvisioner`, in addition to including the `--spark` option, you will also include the `--crd` option to `jupyter k8s-spec install`. @@ -99,10 +99,10 @@ sparkoperator.k8s.io-v1beta2.yaml.j2 There are a few things worth noting here. -1. The `scripts` directory contains a different set of scripts. This is because the SparkOperator requires a +1. The `scripts` directory contains a different set of scripts. This is because the SparkOperator requires a slightly different launch script and its yaml definitions are different enough to warrant separation. 1. Although this provisioner uses Spark, there is no `run` sub-directory created that contains a `spark-submit` - command. Instead, the appropriate CRD is created which performs the application's submission to Spark directly. + command. Instead, the appropriate CRD is created which performs the application's submission to Spark directly. 1. The yaml template name is a composition of the provisioner's [`group` and `version` attributes](../contributors/system-architecture.md/#sparkoperatorprovisioner). In this case, the `group` is `sparkoperator.k8s.io` and `version` is `v1beta2`. @@ -120,7 +120,7 @@ webhook server is enabled when deploying the helm chart: ### Generating Multiple Specifications -Its common practice to support multiple languages or use different images for kernels of the same language. For each +Its common practice to support multiple languages or use different images for kernels of the same language. For each of those differences, a separate installation command should be provided: ```dockerfile @@ -136,7 +136,7 @@ When kernels are launched, Gateway Provisioners is responsible for creating the entity created is a function of whether the kernel is a regular or spark-based kernel specification. Regular kernels are launched as a kernel pod based on the `scripts/kernel-pod.yaml.j2` template via the -`scripts/launch_kubernetes.py` script, both of which are located in the `scripts` directory. Spark-based kernels are +`scripts/launch_kubernetes.py` script, both of which are located in the `scripts` directory. Spark-based kernels are launched via `spark-submit` in the `bin/run.sh` script triggering the creation of a _driver_ pod and one or more _executor_ pods. @@ -145,7 +145,7 @@ Items worth noting: 1. The launched pods' names will be composed of the launching username (`KERNEL_USERNAME`) and kernel-id. Some additional information is added to the spark-based pod names. 1. The pods will have 3 labels applied: `"kernel_id="`, `"component=kernel"`, and - `"app=gateway-provisioners"` - similar to Docker. The `component` label on the spark-based executor pods will hold a + `"app=gateway-provisioners"` - similar to Docker. The `component` label on the spark-based executor pods will hold a value of `worker` to distinguish them from the driver. 1. The pods will be launched within the same Kubernetes network as the host application. @@ -158,17 +158,17 @@ and Automatic Namespace. ### Shared Namespace Because Gateway Provisioners is a _library package_ and not its own application, the default namespace behavior is to -use a _shared namespace_. That is, kernel pods launched by the application are placed into the same namespace as the -hosting application. This option is controlled by two environment variables: `GP_SHARED_NAMESPACE` and `GP_NAMESPACE`. +use a _shared namespace_. That is, kernel pods launched by the application are placed into the same namespace as the +hosting application. This option is controlled by two environment variables: `GP_SHARED_NAMESPACE` and `GP_NAMESPACE`. #### `GP_SHARED_NAMESPACE` -This environment variable defaults to `True`. When enabled, all kernel pods will be launched into the namespace -identified by `GP_NAMESPACE`. No attempt is made to alter the configuration of `GP_NAMESPACE`. +This environment variable defaults to `True`. When enabled, all kernel pods will be launched into the namespace +identified by `GP_NAMESPACE`. No attempt is made to alter the configuration of `GP_NAMESPACE`. #### `GP_NAMESPACE` -This environment variable identifies the namespace in which the host application is running. It defaults to the +This environment variable identifies the namespace in which the host application is running. It defaults to the namespace named `default`, but its recommended that host application deployment configure its own namespace and this environment variable be set to that value. @@ -182,8 +182,8 @@ application or, for example, in JupyterHub situations where Hub launches noteboo modeling_, would have the same effect as setting `KERNEL_NAMESPACE` using the _Bring-Your-Own Namespace modeling_.) When configured, Gateway Provisioners assumes the namespace identified by `KERNEL_NAMESPACE` already exists and is -properly configured from a resources and privileges standpoint. In addition to providing `KERNEL_NAMESPACE`, users -must also provide the name of the service account privileged to create resources within the namespace. The service +properly configured from a resources and privileges standpoint. In addition to providing `KERNEL_NAMESPACE`, users +must also provide the name of the service account privileged to create resources within the namespace. The service account name is conveyed via `KERNEL_SERVICE_ACCOUNT_NAME`. Gateway Provisioners will not attempt to delete this namespace upon the kernel's termination. @@ -191,7 +191,7 @@ Gateway Provisioners will not attempt to delete this namespace upon the kernel's ### Automatic Namespace Operators wanting the finest isolation level, where each kernel pod is launched into their own namespace, can do so by -disabling `GP_SHARED_NAMESPACE` and not setting `KERNEL_NAMESPACE`. In these configurations, Gateway Provisioners will +disabling `GP_SHARED_NAMESPACE` and not setting `KERNEL_NAMESPACE`. In these configurations, Gateway Provisioners will _create_ a namespace corresponding to the values of `KERNEL_USERNAME`-`KERNEL_ID`, just as the pods are named. This option requires higher-level privileges on the RBAC settings of the host application as the namespace must be @@ -200,9 +200,9 @@ created on startup and deleted on termination. #### Required RBAC Settings As noted above, the ability to create and destroy namespaces at the time of kernel launch requires additional privileges -in the form of a `cluster-role`. This role will require privileges to create and delete namespaces, services, and -role-bindings, among other things. Because a new namespace will be created, a subsequent cluster-role should exist -that is for use by the kernel pod itself. These two role definitions are provided below: +in the form of a `cluster-role`. This role will require privileges to create and delete namespaces, services, and +role-bindings, among other things. Because a new namespace will be created, a subsequent cluster-role should exist +that is for use by the kernel pod itself. These two role definitions are provided below: ##### Application Cluster Role @@ -238,7 +238,7 @@ support Spark Operators - but plans to in the near future. The name of this role is conveyed from the host application to Gateway Provisioners via the `GP_KERNEL_CLUSTER_ROLE` environment variable and defaults to `cluster-admin`, so operators are advised to create -a specific role for these purposes. This role is responsible for managing resources within the newly-created namespace.\ +a specific role for these purposes. This role is responsible for managing resources within the newly-created namespace.\ While the role is a `cluster-role`, it is only _bound_ via a `role-binding` binding the cluster role to the namespace and its service account referenced by `KERNEL_SERVICE_ACCOUNT_NAME`. @@ -367,9 +367,9 @@ When defined, these variables are then substituted into the appropriate location ## Other Configuration Items There are some environment variables that can be set in the host application's environment that affect how Gateway -Provisioners operate within a Kubernetes environment. For example, `GP_MIRROR_WORKING_DIRS` can be set +Provisioners operate within a Kubernetes environment. For example, `GP_MIRROR_WORKING_DIRS` can be set to `True`, instructing Gateway Provisioners to set the launched container's working directory to the value of -`KERNEL_WORKING_DIR`. When this environment variable is enabled, it usually implies that volume mounts are in play +`KERNEL_WORKING_DIR`. When this environment variable is enabled, it usually implies that volume mounts are in play such that the per-user volumes are then available to the launched container. Other [environment variables](config-add-env.md#additional-environment-variables) applicable to Kubernetes diff --git a/docs/source/operators/deploy-yarn-cluster.md b/docs/source/operators/deploy-yarn-cluster.md index a4ae354..a4d3b85 100644 --- a/docs/source/operators/deploy-yarn-cluster.md +++ b/docs/source/operators/deploy-yarn-cluster.md @@ -1,7 +1,7 @@ # Hadoop YARN deployments Hadoop YARN deployments will utilize the `YarnProvisioner` to launch kernels across the cluster via -the YARN resource manager. The following assumes a Hadoop Yarn cluster has already been provisioned. +the YARN resource manager. The following assumes a Hadoop Yarn cluster has already been provisioned. ```{note} In some cases, where a Spark "client mode" is desired, use of the `DistributedProvisioner` @@ -92,12 +92,12 @@ where each provides the following function: its display name (`display_name`) and language (`language`), as well as its kernel provisioner's configuration (`metadata.kernel_provisioner`) - which, in this case, will reflect the `YarnProvisioner`. -- `logo-64x64.png` - the icon resource corresponding to this kernel specification. Icon resource files must be start +- `logo-64x64.png` - the icon resource corresponding to this kernel specification. Icon resource files must be start with the `logo-` prefix to be included in the kernel specification. - `bin/run.sh` - the first entry in the `kernel.json`'s `argv` stanza, this script sets up and invokes the `spark-submit` - command that is responsible for interacting with the Hadoop Yarn Resource Manager. The `YarnProvisioner` then + command that is responsible for interacting with the Hadoop Yarn Resource Manager. The `YarnProvisioner` then _discovers_ the location of where the kernel (Spark driver) was scheduled to run to complete the kernel's startup. -- `scripts/launch_ipykernel.py` - the "launcher" for the IPyKernel kernel (or subclasses thereof). This file is typically +- `scripts/launch_ipykernel.py` - the "launcher" for the IPyKernel kernel (or subclasses thereof). This file is typically implemented in the language of the kernel and is responsible for creating the local connection information, asynchronously starting a SparkContext (if asked), spawning a listener process to receive interrupts and shutdown requests, and starting the IPyKernel itself. diff --git a/docs/source/operators/installing-gp-container.md b/docs/source/operators/installing-gp-container.md index 61ea0b5..8172029 100644 --- a/docs/source/operators/installing-gp-container.md +++ b/docs/source/operators/installing-gp-container.md @@ -1,7 +1,7 @@ # Installing Gateway Provisioners (Containers) -These instructions are relative to the host application _image_. For configuring kernel images for use with Gateway -Provisioners see [Installing Supported Kernels (Containers)](installing-kernels-container.md). For instructions on installing +These instructions are relative to the host application _image_. For configuring kernel images for use with Gateway +Provisioners see [Installing Supported Kernels (Containers)](installing-kernels-container.md). For instructions on installing Gateway Provisioners on host application _servers_, please see [Installing Gateway Provisioners (Servers)](installing-gp.md). ```{attention} @@ -53,7 +53,7 @@ RUN mamba install -c conda-forge gateway_provisioners[docker] ## Mixed Environments If you are unsure in which environment this image will be used, you can install both Kubernetes and Docker client -libraries. In addition, in some cases, you may wish to support Yarn from within a Kubernetes or Docker cluster, in +libraries. In addition, in some cases, you may wish to support Yarn from within a Kubernetes or Docker cluster, in which case, you may even wish to add its client libraries as well: ```dockerfile diff --git a/docs/source/operators/installing-gp.md b/docs/source/operators/installing-gp.md index d5c5dfa..6cb0ef9 100644 --- a/docs/source/operators/installing-gp.md +++ b/docs/source/operators/installing-gp.md @@ -1,6 +1,6 @@ # Installing Gateway Provisioners (Servers) -These instructions are relative to the host application _server_. For _container-based_ installations, see +These instructions are relative to the host application _server_. For _container-based_ installations, see [Installing Gateway Provisioners (Containers)](installing-gp-container.md). ```{attention} diff --git a/docs/source/operators/installing-kernels-container.md b/docs/source/operators/installing-kernels-container.md index 1208fda..db65f1b 100644 --- a/docs/source/operators/installing-kernels-container.md +++ b/docs/source/operators/installing-kernels-container.md @@ -7,7 +7,7 @@ Gateway Provisioners includes tooling to generate kernel specifications that sup - IRKernel (R) Kernel specifications reside within the host application image (or volume mount) and their generation is addressed -in the applicable sections for [Kubernetes](deploy-kubernetes.md) and [Docker/DockerSwarm](deploy-docker.md). What +in the applicable sections for [Kubernetes](deploy-kubernetes.md) and [Docker/DockerSwarm](deploy-docker.md). What follows are instructions for how to build the kernel-based image. ```{tip} @@ -19,7 +19,7 @@ these tasks. ## Kernel-image Dockerfiles There are two forms of kernel images that can be created, one for regular or _vanilla_ kernels, another -for Spark-based kernels. Both use the same Dockerfile, but Spark-based images will specify a different base image +for Spark-based kernels. Both use the same Dockerfile, but Spark-based images will specify a different base image from which the target image is derived. ### Spark Base Image @@ -27,7 +27,7 @@ from which the target image is derived. Spark-based images will be built upon the `elyra/gp-spark-base` image, whose [`Dockerfile`](https://github.com/jupyter-server/gateway_provisioners/tree/main/gateway_provisioners/docker/gp-spark-base/Dockerfile) is located in [`gateway_provisioners/docker/gp-spark-base`](https://github.com/jupyter-server/gateway_provisioners/tree/main/gateway_provisioners/docker/gp-spark-base). -This image builds on `jupyter/docker-stacks-foundation:2022-11-15` by installing Spark. However, that base can also be +This image builds on `jupyter/docker-stacks-foundation:2022-11-15` by installing Spark. However, that base can also be substituted via the build argument `BASE_CONTAINER`. To build the spark base image use: @@ -36,8 +36,8 @@ To build the spark base image use: make gp-spark-base ``` -This will create an image named, by default, `elyra/gp-spark-base:dev`. The organization (`elyra`) and tag (`dev`) can -be controlled using variables `DOCKER_ORG` and `TAG`, respectively. For example, to create a `gp-spark-base` image +This will create an image named, by default, `elyra/gp-spark-base:dev`. The organization (`elyra`) and tag (`dev`) can +be controlled using variables `DOCKER_ORG` and `TAG`, respectively. For example, to create a `gp-spark-base` image with a custom organization and tag of the form `my-custom-org/gp-spark-base:my-custom-tag`, you can use: ```bash @@ -58,13 +58,13 @@ used to create a kernel image representing any of the supported kernels is locat [`gateway_provisioners/docker/kernel-image`](https://github.com/jupyter-server/gateway_provisioners/tree/main/gateway_provisioners/docker/kernel-image). This is a [_multi-stage build_](https://docs.docker.com/build/building/multi-stage/) Dockerfile whose build options are -driven by [`docker build` arguments](https://docs.docker.com/engine/reference/builder/#arg). There are three primary +driven by [`docker build` arguments](https://docs.docker.com/engine/reference/builder/#arg). There are three primary arguments that drive a kernel's image build, `BASE_CONTAINER`, `KERNEL_LANG`, and `PACKAGE_SOURCE`. #### `BASE_CONTAINER` Using a `BASE_CONTAINER` build argument (e.g., `--build-arg BASE_CONTAINER=xxx`) controls from which image the kernel -image will be derived. By default, _vanilla_ kernel images are derived from `jupyter/docker-stacks-foundation:2022-11-15` +image will be derived. By default, _vanilla_ kernel images are derived from `jupyter/docker-stacks-foundation:2022-11-15` while _spark-based_ kernel images should specify a `BASE_CONTAINER` of either `elyra/gp-spark-base:dev` or another applicable image. Gateway Provisioner's `Makefile` supports targets for three spark-based images: `gp-kernel-spark-py`, @@ -90,8 +90,8 @@ The _helper-target_ `kernel-images` can be used to create all five kernel images #### `KERNEL_LANG` The second primary build argument to consider is the kernel language. Each of the previously mentioned Makefile targets -automatically set `--build-arg KERNEL_LANG=` to the expected argument. If not specified, `KERNEL_LANG` defaults -to `python`. `KERNEL_LANG` must be one of the following values: `python`, `r`, or `scala`. +automatically set `--build-arg KERNEL_LANG=` to the expected argument. If not specified, `KERNEL_LANG` defaults +to `python`. `KERNEL_LANG` must be one of the following values: `python`, `r`, or `scala`. It is this build argument that determines which kernel package to install, all of which is handled in the `Dockerfile`. @@ -100,12 +100,12 @@ or `Scala`, as well as a `LABEL` entry of `KERNEL_LANG` with the respective lowe #### `PACKAGE_SOURCE` -The third primary build argument is `PACKAGE_SOURCE`. This identifies from where the Gateway Provisioners installation -should come. By default, `PACKAGE_SOURCE=release`, meaning that the Gateway Provisioners package will be installed -from the latest release. If a build argument for `PACKAGE_SOURCE` specifies `local`, then the -locally built wheel file for Gateway Provisioners will be installed. This option is ideal for development environments. +The third primary build argument is `PACKAGE_SOURCE`. This identifies from where the Gateway Provisioners installation +should come. By default, `PACKAGE_SOURCE=release`, meaning that the Gateway Provisioners package will be installed +from the latest release. If a build argument for `PACKAGE_SOURCE` specifies `local`, then the +locally built wheel file for Gateway Provisioners will be installed. This option is ideal for development environments. -Because the `Makefile` is _plumbed_ for `PACKAGE_SOURCE`, it can be specified directly in the `make` command. For example, +Because the `Makefile` is _plumbed_ for `PACKAGE_SOURCE`, it can be specified directly in the `make` command. For example, the following will build a vanilla kernel for python using a locally built wheel file: ```bash diff --git a/docs/source/users/connecting-to-a-gateway.md b/docs/source/users/connecting-to-a-gateway.md index 74e0358..7030cd6 100644 --- a/docs/source/users/connecting-to-a-gateway.md +++ b/docs/source/users/connecting-to-a-gateway.md @@ -75,7 +75,7 @@ where the kernel will run. ### Creating a Papermill Engine Connecting Papermill to a Gateway actually leverages the same `KernelManager` that is used when JupyterLab connects to -a Gateway server. However, to set the class of a `KernelManager` to `jupyter_server.gateway.managers.GatewayKernelManager` +a Gateway server. However, to set the class of a `KernelManager` to `jupyter_server.gateway.managers.GatewayKernelManager` one must implement, and register (via entrypoints), a ["Papermill engine"](https://papermill.readthedocs.io/en/latest/extending-entry-points.html#developing-a-new-engine). Once the engine is implemented and set to use the `GatewayKernelManager` class, Papermill can be invoked using: diff --git a/gateway_provisioners/_version.py b/gateway_provisioners/_version.py index 0d244b1..285d24c 100644 --- a/gateway_provisioners/_version.py +++ b/gateway_provisioners/_version.py @@ -1,7 +1,6 @@ # Copyright (c) Jupyter Development Team. # Distributed under the terms of the Modified BSD License. import re -from typing import List # Version string must appear intact for automatic versioning __version__ = "0.5.0.dev0" @@ -10,7 +9,7 @@ pattern = r"(?P\d+).(?P\d+).(?P\d+)(?P.*)" match = re.match(pattern, __version__) assert match is not None -parts: List[object] = [int(match[part]) for part in ["major", "minor", "patch"]] +parts: list[object] = [int(match[part]) for part in ["major", "minor", "patch"]] if match["rest"]: parts.append(match["rest"]) version_info = tuple(parts) diff --git a/gateway_provisioners/app-support/README.md b/gateway_provisioners/app-support/README.md index 4ee4965..d9ccc1b 100644 --- a/gateway_provisioners/app-support/README.md +++ b/gateway_provisioners/app-support/README.md @@ -1,6 +1,6 @@ # Application Support -Welcome to Gateway Provisioner's application support page. Here, we document helpful +Welcome to Gateway Provisioner's application support page. Here, we document helpful approaches for how to deploy _applications_ that utilize Gateway Provisioners. This information is primarily suited to container-based environments (e.g., Docker & @@ -26,7 +26,7 @@ The Dockerfile located in docker/Dockerfile contains a multi-stage build that: ### Building Application Images -Support for building application images is built into the `Makefile`. The following targets +Support for building application images is built into the `Makefile`. The following targets apply to application builds: #### `app-images` @@ -44,12 +44,12 @@ Use `make gp-lab` to build only `elyra/gp-lab:dev` containing Jupyter Lab. #### `clean-app-images` -Use `make clean-app-images` to remove BOTH `elyra/gp-jkg:dev` and `elyra/gp-lab:dev`. Individual +Use `make clean-app-images` to remove BOTH `elyra/gp-jkg:dev` and `elyra/gp-lab:dev`. Individual images can be cleaned using either `make clean-gp-jkg` or `make clean-gp-lab`. #### `push-app-images` -Use `make push-app-images` to push BOTH `elyra/gp-jkg:dev` and `elyra/gp-lab:dev` to dockerhub. Individual +Use `make push-app-images` to push BOTH `elyra/gp-jkg:dev` and `elyra/gp-lab:dev` to dockerhub. Individual images can be pushed using either `make push-gp-jkg` or `make push-gp-lab`. #### Available Adjustments @@ -59,29 +59,29 @@ there are build arguments that can be specified in the `make` command that influ ##### `BASE_CONTAINER` -`BASE_CONTAINER` can be specified to override the base container upon which the application is built. By +`BASE_CONTAINER` can be specified to override the base container upon which the application is built. By default, the application images are built upon `elyra/gp-spark-base` since they will require Spark. ##### `PACKAGE_SOURCE` -`PACKAGE_SOURCE` supports two values `release` and `local`. A value of `release` indicates that +`PACKAGE_SOURCE` supports two values `release` and `local`. A value of `release` indicates that the gateway_provisioners package be installed from PyPi.org using `pip` - thereby installing the -latest release. While a value of `local` indicates that the local wheel distribution be installed. +latest release. While a value of `local` indicates that the local wheel distribution be installed. ##### `SERVER_APP` `SERVER_APP` can take one of two values (`jkg` or `lab`) to identify which application should be installed -and invoked. A value of `jkg` indicates that `jupyter-kernel-gateway` be installed and `jupyter-kernelgateway` -be invoked with a display name of `Jupyter Kernel Gateway`. While a value of `lab` indicates that `jupyterlab` +and invoked. A value of `jkg` indicates that `jupyter-kernel-gateway` be installed and `jupyter-kernelgateway` +be invoked with a display name of `Jupyter Kernel Gateway`. While a value of `lab` indicates that `jupyterlab` be installed and `jupyter-lab` be invoked with a display name of `Jupyter Lab`. ##### `DOCKER_ORG` -`DOCKER_ORG` can be specified to determine the docker organization. The default value is `elyra`. +`DOCKER_ORG` can be specified to determine the docker organization. The default value is `elyra`. ##### `TAG` -`TAG` can be specified to determine the tag to apply to the image. The default is `dev` when the +`TAG` can be specified to determine the tag to apply to the image. The default is `dev` when the Gateway Provisioners version indicates a development version cycle (e.g., `0.2.0.dev0`) or the actual version value if the there is no `devN` suffix. @@ -91,18 +91,18 @@ Deployment into Kubernetes environments first requires the application reside wi Once an image exists, deployment consists of updating the `values.yaml` file and deploying the helm chart. Much of the information (like environment variables) in the docker image can be overridden -in the helm chart. For example, the list of allowed kernelspecs is specified in the `Dockerfile` +in the helm chart. For example, the list of allowed kernelspecs is specified in the `Dockerfile` via the `APP_ALLOWED_KERNELS` environment variable and reflects the _docker-based_ kernel specifications. However, the `APP_ALLOWED_KERNELS` in the helm charts will be overridden to reflect the _kubernetes-based_ kernel specifications. The primary portion of [`values.yaml`](https://github.com/jupyter-server/gateway_provisioners/tree/main/gateway_provisioners/app-support/kubernetes/helm/gateway-provisioners/values.yaml) -that applies to application-specific information is in the `application` stanza. Here, things +that applies to application-specific information is in the `application` stanza. Here, things like the application name, command, and image can be specified, in addition to the allowed kernels and culling intervals, etc. Other values that are more applicable to the _application_ rather than the _kernel_ are located -in the `provisioner` stanza. These are values that are used by Gateway Provisioners _within_ +in the `provisioner` stanza. These are values that are used by Gateway Provisioners _within_ the application. ### Deploying Kubernetes Applications @@ -117,7 +117,7 @@ that value is up to you: kubectl create namespace gateway-provisioners ``` -Next, you are ready to deploy the application. From the repo's root, issue the following (note: +Next, you are ready to deploy the application. From the repo's root, issue the following (note: we're using a deployment name of `gateway-provisioners`: ```bash diff --git a/gateway_provisioners/cli/docker_specapp.py b/gateway_provisioners/cli/docker_specapp.py index 61799fc..8d2ad2b 100644 --- a/gateway_provisioners/cli/docker_specapp.py +++ b/gateway_provisioners/cli/docker_specapp.py @@ -89,7 +89,7 @@ def _image_name_default(self): def detect_missing_extras(self): super().detect_missing_extras() try: - import docker # type:ignore[import-untyped] # noqa: F401 + import docker # noqa: F401 except ImportError: self.log.warning( "The extra package 'docker' is not installed in this environment and is required. " diff --git a/gateway_provisioners/crd.py b/gateway_provisioners/crd.py index 5e50caa..c401407 100644 --- a/gateway_provisioners/crd.py +++ b/gateway_provisioners/crd.py @@ -5,7 +5,7 @@ import logging import re from contextlib import suppress -from typing import Any, Dict, List, Optional, Set +from typing import Any, Optional from overrides import overrides @@ -38,7 +38,7 @@ def __init__(self, **kwargs): self.kernel_resource_name = None @overrides - async def pre_launch(self, **kwargs: Any) -> Dict[str, Any]: + async def pre_launch(self, **kwargs: Any) -> dict[str, Any]: """Launch the process for a kernel.""" kwargs = await super().pre_launch(**kwargs) self.kernel_resource_name = self._determine_kernel_pod_name(**kwargs) @@ -105,7 +105,7 @@ def get_container_status(self, iteration: Optional[str]) -> str: return application_state @overrides - def delete_managed_object(self, termination_stati: List[str]) -> bool: + def delete_managed_object(self, termination_stati: list[str]) -> bool: """Deletes the object managed by this provisioner A return value of True indicates the object is considered deleted, @@ -128,7 +128,7 @@ def delete_managed_object(self, termination_stati: List[str]) -> bool: return result @overrides - def get_initial_states(self) -> Set[str]: + def get_initial_states(self) -> set[str]: """Return list of states in lowercase indicating container is starting (includes running).""" return {"submitted", "pending", "running"} diff --git a/gateway_provisioners/docker_swarm.py b/gateway_provisioners/docker_swarm.py index 14cb941..2acfedf 100644 --- a/gateway_provisioners/docker_swarm.py +++ b/gateway_provisioners/docker_swarm.py @@ -5,15 +5,15 @@ import logging import os -from typing import Any +from typing import Any, cast from overrides import overrides try: - from docker.client import DockerClient # type:ignore[import-untyped] - from docker.errors import NotFound # type:ignore[import-untyped] - from docker.models.containers import Container # type:ignore[import-untyped] - from docker.models.services import Service # type:ignore[import-untyped] + from docker.client import DockerClient + from docker.errors import NotFound + from docker.models.containers import Container + from docker.models.services import Service except ImportError: logging.warning( "The extra package 'docker' is not installed in this environment and is required. " @@ -94,7 +94,7 @@ def terminate_container_resources(self, restart: bool = False) -> bool | None: result = True # We'll be optimistic service = self._get_service() - if service: + if service is not None: try: service.remove() # Service still exists, attempt removal except Exception as err: @@ -138,7 +138,7 @@ def _get_service(self) -> Service: else: service = services[0] self.container_name = service.name - return service + return cast(Service, service) def _get_task(self) -> dict | None: """ @@ -147,7 +147,7 @@ def _get_task(self) -> dict | None: """ task = None service = self._get_service() - if service: + if service is not None: tasks = service.tasks(filters={"desired-state": "running"}) num_tasks = len(tasks) if num_tasks != 1: @@ -193,7 +193,7 @@ def get_container_status(self, iteration: str | None) -> str: container_status = "" container = self._get_container() - if container: + if container is not None: self.container_name = container.name if container.status: container_status = container.status.lower() @@ -201,8 +201,8 @@ def get_container_status(self, iteration: str | None) -> str: # Container is running, capture IP # we'll use this as a fallback in case we don't find our network - self.assigned_ip = container.attrs.get("NetworkSettings").get("IPAddress") - networks = container.attrs.get("NetworkSettings").get("Networks") + self.assigned_ip = container.attrs.get("NetworkSettings", {}).get("IPAddress") + networks = container.attrs.get("NetworkSettings", {}).get("Networks") if len(networks) > 0: self.assigned_ip = networks.get(docker_network).get("IPAddress") self.log.debug( @@ -231,7 +231,7 @@ def terminate_container_resources(self, restart: bool = False) -> bool: result = True # Since we run containers with remove=True, we'll be optimistic container = self._get_container() - if container: + if container is not None: try: container.remove(force=True) # Container still exists, attempt forced removal except Exception as err: @@ -276,4 +276,4 @@ def _get_container(self) -> Container: raise RuntimeError(err_msg) else: container = containers[0] - return container + return cast(Container, container) diff --git a/gateway_provisioners/k8s.py b/gateway_provisioners/k8s.py index 85aae8c..c3b36b1 100644 --- a/gateway_provisioners/k8s.py +++ b/gateway_provisioners/k8s.py @@ -5,7 +5,7 @@ import logging import os import re -from typing import Any, Dict, List, Optional, Set +from typing import Any, Optional import urllib3 from overrides import overrides @@ -74,7 +74,7 @@ def __init__(self, **kwargs): self.restarting = False @overrides - async def pre_launch(self, **kwargs: Any) -> Dict[str, Any]: + async def pre_launch(self, **kwargs: Any) -> dict[str, Any]: kwargs = await super().pre_launch(**kwargs) # Kubernetes relies on internal env variables to determine its configuration. When @@ -89,7 +89,7 @@ async def pre_launch(self, **kwargs: Any) -> Dict[str, Any]: return kwargs @overrides - async def get_provisioner_info(self) -> Dict[str, Any]: + async def get_provisioner_info(self) -> dict[str, Any]: provisioner_info = await super().get_provisioner_info() provisioner_info.update( { @@ -106,11 +106,11 @@ async def load_provisioner_info(self, provisioner_info: dict) -> None: self.delete_kernel_namespace = provisioner_info["delete_ns"] @overrides - def get_initial_states(self) -> Set[str]: + def get_initial_states(self) -> set[str]: return {"pending", "running"} @overrides - def get_error_states(self) -> Set[str]: + def get_error_states(self) -> set[str]: return {"failed"} @overrides @@ -144,7 +144,7 @@ def get_container_status(self, iteration: Optional[str]) -> str: return pod_status - def delete_managed_object(self, termination_stati: List[str]) -> bool: + def delete_managed_object(self, termination_stati: list[str]) -> bool: """Deletes the object managed by this provisioner A return value of True indicates the object is considered deleted, diff --git a/gateway_provisioners/kernel-launchers/docker/scripts/launch_docker.py b/gateway_provisioners/kernel-launchers/docker/scripts/launch_docker.py index e44db43..e2b63be 100644 --- a/gateway_provisioners/kernel-launchers/docker/scripts/launch_docker.py +++ b/gateway_provisioners/kernel-launchers/docker/scripts/launch_docker.py @@ -3,8 +3,8 @@ import sys import urllib3 -from docker.client import DockerClient # type:ignore[import-untyped] -from docker.types import EndpointSpec, RestartPolicy # type:ignore[import-untyped] +from docker.client import DockerClient +from docker.types import EndpointSpec, RestartPolicy urllib3.disable_warnings() diff --git a/gateway_provisioners/kernel-launchers/kubernetes/scripts/launch_kubernetes.py b/gateway_provisioners/kernel-launchers/kubernetes/scripts/launch_kubernetes.py index 1ac71ad..1f8ca01 100644 --- a/gateway_provisioners/kernel-launchers/kubernetes/scripts/launch_kubernetes.py +++ b/gateway_provisioners/kernel-launchers/kubernetes/scripts/launch_kubernetes.py @@ -5,7 +5,6 @@ import argparse import os import sys -from typing import List import urllib3 import yaml @@ -52,7 +51,7 @@ def extend_pod_env(pod_def: dict) -> dict: # Walk current set of template env entries and replace those found in the current # env with their values (and record those items). Then add all others from the env # that were not already. - processed_entries: List[str] = [] + processed_entries: list[str] = [] for item in env_stanza: item_name = item.get("name") if item_name in os.environ: diff --git a/gateway_provisioners/kernel-launchers/scala/toree-launcher/project/build.properties b/gateway_provisioners/kernel-launchers/scala/toree-launcher/project/build.properties index ac52722..d112f93 100644 --- a/gateway_provisioners/kernel-launchers/scala/toree-launcher/project/build.properties +++ b/gateway_provisioners/kernel-launchers/scala/toree-launcher/project/build.properties @@ -2,4 +2,4 @@ # Copyright (c) Jupyter Development Team. # Distributed under the terms of the Modified BSD License. # -sbt.version = 1.3.12 +sbt.version = 1.10.11 diff --git a/gateway_provisioners/kernel-launchers/shared/scripts/server_listener.py b/gateway_provisioners/kernel-launchers/shared/scripts/server_listener.py index ea180e8..bc95f72 100644 --- a/gateway_provisioners/kernel-launchers/shared/scripts/server_listener.py +++ b/gateway_provisioners/kernel-launchers/shared/scripts/server_listener.py @@ -141,7 +141,7 @@ def return_connection_info(self, comm_port) -> None: def prepare_comm_socket(self, comm_port) -> None: """Prepares the socket to which the server will send signal and shutdown requests.""" sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - sock.bind(("0.0.0.0", comm_port)) + sock.bind(("0.0.0.0", comm_port)) # noqa: S104 self.comm_socket = sock logger.info( f"Signal socket bound to host: " diff --git a/gateway_provisioners/remote_provisioner.py b/gateway_provisioners/remote_provisioner.py index c221bb4..03fba0d 100644 --- a/gateway_provisioners/remote_provisioner.py +++ b/gateway_provisioners/remote_provisioner.py @@ -13,9 +13,9 @@ from abc import abstractmethod from enum import Enum from socket import AF_INET, SHUT_WR, SOCK_STREAM, socket, timeout -from typing import Any, Dict, List, Optional, Tuple +from typing import Any, Optional, cast -import pexpect # type:ignore[import-untyped] +import pexpect from jupyter_client import ( KernelConnectionInfo, KernelProvisionerBase, @@ -109,7 +109,7 @@ def has_process(self) -> bool: pass @overrides - async def pre_launch(self, **kwargs: Any) -> Dict[str, Any]: + async def pre_launch(self, **kwargs: Any) -> dict[str, Any]: self.response_manager.register_event(self.kernel_id) cmd = self.kernel_spec.argv # Build launch command, provide substitutions @@ -144,7 +144,7 @@ def from_ns(match): return kwargs @overrides - async def launch_kernel(self, cmd: List[str], **kwargs: Any) -> KernelConnectionInfo: + async def launch_kernel(self, cmd: list[str], **kwargs: Any) -> KernelConnectionInfo: launch_kwargs = RemoteProvisionerBase._scrub_kwargs(kwargs) self.log.debug(f"RemoteProvisionerBase.launch_kernel() env: {launch_kwargs.get('env', {})}") self.local_proc = gp_launch_kernel(cmd, **launch_kwargs) @@ -217,7 +217,7 @@ async def shutdown_requested(self, restart: bool = False) -> None: await self.shutdown_listener(restart) @overrides - async def get_provisioner_info(self) -> Dict[str, Any]: + async def get_provisioner_info(self) -> dict[str, Any]: provisioner_info = await super().get_provisioner_info() provisioner_info.update( { @@ -250,7 +250,7 @@ def get_shutdown_wait_time(self, recommended: float = 5.0) -> float: # type;ign return recommended @overrides - def _finalize_env(self, env: Dict[str, str]) -> None: + def _finalize_env(self, env: dict[str, str]) -> None: # add the applicable kernel_id and language to the env dict env["KERNEL_ID"] = self.kernel_id @@ -265,9 +265,9 @@ def _finalize_env(self, env: Dict[str, str]) -> None: env.pop(k, None) @staticmethod - def _scrub_kwargs(kwargs: Dict[str, Any]) -> Dict[str, Any]: + def _scrub_kwargs(kwargs: dict[str, Any]) -> dict[str, Any]: """Remove any keyword arguments that Popen does not tolerate.""" - keywords_to_scrub: List[str] = ["extra_arguments", "kernel_id"] + keywords_to_scrub: list[str] = ["extra_arguments", "kernel_id"] scrubbed_kwargs = kwargs.copy() for kw in keywords_to_scrub: scrubbed_kwargs.pop(kw, None) @@ -275,7 +275,7 @@ def _scrub_kwargs(kwargs: Dict[str, Any]) -> Dict[str, Any]: return scrubbed_kwargs @abstractmethod - def log_kernel_launch(self, cmd: List[str]) -> None: + def log_kernel_launch(self, cmd: list[str]) -> None: """Logs the kernel launch from the respective remote provisioner""" pass @@ -486,7 +486,7 @@ def _raise_authorization_error(self, differentiator_clause): ) self.log_and_raise(PermissionError(error_message)) - def _validate_port_range(self) -> Tuple[int, int]: + def _validate_port_range(self) -> tuple[int, int]: """Validates the port range configuration option to ensure appropriate values.""" lower_port = upper_port = 0 @@ -854,9 +854,11 @@ def _spawn_ssh_tunnel( remote_port, server, ) - return pexpect.spawn(cmd, env=os.environ.copy().pop("SSH_ASKPASS", None)) + env = os.environ.copy() + env.pop("SSH_ASKPASS", None) + return pexpect.spawn(cmd, env=cast(os._Environ, env)) - def _select_ports(self, count: int) -> List[int]: + def _select_ports(self, count: int) -> list[int]: """ Selects and returns n random ports that adhere to the configured port range, if applicable. @@ -869,8 +871,8 @@ def _select_ports(self, count: int) -> List[int]: ------- List - ports available and adhering to the configured port range """ - ports: List[int] = [] - sockets: List[socket] = [] + ports: list[int] = [] + sockets: list[socket] = [] for _i in range(count): sock = self._select_socket() ports.append(sock.getsockname()[1]) diff --git a/pyproject.toml b/pyproject.toml index e6ba760..bdb5731 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -14,7 +14,7 @@ classifiers = [ "Programming Language :: Python", "Programming Language :: Python :: 3" ] -requires-python = ">=3.8" +requires-python = ">=3.9" dependencies = [ "jupyter_client>=7.4", "overrides", @@ -174,7 +174,7 @@ post-version-spec = "dev" [tool.mypy] files = "gateway_provisioners" exclude = "gateway_provisioners/kernel-launchers/shared/scripts/server_listener.py" -python_version = "3.8" +python_version = "3.9" strict = true show_error_codes = true enable_error_code = ["ignore-without-code", "redundant-expr", "truthy-bool"] @@ -202,10 +202,10 @@ filterwarnings= [ [tool.black] line-length = 100 -target-version = ["py38"] +target-version = ["py39"] [tool.ruff] -target-version = "py38" +target-version = "py39" line-length = 100 select = [ "A", "B", "C", "E", "EM", "F", "FBT", "I", "N", "Q", "RUF", "S", "T",