Demonstrate working with private wheel packages (#47)

There is no native support for private repositories and this offers a workaround.

Author: pietern

knowledge_base/private_wheel_packages/.gitignore

@@ -0,0 +1,54 @@
+# Private wheel packages
+
+This example demonstrates how to use a private wheel package from a job in a Databricks Asset Bundle.
+
+If you are using notebooks, you can use the approach documented in [Notebook-scoped Python libraries][doc] to install
+wheels from a private repository in a notebook. You can use the workaround documented here if you are not using notebooks.
+
+[doc]: https://docs.databricks.com/en/libraries/notebooks-python-libraries.html#install-a-private-package-with-credentials-managed-by-databricks-secrets-with-pip
+
+## Prerequisites
+
+* Databricks CLI v0.235.0 or above
+* Python 3.10 or above
+
+# Usage
+
+You can refer to private wheel files from job libraries or serverless environments by downloading the wheel
+and making it part of your Databricks Asset Bundle deployment.
+
+To emulate this for this example, we will download a wheel from PyPI, include it in deployment, and refer to it from job configuration.
+
+## Downloading a wheel
+
+First, download the wheel to the `dist` directory:
+
+```shell
+pip download -d dist cowsay==6.1
+```
+
+## Deploying the example
+
+Next, update the `host` field under `workspace` in `databricks.yml` to the Databricks workspace you wish to deploy to.
+
+Run `databricks bundle deploy` to upload the wheel and deploy the jobs.
+
+Run `databricks bundle run` to run either job.
+
+Example output:
+```
+$ databricks bundle run
+Run URL: https://...
+
+2024-11-27 13:23:01 "[dev pieter_noordhuis] Example to demonstrate using a private wheel package on serverless" TERMINATED SUCCESS
+  _____________
+| Hello, world! |
+  =============
+             \
+              \
+                ^__^
+                (oo)\_______
+                (__)\       )\/\
+                    ||----w |
+                    ||     ||
+```

knowledge_base/private_wheel_packages/README.md

@@ -0,0 +1,13 @@
+bundle:
+  name: private_wheel_packages
+
+include:
+  - resources/*.yml
+
+workspace:
+  # host: https://myworkspace.cloud.databricks.com
+
+targets:
+  dev:
+    default: true
+    mode: development

knowledge_base/private_wheel_packages/databricks.yml

@@ -0,0 +1 @@
+*

knowledge_base/private_wheel_packages/dist/.gitignore

@@ -0,0 +1,35 @@
+resources:
+  jobs:
+    cluster:
+      name: Example to demonstrate using a private wheel package on a job cluster
+
+      tasks:
+        - task_key: task
+          job_cluster_key: default
+
+          spark_python_task:
+            python_file: ../src/main.py
+
+          # Relative local path reference to the private wheel package.
+          # This references alone is sufficient to upload it on deployment.
+          #
+          # It is uploaded to `${workspace.artifact_path}`, which can point to
+          # a workspace directory (under `${workspace.bundle_root}`, by default),
+          # or a Unity Catalog Volume.
+          #
+          # This value is automatically rewritten to the fully qualified remote path.
+          libraries:
+            - whl: ../dist/cowsay-6.1-*.whl
+
+      job_clusters:
+        - job_cluster_key: default
+          new_cluster:
+            spark_version: 15.4.x-scala2.12
+            node_type_id: i3.xlarge
+            data_security_mode: SINGLE_USER
+            num_workers: 0
+            spark_conf:
+                "spark.databricks.cluster.profile": "singleNode"
+                "spark.master": "local[*]"
+            custom_tags:
+                "ResourceClass": "SingleNode"

knowledge_base/private_wheel_packages/resources/cluster.job.yml

@@ -0,0 +1,27 @@
+resources:
+  jobs:
+    serverless:
+      name: Example to demonstrate using a private wheel package on serverless
+
+      tasks:
+        - task_key: task
+          environment_key: default
+
+          spark_python_task:
+            python_file: ../src/main.py
+
+      environments:
+        - environment_key: default
+
+          spec:
+            client: "1"
+            dependencies:
+              # Relative local path reference to the private wheel package.
+              # This references alone is sufficient to upload it on deployment.
+              #
+              # It is uploaded to `${workspace.artifact_path}`, which can point to
+              # a workspace directory (under `${workspace.bundle_root}`, by default),
+              # or a Unity Catalog Volume.
+              #
+              # This value is automatically rewritten to the fully qualified remote path.
+              - ../dist/cowsay-6.1-*.whl

knowledge_base/private_wheel_packages/resources/serverless.job.yml

@@ -0,0 +1,6 @@
+# By successfully importing the previously downloaded wheel, we demonstrate that it is possible to
+# include an arbitrary wheel file in your bundle directory, deploy it, and use it from within your code.
+import cowsay
+
+if __name__ == '__main__':
+    cowsay.cow('Hello, world!')