Building MloPS + Data Science Scripts

Author: ciaran28

.dbx/lock.json

@@ -1,3 +1,3 @@
-{
-    "context_id": "6505588973318184615"
+{
+    "context_id": "6291623212386950857"
 }

.dbx/project.json

@@ -1,20 +1,20 @@
-{
-    "environments": {
-        "default": {
-            "profile": "DEFAULT",
-            "storage_type": "mlflow",
-            "properties": {
-                "workspace_directory": "/Shared/cicd_dbx_workflows",
-                "artifact_location": "dbfs:/Shared/cicd_dbx_workflows"
-            }
-        },
-        "ciaran_sandbox": {
-            "profile": "ciaran_sandbox",
-            "workspace_dir": "/Shared/ciaran_sandbox",
-            "artifact_location": "dbfs:/Shared/cicd_workflows/ciaran_sandbox"
-        }
-    },
-    "inplace_jinja_support": true,
-    "failsafe_cluster_reuse_with_assets": false,
-    "context_based_upload_for_execute": false
+{
+    "environments": {
+        "default": {
+            "profile": "DEFAULT",
+            "storage_type": "mlflow",
+            "properties": {
+                "workspace_directory": "/Shared/dbx/projects/dstoolkit-mlops-databricks",
+                "artifact_location": "dbfs:/dbx/dstoolkit-mlops-databricks"
+            }
+        },
+        "ciaran_sandbox": {
+            "profile": "ciaran_sandbox",
+            "workspace_dir": "/Shared/ciaran_sandbox",
+            "artifact_location": "dbfs:/Shared/cicd_workflows/ciaran_sandbox"
+        }
+    },
+    "inplace_jinja_support": true,
+    "failsafe_cluster_reuse_with_assets": false,
+    "context_based_upload_for_execute": false
 }

.devcontainer/Dockerfile

@@ -0,0 +1,18 @@
+# https://github.com/microsoft/vscode-dev-containers/blob/main/containers/python-3/.devcontainer/Dockerfile
+
+# [Choice] Python version (use -bullseye variants on local arm64/Apple Silicon): 3, 3.10, 3.9, 3.8, 3.7, 3.6, 3-bullseye, 3.10-bullseye, 3.9-bullseye, 3.8-bullseye, 3.7-bullseye, 3.6-bullseye, 3-buster, 3.10-buster, 3.9-buster, 3.8-buster, 3.7-buster, 3.6-buster
+
+ARG VARIANT="3.10"
+FROM mcr.microsoft.com/vscode/devcontainers/python:${VARIANT}
+
+ARG DEV_DATABRICKS_TOKEN
+
+COPY .databricks-connect.template /home/vscode/.databricks-connect
+
+RUN && sudo apt update \
+    && sudo apt-get install -y default-jre \
+    && pip install databricks-connect \
+    && pip install -U pip \
+    && sed -i "s/replacetoken/${DEV_DATABRICKS_TOKEN}/g" /home/vscode/.databricks-connect
+
+ENV SPARK_HOME /usr/local/lib/python3.10/site-packages/pyspark

.devcontainer/devcontainer.json

@@ -0,0 +1,52 @@
+// Config options: https://aka.ms/devcontainer.json
+
+// File example: https://github.com/microsoft/vscode-dev-containers/blob/main/containers/python-3/.devcontainer/devcontainer.json
+{
+
+	"name": "Python 3",
+	"build": {
+	  "dockerfile": "Dockerfile",
+	  "context": "..",
+	  "args": {
+		// Update 'VARIANT' to pick a Python version: 3, 3.10, 3.9, 3.8, 3.7, 3.6
+		// Append -bullseye or -buster to pin to an OS version.
+		// Use -bullseye variants on local on arm64/Apple Silicon.
+		"VARIANT": "3.10-bullseye",
+		"DEV_DATABRICKS_TOKEN": "${localEnv:DEV_DATABRICKS_TOKEN}"
+	  }
+	},
+  
+	// Configure tool-specific properties.
+	"customizations": {
+	  // Configure properties specific to VS Code.
+	  "vscode": {
+		// Set *default* container specific settings.json values on container create.
+		"settings": {
+		  "python.defaultInterpreterPath": "/usr/local/bin/python",
+		  "python.linting.enabled": true,
+		  "python.linting.pylintEnabled": true,
+		  "python.formatting.autopep8Path": "/usr/local/py-utils/bin/autopep8",
+		  "python.formatting.blackPath": "/usr/local/py-utils/bin/black",
+		  "python.formatting.yapfPath": "/usr/local/py-utils/bin/yapf",
+		  "python.linting.banditPath": "/usr/local/py-utils/bin/bandit",
+		  "python.linting.flake8Path": "/usr/local/py-utils/bin/flake8",
+		  "python.linting.mypyPath": "/usr/local/py-utils/bin/mypy",
+		  "python.linting.pycodestylePath": "/usr/local/py-utils/bin/pycodestyle",
+		  "python.linting.pydocstylePath": "/usr/local/py-utils/bin/pydocstyle",
+		  "python.linting.pylintPath": "/usr/local/py-utils/bin/pylint"
+		},
+  
+		// Add the IDs of extensions you want installed when the container is created.
+		"extensions": ["ms-python.python", "ms-python.vscode-pylance"]
+	  }
+	},
+  
+	// Use 'forwardPorts' to make a list of ports inside the container available locally.
+	// "forwardPorts": [],
+  
+	// Use 'postCreateCommand' to run commands after the container is created.
+	// "postCreateCommand": "pip3 install --user -r requirements.txt",
+  
+	// Comment out to connect as root instead. More info: https://aka.ms/vscode-remote/containers/non-root.
+	"remoteUser": "vscode"
+  }

.gitignore

@@ -13,6 +13,8 @@ dev.env
 data_science/src_nyc_taxi/build
 data_science/src_nyc_taxi/dist
 data_science/src_nyc_taxi/nyc_taxi_src.egg-info
+.linux_venv
+experiments