liquibase generated from template test

.github/workflows/flights_liquibase.yml

@@ -0,0 +1,127 @@
+
+name: Deploy flights_project to TEST with liquibase
+
+concurrency: 1
+
+on:
+  workflow_dispatch:
+
+  pull_request:
+    types:
+      - opened
+      - synchronize
+    branches:
+      - dev
+    paths:
+      - "**/*.yml"
+      - "**/*.py"
+
+  push:
+    branches:
+      - test/liquibase*
+    paths:
+      - "**/*.yml"
+      - "**/*.py"
+      - "**/*.xml"
+      - "**/*.txt"
+
+jobs:
+  deploy:
+    name: "Test and deploy bundle"
+    runs-on: ubuntu-latest
+    environment: test
+    env:
+      DATABRICKS_HOST: ${{ vars.DATABRICKS_HOST }}
+      DATABRICKS_TOKEN: ${{ vars.DATABRICKS_TOKEN_TST }}
+      DATABRICKS_BUNDLE_ENV: dev  # should be TST, 'dev' is set for demo purposes only
+    defaults:
+      run:
+        working-directory: .
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Set up Python 3.12
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.12'
+
+      #----------------------------------------------
+      #         Set up Java (Temurin JDK 8)
+      #----------------------------------------------
+      - name: Set up Java 8
+        uses: actions/setup-java@v4
+        with:
+          java-version: '8'
+          distribution: 'temurin'
+
+      #----------------------------------------------
+      #    Download Liquibase CLI and make executable
+      #----------------------------------------------
+      - name: Download Liquibase CLI
+        run: |
+          LIQUIBASE_VERSION=4.33.0
+          wget https://github.com/liquibase/liquibase/releases/download/v${LIQUIBASE_VERSION}/liquibase-${LIQUIBASE_VERSION}.tar.gz
+          mkdir liquibase_download
+          tar -xzf liquibase-${LIQUIBASE_VERSION}.tar.gz -C liquibase_download
+
+      #----------------------------------------------
+      #   Download Databricks JDBC and extension JARs
+      #----------------------------------------------
+      - name: Download Databricks JDBC Driver and Liquibase Extension
+        run: |
+          # Download Databricks JDBC driver
+          wget https://databricks-bi-artifacts.s3.us-east-2.amazonaws.com/simbaspark-drivers/jdbc/2.7.3/DatabricksJDBC42-2.7.3.1010.zip
+          unzip DatabricksJDBC42-2.7.3.1010.zip
+          mv DatabricksJDBC-2.7.3.1010/DatabricksJDBC42.jar liquibase_download/lib
+
+          # Download Liquibase-Databricks extension
+          wget https://github.com/liquibase/liquibase-databricks/releases/download/v1.4.2/liquibase-databricks-1.4.2.jar
+          mv liquibase-databricks-1.4.2.jar liquibase_download/lib
+
+      #----------------------------------------------
+      #   Run Liquibase connection test & update
+      #----------------------------------------------
+      - name: Run Liquibase connection test & update
+        env:
+          LIQUIBASE_COMMAND_URL: ${{ vars.DATABRICKS_JDBC_URL }}
+          LIQUIBASE_COMMAND_USERNAME: "token"
+          LIQUIBASE_COMMAND_PASSWORD: ${{ secrets.DATABRICKS_TOKEN_TST }}
+          LIQUIBASE_COMMAND_DEFAULT_CATALOG_NAME: ${{ vars.LIQUIBASE_CATALOG_NAME }}
+          LIQUIBASE_COMMAND_DEFAULT_SCHEMA_NAME: ${{ vars.LIQUIBASE_SCHEMA_NAME }}
+        run: |
+          liquibase_download/liquibase update --changelog-file=./liquibase/root.changelog.xml --log-level INFO
+
+      #----------------------------------------------
+      #   Install dependencies and package
+      #----------------------------------------------
+      - run: python -m pip install --upgrade pip
+      - run: pip install -r ./requirements.txt -r ./dev-requirements.txt
+      - run: pip install -e .
+
+      #----------------------------------------------
+      #              run test suite
+      #----------------------------------------------
+      - name: Run unit tests
+        env:
+          DATABRICKS_HOST: ${{ vars.DATABRICKS_HOST }}
+          DATABRICKS_TOKEN: ${{ secrets.DATABRICKS_TOKEN_TST }}
+          DATABRICKS_SERVERLESS_COMPUTE_ID: "auto"
+        run: |
+          pytest ./tests/
+
+      - uses: databricks/setup-cli@main
+
+      #----------------------------------------------
+      #              run job (in staging)
+      #----------------------------------------------
+      - name: Run job (in staging)
+        env:
+          DATABRICKS_HOST: ${{ vars.DATABRICKS_HOST }}
+          DATABRICKS_TOKEN: ${{ secrets.DATABRICKS_TOKEN_TST }}
+          DATABRICKS_BUNDLE_ENV: dev  # should be TST, 'dev' is set for demo purposes only
+          BUNDLE_VAR_liquibase_catalog: ${{ vars.LIQUIBASE_CATALOG_NAME }}
+          BUNDLE_VAR_liquibase_schema: ${{ vars.LIQUIBASE_SCHEMA_NAME }}
+        run: |
+          databricks bundle deploy; databricks bundle run flights_notebook; databricks bundle destroy --auto-approve
+

databricks.yml

@@ -0,0 +1,110 @@
+# This is the configuration for the Databricks Asset Bundle flights-gen-bundle.
+
+bundle:
+  name: flights-gen-bundle
+
+
+
+include:
+  - resources/*.yml
+  - resources/dlt/*.yml
+
+variables:
+
+  liquibase_catalog:
+  liquibase_schema:
+
+  catalog:
+    default: ${var.liquibase_catalog}
+  database:
+    default: ${var.liquibase_schema}
+  flights_dlt_schema:
+    default: ${resources.schemas.project_schema.name}_dlt
+  flights_test_schema:
+    default: "${resources.schemas.project_schema.name}_validation"
+  spark_version:
+    default: "15.4.x-scala2.12"
+  node_type_id:
+    default: "m6gd.xlarge"
+  artifact_dir:
+    default: "/Workspace/Users/${workspace.current_user.userName}/shared_code/${bundle.target}"
+  shared_cluster_config:
+    type: "complex"
+    default:
+      spark_version: ${var.spark_version}
+      node_type_id: ${var.node_type_id}
+      data_security_mode: USER_ISOLATION
+      autoscale:
+          min_workers: 1
+          max_workers: 2
+
+targets:
+  # The 'dev' target, used for development purposes.
+  # Whenever a developer deploys using 'dev', they get their own copy.
+  dev:
+    # We use 'mode: development' to make sure everything deployed to this target gets a prefix
+    # like '[dev my_user_name]'. Setting this mode also disables any schedules and
+    # automatic triggers for jobs and enables the 'development' mode for Delta Live Tables pipelines.
+    mode: development
+    default: true
+
+    # RESOURCE OVERRIDE
+    ## Only create this schema in dev, for other targets manage with Terraform.
+    resources:
+      schemas:
+        project_schema:
+          name: ${var.database}
+          catalog_name: ${var.catalog}
+          comment: "Schema for flight data"
+
+  test:
+    # For test/staging deployments, we only have a single copy, so we should deploy as 
+    # the same user (Service Principal) ever time.
+    mode: production
+    workspace:
+      root_path: /Users/${workspace.current_user.userName}/.bundle/${bundle.name}/${bundle.target}
+    run_as:
+      user_name: ${workspace.current_user.userName}
+    variables:
+      database: flights_${bundle.target}
+      flights_dlt_schema: flights_dlt_${bundle.target}
+      flights_test_schema: flights_validation_test
+
+  test_automated:
+    # For test/staging deployments, we only have a single copy, so we should deploy as 
+    # the same user (Service Principal) ever time.
+    mode: production
+    workspace:
+      root_path: /Users/${workspace.current_user.userName}/.bundle/${bundle.name}/${bundle.target}
+    run_as:
+      user_name: ${workspace.current_user.userName}
+    variables:
+      database: flights_${bundle.target}
+      flights_dlt_schema: flights_dlt_${bundle.target}
+      flights_test_schema: flights_validation_test
+      node_type_id: Standard_DS3_v2
+
+  staging:
+    mode: production
+    workspace:
+      root_path: /Users/${workspace.current_user.userName}/.bundle/${bundle.name}/${bundle.target}
+    run_as:
+      user_name: ${workspace.current_user.userName}
+    variables:
+      database: flights_${bundle.target}
+      flights_dlt_schema: flights_dlt_${bundle.target}
+      flights_test_schema: flights_validation_staging
+
+  # The 'prod' target, used for production deployment.
+  prod:
+    mode: production
+    workspace:
+      root_path: /Users/${workspace.current_user.userName}/.bundle/${bundle.name}/${bundle.target}
+    run_as:
+      # This can run as a specific user or service principal in production.
+      # To run as service principal use service_principal_name (see Databricks documentation).
+      user_name: ${workspace.current_user.userName}
+    variables:
+      database: flights_${bundle.target}
+      flights_dlt_schema: flights_dlt_${bundle.target}
+      flights_test_schema: flights_validation_prod

dev-requirements.txt

@@ -0,0 +1,4 @@
+wheel
+pytest
+databricks-sdk[notebook]>=0.46.0
+pandas==2.2.3

flights/flights-bundle-template/README.md

@@ -3,6 +3,7 @@
 Template on top of the [flights_simple](../flights-simple) project to enable the following deployment options:
 1. Wheels or relative imports for the project's Python modules
 2. Serverless compute or classic compute for workflows
+3. Liquibase example setup
 
 ## Resources
 A subset of [flights_simple](../flights-simple) resources are currently demonstrated at the [template resources dir](template/resources/).
@@ -49,3 +50,20 @@ A subset of [flights_simple](../flights-simple) resources are currently demonstr
    ```
    $ databricks bundle run flights_notebook --profile <your CLI profile>
    ```
+
+## Liquibase
+If Liquibase is enabled, the template creates a GH action under `./.github/workflows/flights_liquibase.yml` which
+- Downloads the Liquibase CLI + required JDBC driver and jars
+- Runs a liquibase connection test and updates the changelog
+- Runs local tests
+- Deploys the bundle in target env and runs a job
+
+### GH setup
+Liquibase requires the following variables set in GitHub:
+- `DATABRICKS_HOST`: e.g. `dbc-eae35cd0-9e95.cloud.databricks.com`
+- `DATABRICKS_JDBC_URL`: DWH url e.g. `jdbc:databricks://dbc-eae35cd0-9e95.cloud.databricks.com:443/default;transportMode=http;ssl=1;AuthMech=3;httpPath=/sql/1.0/warehouses/863bcc7ff8fea25b;`
+- `LIQUIBASE_CATALOG_NAME`: a catalog name e.g. `main`
+- `LIQUIBASE_SCHEMA_NAME`: a schema name e.g. `liquibase_test`
+
+Plus the secret:
+- `DATABRICKS_TOKEN_TST`: Databricks token for deployment environment

flights/flights-bundle-template/databricks_template_schema.json

@@ -16,19 +16,25 @@
          "type": "boolean",
          "default": false,
          "description": "Do you want the Databricks workflows to run on serverless?",
-         "order": 2
+         "order": 3
       },
       "catalog": {
          "type": "string",
          "default": "main",
          "description": "Name of your UC catalog for your workflows/DLT?",
-         "order": 2
+         "order": 4
       },
       "database": {
          "type": "string",
          "default": "flights",
          "description": "Name of your UC database for your workflows/DLT?",
-         "order": 2
+         "order": 5
+      },
+      "include_liquibase": {
+         "type": "boolean",
+         "default": false,
+         "description": "Do you want to use liquibase for DataOps data evolution?",
+         "order": 6
       }
    },
    "success_message": "\nYour bundle '{{.project_name}}' has been created."