liquibase example with reduced version of flights-simple

.github/workflows/flights_liquibase.yml

@@ -0,0 +1,117 @@
+name: Deploy flights_project to TEST with liquibase
+
+concurrency: 1
+
+on:
+  workflow_dispatch:
+
+  pull_request:
+    types:
+      - opened
+      - synchronize
+    branches:
+      - dev
+    paths:
+      - "**/*.yml"
+      - "**/*.py"
+
+  push:
+    branches:
+      - feat/liquibase
+    paths:
+      - "**/*.yml"
+      - "**/*.py"
+      - "**/*.xml"
+
+jobs:
+  deploy:
+    name: "Test and deploy bundle"
+    runs-on: ubuntu-latest
+    environment: test
+    env:
+      DATABRICKS_HOST: ${{ secrets.DATABRICKS_HOST }}
+      DATABRICKS_TOKEN: ${{ secrets.DATABRICKS_TOKEN_TST }}
+      DATABRICKS_CLIENT_ID: ${{ secrets.DATABRICKS_CLIENT_ID }}
+      DATABRICKS_CLIENT_SECRET: ${{ secrets.DATABRICKS_CLIENT_SECRET }}
+      DATABRICKS_BUNDLE_ENV: tst
+    defaults:
+      run:
+        working-directory: .
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Set up Python 3.12
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.12'
+
+      #----------------------------------------------
+      #         Set up Java (Temurin JDK 8)
+      #----------------------------------------------
+      - name: Set up Java 8
+        uses: actions/setup-java@v4
+        with:
+          java-version: '8'
+          distribution: 'temurin'
+
+      #----------------------------------------------
+      #    Download Liquibase CLI and make executable
+      #----------------------------------------------
+      - name: Download Liquibase CLI
+        run: |
+          LIQUIBASE_VERSION=4.33.0
+          wget https://github.com/liquibase/liquibase/releases/download/v${LIQUIBASE_VERSION}/liquibase-${LIQUIBASE_VERSION}.tar.gz
+          mkdir liquibase
+          tar -xzf liquibase-${LIQUIBASE_VERSION}.tar.gz -C liquibase
+
+      #----------------------------------------------
+      #   Download Databricks JDBC and extension JARs
+      #----------------------------------------------
+      - name: Download Databricks JDBC Driver and Liquibase Extension
+        run: |
+          # Download Databricks JDBC driver
+          wget https://databricks-bi-artifacts.s3.us-east-2.amazonaws.com/simbaspark-drivers/jdbc/2.7.3/DatabricksJDBC42-2.7.3.1010.zip
+          unzip DatabricksJDBC42-2.7.3.1010.zip
+          mv DatabricksJDBC-2.7.3.1010/DatabricksJDBC42.jar liquibase/lib
+
+          # Download Liquibase-Databricks extension
+          wget https://github.com/liquibase/liquibase-databricks/releases/download/v1.4.2/liquibase-databricks-1.4.2.jar
+          mv liquibase-databricks-1.4.2.jar liquibase/lib
+
+      #----------------------------------------------
+      #   Run Liquibase connection test
+      #----------------------------------------------
+      - name: Run Liquibase connection test
+        env:
+          LIQUIBASE_COMMAND_URL: ${{ secrets.LIQUIBASE_COMMAND_URL }}  # alternative: store only the workspace url and compose the Liquibase URL
+          LIQUIBASE_COMMAND_USERNAME: "token"
+          LIQUIBASE_COMMAND_PASSWORD: ${{ secrets.DATABRICKS_TOKEN_TST }}
+        run: |
+          liquibase/liquibase update --changelog-file=flights/flights-liquibase/liquibase/root.changelog.xml --log-level INFO --defaultsFile=flights/flights-liquibase/liquibase.properties
+
+      #----------------------------------------------
+      #   Install dependencies and package
+      #----------------------------------------------
+      - run: python -m pip install --upgrade pip
+      - run: pip install -r flights/flights-liquibase/requirements.txt -r flights/flights-liquibase/dev-requirements.txt
+      - run: pip install -e flights/flights-liquibase/
+
+      #----------------------------------------------
+      #              run test suite
+      #----------------------------------------------
+      - name: Run unit tests
+        env:
+          DATABRICKS_HOST: ${{ secrets.DATABRICKS_HOST }}
+          DATABRICKS_TOKEN: ${{ secrets.DATABRICKS_TOKEN_TST }}
+          DATABRICKS_CLUSTER_ID: ${{ secrets.DATABRICKS_CLUSTER_ID }}
+        run: |
+          pytest flights/flights-liquibase/tests/ 
+
+      - uses: databricks/setup-cli@main
+
+#      - run: cd flights/flights-liquibase; databricks bundle deploy --target test_automated
+#        env:
+#          DATABRICKS_HOST: ${{ secrets.DATABRICKS_HOST }}
+#          DATABRICKS_TOKEN: ${{ secrets.DATABRICKS_TOKEN_TST }}
+#          DATABRICKS_BUNDLE_ENV: tst

flights/flights-liquibase/databricks.yml

@@ -0,0 +1,108 @@
+# This is a Databricks asset bundle definition for databricks-dab-examples.
+# See https://docs.databricks.com/dev-tools/bundles/index.html for documentation.
+bundle:
+  name: flights_simple
+
+include:
+  - resources/*.yml
+  - resources/dlt/*.yml
+
+variables:
+  catalog:
+    default: "main"
+  database:
+    default: ${resources.schemas.project_schema.name}
+  flights_dlt_schema:
+    default: ${resources.schemas.project_schema.name}_dlt
+  flights_test_schema:
+    default: "${resources.schemas.project_schema.name}_validation"
+  spark_version:
+    default: "15.3.x-scala2.12"
+  node_type_id:
+    default: "m6gd.xlarge"
+  shared_cluster_config:
+    type: "complex"
+    default:
+      spark_version: ${var.spark_version}
+      node_type_id: ${var.node_type_id}
+      data_security_mode: USER_ISOLATION
+      autoscale:
+          min_workers: 1
+          max_workers: 2
+
+targets:
+  # The 'dev' target, used for development purposes.
+  # Whenever a developer deploys using 'dev', they get their own copy.
+  dev:
+    # We use 'mode: development' to make sure everything deployed to this target gets a prefix
+    # like '[dev my_user_name]'. Setting this mode also disables any schedules and
+    # automatic triggers for jobs and enables the 'development' mode for Delta Live Tables pipelines.
+    mode: development
+    default: true
+    workspace:
+      host: https://e2-demo-field-eng.cloud.databricks.com/
+      root_path: /Users/${workspace.current_user.userName}/.bundle/${bundle.name}/${bundle.target}
+    # resource override
+    ## Only create this schema in dev, for other targets manage with Terraform.
+
+    resources:
+      schemas:
+        project_schema:
+          name: flights
+          catalog_name: ${var.catalog}
+          comment: "Schema for flight data"
+
+  test:
+    # For test/staging deployments, we only have a single copy, so we should deploy as 
+    # the same user (Service Principal) ever time.
+    mode: production
+    workspace:
+      host: https://company-tst.cloud.databricks.com/
+      root_path: /Users/${workspace.current_user.userName}/.bundle/${bundle.name}/${bundle.target}
+    run_as:
+      user_name: ${workspace.current_user.userName}
+    variables:
+      database: flights_${bundle.target}
+      flights_dlt_schema: flights_dlt_${bundle.target}
+      flights_test_schema: flights_validation_${bundle.target}
+
+  test_automated:
+    # For test/staging deployments, we only have a single copy, so we should deploy as 
+    # the same user (Service Principal) ever time.
+    mode: production
+    workspace:
+      root_path: /Users/${workspace.current_user.userName}/.bundle/${bundle.name}/${bundle.target}
+    run_as:
+      user_name: ${workspace.current_user.userName}
+    variables:
+      database: flights_${bundle.target}
+      flights_dlt_schema: flights_dlt_${bundle.target}
+      flights_test_schema: flights_validation_${bundle.target}
+      node_type_id: Standard_DS3_v2
+
+  staging:
+    mode: production
+    workspace:
+      host: https://company-stg.cloud.databricks.com/
+      root_path: /Users/${workspace.current_user.userName}/.bundle/${bundle.name}/${bundle.target}
+    run_as:
+      user_name: ${workspace.current_user.userName}
+    variables:
+      database: flights_${bundle.target}
+      flights_dlt_schema: flights_dlt_${bundle.target}
+      flights_test_schema: flights_validation_${bundle.target}
+
+  # The 'prod' target, used for production deployment.
+  prod:
+    mode: production
+    workspace:
+      host: https://company-prd.cloud.databricks.com/
+      root_path: /Users/${workspace.current_user.userName}/.bundle/${bundle.name}/${bundle.target}
+    run_as:
+      # This can run as a specific user or service principal in production.
+      # To run as service principal use service_principal_name (see Databricks documentation).
+      user_name: ${workspace.current_user.userName}
+    variables:
+      database: flights_${bundle.target}
+      flights_dlt_schema: flights_dlt_${bundle.target}
+      flights_test_schema: flights_validation_${bundle.target}

flights/flights-liquibase/dev-requirements.txt

@@ -0,0 +1,4 @@
+wheel
+pytest
+databricks-sdk[notebook]>=0.46.0
+pandas==2.2.3

flights/flights-liquibase/liquibase.properties

@@ -0,0 +1,2 @@
+liquibase.databricks.catalog: liquibase
+liquibase.databricks.schema: lr_liquibase_test

flights/flights-liquibase/liquibase/changelogs/changelog.xml

@@ -0,0 +1,65 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<databaseChangeLog
+        xmlns="http://www.liquibase.org/xml/ns/dbchangelog"
+        xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+        xmlns:ext="http://www.liquibase.org/xml/ns/dbchangelog-ext"
+        xmlns:pro="http://www.liquibase.org/xml/ns/pro"
+        xsi:schemaLocation="http://www.liquibase.org/xml/ns/dbchangelog
+        http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-latest.xsd
+        http://www.liquibase.org/xml/ns/dbchangelog-ext
+        http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-ext.xsd
+        http://www.liquibase.org/xml/ns/pro
+        http://www.liquibase.org/xml/ns/pro/liquibase-pro-latest.xsd">
+
+    <changeSet id="1" author="your.name">
+        <createTable tableName="test_table">
+            <column name="test_id1" type="int">
+                <constraints primaryKey="true" nullable="false" />
+            </column>
+            <column name="test_column" type="int"/>
+        </createTable>
+    </changeSet>
+
+    <changeSet id="2" author="your.name">
+        <createTable tableName="test_table_2">
+            <column name="test_column2" type="int"/>
+        </createTable>
+    </changeSet>
+
+    <changeSet id="3" author="your.name">
+        <createTable tableName="flights_raw">
+            <column name="Year" type="INT"/>
+            <column name="Month" type="INT"/>
+            <column name="DayofMonth" type="INT"/>
+            <column name="DayOfWeek" type="INT"/>
+            <column name="DepTime" type="STRING"/>
+            <column name="CRSDepTime" type="INT"/>
+            <column name="ArrTime" type="STRING"/>
+            <column name="CRSArrTime" type="INT"/>
+            <column name="UniqueCarrier" type="STRING"/>
+            <column name="FlightNum" type="INT"/>
+            <column name="TailNum" type="STRING"/>
+            <column name="ActualElapsedTime" type="STRING"/>
+            <column name="CRSElapsedTime" type="INT"/>
+            <column name="AirTime" type="STRING"/>
+            <column name="ArrDelay" type="STRING"/>
+            <column name="DepDelay" type="STRING"/>
+            <column name="Origin" type="STRING"/>
+            <column name="Dest" type="STRING"/>
+            <column name="Distance" type="STRING"/>
+            <column name="TaxiIn" type="STRING"/>
+            <column name="TaxiOut" type="STRING"/>
+            <column name="Cancelled" type="INT"/>
+            <column name="CancellationCode" type="STRING"/>
+            <column name="Diverted" type="INT"/>
+            <column name="CarrierDelay" type="STRING"/>
+            <column name="WeatherDelay" type="STRING"/>
+            <column name="NASDelay" type="STRING"/>
+            <column name="SecurityDelay" type="STRING"/>
+            <column name="LateAircraftDelay" type="STRING"/>
+            <column name="IsArrDelayed" type="STRING"/>
+            <column name="IsDepDelayed" type="STRING"/>
+        </createTable>
+    </changeSet>
+
+</databaseChangeLog>

flights/flights-liquibase/liquibase/root.changelog.xml

@@ -0,0 +1,24 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<databaseChangeLog
+        xmlns="http://www.liquibase.org/xml/ns/dbchangelog"
+        xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+        xmlns:ext="http://www.liquibase.org/xml/ns/dbchangelog-ext"
+        xmlns:pro="http://www.liquibase.org/xml/ns/pro"
+        xsi:schemaLocation="http://www.liquibase.org/xml/ns/dbchangelog
+        http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-latest.xsd
+        http://www.liquibase.org/xml/ns/dbchangelog-ext
+        http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-ext.xsd
+        http://www.liquibase.org/xml/ns/pro
+        http://www.liquibase.org/xml/ns/pro/liquibase-pro-latest.xsd">
+
+    <!-- Root changelog file that automatically includes all other changelog files in the liquibase folder -->
+    <!-- This provides a single entry point for managing all database changes -->
+
+    <!-- Files are included in alphabetical order -->
+    <includeAll path="changelogs" 
+                relativeToChangelogFile="true"
+                errorIfMissingOrEmpty="false"/>
+
+    <!-- The filter ensures only changelog XML files are included, and this root file is excluded by naming convention -->
+
+</databaseChangeLog>

flights/flights-liquibase/pyproject.toml

@@ -0,0 +1,12 @@
+[project]
+name = "flights"
+version = "0.0.1"
+authors = [{"name" = "Databricks Field Eng"}]
+description = "Flights project with Liquibase integration"
+
+[build-system]
+requires = ["setuptools", "wheel"]
+build-backend = "setuptools.build_meta"
+
+[tool.setuptools]
+package-dir = {"flights" = "./src/flights"}

flights/flights-liquibase/requirements.txt

@@ -0,0 +1 @@
+databricks-connect>=17.0.0

flights/flights-liquibase/resources/flights_notebook_job_serverless.yml

@@ -0,0 +1,17 @@
+# The main job for flights_project
+resources:
+  jobs:
+    flights_notebook:
+      name: flights_notebook_job_${bundle.target}
+      max_concurrent_runs: 1
+
+      tasks:
+        - task_key: notebook_task
+          notebook_task:
+            notebook_path: ../src/flights_main_notebook.py
+            base_parameters:
+                catalog: ${var.catalog}
+                database: ${var.database}
+                artifact_path: ${workspace.artifact_path}
+          max_retries: 0
+

flights/flights-liquibase/src/flights/__init__.py

@@ -0,0 +1 @@
+__version__ = "0.0.1"