Merge remote-tracking branch 'refs/remotes/origin/master' into feature_textdata_changes

Author: ghanse

.github/dependabot.yml

@@ -0,0 +1,10 @@
+version: 2
+updates:
+  - package-ecosystem: "pip"
+    directory: "/"
+    schedule:
+      interval: "daily"
+  - package-ecosystem: "github-actions"
+    directory: "/"
+    schedule:
+      interval: "daily"

.github/workflows/push.yml

@@ -7,12 +7,24 @@ on:
     branches: [master]
 
 jobs:
+  fmt:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/[email protected]
+
+      - name: Format files
+        run: make dev fmt
+
+      - name: Fail on differences
+        run: git diff --exit-code
+
   tests:
     # Ubuntu latest no longer installs Python 3.9 by default so install it
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-latest
     steps:
       - name: Checkout
-        uses: actions/checkout@v4
+        uses: actions/checkout@v4.2.2
         with:
           fetch-depth: 0
 
@@ -26,35 +38,33 @@ jobs:
       #     key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
       #     restore-keys: |
       #       ${{ runner.os }}-go-
-      - name: Set Java 8
-        run: |
-          sudo update-alternatives --set java /usr/lib/jvm/temurin-8-jdk-amd64/bin/java
-          java -version
 
-      - name: Set up Python 3.8
-        uses: actions/setup-python@v5
+      - name: Set up JDK 17
+        uses: actions/setup-java@v4
         with:
-          python-version: '3.8.12'
-          cache: 'pipenv'
-
-      - name: Check Python version
-        run: python --version
+          distribution: 'temurin'   # Can also use 'zulu', 'adopt', etc.
+          java-version: '17'
 
-      - name: Install pip
-        run: python -m pip install --upgrade pip
+      - name: Get Java version
+        run: java -version
 
-      - name: Install 
-        run: pip install pipenv
+      #- name: Set Java 8
+      #  run: |
+      #    sudo update-alternatives --set java /usr/lib/jvm/temurin-8-jdk-amd64/bin/java
+      #    java -version
 
-      - name: Install dependencies
-        run: pipenv install --dev
+      - name: Install Python
+        uses: actions/setup-python@v5
+        with:
+          cache: 'pip'
+          cache-dependency-path: '**/pyproject.toml'
+          python-version: '3.10'
 
-      - name: Lint
-        run: |
-          pipenv run prospector --profile prospector.yaml
+      - name: Install Hatch
+        run: pip install hatch
 
-      - name: Run tests
-        run: make test
+      - name: Run unit tests
+        run: make dev test
 
       - name: Publish test coverage to coverage site
         uses: codecov/codecov-action@v4

.github/workflows/release.yml

@@ -24,10 +24,10 @@ jobs:
           sudo update-alternatives --set java /usr/lib/jvm/temurin-8-jdk-amd64/bin/java
           java -version
 
-      - name: Set up Python 3.8
+      - name: Set up Python 3.10.12
         uses: actions/setup-python@v5
         with:
-          python-version: '3.8.12'
+          python-version: '3.10.12'
           cache: 'pipenv'
 
       - name: Check Python version

.gitignore

@@ -36,3 +36,15 @@ docs/source/reference/api/*.rst
 .coverage
 htmlcov/
 .coverage.xml
+
+# IDE-specific folders — prevent local/editor config files from polluting source control.
+# PyCharm
+#  JetBrains specific template is maintained in a separate JetBrains.gitignore that can
+#  be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
+#  and can be added to the global gitignore or merged into this file.  For a more nuclear
+#  option (not recommended) you can uncomment the following to ignore the entire idea folder.
+.idea/
+# Cursor IDE
+# Cursor is an AI-powered code editor. The .cursor/ directory contains IDE-specific
+# settings and configurations similar to other IDEs.
+.cursor/

CHANGELOG.md

@@ -7,6 +7,23 @@ All notable changes to the Databricks Labs Data Generator will be documented in
 
 #### Fixed 
 * Updated build scripts to use Ubuntu 22.04 to correspond to environment in Databricks runtime
+* Refactored `DataAnalyzer` and `BasicStockTickerProvider` to comply with ANSI SQL standards
+* Removed internal modification of `SparkSession`
+
+#### Changed
+* Changed base Databricks runtime version to DBR 13.3 LTS (based on Apache Spark 3.4.1) - minimum supported version
+  of Python is now 3.10.12
+* Updated build tooling to use [hatch](https://hatch.pypa.io/latest/)
+* Moved dependencies and tool configuration to [pyproject.toml](pyproject.toml)
+* Removed dependencies provided by the Databricks Runtime
+* Updated Git actions
+* Updated [makefile](makefile)
+* Updated [CONTRIBUTING.md](CONTRIBUTING.md)
+
+#### Added
+* Added support for serialization to/from JSON format
+* Added Ruff and mypy tooling
+
 
 ### Version 0.4.0 Hotfix 2