Merge remote-tracking branch 'refs/remotes/origin/master' into build_tools

# Conflicts:
#	.github/workflows/push.yml
#	CONTRIBUTING.md
#	Pipfile
#	makefile
#	setup.py

Author: ghanse

.github/workflows/release.yml

@@ -24,10 +24,10 @@ jobs:
           sudo update-alternatives --set java /usr/lib/jvm/temurin-8-jdk-amd64/bin/java
           java -version
 
-      - name: Set up Python 3.9.21
+      - name: Set up Python 3.10.12
         uses: actions/setup-python@v5
         with:
-          python-version: '3.9.21'
+          python-version: '3.10.12'
           cache: 'pipenv'
 
       - name: Check Python version

CHANGELOG.md

@@ -9,7 +9,8 @@ All notable changes to the Databricks Labs Data Generator will be documented in
 * Updated build scripts to use Ubuntu 22.04 to correspond to environment in Databricks runtime
 
 #### Changed
-* Changed base Databricks runtime version to DBR 11.3 LTS (based on Apache Spark 3.3.0)
+* Changed base Databricks runtime version to DBR 13.3 LTS (based on Apache Spark 3.4.1) - minimum supported version
+  of Python is now 3.10.12
 
 #### Added
 * Added support for serialization to/from JSON format

README.md

@@ -83,23 +83,26 @@ The documentation [installation notes](https://databrickslabs.github.io/dbldatag
 contains details of installation using alternative mechanisms.
 
 ## Compatibility 
-The Databricks Labs Data Generator framework can be used with Pyspark 3.3.0 and Python 3.9.21 or later. These are 
-compatible with the Databricks runtime 11.3 LTS and later releases. For full Unity Catalog support, 
-we recommend using Databricks runtime 13.2 or later (Databricks 13.3 LTS or above preferred)
+The Databricks Labs Data Generator framework can be used with Pyspark 3.4.1 and Python 3.10.12 or later. These are 
+compatible with the Databricks runtime 13.3 LTS and later releases. This version also provides Unity Catalog
+compatibily.
 
 For full library compatibility for a specific Databricks Spark release, see the Databricks 
 release notes for library compatibility
 
 - https://docs.databricks.com/release-notes/runtime/releases.html
 
-When using the Databricks Labs Data Generator on "Unity Catalog" enabled Databricks environments, 
+In older releases, when using the Databricks Labs Data Generator on "Unity Catalog" enabled Databricks environments, 
 the Data Generator requires the use of `Single User` or `No Isolation Shared` access modes when using Databricks 
 runtimes prior to release 13.2. This is because some needed features are not available in `Shared` 
 mode (for example, use of 3rd party libraries, use of Python UDFs) in these releases. 
-Depending on settings, the `Custom` access mode may be supported.
+Depending on settings, the `Custom` access mode may be supported for those releases.
 
 The use of Unity Catalog `Shared` access mode is supported in Databricks runtimes from Databricks runtime release 13.2
-onwards.
+onwards. 
+
+*This version of the data generator uses the Databricks runtime 13.3 LTS as the minimum supported 
+version and alleviates these issues.*
 
 See the following documentation for more information:
 
@@ -155,7 +158,7 @@ The GitHub repository also contains further examples in the examples directory.
 
 ## Spark and Databricks Runtime Compatibility
 The `dbldatagen` package is intended to be compatible with recent LTS versions of the Databricks runtime, including 
-older LTS versions at least from 11.3 LTS and later. It also aims to be compatible with Delta Live Table runtimes, 
+older LTS versions at least from 13.3 LTS and later. It also aims to be compatible with Delta Live Table runtimes, 
 including `current` and `preview`. 
 
 While we don't specifically drop support for older runtimes, changes in Pyspark APIs or

python/dev_require.txt

@@ -1,19 +1,19 @@
 # The following packages are used in building the test data generator framework.
 # All packages used are already installed in the Databricks runtime environment for version 6.5 or later
 numpy==1.22.0
-pandas==1.3.4
+pandas==1.4.4
 pickleshare==0.7.5
 py4j>=0.10.9.3
-pyarrow==7.0.0
-pyspark==3.3.0
+pyarrow==8.0.0
+pyspark==3.4.1
 python-dateutil==2.8.2
 six==1.16.0
-pyparsing==3.0.4
+pyparsing==3.0.9
 jmespath==0.10.0
 
 # The following packages are required for development only
-wheel==0.37.0
-setuptools==58.0.4
+wheel==0.37.1
+setuptools==63.4.1
 bumpversion
 pytest
 pytest-cov
@@ -28,9 +28,9 @@ sphinx_rtd_theme
 nbsphinx
 numpydoc==0.8
 pypandoc
-ipython==7.32.0
+ipython==8.10.0
 recommonmark
 sphinx-markdown-builder
-Jinja2 < 3.1
+Jinja2 < 3.1, >= 2.11.3
 sphinx-copybutton
 

python/require.txt

@@ -1,19 +1,19 @@
 # The following packages are used in building the test data generator framework.
 # All packages used are already installed in the Databricks runtime environment for version 6.5 or later
 numpy==1.22.0
-pandas==1.3.4
+pandas==1.4.4
 pickleshare==0.7.5
 py4j==0.10.9
-pyarrow==7.0.0
-pyspark==3.3.0
+pyarrow==8.0.0
+pyspark==3.4.1
 python-dateutil==2.8.2
 six==1.16.0
-pyparsing==3.0.4
+pyparsing==3.0.9
 jmespath==0.10.0
 
 # The following packages are required for development only
-wheel==0.37.0
-setuptools==58.0.4
+wheel==0.37.1
+setuptools==63.4.1
 bumpversion
 pytest
 pytest-cov
@@ -27,9 +27,9 @@ sphinx_rtd_theme
 nbsphinx
 numpydoc==0.8
 pypandoc
-ipython==7.32.0
+ipython==8.10.0
 recommonmark
 sphinx-markdown-builder
-Jinja2 < 3.1
+Jinja2 < 3.1, >= 2.11.3
 sphinx-copybutton