Feature templates bugfix (#175)

ronanstokes-db · web-flow · commit 070bb813936d · 2023-03-22T16:13:15.000-07:00
* wip

* merged updates to master

* wip

* wip

* wip

* wip

* wip

* wip

* wip

* wip

* wip

* wip

* wip

* wip

* wip

* wip

* wip

* wip

* wip

* wip

* wip

* wip

* wip

* wip

* wip

* wip
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -7,6 +7,7 @@ All notable changes to the Databricks Labs Data Generator will be documented in
 
 #### Changed
 * Fixed use of logger in _version.py and in spark_singleton.py
+* Fixed template issues 
 * Added use of prospector to build process to validate common code issues
 
 ### Version 0.3.2
diff --git a/README.md b/README.md
@@ -62,7 +62,7 @@ details of use and many examples.
 
 Release notes and details of the latest changes for this specific release
 can be found in the Github repository
-[here](https://github.com/databrickslabs/dbldatagen/blob/release/v0.3.2/CHANGELOG.md)
+[here](https://github.com/databrickslabs/dbldatagen/blob/release/v0.3.2post2/CHANGELOG.md)
 
 # Installation
 
diff --git a/dbldatagen/_version.py b/dbldatagen/_version.py
@@ -33,7 +33,7 @@ def get_version(version):
     return version_info
 
 
-__version__ = "0.3.2"  # DO NOT EDIT THIS DIRECTLY!  It is managed by bumpversion
+__version__ = "0.3.2post2"  # DO NOT EDIT THIS DIRECTLY!  It is managed by bumpversion
 __version_info__ = get_version(__version__)
 
 
diff --git a/dbldatagen/text_generators.py b/dbldatagen/text_generators.py
diff --git a/docs/source/conf.py b/docs/source/conf.py
@@ -28,7 +28,7 @@
 author = 'Databricks Inc'
 
 # The full version, including alpha/beta/rc tags
-release = "0.3.2"  # DO NOT EDIT THIS DIRECTLY!  It is managed by bumpversion
+release = "0.3.2post2"  # DO NOT EDIT THIS DIRECTLY!  It is managed by bumpversion
 
 
 # -- General configuration ---------------------------------------------------
diff --git a/docs/source/textdata.rst b/docs/source/textdata.rst
@@ -127,7 +127,7 @@ It uses the following special chars:
     Chars     Meaning
     ========  ======================================
     ``\``     Apply escape to next char.
-    0,1,..9   Use base value as an array of values and substitute the `nth` element ( 0 .. 9). Always escaped.
+    v0,..v9   Use base value as an array of values and substitute the `nth` element ( 0 .. 9). Always escaped.
     x         Insert a random lowercase hex digit
     X         Insert an uppercase random hex digit
     d         Insert a random lowercase decimal digit
@@ -149,7 +149,7 @@ It uses the following special chars:
           If the ``escapeSpecialChars`` option is set to True, then the following char only has its special
           meaning when preceded by an escape.
 
-          Some options must be always escaped for example ``\\0``, ``\\v``, ``\\n`` and ``\\w``.
+          Some options must be always escaped for example  ``\\v``, ``\\n`` and ``\\w``.
 
           A special case exists for ``\\v`` - if immediately followed by a digit 0 - 9, the underlying base value
           is interpreted as an array of values and the nth element is retrieved where `n` is the digit specified.
diff --git a/prospector.yaml b/prospector.yaml
@@ -11,6 +11,9 @@ ignore-paths:
 max-line-length: 120
 
 pylint:
+  options:
+    bad-functions: print
+
   disable:
     - too-many-instance-attributes
     - invalid-name
diff --git a/python/.bumpversion.cfg b/python/.bumpversion.cfg
@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 0.3.2
+current_version = 0.3.2post2
 commit = False
 tag = False
 parse = (?P<major>\d+)\.(?P<minor>\d+)\.(?P<patch>\d+){0,1}(?P<release>\D*)(?P<build>\d*)
diff --git a/python/dev_require.txt b/python/dev_require.txt
@@ -19,6 +19,7 @@ pytest-cov
 pytest-timeout
 rstcheck
 prospector
+black
 
 # The following packages are only required for building documentation and are not required at runtime
 sphinx==5.0.0
diff --git a/setup.py b/setup.py
@@ -31,7 +31,7 @@
 
 setuptools.setup(
     name="dbldatagen",
-    version="0.3.2",
+    version="0.3.2post2",
     author="Ronan Stokes, Databricks",
     description="Databricks Labs -  PySpark Synthetic Data Generator",
     long_description=long_description,
diff --git a/tests/test_text_templates.py b/tests/test_text_templates.py
@@ -0,0 +1,295 @@
+import re
+import pytest
+import pandas as pd
+import numpy as np
+
+import pyspark.sql.functions as F
+from pyspark.sql.types import BooleanType, DateType
+from pyspark.sql.types import StructType, StructField, IntegerType, StringType, TimestampType
+
+import dbldatagen as dg
+from dbldatagen import TemplateGenerator, TextGenerator
+
+# add the following if using pandas udfs
+#    .config("spark.sql.execution.arrow.maxRecordsPerBatch", "1000") \
+
+
+spark = dg.SparkSingleton.getLocalInstance("unit tests")
+
+spark.conf.set("spark.sql.execution.arrow.maxRecordsPerBatch", "20000")
+spark.conf.set("spark.sql.execution.arrow.pyspark.enabled", "true")
+
+
+# Test manipulation and generation of test data for a large schema
+class TestTextTemplates:
+    testDataSpec = None
+    row_count = 100000
+    partitions_requested = 4
+
+    @pytest.mark.parametrize("templates, splitTemplates",
+                             [
+                                 (r"a|b", ['a', 'b']),
+                                 (r"a|b|", ['a', 'b', '']),
+                                 (r"a", ['a']),
+                                 (r"", ['']),
+                                 (r"a\|b", [r'a|b']),
+                                 (r"a\\|b", [r'a\\', 'b']),
+                                 (r"a\|b|c", [r'a|b', 'c']),
+                                 (r"123,$456|test test2 |\|\a\\a |021 \| 123",
+                                  ['123,$456', 'test test2 ', '|\\a\\\\a ', '021 | 123']),
+                                 (
+                                         r"123 \\|  123 \|123 | 123|123|123 |asd023,\|23|",
+                                         ['123 \\\\', '  123 |123 ', ' 123', '123', '123 ', 'asd023,|23', '']),
+                                 (r" 123|123|123 |asd023,\|23", [' 123', '123', '123 ', 'asd023,|23']),
+                                 (r'', [''])
+                             ])
+    def test_split_templates(self, templates, splitTemplates):
+        tg1 = TemplateGenerator("test", escapeSpecialChars=False)
+
+        results = tg1._splitTemplates(templates)
+
+        assert results == splitTemplates
+
+    @pytest.mark.parametrize("templateProvided, escapeSpecial, useTemplateObject",
+                             [  # (r'\w \w|\w \v. \w', False, False),
+                                 (r'A', False, True),
+                                 (r'D', False, True),
+                                 (r'K', False, True),
+                                 (r'X', False, True),
+                                 (r'\W', False, True),
+                                 (r'\W', True, True),
+                                 (r'\\w A. \\w|\\w \\w', False, False),
+                                 (r'\\w \\w|\\w A. \\w', False, False),
+                                 (r'\\w \\w|\\w A. \\w', False, True),
+                                 (r'\\w \\w|\\w A. \\w', True, True),
+                                 (r'\\w \\w|\\w A. \\w|\w n n \w', False, False),
+                                 (r'\\w \\w|\\w K. \\w', False, False),
+                                 (r'\\w \\w|\\w K. \\w', False, True),
+                                 (r'\\w \\w|\\w K. \\w', True, True),
+                                 (r'\\w \\w|\\w X. \\w', False, False),
+                                 (r'\\w \\w|\\w X. \\w', False, True),
+                                 (r'\\w \\w|\\w X. \\w', True, True),
+                                 (r'\\w \\w|\\w a. \\w', False, False),
+                                 (r'\\w \\w|\\w a. \\w', False, True),
+                                 (r'\\w \\w|\\w a. \\w', True, True),
+                                 (r'\\w \\w|\\w k. \\w', False, False),
+                                 (r'\\w \\w|\\w k. \\w', False, True),
+                                 (r'\\w \\w|\\w k. \\w', True, True),
+                                 (r'\\w \\w|\\w x. \\w', False, False),
+                                 (r'\\w \\w|\\w x. \\w', False, True),
+                                 (r'\\w \\w|\\w x. \\w', True, True),
+                                 (r'\\w a. \\w', False, True),
+                                 (r'\\w a. \\w|\\w \\w', False, False),
+                                 (r'\\w k. \\w', False, True),
+                                 (r'\\w k. \\w|\\w \\w', False, False),
+                                 (r'\n', False, True),
+                                 (r'\n', True, True),
+                                 (r'\v', False, True),
+                                 (r'\v', True, True),
+                                 (r'\w A. \w', False, False),
+                                 (r'\w \a. \w', True, True),
+                                 (r'\w \k. \w', True, True),
+                                 (r'\w \n \w', True, True),
+                                 (r'\w \w|\w A. \w', False, False),
+                                 (r'\w \w|\w \A. \w', True, True),
+                                 (r'\w \w|\w \a. \w', True, True),
+                                 (r'\w \w|\w \w \w|\w \n \w|\w \w \w \w', True, True),
+                                 (r'\w aAdDkK \w', False, False),
+                                 (r'\w aAdDkKxX \n \N \w', False, False),
+                                 (r'\w', False, False),
+                                 (r'\w', False, True),
+                                 (r'\w', True, True),
+                                 (r'a', False, True),
+                                 (r'b', False, False),
+                                 (r'b', False, True),
+                                 (r'b', True, True),
+                                 (r'd', False, True),
+                                 (r'k', False, True),
+                                 (r'x', False, True),
+                                 ('', False, False),
+                                 ('', False, True),
+                                 (r'', True, True),
+                             ])
+    def test_rnd_compute(self, templateProvided, escapeSpecial, useTemplateObject):
+        template1 = TemplateGenerator(templateProvided, escapeSpecialChars=escapeSpecial)
+        print(f"template [{templateProvided}]")
+
+        arr = np.arange(100)
+
+        template_choices, template_rnd_bounds, template_rnds = template1._prepare_random_bounds(arr)
+
+        assert template_choices is not None
+        assert template_rnd_bounds is not None
+        assert template_rnds is not None
+        assert len(template_choices) == len(template_rnds)
+        assert len(template_choices) == len(template_rnd_bounds)
+
+        for ix, _ in enumerate(template_choices):
+            bounds = template_rnd_bounds[ix]
+            rnds = template_rnds[ix]
+
+            assert len(bounds) == len(rnds)
+
+            for iy, bounds_value in enumerate(bounds):
+                assert bounds_value == -1 or (rnds[iy] < bounds_value)
+
+    @pytest.mark.parametrize("templateProvided, escapeSpecial, useTemplateObject",
+                             [  # (r'\w \w|\w \v. \w', False, False),
+                                 (r'\\w \\w|\\w a. \\w', False, False),
+                                 (r'\\w \\w|\\w a. \\w', False, True),
+                                 (r'\\w \\w|\\w a. \\w', True, True),
+                                 (r'\w \w|\w a. \w', False, False),
+                                 (r'\w.\w@\w.com', False, False),
+                                 (r'\n-\n', False, False),
+                                 (r'A', False, True),
+                                 (r'D', False, True),
+                                 (r'K', False, True),
+                                 (r'X', False, True),
+                                 (r'\W', False, True),
+                                 (r'\W', True, True),
+                                 (r'\\w A. \\w|\\w \\w', False, False),
+                                 (r'\\w \\w|\\w A. \\w', False, False),
+                                 (r'\\w \\w|\\w A. \\w', False, True),
+                                 (r'\\w \\w|\\w A. \\w', True, True),
+                                 (r'\\w \\w|\\w A. \\w|\w n n \w', False, False),
+                                 (r'\\w \\w|\\w K. \\w', False, False),
+                                 (r'\\w \\w|\\w K. \\w', False, True),
+                                 (r'\\w \\w|\\w K. \\w', True, True),
+                                 (r'\\w \\w|\\w X. \\w', False, False),
+                                 (r'\\w \\w|\\w X. \\w', False, True),
+                                 (r'\\w \\w|\\w X. \\w', True, True),
+                                 (r'\\w \\w|\\w a. \\w', False, False),
+                                 (r'\\w \\w|\\w a. \\w', False, True),
+                                 (r'\\w \\w|\\w a. \\w', True, True),
+                                 (r'\\w \\w|\\w k. \\w', False, False),
+                                 (r'\\w \\w|\\w k. \\w', False, True),
+                                 (r'\\w \\w|\\w k. \\w', True, True),
+                                 (r'\\w \\w|\\w x. \\w', False, False),
+                                 (r'\\w \\w|\\w x. \\w', False, True),
+                                 (r'\\w \\w|\\w x. \\w', True, True),
+                                 (r'\\w a. \\w', False, True),
+                                 (r'\\w a. \\w|\\w \\w', False, False),
+                                 (r'\\w k. \\w', False, True),
+                                 (r'\\w k. \\w|\\w \\w', False, False),
+                                 (r'\n', False, True),
+                                 (r'\n', True, True),
+                                 (r'\v', False, True),
+                                 (r'\v', True, True),
+                                 (r'\v|\v-\v', False, True),
+                                 (r'\v|\v-\v', True, True),
+                                 (r'short string|a much longer string which is bigger than short string', False, True),
+                                 (r'short string|a much longer string which is bigger than short string', True, True),
+                                 (r'\w A. \w', False, False),
+                                 (r'\w \a. \w', True, True),
+                                 (r'\w \k. \w', True, True),
+                                 (r'\w \n \w', True, True),
+                                 (r'\w \w|\w A. \w', False, False),
+                                 (r'\w \w|\w \A. \w', True, True),
+                                 (r'\w \w|\w \a. \w', True, True),
+                                 (r'\w \w|\w \w \w|\w \n \w|\w \w \w \w', True, True),
+                                 (r'\w aAdDkK \w', False, False),
+                                 (r'\w aAdDkKxX \n \N \w', False, False),
+                                 (r'\w', False, False),
+                                 (r'\w', False, True),
+                                 (r'\w', True, True),
+                                 (r'a', False, True),
+                                 (r'b', False, False),
+                                 (r'b', False, True),
+                                 (r'b', True, True),
+                                 (r'd', False, True),
+                                 (r'k', False, True),
+                                 (r'x', False, True),
+                                 ('', False, False),
+                                 ('', False, True),
+                                 (r'', True, True),
+                                 ('|', False, False),
+                                 ('|', False, True),
+                                 (r'|', True, True),
+                                 (r'\ww - not e\xpecting two wor\ds', False, False),
+                                 (r'\ww - not expecting two words', True, True)
+                             ])
+    def test_use_pandas(self, templateProvided, escapeSpecial, useTemplateObject):
+        template1 = TemplateGenerator(templateProvided, escapeSpecialChars=escapeSpecial)
+
+        TEST_ROWS = 100
+
+        arr1 = np.arange(TEST_ROWS)
+        arr = pd.Series(arr1)
+
+        template_choices, template_rnd_bounds, template_rnds = template1._prepare_random_bounds(arr)
+
+        assert len(template_choices) == len(template_rnds)
+        assert len(template_choices) == len(template_rnd_bounds)
+
+        for ix, _ in enumerate(template_choices):
+            bounds = template_rnd_bounds[ix]
+            rnds = template_rnds[ix]
+
+            assert len(bounds) == len(rnds)
+
+            for iy, bounds_value in enumerate(bounds):
+                assert bounds_value == -1 or (rnds[iy] < bounds_value)
+
+        results = template1.pandasGenerateText(arr)
+        assert results is not None
+
+        results_list = results.tolist()
+
+        results_rows = len(results_list)
+        assert results_rows == TEST_ROWS
+
+        for r, result_str in enumerate(results):
+            assert result_str is not None and isinstance(result_str, str)
+            assert len(result_str) >= 0
+
+        print("results")
+        for i, result_value in enumerate(results):
+            print(f"{i}: '{result_value}'")
+
+    @pytest.mark.parametrize("templateProvided, escapeSpecial, useTemplateObject",
+                             [(r'\n', False, True),
+                              (r'\n', True, True),
+                              (r'\v', False, True),
+                              (r'\v', True, True),
+                              (r'\v|\v-\v', False, True),
+                              (r'\v|\v-\v', True, True),
+                              ])
+    def test_sub_value1(self, templateProvided, escapeSpecial, useTemplateObject):
+        template1 = TemplateGenerator(templateProvided, escapeSpecialChars=escapeSpecial)
+
+        TEST_ROWS = 100
+
+        arr1 = np.arange(TEST_ROWS)
+        arr = pd.Series(arr1)
+
+        print(type(arr1), type(arr))
+
+        template_choices, template_rnd_bounds, template_rnds = template1._prepare_random_bounds(arr)
+
+        assert len(template_choices) == len(template_rnds)
+        assert len(template_choices) == len(template_rnd_bounds)
+
+        for ix, _ in enumerate(template_choices):
+            bounds = template_rnd_bounds[ix]
+            rnds = template_rnds[ix]
+
+            assert len(bounds) == len(rnds)
+
+            for iy, bounds_value in enumerate(bounds):
+                assert bounds_value == -1 or (rnds[iy] < bounds_value)
+
+        results = template1.pandasGenerateText(arr)
+        assert results is not None
+
+        results_list = results.tolist()
+
+        results_rows = len(results_list)
+        assert results_rows == TEST_ROWS
+
+        for r, result_str in enumerate(results):
+            assert result_str is not None and isinstance(result_str, str)
+            assert len(result_str) >= 0
+
+        print("results")
+        for i, result_value in enumerate(results):
+            print(f"{i}: '{result_value}'")
