wip

Author: ronanstokes-db

README.md

@@ -62,7 +62,7 @@ details of use and many examples.
 
 Release notes and details of the latest changes for this specific release
 can be found in the Github repository
-[here](https://github.com/databrickslabs/dbldatagen/blob/release/v0.3.2/CHANGELOG.md)
+[here](https://github.com/databrickslabs/dbldatagen/blob/release/v0.3.3/CHANGELOG.md)
 
 # Installation
 

dbldatagen/_version.py

@@ -33,7 +33,7 @@ def get_version(version):
     return version_info
 
 
-__version__ = "0.3.2"  # DO NOT EDIT THIS DIRECTLY!  It is managed by bumpversion
+__version__ = "0.3.3"  # DO NOT EDIT THIS DIRECTLY!  It is managed by bumpversion
 __version_info__ = get_version(__version__)
 
 

dbldatagen/text_generators.py

@@ -477,14 +477,17 @@ def _get_values_subelement(elem):
         num_placeholders = 0
         rnd_offset = 0
 
-        masked_rows = None
+        unmasked_rows = None  # unmasked_rows is None, indicates that all rows are unmasked
 
         assert isinstance(placeholders, np.ma.MaskedArray), "expecting MaskArray"
 
         # if template is empty, then nothing needs to be done
         if template_len > 0 and isinstance(placeholders, np.ma.MaskedArray):
             active_rows = ~placeholders.mask
-            masked_rows = active_rows[:, 0]
+            unmasked_rows = active_rows[:, 0]
+
+            if np.all(active_rows):
+                unmasked_rows = None
 
         # in the following code, the construct `(not escape) ^ self._escapeSpecialMeaning` means apply
         # special meaning if either escape is not true or the option `self._escapeSpecialMeaning` is true.
@@ -506,8 +509,8 @@ def _get_values_subelement(elem):
                 # random numbers from `rnds` 2d array
                 bound, valueMappings = self._templateMappings[char]
 
-                if masked_rows is not None:
-                    placeholders[masked_rows, num_placeholders] = valueMappings[rnds[masked_rows, rnd_offset]]
+                if unmasked_rows is not None:
+                    placeholders[unmasked_rows, num_placeholders] = valueMappings[rnds[unmasked_rows, rnd_offset]]
                 else:
                     placeholders[:, num_placeholders] = valueMappings[rnds[:, rnd_offset]]
 
@@ -519,13 +522,13 @@ def _get_values_subelement(elem):
                 bound, valueMappings = self._templateEscapedMappings[char]
 
                 if valueMappings is not None:
-                    if masked_rows is not None:
-                        placeholders[masked_rows, num_placeholders] = valueMappings[rnds[masked_rows, rnd_offset]]
+                    if unmasked_rows is not None:
+                        placeholders[unmasked_rows, num_placeholders] = valueMappings[rnds[unmasked_rows, rnd_offset]]
                     else:
                         placeholders[:, num_placeholders] = valueMappings[rnds[:, rnd_offset]]
                 else:
-                    if masked_rows is not None:
-                        placeholders[masked_rows, num_placeholders] = rnds[masked_rows, rnd_offset]
+                    if unmasked_rows is not None:
+                        placeholders[unmasked_rows, num_placeholders] = rnds[unmasked_rows, rnd_offset]
                     else:
                         placeholders[:, num_placeholders] = rnds[:, rnd_offset]
                 num_placeholders += 1

docs/source/conf.py

@@ -28,7 +28,7 @@
 author = 'Databricks Inc'
 
 # The full version, including alpha/beta/rc tags
-release = "0.3.2"  # DO NOT EDIT THIS DIRECTLY!  It is managed by bumpversion
+release = "0.3.3"  # DO NOT EDIT THIS DIRECTLY!  It is managed by bumpversion
 
 
 # -- General configuration ---------------------------------------------------

python/.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 0.3.2
+current_version = 0.3.3
 commit = False
 tag = False
 parse = (?P<major>\d+)\.(?P<minor>\d+)\.(?P<patch>\d+){0,1}(?P<release>\D*)(?P<build>\d*)

python/dev_require.txt

@@ -19,6 +19,7 @@ pytest-cov
 pytest-timeout
 rstcheck
 prospector
+black
 
 # The following packages are only required for building documentation and are not required at runtime
 sphinx==5.0.0

setup.py

@@ -31,7 +31,7 @@
 
 setuptools.setup(
     name="dbldatagen",
-    version="0.3.2",
+    version="0.3.3",
     author="Ronan Stokes, Databricks",
     description="Databricks Labs -  PySpark Synthetic Data Generator",
     long_description=long_description,