Update tests and documentation

Author: ghanse

docs/source/generating_column_data.rst

@@ -182,3 +182,48 @@ This has several implications:
   SQL expression.
   To enforce the dependency, you must use the `baseColumn` attribute to indicate the dependency.
 
+Creating data generation specs from files
+-----------------------------------------
+
+``DataGenerator.fromFile("file_path")`` will return a ``DataGenerator`` with ``ColumnGenerationSpecs`` from definitions
+in a JSON or YAML file. Use the ``"generator"`` key to specify ``DataGenerator`` options and the ``"columns"`` key to
+specify ``ColumnGenerationSpec`` options.
+
+**JSON Example:**
+
+.. code-block:: JSON
+   {
+     "generator": {
+         "name": "test_data_generator",
+         "rows": 1000,
+         "partitions": 10
+     },
+     "columns": [
+         {"colName": "col1", "colType": "int", "minValue": 0, "maxValue": 100},
+         {"colName": "col2", "colType": "float", "minValue": 0.0, "maxValue": 100.0},
+         {"colName": "col3", "colType": "string", "values": ["a", "b", "c"], "random": true}
+     ]
+   }
+
+**YAML Example:**
+.. code-block:: YAML
+generator:
+  name: test_data_generator
+  rows: 1000
+  partitions: 10
+columns:
+  - colName: col1
+    colType: int
+    minValue: 0
+    maxValue: 1000
+  - colName: col2
+    colType: float
+    minValue: -10.0
+    maxValue: 10.0
+  - colName: col3
+    colType: string
+    values:
+      - a
+      - b
+      - c
+    random: true

docs/source/options_and_features.rst

@@ -128,6 +128,12 @@ representing the column - for example "email_0", "email_1" etc.
 If you specify the attribute ``structType="array"``, the multiple columns will be combined into a single array valued
 column.
 
+Generating columns from Python dictionaries
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+You can generate columns from Python dictionaries using ``withColumns(column_options)``. Each dictionary should contain
+keys which match the ``withColumn`` arguments (e.g. ``"colName"``, ``"colType"``).
+
 Generating random values
 ^^^^^^^^^^^^^^^^^^^^^^^^
 

tests/files/test_generator_spec.json

@@ -8,8 +8,8 @@
       "random": true
   },
   "columns": [
-      {"colName": "col1", "colType": "int", "min": 0, "max": 100},
-      {"colName": "col2", "colType": "float", "min": 0.0, "max": 100.0},
+      {"colName": "col1", "colType": "int", "minValue": 0, "maxValue": 1000},
+      {"colName": "col2", "colType": "float", "minValue": -10.0, "maxValue": 10.0},
       {"colName": "col3", "colType": "string", "values": ["a", "b", "c"], "random": true}
   ]
 }

tests/files/test_generator_spec.yml

@@ -8,12 +8,12 @@ generator:
 columns:
   - colName: col1
     colType: int
-    min: 0
-    max: 100
+    minValue: 0
+    maxValue: 1000
   - colName: col2
     colType: float
-    min: 0.0
-    max: 100.0
+    minValue: -10.0
+    maxValue: 10.0
   - colName: col3
     colType: string
     values:

tests/test_quick_tests.py

@@ -1,7 +1,6 @@
 from datetime import timedelta, datetime
-
-import pytest
 import json
+import pytest
 import yaml
 from pyspark.sql.types import (
     StructType, StructField, IntegerType, StringType, FloatType, DateType, DecimalType, DoubleType, ByteType,
@@ -784,7 +783,7 @@ def test_generation_from_dictionary(self):
 
     def test_generation_from_file(self):
         path = "tests/files/test_generator_spec.json"
-        with open(path, "r") as f:
+        with open(path, "r", encoding="utf-8") as f:
             options = json.load(f)
             gen_options = options.get("generator")
             gen_from_json = DataGenerator.fromFile(path)
@@ -798,7 +797,7 @@ def test_generation_from_file(self):
             assert df_from_json.columns == ["col1", "col2", "col3"]
 
         path = "tests/files/test_generator_spec.yml"
-        with open(path, "r") as f:
+        with open(path, "r", encoding="utf-8") as f:
             options = yaml.safe_load(f)
             gen_options = options.get("generator")
             gen_from_yaml = DataGenerator.fromFile(path)