Ak/spec

.github/workflows/push.yml

@@ -61,7 +61,8 @@ jobs:
           python-version: '3.10'
 
       - name: Install Hatch
-        run: pip install hatch
+        # click 8.3+ introduced bug for hatch
+        run: pip install "hatch==1.13.0" "click<8.3"
 
       - name: Run unit tests
         run: make dev test

PULL_REQUEST_TEMPLATE.md

@@ -1,34 +1,15 @@
-## Proposed changes
+## Changes
+<!-- Summarize your changes. Add code examples or screenshots when necessary. -->
 
-Describe the big picture of your changes here to communicate to the maintainers. 
-If it fixes a bug or resolves a feature request, please provide a link to that issue.
+### Linked issues
+<!-- DOC: Link issue with a keyword: close, closes, closed, fix, fixes, fixed, resolve, resolves, resolved. See https://docs.github.com/en/issues/tracking-your-work-with-issues/linking-a-pull-request-to-an-issue#linking-a-pull-request-to-an-issue-using-a-keyword -->
 
-## Types of changes
+Resolves #..
 
-What types of changes does your code introduce to dbldatagen?
-_Put an `x` in the boxes that apply_
+### Requirements
+<!-- How are your changes documented and tested? Please see the checklist below. -->
 
-- [ ] Bug fix (non-breaking change which fixes an issue)
-- [ ] New feature (non-breaking change which adds functionality)
-- [ ] Breaking change (fix or feature that would cause existing functionality to not work as expected)
-- [ ] Change to tutorials, tests or examples
-- [ ] Non code change (readme, images or other non-code assets)
-- [ ] Documentation Update (if none of the other choices apply)
-
-## Checklist
-
-_Put an `x` in the boxes that apply. You can also fill these out after creating the PR. 
-If you're unsure about any of them, don't hesitate to ask. We're here to help! 
-This is simply a reminder of what we are going to look for before merging your code._
-
-- [ ] Lint and unit tests pass locally with my changes
-- [ ] I have added tests that prove my fix is effective or that my feature works
-- [ ] I have added necessary documentation (if appropriate)
-- [ ] Any dependent changes have been merged and published in downstream modules
-- [ ] Submission does not reduce code coverage numbers
-- [ ] Submission does not increase alerts or messages from prospector / lint
-
-## Further comments
-
-If this is a relatively large or complex change, kick off the discussion by explaining why you chose the solution you 
-did and what alternatives you considered, etc...
+- [ ] manually tested
+- [ ] updated documentation
+- [ ] updated demos
+- [ ] updated tests

dbldatagen/__init__.py

@@ -47,11 +47,12 @@
 from .text_generator_plugins import PyfuncText, PyfuncTextFactory, FakerTextFactory, fakerText
 from .html_utils import HtmlUtils
 from .datasets_object import Datasets
+from .config import OutputDataset
 
 __all__ = ["data_generator", "data_analyzer", "schema_parser", "daterange", "nrange",
            "column_generation_spec", "utils", "function_builder",
            "spark_singleton", "text_generators", "datarange", "datagen_constants",
-           "text_generator_plugins", "html_utils", "datasets_object", "constraints"
+           "text_generator_plugins", "html_utils", "datasets_object", "constraints", "config"
            ]
 
 

dbldatagen/config.py

@@ -0,0 +1,36 @@
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+"""
+This module implements configuration classes for writing generated data.
+"""
+from dataclasses import dataclass
+
+
+@dataclass(frozen=True, slots=True)
+class OutputDataset:
+    """
+    This class implements an output sink configuration used to write generated data. An output location must be
+    provided. The output mode, format, and options can be provided.
+
+    :param location: Output location for writing data. This could be an absolute path, a relative path to a Databricks
+        Volume, or a full table location using Unity catalog's 3-level namespace.
+    :param output_mode: Output mode for writing data (default is ``"append"``).
+    :param format: Output data format (default is ``"delta"``).
+    :param options: Optional dictionary of options for writing data (e.g. ``{"mergeSchema": "true"}``)
+    """
+    location: str
+    output_mode: str = "append"
+    format: str = "delta"
+    options: dict[str, str] | None = None
+    trigger: dict[str, str] | None = None
+
+    def __post_init__(self) -> None:
+        if not self.trigger:
+            return
+
+        # Only processingTime is currently supported
+        if "processingTime" not in self.trigger:
+            valid_trigger_format = '{"processingTime": "10 SECONDS"}'
+            raise ValueError(f"Attribute 'trigger' must be a dictionary of the form '{valid_trigger_format}'")