renaming

Author: andreatgretel

src/data_designer/config/exports.py

@@ -33,7 +33,7 @@
     UniformDistributionParams,
 )
 from data_designer.config.processors import (
-    AncillaryDatasetProcessorConfig,
+    SchemaTransformProcessorConfig,
     DropColumnsProcessorConfig,
     ProcessorType,
 )
@@ -73,7 +73,7 @@
 
 def get_config_exports() -> list[str]:
     return [
-        AncillaryDatasetProcessorConfig.__name__,
+        SchemaTransformProcessorConfig.__name__,
         BernoulliMixtureSamplerParams.__name__,
         BernoulliSamplerParams.__name__,
         BinomialSamplerParams.__name__,

src/data_designer/config/processors.py

@@ -15,7 +15,7 @@
 
 class ProcessorType(str, Enum):
     DROP_COLUMNS = "drop_columns"
-    ANCILLARY_DATASET = "ancillary_dataset"
+    SCHEMA_TRANSFORM = "schema_transform"
 
 
 class ProcessorConfig(ConfigBase, ABC):
@@ -39,20 +39,20 @@ def validate_build_stage(cls, v: BuildStage) -> BuildStage:
 def get_processor_config_from_kwargs(processor_type: ProcessorType, **kwargs) -> ProcessorConfig:
     if processor_type == ProcessorType.DROP_COLUMNS:
         return DropColumnsProcessorConfig(**kwargs)
-    elif processor_type == ProcessorType.ANCILLARY_DATASET:
-        return AncillaryDatasetProcessorConfig(**kwargs)
+    elif processor_type == ProcessorType.SCHEMA_TRANSFORM:
+        return SchemaTransformProcessorConfig(**kwargs)
 
 
 class DropColumnsProcessorConfig(ProcessorConfig):
     column_names: list[str]
     processor_type: Literal[ProcessorType.DROP_COLUMNS] = ProcessorType.DROP_COLUMNS
 
 
-class AncillaryDatasetProcessorConfig(ProcessorConfig):
+class SchemaTransformProcessorConfig(ProcessorConfig):
     template: dict[str, Any] = Field(
         ...,
         description="""
-        Dictionary specifying columns and templates to use in the ancillary dataset.
+        Dictionary specifying columns and templates to use in the new dataset with transformed schema.
 
         Each key is a new column name, and each value is an object containing Jinja2 templates - for instance, a string or a list of strings.
         Values must be JSON-serializable.
@@ -67,8 +67,8 @@ class AncillaryDatasetProcessorConfig(ProcessorConfig):
         }
         ```
 
-        The above templates will create an ancillary dataset with three columns: "list_of_strings", "uppercase_string", and "lowercase_string".
+        The above templates will create an new dataset with three columns: "list_of_strings", "uppercase_string", and "lowercase_string".
         References to columns "col1" and "col2" in the templates will be replaced with the actual values of the columns in the dataset.
         """,
     )
-    processor_type: Literal[ProcessorType.ANCILLARY_DATASET] = ProcessorType.ANCILLARY_DATASET
+    processor_type: Literal[ProcessorType.SCHEMA_TRANSFORM] = ProcessorType.SCHEMA_TRANSFORM

src/data_designer/config/utils/validation.py

@@ -68,7 +68,7 @@ def validate_data_designer_config(
     violations.extend(validate_expression_references(columns=columns, allowed_references=allowed_references))
     violations.extend(validate_columns_not_all_dropped(columns=columns))
     violations.extend(validate_drop_columns_processor(columns=columns, processor_configs=processor_configs))
-    violations.extend(validate_ancillary_dataset_processor(columns=columns, processor_configs=processor_configs))
+    violations.extend(validate_schema_transform_processor(columns=columns, processor_configs=processor_configs))
     if not can_run_data_designer_locally():
         violations.extend(validate_local_only_columns(columns=columns))
     return violations
@@ -294,15 +294,15 @@ def validate_drop_columns_processor(
     return []
 
 
-def validate_ancillary_dataset_processor(
+def validate_schema_transform_processor(
     columns: list[ColumnConfigT],
     processor_configs: list[ProcessorConfig],
 ) -> list[Violation]:
     violations = []
 
     all_column_names = {c.name for c in columns}
     for processor_config in processor_configs:
-        if processor_config.processor_type == ProcessorType.ANCILLARY_DATASET:
+        if processor_config.processor_type == ProcessorType.SCHEMA_TRANSFORM:
             try:
                 json.dumps(processor_config.template)
             except TypeError as e:

src/data_designer/engine/processing/processors/registry.py

@@ -3,11 +3,11 @@
 
 from data_designer.config.base import ConfigBase
 from data_designer.config.processors import (
-    AncillaryDatasetProcessorConfig,
+    SchemaTransformProcessorConfig,
     DropColumnsProcessorConfig,
     ProcessorType,
 )
-from data_designer.engine.processing.processors.ancillary_dataset import AncillaryDatasetProcessor
+from data_designer.engine.processing.processors.schema_transform import SchemaTransformProcessor
 from data_designer.engine.processing.processors.base import Processor
 from data_designer.engine.processing.processors.drop_columns import DropColumnsProcessor
 from data_designer.engine.registry.base import TaskRegistry
@@ -19,7 +19,7 @@ class ProcessorRegistry(TaskRegistry[str, Processor, ConfigBase]): ...
 def create_default_processor_registry() -> ProcessorRegistry:
     registry = ProcessorRegistry()
     registry.register(
-        ProcessorType.ANCILLARY_DATASET, AncillaryDatasetProcessor, AncillaryDatasetProcessorConfig, False
+        ProcessorType.SCHEMA_TRANSFORM, SchemaTransformProcessor, SchemaTransformProcessorConfig, False
     )
     registry.register(ProcessorType.DROP_COLUMNS, DropColumnsProcessor, DropColumnsProcessorConfig, False)
     return registry

…ocessing/processors/ancillary_dataset.py …rocessing/processors/schema_transform.pysrc/data_designer/engine/processing/processors/ancillary_dataset.py renamed to src/data_designer/engine/processing/processors/schema_transform.py src/data_designer/engine/processing/processors/ancillary_dataset.py renamed to src/data_designer/engine/processing/processors/schema_transform.py

@@ -6,7 +6,7 @@
 
 import pandas as pd
 
-from data_designer.config.processors import AncillaryDatasetProcessorConfig
+from data_designer.config.processors import SchemaTransformProcessorConfig
 from data_designer.engine.configurable_task import ConfigurableTaskMetadata
 from data_designer.engine.dataset_builders.artifact_storage import BatchStage
 from data_designer.engine.processing.ginja.environment import WithJinja2UserTemplateRendering
@@ -16,12 +16,12 @@
 logger = logging.getLogger(__name__)
 
 
-class AncillaryDatasetProcessor(WithJinja2UserTemplateRendering, Processor[AncillaryDatasetProcessorConfig]):
+class SchemaTransformProcessor(WithJinja2UserTemplateRendering, Processor[SchemaTransformProcessorConfig]):
     @staticmethod
     def metadata() -> ConfigurableTaskMetadata:
         return ConfigurableTaskMetadata(
-            name="ancillary_dataset_processor",
-            description="Generate an ancillary dataset using a Jinja2 template.",
+            name="schema_transform_processor",
+            description="Generate dataset with transformed schema using a Jinja2 template.",
             required_resources=None,
         )
 

tests/config/test_processors.py

@@ -6,7 +6,7 @@
 
 from data_designer.config.dataset_builders import BuildStage
 from data_designer.config.processors import (
-    AncillaryDatasetProcessorConfig,
+    SchemaTransformProcessorConfig,
     DropColumnsProcessorConfig,
     ProcessorConfig,
     ProcessorType,
@@ -53,35 +53,35 @@ def test_drop_columns_processor_config_serialization():
     assert config_restored.column_names == config.column_names
 
 
-def test_ancillary_dataset_processor_config_creation():
-    config = AncillaryDatasetProcessorConfig(
+def test_schema_transform_processor_config_creation():
+    config = SchemaTransformProcessorConfig(
         name="output_format_processor",
         build_stage=BuildStage.POST_BATCH,
         template={"text": "{{ col1 }}"},
     )
 
     assert config.build_stage == BuildStage.POST_BATCH
     assert config.template == {"text": "{{ col1 }}"}
-    assert config.processor_type == ProcessorType.ANCILLARY_DATASET
+    assert config.processor_type == ProcessorType.SCHEMA_TRANSFORM
     assert isinstance(config, ProcessorConfig)
 
 
-def test_ancillary_dataset_processor_config_validation():
+def test_schema_transform_processor_config_validation():
     # Test unsupported stage raises error
     with pytest.raises(ValidationError, match="Invalid dataset builder stage"):
-        AncillaryDatasetProcessorConfig(
-            name="ancillary_dataset_processor",
+        SchemaTransformProcessorConfig(
+            name="schema_transform_processor",
             build_stage=BuildStage.PRE_BATCH,
             template={"text": "{{ col1 }}"},
         )
 
     # Test missing required field raises error
     with pytest.raises(ValidationError, match="Field required"):
-        AncillaryDatasetProcessorConfig(name="ancillary_dataset_processor", build_stage=BuildStage.POST_BATCH)
+        SchemaTransformProcessorConfig(name="schema_transform_processor", build_stage=BuildStage.POST_BATCH)
 
 
 def test_output_format_processor_config_serialization():
-    config = AncillaryDatasetProcessorConfig(
+    config = SchemaTransformProcessorConfig(
         name="output_format_processor",
         build_stage=BuildStage.POST_BATCH,
         template={"text": "{{ col1 }}"},
@@ -93,7 +93,7 @@ def test_output_format_processor_config_serialization():
     assert config_dict["template"] == {"text": "{{ col1 }}"}
 
     # Deserialize from dict
-    config_restored = AncillaryDatasetProcessorConfig.model_validate(config_dict)
+    config_restored = SchemaTransformProcessorConfig.model_validate(config_dict)
     assert config_restored.build_stage == config.build_stage
     assert config_restored.template == config.template
 
@@ -110,15 +110,15 @@ def test_get_processor_config_from_kwargs():
     assert config_drop_columns.column_names == ["col1"]
     assert config_drop_columns.processor_type == ProcessorType.DROP_COLUMNS
 
-    config_ancillary_dataset = get_processor_config_from_kwargs(
-        ProcessorType.ANCILLARY_DATASET,
+    config_schema_transform = get_processor_config_from_kwargs(
+        ProcessorType.SCHEMA_TRANSFORM,
         name="output_format_processor",
         build_stage=BuildStage.POST_BATCH,
         template={"text": "{{ col1 }}"},
     )
-    assert isinstance(config_ancillary_dataset, AncillaryDatasetProcessorConfig)
-    assert config_ancillary_dataset.template == {"text": "{{ col1 }}"}
-    assert config_ancillary_dataset.processor_type == ProcessorType.ANCILLARY_DATASET
+    assert isinstance(config_schema_transform, SchemaTransformProcessorConfig)
+    assert config_schema_transform.template == {"text": "{{ col1 }}"}
+    assert config_schema_transform.processor_type == ProcessorType.SCHEMA_TRANSFORM
 
     # Test with unknown processor type returns None
     from enum import Enum

tests/config/utils/test_validation.py

@@ -15,7 +15,7 @@
 from data_designer.config.dataset_builders import BuildStage
 from data_designer.config.models import ImageContext, ModalityDataType
 from data_designer.config.processors import (
-    AncillaryDatasetProcessorConfig,
+    SchemaTransformProcessorConfig,
     DropColumnsProcessorConfig,
 )
 from data_designer.config.utils.code_lang import CodeLang
@@ -24,7 +24,7 @@
     ViolationLevel,
     ViolationType,
     rich_print_violations,
-    validate_ancillary_dataset_processor,
+    validate_schema_transform_processor,
     validate_code_validation,
     validate_columns_not_all_dropped,
     validate_data_designer_config,
@@ -106,13 +106,13 @@
         column_names=["inexistent_column"],
         build_stage=BuildStage.POST_BATCH,
     ),
-    AncillaryDatasetProcessorConfig(
-        name="ancillary_dataset_processor_invalid_reference",
+    SchemaTransformProcessorConfig(
+        name="schema_transform_processor_invalid_reference",
         template={"text": "{{ invalid_reference }}"},
         build_stage=BuildStage.POST_BATCH,
     ),
-    AncillaryDatasetProcessorConfig(
-        name="ancillary_dataset_processor_invalid_template",
+    SchemaTransformProcessorConfig(
+        name="schema_transform_processor_invalid_template",
         template={"text": {1, 2, 3}},
         build_stage=BuildStage.POST_BATCH,
     ),
@@ -125,14 +125,14 @@
 @patch("data_designer.config.utils.validation.validate_expression_references")
 @patch("data_designer.config.utils.validation.validate_columns_not_all_dropped")
 @patch("data_designer.config.utils.validation.validate_drop_columns_processor")
-@patch("data_designer.config.utils.validation.validate_ancillary_dataset_processor")
+@patch("data_designer.config.utils.validation.validate_schema_transform_processor")
 def test_validate_data_designer_config(
     mock_validate_columns_not_all_dropped,
     mock_validate_expression_references,
     mock_validate_code_validation,
     mock_validate_prompt_templates,
     mock_validate_drop_columns_processor,
-    mock_validate_ancillary_dataset_processor,
+    mock_validate_schema_transform_processor,
 ):
     mock_validate_columns_not_all_dropped.return_value = [
         Violation(
@@ -174,7 +174,7 @@ def test_validate_data_designer_config(
             level=ViolationLevel.ERROR,
         )
     ]
-    mock_validate_ancillary_dataset_processor.return_value = [
+    mock_validate_schema_transform_processor.return_value = [
         Violation(
             column="text",
             type=ViolationType.INVALID_REFERENCE,
@@ -196,7 +196,7 @@ def test_validate_data_designer_config(
     mock_validate_code_validation.assert_called_once()
     mock_validate_prompt_templates.assert_called_once()
     mock_validate_drop_columns_processor.assert_called_once()
-    mock_validate_ancillary_dataset_processor.assert_called_once()
+    mock_validate_schema_transform_processor.assert_called_once()
 
 
 def test_validate_prompt_templates():
@@ -281,8 +281,8 @@ def test_validate_expression_references():
     assert violations[0].type == ViolationType.EXPRESSION_REFERENCE_MISSING
 
 
-def test_validate_ancillary_dataset_processor():
-    violations = validate_ancillary_dataset_processor(COLUMNS, PROCESSOR_CONFIGS)
+def test_validate_schema_transform_processor():
+    violations = validate_schema_transform_processor(COLUMNS, PROCESSOR_CONFIGS)
     assert len(violations) == 2
     assert violations[0].type == ViolationType.INVALID_REFERENCE
     assert violations[0].column is None
@@ -295,7 +295,7 @@ def test_validate_ancillary_dataset_processor():
     assert violations[1].column is None
     assert (
         violations[1].message
-        == "Ancillary dataset processor ancillary_dataset_processor_invalid_template template is not a valid JSON object."
+        == "Ancillary dataset processor schema_transform_processor_invalid_template template is not a valid JSON object."
     )
     assert violations[1].level == ViolationLevel.ERROR