some bug fix in basic utterance generation

Author: voorhs

autointent/_dataset/_dataset.py

@@ -144,7 +144,10 @@ def to_json(self, filepath: str | Path) -> None:
 
         :param filepath: The path to the file where the JSON data will be saved.
         """
-        with Path(filepath).open("w") as file:
+        path = Path(filepath)
+        if not path.parent.exists():
+            path.parent.mkdir(parents=True)
+        with path.open("w") as file:
             json.dump(self.to_dict(), file, indent=4, ensure_ascii=False)
 
     def push_to_hub(self, repo_id: str, private: bool = False) -> None:

autointent/generation/utterances/basic/cli.py

@@ -28,11 +28,7 @@ def main() -> None:
         default=None,
         help="Local path where to save result",
     )
-    parser.add_argument(
-        "--private",
-        action="store_true",
-        help="Publish privately if --output-repo option is used"
-    )
+    parser.add_argument("--private", action="store_true", help="Publish privately if --output-repo option is used")
     parser.add_argument(
         "--n-generations",
         type=int,
@@ -72,11 +68,18 @@ def main() -> None:
     args = parser.parse_args()
 
     dataset = load_dataset(args.input_path)
-    generator = UtteranceGenerator(Generator(), args.custom_instruction, args.length, args.style, args.same_punctuation)
-    generator.augment(dataset, n_generations=args.n_generations, max_sample_utterances=args.n_sample_utterances)
+    generator = UtteranceGenerator(
+        Generator(), args.custom_instruction or [], args.length, args.style, args.same_punctuation
+    )
+    generator.augment(
+        dataset, n_generations=args.n_generations, max_sample_utterances=args.n_sample_utterances
+    )
+
+    dataset.to_json(args.output_path)
 
     if args.output_repo is not None:
-        dataset.push_to_hub(args.output_repo)
+        dataset.push_to_hub(args.output_repo, private=args.private)
+
 
 if __name__ == "__main__":
     main()

autointent/generation/utterances/basic/utterance_generator.py

@@ -89,17 +89,17 @@ def augment(
             )
         if update_split:
             generated_split = HFDataset.from_list(new_samples)
-            dataset[split_name] = concatenate_datasets(original_split, generated_split)
+            dataset[split_name] = concatenate_datasets([original_split, generated_split])
         return [Sample(**sample) for sample in new_samples]
 
 
 def _load_prompt() -> str:
-    with ires.files("autointent.generation.basic").joinpath("chat_template.yaml").open() as file:
+    with ires.files("autointent.generation.utterances.basic").joinpath("chat_template.yaml").open() as file:
         return file.read()
 
 
 def _load_extra_instructions() -> dict[str, Any]:
-    with ires.files("autointent.generation.basic").joinpath("extra_instructions.json").open() as file:
+    with ires.files("autointent.generation.utterances.basic").joinpath("extra_instructions.json").open() as file:
         return json.load(file)
 
 

pyproject.toml

@@ -57,7 +57,7 @@ Documentation = "https://deeppavlov.github.io/AutoIntent/"
 "autointent" = "autointent._pipeline._cli_endpoint:optimize"
 "autointent-inference" = "autointent.pipeline.inference.cli_endpoint:main"
 "clear-cache" = "autointent.context.vector_index_client.cache:clear_chroma_cache"
-
+"basic-aug" = "autointent.generation.utterances.basic.cli:main"
 
 [tool.poetry.group.dev]
 optional = true