Semantic Dedup Tutorial + bug fixes (#1067)

* bug fixes

Signed-off-by: Praateek <praateekm@gmail.com>

* add notebooks

Signed-off-by: Praateek <praateekm@gmail.com>

* change input path

Signed-off-by: Praateek <praateekm@gmail.com>

* add comment about input filetype

Signed-off-by: Praateek <praateekm@gmail.com>

* add download dataset too

Signed-off-by: Praateek <praateekm@gmail.com>

* pr comments

Signed-off-by: Praateek <praateekm@gmail.com>

* json -> jsonl

Signed-off-by: Praateek <praateekm@gmail.com>

* fc

Signed-off-by: Praateek <praateekm@gmail.com>

* pr comments

Signed-off-by: Praateek <praateekm@gmail.com>

* ..

Signed-off-by: Praateek <praateekm@gmail.com>

* change graph

Signed-off-by: Praateek <praateekm@gmail.com>

* pr reveiw

Signed-off-by: Praateek <praateekm@gmail.com>

* ..

Signed-off-by: Praateek <praateekm@gmail.com>

---------

Signed-off-by: Praateek <praateekm@gmail.com>

Author: praateekmahajan

nemo_curator/stages/deduplication/semantic/workflow.py

@@ -380,7 +380,7 @@ def run(self, pairwise_executor: BaseExecutor | None = None) -> dict[str, Any]:
             if total_duplicates > 0:
                 logger.success(f"Total documents identified as duplicates: {total_duplicates}")
                 logger.info(f"Similarity threshold used: {1.0 - self.eps:.3f} (eps={self.eps})")
-            else:
+            elif self.eps is not None:
                 logger.info(
                     f"No duplicates identified with similarity threshold of {1.0 - self.eps:.3f} (eps={self.eps})"
                 )
@@ -396,5 +396,5 @@ def run(self, pairwise_executor: BaseExecutor | None = None) -> dict[str, Any]:
                 "pairwise_execution_time": pairwise_time,
                 "kmeans_results": kmeans_results,
                 "pairwise_results": pairwise_results,
-                "total_duplicates_identified": total_duplicates,
+                **({"total_duplicates_identified": total_duplicates} if self.eps is not None else {}),
             }

nemo_curator/stages/text/io/reader/base.py

@@ -52,7 +52,12 @@ def inputs(self) -> tuple[list[str], list[str]]:
         return [], []
 
     def outputs(self) -> tuple[list[str], list[str]]:
-        return ["data"], self.fields or []
+        output_fields = self.fields or []
+        if self._generate_ids or self._assign_ids:
+            from nemo_curator.stages.deduplication.id_generator import CURATOR_DEDUP_ID_STR
+
+            output_fields.append(CURATOR_DEDUP_ID_STR)
+        return ["data"], output_fields
 
     def setup(self, _: WorkerMetadata | None = None) -> None:
         if self._generate_ids or self._assign_ids: