Bump version to v0.8.0 for release

Author: gagan3012

CHANGELOG.md

@@ -7,6 +7,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## Unreleased
 
+## [v0.8.0](https://github.com/gagan3012/PolyDeDupe/releases/tag/v0.8.0) - 2023-11-27
+
 ## [v0.7.0](https://github.com/gagan3012/PolyDeDupe/releases/tag/v0.7.0) - 2023-11-27
 
 ## [v0.6.0](https://github.com/gagan3012/PolyDeDupe/releases/tag/v0.6.0) - 2023-11-27

PolyDeDupe/polydedupe.py

@@ -5,23 +5,27 @@
 from functools import partial
 from typing import Dict, List, Optional, Set, Tuple, Type
 import os
-from datasets import Dataset, load_dataset, concatenate_datasets
+from datasets import Dataset
 from tqdm.auto import tqdm
 
 from datasketch import MinHash, MinHashLSH
 from dpu_utils.utils.iterators import ThreadedIterator
 from sklearn.metrics.pairwise import cosine_similarity
 
-NON_ALPHA = re.compile("[^\u0080-\u00FF\u0100-\u017F\u0600-\u06FF\u07C0-\u07FF\u0900-\u097F\u1200-\u137F\u2D30-\u2D7F\uA500-\uA63FA-Za-z_0-9]")
+NON_ALPHA = re.compile(
+    "[^\u0080-\u00FF\u0100-\u017F\u0600-\u06FF\u07C0-\u07FF\u0900-\u097F\u1200-\u137F\u2D30-\u2D7F\uA500-\uA63FA-Za-z_0-9]"
+)
 # parameters used in DuplicationIndex
 MIN_NUM_TOKENS = 10
 NUM_PERM = 256
 
 
 def get_model(model_name="sentence-transformers/all-MiniLM-L12-v2"):
-  from sentence_transformers import SentenceTransformer
-  model = SentenceTransformer(model_name)  
-  return model
+    from sentence_transformers import SentenceTransformer
+
+    model = SentenceTransformer(model_name)
+    return model
+
 
 def get_min_hash(tokens: List[str]) -> Optional[MinHash]:
     """Compute the MinHash of a code snippet."""
@@ -37,10 +41,10 @@ def get_tokens(code: str) -> Set[str]:
     """Tokenize a code snippet."""
     return set(get_data(code))
 
-  
+
 def get_data(data):
-  return [t for t in NON_ALPHA.split(data) if len(t.strip()) > 0]
-  # return tokenizer(data)
+    return [t for t in NON_ALPHA.split(data) if len(t.strip()) > 0]
+    # return tokenizer(data)
 
 
 class DuplicationIndex:
@@ -72,7 +76,6 @@ def add(self, code_key: Tuple, min_hash: MinHash) -> None:
 
         self._index.insert(code_key, min_hash)
         if len(close_duplicates) > 0:
-
             for base_duplicate in close_duplicates:
                 if base_duplicate in self._duplicate_clusters:
                     self._duplicate_clusters[base_duplicate].add(code_key)
@@ -104,7 +107,7 @@ def save(self, filepath) -> None:
 
 def _compute_min_hash(element):
     index, data = element
-    min_hash = get_min_hash(get_data(data['text']))
+    min_hash = get_min_hash(get_data(data["text"]))
     if min_hash is not None:
         return index, min_hash
 
@@ -129,7 +132,11 @@ def make_duplicate_clusters(dataset_iterator: Type[Dataset], jaccard_threshold:
     """
     di = DuplicationIndex(duplication_jaccard_threshold=jaccard_threshold)
 
-    for filename, min_hash in tqdm(ThreadedIterator(minhash_iter(enumerate(dataset_iterator)), max_queue_size=10000), total=len(dataset_iterator), desc="Deduplication"):
+    for filename, min_hash in tqdm(
+        ThreadedIterator(minhash_iter(enumerate(dataset_iterator)), max_queue_size=10000),
+        total=len(dataset_iterator),
+        desc="Deduplication",
+    ):
         di.add(filename, min_hash)
 
     # Returns a List[Cluster] where Cluster is List[str] with the filenames.
@@ -142,8 +149,10 @@ def jaccard_similarity(code1: str, code2: str) -> float:
     tokens2 = get_tokens(code2)
     return len(tokens1 & tokens2) / len(tokens1 | tokens2)
 
+
 _shared_dataset = None
 
+
 def _find_cluster_extremes_shared(cluster, jaccard_threshold):
     """Find a reduced cluster such that each code in the origin cluster is similar to at least one code in the reduced cluster.
     Two codes are similar if their Jaccard similarity is above the threshold.
@@ -175,13 +184,15 @@ def _find_cluster_extremes_shared(cluster, jaccard_threshold):
             extremes.append(element1)
     return extremes
 
+
 def _find_cluster_extremes_shared_semantic(cluster, jaccard_threshold):
     extremes = []
     # Convert code snippets to embeddings
     model = get_model()
-    code_embeddings = {element["base_index"]: model.encode(_shared_dataset[element["base_index"]]["text"])
-                       for element in cluster}
-    
+    code_embeddings = {
+        element["base_index"]: model.encode(_shared_dataset[element["base_index"]]["text"]) for element in cluster
+    }
+
     for element1 in cluster:
         embedding1 = code_embeddings[element1["base_index"]]
         for element2 in extremes:
@@ -195,6 +206,7 @@ def _find_cluster_extremes_shared_semantic(cluster, jaccard_threshold):
             extremes.append(element1)
     return extremes
 
+
 def find_extremes(cluster_list, dataset, jaccard_threshold):
     """Call the _find_cluster_extremes_shared function in a parallel fashion.
     Args:
@@ -224,11 +236,12 @@ def find_extremes(cluster_list, dataset, jaccard_threshold):
                 cluster_list,
             ),
             total=len(cluster_list),
-            desc="Finding overlaps"
+            desc="Finding overlaps",
         ):
             extremes_list.append(extremes)
     return extremes_list
 
+
 def find_extremes_semantic(cluster_list, dataset, jaccard_threshold):
     """Call the _find_cluster_extremes_shared_semantic function in a parallel fashion.
     Args:
@@ -258,18 +271,19 @@ def find_extremes_semantic(cluster_list, dataset, jaccard_threshold):
                 cluster_list,
             ),
             total=len(cluster_list),
-            desc="Finding overlaps semantically"
+            desc="Finding overlaps semantically",
         ):
             extremes_list.append(extremes)
     return extremes_list
 
+
 def display_dataset_entries(dataset, duplicate_clusters):
     """
     Fetches and displays the dataset entries for given base_index values in duplicate clusters.
 
     Args:
         dataset (Dataset): The dataset object containing the code snippets.
-        duplicate_clusters (List[List[Dict]]): List of duplicate clusters, each containing dictionaries 
+        duplicate_clusters (List[List[Dict]]): List of duplicate clusters, each containing dictionaries
                                                with 'base_index', 'is_extreme', and 'copies' keys.
 
     Returns:
@@ -278,12 +292,13 @@ def display_dataset_entries(dataset, duplicate_clusters):
     for cluster in duplicate_clusters:
         print("Cluster:")
         for item in cluster:
-            base_index = item['base_index']
+            base_index = item["base_index"]
             data_entry = dataset[base_index]  # Assuming the dataset can be accessed by index
             print(f"Base Index: {base_index}, Data: {data_entry}")
 
         print("\n")  # Separate clusters by a newline for clarity
 
+
 def deduplicate_dataset(
     dataset: Type[Dataset], jaccard_threshold: float = 0.85
 ) -> Tuple[Type[Dataset], List[List[Dict]]]:
@@ -343,4 +358,4 @@ def deduplicate_dataset(
     print(f"Unique files in duplicate cluster: {len(extreme_dict)}")
     print(f"Filtered dataset size: {len(ds_filter)}")
 
-    return ds_filter, duplicate_clusters
+    return ds_filter, duplicate_clusters

PolyDeDupe/version.py

@@ -1,5 +1,5 @@
 _MAJOR = "0"
-_MINOR = "7"
+_MINOR = "8"
 # On main and in a nightly release the patch should be one ahead of the last
 # released build.
 _PATCH = "0"