Merge pull request #86 from zStupan/main

squashing refactor

Author: zStupan

niaarm/preprocessing.py

@@ -4,8 +4,18 @@
 from niaarm.dataset import Dataset
 
 
-def _euclidean(u, v):
-    return 1 - np.linalg.norm(u - v)
+def _euclidean(u, v, features):
+    dist = 0
+    for f in features:
+        if f.dtype == 'cat':
+            weight = 1 / len(f.categories)
+            if u[f.name] != v[f.name]:
+                dist += weight * weight
+        else:
+            weight = 1 / (f.max_val - f.min_val)
+            dist += (u[f.name] - v[f.name]) * (u[f.name] - v[f.name]) * weight * weight
+
+    return 1 - (dist ** 0.5)
 
 
 def _cosine_similarity(u, v):
@@ -37,41 +47,30 @@ def squash(dataset, threshold, similarity='euclidean'):
     transactions_dummies = pd.get_dummies(dataset.transactions).to_numpy()
     num_transactions = len(transactions)
 
-    if similarity == 'euclidean':
-        features_min = np.min(transactions_dummies, axis=0)
-        features_max = np.max(transactions_dummies, axis=0)
-        transactions_dummies = transactions_dummies / (features_max - features_min)
-
-    distance = _euclidean if similarity == 'euclidean' else _cosine_similarity
     squashed = np.zeros(num_transactions, dtype=bool)
     squashed_transactions = pd.DataFrame(columns=transactions.columns, dtype=int)
 
-    pos = 0
-    while pos < num_transactions:
-        squashed_set = pd.DataFrame(columns=transactions.columns, dtype=int)
-        while pos < num_transactions and squashed[pos]:
-            pos += 1
-        if pos + 1 < num_transactions:
-            transaction = pd.DataFrame(transactions.iloc[pos].to_dict(), index=[0])
-            squashed_set = pd.concat([squashed_set, transaction], ignore_index=True)
-            squashed[pos] = True
-
-            i = pos + 1
-            while i < num_transactions:
-                while i < num_transactions and squashed[i]:
-                    i += 1
-
-                if i < num_transactions:
-                    if distance(transactions_dummies[pos], transactions_dummies[i]) >= threshold:
-                        transaction = pd.DataFrame(transactions.iloc[i].to_dict(), index=[0])
-                        squashed_set = pd.concat([squashed_set, transaction], ignore_index=True)
-                        squashed[i] = True
-                i += 1
+    for pos in range(num_transactions):
+        if squashed[pos]:
+            continue
+
+        squashed_set = transactions.iloc[pos:pos + 1]
+        squashed[pos] = True
+
+        for i in range(pos + 1, num_transactions):
+            if squashed[i]:
+                continue
+            if similarity == 'euclidean':
+                distance = _euclidean(transactions.iloc[pos], transactions.iloc[i], dataset.features)
+            else:
+                distance = _cosine_similarity(transactions_dummies[pos], transactions_dummies[i])
+
+            if distance >= threshold:
+                squashed_set = pd.concat([squashed_set, transactions.iloc[i:i + 1]], ignore_index=True)
+                squashed[i] = True
 
         if not squashed_set.empty:
             squashed_transaction = squashed_set.agg(_mean_or_mode)
             squashed_transactions = pd.concat([squashed_transactions, squashed_transaction], ignore_index=True)
 
-        pos += 1
-
     return Dataset(squashed_transactions)