Let apriori always use low_memory processing

Thanks to previous optimizations, processing with low_memory=True is
now almost as efficient as with low_memory=False, and allows to process
much larger datasets.

Removing processing with low_memory=False makes code simpler and allows
to generate itemsets by a generator, which saves more memory.

The downside is that we do not know in advance the number of itemsets to
process, thus it is displayed afterwards.  Note that commit 2f928cb
introduced a bug, the number of processing combinations was multiplied
by itemset's length, which explains why output is different now.

Author: dbarbier

mlxtend/frequent_patterns/apriori.py

@@ -61,40 +61,36 @@ def generate_new_combinations(old_combinations):
                     # early exit from for-loop skips else clause just below
                     break
             else:
-                yield from candidate
+                yield candidate
             j = j + 1
 
 
-def compute_supports_low_memory(X, is_sparse, combin):
-    supports = np.zeros(combin.shape[0])
-    ncomb, nitems = combin.shape
+def generate_supports_and_itemsets(X, is_sparse, combin, min_support):
+    counter = 0
     if is_sparse:
-        _bools = X[:, 0].toarray()
-        for c in range(ncomb):
-            _bools[:] = X[:, combin[c, 0]].toarray()
-            for j in range(1, nitems):
-                np.logical_and(_bools, X[:, combin[c, j]].toarray(), out=_bools)
-            supports[c] = np.count_nonzero(_bools)
+        count = np.empty(X.shape[0], dtype=int)
+        for itemset in combin:
+            counter += 1
+            count[:] = 0
+            for item in itemset:
+                # faster than X[:, item].toarray() or X.getcol(item).indices
+                count[X.indices[X.indptr[item]:X.indptr[item+1]]] += 1
+            support = np.count_nonzero(count == len(itemset))
+            if support >= min_support:
+                yield support
+                yield from itemset
     else:
-        _bools = np.copy(X[:, 0])
-        for c in range(ncomb):
-            _bools[:] = X[:, combin[c, 0]]
-            for j in range(1, nitems):
-                np.logical_and(_bools, X[:, combin[c, j]], out=_bools)
-            supports[c] = np.count_nonzero(_bools)
-    return supports
-
-
-def compute_supports(X, is_sparse, combin):
-    if is_sparse:
-        all_ones = np.ones((X.shape[0], 1))
-        _bools = np.array(X[:, combin[:, 0]] == all_ones)
-        for n in range(1, combin.shape[1]):
-            _bools = _bools & np.array(X[:, combin[:, n]] == all_ones)
-    else:
-        _bools = np.all(X[:, combin], axis=2)
-
-    return np.sum(_bools, axis=0)
+        for itemset in combin:
+            counter += 1
+            _bools = np.ones(X.shape[0], dtype=bool)
+            for item in itemset:
+                np.logical_and(_bools, X[:, item], out=_bools)
+            support = np.count_nonzero(_bools)
+            if support >= min_support:
+                yield support
+                yield from itemset
+    # return the total of processed itemsets as last element
+    yield counter
 
 
 def apriori(df, min_support=0.5, use_colnames=False, max_len=None, verbose=0,
@@ -206,13 +202,15 @@ def _support(_x, _n_rows, _is_sparse):
             X = df.values
         else:
             X = df.to_coo().tocsc()
+            X.eliminate_zeros()
         is_sparse = True
     elif hasattr(df, "sparse"):
         # DataFrame with SparseArray (pandas >= 0.24)
         if df.size == 0:
             X = df.values
         else:
             X = df.sparse.to_coo().tocsc()
+            X.eliminate_zeros()
         is_sparse = True
     else:
         # dense DataFrame
@@ -223,37 +221,26 @@ def _support(_x, _n_rows, _is_sparse):
     support_dict = {1: support[support >= min_support]}
     itemset_dict = {1: ary_col_idx[support >= min_support].reshape(-1, 1)}
     max_itemset = 1
-    rows_count = float(X.shape[0])
-
-    all_ones = np.ones((int(rows_count), 1))
 
     while max_itemset and max_itemset < (max_len or float('inf')):
         next_max_itemset = max_itemset + 1
 
         combin = generate_new_combinations(itemset_dict[max_itemset])
-        combin = np.fromiter(combin, dtype=int)
-        combin = combin.reshape(-1, next_max_itemset)
-
-        if combin.size == 0:
-            break
-        if verbose:
-            print(
-                '\rProcessing %d combinations | Sampling itemset size %d' %
-                (combin.size, next_max_itemset), end="")
-
-        # With exceptionally large datasets, the matrix operations can use a
-        # substantial amount of memory. For low memory applications or large
-        # datasets, set `low_memory=True` to use a slower but more memory-
-        # efficient implementation.
-        if low_memory:
-            support = compute_supports_low_memory(X, is_sparse, combin) / rows_count
-        else:
-            support = compute_supports(X, is_sparse, combin) / rows_count
-
-        _mask = (support >= min_support)
-        if any(_mask):
-            itemset_dict[next_max_itemset] = np.array(combin[_mask])
-            support_dict[next_max_itemset] = np.array(support[_mask])
+        min_rows = np.math.ceil(min_support * X.shape[0])
+        gen_itemsets = generate_supports_and_itemsets(
+            X, is_sparse, combin, min_rows)
+
+        support_valid_itemsets = np.fromiter(gen_itemsets, dtype=int)
+        processed_itemsets = support_valid_itemsets[-1]
+        support_valid_itemsets = support_valid_itemsets[:-1]
+        if support_valid_itemsets.size > 0:
+            if verbose:
+                print(
+                    '\rProcessed %d combinations | Sampling itemset size %d' %
+                    (processed_itemsets, next_max_itemset), end="")
+            support_valid_itemsets.shape = (-1, 1 + next_max_itemset)
+            itemset_dict[next_max_itemset] = support_valid_itemsets[:, 1:]
+            support_dict[next_max_itemset] = support_valid_itemsets[:, 0] / X.shape[0]
             max_itemset = next_max_itemset
         else:
             # Exit condition

mlxtend/frequent_patterns/tests/test_fpbase.py

@@ -229,7 +229,7 @@ def test_low_memory_flag(self):
                 _ = self.fpalgo(self.df, low_memory=True, verbose=1)
 
             # Only get the last value of the stream to reduce test noise
-            expect = 'Processing 3 combinations | Sampling itemset size 3\n'
+            expect = 'Processed 1 combinations | Sampling itemset size 3\n'
             out = out.getvalue().split('\r')[-1]
             assert out == expect
         else: