improve memory estimation for interactions

Author: paulbkoch

docs/benchmarks/ebm-benchmark.ipynb

@@ -832,6 +832,12 @@
     "    \n",
     "    # Train\n",
     "    print(f\"FIT: {global_counter}, {trial.task.origin}, {trial.task.name}, {trial.method}, {trial.meta}, classes:{trial.task.n_classes}, features:{fit_params['X'].shape[1]}, train_samples:{fit_params['X'].shape[0]}, orig_samples:{trial.task.n_samples}\")\n",
+    "\n",
+    "    if isinstance(est, (ExplainableBoostingClassifier, ExplainableBoostingRegressor)):\n",
+    "        n_bytes = est.estimate_mem(fit_params[\"X\"])\n",
+    "        print(f\"EBM Memory Required: {n_bytes}\")\n",
+    "        trial.log(\"mem\", n_bytes)\n",
+    "    \n",
     "    with warnings.catch_warnings():\n",
     "        warnings.filterwarnings(\"ignore\")\n",
     "        gc.collect()  # clean out garbage to have as much memory available as possible\n",

python/interpret-core/interpret/glassbox/_ebm/_ebm.py

@@ -1701,6 +1701,7 @@ def estimate_mem(self, X):
             Estimated memory usage in bytes.
             The estimate does not include the memory from the
             caller's copy of X, nor the process's code or other data.
+            The estimate will be more accurate for larger datasets.
         """
 
         # number of classes does not affect memory much, so choose a sensible default
@@ -1771,6 +1772,11 @@ def estimate_mem(self, X):
             max_bytes = max(max_bytes, interaction_detection_bytes)
 
             interaction_multiple = float(interactions) / float(n_features_in)
+            # We merge the interactions together to make a combined interaction
+            # dataset, so if feature1 takes 4 bits and feature2 takes 10 bits
+            # then the resulting data storage should take approx 14 bits in total,
+            # so as a loose approximation we can add the bits in a pair.
+            interaction_multiple *= 2.0
             interaction_boosting_bytes = n_bytes_pairs + int(
                 n_bytes_pairs * interaction_multiple * self.outer_bags
             )