add: runtime comparison script across libraries added

Author: cosmic-cortex

examples/runtime_comparison.py

@@ -0,0 +1,218 @@
+import numpy as np
+
+from time import time
+
+from sklearn.datasets import load_iris
+
+from acton.acton import main as acton_main
+
+from alp.active_learning.active_learning import ActiveLearner as ActiveLearnerALP
+
+from libact.base.dataset import Dataset
+from libact.labelers import IdealLabeler
+from libact.query_strategies import UncertaintySampling, QueryByCommittee
+from libact.query_strategies.multiclass.expected_error_reduction import EER
+from libact.models.logistic_regression import LogisticRegression as LogisticRegressionLibact
+
+from modAL.models import ActiveLearner, Committee
+from modAL.expected_error import expected_error_reduction
+
+from sklearn.linear_model import LogisticRegression
+
+
+runtime = {}
+
+
+def timeit(n_reps=10):
+
+    def timer(func):
+
+        def timed_func(*args, **kwargs):
+            start = time()
+            for _ in range(n_reps):
+                result = func(*args, **kwargs)
+            end = time()
+            print("%s has been executed in %f s avg for %d reps" % (func.__name__, (end - start)/n_reps, n_reps))
+            runtime[func.__name__] = (end - start)/n_reps
+            return result
+
+        return timed_func
+
+    return timer
+
+
+@timeit()
+def libact_uncertainty(X, y, n_queries):
+    y_train = np.array([None for _ in range(len(y))])
+    y_train[0], y_train[50], y_train[100] = 0, 1, 2
+    libact_train_dataset = Dataset(X, y_train)
+    libact_full_dataset = Dataset(X, y)
+    libact_learner = LogisticRegressionLibact(solver='liblinear', n_jobs=1, multi_class='ovr') #SVM(gamma='auto', probability=True)
+    libact_qs = UncertaintySampling(libact_train_dataset, model=libact_learner, method='lc')
+    libact_labeler = IdealLabeler(libact_full_dataset)
+    libact_learner.train(libact_train_dataset)
+
+    for _ in range(n_queries):
+        query_idx = libact_qs.make_query()
+        query_label = libact_labeler.label(X[query_idx])
+        libact_train_dataset.update(query_idx, query_label)
+        libact_learner.train(libact_train_dataset)
+
+
+@timeit()
+def libact_EER(X, y, n_queries):
+    y_train = np.array([None for _ in range(len(y))])
+    y_train[0], y_train[50], y_train[100] = 0, 1, 2
+    libact_train_dataset = Dataset(X, y_train)
+    libact_full_dataset = Dataset(X, y)
+    libact_learner = LogisticRegressionLibact(solver='liblinear', n_jobs=1, multi_class='ovr') #SVM(gamma='auto', probability=True)
+    libact_qs = EER(libact_train_dataset, model=libact_learner, loss='01')
+    libact_labeler = IdealLabeler(libact_full_dataset)
+    libact_learner.train(libact_train_dataset)
+
+    for _ in range(n_queries):
+        query_idx = libact_qs.make_query()
+        query_label = libact_labeler.label(X[query_idx])
+        libact_train_dataset.update(query_idx, query_label)
+        libact_learner.train(libact_train_dataset)
+
+
+@timeit()
+def libact_QBC(X, y, n_queries):
+    y_train = np.array([None for _ in range(len(y))])
+    y_train[0], y_train[50], y_train[100] = 0, 1, 2
+    libact_train_dataset = Dataset(X, y_train)
+    libact_full_dataset = Dataset(X, y)
+    libact_learner_list = [LogisticRegressionLibact(solver='liblinear', n_jobs=1, multi_class='ovr'),
+                           LogisticRegressionLibact(solver='liblinear', n_jobs=1, multi_class='ovr')]
+    libact_qs = QueryByCommittee(libact_train_dataset, models=libact_learner_list,
+                                 method='lc')
+    libact_labeler = IdealLabeler(libact_full_dataset)
+    for libact_learner in libact_learner_list:
+        libact_learner.train(libact_train_dataset)
+
+    for _ in range(n_queries):
+        query_idx = libact_qs.make_query()
+        query_label = libact_labeler.label(X[query_idx])
+        libact_train_dataset.update(query_idx, query_label)
+        for libact_learner in libact_learner_list:
+            libact_learner.train(libact_train_dataset)
+
+
+@timeit()
+def modAL_uncertainty(X, y, n_queries):
+    modAL_learner = ActiveLearner(LogisticRegression(solver='liblinear', n_jobs=1, multi_class='ovr'),
+                                  X_training=X[[0, 50, 100]], y_training=y[[0, 50, 100]])
+
+    for _ in range(n_queries):
+        query_idx, query_inst = modAL_learner.query(X)
+        modAL_learner.teach(X[query_idx], y[query_idx])
+
+
+@timeit()
+def modAL_QBC(X, y, n_queries):
+    learner_list = [ActiveLearner(LogisticRegression(solver='liblinear', n_jobs=1, multi_class='ovr'),
+                                  X_training=X[[0, 50, 100]], y_training=y[[0, 50, 100]]),
+                    ActiveLearner(LogisticRegression(solver='liblinear', n_jobs=1, multi_class='ovr'),
+                                  X_training=X[[0, 50, 100]], y_training=y[[0, 50, 100]])]
+
+    modAL_learner = Committee(learner_list)
+
+    for _ in range(n_queries):
+        query_idx, query_inst = modAL_learner.query(X)
+        modAL_learner.teach(X[query_idx], y[query_idx])
+
+
+@timeit()
+def modAL_EER(X, y, n_queries):
+    modAL_learner = ActiveLearner(LogisticRegression(solver='liblinear', n_jobs=1, multi_class='ovr'),
+                                  query_strategy=expected_error_reduction,
+                                  X_training=X[[0, 50, 100]], y_training=y[[0, 50, 100]])
+
+    for _ in range(n_queries):
+        query_idx, query_inst = modAL_learner.query(X)
+        modAL_learner.teach(X[query_idx], y[query_idx])
+
+
+@timeit()
+# acton requires a txt format for data
+def acton_uncertainty(data_path, n_queries):
+    # acton has no SVM support, so the LogisticRegression model is used
+    acton_main(
+        data_path=data_path,
+        feature_cols=['feat01', 'feat02', 'feat03', 'feat04'],
+        label_col='label',
+        output_path='out.csv',
+        n_epochs=n_queries,
+        initial_count=3,
+        recommender='UncertaintyRecommender',
+        predictor='LogisticRegression')
+
+
+@timeit()
+# acton requires a txt format for data
+def acton_QBC(data_path, n_queries):
+    # acton has no SVM support, so the LogisticRegression model is used
+    acton_main(
+        data_path=data_path,
+        feature_cols=['feat01', 'feat02', 'feat03', 'feat04'],
+        label_col='label',
+        output_path='out.csv',
+        n_epochs=n_queries,
+        initial_count=3,
+        recommender='QBCRecommender',
+        predictor='LogisticRegressionCommittee')
+
+
+@timeit()
+def alp_uncertainty(X, y, n_queries):
+    X_labeled, y_labeled = X[[0, 50, 100]], y[[0, 50, 100]]
+    estimator = LogisticRegression(solver='liblinear', n_jobs=1, multi_class='ovr')
+    estimator.fit(X_labeled, y_labeled)
+    learner = ActiveLearnerALP(strategy='least_confident')
+
+    for _ in range(n_queries):
+        query_idx = learner.rank(estimator, X, num_queries=1)
+        X_labeled = np.concatenate((X_labeled, X[query_idx]), axis=0)
+        y_labeled = np.concatenate((y_labeled, y[query_idx]), axis=0)
+        estimator.fit(X_labeled, y_labeled)
+
+
+@timeit()
+def alp_QBC(X, y, n_queries):
+    X_labeled, y_labeled = X[[0, 50, 100]], y[[0, 50, 100]]
+    estimators = [LogisticRegression(solver='liblinear', n_jobs=1, multi_class='ovr'),
+                  LogisticRegression(solver='liblinear', n_jobs=1, multi_class='ovr')]
+
+    for estimator in estimators:
+        estimator.fit(X_labeled, y_labeled)
+
+    learner = ActiveLearnerALP(strategy='vote_entropy')
+
+    for _ in range(n_queries):
+        query_idx = learner.rank(estimators, X, num_queries=1)
+        X_labeled = np.concatenate((X_labeled, X[query_idx]), axis=0)
+        y_labeled = np.concatenate((y_labeled, y[query_idx]), axis=0)
+        for estimator in estimators:
+            estimator.fit(X_labeled, y_labeled)
+
+
+def comparisons(n_queries=10):
+    # loading the data
+    X, y = load_iris(return_X_y=True)
+
+    libact_uncertainty(X, y, n_queries)
+    libact_QBC(X, y, n_queries)
+    libact_EER(X, y, n_queries)
+    acton_uncertainty('iris.csv', n_queries)
+    acton_QBC('iris.csv', n_queries)
+    alp_uncertainty(X, y, n_queries)
+    alp_QBC(X, y, n_queries)
+    modAL_uncertainty(X, y, n_queries)
+    modAL_QBC(X, y, n_queries)
+    modAL_EER(X, y, n_queries)
+
+
+if __name__ == '__main__':
+    comparisons()
+    print(runtime)