higgsml/extratrees_submission.py at master · bourcierj/higgsml · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
"""Submission on Higgs ML Data with extra trees"""

import os
import argparse
import pickle
from pprint import pprint

import matplotlib.pyplot as plt
plt.style.use('ggplot')
from sklearn.ensemble import RandomForestClassifier
import numpy as np
import pandas as pd
from hyperopt import space_eval
from sklearn.impute import SimpleImputer

from extratrees_hyperopt import PARAMS_SPACE
from utils import *


def parse_args():
    parser = argparse.ArgumentParser()
    parser.add_argument('--seed', type=int, help='random generators seed (default: None)')
    parser.add_argument('--logdir', type=str, default='./', help='save directory')
    args = parser.parse_args()
    return args

if __name__ == '__main__':

    args = parse_args()

    if args.seed is not None:
        # random seed for reproducibility
        np.random.seed(42)

    df_train, df_test = get_kaggle_datasets()

    # prepare datasets
    imputer = SimpleImputer(strategy='median')
    train_data, train_target, train_weights = prepare_kaggle_dataset(df_train, True, imputer)
    test_data = prepare_kaggle_dataset(df_test, False, imputer)

    # load Trials object from hyperopt search in log dir
    with open(os.path.join(args.logdir, 'Trials-extratrees.pkl'), 'rb') as file:
        trials = pickle.load(file)

    #print("All Losses:\n", trials.losses())
    #print("First result\n:", trials.results[0])
    # get all losses
    all_ams = [-ams for ams in trials.losses()]
    # extract the best Trial among all
    bidx = np.argmax(all_ams)
    bresult = trials.results[bidx]
    ams, ams_var = -bresult['loss'], bresult['loss_variance'],
    threshold, threshold_var = bresult['threshold'], bresult['threshold_variance']

    bparams = trials.argmin
    bparams = space_eval(PARAMS_SPACE, bparams)
    # retrain the model with best set of parameters on full training set
    print('Training on train set...')
    clf = ExtraTreesClassifier(**bparams)
    clf.fit(train_data, train_target)
    print('Done\n')

    # predictions with best threshold on the test set
    test_preds = clf.predict_proba(test_data)[:, 1]

    # Write submission to a file
    rank_orders = np.argsort(test_preds) + 1
    # print('Test preds scores sorted: ', test_preds[rank_orders-1])
    # print('Test predictions sorted:', round_predictions(test_preds[rank_orders-1], threshold))
    df_submission = pd.DataFrame({'EventId': df_test['EventId'],
                                  'RankOrder': rank_orders,
                                  'Class': ['s' if y == 1 else 'b'
                                            for y in round_predictions(test_preds, threshold)]})
    df_submission.to_csv('submissions/randomforest_submission.csv', index=False)


    solution_path = 'data/solution_from_cern.csv'
    submission_path = 'submissions/randomforest_submission.csv'

    from HiggsBosonCompetition_AMSMetric_rev1 import AMS_metric
    AMS_metric(solution_path, submission_path)

    # From the AMS_metric() in HiggsBosonCompetition_AMSMetric_rev1.py:

    # signal = 400.6366116965329, background = 6524.502478690059
    # AMS = 4.906756338440252
    # => The scores are wrong, they are way too high!!

    # The submission on Kaggle showed only the score in private leaderboard, which is
    # the same as ours. Therefore we haven't made any errors in our evaluations.

    # RANK AGAINST THE PARTICIPANTS ON KAGGLE (exluding the late submissions we can't see):

    # On private leaderboard: 739 / 1785
    # On public leaderboard: 842 / 1785