

In [1]:

    
%load_ext autoreload
%autoreload 2
%matplotlib inline



In [2]:

    
import random

import numpy as np
import matplotlib.pyplot as plt

from sklearn.externals import joblib
from sklearn.neural_network import MLPClassifier
from sklearn.metrics import classification_report, roc_curve, auc
from sklearn.model_selection import RandomizedSearchCV, train_test_split

from game.utils import get_database_cursor
from game.constants import DIM, CLFILE



In [3]:

    
import warnings
warnings.filterwarnings('ignore')

Load and pre-process data:



In [4]:

    
with get_database_cursor() as cur:
    # Fetch data from the database and preprocess it:
    # 1. Average the 'winning' value over the equal 'state's
    # 2. Retrieve 'state' and 'winning' values from the averaged 
    #    ones such that the 'winning' value becomes equal to 0 if
    #    it's less than 0.5, and equal to 1 otherwise.
    cur.execute("""
        SELECT
            state,
            CASE WHEN winning < 0.5 THEN 0 ELSE 1 END AS winning
        FROM (
            SELECT
                state,
                SUM(winning) / COUNT(winning) AS winning
            FROM states
            GROUP BY state
        );
    """)
    rows = cur.fetchall()

if rows is None:
    raise TypeError('no training data found')

Define training X and target y vectors:



In [5]:

    
n = len(rows)
X = np.zeros((n, 2 * DIM * DIM))
y = np.zeros(n)

for i, row in enumerate(rows):
    state_str, winning = row
    X[i, :] = np.fromstring(state_str, dtype=int)
    y[i] = int(winning)

X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2)

Training examples:



In [6]:

    
pos_n = int(y.sum())
print('positives:', pos_n)
print('negatives:', len(y) - pos_n)









    



positives: 2845
negatives: 2768

Define a classifier model as a simple feed-forward neural network (MLP, multilayer perceptron):

Generate random network layer architectures:



In [7]:

    
N_ARCH = 1000
N_LAYERS_MIN = 1
N_LAYERS_MAX = 2
N_NEURONS_MIN = 100
N_NEURONS_MAX = 500

hidden_layer_sizes = []

for i in range(N_ARCH):
    n_layers = random.randint(N_LAYERS_MIN, N_LAYERS_MAX)
    random_size = []
    
    for layer in range(n_layers):
        n_neurons = random.randint(N_NEURONS_MIN, N_NEURONS_MAX)
        random_size.append(n_neurons)
        
    hidden_layer_sizes.append(tuple(random_size))

Train the classifier model:



In [8]:

    
# Number of random search iterations:
N_ITER = 100

clf = MLPClassifier(early_stopping=True)
params = {
    'hidden_layer_sizes': hidden_layer_sizes,
    'alpha': np.linspace(0.001, 0.05, num=100),
    'activation': ('logistic', 'tanh', 'relu')
}

rs = RandomizedSearchCV(estimator=clf, param_distributions=params,
                        n_jobs=2, n_iter=N_ITER, cv=5, scoring='f1', verbose=1)
rs.fit(X_train, y_train)

clf = rs.best_estimator_

# Save the best classifier model:
joblib.dump(clf, CLFILE);









    



Fitting 5 folds for each of 100 candidates, totalling 500 fits






    



[Parallel(n_jobs=2)]: Done  46 tasks      | elapsed:  9.2min
[Parallel(n_jobs=2)]: Done 196 tasks      | elapsed: 28.8min
[Parallel(n_jobs=2)]: Done 446 tasks      | elapsed: 61.2min
[Parallel(n_jobs=2)]: Done 500 out of 500 | elapsed: 66.4min finished



In [9]:

    
print('Best score:', rs.best_score_)
print('Best NN parameters found:', rs.best_params_)









    



Best score: 0.88112092273
Best NN parameters found: {'alpha': 0.001494949494949495, 'hidden_layer_sizes': (473, 461), 'activation': 'relu'}

Evaluate the predictions' quality:

Calculate main metrics:



In [10]:

    
print(classification_report(y_test, clf.predict(X_test)))









    



             precision    recall  f1-score   support

        0.0       0.84      0.89      0.87       550
        1.0       0.89      0.84      0.87       573

avg / total       0.87      0.87      0.87      1123

Plot a ROC curve:



In [11]:

    
y_score = clf.predict_proba(X_test)[:, 1]

fpr, tpr, _ = roc_curve(y_test, y_score)
roc_auc = auc(fpr, tpr)

plt.figure()
plt.plot(fpr, tpr, label='ROC curve (area = {:.2f})'.format(roc_auc))
plt.plot([0, 1], [0, 1], 'k--')
plt.xlim([0.0, 1.0])
plt.ylim([0.0, 1.0])
plt.xlabel('False Positive Rate')
plt.ylabel('True Positive Rate')
plt.title('Receiver operating characteristic')
plt.legend(loc="lower right")
plt.show()