In [113]:

    

%matplotlib inline
from pprint import pprint
from matplotlib.pyplot import *
import json
with open('/home/rmcgibbo/biox3/projects/hyperopt-mixtape/met-enk-tpe/results-0e18519.json') as f:
    results = [e['result'] for e in json.load(f)]


    

In [114]:

    




    

In [121]:

    

score = [-e['loss'] for e in results]
cumulative_best_indices = [np.argmax(score[:i]) for i in range(1, len(results))]
best_score_by_iter = [score[i] for i in cumulative_best_indices]


pprint(results[cumulative_best_indices[-1]])
plot(best_score_by_iter)
plot(xlim(), [best_score_by_iter[-1]]*2, 'k--')


    

    




{u'fit_times': [37.437878131866455, 49.7844820022583, 54.609161138534546],
 u'loss': -3.9761071083545274,
 u'parameters': {u'DihedralFeaturizer__sincos': True,
                 u'DihedralFeaturizer__types': [u'phi', u'psi', u'chi1'],
                 u'KMeans__copy_x': True,
                 u'KMeans__init': u'k-means++',
                 u'KMeans__max_iter': 300,
                 u'KMeans__n_clusters': 500,
                 u'KMeans__n_init': 1,
                 u'KMeans__n_jobs': 1,
                 u'KMeans__precompute_distances': True,
                 u'KMeans__random_state': None,
                 u'KMeans__tol': 0.0001,
                 u'KMeans__verbose': 0,
                 u'MarkovStateModel__ergodic_cutoff': 1,
                 u'MarkovStateModel__lag_time': 1,
                 u'MarkovStateModel__n_timescales': 3,
                 u'MarkovStateModel__prior_counts': 0,
                 u'MarkovStateModel__reversible_type': u'transpose',
                 u'MarkovStateModel__verbose': False,
                 u'tICA__gamma': 1e-05,
                 u'tICA__lag_time': 1,
                 u'tICA__n_components': 4,
                 u'tICA__weighted_transform': True},
 u'status': u'ok',
 u'test_scores': [3.9779918200010242, 3.9722745422400343, 3.978054962822524],
 u'train_scores': [3.985601321342712, 3.9779289783568026, 3.980673795374737]}






    
Out[121]:





[<matplotlib.lines.Line2D at 0x7fbc1e5be210>]






    

In [206]:

    

def plot_vs_score(key, results=results, type='scatter'):
    keys = key.split('.')
    score = [-e['loss'] for e in results]
    if len(keys) == 1:
        x = [e.get(keys[0], np.nan) for e in results]
    else:
        x = [e.get(keys[0]).get(keys[1], np.nan) for e in results]

    x = np.asarray(x)
    y = np.asarray(score)

    if type == 'scatter':
        scatter(x, score)
        ylim(3.8, 4.0)    
    elif type == 'boxplot':
        unique_x = np.unique(x[np.isfinite(x)])
        boxplot([y[x==xx] for xx in unique_x])
        xticks(1+np.arange(len(unique_x)), unique_x)
        
    xlabel(key)
    ylabel('Score')


    

In [212]:

    

figure()
plot_vs_score('parameters.KMeans__n_clusters', type='scatter')

figure()
plot_vs_score('parameters.tICA__n_components', type='boxplot')

figure()
plot_vs_score('parameters.tICA__weighted_transform', type='boxplot')


    

In [202]:

    

with open('/home/rmcgibbo/biox3/projects/hyperopt-mixtape/met-enk-rand/results-e8ada65.json') as f:
    results_rand= [e['result'] for e in json.load(f)]


    

In [203]:

    

score = [-e['loss'] for e in results_rand]
cumulative_best_indices = [np.argmax(score[:i]) for i in range(1, len(results_rand))]
best_score_by_iter = [score[i] for i in cumulative_best_indices]


pprint(results[cumulative_best_indices[-1]])
plot(best_score_by_iter)
plot(xlim(), [best_score_by_iter[-1]]*2, 'k--')


    

    




{u'fit_times': [25.993999004364014, 35.85835385322571, 34.88067579269409],
 u'loss': -3.9650002221958487,
 u'parameters': {u'DihedralFeaturizer__sincos': True,
                 u'DihedralFeaturizer__types': [u'phi', u'psi', u'chi1'],
                 u'KMeans__copy_x': True,
                 u'KMeans__init': u'k-means++',
                 u'KMeans__max_iter': 300,
                 u'KMeans__n_clusters': 290,
                 u'KMeans__n_init': 1,
                 u'KMeans__n_jobs': 1,
                 u'KMeans__precompute_distances': True,
                 u'KMeans__random_state': None,
                 u'KMeans__tol': 0.0001,
                 u'KMeans__verbose': 0,
                 u'MarkovStateModel__ergodic_cutoff': 1,
                 u'MarkovStateModel__lag_time': 1,
                 u'MarkovStateModel__n_timescales': 3,
                 u'MarkovStateModel__prior_counts': 0,
                 u'MarkovStateModel__reversible_type': u'transpose',
                 u'MarkovStateModel__verbose': False,
                 u'tICA__gamma': 1e-07,
                 u'tICA__lag_time': 1,
                 u'tICA__n_components': 3,
                 u'tICA__weighted_transform': False},
 u'status': u'ok',
 u'test_scores': [3.9744250446639784, 3.9596017453175776, 3.96097387660599],
 u'train_scores': [3.984794237656074, 3.9660418320770363, 3.9653179121909816]}






    
Out[203]:





[<matplotlib.lines.Line2D at 0x7fbc1e11b710>]






    

In [213]:

    

figure()
plot_vs_score('parameters.KMeans__n_clusters', results=results_rand, type='scatter')

figure()
plot_vs_score('parameters.tICA__n_components', results=results_rand, type='scatter')

figure()
plot_vs_score('parameters.tICA__weighted_transform', results=results_rand, type='scatter')


    

In [ ]: