

In [ ]:

    
import data_science.lendingclub.dataprep_and_modeling.modeling_utils.data_prep_new as data_prep
import dir_constants as dc
from sklearn.externals import joblib
import torch
import torch.nn as nn
import torch.optim as optim
from torch.autograd import Variable
import torch.nn.functional as F
from torch.utils.data import Dataset, DataLoader
import time
from sklearn.metrics import mean_squared_error
from tqdm import tqdm_notebook
import matplotlib.pyplot as plt
from  lendingclub.dataprep_and_modeling.model_dump.nn_1_0_0 import net_class
import torch
%matplotlib inline



In [ ]:

    
platform = 'lendingclub'
model_path = '/home/justin/justin_tinkering/data_science/lendingclub/dataprep_and_modeling/model_dump/nn_1_0_0/1.0.0_e500'

store = pd.HDFStore(
    dc.home_path+'/justin_tinkering/data_science/lendingclub/{0}_store.h5'.
    format(platform),
    append=True)

Make sure no loan in test set was in train set



In [ ]:

    
store.open()
train = store['train_filtered_columns']
test = store['test_filtered_columns']
loan_npv_rois = store['loan_npv_rois']
default_series = test['target_strict']
results = store['results']
store.close()

train_ids = set(train.index.values)
test_ids = set(test.index.values)
assert len(train_ids.intersection(test_ids)) == 0

Examine performance on test set



In [ ]:

    
test_X, test_y = data_prep.process_data_test(test)
test_y = test_y['npv_roi_10'].values
net = net_class.Net()
net.load_state_dict(torch.load(model_path))
regr_version = '1.0.0'
test_yhat = net_class.torch_version(test_X,net)



In [ ]:

    
for col in test_X.columns:
    if len(test_X[test_X[col].isnull()]) > 0:
        print(col)

Short digression on examining feature importances (N/A)



In [ ]:

    
test['1.0.0_scores'] = test_yhat



In [ ]:

    
import matplotlib as mpl
mpl.rcParams['agg.path.chunksize'] = 10000
plt.figure(figsize=(12,9))
plt.plot(test['1.0.0_scores'], test['npv_roi_10'], 'o')
plt.show()

at scores above 0, check the pctage of positive returns



In [ ]:

    
pos_scores = test[test['1.0.0_scores'] >= 0]
ps_pos_returns = pos_scores[pos_scores['npv_roi_10'] > 0]



In [ ]:

    
len(ps_pos_returns)/len(pos_scores)

checking return per percentile below



In [ ]:

    
percentiles = np.arange(0,100,1)



In [ ]:

    
def eval_models(trials, port_size, available_loans, test, percentiles):
    results = {}
    pct_default = {}
    test_copy = test.copy()
    for trial in tqdm_notebook(np.arange(trials)):
        loan_ids = np.random.choice(
            test_copy.index.values, available_loans, replace=False)
        loans_to_pick_from = test_copy.loc[loan_ids, :]
        loans_to_pick_from.sort_values('1.0.0_scores', ascending=True, inplace = True)
        chunksize = int(len(loans_to_pick_from)/100)
        results_dict = {}
        for k,perc in enumerate(percentiles):
            subset = loans_to_pick_from[k*chunksize:(k+1)*chunksize]
            results_dict[perc] = subset['npv_roi_10'].mean()

        results[trial] = pd.Series(results_dict)
        
    return pd.DataFrame.from_dict(results).T


#         picks = scores_series[:900].index.values
#         results[trial] = loan_npv_rois.loc[picks, :].mean().to_dict()
#         pct_default[trial] = (default_series.loc[picks].sum()) / port_size
#     pct_default_series = pd.Series(pct_default)
#     results_df = pd.DataFrame(results).T
#     results_df['pct_def'] = pct_default_series
#     return results_df



In [ ]:

    
# as per done with baseline models, say 3000 loans available
# , pick 900 of them
trials = 20000
port_size = 900
available_loans = 3000
results = eval_models(trials, port_size, available_loans, test, percentiles)



In [ ]:

    
summaries = results.describe()



In [ ]:

    
summaries



In [ ]:

    
plt.figure(figsize=(12,9))
plt.plot(summaries.columns.values, summaries.loc['mean',:], 'o', label='mean')
plt.plot(summaries.columns.values, summaries.loc['25%',:], 'ro', label='25%')
# plt.plot(summaries.columns.values, summaries.loc['50%',:], '-.')
plt.plot(summaries.columns.values, summaries.loc['75%',:], 'ko', label='75%')
plt.title('return per percentile over batches')
plt.legend(loc='best')
plt.xlabel('percentile of 1.0.0_score')
plt.ylabel('npv_roi_10')
plt.show()



In [ ]:

    
store.open()
store['percentiles_for_1.0.0'] = results
store.close()



In [ ]: