In [1]:
import ml_insights as mli
import numpy as np
import scipy as sp
import pandas as pd
import psycopg2
from sklearn.cross_validation import train_test_split
from sklearn.ensemble import RandomForestClassifier
import warnings
%matplotlib inline
warnings.filterwarnings('ignore')
In [2]:
def shuffle(df, n=1, axis=0):
df = df.copy()
for _ in range(n):
df.apply(np.random.shuffle, axis=axis)
return df
def myround(x, prec=2, base=.05):
return round(base * round(float(x)/base),prec)
def flatten(l):
return [item for sublist in l for item in sublist]
In [3]:
# cell 2: connection
conn = psycopg2.connect(
database='tradesy',
port=os.environ['REDSHIFT_PORT'],
password=os.environ['REDSHIFT_PASS'],
user=os.environ['REDSHIFT_USER'],
host=os.environ['REDSHIFT_HOST']
)
In [4]:
query = """
select * from saleability_model_v2 limit 1000
"""
df = pd.read_sql(query, conn)
In [5]:
domain = [u'shipping_price_ratio',
u'asking_price', u'price_level', u'brand_score', u'a_over_b', u'a',
u'favorite_count', u'b', u'purchase_count', u'has_blurb', u'has_image',
u'seasonal_component', u'description_length', u'sold_similarity',
u'unsold_similarity', u'description_similarity_delta']
In [6]:
categorical_features = [u'has_blurb', u'has_image']
In [7]:
y = df.purchase_dummy
X = df[domain]
X_train, X_test, y_train, y_test = train_test_split(X,y, test_size = .2)
In [8]:
rf = RandomForestClassifier(n_estimators=250, class_weight='auto')
rf.fit(X_train, y_train)
Out[8]:
In [9]:
pred = rf.predict(X_test)
from sklearn.metrics import classification_report
print classification_report(y_test, pred)
In [10]:
#need to convert integers to floats for lime to work
for column in X_train.columns:
X_train[column] = X_train[column].astype(float)
X_test[column] = X_test[column].astype(float)
In [11]:
# This took 23 minutes!!!
mxr = mli.ModelXRay(rf, X_train)
In [23]:
# Partial dependence plots as described in Goldstein (Individual Conditional Expectation plots)
# num_pts = number of observations to plot
# Each line represents effect of varying feature X for that particular observation. Base value is marked by the blue dot.
# Y axis is predicted target
indices = mxr.feature_dependence_plots(num_pts=10)
In [21]:
# shows disctribution of feature effects
mxr.feature_effect_summary()
In [22]:
# gven the two points evaluates target in both scenarios and determines which feature creates the largers change in target
# args: index_1, index_2
diff_path_obj = mxr.explain_prediction_difference(193,300, tol=.05)
In [ ]:
In [ ]: