Copyright 2017 J. Patrick Hall, jphall@gwu.edu
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
Based on:
Goldstein, Alex, Kapelner, Adam, Bleich, Justin, and Pitkin, Emil. Peeking inside the black box: Visualizing statistical learning with plots of individual conditional expectation. Journal of Computational and Graphical Statistics, 24(1):44–65, 2015. https://arxiv.org/pdf/1309.6392.pdf
Hastie, Trevor, Tibshirani, Robert, and Friedman, Jerome. The Elements of Statistical Learning. Springer, 2008. https://statweb.stanford.edu/~tibs/ElemStatLearn/printings/ESLII_print10.pdf
In [1]:
# imports
import h2o
import numpy as np
import pandas as pd
from h2o.estimators.gbm import H2OGradientBoostingEstimator
# display matplotlib graphics in notebook
import matplotlib.pyplot as plt
%matplotlib inline
In [2]:
# start h2o
h2o.init()
h2o.remove_all()
In [ ]:
In [3]:
# load clean data
path = '../../03_regression/data/train.csv'
frame = h2o.import_file(path=path)
In [4]:
# assign target and inputs
y = 'SalePrice'
X = [name for name in frame.columns if name not in [y, 'Id']]
In [5]:
# determine column types
# impute
reals, enums = [], []
for key, val in frame.types.items():
if key in X:
if val == 'enum':
enums.append(key)
else:
reals.append(key)
_ = frame[reals].impute(method='median')
_ = frame[enums].impute(method='mode')
In [6]:
# split into training an validation, and 30% test
train, valid = frame.split_frame([0.7])
In [7]:
# train GBM model
model = H2OGradientBoostingEstimator(ntrees=100,
max_depth=10,
distribution='huber',
learn_rate=0.1,
stopping_rounds=5,
seed=12345)
model.train(y=y, x=X, training_frame=train, validation_frame=valid)
In [8]:
model.varimp_plot()
In [9]:
# manually calculate 1-D partial dependence
# for educational purposes
def par_dep(xs, frame, model, resolution=20, bins=None):
""" Creates Pandas dataframe containing partial dependence for a single variable.
Args:
xs: Variable for which to calculate partial dependence.
frame: Data for which to calculate partial dependence.
model: Model for which to calculate partial dependence.
resolution: The number of points across the domain of xs for which to calculate partial dependence.
Returns:
Pandas dataframe containing partial dependence values.
"""
# init empty Pandas frame w/ correct col names
par_dep_frame = pd.DataFrame(columns=[xs, 'partial_dependence'])
# cache original data
col_cache = h2o.deep_copy(frame[xs], xid='col_cache')
# determine values at which to calculate partial dependency
if bins == None:
min_ = frame[xs].min()
max_ = frame[xs].max()
by = (max_ - min_)/resolution
bins = np.arange(min_, max_, by)
# calculate partial dependency
# by setting column of interest to constant
for j in bins:
frame[xs] = j
par_dep_i = model.predict(frame)
par_dep_j = par_dep_i.mean()[0]
par_dep_frame = par_dep_frame.append({xs:j,
'partial_dependence': par_dep_j},
ignore_index=True)
# return input frame to original cached state
frame[xs] = h2o.get_frame('col_cache')
return par_dep_frame
# show some output
par_dep_OverallQual = par_dep('OverallQual', valid, model)
par_dep_OverallQual.plot.line(x='OverallQual', y='partial_dependence')
print()
print(par_dep_OverallQual)
In [10]:
# use h2o to calculate 1-D partial dependence
# (easy, fast)
model.partial_plot(data=valid, cols=['OverallQual'], plot=True, plot_stddev=True)
Out[10]:
In [11]:
GrLivArea_housingMedianAge = model.partial_plot(data=valid, cols=['GrLivArea'], plot=True, plot_stddev=True)
In [12]:
# manually calculate 2-D partial dependence
def par_dep_2d(xs1, xs2, frame, model, resolution=20):
""" Creates Pandas dataframe containing partial dependence for a two variables.
Args:
xs1: First variable for which to calculate partial dependence.
xs2: Second variable for which to calculate partial dependence.
frame: Data for which to calculate partial dependence.
model: Model for which to calculate partial dependence.
resolution: The number of points across the domain of xs for which to calculate partial dependence.
Returns:
Pandas dataframe containing partial dependence values.
"""
# init empty Pandas frame w/ correct col names
par_dep_frame = pd.DataFrame(columns=[xs1, xs2, 'partial_dependence'])
# cache original data
col_cache1 = frame[xs1]
col_cache2 = frame[xs2]
# determine values at which to calculate partial dependency
# for xs1
min1_ = frame[xs1].min()
max1_ = frame[xs1].max()
by1 = float((max1_ - min1_)/resolution)
range1 = np.arange(min1_, max1_, by1)
# determine values at which to calculate partial dependency
# for xs2
min2_ = frame[xs2].min()
max2_ = frame[xs2].max()
by2 = float((max2_ - min2_)/resolution)
range2 = np.arange(min2_, max2_, by2)
# calculate partial dependency
for j in range1:
for k in range2:
frame[xs1] = j
frame[xs2] = k
par_dep_i = model.predict(frame)
par_dep_j = par_dep_i.mean()[0]
std_j = model.predict(frame).sd()[0]
pos_std, neg_std = par_dep_j + std_j, par_dep_j - std_j
par_dep_frame = par_dep_frame.append({xs1:j,
xs2:k,
'partial_dependence': par_dep_j},
ignore_index=True)
# return input frame to original cached state
frame[xs1] = col_cache1
frame[xs2] = col_cache2
return par_dep_frame
# calculate 2-D partial dependence
h2o.no_progress()
resolution = 20
par_dep_OverallQual_v_GrLivArea = par_dep_2d('OverallQual',
'GrLivArea',
valid,
model,
resolution=resolution)
print()
print(par_dep_OverallQual_v_GrLivArea)
In [13]:
# create 2-D partial dependence plot
# imports
from mpl_toolkits.mplot3d import Axes3D
from matplotlib import cm
from matplotlib.ticker import LinearLocator, FormatStrFormatter
# create 3-D grid
new_shape = (resolution, resolution)
x = np.asarray(par_dep_OverallQual_v_GrLivArea['OverallQual']).reshape(new_shape)
y = np.asarray(par_dep_OverallQual_v_GrLivArea['GrLivArea']).reshape(new_shape)
z = np.asarray(par_dep_OverallQual_v_GrLivArea['partial_dependence']).reshape(new_shape)
fig = plt.figure(figsize=(8,6))
ax = plt.axes(projection='3d')
# set axes labels
ax.set_title('Partial Dependence for Sale Price')
ax.set_xlabel('OverallQual')
ax.set_ylabel('GrLivArea')
ax.set_zlabel('\nSale Price')
# axis decorators/details
#ax.zaxis.set_major_locator(LinearLocator(10))
#ax.zaxis.set_major_formatter(FormatStrFormatter('%.1f'))
# surface
surf = ax.plot_surface(x, y, z,
cmap=cm.coolwarm,
linewidth=0.05,
rstride=1,
cstride=1,
antialiased=True)
plt.tight_layout()
_ = plt.show()
In [14]:
def get_quantile_dict(y, id_, frame):
""" Returns the percentiles of a column y as the indices for another column id_.
Args:
y: Column in which to find percentiles.
id_: Id column that stores indices for percentiles of y.
frame: H2OFrame containing y and id_.
Returns:
Dictionary of percentile values and index column values.
"""
quantiles_df = frame.as_data_frame()
quantiles_df.sort_values(y, inplace=True)
quantiles_df.reset_index(inplace=True)
percentiles_dict = {}
percentiles_dict[0] = quantiles_df.loc[0, id_]
percentiles_dict[99] = quantiles_df.loc[quantiles_df.shape[0]-1, id_]
inc = quantiles_df.shape[0]//10
for i in range(1, 10):
percentiles_dict[i * 10] = quantiles_df.loc[i * inc, id_]
return percentiles_dict
quantile_dict = get_quantile_dict('SalePrice', 'Id', valid)
In [15]:
bins = list(par_dep_OverallQual['OverallQual'])
for i in sorted(quantile_dict.keys()):
col_name = 'Percentile_' + str(i)
par_dep_OverallQual[col_name] = par_dep('OverallQual',
valid[valid['Id'] == int(quantile_dict[i])],
model,
bins=bins)['partial_dependence']
par_dep_OverallQual
Out[15]:
In [16]:
fig, ax = plt.subplots()
par_dep_OverallQual.drop('partial_dependence', axis=1).plot(x='OverallQual', colormap='gnuplot', ax=ax)
par_dep_OverallQual.plot(title='Partial Dependence and ICE for Sales Price',
x='OverallQual',
y='partial_dependence',
style='r-',
linewidth=3,
ax=ax)
_ = plt.legend(bbox_to_anchor=(1.05, 0),
loc=3,
borderaxespad=0.)
From this partial dependence and ICE plot, it can be seen that most individual percentiles of predicted sale price are well-represented by partial dependence, except for the most expensive houses, above the 90th percentile. Pricing for these homes appears to behave differently than the average behavior of other homes under the model.
In [17]:
h2o.cluster().shutdown(prompt=True)