Comparing Keras and Scikit models deployed on Cloud AI Platform with the What-if Tool

In [ ]:

    

import sys
python_version = sys.version_info[0]


    

In [0]:

    

# If you're running on Colab, you'll need to install the What-if Tool package and authenticate
def pip_install(module):
    if python_version == '2':
        !pip install {module} --quiet
    else:
        !pip3 install {module} --quiet

try:
    import google.colab
    IN_COLAB = True
except:
    IN_COLAB = False

if IN_COLAB:
    pip_install('witwidget')

    from google.colab import auth
    auth.authenticate_user()


    

In [0]:

    

import pandas as pd
import numpy as np
import tensorflow as tf
import witwidget
import os
import pickle

from tensorflow.keras.layers import Dense
from tensorflow.keras.models import Sequential

from sklearn.utils import shuffle
from sklearn.linear_model import LinearRegression
from witwidget.notebook.visualization import WitWidget, WitConfigBuilder

# This has been tested on TF 1.14
print(tf.__version__)


    

In [0]:

    

!wget 'http://archive.ics.uci.edu/ml/machine-learning-databases/wine-quality/winequality-white.csv'


    

In [0]:

    

data = pd.read_csv('winequality-white.csv', index_col=False, delimiter=';')
data = shuffle(data, random_state=4)


    

In [0]:

    

data.head()


    

In [0]:

    

labels = data['quality']


    

In [0]:

    

print(labels.value_counts())


    

In [0]:

    

data = data.drop(columns=['quality'])


    

In [0]:

    

train_size = int(len(data) * 0.8)
train_data = data[:train_size]
train_labels = labels[:train_size]

test_data = data[train_size:]
test_labels = labels[train_size:]


    

In [0]:

    

train_data.head()


    

In [0]:

    

# This is the size of the array we'll be feeding into our model for each wine example
input_size = len(train_data.iloc[0])
print(input_size)


    

In [0]:

    

model = Sequential()
model.add(Dense(200, input_shape=(input_size,), activation='relu'))
model.add(Dense(50, activation='relu'))
model.add(Dense(25, activation='relu'))
model.add(Dense(1))

model.compile(loss='mean_squared_error', optimizer='adam')


    

In [0]:

    

model.summary()


    

In [0]:

    

model.fit(train_data.values,train_labels.values, epochs=4, batch_size=32, validation_split=0.1)


    

In [0]:

    

# Update these to your own GCP project + model names
GCP_PROJECT = 'your_gcp_project'
KERAS_MODEL_BUCKET = 'gs://your_storage_bucket'
KERAS_VERSION_NAME = 'v1'


    

In [0]:

    

# Add the serving input layer below in order to serve our model on AI Platform
class ServingInput(tf.keras.layers.Layer):
  # the important detail in this boilerplate code is "trainable=False"
  def __init__(self, name, dtype, batch_input_shape=None):
    super(ServingInput, self).__init__(trainable=False, name=name, dtype=dtype, batch_input_shape=batch_input_shape)
  def get_config(self):
    return {'batch_input_shape': self._batch_input_shape, 'dtype': self.dtype, 'name': self.name }

restored_model = model

serving_model = tf.keras.Sequential()
serving_model.add(ServingInput('serving', tf.float32, (None, input_size)))
serving_model.add(restored_model)
tf.contrib.saved_model.save_keras_model(serving_model, os.path.join(KERAS_MODEL_BUCKET, 'keras_export'))  # export the model to your GCS bucket
export_path = KERAS_MODEL_BUCKET + '/keras_export'


    

In [0]:

    

# Configure gcloud to use your project
!gcloud config set project $GCP_PROJECT


    

In [0]:

    

# Create a new model in our project, you only need to run this once
!gcloud ai-platform models create keras_wine


    

In [0]:

    

# Deploy the model to Cloud AI Platform
!gcloud beta ai-platform versions create $KERAS_VERSION_NAME --model keras_wine \
--origin=$export_path \
--python-version=3.5 \
--runtime-version=1.14 \
--framework='TENSORFLOW'


    

In [0]:

    

%%writefile predictions.json
[7.8, 0.21, 0.49, 1.2, 0.036, 20.0, 99.0, 0.99, 3.05, 0.28, 12.1]


    

In [0]:

    

# Test the deployed model on an example from our test set
# The correct score for this prediction is 7
prediction = !gcloud ai-platform predict --model=keras_wine --json-instances=predictions.json --version=$KERAS_VERSION_NAME
print(prediction[1])


    

In [0]:

    

SKLEARN_VERSION_NAME = 'v1'
SKLEARN_MODEL_BUCKET = 'gs://sklearn_model_bucket'


    

In [0]:

    

scikit_model = LinearRegression().fit(train_data.values, train_labels.values)


    

In [0]:

    

# Export the model to a local file using pickle
pickle.dump(scikit_model, open('model.pkl', 'wb'))


    

In [0]:

    

# Copy the saved model to Cloud Storage
!gsutil cp ./model.pkl gs://wine_sklearn/model.pkl


    

In [0]:

    

# Create a new model in our project, you only need to run this once
!gcloud ai-platform models create sklearn_wine


    

In [0]:

    

!gcloud beta ai-platform versions create $SKLEARN_VERSION_NAME --model=sklearn_wine \
--origin=$SKLEARN_MODEL_BUCKET \
--runtime-version=1.14 \
--python-version=3.5 \
--framework='SCIKIT_LEARN'


    

In [0]:

    

# Test the model usnig the same example instance from above
!gcloud ai-platform predict --model=sklearn_wine --json-instances=predictions.json --version=$SKLEARN_VERSION_NAME


    

In [0]:

    

# Create np array of test examples + their ground truth labels
test_examples = np.hstack((test_data[:200].values,test_labels[:200].values.reshape(-1,1)))
print(test_examples.shape)


    

In [0]:

    

# Create a What-if Tool visualization, it may take a minute to load
# See the cell below this for exploration ideas

# We use `set_predict_output_tensor` here becuase our tf.keras model returns a dict with a 'sequential' key

config_builder = (WitConfigBuilder(test_examples.tolist(), data.columns.tolist() + ['quality'])
  .set_ai_platform_model(GCP_PROJECT, 'keras_wine', KERAS_VERSION_NAME).set_predict_output_tensor('sequential').set_uses_predict_api(True)
  .set_target_feature('quality')
  .set_model_type('regression')
  .set_compare_ai_platform_model(GCP_PROJECT, 'sklearn_wine', SKLEARN_VERSION_NAME))
WitWidget(config_builder, height=800)