In case of doubts refer to the documentation on https://clipper.ai.


In [2]:
import torch
import torch.nn as nn
import numpy as np

In [3]:
# Hyper-parameters
input_size = 1
output_size = 1
num_epochs = 60
learning_rate = 0.001

# Toy dataset
x_train = np.array([[3.3], [4.4], [5.5], [6.71], [6.93], [4.168], 
                    [9.779], [6.182], [7.59], [2.167], [7.042], 
                    [10.791], [5.313], [7.997], [3.1]], dtype=np.float32)

y_train = np.array([[1.7], [2.76], [2.09], [3.19], [1.694], [1.573], 
                    [3.366], [2.596], [2.53], [1.221], [2.827], 
                    [3.465], [1.65], [2.904], [1.3]], dtype=np.float32)

# Linear regression model
model = nn.Linear(input_size, output_size)

# Loss and optimizer
criterion = nn.MSELoss()
optimizer = torch.optim.SGD(model.parameters(), lr=learning_rate)  

# Train the model
for epoch in range(num_epochs):
    # Convert numpy arrays to torch tensors
    inputs = torch.from_numpy(x_train)
    targets = torch.from_numpy(y_train)

    # Forward pass
    outputs = model(inputs)
    loss = criterion(outputs, targets)
    
    # Backward and optimize
    optimizer.zero_grad()
    loss.backward()
    optimizer.step()
    
    if (epoch+1) % 5 == 0:
        print ('Epoch [{}/{}], Loss: {:.4f}'.format(
            epoch+1, num_epochs, loss.item()))


Epoch [5/60], Loss: 20.1887
Epoch [10/60], Loss: 8.3919
Epoch [15/60], Loss: 3.6126
Epoch [20/60], Loss: 1.6761
Epoch [25/60], Loss: 0.8914
Epoch [30/60], Loss: 0.5731
Epoch [35/60], Loss: 0.4439
Epoch [40/60], Loss: 0.3913
Epoch [45/60], Loss: 0.3697
Epoch [50/60], Loss: 0.3607
Epoch [55/60], Loss: 0.3567
Epoch [60/60], Loss: 0.3548

In [4]:
# Create a directory for the model
%mkdir -p /tmp/pytorch

In [5]:
# Save the model to a file and remove it, we will reload it in the next step
torch.save(model.state_dict(), "/tmp/pytorch/model.pt")
print("Model saved")
del(model)
# Let's check if the model variable is really gone
try: 
    model
except NameError:
    print("Model variable removal successful")


Model saved
Model variable removal successful

In [6]:
# Load the model from the file, remember you must define the model type first 
# (for example nn.Linear)
model = nn.Linear(input_size, output_size)
model.load_state_dict(torch.load("/tmp/pytorch/model.pt"))
# Check model type
model.eval()


Out[6]:
Linear(in_features=1, out_features=1, bias=True)

In [7]:
# Predict function needs the model variable and inputs
def predict(model, inputs):
    pred = torch.from_numpy(np.array(inputs, dtype=np.float32))
    pred = model(pred)
    pred = pred.data.numpy()
    return [str(x) for x in pred]
# Test on 2 values
predict(model, [[2.0], [4.0]])


Out[7]:
['[ 0.41221234]', '[ 1.26506734]']

In [8]:
from clipper_admin import ClipperConnection, DockerContainerManager
from clipper_admin.deployers.pytorch import deploy_pytorch_model
clipper_conn = ClipperConnection(DockerContainerManager())

In [9]:
clipper_conn.start_clipper()


19-05-30:16:07:11 INFO     [docker_container_manager.py:154] [default-cluster] Starting managed Redis instance in Docker
19-05-30:16:07:14 INFO     [docker_container_manager.py:232] [default-cluster] Metric Configuration Saved at /tmp/tmpfgf70_7q.yml
19-05-30:16:07:14 INFO     [clipper_admin.py:143] [default-cluster] Clipper is running

In [10]:
# You can see some Clipper containers by simply running:
!docker ps


CONTAINER ID        IMAGE                                 COMMAND                  CREATED             STATUS              PORTS                                            NAMES
26534b53dd6e        prom/prometheus:v2.1.0                "/bin/prometheus --c…"   2 seconds ago       Up 2 seconds        0.0.0.0:9090->9090/tcp                           metric_frontend-75355
dcf4a6516ccf        clipper/frontend-exporter:develop     "python /usr/src/app…"   3 seconds ago       Up 2 seconds                                                         query_frontend_exporter-11466
44e204a13566        clipper/query_frontend:develop        "/clipper/release/sr…"   4 seconds ago       Up 3 seconds        0.0.0.0:1337->1337/tcp, 0.0.0.0:7000->7000/tcp   query_frontend-11466
a8ef297e3f8e        clipper/management_frontend:develop   "/clipper/release/sr…"   4 seconds ago       Up 3 seconds        0.0.0.0:1338->1338/tcp                           mgmt_frontend-29672
1cd20d87314b        redis:alpine                          "docker-entrypoint.s…"   5 seconds ago       Up 4 seconds        0.0.0.0:6379->6379/tcp                           redis-41206

In [13]:
clipper_conn.connect()


19-05-30:16:07:48 INFO     [clipper_admin.py:156] [default-cluster] Successfully connected to Clipper cluster at localhost:1337

In [14]:
# List all applications
clipper_conn.get_all_apps()


Out[14]:
[]

In [15]:
# List all models
clipper_conn.get_all_models()


Out[15]:
[]

In [16]:
# Add an application with a name and an input type
clipper_conn.register_application(name="pytorch-app", input_type="doubles", default_output="-1.0", slo_micros=100000)


19-05-30:16:07:56 INFO     [clipper_admin.py:220] [default-cluster] Application pytorch-app was successfully registered

In [17]:
# Deploy a model, to check what arguments you need run 
# "?deploy_pytorch_model" to let the notebook show you the definition of the method
deploy_pytorch_model(
    clipper_conn,
    name="pytorch-mod",
    version=1,
    input_type="doubles",
    func=predict,
    pytorch_model=model)


19-05-30:16:08:03 INFO     [deployer_utils.py:41] Saving function to /tmp/tmp9tsr3fy9clipper
19-05-30:16:08:03 INFO     [deployer_utils.py:51] Serialized and supplied predict function
19-05-30:16:08:03 INFO     [pytorch.py:204] Torch model saved
19-05-30:16:08:03 INFO     [pytorch.py:218] Using Python 3.6 base image
19-05-30:16:08:03 INFO     [clipper_admin.py:513] [default-cluster] Building model Docker image with model data from /tmp/tmp9tsr3fy9clipper
19-05-30:16:08:03 INFO     [clipper_admin.py:518] [default-cluster] Step 1/2 : FROM clipper/pytorch36-container:develop
19-05-30:16:08:03 INFO     [clipper_admin.py:518] [default-cluster]  ---> dd1bbe55c6a3
19-05-30:16:08:03 INFO     [clipper_admin.py:518] [default-cluster] Step 2/2 : COPY /tmp/tmp9tsr3fy9clipper /model/
19-05-30:16:08:03 INFO     [clipper_admin.py:518] [default-cluster]  ---> 79ed504922d7
19-05-30:16:08:03 INFO     [clipper_admin.py:518] [default-cluster] Successfully built 79ed504922d7
19-05-30:16:08:03 INFO     [clipper_admin.py:518] [default-cluster] Successfully tagged default-cluster-pytorch-mod:1
19-05-30:16:08:03 INFO     [clipper_admin.py:520] [default-cluster] Pushing model Docker image to default-cluster-pytorch-mod:1
19-05-30:16:08:06 INFO     [docker_container_manager.py:356] [default-cluster] Found 0 replicas for pytorch-mod:1. Adding 1
19-05-30:16:08:06 INFO     [clipper_admin.py:697] [default-cluster] Successfully registered model pytorch-mod:1
19-05-30:16:08:06 INFO     [clipper_admin.py:615] [default-cluster] Done deploying model pytorch-mod:1.

In [18]:
# Link the model and the app
clipper_conn.link_model_to_app(
    app_name="pytorch-app",
    model_name="pytorch-mod")


19-05-30:16:08:11 INFO     [clipper_admin.py:282] [default-cluster] Model pytorch-mod is now linked to application pytorch-app

In [19]:
# Show the apps again, should list one
clipper_conn.get_all_apps()


Out[19]:
['pytorch-app']

In [20]:
# Get query address
query_address = clipper_conn.get_query_addr()
print(query_address)


localhost:1337

In [21]:
# Run a query
import requests, json, numpy as np
headers = {"Content-type": "application/json"}
requests.post("http://"+query_address+"/pytorch-app/predict", headers=headers, data=json.dumps({
    "input": [2.0]})).json()
# The result is the same as in the local prediction!


Out[21]:
{'query_id': 0, 'output': [0.41221234], 'default': False}

In [22]:
# Unlink the model and the app
clipper_conn.unlink_model_from_app(model_name="pytorch-mod", app_name="pytorch-app")


19-05-30:16:08:24 INFO     [clipper_admin.py:323] Model pytorch-mod is now removed to application pytorch-app

In [23]:
# Stop the model
clipper_conn.stop_models('pytorch-mod')


19-05-30:16:08:35 INFO     [clipper_admin.py:1238] [default-cluster] Stopped all containers for these models and versions:
{'pytorch-mod': ['1']}

In [24]:
# Remove the app
clipper_conn.delete_application('pytorch-app')


19-05-30:16:08:38 INFO     [clipper_admin.py:239] [default-cluster] Application pytorch-app was successfully deleted

In [25]:
# Stop Clipper
clipper_conn.stop_all()


19-05-30:16:09:13 INFO     [clipper_admin.py:1324] [default-cluster] Stopped all Clipper cluster and all model containers

In [26]:
# Check that all docker processes are stopped
!docker ps


CONTAINER ID        IMAGE               COMMAND             CREATED             STATUS              PORTS               NAMES

In [ ]: