Use Amazon Elastic Container Registry (ECR) in faring

This example will show you how to remotely train a tensorflow model and use ECR to host fairing-job container image.

Develop training algorithm


In [ ]:
import os
import sys
import tensorflow as tf
import numpy as np

def train():
    # Genrating random linear data 
    # There will be 50 data points ranging from 0 to 50 
    x = np.linspace(0, 50, 50) 
    y = np.linspace(0, 50, 50) 

    # Adding noise to the random linear data 
    x += np.random.uniform(-4, 4, 50) 
    y += np.random.uniform(-4, 4, 50) 

    n = len(x) # Number of data points 

    X = tf.placeholder("float") 
    Y = tf.placeholder("float")
    W = tf.Variable(np.random.randn(), name = "W") 
    b = tf.Variable(np.random.randn(), name = "b") 
    learning_rate = 0.01
    training_epochs = 1000
    
    # Hypothesis 
    y_pred = tf.add(tf.multiply(X, W), b) 

    # Mean Squared Error Cost Function 
    cost = tf.reduce_sum(tf.pow(y_pred-Y, 2)) / (2 * n)

    # Gradient Descent Optimizer 
    optimizer = tf.train.GradientDescentOptimizer(learning_rate).minimize(cost) 

    # Global Variables Initializer 
    init = tf.global_variables_initializer() 


    sess = tf.Session()
    sess.run(init) 
      
    # Iterating through all the epochs 
    for epoch in range(training_epochs): 
          
        # Feeding each data point into the optimizer using Feed Dictionary 
        for (_x, _y) in zip(x, y): 
            sess.run(optimizer, feed_dict = {X : _x, Y : _y}) 
          
        # Displaying the result after every 50 epochs 
        if (epoch + 1) % 50 == 0: 
            # Calculating the cost a every epoch 
            c = sess.run(cost, feed_dict = {X : x, Y : y}) 
            print("Epoch", (epoch + 1), ": cost =", c, "W =", sess.run(W), "b =", sess.run(b)) 
      
    # Storing necessary values to be used outside the Session 
    training_cost = sess.run(cost, feed_dict ={X: x, Y: y}) 
    weight = sess.run(W) 
    bias = sess.run(b) 

    print('Weight: ', weight, 'Bias: ', bias)

Get ECR Access


In [ ]:
# Authenticate ECR
# This command retrieves a token that is valid for a specified registry for 12 hours, 
# and then it prints a docker login command with that authorization token. 
# Then we executate this command to login ECR

AWS_REGION='us-west-2'
!eval $(aws ecr get-login --region $AWS_REGION --no-include-email)

Use ECR to host your fairing job

Notice: ECR will work out of the box on Amazon EKS.


In [ ]:
# Setting up AWS Elastic Container Registry (ECR) for storing output containers
from kubeflow import fairing

AWS_ACCOUNT_ID=fairing.cloud.aws.guess_account_id()
AWS_REGION='us-west-2'
DOCKER_REGISTRY = '{}.dkr.ecr.{}.amazonaws.com'.format(AWS_ACCOUNT_ID, AWS_REGION)

fairing.config.set_builder('append', base_image='tensorflow/tensorflow:1.14.0-py3', registry=DOCKER_REGISTRY, push=True)
fairing.config.set_deployer('job')
    
if __name__ == '__main__':
    remote_train = fairing.config.fn(train)
    remote_train()