Run training on CMLE and test the online prediction service

Assumes preprocessed data is available in GCS



In [ ]:

    
import os

REGION = ""
TFVERSION = "1.8"
BUCKET = ""
DATA_DIR = ""
MODEL_DIR = ""
MODEL_NAME = "" # no hyphens in model name
MODEL_VERSION = ""
JOBDIR = "gs://$BUCKET/$MODEL_DIR/job"
JOBNAME= "keras_mnist_$(date -u +%y%m%d_%H%M%S)"
TRAIN_STEPS=1000

os.environ['REGION'] = REGION
os.environ['TFVERSION'] = TFVERSION
os.environ['BUCKET'] = BUCKET
os.environ['DATA_DIR'] = DATA_DIR
os.environ['MODEL_DIR'] = MODEL_DIR
os.environ['MODEL_NAME'] = MODEL_NAME
os.environ['MODEL_VERSION'] = MODEL_VERSION
os.environ['JOBDIR'] = JOBDIR
os.environ['JOBNAME'] = JOBNAME

Run the training job. Adjust train_steps as needed



In [ ]:

    
!gcloud ml-engine jobs submit training $JOBNAME \
   --region=$REGION \
   --module-name=trainer.main \
   --package-path="$(pwd)/trainer" \
   --job-dir=$JOBDIR \
   --staging-bucket=gs://$BUCKET \
   --runtime-version=$TFVERSION \
   -- \
   --data_dir=gs://$BUCKET/$DATA_DIR \
   --model_dir=gs://$BUCKET/$MODEL_DIR \
   --export_dir=gs://$BUCKET/$MODEL_DIR \
   --train_steps=$TRAIN_STEPS

Export the model location



In [ ]:

    
%%bash --out MODEL_LOCATION

MODEL_LOCATION=$(gsutil ls gs://$BUCKET/$MODEL_DIR/export | tail -1)
echo $MODEL_LOCATION



In [ ]:

    
print(MODEL_LOCATION)

Create the model in CMLE



In [ ]:

    
!gcloud ml-engine models create $MODEL_NAME --regions $REGION

Create the versioned model with the model location



In [ ]:

    
!gcloud ml-engine versions create $MODEL_VERSION --model $MODEL_NAME --runtime-version $TFVERSION --origin $MODEL_LOCATION

Generate sample data to test



In [ ]:

    
import json, codecs
import matplotlib.pyplot as plt
from tensorflow.examples.tutorials.mnist import input_data

HEIGHT=28
WIDTH=28

mnist = input_data.read_data_sets('mnist/data', one_hot=True, reshape=False)
IMGNO=5 #CHANGE THIS to get different images
jsondata = {'X': mnist.test.images[IMGNO].reshape(HEIGHT, WIDTH).tolist()}
json.dump(jsondata, codecs.open('test.json', 'w', encoding='utf-8'))
plt.imshow(mnist.test.images[IMGNO].reshape(HEIGHT, WIDTH));



In [ ]:

    
!gcloud ml-engine predict \
   --model=$MODEL_NAME \
   --version=$MODEL_VERSION \
   --json-instances=./test.json



In [ ]: