You should prepare data volume user-susan by following docs.
And run arena data list to check if it's created.
In [10]:
! arena data list
We assume this notebook kernel has access to Python's site-packages and is in Python3.
Please fill in the below environment variables with you own settings.
In [3]:
KFP_SERVICE="ml-pipeline.kubeflow.svc.cluster.local:8888"
KFP_PACKAGE = 'http://kubeflow.oss-cn-beijing.aliyuncs.com/kfp/0.1.14/kfp.tar.gz'
KFP_ARENA_PACKAGE = 'http://kubeflow.oss-cn-beijing.aliyuncs.com/kfp-arena/kfp-arena-0.3.tar.gz'
KUBEFLOW_PIPELINE_LINK = ''
MOUNT="['user-susan:/training']"
GPUs=1
In [20]:
!pip3 install $KFP_PACKAGE --upgrade
Note: Install arena's python package
In [4]:
!pip3 install $KFP_ARENA_PACKAGE --upgrade
In [10]:
import arena
import kfp.dsl as dsl
@dsl.pipeline(
name='pipeline to run jobs',
description='shows how to run pipeline jobs.'
)
def sample_pipeline(learning_rate='0.01',
dropout='0.9',
model_version='1'):
"""A pipeline for end to end machine learning workflow."""
# 1. prepare data
prepare_data = arena.StandaloneOp(
name="prepare-data",
image="byrnedo/alpine-curl",
data=MOUNT,
command="mkdir -p /training/dataset/mnist && \
cd /training/dataset/mnist && \
curl -O https://code.aliyun.com/xiaozhou/tensorflow-sample-code/raw/master/data/t10k-images-idx3-ubyte.gz && \
curl -O https://code.aliyun.com/xiaozhou/tensorflow-sample-code/raw/master/data/t10k-labels-idx1-ubyte.gz && \
curl -O https://code.aliyun.com/xiaozhou/tensorflow-sample-code/raw/master/data/train-images-idx3-ubyte.gz && \
curl -O https://code.aliyun.com/xiaozhou/tensorflow-sample-code/raw/master/data/train-labels-idx1-ubyte.gz")
# 2. prepare source code
prepare_code = arena.StandaloneOp(
name="source-code",
image="alpine/git",
data=MOUNT,
command="mkdir -p /training/models/ && \
cd /training/models/ && \
if [ ! -d /training/models/tensorflow-sample-code ]; then https://github.com/cheyang/tensorflow-sample-code.git; else echo no need download;fi")
# 3. train the models
train = arena.StandaloneOp(
name="train",
image="tensorflow/tensorflow:1.11.0-gpu-py3",
gpus=GPUs,
data=MOUNT,
command="echo %s; \
echo %s; \
python /training/models/tensorflow-sample-code/tfjob/docker/mnist/main.py --max_steps 500 --data_dir /training/dataset/mnist --log_dir /training/output/mnist" % (prepare_data.output, prepare_code.output),
metric_name="Train-accuracy",
metric_unit="PERCENTAGE",
)
# 4. export the model
export_model = arena.StandaloneOp(
name="export-model",
image="tensorflow/tensorflow:1.11.0-py3",
data=MOUNT,
command="echo %s; \
python /training/models/tensorflow-sample-code/tfjob/docker/mnist/export_model.py --model_version=%s --checkpoint_step=400 --checkpoint_path=/training/output/mnist /training/output/models" % (train.output,model_version))
In [11]:
learning_rate = "0.001"
dropout = "0.8"
model_verison = "1"
arguments = {
'learning_rate': learning_rate,
'dropout': dropout,
'model_version': model_version,
}
import kfp
client = kfp.Client(host=KUBEFLOW_PIPELINE_LINK)
run = client.create_run_from_pipeline_func(sample_pipeline, arguments=arguments).run_info
print('The above run link is assuming you ran this cell on JupyterHub that is deployed on the same cluster. ' +
'The actual run link is ' + KUBEFLOW_PIPELINE_LINK + '/#/runs/details/' + run.id)
In [ ]: