Kubeflow Fairing is a Python package that makes training and deploying machine learning models on Kubeflow easier.
Here, we use the preprocessor in Kubeflow Fairing to convert a notebook to be a Python script and create an entry point for that script. After preprocessing the notebook, we can call the command in the command line like the following to run
$ python repo_mlp.py train
In [1]:
import sys
sys.path.append("../../py")
In [2]:
import os
from fairing.preprocessors.converted_notebook import ConvertNotebookPreprocessorWithFire
In [3]:
preprocessor = ConvertNotebookPreprocessorWithFire('IssuesLoader', notebook_file='issues_loader.ipynb')
if not preprocessor.input_files:
preprocessor.input_files = set()
input_files = ['../../py/code_intelligence/embeddings.py',
'../../py/code_intelligence/inference.py',
'../../py/label_microservice/repo_config.py']
preprocessor.input_files = set([os.path.normpath(f) for f in input_files])
preprocessor.preprocess()
Out[3]:
In [4]:
preprocessor = ConvertNotebookPreprocessorWithFire('RepoMLP', notebook_file='repo_mlp.ipynb')
if not preprocessor.input_files:
preprocessor.input_files = set()
input_files = ['../../py/label_microservice/mlp.py',
'../../py/label_microservice/repo_config.py']
preprocessor.input_files = set([os.path.normpath(f) for f in input_files])
preprocessor.preprocess()
Out[4]:
In [5]:
import os
import sys
import fairing
from fairing.builders import append
from fairing.builders import cluster
from fairing.deployers import job
from fairing.preprocessors.converted_notebook import ConvertNotebookPreprocessorWithFire
In [6]:
# Setting up google container repositories (GCR) for storing output containers
# You can use any docker container registry istead of GCR
GCP_PROJECT = fairing.cloud.gcp.guess_project_name()
print(GCP_PROJECT)
DOCKER_REGISTRY = 'gcr.io/{}/training'.format(GCP_PROJECT)
print(DOCKER_REGISTRY)
PY_VERSION = ".".join([str(x) for x in sys.version_info[0:3]])
BASE_IMAGE = 'python:{}'.format(PY_VERSION)
# ucan use Dockerfile in this repo to build and use the base_image
base_image = 'gcr.io/issue-label-bot-dev/ml-gpu-lite-py3.6'
In [7]:
preprocessor = ConvertNotebookPreprocessorWithFire('RepoMLP', notebook_file='repo_mlp.ipynb')
if not preprocessor.input_files:
preprocessor.input_files = set()
input_files = ['../../py/label_microservice/mlp.py',
'../../py/label_microservice/repo_config.py',
'../../py/code_intelligence/embeddings.py',
'../../py/code_intelligence/inference.py',
'issues_loader.py']
preprocessor.input_files = set([os.path.normpath(f) for f in input_files])
preprocessor.preprocess()
Out[7]:
In [8]:
cluster_builder = cluster.cluster.ClusterBuilder(registry=DOCKER_REGISTRY,
base_image=base_image,
namespace='chunhsiang',
preprocessor=preprocessor,
pod_spec_mutators=[fairing.cloud.gcp.add_gcp_credentials_if_exists],
context_source=cluster.gcs_context.GCSContextSource())
cluster_builder.build()
In [9]:
builder = append.append.AppendBuilder(registry=DOCKER_REGISTRY,
base_image=cluster_builder.image_tag,
preprocessor=preprocessor)
builder.build()
In [ ]:
Kubeflow Pipelines builds reusable end-to-end machine learning workflows.
Define the pipeline as a Python function. "@kfp.dsl.pipeline" is a required decoration including name and description properties.
We define two steps for our training pipelines, including scrapping issues and training model, both of which will be executed in our built image from Kubeflow Fairing. Also, we use GPU and add GCP credentials to the pipelines.
In [10]:
import kfp
import kfp.components as comp
import kfp.gcp as gcp
import kfp.dsl as dsl
import kfp.compiler as compiler
In [11]:
# need to modify it if build a new one
target_image = 'gcr.io/issue-label-bot-dev/training/fairing-job:5350A3D3'
In [12]:
@dsl.pipeline(
name='Training pipeline',
description='A pipeline that loads embeddings and trains a model for a github repo.'
)
def train_pipeline(owner, repo):
scrape_op = dsl.ContainerOp(
name='scrape issues',
image=target_image,
command=['python', 'issues_loader.py', 'save_issue_embeddings', f'--owner={owner}', f'--repo={repo}'],
).set_gpu_limit(1).apply(
gcp.use_gcp_secret('user-gcp-sa'),
)
scrape_op.container.working_dir = '/app'
train_op = dsl.ContainerOp(
name='train',
image=target_image,
command=['python', 'repo_mlp.py', 'train', f'--owner={owner}', f'--repo={repo}'],
).set_gpu_limit(1).apply(
gcp.use_gcp_secret('user-gcp-sa'),
)
train_op.container.working_dir = '/app'
train_op.after(scrape_op)
In [13]:
pipeline_func = train_pipeline
pipeline_filename = pipeline_func.__name__ + '.pipeline.zip'
compiler.Compiler().compile(pipeline_func, pipeline_filename)
In [ ]:
In [14]:
EXPERIMENT_NAME = 'TrainModel'
client = kfp.Client()
experiment = client.create_experiment(EXPERIMENT_NAME)
In [15]:
#Specify pipeline argument values
arguments = {'owner': '', 'repo': ''}
#Submit a pipeline run
run_name = pipeline_func.__name__ + ' run'
run_result = client.run_pipeline(experiment.id, run_name, pipeline_filename, arguments)
In [ ]:
In [ ]:
In [ ]:
In [ ]: