ML with TensorFlow Extended (TFX) -- Part 1

In [ ]:

    

import tensorflow as tf
import tensorflow_data_validation as tfdv

print('TF version: {}'.format(tf.__version__))
print('TFDV version: {}'.format(tfdv.__version__))


    

In [ ]:

    

PROJECT = 'cloud-training-demos'    # Replace with your PROJECT
BUCKET = 'cloud-training-demos-ml'  # Replace with your BUCKET
REGION = 'us-central1'              # Choose an available region for Cloud MLE

import os

os.environ['PROJECT'] = PROJECT
os.environ['BUCKET'] = BUCKET
os.environ['REGION'] = REGION


    

In [ ]:

    

%%bash
gcloud config set project $PROJECT
gcloud config set compute/region $REGION

## ensure we predict locally with our current Python environment
gcloud config set ml_engine/local_python `which python`


    

In [ ]:

    

DATA_DIR='gs://cloud-samples-data/ml-engine/census/data'


    

In [ ]:

    

import os

TRAIN_DATA_FILE = os.path.join(DATA_DIR, 'adult.data.csv')
EVAL_DATA_FILE = os.path.join(DATA_DIR, 'adult.test.csv')
!gsutil ls -l $TRAIN_DATA_FILE
!gsutil ls -l $EVAL_DATA_FILE


    

In [ ]:

    

HEADER = ['age', 'workclass', 'fnlwgt', 'education', 'education_num',
               'marital_status', 'occupation', 'relationship', 'race', 'gender',
               'capital_gain', 'capital_loss', 'hours_per_week',
               'native_country', 'income_bracket']

TARGET_FEATURE_NAME = 'income_bracket'
TARGET_LABELS = [' <=50K', ' >50K']
WEIGHT_COLUMN_NAME = 'fnlwgt'


    

In [ ]:

    

# This is a convenience function for CSV. We can write a Beam pipeline for other formats.
# https://www.tensorflow.org/tfx/data_validation/api_docs/python/tfdv/generate_statistics_from_csv
train_stats = tfdv.generate_statistics_from_csv(
    data_location=TRAIN_DATA_FILE, 
    column_names=HEADER,
    stats_options=tfdv.StatsOptions(
        weight_feature=WEIGHT_COLUMN_NAME,
        sample_rate=1.0
    )
)


    

In [ ]:

    

tfdv.visualize_statistics(train_stats)


    

In [ ]:

    

schema = tfdv.infer_schema(statistics=train_stats)
tfdv.display_schema(schema=schema)


    

In [ ]:

    

print(tfdv.get_feature(schema, 'age'))


    

In [ ]:

    

# Relax the minimum fraction of values that must come from the domain for feature occupation.
occupation = tfdv.get_feature(schema, 'occupation')
occupation.distribution_constraints.min_domain_mass = 0.9

# Add new value to the domain of feature native_country, assuming that we start receiving this
# we won't be able to make great predictions of course, because this country is not part of our
# training data.
native_country_domain = tfdv.get_domain(schema, 'native_country')
native_country_domain.value.append('Egypt')

# All features are by default in both TRAINING and SERVING environments.
schema.default_environment.append('TRAINING')
schema.default_environment.append('EVALUATION')
schema.default_environment.append('SERVING')

# Specify that the class feature is not in SERVING environment.
tfdv.get_feature(schema, TARGET_FEATURE_NAME).not_in_environment.append('SERVING')


    

In [ ]:

    

tfdv.display_schema(schema=schema)


    

In [ ]:

    

eval_stats = tfdv.generate_statistics_from_csv(
    EVAL_DATA_FILE, 
    column_names=HEADER,
    stats_options=tfdv.StatsOptions(
        weight_feature=WEIGHT_COLUMN_NAME)
)

eval_anomalies = tfdv.validate_statistics(eval_stats, schema, environment='EVALUATION')
tfdv.display_anomalies(eval_anomalies)


    

In [ ]:

    

RAW_SCHEMA_LOCATION = 'raw_schema.pbtxt'


    

In [ ]:

    

from tensorflow.python.lib.io import file_io
from google.protobuf import text_format

tfdv.write_schema_text(schema, RAW_SCHEMA_LOCATION)


    

In [ ]:

    

!cat {RAW_SCHEMA_LOCATION}