This script show a simple example of using tf.contrib.learn library to create our model.
The code is divided in following steps:
v0.1: Added code for data loading, modeling and prediction model.
v0.2: Removed unnecessary output logs.
PS: I was able to get a score of 1295.07972 using this script with 70% (of train.csv) data used for training and rest for evaluation. Script took 2hrs for training and 3000 steps were used.
In [1]:
import pandas as pd
import numpy as np
import tensorflow as tf
tf.logging.set_verbosity(tf.logging.ERROR)
import warnings
warnings.filterwarnings("ignore")
%matplotlib inline
from matplotlib import pyplot as plt
from sklearn.cross_validation import train_test_split
from sklearn.preprocessing import StandardScaler, MinMaxScaler, minmax_scale
from sklearn.metrics import r2_score
from time import time
start0 = time()
plt.rcParams['figure.dpi'] = 300
In [2]:
nSkip = 20
spitzerDataRaw = pd.read_csv('pmap_ch2_0p1s_x4_rmulti_s3_7.csv')[::nSkip]
In [3]:
PLDpixels = pd.DataFrame({key:spitzerDataRaw[key] for key in spitzerDataRaw.columns.values if 'pix' in key})
PLDpixels
Out[3]:
In [4]:
PLDnorm = np.sum(np.array(PLDpixels),axis=1)
In [5]:
PLDpixels = (PLDpixels.T / PLDnorm).T
PLDpixels
Out[5]:
In [6]:
spitzerData = spitzerDataRaw.copy()
for key in spitzerDataRaw.columns:
if key in PLDpixels.columns:
spitzerData[key] = PLDpixels[key]
In [7]:
testPLD = np.array(pd.DataFrame({key:spitzerData[key] for key in spitzerData.columns.values if 'pix' in key}))
assert(not sum(abs(testPLD - np.array(PLDpixels))).all())
print('Confirmed that PLD Pixels have been Normalized to Spec')
In [8]:
notFeatures = ['flux', 'fluxerr', 'xerr', 'yerr', 'xycov']
feature_columns = spitzerData.drop(notFeatures,axis=1).columns.values
features = spitzerData.drop(notFeatures,axis=1).values
labels = spitzerData['flux'].values
In [9]:
stdScaler = StandardScaler()
In [11]:
features_scaled = stdScaler.fit_transform(features)
labels_scaled = stdScaler.fit_transform(labels[:,None]).ravel()
x_valtest, x_train, y_valtest, y_train = train_test_split(features_scaled, labels_scaled, test_size=0.6, random_state=42)
x_val, x_test, y_val, y_test = train_test_split(x_valtest, y_valtest, test_size=0.5, random_state=42)
# x_val = minmax_scale(x_val.astype('float32'))
# x_train = minmax_scale(x_train.astype('float32'))
# x_test = minmax_scale(x_test.astype('float32'))
# y_val = minmax_scale(y_val.astype('float32'))
# y_train = minmax_scale(y_train.astype('float32'))
# y_test = minmax_scale(y_test.astype('float32'))
print(x_val.shape[0] , 'validation samples')
print(x_train.shape[0], 'train samples')
print(x_test.shape[0] , 'test samples')
In [12]:
train_df = pd.DataFrame(np.c_[x_train, y_train], columns=list(feature_columns) + ['flux'])
test_df = pd.DataFrame(np.c_[x_test , y_test ], columns=list(feature_columns) + ['flux'])
evaluate_df = pd.DataFrame(np.c_[x_val , y_val ], columns=list(feature_columns) + ['flux'])
In [14]:
# categorical_features = [feature for feature in features if 'cat' in feature]
categorical_features = []
continuous_features = [feature for feature in train_df.columns]# if 'cat' in feature]
LABEL_COLUMN = 'flux'
When building a TF.Learn model, the input data is specified by means of an Input Builder function. This builder function will not be called until it is later passed to TF.Learn methods such as fit and evaluate. The purpose of this function is to construct the input data, which is represented in the form of Tensors or SparseTensors.
Note that input_fn will be called while constructing the TensorFlow graph, not while running the graph. What it is returning is a representation of the input data as the fundamental unit of TensorFlow computations, a Tensor (or SparseTensor).
More detail on input_fn.
In [15]:
# Converting Data into Tensors
def input_fn(df, training = True):
# Creates a dictionary mapping from each continuous feature column name (k) to
# the values of that column stored in a constant Tensor.
continuous_cols = {k: tf.constant(df[k].values)
for k in continuous_features}
# Creates a dictionary mapping from each categorical feature column name (k)
# to the values of that column stored in a tf.SparseTensor.
# categorical_cols = {k: tf.SparseTensor(
# indices=[[i, 0] for i in range(df[k].size)],
# values=df[k].values,
# shape=[df[k].size, 1])
# for k in categorical_features}
# Merges the two dictionaries into one.
feature_cols = continuous_cols
# feature_cols = dict(list(continuous_cols.items()) + list(categorical_cols.items()))
if training:
# Converts the label column into a constant Tensor.
label = tf.constant(df[LABEL_COLUMN].values)
# Returns the feature columns and the label.
return feature_cols, label
# Returns the feature columns
return feature_cols
def train_input_fn():
return input_fn(train_df, training=True)
def eval_input_fn():
return input_fn(evaluate_df, training=True)
# def test_input_fn():
# return input_fn(test_df.drop(LABEL_COLUMN,axis=1), training=False)
def test_input_fn():
return input_fn(test_df, training=False)
We use tf.learn's concept of FeatureColumn which help in transforming raw data into suitable input features.
These engineered features will be used when we construct our model.
In [16]:
engineered_features = []
for continuous_feature in continuous_features:
engineered_features.append(
tf.contrib.layers.real_valued_column(continuous_feature))
# for categorical_feature in categorical_features:
# sparse_column = tf.contrib.layers.sparse_column_with_hash_bucket(
# categorical_feature, hash_bucket_size=1000)
# engineered_features.append(tf.contrib.layers.embedding_column(sparse_id_column=sparse_column, dimension=16,
# combiner="sum"))
Following is the simple DNNRegressor model. More detail about hidden_units, etc can be found here.
model_dir is used to save and restore our model. This is because once we have trained the model we don't want to train it again, if we only want to predict on new data-set.
In [24]:
# train_df = df_train_ori.head(1000)
# evaluate_df = df_train_ori.tail(500)
# test_df = df_test_ori.head(1000)
# MODEL_DIR = "tf_model_spitzer/withNormalization_drop50/relu"
MODEL_DIR = "tf_model_spitzer/adamOptimizer_with_drop50/tanh/"
# MODEL_DIR = "tf_model_spitzer/xgf"
print("train_df.shape = " , train_df.shape)
print("test_df.shape = " , test_df.shape)
print("evaluate_df.shape = ", evaluate_df.shape)
In [28]:
nHidden1 = 10
nHidden2 = 5
nHidden3 = 10
regressor = tf.contrib.learn.DNNRegressor(activation_fn=tf.nn.relu, dropout=0.5, optimizer=tf.train.AdamOptimizer,
feature_columns=engineered_features, hidden_units=[nHidden1, nHidden2, nHidden3], model_dir=MODEL_DIR)
add progress bar through python logging
In [ ]:
import logging
logging.getLogger().setLevel(logging.INFO)
In [96]:
# Training Our Model
nFitSteps = 100000
start = time()
wrap = regressor.fit(input_fn=train_input_fn, steps=nFitSteps)
print('TF Regressor took {} seconds'.format(time()-start))
In [29]:
# Evaluating Our Model
print('Evaluating ...')
results = regressor.evaluate(input_fn=eval_input_fn, steps=1)
for key in sorted(results):
print("{}: {}".format(key, results[key]))
print("Val Acc: {:.3f}".format((1-results['loss'])*100))
Track Scalable Growth
Shrunk data set to 23559 Training samples and 7853 Val/Test samples
| n_iters | time (s) | val acc | multicore | gpu |
|---|---|---|---|---|
| 100 | 5.869 | 6.332 | yes | no |
| 200 | 6.380 | 13.178 | yes | no |
| 500 | 8.656 | 54.220 | yes | no |
| 1000 | 12.170 | 66.596 | yes | no |
| 2000 | 19.891 | 62.996 | yes | no |
| 5000 | 43.589 | 76.586 | yes | no |
| 10000 | 80.581 | 66.872 | yes | no |
| 20000 | 162.435 | 78.927 | yes | no |
| 50000 | 535.584 | 75.493 | yes | no |
| 100000 | 1062.656 | 73.162 | yes | no |
In [98]:
nItersList = [100,200,500,1000,2000,5000,10000,20000,50000,100000]
rtimesList = [5.869, 6.380, 8.656, 12.170, 19.891, 43.589, 80.581, 162.435, 535.584, 1062.656]
valAccList = [6.332, 13.178, 54.220, 66.596, 62.996, 76.586, 66.872, 78.927, 75.493, 73.162]
In [106]:
plt.loglog(nItersList, rtimesList,'o-');
plt.twinx()
plt.semilogx(nItersList, valAccList,'o-', color='orange');
In [47]:
def de_median(x):
return x - np.median(x)
In [ ]:
predicted_output = list(regressor.predict(input_fn=test_input_fn))
# x = list(predicted_output)
In [ ]:
r2_score(test_df['flux'].values,predicted_output)*100
In [ ]:
print('Full notebook took {} seconds'.format(time()-start0))