Deep and Wide paper implemented in Keras

In [5]:

    

from keras.utils import get_file

fTrain = get_file('adult.data',
         'http://mlr.cs.umass.edu/ml/machine-learning-databases/adult/adult.data',
         cache_dir='.')

fTest = get_file('adult.test',
         'http://mlr.cs.umass.edu/ml/machine-learning-databases/adult/adult.test',
         cache_dir='.')


    

In [6]:

    

import pandas as pd
from keras.models import Sequential
from keras.layers import Dense, Merge
from sklearn.preprocessing import MinMaxScaler

COLUMNS = [
    "age", "workclass", "fnlwgt", "education", "education_num", "marital_status", 
    "occupation", "relationship", "race", "gender", "capital_gain", "capital_loss", 
    "hours_per_week", "native_country", "income_bracket"
]

LABEL_COLUMN = "label"

CATEGORICAL_COLUMNS = [
    "workclass", "education", "marital_status", "occupation", "relationship", 
    "race", "gender", "native_country"
]

CONTINUOUS_COLUMNS = [
    "age", "education_num", "capital_gain", "capital_loss", "hours_per_week"
]

def load(filename):
    with open(filename, 'r') as f:
        skiprows = 1 if 'test' in filename else 0
        df = pd.read_csv(
            f, names=COLUMNS, skipinitialspace=True, skiprows=skiprows, engine='python'
        )
        df = df.dropna(how='any', axis=0)
    return df

def preprocess(df):
    df[LABEL_COLUMN] = df['income_bracket'].apply(lambda x: ">50K" in x).astype(int)
    df.pop("income_bracket")
    y = df[LABEL_COLUMN].values
    df.pop(LABEL_COLUMN)
    
    df = pd.get_dummies(df, columns=[x for x in CATEGORICAL_COLUMNS])

    # TODO: select features for wide & deep parts
    
    # TODO: transformations (cross-products)
    # from sklearn.preprocessing import PolynomialFeatures
    # X = PolynomialFeatures(degree=2, interaction_only=True, include_bias=False).fit_transform(X)
    
    df = pd.DataFrame(MinMaxScaler().fit_transform(df), columns=df.columns)

    X = df.values
    return X, y

def main():
    df_train = load('datasets/adult.data')
    df_test = load('datasets/adult.test')
    df = pd.concat([df_train, df_test])
    train_len = len(df_train)
    
    X, y = preprocess(df)
    X_train = X[:train_len]
    y_train = y[:train_len]
    X_test = X[train_len:]
    y_test = y[train_len:]
    
    wide = Sequential()
    wide.add(Dense(1, input_dim=X_train.shape[1]))
    
    deep = Sequential()
    # TODO: add embedding
    deep.add(Dense(input_dim=X_train.shape[1], output_dim=100, activation='relu'))
    deep.add(Dense(100, activation='relu'))
    deep.add(Dense(50, activation='relu'))
    deep.add(Dense(1, activation='sigmoid'))
    
    model = Sequential()
    model.add(Merge([wide, deep], mode='concat', concat_axis=1))
    model.add(Dense(1, activation='sigmoid'))
    
    model.compile(
        optimizer='rmsprop',
        loss='binary_crossentropy',
        metrics=['accuracy']
    )
    
    model.fit([X_train, X_train], y_train, nb_epoch=10, batch_size=32)
    
    loss, accuracy = model.evaluate([X_test, X_test], y_test)
    print('\n', 'test accuracy:', accuracy)
    
if __name__ == '__main__':
    main()


    

    




/Users/anatoly/anaconda3/lib/python3.6/site-packages/ipykernel/__main__.py:68: UserWarning: Update your `Dense` call to the Keras 2 API: `Dense(input_dim=108, activation="relu", units=100)`
/Users/anatoly/anaconda3/lib/python3.6/site-packages/ipykernel/__main__.py:74: UserWarning: The `Merge` layer is deprecated and will be removed after 08/2017. Use instead layers from `keras.layers.merge`, e.g. `add`, `concatenate`, etc.
/Users/anatoly/anaconda3/lib/python3.6/site-packages/keras/models.py:844: UserWarning: The `nb_epoch` argument in `fit` has been renamed `epochs`.
  warnings.warn('The `nb_epoch` argument in `fit` '






    




Epoch 1/10
32561/32561 [==============================] - 1s - loss: 0.3832 - acc: 0.8254     
Epoch 2/10
32561/32561 [==============================] - 1s - loss: 0.3412 - acc: 0.8444     
Epoch 3/10
32561/32561 [==============================] - 1s - loss: 0.3336 - acc: 0.8461     
Epoch 4/10
32561/32561 [==============================] - 1s - loss: 0.3284 - acc: 0.8490     
Epoch 5/10
32561/32561 [==============================] - 1s - loss: 0.3252 - acc: 0.8512     
Epoch 6/10
32561/32561 [==============================] - 1s - loss: 0.3229 - acc: 0.8518     
Epoch 7/10
32561/32561 [==============================] - 1s - loss: 0.3201 - acc: 0.8552     
Epoch 8/10
32561/32561 [==============================] - 1s - loss: 0.3185 - acc: 0.8553     
Epoch 9/10
32561/32561 [==============================] - 1s - loss: 0.3161 - acc: 0.8576     
Epoch 10/10
32561/32561 [==============================] - 1s - loss: 0.3163 - acc: 0.8572     
11264/16281 [===================>..........] - ETA: 0s
 test accuracy: 0.85179043053

In [ ]: