In [1]:

    

%matplotlib inline
import os,random
import numpy as np
import theano as th
import theano.tensor as T
from keras.utils import np_utils
import keras.models as models
from keras.layers import Input,merge
from keras.layers.core import Reshape,Dense,Dropout,Activation,Flatten
from keras.layers.advanced_activations import LeakyReLU
from keras.activations import *
from keras.layers.wrappers import TimeDistributed
from keras.layers.noise import GaussianNoise
from keras.layers.convolutional import Convolution2D, MaxPooling2D, ZeroPadding2D, UpSampling2D
from keras.layers.recurrent import LSTM
from keras.regularizers import *
from keras.layers.normalization import *
from keras.optimizers import *
from keras.datasets import mnist
import matplotlib.pyplot as plt
import seaborn as sns
import cPickle, random, sys, keras
from keras.models import Model
from IPython import display

from keras.utils import np_utils
from tqdm import tqdm


    

    




Using Theano backend.

In [2]:

    

img_rows, img_cols = 28, 28

# the data, shuffled and split between train and test sets
(X_train, y_train), (X_test, y_test) = mnist.load_data()

X_train = X_train.reshape(X_train.shape[0], 1, img_rows, img_cols)
X_test = X_test.reshape(X_test.shape[0], 1, img_rows, img_cols)
X_train = X_train.astype('float32')
X_test = X_test.astype('float32')
X_train /= 255
X_test /= 255

print np.min(X_train), np.max(X_train)

print('X_train shape:', X_train.shape)
print(X_train.shape[0], 'train samples')
print(X_test.shape[0], 'test samples')


    

    




0.0 1.0
('X_train shape:', (60000L, 1L, 28L, 28L))
(60000L, 'train samples')
(10000L, 'test samples')

In [3]:

    

def make_trainable(net, val):
    net.trainable = val
    for l in net.layers:
        l.trainable = val


    

In [4]:

    

shp = X_train.shape[1:]
dropout_rate = 0.25
opt = Adam(lr=1e-4)
dopt = Adam(lr=1e-3)

# Build Generative model ...
nch = 200
g_input = Input(shape=[100])
H = Dense(nch*14*14)(g_input)
H = BatchNormalization()(H)
H = Activation('relu')(H)
H = Reshape( [nch, 14, 14] )(H)
H = UpSampling2D(size=(2, 2))(H)
H = Convolution2D(nch/2, 3, 3, border_mode='same', init='glorot_uniform')(H)
H = BatchNormalization()(H)
H = Activation('relu')(H)
H = Convolution2D(nch/4, 3, 3, border_mode='same', init='glorot_uniform')(H)
H = BatchNormalization()(H)
H = Activation('relu')(H)
H = Convolution2D(1, 1, 1, border_mode='same', init='glorot_uniform')(H)
g_V = Activation('sigmoid')(H)
generator = Model(g_input,g_V)
generator.compile(loss='binary_crossentropy', optimizer=opt)
generator.summary()


# Build Discriminative model ...
d_input = Input(shape=shp)
H = Convolution2D(256, 5, 5, subsample=(2, 2), border_mode = 'same', activation='relu')(d_input)
H = LeakyReLU(0.2)(H)
H = Dropout(dropout_rate)(H)
H = Convolution2D(512, 5, 5, subsample=(2, 2), border_mode = 'same', activation='relu')(H)
H = LeakyReLU(0.2)(H)
H = Dropout(dropout_rate)(H)
H = Flatten()(H)
H = Dense(256)(H)
H = LeakyReLU(0.2)(H)
H = Dropout(dropout_rate)(H)
d_V = Dense(2,activation='softmax')(H)
discriminator = Model(d_input,d_V)
discriminator.compile(loss='categorical_crossentropy', optimizer=dopt)
discriminator.summary()

# Freeze weights in the discriminator for stacked training
make_trainable(discriminator, False)

# Build stacked GAN model
gan_input = Input(shape=[100])
H = generator(gan_input)
gan_V = discriminator(H)
GAN = Model(gan_input, gan_V)
GAN.compile(loss='categorical_crossentropy', optimizer=opt)
GAN.summary()


    

    




C:\Users\Omar Saleem Mohammed\AppData\Local\Continuum\Anaconda2\lib\site-packages\ipykernel\__main__.py:14: UserWarning: Update your `Conv2D` call to the Keras 2 API: `Conv2D(100, (3, 3), padding="same", kernel_initializer="glorot_uniform")`
C:\Users\Omar Saleem Mohammed\AppData\Local\Continuum\Anaconda2\lib\site-packages\ipykernel\__main__.py:17: UserWarning: Update your `Conv2D` call to the Keras 2 API: `Conv2D(50, (3, 3), padding="same", kernel_initializer="glorot_uniform")`
C:\Users\Omar Saleem Mohammed\AppData\Local\Continuum\Anaconda2\lib\site-packages\ipykernel\__main__.py:20: UserWarning: Update your `Conv2D` call to the Keras 2 API: `Conv2D(1, (1, 1), padding="same", kernel_initializer="glorot_uniform")`






    




_________________________________________________________________
Layer (type)                 Output Shape              Param #   
=================================================================
input_1 (InputLayer)         (None, 100)               0         
_________________________________________________________________
dense_1 (Dense)              (None, 39200)             3959200   
_________________________________________________________________
batch_normalization_1 (Batch (None, 39200)             156800    
_________________________________________________________________
activation_1 (Activation)    (None, 39200)             0         
_________________________________________________________________
reshape_1 (Reshape)          (None, 200, 14, 14)       0         
_________________________________________________________________
up_sampling2d_1 (UpSampling2 (None, 200, 28, 28)       0         
_________________________________________________________________
conv2d_1 (Conv2D)            (None, 100, 28, 28)       180100    
_________________________________________________________________
batch_normalization_2 (Batch (None, 100, 28, 28)       112       
_________________________________________________________________
activation_2 (Activation)    (None, 100, 28, 28)       0         
_________________________________________________________________
conv2d_2 (Conv2D)            (None, 50, 28, 28)        45050     
_________________________________________________________________
batch_normalization_3 (Batch (None, 50, 28, 28)        112       
_________________________________________________________________
activation_3 (Activation)    (None, 50, 28, 28)        0         
_________________________________________________________________
conv2d_3 (Conv2D)            (None, 1, 28, 28)         51        
_________________________________________________________________
activation_4 (Activation)    (None, 1, 28, 28)         0         
=================================================================
Total params: 4,341,425
Trainable params: 4,262,913
Non-trainable params: 78,512
_________________________________________________________________






    




C:\Users\Omar Saleem Mohammed\AppData\Local\Continuum\Anaconda2\lib\site-packages\ipykernel\__main__.py:29: UserWarning: Update your `Conv2D` call to the Keras 2 API: `Conv2D(256, (5, 5), padding="same", strides=(2, 2), activation="relu")`
C:\Users\Omar Saleem Mohammed\AppData\Local\Continuum\Anaconda2\lib\site-packages\ipykernel\__main__.py:32: UserWarning: Update your `Conv2D` call to the Keras 2 API: `Conv2D(512, (5, 5), padding="same", strides=(2, 2), activation="relu")`






    




_________________________________________________________________
Layer (type)                 Output Shape              Param #   
=================================================================
input_2 (InputLayer)         (None, 1L, 28L, 28L)      0         
_________________________________________________________________
conv2d_4 (Conv2D)            (None, 256, 14L, 14L)     6656      
_________________________________________________________________
leaky_re_lu_1 (LeakyReLU)    (None, 256, 14L, 14L)     0         
_________________________________________________________________
dropout_1 (Dropout)          (None, 256, 14L, 14L)     0         
_________________________________________________________________
conv2d_5 (Conv2D)            (None, 512, 7L, 7L)       3277312   
_________________________________________________________________
leaky_re_lu_2 (LeakyReLU)    (None, 512, 7L, 7L)       0         
_________________________________________________________________
dropout_2 (Dropout)          (None, 512, 7L, 7L)       0         
_________________________________________________________________
flatten_1 (Flatten)          (None, 25088)             0         
_________________________________________________________________
dense_2 (Dense)              (None, 256)               6422784   
_________________________________________________________________
leaky_re_lu_3 (LeakyReLU)    (None, 256)               0         
_________________________________________________________________
dropout_3 (Dropout)          (None, 256)               0         
_________________________________________________________________
dense_3 (Dense)              (None, 2)                 514       
=================================================================
Total params: 9,707,266
Trainable params: 9,707,266
Non-trainable params: 0
_________________________________________________________________
_________________________________________________________________
Layer (type)                 Output Shape              Param #   
=================================================================
input_3 (InputLayer)         (None, 100)               0         
_________________________________________________________________
model_1 (Model)              (None, 1, 28, 28)         4341425   
_________________________________________________________________
model_2 (Model)              (None, 2)                 9707266   
=================================================================
Total params: 14,048,691
Trainable params: 4,262,913
Non-trainable params: 9,785,778
_________________________________________________________________

In [5]:

    

def plot_loss(losses):
        display.clear_output(wait=True)
        display.display(plt.gcf())
        plt.figure(figsize=(10,8))
        plt.plot(losses["d"], label='discriminitive loss')
        plt.plot(losses["g"], label='generative loss')
        plt.legend()
        plt.show()


    

In [6]:

    

def plot_gen(n_ex=16,dim=(4,4), figsize=(10,10) ):
    noise = np.random.uniform(0,1,size=[n_ex,100])
    generated_images = generator.predict(noise)

    plt.figure(figsize=figsize)
    for i in range(generated_images.shape[0]):
        plt.subplot(dim[0],dim[1],i+1)
        img = generated_images[i,0,:,:]
        plt.imshow(img)
        plt.axis('off')
    plt.tight_layout()
    plt.show()


    

In [7]:

    

ntrain = 10
trainidx = random.sample(range(0,X_train.shape[0]), ntrain)
XT = X_train[trainidx,:,:,:]

# Pre-train the discriminator network ...
noise_gen = np.random.uniform(0,1,size=[XT.shape[0],100])
generated_images = generator.predict(noise_gen)
print(generated_images.shape,XT.shape)

X = np.concatenate((XT, generated_images))
n = XT.shape[0]
y = np.zeros([2*n,2])
y[:n,1] = 1
y[n:,0] = 1

make_trainable(discriminator,True)
discriminator.fit(X,y, epochs=1, batch_size=128)
y_hat = discriminator.predict(X)


    

    




((10L, 1L, 28L, 28L), (10L, 1L, 28L, 28L))
Epoch 1/1
20/20 [==============================] - 1s - loss: 0.6914

In [8]:

    

y_hat_idx = np.argmax(y_hat,axis=1)
y_idx = np.argmax(y,axis=1)
diff = y_idx-y_hat_idx
n_tot = y.shape[0]
n_rig = (diff==0).sum()
acc = n_rig*100.0/n_tot
print "Accuracy: %0.02f pct (%d of %d) right"%(acc, n_rig, n_tot)


    

    




Accuracy: 100.00 pct (20 of 20) right

In [9]:

    

# set up loss storage vector
losses = {"d":[], "g":[]}


    

In [10]:

    

def train_for_n(nb_epoch=5000, plt_frq=25,BATCH_SIZE=32):

    for e in tqdm(range(nb_epoch)):  
        
        # Make generative images
        image_batch = X_train[np.random.randint(0,X_train.shape[0],size=BATCH_SIZE),:,:,:]    
        noise_gen = np.random.uniform(0,1,size=[BATCH_SIZE,100])
        generated_images = generator.predict(noise_gen)
        
        # Train discriminator on generated images
        X = np.concatenate((image_batch, generated_images))
        y = np.zeros([2*BATCH_SIZE,2])
        y[0:BATCH_SIZE,1] = 1
        y[BATCH_SIZE:,0] = 1
        
        #make_trainable(discriminator,True)
        d_loss  = discriminator.train_on_batch(X,y)
        losses["d"].append(d_loss)
    
        # train Generator-Discriminator stack on input noise to non-generated output class
        noise_tr = np.random.uniform(0,1,size=[BATCH_SIZE,100])
        y2 = np.zeros([BATCH_SIZE,2])
        y2[:,1] = 1
        
        #make_trainable(discriminator,False)
        g_loss = GAN.train_on_batch(noise_tr, y2 )
        losses["g"].append(g_loss)
        
        # Updates plots
        if e%plt_frq==plt_frq-1:
            plot_loss(losses)
            plot_gen()


    

In [20]:

    

train_for_n(nb_epoch=50, plt_frq=1,BATCH_SIZE=32)


    

    






<matplotlib.figure.Figure at 0x386977b8>






    






<matplotlib.figure.Figure at 0x386977b8>






    













    













    




100%|██████████████████████████████████████████| 50/50 [06:30<00:00,  8.19s/it]

In [14]:

    

opt.lr.set_value(1e-5)
dopt.lr.set_value(1e-4)
train_for_n(nb_epoch=5, plt_frq=1,BATCH_SIZE=32)


    

    






<matplotlib.figure.Figure at 0x30a50048>






    






<matplotlib.figure.Figure at 0x30a50048>






    













    













    




100%|████████████████████████████████████████████| 5/5 [00:38<00:00,  7.63s/it]

In [ ]:

    

opt.lr.set_value(1e-6)
dopt.lr.set_value(1e-5)
train_for_n(nb_epoch=5000, plt_frq=50,BATCH_SIZE=32)


    

    






<matplotlib.figure.Figure at 0x378e5da0>






    






<matplotlib.figure.Figure at 0x378e5da0>






    













    













    




  5%|█▋                                  | 233/5000 [43:36<94:50:54, 71.63s/it]

In [ ]:

    

plot_loss(losses)


    

In [ ]:

    

plot_gen(25,(5,5),(12,12))


    

In [ ]:

    

def plot_real(n_ex=16,dim=(4,4), figsize=(10,10) ):
    
    idx = np.random.randint(0,X_train.shape[0],n_ex)
    generated_images = X_train[idx,:,:,:]

    plt.figure(figsize=figsize)
    for i in range(generated_images.shape[0]):
        plt.subplot(dim[0],dim[1],i+1)
        img = generated_images[i,0,:,:]
        plt.imshow(img)
        plt.axis('off')
    plt.tight_layout()
    plt.show()


    

In [ ]:

    

plot_real()


    

In [ ]: