In [ ]:
import numpy as np
import matplotlib.pyplot as plt
% matplotlib inline
import tensorflow as tf
In [ ]:
# True distribution
def get_data_samples(N,type=0):
if type == 0:
# Gaussian
mu = 5
sigma = 2
samples = np.random.normal(mu, sigma, N)
samples = samples.reshape(-1,1)
if type == 1:
# Mixture of Gaussians
mu1 = 3
sigma1 = 1
mu2 = -3
sigma2 = 2
select = np.random.binomial(1,0.5,N)
samples = select*np.random.normal(mu1, sigma1, N) + (1-select)*np.random.normal(mu2, sigma2, N)
samples = samples.reshape(-1,1)
if type == 2:
# Two dimensional distribution
means = np.array([0,0])
cov = np.array([[2,1],[1,2]])
samples = np.random.multivariate_normal(means, cov, N)
samples = samples.reshape(-1,2)
return samples
In [ ]:
data_samples = get_data_samples(100000,2)
data_samples.shape
In [ ]:
if data_samples.shape[1] == 2:
plt.hexbin(data_samples[:,0],data_samples[:,1])
plt.colorbar()
else:
plt.hist(data_samples,100)
xlims = 10
plt.xlim(-xlims,xlims)
plt.grid()
In [ ]:
# Noise for generator input
def get_noise_samples(N):
# low = 0
# high = 1
# samples = np.random.uniform(low, high, N)
mu = 0
sigma = 1
samples = np.random.normal(mu, sigma, N)
samples = samples.reshape(-1,1)
return samples
noise_samples = get_noise_samples(100000)
In [ ]:
plt.hist(noise_samples,100)
xlims = 5
plt.xlim(-xlims,xlims)
plt.grid()
In [ ]:
# Generator/Discriminator network
def Network(inp, hidden_sizes, I,O):
weights = {}
biases = {}
inp_dim = I
for i in xrange(len(hidden_sizes)):
op_dim = hidden_sizes[i]
weights[i] = tf.get_variable(name="w" + `i`,shape=[inp_dim, op_dim],
initializer=tf.random_normal_initializer(0, 0.1))
biases[i] = tf.get_variable(name="b" + `i`,shape=(op_dim,),initializer=tf.constant_initializer(0))
tf.summary.histogram("w" + `i`,weights[i])
tf.summary.histogram("b"+`i`,biases[i])
op = tf.nn.relu(tf.matmul(inp,weights[i]) + biases[i])
inp = op
inp_dim = op_dim
op_dim = O
weights[i+1] = tf.get_variable(name="w" + `i+1`,shape=(inp_dim, op_dim),
initializer=tf.random_normal_initializer(0, 0.1))
biases[i+1] = tf.get_variable(name="b" + `i+1`,shape=(op_dim,),initializer=tf.constant_initializer(0))
tf.summary.histogram("w"+`i+1`,weights[i+1])
tf.summary.histogram("b"+`i+1`,biases[i+1])
op = tf.matmul(inp,weights[i+1]) + biases[i+1]
tf.summary.histogram("op",op)
return op
In [ ]:
# sigmoid function to convert to a probability
def sigmoid(x):
return 1./(1+tf.exp(-x))
In [ ]:
G_hidden_sizes = [50,50]
D_hidden_sizes = [50,50,50] # D uses a larger network since it should be able to discriminate
lr = 0.001
# Clear graph if it already exists
tf.reset_default_graph()
GANgraph = tf.Graph()
with GANgraph.as_default():
data = tf.placeholder(tf.float32,shape=[None,None])
noise = tf.placeholder(tf.float32,shape=[None,None])
with tf.variable_scope("D"):
I = 2
O = 1
D = sigmoid(Network(data,D_hidden_sizes,I,O))
with tf.variable_scope("G"):
I = 1
O = 2
G = Network(noise,G_hidden_sizes,I,O)
with tf.variable_scope("D",reuse=True):
I = 2
O = 1
DG = sigmoid(Network(G,D_hidden_sizes,I,O))
# Collecting variables since we need to use SGD separately
D_vars = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES, scope='D')
G_vars = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES, scope='G')
# Also, as per the paper’s suggestion, it’s better to maximize tf.reduce_mean(tf.log(D_fake))
# instead of minimizing tf.reduce_mean(1 - tf.log(D_fake)).
lossD = -tf.reduce_mean(tf.log(D)) + tf.reduce_mean(tf.log(1-DG))
lossG = -tf.reduce_mean(tf.log(DG))
tf.summary.scalar("LossD",lossD)
tf.summary.scalar("LossG",lossG)
SGDoptimizer = tf.train.GradientDescentOptimizer(lr)
Doptimizer = SGDoptimizer.minimize(lossD,var_list=D_vars)
Goptimizer = SGDoptimizer.minimize(lossG,var_list=G_vars)
data_mean = tf.reduce_mean(data)
G_mean = tf.reduce_mean(G)
# KL_divergence = tf.contrib.distributions.kl(data_dist,G_dist)
mean_divergence = data_mean-G_mean
tf.summary.scalar("mean_divergence",mean_divergence)
summary_op = tf.summary.merge_all()
In [ ]:
sess = tf.Session(graph=GANgraph)
LOGDIR = './logdir'
writer = tf.summary.FileWriter(LOGDIR,GANgraph)
k = 1
n_iter = 10000
batch_size = 1000
type = 1
with sess:
tf.global_variables_initializer().run()
step = 0
for iter in xrange(n_iter):
for idx in xrange(k):
_, summary, _, _ = sess.run([Doptimizer, summary_op, lossD, lossG],
feed_dict={data:get_data_samples(batch_size,type=type), noise:get_noise_samples(batch_size)})
writer.add_summary(summary,global_step=step)
step += 1
_, md, summary = sess.run([Goptimizer, mean_divergence, summary_op],
feed_dict={data:get_data_samples(batch_size,type=type), noise:get_noise_samples(batch_size)})
writer.add_summary(summary,global_step=step)
step += 1
# Generating new samples
N = 100000
generated_samples = sess.run(G, feed_dict={noise:get_noise_samples(N)})
In [ ]:
if data_samples.shape[1] == 2:
plt.hexbin(generated_samples[:,0],generated_samples[:,1])
plt.colorbar()
In [ ]:
plt.hist(generated_samples[:,0])
In [ ]:
# Plotting histogram of the generated samples and comparing with original
plt.figure(figsize=(12,5))
num_bins = 100
y,x = np.histogram(get_data_samples(N,type),num_bins)
x = x[:-1] + (x[1] - x[0])/2
plt.plot(x,y,linewidth=3)
y,x = np.histogram(get_noise_samples(N),num_bins)
x = x[:-1] + (x[1] - x[0])/2
plt.plot(x,y,'r',linewidth=3)
y,x = np.histogram(generated_samples,num_bins)
x = x[:-1] + (x[1] - x[0])/2
plt.plot(x,y,'k--',linewidth=3)
plt.legend(['Original Data','Noisy Data','Generated Data'])
plt.title('Histograms',fontsize=24)
plt.grid()
In [ ]: