In [ ]:
##################Build Essential DenseNet to Load Pretrained Parameters###############################################
# encoding=utf8
import numpy as np
import tensorflow as tf
def unpickle(file):
import _pickle as cPickle
fo = open(file, 'rb')
dict = cPickle.load(fo,encoding='latin1')
fo.close()
if 'data' in dict:
dict['data'] = dict['data'].reshape((-1, 3, 32, 32)).swapaxes(1, 3).swapaxes(1, 2).reshape(-1, 32*32*3) / 256.
return dict
In [2]:
def load_data_one(f):
batch = unpickle(f)
data = batch['data']
labels = batch['labels']
print ("Loading %s: %d" % (f, len(data)))
return data, labels
In [3]:
def load_data(files, data_dir, label_count):
data, labels = load_data_one(data_dir + '/' + files[0])
for f in files[1:]:
data_n, labels_n = load_data_one(data_dir + '/' + f)
data = np.append(data, data_n, axis=0)
labels = np.append(labels, labels_n, axis=0)
labels = np.array([ [ float(i == label) for i in range(label_count) ] for label in labels ])
return data, labels
In [4]:
def run_in_batch_avg(session, tensors, batch_placeholders, feed_dict={}, batch_size=200):
res = [ 0 ] * len(tensors)
batch_tensors = [ (placeholder, feed_dict[ placeholder ]) for placeholder in batch_placeholders ]
total_size = len(batch_tensors[0][1])
batch_count = (total_size + batch_size - 1) / batch_size
for batch_idx in range(batch_count):
current_batch_size = None
for (placeholder, tensor) in batch_tensors:
batch_tensor = tensor[ batch_idx*batch_size : (batch_idx+1)*batch_size ]
current_batch_size = len(batch_tensor)
feed_dict[placeholder] = tensor[ batch_idx*batch_size : (batch_idx+1)*batch_size ]
tmp = session.run(tensors, feed_dict=feed_dict)
res = [ r + t * current_batch_size for (r, t) in zip(res, tmp) ]
return [ r / float(total_size) for r in res ]
In [5]:
def weight_variable(shape):
initial = tf.truncated_normal(shape, stddev=0.01)
return tf.Variable(initial)
def bias_variable(shape):
initial = tf.constant(0.01, shape=shape)
return tf.Variable(initial)
def conv2d(input, in_features, out_features, kernel_size, with_bias=False):
W = weight_variable([ kernel_size, kernel_size, in_features, out_features ])
conv = tf.nn.conv2d(input, W, [ 1, 1, 1, 1 ], padding='SAME')
if with_bias:
return conv + bias_variable([ out_features ])
return conv
def batch_activ_conv(current, in_features, out_features, kernel_size, is_training, keep_prob):
current = tf.contrib.layers.batch_norm(current, scale=True, is_training=is_training, updates_collections=None)
current = tf.nn.relu(current)
current = conv2d(current, in_features, out_features, kernel_size)
current = tf.nn.dropout(current, keep_prob)
return current
def block(input, layers, in_features, growth, is_training, keep_prob):
current = input
features = in_features
for idx in range(layers):
tmp = batch_activ_conv(current, features, growth, 3, is_training, keep_prob)
current = tf.concat((current, tmp),3)
features += growth
return current, features
def avg_pool(input, s):
return tf.nn.avg_pool(input, [ 1, s, s, 1 ], [1, s, s, 1 ], 'VALID')
In [6]:
data_dir = './data'
image_size = 32
image_dim = image_size * image_size * 3
# meta = unpickle(data_dir + '/batches.meta')
# label_names = meta['label_names']
# label_count = len(label_names)
label_count = 10
# train_files = [ 'data_batch_%d' % d for d in range(1, 6) ]
# train_data, train_labels = load_data(train_files, data_dir, label_count)
# pi = np.random.permutation(len(train_data))
# train_data, train_labels = train_data[pi], train_labels[pi]
# test_data, test_labels = load_data([ 'test_batch' ], data_dir, label_count)
# print ("Train:", np.shape(train_data), np.shape(train_labels))
# print ("Test:", np.shape(test_data), np.shape(test_labels))
# data = { 'train_data': train_data,
# 'train_labels': train_labels,
# 'test_data': test_data,
# 'test_labels': test_labels }
depth = 40
In [ ]:
weight_decay = 1e-4
layers = int((depth - 4) / 3)
graph = tf.Graph()
xs = tf.placeholder("float", shape=[None, image_dim])
ys = tf.placeholder("float", shape=[None, label_count])
lr = tf.placeholder("float", shape=[])
keep_prob = tf.placeholder(tf.float32)
is_training = tf.placeholder("bool", shape=[])
current = tf.reshape(xs, [ -1, 32, 32, 3 ])
current = conv2d(current, 3, 16, 3)
current, features = block(current, layers, 16, 12, is_training, keep_prob)
current = batch_activ_conv(current, features, features, 1, is_training, keep_prob)
current = avg_pool(current, 2)
current, features = block(current, layers, features, 12, is_training, keep_prob)
current = batch_activ_conv(current, features, features, 1, is_training, keep_prob)
current = avg_pool(current, 2)
current, features = block(current, layers, features, 12, is_training, keep_prob)
current = tf.contrib.layers.batch_norm(current, scale=True, is_training=is_training, updates_collections=None)
current = tf.nn.relu(current)
current = avg_pool(current, 8)
final_dim = features
current = tf.reshape(current, [ -1, final_dim ])
Wfc = weight_variable([ final_dim, label_count ])
bfc = bias_variable([ label_count ])
ys_ = tf.nn.softmax( tf.matmul(current, Wfc) + bfc )
cross_entropy = -tf.reduce_mean(ys * tf.log(ys_ + 1e-12))
l2 = tf.add_n([tf.nn.l2_loss(var) for var in tf.trainable_variables()])
train_step = tf.train.MomentumOptimizer(lr, 0.9, use_nesterov=True).minimize(cross_entropy + l2 * weight_decay)
correct_prediction = tf.equal(tf.argmax(ys_, 1), tf.argmax(ys, 1))
accuracy = tf.reduce_mean(tf.cast(correct_prediction, "float"))
In [ ]:
para_dict={}
for k in tf.global_variables():
if k not in tf.contrib.framework.get_variables_by_suffix('Momentum'): #Load all parameters except ones of optimization functions
para_dict[k.name[:-2]] = k
In [ ]:
sess=tf.InteractiveSession()
saver = tf.train.Saver(para_dict)
#saver.restore(sess,'./inqmodel/stage2/64pinq80/64pinq80ok_93149_7.ckpt')
#saver.restore(sess,'./modellog/weightonlypara93.ckpt')
saver.restore(sess,'./inqmodel/stage2/inq16_97/inq1697_92729_5.ckpt')
#saver.restore(sess,'./prunemodel/stage2/inc100adj/prune100ar_92969_10ok.ckpt')
##################End of Pretrained Parameters Loading###############################################
In [ ]:
import config
#Nearly all hyperparameters are set in config.py
In [11]:
def apply_inq(weights, inq_dict): # Apply INQ
for target in config.all_para:
wl = target
bit = config.inq_para[wl]
# Get target layer's weights
weight_obj = weights[wl]
weight_arr = weight_obj.eval()
weight_rest = np.reshape(weight_arr,[-1])
dic_tem = np.reshape(inq_dict[wl],[-1])
idx_rest = np.flip(np.argsort(abs(np.reshape(weight_rest,[-1]))),0) #choose which weights to be INQed
num_prune = int(len(weight_rest)*config.inqpercen_para[wl]) #how many weights to be INQed
weight_toINQ = weight_rest[idx_rest[:num_prune]]
#calculate INQ bounds
n1 = (np.floor(np.log2(max(abs(np.reshape(weight_arr,[-1])))*4/3)))
n2 = n1 +1 - bit/4
upper_bound = 2**(np.floor(np.log2(max(abs(np.reshape(weight_arr,[-1])))*4/3)))
lower_bound = 2**(n1 +1 - bit/4)
#INQ
weight_toINQ[abs(weight_toINQ) < lower_bound] = 0
weight_toINQ[weight_toINQ != 0] = 2**(np.floor(np.log2(abs(weight_toINQ[weight_toINQ != 0]*4/3))))*np.sign(weight_toINQ[weight_toINQ != 0])
weight_rest[idx_rest[:num_prune]] = weight_toINQ
weight_arr = np.reshape(weight_rest,np.shape(weight_arr))
dic_tem [idx_rest[:num_prune]] = np.zeros_like(dic_tem [idx_rest[:num_prune]])
inq_dict[wl] = np.reshape(dic_tem,np.shape(inq_dict[wl]))
# Store INQed weights as tensorflow objects
sess.run(weight_obj.assign(weight_arr))
return inq_dict
In [12]:
prune_dict = {}
for target in config.all_para: #choose which layers
wl =target
weight_obj = para_dict[wl]
prune_dict[wl] = np.ones_like(weight_obj.eval())
In [13]:
prune_dict = apply_inq(para_dict, prune_dict)
In [14]:
saver.save(sess,'./inqmodel/stage1/inq1697.ckpt') #save INQed parameters
Out[14]:
In [ ]:
#save INQ mask
import pickle
# create dict
# save dict
f1 = open("C:/Users/lhlne/Desktop/project/densenet/inqmodel/stage1/inq1697.txt","wb")
pickle.dump(prune_dict, f1)
f1.close()
# load dict
f2 = open("C:/Users/lhlne/Desktop/project/densenet/inqmodel/stage1/inq1697.txt","rb")
load_list = pickle.load(f2)
f2.close()
# print
print(load_list)