In [1]:
import tensorflow as tf
import rfho as rf
from rfho.datasets import load_mnist
In [2]:
mnist = load_mnist(partitions=(.05, .01)) # 5% of data in training set, 1% in validation
# remaining in test set (change these percentages and see the effect on regularization hyperparameter)
In [3]:
x, y = tf.placeholder(tf.float32, name='x'), tf.placeholder(tf.float32, name='y')
# define the model (here use a linear model from rfho.models)
model = rf.LinearModel(x, mnist.train.dim_data, mnist.train.dim_target)
# vectorize the model, and build the state vector (augment by 1 since we are
# going to optimize the weights with momentum)
s, out, w_matrix = rf.vectorize_model(model.var_list, model.inp[-1], model.Ws[0],
augment=1)
# (this function will print also some tensorflow infos and warnings about variables
# collections... we'll solve this)
In [4]:
# define error
error = tf.reduce_mean(rf.cross_entropy_loss(labels=y, logits=out), name='error')
constraints = []
# define training error by error + L2 weights penalty
rho = tf.Variable(0., name='rho') # regularization hyperparameter
training_error = error + rho*tf.reduce_sum(tf.pow(w_matrix, 2))
constraints.append(rf.positivity(rho)) # regularization coefficient should be positive
accuracy = tf.reduce_mean(tf.cast(tf.equal(tf.argmax(out, 1), tf.argmax(y, 1)),
"float"), name='accuracy')
# define learning rates and momentum factor as variables, to be optimized
eta = tf.Variable(.01, name='eta')
mu = tf.Variable(.5, name='mu')
# now define the training dynamics (similar to tf.train.Optimizer)
optimizer = rf.MomentumOptimizer.create(s, eta, mu, loss=training_error)
# add constraints for learning rate and momentum factor
constraints += optimizer.get_natural_hyperparameter_constraints()
In [5]:
# we want to optimize the weights w.r.t. training_error
# and hyperparameters w.r.t. validation error (that in this case is
# error evaluated on the validation set)
# we are going to use ReverseMode
hyper_dict = {error: [rho, eta, mu]}
hyper_opt = rf.HyperOptimizer(optimizer, hyper_dict, method=rf.ReverseHG)
In [6]:
# define helper for stochastic descent
ev_data = rf.ExampleVisiting(mnist.train, batch_size=2**8, epochs=200)
tr_suppl = ev_data.create_supplier(x, y)
val_supplier = mnist.validation.create_supplier(x, y)
test_supplier = mnist.test.create_supplier(x, y)
In [7]:
# Run all for some hyper-iterations and print progresses
def run(hyper_iterations):
with tf.Session().as_default() as ss:
ev_data.generate_visiting_scheme() # needed for remembering the example visited in forward pass
for hyper_step in range(hyper_iterations):
hyper_opt.initialize() # initializes all variables or reset weights to initial state
hyper_opt.run(ev_data.T, train_feed_dict_supplier=tr_suppl,
val_feed_dict_suppliers=val_supplier,
hyper_constraints_ops=constraints)
#
# print('Concluded hyper-iteration', hyper_step)
# print('Test accuracy:', ss.run(accuracy, feed_dict=test_supplier()))
# print('Validation error:', ss.run(error, feed_dict=val_supplier()))
In [8]:
saver = rf.Saver('Staring example', collect_data=False)
with saver.record(rf.Records.tensors('error', fd=('x', 'y', mnist.validation), rec_name='valid'),
rf.Records.tensors('error', fd=('x', 'y', mnist.test), rec_name='test'),
rf.Records.tensors('accuracy', fd=('x', 'y', mnist.validation), rec_name='valid'),
rf.Records.tensors('accuracy', fd=('x', 'y', mnist.test), rec_name='test'),
rf.Records.hyperparameters(),
rf.Records.hypergradients(),
): # a context to print some statistics.
# If you execute again any cell containing the model construction,
# restart the notebook or reset tensorflow graph in order to prevent errors
# due to tensor namings
run(50) # this will take some time... run it for less hyper-iterations for a qucker look
In [ ]: