Hyperparameter Scaling



In [7]:

    
%matplotlib inline
import matplotlib.pyplot as plt
import pandas as pd



In [2]:

    
import numpy as np
np.random.seed(0)

Learning Rate $ \alpha $

Pick values between 0 and 1

Problem is that 1e-4 to 1e-3 regions, 1e-3 to 1e-2, 1e-2 to 1e-1 regions are not equally represented.



In [3]:

    
random_values = np.random.rand(100)
r = -4 * random_values
alpha = 10 ** r



In [4]:

    
plt.scatter(r, alpha);



In [22]:

    
ranges_alpha = [1e-4, 1e-3, 1e-2, 1e-1, 5e-1, 1]



In [18]:

    
# There's a very small number of values sampled from 1e-4 to 1e-3 and 1e-3 to 1e-2 ranges (in fact 1 in total)
pd.cut(random_values, ranges_alpha).value_counts()









    Out[18]:





(0.0001, 0.001]     0
(0.001, 0.01]       1
(0.01, 0.1]        11
(0.1, 0.5]         39
(0.5, 1]           49
dtype: int64



In [19]:

    
# Whereas this sampling allows for more uniform sampling
pd.cut(alpha, ranges_alpha).value_counts()









    Out[19]:





(0.0001, 0.001]    20
(0.001, 0.01]      29
(0.01, 0.1]        23
(0.1, 0.5]         20
(0.5, 1]            8
dtype: int64

Beta $\beta$

Ranging between 0.9 and 0.999



In [69]:

    
random_values_beta = 2 * random_values - 3
beta = 1 - 10 ** random_values_beta



In [70]:

    
pd.Series(random_values_beta).hist();



In [71]:

    
plt.scatter(r, beta);



In [72]:

    
ranges_beta = [0.9, 0.99, 0.999]



In [73]:

    
pd.cut(beta, ranges_beta).value_counts()









    Out[73]:





(0.9, 0.99]      49
(0.99, 0.999]    51
dtype: int64