Entropy tradeoffs

Experiments for entropy vs Kullback-Liebler divergence.



In [1]:

    
import math._
def hDiff(x: Double, h0: Double) = (-x * h0) - (x * log(x)) - ((1 - x) * log(1 - x))









    Out[1]:





import math._

defined function hDiff



In [2]:

    
hDiff(0.1, 3)









    Out[2]:





res1: Double = 0.025082973391448174



In [3]:

    
def klDiff(x: Double, p0: Double, q0: Double) = (1 - p0) * log(1 - x) + p0 * log((q0 + x)/q0)









    Out[3]:





defined function klDiff



In [4]:

    
klDiff(0.001, 0.1, 0.0001)









    Out[4]:





res3: Double = 0.2388890769796119



In [5]:

    
def geomStep(a : Double, b: Double, j: Int, n: Int) = math.pow(a, (n.toDouble -  j)/n) * math.pow(b, j.toDouble /n)

def ratioRange(h0: Double, p0: Double, q0: Double, steps: Int, hW: Double = 5) = {
    (-steps to steps).map{j =>
       val x = geomStep(q0, p0, j, steps)
       (j, x, klDiff(x, p0, q0)/ hDiff(x, h0), klDiff(x, p0, q0) - (hW *  hDiff(x, h0)))}
}









    Out[5]:





defined function geomStep
defined function ratioRange



In [6]:

    
val r1 = ratioRange(3, 0.01, 0.001, 10)









    Out[6]:





r1: collection.immutable.IndexedSeq[(Int, Double, Double, Double)] = Vector(
  (-10, 9.999999999999999E-5, 1.1845526377554119, -0.0027510483374414262),
  (-9, 1.258925411794168E-4, 1.2075570983246235, -0.0033325421438979387),
  (-8, 1.5848931924611134E-4, 1.2285453818660486, -0.004034560148472095),
  (-7, 1.9952623149688798E-4, 1.2467025534323188, -0.004882305448876259),
  (-6, 2.511886431509578E-4, 1.261089379778421, -0.005906621724741096),
  (-5, 3.1622776601683794E-4, 1.2706668624768478, -0.007145361838556817),
  (-4, 3.981071705534975E-4, 1.2743423073851317, -0.00864502940025556),
  (-3, 5.011872336272721E-4, 1.2710386316882742, -0.01046266966076477),
  (-2, 6.309573444801935E-4, 1.259783757292165, -0.012667931903230047),
  (-1, 7.943282347242811E-4, 1.239810550567618, -0.015345159648736971),
  (0, 0.001, 1.2106516452635523, -0.018595299085808682),
  (1, 0.001258925411794167, 1.1722102433663446, -0.02253736298981176),
  (2, 0.0015848931924611132, 1.1247898811701507, -0.02730915225493906),
  (3, 0.0019952623149688802, 1.0690735875987332, -0.03306690942358154),
  (4, 0.0025118864315095803, 1.0060536977969712, -0.03998353047584451),
  (5, 0.0031622776601683794, 0.9369239038657533, -0.04824485546174627),
  (6, 0.0039810717055349725, 0.862951211867383, -0.058043360685566384),
  (7, 0.005011872336272725, 0.7853455929736037, -0.06956826017509153),
  (8, 0.006309573444801932, 0.7051399292716359, -0.0829905742053144),
  (9, 0.007943282347242814, 0.6230844139822886, -0.09844111640019926),
  (10, 0.01, 0.5395497066272893, -0.11597855154121946)
)



In [7]:

    
val r2 = ratioRange(3, 0.01, 0.0001, 20)









    Out[7]:





r2: collection.immutable.IndexedSeq[(Int, Double, Double, Double)] = Vector(
  (-20, 1.0E-6, 8.337626353253418, 3.94357577466878E-5),
  (-19, 1.2589254117941674E-6, 8.49232931627923, 5.093550927573794E-5),
  (-18, 1.584893192461115E-6, 8.65045266711772, 6.569519147486084E-5),
  (-17, 1.995262314968878E-6, 8.811486399963552, 8.460265869278623E-5),
  (-16, 2.51188643150958E-6, 8.974704761016847, 1.0877051685274965E-4),
  (-15, 3.1622776601683796E-6, 9.139107640218397, 1.3958401015730097E-4),
  (-14, 3.981071705534973E-6, 9.303350437664125, 1.787540036084948E-4),
  (-13, 5.0118723362727275E-6, 9.465662362215474, 2.2837233954958622E-4),
  (-12, 6.309573444801927E-6, 9.62375473801637, 2.9096432220240506E-4),
  (-11, 7.943282347242815E-6, 9.774723567461809, 3.6952936802033323E-4),
  (-10, 1.0000000000000003E-5, 9.914954634785962, 4.6755572529551384E-4),
  (-9, 1.258925411794168E-5, 10.040045009681085, 5.88988600630176E-4),
  (-8, 1.584893192461115E-5, 10.144761715072436, 7.381234887242816E-4),
  (-7, 1.9952623149688776E-5, 10.223065589884499, 9.193892534803529E-4),
  (-6, 2.511886431509579E-5, 10.268233848651827, 0.0011369809545059165),
  (-5, 3.1622776601683795E-5, 10.273114936138729, 0.0013943041312325715),
  (-4, 3.981071705534974E-5, 10.230539415565767, 0.0016932043463130284),
  (-3, 5.0118723362727265E-5, 10.133886852512969, 0.002032981131360591),
  (-2, 6.309573444801928E-5, 9.977770602864435, 0.002409223003660641),
  (-1, 7.943282347242813E-5, 9.758756665733248, 0.0028125423623877287),
  (0, 1.0E-4, 9.47599403680649, 0.0032273216701147788),
  (1, 1.2589254117941677E-4, 9.131621264069805, 0.0036305891326814423),
  (2, 1.584893192461113E-4, 8.73084105408975, 0.003991113286830888),
  (3, 1.9952623149688798E-4, 8.281620426372656, 0.004268745953901029),
  (4, 2.511886431509579E-4, 7.794058218730942, 0.004413971515042409),
  (5, 3.1622776601683794E-4, 7.279534631029139, 0.004367563626413409),
  (6, 3.981071705534974E-4, 6.749794021706447, 0.004060228289901394),
  (7, 5.011872336272723E-4, 6.216104576366052, 0.003412129866398401),
  (8, 6.309573444801934E-4, 5.688597231939777, 0.00233224559140743),
  (9, 7.943282347242813E-4, 5.175831442649361, 7.175600046240987E-4),
  (10, 0.001, 4.684585755575198, -0.0015478181634244297),
  (11, 0.001258925411794167, 4.2198374820090665, -0.004593461756496883),
  (12, 0.0015848931924611134, 3.784880077755565, -0.008563121468779387),
  (13, 0.00199526231496888, 3.3815255596338396, -0.013614589058480332),
  (14, 0.0025118864315095807, 3.0103467346335098, -0.019918485616159547),
  (15, 0.0031622776601683794, 2.6709250603243184, -0.027655372718994244),
  (16, 0.003981071705534972, 2.3620810288731615, -0.0370103641851238),
  (17, 0.005011872336272724, 2.0820731239234385, -0.04816411417941318),
...



In [8]:

    
val r3 = ratioRange(3, 0.01, 0.001, 10, 1)









    Out[8]:





r3: collection.immutable.IndexedSeq[(Int, Double, Double, Double)] = Vector(
  (-10, 9.999999999999999E-5, 1.1845526377554119, 1.3306781068230314E-4),
  (-9, 1.258925411794168E-4, 1.2075570983246235, 1.8238713023903646E-4),
  (-8, 1.5848931924611134E-4, 1.2285453818660486, 2.4448924437816123E-4),
  (-7, 1.9952623149688798E-4, 1.2467025534323188, 3.2091174174745155E-4),
  (-6, 2.511886431509578E-4, 1.261089379778421, 4.124613715978606E-4),
  (-5, 3.1622776601683794E-4, 1.2706668624768478, 5.185947725196939E-4),
  (-4, 3.981071705534975E-4, 1.2743423073851317, 6.365848687010815E-4),
  (-3, 5.011872336272721E-4, 1.2710386316882742, 7.604765479091026E-4),
  (-2, 6.309573444801935E-4, 1.259783757292165, 8.798750482297874E-4),
  (-1, 7.943282347242811E-4, 1.239810550567618, 9.786557920551352E-4),
  (0, 0.001, 1.2106516452635523, 0.0010337213631196667),
  (1, 0.001258925411794167, 1.1722102433663446, 0.0010139440805454307),
  (2, 0.0015848931924611132, 1.1247898811701507, 8.794118925815669E-4),
  (3, 0.0019952623149688802, 1.0690735875987332, 5.810462534947044E-4),
  (4, 0.0025118864315095803, 1.0060536977969712, 6.060377181917563E-5),
  (5, 0.0031622776601683794, 0.9369239038657533, -7.489638562229381E-4),
  (6, 0.0039810717055349725, 0.862951211867383, -0.0019228132537184531),
  (7, 0.005011872336272725, 0.7853455929736037, -0.003543145461902354),
  (8, 0.006309573444801932, 0.7051399292716359, -0.005697649324304201),
  (9, 0.007943282347242814, 0.6230844139822886, -0.00847720051872819),
  (10, 0.01, 0.5395497066272893, -0.011972414121830076)
)



In [9]:

    
r1.maxBy(_._3)
r2.maxBy(_._3)
r3.maxBy(_._3)









    Out[9]:





res8_0: (Int, Double, Double, Double) = (
  -4,
  3.981071705534975E-4,
  1.2743423073851317,
  -0.00864502940025556
)
res8_1: (Int, Double, Double, Double) = (
  -5,
  3.1622776601683795E-5,
  10.273114936138729,
  0.0013943041312325715
)
res8_2: (Int, Double, Double, Double) = (
  -4,
  3.981071705534975E-4,
  1.2743423073851317,
  6.365848687010815E-4
)



In [10]:

    
r1.maxBy(_._4)
r2.maxBy(_._4)
r3.maxBy(_._4)









    Out[10]:





res9_0: (Int, Double, Double, Double) = (
  -10,
  9.999999999999999E-5,
  1.1845526377554119,
  -0.0027510483374414262
)
res9_1: (Int, Double, Double, Double) = (
  4,
  2.511886431509579E-4,
  7.794058218730942,
  0.004413971515042409
)
res9_2: (Int, Double, Double, Double) = (
  0,
  0.001,
  1.2106516452635523,
  0.0010337213631196667
)



In [13]:

    
val r4 = ratioRange(10, 0.01, 0.001, 10)









    Out[13]:





r4: collection.immutable.IndexedSeq[(Int, Double, Double, Double)] = Vector(
  (-10, 9.999999999999999E-5, 40.61511929704236, 7.489516625585742E-4),
  (-9, 1.258925411794168E-4, -421.8374960471386, 0.0010736967973816493),
  (-8, 1.5848931924611134E-4, -33.13555084429469, 0.0015125660251418013),
  (-7, 1.9952623149688798E-4, -16.91413739341092, 0.0021011126535148195),
  (-6, 2.511886431509578E-4, -11.157855955935197, 0.002884980785542425),
  (-5, 3.1622776601683794E-4, -8.180582358139926, 0.00392260997203251),
  (-4, 3.981071705534975E-4, -6.340752279639776, 0.005288721569116854),
  (-3, 5.011872336272721E-4, -5.0763570747914155, 0.007078883516189755),
  (-2, 6.309573444801935E-4, -4.1435573090372815, 0.009415575153576727),
  (-1, 7.943282347242811E-4, -3.4201716401786144, 0.012456328566612867),
  (0, 0.001, -2.8388440989997124, 0.016404700914191318),
  (1, 0.001258925411794167, -2.3598612541536688, 0.02152502642298409),
  (2, 0.0015848931924611132, -1.9585705263888231, 0.028162109481199897),
  (3, 0.0019952623149688802, -1.6189527291267525, 0.03676727160032927),
  (4, 0.0025118864315095803, -1.3300857473622503, 0.04793249462699078),
  (5, 0.0031622776601683794, -1.0841006419649069, 0.062434862644147),
  (6, 0.0039810717055349725, -0.8749731999958605, 0.08129414900815767),
  (7, 0.005011872336272725, -0.6978119969004899, 0.10584727159445384),
  (8, 0.006309573444801932, -0.548450369029329, 0.13784449636275317),
  (9, 0.007943282347242814, -0.42322359043509955, 0.17957376575329925),
  (10, 0.01, -0.318854760667384, 0.2340214484587806)
)



In [15]:

    
val r5 = ratioRange(10, 0.01, 0.0001, 10)









    Out[15]:





r5: collection.immutable.IndexedSeq[(Int, Double, Double, Double)] = Vector(
  (-10, 1.0E-6, 20.457502289531742, 7.44357577466878E-5),
  (-9, 1.584893192461115E-6, 22.55476254516544, 1.2116645321099987E-4),
  (-8, 2.51188643150958E-6, 25.10600077783788, 1.9668654195558494E-4),
  (-7, 3.981071705534973E-6, 28.26789873017816, 3.1809151330221893E-4),
  (-6, 6.309573444801927E-6, 32.27976877337067, 5.117993927704725E-4),
  (-5, 1.0000000000000003E-5, 37.53408680055866, 8.175557252955138E-4),
  (-4, 1.584893192461115E-5, 44.744895192420095, 0.001292836106085672),
  (-3, 2.511886431509579E-5, 55.42093885328467, 0.0020161412055342692),
  (-2, 3.981071705534974E-5, 73.52968719295646, 0.0030865794432502693),
  (-1, 6.309573444801928E-5, 114.09480045240937, 0.0046175737093413155),
  (0, 1.0E-4, 324.90631145969195, 0.006727321670114778),
  (1, 1.584893192461113E-4, -235.48273586916855, 0.009538239460444783),
  (2, 2.511886431509579E-4, -68.9602024339089, 0.013205574025325935),
  (3, 3.981071705534974E-4, -33.58498857191237, 0.017993979259273804),
  (4, 6.309573444801934E-4, -18.710376683406313, 0.024415752648214196),
  (5, 0.001, -10.984835051851503, 0.03345218183657557),
  (6, 0.0015848931924611134, -6.590523874998215, 0.04690814026735958),
  (7, 0.0025118864315095807, -3.979926016993338, 0.06799753948667578),
  (8, 0.003981071705534972, -2.3949877676286073, 0.10232714550860023),
  (9, 0.006309573444801932, -1.4223884910876414, 0.1595564253294015),
  (10, 0.01, -0.8227871618390059, 0.25619370089920945)
)



In [16]:

    
val r6 = ratioRange(3, 0.0001, 0.000001, 10)









    Out[16]:





r6: collection.immutable.IndexedSeq[(Int, Double, Double, Double)] = Vector(
  (-10, 1.0E-8, 5.998740859619163, 1.6400004801771548E-7),
  (-9, 1.5848931924611156E-8, 6.153817078518452, 2.9185981644682834E-7),
  (-8, 2.511886431509579E-8, 6.307550241999972, 5.09073141236953E-7),
  (-7, 3.981071705534973E-8, 6.453888441284274, 8.704699590474054E-7),
  (-6, 6.309573444801923E-8, 6.5830706579122245, 1.456184808493852E-6),
  (-5, 9.999999999999998E-8, 6.6800992445447465, 2.3719801752696242E-6),
  (-4, 1.5848931924611157E-7, 6.723340719128838, 3.7303094180534433E-6),
  (-3, 2.511886431509578E-7, 6.684331493584441, 5.583478000683879E-6),
  (-2, 3.9810717055349756E-7, 6.5306773698809, 7.761314378343795E-6),
  (-1, 6.309573444801924E-7, 6.23392811673463, 9.557574776213051E-6),
  (0, 1.0E-6, 5.78179166372952, 9.237267266051121E-6),
  (1, 1.5848931924611128E-6, 5.1890055690749435, 3.40142940683499E-6),
  (2, 2.5118864315095785E-6, 4.498469423201515, -1.3724727579985822E-5),
  (3, 3.981071705534975E-6, 3.769622275374925, -5.110783967359717E-5),
  (4, 6.309573444801933E-6, 3.0607849727822924, -1.2203121012451971E-4),
  (5, 1.0E-5, 2.4155618469767264, -2.4585554596290125E-4),
  (6, 1.584893192461113E-5, 1.8580861484517601, -4.5077314398870596E-4),
  (7, 2.5118864315095815E-5, 1.3953830557018, -7.779420830009608E-4),
  (8, 3.98107170553497E-5, 1.0227727753675204, -0.0012874883234760587),
  (9, 6.30957344480193E-5, 0.7292334682715141, -0.002067035585284577),
  (10, 1.0E-4, 0.5013904201964933, -0.0032436281333038486)
)



In [17]:

    
r6.maxBy(_._3)
r6.maxBy(_._4)









    Out[17]:





res16_0: (Int, Double, Double, Double) = (
  -4,
  1.5848931924611157E-7,
  6.723340719128838,
  3.7303094180534433E-6
)
res16_1: (Int, Double, Double, Double) = (
  -1,
  6.309573444801924E-7,
  6.23392811673463,
  9.557574776213051E-6
)

Conclusions

The largest entropy ratio in all these cases is at $q_0$ or even below.
As adding a weight below $q_0$ is clearly useless, we can compare largest terms at $q_0$ (or at $kq_0$, $k > 1$).
This gives the criterion we had seen: $\frac{p}{qlog(q)}$ (or $\frac{p}{q(log(q) + log(k))}$)