Plotting code for noise experiments

This notebook contains a great deal of plotting code for noise experiments conducted in May 2018 by Max Schwarzer. It assumes that results will be a set of .json files placed in results/, which are laid out in the format created by noise_simulation.py. This code employs matplotlib, operating on top of Pandas. As such, it should be easily modified by a user.


In [1]:
import cPickle as pkl
import matplotlib.pyplot as plt
from mpl_toolkits.mplot3d import Axes3D
from matplotlib.ticker import LinearLocator, FormatStrFormatter
from matplotlib import cm
plt.style.use('seaborn')
import seaborn as sns
import numpy as np
import pandas as pd
import json
import glob

# Helper functions for parsing the simulation output.
def dict_to_mean(failure_dict):
    items = 0
    counter = 0
    for key, value in failure_dict.items():
        if key is None or key == "null":
            continue
        items += int(value)
        counter += int(key)*int(value)
    if items == 0:
        return 40
    return(counter/items)

def failureRate(failure_dict):
    items = 0.
    failed = 0.
    for key, value in failure_dict.items():
        if key is None or key == "null":
            failed += int(value)
            items += int(value)
        else:
            items += int(value)
    return(failed/items)

In [2]:
resultsFiles = glob.glob("results/*.json")
results = []
for f in resultsFiles:
    with open(f, "rb") as ff:
        results += json.load(ff)

# Some old data may not include a threshold, so set it to the default.
for experiment in results:
    if "threshold" not in experiment[0]:
        experiment[0]["threshold"] = 16
        
        
jointDfResults = pd.DataFrame([r[0] for r in results])
jointDfResults["convergence"] = [r[1] for r in results]
jointDfResults["failureRate"] = [failureRate(r) for r in jointDfResults["convergence"]]
jointDfResults["meanTime"] = [dict_to_mean(r) for r in jointDfResults["convergence"]]


# Split by method and threshold, and compare pairwise results
thresh12DfResults = jointDfResults.query("threshold==12")
thresh16DfResults = jointDfResults.query("threshold==16")
pairedThresholdResults = thresh12DfResults.merge(thresh16DfResults, on = ["numObjects",
                                                                  "moduleNoiseFactor",
                                                                  "noiseFactor",
                                                                  "anchoringMethod"], how = "inner")
pairedThresholdResults["difference"] = pairedThresholdResults["failureRate_x"] - pairedThresholdResults["failureRate_y"]

narrowingDfResults = jointDfResults.query("anchoringMethod=='narrowing'")
reanchoringDfResults = jointDfResults.query("anchoringMethod=='reanchoring'")

pairedMethodResults = narrowingDfResults.merge(reanchoringDfResults, on = ["numObjects",
                                                                  "moduleNoiseFactor",
                                                                  "noiseFactor",
                                                                  "threshold"], how = "inner")
pairedMethodResults["difference"] = pairedMethodResults["failureRate_y"] - pairedMethodResults["failureRate_x"]

In [3]:
thresh16fixedObjectResults = thresh16DfResults.query("numObjects==600")
thresh16noiseResults = thresh16fixedObjectResults.query("moduleNoiseFactor == 0")
thresh16moduleNoiseResults = thresh16fixedObjectResults.query("noiseFactor == 0")

thresh12fixedObjectResults = thresh12DfResults.query("numObjects==600")
thresh12noiseResults = thresh12fixedObjectResults.query("moduleNoiseFactor == 0")
thresh12moduleNoiseResults = thresh12fixedObjectResults.query("noiseFactor == 0")

thresh16Results = thresh16moduleNoiseResults.query("anchoringMethod == 'reanchoring'") 
plt.plot(thresh16Results["moduleNoiseFactor"], thresh16Results["failureRate"], color = "blue", label = "Threshold=16 module noise")

thresh12Results = thresh12moduleNoiseResults.query("anchoringMethod == 'reanchoring'") 
plt.plot(thresh12Results["moduleNoiseFactor"][1:], thresh12Results["failureRate"][1:], color = "red", label = "Threshold=12 module noise")


plt.legend(loc = "best")
plt.xlabel("Module Noise")
plt.ylabel("Failure Rate")
plt.title("Failure rate vs module noise")
plt.savefig("ModuleNoiseThresholdComparisonLine.png")
plt.show()

thresh16Results = thresh16noiseResults.query("anchoringMethod == 'reanchoring'") 
plt.plot(thresh16Results["noiseFactor"][:-1], thresh16Results["failureRate"][:-1], color = "blue", label = "Threshold=16 global noise")

thresh12Results = thresh12noiseResults.query("anchoringMethod == 'reanchoring'") 
plt.plot(thresh12Results["noiseFactor"][:-1], thresh12Results["failureRate"][:-1], color = "red", label = "Threshold=12 global noise")


plt.legend(loc = "best")
plt.xlabel("Global Noise")
plt.ylabel("Failure Rate")
plt.title("Failure rate vs global noise")
plt.savefig("GlobalNoiseThresholdComparisonLine.png")
plt.show()



In [4]:
thresh16fixedObjectResults = thresh16DfResults.query("numObjects==600")
thresh16noiseResults = thresh16fixedObjectResults.query("moduleNoiseFactor == 0")
thresh16moduleNoiseResults = thresh16fixedObjectResults.query("noiseFactor == 0")

thresh12fixedObjectResults = thresh12DfResults.query("numObjects==600")
thresh12noiseResults = thresh12fixedObjectResults.query("moduleNoiseFactor == 0")
thresh12moduleNoiseResults = thresh12fixedObjectResults.query("noiseFactor == 0")

thresh16Results = thresh16moduleNoiseResults.query("anchoringMethod == 'narrowing'") 
plt.plot(thresh16Results["moduleNoiseFactor"], thresh16Results["failureRate"], color = "blue", label = "Module noise")

thresh16Results = thresh16noiseResults.query("anchoringMethod == 'narrowing'") 
plt.plot(thresh16Results["noiseFactor"][:-1], thresh16Results["failureRate"][:-1], color = "red", label = "Global noise")


plt.legend(loc = "best")
plt.xlabel("Noise Scale")
plt.ylabel("Failure Rate")
plt.title("t=16, n=600 Failure rate vs noise")
plt.savefig("600ojbT16NoiseComparison.png", dpi = 250)
plt.show()


thresh12Results = thresh12moduleNoiseResults.query("anchoringMethod == 'narrowing'") 
plt.plot(thresh12Results["moduleNoiseFactor"], thresh12Results["failureRate"], color = "blue", label = "Module noise")

thresh12Results = thresh12noiseResults.query("anchoringMethod == 'narrowing'") 
plt.plot(thresh12Results["noiseFactor"][:-1], thresh12Results["failureRate"][:-1], color = "red", label = "Global noise")


plt.legend(loc = "best")
plt.xlabel("Noise Scale")
plt.ylabel("Failure Rate")
plt.title("t=12, n=600 Failure rate vs noise")
plt.savefig("600ojbT12NoiseComparison.png", dpi = 250)
plt.show()



In [5]:
thresh16Results = thresh16moduleNoiseResults.query("anchoringMethod == 'narrowing'") 
plt.plot(thresh16Results["moduleNoiseFactor"], thresh16Results["meanTime"], color = "blue", label = "Module noise")

thresh16Results = thresh16noiseResults.query("anchoringMethod == 'narrowing'") 
plt.plot(thresh16Results["noiseFactor"][:-1], thresh16Results["meanTime"][:-1], color = "red", label = "Global noise")


plt.legend(loc = "best")
plt.xlabel("Noise Scale")
plt.ylabel("Convergence time")
plt.title("t=16, n=600 Convergence time vs noise")
plt.savefig("600ojbT16NoiseTimeComparison.png", dpi = 250)
plt.show()

thresh12Results = thresh12moduleNoiseResults.query("anchoringMethod == 'narrowing'") 
plt.plot(thresh12Results["moduleNoiseFactor"], thresh12Results["meanTime"], color = "blue", label = "Module noise")

thresh12Results = thresh12noiseResults.query("anchoringMethod == 'narrowing'") 
plt.plot(thresh12Results["noiseFactor"][:-1], thresh12Results["meanTime"][:-1], color = "red", label = "Global noise")


plt.legend(loc = "best")
plt.xlabel("Noise Scale")
plt.ylabel("Convergence time")
plt.title("t=12, n=600 Convergence time vs noise")
plt.savefig("600ojbT12NoiseTimeComparison.png", dpi = 250)
plt.show()



In [6]:
fig = plt.figure()
ax = plt.subplot(111)

thresh16noiseResults = thresh16DfResults.query("moduleNoiseFactor == 0")
bound = len(thresh16noiseResults.groupby(["noiseFactor"])) + 0.
for i, (noise, data) in enumerate(thresh16noiseResults.groupby(["noiseFactor"])):
    data = data.query("anchoringMethod == 'narrowing'")
    plt.plot(data["numObjects"],
             data["failureRate"],
             color = cm.coolwarm(i/bound),
             label = "Global Noise {}".format(noise))

# Shrink current axis by 20%
box = ax.get_position()
ax.set_position([box.x0, box.y0, box.width * 0.70, box.height])

# Put a legend to the right of the current axis
ax.legend(loc='center left', bbox_to_anchor=(1, 0.5))

plt.xlabel("Number of objects")
plt.ylabel("Failure rate")
plt.title("Failure with global noise")
plt.savefig("GlobalNoiseLinePlot.png", dpi = 250)
plt.show()



In [7]:
fig = plt.figure()
ax = plt.subplot(111)

moduleNoiseResults = thresh16DfResults.query("noiseFactor == 0")
bound = len(moduleNoiseResults.groupby(["moduleNoiseFactor"])) + 0.
for i, (noise, data) in enumerate(moduleNoiseResults.groupby(["moduleNoiseFactor"])):
    data = data.query("anchoringMethod == 'narrowing'")
    plt.plot(data["numObjects"][:5],
             data["failureRate"][:5],
             color = cm.coolwarm(i/bound),
             label = "Module Noise {}".format(noise))

# Shrink current axis by 20%
box = ax.get_position()
ax.set_position([box.x0, box.y0, box.width * 0.7, box.height])

# Put a legend to the right of the current axis
ax.legend(loc='center left', bbox_to_anchor=(1, 0.5))

plt.xlabel("Number of Objects")
plt.ylabel("Failure rate")
plt.title("Failure with module noise")
plt.savefig("ModuleNoiseLinePlot.png", dpi = 250)
plt.show()



In [8]:
fig = plt.figure()
ax = fig.gca(projection='3d')

# Make data.
for method, group in pairedThresholdResults.groupby(["anchoringMethod"]):
    if method not in ["reanchoring"]: continue
    
# Plot the surface.
    surf = ax.plot_trisurf(group.noiseFactor,
                           group.numObjects,
                           group.difference,
                           cmap=cm.coolwarm,
                           linewidth=0,
                           vmin = -.1,
                           vmax = .1,
                           antialiased=True)

ax.set_xlabel("Global Noise")
ax.set_ylabel("Number of Objects")
ax.set_zlabel("Failure rate change")
plt.title("Threshold Comparison")


# Customize the z axis.
ax.zaxis.set_major_locator(LinearLocator(10))
ax.zaxis.set_major_formatter(FormatStrFormatter('%.02f'))


ax.view_init(30, 90 + 210)
plt.savefig("ThresholdComparisonGlobalNoise.png", dpi = 250)
plt.draw()
plt.show()



In [9]:
fig = plt.figure()
ax = fig.gca(projection='3d')

# Make data.
for method, group in pairedThresholdResults.groupby(["anchoringMethod"]):
    if method not in ["reanchoring"]: continue
    
# Plot the surface.
    surf = ax.plot_trisurf(group.moduleNoiseFactor,
                           group.numObjects,
                           group.difference,
                           cmap=cm.coolwarm,
                           linewidth=0,
                           vmin = -.1,
                           vmax = .1,
                           antialiased=True)

ax.set_xlabel("Module Noise")
ax.set_ylabel("Number of Objects")
ax.set_zlabel("Failure rate change")
plt.title("Threshold Comparison")


# Customize the z axis.
ax.zaxis.set_major_locator(LinearLocator(10))
ax.zaxis.set_major_formatter(FormatStrFormatter('%.02f'))


ax.view_init(30, 90 + 210)
plt.savefig("ThresholdComparisonModuleNoise.png", dpi = 250)
plt.draw()
plt.show()



In [10]:
fig = plt.figure()
ax = fig.gca(projection='3d')

# Make data.
data = pairedMethodResults.query("threshold==16")
surf = ax.plot_trisurf(data.noiseFactor,
                       data.numObjects,
                       data.difference,
                       cmap=cm.coolwarm,
                       linewidth=0,
                       vmin = -.1,
                       vmax = .1,
                       antialiased=True)

ax.set_xlabel("Global Noise")
ax.set_ylabel("Number of Objects")
ax.set_zlabel("Failure rate change")
plt.title("Method Comparison")


# Customize the z axis.
ax.zaxis.set_major_locator(LinearLocator(10))
ax.zaxis.set_major_formatter(FormatStrFormatter('%.02f'))


ax.view_init(30, 90 + 210)
plt.savefig("MethodGlobalNoiseComparison.png", dpi = 250)
plt.draw()
plt.show()



In [11]:
fig = plt.figure()
ax = fig.gca(projection='3d')

# Make data.
data = pairedMethodResults.query("threshold==16")
surf = ax.plot_trisurf(data.moduleNoiseFactor,
                       data.numObjects,
                       data.difference,
                       cmap=cm.coolwarm,
                       linewidth=0,
                       vmin = -.1,
                       vmax = .1,
                       antialiased=True)

ax.set_xlabel("Module Noise")
ax.set_ylabel("Number of Objects")
ax.set_zlabel("Failure rate change")
plt.title("Method Comparison")


# Customize the z axis.
ax.zaxis.set_major_locator(LinearLocator(10))
ax.zaxis.set_major_formatter(FormatStrFormatter('%.02f'))

plt.tight_layout()
ax.view_init(45, 90 + 210)
plt.savefig("MethodModuleNoiseComparison.png", dpi = 250)
plt.draw()
plt.show()



In [12]:
fig = plt.figure()
ax = fig.gca(projection='3d')


# Make data.
data = thresh16DfResults.query("noiseFactor==0")
data = data.query("anchoringMethod=='reanchoring'")
# Plot the surface.
surf = ax.plot_trisurf(data.moduleNoiseFactor,
                       data.numObjects,
                       data.failureRate,
                       linewidth=0,
                       cmap=cm.coolwarm,
                       vmin = 0,
                       vmax = 1,
                       antialiased=True)

ax.set_xlabel("Module Noise")
ax.set_ylabel("Number of Objects")
ax.set_zlabel("Failure Rate")
plt.title("T=16 Reanchoring Results")

# Customize the z axis.
ax.set_zlim(0, 1.0)
ax.zaxis.set_major_locator(LinearLocator(10))
ax.zaxis.set_major_formatter(FormatStrFormatter('%.02f'))


ax.view_init(30, 0 + 210)
plt.savefig("T16ReanchoringModule3DFailureSurface.png", dpi = 250)
plt.draw()
plt.show()

fig = plt.figure()
ax = fig.gca(projection='3d')
# Make data.
data = thresh16DfResults.query("noiseFactor==0")
data = data.query("anchoringMethod=='narrowing'")
# Plot the surface.
surf = ax.plot_trisurf(data.moduleNoiseFactor,
                       data.numObjects,
                       data.failureRate,
                       linewidth=0,
                       cmap=cm.coolwarm,
                       vmin = 0,
                       vmax = 1,
                       antialiased=True)

ax.set_xlabel("Module Noise")
ax.set_ylabel("Number of Objects")
ax.set_zlabel("Failure Rate")
plt.title("T=16 Narrowing Results")

# Customize the z axis.
ax.set_zlim(0, 1.0)
ax.zaxis.set_major_locator(LinearLocator(10))
ax.zaxis.set_major_formatter(FormatStrFormatter('%.02f'))


ax.view_init(30, 0 + 210)
plt.savefig("T16NarrowingModule3DFailureSurface.png", dpi = 250)
plt.draw()
plt.show()



In [13]:
fig = plt.figure()
ax = fig.gca(projection='3d')


# Make data.
data = thresh12DfResults.query("noiseFactor==0")
data = data.query("anchoringMethod=='reanchoring'")
# Plot the surface.
surf = ax.plot_trisurf(data.moduleNoiseFactor,
                       data.numObjects,
                       data.failureRate,
                       linewidth=0,
                       cmap=cm.coolwarm,
                       vmin = 0,
                       vmax = 1,
                       antialiased=True)

ax.set_xlabel("Module Noise")
ax.set_ylabel("Number of Objects")
ax.set_zlabel("Failure Rate")
plt.title("T=12 Reanchoring Results")

# Customize the z axis.
ax.set_zlim(0, 1.0)
ax.zaxis.set_major_locator(LinearLocator(10))
ax.zaxis.set_major_formatter(FormatStrFormatter('%.02f'))


ax.view_init(30, 0 + 210)
plt.savefig("T12ReanchoringModule3DFailureSurface.png", dpi = 250)
plt.draw()
plt.show()

fig = plt.figure()
ax = fig.gca(projection='3d')
# Make data.
data = thresh12DfResults.query("noiseFactor==0")
data = data.query("anchoringMethod=='narrowing'")
# Plot the surface.
surf = ax.plot_trisurf(data.moduleNoiseFactor,
                       data.numObjects,
                       data.failureRate,
                       linewidth=0,
                       cmap=cm.coolwarm,
                       vmin = 0,
                       vmax = 1,
                       antialiased=True)

ax.set_xlabel("Module Noise")
ax.set_ylabel("Number of Objects")
ax.set_zlabel("Failure Rate")
plt.title("T=12 Narrowing Results")

# Customize the z axis.
ax.set_zlim(0, 1.0)
ax.zaxis.set_major_locator(LinearLocator(10))
ax.zaxis.set_major_formatter(FormatStrFormatter('%.02f'))


ax.view_init(30, 0 + 210)
plt.savefig("T12NarrowingModule3DFailureSurface.png", dpi = 250)
plt.draw()
plt.show()



In [14]:
fig = plt.figure()
ax = fig.gca(projection='3d')


# Make data.
data = thresh16DfResults.query("moduleNoiseFactor==0")
data = data.query("anchoringMethod=='reanchoring'")
# Plot the surface.
surf = ax.plot_trisurf(data.noiseFactor,
                       data.numObjects,
                       data.failureRate,
                       linewidth=0,
                       cmap=cm.coolwarm,
                       vmin = 0,
                       vmax = 1,
                       antialiased=True)

ax.set_xlabel("Global Noise")
ax.set_ylabel("Number of Objects")
ax.set_zlabel("Failure Rate")
plt.title("T=16 Reanchoring Results")

# Customize the z axis.
ax.set_zlim(0, 1.0)
ax.zaxis.set_major_locator(LinearLocator(10))
ax.zaxis.set_major_formatter(FormatStrFormatter('%.02f'))


ax.view_init(30, 0 + 210)
plt.savefig("T16ReanchoringGlobal3DFailureSurface.png", dpi = 250)
plt.draw()
plt.show()

fig = plt.figure()
ax = fig.gca(projection='3d')
# Make data.
data = thresh16DfResults.query("moduleNoiseFactor==0")
data = data.query("anchoringMethod=='narrowing'")
# Plot the surface.
surf = ax.plot_trisurf(data.noiseFactor,
                       data.numObjects,
                       data.failureRate,
                       linewidth=0,
                       cmap=cm.coolwarm,
                       vmin = 0,
                       vmax = 1,
                       antialiased=True)

ax.set_xlabel("Global Noise")
ax.set_ylabel("Number of Objects")
ax.set_zlabel("Failure Rate")
plt.title("T=16 Narrowing Results")

# Customize the z axis.
ax.set_zlim(0, 1.0)
ax.zaxis.set_major_locator(LinearLocator(10))
ax.zaxis.set_major_formatter(FormatStrFormatter('%.02f'))


ax.view_init(30, 0 + 210)
plt.savefig("T16NarrowingGlobal3DFailureSurface.png", dpi = 250)
plt.draw()
plt.show()



In [15]:
fig = plt.figure()
ax = fig.gca(projection='3d')


# Make data.
data = thresh12DfResults.query("moduleNoiseFactor==0")
data = data.query("anchoringMethod=='reanchoring'")
# Plot the surface.
surf = ax.plot_trisurf(data.noiseFactor,
                       data.numObjects,
                       data.failureRate,
                       linewidth=0,
                       cmap=cm.coolwarm,
                       vmin = 0,
                       vmax = 1,
                       antialiased=True)

ax.set_xlabel("Global Noise")
ax.set_ylabel("Number of Objects")
ax.set_zlabel("Failure Rate")
plt.title("T=12 Reanchoring Results")

# Customize the z axis.
ax.set_zlim(0, 1.0)
ax.zaxis.set_major_locator(LinearLocator(10))
ax.zaxis.set_major_formatter(FormatStrFormatter('%.02f'))


ax.view_init(30, 0 + 210)
plt.savefig("T12ReanchoringGlobal3DFailureSurface.png", dpi = 250)
plt.draw()
plt.show()

fig = plt.figure()
ax = fig.gca(projection='3d')
# Make data.
data = thresh12DfResults.query("moduleNoiseFactor==0")
data = data.query("anchoringMethod=='narrowing'")
# Plot the surface.
surf = ax.plot_trisurf(data.noiseFactor,
                       data.numObjects,
                       data.failureRate,
                       linewidth=0,
                       cmap=cm.coolwarm,
                       vmin = 0,
                       vmax = 1,
                       antialiased=True)

ax.set_xlabel("Global Noise")
ax.set_ylabel("Number of Objects")
ax.set_zlabel("Failure Rate")
plt.title("T=12 Narrowing Results")

# Customize the z axis.
ax.set_zlim(0, 1.0)
ax.zaxis.set_major_locator(LinearLocator(10))
ax.zaxis.set_major_formatter(FormatStrFormatter('%.02f'))


ax.view_init(30, 0 + 210)
plt.savefig("T12NarrowingGlobal3DFailureSurface.png", dpi = 250)
plt.draw()
plt.show()