In [4]:

    

one_kb = 1024
one_mb = one_kb * 1000
ten_mb = 10 * one_mb
twenty_mb = 2 * ten_mb
thirdtwo_mb = 32 * one_mb
six = 64 * one_mb
oneh = 128 * one_mb
twof = 256 * one_mb
fivet = 512 * one_mb

sizes = [one_kb, one_mb, ten_mb, twenty_mb, thirdtwo_mb, six, oneh, twof, fivet]
generation_sizes = [8, 16, 13]

def calc_symbol_size(ds, gs):
    res = list()
    
    for d in ds:
        t = list()
        for g in gs:
            t.append(d/g)
        res.append(t)
    return res

ss = calc_symbol_size(sizes, generation_sizes)

m = ''
for s in ss:
    
    for x in s:
        m = m + ' {},'.format(x)
    m = m[:-1] + '\n'
print m


    

    




 128, 64, 78
 128000, 64000, 78769
 1280000, 640000, 787692
 2560000, 1280000, 1575384
 4096000, 2048000, 2520615
 8192000, 4096000, 5041230
 16384000, 8192000, 10082461
 32768000, 16384000, 20164923
 65536000, 32768000, 40329846

In [29]:

    

import numpy as np

one_kb = 1024
one_mb = one_kb * 1000
ten_mb = 10 * one_mb
twenty_mb = 2 * ten_mb
thirdtwo_mb = 32 * one_mb
six = 64 * one_mb
oneh = 128 * one_mb
twof = 256 * one_mb
fivet = 512 * one_mb

#one_gb = 1024 * one_mb
#two_gb = 2 * one_gb

sizes = [one_kb, one_mb, ten_mb, twenty_mb, thirdtwo_mb, six, oneh, twof, fivet]


def symbol_size(g, data_sizes):
    symbol_sizes = list()
    for ds in data_sizes:
        symbol_sizes.append(ds/g)
    return symbol_sizes

def on_the_fly(g,k,r):
    operations = 0
    for i in range(0, g):
        operations = operations + (i*k)
    operations = operations + (r*(g*k))

    for i in range(0, g):
        operations = operations + (i*k)
    operations = operations + (r*(k*(g-1)))
    return operations
    
def diff_on_the_fly(g,k,r,gg,kk,rr):
    return on_the_fly(gg,kk,rr)/float((on_the_fly(g,k,r)))

def full_vector(g,k,r):
    return(2*g-1)*k*(g+r)

def diff_full_vector(g,k,r,gg,kk,rr):
    return full_vector(gg,kk,rr)/float(full_vector(g,k,r))

def diff_two_codes(g,k,r):
    return full_vector(g,k,r)/float(on_the_fly(g,k,r))

def calculate_avg_on_the_fly():
    print('On-The-Fly')
    # Gen 8,16:
    diff = list()
    g = 8
    r = 0
    s = symbol_size(g, sizes)

    gg = 16
    rr = 0
    ss = symbol_size(gg, sizes)
    
    for k, kk in zip(s,ss):
        diff.append(diff_on_the_fly(g,k,r,gg,kk,rr))
        
    dd = np.average(diff)
    print('Diff: {} for g: {} and gg: {}'.format(dd,g,gg))
    #for d in diff:
    #    print(d)
    
    difff = list()
    ggg = 32 
    rrr = 0
    sss = symbol_size(ggg, sizes)
    for kk, kkk in zip(ss,sss):
        difff.append(diff_on_the_fly(gg,kk,rr,ggg,kkk,rrr))
    
    dd = np.average(difff)
    print('Diff: {} for g: {} and gg: {}'.format(dd,gg,ggg))    
    #for d in difff:
    #    print(d)  
        
def calculate_avg_full_vector():
    print('Full Vector')
    # Gen 8,16:
    diff = list()
    g = 8
    r = 0
    s = symbol_size(g, sizes)

    gg = 16
    rr = 0
    ss = symbol_size(gg, sizes)
    
    for k, kk in zip(s,ss):
        diff.append(diff_full_vector(g,k,r,gg,kk,rr))
        
    dd = np.average(diff)
    print('Diff: {} for g: {} and gg: {}'.format(dd,g,gg))
    #for d in diff:
    ##    print(d)
    
    difff = list()
    ggg = 32 
    rrr = 0
    sss = symbol_size(ggg, sizes)
    for kk, kkk in zip(ss,sss):
        difff.append(diff_full_vector(gg,kk,rr,ggg,kkk,rrr))
    
    dd = np.average(difff)
    print('Diff: {} for g: {} and gg: {}'.format(dd,gg,ggg))    
    # for d in difff:
    #    print(d)     
    
def calc():
    print('llll')
    g = 8
    r = 0
    s = symbol_size(g, sizes)
    diff = list()
    for k in s:
        diff.append(diff_two_codes(g,k,r))
    d = np.average(diff)
    print('g: {} diff: {}'.format(g,d))
    
    g = 16
    r = 0
    s = symbol_size(g, sizes)
    diff = list()
    for k in s:
        diff.append(diff_two_codes(g,k,r))
    d = np.average(diff)
    print('g: {} diff: {}'.format(g,d))    

    g = 32
    r = 0
    s = symbol_size(g, sizes)
    diff = list()
    for k in s:
        diff.append(diff_two_codes(g,k,r))
    d = np.average(diff)
    print('g: {} diff: {}'.format(g,d))
    
calculate_avg_full_vector()
calculate_avg_on_the_fly()
calc()


    

    




Full Vector
Diff: 2.06666666667 for g: 8 and gg: 16
Diff: 2.03225806452 for g: 16 and gg: 32
On-The-Fly
Diff: 2.14285714286 for g: 8 and gg: 16
Diff: 2.06666666667 for g: 16 and gg: 32
llll
g: 8 diff: 2.14285714286
g: 16 diff: 2.06666666667
g: 32 diff: 2.03225806452

In [14]:

    

, value):
    if index == 0 or index == 1 or index == 2 or index == 3 or index == 4:
        return int(value)
    else:
        return float(value)

f = open('results/1510670606_full_vector_encoder_1_0_8_134217728')

data = []

for line in f:
    temp = line.split(',')
    index = 0
    for item in temp: 
        temp[index] = parseData(index, item)
        index = index + 1
    data.append(temp)
    
        
    
    
throughput = []
latency = []

for entry in data: 
    throughput.append(entry[5])
    latency.append(entry[6])
latency = [lat/(1000) for lat in latency]
throughput = [th/(1000) for th in throughput] 
print(throughput[0])

avg_latency = 0.0
for val in latency:
    avg_latency = avg_latency + val
    
avg_latency = avg_latency / len(latency)
print avg_latency

#df = pd.DataFrame(data, columns=['a'])
#df.plot(kind='scatter', x='a')
pd.DataFrame
fig = plt.figure()
ts = pd.Series(throughput)
pd.to_numeric(ts)
throughput_line = ts.plot(label='test', marker='o', linestyle='--')

ts_2 = pd.Series(latency)
pd.to_numeric(ts_2)
latency_line  = ts_2.plot(marker='^', linestyle='--')
plt.legend(['bytes/ms', 'Latency (Millisecons)'])
plt.ylabel('bytes/ms | microseconds')
plt.xlabel('# Data point')
fig.savefig('results/graphs/full_vector_0_64_16.eps')
plt.show()


    

    




  File "<ipython-input-14-bd53d30ef41c>", line 2
    if index == 0 or index == 1 or index == 2 or index == 3 or index == 4:
    ^
IndentationError: unexpected indent

In [16]:

    

## Generation Size: 8 
## Full Vector - latency 
import pandas as pd
import numpy as np 
%matplotlib inline
import matplotlib
matplotlib.use('Agg')
import matplotlib.pyplot as plt 

def data_size_in_bytes(data_size, mb=True):
    kb = 1024
    if mb:
        return data_size * kb * kb
    else:
        return data_size * kb

def  calulate_theorectical(gen, data_size, redundancy=0):
    symbols = data_size / gen
    return symbols * gen * (gen + redundancy)

def parseData(index, value):
    if index == 0 or index == 1 or index == 2 or index == 3 or index == 4:
        return int(value)
    else:
        return float(value)

def get_latency(file_name):
    file = open(file_name)
    data = []
    for line in file:
        temp = line.split(',')
        index = 0
        for item in temp: 
            temp[index] = parseData(index, item)
            index = index + 1
        data.append(temp)
    latency = [entry[6]/1000 for entry in data] # convert to milliseconds 
    return np.average(latency)

one_kb = 'results/encoder/thousand/1512452623_full_vector_encoder_binary8_0_8_128'
one_mb = 'results/encoder/thousand/1512452934_full_vector_encoder_binary8_0_8_131072'
ten_mb = 'results/encoder/thousand/1512453317_full_vector_encoder_binary8_0_8_1310720'
twenty_mb = 'results/encoder/thousand/1512453778_full_vector_encoder_binary8_0_8_2621440'
thirdy_two_mb = 'results/encoder/thousand/1512454344_full_vector_encoder_binary8_0_8_4194304'
sixty_four_mb = 'results/encoder/thousand/1512455182_full_vector_encoder_binary8_0_8_8388608'
one_hundre_and_twenty_eight_mb = 'results/encoder/thousand/1512456573_full_vector_encoder_binary8_0_8_16777216'
two_hundre_and_fifty_six_mb = 'results/encoder/thousand/1512459046_full_vector_encoder_binary8_0_8_33554432'
five_hundre_and_twelve_mb = 'results/encoder/thousand/1512463692_full_vector_encoder_binary8_0_8_67108864'


def calc_delta(data):
    for i in range(0, len(data)-1):
        print('d: {}'.format(((data[i+1]-data[i])/float(data[i]))))


latencies = {}

latencies[0] = get_latency(one_kb)
latencies[1] = get_latency(one_mb)
latencies[2] = get_latency(ten_mb)
latencies[3] = get_latency(twenty_mb)
latencies[4] = get_latency(thirdy_two_mb)
latencies[5] = get_latency(sixty_four_mb)
latencies[6] = get_latency(one_hundre_and_twenty_eight_mb)
latencies[7] = get_latency(two_hundre_and_fifty_six_mb)
latencies[8] = get_latency(five_hundre_and_twelve_mb)

print latencies[1]/float(latencies[0])
print latencies[2]/float(latencies[1])
print latencies[3]/float(latencies[2])
print 't'
i = 4
for x in range(i, (len(latencies)-1)):
    print (latencies[x+1]/float(latencies[x]))

theory = {}

print calulate_theorectical(8, data_size_in_bytes(256))
theory[0] = calulate_theorectical(8, data_size_in_bytes(1, False)) 
theory[1] = calulate_theorectical(8, data_size_in_bytes(1))
theory[2] = calulate_theorectical(8, data_size_in_bytes(10))
theory[3] = calulate_theorectical(8, data_size_in_bytes(20))
theory[4] = calulate_theorectical(8, data_size_in_bytes(32))
theory[5] = calulate_theorectical(8, data_size_in_bytes(64))
theory[6] = calulate_theorectical(8, data_size_in_bytes(128))
theory[7] = calulate_theorectical(8, data_size_in_bytes(256))
theory[8] = calulate_theorectical(8, data_size_in_bytes(512))

print theory.values()

data_size = np.array(latencies.keys())
latency = np.array(latencies.values())
print(latency[8])
theory = np.array(theory.values())
pd.DataFrame
fig = plt.figure()
my_xticks = ['1kB','1MB','10MB','20MB','32MB','64MB','128MB','256MB','512MB']
plt.xticks(data_size, my_xticks)

plt.plot(data_size, latency)
#plt.plot(data_size, theory)
plt.yscale("log", nonposy="clip")
plt.ylabel('Latency in ms')
plt.xlabel('Data size')
fig.savefig('results/graphs/full_vector_gen_8.eps')
plt.show()


    

    




185.091932458
13.7908244977
2.11918413548
t
2.41145573908
2.30046693432
1.97750089697
1.99841566616
2147483648
[8192, 8388608, 83886080, 167772160, 268435456, 536870912, 1073741824, 2147483648, 4294967296]
534.278127






    

In [22]:

    

## Generation Size: 16
## Full Vector - latency 
import pandas as pd
import numpy as np 
%matplotlib inline
import matplotlib
matplotlib.use('Agg')
import matplotlib.pyplot as plt 

def data_size_in_bytes(data_size, mb=True):
    kb = 1024
    if mb:
        return data_size * kb * kb
    else:
        return data_size * kb

def  calulate_theorectical(gen, data_size, redundancy=0):
    symbols = data_size / gen
    return symbols * gen * (gen + redundancy)

def parseData(index, value):
    if index == 0 or index == 1 or index == 2 or index == 3 or index == 4:
        return int(value)
    else:
        return float(value)

def get_latency(file_name):
    file = open(file_name)
    data = []
    for line in file:
        temp = line.split(',')
        index = 0
        for item in temp: 
            temp[index] = parseData(index, item)
            index = index + 1
        data.append(temp)
    latency = [entry[6]/1000 for entry in data] # convert to milliseconds 
    return np.average(latency)

def get_latency_t(file_name):
    file = open(file_name)
    data = []
    for line in file:
        temp = line.split(',')
        index = 0
        for item in temp: 
            temp[index] = parseData(index, item)
            index = index + 1
        data.append(temp)
    latency = [entry[6]/1000 for entry in data] # convert to milliseconds 
    return latency

five_hundre_and_twelve_mb = 'results/encoder/thousand/1512475943_full_vector_encoder_binary8_0_16_33554432'
two_hundre_and_fifty_six_mb = 'results/encoder/thousand/1512470804_full_vector_encoder_binary8_0_16_16777216'
one_hundre_and_twenty_eight_mb = 'results/encoder/thousand/1512468085_full_vector_encoder_binary8_0_16_8388608'
sixty_four_mb = 'results/encoder/thousand/1512466613_full_vector_encoder_binary8_0_16_4194304'
thirdy_two_mb = 'results/encoder/thousand/1512465749_full_vector_encoder_binary8_0_16_2097152'
twenty_mb = 'results/encoder/thousand/1512465168_full_vector_encoder_binary8_0_16_1310720'
ten_mb = 'results/encoder/thousand/1512464696_full_vector_encoder_binary8_0_16_655360'
one_mb = 'results/encoder/thousand/1512464308_full_vector_encoder_binary8_0_16_65536'
one_kb = 'results/encoder/thousand/1512463997_full_vector_encoder_binary8_0_16_64'

latencies = {}

latencies[0] = get_latency(one_kb)
latencies[1] = get_latency(one_mb)
latencies[2] = get_latency(ten_mb)
latencies[3] = get_latency(twenty_mb)
latencies[4] = get_latency(thirdy_two_mb)
latencies[5] = get_latency(sixty_four_mb)
latencies[6] = get_latency(one_hundre_and_twenty_eight_mb)
latencies[7] = get_latency(two_hundre_and_fifty_six_mb)
latencies[8] = get_latency(five_hundre_and_twelve_mb)

mm = get_latency_t(one_mb)
nn = get_latency_t(ten_mb)

def calulate_avg_diffff(a1, a2):
    ll = list()
    for x,y in zip(a1, a2):
        ll.append((y-x)/float(x))
    return np.average(ll)

#print(calulate_avg_diffff(mm, nn))

print latencies[1]/float(latencies[0])
print (latencies[2]-float(latencies[1]))/float(latencies[1])
print latencies[3]/float(latencies[2])
print 't'
i = 4
for x in range(i, (len(latencies)-1)):
    print (latencies[x+1]/float(latencies[x]))
    
theory = {}

print calulate_theorectical(8, data_size_in_bytes(256))
theory[0] = calulate_theorectical(8, data_size_in_bytes(1, False)) 
theory[1] = calulate_theorectical(8, data_size_in_bytes(1))
theory[2] = calulate_theorectical(8, data_size_in_bytes(10))
theory[3] = calulate_theorectical(8, data_size_in_bytes(20))
theory[4] = calulate_theorectical(8, data_size_in_bytes(32))
theory[5] = calulate_theorectical(8, data_size_in_bytes(64))
theory[6] = calulate_theorectical(8, data_size_in_bytes(128))
theory[7] = calulate_theorectical(8, data_size_in_bytes(256))
theory[8] = calulate_theorectical(8, data_size_in_bytes(512))

print theory.values()

data_size = np.array(latencies.keys())
latency = np.array(latencies.values())
theory = np.array(theory.values())
pd.DataFrame
fig = plt.figure()
my_xticks = ['1KiB','1MB', '10MB','20MB','32MB','64MB','128MB','256MB','512MB']
plt.xticks(data_size, my_xticks)
plt.xticks(data_size, my_xticks)

plt.plot(data_size, latency)
#plt.plot(data_size, theory)
plt.yscale("log", nonposy="clip")
plt.ylabel('Latency in ms')
plt.xlabel('Data size')
fig.savefig('results/graphs/full_vector_gen_16.eps')
plt.show()


    

    




13.1061896437
101.322140355
13.1048011196
2.10868445901
t
2.11739018959
2.51501772173
2.35151874031
2.0107896958
2147483648
[8192, 8388608, 83886080, 167772160, 268435456, 536870912, 1073741824, 2147483648, 4294967296]






    

In [15]:

    

## Generation Size: 32 
## Full Vector - latency 
import pandas as pd
import numpy as np 
%matplotlib inline
import matplotlib
matplotlib.use('Agg')
import matplotlib.pyplot as plt 

def parseData(index, value):
    if index == 0 or index == 1 or index == 2 or index == 3 or index == 4:
        return int(value)
    else:
        return float(value)

def get_latency(file_name):
    file = open(file_name)
    data = []
    for line in file:
        temp = line.split(',')
        index = 0
        for item in temp: 
            temp[index] = parseData(index, item)
            index = index + 1
        data.append(temp)
    latency = [entry[6]/1000 for entry in data] # convert to milliseconds 
    return np.average(latency)

one_kb = 'results/encoder/thousand/1512476247_full_vector_encoder_binary8_0_32_32'
one_mb = 'results/encoder/thousand/1512476560_full_vector_encoder_binary8_0_32_32768'
ten_mb = 'results/encoder/thousand/1512476958_full_vector_encoder_binary8_0_32_327680'
twenty_mb = 'results/encoder/thousand/1512477451_full_vector_encoder_binary8_0_32_655360'
thirdy_two_mb = 'results/encoder/thousand/1512478066_full_vector_encoder_binary8_0_32_1048576'
sixty_four_mb = 'results/encoder/thousand/1512478997_full_vector_encoder_binary8_0_32_2097152'
one_hundre_and_twenty_eight_mb = 'results/encoder/thousand/1512480568_full_vector_encoder_binary8_0_32_4194304'
two_hundre_and_fifty_six_mb = 'results/encoder/thousand/1512483577_full_vector_encoder_binary8_0_32_8388608'
five_hundre_and_twelve_mb = 'results/encoder/thousand/1512489653_full_vector_encoder_binary8_0_32_16777216'

latencies = {}

latencies[0] = get_latency(one_kb)
latencies[1] = get_latency(one_mb)
latencies[2] = get_latency(ten_mb)
latencies[3] = get_latency(twenty_mb)
latencies[4] = get_latency(thirdy_two_mb)
latencies[5] = get_latency(sixty_four_mb)
latencies[6] = get_latency(one_hundre_and_twenty_eight_mb)
latencies[7] = get_latency(two_hundre_and_fifty_six_mb)
latencies[8] = get_latency(five_hundre_and_twelve_mb)

print latencies[1]/float(latencies[0])
print latencies[2]/float(latencies[1])
print latencies[3]/float(latencies[2])
print 't'
i = 4
for x in range(i, (len(latencies)-1)):
    print (latencies[x+1]/float(latencies[x]))

data_size = np.array(latencies.keys())
latency = np.array(latencies.values())
pd.DataFrame
fig = plt.figure()
my_xticks = ['1kB','1MB','10MB','20MB','32MB','64MB','128MB','256MB','512MB']
plt.xticks(data_size, my_xticks)

plt.plot(data_size, latency)
plt.yscale("log", nonposy="clip")
plt.ylabel('Latency in ms')
plt.xlabel('Data size')
fig.savefig('results/graphs/full_vector_gen_32.eps')
plt.show()


    

    




82.5467896718
14.1233696123
2.10868713404
t
2.05370177557
2.0735710227
2.5096788308
2.43921241017






    

In [26]:

    

## Generation Size: 8,16,32 
## Full Vector - latency comparison
import pandas as pd
import numpy as np 
%matplotlib inline
import matplotlib
matplotlib.use('Agg')
import matplotlib.pyplot as plt 

def parseData(index, value):
    if index == 0 or index == 1 or index == 2 or index == 3 or index == 4:
        return int(value)
    else:
        return float(value)

def get_latency(file_name):
    file = open(file_name)
    data = []
    for line in file:
        temp = line.split(',')
        index = 0
        for item in temp: 
            temp[index] = parseData(index, item)
            index = index + 1
        data.append(temp)
    latency = [entry[6]/1000 for entry in data] # convert to milliseconds 
    return np.average(latency)

def read_gen_eight():
    one_kb = 'results/encoder/thousand/1512452623_full_vector_encoder_binary8_0_8_128'
    one_mb = 'results/encoder/thousand/1512452934_full_vector_encoder_binary8_0_8_131072'
    ten_mb = 'results/encoder/thousand/1512453317_full_vector_encoder_binary8_0_8_1310720'
    twenty_mb = 'results/encoder/thousand/1512453778_full_vector_encoder_binary8_0_8_2621440'
    thirdy_two_mb = 'results/encoder/thousand/1512454344_full_vector_encoder_binary8_0_8_4194304'
    sixty_four_mb = 'results/encoder/thousand/1512455182_full_vector_encoder_binary8_0_8_8388608'
    one_hundre_and_twenty_eight_mb = 'results/encoder/thousand/1512456573_full_vector_encoder_binary8_0_8_16777216'
    two_hundre_and_fifty_six_mb = 'results/encoder/thousand/1512459046_full_vector_encoder_binary8_0_8_33554432'
    five_hundre_and_twelve_mb = 'results/encoder/thousand/1512463692_full_vector_encoder_binary8_0_8_67108864'

    latencies = {}

    latencies[0] = get_latency(one_kb)
    latencies[1] = get_latency(one_mb)
    latencies[2] = get_latency(ten_mb)
    latencies[3] = get_latency(twenty_mb)
    latencies[4] = get_latency(thirdy_two_mb)
    latencies[5] = get_latency(sixty_four_mb)
    latencies[6] = get_latency(one_hundre_and_twenty_eight_mb)
    latencies[7] = get_latency(two_hundre_and_fifty_six_mb)
    latencies[8] = get_latency(five_hundre_and_twelve_mb)
    return latencies

def read_gen_sixteen():
    five_hundre_and_twelve_mb = 'results/encoder/thousand/1512475943_full_vector_encoder_binary8_0_16_33554432'
    two_hundre_and_fifty_six_mb = 'results/encoder/thousand/1512470804_full_vector_encoder_binary8_0_16_16777216'
    one_hundre_and_twenty_eight_mb = 'results/encoder/thousand/1512468085_full_vector_encoder_binary8_0_16_8388608'
    sixty_four_mb = 'results/encoder/thousand/1512466613_full_vector_encoder_binary8_0_16_4194304'
    thirdy_two_mb = 'results/encoder/thousand/1512465749_full_vector_encoder_binary8_0_16_2097152'
    twenty_mb = 'results/encoder/thousand/1512465168_full_vector_encoder_binary8_0_16_1310720'
    ten_mb = 'results/encoder/thousand/1512464696_full_vector_encoder_binary8_0_16_655360'
    one_mb = 'results/encoder/thousand/1512464308_full_vector_encoder_binary8_0_16_65536'
    one_kb = 'results/encoder/thousand/1512463997_full_vector_encoder_binary8_0_16_64'
    
    latencies = {}

    latencies[0] = get_latency(one_kb)
    latencies[1] = get_latency(one_mb)
    latencies[2] = get_latency(ten_mb)
    latencies[3] = get_latency(twenty_mb)
    latencies[4] = get_latency(thirdy_two_mb)
    latencies[5] = get_latency(sixty_four_mb)
    latencies[6] = get_latency(one_hundre_and_twenty_eight_mb)
    latencies[7] = get_latency(two_hundre_and_fifty_six_mb)
    latencies[8] = get_latency(five_hundre_and_twelve_mb)
    return latencies

def read_gen_thirdy_two():
    one_kb = 'results/encoder/thousand/1512476247_full_vector_encoder_binary8_0_32_32'
    one_mb = 'results/encoder/thousand/1512476560_full_vector_encoder_binary8_0_32_32768'
    ten_mb = 'results/encoder/thousand/1512476958_full_vector_encoder_binary8_0_32_327680'
    twenty_mb = 'results/encoder/thousand/1512477451_full_vector_encoder_binary8_0_32_655360'
    thirdy_two_mb = 'results/encoder/thousand/1512478066_full_vector_encoder_binary8_0_32_1048576'
    sixty_four_mb = 'results/encoder/thousand/1512478997_full_vector_encoder_binary8_0_32_2097152'
    one_hundre_and_twenty_eight_mb = 'results/encoder/thousand/1512480568_full_vector_encoder_binary8_0_32_4194304'
    two_hundre_and_fifty_six_mb = 'results/encoder/thousand/1512483577_full_vector_encoder_binary8_0_32_8388608'
    five_hundre_and_twelve_mb = 'results/encoder/thousand/1512489653_full_vector_encoder_binary8_0_32_16777216'

    latencies = {}

    latencies[0] = get_latency(one_kb)
    latencies[1] = get_latency(one_mb)
    latencies[2] = get_latency(ten_mb)
    latencies[3] = get_latency(twenty_mb)
    latencies[4] = get_latency(thirdy_two_mb)
    latencies[5] = get_latency(sixty_four_mb)
    latencies[6] = get_latency(one_hundre_and_twenty_eight_mb)
    latencies[7] = get_latency(two_hundre_and_fifty_six_mb)
    latencies[8] = get_latency(five_hundre_and_twelve_mb)
    return latencies


def calc_diff(x,y):
    print (y/float(x))
    
def diff_gen(g, gg):
    for x, y in zip(g,gg):
        calc_diff(g[x],gg[y])

eight_latencies = read_gen_eight()
eight_data_size = np.array(eight_latencies.keys())
eight_latency = np.array(eight_latencies.values())

sixteen_latencies = read_gen_sixteen()
diff_gen(eight_latencies,sixteen_latencies)
sixteen_data_size = np.array(sixteen_latencies.keys())
sixteen_latency = np.array(sixteen_latencies.values())

thridy_two_latencies = read_gen_thirdy_two()
print('t')
diff_gen(sixteen_latencies,thridy_two_latencies)
thridy_two_data_size = np.array(thridy_two_latencies.keys())
thridy_two_latency = np.array(thridy_two_latencies.values())


pd.DataFrame
fig = plt.figure()
my_xticks = ['1kB','1MB','10MB','20MB','32MB','64MB','128MB','256MB','512MB']
plt.xticks(data_size, my_xticks)

plt.plot(eight_data_size, eight_latency)

plt.xticks(sixteen_data_size, my_xticks)
plt.plot(sixteen_data_size, sixteen_latency)

plt.xticks(thridy_two_data_size, my_xticks)
plt.plot(thridy_two_data_size, thridy_two_latency)

plt.yscale("log", nonposy="clip")
plt.legend(['G=8', 'G=16', 'G=32'])
plt.ylabel('Latency in ms')
plt.xlabel('Data size')
fig.savefig('results/graphs/full_vector_gen_comparison.eps')
plt.show()


    

    




3.10656660413
1.70058183145
1.73929909152
1.73068158753
1.68904305888
1.48307229725
1.62138957729
1.9280537279
1.93999208206
t
2.27817369247
1.85602005612
1.85846344363
1.85846580124
1.83780143833
1.78252269969
1.46964666906
1.5684931916
1.90267936333






    

In [21]:

    

## Generation Size: 8 
## Full Vector - throughput 
import pandas as pd
import numpy as np 
%matplotlib inline
import matplotlib
matplotlib.use('Agg')
import matplotlib.pyplot as plt 

def parseData(index, value):
    if index == 0 or index == 1 or index == 2 or index == 3 or index == 4:
        return int(value)
    else:
        return float(value)

def get_latency(file_name):
    file = open(file_name)
    data = []
    for line in file:
        temp = line.split(',')
        index = 0
        for item in temp: 
            temp[index] = parseData(index, item)
            index = index + 1
        data.append(temp)
    latency = [entry[5] for entry in data] # convert to milliseconds 
    return np.average(latency)
    
one_kb = 'results/1510751197_full_vector_encoder_1_0_8_128'
one_mb = 'results/1510751438_full_vector_encoder_1_0_8_131072'
ten_mb = 'results/1510752389_full_vector_encoder_1_0_8_1310720'
twenty_mb = 'results/1510822030_full_vector_encoder_1_0_8_2621440'
thirdy_two_mb = 'results/1510824925_full_vector_encoder_1_0_8_4194304'
sixty_four_mb = 'results/1510836357_full_vector_encoder_1_0_8_8388608'
one_hundre_and_twenty_eight_mb = 'results/1510847597_full_vector_encoder_1_0_8_16777216'
two_hundre_and_fifty_six_mb = 'results/1510927849_full_vector_encoder_1_0_8_33554432'
five_hundre_and_twelve_mb = 'results/1510972685_full_vector_encoder_1_0_8_67108864'

latencies = {}

latencies[0] = get_latency(one_kb)
latencies[1] = get_latency(one_mb)
latencies[2] = get_latency(ten_mb)
latencies[3] = get_latency(twenty_mb)
latencies[4] = get_latency(thirdy_two_mb)
latencies[5] = get_latency(sixty_four_mb)
latencies[6] = get_latency(one_hundre_and_twenty_eight_mb)
latencies[7] = get_latency(two_hundre_and_fifty_six_mb)
latencies[8] = get_latency(five_hundre_and_twelve_mb)

data_size = np.array(latencies.keys())
latency = np.array(latencies.values())
pd.DataFrame
fig = plt.figure()
my_xticks = ['1kB','1MiB','10MiB','20MiB','32MiB','64MiB','128MiB','256MiB','512MiB']
plt.xticks(data_size, my_xticks)
plt.plot(data_size, latency)
plt.legend(['G=8'])
plt.ylabel('Bytes Per microsecond')
plt.xlabel('Data size')
fig.savefig('results/graphs/full_vector_through_put_gen_8.eps')
plt.show()


    

In [9]:

    

## Generation Size: 16 
## Full Vector - throughput 
import pandas as pd
import numpy as np 
%matplotlib inline
import matplotlib
matplotlib.use('Agg')
import matplotlib.pyplot as plt 

def parseData(index, value):
    if index == 0 or index == 1 or index == 2 or index == 3 or index == 4:
        return int(value)
    else:
        return float(value)

def get_latency(file_name):
    file = open(file_name)
    data = []
    for line in file:
        temp = line.split(',')
        index = 0
        for item in temp: 
            temp[index] = parseData(index, item)
            index = index + 1
        data.append(temp)
    latency = [entry[5] for entry in data] # convert to milliseconds 
    return np.average(latency)
    
    five_hundre_and_twelve_mb = 'results/1511427894_full_vector_encoder_1_0_16_33554432'
    two_hundre_and_fifty_six_mb = 'results/1511454980_full_vector_encoder_1_0_16_16777216'
    one_hundre_and_twenty_eight_mb = 'results/1511470001_full_vector_encoder_1_0_16_8388608'
    sixty_four_mb = 'results/1511479334_full_vector_encoder_1_0_16_4194304'
    thirdy_two_mb = 'results/1511516065_full_vector_encoder_1_0_16_2097152'
    twenty_mb = 'results/1511524913_full_vector_encoder_1_0_16_1310720'
    ten_mb = 'results/1511529363_full_vector_encoder_1_0_16_655360'
    one_mb = 'results/1511534339_full_vector_encoder_1_0_16_65536'
    one_kb = 'results/1511534688_full_vector_encoder_1_0_16_64'

latencies = {}

latencies[0] = get_latency(one_kb)
latencies[1] = get_latency(one_mb)
latencies[2] = get_latency(ten_mb)
latencies[3] = get_latency(twenty_mb)
latencies[4] = get_latency(thirdy_two_mb)
latencies[5] = get_latency(sixty_four_mb)
latencies[6] = get_latency(one_hundre_and_twenty_eight_mb)
latencies[7] = get_latency(two_hundre_and_fifty_six_mb)
latencies[8] = get_latency(five_hundre_and_twelve_mb)

data_size = np.array(latencies.keys())
latency = np.array(latencies.values())
pd.DataFrame
fig = plt.figure()
my_xticks = ['1kB','1MiB','10MiB','20MiB','32MiB','64MiB','128MiB','256MiB','512MiB']
plt.xticks(data_size, my_xticks)
plt.plot(data_size, latency)
plt.legend(['G=16'])
plt.ylabel('Bytes Per microsecond')
plt.xlabel('Data size')
fig.savefig('results/graphs/full_vector_through_put_gen_16.eps')
plt.show()


    

In [11]:

    

## Generation Size: 32 
## Full Vector - throughput 
import pandas as pd
import numpy as np 
%matplotlib inline
import matplotlib
matplotlib.use('Agg')
import matplotlib.pyplot as plt 

def parseData(index, value):
    if index == 0 or index == 1 or index == 2 or index == 3 or index == 4:
        return int(value)
    else:
        return float(value)

def get_latency(file_name):
    file = open(file_name)
    data = []
    for line in file:
        temp = line.split(',')
        index = 0
        for item in temp: 
            temp[index] = parseData(index, item)
            index = index + 1
        data.append(temp)
    latency = [entry[5] for entry in data] # convert to milliseconds 
    return np.average(latency)
    
one_kb = 'results/1511252697_full_vector_encoder_1_0_32_32'
one_mb = 'results/1511252611_full_vector_encoder_1_0_32_32768'
ten_mb = 'results/1511252293_full_vector_encoder_1_0_32_327680'
twenty_mb = 'results/1511221602_full_vector_encoder_1_0_32_655360'
thirdy_two_mb = 'results/1511217072_full_vector_encoder_1_0_32_1048576'
sixty_four_mb = 'results/1511210497_full_vector_encoder_1_0_32_2097152'
one_hundre_and_twenty_eight_mb = 'results/1511201901_full_vector_encoder_1_0_32_4194304'
two_hundre_and_fifty_six_mb = 'results/1511187820_full_vector_encoder_1_0_32_8388608'
five_hundre_and_twelve_mb = 'results/1511156569_full_vector_encoder_1_0_32_16777216'

latencies = {}

latencies[0] = get_latency(one_kb)
latencies[1] = get_latency(one_mb)
latencies[2] = get_latency(ten_mb)
latencies[3] = get_latency(twenty_mb)
latencies[4] = get_latency(thirdy_two_mb)
latencies[5] = get_latency(sixty_four_mb)
latencies[6] = get_latency(one_hundre_and_twenty_eight_mb)
latencies[7] = get_latency(two_hundre_and_fifty_six_mb)
latencies[8] = get_latency(five_hundre_and_twelve_mb)

data_size = np.array(latencies.keys())
latency = np.array(latencies.values())
pd.DataFrame
fig = plt.figure()
my_xticks = ['1kB','1MB','10MB','20MB','32MB','64MB','128MB','256MB','512MB']
plt.xticks(data_size, my_xticks)
plt.plot(data_size, latency)
plt.legend(['G=32'])
plt.ylabel('Bytes Per microsecond')
plt.xlabel('Data size')
fig.savefig('results/graphs/full_vector_through_gen_32.eps')
plt.show()


    

In [10]:

    

## Generation Size: 8,16,32 
## Full Vector - throughput comparison
import pandas as pd
import numpy as np 
%matplotlib inline
import matplotlib
matplotlib.use('Agg')
import matplotlib.pyplot as plt 

def parseData(index, value):
    if index == 0 or index == 1 or index == 2 or index == 3 or index == 4:
        return int(value)
    else:
        return float(value)

def get_latency(file_name):
    file = open(file_name)
    data = []
    for line in file:
        temp = line.split(',')
        index = 0
        for item in temp: 
            temp[index] = parseData(index, item)
            index = index + 1
        data.append(temp)
    latency = [entry[5] for entry in data] # convert to milliseconds 
    print(np.average(latency))
    return np.average(latency)

def read_gen_eight():
    one_kb = 'results/1510751197_full_vector_encoder_1_0_8_128'
    one_mb = 'results/1510751438_full_vector_encoder_1_0_8_131072'
    ten_mb = 'results/1510752389_full_vector_encoder_1_0_8_1310720'
    twenty_mb = 'results/1510822030_full_vector_encoder_1_0_8_2621440'
    thirdy_two_mb = 'results/1510824925_full_vector_encoder_1_0_8_4194304'
    sixty_four_mb = 'results/1510836357_full_vector_encoder_1_0_8_8388608'
    one_hundre_and_twenty_eight_mb = 'results/1510847597_full_vector_encoder_1_0_8_16777216'
    two_hundre_and_fifty_six_mb = 'results/1510927849_full_vector_encoder_1_0_8_33554432'
    five_hundre_and_twelve_mb = 'results/1510972685_full_vector_encoder_1_0_8_67108864'

    latencies = {}

    latencies[0] = get_latency(one_kb)
    latencies[1] = get_latency(one_mb)
    latencies[2] = get_latency(ten_mb)
    latencies[3] = get_latency(twenty_mb)
    latencies[4] = get_latency(thirdy_two_mb)
    latencies[5] = get_latency(sixty_four_mb)
    latencies[6] = get_latency(one_hundre_and_twenty_eight_mb)
    latencies[7] = get_latency(two_hundre_and_fifty_six_mb)
    latencies[8] = get_latency(five_hundre_and_twelve_mb)
    return latencies

def read_gen_sixteen():
    five_hundre_and_twelve_mb = 'results/1511427894_full_vector_encoder_1_0_16_33554432'
    two_hundre_and_fifty_six_mb = 'results/1511454980_full_vector_encoder_1_0_16_16777216'
    one_hundre_and_twenty_eight_mb = 'results/1511470001_full_vector_encoder_1_0_16_8388608'
    sixty_four_mb = 'results/1511479334_full_vector_encoder_1_0_16_4194304'
    thirdy_two_mb = 'results/1511516065_full_vector_encoder_1_0_16_2097152'
    twenty_mb = 'results/1511524913_full_vector_encoder_1_0_16_1310720'
    ten_mb = 'results/1511529363_full_vector_encoder_1_0_16_655360'
    one_mb = 'results/1511534339_full_vector_encoder_1_0_16_65536'
    one_kb = 'results/1511534688_full_vector_encoder_1_0_16_64'
    
    latencies = {}

    latencies[0] = get_latency(one_kb)
    latencies[1] = get_latency(one_mb)
    latencies[2] = get_latency(ten_mb)
    latencies[3] = get_latency(twenty_mb)
    latencies[4] = get_latency(thirdy_two_mb)
    latencies[5] = get_latency(sixty_four_mb)
    latencies[6] = get_latency(one_hundre_and_twenty_eight_mb)
    latencies[7] = get_latency(two_hundre_and_fifty_six_mb)
    latencies[8] = get_latency(five_hundre_and_twelve_mb)
    return latencies

def read_gen_thirdy_two():
    one_kb = 'results/1511252697_full_vector_encoder_1_0_32_32'
    one_mb = 'results/1511252611_full_vector_encoder_1_0_32_32768'
    ten_mb = 'results/1511252293_full_vector_encoder_1_0_32_327680'
    twenty_mb = 'results/1511221602_full_vector_encoder_1_0_32_655360'
    thirdy_two_mb = 'results/1511217072_full_vector_encoder_1_0_32_1048576'
    sixty_four_mb = 'results/1511210497_full_vector_encoder_1_0_32_2097152'
    one_hundre_and_twenty_eight_mb = 'results/1511201901_full_vector_encoder_1_0_32_4194304'
    two_hundre_and_fifty_six_mb = 'results/1511187820_full_vector_encoder_1_0_32_8388608'
    five_hundre_and_twelve_mb = 'results/1511156569_full_vector_encoder_1_0_32_16777216'
    
    latencies = {}

    latencies[0] = get_latency(one_kb)
    latencies[1] = get_latency(one_mb)
    latencies[2] = get_latency(ten_mb)
    latencies[3] = get_latency(twenty_mb)
    latencies[4] = get_latency(thirdy_two_mb)
    latencies[5] = get_latency(sixty_four_mb)
    latencies[6] = get_latency(one_hundre_and_twenty_eight_mb)
    latencies[7] = get_latency(two_hundre_and_fifty_six_mb)
    latencies[8] = get_latency(five_hundre_and_twelve_mb)
    return latencies

eight_latencies = read_gen_eight()
eight_data_size = np.array(eight_latencies.keys())
eight_latency = np.array(eight_latencies.values())

sixteen_latencies = read_gen_sixteen()
sixteen_data_size = np.array(sixteen_latencies.keys())
sixteen_latency = np.array(sixteen_latencies.values())

thridy_two_latencies = read_gen_thirdy_two()
thridy_two_data_size = np.array(thridy_two_latencies.keys())
thridy_two_latency = np.array(thridy_two_latencies.values())


pd.DataFrame
fig = plt.figure()
my_xticks = ['1kB','1MB','10MB','20MB','32MB','64MB','128MB','256MB','512MB']
plt.xticks(data_size, my_xticks)
plt.plot(eight_data_size, eight_latency)

plt.xticks(sixteen_data_size, my_xticks)
plt.plot(sixteen_data_size, sixteen_latency)

plt.xticks(thridy_two_data_size, my_xticks)
plt.plot(thridy_two_data_size, thridy_two_latency)

plt.yscale("log", nonposy="clip")
plt.legend(['G=8', 'G=16', 'G=32'])
plt.ylabel('Bytes Per microsecond')
plt.xlabel('Data size')
fig.savefig('results/graphs/full_vector_throughput_gen_comparison.eps')
plt.show()


    

    




336.3206
2043.096
1494.6857
1437.9984
1407.4016
1142.202
989.0439
984.4317
982.3868
126.422
1217.3281
858.5034
821.6987
800.3391
749.1728
603.6961
507.4595
506.3925
54.3936
660.137
466.4345
440.2093
431.6178
422.3622
404.3491
326.4833
264.8809






    

In [8]:

    

## Generation Size: 8 
## On The fly - latency 

import pandas as pd
import numpy as np 
%matplotlib inline
import matplotlib
matplotlib.use('Agg')
import matplotlib.pyplot as plt 

def data_size_in_bytes(data_size, mb=True):
    kb = 1024
    if mb:
        return data_size * kb * kb
    else:
        return data_size * kb

def  calulate_theorectical(gen, data_size, redundancy=0):
    symbols = data_size / gen
    return symbols * gen * (gen + redundancy)

def parseData(index, value):
    if index == 0 or index == 1 or index == 2 or index == 3 or index == 4:
        return int(value)
    else:
        return float(value)

def get_latency(file_name):
    file = open(file_name)
    data = []
    for line in file:
        temp = line.split(',')
        index = 0
        for item in temp: 
            temp[index] = parseData(index, item)
            index = index + 1
        data.append(temp)
    latency = [entry[6]/1000 for entry in data] # convert to milliseconds
    return np.average(latency)

five_hundre_and_twelve_mb = 'results/encoder/thousand/1512501259_on_the_fly_encoder_binary8_0_8_67108864'
two_hundre_and_fifty_six_mb = 'results/encoder/thousand/1512496496_on_the_fly_encoder_binary8_0_8_33554432'
one_hundre_and_twenty_eight_mb = 'results/encoder/thousand/1512493964_on_the_fly_encoder_binary8_0_8_16777216'
sixty_four_mb = 'results/encoder/thousand/1512492545_on_the_fly_encoder_binary8_0_8_8388608'
thirdy_two_mb = 'results/encoder/thousand/1512491693_on_the_fly_encoder_binary8_0_8_4194304'
twenty_mb = 'results/encoder/thousand/1512491120_on_the_fly_encoder_binary8_0_8_2621440'
ten_mb = 'results/encoder/thousand/1512490653_on_the_fly_encoder_binary8_0_8_1310720'
one_mb = 'results/encoder/thousand/1512490269_on_the_fly_encoder_binary8_0_8_131072'
one_kb = 'results/encoder/thousand/1512489957_on_the_fly_encoder_binary8_0_8_128'

latencies = {}

latencies[0] = get_latency(one_kb)
latencies[1] = get_latency(one_mb)
latencies[2] = get_latency(ten_mb)
latencies[3] = get_latency(twenty_mb)
latencies[4] = get_latency(thirdy_two_mb)
latencies[5] = get_latency(sixty_four_mb)
latencies[6] = get_latency(one_hundre_and_twenty_eight_mb)
latencies[7] = get_latency(two_hundre_and_fifty_six_mb)
latencies[8] = get_latency(five_hundre_and_twelve_mb)

print latencies[1]/float(latencies[0])
print (latencies[2]-float(latencies[1]))/float(latencies[1])
print latencies[3]/float(latencies[2])
print 't'
i = 4
for x in range(i, (len(latencies)-1)):
    print (latencies[x+1]/float(latencies[x]))

theory = {}

print calulate_theorectical(8, data_size_in_bytes(256))
theory[0] = calulate_theorectical(8, data_size_in_bytes(1, False)) 
theory[1] = calulate_theorectical(8, data_size_in_bytes(1))
theory[2] = calulate_theorectical(8, data_size_in_bytes(10))
theory[3] = calulate_theorectical(8, data_size_in_bytes(20))
theory[4] = calulate_theorectical(8, data_size_in_bytes(32))
theory[5] = calulate_theorectical(8, data_size_in_bytes(64))
theory[6] = calulate_theorectical(8, data_size_in_bytes(128))
theory[7] = calulate_theorectical(8, data_size_in_bytes(256))
theory[8] = calulate_theorectical(8, data_size_in_bytes(512))

print theory.values()

data_size = np.array(latencies.keys())
latency = np.array(latencies.values())
theory = np.array(theory.values())
pd.DataFrame
fig = plt.figure()
my_xticks = ['1kB','1MB','10MB','20MB','32MB','64MB','128MB','256MB','512MB']
plt.xticks(data_size, my_xticks)

plt.plot(data_size, latency)
#plt.plot(data_size, theory)
plt.yscale("log", nonposy="clip")
plt.ylabel('Latency in ms')
plt.xlabel('Data size')
fig.savefig('results/graphs/on_the_fly_gen_8.eps')
plt.show()


    

    




170.601161996
12.5145348663
2.11998107964
t
2.39688843232
2.31317858658
1.9797070488
2.00283028044
2147483648
[8192, 8388608, 83886080, 167772160, 268435456, 536870912, 1073741824, 2147483648, 4294967296]






    

In [9]:

    

## Generation Size: 16 
## On The fly - latency 

import pandas as pd
import numpy as np 
%matplotlib inline
import matplotlib
matplotlib.use('Agg')
import matplotlib.pyplot as plt 

def data_size_in_bytes(data_size, mb=True):
    kb = 1024
    if mb:
        return data_size * kb * kb
    else:
        return data_size * kb

def  calulate_theorectical(gen, data_size, redundancy=0):
    symbols = data_size / gen
    return symbols * gen * (gen + redundancy)

def parseData(index, value):
    if index == 0 or index == 1 or index == 2 or index == 3 or index == 4:
        return int(value)
    else:
        return float(value)

def get_latency(file_name):
    file = open(file_name)
    data = []
    for line in file:
        temp = line.split(',')
        index = 0
        for item in temp: 
            temp[index] = parseData(index, item)
            index = index + 1
        data.append(temp)
    latency = [entry[6]/1000 for entry in data] # convert to milliseconds
    return np.average(latency)

five_hundre_and_twelve_mb = 'results/encoder/thousand/1512513739_on_the_fly_encoder_binary8_0_16_33554432'
two_hundre_and_fifty_six_mb = 'results/encoder/thousand/1512508485_on_the_fly_encoder_binary8_0_16_16777216'
one_hundre_and_twenty_eight_mb = 'results/encoder/thousand/1512505713_on_the_fly_encoder_binary8_0_16_8388608'
sixty_four_mb = 'results/encoder/thousand/1512504214_on_the_fly_encoder_binary8_0_16_4194304'
thirdy_two_mb = 'results/encoder/thousand/1512503337_on_the_fly_encoder_binary8_0_16_2097152'
twenty_mb = 'results/encoder/thousand/1512502740_on_the_fly_encoder_binary8_0_16_1310720'
ten_mb = 'results/encoder/thousand/1512502264_on_the_fly_encoder_binary8_0_16_655360'
one_mb = 'results/encoder/thousand/1512501875_on_the_fly_encoder_binary8_0_16_65536'
one_kb = 'results/encoder/thousand/1512501563_on_the_fly_encoder_binary8_0_16_64'


latencies = {}

latencies[0] = get_latency(one_kb)
latencies[1] = get_latency(one_mb)
latencies[2] = get_latency(ten_mb)
latencies[3] = get_latency(twenty_mb)
latencies[4] = get_latency(thirdy_two_mb)
latencies[5] = get_latency(sixty_four_mb)
latencies[6] = get_latency(one_hundre_and_twenty_eight_mb)
latencies[7] = get_latency(two_hundre_and_fifty_six_mb)
latencies[8] = get_latency(five_hundre_and_twelve_mb)

print latencies[1]/float(latencies[0])
print (latencies[2]-float(latencies[1]))/float(latencies[1])
print latencies[3]/float(latencies[2])
print 't'
i = 4
for x in range(i, (len(latencies)-1)):
    print (latencies[x+1]/float(latencies[x]))

theory = {}

print calulate_theorectical(8, data_size_in_bytes(256))
theory[0] = calulate_theorectical(8, data_size_in_bytes(1, False)) 
theory[1] = calulate_theorectical(8, data_size_in_bytes(1))
theory[2] = calulate_theorectical(8, data_size_in_bytes(10))
theory[3] = calulate_theorectical(8, data_size_in_bytes(20))
theory[4] = calulate_theorectical(8, data_size_in_bytes(32))
theory[5] = calulate_theorectical(8, data_size_in_bytes(64))
theory[6] = calulate_theorectical(8, data_size_in_bytes(128))
theory[7] = calulate_theorectical(8, data_size_in_bytes(256))
theory[8] = calulate_theorectical(8, data_size_in_bytes(512))

print theory.values()

data_size = np.array(latencies.keys())
latency = np.array(latencies.values())
theory = np.array(theory.values())
pd.DataFrame
fig = plt.figure()
my_xticks = ['1kB','1MB','10MB','20MB','32MB','64MB','128MB','256MB','512MB']
plt.xticks(data_size, my_xticks)

plt.plot(data_size, latency)
#plt.plot(data_size, theory)
plt.yscale("log", nonposy="clip")
plt.ylabel('Latency in ms')
plt.xlabel('Data size')
fig.savefig('results/graphs/on_the_fly_gen_16.eps')
plt.show()


    

    




102.18009828
13.0392211084
2.12470880956
t
2.10778455771
2.52915881181
2.35885300174
2.0260116244
2147483648
[8192, 8388608, 83886080, 167772160, 268435456, 536870912, 1073741824, 2147483648, 4294967296]






    

In [10]:

    

## Generation Size: 32 
## On The fly - latency 

import pandas as pd
import numpy as np 
%matplotlib inline
import matplotlib
matplotlib.use('Agg')
import matplotlib.pyplot as plt 

def data_size_in_bytes(data_size, mb=True):
    kb = 1024
    if mb:
        return data_size * kb * kb
    else:
        return data_size * kb

def  calulate_theorectical(gen, data_size, redundancy=0):
    symbols = data_size / gen
    return symbols * gen * (gen + redundancy)

def parseData(index, value):
    if index == 0 or index == 1 or index == 2 or index == 3 or index == 4:
        return int(value)
    else:
        return float(value)

def get_latency(file_name):
    file = open(file_name)
    data = []
    for line in file:
        temp = line.split(',')
        index = 0
        for item in temp: 
            temp[index] = parseData(index, item)
            index = index + 1
        data.append(temp)
    latency = [entry[6]/1000 for entry in data] # convert to milliseconds
    return np.average(latency)

five_hundre_and_twelve_mb = 'results/encoder/thousand/1512527656_on_the_fly_encoder_binary8_0_32_16777216'
two_hundre_and_fifty_six_mb = 'results/encoder/thousand/1512521487_on_the_fly_encoder_binary8_0_32_8388608'
one_hundre_and_twenty_eight_mb = 'results/encoder/thousand/1512518431_on_the_fly_encoder_binary8_0_32_4194304'
sixty_four_mb = 'results/encoder/thousand/1512516828_on_the_fly_encoder_binary8_0_32_2097152'
thirdy_two_mb = 'results/encoder/thousand/1512515884_on_the_fly_encoder_binary8_0_32_1048576'
twenty_mb = 'results/encoder/thousand/1512515252_on_the_fly_encoder_binary8_0_32_655360'
ten_mb = 'results/encoder/thousand/1512514755_on_the_fly_encoder_binary8_0_32_327680'
one_mb = 'results/encoder/thousand/1512514356_on_the_fly_encoder_binary8_0_32_32768'
one_kb = 'results/encoder/thousand/1512514043_on_the_fly_encoder_binary8_0_32_32'


latencies = {}

latencies[0] = get_latency(one_kb)
latencies[1] = get_latency(one_mb)
latencies[2] = get_latency(ten_mb)
latencies[3] = get_latency(twenty_mb)
latencies[4] = get_latency(thirdy_two_mb)
latencies[5] = get_latency(sixty_four_mb)
latencies[6] = get_latency(one_hundre_and_twenty_eight_mb)
latencies[7] = get_latency(two_hundre_and_fifty_six_mb)
latencies[8] = get_latency(five_hundre_and_twelve_mb)

print latencies[1]/float(latencies[0])
print (latencies[2]-float(latencies[1]))/float(latencies[1])
print latencies[3]/float(latencies[2])
print 't'
i = 4
for x in range(i, (len(latencies)-1)):
    print (latencies[x+1]/float(latencies[x]))

theory = {}

print calulate_theorectical(8, data_size_in_bytes(256))
theory[0] = calulate_theorectical(8, data_size_in_bytes(1, False)) 
theory[1] = calulate_theorectical(8, data_size_in_bytes(1))
theory[2] = calulate_theorectical(8, data_size_in_bytes(10))
theory[3] = calulate_theorectical(8, data_size_in_bytes(20))
theory[4] = calulate_theorectical(8, data_size_in_bytes(32))
theory[5] = calulate_theorectical(8, data_size_in_bytes(64))
theory[6] = calulate_theorectical(8, data_size_in_bytes(128))
theory[7] = calulate_theorectical(8, data_size_in_bytes(256))
theory[8] = calulate_theorectical(8, data_size_in_bytes(512))

print theory.values()

data_size = np.array(latencies.keys())
latency = np.array(latencies.values())
theory = np.array(theory.values())
pd.DataFrame
fig = plt.figure()
my_xticks = ['1kB','1MB','10MB','20MB','32MB','64MB','128MB','256MB','512MB']
plt.xticks(data_size, my_xticks)

plt.plot(data_size, latency)
#plt.plot(data_size, theory)
plt.yscale("log", nonposy="clip")
plt.ylabel('Latency in ms')
plt.xlabel('Data size')
fig.savefig('results/graphs/on_the_fly_gen_32.eps')
plt.show()


    

    




85.693500136
12.8405712563
2.12056889935
t
2.03719422218
2.07883043154
2.48792871386
2.45240696245
2147483648
[8192, 8388608, 83886080, 167772160, 268435456, 536870912, 1073741824, 2147483648, 4294967296]






    

In [14]:

    

## Generation Size: 8,16,32 
## On The Fly - latency comparison
import pandas as pd
import numpy as np 
%matplotlib inline
import matplotlib
matplotlib.use('Agg')
import matplotlib.pyplot as plt 

def parseData(index, value):
    if index == 0 or index == 1 or index == 2 or index == 3 or index == 4:
        return int(value)
    else:
        return float(value)

def get_latency(file_name):
    file = open(file_name)
    data = []
    for line in file:
        temp = line.split(',')
        index = 0
        for item in temp: 
            temp[index] = parseData(index, item)
            index = index + 1
        data.append(temp)
    latency = [entry[6]/1000 for entry in data] # convert to milliseconds 
    return np.average(latency)

def read_gen_eight():
    five_hundre_and_twelve_mb = 'results/encoder/thousand/1512501259_on_the_fly_encoder_binary8_0_8_67108864'
    two_hundre_and_fifty_six_mb = 'results/encoder/thousand/1512496496_on_the_fly_encoder_binary8_0_8_33554432'
    one_hundre_and_twenty_eight_mb = 'results/encoder/thousand/1512493964_on_the_fly_encoder_binary8_0_8_16777216'
    sixty_four_mb = 'results/encoder/thousand/1512492545_on_the_fly_encoder_binary8_0_8_8388608'
    thirdy_two_mb = 'results/encoder/thousand/1512491693_on_the_fly_encoder_binary8_0_8_4194304'
    twenty_mb = 'results/encoder/thousand/1512491120_on_the_fly_encoder_binary8_0_8_2621440'
    ten_mb = 'results/encoder/thousand/1512490653_on_the_fly_encoder_binary8_0_8_1310720'
    one_mb = 'results/encoder/thousand/1512490269_on_the_fly_encoder_binary8_0_8_131072'
    one_kb = 'results/encoder/thousand/1512489957_on_the_fly_encoder_binary8_0_8_128'


    latencies = {}

    latencies[0] = get_latency(one_kb)
    latencies[1] = get_latency(one_mb)
    latencies[2] = get_latency(ten_mb)
    latencies[3] = get_latency(twenty_mb)
    latencies[4] = get_latency(thirdy_two_mb)
    latencies[5] = get_latency(sixty_four_mb)
    latencies[6] = get_latency(one_hundre_and_twenty_eight_mb)
    latencies[7] = get_latency(two_hundre_and_fifty_six_mb)
    latencies[8] = get_latency(five_hundre_and_twelve_mb)
    return latencies

def read_gen_sixteen():
    five_hundre_and_twelve_mb = 'results/encoder/thousand/1512513739_on_the_fly_encoder_binary8_0_16_33554432'
    two_hundre_and_fifty_six_mb = 'results/encoder/thousand/1512508485_on_the_fly_encoder_binary8_0_16_16777216'
    one_hundre_and_twenty_eight_mb = 'results/encoder/thousand/1512505713_on_the_fly_encoder_binary8_0_16_8388608'
    sixty_four_mb = 'results/encoder/thousand/1512504214_on_the_fly_encoder_binary8_0_16_4194304'
    thirdy_two_mb = 'results/encoder/thousand/1512503337_on_the_fly_encoder_binary8_0_16_2097152'
    twenty_mb = 'results/encoder/thousand/1512502740_on_the_fly_encoder_binary8_0_16_1310720'
    ten_mb = 'results/encoder/thousand/1512502264_on_the_fly_encoder_binary8_0_16_655360'
    one_mb = 'results/encoder/thousand/1512501875_on_the_fly_encoder_binary8_0_16_65536'
    one_kb = 'results/encoder/thousand/1512501563_on_the_fly_encoder_binary8_0_16_64'
    
    latencies = {}

    latencies[0] = get_latency(one_kb)
    latencies[1] = get_latency(one_mb)
    latencies[2] = get_latency(ten_mb)
    latencies[3] = get_latency(twenty_mb)
    latencies[4] = get_latency(thirdy_two_mb)
    latencies[5] = get_latency(sixty_four_mb)
    latencies[6] = get_latency(one_hundre_and_twenty_eight_mb)
    latencies[7] = get_latency(two_hundre_and_fifty_six_mb)
    latencies[8] = get_latency(five_hundre_and_twelve_mb)
    return latencies

def read_gen_thirdy_two():
    five_hundre_and_twelve_mb = 'results/encoder/thousand/1512527656_on_the_fly_encoder_binary8_0_32_16777216'
    two_hundre_and_fifty_six_mb = 'results/encoder/thousand/1512521487_on_the_fly_encoder_binary8_0_32_8388608'
    one_hundre_and_twenty_eight_mb = 'results/encoder/thousand/1512518431_on_the_fly_encoder_binary8_0_32_4194304'
    sixty_four_mb = 'results/encoder/thousand/1512516828_on_the_fly_encoder_binary8_0_32_2097152'
    thirdy_two_mb = 'results/encoder/thousand/1512515884_on_the_fly_encoder_binary8_0_32_1048576'
    twenty_mb = 'results/encoder/thousand/1512515252_on_the_fly_encoder_binary8_0_32_655360'
    ten_mb = 'results/encoder/thousand/1512514755_on_the_fly_encoder_binary8_0_32_327680'
    one_mb = 'results/encoder/thousand/1512514356_on_the_fly_encoder_binary8_0_32_32768'
    one_kb = 'results/encoder/thousand/1512514043_on_the_fly_encoder_binary8_0_32_32'

    latencies = {}

    latencies[0] = get_latency(one_kb)
    latencies[1] = get_latency(one_mb)
    latencies[2] = get_latency(ten_mb)
    latencies[3] = get_latency(twenty_mb)
    latencies[4] = get_latency(thirdy_two_mb)
    latencies[5] = get_latency(sixty_four_mb)
    latencies[6] = get_latency(one_hundre_and_twenty_eight_mb)
    latencies[7] = get_latency(two_hundre_and_fifty_six_mb)
    latencies[8] = get_latency(five_hundre_and_twelve_mb)
    return latencies

def calc_diff(x,y):
    print (y/float(x))
    
def diff_gen(g, gg):
    for x, y in zip(g,gg):
        calc_diff(g[x],gg[y])

eight_latencies = read_gen_eight()
eight_data_size = np.array(eight_latencies.keys())
eight_latency = np.array(eight_latencies.values())

sixteen_latencies = read_gen_sixteen()
diff_gen(eight_latencies,sixteen_latencies)
sixteen_data_size = np.array(sixteen_latencies.keys())
sixteen_latency = np.array(sixteen_latencies.values())

print('t')
thridy_two_latencies = read_gen_thirdy_two()
diff_gen(sixteen_latencies,thridy_two_latencies)
thridy_two_data_size = np.array(thridy_two_latencies.keys())
thridy_two_latency = np.array(thridy_two_latencies.values())


pd.DataFrame
fig = plt.figure()
my_xticks = ['1kB','1MB','10MB','20MB','32MB','64MB','128MB','256MB','512MB']
plt.xticks(eight_data_size, my_xticks)
plt.plot(eight_data_size, eight_latency)

plt.xticks(sixteen_data_size, my_xticks)
plt.plot(sixteen_data_size, sixteen_latency)

plt.xticks(thridy_two_data_size, my_xticks)
plt.plot(thridy_two_data_size, thridy_two_latency)

plt.yscale("log", nonposy="clip")
plt.legend(['G=8', 'G=16', 'G=32'])
plt.ylabel('Latency in ms')
plt.xlabel('Data size')
fig.savefig('results/graphs/on_the_fly_gen_comparison.eps')
plt.show()