notebook.community

Edit and run

Aggregate NDT Metrics for USA & Greece over last three months





In [2]:



    


import pandas
import MySQLdb
import pandas.io.sql as psql
import matplotlib
from datetime import datetime as dt
from bq2ipy import bigquery
import socket
import os

def get_ip(host):
    return socket.gethostbyname(host)

def get_ndt_ip(host):
    h = "ndt.iupui."+host+".measurement-lab.org"
    return get_ip(h)

Queries





In [9]:



    


def get_ndt_case_bins():
    args = {}
    US_IPS="""'216.156.197.139','216.156.197.152','216.156.197.165','64.9.225.141','64.9.225.154','64.9.225.167','64.9.225.187','74.63.50.19','74.63.50.32','74.63.50.47','38.98.51.20','38.98.51.33','38.98.51.45','38.102.0.83','38.102.0.97','38.102.0.109','38.106.70.147','38.106.70.160','38.106.70.173','4.71.251.147','4.71.251.160','4.71.251.173','4.71.210.211','4.71.210.224','4.71.210.237','4.71.254.147','4.71.254.173','4.71.254.160','38.107.216.17','38.107.216.32','38.107.216.45'"""
    GR_IPS="""'83.212.4.24','83.212.4.10','83.212.4.37','83.212.5.139','83.212.5.152','83.212.5.164'"""
    months=['m_lab.2012_07', 'm_lab.2012_08', 'm_lab.2012_09', 'm_lab.2012_10' ]
    directions = [ 0, 1] # up, down

    # sum on HCThruOctetsAcked, where is_last_entry is true group by test_id
    # HCThruOctetsAcked
    # 
    ret = { 'us' : {'up' : None, 'down' : None}, 
            'gr' : {'up' : None, 'down' : None} }
    for country,ip_list in [ ('us', US_IPS), ('gr', GR_IPS) ]:
        up_query="""
select  
    connection_spec.data_direction as direction,
    UTC_USEC_TO_DAY(web100_log_entry.log_time*1000000) as date,
    count(test_id) as test_count,
    count(distinct web100_log_entry.connection_spec.remote_ip) as unique_clients,
    SUM(web100_log_entry.snap.HCThruOctetsReceived) as total_bytes
from    
    m_lab.2012_07,m_lab.2012_08,m_lab.2012_09,m_lab.2012_10
where 
        IS_EXPLICITLY_DEFINED(project)
    AND project = 0
    AND IS_EXPLICITLY_DEFINED(connection_spec.data_direction)
    AND connection_spec.data_direction = 0
    AND IS_EXPLICITLY_DEFINED(web100_log_entry.is_last_entry)
    AND web100_log_entry.is_last_entry = True
    AND IS_EXPLICITLY_DEFINED(web100_log_entry.snap.HCThruOctetsReceived)
    AND web100_log_entry.snap.HCThruOctetsReceived >= 8192
    AND web100_log_entry.snap.HCThruOctetsReceived < 1000000000
    AND IS_EXPLICITLY_DEFINED(web100_log_entry.snap.Duration)
    AND web100_log_entry.snap.Duration >= 9000000
    AND web100_log_entry.snap.Duration < 3600000000
    AND IS_EXPLICITLY_DEFINED(web100_log_entry.connection_spec.remote_ip)
    AND IS_EXPLICITLY_DEFINED(web100_log_entry.connection_spec.local_ip)
    AND web100_log_entry.connection_spec.local_ip IN(%(ip_list)s) 
group by
    date, direction
order by 
    date;
""" % { 'ip_list' : ip_list }
        
        down_query="""
select  
    connection_spec.data_direction as direction,
    UTC_USEC_TO_DAY(web100_log_entry.log_time*1000000) as date,
    count(test_id) as test_count,
    count(distinct web100_log_entry.connection_spec.remote_ip) as unique_clients,
    SUM(web100_log_entry.snap.HCThruOctetsAcked) as total_bytes
from    
    m_lab.2012_07,m_lab.2012_08,m_lab.2012_09,m_lab.2012_10
where 
        IS_EXPLICITLY_DEFINED(project)
    AND project = 0
    AND IS_EXPLICITLY_DEFINED(connection_spec.data_direction)
    AND connection_spec.data_direction = 1
    AND IS_EXPLICITLY_DEFINED(web100_log_entry.is_last_entry)
    AND web100_log_entry.is_last_entry = True
    AND IS_EXPLICITLY_DEFINED(web100_log_entry.snap.HCThruOctetsAcked)
    AND web100_log_entry.snap.HCThruOctetsAcked >= 8192
    AND web100_log_entry.snap.HCThruOctetsAcked < 1000000000
    AND (web100_log_entry.snap.SndLimTimeRwin +
         web100_log_entry.snap.SndLimTimeCwnd +
         web100_log_entry.snap.SndLimTimeSnd) >= 9000000
    AND (web100_log_entry.snap.SndLimTimeRwin +
         web100_log_entry.snap.SndLimTimeCwnd +
         web100_log_entry.snap.SndLimTimeSnd) < 3600000000
    AND IS_EXPLICITLY_DEFINED(web100_log_entry.snap.CongSignals)
    AND web100_log_entry.snap.CongSignals > 0
    AND IS_EXPLICITLY_DEFINED(web100_log_entry.connection_spec.remote_ip)
    AND IS_EXPLICITLY_DEFINED(web100_log_entry.connection_spec.local_ip)
    AND web100_log_entry.connection_spec.local_ip IN(%(ip_list)s) 
group by
    date, direction
order by
    date; 
""" % { 'ip_list' : ip_list }

        print up_query
        ret[country]['up'] = bigquery(up_query)
        ret[country]['down'] = bigquery(down_query)
    return ret



    











In [10]:



    


ret = get_ndt_case_bins()
print ret



    


















    






select  
    connection_spec.data_direction as direction,
    UTC_USEC_TO_DAY(web100_log_entry.log_time*1000000) as date,
    count(test_id) as test_count,
    count(distinct web100_log_entry.connection_spec.remote_ip) as unique_clients,
    SUM(web100_log_entry.snap.HCThruOctetsReceived) as total_bytes
from    
    m_lab.2012_07,m_lab.2012_08,m_lab.2012_09,m_lab.2012_10
where 
        IS_EXPLICITLY_DEFINED(project)
    AND project = 0
    AND IS_EXPLICITLY_DEFINED(connection_spec.data_direction)
    AND connection_spec.data_direction = 0
    AND IS_EXPLICITLY_DEFINED(web100_log_entry.is_last_entry)
    AND web100_log_entry.is_last_entry = True
    AND IS_EXPLICITLY_DEFINED(web100_log_entry.snap.HCThruOctetsReceived)
    AND web100_log_entry.snap.HCThruOctetsReceived >= 8192
    AND web100_log_entry.snap.HCThruOctetsReceived < 1000000000
    AND IS_EXPLICITLY_DEFINED(web100_log_entry.snap.Duration)
    AND web100_log_entry.snap.Duration >= 9000000
    AND web100_log_entry.snap.Duration < 3600000000
    AND IS_EXPLICITLY_DEFINED(web100_log_entry.connection_spec.remote_ip)
    AND IS_EXPLICITLY_DEFINED(web100_log_entry.connection_spec.local_ip)
    AND web100_log_entry.connection_spec.local_ip IN('216.156.197.139','216.156.197.152','216.156.197.165','64.9.225.141','64.9.225.154','64.9.225.167','64.9.225.187','74.63.50.19','74.63.50.32','74.63.50.47','38.98.51.20','38.98.51.33','38.98.51.45','38.102.0.83','38.102.0.97','38.102.0.109','38.106.70.147','38.106.70.160','38.106.70.173','4.71.251.147','4.71.251.160','4.71.251.173','4.71.210.211','4.71.210.224','4.71.210.237','4.71.254.147','4.71.254.173','4.71.254.160','38.107.216.17','38.107.216.32','38.107.216.45') 
group by
    date, direction
order by 
    date;


select  
    connection_spec.data_direction as direction,
    UTC_USEC_TO_DAY(web100_log_entry.log_time*1000000) as date,
    count(test_id) as test_count,
    count(distinct web100_log_entry.connection_spec.remote_ip) as unique_clients,
    SUM(web100_log_entry.snap.HCThruOctetsReceived) as total_bytes
from    
    m_lab.2012_07,m_lab.2012_08,m_lab.2012_09,m_lab.2012_10
where 
        IS_EXPLICITLY_DEFINED(project)
    AND project = 0
    AND IS_EXPLICITLY_DEFINED(connection_spec.data_direction)
    AND connection_spec.data_direction = 0
    AND IS_EXPLICITLY_DEFINED(web100_log_entry.is_last_entry)
    AND web100_log_entry.is_last_entry = True
    AND IS_EXPLICITLY_DEFINED(web100_log_entry.snap.HCThruOctetsReceived)
    AND web100_log_entry.snap.HCThruOctetsReceived >= 8192
    AND web100_log_entry.snap.HCThruOctetsReceived < 1000000000
    AND IS_EXPLICITLY_DEFINED(web100_log_entry.snap.Duration)
    AND web100_log_entry.snap.Duration >= 9000000
    AND web100_log_entry.snap.Duration < 3600000000
    AND IS_EXPLICITLY_DEFINED(web100_log_entry.connection_spec.remote_ip)
    AND IS_EXPLICITLY_DEFINED(web100_log_entry.connection_spec.local_ip)
    AND web100_log_entry.connection_spec.local_ip IN('83.212.4.24','83.212.4.10','83.212.4.37','83.212.5.139','83.212.5.152','83.212.5.164') 
group by
    date, direction
order by 
    date;

{'gr': {'down': {'date': [1341100800000000.0, 1341187200000000.0, 1341273600000000.0, 1341360000000000.0, 1341446400000000.0, 1341532800000000.0, 1341619200000000.0, 1341705600000000.0, 1341792000000000.0, 1341878400000000.0, 1341964800000000.0, 1342051200000000.0, 1342137600000000.0, 1342224000000000.0, 1342310400000000.0, 1342396800000000.0, 1342483200000000.0, 1342569600000000.0, 1342656000000000.0, 1342742400000000.0, 1342828800000000.0, 1342915200000000.0, 1343001600000000.0, 1343088000000000.0, 1343174400000000.0, 1343260800000000.0, 1343347200000000.0, 1343433600000000.0, 1343520000000000.0, 1343606400000000.0, 1343692800000000.0, 1343779200000000.0, 1343865600000000.0, 1343952000000000.0, 1344038400000000.0, 1344124800000000.0, 1344211200000000.0, 1344297600000000.0, 1344384000000000.0, 1344470400000000.0, 1344556800000000.0, 1344643200000000.0, 1344729600000000.0, 1344816000000000.0, 1344902400000000.0, 1344988800000000.0, 1345075200000000.0, 1345161600000000.0, 1345248000000000.0, 1345334400000000.0, 1345420800000000.0, 1345507200000000.0, 1345593600000000.0, 1345680000000000.0, 1345766400000000.0, 1345852800000000.0, 1345939200000000.0, 1346025600000000.0, 1346112000000000.0, 1346198400000000.0, 1346284800000000.0, 1346371200000000.0, 1346457600000000.0, 1346544000000000.0, 1346630400000000.0, 1346716800000000.0, 1346803200000000.0, 1346889600000000.0, 1346976000000000.0, 1347062400000000.0, 1347148800000000.0, 1347235200000000.0, 1347321600000000.0, 1347408000000000.0, 1347494400000000.0, 1347580800000000.0, 1347667200000000.0, 1347753600000000.0, 1347840000000000.0, 1347926400000000.0, 1348012800000000.0, 1348099200000000.0, 1348185600000000.0, 1348272000000000.0, 1348358400000000.0, 1348444800000000.0, 1348531200000000.0, 1348617600000000.0, 1348704000000000.0, 1348790400000000.0, 1348876800000000.0, 1348963200000000.0, 1349049600000000.0, 1349136000000000.0, 1349222400000000.0, 1349308800000000.0, 1349395200000000.0, 1349481600000000.0, 1349568000000000.0, 1349654400000000.0, 1349740800000000.0], 'direction': [1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0], 'total_bytes': [71045876278.0, 64307481540.0, 65696425976.0, 65440769401.0, 63284912110.0, 65352433045.0, 68066601053.0, 67249744657.0, 63248009137.0, 65127472626.0, 64547335237.0, 41868664898.0, 42539944264.0, 40462523106.0, 43957907132.0, 47942998299.0, 61019517363.0, 64363946402.0, 63320588954.0, 62675912406.0, 68794657221.0, 73008036230.0, 64716905960.0, 71443666082.0, 69712990706.0, 63984220958.0, 66533978779.0, 69128010422.0, 65199469168.0, 64248977388.0, 63879585638.0, 70347453691.0, 66354345632.0, 66753933225.0, 63474722014.0, 71365619383.0, 66393619145.0, 67867360277.0, 64647577039.0, 62698051143.0, 64958424743.0, 64019429860.0, 68954070879.0, 63105206784.0, 66885235986.0, 66757949548.0, 70320288649.0, 57300408329.0, 50134511049.0, 48067160925.0, 21721130868.0, 49312448107.0, 65416650412.0, 52611504393.0, 71504969542.0, 71520067575.0, 71168349621.0, 68630733914.0, 73207509613.0, 69242358581.0, 69578075221.0, 71957901282.0, 74573176985.0, 72655397755.0, 69965003208.0, 69345908436.0, 68533704316.0, 71145650758.0, 68577774854.0, 72416251505.0, 75020490802.0, 70288585454.0, 69545661250.0, 66023057114.0, 65431693995.0, 73115541949.0, 73952274955.0, 71691899770.0, 47737704270.0, 49231656825.0, 58175433860.0, 76581043975.0, 72140373619.0, 75088904830.0, 59121821225.0, 65631514893.0, 61425171746.0, 69280082287.0, 70102770440.0, 76970355287.0, 73584602072.0, 78067504877.0, 79084681962.0, 76979575206.0, 28402488378.0, 27117379193.0, 2096971686.0, 2613976332.0, 3605657804.0, 2029890246.0, 2833612808.0], 'unique_clients': [13065.0, 12130.0, 12545.0, 12045.0, 12466.0, 12058.0, 12799.0, 12635.0, 12310.0, 13033.0, 12108.0, 8822.0, 8854.0, 9165.0, 8647.0, 9589.0, 12017.0, 12345.0, 11779.0, 12246.0, 12305.0, 13005.0, 12347.0, 12699.0, 12655.0, 12601.0, 11934.0, 12075.0, 12059.0, 12124.0, 12956.0, 13243.0, 12047.0, 11834.0, 11434.0, 12920.0, 12488.0, 12685.0, 11900.0, 11900.0, 12786.0, 12301.0, 12591.0, 12299.0, 12982.0, 13033.0, 14179.0, 10712.0, 9867.0, 9181.0, 4444.0, 8409.0, 11847.0, 9695.0, 14404.0, 13675.0, 13595.0, 13001.0, 13800.0, 13573.0, 13119.0, 12406.0, 13392.0, 12818.0, 12268.0, 13007.0, 12557.0, 12601.0, 12492.0, 13475.0, 13800.0, 12943.0, 12569.0, 12627.0, 13373.0, 14073.0, 13955.0, 13923.0, 9816.0, 9555.0, 11214.0, 14341.0, 14272.0, 13914.0, 10767.0, 12335.0, 11931.0, 14002.0, 12630.0, 13863.0, 14111.0, 14118.0, 14331.0, 13587.0, 5357.0, 5475.0, 515.0, 529.0, 715.0, 465.0, 624.0], 'test_count': [17217.0, 15669.0, 16346.0, 16264.0, 16282.0, 16895.0, 16941.0, 16913.0, 16244.0, 16943.0, 16442.0, 11023.0, 10857.0, 10863.0, 11103.0, 11950.0, 16356.0, 16056.0, 16023.0, 16271.0, 16624.0, 17627.0, 16204.0, 17124.0, 16615.0, 16058.0, 16154.0, 17188.0, 16706.0, 16345.0, 16019.0, 16582.0, 15904.0, 16191.0, 15241.0, 17081.0, 16866.0, 16691.0, 15791.0, 15785.0, 16120.0, 16254.0, 17219.0, 16005.0, 16767.0, 16934.0, 17530.0, 14436.0, 11825.0, 11603.0, 5314.0, 10666.0, 15477.0, 13062.0, 17828.0, 18140.0, 18374.0, 17378.0, 18311.0, 17236.0, 17765.0, 17702.0, 18225.0, 17590.0, 16434.0, 17042.0, 17108.0, 17113.0, 17229.0, 18311.0, 18667.0, 17312.0, 17140.0, 17371.0, 17319.0, 18546.0, 18974.0, 18582.0, 12311.0, 12012.0, 13891.0, 19720.0, 18524.0, 19092.0, 14271.0, 16039.0, 15192.0, 17061.0, 17325.0, 18179.0, 18508.0, 19047.0, 18848.0, 18133.0, 6509.0, 6523.0, 554.0, 586.0, 823.0, 513.0, 675.0]}, 'up': {'date': [1341100800000000.0, 1341187200000000.0, 1341273600000000.0, 1341360000000000.0, 1341446400000000.0, 1341532800000000.0, 1341619200000000.0, 1341705600000000.0, 1341792000000000.0, 1341878400000000.0, 1341964800000000.0, 1342051200000000.0, 1342137600000000.0, 1342224000000000.0, 1342310400000000.0, 1342396800000000.0, 1342483200000000.0, 1342569600000000.0, 1342656000000000.0, 1342742400000000.0, 1342828800000000.0, 1342915200000000.0, 1343001600000000.0, 1343088000000000.0, 1343174400000000.0, 1343260800000000.0, 1343347200000000.0, 1343433600000000.0, 1343520000000000.0, 1343606400000000.0, 1343692800000000.0, 1343779200000000.0, 1343865600000000.0, 1343952000000000.0, 1344038400000000.0, 1344124800000000.0, 1344211200000000.0, 1344297600000000.0, 1344384000000000.0, 1344470400000000.0, 1344556800000000.0, 1344643200000000.0, 1344729600000000.0, 1344816000000000.0, 1344902400000000.0, 1344988800000000.0, 1345075200000000.0, 1345161600000000.0, 1345248000000000.0, 1345334400000000.0, 1345420800000000.0, 1345507200000000.0, 1345593600000000.0, 1345680000000000.0, 1345766400000000.0, 1345852800000000.0, 1345939200000000.0, 1346025600000000.0, 1346112000000000.0, 1346198400000000.0, 1346284800000000.0, 1346371200000000.0, 1346457600000000.0, 1346544000000000.0, 1346630400000000.0, 1346716800000000.0, 1346803200000000.0, 1346889600000000.0, 1346976000000000.0, 1347062400000000.0, 1347148800000000.0, 1347235200000000.0, 1347321600000000.0, 1347408000000000.0, 1347494400000000.0, 1347580800000000.0, 1347667200000000.0, 1347753600000000.0, 1347840000000000.0, 1347926400000000.0, 1348012800000000.0, 1348099200000000.0, 1348185600000000.0, 1348272000000000.0, 1348358400000000.0, 1348444800000000.0, 1348531200000000.0, 1348617600000000.0, 1348704000000000.0, 1348790400000000.0, 1348876800000000.0, 1348963200000000.0, 1349049600000000.0, 1349136000000000.0, 1349222400000000.0, 1349308800000000.0, 1349395200000000.0, 1349481600000000.0, 1349568000000000.0, 1349654400000000.0, 1349740800000000.0], 'direction': [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], 'total_bytes': [54847370067.0, 50709088625.0, 52190196179.0, 52440297602.0, 50245811281.0, 51120258526.0, 52670160860.0, 51576087017.0, 50593925255.0, 51257073106.0, 49514617293.0, 33451273099.0, 33378510339.0, 32581770459.0, 33060688048.0, 37394179767.0, 49469047578.0, 50933741109.0, 48294957332.0, 51517036416.0, 53415889088.0, 56117530549.0, 50033740735.0, 54611403753.0, 52215316948.0, 50737910560.0, 49909151589.0, 53055590350.0, 50089887982.0, 50463144073.0, 50403469228.0, 56745440664.0, 49713548870.0, 52510041969.0, 49470150070.0, 55021120403.0, 50952407183.0, 52565736266.0, 51158115584.0, 49770346647.0, 49833687174.0, 48342682330.0, 53018085417.0, 49360655330.0, 53236180210.0, 51744580074.0, 52587866804.0, 44093293135.0, 37099082956.0, 36107170677.0, 17198722265.0, 37647430277.0, 48942162534.0, 40453731789.0, 54816474121.0, 53439072683.0, 53718593660.0, 52203490635.0, 56881251802.0, 53719564907.0, 53044756385.0, 55304346626.0, 57174568852.0, 56288121342.0, 52637543952.0, 54457569501.0, 53144187156.0, 55534302279.0, 54802477934.0, 56306722809.0, 57286879523.0, 55163712768.0, 54810839933.0, 53823890390.0, 52832166082.0, 56159437761.0, 56779389469.0, 54519720829.0, 38114626873.0, 38382402870.0, 43027134681.0, 58353168580.0, 57064796297.0, 58134912460.0, 44503025468.0, 51728454113.0, 51131669218.0, 54045048023.0, 54345789599.0, 56962993091.0, 56329122583.0, 58804127193.0, 62981459425.0, 58317509772.0, 22078708977.0, 20641777662.0, 1286333697.0, 1371983336.0, 2181877867.0, 1369359212.0, 1951354520.0], 'unique_clients': [25089.0, 23088.0, 23832.0, 23588.0, 23844.0, 22326.0, 24128.0, 22949.0, 23419.0, 23795.0, 22815.0, 17022.0, 16943.0, 18105.0, 15958.0, 17729.0, 22672.0, 23419.0, 22388.0, 23501.0, 23606.0, 24493.0, 22605.0, 23768.0, 23458.0, 21946.0, 21907.0, 23285.0, 22913.0, 22601.0, 23133.0, 23752.0, 22827.0, 21747.0, 21656.0, 23897.0, 23682.0, 23454.0, 22549.0, 22314.0, 23225.0, 23030.0, 23840.0, 23087.0, 23461.0, 23948.0, 23810.0, 20762.0, 18001.0, 17147.0, 8317.0, 14702.0, 21878.0, 18382.0, 25581.0, 23793.0, 25300.0, 23844.0, 25273.0, 23467.0, 24056.0, 22930.0, 24944.0, 23344.0, 22435.0, 23143.0, 22642.0, 23852.0, 22224.0, 24469.0, 25335.0, 23371.0, 22674.0, 21921.0, 24054.0, 25297.0, 24735.0, 24561.0, 17126.0, 17626.0, 19609.0, 26592.0, 24594.0, 25586.0, 19275.0, 21605.0, 20905.0, 24379.0, 23243.0, 25682.0, 25694.0, 25051.0, 26811.0, 24573.0, 9778.0, 9733.0, 778.0, 747.0, 1061.0, 725.0, 920.0], 'test_count': [34562.0, 32221.0, 33184.0, 33138.0, 33481.0, 34009.0, 34461.0, 33948.0, 32976.0, 33819.0, 33221.0, 22564.0, 21856.0, 22549.0, 22424.0, 24145.0, 33274.0, 32609.0, 32240.0, 33125.0, 33175.0, 35608.0, 32778.0, 34695.0, 33446.0, 32873.0, 32410.0, 34574.0, 33866.0, 32835.0, 32211.0, 33658.0, 32367.0, 33309.0, 31189.0, 35198.0, 34561.0, 33924.0, 32517.0, 32377.0, 32403.0, 33477.0, 34868.0, 31812.0, 33324.0, 33861.0, 34907.0, 28795.0, 23604.0, 22877.0, 10517.0, 21033.0, 30425.0, 25918.0, 35259.0, 35564.0, 36096.0, 34494.0, 35956.0, 33221.0, 34607.0, 34852.0, 36967.0, 35171.0, 33295.0, 33910.0, 33688.0, 34106.0, 33873.0, 35538.0, 36693.0, 33422.0, 33328.0, 33550.0, 33585.0, 35910.0, 36619.0, 35711.0, 24197.0, 23326.0, 26650.0, 37903.0, 35938.0, 37149.0, 27686.0, 31408.0, 29637.0, 33350.0, 33505.0, 34768.0, 35916.0, 36928.0, 37335.0, 35679.0, 12665.0, 12876.0, 849.0, 845.0, 1251.0, 809.0, 1026.0]}}, 'us': {'down': {'date': [1341100800000000.0, 1341187200000000.0, 1341273600000000.0, 1341360000000000.0, 1341446400000000.0, 1341532800000000.0, 1341619200000000.0, 1341705600000000.0, 1341792000000000.0, 1341878400000000.0, 1341964800000000.0, 1342051200000000.0, 1342137600000000.0, 1342224000000000.0, 1342310400000000.0, 1342396800000000.0, 1342483200000000.0, 1342569600000000.0, 1342656000000000.0, 1342742400000000.0, 1342828800000000.0, 1342915200000000.0, 1343001600000000.0, 1343088000000000.0, 1343174400000000.0, 1343260800000000.0, 1343347200000000.0, 1343433600000000.0, 1343520000000000.0, 1343606400000000.0, 1343692800000000.0, 1343779200000000.0, 1343865600000000.0, 1343952000000000.0, 1344038400000000.0, 1344124800000000.0, 1344211200000000.0, 1344297600000000.0, 1344384000000000.0, 1344470400000000.0, 1344556800000000.0, 1344643200000000.0, 1344729600000000.0, 1344816000000000.0, 1344902400000000.0, 1344988800000000.0, 1345075200000000.0, 1345161600000000.0, 1345248000000000.0, 1345334400000000.0, 1345420800000000.0, 1345507200000000.0, 1345593600000000.0, 1345680000000000.0, 1345766400000000.0, 1345852800000000.0, 1345939200000000.0, 1346025600000000.0, 1346112000000000.0, 1346198400000000.0, 1346284800000000.0, 1346371200000000.0, 1346457600000000.0, 1346544000000000.0, 1346630400000000.0, 1346716800000000.0, 1346803200000000.0, 1346889600000000.0, 1346976000000000.0, 1347062400000000.0, 1347148800000000.0, 1347235200000000.0, 1347321600000000.0, 1347408000000000.0, 1347494400000000.0, 1347580800000000.0, 1347667200000000.0, 1347753600000000.0, 1347840000000000.0, 1347926400000000.0, 1348012800000000.0, 1348099200000000.0, 1348185600000000.0, 1348272000000000.0, 1348358400000000.0, 1348444800000000.0, 1348531200000000.0, 1348617600000000.0, 1348704000000000.0, 1348790400000000.0, 1348876800000000.0, 1348963200000000.0, 1349049600000000.0, 1349136000000000.0, 1349222400000000.0, 1349308800000000.0, 1349395200000000.0, 1349481600000000.0, 1349568000000000.0, 1349654400000000.0, 1349740800000000.0, 1349827200000000.0], 'direction': [1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0], 'total_bytes': [181230681377.0, 186133430144.0, 179104609695.0, 179733140976.0, 156821605879.0, 170542420411.0, 175365410330.0, 182537266969.0, 179848805185.0, 181028795867.0, 177077046755.0, 192018763666.0, 189627567543.0, 189337611351.0, 198546140277.0, 192678239426.0, 193566088440.0, 187073208629.0, 179201694583.0, 194846456667.0, 193757754007.0, 188053160031.0, 191237403258.0, 186264017554.0, 186294666412.0, 186523997965.0, 177527063122.0, 190406631552.0, 190400652213.0, 197804970634.0, 190822488185.0, 191485006704.0, 190173319910.0, 184547117338.0, 194572650950.0, 202302136458.0, 200757594891.0, 186308047533.0, 166018295438.0, 191762474594.0, 198246018304.0, 207357910393.0, 209297931584.0, 214061990578.0, 213886107514.0, 221841680702.0, 215477243819.0, 213478494660.0, 227125182584.0, 221933755007.0, 232271113265.0, 226620123146.0, 237701022966.0, 228594248887.0, 217709911364.0, 222365329859.0, 224456271767.0, 231649605444.0, 215809940155.0, 206236033161.0, 196733847382.0, 214399657926.0, 227265283684.0, 223647597064.0, 237247023909.0, 217522349522.0, 217605228206.0, 203878002404.0, 207591868691.0, 227532807448.0, 219736444641.0, 202124234885.0, 202695429734.0, 210304405876.0, 204283374121.0, 218973217223.0, 225126564221.0, 219064935950.0, 218947974736.0, 220733875096.0, 231707442713.0, 224057026799.0, 224887605427.0, 247920224829.0, 240926015887.0, 218670128927.0, 226635416955.0, 235875493759.0, 215568110699.0, 217769181000.0, 237575507041.0, 240450620897.0, 228741342611.0, 200069984067.0, 221328851048.0, 215604599048.0, 212587198405.0, 231045602265.0, 220339524876.0, 211642995880.0, 139421089911.0, 68940156.0], 'unique_clients': [16307.0, 17220.0, 16544.0, 15820.0, 14034.0, 15026.0, 15444.0, 17247.0, 16096.0, 15937.0, 16309.0, 17321.0, 18320.0, 18464.0, 18017.0, 16741.0, 17474.0, 16811.0, 15990.0, 17956.0, 17715.0, 16902.0, 16925.0, 17608.0, 16014.0, 17294.0, 16325.0, 17382.0, 17005.0, 17272.0, 15170.0, 16551.0, 16622.0, 16612.0, 16978.0, 16921.0, 17940.0, 15302.0, 13449.0, 15870.0, 15572.0, 17245.0, 16805.0, 16819.0, 17094.0, 19708.0, 17665.0, 17420.0, 18941.0, 20061.0, 20003.0, 18317.0, 18108.0, 17308.0, 16487.0, 17728.0, 18999.0, 19527.0, 17802.0, 16624.0, 16717.0, 17205.0, 18488.0, 17406.0, 20440.0, 17521.0, 16764.0, 16833.0, 16907.0, 18123.0, 18662.0, 16655.0, 17265.0, 16728.0, 16503.0, 17086.0, 17315.0, 19844.0, 18041.0, 18274.0, 18804.0, 18807.0, 18595.0, 20100.0, 20061.0, 18383.0, 19361.0, 18146.0, 17769.0, 18193.0, 20190.0, 20478.0, 18637.0, 18127.0, 17890.0, 17489.0, 17472.0, 19838.0, 19844.0, 18951.0, 12387.0, 10.0], 'test_count': [22425.0, 22745.0, 22420.0, 22328.0, 20010.0, 21339.0, 22120.0, 22882.0, 22414.0, 22950.0, 22986.0, 24165.0, 23967.0, 24705.0, 25263.0, 24832.0, 25110.0, 23020.0, 22791.0, 23533.0, 23849.0, 24360.0, 24099.0, 23995.0, 23301.0, 22823.0, 22278.0, 23391.0, 23413.0, 24036.0, 23008.0, 22938.0, 23176.0, 21855.0, 22366.0, 23452.0, 22888.0, 20528.0, 19040.0, 21863.0, 21797.0, 22509.0, 23310.0, 23512.0, 24356.0, 26009.0, 24697.0, 24397.0, 26235.0, 26630.0, 26122.0, 25736.0, 26346.0, 25398.0, 24308.0, 25543.0, 26757.0, 25768.0, 24483.0, 22927.0, 22395.0, 23771.0, 25612.0, 25251.0, 26937.0, 24498.0, 23767.0, 22381.0, 22672.0, 25626.0, 25442.0, 23036.0, 22935.0, 22644.0, 22789.0, 23338.0, 24970.0, 25200.0, 24796.0, 24346.0, 25458.0, 25854.0, 25613.0, 28588.0, 27790.0, 25438.0, 27040.0, 26904.0, 25410.0, 25812.0, 28257.0, 29377.0, 26237.0, 24087.0, 25555.0, 24913.0, 25284.0, 28119.0, 27515.0, 27110.0, 16442.0, 10.0]}, 'up': {'date': [1341100800000000.0, 1341187200000000.0, 1341273600000000.0, 1341360000000000.0, 1341446400000000.0, 1341532800000000.0, 1341619200000000.0, 1341705600000000.0, 1341792000000000.0, 1341878400000000.0, 1341964800000000.0, 1342051200000000.0, 1342137600000000.0, 1342224000000000.0, 1342310400000000.0, 1342396800000000.0, 1342483200000000.0, 1342569600000000.0, 1342656000000000.0, 1342742400000000.0, 1342828800000000.0, 1342915200000000.0, 1343001600000000.0, 1343088000000000.0, 1343174400000000.0, 1343260800000000.0, 1343347200000000.0, 1343433600000000.0, 1343520000000000.0, 1343606400000000.0, 1343692800000000.0, 1343779200000000.0, 1343865600000000.0, 1343952000000000.0, 1344038400000000.0, 1344124800000000.0, 1344211200000000.0, 1344297600000000.0, 1344384000000000.0, 1344470400000000.0, 1344556800000000.0, 1344643200000000.0, 1344729600000000.0, 1344816000000000.0, 1344902400000000.0, 1344988800000000.0, 1345075200000000.0, 1345161600000000.0, 1345248000000000.0, 1345334400000000.0, 1345420800000000.0, 1345507200000000.0, 1345593600000000.0, 1345680000000000.0, 1345766400000000.0, 1345852800000000.0, 1345939200000000.0, 1346025600000000.0, 1346112000000000.0, 1346198400000000.0, 1346284800000000.0, 1346371200000000.0, 1346457600000000.0, 1346544000000000.0, 1346630400000000.0, 1346716800000000.0, 1346803200000000.0, 1346889600000000.0, 1346976000000000.0, 1347062400000000.0, 1347148800000000.0, 1347235200000000.0, 1347321600000000.0, 1347408000000000.0, 1347494400000000.0, 1347580800000000.0, 1347667200000000.0, 1347753600000000.0, 1347840000000000.0, 1347926400000000.0, 1348012800000000.0, 1348099200000000.0, 1348185600000000.0, 1348272000000000.0, 1348358400000000.0, 1348444800000000.0, 1348531200000000.0, 1348617600000000.0, 1348704000000000.0, 1348790400000000.0, 1348876800000000.0, 1348963200000000.0, 1349049600000000.0, 1349136000000000.0, 1349222400000000.0, 1349308800000000.0, 1349395200000000.0, 1349481600000000.0, 1349568000000000.0, 1349654400000000.0, 1349740800000000.0, 1349827200000000.0], 'direction': [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], 'total_bytes': [110460682667.0, 117208037621.0, 114502189403.0, 113746059546.0, 105125944654.0, 111542510039.0, 112026667920.0, 112718732904.0, 116198636682.0, 116976345738.0, 113176102744.0, 119523696571.0, 119735112933.0, 120392189513.0, 121856656563.0, 121026847395.0, 125998643557.0, 124680159442.0, 120828833805.0, 123848088974.0, 118396383582.0, 117766815564.0, 119036160522.0, 118227161532.0, 116505121018.0, 121357642092.0, 114742528387.0, 115713224395.0, 115838405116.0, 120646239255.0, 118676092491.0, 122385328383.0, 122020963183.0, 111865081546.0, 117497966077.0, 120765110652.0, 126310972093.0, 114732108775.0, 106430999887.0, 126676871360.0, 120116846576.0, 131185506454.0, 125115818902.0, 133387560817.0, 136973025011.0, 135047489806.0, 130390887004.0, 126276809086.0, 133922473060.0, 123375659068.0, 137084492048.0, 137307095610.0, 145710132084.0, 139225451536.0, 129960373887.0, 132495924990.0, 140641603084.0, 142864223071.0, 138506250494.0, 131253784749.0, 124456460808.0, 139285136980.0, 135578128058.0, 137597645707.0, 143019743719.0, 142377726557.0, 138684611050.0, 130573886049.0, 136543192584.0, 137749201514.0, 131849628588.0, 133174643464.0, 133601410534.0, 131072144716.0, 132807304863.0, 140708905774.0, 135344520090.0, 135762253836.0, 144058983128.0, 148263016429.0, 157841001700.0, 144188943587.0, 144887995523.0, 150804485298.0, 142407210199.0, 139070082271.0, 146145463251.0, 146375418308.0, 139587072815.0, 140195311055.0, 140049684443.0, 143548618484.0, 144919808778.0, 120675896882.0, 136522269919.0, 135058992173.0, 135282921208.0, 135953239743.0, 129823283320.0, 128611312057.0, 89082127103.0, 5981978.0], 'unique_clients': [27938.0, 29163.0, 27320.0, 28873.0, 24167.0, 25998.0, 27686.0, 28101.0, 26068.0, 26831.0, 27412.0, 29831.0, 31019.0, 31995.0, 30168.0, 28717.0, 30043.0, 28574.0, 27866.0, 31138.0, 28449.0, 30206.0, 28362.0, 29218.0, 26772.0, 29108.0, 28762.0, 31374.0, 30322.0, 29158.0, 27241.0, 27293.0, 27873.0, 27224.0, 28782.0, 28895.0, 28174.0, 25706.0, 23830.0, 26642.0, 27292.0, 28151.0, 28904.0, 28005.0, 28210.0, 32265.0, 30951.0, 28771.0, 32083.0, 32702.0, 31447.0, 32548.0, 32099.0, 30688.0, 30513.0, 30601.0, 33399.0, 33516.0, 29597.0, 28541.0, 29788.0, 30861.0, 34605.0, 31243.0, 33731.0, 30757.0, 28443.0, 29072.0, 28570.0, 33314.0, 30663.0, 29194.0, 29407.0, 28985.0, 29084.0, 29437.0, 31037.0, 32518.0, 29165.0, 32417.0, 30878.0, 32175.0, 33397.0, 34761.0, 32932.0, 30072.0, 31919.0, 29831.0, 28499.0, 30062.0, 32690.0, 35030.0, 31549.0, 30301.0, 30652.0, 28577.0, 28554.0, 34319.0, 32184.0, 32105.0, 20680.0, 6.0], 'test_count': [43223.0, 43777.0, 42535.0, 43070.0, 38834.0, 40647.0, 43326.0, 44237.0, 42334.0, 43339.0, 43090.0, 46390.0, 46159.0, 47847.0, 49006.0, 46546.0, 47629.0, 44050.0, 43242.0, 45188.0, 47368.0, 47883.0, 46603.0, 46067.0, 45309.0, 44295.0, 43161.0, 46128.0, 46387.0, 45752.0, 44829.0, 43821.0, 44475.0, 41994.0, 43297.0, 45984.0, 42974.0, 38880.0, 36832.0, 41574.0, 41623.0, 43696.0, 44960.0, 44407.0, 46365.0, 47899.0, 46304.0, 45858.0, 50511.0, 50501.0, 49317.0, 49019.0, 52896.0, 51820.0, 49333.0, 53196.0, 54625.0, 52271.0, 49846.0, 46735.0, 45696.0, 48819.0, 52672.0, 52145.0, 52810.0, 48762.0, 47898.0, 45777.0, 47349.0, 52379.0, 51899.0, 46618.0, 46915.0, 46143.0, 45961.0, 47371.0, 51107.0, 51901.0, 49248.0, 49254.0, 50656.0, 50911.0, 50924.0, 57046.0, 53958.0, 46665.0, 49835.0, 49801.0, 47012.0, 48681.0, 53336.0, 55148.0, 48986.0, 45715.0, 47982.0, 46759.0, 47763.0, 53312.0, 52526.0, 50151.0, 31089.0, 6.0]}}}

















In [72]:



    


f=figure(figsize=(12,6))           
grid()

ylim(ymin=0,ymax=300)
xmin = datetime.datetime.strptime("2012-07-01", "%Y-%m-%d") 
xmax = datetime.datetime.strptime("2012-10-01", "%Y-%m-%d") 
xlim(xmin=xmin,xmax=xmax)

#title("%s) Package %s, %s samples from %s, %2.1f%% pass" % (i, 
#                                                        package_list[affected_host][package_id], 
#                                                        total_samples, affected_host, percent_pass))
#ylabel("y-label") # this depends on the data set
            
#                # Plot raw-good, pass-bad, fail-bad, and average-good
title("NDT Data Uploaded or Downloaded per Day in USA and Greece")
ylabel("Data Transferred (Gigabytes)", fontsize="large")
plot_date(matplotlib.dates.epoch2num([ x/1e6 for x in ret['us']['up']['date'] ]),
          [ x/1e9 for x in ret['us']['up']['total_bytes'] ],
          xdate=True, ydate=False, 
          color='red', mec='white', mew=1.5,
          linestyle='-', linewidth=3,markevery=2,
          marker='^', figure=f, markersize=10, label="USA upload")
plot_date(matplotlib.dates.epoch2num([ x/1e6 for x in ret['us']['down']['date'] ]),
          [ x/1e9 for x in ret['us']['down']['total_bytes'] ],
          xdate=True, ydate=False, 
          color='blue', mec='white', mew=1.5,
          linestyle='-', linewidth=3,markevery=2,
          marker='v', figure=f, markersize=10, label="USA download")

plot_date(matplotlib.dates.epoch2num([ x/1e6 for x in ret['gr']['up']['date'] ]),
          [ x/1e9 for x in ret['gr']['up']['total_bytes'] ],
          xdate=True, ydate=False, 
          color='navy', mec='navy',
          linestyle='-', linewidth=1,markevery=2,
          marker='^', figure=f, markersize=8, label="Greece upload")
plot_date(matplotlib.dates.epoch2num([ x/1e6 for x in ret['gr']['down']['date'] ]),
          [ x/1e9 for x in ret['gr']['down']['total_bytes'] ],
          xdate=True, ydate=False, 
          color='orange', mec='orange',
          linestyle='-', linewidth=1, markevery=2,
          marker='v', figure=f, markersize=8, label="Greece download")
legend(ncol=4, loc=2)



    


















    
Out[72]:








<matplotlib.legend.Legend at 0xe845d2c>










    
























In [73]:



    


f=figure(figsize=(12,6))           
grid()

#ylim(ymin=0,ymax=300)
xmin = datetime.datetime.strptime("2012-07-01", "%Y-%m-%d") 
xmax = datetime.datetime.strptime("2012-10-01", "%Y-%m-%d") 
xlim(xmin=xmin,xmax=xmax)

#title("%s) Package %s, %s samples from %s, %2.1f%% pass" % (i, 
#                                                        package_list[affected_host][package_id], 
#                                                        total_samples, affected_host, percent_pass))
#ylabel("y-label") # this depends on the data set
            
#                # Plot raw-good, pass-bad, fail-bad, and average-good
title("NDT Tests per Day in USA and Greece")
ylabel("Tests per Day (Thousands)", fontsize="large")
plot_date(matplotlib.dates.epoch2num([ x/1e6 for x in ret['us']['up']['date'] ]),
          [ x/1e3 for x in ret['us']['up']['test_count'] ],
          xdate=True, ydate=False, 
          color='red', mec='white', mew=1.5,
          linestyle='-', linewidth=3,markevery=2,
          marker='^', figure=f, markersize=10, label="USA upload")
plot_date(matplotlib.dates.epoch2num([ x/1e6 for x in ret['us']['down']['date'] ]),
          [ x/1e3 for x in ret['us']['down']['test_count'] ],
          xdate=True, ydate=False, 
          color='blue', mec='white', mew=1.5,
          linestyle='-', linewidth=3,markevery=2,
          marker='v', figure=f, markersize=10, label="USA download")

plot_date(matplotlib.dates.epoch2num([ x/1e6 for x in ret['gr']['up']['date'] ]),
          [ x/1e3 for x in ret['gr']['up']['test_count'] ],
          xdate=True, ydate=False, 
          color='navy', mec='navy',
          linestyle='-', linewidth=1,markevery=2,
          marker='^', figure=f, markersize=8, label="Greece upload")
plot_date(matplotlib.dates.epoch2num([ x/1e6 for x in ret['gr']['down']['date'] ]),
          [ x/1e3 for x in ret['gr']['down']['test_count'] ],
          xdate=True, ydate=False, 
          color='orange', mec='orange',
          linestyle='-', linewidth=1, markevery=2,
          marker='v', figure=f, markersize=8, label="Greece download")
legend(ncol=4, loc=2)



    


















    
Out[73]:








<matplotlib.legend.Legend at 0xe97884c>










    
























In [ ]:

Content source: m-lab/analysis

Similar notebooks:

notebook.community | gallery | about