notebook.community

Edit and run

moving the work to jupyter

This notebook demonstrates the use of single-line calls to D3 visualizations via the simple d3_lib.py file and referenced css and js files.

Configuration for D3





In [13]:



    


%matplotlib inline
import pandas as pd
from korg import korg
from korg.pattern import PatternRepo
import tarfile

from loganalyser import plot

Aogaeru load





In [14]:



    


pr = PatternRepo(['./korg_patterns/'], False)
lg = korg.LineGrokker('%{AOGAERU_LOAD}', pr)

df = pd.DataFrame()

# now grok the aogaeru load log
with open("./data/results.csv") as infile:
    for line in infile:
        #print line
        #print lg.grok(line)
        df = df.append(lg.grok(line), ignore_index=True)
df.timestamp = pd.to_datetime(df.timestamp)  # convert to timestamp
df.head()



    


















    
Out[14]:










  
    

      
      elapsed
      iteration
      name
      status
      step
      timestamp
      user
    

  
  
    

      0
      None
      00000
      testcase01
      start
      user
      2013-10-29 10:47:55.528
      0001
    

    

      1
      None
      00001
      testcase01
      start
      iteration
      2013-10-29 10:47:55.978
      0001
    

    

      2
      None
      00001
      testcase01
      start
      start_page
      2013-10-29 10:47:55.979
      0001
    

    

      3
      0.665
      00001
      testcase01
      end
      start_page
      2013-10-29 10:47:56.644
      0001
    

    

      4
      None
      00000
      testcase02
      start
      user
      2013-10-29 10:47:57.532
      0001
    

  




















In [15]:



    


df.dtypes



    


















    
Out[15]:








elapsed              object
iteration            object
name                 object
status               object
step                 object
timestamp    datetime64[ns]
user                 object
dtype: object

















In [ ]:

LINUX_VMSTAT





In [16]:



    


pr = PatternRepo(['./korg_patterns/'], False)
lg = korg.LineGrokker('%{LINUX_VMSTAT}', pr)

vm = pd.DataFrame()

with tarfile.open('data/supercars-oscounters-13102912.tgz', mode="r:gz") as tar:
    #infile = tar.extractfile('./vmstat.txt.%s' % testrun)
    infile = tar.extractfile('./vmstat.txt.13102912')
    # read first line start and interval
    # 20131029 124500 interval 5 sec
    for line in infile:
        #print line
        #print lg.grok(line)
        vm = vm.append(lg.grok(line), ignore_index=True)

#vm.timestamp = pd.to_datetime(vm.timestamp)  # convert to timestamp
# tz is timezone of data
rng = pd.date_range('10/29/2013 12:45:00', periods=400, freq='5S', tz="Europe/Berlin")
#rng = pd.date_range('10/29/2013 12:45:00', periods=400, freq='5S')
cpu = pd.DataFrame()
cpu['idle'] = vm['id'].astype(int)
cpu['system'] = vm['sy'].astype(int)
cpu['user'] = vm['us'].astype(int)
cpu['wait'] = vm['wa'].astype(int)
# we do not use index since we "to_json" next
#cpu['timestamp'] = rng
#cpu.set_index(['timestamp'], inplace=True)
#cpu.index = pd.to_datetime(rng, utc=True)
cpu['timestamp'] = pd.to_datetime(rng, utc=True) # timestamp in UTC !
cpu.head()



    


















    
Out[16]:










  
    

      
      idle
      system
      user
      wait
      timestamp
    

  
  
    

      0
      89
      2
      3
      6
      2013-10-29 11:45:00
    

    

      1
      99
      1
      0
      0
      2013-10-29 11:45:05
    

    

      2
      99
      0
      0
      1
      2013-10-29 11:45:10
    

    

      3
      100
      0
      0
      0
      2013-10-29 11:45:15
    

    

      4
      100
      0
      0
      0
      2013-10-29 11:45:20
    

  




















In [17]:



    


cpu.dtypes



    


















    
Out[17]:








idle                  int64
system                int64
user                  int64
wait                  int64
timestamp    datetime64[ns]
dtype: object

convert the dataframe data into Json format to_csv does not work since it is necessary to parse it in d3 where it ends up as json format anyways





In [18]:



    


#print  cpu.to_csv(orient='records')
print cpu.to_json(orient='records', date_format='epoch')



    


















    






[{"idle":89,"system":2,"user":3,"wait":6,"timestamp":1383047100000},{"idle":99,"system":1,"user":0,"wait":0,"timestamp":1383047105000},{"idle":99,"system":0,"user":0,"wait":1,"timestamp":1383047110000},{"idle":100,"system":0,"user":0,"wait":0,"timestamp":1383047115000},{"idle":100,"system":0,"user":0,"wait":0,"timestamp":1383047120000},{"idle":100,"system":0,"user":0,"wait":0,"timestamp":1383047125000},{"idle":100,"system":0,"user":0,"wait":0,"timestamp":1383047130000},{"idle":100,"system":0,"user":0,"wait":0,"timestamp":1383047135000},{"idle":100,"system":0,"user":0,"wait":0,"timestamp":1383047140000},{"idle":100,"system":0,"user":0,"wait":0,"timestamp":1383047145000},{"idle":100,"system":0,"user":0,"wait":0,"timestamp":1383047150000},{"idle":100,"system":0,"user":0,"wait":0,"timestamp":1383047155000},{"idle":100,"system":0,"user":0,"wait":0,"timestamp":1383047160000},{"idle":100,"system":0,"user":0,"wait":0,"timestamp":1383047165000},{"idle":100,"system":0,"user":0,"wait":0,"timestamp":1383047170000},{"idle":100,"system":0,"user":0,"wait":0,"timestamp":1383047175000},{"idle":100,"system":0,"user":0,"wait":0,"timestamp":1383047180000},{"idle":100,"system":0,"user":0,"wait":0,"timestamp":1383047185000},{"idle":100,"system":0,"user":0,"wait":0,"timestamp":1383047190000},{"idle":100,"system":0,"user":0,"wait":0,"timestamp":1383047195000},{"idle":100,"system":0,"user":0,"wait":0,"timestamp":1383047200000},{"idle":100,"system":0,"user":0,"wait":0,"timestamp":1383047205000},{"idle":100,"system":0,"user":0,"wait":0,"timestamp":1383047210000},{"idle":100,"system":0,"user":0,"wait":0,"timestamp":1383047215000},{"idle":100,"system":0,"user":0,"wait":0,"timestamp":1383047220000},{"idle":100,"system":0,"user":0,"wait":0,"timestamp":1383047225000},{"idle":100,"system":0,"user":0,"wait":0,"timestamp":1383047230000},{"idle":100,"system":0,"user":0,"wait":0,"timestamp":1383047235000},{"idle":100,"system":0,"user":0,"wait":0,"timestamp":1383047240000},{"idle":100,"system":0,"user":0,"wait":0,"timestamp":1383047245000},{"idle":100,"system":0,"user":0,"wait":0,"timestamp":1383047250000},{"idle":100,"system":0,"user":0,"wait":0,"timestamp":1383047255000},{"idle":100,"system":0,"user":0,"wait":0,"timestamp":1383047260000},{"idle":100,"system":0,"user":0,"wait":0,"timestamp":1383047265000},{"idle":100,"system":0,"user":0,"wait":0,"timestamp":1383047270000},{"idle":100,"system":0,"user":0,"wait":0,"timestamp":1383047275000},{"idle":99,"system":1,"user":0,"wait":0,"timestamp":1383047280000},{"idle":99,"system":0,"user":0,"wait":0,"timestamp":1383047285000},{"idle":99,"system":0,"user":0,"wait":0,"timestamp":1383047290000},{"idle":99,"system":0,"user":0,"wait":0,"timestamp":1383047295000},{"idle":99,"system":0,"user":1,"wait":0,"timestamp":1383047300000},{"idle":99,"system":0,"user":1,"wait":0,"timestamp":1383047305000},{"idle":100,"system":0,"user":0,"wait":0,"timestamp":1383047310000},{"idle":99,"system":0,"user":1,"wait":0,"timestamp":1383047315000},{"idle":99,"system":0,"user":0,"wait":0,"timestamp":1383047320000},{"idle":99,"system":0,"user":0,"wait":0,"timestamp":1383047325000},{"idle":99,"system":0,"user":1,"wait":0,"timestamp":1383047330000},{"idle":99,"system":0,"user":0,"wait":0,"timestamp":1383047335000},{"idle":98,"system":1,"user":1,"wait":0,"timestamp":1383047340000},{"idle":99,"system":0,"user":1,"wait":0,"timestamp":1383047345000},{"idle":99,"system":0,"user":0,"wait":0,"timestamp":1383047350000},{"idle":99,"system":0,"user":0,"wait":0,"timestamp":1383047355000},{"idle":99,"system":0,"user":0,"wait":0,"timestamp":1383047360000},{"idle":100,"system":0,"user":0,"wait":0,"timestamp":1383047365000},{"idle":99,"system":1,"user":0,"wait":0,"timestamp":1383047370000},{"idle":99,"system":0,"user":1,"wait":0,"timestamp":1383047375000},{"idle":99,"system":0,"user":0,"wait":0,"timestamp":1383047380000},{"idle":100,"system":0,"user":0,"wait":0,"timestamp":1383047385000},{"idle":99,"system":0,"user":0,"wait":0,"timestamp":1383047390000},{"idle":99,"system":0,"user":1,"wait":0,"timestamp":1383047395000},{"idle":99,"system":1,"user":1,"wait":0,"timestamp":1383047400000},{"idle":99,"system":0,"user":0,"wait":0,"timestamp":1383047405000},{"idle":99,"system":0,"user":0,"wait":0,"timestamp":1383047410000},{"idle":99,"system":0,"user":0,"wait":0,"timestamp":1383047415000},{"idle":99,"system":0,"user":0,"wait":0,"timestamp":1383047420000},{"idle":99,"system":0,"user":0,"wait":1,"timestamp":1383047425000},{"idle":99,"system":0,"user":0,"wait":0,"timestamp":1383047430000},{"idle":99,"system":0,"user":1,"wait":0,"timestamp":1383047435000},{"idle":98,"system":0,"user":1,"wait":1,"timestamp":1383047440000},{"idle":99,"system":0,"user":1,"wait":0,"timestamp":1383047445000},{"idle":99,"system":0,"user":1,"wait":1,"timestamp":1383047450000},{"idle":99,"system":0,"user":1,"wait":0,"timestamp":1383047455000},{"idle":99,"system":0,"user":1,"wait":0,"timestamp":1383047460000},{"idle":99,"system":0,"user":1,"wait":0,"timestamp":1383047465000},{"idle":99,"system":0,"user":1,"wait":0,"timestamp":1383047470000},{"idle":99,"system":1,"user":0,"wait":0,"timestamp":1383047475000},{"idle":100,"system":0,"user":0,"wait":0,"timestamp":1383047480000},{"idle":99,"system":0,"user":1,"wait":0,"timestamp":1383047485000},{"idle":100,"system":0,"user":0,"wait":0,"timestamp":1383047490000},{"idle":99,"system":0,"user":0,"wait":0,"timestamp":1383047495000},{"idle":99,"system":0,"user":1,"wait":0,"timestamp":1383047500000},{"idle":99,"system":0,"user":0,"wait":0,"timestamp":1383047505000},{"idle":99,"system":0,"user":0,"wait":0,"timestamp":1383047510000},{"idle":99,"system":0,"user":1,"wait":0,"timestamp":1383047515000},{"idle":99,"system":0,"user":0,"wait":0,"timestamp":1383047520000},{"idle":99,"system":0,"user":0,"wait":0,"timestamp":1383047525000},{"idle":100,"system":0,"user":0,"wait":0,"timestamp":1383047530000},{"idle":98,"system":0,"user":1,"wait":0,"timestamp":1383047535000},{"idle":99,"system":0,"user":0,"wait":0,"timestamp":1383047540000},{"idle":99,"system":0,"user":1,"wait":0,"timestamp":1383047545000},{"idle":99,"system":0,"user":1,"wait":0,"timestamp":1383047550000},{"idle":99,"system":0,"user":0,"wait":0,"timestamp":1383047555000},{"idle":98,"system":1,"user":1,"wait":0,"timestamp":1383047560000},{"idle":100,"system":0,"user":0,"wait":0,"timestamp":1383047565000},{"idle":99,"system":0,"user":1,"wait":0,"timestamp":1383047570000},{"idle":99,"system":0,"user":0,"wait":0,"timestamp":1383047575000},{"idle":99,"system":0,"user":1,"wait":0,"timestamp":1383047580000},{"idle":98,"system":1,"user":1,"wait":0,"timestamp":1383047585000},{"idle":99,"system":0,"user":1,"wait":0,"timestamp":1383047590000},{"idle":98,"system":0,"user":1,"wait":0,"timestamp":1383047595000},{"idle":99,"system":0,"user":1,"wait":0,"timestamp":1383047600000},{"idle":99,"system":0,"user":1,"wait":0,"timestamp":1383047605000},{"idle":98,"system":0,"user":1,"wait":0,"timestamp":1383047610000},{"idle":98,"system":0,"user":1,"wait":0,"timestamp":1383047615000},{"idle":98,"system":1,"user":1,"wait":0,"timestamp":1383047620000},{"idle":99,"system":0,"user":1,"wait":0,"timestamp":1383047625000},{"idle":98,"system":0,"user":2,"wait":0,"timestamp":1383047630000},{"idle":99,"system":0,"user":1,"wait":0,"timestamp":1383047635000},{"idle":98,"system":0,"user":1,"wait":0,"timestamp":1383047640000},{"idle":98,"system":0,"user":1,"wait":0,"timestamp":1383047645000},{"idle":99,"system":0,"user":0,"wait":0,"timestamp":1383047650000},{"idle":99,"system":0,"user":1,"wait":0,"timestamp":1383047655000},{"idle":99,"system":0,"user":0,"wait":0,"timestamp":1383047660000},{"idle":99,"system":1,"user":0,"wait":0,"timestamp":1383047665000},{"idle":98,"system":0,"user":1,"wait":0,"timestamp":1383047670000},{"idle":99,"system":0,"user":1,"wait":0,"timestamp":1383047675000},{"idle":98,"system":0,"user":1,"wait":0,"timestamp":1383047680000},{"idle":99,"system":0,"user":1,"wait":0,"timestamp":1383047685000},{"idle":99,"system":0,"user":1,"wait":0,"timestamp":1383047690000},{"idle":99,"system":0,"user":1,"wait":0,"timestamp":1383047695000},{"idle":98,"system":0,"user":1,"wait":0,"timestamp":1383047700000},{"idle":97,"system":1,"user":2,"wait":1,"timestamp":1383047705000},{"idle":99,"system":0,"user":1,"wait":0,"timestamp":1383047710000},{"idle":98,"system":0,"user":1,"wait":0,"timestamp":1383047715000},{"idle":99,"system":0,"user":0,"wait":0,"timestamp":1383047720000},{"idle":98,"system":1,"user":1,"wait":0,"timestamp":1383047725000},{"idle":99,"system":0,"user":1,"wait":0,"timestamp":1383047730000},{"idle":99,"system":0,"user":1,"wait":0,"timestamp":1383047735000},{"idle":98,"system":0,"user":1,"wait":0,"timestamp":1383047740000},{"idle":99,"system":0,"user":1,"wait":0,"timestamp":1383047745000},{"idle":98,"system":0,"user":2,"wait":0,"timestamp":1383047750000},{"idle":99,"system":0,"user":1,"wait":0,"timestamp":1383047755000},{"idle":98,"system":1,"user":1,"wait":0,"timestamp":1383047760000},{"idle":97,"system":0,"user":2,"wait":0,"timestamp":1383047765000},{"idle":99,"system":0,"user":1,"wait":0,"timestamp":1383047770000},{"idle":98,"system":0,"user":2,"wait":0,"timestamp":1383047775000},{"idle":99,"system":0,"user":0,"wait":0,"timestamp":1383047780000},{"idle":97,"system":1,"user":1,"wait":1,"timestamp":1383047785000},{"idle":98,"system":0,"user":1,"wait":0,"timestamp":1383047790000},{"idle":99,"system":0,"user":1,"wait":0,"timestamp":1383047795000},{"idle":98,"system":0,"user":1,"wait":0,"timestamp":1383047800000},{"idle":99,"system":0,"user":0,"wait":0,"timestamp":1383047805000},{"idle":98,"system":1,"user":1,"wait":0,"timestamp":1383047810000},{"idle":98,"system":0,"user":1,"wait":0,"timestamp":1383047815000},{"idle":98,"system":0,"user":1,"wait":0,"timestamp":1383047820000},{"idle":99,"system":1,"user":0,"wait":0,"timestamp":1383047825000},{"idle":98,"system":1,"user":1,"wait":1,"timestamp":1383047830000},{"idle":98,"system":0,"user":2,"wait":0,"timestamp":1383047835000},{"idle":99,"system":0,"user":1,"wait":0,"timestamp":1383047840000},{"idle":99,"system":1,"user":1,"wait":0,"timestamp":1383047845000},{"idle":98,"system":0,"user":1,"wait":0,"timestamp":1383047850000},{"idle":99,"system":0,"user":1,"wait":0,"timestamp":1383047855000},{"idle":97,"system":1,"user":2,"wait":0,"timestamp":1383047860000},{"idle":99,"system":0,"user":1,"wait":0,"timestamp":1383047865000},{"idle":98,"system":0,"user":1,"wait":0,"timestamp":1383047870000},{"idle":98,"system":1,"user":1,"wait":0,"timestamp":1383047875000},{"idle":99,"system":0,"user":1,"wait":0,"timestamp":1383047880000},{"idle":99,"system":0,"user":1,"wait":0,"timestamp":1383047885000},{"idle":99,"system":0,"user":1,"wait":0,"timestamp":1383047890000},{"idle":98,"system":1,"user":1,"wait":0,"timestamp":1383047895000},{"idle":99,"system":0,"user":1,"wait":0,"timestamp":1383047900000},{"idle":98,"system":0,"user":1,"wait":1,"timestamp":1383047905000},{"idle":98,"system":0,"user":1,"wait":0,"timestamp":1383047910000},{"idle":99,"system":0,"user":1,"wait":0,"timestamp":1383047915000},{"idle":98,"system":0,"user":1,"wait":0,"timestamp":1383047920000},{"idle":99,"system":1,"user":1,"wait":0,"timestamp":1383047925000},{"idle":99,"system":0,"user":1,"wait":0,"timestamp":1383047930000},{"idle":99,"system":0,"user":1,"wait":0,"timestamp":1383047935000},{"idle":100,"system":0,"user":0,"wait":0,"timestamp":1383047940000},{"idle":98,"system":1,"user":1,"wait":0,"timestamp":1383047945000},{"idle":99,"system":0,"user":1,"wait":0,"timestamp":1383047950000},{"idle":98,"system":0,"user":1,"wait":0,"timestamp":1383047955000},{"idle":99,"system":0,"user":1,"wait":0,"timestamp":1383047960000},{"idle":99,"system":1,"user":1,"wait":0,"timestamp":1383047965000},{"idle":98,"system":0,"user":1,"wait":0,"timestamp":1383047970000},{"idle":98,"system":0,"user":1,"wait":0,"timestamp":1383047975000},{"idle":99,"system":0,"user":1,"wait":0,"timestamp":1383047980000},{"idle":99,"system":0,"user":1,"wait":0,"timestamp":1383047985000},{"idle":98,"system":0,"user":1,"wait":0,"timestamp":1383047990000},{"idle":99,"system":1,"user":1,"wait":0,"timestamp":1383047995000},{"idle":99,"system":0,"user":1,"wait":0,"timestamp":1383048000000},{"idle":98,"system":0,"user":1,"wait":0,"timestamp":1383048005000},{"idle":99,"system":0,"user":1,"wait":0,"timestamp":1383048010000},{"idle":98,"system":1,"user":1,"wait":0,"timestamp":1383048015000},{"idle":98,"system":1,"user":1,"wait":0,"timestamp":1383048020000},{"idle":99,"system":0,"user":1,"wait":0,"timestamp":1383048025000},{"idle":98,"system":1,"user":1,"wait":0,"timestamp":1383048030000},{"idle":99,"system":0,"user":1,"wait":0,"timestamp":1383048035000},{"idle":98,"system":0,"user":1,"wait":0,"timestamp":1383048040000},{"idle":99,"system":0,"user":1,"wait":0,"timestamp":1383048045000},{"idle":99,"system":0,"user":0,"wait":0,"timestamp":1383048050000},{"idle":98,"system":0,"user":1,"wait":0,"timestamp":1383048055000},{"idle":99,"system":0,"user":0,"wait":0,"timestamp":1383048060000},{"idle":98,"system":0,"user":1,"wait":0,"timestamp":1383048065000},{"idle":99,"system":0,"user":1,"wait":0,"timestamp":1383048070000},{"idle":99,"system":0,"user":1,"wait":0,"timestamp":1383048075000},{"idle":98,"system":1,"user":1,"wait":0,"timestamp":1383048080000},{"idle":99,"system":0,"user":0,"wait":0,"timestamp":1383048085000},{"idle":98,"system":1,"user":1,"wait":0,"timestamp":1383048090000},{"idle":99,"system":1,"user":1,"wait":0,"timestamp":1383048095000},{"idle":98,"system":1,"user":1,"wait":0,"timestamp":1383048100000},{"idle":99,"system":1,"user":1,"wait":0,"timestamp":1383048105000},{"idle":99,"system":0,"user":1,"wait":0,"timestamp":1383048110000},{"idle":98,"system":1,"user":1,"wait":0,"timestamp":1383048115000},{"idle":99,"system":0,"user":1,"wait":0,"timestamp":1383048120000},{"idle":98,"system":1,"user":1,"wait":0,"timestamp":1383048125000},{"idle":98,"system":1,"user":1,"wait":0,"timestamp":1383048130000},{"idle":98,"system":0,"user":1,"wait":0,"timestamp":1383048135000},{"idle":98,"system":1,"user":2,"wait":0,"timestamp":1383048140000},{"idle":99,"system":1,"user":1,"wait":0,"timestamp":1383048145000},{"idle":97,"system":1,"user":2,"wait":0,"timestamp":1383048150000},{"idle":98,"system":0,"user":1,"wait":0,"timestamp":1383048155000},{"idle":98,"system":1,"user":1,"wait":0,"timestamp":1383048160000},{"idle":98,"system":1,"user":1,"wait":0,"timestamp":1383048165000},{"idle":99,"system":0,"user":1,"wait":0,"timestamp":1383048170000},{"idle":97,"system":1,"user":2,"wait":0,"timestamp":1383048175000},{"idle":99,"system":0,"user":1,"wait":0,"timestamp":1383048180000},{"idle":98,"system":0,"user":1,"wait":0,"timestamp":1383048185000},{"idle":98,"system":0,"user":1,"wait":0,"timestamp":1383048190000},{"idle":98,"system":0,"user":1,"wait":0,"timestamp":1383048195000},{"idle":98,"system":1,"user":1,"wait":0,"timestamp":1383048200000},{"idle":99,"system":0,"user":1,"wait":0,"timestamp":1383048205000},{"idle":98,"system":0,"user":1,"wait":0,"timestamp":1383048210000},{"idle":99,"system":0,"user":1,"wait":0,"timestamp":1383048215000},{"idle":98,"system":0,"user":1,"wait":0,"timestamp":1383048220000},{"idle":98,"system":0,"user":1,"wait":0,"timestamp":1383048225000},{"idle":98,"system":1,"user":1,"wait":0,"timestamp":1383048230000},{"idle":98,"system":0,"user":1,"wait":0,"timestamp":1383048235000},{"idle":99,"system":0,"user":1,"wait":0,"timestamp":1383048240000},{"idle":98,"system":1,"user":1,"wait":0,"timestamp":1383048245000},{"idle":99,"system":0,"user":1,"wait":0,"timestamp":1383048250000},{"idle":98,"system":1,"user":1,"wait":0,"timestamp":1383048255000},{"idle":98,"system":0,"user":1,"wait":0,"timestamp":1383048260000},{"idle":99,"system":0,"user":1,"wait":0,"timestamp":1383048265000},{"idle":98,"system":0,"user":2,"wait":0,"timestamp":1383048270000},{"idle":99,"system":0,"user":1,"wait":0,"timestamp":1383048275000},{"idle":98,"system":0,"user":2,"wait":0,"timestamp":1383048280000},{"idle":98,"system":1,"user":1,"wait":0,"timestamp":1383048285000},{"idle":98,"system":0,"user":1,"wait":1,"timestamp":1383048290000},{"idle":98,"system":1,"user":1,"wait":0,"timestamp":1383048295000},{"idle":99,"system":0,"user":1,"wait":0,"timestamp":1383048300000},{"idle":98,"system":1,"user":2,"wait":0,"timestamp":1383048305000},{"idle":98,"system":1,"user":1,"wait":0,"timestamp":1383048310000},{"idle":98,"system":1,"user":1,"wait":0,"timestamp":1383048315000},{"idle":98,"system":0,"user":2,"wait":0,"timestamp":1383048320000},{"idle":99,"system":0,"user":1,"wait":0,"timestamp":1383048325000},{"idle":97,"system":1,"user":1,"wait":1,"timestamp":1383048330000},{"idle":98,"system":1,"user":1,"wait":0,"timestamp":1383048335000},{"idle":99,"system":0,"user":1,"wait":0,"timestamp":1383048340000},{"idle":98,"system":1,"user":1,"wait":0,"timestamp":1383048345000},{"idle":99,"system":0,"user":1,"wait":0,"timestamp":1383048350000},{"idle":98,"system":1,"user":1,"wait":0,"timestamp":1383048355000},{"idle":99,"system":0,"user":1,"wait":0,"timestamp":1383048360000},{"idle":98,"system":0,"user":1,"wait":0,"timestamp":1383048365000},{"idle":98,"system":0,"user":1,"wait":0,"timestamp":1383048370000},{"idle":99,"system":0,"user":1,"wait":0,"timestamp":1383048375000},{"idle":99,"system":0,"user":1,"wait":0,"timestamp":1383048380000},{"idle":99,"system":0,"user":1,"wait":0,"timestamp":1383048385000},{"idle":98,"system":0,"user":1,"wait":0,"timestamp":1383048390000},{"idle":99,"system":0,"user":0,"wait":0,"timestamp":1383048395000},{"idle":99,"system":0,"user":1,"wait":0,"timestamp":1383048400000},{"idle":99,"system":0,"user":1,"wait":0,"timestamp":1383048405000},{"idle":98,"system":1,"user":1,"wait":0,"timestamp":1383048410000},{"idle":99,"system":0,"user":1,"wait":0,"timestamp":1383048415000},{"idle":98,"system":0,"user":1,"wait":0,"timestamp":1383048420000},{"idle":97,"system":1,"user":1,"wait":0,"timestamp":1383048425000},{"idle":98,"system":1,"user":1,"wait":0,"timestamp":1383048430000},{"idle":99,"system":0,"user":1,"wait":0,"timestamp":1383048435000},{"idle":99,"system":0,"user":1,"wait":0,"timestamp":1383048440000},{"idle":98,"system":1,"user":1,"wait":0,"timestamp":1383048445000},{"idle":98,"system":1,"user":1,"wait":0,"timestamp":1383048450000},{"idle":99,"system":0,"user":1,"wait":0,"timestamp":1383048455000},{"idle":98,"system":1,"user":1,"wait":0,"timestamp":1383048460000},{"idle":99,"system":1,"user":0,"wait":0,"timestamp":1383048465000},{"idle":99,"system":1,"user":1,"wait":0,"timestamp":1383048470000},{"idle":98,"system":1,"user":1,"wait":1,"timestamp":1383048475000},{"idle":98,"system":1,"user":1,"wait":0,"timestamp":1383048480000},{"idle":99,"system":0,"user":1,"wait":0,"timestamp":1383048485000},{"idle":99,"system":0,"user":1,"wait":0,"timestamp":1383048490000},{"idle":98,"system":0,"user":1,"wait":0,"timestamp":1383048495000},{"idle":99,"system":0,"user":1,"wait":0,"timestamp":1383048500000},{"idle":99,"system":0,"user":1,"wait":0,"timestamp":1383048505000},{"idle":99,"system":0,"user":1,"wait":0,"timestamp":1383048510000},{"idle":98,"system":1,"user":1,"wait":0,"timestamp":1383048515000},{"idle":98,"system":1,"user":1,"wait":0,"timestamp":1383048520000},{"idle":100,"system":0,"user":0,"wait":0,"timestamp":1383048525000},{"idle":98,"system":0,"user":1,"wait":0,"timestamp":1383048530000},{"idle":99,"system":0,"user":1,"wait":0,"timestamp":1383048535000},{"idle":98,"system":1,"user":1,"wait":0,"timestamp":1383048540000},{"idle":99,"system":0,"user":1,"wait":0,"timestamp":1383048545000},{"idle":99,"system":0,"user":1,"wait":0,"timestamp":1383048550000},{"idle":99,"system":0,"user":1,"wait":0,"timestamp":1383048555000},{"idle":99,"system":0,"user":1,"wait":0,"timestamp":1383048560000},{"idle":98,"system":0,"user":2,"wait":0,"timestamp":1383048565000},{"idle":98,"system":0,"user":1,"wait":0,"timestamp":1383048570000},{"idle":98,"system":1,"user":1,"wait":0,"timestamp":1383048575000},{"idle":99,"system":0,"user":0,"wait":0,"timestamp":1383048580000},{"idle":98,"system":0,"user":1,"wait":0,"timestamp":1383048585000},{"idle":99,"system":0,"user":1,"wait":0,"timestamp":1383048590000},{"idle":99,"system":1,"user":1,"wait":0,"timestamp":1383048595000},{"idle":98,"system":0,"user":1,"wait":0,"timestamp":1383048600000},{"idle":99,"system":0,"user":1,"wait":0,"timestamp":1383048605000},{"idle":98,"system":1,"user":1,"wait":0,"timestamp":1383048610000},{"idle":99,"system":0,"user":1,"wait":0,"timestamp":1383048615000},{"idle":98,"system":0,"user":1,"wait":0,"timestamp":1383048620000},{"idle":99,"system":0,"user":1,"wait":0,"timestamp":1383048625000},{"idle":99,"system":0,"user":1,"wait":0,"timestamp":1383048630000},{"idle":99,"system":0,"user":1,"wait":0,"timestamp":1383048635000},{"idle":99,"system":0,"user":1,"wait":0,"timestamp":1383048640000},{"idle":99,"system":0,"user":1,"wait":0,"timestamp":1383048645000},{"idle":98,"system":0,"user":1,"wait":0,"timestamp":1383048650000},{"idle":98,"system":1,"user":1,"wait":0,"timestamp":1383048655000},{"idle":98,"system":0,"user":1,"wait":0,"timestamp":1383048660000},{"idle":98,"system":0,"user":1,"wait":0,"timestamp":1383048665000},{"idle":99,"system":0,"user":1,"wait":0,"timestamp":1383048670000},{"idle":99,"system":0,"user":1,"wait":0,"timestamp":1383048675000},{"idle":99,"system":1,"user":1,"wait":0,"timestamp":1383048680000},{"idle":98,"system":1,"user":1,"wait":0,"timestamp":1383048685000},{"idle":98,"system":0,"user":1,"wait":1,"timestamp":1383048690000},{"idle":99,"system":0,"user":1,"wait":0,"timestamp":1383048695000},{"idle":99,"system":0,"user":1,"wait":0,"timestamp":1383048700000},{"idle":99,"system":0,"user":1,"wait":0,"timestamp":1383048705000},{"idle":98,"system":1,"user":1,"wait":0,"timestamp":1383048710000},{"idle":98,"system":0,"user":2,"wait":0,"timestamp":1383048715000},{"idle":99,"system":0,"user":1,"wait":0,"timestamp":1383048720000},{"idle":98,"system":0,"user":1,"wait":0,"timestamp":1383048725000},{"idle":98,"system":0,"user":2,"wait":0,"timestamp":1383048730000},{"idle":98,"system":1,"user":1,"wait":0,"timestamp":1383048735000},{"idle":98,"system":1,"user":1,"wait":0,"timestamp":1383048740000},{"idle":98,"system":1,"user":1,"wait":0,"timestamp":1383048745000},{"idle":98,"system":0,"user":1,"wait":0,"timestamp":1383048750000},{"idle":98,"system":0,"user":1,"wait":0,"timestamp":1383048755000},{"idle":99,"system":0,"user":0,"wait":0,"timestamp":1383048760000},{"idle":98,"system":0,"user":1,"wait":0,"timestamp":1383048765000},{"idle":99,"system":0,"user":1,"wait":0,"timestamp":1383048770000},{"idle":98,"system":0,"user":1,"wait":0,"timestamp":1383048775000},{"idle":99,"system":0,"user":1,"wait":0,"timestamp":1383048780000},{"idle":99,"system":1,"user":1,"wait":0,"timestamp":1383048785000},{"idle":99,"system":0,"user":1,"wait":0,"timestamp":1383048790000},{"idle":99,"system":0,"user":0,"wait":0,"timestamp":1383048795000},{"idle":99,"system":0,"user":1,"wait":0,"timestamp":1383048800000},{"idle":100,"system":0,"user":0,"wait":0,"timestamp":1383048805000},{"idle":99,"system":0,"user":1,"wait":0,"timestamp":1383048810000},{"idle":99,"system":1,"user":0,"wait":0,"timestamp":1383048815000},{"idle":99,"system":0,"user":1,"wait":0,"timestamp":1383048820000},{"idle":99,"system":0,"user":0,"wait":0,"timestamp":1383048825000},{"idle":99,"system":0,"user":0,"wait":0,"timestamp":1383048830000},{"idle":100,"system":0,"user":0,"wait":0,"timestamp":1383048835000},{"idle":99,"system":0,"user":0,"wait":0,"timestamp":1383048840000},{"idle":99,"system":0,"user":1,"wait":0,"timestamp":1383048845000},{"idle":99,"system":0,"user":0,"wait":0,"timestamp":1383048850000},{"idle":99,"system":0,"user":0,"wait":0,"timestamp":1383048855000},{"idle":99,"system":0,"user":1,"wait":0,"timestamp":1383048860000},{"idle":99,"system":0,"user":0,"wait":0,"timestamp":1383048865000},{"idle":99,"system":0,"user":1,"wait":0,"timestamp":1383048870000},{"idle":99,"system":0,"user":0,"wait":0,"timestamp":1383048875000},{"idle":100,"system":0,"user":0,"wait":0,"timestamp":1383048880000},{"idle":99,"system":0,"user":0,"wait":0,"timestamp":1383048885000},{"idle":99,"system":0,"user":1,"wait":0,"timestamp":1383048890000},{"idle":99,"system":0,"user":0,"wait":0,"timestamp":1383048895000},{"idle":99,"system":0,"user":1,"wait":0,"timestamp":1383048900000},{"idle":99,"system":1,"user":1,"wait":0,"timestamp":1383048905000},{"idle":99,"system":0,"user":0,"wait":0,"timestamp":1383048910000},{"idle":99,"system":0,"user":1,"wait":0,"timestamp":1383048915000},{"idle":99,"system":0,"user":1,"wait":0,"timestamp":1383048920000},{"idle":99,"system":0,"user":1,"wait":0,"timestamp":1383048925000},{"idle":99,"system":0,"user":0,"wait":0,"timestamp":1383048930000},{"idle":99,"system":0,"user":1,"wait":0,"timestamp":1383048935000},{"idle":99,"system":0,"user":1,"wait":0,"timestamp":1383048940000},{"idle":99,"system":0,"user":1,"wait":0,"timestamp":1383048945000},{"idle":99,"system":0,"user":0,"wait":0,"timestamp":1383048950000},{"idle":100,"system":0,"user":0,"wait":0,"timestamp":1383048955000},{"idle":99,"system":0,"user":1,"wait":0,"timestamp":1383048960000},{"idle":99,"system":0,"user":0,"wait":0,"timestamp":1383048965000},{"idle":99,"system":0,"user":1,"wait":0,"timestamp":1383048970000},{"idle":99,"system":0,"user":1,"wait":0,"timestamp":1383048975000},{"idle":99,"system":0,"user":0,"wait":0,"timestamp":1383048980000},{"idle":99,"system":0,"user":1,"wait":0,"timestamp":1383048985000},{"idle":99,"system":0,"user":0,"wait":0,"timestamp":1383048990000},{"idle":99,"system":0,"user":0,"wait":0,"timestamp":1383048995000},{"idle":99,"system":0,"user":0,"wait":0,"timestamp":1383049000000},{"idle":99,"system":0,"user":1,"wait":0,"timestamp":1383049005000},{"idle":99,"system":0,"user":0,"wait":0,"timestamp":1383049010000},{"idle":99,"system":0,"user":1,"wait":0,"timestamp":1383049015000},{"idle":99,"system":0,"user":0,"wait":0,"timestamp":1383049020000},{"idle":98,"system":1,"user":1,"wait":0,"timestamp":1383049025000},{"idle":99,"system":0,"user":1,"wait":0,"timestamp":1383049030000},{"idle":99,"system":0,"user":0,"wait":0,"timestamp":1383049035000},{"idle":99,"system":0,"user":1,"wait":0,"timestamp":1383049040000},{"idle":99,"system":0,"user":1,"wait":0,"timestamp":1383049045000},{"idle":99,"system":0,"user":1,"wait":0,"timestamp":1383049050000},{"idle":99,"system":0,"user":1,"wait":0,"timestamp":1383049055000},{"idle":99,"system":0,"user":0,"wait":0,"timestamp":1383049060000},{"idle":99,"system":0,"user":0,"wait":0,"timestamp":1383049065000},{"idle":98,"system":0,"user":1,"wait":0,"timestamp":1383049070000},{"idle":99,"system":0,"user":1,"wait":0,"timestamp":1383049075000},{"idle":99,"system":0,"user":1,"wait":1,"timestamp":1383049080000},{"idle":99,"system":0,"user":0,"wait":1,"timestamp":1383049085000},{"idle":100,"system":0,"user":0,"wait":0,"timestamp":1383049090000},{"idle":100,"system":0,"user":0,"wait":0,"timestamp":1383049095000}]

















In [ ]:

TODO: add from - to (with timezone!)





In [19]:



    


#HTML(plot.set_styles(['stacked',]))
plot.set_styles(['stacked',])



    


















    
Out[19]:























In [20]:



    


#HTML(plot.draw_graph('stacked', {'data': cpu.to_json(orient='records')}))
plot.draw_graph('stacked', cpu.to_json(orient='records'))



    


















    
Out[20]:



























In [ ]:

Access logs





In [27]:



    


pr = PatternRepo(['./korg_patterns/'], True)
lg = korg.LineGrokker('%{COMMONAPACHELOG}', pr)

req = pd.DataFrame()

# now grok the aogaeru load log
with open("./data/access_log2") as infile:
    for line in infile:
        req = req.append(lg.grok(line), ignore_index=True)
req.head()



    


















    
Out[27]:










  
    

      
      auth
      bytes
      clientip
      httpversion
      ident
      rawrequest
      request
      response
      timestamp
      verb
    

  
  
    

      0
      -
      3395
      192.168.2.20
      1.0
      -
      None
      /cgi-bin/try/
      200
      28/Jul/2006:10:27:10 -0300
      GET
    

    

      1
      -
      2216
      127.0.0.1
      1.0
      -
      None
      /
      200
      28/Jul/2006:10:22:04 -0300
      GET
    

  




















In [25]:



    


req.timestamp = pd.to_datetime(req.timestamp)  # convert to timestamp



    


















    






---------------------------------------------------------------------------
ValueError                                Traceback (most recent call last)
<ipython-input-25-fa05e4a4b34b> in <module>()
----> 1 req.timestamp = pd.to_datetime(req.timestamp)  # convert to timestamp

/usr/lib/python2.7/dist-packages/pandas/util/decorators.pyc in wrapper(*args, **kwargs)
     87                 else:
     88                     kwargs[new_arg_name] = new_arg_value
---> 89             return func(*args, **kwargs)
     90         return wrapper
     91     return _deprecate_kwarg

/usr/lib/python2.7/dist-packages/pandas/tseries/tools.pyc in to_datetime(arg, errors, dayfirst, yearfirst, utc, box, format, exact, coerce, unit, infer_datetime_format)
    274     return _to_datetime(arg, errors=errors, dayfirst=dayfirst, yearfirst=yearfirst,
    275                         utc=utc, box=box, format=format, exact=exact,
--> 276                         unit=unit, infer_datetime_format=infer_datetime_format)
    277 
    278 

/usr/lib/python2.7/dist-packages/pandas/tseries/tools.pyc in _to_datetime(arg, errors, dayfirst, yearfirst, utc, box, format, exact, unit, freq, infer_datetime_format)
    388         return arg
    389     elif isinstance(arg, Series):
--> 390         values = _convert_listlike(arg._values, False, format)
    391         return Series(values, index=arg.index, name=arg.name)
    392     elif isinstance(arg, ABCIndexClass):

/usr/lib/python2.7/dist-packages/pandas/tseries/tools.pyc in _convert_listlike(arg, box, format, name)
    381                 return DatetimeIndex._simple_new(values, name=name, tz=tz)
    382             except (ValueError, TypeError):
--> 383                 raise e
    384 
    385     if arg is None:

ValueError: Unknown string format
















In [ ]:

Content source: finklabs/loganalyser

Similar notebooks:

notebook.community | gallery | about