In [4]:

    

import pandas as pd


    

In [2]:

    

from engine.engine import Engine


    

In [5]:

    

df = pd.read_csv("data/200log.csv")


    

In [6]:

    

df.head()


    

    
Out[6]:






  

    

      

      
url_request
      
url_status_code
      
header_len
      
response_len
      
name
      
time
      
message
    
  
  

    

      
0
      
http://www.frontpageafricaonline.com/index.php...
      
200
      
16
      
43429
      
frontpageafricaonline
      
2015-05-03 12:33:27.188826
      
production
    
    

      
1
      
http://www.1847post.com/?q=node/8
      
200
      
15
      
33305
      
post1847
      
2015-05-03 12:41:19.267498
      
production
    
    

      
2
      
http://www.1847post.com/?q=node/24
      
200
      
15
      
38303
      
post1847
      
2015-05-03 12:59:00.820873
      
production
    
    

      
3
      
http://www.1847post.com/?q=node/25
      
200
      
15
      
33886
      
post1847
      
2015-05-03 13:00:05.688240
      
production
    
    

      
4
      
http://www.thenewdawnliberia.com/general/39
      
200
      
11
      
27215
      
thenewdawn
      
2015-05-03 13:15:20.277015
      
production
    
  

In [ ]:

    

import csv
from dateutil import parser 
import datetime 
import time

from engine.engine import Engine


    

In [ ]:

    

log = []

with open("data/200log.csv", 'r') as f:
    reader = csv.reader(f)
    next(reader, None)
    for row in reader:
        url_request = unicode(row[0])  
        url_status_code = row[1]
        header_len = row[2]
        response_len = row[3]
        name = unicode(row[4]) 
        timedate = parser.parse(row[5])
        message = unicode(row[6])
        
        scraperlog = timedate, url_request, url_status_code, header_len, response_len, name, message
        
        log.append(scraperlog)

        
# Engine(engine='new', scraperlog=log)
print log[:50
]


    

In [50]:

    

type(log[0][0])


    

    
Out[50]:





datetime.datetime

In [10]:

    

["url_request", "url_status_code", "header_len", "response_len", "name",  "time", "message"]


    

    
Out[10]:





['url_request',
 'url_status_code',
 'header_len',
 'response_len',
 'name',
 'time',
 'message']

In [11]:

    




    

    




  File "<ipython-input-11-9041ad43faf0>", line 1
    (	timedate=timedate, name=name, url_request=url_request, url_status_code=url_status_code,
     	        ^
SyntaxError: invalid syntax

In [61]:

    

log = []

with open("data/sample.csv", 'r') as f:
    reader = csv.reader(f)

    print dir(reader)
    print reader.line_num + 1


    

    




['__class__', '__delattr__', '__doc__', '__format__', '__getattribute__', '__hash__', '__init__', '__iter__', '__new__', '__reduce__', '__reduce_ex__', '__repr__', '__setattr__', '__sizeof__', '__str__', '__subclasshook__', 'dialect', 'line_num', 'next']
1

In [ ]: