In [1]:

    

import csv
from dateutil import parser 
import datetime 
import time

from engine.engine import Engine


    

In [2]:

    

log = []

with open("data/200log.csv", 'r') as f:
    reader = csv.reader(f)
    next(reader, None)
    for row in reader:
        url_request = unicode(row[0])  
        url_status_code = row[1]
        header_len = row[2]
        response_len = row[3]
        name = unicode(row[4]) 
        timedate = parser.parse(row[5])
        message = unicode(row[6])
        
        scraperlog = [timedate, url_request, url_status_code, header_len, response_len, name, message]
        
        log.append(scraperlog)

        
print 'done', log[:2]


    

    




done [[datetime.datetime(2015, 5, 3, 12, 33, 27, 188826), u'http://www.frontpageafricaonline.com/index.php/news/1', '200', '16', '43429', u'frontpageafricaonline', u'production'], [datetime.datetime(2015, 5, 3, 12, 41, 19, 267498), u'http://www.1847post.com/?q=node/8', '200', '15', '33305', u'post1847', u'production']]

In [5]:

    

Engine(engine='new_test', scraperlog=log)


    

    




New DB new_test.db created
DB initialized
Scarper log added to commit list
Ending DB connection






    
Out[5]:





<engine.engine.Engine at 0x7f30804f1310>

In [6]:

    

Engine(engine='new_test').main()


    

    




DB initialized
Ending DB connection

In [6]:

    

print e


    

    




---------------------------------------------------------------------------
NameError                                 Traceback (most recent call last)
<ipython-input-6-65369d5b60a8> in <module>()
----> 1 print e

NameError: name 'e' is not defined

In [ ]: