Model20: refactoring

In [37]:

    

from utils import load_buzz, select, write_result
from features import featurize, get_pos
from containers import Questions, Users, Categories
from nlp import extract_entities


    

In [38]:

    

categories = Categories(load_buzz())
categories.sub_keys()


    

    
Out[38]:





dict_keys(['ave_pos_cat', 'acc_ratio_cat'])

In [39]:

    

questions = Questions(load_buzz())
questions.sub_keys()


    

    
Out[39]:





dict_keys(['group', 'category', 'question', 'acc_ratio_qid', 'ave_pos_qid', 'pos_token', 'answer'])

In [40]:

    

users = Users(load_buzz())
users.sub_keys()


    

    
Out[40]:





dict_keys(['acc_ratio_uid', 'ave_pos_uid'])

In [41]:

    

import pickle


with open('ne_count02.pkl', 'rb') as f:
    nes = pickle.load(f)


    

In [42]:

    

questions.sub_append('ne_tags', nes)


    

In [43]:

    

questions.cal_ne()


    

In [44]:

    

import pickle

for ii in ['questions', 'users', 'categories']:
    file_name = ii + '01.pkl'
    with open(file_name, 'wb') as f:
        nes = pickle.dump(globals()[ii], f)


    

In [45]:

    

ls *01.pkl


    

    




categories01.pkl  ne_count01.pkl  questions01.pkl  users01.pkl

In [46]:

    

questions01 = pickle.load(open('questions01.pkl', 'rb'))


    

In [47]:

    

questions01[1]


    

    
Out[47]:





{'acc_ratio_qid': 0.875,
 'answer': 'thomas cole',
 'ave_pos_qid': 70.5,
 'category': 'Fine Arts',
 'group': 'test',
 'ne_count': 12,
 'ne_mean': 56.5,
 'ne_median': 60.5,
 'ne_mod': 65,
 'ne_nor_mean': 0.7243589743589743,
 'ne_tags': [['CD', '1840', 20],
  ['PERSON', 'Architect', 21],
  ['CD', 'three', 40],
  ['GPE', 'Europe', 44],
  ['CD', '1829', 46],
  ['CD', 'four', 56],
  ['ORGANIZATION', 'Hudson', 65],
  ['ORGANIZATION', 'Catskill', 69],
  ['ORGANIZATION', 'FTP', 72],
  ['ORGANIZATION', 'Oxbow', 79],
  ['ORGANIZATION', 'Voyage', 82],
  ['GPE', 'Life', 84]],
 'pos_token': {0: '',
  1: 'painters',
  2: 'indulgence',
  4: 'visual',
  5: 'fantasy',
  7: 'appreciation',
  9: 'different',
  10: 'historic',
  11: 'architectural',
  12: 'styles',
  15: 'seen',
  18: '1840',
  19: 'architects',
  20: 'dream',
  23: 'series',
  25: 'paintings',
  28: 'last',
  31: 'mohicans',
  33: 'made',
  35: 'three',
  36: 'year',
  37: 'trip',
  39: 'europe',
  41: '1829',
  45: 'better',
  46: 'known',
  49: 'trip',
  50: 'four',
  51: 'years',
  52: 'earlier',
  56: 'journeyed',
  59: 'hudson',
  60: 'river',
  63: 'catskill',
  64: 'mountains',
  65: 'ftp',
  66: 'name',
  68: 'this_painter',
  71: 'oxbow',
  74: 'voyage',
  76: 'life',
  77: 'series'},
 'question': "This painter's indulgence of visual fantasy, and appreciation of different historic architectural styles can be seen in his 1840 Architect's Dream. After a series of paintings on The Last of the Mohicans, he made a three year trip to Europe in 1829, but he is better known for a trip four years earlier in which he journeyed up the Hudson River to the Catskill Mountains. FTP, name this painter of The Oxbow and The Voyage of Life series."}