Build selected Hedonometer words frequency dicts for topic_news and topic_tweets docs

Last modified: 2017-10-23

Roadmap

  1. Check shed words pattern-matching requiremnts
  2. Build shed words freq dicts for topic docs

Steps


In [1]:
"""
Initialization
"""

'''
Standard modules
'''
import os
import pickle
import csv
import time
from pprint import pprint

'''
Analysis modules
'''
import pandas as pd


'''
Custom modules
'''
import config
import utilities

'''
Misc
'''
nb_name = '20171019-daheng-build_shed_words_freq_dicts'

Check shed words pattern-matching requiremnts

Ref:

  • Dodds, P. S., Harris, K. D., Kloumann, I. M., Bliss, C. A., & Danforth, C. M. (2011). Temporal patterns of happiness and information in a global social network: Hedonometrics and Twitter. PloS one, 6(12), e26752.

Notes:

  • See 2.1 Algorithm for Hedonometer P3
  • See Methods P23

Build shed words freq dicts for topic docs


In [2]:
"""
Check all shed words
"""
if 1 == 1:
    ind_shed_word_dict = pd.read_pickle(config.IND_SHED_WORD_DICT_PKL)
    print(ind_shed_word_dict.values())


dict_values(['laughter', 'happiness', 'love', 'happy', 'laughed', 'laugh', 'laughing', 'excellent', 'laughs', 'joy', 'successful', 'win', 'rainbow', 'smile', 'won', 'pleasure', 'smiled', 'rainbows', 'winning', 'celebration', 'enjoyed', 'healthy', 'music', 'celebrating', 'congratulations', 'weekend', 'celebrate', 'comedy', 'jokes', 'rich', 'victory', 'christmas', 'free', 'friendship', 'fun', 'holidays', 'loved', 'loves', 'loving', 'beach', 'hahaha', 'kissing', 'sunshine', 'beautiful', 'delicious', 'friends', 'funny', 'outstanding', 'paradise', 'sweetest', 'vacation', 'butterflies', 'freedom', 'flower', 'great', 'sunlight', 'sweetheart', 'sweetness', 'award', 'chocolate', 'hahahaha', 'heaven', 'peace', 'splendid', 'success', 'enjoying', 'kissed', 'attraction', 'celebrated', 'hero', 'hugs', 'positive', 'sun', 'birthday', 'blessed', 'fantastic', 'winner', 'delight', 'beauty', 'butterfly', 'entertainment', 'funniest', 'honesty', 'sky', 'smiles', 'succeed', 'wonderful', 'glorious', 'kisses', 'promotion', 'family', 'gift', 'humor', 'romantic', 'cupcakes', 'festival', 'hahahahaha', 'honour', 'relax', 'weekends', 'angel', 'b-day', 'bonus', 'brilliant', 'diamonds', 'holiday', 'lucky', 'mother', 'super', 'amazing', 'angels', 'enjoy', 'friend', 'friendly', "mother's", 'profit', 'finest', 'bday', 'champion', 'grandmother', 'haha', 'kiss', 'kitten', 'miracle', 'mom', 'sweet', 'blessings', 'bright', 'cutest', 'entertaining', 'excited', 'excitement', 'joke', 'millionaire', 'prize', 'succeeded', 'successfully', 'winners', 'shines', 'awesome', 'genius', 'achievement', 'cake', 'cheers', 'exciting', 'goodness', 'hug', 'income', 'party', 'puppy', 'smiling', 'song', 'succeeding', 'tasty', 'victories', 'achieved', 'billion', 'cakes', 'easier', 'flowers', 'gifts', 'gold', 'merry', 'families', 'handsome', 'lovers', 'affection', 'candy', 'cute', 'diamond', 'earnings', 'interesting', 'peacefully', 'praise', 'relaxing', 'roses', 'saturdays', 'faithful', 'heavens', 'cherish', 'comfort', 'congrats', 'cupcake', 'earn', 'extraordinary', 'glory', 'hilarious', 'moonlight', 'optimistic', 'peaceful', 'romance', 'feast', 'attractive', 'glad', 'grandma', 'internet', 'pleasant', 'profits', 'smart', 'x-mas', 'babies', 'cheer', 'courage', 'enthusiasm', 'honest', 'loyal', 'opportunities', 'triumph', 'wow', 'jewels', 'forests', 'apple', 'dreams', 'fantasy', 'food', 'honey', 'miracles', 'sex', 'sing', 'starlight', 'thankful', 'wins', 'achieve', 'adored', 'cash', 'dances', 'gorgeous', 'grandchildren', 'incredible', 'lunch', 'mommy', 'parties', 'perfect', 'saturday', 'surprise', 'truth', 'blessing', 'creative', 'dinner', 'kindness', 'pleased', 'sexy', 'strength', 'thank', 'thanks', 'thanksgiving', 'treasure', 'valentine', 'riches', 'awarded', 'fabulous', 'grandfather', 'heavenly', 'hope', 'kids', 'magical', 'million', 'nice', 'sundays', 'wealth', 'fantasies', 'cares', 'dance', 'daughters', 'favorable', "friend's", 'generosity', 'grateful', 'inspired', 'mothers', 'parents', "valentine's", 'intelligent', 'liberation', 'melody', 'wonderland', 'beloved', 'caring', 'homemade', 'inspiring', 'movies', 'precious', 'respect', 'satisfaction', 'satisfy', 'wedding', 'accomplished', 'adorable', 'championship', 'comfortable', 'cuddle', 'games', 'grandson', 'life', 'lovely', 'pretty', 'proud', 'rose', 'united', 'fruits', 'adventure', 'couple', 'dollars', 'eating', 'fortune', 'generous', 'golden', 'hahah', 'hooray', 'intelligence', 'lover', 'luxury', 'money', 'passion', 'prosperity', 'remarkable', 'sweetie', 'valentines', 'educated', 'gently', 'baby', 'books', 'bride', 'cherished', 'cookies', 'dessert', 'employed', 'glow', 'god', 'great-grandchildren', 'helped', 'independence', 'likes', 'luckily', 'moon', 'perfectly', 'satisfied', 'sunday', 'juicy', 'championships', 'divine', 'dreaming', 'foods', 'fresh', 'gladly', 'greatest', 'hearts', 'luck', 'millions', 'musicians', 'play', 'progress', 'savings', 'appreciation', 'bliss', 'bloom', 'book', 'child', 'companion', 'computer', 'gardens', 'gentle', 'hahahah', 'helpful', 'impressed', 'kind', 'knowledge', 'liberty', 'mama', 'nature', 'pal', 'passionate', 'promoted', 'reward', 'warmth', 'xmas', 'danced', 'amazed', 'appreciate', 'brother', 'confidence', 'darling', 'encouraging', 'energy', 'films', 'garden', 'graduated', 'guitar', 'health', 'heart', 'honor', 'like', 'musical', 'pets', 'relaxed', 'salary', 'star', 'sweeter', 'trust', 'yummy', 'ecstasy', 'eternal', 'approved', 'benefits', 'cartoon', 'comforted', 'cool', 'discount', 'good', 'google', 'ladies', 'libraries', 'luv', 'perfection', 'presents', 'prizes', 'special', 'wishes', 'alive', 'awards', 'bed', 'best', 'coffee', 'comfy', 'fiesta', 'genuine', 'helping', 'imagine', 'leisure', 'meal', 'promise', 'respected', 'rest', 'travel', 'abundant', 'attracted', 'devoted', 'favourite', 'granddaughter', 'heroes', 'ideas', 'liked', 'oceans', 'pizza', 'skies', 'sleep', 'spring', 'sunset', 'welcome', '1st', 'adoring', 'brighter', "children's", 'cure', 'fireworks', 'home', 'honored', 'journey', 'lovin', 'opportunity', 'paid', 'parks', 'playing', 'shine', 'strawberry', 'summertime', 'wealthy', 'appreciated', 'artistic', 'birth', 'children', 'fruit', 'inspire', 'juice', 'laptop', 'partners', 'son', 'stronger', 'superman', 'tree', 'valuable', "woman's", 'women', 'glowing', 'admiration', 'carnival', 'computers', 'confident', 'cookie', 'cutie', 'dearest', 'dream', 'freely', 'fridays', 'plants', 'quality', 'rabbit', 'resort', 'shopping', 'sincere', 'snack', 'stars', 'toys', 'useful', 'wise', 'yum', 'desirable', 'sparkle', 'bless', 'comic', 'cooking', 'dancing', 'earned', 'equality', 'faith', 'graduate', 'improvements', 'memories', 'park', 'pet', 'powerful', 'princess', 'qualities', 'thrill', 'true', 'wonder', 'everlasting', 'mamma', 'caress', 'charm', 'clever', 'father', 'grand', 'hehehe', 'idea', 'pearl', 'pictures', 'restaurant', 'sandwich', 'sharing', 'strong', 'talent', 'talented', 'tenderness', 'weddings', 'dove', 'awsome', 'cherry', 'daughter', 'eat', 'favorite', 'girlfriend', 'hoping', 'impressive', 'loyalty', 'parent', 'relationship', 'safe', 'scholarship', 'shining', 'sunrise', 'yoga', 'respects', 'fairy', 'humanity', 'productivity', 'brave', 'colours', 'correct', 'dad', 'daddy', 'dollar', 'easily', 'fans', 'goal', 'hawaii', 'honestly', 'inspiration', 'olympics', 'saints', 'sleeping', 'wisdom', 'believed', 'better', 'color', 'colors', "dad's", 'determination', 'discovered', 'gentlemen', 'girl', 'harmony', 'hello', 'hopes', 'noble', 'praised', 'reliable', 'trip', 'agreed', 'approval', 'brothers', 'concerts', 'cooperation', 'encouraged', 'giving', 'goals', 'ideal', 'intellectual', 'invitation', 'marry', 'musician', 'outdoors', 'photography', 'plenty', 'rome', 'trees', 'trips', 'unique', 'wildlife', 'lullaby', 'thrills', 'abroad', 'bath', 'benefit', 'birds', 'dads', 'elegant', 'eternally', 'fair', 'fancy', 'great-grandfather', 'imagination', 'improving', 'mountains', 'ocean', 'pancakes', 'photograph', 'praying', 'present', 'reunion', 'safely', 'saving', 'singing', 'songs', 'sunny', 'terrific', 'theater', 'vanilla', 'adore', 'gentleman', 'autumn', 'cinema', 'college', 'concert', 'correctly', 'cozy', 'dear', 'earning', 'earns', 'gardening', 'girls', 'massage', 'outdoor', 'photos', 'piano', 'sea', 'trusted', 'albums', 'dignity', 'favored', 'fitness', 'game', 'healing', 'learned', 'learning', 'prayers', 'promote', 'secure', 'spa', 'unity', 'wish', 'youtube', 'favour', 'clean', 'dynamic', 'encourage', 'infant', 'jewelry', 'necklace', 'paintings', 'stability', 'voyage', 'worthy', 'fulfill', 'eternity', 'accuracy', 'bookstores', 'breeze', 'bunny', 'cheese', 'comics', 'donated', 'easter', 'education', 'email', 'farmer', 'female', 'flavor', 'friday', 'moms', 'photo', 'pillow', 'pure', 'saved', 'shakespeare', 'survived', 'taste', 'valued', 'vitamin', 'infants', 'silk', 'dreamed', '#music', 'acceptance', 'banana', 'breakfast', 'cooperative', 'dancer', 'grace', 'greatly', 'guarantee', 'improved', 'improvement', 'independent', 'liking', 'paris', 'pasta', 'photographs', 'recipes', 'relationships', 'relief', 'sailing', 'science', 'seas', 'toast', 'truly', 'platinum', 'superstar', 'understands', 'accurately', 'advantage', 'belonging', 'buddy', 'childhood', 'daylight', 'discover', 'forgiveness', 'great-grandmother', 'hopefully', 'horses', 'interested', 'kid', 'live', 'lol', 'movie', 'popularity', 'solution', 'swim', 'toy', 'understanding', 'universe', 'woman', 'woohoo', 'rivers', 'sail', 'cared', 'active', 'artists', 'babe', 'believes', 'born', 'champagne', 'compassion', 'completed', 'create', 'dedicated', 'experienced', 'fathers', 'first', 'gains', 'heal', 'new', 'significant', 'singer', 'surprisingly', 'young', 'mansion', 'prevail', 'qualified', 'air', 'amazon', 'animal', 'bedroom', 'camera', 'cream', 'dreamer', 'forgiven', 'highest', 'horse', 'magic', 'manners', 'naturally', 'novels', 'performers', 'pies', 'protect', 'santa', 'shared', 'smooth', 'together', 'uncle', 'efficient', 'elevated', 'cafe', 'coke', 'completion', 'coolest', 'creation', 'dogs', 'effectiveness', 'esteemed', 'finished', 'glee', 'green', 'heartbeat', 'island', 'jukebox', 'medal', "mom's", 'museums', 'painting', 'pie', 'pool', 'reading', 'real', 'ruby', 'share', 'sons', 'traveling', 'variety', 'wonders', 'worth', 'guaranteed', 'raindrops', 'visions', 'pearls', 'america', 'easy', 'effective', 'future', 'humans', 'intimate', 'married', 'muffin', 'papa', 'plus', 'popcorn', 'savior', 'seasons', 'shop', 'sister', 'style', 'supporter', 'switzerland', 'tenderly', 'top', 'oxygen', 'rhyme', 'allright', 'american', 'artist', 'capable', 'complete', 'convenient', 'courtesy', 'donate', 'drinks', "father's", 'fine', 'focused', 'guitars', 'hi', 'integrity', 'justice', 'lake', 'mankind', 'mentor', 'merit', 'performance', 'plant', 'prepared', 'raise', 'romeo', 'shiny', 'sugar', 'surprising', 'technology', 'treat', 'university', 'wishing', 'yes', 'desires', 'wished', '4-bedroom', 'attract', 'bike', 'car', 'civilization', 'classy', 'confirmed', 'costumes', 'creating', 'culture', 'finish', 'gallery', 'knowing', 'lifelong', 'momma', 'neat', 'niece', 'online', 'orchestra', 'plays', 'revenue', 'shower', 'spiritual', 'surprised', 'tremendous', 'values', 'villages', 'warm', 'doggy', 'hallelujah', 'candle', 'secured', 'valid', 'agree', 'anniversary', 'antiques', 'believe', 'bucks', 'cruise', 'dancers', 'dine', 'dog', 'florida', 'grandsons', 'grants', 'hired', 'learn', 'marriage', 'mum', 'partner', 'productive', 'rockin', 'teaches', 'treats', 'tv', 'water', 'grin', 'invention', 'virtues', 'brains', 'sensation', 'ability', 'ace', 'animals', 'bake', 'bridegroom', 'desire', 'famous', 'forest', 'fountain', 'goodmorning', 'greater', 'grow', 'heritage', 'landscape', 'liberties', 'living', 'lyrics', 'mercy', 'museum', 'novel', 'palace', 'pianist', 'potential', 'power', 'privilege', 'proceed', 'promised', 'river', 'scotland', 'shares', 'skating', 'thanx', 'theatre', 'tours', 'well', 'acceptable', 'possibilities', 'accurate', 'candles', 'approve', 'assets', 'aunt', 'career', 'charms', 'communicate', 'competent', 'currency', 'dedication', 'dvd', 'eligible', 'fan', 'firefighters', 'greet', 'motivation', 'nieces', 'personality', 'powers', 'raises', 'sculpture', 'survivors', 'tea', 'television', 'tour', 'pony', 'rhythm', 'bird', 'care', 'cat', 'cook', 'corn', 'deposits', 'expert', 'high', 'holy', 'invite', 'leading', 'photographer', 'picture', 'promising', 'recover', 'recovered', 'recovery', 'salad', 'shops', 'solutions', 'sparks', 'sport', 'supreme', 'theaters', 'tunes', 'unite', 'volunteers', 'simplicity', 'attained', "book's", 'cameras', 'chatting', 'crown', 'disney', 'dresses', 'heartfelt', 'homes', 'husband', 'immortal', 'invest', 'kitty', 'offer', 'organized', 'performances', 'perfume', 'pray', 'rescue', 'restaurants', 'salaries', 'sisters', 'slept', 'steak', 'stories', 'varieties', 'vision', 'wife', 'youth', 'zoo', 'stimulation', 'touching', 'furnished', 'suitable', 'album', 'amour', 'art', 'beam', 'captain', 'certainty', "child's", 'clothing', 'conservation', 'desired', 'dress', 'favorited', 'females', 'growth', 'helps', 'highly', 'ideals', 'lady', 'lime', 'popular', 'proposal', 'protected', 'relatives', 'rhymes', 'singers', 'specialty', 'spirit', 'starry', 'stroll', 'supported', 'therapeutic', 'unlimited', 'visiting', 'expressions', 'efficiency', 'sleeps', 'vocals', 'impress', 'sympathetic', 'advance', 'advanced', 'arts', 'available', 'baking', 'classic', 'classical', 'colour', 'drawing', 'english', 'exhibition', 'expecting', 'fish', 'goodnight', 'invented', 'islands', 'language', 'majesty', 'me', 'preferred', 'radio', 'ready', 'relative', 'sale', 'solve', 'springs', 'student', 'symphony', 'traditions', 'understood', 'upgrade', 'usa', 'saviour', 'skill', 'belonged', 'muscles', 'able', 'ahaha', 'butter', 'circus', 'cosmic', 'coupon', 'diploma', 'donations', 'e-mail', 'encore', 'film', 'guidance', 'illustration', 'increase', 'international', 'ipod', 'morning', 'natural', 'okay', 'preservation', 'progressive', 'protection', 'raised', 'showers', 'tacos', 'teach', 'traveler', 'understand', 'universities', 'worldwide', 'privileges', 'accepted', 'adoption', 'asset', 'blanket', 'cats', 'cleaned', 'coin', 'cooked', 'crystal', 'dawn', 'dearly', 'discovery', 'done', 'eager', 'emails', 'exercises', 'found', 'give', 'groovy', 'haven', 'invited', 'iphone', 'moral', 'nephew', 'orange', 'overcome', 'pays', 'potato', 'premiere', 'pride', 'receiving', 'recognition', 'reindeer', 'right', 'rising', 'save', 'scholars', 'shelter', 'solar', 'spontaneous', 'tasting', 'ultimate', 'visit', 'advantages', 'sailed', 'feather', 'ambitious', 'baker', 'brain', 'champ', 'communication', 'compensation', 'ease', 'ethics', 'extra', 'fries', 'growing', 'guest', 'incredibly', 'initiative', 'jesus', 'lips', 'literature', 'nights', 'phenomenon', 'planet', 'poem', 'poet', 'prefer', 'read', 'sang', 'soup', 'surf', 'swimming', 'videos', 'wings', 'world', 'amore', 'bounce', 'cultures', 'eden', 'interaction', 'mercedes', 'velvet', 'balanced', 'agriculture', 'allies', 'americans', 'bells', 'chips', 'contribute', 'couples', 'cousins', 'deals', 'determined', 'eaten', 'fame', 'gives', 'hire', 'innocence', 'ipad', 'leadership', 'legend', 'lounge', 'mature', 'newest', 'newly', 'performing', 'receive', 'recipe', 'roast', 'starting', 'stunning', 'tales', 'elder', 'grows', 'herb', 'illustrations', 'rays', 'relevant', 'sanity', 'acoustic', 'always', 'answers', 'bible', 'boost', 'clap', 'dining', 'electronics', 'exclusive', "family's", 'gathering', 'hehe', 'humble', 'information', 'italian', 'library', 'mate', 'modern', 'offers', 'paperbacks', 'perform', 'poems', 'potatoes', 'prayer', 'pumpkin', 'restored', 'rights', 'scholar', 'screenplay', 'shopper', 'sings', 'soft', 'starbucks', 'story', 'supporting', 'video', 'instrumental', 'backyard', 'drums', 'virtue', 'activities', 'athletic', 'clothes', 'cultivated', 'forever', 'goods', 'grass', 'higher', 'literary', 'london', 'memory', 'mint', 'nephews', 'prime', 'prospect', 'reception', 'recommended', 'research', 'resource', 'resources', 'riverside', 'rocking', 'scored', 'talking', 'believer', 'functioning', 'poets', 'boats', 'remedy', 'tender', 'aaah', 'beatles', 'chance', 'coast', 'draw', 'earth', 'eats', 'effectively', 'familiar', 'fast', 'forgive', 'gained', 'graphics', 'improve', 'increases', 'infinite', 'languages', 'likely', 'nap', 'philosophy', 'phone', 'prince', 'princes', 'professional', 'revival', 'rice', 'rides', 'satisfactory', 'scientific', 'scoring', 'sis', 'soccer', 'supermarkets', 'support', 'teachers', 'teaching', 'wage', 'whale', 'wink', 'wit', 'accept', 'assist', 'band', 'chat', 'composer', 'contribution', 'cousin', 'curves', 'dates', 'delivered', 'environmental', 'evening', 'feed', 'fest', 'gaming', 'india', 'interests', 'jazz', 'novelist', 'panties', 'partnership', "party's", 'portrait', 'remember', 'residence', 'shore', 'simply', 'stream', 'traveled', 'wine', 'wondered', 'farming', 'hats', 'hearted', '1980s', 'actress', 'adopt', 'altogether', 'architecture', 'australia', 'baked', 'buying', 'ceremony', 'charity', 'chicken', 'chorus', 'consciousness', 'cultivation', 'dating', 'deserve', 'destination', 'documentary', 'drawings', 'educational', 'electronic', 'equally', 'europe', 'floating', 'futures', 'gain', 'generations', 'gmail', 'hills', 'increasing', 'kidding', 'launch', 'light', 'mountain', 'participate', 'pics', 'playin', 'poetry', 'possibility', 'provide', 'resolved', 'shores', 'studies', 'summer', 'tennis', 'touch', 'touched', 'tradition', 'twins', 'visits', 'wages', 'waves', 'willing', 'younger', 'exercised', 'enabled', 'greeks', 'purely', 'seeds', 'sixteen', 'softly', 'cradle', "80's", 'americas', 'arose', 'bigger', 'boyfriend', 'breath', 'committed', 'contributing', 'craft', 'designers', 'development', 'distinction', 'faster', 'functional', 'giveaway', 'increased', 'lamb', 'leader', 'lottery', 'maximum', 'meet', 'neighborhood', 'ownership', 'painter', 'played', 'preserve', 'purchased', 'queens', 'reasonable', 'revenues', 'rocket', 'sails', 'saves', 'score', 'seeing', 'silver', 'skills', 'sung', 'tasted', 'tastes', 'thinks', 'thought', 'touches', 'we', 'agricultural', 'belle', 'explore', 'sketch', 'voluntary', 'acquire', 'april', 'architect', 'broadway', 'calm', 'climbed', 'colleagues', 'curious', 'definite', 'democracy', 'deposit', 'developed', 'distinguished', 'dressed', 'drink', 'employment', 'farms', 'fashion', 'gravy', 'guiding', 'imagined', 'innocent', 'instantly', 'interest', 'justified', 'logical', 'mail', 'maintained', 'mario', 'mobile', 'mp3', 'obtained', 'original', 'patience', 'performed', 'please', 'prayed', 'rain', 'rational', 'relation', 'rings', 'rise', 'rudolph', 'teacher', 'technologies', 'value', 'vegas', 'volunteer', 'wifi', 'revealed', 'branches', 'existed', 'spotlight', 'bread', 'castle', 'cheddar', 'clouds', 'clubs', 'colleges', 'completely', 'connected', 'december', 'dew', 'employ', 'exists', 'expedition', 'experience', 'farmers', 'firefox', 'football', 'grant', 'hiring', 'hollywood', 'house', 'illustrated', 'images', 'jeans', 'largest', 'linguistic', 'lord', 'purchase', 'received', 'released', 'saint', 'scientists', 'september', 'soon', 'soul', 'soundtrack', 'studio', 'tickets', 'wave', 'continuity', 'equilibrium', 'activity', 'agreement', 'amor', 'arrival', 'arrive', 'asian', 'bbq', 'bedtime', 'berry', 'brunch', 'commitment', 'date', 'deal', 'democratic', 'design', 'designer', 'devotion', 'experiences', 'fly', 'foxy', 'france', 'handy', 'importance', 'important', 'jamaica', 'jobs', 'june', 'kin', 'lights', 'mornings', 'newspaper', 'offering', 'organic', 'parade', 'pink', 'published', 'reader', 'remembered', 'resolve', 'ring', 'rofl', 'selected', 'snow', 'streams', 'sufficient', 'sufficiently', 'sure', 'universal', 'unlocked', 'visitors', 'waters', "women's", 'worship', 'writers', 'assembled', 'chickens', 'wheat', 'connections', 'scent', 'volumes', 'whistle', 'absolutely', 'atmosphere', 'belongs', 'bought', 'chess', 'christian', 'clear', 'clearer', 'commonwealth', 'conversations', 'designed', 'downloaded', 'earrings', 'engineer', 'epic', 'exercise', 'expansion', 'feeding', 'flowing', 'headphones', 'indians', 'joined', 'lipstick', 'metropolitan', 'mine', 'myself', 'paint', 'painted', 'plane', 'produced', 'protecting', 'reasoning', 'relations', 'salvation', 'sciences', 'sense', 'software', 'suite', 'surplus', 'swing', 'visited', 'cheeks', 'observation', 'calcium', 'conceived', 'rum', 'amigo', 'babes', 'begin', 'breathe', "bridegroom's", 'buy', 'community', 'cooler', 'country', 'disco', 'emerging', 'england', 'experts', 'fairly', 'fix', 'founded', 'globe', 'honorary', 'hoped', 'introduced', 'lead', 'listening', 'lots', 'market', 'monkey', 'olympic', 'pioneer', 'plaza', 'professionals', 'reflect', 'remembering', 'reputation', 'sentimental', 'skype', 'students', 'sweden', 'technological', 'themes', 'thinking', 'tips', 'vehicles', 'village', 'virginia', 'website', 'white', 'wines', 'reasonably', 'uptown', 'aims', 'observe', 'regards', 'allows', 'appropriate', 'australian', 'blackberry', 'breathing', 'camp', 'cars', 'considerable', 'costume', 'degree', 'develop', 'egypt', 'events', 'flag', 'gave', 'gods', 'gr8', 'hotels', 'human', 'indian', 'leap', 'lifetime', 'magnetic', 'mirror', 'mmmm', 'occasion', 'produce', 'prominent', 'promises', 'proved', 'raising', 'school', 'shirt', 'spark', 'surely', 'team', 'travelers', 'upcoming', 'us', 'valley', 'vintage', 'proteins', 'almighty', 'horizon', 'insight', 'ooooh', 'poetic', 'spirits', 'aboard', 'acknowledge', 'actors', 'advances', 'aid', 'answer', 'athletes', 'bowling', 'boy', 'built', 'choice', 'constitution', 'conversation', 'cowboy', 'day', 'deliver', 'developments', 'distinctive', 'dvds', 'edison', 'eighteen', 'enterprise', 'eyes', 'flying', 'grad', 'grammy', 'grill', 'halloween', 'holland', 'jelly', 'jingle', 'legitimate', 'making', 'more', 'options', 'possible', 'practical', 'proceeds', 'proposed', 'provides', 'queen', 'revolutionary', 'rises', 'samsung', 'self', 'show', 'sooner', 'speed', 'strategy', 'tale', 'tip', 'updating', 'vip', 'websites', 'worlds', 'writing', 'xbox', 'you', 'yours', 'yourself', 'collective', 'embrace', 'produces', 'meanings', 'accompanied', 'advice', 'all', 'answered', 'architectural', 'asia', 'authors', 'avid', 'batman', 'big', 'breast', 'bro', 'build', 'chef', 'clowns', 'contacts', 'contributions', 'cotton', 'cowboys', 'decent', 'designs', 'downloading', 'environment', 'evolution', 'farm', 'finishing', 'fit', 'foundations', 'full', 'guys', 'instrument', 'join', 'karma', 'knight', 'lives', 'logic', 'milk', 'most', 'neon', 'night', 'package', 'participation', 'penny', 'pregnant', 'properly', 'quest', 'restoration', 'seventeen', 'social', 'styles', 'supports', 'tech', 'thai', 'thoughts', 'today', 'transformation', 'treaty', 'tribute', 'aesthetic', 'upside', 'behold', 'dough', 'sands', '3-bedroom', 'actor', 'agreements', 'arise', 'assured', 'bubble', 'cereal', 'definitely', 'dime', 'engage', 'erected', 'estate', 'ethical', 'everybody', 'faces', 'feeds', 'haircut', 'halo', 'jacket', 'joining', 'kingdom', 'lifted', 'listened', 'meat', 'menu', 'nurse', 'opening', 'pension', 'phd', 'phones', 'plans', 'premier', 'proposals', 'protein', 'providence', 'recommendations', 'sexual', 'soda', 'spain', 'stable', 'succession', 'supporters', 'taco', 'think', 'trading', 'upward', 'yields', 'sailor', 'dynamics', 'lyrical', 'copper', 'realise', 'righteous', 'transformed', 'venus', '80s', 'advocates', 'aha', 'ate', 'atlantic', 'awareness', 'balance', 'blonde', 'burger', 'buyer', 'certificate', 'chances', 'chief', 'clearly', 'cultural', 'draws', 'driving', 'duck', 'eagle', 'emotions', 'established', 'experiments', 'expression', 'fishing', 'fri', 'fully', 'informed', 'initiated', 'italy', 'king', 'land', 'lion', 'miami', 'midnight', 'mineral', 'nomination', 'oak', 'occasions', 'philosophical', 'playlist', 'profound', 'provided', 'resolution', 'riding', 'safety', 'scientist', 'she', 'sight', 'spice', 'steady', 'survey', 'swiss', 't-shirt', 'tiger', 'tomorrow', 'tourist', 'tournament', 'trade', 'trains', 'tune', 'victor', 'walking', 'wireless', 'www', 'yea', 'beds', 'preference', 'applying', 'crop', 'enable', 'interactions', 'narrative', 'railway', 'afford', 'allowing', 'automobile', 'bands', 'boys', 'cds', 'christ', 'dictionary', 'downloads', 'eagles', 'engaged', 'especially', 'fiction', 'grocery', 'hotel', 'houses', 'hubby', 'included', 'lemon', 'mellow', 'minds', 'my', 'own', 'pacific', 'people', 'planning', 'polish', 'premium', 'providing', 'readers', 'rocked', 'sausage', 'south', 'transportation', 'turkey', 'wed', 'wheels', 'woods', 'yacht', 'livin', 'believing', 'chemistry', 'continuous', 'persons', 'seed', 'sheep', 'successive', 'adult', 'amsterdam', 'arises', 'arrived', 'asleep', 'aviation', 'basketball', 'browser', 'cathedral', 'cd', 'cheek', 'combination', 'conscious', 'cricket', 'debut', 'dividends', 'drinking', 'elizabeth', 'eye', 'generate', 'granted', 'guests', 'huge', 'jumping', 'kindle', 'launches', 'mend', 'models', 'mutual', 'offered', 'places', 'plan', 'principles', 'recovering', 'respectively', 'restore', 'ride', 'rock', 'shirts', 'sony', 'strategies', 'strongly', 'temple', 'thousands', 'tonight', 'trail', 'twin', 'up', 'updates', 'vagina', 'yahoo', 'receives', 'exclusively', 'writings', 'destiny', 'outcomes', 'quicker', 'boulevard', 'chapels', 'consideration', 'digital', 'dish', 'eat-in', 'ensure', 'event', 'everyone', 'face', 'focus', 'funds', 'garlic', 'investing', 'keyboard', 'knows', 'leaf', 'males', 'maps', 'masters', 'networking', 'nursing', 'patiently', 'proceeded', 'proceeding', 'profession', 'robot', 'snowing', 'studied', 'study', 'theme', 'toward', 'traditional', 'treasurer', "university's", 'v-day', 'very', 'voted', 'wii', 'waving', 'extending', 'readily', 'mirrors', 'nearer', 'nurses', 'preserved', 'senses', 'aah', 'acknowledged', 'beers', 'bentley', 'brazil', 'cattle', 'challenging', 'check', 'chili', 'citizens', 'collection', 'comprehend', 'customers', 'elected', 'electricity', 'enters', 'essence', 'fab', 'forthcoming', 'forward', 'guide', 'herself', 'increasingly', 'info', 'investments', 'justification', 'karaoke', 'keeping', 'know', 'launched', "life's", 'madame', 'markets', 'moments', 'nike', 'november', 'open', 'oscar', 'owner', 'practically', 'precise', 'release', 'romans', 'security', 'shade', 'shoulders', 'soap', 'springfield', 'start', 'telecommunications', "tomorrow's", 'trinity', 'western', 'window', 'woof', 'yay', 'roam', 'dawning', 'choir', 'crops', 'elvis', 'significance', 'throne', 'velocity', 'acquainted', 'ahead', 'alright', 'audiences', 'ball', 'belief', 'bff', 'boat', 'boots', 'california', 'centuries', 'cheaper', 'clue', 'coat', 'consensus', 'contact', 'deserved', 'drive', 'facebook', 'freelance', 'greek', 'grown', 'help', 'housing', 'instant', 'integrated', 'introduction', 'legit', 'ma', 'message', 'negotiate', 'neighbor', 'neighborhoods', 'numerous', 'our', 'oven', 'picked', 'reached', 'recognize', 'recognized', 'rider', 'shows', 'significantly', 'specialist', 'suggestions', 'superior', 'tempo', 'tourists', 'ups', 'validity', 'vehicle', 'votes', 'theories', 'associations', 'attachment', 'fluid', 'shells', '1970s', 'adults', 'advocacy', 'bella', 'brazilian', 'bueno', 'certain', 'certainly', 'combinations', 'composed', 'composition', 'couch', 'created', 'creek', 'dimes', 'distinct', 'equal', 'facts', 'flight', 'gaze', 'goodman', 'harbor', 'hey', 'historian', 'host', 'icon', 'influences', 'instruments', 'landmark', 'large', 'latest', 'leads', 'legs', 'liverpool', 'magazines', 'membership', 'muscle', 'nation', 'outlets', 'overseas', 'peanut', 'personal', 'photoshop', 'preparation', 'quantities', 'racing', 'reflection', 'representation', 'respective', 'see', 'servings', 'shoes', 'slim', 'sports', 'starring', 'straight', 'talk', 'towns', 'updated', 'wood', 'solving', 'bridges', 'climbing', 'geographical', 'skirt', '1960s', 'academy', 'accompanying', 'acquired', 'acting', 'alumni', "america's", 'approaches', 'bass', 'beginning', 'bringing', 'campus', 'casino', 'choices', 'contributed', 'exact', 'expand', 'express', 'fave', 'feliz', 'folks', 'fund', 'furniture', 'groove', 'hair', 'hint', 'installed', 'interactive', 'kitchen', 'melbourne', 'mind', 'numbers', 'perspective', 'points', 'prevention', 'professor', 'prospective', 'prospects', 'purple', 'purpose', 'replied', 'sauce', 'signing', 'sofa', 'supplies', 'tops', 'transport', 'union', 'visible', 'vocal', 'washington', 'words', 'xp', 'carriage', 'beings', 'colored', 'considerations', 'nearest', 'porch', 'relate', 'seventeenth', 'vibe', "1980's", 'acres', 'aircraft', 'amen', 'basket', 'blog', 'cards', 'celebrity', 'christians', 'concepts', 'content', 'creates', 'delivery', 'developing', 'doll', 'download', 'eggs', 'engineers', 'essential', 'fixed', 'float', 'fridge', 'fund-raising', 'inn', 'jam', 'japanese', 'male', 'monetary', 'native', 'newspapers', 'objectives', 'pregnancy', 'presence', 'production', 'programs', 'pub', 'quick', 'rare', 'records', 'retire', 'simple', 'sophisticated', 'teams', 'totally', 'try', 'unwind', 'voting', 'walk', 'will', 'windows', 'wondering', 'writes', 'xoxo', 'rains', "1990's", 'act', 'adapted', 'alliance', 'allow', 'applicable', 'archives', 'attend', 'attending', 'automatic', 'automatically', 'avatar', 'beans', 'beliefs', 'bien', 'biggest', 'brew', 'brook', 'cambridge', 'concentrations', 'conscience', 'continent', 'crimson', 'eighteenth', 'exactly', 'extend', 'favor', 'finale', 'find', 'fireplace', 'fixing', 'glance', 'global', 'ha', 'hands', 'heating', 'indeed', 'integral', 'itunes', 'japan', 'jenny', "king's", 'lawn', 'lighting', 'likewise', 'lmfao', 'make', 'meaning', 'mega', 'metals', 'mucho', 'nations', 'network', 'olive', 'opened', 'oregon', 'owns', 'participants', 'pilot', 'principle', 'religion', 'result', 'service', 'sights', 'sites', 'sponsor', 'started', 'stereo', 'stores', 'successor', 'survive', 'surviving', "today's", 'tuned', 'virgin', 'vista', 'walked', "ain't", 'alleged', 'arafat', 'bum', 'ceased', 'cracks', 'creeping', 'defensive', "didn't", 'didnt', 'downs', 'force', 'least', 'limits', 'racial', 'ridiculous', 'rip', 'roughly', 'twit', 'zombies', 'accidentally', 'avoided', 'bite', 'breaking', 'demands', 'diagnosis', 'fled', 'hardly', 'humidity', 'isnt', 'old', 'punks', 'terminal', 'ruins', 'cracked', 'slam', 'argh', 'bang', 'bye', 'closing', 'dagger', 'expense', 'fists', 'iraqi', 'loose', 'minus', 'slugs', 'strike', 'tough', 'trial', 'unclear', 'killa', 'skull', 'charges', 'darker', 'erroneously', 'mess', 'pakistan', 'reluctant', 'slumdog', 'strapped', 'dizzy', 'executed', 'honky', 'homework', 'nixon', 'omitted', 'stained', 'ughh', 'jaded', 'dusty', 'absent', 'alarm', 'artificial', 'defendant', 'dim', 'doesnt', 'impose', 'iraq', 'issues', 'killas', 'misses', 'neediest', 'nothing', 'opponent', 'quit', 'slipping', 'stop', 'bald', 'begged', 'dropped', 'drunk', 'mortgage', 'nooo', 'shout', 'artillery', 'goddamn', 'rags', 'restless', 'uncertain', 'fiends', 'ass', 'farewell', 'fuckin', 'hang', 'not', 'sanctions', 'stopped', 'subjected', 'tremble', 'voodoo', 'wouldnt', 'slipped', 'mold', 'shiver', 'allegations', 'armed', 'ended', 'excuses', 'gripe', 'lawyer', 'messed', 'none', 'offline', 'pleaded', 'rent', "shouldn't", 'snatch', 'ghosts', 'hatin', 'fragile', 'baddest', 'blood', 'creep', 'dark', 'darkness', 'eliminate', 'forgetting', 'gang', 'hanging', 'hardest', "haven't", 'junk', 'loans', 'oppose', 'slip', 'sos', 'thirst', 'erased', 'vain', 'fades', 'aggressive', 'costs', 'critics', 'fire', 'fist', 'interment', 'ow', 'pale', 'protesters', 'witch', 'chronic', 'thirsty', 'thorns', 'sink', 'battles', 'bugs', 'court', 'ends', 'exams', 'predeceased', 'risks', 'rusty', 'slow', "wouldn't", 'bothered', 'unnecessary', 'nothings', 'resigned', 'symptoms', 'yell', 'gutter', 'hangs', 'void', 'bailout', 'boo', "critic's", 'denying', 'last', 'noise', 'obsession', 'reckless', 'shove', 'stomp', 'wait', 'sucka', 'pimp', 'stranded', 'tearing', 'strain', 'crack', 'fewer', 'gross', 'kick', 'oops', 'operation', 'removed', 'withdrawal', 'crowded', 'lacking', 'revenge', 'foolish', 'con', 'crooked', 'demanding', 'dirt', "don't", 'dont', 'goodbye', 'locked', 'remove', 'sentenced', 'wasnt', "won't", 'abnormal', 'hustler', 'controversy', 'disagree', 'fees', 'hitting', 'kicking', 'mean', 'missed', 'rival', 'sucker', 'waiting', 'wrath', 'plead', 'closed', 'deadline', 'down', 'low', 'messy', 'outdated', 'patients', 'pressure', 'snitch', 'sorry', 'stuck', 'anti', 'complications', 'disappear', 'snakes', 'lesions', 'bill', 'blocked', 'bore', 'cuts', 'darkest', 'delete', 'ghost', 'miss', 'nobody', 'nothin', 'shocked', 'swine', 'uncertainty', 'fooled', 'awkward', 'baghdad', 'begging', 'brat', "doesn't", 'haunt', 'hussein', 'incompletely', 'limitations', 'risk', 'tore', 'bacteria', 'crude', 'dust', 'falls', 'flies', 'indicted', 'madness', 'mistaken', 'shattered', 'suspects', 'acid', 'pistol', 'decreased', 'absence', 'couldnt', 'excluded', 'gossip', 'leaving', 'punch', 'shotgun', 'sirens', 'restricted', 'darkened', 'slut', 'servants', 'afghanistan', 'confrontation', 'confusing', 'denial', 'empty', 'fucked', 'gloom', 'misidentified', 'mob', 'offense', 'piss', 'protest', 'runaway', 'shut', 'sorely', 'dire', 'stains', 'taxation', 'flee', 'haunted', 'bug', 'caught', 'chained', 'crushed', 'despise', 'dispute', 'expensive', 'forsaken', 'hospitals', 'owe', "poor's", 'rough', 'shock', 'slug', 'without', 'drunken', 'missin', 'separation', 'spite', 'addicted', 'apart', 'fallen', 'suspected', 'suspicion', 'teardrops', 'tomb', 'ugh', 'warned', 'untrue', 'casket', 'dope', 'foe', 'hospital', 'paranoid', 'snake', 'struck', 'deficiency', 'pressures', 'cant', 'inmates', 'no', 'opponents', 'opposition', 'sucked', 'tobacco', 'unlikely', 'zombie', 'screams', 'sinking', 'swollen', 'deceive', 'monsters', 'urine', 'chaos', 'creepy', 'fee', 'insanity', 'isolated', 'late', 'misspelled', 'misstated', 'misunderstood', 'monster', 'refuse', 'shoot', 'sting', 'thorn', 'wreck', 'fright', 'radiation', 'stab', 'confined', 'delays', 'deny', 'fault', 'forgot', 'ghetto', 'litigation', 'poop', 'seized', 'zero', 'cage', 'disappeared', 'trap', 'diss', 'foes', 'smashed', 'anxious', "can't", 'cut', 'erroneous', 'gangsta', 'gone', 'ignorant', 'invasion', 'lame', 'obsessed', 'raging', 'shatter', 'shouting', 'troubles', 'disturbed', 'zit', 'against', 'condolences', 'muthafucka', 'separated', 'struggle', 'whores', 'deception', 'stain', 'unconscious', 'delay', 'difficulty', 'discontinued', 'eliminated', 'haunting', 'hungry', 'refused', 'wicked', 'blinded', 'hunger', 'torn', 'phony', 'argued', 'beast', 'bullet', 'busted', 'critic', 'dammit', 'deleted', 'dentist', 'forbidden', 'killin', 'syndrome', 'tornado', 'weapon', 'emptiness', 'injection', 'burnt', 'complicated', 'crap', 'never', 'politicians', 'tired', 'traffic', 'unfair', 'vulnerable', 'warning', 'fucker', 'sinner', 'envy', 'whack', 'alone', 'bleeds', 'cannot', 'confusion', "couldn't", 'expenses', 'ignored', 'nigga', 'noose', 'opposed', 'restrictions', 'scars', 'shots', 'savage', 'choke', 'cigarettes', 'doubts', 'fool', 'fury', 'lowest', 'suckers', 'whip', 'helpless', 'rats', 'conspiracy', 'crashing', 'falling', 'fools', 'lazy', 'nuclear', 'scar', 'suspicious', 'scarred', 'screamed', 'cough', 'damned', 'frown', 'pimps', 'vengeance', 'canceled', 'cavity', 'delayed', 'dull', 'fat', 'jerk', 'missile', 'remorse', 'rot', 'screwed', 'gangstas', 'captured', 'critical', 'fell', 'forget', 'freezing', 'ignore', 'losers', 'lynch', 'wasting', 'defect', 'frightened', 'combat', 'convicted', 'defeat', 'dirty', 'dread', 'drug', 'inferior', 'screamin', 'cryin', 'liar', 'aching', 'difficult', 'faggot', 'false', 'forgotten', 'garbage', 'kicked', 'scandal', 'sinners', 'suspension', 'woe', 'accusations', 'complain', 'declined', 'disorders', 'doubt', 'forced', 'lack', 'severe', 'smoke', 'yuck', 'feared', 'gangster', 'argument', 'avoid', 'bitch', 'bruise', 'dismissed', 'disorder', 'exhausted', 'incorrectly', 'isolation', 'scream', 'slapped', 'spit', 'suck', 'sucks', 'suspect', 'whore', 'wrong', 'cursed', 'doom', 'desperate', 'lonesome', 'regret', 'rob', 'defects', 'ambulance', 'annoy', 'conflict', 'criticism', 'execution', 'fought', 'indictment', 'pity', 'smoking', 'stink', 'tear', 'unable', 'cigarette', 'beg', 'prejudice', 'bullshit', 'decay', 'decline', 'deficit', 'difficulties', 'graves', 'regrets', 'suspended', 'trapped', 'yelling', 'aging', 'arguing', 'bullets', 'dumb', 'emergency', 'greed', 'idiot', 'idiots', 'inadequate', 'refugees', 'turmoil', 'rotting', 'greedy', 'havoc', 'arguments', 'bled', 'bored', 'complaints', 'horror', 'insane', 'jealousy', 'lawsuits', 'rat', 'resignation', 'scare', 'anxiety', 'fiend', 'hostile', 'weeping', 'broken', 'criticized', 'offensive', 'trembling', 'argue', 'argues', 'bitter', 'condemned', 'fights', 'muthafuckin', 'vicious', 'battle', 'confused', 'crappy', 'damn', 'guns', 'ignorance', 'missing', 'niggaz', 'problem', 'worthless', 'insecure', 'coffin', 'conflicts', 'damages', 'lawsuit', 'niggas', 'screaming', 'wound', 'bloody', 'cemetery', 'choking', 'explosion', 'foul', 'nervous', 'sore', 'tension', 'thief', 'thug', 'unfortunate', 'weakness', 'breakdown', 'bury', 'accused', 'awful', 'burn', 'cries', 'hangover', 'mistakes', 'problems', 'riot', 'sleepless', 'demon', 'boring', 'bruised', 'burned', 'collapse', 'complained', 'debt', 'fake', 'frustrated', 'impossible', 'ouch', 'deadly', 'disrespect', 'drown', 'badly', 'banned', 'burning', 'cancelled', 'dislike', 'threats', 'sins', 'bombs', 'complaint', 'errors', 'illegal', 'lonely', 'mourns', 'prisoner', 'stress', 'tax', 'violations', 'widow', 'addict', 'buried', 'devils', 'dump', 'hater', 'incorrect', 'infection', 'neglected', 'penalty', 'terrible', 'unkind', 'weak', 'annoying', 'bills', 'blame', 'burden', 'complaining', 'danger', 'demise', 'despair', 'disabled', 'discrimination', 'filthy', 'gun', 'lied', 'missiles', 'mourners', 'obituary', 'prosecution', 'worry', 'mafia', 'wounds', 'burns', 'cowards', 'fever', 'mistake', 'trouble', 'troubled', 'wasted', 'bitches', 'bleeding', 'fighting', 'lose', 'lost', 'pathetic', 'unfortunately', 'neglect', 'defeated', 'loses', 'stressed', 'ugly', 'violation', 'unholy', 'addiction', 'arrests', 'disgrace', 'heartbreaker', 'mourn', 'struggling', 'desperation', 'distress', 'fight', 'spam', 'taxes', 'waste', 'worse', 'sorrows', 'bleed', 'ache', 'bastards', 'fears', 'injuries', 'jealous', 'misery', 'ruin', 'shame', 'stupid', 'trash', 'deaf', 'afraid', 'ban', 'drugs', 'loneliness', 'penalties', 'surgery', 'tensions', 'bad', 'curse', 'demons', 'enemy', 'guilty', 'inflation', 'motherfucking', 'sin', 'heartaches', '#fail', 'beaten', 'lies', 'losing', 'nasty', 'retarded', 'rude', 'threatened', 'violated', 'thugs', 'abortion', 'brutal', 'crash', 'error', 'lie', 'mad', 'selfish', 'stole', 'worries', 'ashamed', 'infections', 'annoyed', 'blind', 'cheated', 'damage', 'disgusting', 'guilt', 'lying', 'motherfuckin', 'rotten', 'scared', 'scary', 'shitty', 'starving', 'stroke', 'betrayed', 'nightmares', 'assault', 'beating', 'grave', 'hopeless', 'loss', 'rage', 'satan', 'upset', 'corpse', 'abandoned', 'broke', 'cocaine', 'denied', 'harm', 'hurricane', 'miserable', 'pissed', 'ruined', 'tumor', 'attacked', 'bastard', 'destroy', 'failing', 'shooting', 'useless', 'motherfuckers', 'betray', 'psycho', 'shit', 'shot', 'stolen', 'crisis', 'damaged', 'haters', 'recession', 'saddam', 'slap', 'attacks', 'crashed', 'losses', 'panic', 'steal', 'stealing', 'tears', 'burial', 'cheat', 'dangerous', 'drowning', 'enemies', 'hating', 'prisoners', 'saddened', 'arrest', 'attack', 'flood', 'ill', 'killer', 'negative', 'worried', 'wounded', 'nigger', 'slaughter', 'asshole', 'flu', 'weapons', 'graveside', 'sad', 'victim', 'hurting', 'threat', 'frustration', 'hate', 'tragic', 'grief', 'accident', 'angry', 'fear', 'nightmare', 'poor', 'victims', 'anger', 'fired', 'fraud', 'theft', 'thieves', 'heartache', 'sadly', 'cheating', 'destruction', 'disappointed', 'bombing', 'devil', 'horrible', 'suffered', 'hatred', 'weep', 'hell', 'holocaust', 'injured', 'suffering', 'cried', 'crime', 'loser', 'depressed', 'divorce', 'hurt', 'robbed', 'tsunami', 'agony', 'drowned', 'homeless', 'pollution', 'corruption', 'crimes', 'hated', 'hurts', 'painful', 'sorrow', 'unemployment', 'unhappy', 'heartbreak', 'dying', 'funeral', 'pain', 'worst', 'dies', 'racist', 'rejected', 'robbery', 'suffer', 'virus', 'bankruptcy', 'fails', 'failure', 'hates', 'prison', 'slave', 'slaves', 'tragedy', 'violent', 'crying', 'destroyed', 'injury', 'rejection', 'motherfucker', 'sick', 'slavery', 'dead', 'disease', 'illness', 'killers', 'punishment', 'criminal', 'depression', 'headache', 'poverty', 'tumors', 'bomb', 'disaster', 'fail', 'poison', 'depressing', 'earthquake', 'evil', 'wars', 'abuse', 'diseases', 'sadness', 'violence', 'cruel', 'cry', 'failed', 'sickness', 'abused', 'tortured', 'fatal', 'killings', 'murdered', 'war', 'kills', 'jail', 'terror', 'die', 'killing', 'arrested', 'deaths', 'raped', 'torture', 'died', 'kill', 'killed', 'cancer', 'death', 'murder', 'terrorism', 'rape', 'suicide', 'terrorist'])

Build single shed words freq dict for topic_news docs

Result single dict format (for all topic_news docs)

{topic_ind_0: {
    news_native_id_0_0: {shed_word_0_ind: shed_word_0_freq,
                         shed_word_1_ind: shed_word_1_freq,
                         ...},
    news_native_id_0_1: {shed_word_0_ind: shed_word_0_freq,
                         shed_word_1_ind: shed_word_1_freq,
                         ...},
    ...},
topic_ind_1: {
    news_native_id_1_0: {shed_word_0_ind: shed_word_0_freq,
                         shed_word_1_ind: shed_word_1_freq,
                         ...},
    news_native_id_1_1: {shed_word_0_ind: shed_word_0_freq,
                         shed_word_1_ind: shed_word_1_freq,
                         ...},
    ...},
...}

Build single shed words freq dict for all topic_news docs


In [4]:
%%time
"""
Build single shed words freq dict for all topic_news docs

Register
    TOPICS_NEWS_SHED_WORDS_FREQ_DICT_PKL = os.path.join(DATA_DIR, 'topics_news_shed_words_freq.dict.pkl')
in config
"""
if 0 == 1:
    topics_news_shed_words_freq_dict = {}
    
    for topic_ind, topic in enumerate(config.MANUALLY_SELECTED_TOPICS_LST):
        localtime = time.asctime(time.localtime(time.time()))
        print('({}/{}) processing topic: {} ... {}'.format(topic_ind+1,
                                                           len(config.MANUALLY_SELECTED_TOPICS_LST),
                                                           topic['name'],
                                                           localtime))
        
        topic_shed_words_freq_dict = {}
        
        '''
        Load shed_word and shed_word_ind mapping pkls
        '''
        ind_shed_word_dict = pd.read_pickle(config.IND_SHED_WORD_DICT_PKL)
        shed_word_ind_dict = pd.read_pickle(config.SHED_WORD_IND_DICT_PKL)
        shed_words_set = set(ind_shed_word_dict.values())
        
        '''
        Load topic_news doc
        '''
        csv.register_dialect('topics_docs_line', delimiter='\t', doublequote=True, quoting=csv.QUOTE_ALL)
        topic_news_csv_file = os.path.join(config.TOPICS_DOCS_DIR, '{}-{}.news.csv'.format(topic_ind, topic['name']))
        with open(topic_news_csv_file, 'r') as f:
            reader = csv.DictReader(f, dialect='topics_docs_line')
            '''
            Count shed words freq for each tweet
            '''
            # lazy load
            for row in reader:
                news_native_id = int(row['news_native_id'])
                news_doc = row['news_doc']
                
                news_doc_shed_words_freq_dict = utilities.count_news_doc_shed_words_freq(news_doc, ind_shed_word_dict, shed_word_ind_dict, shed_words_set)
                
                topic_shed_words_freq_dict[news_native_id] = news_doc_shed_words_freq_dict
        
        topics_news_shed_words_freq_dict[topic_ind] = topic_shed_words_freq_dict
    
    '''
    Make pkl for result single dict
    '''
    with open(config.TOPICS_NEWS_SHED_WORDS_FREQ_DICT_PKL, 'wb') as f:
        pickle.dump(topics_news_shed_words_freq_dict, f)


(1/51) processing topic: Hillary_Clinton_email_controversy ... Sun Oct 22 18:49:53 2017
(2/51) processing topic: Iran_nuclear_deal ... Sun Oct 22 18:49:53 2017
(3/51) processing topic: ISIS_Jihadi_John_identity_reveal ... Sun Oct 22 18:49:53 2017
(4/51) processing topic: Ukraine_cease_fire ... Sun Oct 22 18:49:54 2017
(5/51) processing topic: Egypt_free_Al_Jazeera_journalist ... Sun Oct 22 18:49:54 2017
(6/51) processing topic: Keystone_XL_Pipeline_bill ... Sun Oct 22 18:49:54 2017
(7/51) processing topic: CIA_Torture_Report ... Sun Oct 22 18:49:54 2017
(8/51) processing topic: Obama_cybersecurity_plan ... Sun Oct 22 18:49:54 2017
(9/51) processing topic: DHS_funding_issue ... Sun Oct 22 18:49:54 2017
(10/51) processing topic: US_Cuba_relationship ... Sun Oct 22 18:49:54 2017
(11/51) processing topic: 2015_CPAC ... Sun Oct 22 18:49:54 2017
(12/51) processing topic: Iraq_free_ISIS_Tikrit ... Sun Oct 22 18:49:54 2017
(13/51) processing topic: Nigeria_Boko_Haram_terrorists ... Sun Oct 22 18:49:54 2017
(14/51) processing topic: Ferguson_unrest ... Sun Oct 22 18:49:54 2017
(15/51) processing topic: Hong_Kong_protest ... Sun Oct 22 18:49:55 2017
(16/51) processing topic: Sony_cyberattack ... Sun Oct 22 18:49:55 2017
(17/51) processing topic: Bill_Cosby_sexual_assault_allegation ... Sun Oct 22 18:49:55 2017
(18/51) processing topic: SpaceX_rocket_landing ... Sun Oct 22 18:49:55 2017
(19/51) processing topic: Brian_Williams_fake_story ... Sun Oct 22 18:49:55 2017
(20/51) processing topic: HSBC_tax_scandal ... Sun Oct 22 18:49:55 2017
(21/51) processing topic: David_Carr_death ... Sun Oct 22 18:49:55 2017
(22/51) processing topic: Patriots_Deflategate ... Sun Oct 22 18:49:55 2017
(23/51) processing topic: Delhi_Uber_driver_rape ... Sun Oct 22 18:49:55 2017
(24/51) processing topic: Superbug_spread ... Sun Oct 22 18:49:55 2017
(25/51) processing topic: Rudy_Giuliani_Obama_critique ... Sun Oct 22 18:49:55 2017
(26/51) processing topic: Oscar ... Sun Oct 22 18:49:55 2017
(27/51) processing topic: Super_Bowl ... Sun Oct 22 18:49:55 2017
(28/51) processing topic: Grammy ... Sun Oct 22 18:49:56 2017
(29/51) processing topic: Golden_Globe ... Sun Oct 22 18:49:56 2017
(30/51) processing topic: 500_million_Powerball ... Sun Oct 22 18:49:56 2017
(31/51) processing topic: Thanksgiving ... Sun Oct 22 18:49:56 2017
(32/51) processing topic: Black_Friday_and_Cyber_Monday ... Sun Oct 22 18:49:56 2017
(33/51) processing topic: Christmas ... Sun Oct 22 18:49:56 2017
(34/51) processing topic: New_Year ... Sun Oct 22 18:49:56 2017
(35/51) processing topic: Apple_Watch ... Sun Oct 22 18:49:56 2017
(36/51) processing topic: Yosemite_historic_climb ... Sun Oct 22 18:49:56 2017
(37/51) processing topic: Jon_Stewart_Daily_Show ... Sun Oct 22 18:49:56 2017
(38/51) processing topic: success_of_American_Sniper ... Sun Oct 22 18:49:56 2017
(39/51) processing topic: Ebola_virus_spread ... Sun Oct 22 18:49:56 2017
(40/51) processing topic: Indonesia_AirAsia_Flight_QZ8501_crash ... Sun Oct 22 18:49:56 2017
(41/51) processing topic: Paris_attacks ... Sun Oct 22 18:49:56 2017
(42/51) processing topic: Vanuatu_Cyclone_Pam ... Sun Oct 22 18:49:57 2017
(43/51) processing topic: Malaysia_Airlines_Flight_MH370_crash ... Sun Oct 22 18:49:57 2017
(44/51) processing topic: Colorado_NAACP_bombing ... Sun Oct 22 18:49:57 2017
(45/51) processing topic: FSU_shooting ... Sun Oct 22 18:49:57 2017
(46/51) processing topic: Chapel_Hill_shooting ... Sun Oct 22 18:49:57 2017
(47/51) processing topic: Bobbi_Kristina_Brown_death ... Sun Oct 22 18:49:57 2017
(48/51) processing topic: Taliban_Pakistan_school_massacre ... Sun Oct 22 18:49:57 2017
(49/51) processing topic: American_ISIS_Hostage_Kayla_Mueller ... Sun Oct 22 18:49:57 2017
(50/51) processing topic: TransAsia_Airways_Flight_GE235_crash ... Sun Oct 22 18:49:57 2017
(51/51) processing topic: Germanwings_Flight_9525_crash ... Sun Oct 22 18:49:57 2017
CPU times: user 3.76 s, sys: 124 ms, total: 3.88 s
Wall time: 4.04 s

Check basic statistics


In [6]:
"""
Print out sample news shed_words_freq_dicts inside single topic
"""
if 0 == 1:
    target_topic_ind = 0
    
    with open(config.TOPICS_NEWS_SHED_WORDS_FREQ_DICT_PKL, 'rb') as f:
        topics_news_shed_words_freq_dict = pickle.load(f)
        
    count = 0
    for news_native_id, news_doc_shed_words_freq_dict in topics_news_shed_words_freq_dict[target_topic_ind].items():
        print('news_native_id: {}'.format(news_native_id))
        print('\t{}'.format(news_doc_shed_words_freq_dict))
        news_doc_shed_words_len = sum(news_doc_shed_words_freq_dict.values())
        print('\tLEN: {}'.format(news_doc_shed_words_len))
        count += 1
        if count >= 5:
            break


news_native_id: 30275
	{2228: 1, 1221: 15, 2773: 10, 2575: 9, 2111: 3, 704: 12, 2290: 1, 2451: 7, 1644: 1, 1497: 1, 1341: 3, 788: 1, 791: 1, 2387: 2, 2106: 1, 1361: 1, 1504: 1, 416: 1, 1656: 6, 3718: 1, 1294: 1, 3586: 1, 1986: 1, 2856: 2, 733: 1, 1884: 1, 1849: 1, 1326: 1, 1577: 2, 1767: 1, 2922: 2, 2324: 1, 3555: 1, 3685: 1, 1008: 1, 460: 1, 1198: 1, 1260: 1, 1678: 1, 3393: 1, 1949: 3, 52: 1, 2592: 1, 2416: 1, 2204: 1, 3069: 1, 1916: 1, 1828: 4, 996: 1, 2901: 1, 2637: 1, 2299: 1, 2330: 1, 758: 1, 3674: 1, 779: 1, 3196: 3, 2653: 1, 3032: 1, 1492: 1, 2597: 2, 2908: 1, 849: 1, 1116: 2, 2439: 1, 3122: 1, 3374: 1, 383: 1, 2308: 1, 2718: 1, 1991: 1, 1934: 1, 1195: 1, 2248: 1}
	LEN: 144
news_native_id: 30282
	{1690: 1, 2751: 1, 2451: 2, 704: 6, 791: 1, 3317: 1, 1341: 1, 539: 1, 1257: 1, 1283: 1, 2278: 1, 3217: 1, 3174: 1, 1607: 1, 2139: 1, 3069: 1, 2597: 1, 2575: 3, 2111: 2, 1127: 1, 3110: 1, 1504: 2, 1656: 1, 3570: 1, 1664: 1, 1221: 3, 301: 1, 788: 1, 1118: 1, 1849: 1, 2908: 1, 3134: 1, 2773: 2, 2332: 1, 2412: 1, 2245: 1, 930: 1, 2587: 1, 3233: 1, 2273: 1, 1916: 1, 2653: 1, 2490: 1, 1120: 1, 1892: 1, 2602: 1, 2394: 1}
	LEN: 60
news_native_id: 30290
	{2228: 1, 1221: 15, 2773: 10, 2575: 9, 2111: 3, 704: 12, 2290: 1, 2451: 7, 1644: 1, 1497: 1, 1341: 3, 788: 1, 791: 1, 2387: 2, 2106: 1, 1361: 1, 1504: 1, 416: 1, 1656: 6, 3718: 1, 1294: 1, 3586: 1, 1986: 1, 2856: 2, 733: 1, 1884: 1, 1849: 1, 1326: 1, 1577: 2, 1767: 1, 2922: 2, 2324: 1, 3555: 1, 3685: 1, 1008: 1, 460: 1, 1198: 1, 1260: 1, 1678: 1, 3393: 1, 1949: 3, 52: 1, 2592: 1, 2416: 1, 2204: 1, 3069: 1, 1916: 1, 1828: 4, 996: 1, 2901: 1, 2637: 1, 2299: 1, 2330: 1, 758: 1, 3674: 1, 779: 1, 3196: 3, 2653: 1, 3032: 1, 1492: 1, 2597: 2, 2908: 1, 849: 1, 1116: 2, 2439: 1, 3122: 1, 3374: 1, 383: 1, 2308: 1, 2718: 1, 1991: 1, 1934: 1, 1195: 1, 2248: 1}
	LEN: 144
news_native_id: 30304
	{2773: 1, 1940: 3, 2192: 1, 1249: 1, 455: 1, 2387: 1, 1618: 1, 2368: 1}
	LEN: 10
news_native_id: 30310
	{1336: 1, 2451: 5, 1184: 2, 3504: 1, 791: 1, 2601: 1, 2287: 2, 2773: 2, 704: 2, 3032: 1, 1221: 2, 2575: 2, 1644: 1, 1718: 1, 1171: 1, 2663: 1, 3084: 1, 1127: 1, 459: 1, 550: 1, 11: 1, 1690: 1, 2100: 1, 2111: 1, 1415: 1, 1940: 1, 2459: 1, 2440: 1, 1544: 1, 1031: 1, 1949: 1, 1668: 1, 2102: 1}
	LEN: 43

In [9]:
%%time
"""
Check total shed words length of this topic_news doc
"""  
if 0 == 1:
    topic_news_shed_words_len = sum([sum(news_doc_shed_words_freq_dict.values()) for news_doc_shed_words_freq_dict in topics_news_shed_words_freq_dict[target_topic_ind].values()])
    print('Total shed words length of this topic_news doc: {}'.format(topic_news_shed_words_len))


Total shed words length of this topic_news doc: 22325
CPU times: user 0 ns, sys: 0 ns, total: 0 ns
Wall time: 1.06 ms

Build shed words freq dicts for each topic_tweets doc separately

Result dict format (for each given topic_tweets doc)

{tweet_id_0_0: {shed_word_0_ind: shed_word_0_freq,
                shed_word_1_ind: shed_word_1_freq,
                ...},
tweet_id_0_1: {shed_word_0_ind: shed_word_0_freq,
               shed_word_1_ind: shed_word_1_freq,
               ...},
...}

Build shed words freq dict for each topic separately


In [2]:
%%time
"""
Build shed words freq dict for each topic separately

Register
    TOPICS_TWEETS_SHED_WORDS_FREQ_DICT_PKLS_DIR = os.path.join(DATA_DIR, 'topics_tweets_shed_words_freq_dict_pkls')
in config

Note:
 - Number of tweets is large. Process each topic_tweets doc individually to avoid crash
 - Execute second time for updated topic_tweets docs
"""
if 0 == 1:
    for topic_ind, topic in enumerate(config.MANUALLY_SELECTED_TOPICS_LST):
        localtime = time.asctime(time.localtime(time.time()))
        print('({}/{}) processing topic: {} ... {}'.format(topic_ind+1,
                                                           len(config.MANUALLY_SELECTED_TOPICS_LST),
                                                           topic['name'],
                                                           localtime))
        
        topic_shed_words_freq_dict = {}
        
        '''
        Load shed_word and shed_word_ind mapping pkls
        '''
        ind_shed_word_dict = pd.read_pickle(config.IND_SHED_WORD_DICT_PKL)
        shed_word_ind_dict = pd.read_pickle(config.SHED_WORD_IND_DICT_PKL)
        shed_words_set = set(ind_shed_word_dict.values())
        
        '''
        Load topic_tweets doc
        '''
        csv.register_dialect('topics_docs_line', delimiter='\t', doublequote=True, quoting=csv.QUOTE_ALL)
        topic_tweets_csv_file = os.path.join(config.TOPICS_DOCS_DIR, '{}-{}.updated.tweets.csv'.format(topic_ind, topic['name']))
        with open(topic_tweets_csv_file, 'r') as f:
            reader = csv.DictReader(f, dialect='topics_docs_line')
            
            '''
            Count shed words freq for each tweet
            '''
            # lazy load
            for row in reader:
                tweet_id = int(row['tweet_id'])
                tweet_text = row['tweet_text']
                
                tweet_shed_words_freq_dict = utilities.count_tweet_shed_words_freq(tweet_text, ind_shed_word_dict, shed_word_ind_dict, shed_words_set)
                
                topic_shed_words_freq_dict[tweet_id] = tweet_shed_words_freq_dict
        
        '''
        Make pkl for result dict file
        '''
        topic_tweets_shed_words_freq_dict_pkl_file = os.path.join(config.TOPICS_TWEETS_SHED_WORDS_FREQ_DICT_PKLS_DIR,
                                                                  '{}.updated.dict.pkl'.format(topic_ind))
        with open(topic_tweets_shed_words_freq_dict_pkl_file, 'wb') as f:
            pickle.dump(topic_shed_words_freq_dict, f)


(1/51) processing topic: Hillary_Clinton_email_controversy ... Mon Oct 23 16:00:01 2017
(2/51) processing topic: Iran_nuclear_deal ... Mon Oct 23 16:00:23 2017
(3/51) processing topic: ISIS_Jihadi_John_identity_reveal ... Mon Oct 23 16:01:36 2017
(4/51) processing topic: Ukraine_cease_fire ... Mon Oct 23 16:01:54 2017
(5/51) processing topic: Egypt_free_Al_Jazeera_journalist ... Mon Oct 23 16:02:08 2017
(6/51) processing topic: Keystone_XL_Pipeline_bill ... Mon Oct 23 16:02:10 2017
(7/51) processing topic: CIA_Torture_Report ... Mon Oct 23 16:02:12 2017
(8/51) processing topic: Obama_cybersecurity_plan ... Mon Oct 23 16:02:17 2017
(9/51) processing topic: DHS_funding_issue ... Mon Oct 23 16:02:31 2017
(10/51) processing topic: US_Cuba_relationship ... Mon Oct 23 16:02:35 2017
(11/51) processing topic: 2015_CPAC ... Mon Oct 23 16:03:19 2017
(12/51) processing topic: Iraq_free_ISIS_Tikrit ... Mon Oct 23 16:03:26 2017
(13/51) processing topic: Nigeria_Boko_Haram_terrorists ... Mon Oct 23 16:03:39 2017
(14/51) processing topic: Ferguson_unrest ... Mon Oct 23 16:03:50 2017
(15/51) processing topic: Hong_Kong_protest ... Mon Oct 23 16:05:25 2017
(16/51) processing topic: Sony_cyberattack ... Mon Oct 23 16:05:31 2017
(17/51) processing topic: Bill_Cosby_sexual_assault_allegation ... Mon Oct 23 16:06:24 2017
(18/51) processing topic: SpaceX_rocket_landing ... Mon Oct 23 16:06:38 2017
(19/51) processing topic: Brian_Williams_fake_story ... Mon Oct 23 16:06:47 2017
(20/51) processing topic: HSBC_tax_scandal ... Mon Oct 23 16:06:55 2017
(21/51) processing topic: David_Carr_death ... Mon Oct 23 16:06:57 2017
(22/51) processing topic: Patriots_Deflategate ... Mon Oct 23 16:06:59 2017
(23/51) processing topic: Delhi_Uber_driver_rape ... Mon Oct 23 16:07:08 2017
(24/51) processing topic: Superbug_spread ... Mon Oct 23 16:07:19 2017
(25/51) processing topic: Rudy_Giuliani_Obama_critique ... Mon Oct 23 16:07:28 2017
(26/51) processing topic: Oscar ... Mon Oct 23 16:07:40 2017
(27/51) processing topic: Super_Bowl ... Mon Oct 23 16:08:36 2017
(28/51) processing topic: Grammy ... Mon Oct 23 16:09:30 2017
(29/51) processing topic: Golden_Globe ... Mon Oct 23 16:09:52 2017
(30/51) processing topic: 500_million_Powerball ... Mon Oct 23 16:10:16 2017
(31/51) processing topic: Thanksgiving ... Mon Oct 23 16:10:29 2017
(32/51) processing topic: Black_Friday_and_Cyber_Monday ... Mon Oct 23 16:11:50 2017
(33/51) processing topic: Christmas ... Mon Oct 23 16:12:46 2017
(34/51) processing topic: New_Year ... Mon Oct 23 16:14:50 2017
(35/51) processing topic: Apple_Watch ... Mon Oct 23 16:15:32 2017
(36/51) processing topic: Yosemite_historic_climb ... Mon Oct 23 16:15:47 2017
(37/51) processing topic: Jon_Stewart_Daily_Show ... Mon Oct 23 16:15:48 2017
(38/51) processing topic: success_of_American_Sniper ... Mon Oct 23 16:15:52 2017
(39/51) processing topic: Ebola_virus_spread ... Mon Oct 23 16:16:15 2017
(40/51) processing topic: Indonesia_AirAsia_Flight_QZ8501_crash ... Mon Oct 23 16:16:41 2017
(41/51) processing topic: Paris_attacks ... Mon Oct 23 16:17:07 2017
(42/51) processing topic: Vanuatu_Cyclone_Pam ... Mon Oct 23 16:17:46 2017
(43/51) processing topic: Malaysia_Airlines_Flight_MH370_crash ... Mon Oct 23 16:17:51 2017
(44/51) processing topic: Colorado_NAACP_bombing ... Mon Oct 23 16:18:05 2017
(45/51) processing topic: FSU_shooting ... Mon Oct 23 16:18:12 2017
(46/51) processing topic: Chapel_Hill_shooting ... Mon Oct 23 16:18:18 2017
(47/51) processing topic: Bobbi_Kristina_Brown_death ... Mon Oct 23 16:18:20 2017
(48/51) processing topic: Taliban_Pakistan_school_massacre ... Mon Oct 23 16:18:23 2017
(49/51) processing topic: American_ISIS_Hostage_Kayla_Mueller ... Mon Oct 23 16:18:30 2017
(50/51) processing topic: TransAsia_Airways_Flight_GE235_crash ... Mon Oct 23 16:18:31 2017
(51/51) processing topic: Germanwings_Flight_9525_crash ... Mon Oct 23 16:18:36 2017
CPU times: user 18min 25s, sys: 18.1 s, total: 18min 43s
Wall time: 18min 47s

Check basic statistics


In [2]:
%%time
"""
Print out sample tweet shed_words_freq_dicts inside single topic
"""
if 0 == 1:
    target_topic_ind = 0
    
    topic_tweets_shed_words_freq_dict_pkl_file = os.path.join(config.TOPICS_TWEETS_SHED_WORDS_FREQ_DICT_PKLS_DIR, '{}.updated.dict.pkl'.format(target_topic_ind))
    with open(topic_tweets_shed_words_freq_dict_pkl_file, 'rb') as f:
        topic_tweets_shed_words_freq_dict_tmp = pickle.load(f)
        
    count = 0
    for tweet_id, tweet_shed_words_freq_dict in topic_tweets_shed_words_freq_dict_tmp.items():
        print('tweet_id: {}'.format(tweet_id))
        print('\t{}'.format(tweet_shed_words_freq_dict))
        tweet_shed_words_len = sum(tweet_shed_words_freq_dict.values())
        print('\tLEN: {}'.format(tweet_shed_words_len))
        count += 1
        if count >= 20:
            break


tweet_id: 128954438
	{704: 1}
	LEN: 1
tweet_id: 128954439
	{1916: 1, 677: 1, 2451: 1, 704: 1}
	LEN: 4
tweet_id: 128954440
	{2451: 1, 704: 1}
	LEN: 2
tweet_id: 128954441
	{2451: 1, 704: 1}
	LEN: 2
tweet_id: 128954442
	{2451: 1, 704: 1}
	LEN: 2
tweet_id: 128954443
	{1940: 1, 2884: 1}
	LEN: 2
tweet_id: 128954444
	{2451: 1, 704: 1}
	LEN: 2
tweet_id: 128954445
	{2451: 1, 704: 1}
	LEN: 2
tweet_id: 128954446
	{680: 1, 2387: 1, 3366: 1, 2451: 1, 704: 1}
	LEN: 5
tweet_id: 128954448
	{1577: 1, 2451: 1, 704: 1}
	LEN: 3
tweet_id: 128954451
	{2451: 1, 704: 1}
	LEN: 2
tweet_id: 128954452
	{232: 1, 2451: 1, 704: 1}
	LEN: 3
tweet_id: 128954454
	{2451: 1, 704: 1}
	LEN: 2
tweet_id: 128954459
	{704: 1}
	LEN: 1
tweet_id: 128954460
	{704: 1}
	LEN: 1
tweet_id: 128954461
	{3437: 1, 2451: 1, 704: 1}
	LEN: 3
tweet_id: 128954462
	{2451: 1, 704: 1}
	LEN: 2
tweet_id: 128954463
	{704: 1}
	LEN: 1
tweet_id: 128954464
	{}
	LEN: 0
tweet_id: 128954465
	{2111: 2, 1219: 1, 704: 1}
	LEN: 4
CPU times: user 408 ms, sys: 172 ms, total: 580 ms
Wall time: 576 ms

In [3]:
%%time
"""
Check total shed words length of a topic_tweets doc
"""  
if 0 == 1:
    topic_tweets_shed_words_len = sum([sum(tweet_shed_words_freq_dict.values()) for tweet_shed_words_freq_dict in topic_tweets_shed_words_freq_dict_tmp.values()])
    print('Total shed words length of this topic_tweets_doc: {}'.format(topic_tweets_shed_words_len))


Total shed words length of this topic_tweets_doc: 1105282
CPU times: user 216 ms, sys: 8 ms, total: 224 ms
Wall time: 224 ms

Notes

  • Do NOT try to merge all topic_tweets shed words freq dicts into a single huge dict. This is extremely time-consuming and would leave VM unresponsive.