In [54]:

    

import nltk

nltk.download("wordnet")
nltk.download("averaged_perceptron_tagger")


    

    




[nltk_data] Downloading package wordnet to /Users/makris/nltk_data...
[nltk_data]   Package wordnet is already up-to-date!
[nltk_data] Downloading package averaged_perceptron_tagger to
[nltk_data]     /Users/makris/nltk_data...
[nltk_data]   Package averaged_perceptron_tagger is already up-to-
[nltk_data]       date!






    
Out[54]:





True

In [55]:

    

from nltk.corpus import wordnet
from nltk import word_tokenize, pos_tag
from nltk.stem import WordNetLemmatizer


    

In [56]:

    

# refers to http://linusp.github.io/2016/01/21/lemmatization-survey.html
def get_wordnet_pos(treebank_tag):
    if treebank_tag.startswith('J'):
        return wordnet.ADJ
    elif treebank_tag.startswith('V'):
        return wordnet.VERB
    elif treebank_tag.startswith('N'):
        return wordnet.NOUN
    elif treebank_tag.startswith('R'):
        return wordnet.ADV
    else:
        return None


    

In [57]:

    

def lemmatize_sentence(sentence):
    res = []
    lemmatizer = WordNetLemmatizer()
    for word, pos in pos_tag(word_tokenize(sentence)):
        aWord = []
        
        # original word
        aWord.append(word)
        
        # Part of Class
        aWord.append(pos)
        
        # stemmedWord
        wordnet_pos = get_wordnet_pos(pos) or wordnet.NOUN
        stemmedWord = lemmatizer.lemmatize(word, pos=wordnet_pos)
        aWord.append(stemmedWord)
        
        # lowerCasedStemmedWord
        lowerCasedStemmedWord = stemmedWord.lower()
        aWord.append(lowerCasedStemmedWord)
        
        # save content
        res.append(aWord)

    return res


    

In [58]:

    

totalWords = {}


    

In [77]:

    

fileRoot = 'ES2002a'
speaker = 'D'
fileSuffix = 'words'
fileName = fileRoot + '.' + speaker + '.' + fileSuffix
print fileName


    

    




ES2002a.D.words

In [78]:

    

f = open('../data/wordsCombined_txt/' + fileName + '.txt', 'r+')


    

In [79]:

    

text = f.read()


    

In [80]:

    

# refers to http://www.nltk.org/book/ch05.html
result = lemmatize_sentence(text)


    

In [81]:

    

for aWordList in result:
    stemmedWord = aWordList[3]
    if stemmedWord in totalWords:
        # increment freq with 1
        freq = totalWords[stemmedWord]
        freq = freq + 1
        totalWords[stemmedWord] = freq
    else:
        # set freq = 1
        totalWords[stemmedWord] = 1


    

In [82]:

    

print totalWords


    

    




{'all': 13, 'laura': 1, 'mild': 1, 'go': 8, u'zone': 1, 'to': 52, 'tail': 5, 'th': 5, 'under': 1, 'include': 1, 'friendly': 2, 'premium': 1, 'very': 6, 'choice': 1, 'awful': 1, u'minute': 4, 'cool': 3, 'four': 1, 'cable': 2, 'did': 1, u'button': 2, 'manager': 1, 'try': 1, 'this': 13, 'small': 2, u'guy': 1, 'work': 4, 'ten': 1, 'cost': 5, 'design': 9, 'even': 1, 'will': 4, 'what': 28, 'sum': 1, 'brief': 2, 'designing': 1, 'international': 2, 'appeal': 2, 'thin': 1, 'never': 1, 'here': 6, u'explore': 1, 'let': 2, 'along': 1, 'standard': 1, 'boy': 1, 'great': 2, 'ahead': 1, u'receive': 1, 'example': 2, 'usually': 1, u'streamline': 1, 'useful': 1, 'family': 3, 'extra': 1, 'c_d_': 1, 'gosh': 1, 'market': 2, u'ape': 1, 'everybody': 1, 'cha': 1, u'use': 2, 'from': 3, 'wealth': 1, 'would': 11, 'aye': 1, 'next': 5, u'camera': 1, 'cram': 1, u'tell': 2, 'more': 8, 'sort': 8, 'successful': 1, 'becomes': 1, 'aware': 1, 'basically': 1, 'actual': 1, 'scribble': 1, 'me': 5, 'ma': 1, 'room': 1, 'f': 2, 'mm': 5, 'des': 1, 'worth': 1, 'sudden': 1, 'can': 7, u'arrive': 1, 'my': 14, 'could': 7, 'control': 23, u'compare': 1, 'ach': 1, 'want': 9, u'give': 2, 'process': 1, 'high': 2, "n't": 20, 'mixture': 1, 'something': 14, u'slip': 1, 'sense': 1, 'chic': 1, 'keep': 3, 'information': 2, 'end': 3, 't_v_': 2, u'feature': 2, 'write': 2, 'how': 11, 'animal': 8, u'chase': 3, 'product': 2, 'designer': 3, 'okay': 36, 'may': 1, 'after': 2, 'spot': 1, u'produce': 1, 'lighting': 1, 'such': 4, 'a': 65, 'somethin': 1, 'natural': 1, 'remember': 1, 'maybe': 9, 'so': 46, 'scoot': 1, 'agenda': 1, u'talk': 1, 'thirty': 1, 'cute': 1, 'over': 4, 'industrial': 3, 'london': 1, u'pitch': 1, 'still': 3, 'before': 1, 'machine': 1, u'style': 1, 'thank': 1, 'internationally': 1, 'interesting': 2, ',': 173, 'actually': 10, 'harmless': 1, 'production': 3, 'choose': 1, 'main': 2, 'might': 9, 'then': 10, 'them': 6, 'good': 11, 'finance': 2, u'material': 1, 'introduce': 1, 'commitment': 1, 'they': 17, 'not': 4, 'now': 8, 'discuss': 3, 'assess': 1, u'term': 1, 'always': 1, 'l': 1, u'shoelace': 1, 'reasonable': 1, 'each': 4, 'mean': 9, 'p': 1, 'house': 2, 'fish': 1, 'yeah': 58, u'frequency': 1, 'our': 12, 'beyond': 1, 'really': 8, 'living': 1, 'gon': 12, 'space': 1, 'god': 1, 'health': 1, 'issue': 1, 'uh-huh': 3, 'million': 1, 'little': 4, 'quite': 6, 'reason': 1, 'complicated': 2, 'imagine': 2, 'put': 1, "'round": 1, u'language': 1, 'dunno': 4, 'coulda': 1, 'positioning': 1, 'david': 2, 'thing': 10, 'massive': 1, 'w': 2, 'retail': 1, 'think': 19, 'first': 4, u'feed': 2, 'already': 1, 'feel': 2, u'requirement': 1, 'yourself': 1, 'another': 2, 'message': 1, u'whale': 2, 'guess': 5, 'monkey': 1, 'twenty': 5, 'top': 2, 'system': 1, 'priority': 2, 'their': 3, u'assumption': 2, 'too': 2, 'selling': 1, 'gear': 1, 'that': 56, 'shelf': 1, 'part': 2, u'telephone': 1, 'than': 4, 'kind': 10, 'b': 1, u'instruction': 1, 'see': 5, 'project': 5, 'r': 1, 'video': 1, 'sooner': 1, 'and': 64, 'craig': 2, 'mine': 1, 'say': 1, 'have': 26, 'telly': 1, 'need': 2, 'any': 4, u'sell': 7, 'purchasing': 1, 'couch': 1, u'note': 1, 'amusing': 1, 'which': 2, 'finding': 1, 'blue': 2, 'sure': 6, 'pain': 1, 'though': 1, 'price': 6, 'who': 4, 'most': 1, 'regular': 1, "'cause": 4, 'nothing': 1, 'cap': 1, 'why': 1, 'forever': 1, 'm': 1, 'sale': 3, 'fact': 1, 'twel': 1, 'anyway': 4, 'fifty': 4, u'shoe': 1, 'fine': 1, 'find': 1, 'blender': 1, 'one': 11, 'pe': 1, 'charac': 1, 'should': 2, 'technologically': 1, 'only': 1, 'pretty': 1, 'factor': 2, 'personality': 1, u'do': 40, 'his': 5, 'hit': 2, u'get': 19, 'watch': 3, 'television': 5, 'whistle': 1, 'new': 4, 'him': 2, 'bas': 1, 'stuff': 1, 'wholesale': 1, 'where': 3, 'view': 1, 'consciously': 1, 'seventeen': 1, u'set': 3, 'keypad': 1, u'habit': 1, 'we': 35, 'depends': 2, 'individual': 1, 'greece': 1, u'wonder': 3, 'planet': 1, 'expert': 1, 'away': 3, u'please': 1, 'junky': 1, 'superb': 1, 'behind': 2, 'various': 1, 'between': 1, 'probably': 5, u'email': 2, u'accord': 1, u'parent': 1, 'screen': 1, 'attention': 1, 'willing': 1, 'coffee': 1, 'interface': 2, 'come': 3, 'fit': 1, 'gimmicky': 1, 'country': 1, u'region': 2, 's': 2, 'whole': 1, 'point': 4, 'ca': 1, 'way': 3, 'whatever': 2, 'table': 2, 'three': 3, '.': 299, 'beep': 1, 'much': 6, 'certain': 1, 'lovely': 1, 'meeting': 6, 'affectionate': 1, 'andrew': 2, 'else': 3, 'hmm': 5, 'personally': 1, 'an': 6, 'those': 1, u'pilot': 1, 'sound': 1, 'fur': 1, 'look': 6, 'these': 1, u'aim': 2, 'technical': 1, u'suppose': 5, 'many': 2, u'pound': 2, 'is': 4, 'it': 56, 'rooster': 1, u'player': 4, 'eighteen': 2, 'in': 19, 'ironic': 1, 'technology': 1, 'id': 1, 'if': 5, 'funny': 1, 'different': 6, 'suggest': 1, 'make': 7, 'same': 2, 'when': 5, u'day': 1, 'european': 1, 'satellite': 1, 'finish': 1, 'impressionist': 1, 'inbetween': 1, 'user': 3, 'robust': 1, 'cheery': 1, 'euro': 4, 'off': 1, 'i': 80, 'well': 20, 'anybody': 2, u'thought': 1, 'chara': 1, 'thi': 1, 'y': 1, 'the': 61, 'audio': 1, 'spend': 1, 'just': 31, 'money': 2, 'executive': 1, 'touch': 1, 'yep': 7, 'yes': 2, 'functionally': 1, 'announcement': 1, 'sixteen': 1, 'wee': 1, 'euros': 2, 'furry': 1, u'character': 2, 'wel': 1, 'beagle': 4, 'add': 3, 'combine': 2, u'board': 1, u'kick': 1, 'take': 3, 'real': 1, 'big': 5, 'couple': 1, 'possibly': 1, 'five': 7, 'know': 28, 'background': 1, 'bit': 2, 'unique': 1, 'd': 2, 'like': 47, "'ve": 6, u'lose': 2, u'become': 1, 'page': 1, 'fulfil': 1, 'twelve': 3, 'sketch': 1, 'right': 20, 'people': 5, 'some': 2, 'palm': 2, 'sight': 1, 'trendy': 2, 'home': 3, 'scale': 1, 'for': 14, 'everything': 4, 'does': 1, '?': 33, 'be': 81, 'noise': 1, 'v_c_r_': 1, 'marketing': 4, 'by': 2, u'stage': 2, 'on': 17, 'about': 18, 'ok': 1, 'anything': 3, 'oh': 9, 'of': 53, 'whiteboard': 1, 'd_v_d_': 2, 'dinner': 2, 'or': 20, 'primitive': 1, 'own': 3, 'into': 1, 'retailer': 1, 'functional': 1, 'down': 2, u'device': 1, 'because': 4, 'wrap': 3, 'quickly': 2, 'your': 9, 'wan': 1, u'wag': 1, 'there': 14, 'question': 1, 'long': 1, 'start': 1, 'm_p_': 1, 'lot': 6, 'allergic': 2, u'function': 3, 'head': 1, u'buy': 3, 'basic': 2, "'kay": 2, 'but': 9, 'idea': 1, 'kick-off': 1, 'hi': 1, 'with': 4, 'eat': 1, 'he': 14, 'rush': 1, u'characteristic': 3, 'up': 13, u'us': 1, 'um': 87, 'record': 1, "'re": 18, 'uh': 27, 'piece': 1, 'affection': 1, 'bearing': 1, 'ah': 2, u'condition': 1, 'al': 1, 'pig': 1, 'as': 5, 'at': 9, 'goodness': 1, 'check': 1, 'physical': 1, 'again': 3, "'ll": 8, 'no': 3, 'na': 13, 'whereas': 1, 'percent': 1, 'mm-hmm': 19, 'other': 8, u'functionality': 1, 'you': 62, 'out': 3, 'nice': 2, 'draw': 3, 'repeat': 1, "'s": 51, 'favourite': 6, u'symbol': 1, "'d": 3, 'kinda': 1, "'m": 17, 'remote': 22, 'podi': 1, 'dog': 5, 'together': 2, u'u': 1, 'time': 2, 'alright': 6, 'original': 1}

In [83]:

    

# connect to database
import pymysql
connection = pymysql.connect(host='140.116.112.164',
                             user='iim_project',
                             password='1qaz2wsx3eDC',
                             db='meeting_word_total_freq',
                             charset='utf8mb4',
                             cursorclass=pymysql.cursors.DictCursor)

cursor = connection.cursor()


    

In [84]:

    

sql_create = "create table `%s` (id int(11) NOT NULL AUTO_INCREMENT, stemmedWord char(16), freq int(11), PRIMARY KEY(id)) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_bin AUTO_INCREMENT=1"
print sql_create


    

    




create table `%s` (id int(11) NOT NULL AUTO_INCREMENT, stemmedWord char(16), freq int(11), PRIMARY KEY(id)) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_bin AUTO_INCREMENT=1

In [85]:

    

fileTableName = fileRoot + '_total_freq'
cursor.execute(sql_create, fileTableName)


    

    
Out[85]:





0

In [86]:

    

sql_insert = "insert into `%s` (stemmedWord, freq) values(%s, %s)"
print sql_insert


    

    




insert into `%s` (stemmedWord, freq) values(%s, %s)

In [87]:

    

for key, value in totalWords.iteritems():
    cursor.execute(sql_insert, (fileTableName, key, value))
connection.commit()


    

In [ ]: