In [47]:
from keras.preprocessing import sequence
from keras.models import Sequential
from keras.layers import Dense, Embedding, Dropout
from keras.layers import LSTM
from keras.datasets import imdb
import pandas as pd
import numpy as np
from bs4 import BeautifulSoup
import re
from keras.preprocessing.text import Tokenizer
In [2]:
#(x_train, y_train), (x_test, y_test) = imdb.load_data()
In [48]:
dataset = pd.read_csv(r'movie_reviews.csv')
print(dataset.head())
reviews = np.array(dataset['review'])
sentiments = np.array(dataset['sentiment'])
review sentiment
0 One of the other reviewers has mentioned that ... positive
1 A wonderful little production. <br /><br />The... positive
2 I thought this was a wonderful way to spend ti... positive
3 Basically there's a family where a little boy ... negative
4 Petter Mattei's "Love in the Time of Money" is... positive
In [49]:
def strip_html_tags(text):
soup = BeautifulSoup(text, "html.parser")
stripped_text = soup.get_text()
return stripped_text
clean_docs = []
for doc in reviews:
doc = strip_html_tags(doc)
doc = doc.lower().strip()
doc = re.sub('[^a-zA-z0-9\s]','',doc)
clean_docs.append(doc)
In [5]:
tokenizer = Tokenizer(num_words=10000, split=' ')
tokenizer.fit_on_texts(clean_docs)
In [6]:
tokenizer.word_index
Out[6]:
{'kuttram': 88006,
'payola': 56325,
'viscousif': 161324,
'sheso': 128104,
'vsexy': 206912,
'michalka': 48912,
'intended': 1331,
'rally': 12221,
'coprology': 92641,
'businesses': 17527,
'caveshokey': 200779,
'pieomys': 140773,
'wallah': 44909,
'beginnig': 154141,
'onceto': 200875,
'scenography': 59031,
'storynow': 61116,
'khoo': 136733,
'peopleprudes': 122060,
'dangerfieldi': 150247,
'abbad': 162652,
'pointcounterpoint': 153619,
'houndside': 183681,
'aarika': 73428,
'ushpizin': 167708,
'filmcontrary': 151610,
'goodwin': 42771,
'allthis': 15806,
'monstersontherampage': 160875,
'newell': 36639,
'frequencies': 79149,
'batjacproduction': 107750,
'iswas': 21911,
'bloodtaking': 129314,
'whammo': 128803,
'failsthen': 100142,
'directiongrade': 144701,
'rahims': 214038,
'ramzi': 71792,
'momentsfor': 111870,
'pavloskys': 208507,
'singlemalt': 183751,
'zanzibar': 120214,
'httpowlypttp': 142080,
'spyl': 94961,
'flatulence': 24471,
'30th': 21731,
'sidneyeven': 209922,
'francs': 27098,
'legendtoday': 169917,
'noninvolvement': 64840,
'bonsais': 170995,
'late70searly': 107569,
'kulak': 174180,
'timeshowever': 203314,
'jumpthat': 222011,
'eightiesand': 96822,
'streetbut': 128247,
'fiat': 135418,
'nutshelllight': 124059,
'clouzot': 15880,
'complaints3': 220515,
'mikesthe': 92420,
'timevery': 55435,
'out': 43,
'sketching': 36912,
'cercueils': 88588,
'undeservingly': 156688,
'film1010': 62892,
'riegerthis': 137156,
'irisingthere': 106569,
'childrenwhenever': 110268,
'fanglys': 35965,
'theslimypoliticians': 198752,
'winner': 2454,
'lowbudgetundergroundexperimental': 120516,
'bourneslick': 107542,
'cameraseverything': 120370,
'mangler': 42486,
'locationed': 177067,
'nut9510': 89195,
'offspoiler': 175870,
'robison': 51767,
'animes': 21847,
'outamitabh': 150136,
'indeeddont': 144890,
'dishwashing': 81195,
'sylvie': 29392,
'dramaturgy': 29559,
'roughbut': 152902,
'mindalthough': 71303,
'fantasys': 55022,
'supposedtrue': 154190,
'fiveo': 57197,
'holiest': 172690,
'audiencebut': 62108,
'paysit': 219600,
'janethere': 139805,
'niobe': 89475,
'munter': 68482,
'momentsall': 80487,
'unmasks': 58295,
'knifewhich': 171293,
'chalte': 68435,
'vampireits': 64412,
'fourthwallbusting': 217146,
'halfif': 96244,
'hazer': 118347,
'disadvantages': 26807,
'audley': 76479,
'corpsethe': 161819,
'bernini': 67324,
'imagesi': 66020,
'badlystaged': 191380,
'with9': 205843,
'said3': 141998,
'galiena': 162497,
'accordingly': 13076,
'1887': 133718,
'aroundone': 74350,
'1928s': 163279,
'sadeqi': 156401,
'balrog': 43934,
'beholdwhile': 196624,
'flynns': 11008,
'herovillain': 100562,
'eithergilmore': 192248,
'logicthis': 181231,
'inspector': 3491,
'1975s': 59291,
'arp': 77413,
'boggle': 43918,
'sadi': 82260,
'yvelines': 30787,
'waif': 23156,
'radicalsi': 68454,
'serious': 575,
'mcgraws': 54470,
'friendend': 117743,
'connectivity': 136601,
'pseudonyms': 63281,
'makingand': 169185,
'directedmy': 142927,
'hitsone': 183972,
'determination': 6704,
'priveghi': 73282,
'donorwolf': 79733,
'barryrichard': 106039,
'statesnow': 204131,
'mrinal': 51186,
'afterhowever': 87142,
'lazily': 23727,
'webalthough': 170751,
'battlegroundthe': 90523,
'pairing': 7776,
'brisco': 48692,
'sordie': 84465,
'chalk': 11494,
'hypocrisya': 217240,
'clubkids': 105027,
'appearancethis': 132325,
'nostalgiafirst': 209049,
'manis': 69964,
'rentingbuyingwatching': 85643,
'armsgreat': 188146,
'overintellectualize': 179761,
'rangerit': 75188,
'bumsmy': 70317,
'hangin': 44529,
'ryderall': 152732,
'cultfilm': 191737,
'chivo': 42708,
'idealists': 47846,
'kz': 140177,
'nondescript': 14361,
'comensurate': 198465,
'gaga': 38210,
'subparobvious': 202551,
'brooma': 110296,
'whalehugger': 85509,
'doyes': 154177,
'guessits': 132678,
'kamaljothika': 87993,
'amrish': 30513,
'stalones': 115340,
'bullitt': 16364,
'pictureif': 74299,
'rowdy': 15937,
'fairytale': 9136,
'comedyspoof': 47894,
'jpc': 127979,
'1grade': 212573,
'lloydpack': 161793,
'site': 2063,
'grouping': 43294,
'untalent': 130343,
'grassroots': 38198,
'goateed': 54829,
'annos': 144888,
'keitaro': 28109,
'timeisnt': 119603,
'nopeople': 165538,
'deserveso': 196440,
'bands': 3996,
'farnsworths': 36136,
'mussolini': 17547,
'earththis': 205837,
'doall': 210968,
'rotating': 20174,
'younggoon': 168251,
'preorders': 95539,
'consolations': 204026,
'lessthanten': 193893,
'embankment': 78479,
'fanclub': 108074,
'ironhead': 50034,
'adolescents': 15134,
'pycho': 132637,
'involved1': 89080,
'hennessey': 39788,
'digitized': 48478,
'lunas': 54566,
'cokesnorting': 54257,
'trades': 17012,
'breckinridge': 18336,
'muckerji': 60561,
'curiousity': 38260,
'juxtaposing': 24859,
'worest': 154278,
'collector': 8207,
'yakkity': 159946,
'tomnot': 97325,
'xu': 30764,
'misfortunate': 169719,
'prescottmarshall': 183148,
'languagei': 38957,
'unlikabledownsidenot': 195911,
'stockler': 36677,
'garloupis': 96580,
'finkkeith': 182690,
'exhusbandexwife': 166305,
'lemon': 12682,
'nielsens': 44746,
'harels': 145793,
'scarfaced': 57722,
'oneself': 11144,
'lowtheyre': 144390,
'unprofessionalism': 150358,
'4th': 6018,
'wifetony': 168131,
'teamed': 9222,
'shipscrew': 96482,
'euphoria': 43284,
'didit': 47927,
'stationrather': 96017,
'migratory': 207667,
'cheapcheap': 111825,
'gradethe': 44510,
'nostalgic': 4646,
'hotchickinpanties': 95341,
'filmographymegalomania': 175603,
'linton': 175266,
'where': 112,
'affairscoronation': 108294,
'unscrupulous': 13230,
'wellevenly': 105622,
'ashwini': 194088,
'daydreamingif': 174069,
'gauthamthe': 191198,
'situationacting': 114771,
'mst3kavoid': 92530,
'stingonly': 196230,
'ayin': 139087,
'teamthe': 34039,
'retrained': 120756,
'ducktails2': 215809,
'uninterestingwe': 105202,
'greuesome': 95181,
'subtracted': 34444,
'thirdthe': 74296,
'funnybecause': 87808,
'inspectorelectrocution': 144110,
'laughing': 1017,
'murdereras': 145924,
'fathergetting': 167672,
'goingthis': 110945,
'oftdone': 194541,
'unforgettable': 3193,
'sanitizing': 86220,
'them2': 56682,
'highsociety': 53447,
'pesta': 192967,
'allpresiding': 175500,
'intune': 135567,
'lvres': 50657,
'graveall': 183022,
'devise': 17701,
'midninetieswhile': 133076,
'defection': 49165,
'personalitiesdown': 191727,
'petr': 48457,
'lovedmatt': 139451,
'longhardjourney': 142969,
'mindbogglingly': 30952,
'memore': 190699,
'misdemeanors': 47754,
'banshees': 57799,
'ideologies': 17895,
'operates': 12521,
'harnesses': 82729,
'levelso': 111483,
'umcompromising': 86676,
'overboiled': 136784,
'breathtakingi': 100751,
'doctor': 883,
'scissorsto': 211298,
'21st': 5807,
'op78': 188124,
'flatly': 21149,
'nightmarefrom': 80890,
'partsjean': 152265,
'manhattanbased': 102774,
'connivance': 73831,
'windworshipers': 82991,
'agreeif': 133824,
'apartthe': 73993,
'productionno': 185314,
'aiello': 13028,
'writerswhere': 122602,
'endwith': 42080,
'marriageminded': 70706,
'birol': 179171,
'horsetraders': 169472,
'basisif': 192768,
'creampie': 92978,
'victimizing': 46463,
'except\x85': 145846,
'roomusing': 196689,
'plied': 64864,
'removal': 16366,
'scenewhat': 80790,
'storyboarddirector': 94323,
'quess': 77689,
'detest': 25779,
'skillfully': 9135,
'nosdam': 116964,
'glitchy': 93188,
'kidsheres': 160756,
'continentsjewish': 123375,
'ctrlc': 130254,
'signedpetloveritual': 135603,
'yettocome': 155590,
'followon': 60493,
'secretos': 183347,
'whoattacked': 152421,
'midwaysurya': 136071,
'anybodyto': 162439,
'mundanegentleselfconsciously': 179786,
'akki': 35838,
'smileso': 81678,
'arise': 9739,
'deflector': 74626,
'selfacceptance': 200221,
'nye': 47654,
'therejess': 172309,
'goose': 12064,
'piste': 85034,
'immeasurablythe': 148387,
'dtent': 167668,
'rightman': 153326,
'ga': 28114,
'chappelle': 19912,
'hig': 149905,
'roaringsquawking': 101069,
'discussing': 5605,
'downandout': 31828,
'contemptible': 24566,
'pictorially': 60219,
'reunions': 47195,
'gagshowever': 85976,
'herberts': 33128,
'characterdevelopmentdenouemententire': 92911,
'matterof': 152905,
'mosesat': 184608,
'gumppeter': 178521,
'readi': 73434,
'excomrades': 81198,
'uninspiredi': 203684,
'inreign': 207195,
'burgade': 22718,
'opined': 42019,
'deletion': 62426,
'sakrileg': 142960,
'end8': 185540,
'thank': 1378,
'duvallthe': 186452,
'dismembermentmaybe': 170727,
'christmasrelated': 162040,
'badacted': 45358,
'screensmore': 131285,
'foist': 32930,
'thinkingthrough': 92037,
'candied': 121769,
'planetout': 199985,
'horned': 34841,
'governmentdepartmentalthough': 164951,
'codenamedragonfly': 71846,
'tvwife': 171819,
'cornfieldthis': 139479,
'helli': 51290,
'schoolchiasmus': 135901,
'benvictor': 79273,
'adultsthis': 48394,
'resultdecent': 147623,
'www1tvru': 176197,
'alltooaverage': 126416,
'sizzlingas': 165404,
'boosters': 63686,
'hairi': 54457,
'happenstances': 196637,
'scientistplastic': 184256,
'climaxas': 75022,
'matkondar': 138884,
'elisabeths': 72744,
'usersbut': 113357,
'yardsticks': 129204,
'madhousemade': 135084,
'threateningthe': 72317,
'radiovoice': 165264,
'albany': 51892,
'freaked': 9676,
'hamster': 24866,
'crotchhugging': 126392,
'rodriguezdesperado': 193336,
'janitor': 10157,
'rounding': 16758,
'againbravo': 92374,
'screensjack': 215848,
'sequencesexotic': 114948,
'drzhivago': 130383,
'mitchumwhos': 137988,
'outisnt': 113618,
'healy': 15988,
'emperor': 4444,
'chauvelins': 208048,
'wayah': 128151,
'grabs': 5386,
'mcintyres': 49601,
'thrillerssadly': 169079,
'beginsmy': 204102,
'thisoverall': 54813,
'momentssejin': 218646,
'sentencetalk': 163902,
'cancelled': 8431,
'marsalis': 79245,
'renna': 139148,
'ambitiouscrap': 133162,
'arehairspray': 205414,
'feud': 11980,
'meteorites': 72295,
'really4': 170389,
'runneresque': 167819,
'salena': 25460,
'televisionsubjects': 178667,
'gallaghers': 45442,
'reue': 173363,
'antique': 15511,
'corleones': 31489,
'warbles': 62929,
'thourghly': 157667,
'dayle': 210027,
'sceneshutton': 200504,
'politicallythemed': 201110,
'interpretations': 7999,
'frugality': 191322,
'simplified4': 188346,
'filmsenter': 95806,
'mafioso': 22607,
'mentiontheres': 163417,
'ifwhen': 96199,
'forensic': 15358,
'bowlers': 80822,
'yessusan': 130425,
'knowsome': 117940,
'bleibtreau': 151992,
'leclerc': 47344,
'aurvaag': 33018,
'sbd': 180010,
'hella': 54416,
'typein': 215591,
'crapbatman': 84068,
'tres': 42991,
'suvs': 43669,
'wantssome': 189033,
'resumesi': 93733,
'alljackie': 167164,
'sappylinemaking': 139069,
'aliveif': 186896,
'burnett': 18237,
'lajja2001': 149698,
'cairoso': 207171,
'adulthood': 10770,
'larryjoe76': 136569,
'teasedthe': 168197,
'marple': 30139,
'plotall': 50085,
'recherch': 89323,
'roughies': 57536,
'areconfirmed': 185433,
'peppoire': 74059,
'curly': 6431,
'materialand': 67590,
'workmanlike': 18010,
'journalism': 12967,
'pressureconformity': 127875,
'howeverand': 79160,
'overstylized': 33797,
'lauterthe': 187850,
'13thclone': 144076,
'bogstandard': 43640,
'youthfully': 221332,
'mixedfabric': 221993,
'worthwhilebut': 212396,
'raymarthere': 149530,
'gilscott': 153262,
'10292006': 191347,
'warholsthe': 157188,
'cringesuccessful': 189212,
'republicanismnineties': 196016,
'bulllike': 188296,
'krasinski': 26648,
'eitherannette': 141256,
'wallonia': 184710,
'mighta': 133899,
'excavations': 187228,
'nonconformism': 177504,
'metajokes': 219403,
'ifyoure': 131855,
'muddid': 170501,
'lunchmeat': 186056,
'sssr': 210548,
'lolit': 51110,
'otherits': 169624,
'gromitfan': 181614,
'entertainmentafter': 162664,
'piquant': 185336,
'regularity': 38181,
'incautos': 60102,
'caparoula': 172186,
'8all': 191764,
'ej': 52198,
'spits': 12649,
'funalong': 182494,
'directionset': 41310,
'thingsbushwhacked': 190630,
'monsterthe': 38918,
'postfact': 181443,
'realif': 58484,
'welldisplayed': 178890,
'igniminiously': 113343,
'sidewindows': 122804,
'griffins': 46025,
'amply': 25511,
'websterjimmy': 156626,
'hitmanthere': 89058,
'listenbig': 122730,
'sledding': 42607,
'couture': 59616,
'response': 3407,
'stresses': 17264,
'jeremys': 34900,
'kojo': 132764,
'fuccon': 169057,
'lovecraftwhich': 138863,
'chooses': 4983,
'thismindblowingly': 87699,
'roundtable': 195924,
'colagrandes': 45983,
'rage': 3976,
'holiday': 3113,
'charactersalthough': 144716,
'planethow': 133654,
'bergens': 61223,
'claudia': 12559,
'highestrated': 90487,
'solving': 9173,
'magnums': 67567,
'payoff': 6115,
'mcgangs': 171564,
'apostle': 33603,
'alcoholdrugs': 186393,
'effectanyway': 151205,
'albatrosss': 176723,
'charendoff': 103339,
'lowestcommondenominator': 66131,
'armatures': 74775,
'braccos': 122747,
'doesntif': 156370,
'balletic': 185326,
'uncontrollably': 20564,
'fistycuffs': 187651,
'panache': 13226,
'becomesits': 153357,
'meeropol': 70617,
'ethiers': 73496,
'thinkmunchie': 127569,
'untill': 26563,
'persuading': 31406,
'missionjustice': 167711,
'blankeyed': 152590,
'presson': 38338,
'ethnographic': 164403,
'derns': 46010,
'highestquality': 179201,
'eliminationjoe': 189158,
'harebrained': 57211,
'hardcores': 206758,
'childhoodthere': 69070,
'atin': 151533,
'jackaass': 98214,
'dopers': 50235,
'howweird': 177429,
'pixiedust': 102081,
'eviction': 47375,
'happeningit': 90028,
'playedand': 96795,
'jordana': 20913,
'clos': 188484,
'motivational': 24208,
'hehehe': 54019,
'macfarlane': 47274,
'fourth': 2861,
'speilburg': 189661,
'pteradactyl': 85025,
'shill': 34469,
'prettier': 16886,
'pasqal': 121373,
'1959well': 205875,
'booingwhy': 144272,
'headdoctor': 93373,
'stupidits': 40269,
'wellexcellent': 208786,
'pictureone': 76221,
'goodthree': 101690,
'bayits': 110694,
'teethgnashing': 63010,
'tvok': 160505,
'threeethis': 218750,
'yeiksto': 109357,
'asianlike': 124408,
'nina': 7491,
'civiliansto': 91055,
'roses62': 222170,
'hmmmare': 120543,
'thrilleravoid': 141629,
'bruises': 24229,
'entertainerers': 146611,
'horrendouslywe': 93936,
'pocheas': 170241,
'interviewsnothing': 134568,
'filmby': 33008,
'avenuebut': 97031,
'endureand': 185534,
'wagonmaster': 32134,
'disapears': 222105,
'madelaine': 83076,
'vaudevillesque': 214472,
'gimenez': 85799,
'mercado': 83335,
'mcgarrett': 87139,
'metro': 14091,
'entertainerwhat': 130412,
'laconian': 144804,
'persecutes': 119890,
'racisms': 210367,
'durbervilles': 103696,
'disappointedhopefully': 169419,
'funnist': 185102,
'guideline': 51314,
'thoughtdr': 138024,
'arnt': 36486,
'gawdy': 121072,
'anywayfirst': 64718,
'stevedean': 104353,
'uponand': 88572,
'dated': 1994,
'restgood': 180625,
'prichardrichard': 212458,
'armitages': 188782,
'ridefans': 89037,
'penal': 22677,
'matchwhile': 160725,
'rey': 23619,
'breathtop': 182315,
'agentoh': 142322,
'forgivenessbad': 213308,
'wonderfullythe': 140460,
'dalek': 24086,
'beganlindy': 165785,
'anbthony': 113888,
'awei': 61431,
'raghupati': 132371,
'buttonholes': 89670,
'himherself': 28687,
'vampiremovie': 67570,
'tridevplot': 181362,
'capitaes': 160443,
'hidesonly': 115628,
'recenter': 139481,
'midperformancei': 85804,
'riccardo': 23306,
'tristram': 48248,
'paved': 16261,
'fuyumi': 71852,
'corners': 10121,
'pta': 28441,
'v1': 151767,
'sabretooth': 15521,
'sinatra': 3048,
'quasigoofy': 178732,
'islandbut1': 120927,
'moviesmiles': 134655,
'catlover': 143838,
'everly': 46134,
'echance': 203659,
'scroogedit': 208109,
'analyzed': 15430,
'selfaffirmation': 183228,
'khialdi': 190093,
'coursethis': 48466,
'scriptwise': 36178,
'dancingive': 159058,
'worldalonethis': 121514,
'ogle': 32868,
'malevolent': 11421,
'allard': 55143,
'spoilersit': 67881,
'admirersin': 217326,
'realismthe': 32816,
'baitz': 124969,
'class': 806,
'lousiest': 29697,
'slurantwone': 191650,
'tamper': 38513,
'itgood': 55888,
'sprogin': 167494,
'genreof': 96691,
'fascinates': 25401,
'graphicsnot': 165807,
'everlovely': 175237,
'xiao': 26619,
'uninformed': 20831,
'hellbent': 20992,
'oooohhhscary': 100696,
'bharathi': 137277,
'fcc': 46026,
'turtlenecks': 80827,
'santiniurbano': 211299,
'ignominiously': 151298,
'mrsavage': 217556,
'diversioni': 83552,
'discreet': 17894,
'terrific': 1280,
'northfield': 60642,
'wicket': 143514,
'postsaloon': 124647,
'influencepolanski': 89933,
'torturerape': 193284,
'lofranco': 143987,
'watchable': 1707,
'ageafter': 215185,
'tigron': 200778,
'biographers': 63091,
'envoked': 188271,
'nicethere': 108552,
'dreamsa': 187974,
'colethe': 106559,
'exorcism': 9130,
'premchand': 59890,
'synthesize': 46426,
'movieall': 21587,
'characterone': 46057,
'rightness': 40206,
'expelledmiddle': 117742,
'nonexistence': 41107,
'mcnichols': 103579,
'carlise': 100226,
'seungup': 47139,
'burgess': 11454,
'commentboth': 141252,
'runteldat': 46049,
'tiesfrom': 130807,
'denseand': 184816,
'boozedup': 184278,
'talkingboring': 189318,
'knowns': 60429,
'foundered': 151359,
'pukesif': 176470,
'rioters': 54379,
'spendrating': 165229,
'schmid': 21870,
'denominated': 70218,
'relates': 8344,
'1999it': 104785,
'moneymatrix': 126022,
'7star': 206955,
'prague': 14819,
'polly': 8017,
'liasons': 57419,
'calledfor': 68139,
'inonly': 199602,
'milliagn': 163457,
'irnonicness': 215928,
'kurosawa': 8155,
'animationregarding': 202497,
'almost': 208,
'personathe': 198129,
'susannah': 14934,
'imbues': 25969,
'meateater': 205356,
'astral': 20629,
'nevksy': 180495,
'warnedat': 120659,
'passages': 10993,
'militias': 192025,
'adaptationgielgud': 186158,
'rugrats': 17839,
'entitled': 5278,
'bodieschris': 208658,
'timeswhos': 212035,
'highcontact': 104308,
'marjoke': 115377,
'furor': 210336,
'twooddly': 218430,
'slauther': 107406,
'asgrownup': 204823,
'cryface': 149330,
'hipswinging': 86504,
'phewi': 180759,
'moviecareer': 108735,
'roadchris': 213737,
'luces': 80531,
'shapeshifter': 71705,
'postpaul': 174572,
'allfruity': 89187,
'ooooozes': 165451,
'pepi': 32700,
'capsuleps': 190862,
'certainties': 74278,
'swardson': 77315,
'lolly': 58362,
'interlopers': 44104,
'viel': 32413,
'cullen': 27615,
'mouthlaura': 173902,
'copranger': 182575,
'legacya': 186860,
'harnett': 71693,
'followclocking': 126756,
'treads': 21823,
'musictvcat': 136590,
'reluctantunlikely': 97156,
'humourthis': 63289,
'daysoverall': 154047,
'thoughat': 59879,
'somemore': 67913,
'comedya': 32151,
'tellingthe': 67017,
'vengeanceseeking': 186409,
'idontneedrevengenormyhusbandsmoney': 143099,
'nearpoverty': 201544,
'steinem': 54433,
'oildrained': 169229,
'uruguayan': 45522,
'fretfully': 198666,
'alonzo': 26397,
'fakethe': 47065,
'exalted': 35924,
'scobs': 214346,
'barebreasted': 54568,
'wheatlry': 104222,
'tite': 173383,
'okdirectors': 182511,
'dewy': 46703,
'reformer': 36772,
'finchis': 156272,
'umekis': 53381,
'raf': 14889,
'etcjudy': 187905,
'greene': 8470,
'offthetop': 116995,
'easywith': 153227,
'personification': 22326,
'expensively': 46794,
'awaywhats': 110353,
'gypsy': 5564,
'spoiledthere': 178759,
'wongundergoes': 216441,
'310little': 213741,
'againhe': 60784,
'officethe': 62942,
'carrerahudson': 199840,
'unrealisticsecondly': 125658,
'kellythis': 88628,
'empirical': 61797,
'prejudicesford': 106590,
'discontinuity': 72187,
'sceneryhonest': 157381,
'fadesbest': 96655,
'weakand': 66699,
'cubbyhouse': 166268,
'litja': 44813,
'chasing': 3163,
'mcjannet': 82640,
'spinnable': 80408,
'ozarks': 64054,
'againpaltrow': 147167,
'spotless': 26995,
'interjects': 44928,
'fabios': 139930,
'screwballs': 37573,
'sixthform': 62110,
'hard310': 163057,
'nocount': 43090,
'summerize': 70888,
'difrasso': 159678,
'orgasms': 45145,
'unreadiness': 152121,
'breaktension': 175153,
'arresting': 12593,
'nonprofit': 57713,
'handclawing': 71960,
'etcthere': 75198,
'amazedfrom': 207315,
'lemat': 208777,
'1908': 51053,
'whiners': 45067,
'rentfree': 148579,
'soulfulness': 98595,
'roadsthose': 136307,
'denistouns': 112885,
'dannyher': 219911,
'edittake': 107416,
'invincibleto': 199528,
'sourceat': 202114,
'movieseen': 137781,
'awfulsounding': 190493,
'spaders': 212411,
'bloodgutsgorebonesnapping': 208581,
'ewan': 12379,
'mufla': 153654,
'equaledthe': 138772,
'doublethis': 195999,
'kaleidiscopic': 97853,
'shotin': 99417,
'tokers': 69562,
'essenceexistentialists': 108000,
'darwininan': 163798,
'angelinaif': 86031,
'doorbells': 69394,
'superficiality': 20574,
'tiffani': 26138,
'sharron': 60044,
'3bs': 184107,
'vigilantestyle': 120416,
'tercerothird': 99027,
'edwardsmasterpiece': 191440,
'yatyuksu': 131243,
'greedily': 50115,
'chancewell': 109775,
'whupa': 117639,
'thismatthau': 101524,
'analysed': 63528,
'ratcher': 75123,
'aeclenching': 119301,
'affectts': 179261,
'selfdeprecatingbut': 108291,
'glassmajor': 145278,
'larquey': 163119,
'girlchild': 101327,
'pit': 5437,
'chekhov': 74194,
'effortthese': 170459,
'wordsin': 72050,
'louisedreyfussesque': 113567,
'squanderedif': 106705,
'reprint': 168030,
'charactersvery': 159221,
'ud': 71424,
'emotionlessness': 106448,
'forcegailard': 125168,
'nondiscriminatory': 162105,
'montagethe': 95260,
'vasectomy': 134667,
'q1': 100861,
'maryland': 22121,
'loosejointed': 220095,
'bilateral': 61795,
'movementnorma': 126060,
'rectangularwhich': 184583,
...}
In [7]:
X = tokenizer.texts_to_sequences(clean_docs)
In [8]:
X = sequence.pad_sequences(X)
In [9]:
sentiment_ohe = np.array(pd.get_dummies(sentiments))
sentiment_ohe
Out[9]:
array([[0, 1],
[0, 1],
[0, 1],
...,
[1, 0],
[1, 0],
[1, 0]], dtype=uint8)
In [10]:
train_X = X[:35000]
train_y = sentiment_ohe[:35000]
test_X = X[35000:]
test_y = sentiment_ohe[35000:]
train_X.shape, train_y.shape, test_X.shape, test_y.shape
Out[10]:
((35000, 2087), (35000, 2), (15000, 2087), (15000, 2))
In [19]:
embed_dim = 128
lstm_out = 64
model = Sequential()
model.add(Embedding(10000, embed_dim, input_length = X.shape[1]))
model.add(Dropout(0.2))
model.add(LSTM(lstm_out, dropout=0.2, recurrent_dropout=0.2))
model.add(Dense(2, activation='softmax'))
model.compile(loss = 'categorical_crossentropy', optimizer='adam',metrics = ['accuracy'])
print(model.summary())
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
embedding_3 (Embedding) (None, 2087, 128) 1280000
_________________________________________________________________
dropout_1 (Dropout) (None, 2087, 128) 0
_________________________________________________________________
lstm_2 (LSTM) (None, 64) 49408
_________________________________________________________________
dense_2 (Dense) (None, 2) 130
=================================================================
Total params: 1,329,538
Trainable params: 1,329,538
Non-trainable params: 0
_________________________________________________________________
None
In [22]:
batch_size = 100
model.fit(train_X, train_y, epochs = 5, batch_size=batch_size, verbose=1)
Epoch 1/5
35000/35000 [==============================] - 5115s - loss: 0.4314 - acc: 0.8086
Epoch 2/5
35000/35000 [==============================] - 5328s - loss: 0.3293 - acc: 0.8661
Epoch 3/5
35000/35000 [==============================] - 5458s - loss: 0.2708 - acc: 0.8923
Epoch 4/5
35000/35000 [==============================] - 5855s - loss: 0.2266 - acc: 0.9128
Epoch 5/5
35000/35000 [==============================] - 5681s - loss: 0.1936 - acc: 0.9257
Out[22]:
<keras.callbacks.History at 0x22bbe9760b8>
In [23]:
r = model.predict_classes(test_X)
15000/15000 [==============================] - 503s
In [29]:
a = np.array(pd.DataFrame(test_y).idxmax(axis=1))
a
Out[29]:
array([0, 1, 0, ..., 0, 0, 0], dtype=int64)
In [43]:
from sklearn.metrics import classification_report
from sklearn.metrics import confusion_matrix
In [42]:
print(classification_report(a, r, target_names=['Negative', 'Positive']))
precision recall f1-score support
Negative 0.86 0.90 0.88 7490
Positive 0.89 0.85 0.87 7510
avg / total 0.87 0.87 0.87 15000
In [45]:
confusion_matrix(a, r)
Out[45]:
array([[6717, 773],
[1135, 6375]])
In [46]:
reviews[0]
Out[46]:
"One of the other reviewers has mentioned that after watching just 1 Oz episode you'll be hooked. They are right, as this is exactly what happened with me.<br /><br />The first thing that struck me about Oz was its brutality and unflinching scenes of violence, which set in right from the word GO. Trust me, this is not a show for the faint hearted or timid. This show pulls no punches with regards to drugs, sex or violence. Its is hardcore, in the classic use of the word.<br /><br />It is called OZ as that is the nickname given to the Oswald Maximum Security State Penitentary. It focuses mainly on Emerald City, an experimental section of the prison where all the cells have glass fronts and face inwards, so privacy is not high on the agenda. Em City is home to many..Aryans, Muslims, gangstas, Latinos, Christians, Italians, Irish and more....so scuffles, death stares, dodgy dealings and shady agreements are never far away.<br /><br />I would say the main appeal of the show is due to the fact that it goes where other shows wouldn't dare. Forget pretty pictures painted for mainstream audiences, forget charm, forget romance...OZ doesn't mess around. The first episode I ever saw struck me as so nasty it was surreal, I couldn't say I was ready for it, but as I watched more, I developed a taste for Oz, and got accustomed to the high levels of graphic violence. Not just violence, but injustice (crooked guards who'll be sold out for a nickel, inmates who'll kill on order and get away with it, well mannered, middle class inmates being turned into prison bitches due to their lack of street skills or prison experience) Watching Oz, you may become comfortable with what is uncomfortable viewing....thats if you can get in touch with your darker side."
Content source: dipanjanS/text-analytics-with-python
Similar notebooks: