In [10]:
import os
import nltk
from nltk.corpus import stopwords
files = os.listdir('aclimdb/train/pos')
In [11]:
words =[]
for file in files:
with open('aclimdb/train/pos/{}'.format(file), 'r' , encoding = 'utf-8') as f:
review = nltk.word_tokenize(f.read())
for token in review:
words.append(token)
print(len(words))
3555958
In [12]:
files = os.listdir('aclimdb/train/neg')
In [13]:
for file in files:
with open('aclimdb/train/neg/{}'.format(file), 'r' , encoding = 'utf-8') as f:
review = nltk.word_tokenize(f.read())
for token in review:
words.append(token)
print(len(words))
7056193
In [14]:
words = nltk.FreqDist(words)
word_features = list(words.keys())[:3000]
In [15]:
def find_features(doc):
words = set(doc)
features = {}
for w in word_features:
features[w] = (w in words)
return features
In [17]:
with open('aclimdb/train/neg/{}'.format(files[0]), 'r' , encoding = 'utf-8') as f:
review = nltk.word_tokenize(f.read())
find_features(review)
Out[17]:
{'heck-of-a': False,
'big-and-little': False,
'..need': False,
'Pooh': False,
'Jeanane': False,
'smoke-effects': False,
"'Handicapped": False,
'Challenge': False,
'inundated': False,
'literal': False,
'ravenous': False,
'Nabokov': False,
'arabic': False,
'WAS': False,
'sounds': False,
'bone-dry': False,
'doubly': False,
'Eldard': False,
'TV-looking': False,
'tricky.': False,
'overdrive': False,
'dispensed': False,
'Marie-Paul': False,
'beta': False,
'VERSION': False,
'FALLS': False,
'Condensing': False,
'Misato': False,
'ROMANTIC': False,
'advertisers': False,
'exertion': False,
'worshiped': False,
'Yokozuna': False,
'shoo': False,
'ass-ed': False,
'concurrent': False,
'slant': False,
'already-harvested': False,
'botched.': False,
'info': False,
'Blitzkrieg': False,
'man-loving': False,
'puh-leeze': False,
'recuperating': False,
'Rendez-Vous': False,
'MATRIX-LIKE': False,
'Nader': False,
'accomplish..The': False,
'Papamoschou': False,
'well-fleshed': False,
'Chalie': False,
'starkness': False,
'Cleo': False,
'documentary': False,
'`Kiss': False,
"'nervous": False,
'added/altered': False,
'apparently': False,
'overflow': False,
'female-written': False,
'Rakastin': False,
'stakes.': False,
'Carvan': False,
'Barbet': False,
'It-girl': False,
'self-harming.': False,
'conniving': False,
"rock'em": False,
'Karlsson': False,
'burp': False,
'fabrics': False,
'biker-esque': False,
'tamed': False,
'staunchly': False,
'Expcept': False,
'statesmanship': False,
'confab': False,
'Unfortuneatly': False,
'organizers': False,
'Papel': False,
'Hurry': False,
'tarring': False,
'staffs': False,
'Rains-as-Belasco': False,
'rabified': False,
'warrant': False,
'enthusiast.': False,
'miners': False,
'Russo-Japanese': False,
'downfall.OK': False,
'Rip-off': False,
'loveability': False,
'Philly': False,
'uneffective': False,
'gyudon': False,
'allsuperb': False,
'keaton': False,
'Stripe': False,
'high-cheekboned': False,
'Virtuous': False,
'.how': False,
'rocky': False,
'451': False,
'Anti-Fascist': False,
'tastes.': False,
'initiating': False,
'favoured': False,
'Frustrating': False,
'late-spring/summer': False,
'yrs': False,
'flung': False,
"'Straight": False,
'Ovies': False,
'one-shouldered': False,
'up-to-the-minute': False,
'explained.': False,
'Dragonlord': False,
'dragnet': False,
'knows.': False,
'`Second': False,
'Harvard-educated': False,
'1415': False,
'producer/writer/actor/impresario': False,
'Monique': False,
'convoked': False,
'revivals': False,
'unbearable.Most': False,
'Quebec': False,
'all-too-real': False,
'unlikley': False,
'badmen': False,
'attendance': False,
'WW1': False,
'preconceive': False,
'Sahibjaan': False,
'solarisation': False,
'idris': False,
'fooled': False,
"sock'em": False,
'Pujari': False,
'enormity': False,
'pre-modern': False,
'gamma': False,
'wabbit': False,
'afar': False,
'Fed': False,
'checklist': False,
'white-tip': False,
'fire-and-brimstone': False,
'breakthrough.': False,
'Rambos': False,
'buildup': False,
'Nintendo': False,
'Kavalier': False,
'lusted': False,
'Earned': False,
'comment..': False,
'MAGNOLIAS': False,
'debate': False,
'Crank': False,
'diatribe': False,
'agonisingly': False,
'1890': False,
'Reymond': False,
'trusty': False,
'hopefully': False,
'adviser/lover': False,
'delirious': False,
'Lemarit': False,
'Stirba': False,
'Antes': False,
'STORYLINE.': False,
'elegance”': False,
'unprocessed': False,
'BOTHER': False,
'turn-': False,
'Holi': False,
'shoulder-padded': False,
'cheesy/campy': False,
'T-Shirt': False,
'worrisome': False,
'Pages': False,
'character-in': False,
'excluded': False,
'Girlfriends': False,
'fast-food-mainstream': False,
'Akira': False,
'travel': False,
'Dr': False,
'Tangier': False,
'epochs\x97in': False,
'mountain-dwellers': False,
'Gershuni-directed': False,
'Carmus': False,
'Pomeranian': False,
'Frisbee': False,
'Schwarznegger': False,
'fixed': False,
'zombie.': False,
'UNtalented': False,
'Surprising': False,
'him/': False,
'ROGERS': False,
'straightened': False,
'premiering': False,
"munnera'na": False,
'TALENT': False,
'more.Of': False,
'cogent': False,
'Went': False,
'Baraka': False,
'Shimomo': False,
'it.i': False,
'Sister-': False,
'Berghe': False,
'opinions': False,
'Tawny': False,
'fuselage': False,
'gnomes': False,
'TAILS': False,
'overrun': False,
'repellant': False,
'POOR': False,
'Majelewski': False,
'Part': False,
'teen-age': False,
'CREEPSHOW': False,
'incoherent.Why': False,
'big-name': False,
'hunt': False,
'flapper/Jazz': False,
'LAN': False,
'reality-show': False,
'RAIDER': False,
'Aylmer': False,
'motherfockers': False,
'Oscar': False,
'Wesker': False,
'poster/stills': False,
'producer.And': False,
'dress-rehearsal': False,
'passages': False,
'hulled': False,
'periodically': False,
'Hernando': False,
'Solyaris': False,
'tho.': False,
'AMITYVILLE': False,
'passwords': False,
"'yet": False,
'M15+': False,
'soon.I': False,
'Kristoferson': False,
'HOOD': False,
'Redneck': False,
'2.75': False,
'dozens': False,
'remastering': False,
'wash-up': False,
'literally': False,
'snoopy': False,
'floodwaters': False,
'subtleties': False,
'Showcase': False,
'ease': False,
'Corral.': False,
'counterbalances': False,
'annoying.This': False,
'grainy/noisy': False,
"'cradle": False,
'Extremities': False,
'path': False,
'Assigned': False,
'smartassy': False,
'well-developed.It': False,
'winos': False,
'phobic': False,
'Renoir': False,
'whine': False,
'Chatham': False,
'MR': False,
'ulster': False,
'Northwest': False,
'hypocritic': False,
'Unrelated': False,
'Gestaldi': False,
'Youngs': False,
'Kou': False,
'FREE': False,
'gremlins': False,
'Rumors': False,
'miniseries.': False,
'Doors': False,
'Ghatak': False,
'Key': False,
'Debuted': False,
'not-too-helpful': False,
'in-dept': False,
'misinterpreted': False,
'predatory': False,
'Billys': False,
'Vietnamese': False,
'bonhomie': False,
'STORIES': False,
'Vanilla': False,
'Comedylooser': False,
'Claim': False,
'Zapata': False,
'Jong-chan': False,
'Society®': False,
'languishing': False,
'cowhand': False,
'SLIDING': False,
'barricaded': False,
'uplifting.': False,
'Festival.': False,
'reinvent': False,
'highschool': False,
'sanitized': False,
'ASL': False,
"'Fatal": False,
'arguments.': False,
'Madonna': False,
'pretenders': False,
'Tanushree': False,
'Joxs': False,
're-confirm': False,
'rumbles': False,
'Dinotopia': False,
'Einstien': False,
'Module': False,
'Fate/Stay': False,
'nonbelieveability': False,
'Clement': False,
'Melora': False,
'pungee': False,
'uncooked': False,
'imbues': False,
'unrevealed': False,
'tape.Of': False,
'guttersnipe': False,
'Ex-Army': False,
'pimply': False,
'Spanish-Italian': False,
'EDISON': False,
'trillion': False,
'brigades': False,
'Bentley': False,
'FRIDGE': False,
'conforms': False,
'cheese.The': False,
'Borg': False,
'center-stage': False,
'Exceedingly': False,
'high-stepping': False,
'BLEEDING': False,
'deeply': False,
'saloon-owner': False,
'denver': False,
'2/': False,
'equator': False,
'empathetic': False,
'Mardi': False,
'performace': False,
'//www.intuitor.com/moviephysics/bleep.html': False,
'pimples': False,
'say.Would': False,
'heightens.': False,
'pride.': False,
'Chegwin': False,
'Galveston': False,
'Silvestre': False,
'Seagals/Willis/Van': False,
'nudes.': False,
'bitch-slapping': False,
'para-military': False,
'Amazon-ic': False,
'apologising': False,
'inroads': False,
'Graces': False,
'step-child': False,
'expectant': False,
'odd': False,
'adv': False,
'MAIDEN': False,
'Transylvania': False,
'saner': False,
'tell-all': False,
'half-Korean': False,
'smelling': False,
'480m': False,
'Southside': False,
'car-washes': False,
'condensed.': False,
'Manji': False,
'Rusell': False,
'false.': False,
'penis-strangulation': False,
'pseudo-religious': False,
'patrician': False,
'cornball': False,
'thunderously': False,
'H.R': False,
'copout': False,
'apalling': False,
'vacuum-cleaner': False,
'unnameable': False,
'RB': False,
'scream': False,
'bitch-nemeses/teammates': False,
'post-impressionist': False,
'Thing': False,
'discipline.': False,
'Meistersinger': False,
'amuck': False,
'phrases': False,
'Erotica': False,
'Berry': False,
'DeLuca': False,
'Cunty': False,
'disappointed.': False,
'CAMERAMAN': False,
'Rosie': False,
'Barbi': False,
'mod': False,
'Hellborn': False,
'aristocracy.': False,
'Based': False,
'fugace': False,
'Rowe': False,
'embarks': False,
'faux-infant': False,
'mundane': False,
'closure.': False,
'10-12': False,
'Manics': False,
'personae': False,
'writhe': False,
'CRAP-O-LANTERN': False,
'sex-bots': False,
'foretell': False,
'stout': False,
'Lip': False,
'Sox': False,
'co-starrer': False,
'millar': False,
'concentrate': False,
'feline-like': False,
'nuances': False,
'screwy': False,
'Clutter': False,
"'Steamboat": False,
'30,00': False,
'top-level': False,
'inhaled': False,
'test-tube': False,
'Adjani': False,
'Hilda': False,
'Alternatively': False,
'repulsiveness': False,
'Crosseyed': False,
'Kller': False,
'sorbet': False,
'Ricardo': False,
'Truly': False,
'NORD': False,
'Davy': False,
'NASCAR': False,
"'slight": False,
'pah-pah-tah-dah': False,
'music-related': False,
'cont': False,
'unfold': False,
'silencers': False,
'proto-feminist': False,
'Nirgendwo': False,
'bidding': False,
'therese': False,
'updated.': False,
'movie.the': False,
'Bale': False,
'differently..': False,
'motto': False,
'Sembene': False,
'Catalina': False,
'Croaziera': False,
'enigma': False,
'Fed-Ex': False,
'undying': False,
'Plaza': False,
'Lunceford': False,
"'Fashions": False,
'inhibit': False,
'notch': False,
'thumbs/tails': False,
'Beatlemaniac': False,
'plush-toys': False,
'sub-humans': False,
'shower.': False,
'HJelmet': False,
'creme-de-la-creme': False,
'privation': False,
'adenine': False,
'contrast.Also': False,
'nicer': False,
'Posing': False,
'Eccelston': False,
'MUSIC': False,
'generator': False,
'decide': False,
'nurses': False,
'Barefoot': False,
'Ambassador': False,
'D-Day': False,
'alleging': False,
'malarkey': False,
'Ramadan': False,
'well-integrated': False,
'punctuations': False,
'rise': False,
'discomfiting': False,
'Franziska': False,
'breaks.': False,
'execution-scenes': False,
'Pickles': False,
"'ten": False,
'Silvermen': False,
'Furdion': False,
'tuner': False,
'Kitaen': False,
'KYLE': False,
'cinema.': False,
'almighty': False,
'direction.I': False,
'rent.': False,
'screenplay.': False,
'punishments.': False,
'lot.I': False,
'lizzie': False,
'curiosity.': False,
'divorced': False,
'Andrus': False,
'Ruppert': False,
'traffickers': False,
'Him': False,
'Snoozefest': False,
'1905': False,
'Wondering': False,
'witty.': False,
'reissues': False,
'grabbing': False,
'Steers': False,
'everyman': False,
'method': False,
'Kool': False,
'PUMPKIN': False,
'husband-to-be': False,
'unseated': False,
'works.': False,
'taker': False,
'Nozaki': False,
'Sherawali': False,
'ultra-loyalist': False,
"'novelty": False,
'Achra': False,
'canyon': False,
'woodland': False,
'phantasm': False,
'semi-graphic': False,
"'what-the-hell": False,
'Merciless': False,
'Das': False,
'Jeff': False,
'increases': False,
'sarkar': False,
'Shvollenpecker': False,
'grinch': False,
'Shogun': False,
'087': False,
'solved.': False,
'comparing': False,
'winces': False,
"'sneak": False,
'Needless': False,
'Marple': False,
'removes': False,
'THIEVES': False,
'rancher': False,
'Lampoons': False,
'Shojo': False,
'geniusly': False,
'**********': False,
'Jerichow': False,
"'seducing": False,
'hell-fire': False,
'legacy.': False,
'Toland': False,
'Arkush': False,
'Tina': False,
'Trombone': False,
'AHHHHH': False,
'fightfest': False,
'Chomet': False,
'overripe': False,
'Baise': False,
'unbilled.': False,
'anti-vampire': False,
'Primarily': False,
'totally-deranged': False,
'Bryant': False,
'Shock-o-Rama': False,
'gladiatorial': False,
'another=-her': False,
'Sailormoon': False,
'Rodgers': False,
'Livington': False,
'Muñoz': False,
'procrastination': False,
'Than': False,
'idiosyncratically': False,
'Chriterion': False,
'Internet': False,
'stationary': False,
'partake.': False,
'leap': False,
'suborned': False,
'average.There': False,
'Ciff': False,
'Finnerty.': False,
"70's.Hippie": False,
'Beauty.': False,
'munchies.': False,
'maiden': False,
'kaante': False,
'Angeles.': False,
'Ground': False,
'improvisations': False,
'three-dimensionality': False,
'mind-numbed': False,
'25yo': False,
'Releasing': False,
'Uneasy': False,
'Kohara': False,
'Lousy': False,
'off-day': False,
'Operatic': False,
'negative': False,
'banishses': False,
'F22': False,
'Damascus': False,
'feminine/masculine/ying-yang': False,
'Weskit': False,
'bartender': False,
'well-thought-out': False,
'accelerate': False,
'Fardeen': False,
'out-of-reach': False,
'Epstein': False,
'six-month': False,
'boys/young': False,
'wow': False,
'magisterial': False,
'1975-': False,
'Seachd': False,
"'Tooth": False,
'papaya': False,
'infusing': False,
'1-6': False,
'Barge': False,
'overflowing': False,
'hop': False,
'Meth': False,
'trailer': False,
'Hate': False,
'two-and-two': False,
'atmosphere.': False,
'ATTEMPTED': False,
'nail.': False,
'ratio': False,
'Yeung': False,
'ineptness': False,
'gonads': False,
'avenges': False,
'vasectomies': False,
'nacho': False,
'Mao`s': False,
'exterior.': False,
'Fujiko': False,
'Feeney': False,
'husband-': False,
'woodlands': False,
'Heroo': False,
'nephews': False,
'overdub': False,
'overseen': False,
'Solo': False,
'BURMA': False,
'intersected': False,
'Francoisa': False,
'Guerrero': False,
'corporate-done': False,
'1/21/06': False,
'Katharine': False,
'PCs': False,
'relations.': False,
'paternalistic': False,
'Espe': False,
'Tobruk': False,
'Lamarr.': False,
'western/social': False,
'seven-foot': False,
'Gwar-costume': False,
'decorous': False,
'Steadican': False,
'BAFTAs': False,
"Je'Taime": False,
'YIIII': False,
'scumbag': False,
'Intervention': False,
'Jarvis': False,
'causal': False,
'Zu': False,
'Saltcoats': False,
'Renegade': False,
'valuation': False,
'waged': False,
'six-armed': False,
'Damn': False,
'Bukvic': False,
'costumes.': False,
'investogate': False,
'chunk-city': False,
'frothing': False,
'Inom': False,
'REVOLUTIONARIES': False,
'Alba.The': False,
'MUDD': False,
'15/16': False,
'segment`s': False,
'Bonham-Carter': False,
'Ossana': False,
'stiffer': False,
'initially.it': False,
'pneumonia': False,
'misadventure': False,
'BB': False,
'arkin': False,
'Siege': False,
'lapels': False,
'Goethe': False,
'living-room': False,
'exterminators': False,
'reconstitution': False,
'lawyer': False,
'near-hysterical': False,
'revenge-minded': False,
'INSPECTOR': False,
'tripled': False,
'sexagenarians': False,
'adequate.': False,
'barters': False,
'loach': False,
'09': False,
'kid/hero': False,
'riverboat': False,
'store.hahahahaha.she': False,
'Eick': False,
'Gale': False,
'puppeteer': False,
'doers': False,
'buffoonish': False,
'middling': False,
'furthest': False,
'sleep-deprived': False,
'spoof': False,
'irresistible.': False,
'christmas-tree': False,
'velveeta': False,
'Sci-Fi/Horror': False,
'daughter.After': False,
'Eberhard-if': False,
'Lavin': False,
'impressed.': False,
'brining': False,
'Writer-director': False,
'assassination.': False,
'tiring': False,
'ambiguity\x96this': False,
'Purbbs': False,
'Uneven': False,
'challnges': False,
'space/time': False,
'*very*': False,
'behalf.': False,
'headline': False,
"'oldest": False,
'beehive': False,
'.its': False,
'CAME': False,
'pullers': False,
'Outrageously': False,
'HerInteractive': False,
'settling': False,
'here.in': False,
'foo': False,
'kicks.': False,
'lizard': False,
'bonafide': False,
'wisdom/knowledge': False,
'Hindi': False,
'then-boyfriend': False,
'maiming': False,
'Sartain': False,
'client': False,
'Cooler.': False,
'Taverner.': False,
'war..': False,
'hooker-with-a-heart-of-gold': False,
'4th': False,
'raptor': False,
'Baston': False,
'spy-movie': False,
'foray': False,
'epps': False,
'teddy': False,
'Disliking': False,
'GLACIALLY': False,
'trapping': False,
'screen-': False,
'on-street': False,
'wroting': False,
'Meatwad': False,
'curtains': False,
'threaten': False,
'dibley': False,
'station': False,
'COUNTY': False,
'It-is-excellent': False,
'kick-boxer': False,
'lugubrious': False,
"'L'Age": False,
'Proog': False,
'granddaughters': False,
'Yashiro': False,
'bemoans': False,
'enhancement': False,
'attemps': False,
'decadents': False,
'embellishes': False,
'Jigen': False,
'tracks.It': False,
'Jotted': False,
'thailand': False,
'sewn': False,
'Erickson': False,
'maneuvering': False,
'Eisenstein': False,
'chillout': False,
'Worms': False,
"'gratitude": False,
'bullcrap': False,
'Throughout': False,
'stare': False,
'propagandist': False,
'sexually-frustrated': False,
'flawless': False,
'ISD': False,
'foxs': False,
'broaden': False,
'lodger': False,
'10min': False,
'timing': False,
'low-angles': False,
'withholds': False,
'Conquistador': False,
'lutz': False,
'expounds': False,
'sensible': False,
'fakey-looking': False,
'Mutha': False,
'off-off': False,
'Birnam': False,
"'96": False,
'perplexing': False,
'-D': False,
'Secession.': False,
'injuring': False,
'SUGAR': False,
'hero.This': False,
'blamed': False,
'Berlusconi': False,
'Hanoverian': False,
'hallway': False,
'whittle': False,
'impulses': False,
'demon-kicking': False,
're-work': False,
'Hedley': False,
'reverence': False,
'Punks': False,
'pertinence': False,
'Consisting': False,
'predictible': False,
'iniquity': False,
'unbelievers': False,
'meant': False,
'figuring': False,
'doncha': False,
'unmarked': False,
'feel.Carne': False,
'runaway': False,
'kill-fest': False,
'Belmondo': False,
'top-billing': False,
'larry': False,
'crucify': False,
'motorcycles': False,
'changes-of-heart': False,
'Muddled': False,
'ted': False,
'12th-rate': False,
'GV': False,
'duos': False,
'Homeland': False,
'government': False,
'Mahattan': False,
'sunburst': False,
'WTSE': False,
'Klemperer': False,
'Paravasam': False,
'basement': False,
'boite': False,
'film-as': False,
'chambara': False,
'skeleton-guide-': False,
'precedents': False,
'mid-afternoon': False,
'caste': False,
'heaviest': False,
'killer': False,
'nicely-written': False,
'bible/water': False,
'Silvers': False,
'accusing': False,
'82-years-old': False,
'kudos.': False,
'hassling': False,
'Varotto': False,
'echelon': False,
'television-to-big': False,
'romania': False,
'intercut': False,
'.Hobgoblins': False,
'profanity.': False,
'GOP': False,
'volume': False,
'Mecha-Ghidora': False,
'pictures': False,
'v.': False,
'Spratt': False,
'communal': False,
'taunts': False,
'redeemiing': False,
'Raft': False,
'Warlock': False,
'settle': False,
"could't": False,
'Loew': False,
'COLLECTOR': False,
'primordial': False,
'Batfan': False,
'*Vomit*': False,
'forgettable': False,
'Timing': False,
"'care": False,
'opinion.Some': False,
'Obs': False,
'Ivory.': False,
'performances.Now': False,
'TOUR': False,
'Galiano': False,
'wastebasket': False,
'Bio-Terror': False,
'0-5': False,
'faxes': False,
'fastened': False,
'horticulturalist': False,
'Avocado': False,
'Edward': False,
'Mace': False,
'Citroen': False,
'tor': False,
'Sensational': False,
'servings': False,
'C.O.R.E': False,
'Teesh': False,
'rays.': False,
'briefly-appearing': False,
'Cs': False,
'centralized': False,
...}
In [20]:
feature_sets=[]
files = os.listdir('aclimdb/train/pos')[:1000]
for file in files:
with open('aclimdb/train/pos/{}'.format(file), 'r' , encoding = 'utf-8') as f:
review = nltk.word_tokenize(f.read())
feature_sets.append((find_features(review), 'pos'))
In [21]:
clf = nltk.NaiveBayesClassifier.train(training_set)
---------------------------------------------------------------------------
NameError Traceback (most recent call last)
<ipython-input-21-b1229196e194> in <module>()
----> 1 clf = nltk.NaiveBayesClassifier.train(training_set)
NameError: name 'training_set' is not defined
In [ ]:
Content source: kiseyno92/SNU_ML
Similar notebooks: