In [1]:

    

import pandas as pd
import numpy as np
import re


    

In [3]:

    

# Preprocess dataset
def clean_str(string):
    """
    Tokenization/string cleaning
    Original taken from https://github.com/yoonkim/CNN_sentence/blob/master/process_data.py
    """
    string = re.sub(r"[^A-Za-z0-9(),!?\'\`]", " ", string)
    string = re.sub(r"\'s", " \'s", string)
    string = re.sub(r"\'ve", " \'ve", string)
    string = re.sub(r"n\'t", " n\'t", string)
    string = re.sub(r"\'re", " \'re", string)
    string = re.sub(r"\'d", " \'d", string)
    string = re.sub(r"\'ll", " \'ll", string)
    string = re.sub(r",", " , ", string)
    string = re.sub(r"!", " ! ", string)
    string = re.sub(r"\(", " \( ", string)
    string = re.sub(r"\)", " \) ", string)
    string = re.sub(r"\?", " \? ", string)
    string = re.sub(r"\s{2,}", " ", string)
    
    return string.strip().lower()


def load_data_and_labels(csv_file, text_col, target_col):
    """
    Loads IMDb plot data from csv, splits the data into words and generates labels.
    Returns split sentences and labels.
    """
    # Load CSV file, indicate that the last column represents labels
    df = pd.read_csv(csv_file)
    features = df[text_col]
    labels = df[target_col]

    # Split by words
    x_text = [clean_str(str(sent)) for sent in features]
    
    # Generate labels as [1, 0] for oscar win, [0, 1] otherwise
    y = [[1, 0] if row == 1 else [0, 1] for row in labels]
    y = np.array(y)
    
    return [x_text, y]

x_text, y = load_data_and_labels('movies_plot.csv', 'Plot Summary', 'Won')

print x_text[0], y[0]


    

    




the film opens with the rabid ranting and raving of arthur eden 's \( tom wilkinson \) most of this speech is rambling and babbling he begins talking about how one day he was leaving work and he walked out into the street into traffic and suddenly he found he could not move , even though there was traffic incoming essentially he tells us he was frozen in a moment of utter consternation brought on by the shambles he has made of his life doing what it is that he does his work as an attorney helping a corrupt company defend the indefensible has left him where must break the shackles with his old life and try something new in an office a legal team of dozens of people are diligently taking phone calls , shedding documents , comparing notes , etc a lawyer talking to a reporter on the phone and walks over to marty bach \( sydney pollack \) and hands him the phone he tells bach that it 's a reporter from the wall street journal inquiring about the settlement bach looks infuriated but takes the phone he simply tells the reporter that u north is not settling at this time and that nothing has changed in the legal case the reporter remains dubious , but bach says he does not care and simply hangs up the phone bach then asks about where crowder went to we then see karen crowder \( tilda swinton \) in the bathroom having what looks like a panic attack the film cuts to a card game where we see michael clayton \( george clooney \) at the table with a handful of other men one of the men recognizes clayton he tells clayton he knows him from a game they played some years back clayton looks up and admits he recognizes him the player says he won a whole bunch of money off clayton the player continues to brag about winning a whole bunch of cash off clayton and it clearly annoys him clayton says i see you 've bought some hair since then , but the other gambler retorts , yeah , with your money clayton leaves clayton is sent by a legal friend of his to the house of a man who desperately needs legal counsel clayton learns that the man hit a pedestrian and may have killed him earlier that night clayton explains to the man and his wife that his options are limited even though it may have been an accident , there are all sorts of variables that could affect the criminality of the case clayton tells the man he needs to find a good trial lawyer , because he is going to need one the man becomes irate and asks , what the hell are you supposed to be \? he wants to know why clayton ca n't be his lawyer and he tells clayton that he was promised a miracle worker clayton tells him that is not the case he explains what he does by calling himself a janitor clayton is an overall strategic legal adviser he has a tremendous acumen for understanding how to plan and execute a legal defense from start to finish even though he actually does not lead any of it he merely advises clayton finally realizes there is nothing more he can give the man and leaves as clayton is driving he notices his car 's gps system is behaving erratically as clayton drives an intriguing sight catches his eye he sees three horses standing on a hilltop together by themselves clayton is so captivated by this event he stops the car clayton approaches the horses and is fascinated by their presence in the middle of nowhere clayton 's car suddenly explodes behind him and the horses charge off clayton is stunned and petrified for a few moments as he looks at the flaming wreck when he finally regains his wits he runs away the film cuts to four days earlier clayton 's son henry \( austin williams \) is getting ready to go to school clayton is divorced from his wife and his son lives with her and his step dad henry leaves the house and joins clayton in his car clayton drives henry to school on the way henry tells clayton about all the fantasy novels he likes to read he describes how the current one is about a bunch of characters that are in a universe where they each have a nemesis , but do n't know who that person is so they ca n't trust anyone and just live for themselves clayton laughs and says , that sounds familiar , obviously alluding to his own legal work we next see crowder , who is a head executive for u north , is a tremendously large , corporate agricultural conglomerate that creates various types of substances to help better cultivate land for farming as we observer her , the film cuts back and forth between her giving an interview on camera with her boss don jefferies \( ken howard \) and crowder anxiously rehearsing the interview in her hotel room as she tries to prepare herself for it the subject of crowder 's interview is about how she is taking jefferies' old job as one of the head corporate managers as she is giving the interview , an assistant walks in and interrupts them with what clearly appears to be a urgent matter she tells them that eden just stripped down in the middle of a deposition for their current legal battle this is a dire matter , because eden is handling the legal defense for this case involving a weed killer which is potentially toxic and lethal which stands to lose u north billions of dollars we then cut back to clayton who is visiting an older man the man is a bookmaker who takes bets and handles local gambling operations we learn of clayton 's severe gambling addiction clayton is in debt for a massive amount of money to the gamblers he has sold off much of his possessions , but yet still is approximately 80 , 000 short of what he needs to settle his debt the man tells clayton that he had better pay up in a week or the consequences could be catastrophic clayton gets in contact with the head of his legal firm , marty bach bach informs clayton of the situation with arthur eden who also works for the firm bach tells him of the disastrous ramifications of eden 's mental breakdown clayton says that eden 's episode is understandable given the amount of time and energy eden has invested into this case he basically says that for the last decade eden has basically lived this case and his life is tumultuous and fractured because of it this case is in many ways just as big of a deal for the legal firm as it is for u north , because it coincides with a vital impending merger with a london based firm clayton then agrees to head to minnesota to meet with eden who is currently being held in a police station as clayton arrives at the station we hear crowder \( who is at one of the u north offices \) discussing with some of her assistants clayton 's background she learns from documents that clayton used to be a major criminal prosecutor , but now has been seemingly relegated to a more menial task of handling wills and trusts crowder seems baffled that michael is the one being sent to handle the crisis with eden as she has been saying all this , we are watching clayton arrive at the station and visiting eden in a holding cell he meets with eden and learn more about eden 's crazed rambling from the start of the film as eden finishes his lengthy and deranged claims , clayton becomes completely livid and says , you stopped taking your pills again \? did n't you \? , as he shows him the still clearly full bottle eden says he is no longer going to be a slave to his medication and that his eyes have been opened by a woman who is one of the plaintiffs in the lawsuit against u north named anna he claims that anna is a beautiful , kind , and graceful soul and that he refuses to assist in the defense of a product against her and all the other people involved in the law suit knowing that it has really is a killer and how much pain it has caused them clayton can clearly see how grievous the situation is eden is potentially infatuated with one of the people he is supposed to be defending u north from and may have caused irreparable damage to the suit in the process he tries to remind eden of his responsibilities and what he means to the firm , but eden only responds to this by proclaiming , i am shiva the god of death ! realizing that eden is incorrigible at this point and is causing a scene at the precinct which could potentially make things even worse than they are , clayton persuades eden to leave with him and return to his hotel room the next day clayton meets with crowder who is extremely upset over all that has transpired with eden clayton reassures crowder that all will be fine he tells her that eden does have a history of mental illness , but that it has been largely regulated in recent years he tells her that his last episode was several years ago and that it was not a major issue and eden was able to continue his war largely unhindered clayton reminds crowder how hard u north has been pushing him on this case and how he has been near singularly devoted to this case for a long time now crowder however , seems full of antipathy over clayton 's claims and says that u north has been paying clayton 's legal firm a very lucrative sum for eden 's time clayton simply reassures her that he will straighten eden out and everything will return to its previous status crowder however , seems completely dubious about this we then see crowder back at the u north office watching a videotape of eden 's episode from the deposition everything is going routinely until eden suddenly stands up and begins spouting a bunch of rhetoric in a obstreperous tone about how anna was right and how wrong he was he tells her he does n't want to continue this charade any longer as he is saying all this he is stripping off all his clothes the tape finally ends and we are informed that eden then followed anna into the parking lot outside totally naked crowder is deeply mortified and aghast as she realizes how damaging this tape could be to further legal proceedings especially given how eden seemingly admits to u norths guilt in the case the scene shifts back again to eden in the hotel room at night , who has managed to contact clayton 's son henry henry tells him about another new book he is reading called realm conquest he says in this book the characters are experiencing visions of things that would seem too fantastic to possibly be real , but they challenge conventional opinions and choose to believe that these illusions are real and that they must believe in them regardless of what others deem to be true , naturally eden hears this books plot and it compels him to do the same and keep believing what he has come view u north as , a corrupt company attempt to insulate themselves from their crimes in the morning eden tells michael that he should go see anna and he too will feel stimulated to stop trying to help u north in perpetuating their lies clayton tries to assuage eden and tells him to just take a shower so they can get ready to return to the legal firm in new york however , clayton gets occupied for a second and leaves eden alone , when he comes back the bathroom door is locked clayton can hear the shower running , but when he knocks on the door there is no response finally , clayton kicks the door down to discover eden has fled he goes over to the window and sees that eden has left back at u north crowder learns of the new situation we see her sitting at a desk looking over a document with an agitated expression we see the document she is reading it is an old scientific report from one of u north 's research teams about the hazardous effects of exposure to the weed killer involved in the law suit it clearly documents that prolonged exposure to the substance can lead to cancer as we see crowder reading this document , she is contacting someone on the phone it becomes immediately evident from the surreptitious tone of the discussion that this is some sort of private cabal of operatives that u north uses to deal with problems in all sorts of nefarious ways meanwhile , clayton returns to new york to meet with bach bach is absolutely incensed over what has transpired he tells clayton that this affair could mean the end of the firm he says that if the case goes under that u north could potentially sue the firm for their legal incompetence and ruin them with the merger coming the situation is even more grave bach asks clayton why he just did n't have eden committed in minnesota and says that would 've helped quell the problems for the time being until things could be ameliorated clayton tells bach that it is not that simple and that involuntary commitment to a mental facility is a very arduous procedure mary asks clayton well do n't we have anyone who knows a lot about these types of commitment cases \? clayton says yes , but informs bach that the person who is the expert unfortunately is eden eden has also returned to new york and is now roaming freely he is living a reclusive lifestyle in a private apartment however , the men hired by u north are already watching him they are tracking his movements around the city , have bugged his phone and are surveilling his actions all the while they are keeping crowder abreast of important developments during this time michael pays a visit to anna in an attempt to see if he can gain more information about the case he arrives at a farm in minnesota after introducing himself to the family , he taken to meet the woman with whom eden has become so enamored anna \( merritt wever \) clayton gains some more information about eden and the case she seems puzzled as to why eden seems to have been selected to communicate with her when there are over four hundred other people suing u north anna seems rather oblivious of how personally connected eden is to her soon thereafter eden again contacts anna over the telephone he hints to her that he has more information about her case and he will contact her again in the near future about the matter anna still seems perplexed and feels awkward about being contacted by the man formerly leading the u north defense assisting her back in new york bach notifies clayton about eden getting in contact with anna again it has now become clear that eden wont abandon this matter and he sends clayton to go to eden 's location and try to reason with him one last time clayton confronts eden on the street eden seems startled , but greets clayton clayton however , is in no mood for pleasantries and is still enraged over eden 's continuing erratic and irrational behavior clayton mentions about eden going to visit anna again and how idiotic this was eden immediately becomes suspicious of how clayton could possibly have become privy to a private phone call he made to anna he inquires to clayton about this and clayton responds that bach told him he then accuses u north and the legal firm of colluding together to tap his phone clayton tells eden this is absolutely preposterous and says the most logical answer is that anna contacted the police herself after eden called her eden disputes this saying that anna would never betray him like that , but michael tells eden this is lunacy he says after the crazy antics eden pulled at the deposition , stripping down before anna and shouting all sorts of psychotic things , it would only be expected she would contact the police eden as previously seems totally innocuous to clayton 's continuing reproach and his attempts to reason with him finally clayton makes threats about the state of eden 's sanity and how his recent behavior could lead to him being hospitalized , but eden still clearly retains his knowledge of legal matters he tells clayton that this is new york and here he has no criminal record or history of threatening behavior both of which are critical for compulsory treatment he says that now that he is out of minnesota , that clayton can no longer use prior incidents there to try and go after him in this state clayton finally concedes that he cannot win here and leaves later that night eden is in his apartment he makes a tape of one of u norths commercials he then plays this tape as he makes a phone call , correctly knowing that the u north agents are listening he plays the commercial and then begins reading in a sardonic tone to them the scientific report from earlier in the film as he reads it , he mocks the company and the consequences the report could have the agents are shocked to learn that eden has this document in his possession the agents meet with crowder when she hears the tape she becomes very perturbed she asks the agents what can be done they make some suggestions about additional measures , but then crowder subtly implies she wants them to go much further than just mere subterfuge u north agents enter eden 's apartment , drug him , and inject a drug under his toenail , killing him they then arrange a suicide scene michael clayton is notified about eden 's death at a police station the police detective tells clayton that it looks like a possible suicide or accidental overdose clayton is confounded and does n't believe that eden would kill himself , but the policemen reminds him about eden s perverse behavior in recent weeks clayton asks if he can go to eden 's apartment , but the officer says this is impossible and the place has been sealed clayton meets with his fellow legal employees there he tells bach that eden was onto something and clayton feels it is linked to his death clayton tells bach that he desperately needs a loan bach promises clayton he will look into the loan and given what a valuable asset clayton has been to the firm he should be able to convince them to help him out bach also notifies clayton that eden was scheduled to meet with anna in new york after contacting anna 's sister and learning of anna 's location clayton arrives at a hotel where she is staying clayton inquires with anna about the final conversation she and eden had anna says eden told her he has an item in his possession which could win the case for her clayton now becomes more convinced eden 's death was no mere suicide clayton goes to a family gathering at his sister 's house his brothers timmy clayton \( david lansbury \) and raymond clayton \( kevin hagan \) are there timmy is a recovering substance abuser and the two have a rather uncomfortable confrontation over this , which clayton 's son henry witnesses clayton also talks with his brother ray \( who is a policeman \) about needing a possible favor regarding eden 's death after the party , clayton talks with henry and says that his son does n't have to worry about going astray in life like timmy did and that he believes in him clayton then meets with his brother ray who provides him with keys to eden 's apartment ray reminds clayton that this is a serious issue and clayton cannot get caught eden 's apartment is still watched by the u north operatives and they watch him enter clayton searches eden 's apartment and finds a copy of realm and conquest , the book clayton 's son henry was reading when clayton opens it , a slip of paper falls out it is a receipt from a copy store which shows eden ordered copies of a memorandum shortly afterward cops arrest clayton raymond gets clayton out of jail , but is angry because clayton 's illegal entry might be connected to ray clayton tells ray someone set him up at eden 's place and that they called the cops following this clayton goes to the printing place with eden 's receipt there he finds out that eden has ordered 3000 copies of the confidential u north report he tells the clerk to hold onto the documents for him and takes one with him as he leaves , the u north operatives arrive and learn about the copies they then tell crowder about the copies she realizes she must do something immediately before clayton finishes what eden started clayton meets with bach who gives him a check with the money needed to pay off his gambling debt with the check , clayton signs a 3 year contract that binds him to secrecy about past and future events clayton visits the man to pay what his gambling debt , and then plays cards in a underground card game for a few minutes from this point the film recaps the opening moments of the film clayton leaving the card game , but this time we also see the u north agents putting an explosive bomb into his car since clayton leaves the game early they did n't finish hooking the bomb up , which apparently interfered with his gps signal the two u north operatives following him experience difficulty tracking the device 's signal clayton reverses direction and then takes a side road , and they agents lose him they trigger the bomb , not knowing clayton is not in the car clayton runs back to the flaming car and throws his wallet , watch and other personal belongings inside a short time later clayton 's brother timmy picks him up the staff at clayton 's law firm is notified of clayton 's supposed death from a car bomb in a hotel theatre , crowder presents a case for settling the law suit to u north executives she says the timing for a settlement has improved due to recent events and that the cost will be offset by tax benefits jefferies asks her to leave while they discuss the proposed settlement crowder steps outside and is slack jawed to see michael clayton standing before her , thinking he is dead clayton tells her that arthur is around here somewhere he shows crowder the report and tells her how foolish she was to try and eliminate him he tells her it would 've been far easier and more prudent to just buy him off and tells crowder he wants 10 clayton tells her to either pay up or he will go public with the report crowder asks him what he wants she asks him if there is a number \? she says , 10 million \? he offers 5 to forget about eden , which she says would be easier and then he says the other 5 is to forget about the other 468 people you knocked off with your weed killer he forces her to repeat the terms of the bribe , and she tells him , you have a deal jefferies emerges from the conference room and asks crowder to rejoin the board meeting she finally agrees to clayton 's terms for the payoff , he promptly responds , you 're so fucked crowder , confused , asks , you do n't want the money \? clayton says wait , let me get a picture of this ! and takes a picture of her with his cell phone he then begins to walk away jefferies asks who clayton is , and clayton responds , quoting eden , i am shiva the god of death ! jefferies becomes irate and calls for security to seize clayton crowder slumps to the floor in shock and dismay a group of police officers walk past clayton and instead move to arrest crowder and jefferies clayton asks his brother ray , did you get all that \? clayton thanks him for helping him out clayton walks outside and gets in a taxi he gives the driver 50 and tells him to just drive as the taxi drives away the credits roll [0 1]

In [4]:

    

# Build vocabulary
from tensorflow.contrib import learn

max_document_length = max([len(x.split(" ")) for x in x_text])
vocab_processor = learn.preprocessing.VocabularyProcessor(max_document_length)
x = np.array(list(vocab_processor.fit_transform(x_text)))
print("Vocabulary size: {:d}".format(len(vocab_processor.vocabulary_)))


    

    




Vocabulary size: 15299

In [5]:

    

# Randomly shuffle data
np.random.seed(42)
shuffle_indices = np.random.permutation(np.arange(len(y)))
x_shuffled = x[shuffle_indices]
y_shuffled = y[shuffle_indices]

# Saves preprocessed data into a pickle file
import pickle

with open('preprocess_x_2.pickle', 'wb') as handle:
    pickle.dump(x_shuffled, handle, protocol=pickle.HIGHEST_PROTOCOL)
    
with open('preprocess_y_2.pickle', 'wb') as handle:
    pickle.dump(y_shuffled, handle, protocol=pickle.HIGHEST_PROTOCOL)

print "Files saved."


    

    




Files saved.

In [7]:

    

# Preprocess test dataset

def preprocess_test_set(csv_file, text_col):
    # Load csv test file
    df = pd.read_csv(csv_file)
    features_test = df[text_col]
    
    # Split by words
    x_text_test = [clean_str(sent) for sent in features_test]
    
    # Convert text to lower representations
    x = np.array(list(vocab_processor.transform(x_text_test)))
    return x

x = preprocess_test_set('movies_plot_pred.csv', 'Plot Summary')

with open('preprocess_test_2.pickle', 'wb') as handle:
    pickle.dump(x, handle, protocol=pickle.HIGHEST_PROTOCOL)

print "File saved."


    

    




File saved.

In [ ]: