notebook.community

Edit and run 





In [34]:



    


import pandas as pd
import numpy as np
import seaborn as sns
import matplotlib.pyplot as plt
from matplotlib.pyplot import show
%matplotlib inline

import glob # to find all files in folder

import pycountry
import re # regex
from nltk.sentiment.util import *
import nltk as nl
from nltk.corpus import stopwords as nlstopw
import string
from matplotlib import cm

Import data





In [2]:



    


folder = 'hillary-clinton-emails/'



    











In [3]:



    


emails = pd.read_csv(folder + 'Emails.csv', index_col='Id')
emails.head(5)



    


















    
Out[3]:










  
    

      
      DocNumber
      MetadataSubject
      MetadataTo
      MetadataFrom
      SenderPersonId
      MetadataDateSent
      MetadataDateReleased
      MetadataPdfLink
      MetadataCaseNumber
      MetadataDocumentClass
      ...
      ExtractedTo
      ExtractedFrom
      ExtractedCc
      ExtractedDateSent
      ExtractedCaseNumber
      ExtractedDocNumber
      ExtractedDateReleased
      ExtractedReleaseInPartOrFull
      ExtractedBodyText
      RawText
    

    

      Id
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
    

  
  
    

      1
      C05739545
      WOW
      H
      Sullivan, Jacob J
      87.0
      2012-09-12T04:00:00+00:00
      2015-05-22T04:00:00+00:00
      DOCUMENTS/HRC_Email_1_296/HRCH2/DOC_0C05739545...
      F-2015-04841
      HRC_Email_296
      ...
      NaN
      Sullivan, Jacob J <Sullivan11@state.gov>
      NaN
      Wednesday, September 12, 2012 10:16 AM
      F-2015-04841
      C05739545
      05/13/2015
      RELEASE IN FULL
      NaN
      UNCLASSIFIED\nU.S. Department of State\nCase N...
    

    

      2
      C05739546
      H: LATEST: HOW SYRIA IS AIDING QADDAFI AND MOR...
      H
      NaN
      NaN
      2011-03-03T05:00:00+00:00
      2015-05-22T04:00:00+00:00
      DOCUMENTS/HRC_Email_1_296/HRCH1/DOC_0C05739546...
      F-2015-04841
      HRC_Email_296
      ...
      NaN
      NaN
      NaN
      NaN
      F-2015-04841
      C05739546
      05/13/2015
      RELEASE IN PART
      B6\nThursday, March 3, 2011 9:45 PM\nH: Latest...
      UNCLASSIFIED\nU.S. Department of State\nCase N...
    

    

      3
      C05739547
      CHRIS STEVENS
      ;H
      Mills, Cheryl D
      32.0
      2012-09-12T04:00:00+00:00
      2015-05-22T04:00:00+00:00
      DOCUMENTS/HRC_Email_1_296/HRCH2/DOC_0C05739547...
      F-2015-04841
      HRC_Email_296
      ...
      B6
      Mills, Cheryl D <MillsCD@state.gov>
      Abedin, Huma
      Wednesday, September 12, 2012 11:52 AM
      F-2015-04841
      C05739547
      05/14/2015
      RELEASE IN PART
      Thx
      UNCLASSIFIED\nU.S. Department of State\nCase N...
    

    

      4
      C05739550
      CAIRO CONDEMNATION - FINAL
      H
      Mills, Cheryl D
      32.0
      2012-09-12T04:00:00+00:00
      2015-05-22T04:00:00+00:00
      DOCUMENTS/HRC_Email_1_296/HRCH2/DOC_0C05739550...
      F-2015-04841
      HRC_Email_296
      ...
      NaN
      Mills, Cheryl D <MillsCD@state.gov>
      Mitchell, Andrew B
      Wednesday, September 12,2012 12:44 PM
      F-2015-04841
      C05739550
      05/13/2015
      RELEASE IN PART
      NaN
      UNCLASSIFIED\nU.S. Department of State\nCase N...
    

    

      5
      C05739554
      H: LATEST: HOW SYRIA IS AIDING QADDAFI AND MOR...
      Abedin, Huma
      H
      80.0
      2011-03-11T05:00:00+00:00
      2015-05-22T04:00:00+00:00
      DOCUMENTS/HRC_Email_1_296/HRCH1/DOC_0C05739554...
      F-2015-04841
      HRC_Email_296
      ...
      NaN
      NaN
      NaN
      NaN
      F-2015-04841
      C05739554
      05/13/2015
      RELEASE IN PART
      H <hrod17@clintonemail.com>\nFriday, March 11,...
      B6\nUNCLASSIFIED\nU.S. Department of State\nCa...
    

  



5 rows × 21 columns

Analyse Emails





In [4]:



    


emails.head()



    


















    
Out[4]:










  
    

      
      DocNumber
      MetadataSubject
      MetadataTo
      MetadataFrom
      SenderPersonId
      MetadataDateSent
      MetadataDateReleased
      MetadataPdfLink
      MetadataCaseNumber
      MetadataDocumentClass
      ...
      ExtractedTo
      ExtractedFrom
      ExtractedCc
      ExtractedDateSent
      ExtractedCaseNumber
      ExtractedDocNumber
      ExtractedDateReleased
      ExtractedReleaseInPartOrFull
      ExtractedBodyText
      RawText
    

    

      Id
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
    

  
  
    

      1
      C05739545
      WOW
      H
      Sullivan, Jacob J
      87.0
      2012-09-12T04:00:00+00:00
      2015-05-22T04:00:00+00:00
      DOCUMENTS/HRC_Email_1_296/HRCH2/DOC_0C05739545...
      F-2015-04841
      HRC_Email_296
      ...
      NaN
      Sullivan, Jacob J <Sullivan11@state.gov>
      NaN
      Wednesday, September 12, 2012 10:16 AM
      F-2015-04841
      C05739545
      05/13/2015
      RELEASE IN FULL
      NaN
      UNCLASSIFIED\nU.S. Department of State\nCase N...
    

    

      2
      C05739546
      H: LATEST: HOW SYRIA IS AIDING QADDAFI AND MOR...
      H
      NaN
      NaN
      2011-03-03T05:00:00+00:00
      2015-05-22T04:00:00+00:00
      DOCUMENTS/HRC_Email_1_296/HRCH1/DOC_0C05739546...
      F-2015-04841
      HRC_Email_296
      ...
      NaN
      NaN
      NaN
      NaN
      F-2015-04841
      C05739546
      05/13/2015
      RELEASE IN PART
      B6\nThursday, March 3, 2011 9:45 PM\nH: Latest...
      UNCLASSIFIED\nU.S. Department of State\nCase N...
    

    

      3
      C05739547
      CHRIS STEVENS
      ;H
      Mills, Cheryl D
      32.0
      2012-09-12T04:00:00+00:00
      2015-05-22T04:00:00+00:00
      DOCUMENTS/HRC_Email_1_296/HRCH2/DOC_0C05739547...
      F-2015-04841
      HRC_Email_296
      ...
      B6
      Mills, Cheryl D <MillsCD@state.gov>
      Abedin, Huma
      Wednesday, September 12, 2012 11:52 AM
      F-2015-04841
      C05739547
      05/14/2015
      RELEASE IN PART
      Thx
      UNCLASSIFIED\nU.S. Department of State\nCase N...
    

    

      4
      C05739550
      CAIRO CONDEMNATION - FINAL
      H
      Mills, Cheryl D
      32.0
      2012-09-12T04:00:00+00:00
      2015-05-22T04:00:00+00:00
      DOCUMENTS/HRC_Email_1_296/HRCH2/DOC_0C05739550...
      F-2015-04841
      HRC_Email_296
      ...
      NaN
      Mills, Cheryl D <MillsCD@state.gov>
      Mitchell, Andrew B
      Wednesday, September 12,2012 12:44 PM
      F-2015-04841
      C05739550
      05/13/2015
      RELEASE IN PART
      NaN
      UNCLASSIFIED\nU.S. Department of State\nCase N...
    

    

      5
      C05739554
      H: LATEST: HOW SYRIA IS AIDING QADDAFI AND MOR...
      Abedin, Huma
      H
      80.0
      2011-03-11T05:00:00+00:00
      2015-05-22T04:00:00+00:00
      DOCUMENTS/HRC_Email_1_296/HRCH1/DOC_0C05739554...
      F-2015-04841
      HRC_Email_296
      ...
      NaN
      NaN
      NaN
      NaN
      F-2015-04841
      C05739554
      05/13/2015
      RELEASE IN PART
      H <hrod17@clintonemail.com>\nFriday, March 11,...
      B6\nUNCLASSIFIED\nU.S. Department of State\nCa...
    

  



5 rows × 21 columns

The columns ExtractedBodyText is supposed to be the content of the mail but some of the mail have a ExtractedBodyText = NaN but the Rawtext seems to contains something





In [5]:



    


emails.columns



    


















    
Out[5]:








Index(['DocNumber', 'MetadataSubject', 'MetadataTo', 'MetadataFrom',
       'SenderPersonId', 'MetadataDateSent', 'MetadataDateReleased',
       'MetadataPdfLink', 'MetadataCaseNumber', 'MetadataDocumentClass',
       'ExtractedSubject', 'ExtractedTo', 'ExtractedFrom', 'ExtractedCc',
       'ExtractedDateSent', 'ExtractedCaseNumber', 'ExtractedDocNumber',
       'ExtractedDateReleased', 'ExtractedReleaseInPartOrFull',
       'ExtractedBodyText', 'RawText'],
      dtype='object')

















In [6]:



    


print('Number of emails: ', len(emails))



    


















    






Number of emails:  7945

















In [7]:



    


bodyNaN = emails.ExtractedBodyText.isnull().sum()
print('Number of emails with ExtractedBodyText=NaN: {}, ({:.2f}%)'.format(emails.ExtractedBodyText.isnull().sum(), bodyNaN/ len(emails)))



    


















    






Number of emails with ExtractedBodyText=NaN: 1203, (0.15%)

We could also use the subject since it is usually a summary of the mail





In [8]:



    


bodyNaN = emails.ExtractedSubject.isnull().sum()
print('Number of emails with ExtractedSubject=NaN: {}, ({:.2f}%)'.format(emails.ExtractedBodyText.isnull().sum(), bodyNaN/ len(emails)))



    


















    






Number of emails with ExtractedSubject=NaN: 1203, (0.21%)

Now let's try to combine the subject and the body and drop the mail that have both subject= NaN and body = Nan





In [9]:



    


subBodyNan = emails[np.logical_and(emails.ExtractedBodyText.isnull(),emails.ExtractedSubject.isnull())]
print('Number of email where both subject and body is NaN: {}({:.2f})'.format(len(subBodyNan), len(subBodyNan)/ len(emails)))



    


















    






Number of email where both subject and body is NaN: 368(0.05)

Well, that number is small enough to drop all email where both Extracted subject and Extracted body is NaN.

Let's drop them and create a new columns subjectBody that is the concatenation of the 2 columns ExtractedSubject and ExtractedBody. From now we will work with that columns





In [10]:



    


emails = emails[~ np.logical_and(emails.ExtractedBodyText.isnull(), emails.ExtractedSubject.isnull())]
len(emails)



    


















    
Out[10]:








7577

















In [11]:



    


emails.ExtractedBodyText.fillna('',inplace=True)
emails.ExtractedSubject.fillna('',inplace=True)
emails['SubjectBody'] = emails.ExtractedBodyText + emails.ExtractedSubject
emails.SubjectBody.head()



    


















    
Out[11]:








Id
1                                              FW: Wow
2    B6\nThursday, March 3, 2011 9:45 PM\nH: Latest...
3                                 ThxRe: Chris Stevens
4                      FVV: Cairo Condemnation - Final
5    H <hrod17@clintonemail.com>\nFriday, March 11,...
Name: SubjectBody, dtype: object

Last check to be sur that our columns of interest don't have anymore NaN





In [12]:



    


print('Number of NaN in columns SubjectBody: ' ,emails.SubjectBody.isnull().sum())



    


















    






Number of NaN in columns SubjectBody:  0

Keep only mail that mentions a country

Structure of a country in pycountry.countres





In [13]:



    


list(pycountry.countries)[0]



    


















    
Out[13]:








Country(alpha_2='AW', alpha_3='ABW', name='Aruba', numeric='533')

First we create a dataframe with one line by countries and we count for each countries its occurences in the mail.

Since a country can be reference in many way (Switzerland, switzerland, CH), we need to consider all the possible form.

We may have a problem with word that have many meaning like US(country) and us (pronoun) so we can't just take all the country name in loer case and all the mail in lower case and just compare.

We first try to use that technique:

1. the country name can appear either in lower case, with the first letter in uppercase or all in uppercase
2. alpha_2 and alpha_3 are always used in uppercase

But we still have a lot of problem. Indeed a lot of mail contain sentences all in upper cas (see below):

   - SUBJECT TO AGREEMENT ON SENSITIVE INFORMATION & REDACTIONS. NO FOIA WAIVER. STATE-SCB0045012

For example this sentence will match for Togo because of the TO and also for norway because of the NO. An other example is Andorra that appears in 55 mails thanks to AND

At first we also wanted to keep the upper case since it can be helpfull to do the sentiment analysis. Look at those 2 sentence and their corresponding score:

- VADER is very smart, handsome, and funny.: compound: 0.8545, neg: 0.0, neu: 0.299, pos: 0.701,
- VADER is VERY SMART, handsome, and FUNNY.: compound: 0.9227, neg: 0.0, neu: 0.246, pos: 0.754,

The score is not the same. But since we have a lot of information in Upper case and it nothing to do with sentiment, we will put all mails in lower case. And we will also remove the stopwords.

We know that we remove the occurance of USA under 'us' but it will also remove the 'and', 'can', 'it',...





In [14]:



    


emails.SubjectBody.head(100).apply(print)



    


















    






FW: Wow
B6
Thursday, March 3, 2011 9:45 PM
H: Latest How Syria is aiding Qaddafi and more... Sid
hrc memo syria aiding libya 030311.docx; hrc memo syria aiding libya 030311.docx
March 3, 2011
For: Hillary
ThxRe: Chris Stevens
FVV: Cairo Condemnation - Final
H <hrod17@clintonemail.com>
Friday, March 11, 2011 1:36 PM
Huma Abedin
Fw: H: Latest: How Syria is aiding Qaddafi and more... Sid
hrc memo syria aiding libya 030311.docx
Pis print.
Pis print.
-•-...-^
H < hrod17@clintonernailcom>
Wednesday, September 12, 2012 2:11 PM
°Russorv@state.gov'
Fw: Meet The Right-Wing Extremist Behind Anti-fvluslim Film That Sparked Deadly Riots
From [meat)
Sent: Wednesday, September 12, 2012 01:00 PM
To: 11
Subject: Meet The Right Wing Extremist Behind Anti-Muslim Film That Sparked Deadly Riots
htte/maxbiumenthal.com12012/09/meet-the-right-wing-extremist-behind-anti-musiim-tihn-that-sparked-
deadly-riots/
Sent from my Verizon Wireless 4G LTE DROID
U.S. Department of State
Case No. F-2015-04841
Doc No. C05739559
Date: 05/13/2015
STATE DEPT. - PRODUCED TO HOUSE SELECT BENGHAZI COMM.
SUBJECT TO AGREEMENT ON SENSITIVE INFORMATION & REDACTIONS. NO FOIA WAIVER. STATE-5CB0045251Meet The Right Wing Extremist Behind Anti-Muslim Film That Sparked Deadly Riots
FW: Anti-Muslim film director in hiding, following Libya, Egypt violence
H <hrod17@clintonemail.corn>
Friday, March 11, 2011 1:36 PM
Huma Abedin
Fw: H: Latest: How Syria is aiding Qaddafi and more... Sid
hrc memo Syria aiding libya 030311.docx
Pis print.
FYIFVV: Secretary's remarks
B6
Wednesday, September 12, 2012 6:16 PM
Fwd: more on libya
Libya 37 sept 12 12.docx
Sending direct. Just in.
Sent from my Verizon Wireless 4G LTE DRUIDmore on Libya
Fyi
B6
— —AbZ and Hb3 on Libya and West Bank/Gaza
B6
Wednesday, September 12, 2012 6:16 PM
Fwd: more on libya
Libya 37 sept 12 12.docx
Sending direct. Just in.
Sent from my Verizon Wireless 4G LTE DRUID
Fyihey
Anne-Marie Slaughter
Sunday, March 13, 2011 9:39 PM
Jacob Mills, Cheryl D; Reinesp@stategov; Abedin, Hurtle
piece on libya nfz that will appear on NYT op-ed page tomorrow
B6
wanted to make sure you had a heads up. AM
Anne-Marie Slaughter is a professor of politics and international affairs at Princeton.
By Anne-Marie Slaughter
PRESIDENT Obama says the noose is tightening around Col. Muammar al-Qaddafi. In fact, it is
tightening around the Libyan rebels, as Colonel Qaddafi makes the most of the world's dithering and
steadily retakes rebel-held tow.ns, The United States and Europe are temporizing on a no-flight zone
while the Organization of the Islamic Conference, the Gulf Cooperation Council and now the Arab
League have all called on the United Nations Security Council to authorize one. Opponents of a no-
flight zone have put forth five main arguments, none of which, on closer examination, hold up.
It's not in our interest. Gen. Wesley K. Clark argues that "Libya doesn't sell much oil to the United
States" and that while Americans "want to support democratic movements in the region," they are
already doing that in Iraq and Afghanistan. Framing this issue in terms of oil is exactly what Arab
populations and indeed much of the world expect, which is why they are so cynical about our
professions of support for democracy and human rights. Now we have a chance to support a new
beginning in the Muslim world — a new beginning of accountable governments that can provide
services and opportunities for their citizens in ways that could dramatically decrease support for
terrorist groups and violent extremism. It's hard to imagine something more in our strategic interest.
It will be counter-productive. Many thoughtful commentators, including Al Jazeera's director
general, Wadah Khanfar, argue that what is most important about the Arab spring is that it is coming
from Arabs themselves. From this perspective, Western military intervention will play right into
Qaddati's hands, allowing him to broadcast pictures of Western bombs falling on Arab civilians. But
these arguments, while important, must be weighed against the appeals of of Libyan opposition
fighters asking for international help, and now, astonishingly, against support for a no-flight zone by
some of the same governments that have kept their populations quiescent by holding up the specter of
foreign intervention. Assuming that a no-flight zone can be imposed by an international coalition that
includes Arab states, we have an opportunity to establish a new narrative of Western support for Arab
democrats.
U.S. Department of State
Case No. F-2015-04841
Doc No. C05739568
Date: 05/13/2015
STATE DEPT. - PRODUCED TO HOUSE SELECT BENGHAZI COMM.
SUBJECT TO AGREEMENT ON SENSITIVE INFORMATION & REDACTIONS. NO FOIA WAIVER. STATE-SCB0045012
U.S. Department of State
Case No. F-2015-04841
Doc No. C05739568
Date: 05/13/2015
STATE DEPT. - PRODUCED TO HOUSE SELECT BENGHAZI COMM.
SUBJECT TO AGREEMENT ON SENSITIVE INFORMATION & REDACTIONS. NO FOIA WAIVER.
It won't work. The United States ambassador to NATO, Ivo H. Daaider, argues that stopping
Colonel Qaddafi's air force will not be decisive; he will continue to inflict damage with tanks and
helicopters, bombing oil refineries and depots on his way to retaking key towns. But the potential
effect of a no-flight zone must also be assessed in terms of Colonel Qaddafi's own calculations about
his future. Richard Downie of the Center for Strategic and International Studies argues that although
Colonel Qaddafi cultivates a mad-dictator image, he has been a canny survivor and political
manipulator for 40 years. He is aware of debates with regard to a no-flight zone and is timing his
military campaign accordingly; he is also capable of using his air force just enough to gain strategic
advantage, but not enough to trigger a no-flight zone. If the international community lines up against
him and is willing to crater his runways and take out his antiaircraft weapons, he might well renew his
offer of a negotiated departure.
If it does work, we don't know what we will get. Revolutions are almost always followed by internal
divisions among the revolutionaries. We should not expect a rosy, Jeffersonian Libya. But the choice
is between uncertainty and the certainty that if Colonel Qaddafi wins, regimes acrosS the region will
conclude that force is the livay to answer protests. And when Colonel Qaddafi massacres the
opposition, young protesters across the Middle East will conclude that when we were asked to support
their cause with more than words, we blinked. Americans in turn will read the words of Mr. Obama's
June 2009 speech in Cairo., with its lofty promises to stand for universal human rights, and cringe.
Let's arm the rebels instead. Some commentators who agree with the analysis above say we could
'better accomplish our goals by providing intelligence and arms to the opposition. That would, of
course, be much easier for us. It undoubtedly appeals to Mn Obama as a neat compromise between
the desire to help the protesters and the desire not to overrule his defense secretary's reluctance to
participate in a no-flight zone. But we would be providing arms not to a disciplined military, but to
ragged groups of brave volunteers who barely know how to use the weapons they have. They need
action that will change the situation on. the ground for Colonel Qaddafi, as well as his the calculations.
Moreover, by the time arms and intelligence could take effect, it is quite likely that Colonel Qaddafi
will have retaken or at least besieged .Benghazi, the opposition stronghold.
The United States should immediately ask the Security Council to authorize a no-flight zOne and
make clear to Russia and China that if they block the resolution, the blood of the Libyan opposition
will be on their hands. We should push them at least to abstain and bring the issue to a vote as soon as
possible. If we get a resolution, we should work with the Arab League to quickly assemble an
international coalition to impose the no-flight zone. If the Security Council fails toact, then we should
recognize the opposition Interim National Council as the legitimate Libyan government, as France has
done, and work with the Arab League to give the council any assistance it requests:
U.S. Department of State
Case No. F-2015-04841
Doc No. C05739568
Date: 05/13/2015
STATE DEPT. - PRODUCED TO HOUSE SELECT BENGHAZI COMM.
SUBJECT TO AGREEMENT ON SENSITIVE INFORMATION & REDACTIONS. NO FOIA WAIVER. STATE-SCB0045013
U.S. Department of State
Case No. F-2015-04841
Doc No. C05739568
Date: 05/13/2015
STATE DEPT. - PRODUCED TO HOUSE SELECT BENGHAZI COMM.
SUBJECT TO AGREEMENT ON SENSITIVE INFORMATION & REDACTIONS. NO FOIA WAIVER.
Any use of force must be carefully and fully debated, but that debate has now been had. Ifs been
raging for a week, during which almost every Arab country has come on board calling for a no-flight
zone and Colonel Qaddafi has retaken two key cities and is moving toward three more. It is time to
act. $
U.S. Department of State
Case No. F-2015-04841
Doc No. C05739568
Date: 05/13/2015
STATE DEPT. - PRODUCED TO HOUSE SELECT BENGHAZI COMM.
SUBJECT TO AGREEMENT ON SENSITIVE INFORMATION & REDACTIONS. NO FOIA WAIVER. STATE-5CB0045014
_ .....
From Randolph, Lawrence M
Sent: Wednesday, September 12, 2012 04:33 PM
To: Mills, Cheryl D
Subject: RE: Not a dry eye in NEA
Including mine. Her remarks were really moving. Chriswas an amazing man. Such a huge loss. You know, I was in Libya
before coming here and in my almost ten years, I have never worked with such a calm, cool headed, funny
diplomat. Made it all seem really easy- even in one of the hardest places to work in the world.RE: Not a dry eye in NEA
I asked to attend your svtc today with Embassy Tripoli, because had first met so many of that staff when I went with
you from Malta to Tripoli for the reopening of our Embassy.
Today's deaths hit me much harder than I would have guessed. I am always proud to serve under you, but never have
your words been more meaningful than on today's svtc. Every day of your tenure has been extraordinary, but none
more so than today. Thank you again for your inspirational leadership and example.
As ever,
Harold
U.S. Department of State
Case No. F-2015-04841
Doc No. C05739571
Date: 05/13/2015
STATE DEPT. - PRODUCED TO HOUSE SELECT BENGHAZI COMM.
SUBJECT TO AGREEMENT ON SENSITIVE INFORMATION & REDACTIONS. NO FOIA WAIVER. STATE-SCB0045269
Hope. See picture below Kamala sent.Fw: The Youth of Libya
Another photo.Fw: One More Photo
This is nice.Fw: S today
Amazing.
Sullivan, Jacob J <Sullivanii@state,gov>
Wednesday, September 12, 2012 10:34 PM
Sherman, Wendy R; H; Mills, Cheryl D; Nuland, Victoria J; Reines, Philippe
Re: The Youth of Libya
H <hrod17@clintonernaii.com›
Wednesday, September 12, 2012 11:26 PM
esullivanjj@state.gov'
Fw: Fwd: more on libya
Libya 37 sept 12 12,docx
We should get this around asap.Fwd: more on libya
Pis print.
H < hrod17@clintoriernail.corn>
Wednesday, September 12, 2012 11:28 PM
°Russont@state.gov°
Fw: Fwd: more on libya
Libya 37 sept 12 12.dacxFwd: more on libya
Pis print.Fw: H: Magariaf on attack on US in Libya. Sid
Follow Up Flag: Follow up
Flag Status: Flagged
More info,Fw: H: Magariaf on attack on US in Libya. Sid
B5Re: Proposed Quad Deal
Sidney Blumenthal
Thursday, September 13, 2012 12:29 AM
H: More Magariaf privat reax (sent now as memo). Sid
hre memo more magariaf private reax 091312.docx
B6
September 13, 2012
For: Hillary
Will do.Re: Fwd: more on libya
Remind me to discussFw: Amb Stevens
http://religion.b1ogs.cnn.com/20 1 2/09/13/my-take-a-deadlv-link-between-islamic-and-anti-islam c-extremists/
Editor's Note: Stephen Prothero, a Boston University religion scholar and author of "The American Bible:
How Our Words Unite, Divide, and Define a Nation," is a regular CNN Belief Blog contributor.
By Stephen Prothero, Special to CNN
What should we make of the attacks on the U.S. consulates in Egypt, Libya and Yemen, and the deaths of
four Americans, including U.S. ambassador to Libya Chris Stevens?
It depends on who you mean by "we."
From the perspective of those who stormed the embassies, taking down the American flag in Egypt and
replacing it with a black flag reading, "There is no God but Allah and Mohammed is his messenger," theirs was
an act of justice on behalf of the One True Faith.
The Americans had it coming, according to this view, because Americans are on the wrong side in the clash of
civilizations between the Judeo-Christian West and the Muslim world. And the creation of an anti-Islamic video
(ironically entitled "Innocence of Muslims") by a producer thought to be an American provided the catalyst
(or excuse) for the protests.
Why Muslims are sensitive about portraying the Prophet Mohammed
Most Americans operate by a different "we." From our perspective, the assault was an unjust and cowardly
attack on (among other things) the rule of law, the institution of diplomacy and the right of free speech.
Libyan Deputy Prime Minister Mustafa Abushagur gave voice to this perspective when he said on Twitter, "I
condemn these barbaric attacks in the strongest possible terms. This is an attack on America, Libya and free
people everywhere."
"Some have sought to justify this vicious behavior as a response to inflammatory material posted on the
Internet," Secretary of State Hillary Clinton added. "The United States deplores any intentional effort to
denigrate the religious beliefs of others. Our commitment to religious tolerance goes back to the very beginning
of our nation."
In a commencement speech at Kenyon College in 2005, later published as"This Is Water," novelist David
Foster Wallace urged student to scrutinize the "natural, basic self-centeredness" that puts "me" at the center of
the world.
This unconscious "default setting" fosters the sort of "blind certainty" that manifests as arrogant atheism and
fundamentalism alike, Wallace said. And the point of education is to shake us out of that "default setting" — to
begin to see things from the perspectives of others.
Follow the CNN Belief Blog on Twitter
U.S. Department of State
Case No. F-2015-04841
Doc No. C05739590
Date: 05/13/2015
STATE DEPT. - PRODUCED TO HOUSE SELECT BENGHAZI COMM.
SUBJECT TO AGREEMENT ON SENSITIVE INFORMATION & REDACTIONS. NO FOIA WAIVER. STATE-SCB0045293
B6
U.S. Department of State
Case No. F-2015-04841
Doc No. C05739590
Date: 05/13/2015
STATE DEPT. - PRODUCED TO HOUSE SELECT BENGHAZI COMM.
SUBJECT TO AGREEMENT ON SENSITIVE INFORMATION & REDACTIONS. NO FOIA WAIVER.
Wallace focused in his speech on the individual "I." But his critique can be applied as well to another "default
setting" (this one more cultural than natural): our tendency to understand the world from the perspective of
Give:5
According to those who stormed the embassies, "Innocence of Muslims" was not made by an individual
(whoever the shadowy "Sam Bacile" may — or may not — be). It was made by anAmerican.
Moreover, according to this groupthink, Americans as a group are responsible for the sins of individual
Americans, so it makes perfect sense to strike out at embassy personnel who issued a statement before the
protests turned deadly, condemning "the continuing efforts by misguided individuals to hurt the religious
feelings of Muslims, as we condemn efforts to offend believers of all religions."
CNN's Belief Blog: The faith angles behind the biggest stories
Meanwhile, the shadowy figures who produced and distributed "Innocence of Muslims" are imprisoned in some
groupthink of their own, which gathers all the world's Muslims under the category of an evil Islamic empire.
Their film doesn't just depict Mohammed as a con man, philanderer, and pedophile. It portrays his followers as
dupes.
Unfortunately, the events of recent days have done nothing to jar the alleged producer out of this default setting.
When questioned about his film in the aftermath of the Libyan and Egyptian attacks, a man identifying himself
as "Sam Bacile" reportedly told the Wall Street Journal that "Islam is a cancer."
You don't need a Kenyon college degree to see how this vicious cycle of provocation and violence is fueled,
and you don't need to be a moral relativist to see that it takes two sides to keep it running.
I am an American who is justly proud of the First Amendment and its protections of freedom of speech and
religion. And I believe that killing your opponents in the name of God or nation is a far greater offense than
offending them with an ignorant movie.
Nonetheless, the root problem on display here is in my view what Wallace denounced as "blind certainty."
Our world is not divided first and foremost along religious or national lines, into Muslims and Christians and
Hindus or Israelis and Americans and Egyptians.
Neither is it divided into secularists and people of faith — the New Atheists and partisans of old-time religion.
The real "red line" in the modem world divides the certain from the uncertain. And in the tragic events of recent
days, those who produced and distributed this hateful movie stand on the same side of this line as the killers of
innocents in Libya and Egypt.
Both groups are possessed by the unholy spirit of "blind certainty," unacquainted with both the complexity and
contradictions of life on Earth and the "cloud of unknowing" that hovers over every major religion.
The great religious thinkers in Judaism, Christianity and Islam always knew enough to know that they did not
know everything.
They knew the difference between the eternal decrees of the God of Abraham and the all-too-flawed thoughts of
ordinary people. They knew, in short, that they are not God, who alone is charged with creating and destroying
the world and discriminating along the way between the real and the illusory.
The killers in Libya have forgotten that. So have the not-so-innocent producers and distributors of Innocence
of Islam."
The opinions expressed in this commentary are solely those of Stephen Prothero.
U.S. Department of State
Case No. F-2015-04841
Doc No. C05739590
Date: 05/13/2015
STATE DEPT. - PRODUCED TO HOUSE SELECT BENGHAZI COMM.
SUBJECT TO AGREEMENT ON SENSITIVE INFORMATION & REDACTIONS. NO FOIA WAIVER. STATE-SCB0045294
U.S. Department of State
Case No. F-2015-04841
Doc No. C05739590
Date: 05/13/2015
STATE DEPT. - PRODUCED TO HOUSE SELECT BENGHAZI COMM.
SUBJECT TO AGREEMENT ON SENSITIVE INFORMATION & REDACTIONS. NO FOIA WAIVER.
CNN's Belief Blog: The faith angles behind the biggest stories
Follow the CNN Belief flog on Twitter
U.S. Department of State
Case No. F-2015-04841
Doc No. C05739590
Date: 05/13/2015
STATE DEPT. - PRODUCED TO HOUSE SELECT BENGHAZI COMM.
SUBJECT TO AGREEMENT ON SENSITIVE INFORMATION & REDACTIONS. NO FOIA WAIVER. STATE-5CB0045295CNN Belief Biog. Prothero
See note belowFw: chris Stevens mission
FYIFW: Quick Summary of POTUS Calls to Presidents of Libya and Egypt
embassy in YemenFW: Wash Post: Hillary Clinton tries to calm at i-U S sentiment as protesters storm
B6
Sunday, March 27, 201i 9:05 PM
H: Lots of new intel; possible Libyan collapse. Sid
hic memo lots of new Intel; possible imminent collapse 032711.docx; hrc memo lots of
new intel; possible imminent collapse 032711.docx
March 27. 2011
For: Hillary
From Sid
Re: Lots of new Intel; Libyan army possibly on verge of collapse
Latest:
During the evening of March 27, 2011, individuals with direct access to the military committee of the National
Libyan Council (NLC) stated in confidence that while the rebel forces continue to have organization and
communications problems, their morale has improved drastically, and they believe the Libyan Army is on the
verge of collapse. Under attack from allied Air and Naval forces, the Libyan Army troops have begun to desert
to the rebel side in increasing numbers. The rebels are making an effort togeet these troops as fellow Libyans,
in an effort to encourage additional defections.
(Source Comment: Speaking in strict confidence, one rebel commander stated that his troops continue to
summarily execute all foreign mercenaries captured in the fighting. At the same time Colonel Khalifa Haftzu•
has reportedly joined the rebel command structure, in an effort to help organization the rebel forces. Haftar was
the commander of the Libyan National Army ([NA), a 1500 man force of anti-Qaddafi troops based in
N'djamena, Chad, until current President Idryss Debi overthrew Chadian President Hussein Habre in the fall of
1990, with the assistance of Libyan and French troops. Haftar and his troops lied to a cotaatry in central Africa
and many, including Haftar, later settled in the United States.)
An extremely sensitive source added that the rebels are receiving direct assistance and training from a small
number of Egyptian Special Forces units, while French and British Special Operations troops are working out of
bases in Egypt, along the Libyan border. These troops are overseeing the transfer of weapons and supplies to
the rebels.
U.S. Department of State
Case No. F-2015-04841
Doc No. C05739594
Date: 05/13/2015
STATE DEPT. - PRODUCED TO HOUSE SELECT BENGHAZI COMM.
SUBJECT TO AGREEMENT ON SENSITIVE INFORMATION & REDACTIONS. NO FOIA WAIVER. STATE-SCB0045018
U.S. Department of State
Case No. F-2015-04841
Doc No. C05739594
Date: 05/13/2015
STATE DEPT. - PRODUCED TO HOUSE SELECT BENGHAZI COMM.
SUBJECT TO AGREEMENT ON SENSITIVE INFORMATION & REDACTIONS. NO FOIA WAIVER.
(Source Comment: The rebels are moving back into 'areas that they lost in mid- March fighting, and using the
allied air cover, they are confident they can move into the Tripoli district in a matter of days. One rebel source
reported that they had reliable intelligence that people were evacuating Qaddafi's hometown, Sirte, anticipating
its capture by the rebels. Sirte is located mid-way between Tripoli and Benghazi; its loss will be a
psychological blow for Qaddall.)
Separately, a sensitive European source stated in strict confidence that French military commanders anticipate a
complete collapse of the Libyan military command and control structure in the next week. These officers, who
have experience with the Libyan Army during its invasion of Chad in the late 1980s stated that the Libyan
troops did not perform well under concentrated fire from troops armed with modern weapons.
This source also noted that information from Tripoli has become increasingly difficult to obtain, but there are
reports of deaths among some Qaddafi's senior security officials, including Mansour Daw, and that the Libyan
leader himself may have experienced a health problem. Unfortunately, the European Intelligence services have
been unable to confirm or discredit these reports. This situation has become increasingly frustrating for French
President Nicolas Sarkozy, who, according to knowledgeable individuals, is pressing to have France emerge
from this crisis as the principal foreign ally of any new government that takes power.
Sarkozy is also concerned about continuing reports that radical/terrorist groups such as the Libyan Fighting
Groups and Al Qa'ida in the Islamic Maghreb (AQIM) are infiltrating the NLC. and its military command.
Accordingly, he asked sociologist who has long established ties to Israel, Syria, and other B6
nations in the Middle East, to use his contacts to determine the level of influence AQIM and other terrorist
groups have inside of the NLC. Sarkozy also asked for reports setting out a clear picture of the role of the
Muslim Brotherhood in the rebel leadership.
These sources note that French diplomats and intelligence officers in Egypt are in contact with the following
rebel figures in Benghazi:
MESMAR1Nouri (Qaddafi's former head of protocol, who along with Mousa Kousa was considered the Libyan
leaders most loyal follower. Noun currently lives in France;
.Gehan (GI-LEI-LAND Abdallah (colonel);
CHARRA.NT Faraj,
BOUKHRIS Fathi,
General Abdelfateh Younis (commander of the rebel forces)
(Source Comment: Senior European security officials caution that AQIM is watching developments in Libya,
and elements of that organization have been in touch with tribes in the southeastern part of the country. These
officials are concerned that in a post-Qaddafi Libya, France and other western European countries must move
quickly to ensure that the new government does not allow AQUA and others to set up small, semi- autonomous
local entities—or "Caliphates"—in the oil and gas producing regions of southeastern Libya.)
These knowledgeable sources add that the insurgents have the following weapons stockpiled in Benghazi:
82 and 120 mm. mortars;
GPZ type machine guns;
12.7mm. machine guns mounted on 4x4 vehicles;
some anti-aircraft batteries type ZSU 23/2 and 23/4 as well as Man-Portable Air-Defense Systems
(MANPADS) type SAM?;
some tanks type T-72;
U.S. Department of State
Case No. F-2015-04841
Doc No. C05739594
Date: 05/13/2015
STATE DEPT. - PRODUCED TO HOUSE SELECT BENGHAZI COMM.
SUBJECT TO AGREEMENT ON SENSITIVE INFORMATION & REDACTIONS. NO FOIA WAIVER. STATE-SCB0045019
U.S. Department of State
Case No. F-2015-04841
Doc No. C05739594
Date: 05/13/2015
STATE DEPT. - PRODUCED TO HOUSE SELECT BENGHAZI COMM.
SUBJECT TO AGREEMENT ON SENSITIVE INFORMATION & REDACTIONS. NO FOIA WAIVER.
Possibly some fixed wing aircraft, and some light transporthnedim helicopters.
A seemingly endless supply of AK47 assault rifles and ammunition (even for systems ZSU 23/4 and 23/2).
French, British and Egyptian Special Forces troops are training the rebels inside of western Egypt, and to a
limited degree in the western suburbs of Benghazi.
(Source Comment: These sources add that the MANPAD SAM7 systems appear to be old and badly preserved.
and have been judged useless by the rebels.)
U.S. Department of State
Case No. F-2015-04841
Doc No. C05739594
Date: 05/13/2015
STATE DEPT. - PRODUCED TO HOUSE SELECT BENGHAZI COMM.
SUBJECT TO AGREEMENT ON SENSITIVE INFORMATION & REDACTIONS. NO FOIA WAIVER. STATE-SCB0045020
Pis print.
From Sullivan, Jacob 3 [mailto:SullivanJJ@state.gov]
Sent: Thursday, September 13, 2012 01:25 PM
To: I-I
Subject: FW: Quick Summary of POTUS Calls to Presidents of Libya and Egypt
FYIFW: Quick Summary of POTUS Calls to Presidents of Libya and Egypt
FYIFW: Condolences from PM Netanyahu
FW: American Killed in Libya Was on Intel Mission to Track Weapons
8.25 am DEPART Private Residence En route State Department
8:35 am ARRIVE State Department
8,35 am PRESIDENTIAL DAILY BRIEFING
8.40 am Secretary's Office
8:45 am DAILY SENIOR STAFF MEETING
9:15 am Secretary's Conference Room
9:20 am GROUP PHOTO wfFEMALlE MILITARY OFFICERS FROM AFRICA
9:25 am Treaty Ksom, 7 Floor
Closed Press (Official Photographer Only), Staff: Monica , WS Carson will greet and escort to the Treaty Room
-Approx. 25-30 people attending; consecutive French and Portuguese interpretation
9:30 tun WEEKLY MEETING w/UN AMBASSADOR SUSAN RICE
10:00 am Secretary's Office
10-05 am AWARD PRESENTATION TO DEPUTY EXECUTIVE SECRETARY
10:10 am PAM QUANRUD
Secretary's Outer Office
Closed Press (Official Photographer only), Staff Rob and lAnta
-15 people expected, YOU present the Superior Honor Award to Pam
10:15 am MEETING w/ASSISTANT SECRETARY MIKE POSNER AND CHERYL
10:45 am Secretary's Outer Office
10:45 am PRE-BRIEF wfWENDY AND JAKE FOR WH MEETING
11:15 am Secretary's Outer Office
11:20 am DEPART State Department *En Route White House
11.25 am ARRIVE White House
II:30 am MEETING
1-00 pm White House Situation Room
1.00 pm DEPART White House En Route Andrews Air Force Base
1:30 pm ARRIVE Andrews Air Force Base
2:15 pm CEREMONY FOR RETURN OF THE REMAINS OF DEPARTMENT
2.45 pm COLLEAGUES FROM BENGHAZI, LIBYA
Andrews Air Force Base
Press TOO, Scenario TBD, Staff: Monica
3:00 pm (t)DEPART Andrews Air Force Base * En Route State Department
3:30 pm(t)ARRIVE State Department
3.30 pm OFFICE TIME
TBD Secretary's Office
TBD DEPART State Department *En Route Andrews Air Force Base
TBD ARRIVE Andrews Air Force Base
TBD DEPART Andrews Air Force Base via Air Force Aircraft Tail i160204
En route White Plains, NY
TBD ARRIVE Westchester County Airport
U.S. Department of State
Case No. F-2015-04841
Doc No. C05739600
Date: 05/13/2015
STATE DEPT. - PRODUCED TO HOUSE SELECT BENGHAZI COMM.
SUBJECT TO AGREEMENT ON SENSITIVE INFORMATION & REDACTIONS. NO FOIA WAIVER. STATE-SCB0045306
U.S. Department of State
Case No. F-2015-04841
Doc No. C05739600
Date: 05/13/2015
STATE DEPT. - PRODUCED TO HOUSE SELECT BENGHAZI COMM.
SUBJECT TO AGREEMENT ON SENSITIVE INFORMATION & REDACTIONS. NO FOIA WAIVER.
TBD DEPART Westchester County Aimed* Ea route Private Residence
113D ARRIVE Private Residence
U.S. Department of State
Case No. F-2015-04841
Doc No. C05739600
Date: 05/13/2015
STATE DEPT. - PRODUCED TO HOUSE SELECT BENGHAZI COMM.
SUBJECT TO AGREEMENT ON SENSITIVE INFORMATION & REDACTIONS. NO FOIA WAIVER. STATE-5CB0045307
FW: SITUATION REPORT No. 1 09/14/I (SBU)
I'm giving you credit for inspiring the "peaceful" protests.Fw: SITUATION REPORT No 1 09/14/1 (SBU)
U.S. Department of State
Case No. F-2015-04841
Doc No. C05739607
Date: 05/13/2015
STATE DEPT. - PRODUCED TO HOUSE SELECT BENGHAZI COMM.
SUBJECT TO AGREEMENT ON SENSITIVE INFORMATION & REDACTIONS. NO FOIA WAIVER. STATE-5CB0045314You do great work - thanks for making our heros have the homecoming they deserved.
Fyi
--Fw: Libyan
deserved.
Great teamwork -- great leadership!
Pis print.Fw: Benhazi/protest statements
Both Gates and I said the same on the Sunday shows.Re: Libyan
FyiFav: Pretty powerful piece on standing up for diplomats
My lYi was referring to the Libyan. The arming the rebels breathlessness is silly.
—Re: Libyan
http://www.salon.comi2o12/1o/or/gops_october_surprisei
Monday, Oct 1, 2012 09:30 AM EDT
GOP's October surprise?
They're calling it the "Jimmy Carter Strategy": Exploit
Libya attack to paint Obama as weak on terrorism
By cY'aia?,_
(Credit: AP/AI Behrman/Salort)
According to a highly reliable source, as Mitt Romney and President Barack Obama prepare for the
first presidential debate Wednesday night, top Republican operatives are primed to unleash a new
two-pronged offensive that will attack Obama as weak on national security, and will be based, in part,
on inW !ia:en,y-.3iTeForrnation regarding the attacks in Libya that killed U.S. ambassador Chris
Stevens on September
The source, who has first-hand knowledge of private, high-level conversations in the Romney tamp
that took place in Washington, DC last week, said that at various times the GOP strategists referred to
their new operation as the Jimmy Carter Strategy or the October Surprise.
He added that they planned to release what they hoped would be "a bombshell" that would make
Libya and Obama's foreign policy a major issue in the campaign. "My understanding is that they have
come up with evidence that the Obama administration had positive intelligence that there was going
to be a terrorist attack on the intelligence."
The source described the Republicans as chortling with glee that the Obama administration
"definitely had intel" about the attack before it happened. "Intelligence can be graded in different
ways," he added, "and sometimes A and B don't get connected. But [the Romney campaign] will try to
paint it to look like Obama had advance knowledge of the attack and is weak on terrorism."
He said they were jubilant about their new strategy and said they intended to portray Obama as a
helpless, Jimmy Carter-like president and to equate the tragedy in Libya with President Carter's failed
attempt to rescue American hostages in Iran in t980. "They are so excited about it," he said. "Over
and over again they talked about bow it would be just like Jimmy Carter's failed raid. They feel it is
going to give them .a last-minute landslide in the election."
The source, however, said lie was dubious about the tactic. "To me, it is indicative that they have lost
touch with a huge portion of the electorate," he said.
The source declined to reveal the names of the GOP operatives who were present. But he said, "These
were the top guys in the party. It was a .private, unguarded planning conversation." He further
U.S. Department of State
Case No. F-2015-04841
Doc No. C05739615
Date: 05/13/2015
STATE DEPT. - PRODUCED TO HOUSE SELECT BENGHAZI COMM.
SUBJECT TO AGREEMENT ON SENSITIVE INFORMATION & REDACTIONS. NO FOIA WAIVER. STATE-SCB0045540
U.S. Department of State
Case No. F-2015-04841
Doc No. C05739615
Date: 05/13/2015
STATE DEPT. - PRODUCED TO HOUSE SELECT BENGHAZI COMM.
SUBJECT TO AGREEMENT ON SENSITIVE INFORMATION & REDACTIONS. NO FOIA WAIVER.
described participants in the meeting as consisting of well-known names tied to the big Republican
super PACs and people who had access to high-level national security intelligence.
"As usual, Karl Rove wasn't present," he said, "but some were close to Rove."
The source said that "there was quite a bit more" to the operation than simply revealing the
intelligence regarding Libya. He declined to discuss what he described as the second phase of the
operation.
However, already there is evidence that the first phase of the operation is underway. On Wednesday,
September 26, on Fox News, Karl Rove called Obama's reaction to what happened in Libya
"inexplicable" and added that Obama is "more intent on campaigning than meeting his
responsibilities as commander-in-chief." He went on to warn that Obama's policies in Libya
endangered the U.S. with regard to Iran. "I am astonished," Rove said. "We are projecting weakness
when we need to be projecting strength."
Similarly, former New York Mayor Rudolph Giuliani, a one-time GOP presidential hopeful who
became known as the mayor of 9/11, followed Rove in criticizing Obama on the attack in Libya. "This
is a deliberate attempt to cover up the truth, from an administration that claimed it wanted to be the
most transparent in history," he told Fox News. "And it's the worst kind of cover-up: the kind of
cover-up that involves our national security. This is a cover-up that involves the slaughter of four
Americans."
Giuliani added that the Obama White House wanted to dismiss the role played by al-Qaeda because it
seemed to diminish the triumph of having killed Osama bin Laden. "I think it's because they have this
narrative that they defeated al-Qaeda," he said. "They never say the words 'Islamic fundamentalist
terrorism'. They want to wish it away. The president was moving on to Asia — he was going to declare
this a great victory for himself and unfortunately, this terrible act of terror intervened in their very
convenient narrative."
Likewise, Bob Corker, Republican senator from Tennessee, described the administration's response to
"the terrorist attack" in Libya as And on Sunday, September 30, both Mitt Romney and
vice-presidential nominee Paul Ryan joined in the attacks. "I think they want to do their very best to
keep the people of America from understanding exactly what happened. We expect candor, we expect
transparency, particularly as it relates to terrorism," Romney said.
Added Ryan, "Why is he (Obama) not on the same page with his own administration officials who are
saying that this is a terrorist attack? We'll leave it up to you to decide whether it's a cover-up or not."
In an apparently related development, a Politico artkle posted last night said Romney advisers were
divided over hovv aggressively to attack President Obama over his handling of the events in Libya. One
unnamed Romney adviser described them as "a huge gift" to the campaign. "Across the board —
domestic, economic and foreign-policy issues — President Obama has been outmatched by events," he
said. "He's an observer of events, not a shaper of events. Everywhere you look, he's been outmatched."
But other Romney advisers, led by media strategist Stuart Stevens, wanted to stick to the economy as
the central issue. "They'll tell you that you've got to focus people on the fact that their economic
prospects are not very good and all that," the aide told Politico. "Well, Romney's been trying to do that
now since he clinched the nomination at the end of April — and he's failed. The president is better at
deflecting attention from the bad news than Romney is at driving home the impact of the bad news on
individual voters."
Craig Unger is the New York Times bestselling author of "House of Bush, House of Saud," and a
frequent analyst on CNN, ABC Radio, Air America, The Charlie Rose Show, NBC's Today Show and
other broadcast outlets. He has written for The New Yorker, Esquire and many other publications and
is currently a contributing editor at Vanity Fair.
U.S. Department of State
Case No. F-2015-04841
Doc No. C05739615
Date: 05/13/2015
STATE DEPT. - PRODUCED TO HOUSE SELECT BENGHAZI COMM.
SUBJECT TO AGREEMENT ON SENSITIVE INFORMATION & REDACTIONS. NO FOIA WAIVER. STATE-SCB0045541H: Romney's last gambit. Got done and published. Sid
Pis let me know what Phil finds out asap. Thx.Re. Libyan
B5H: Romney's last gambit. Got done and published. Sid
deserved.
By leadership - let me be dear- meant the 3 of you -- especially our Secretary- lovely speech!You do great work - thanks for making our hems have the homecoming they deserved.
Will do.
Sullivan, Jacob J <SullivanJJ@state.gov>
Monday, October 1, 2012 3:37 PM
RE: H: Romney's last gambit Got done and published. Sid
One for the wall...
For what it's worth, I am VERY dubious about arming the Libyan rebels. Our best bet is to keep pressure on BOTH sides
to force a diplo solution. Aryls
U.S. Department of State
Case No. F-2015-04841
Doc No. C05739622
Date: 05/13/2015
STATE DEPT. - PRODUCED TO HOUSE SELECT BENGHAZI COMM.
SUBJECT TO AGREEMENT ON SENSITIVE INFORMATION & REDACTIONS. NO FOIA WAIVER. STATE-5CB0045026gorgeous plc on the front page of the NYTI
Romney has an op-ed in today's Wall Street Journal playing off Stevens' murder to say Obama has put
"security at risk," etc.
According to Politico yesterday, there was an internal argument within the Romney campaign over
Libya. Obviously, the neocons and the Rove oriented faction (Ed Gillespie, Rove's surrogate is now a
Romney campaign adviser) beat Stuart Stevens.
Romney is careening from tactic to tactic, hoping something will catch.
I wouldn't be surprised to see Romney raise the specter of the Muslim Brotherhood in charge of Egypt
& Libya as a bogeyman. I have heard him do this within the last week.
----Re: H: Romney's last gambit. Got done and published_ Sid
Sidney Blumenthat
Friday, September 14, 2012 6:32 PM
H: Morsi's private conversations today. Sid
hrc memo morsi's private conversations 091412.docx
September14, 2012
For: Hillary
Your remarks this afternoon at Andrews were wonderful. Thanks for doing so much for our institution on such a sad
day.
Just sent you a hi side note on visits taJordan and Iraq, and look forward to catching up. Enjoy the weekend in the
meantime.
Bill
U.S. Department of State
Case No. F-2015-04841
Doc No. C05739626
Date: 05/13/2015
STATE DEPT. - PRODUCED TO HOUSE SELECT BENGHAZI COMM.
SUBJECT TO AGREEMENT ON SENSITIVE INFORMATION & REDACTIONS. NO FOIA WAIVER. STATE-SCB0045325Thanks
More info.
From Sidney Blumenthal [mailto
Sent: Monday, October 01, 2012 03:47 PM
To: H
Subject: Re: Romney s last gambit. Got done arid published. Sid
Romney has an op-ed in today's Wall Street Journal playing off Stevens' murder to say Obama has put
"security at risk," etc.
According to Politico yesterday, there was an internal argument within the Romney campaign over
Libya. Obviously, the neocons and the Rove oriented faction (Ed Gillespie, Rove's surrogate is now a
Romney campaign adviser) beat Stuart Stevens.
Romney is careening from tactic to tactic, hoping something will catch.
I wouldn't be surprised to see Romney raise the specter of the Muslim Brotherhood in charge of Egypt
& Libya as a bogeyman. I have heard him do this within the last week.
B6Fw: H: Romney's last gambit Got done and published. Sid
FW: thanks
This is it. Pretty uncompelling piece if you ask me, but of course I'm biased.
Mitt Romney: A New Course for the Middle East
Restore the three sinews of American influence: our economic strength, our military strength and the strength of our values.
Disturbing developments are sweeping across the greater Middle East. In Syria, tens of thousands of innocent people have been slaughtered. In Egypt, the Muslim
Brotherhood has come to power, and the country's peace treaty with Israel hangs In the balance. In Libya, our ambassador was murdered In a terrorist attack. U.S.
embassies throughout the region have been stormed in violent protests. And in Iran, the ayatollahs continue to move full tilt toward nuclear-weapons capability, all
the while promising to annihilate Israel.
These developments are not, as President Obama says, mere "bumps in the road." They are major issues that put our security at risk.
Yet amid this upheaval, our country seems to be at the mercy of events rather than shaping them. We're not moving them in a direction that protects our people or
our allies.
And that's dangerous. If the Middle East descends into chaos, if Iran moves toward nuclear breakout, or if Israel's security is compromised, America could be
pulled into the maelstrom.
We still have time to address these threats, but it will require a new strategy toward the Middle East
The first step is to understand how we got here. Since World War II, America has been the leader of the Free World. We're unique in having earned that role not
through conquest but through promoting human rights, free markets and the rule of law. We ally ourselves with like-minded countries, expand prosperity through
trade and keep the peace by maintaining a military second to none.
But in recent years, President Obama has allowed our leadership to atrophy. Our economy is stuck in a "recovery" that barely deserves the name. Our national
debt has risen to record levels. Our military, tested by a decade of war, is facing devastating cuts thanks to the budgetary games played by the White House.
Finally, our values have been misapplied—and misunderstood—by a president who thinks that weakness will win favor with our adversaries.
By failing to maintain the elements of our influence and by stepping away from our allies, President Obama has heightened the prospect of conflict and instability.
He does not understand that an American policy that lacks resolve can provoke aggression and encourage disorder.
The Middle East is a case in point. The Arab Spring presented an opportunity to help move millions of people from oppression to freedom. But it also presented
grave risks. We needed a strategy for success, but the president offered none. And now he seeks to downplay the significance of the calamities of the past few
weeks
The same incomprehension afflicts the president's policy toward Israel. The president began his term with the explicit policy of creating "daylight" between our two
countries. He recently downgraded Israel from being our "closest ally" in the Middle East to being only "one of our closest allies." It's a diplomatic message that will
be received clearly by Israel and its adversaries alike. He dismissed Israel's concerns about Iran as mere "noise" that he prefers to "block out." And at a time when
Israel needs America to stand with it, he declined to meet with Prime Minister Benjamin Netanyahu.
In this period of uncertainty, we need to apply a coherent strategy of supporting our partners in the Middle East—that is, both governments and individuals who
share our values.
This means restoring our credibility with Iran. When we say an Iranian nuclear-weapons capability—and the regional instability that comes with it--is unacceptable,
the ayatollahs must be made to believe us.
It means placing no daylight between the United States and Israel, And it means using the full spectrum of our soft power to encourage liberty and opportunity for
those who have for too long known only corruption and oppression. The dignity of work and the ability to steer the course of their lives are the best alternatives to
extremism.
But this Middle East policy will be undermined unless we restore the three sinews of our influence: our economic strength, our military strength and the strength Of
our values. That will require a very different set of policies from those President Obama is pursuing.
U.S. Department of State
Case No. F-2015-04841
Doc No. C05739629
Date: 05/13/2015
STATE DEPT. - PRODUCED TO HOUSE SELECT BENGHAZI COMM.
SUBJECT TO AGREEMENT ON SENSITIVE INFORMATION & REDACTIONS. NO FOIA WAIVER. STATE-SCB0045554
U.S. Department of State
Case No. F-2015-04841
Doc No. C05739629
Date: 05/13/2015
STATE DEPT. - PRODUCED TO HOUSE SELECT BENGHAZI COMM.
SUBJECT TO AGREEMENT ON SENSITIVE INFORMATION & REDACTIONS. NO FOIA WAIVER.
The 20th century became an American Century because we were steadfast in defense of freedom. We made the painful sacrifices necessary to defeat
totalitarianism In all of Its guises.. To defend ourselves and our allies, we paid the price in treasure and in soldiers who never came home.
Our challenges are different now, but if the 21st century is to be another American Century, we need leaders who understand that keeping the peace requires
American strength in all of its dimensions.
Mr. Romney is the Republican Pady candidate for president
A version of this article appeared October 1,2012, on page .A.15 in the U.S. edition of The WallStreet Journal, with the headline: A
New course for the Middle East.RE: H: Romney's last gambit. Got done and published. Sid
FYI — should take care of the job.
Prom: Rhodes, Benjamin 1
Sent Wednesday, March 30, 2011 8:15 PM
To: Sullivan, Jacob]; Reines, Philippe
Subject FIN: Statement by the Press Secretary
B6Statement by the Press Secretary
FW:
STATE DEPT. - PRODUCED TO HOUSE SELECT BENGHAZI COMM.
SUBJECT TO AGREEMENT ON SENSITIVE INFORMATION & REDACTIONS. NO FOIA WAIVER.
RELEASE IN
PART B6
Anne-Marie Slaughter _
Friday. April 1,2011 11:53 AM
RE: gorgeous pie on the front page of the NWT!
Bic sending more arms into a society generally — particularly when they are as disorganized and fragmented as they are
— will result in more violence — against each other. Boys like to play with guns (trust me as the mother of sons). I am all
for saying we have no objection to French doing it to increase pressure on Gaddafi, but in a tribal society where conflicts
have been repressed for so long, adding even more weapons does not make sense. Obviously they can get weapons
from other places, but nb that they are not awash in weapons now. I wrote something for New York Review of Books
this week that builds on my farewell remarks and I think you will like it; will send it to you for weekend reading,Re: gorgeous pie on the front page of the NYTI
Pis print
H <hrod17@clintonemail.com>
Wednesday, October 3, 2012 12:05 PM
'Russorv@state.gov'
Fw: Fwd: Benghazi was obama's 3 am call
Remarks
Tripoli trip
From PA Clips [mailtoTaclips@state.govl
Sent; Saturday, October 06, 2012 12:26 AM
To PA-Monitoring-Group-DI; NEA-Press-DL; DS PA Media; Shore, Rhonda H; Atchison, Mark C; Moe, Grace T;
Coordination; Mills, Cheryl D; Tow, Nora F; 10 PG
Subject: Daily Beast: A diplomat under fire.
Person of Interest: United States UN Ambassador Susan Rice
Oct 8, 2012 iloo AM EDT
Leslie H. Gelb
A diplomat under fire.
With the presidential campaign drowning in boredom, commentators were desperately
searching for news, preferably a new victim, to brighten their days. And along came Susan
Rice. A couple Sundays ago, the U.N. ambassador made the mistake of appearing on the talk
shows to insist that the attack on the U.S. consulate in Benghazi, Libya, was "spontaneous"
rather than a terrorist plot. You would have thought she had said that the pope was a
Mormon.
Out came the Washington daggers. She lied. She was a shill. Benghazi was no spontaneous
event. It was the work of an al Qaeda offshoot. The White House had fumbled key
intelligence information and was now trying to cover up its unforgivable mistake. Rice was
just covering up for the president. Her reward would be Obama's nominating her to succeed
Hillary Clinton as secretary of State.
The commentariat immediately rolled out plans to nail Rice and the Obama team to the
wall. Peter King, the Grand Inquisitor of the Republicans in the House of Representatives,
pledged to "investigate" and hold hearings. And that, said the commentariat, would be the
end of Rice's hopes to become the new Hillary.
U.S. Department of State
Case No. F-2015-04841
Doc No. C05739637
Date: 05/13/2015
STATE DEPT. - PRODUCED TO HOUSE SELECT BENGHAZI COMM.
SUBJECT TO AGREEMENT ON SENSITIVE INFORMATION & REDACTIONS. NO FOIA WAIVER. STATE-SCB0045563
U.S. Department of State
Case No. F-2015-04841
Doc No. C05739637
Date: 05/13/2015
STATE DEPT. - PRODUCED TO HOUSE SELECT BENGHAZI COMM.
SUBJECT TO AGREEMENT ON SENSITIVE INFORMATION & REDACTIONS. NO FOIA WAIVER.
Now, let's take an unfashionable step back and examine the facts, such as they are. Rice did
not make up the "spontaneous" story or have it handed to her by White House conspirators.
She got it from the same daily intelligence brief given to the president. Those daily intel
reports told the very story Rice repeated publicly, i.e., that the Benghazi mob formed more
or less on its own, mainly in reaction to the American video cartooning sacred Muslim
figures, and that extremists then took advantage to kill, including Ambassador Christopher
Stevens. Rice didn't make up that story. The problem was not with Rice; it was with the
inevitably complicated U.S. intelligence process.
Her mistake was taking the initial intelligence at face value. The White House biiefers made
the same error, and so did I. I too repeated in an op-ed on The Daily Beast what the
intelligence briefers told me, but I knew very well that more and contradictory information
would appear soon enough. It always does.
Sure, the CIA had additional intel from the start, but it hadn't begun to sift through all the
communications intercepts, satellite photos, and human reports. It always takes time to sort
out facts from fictions and find consistent threads. Seemingly, this gossamer information is
now taking shape.
Meantime, Rice is still very much in the running for the job at Fogg' Bottom, along with
Senator John Kerry, NSC Adviser Tom Donilon, and former diplomat Nicholas Burns, now
of Harvard. She was Obama's earliest loyalist in the foreign-policy crowd, and he still likes
her, despite lackluster reports on her performance at the U.N. The smears, though flatly
inaccurate, won't help her either. What will help her and, more importantly a flagging
Obama after the first debate, is hunting down the Benghazi culprits.
U.S. Department of State
Case No. F-2015-04841
Doc No. C05739637
Date: 05/13/2015
STATE DEPT. - PRODUCED TO HOUSE SELECT BENGHAZI COMM.
SUBJECT TO AGREEMENT ON SENSITIVE INFORMATION & REDACTIONS. NO FOIA WAIVER. STATE-SCB0045564Fw: Daily Beast: A diplomat under fire.
When is Chris Stevens going?
---,Re: Brits and Libya
Was supposed to go today. Will check.
---Re: Brits and Libya
Pis print.
H <hrod17@clintonemail.com>
Friday, September 14, 2012 8:56 PM
`Russorv@state.gov'
Fw: CNN Belief Blog. Prothero
FyiMission to Benghazi
October 6, 2012
For: Hillary
Pis print.
..... ....Fw: H: Great to see you. Drop in again. Here's Libya, Sid
A pouch with all of your call sheets and the schedule in en route to you. Here it is below as well
Also in the pouch are a few read items, and an action memo authorizing the War Powers resolution for Tunisia that the
office would like you to approve today. Ops can send a courier over to pick up the action memo later today.
12:00 UK FM Hague
12:15 Egyptian FM Amr
12:30 Israeli PM Netanyahu
1:15 French FM Fabius
1:30 Saudi FM Saud al-Faisal
2:00 Somali Former Transitional President Sherif
2:15 Libyan PM-elect Abu-Shakour
2:30 Turkish FM Davutoglu
3:00 Somali President Mohamoud (T)
- Moroccan King is still pending.
- NEW CALL King Juan Carlos of Spain called today and offered anytime today or tomorrow. His office relayed that it is a
personal call inquiring after the status of the Embassies in the Middle East We are working on a call sheet
----Re: PDB
Prom: Nora ToIv (mato
Sent Sun ay April 03, 2011 09:25 PM
To: Mills, ateryl D
Subject: EXCLUSIVE: Bill Clinton Says He Wouldn't Rule Out Arming Uhya's Rebels
EXCLUSIVE: Bill Clinton Says He Wouldn't Rule Out Arming
Libya's Rebels
Pm Just Speaking For Myself, Former President Says
By SUZAN CLARICE
April 3, 2011 —
Former U.S. President Bill Clinton said he thinks the United States should consider arming the rebels in Libya's
civil war.
"It would depend ... and I might need to know a little more, but I would be inclined to do it," Clinton told
correspondent Bianna Golodryga today in San Diego during an exclusive interview with ABC News.
Clinton was in San Diego for his annual Clinton Global Initiative University Meeting.
The uprising in Libya stared in February during a wave of social and political unrest in the Middle East.
Fighting has escalated all across the country, but the rebels have found themselves outgunned by Libyan leader
Col. Moamrnar Gadhafi's forces, which are better equipped.
Last month the United Nations authorized a no-fly zone over the North African nation to prevent further attacks
on civilians from Gadhafiss troops in the air.
Clinton on Arming Rebels: 'I Sure Wouldn't Shut the Door to It'
The United States backed the no-fly zone and has also authorized additional support for the anti-Gad hafi
fighters, but key U.S. lawmakers and top officials in the government of President Barack Obanaa have said they
are not comfortable with arming the rebels -- apparently because not enough is known about then
But Clinton said he wouldn't completely rule out the idea of supplying arms to Libya's rebels.
U.S. Department of State
Case No. F-2015-04841
Doc No. C05739646
Date: 05/13/2015
STATE DEPT. - PRODUCED TO HOUSE SELECT BENGHAZI COMM.
SUBJECT TO AGREEMENT ON SENSITIVE INFORMATION & REDACTIONS. NO FOIA WAIVER. STATE-SCB0045034
U.S. Department of State
Case No. F-2015-04841
Doc No. C05739646
Date: 05/13/2015
STATE DEPT. - PRODUCED TO HOUSE SELECT BENGHAZI COMM.
SUBJECT TO AGREEMENT ON SENSITIVE INFORMATION & REDACTIONS. NO FOIA WAIVER.
"Let me just say this. I sure wouldn't shut the door to it. I think ... we may need to know a little more," he said.
Clinton, husband of Secretary of State Hillary Clinton, stressed that he was speaking without "any official
sanction" whatsoever.
"I'm just speaking from myself. But I certainly wouldn't take that off the table, too," he said.
Watch More of Interview With Bill Clinton Monday on 'Good Morning America'
Gadhafi is the focus of an investigation by the International Criminal Court in the Hague.
Watch "Good Morning America"i0114)/TOW for the full interview and hear Clinton speak about his
summit, the looming government shutdown and the controversy over President Obama's birth certificate.
Copyright © 2011 ABC News Internet Ventures
U.S. Department of State
Case No. F-2015-04841
Doc No. C05739646
Date: 05/13/2015
STATE DEPT. - PRODUCED TO HOUSE SELECT BENGHAZI COMM.
SUBJECT TO AGREEMENT ON SENSITIVE INFORMATION & REDACTIONS. NO FOIA WAIVER. STATE-SCB0045035EXCLUSIVE: Bill Clinton Says He Wouldn't Rule Out Arming Uhya's Rebels
Useful insight. Pls circulate.Fw: H: Great to see you. Drop in again. Here's Libya. Sid
J .........FW: Item
FYI —Jeff and I discussed and I fully agree that a NATO liaison is a good idea.
Prom: Feibran, Jeffrey D
Sent: Friday, April 08, 2011 3:06 PM
To: Daaider, Ivo H; 0112, Gene A; Polaschlk, Joan A; Gordon, Philip H; Roebuck, William V; Keene, Matt E; Dibble,
Elizabeth L
CC: Sullivan. Jacob); Schikher, Ronald L; Sanderson, Janet A; Djerassi, Alexander M;
Heffern, John A; Degnan, Kelly C
Subject: RE: Itenghazi and NATO
Ivo, thanks for the helpful response. From NEA's perspective, an operational liaison with the TNC does
not constitute formal, legal recognition of the INC as the government of Libya. We don't consider
Chris Stevens' presence in Benghazi to trigger formal recognition, and we've never broken relations
with Libya despite suspending our embassy operations.
Without question, having a NATO liaison in Benghazi would have the appearance of NATO leaning in
the direction of the INC. But NATO's operations already show a considerable lean away from
Qadhati. Speaking for NEA, I don't mind the appearance. But the real purpose would be to avoid
friendly fire incidents, of course.
Cheers,
Jeff
Jeffrey Fettman
Assistant Secretary
Bureau of Near Eastern Affairs
Department of State
202-6474209
,,,, .....RE: Itenghazi and NATO
Postelection, we'd like to have you over for dinner. Bill can come, too, if he's in town. Whatever
works,
--H: Great to see you. Drop in again. Here's Libya. Ski
Will do.
Sullivan, Jacob J <SullivanJJ@state.gov>
Sunday, October 7, 2012 12:29 PM
Re: H: Great to see you. Drop in again. Here's Libya. Sid
I thanked Prince Saud.Fw; Saudi Grand Mufti calls attacks on foreign embassies tin Islamic (Reuters)
_Fw: The Cable: Romney to compare Benghazi to original 9/11 attacks
B6
Friday, April 8, 2011 5:28 PM
H: UK game playing; new rebel strategists; Egypt moves in. Sid -
hre memo UK games new rebel strategists; Egypt moves 040811.docx; hrc memo UK
games new rebel strategists; Egypt moves 040811.clocx
For: Hillary
Hillary,
I just want to offer my personal condolences over the loss of four fine foreign service officers. The Ambassador
was a perfect role model of the kind of person we need representing us around the world, and the others had so
much to give -- and already had given so much.
I know it has been a hard week for you, and I wanted simply to express my sympathy and tell you I was
thinking of you.
Bob Gates
U.S. Department of State
Case No. F-2015-04841
Doc No. C05739655
Date: 05/13/2015
STATE DEPT. - PRODUCED TO HOUSE SELECT BENGHAZI COMM.
SUBJECT TO AGREEMENT ON SENSITIVE INFORMATION & REDACTIONS. NO FOIA WAIVER. STATE-SCB0045336Condolences
Can you call me?Re: The Cable: Romney to compare Benghazi to original 9/11 attacks
FW: US intelligence warned embassy in Egypt of concern about anti Muslim film
Fw: The Hill: Issa targets Hillary: Big mistake
H <hrod17@clintonernail.com>
Friday, April 8, 2011 5:35 PM
'sullivanjj@state.govi
Fvv: H: UK game playing; new rebel strategists; Egypt moves in Sid
hit memo UK games; new rebel strategists; Egypt moves 040811.docx
Fyi, B5H: UK game playing; new rebel strategists; Egypt moves in Sid
Madam Secretary, here is our latest draft of Friday's speech. It's long — about 30 minutes in this version — but if
you're comfortable with the general direction we'll streamline it.
Thanks
Dan
U.S. Department of State
Case No. F-2015-04841
Doc No. C05739661
Date: 05/13/2015
STATE DEPT. - PRODUCED TO HOUSE SELECT BENGHAZI COMM.
SUBJECT TO AGREEMENT ON SENSITIVE INFORMATION & REDACTIONS. NO FOIA WAIVER. STATE-5CB0045603
U.S. Department of State
Case No. F-2015-04841
Doc No. C05739661
Date: 05/13/2015
STATE DEPT. - PRODUCED TO HOUSE SELECT BENGHAZI COMM.
SUBJECT TO AGREEMENT ON SENSITIVE INFORMATION & REDACTIONS. NO FOIA WAIVER.
SECRETARY OF STATE HILLARY RODHAM CLINTON
REMARKS ON THE MAGHREB IN TRANSITION
AT THE CENTER FOR STRATEGIC AND INTERNATIONAL STUDIES
WASHINGTON, DC
OCMBER 12, 2012
B5
U.S. Department of State
Case No. F-2015-04841
Doc No. C05739661
Date: 05/13/2015
STATE DEPT. - PRODUCED TO HOUSE SELECT BENGHAZI COMM.
SUBJECT TO AGREEMENT ON SENSITIVE INFORMATION & REDACTIONS. NO FOIA WAIVER. STATE-5CB0045604
B5
U.S. Department of State
Case No. F-2015-04841
Doc No. C05739661
Date: 05/13/2015
STATE DEPT. - PRODUCED TO HOUSE SELECT BENGHAZI COMM.
SUBJECT TO AGREEMENT ON SENSITIVE INFORMATION & REDACTIONS. NO FOIA WAIVER.
U.S. Department of State
Case No. F-2015-04841
Doc No. C05739661
Date: 05/13/2015
STATE DEPT. - PRODUCED TO HOUSE SELECT BENGHAZI COMM.
SUBJECT TO AGREEMENT ON SENSITIVE INFORMATION & REDACTIONS. NO FOIA WAIVER. STATE-5CB0045605
U.S. Department of State
Case No. F-2015-04841
Doc No. C05739661
Date: 05/13/2015
STATE DEPT. - PRODUCED TO HOUSE SELECT BENGHAZI COMM.
SUBJECT TO AGREEMENT ON SENSITIVE INFORMATION & REDACTIONS. NO FOIA WAIVER.
B5
U.S. Department of State
Case No. F-2015-04841
Doc No. C05739661
Date: 05/13/2015
STATE DEPT. - PRODUCED TO HOUSE SELECT BENGHAZI COMM.
SUBJECT TO AGREEMENT ON SENSITIVE INFORMATION & REDACTIONS. NO FOIA WAIVER. STATE-5CB0045606
U.S. Department of State
Case No. F-2015-04841
Doc No. C05739661
Date: 05/13/2015
STATE DEPT. - PRODUCED TO HOUSE SELECT BENGHAZI COMM.
SUBJECT TO AGREEMENT ON SENSITIVE INFORMATION & REDACTIONS. NO FOIA WAIVER.
B5
U.S. Department of State
Case No. F-2015-04841
Doc No. C05739661
Date: 05/13/2015
STATE DEPT. - PRODUCED TO HOUSE SELECT BENGHAZI COMM.
SUBJECT TO AGREEMENT ON SENSITIVE INFORMATION & REDACTIONS. NO FOIA WAIVER. STATE-5CB0045607
U.S. Department of State
Case No. F-2015-04841
Doc No. C05739661
Date: 05/13/2015
STATE DEPT. - PRODUCED TO HOUSE SELECT BENGHAZI COMM.
SUBJECT TO AGREEMENT ON SENSITIVE INFORMATION & REDACTIONS. NO FOIA WAIVER.
B5
U.S. Department of State
Case No. F-2015-04841
Doc No. C05739661
Date: 05/13/2015
STATE DEPT. - PRODUCED TO HOUSE SELECT BENGHAZI COMM.
SUBJECT TO AGREEMENT ON SENSITIVE INFORMATION & REDACTIONS. NO FOIA WAIVER. STATE-5CB0045608
U.S. Department of State
Case No. F-2015-04841
Doc No. C05739661
Date: 05/13/2015
STATE DEPT. - PRODUCED TO HOUSE SELECT BENGHAZI COMM.
SUBJECT TO AGREEMENT ON SENSITIVE INFORMATION & REDACTIONS. NO FOIA WAIVER.
B5
U.S. Department of State
Case No. F-2015-04841
Doc No. C05739661
Date: 05/13/2015
STATE DEPT. - PRODUCED TO HOUSE SELECT BENGHAZI COMM.
SUBJECT TO AGREEMENT ON SENSITIVE INFORMATION & REDACTIONS. NO FOIA WAIVER. STATE-5CB0045609
U.S. Department of State
Case No. F-2015-04841
Doc No. C05739661
Date: 05/13/2015
STATE DEPT. - PRODUCED TO HOUSE SELECT BENGHAZI COMM.
SUBJECT TO AGREEMENT ON SENSITIVE INFORMATION & REDACTIONS. NO FOIA WAIVER.
B5
U.S. Department of State
Case No. F-2015-04841
Doc No. C05739661
Date: 05/13/2015
STATE DEPT. - PRODUCED TO HOUSE SELECT BENGHAZI COMM.
SUBJECT TO AGREEMENT ON SENSITIVE INFORMATION & REDACTIONS. NO FOIA WAIVER. STATE-5CB0045610
U.S. Department of State
Case No. F-2015-04841
Doc No. C05739661
Date: 05/13/2015
STATE DEPT. - PRODUCED TO HOUSE SELECT BENGHAZI COMM.
SUBJECT TO AGREEMENT ON SENSITIVE INFORMATION & REDACTIONS. NO FOIA WAIVER.
B5
U.S. Department of State
Case No. F-2015-04841
Doc No. C05739661
Date: 05/13/2015
STATE DEPT. - PRODUCED TO HOUSE SELECT BENGHAZI COMM.
SUBJECT TO AGREEMENT ON SENSITIVE INFORMATION & REDACTIONS. NO FOIA WAIVER. STATE-5CB0045611
U.S. Department of State
Case No. F-2015-04841
Doc No. C05739661
Date: 05/13/2015
STATE DEPT. - PRODUCED TO HOUSE SELECT BENGHAZI COMM.
SUBJECT TO AGREEMENT ON SENSITIVE INFORMATION & REDACTIONS. NO FOIA WAIVER.
B5
U.S. Department of State
Case No. F-2015-04841
Doc No. C05739661
Date: 05/13/2015
STATE DEPT. - PRODUCED TO HOUSE SELECT BENGHAZI COMM.
SUBJECT TO AGREEMENT ON SENSITIVE INFORMATION & REDACTIONS. NO FOIA WAIVER. STATE-5CB0045612
U.S. Department of State
Case No. F-2015-04841
Doc No. C05739661
Date: 05/13/2015
STATE DEPT. - PRODUCED TO HOUSE SELECT BENGHAZI COMM.
SUBJECT TO AGREEMENT ON SENSITIVE INFORMATION & REDACTIONS. NO FOIA WAIVER.
B5
U.S. Department of State
Case No. F-2015-04841
Doc No. C05739661
Date: 05/13/2015
STATE DEPT. - PRODUCED TO HOUSE SELECT BENGHAZI COMM.
SUBJECT TO AGREEMENT ON SENSITIVE INFORMATION & REDACTIONS. NO FOIA WAIVER. STATE-5CB0045613
U.S. Department of State
Case No. F-2015-04841
Doc No. C05739661
Date: 05/13/2015
STATE DEPT. - PRODUCED TO HOUSE SELECT BENGHAZI COMM.
SUBJECT TO AGREEMENT ON SENSITIVE INFORMATION & REDACTIONS. NO FOIA WAIVER.
B5
U.S. Department of State
Case No. F-2015-04841
Doc No. C05739661
Date: 05/13/2015
STATE DEPT. - PRODUCED TO HOUSE SELECT BENGHAZI COMM.
SUBJECT TO AGREEMENT ON SENSITIVE INFORMATION & REDACTIONS. NO FOIA WAIVER. STATE-5CB0045614
U.S. Department of State
Case No. F-2015-04841
Doc No. C05739661
Date: 05/13/2015
STATE DEPT. - PRODUCED TO HOUSE SELECT BENGHAZI COMM.
SUBJECT TO AGREEMENT ON SENSITIVE INFORMATION & REDACTIONS. NO FOIA WAIVER.
B5
U.S. Department of State
Case No. F-2015-04841
Doc No. C05739661
Date: 05/13/2015
STATE DEPT. - PRODUCED TO HOUSE SELECT BENGHAZI COMM.
SUBJECT TO AGREEMENT ON SENSITIVE INFORMATION & REDACTIONS. NO FOIA WAIVER. STATE-5CB0045615Speech draft for Friday at CSIS
- do you approve the War Powers Act action memo?
- The King of Morocco has not gotten' back to us with a time but ops will try again when its a reasonable hour for the
Moroccans.
- Attorney General Holder has requested to speak with you regarding the Department of Justice's Libya investigation. He
is standing by Would you like to take this tonight?
U.S. Department of State
Case No. F-2015-04841
Doc No. C05739662
Date: 05/13/2015
STATE DEPT. - PRODUCED TO HOUSE SELECT BENGHAZI COMM.
SUBJECT TO AGREEMENT ON SENSITIVE INFORMATION & REDACTIONS. NO FOIA WAIVER. STATE-SCB0045339Question and update
Attachment= HRC @ CSIS - v4.docx
Sorry we were a couple hours late!Speech draft for Friday at CSIS
--I approved. Will it be picked up?
--call w King should be scheduled so keep working w Huma
--I will speak to Holder. Does he need to speak secure? I can do either.Re: Question and update
We will send a printed copy to you with the book tonight. Should be there shortly.Re: Speech draft for Friday at CSIS
Importance: High
Jake may have sent but In case not..Fw: Stevens Update (important)
Yes, II give memo to Service downstairs.
What about Holder? And the King?
---Re: PDB
Survive, yes. Pat helped level set things tonight and we'll see where we are in the morning.Re: Speech draft for Friday at CSIS
Fw: update on situation in Benghazi: McCain visit
Benghazi staffing
Thank you.
We are offering 7pm to Holder and waiting to hear if he would like to speak secure.
We likely won't hear from the King for several hours but will keep you posted as soon as we do.Re: PDS
+Hrc
Tom, she moved that yellow chair as close as it went. Knee to knee. Amazed she didn't try knee in between knee. And if
that wasn't enough, she leaned forward. More like a pivot, as far as her torso could fold forward to minimize the space
between their heads. Was like the dental hygienist rolling around the floor to get the best access to your mouth
depending on what tooth she was trying to get access to I've never seen a Westerner invade her space like that And
even the non Westerners I've seen do it based on cultural differences have been only briefly to greet, This went on like
that for 51 minutes - unacceptable in any culture. I don't even think you see that behavior among any type of mammal.
The touching the leg and repeatedly calling her 'Hillary' was just gravy.
But it was wonderful. One of the best interviews I've ever witnessed. Wish it were on live tv.RE: Monica Langley Transcript
He can speak at 7pm. He said that it does not have to be secure. Ops will connect to your residence.Re: PDB
----Fw: McCain update










    
Out[14]:








Id
1      None
2      None
3      None
4      None
5      None
6      None
7      None
8      None
9      None
10     None
11     None
12     None
13     None
14     None
15     None
16     None
17     None
18     None
19     None
20     None
21     None
22     None
23     None
24     None
25     None
26     None
27     None
28     None
29     None
30     None
       ... 
71     None
72     None
73     None
74     None
75     None
76     None
77     None
78     None
79     None
80     None
81     None
82     None
83     None
84     None
85     None
86     None
87     None
88     None
89     None
90     None
91     None
92     None
93     None
94     None
95     None
96     None
97     None
98     None
99     None
100    None
Name: SubjectBody, dtype: object

Tokenize and remove stopwords





In [15]:



    


from gensim import corpora, models, utils
from nltk.corpus import stopwords
sw = stopwords.words('english') + ['re', 'fw', 'fvv', 'fwd']
sw = sw + ['pm', "a", "about", "above", "above", "across", "after", "afterwards", "again", "against", "all", "almost", "alone", "along", "already", "also","although","always","am","among", "amongst", "amoungst", "amount",  "an", "and", "another", "any","anyhow","anyone","anything","anyway", "anywhere", "are", "around", "as",  "at", "back","be","became", "because","become","becomes", "becoming", "been", "before", "beforehand", "behind", "being", "below", "beside", "besides", "between", "beyond", "bill", "both", "bottom","but", "by", "call", "can", "cannot", "cant", "co", "con", "could", "couldnt", "cry", "de", "describe", "detail", "do", "done", "down", "due", "during", "each", "eg", "eight", "either", "eleven","else", "elsewhere", "empty", "enough", "etc", "even", "ever", "every", "everyone", "everything", "everywhere", "except", "few", "fifteen", "fify", "fill", "find", "fire", "first", "five", "for", "former", "formerly", "forty", "found", "four", "from", "front", "full", "further", "get", "give", "go", "had", "has", "hasnt", "have", "he", "hence", "her", "here", "hereafter", "hereby", "herein", "hereupon", "hers", "herself", "him", "himself", "his", "how", "however", "hundred", "ie", "if", "in", "inc", "indeed", "interest", "into", "is", "it", "its", "itself", "keep", "last", "latter", "latterly", "least", "less", "ltd", "made", "many", "may", "me", "meanwhile", "might", "mill", "mine", "more", "moreover", "most", "mostly", "move", "much", "must", "my", "myself", "name", "namely", "neither", "never", "nevertheless", "next", "nine", "no", "nobody", "none", "noone", "nor", "not", "nothing", "now", "nowhere", "of", "off", "often", "on", "once", "one", "only", "onto", "or", "other", "others", "otherwise", "our", "ours", "ourselves", "out", "over", "own","part", "per", "perhaps", "please", "put", "rather", "re", "same", "see", "seem", "seemed", "seeming", "seems", "serious", "several", "she", "should", "show", "side", "since", "sincere", "six", "sixty", "so", "some", "somehow", "someone", "something", "sometime", "sometimes", "somewhere", "still", "such", "system", "take", "ten", "than", "that", "the", "their", "them", "themselves", "then", "thence", "there", "thereafter", "thereby", "therefore", "therein", "thereupon", "these", "they", "thickv", "thin", "third", "this", "those", "though", "three", "through", "throughout", "thru", "thus", "to", "together", "too", "top", "toward", "towards", "twelve", "twenty", "two", "un", "under", "until", "up", "upon", "us", "very", "via", "was", "we", "well", "were", "what", "whatever", "when", "whence", "whenever", "where", "whereafter", "whereas", "whereby", "wherein", "whereupon", "wherever", "whether", "which", "while", "whither", "who", "whoever", "whole", "whom", "whose", "why", "will", "with", "within", "without", "would", "yet", "you", "your", "yours", "yourself", "yourselves", "the"]



    


















    






/Users/baptisteraemy/anaconda3/lib/python3.5/site-packages/gensim/utils.py:1015: UserWarning: Pattern library is not installed, lemmatization won't be available.
  warnings.warn("Pattern library is not installed, lemmatization won't be available.")

















In [16]:



    


def fil(row):
    t = utils.simple_preprocess(row.SubjectBody)
    filt = list(filter(lambda x: x not in sw, t))
    return ' '.join(filt)
emails['SubjectBody'] = emails.apply(fil, axis=1)
emails.head(10)



    


















    
Out[16]:










  
    

      
      DocNumber
      MetadataSubject
      MetadataTo
      MetadataFrom
      SenderPersonId
      MetadataDateSent
      MetadataDateReleased
      MetadataPdfLink
      MetadataCaseNumber
      MetadataDocumentClass
      ...
      ExtractedFrom
      ExtractedCc
      ExtractedDateSent
      ExtractedCaseNumber
      ExtractedDocNumber
      ExtractedDateReleased
      ExtractedReleaseInPartOrFull
      ExtractedBodyText
      RawText
      SubjectBody
    

    

      Id
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
    

  
  
    

      1
      C05739545
      WOW
      H
      Sullivan, Jacob J
      87.0
      2012-09-12T04:00:00+00:00
      2015-05-22T04:00:00+00:00
      DOCUMENTS/HRC_Email_1_296/HRCH2/DOC_0C05739545...
      F-2015-04841
      HRC_Email_296
      ...
      Sullivan, Jacob J <Sullivan11@state.gov>
      NaN
      Wednesday, September 12, 2012 10:16 AM
      F-2015-04841
      C05739545
      05/13/2015
      RELEASE IN FULL
      
      UNCLASSIFIED\nU.S. Department of State\nCase N...
      wow
    

    

      2
      C05739546
      H: LATEST: HOW SYRIA IS AIDING QADDAFI AND MOR...
      H
      NaN
      NaN
      2011-03-03T05:00:00+00:00
      2015-05-22T04:00:00+00:00
      DOCUMENTS/HRC_Email_1_296/HRCH1/DOC_0C05739546...
      F-2015-04841
      HRC_Email_296
      ...
      NaN
      NaN
      NaN
      F-2015-04841
      C05739546
      05/13/2015
      RELEASE IN PART
      B6\nThursday, March 3, 2011 9:45 PM\nH: Latest...
      UNCLASSIFIED\nU.S. Department of State\nCase N...
      thursday march latest syria aiding qaddafi sid...
    

    

      3
      C05739547
      CHRIS STEVENS
      ;H
      Mills, Cheryl D
      32.0
      2012-09-12T04:00:00+00:00
      2015-05-22T04:00:00+00:00
      DOCUMENTS/HRC_Email_1_296/HRCH2/DOC_0C05739547...
      F-2015-04841
      HRC_Email_296
      ...
      Mills, Cheryl D <MillsCD@state.gov>
      Abedin, Huma
      Wednesday, September 12, 2012 11:52 AM
      F-2015-04841
      C05739547
      05/14/2015
      RELEASE IN PART
      Thx
      UNCLASSIFIED\nU.S. Department of State\nCase N...
      thxre chris stevens
    

    

      4
      C05739550
      CAIRO CONDEMNATION - FINAL
      H
      Mills, Cheryl D
      32.0
      2012-09-12T04:00:00+00:00
      2015-05-22T04:00:00+00:00
      DOCUMENTS/HRC_Email_1_296/HRCH2/DOC_0C05739550...
      F-2015-04841
      HRC_Email_296
      ...
      Mills, Cheryl D <MillsCD@state.gov>
      Mitchell, Andrew B
      Wednesday, September 12,2012 12:44 PM
      F-2015-04841
      C05739550
      05/13/2015
      RELEASE IN PART
      
      UNCLASSIFIED\nU.S. Department of State\nCase N...
      cairo condemnation final
    

    

      5
      C05739554
      H: LATEST: HOW SYRIA IS AIDING QADDAFI AND MOR...
      Abedin, Huma
      H
      80.0
      2011-03-11T05:00:00+00:00
      2015-05-22T04:00:00+00:00
      DOCUMENTS/HRC_Email_1_296/HRCH1/DOC_0C05739554...
      F-2015-04841
      HRC_Email_296
      ...
      NaN
      NaN
      NaN
      F-2015-04841
      C05739554
      05/13/2015
      RELEASE IN PART
      H <hrod17@clintonemail.com>\nFriday, March 11,...
      B6\nUNCLASSIFIED\nU.S. Department of State\nCa...
      hrod clintonemail com friday march huma abedin...
    

    

      6
      C05739559
      MEET THE RIGHT-WING EXTREMIST BEHIND ANTI-MUSL...
      Russorv@state.gov
      H
      80.0
      2012-09-12T04:00:00+00:00
      2015-05-22T04:00:00+00:00
      DOCUMENTS/HRC_Email_1_296/HRCH2/DOC_0C05739559...
      F-2015-04841
      HRC_Email_296
      ...
      NaN
      NaN
      Wednesday, September 12, 2012 01:00 PM
      F-2015-04841
      C05739559
      05/13/2015
      RELEASE IN PART
      Pis print.\n-•-...-^\nH < hrod17@clintonernail...
      B6\nUNCLASSIFIED\nU.S. Department of State\nCa...
      pis print hrod wednesday september russorv sta...
    

    

      7
      C05739560
      ANTI-MUSLIM FILM DIRECTOR IN HIDING, FOLLOWING...
      H
      Mills, Cheryl D
      32.0
      2012-09-12T04:00:00+00:00
      2015-05-22T04:00:00+00:00
      DOCUMENTS/HRC_Email_1_296/HRCH2/DOC_0C05739560...
      F-2015-04841
      HRC_Email_296
      ...
      Mills, Cheryl D <MillsCD@state.gov>
      NaN
      Wednesday, September 12, 2012 4:00 PM
      F-2015-04841
      C05739560
      05/13/2015
      RELEASE IN FULL
      
      UNCLASSIFIED\nU.S. Department of State\nCase N...
      anti muslim film director hiding following lib...
    

    

      8
      C05739561
      H: LATEST: HOW SYRIA IS AIDING QADDAFI AND MOR...
      Abedin, Huma
      H
      80.0
      2011-03-11T05:00:00+00:00
      2015-05-22T04:00:00+00:00
      DOCUMENTS/HRC_Email_1_296/HRCH1/DOC_0C05739561...
      F-2015-04841
      HRC_Email_296
      ...
      NaN
      NaN
      NaN
      F-2015-04841
      C05739561
      05/13/2015
      RELEASE IN PART
      H <hrod17@clintonemail.corn>\nFriday, March 11...
      B6\nUNCLASSIFIED\nU.S. Department of State\nCa...
      hrod clintonemail corn friday march huma abedi...
    

    

      9
      C05739562
      SECRETARY'S REMARKS
      H
      Sullivan, Jacob J
      87.0
      2012-09-12T04:00:00+00:00
      2015-05-22T04:00:00+00:00
      DOCUMENTS/HRC_Email_1_296/HRCH2/DOC_0C05739562...
      F-2015-04841
      HRC_Email_296
      ...
      Sullivan, Jacob J <Sullivanli@stategov>
      NaN
      Wednesday, September 12, 2012 6:08 PM
      F-2015-04841
      C05739562
      05/13/2015
      RELEASE IN FULL
      FYI
      UNCLASSIFIED\nU.S. Department of State\nCase N...
      fyifvv secretary remarks
    

    

      10
      C05739563
      MORE ON LIBYA
      H
      NaN
      NaN
      2012-09-12T04:00:00+00:00
      2015-05-22T04:00:00+00:00
      DOCUMENTS/HRC_Email_1_296/HRCH2/DOC_0C05739563...
      F-2015-04841
      HRC_Email_296
      ...
      NaN
      NaN
      NaN
      F-2015-04841
      C05739563
      05/13/2015
      RELEASE IN PART
      B6\nWednesday, September 12, 2012 6:16 PM\nFwd...
      UNCLASSIFIED\nU.S. Department of State\nCase N...
      wednesday september libya libya sept docx send...
    

  



10 rows × 22 columns



















In [17]:



    


countries = np.array([[country.name.lower(), country.alpha_2.lower(), country.alpha_3.lower()] for country in list(pycountry.countries)])
countries[:5]



    


















    
Out[17]:








array([['aruba', 'aw', 'abw'],
       ['afghanistan', 'af', 'afg'],
       ['angola', 'ao', 'ago'],
       ['anguilla', 'ai', 'aia'],
       ['åland islands', 'ax', 'ala']], 
      dtype='<U44')

















In [18]:



    


countries.shape



    


















    
Out[18]:








(249, 3)

















In [19]:



    


countries = pd.DataFrame(countries, columns=['Name', 'Alpha_2', 'Alpha_3'])
countries.head()



    


















    
Out[19]:










  
    

      
      Name
      Alpha_2
      Alpha_3
    

  
  
    

      0
      aruba
      aw
      abw
    

    

      1
      afghanistan
      af
      afg
    

    

      2
      angola
      ao
      ago
    

    

      3
      anguilla
      ai
      aia
    

    

      4
      åland islands
      ax
      ala
    

  




















In [20]:



    


countries.shape



    


















    
Out[20]:








(249, 3)

















In [21]:



    


countries.isin(['aruba']).any().any()



    


















    
Out[21]:








True

















In [22]:



    


def check_country(row):
    return countries.isin(row.SubjectBody.split()).any().any()



    











In [23]:



    


emails_country = emails[emails.apply(check_country, axis=1)]
len(emails_country)



    


















    
Out[23]:








2208

Sentiment analysis

We explain before our precessing. Now we will do the sentiment analysis only on the subject and the body So we will only consider the subject and the body





In [24]:



    


sentiments = pd.DataFrame(emails_country.SubjectBody)
sentiments.head()



    


















    
Out[24]:










  
    

      
      SubjectBody
    

    

      Id
      
    

  
  
    

      2
      thursday march latest syria aiding qaddafi sid...
    

    

      5
      hrod clintonemail com friday march huma abedin...
    

    

      6
      pis print hrod wednesday september russorv sta...
    

    

      7
      anti muslim film director hiding following lib...
    

    

      8
      hrod clintonemail corn friday march huma abedi...
    

  




















In [25]:



    


sentiments.shape



    


















    
Out[25]:








(2208, 1)

Analysis

We will do a sentiment analysis on each sentense and then compute a socre for each country

We will compare different module:

- nltk.sentiment.vader that attribute a score to each sentence
- liuhu that has a set of positive word and one of neg word. We count the positif word and neg word in each sentence and compute the mean

Vader (That part takes time)





In [26]:



    


sentiments.head()



    


















    
Out[26]:










  
    

      
      SubjectBody
    

    

      Id
      
    

  
  
    

      2
      thursday march latest syria aiding qaddafi sid...
    

    

      5
      hrod clintonemail com friday march huma abedin...
    

    

      6
      pis print hrod wednesday september russorv sta...
    

    

      7
      anti muslim film director hiding following lib...
    

    

      8
      hrod clintonemail corn friday march huma abedi...
    

  




















In [40]:



    


from nltk.sentiment.vader import SentimentIntensityAnalyzer
sid = SentimentIntensityAnalyzer()
def sentiment_analysis(row):
    score = sid.polarity_scores(row)
    return pd.Series({'Pos': score['pos'], 'Neg': score['neg'], 'Compound_':score['compound'] })

sentiments = pd.concat([sentiments, sentiments.SubjectBody.apply(sentiment_analysis)], axis=1)

sentiments.to_csv('mailScore.csv')
sentiments.head()



    


















    
Out[40]:










  
    

      
      SubjectBody
      Compound_
      Neg
      Pos
    

    

      Id
      
      
      
      
    

  
  
    

      2
      thursday march latest syria aiding qaddafi sid...
      0.0000
      0.000
      0.000
    

    

      5
      hrod clintonemail com friday march huma abedin...
      0.0000
      0.000
      0.000
    

    

      6
      pis print hrod wednesday september russorv sta...
      -0.9531
      0.227
      0.032
    

    

      7
      anti muslim film director hiding following lib...
      -0.8225
      0.589
      0.000
    

    

      8
      hrod clintonemail corn friday march huma abedi...
      0.0000
      0.000
      0.000

Liuhu





In [27]:



    


from nltk.corpus import opinion_lexicon
sentimentsLihuh = pd.read_csv('mailScore.csv', index_col='Id')
#transform the array of positiv and negatif word in dict
dicPosNeg =  dict()
for word in opinion_lexicon.positive():
    dicPosNeg[word] = 1
    
for word in opinion_lexicon.negative():
    dicPosNeg[word] = -1   
    

def sentiment_liuhu(sentence):
    counter = []
    for word in sentence.split():
        value = dicPosNeg.get(word, -999)
        if value != -999:
            counter.append(value)
            
    if len(counter) == 0 :
        return pd.Series({'Sum_': int(0), 'Mean_': int(0) })
    return pd.Series({'Sum_': np.sum(counter), 'Mean_': np.mean(counter) })

sentimentsLihuh = pd.concat([sentimentsLihuh, sentimentsLihuh.SubjectBody.apply(sentiment_liuhu)], axis=1)

sentimentsLihuh.to_csv('mailScore2.csv')
sentimentsLihuh



    


















    
Out[27]:










  
    

      
      SubjectBody
      Compound_
      Neg
      Pos
      Mean_
      Sum_
    

    

      Id
      
      
      
      
      
      
    

  
  
    

      2
      thursday march latest syria aiding qaddafi sid...
      0.0000
      0.000
      0.000
      0.000000
      0.0
    

    

      5
      hrod clintonemail com friday march huma abedin...
      0.0000
      0.000
      0.000
      0.000000
      0.0
    

    

      6
      pis print hrod wednesday september russorv sta...
      -0.9531
      0.227
      0.032
      -0.230769
      -3.0
    

    

      7
      anti muslim film director hiding following lib...
      -0.8225
      0.589
      0.000
      0.000000
      0.0
    

    

      8
      hrod clintonemail corn friday march huma abedi...
      0.0000
      0.000
      0.000
      0.000000
      0.0
    

    

      10
      wednesday september libya libya sept docx send...
      0.0000
      0.000
      0.000
      0.000000
      0.0
    

    

      11
      fyi abz hb libya west bank gaza
      0.2023
      0.000
      0.231
      0.000000
      0.0
    

    

      12
      wednesday september libya libya sept docx send...
      0.0000
      0.000
      0.000
      0.000000
      0.0
    

    

      14
      anne marie slaughter sunday march jacob mills ...
      0.9956
      0.103
      0.188
      0.159420
      11.0
    

    

      15
      randolph lawrence sent wednesday september mil...
      0.9294
      0.045
      0.332
      0.500000
      4.0
    

    

      16
      asked attend svtc today embassy tripoli met st...
      0.9246
      0.000
      0.227
      1.000000
      6.0
    

    

      17
      hope picture kamala sent youth libya
      0.4404
      0.000
      0.367
      0.000000
      0.0
    

    

      20
      amazing sullivan jacob sullivanii state gov we...
      0.5859
      0.000
      0.183
      1.000000
      1.0
    

    

      21
      hrod clintonernaii com wednesday september esu...
      0.0000
      0.000
      0.000
      0.000000
      0.0
    

    

      22
      pis print hrod clintoriernail corn wednesday s...
      0.0000
      0.000
      0.000
      0.000000
      0.0
    

    

      23
      pis print magariaf attack libya sid
      -0.4767
      0.383
      0.000
      -1.000000
      -1.0
    

    

      24
      follow flag follow flag status flagged info ma...
      -0.4767
      0.237
      0.000
      -1.000000
      -1.0
    

    

      27
      libya
      0.0000
      0.000
      0.000
      0.000000
      0.0
    

    

      29
      http religion ogs cnn com deadlv link islamic ...
      -0.9136
      0.172
      0.161
      -0.408451
      -29.0
    

    

      31
      fyifw quick summary potus calls presidents lib...
      0.0000
      0.000
      0.000
      0.000000
      0.0
    

    

      33
      sunday march lots new intel possible libyan co...
      -0.9846
      0.155
      0.102
      -0.142857
      -8.0
    

    

      34
      pis print sullivan jacob mailto sullivanjj sta...
      0.0000
      0.000
      0.000
      0.000000
      0.0
    

    

      36
      american killed libya intel mission track weapons
      -0.8126
      0.597
      0.000
      -1.000000
      -1.0
    

    

      37
      depart private residence en route state depart...
      0.9682
      0.012
      0.088
      1.000000
      7.0
    

    

      47
      http www salon comi monday oct edt gop october...
      -0.9601
      0.181
      0.167
      -0.184211
      -14.0
    

    

      53
      romney op ed today wall street journal playing...
      -0.3400
      0.131
      0.125
      -1.000000
      -2.0
    

    

      55
      remarks afternoon andrews wonderful thanks ins...
      0.9153
      0.056
      0.289
      0.500000
      2.0
    

    

      56
      info sidney blumenthal mailto sent monday octo...
      -0.3400
      0.109
      0.104
      -1.000000
      -2.0
    

    

      58
      pretty uncompelling piece ask course biased mi...
      0.9949
      0.171
      0.266
      -0.064935
      -5.0
    

    

      59
      fyi care job prom rhodes benjamin sent wednesd...
      0.6124
      0.000
      0.208
      0.000000
      0.0
    

    

      ...
      ...
      ...
      ...
      ...
      ...
      ...
    

    

      7841
      verveer melanne verveerms state gov monday dec...
      0.0000
      0.000
      0.000
      0.000000
      0.0
    

    

      7844
      dear hillary hear dianne staff new start play ...
      0.8020
      0.119
      0.414
      0.000000
      0.0
    

    

      7847
      meetings today wanted know norwegians agreed m...
      0.9816
      0.035
      0.341
      0.846154
      11.0
    

    

      7850
      opinion december new start ratify caveats trea...
      0.3384
      0.178
      0.215
      -0.101449
      -7.0
    

    

      7856
      fyi wed china pc recommend
      0.5106
      0.000
      0.589
      1.000000
      1.0
    

    

      7859
      good public movement start judd gregg support ...
      0.9605
      0.035
      0.179
      0.500000
      6.0
    

    

      7862
      interesting approach influencing popular attit...
      0.8625
      0.000
      0.231
      1.000000
      4.0
    

    

      7866
      points motho meeting sen mitchell handed today...
      0.3182
      0.000
      0.133
      0.000000
      0.0
    

    

      7874
      depart private residence en route state depart...
      0.6808
      0.012
      0.044
      1.000000
      1.0
    

    

      7876
      http www nybooks com articles archives dec bit...
      0.6179
      0.151
      0.156
      -0.100000
      -6.0
    

    

      7877
      mchale judith mchaleja state gov thursday dece...
      0.7096
      0.000
      0.134
      0.333333
      1.0
    

    

      7879
      ap palin visit haiti humanitarian effort
      0.0000
      0.000
      0.000
      0.000000
      0.0
    

    

      7880
      fyi st century engagement work progress good s...
      0.9136
      0.000
      0.555
      1.000000
      4.0
    

    

      7883
      snowe supports senate ratification new start t...
      0.9903
      0.020
      0.340
      0.733333
      11.0
    

    

      7885
      mills cheryl millscd state gov saturday decemb...
      0.0516
      0.000
      0.059
      -1.000000
      -1.0
    

    

      7886
      government travel visas government leaders cur...
      0.0516
      0.061
      0.113
      -0.500000
      -2.0
    

    

      7889
      ap sarah palin arrives haiti mercy mission
      0.3612
      0.000
      0.294
      1.000000
      1.0
    

    

      7890
      ni mills cheryl millscd state gov saturday dec...
      0.6369
      0.000
      0.271
      0.000000
      0.0
    

    

      7891
      government leaders interestingre leahy calls f...
      0.0516
      0.000
      0.098
      -1.000000
      -1.0
    

    

      7898
      mexican fm espinosa
      0.0000
      0.000
      0.000
      0.000000
      0.0
    

    

      7904
      fyifw iran daily special report fm mottaki rem...
      0.4019
      0.000
      0.278
      0.000000
      0.0
    

    

      7910
      huma abedin huma clintonemail com monday decem...
      -0.2263
      0.163
      0.096
      0.000000
      0.0
    

    

      7912
      according reid staff tentatively start treaty ...
      0.8442
      0.000
      0.205
      0.000000
      0.0
    

    

      7917
      death holbrookell ydalo scrolling english rema...
      -0.5994
      0.281
      0.000
      -1.000000
      -1.0
    

    

      7929
      fyi forwarded messagefwd bb
      0.2023
      0.000
      0.375
      0.000000
      0.0
    

    

      7934
      unsigned editorials appear bloomberg online pr...
      0.9965
      0.026
      0.155
      0.407407
      11.0
    

    

      7935
      ap germany pullout afghanistan start
      0.0000
      0.000
      0.000
      0.000000
      0.0
    

    

      7937
      mills cheryl millscd state gov thursday decemb...
      0.0000
      0.000
      0.000
      0.000000
      0.0
    

    

      7938
      nbc yorker december envoy posted hendrik hertz...
      0.9878
      0.161
      0.249
      0.083333
      4.0
    

    

      7939
      hi sorry chance want hear directly great resul...
      0.9750
      0.033
      0.664
      0.666667
      4.0
    

  



2208 rows × 6 columns

Aggregate by countries

We groupe by country and compute the mean of each score





In [28]:



    


sentiments = pd.read_csv('mailScore2.csv', index_col='Id')
sentiments.head()



    


















    
Out[28]:










  
    

      
      SubjectBody
      Compound_
      Neg
      Pos
      Mean_
      Sum_
    

    

      Id
      
      
      
      
      
      
    

  
  
    

      2
      thursday march latest syria aiding qaddafi sid...
      0.0000
      0.000
      0.000
      0.000000
      0.0
    

    

      5
      hrod clintonemail com friday march huma abedin...
      0.0000
      0.000
      0.000
      0.000000
      0.0
    

    

      6
      pis print hrod wednesday september russorv sta...
      -0.9531
      0.227
      0.032
      -0.230769
      -3.0
    

    

      7
      anti muslim film director hiding following lib...
      -0.8225
      0.589
      0.000
      0.000000
      0.0
    

    

      8
      hrod clintonemail corn friday march huma abedi...
      0.0000
      0.000
      0.000
      0.000000
      0.0
    

  




















In [30]:



    


def aggScoreByCountry(country):
    condition = sentiments.apply(lambda x: np.any(country.isin(x.SubjectBody.split())), axis=1)
    sent = sentiments[condition]
    if len(sent) == 0:
        print(country.Name, -999)
        return pd.Series({'Compound_':-999, 'Mean_':-999, 'Appearance': int(len(sent))})
    compound_ = np.mean(sent.Compound_)
    mean_ = np.mean(sent.Mean_)
    print(country.Name, compound_)
    return pd.Series({'Compound_': compound_, 'Mean_': mean_, 'Appearance': int(len(sent))})

countries = pd.concat([countries, countries.apply(lambda x: aggScoreByCountry(x), axis=1)],axis=1)
countries.to_csv('countriesScore.csv')



    


















    






aruba -999
aruba -999
afghanistan 0.3221980392156861
angola 0.4339446153846152
anguilla 0.49756
åland islands 0.033333333333333326
albania 0.11405576923076924
andorra 0.3122958333333334
united arab emirates 0.20095
argentina 0.319656
armenia 0.293725641025641
american samoa -999
antarctica 0.10395000000000001
french southern territories -0.026650000000000007
antigua and barbuda 0.2629
australia 0.39453448275862074
austria -0.19118
azerbaijan 0.5881571428571429
burundi 0.2767
belgium -0.18862500000000001
benin 0.40291724137931034
bonaire, sint eustatius and saba -999
burkina faso -999
bangladesh 0.23368333333333333
bulgaria -999
bahrain 0.40132500000000004
bahamas 0.9961
bosnia and herzegovina 0.96235
saint barthélemy 0.9413
belarus 0.17
belize -999
bermuda -0.09228
bolivia, plurinational state of 0.9865
brazil 0.3331512195121952
barbados 0.15165
brunei darussalam -0.055454545454545486
bhutan 0.76915
bouvet island -999
botswana -999
central african republic 0.9949
canada 0.25808611111111107
cocos (keeling) islands 0.4999428571428572
switzerland 0.6047875
chile 0.2748
china 0.34194464285714293
côte d'ivoire 0.3410999999999999
cameroon 0.38001666666666667
congo, the democratic republic of the 0.6281333333333333
congo 0.21072500000000002
cook islands 0.5366333333333334
colombia 0.28768000000000005
comoros 0.2979624338624339
cabo verde 0.22777999999999995
costa rica 0.6193600000000001
cuba 0.28931250000000003
curaçao -999
christmas island -999
cayman islands 0.9938
cyprus 0.4986857142857143
czechia 0.993
germany -0.06871923076923075
djibouti -999
dominica -999
denmark 0.47541666666666665
dominican republic 0.9726
algeria -0.02369999999999997
ecuador 0.4383846153846154
egypt 0.31535777777777774
eritrea 0.9739999999999999
western sahara -999
spain 0.4957900000000001
estonia 0.2524
ethiopia 0.28129333333333334
finland 0.46505714285714295
fiji -999
falkland islands (malvinas) -999
france 0.06398285714285713
faroe islands -0.10121249999999998
micronesia, federated states of 0.22702222222222226
gabon 0.35255
united kingdom 0.8530666666666668
georgia 0.7904833333333333
guernsey -999
ghana 0.23518000000000003
gibraltar -0.017199999999999993
guinea 0.15567826086956524
guadeloupe -999
gambia 0.40722500000000006
guinea-bissau -0.1027
equatorial guinea -999
greece 0.18003999999999998
grenada -999
greenland 0.8046
guatemala 0.27515
french guiana 0.6124
guam -0.25
guyana 0.5370117647058823
hong kong 0.7964
heard island and mcdonald islands -999
honduras 0.17134705882352938
croatia 0.0018250000000000072
haiti 0.2612948186528496
hungary 0.34485
indonesia 0.4832783783783784
isle of man 0.5904333333333334
india 0.6330043478260869
british indian ocean territory -0.013800000000000034
ireland 0.2577553191489361
iran, islamic republic of 0.9252
iraq 0.22957215189873426
iceland 0.47324
israel 0.2718496062992127
italy 0.10881176470588236
jamaica 0.33277500000000004
jersey 0.5085928571428571
jordan 0.3879666666666666
japan 0.35972666666666675
kazakhstan 0.3963
kenya 0.4577166666666667
kyrgyzstan 0.04728461538461537
cambodia 0.4986
kiribati -0.07601666666666666
saint kitts and nevis -999
korea, republic of 0.31845
kuwait 0.14969999999999997
lao people's democratic republic 0.44334117647058824
lebanon 0.20733846153846155
liberia 0.11238888888888893
libya 0.14394842105263164
saint lucia 0.2641666666666667
liechtenstein 0.32991764705882354
sri lanka -999
lesotho 0.0
lithuania 0.5784578947368422
luxembourg 0.536925
latvia -0.9888
macao 0.1022777777777778
saint martin (french part) 0.9403
morocco 0.5683925925925927
monaco 0.75328
moldova, republic of 0.34950000000000003
madagascar 0.17469999999999997
maldives -0.9513
mexico 0.3275980392156863
marshall islands 0.9877
macedonia, republic of -999
mali -999
malta -0.037349999999999994
myanmar 0.5817375
montenegro -999
mongolia 0.7705
northern mariana islands 0.4908272727272728
mozambique 0.1531
mauritania 0.35433176470588235
montserrat 0.5207087719298246
martinique -999
mauritius 0.79895
malawi 0.42205000000000004
malaysia 0.6925250000000001
mayotte -999
namibia 0.42325
new caledonia 0.6774
niger 0.19152499999999997
norfolk island -999
nigeria 0.6552555555555556
nicaragua 0.4037458333333334
niue 0.7351
netherlands -0.0010000000000000009
norway 0.3726916666666667
nepal 0.42523333333333335
nauru 0.514675
new zealand 0.7336750000000001
oman 0.362475
pakistan 0.23526710526315783
panama 0.4321825
pitcairn -999
peru 0.4544615384615385
philippines 0.7230083333333334
palau 0.5252454545454545
papua new guinea 0.4387
poland -0.09023461538461541
puerto rico 0.12696875
korea, democratic people's republic of -999
portugal 0.47938000000000003
paraguay 0.9989
palestine, state of 0.5469666666666667
french polynesia 0.9968
qatar 0.44684545454545466
réunion -999
romania 0.7233714285714286
russian federation 0.0
rwanda 0.47523333333333334
saudi arabia 0.679
sudan 0.25760937500000003
senegal 0.3509923076923077
singapore 0.7621619047619048
south georgia and the south sandwich islands 0.0
saint helena, ascension and tristan da cunha 0.0972
svalbard and jan mayen 0.9989
solomon islands 0.4713
sierra leone 0.30872
el salvador 0.5994
san marino 0.9988
somalia 0.1571384615384615
saint pierre and miquelon -999
serbia 0.33043333333333336
south sudan -0.76072
sao tome and principe 0.48067272727272764
suriname 0.6681625
slovakia 0.8811500000000001
slovenia 0.4392600000000001
sweden 0.12425555555555556
swaziland 0.9991
sint maarten (dutch part) -999
seychelles 0.37533999999999995
syrian arab republic 0.987
turks and caicos islands -0.9595
chad 0.3374333333333333
togo -999
thailand 0.36315243902439015
tajikistan 0.9919
tokelau -999
turkmenistan 0.9934000000000001
timor-leste 0.0
tonga -0.029700000000000004
trinidad and tobago 0.66965
tunisia 0.4783454545454546
turkey 0.25975901639344257
tuvalu 0.4502240000000002
taiwan, province of china -0.7351
tanzania, united republic of -999
uganda -0.07911111111111112
ukraine 0.71557
united states minor outlying islands -999
uruguay 0.19649
united states 0.664324
uzbekistan 0.79808
holy see (vatican city state) 0.44155454545454553
saint vincent and the grenadines -999
venezuela, bolivarian republic of -999
virgin islands, british -999
virgin islands, u.s. 0.4958666666666667
viet nam -0.9721
vanuatu 0.45715
wallis and futuna -999
samoa 0.653225
yemen -0.02473157894736841
south africa -999
zambia -0.08233333333333333
zimbabwe 0.9968

Drop all country that have a score of -999 (they never appear in the mails)





In [31]:



    


countries = countries[countries.Compound_ != -999]
len(countries)



    


















    
Out[31]:








207

It's a lot of country. We will also use a thresold for the appearance. We only keep mails that are mentioned in a minimum number of emails





In [32]:



    


minimum_appearance = 15
countries_min = countries[countries.Appearance >= minimum_appearance]
len(countries_min)



    


















    
Out[32]:








66

Plot

We plot the 2 analysis. The first plot show an historgram with the vador score and in color the appearance in the mail.

In the second plot the histogram shows the liuhu score and in color the appearance in the mail

we only consider countries that are at least mention 15 times. Otherwise we end up with to much countries





In [36]:



    


# Set up colors : red to green
countries_sorted = countries_min.sort(columns=['Compound_'])
plt.figure(figsize=(16, 6), dpi=80)


appearance = np.array(countries_sorted.Appearance)
colors = cm.RdYlGn((appearance / float(max(y))))
plot = plt.scatter(appearance, appearance, c=appearance, cmap = 'RdYlGn')
plt.clf()
colorBar = plt.colorbar(plot)
colorBar.ax.set_title("Appearance")


index = np.arange(len(countries_sorted))
bar_width = 0.95
plt.bar(range(countries_sorted.shape[0]), countries_sorted.Compound_, align='center', tick_label=countries_sorted.Name, color=colors)
plt.xticks(rotation=90, ha='center')
plt.title('Using Vader')
plt.xlabel('Countries')
plt.ylabel('Vador Score')



    


















    






/Users/baptisteraemy/anaconda3/lib/python3.5/site-packages/ipykernel/__main__.py:2: FutureWarning: sort(columns=....) is deprecated, use sort_values(by=.....)
  from ipykernel import kernelapp as app










    
Out[36]:








<matplotlib.text.Text at 0x11aa62eb8>










    
























In [37]:



    


countries_sorted = countries_min.sort(columns=['Mean_'])

plt.figure(figsize=(16, 6), dpi=80)

appearance = np.array(countries_sorted.Appearance)
colors = cm.RdYlGn((appearance / float(max(y))))
plot = plt.scatter(appearance, appearance, c=appearance, cmap = 'RdYlGn')
plt.clf()
colorBar = plt.colorbar(plot)
colorBar.ax.set_title("Appearance")


index = np.arange(len(countries_sorted))
bar_width = 0.95
plt.bar(range(countries_sorted.shape[0]), countries_sorted.Mean_, align='center', tick_label=countries_sorted.Name, color=colors)
plt.xticks(rotation=90, ha='center')
plt.title('Liuhu Score')
plt.xlabel('Countries')
plt.ylabel('Liuhu Score')



    


















    






/Users/baptisteraemy/anaconda3/lib/python3.5/site-packages/ipykernel/__main__.py:1: FutureWarning: sort(columns=....) is deprecated, use sort_values(by=.....)
  if __name__ == '__main__':










    
Out[37]:








<matplotlib.text.Text at 0x11cf1a438>

We can see that our result are not very concluding. We coul expect to have USA with a high score and it doesn't appear in our graph. As explain before it's because of the us pronoum.

One improvment is to use a Part-of-speech tagger and that way maybe we would be able to distinguish the country 'US' and the pronoun 'us'





In [ ]:

Content source: christophebertrand/ada-epfl

Similar notebooks:

notebook.community | gallery | about