In [1]:

    

from datetime import datetime,timedelta
from collections import defaultdict,Counter
from pprint import pprint
from tqdm import tqdm
import re

import pymongo
from pymongo import InsertOne, DeleteMany, ReplaceOne, UpdateOne
from pymongo.errors import BulkWriteError


    

In [54]:

    

from fuzzywuzzy import fuzz


    

In [2]:

    

from nltk.corpus import stopwords


    

In [3]:

    

from sklearn.feature_extraction.text import CountVectorizer, TfidfTransformer
import itertools


    

In [4]:

    

list_stopWords=list(set(stopwords.words('english')))


    

In [5]:

    

client = pymongo.MongoClient('localhost:27017')
db = client.tweet


    

In [6]:

    

#db.authenticate('admin','lixiepeng')


    

In [7]:

    

import pandas as pd
import spacy


    

In [8]:

    

nlp = spacy.load('en_core_web_md')


    

In [9]:

    

events = [e for e in db.current_event.find({'event.date':{'$gt':'2017-09-01','$lt':'2017-09-05'}},{'_id':1,'event.class':1,'event.date':1,'event.title':1,'event.description':1})]


    

In [10]:

    

events[0]


    

    
Out[10]:





{'_id': ObjectId('59fbc9fb60b18848c5a55712'),
 'event': {'class': 'Armed attacks and conflicts',
  'date': '2017-09-02',
  'description': 'Seven people are killed and 13 are injured after suicide bombers hit a state-run power station near the northern city of Samarra, Iraq. The Islamic State of Iraq and the Levant claims responsibility for the attack. (Reuters)',
  'title': 'Iraqi Civil War'}}

In [11]:

    

events = [{'id':e['_id'],'class':e['event']['class'],'date':e['event']['date'],'title':e['event']['title'],'description':e['event']['description']} for e in events]


    

In [12]:

    

df_events = pd.DataFrame.from_records(events)


    

In [13]:

    

df_events.head()


    

    
Out[13]:






  

    

      

      
class
      
date
      
description
      
id
      
title
    
  
  

    

      
0
      
Armed attacks and conflicts
      
2017-09-02
      
Seven people are killed and 13 are injured aft...
      
59fbc9fb60b18848c5a55712
      
Iraqi Civil War
    
    

      
1
      
Business and economy
      
2017-09-02
      
Both the FBI and NSD declare that they possess...
      
59fbc9fb60b18848c5a55713
      
Trump Tower wiretapping allegations
    
    

      
2
      
Disasters and accidents
      
2017-09-02
      
Seven schoolgirls are killed and ten hospitali...
      
59fbc9fb60b18848c5a55714
      
2017 disasters in Kenya
    
    

      
3
      
International relations
      
2017-09-02
      
Australia and East Timor settle a dispute betw...
      
59fbc9fb60b18848c5a55715
      
Timor Gap
    
    

      
4
      
Law and crime
      
2017-09-02
      
Cambodian opposition leader Kem Sokha is arres...
      
59fbc9fb60b18848c5a55716
      
Law enforcement in Cambodia
    
  

In [18]:

    

df_events.iloc[0]['id']


    

    
Out[18]:





ObjectId('59fbc9fb60b18848c5a55712')

In [19]:

    

tweets =  [tweet for tweet in db.paper.find({'event_id':df_events.iloc[0]['id']},{'tweet.standard_text':1})]


    

In [20]:

    

tweets


    

    
Out[20]:





[{'_id': '903363851781099522',
  'tweet': {'standard_text': "IS leader al-Baghdadi 'probably still alive' claims US army commander in Iraq"}},
 {'_id': '903368426722381826',
  'tweet': {'standard_text': "IS leader alBaghdadi 'probably still alive' claims US army commander in Iraq via reddit"}},
 {'_id': '903396974715559936',
  'tweet': {'standard_text': 'ISIS claims suicide bombing that killed 29 at soccer match in Iraq'}},
 {'_id': '903400060377030656',
  'tweet': {'standard_text': "realDonaldTrump Why won't you talk to Amb. Bolton about Iraq he claims no one will let him near you? He says he gets stonewalled by staff."}},
 {'_id': '903438029041410048',
  'tweet': {'standard_text': "News IS leader al-Baghdadi 'probably still alive' claims US army commander in Iraq"}},
 {'_id': '903454755699138560',
  'tweet': {'standard_text': 'Iraq claims victory over ISIS in Tal Afar, but more battles loom via skinnergj'}},
 {'_id': '903470373886353408',
  'tweet': {'standard_text': 'IS claims suicide bombing on stadium in Iraq that killed 41 - World - The Jakarta Post'}},
 {'_id': '903477440927809536',
  'tweet': {'standard_text': 'IRAQ IslamicState Claims Killing 6 PMU Fighters In The North Of Baghdad . TerrorMonitor'}},
 {'_id': '903504971122917376',
  'tweet': {'standard_text': 'Islamic State claims suicide bombing on stadium in Iraq that killed 29 - Chicago Tribune'}},
 {'_id': '903549290676494336',
  'tweet': {'standard_text': 'Iraq claims another major victory over ISIS - TheSiasatDaily'}},
 {'_id': '903580512408633344',
  'tweet': {'standard_text': 'Majority of these claims are from IRAQ /AFGHAN Vets claiming PTSD . No one could prepare for the influx of PTSD claims filed by these Vets'}},
 {'_id': '903586930427879424',
  'tweet': {'standard_text': 'Military victory against IS Iraq is far from peace Car bombs underscore insurgency as territorial claims diminishing'}},
 {'_id': '903592275636342784',
  'tweet': {'standard_text': "Donald Trump supported the Iraq war and now claims he didn't."}},
 {'_id': '903614444714037248',
  'tweet': {'standard_text': 'IS claims to have killed 3 Iraqi soldiers in a bomb attack at Tarmiyyah to the North of Baghdad. Iraq'}},
 {'_id': '903620387778519041',
  'tweet': {'standard_text': 'IslamicState claims outside Syria & Iraq have reached 186 in 95 days in 21 different nations'}},
 {'_id': '903637901707059200',
  'tweet': {'standard_text': 'IRAQ IslamicState Claims Killing 3 ISF Soldiers By IED Attack In The North Of Baghdad . TerrorMonitor'}},
 {'_id': '903644089349922817',
  'tweet': {'standard_text': 'ISIS claims suicide bombing that killed 29 at soccer match in Iraq'}},
 {'_id': '903708754113429505',
  'tweet': {'standard_text': "Iraq 's PM Abadi says 36 of security forces killed, 350 injured in the battle of TalAfar , claims of agreement with ISIS are 'unfounded'"}},
 {'_id': '903734420976222208',
  'tweet': {'standard_text': 'Casualty rates in the IRAQ /AFGHAN Wars. Do they help explain high PTSD claims ?'}},
 {'_id': '903761603270631424',
  'tweet': {'standard_text': 'IS claims suicide bombing on stadium in Iraq that killed 41 - World - The Jakarta Post'}},
 {'_id': '904116541830172674',
  'tweet': {'standard_text': 'New video by CBS News: 2 killed , 4 injured after inhaling toxic fumes at power plant'}},
 {'_id': '904116893820342275',
  'tweet': {'standard_text': '2 killed , 4 injured after inhaling toxic fumes at power plant'}},
 {'_id': '904116986380132352',
  'tweet': {'standard_text': '2 killed , 4 injured after inhaling toxic fumes at power plant'}},
 {'_id': '904117018672197632',
  'tweet': {'standard_text': '2 killed , 4 injured after inhaling toxic fumes at power plant'}},
 {'_id': '904117235924459520',
  'tweet': {'standard_text': '2 killed , 4 injured after inhaling toxic fumes at Pennsylvania power plant - CBS News'}},
 {'_id': '904119632600162304',
  'tweet': {'standard_text': 'mystery shrouds dangerous murdetous attack on Khowaja Izhar, as slightly injured one of attacker was killed in fake "encounter" by police'}},
 {'_id': '904120824688508928',
  'tweet': {'standard_text': 'One killed , two injured in Jindal Steel & Power plant mishap in Chhattisgarh'}},
 {'_id': '904122282121064448',
  'tweet': {'standard_text': '2 killed , 4 injured after inhaling toxic fumes at power plant breaking usa world news video today watch cb …'}},
 {'_id': '904122617782816768',
  'tweet': {'standard_text': 'She shot 30 rounds from her home towards the elementary school across the street. She killed the principal and janitor and injured 8 kids.'}},
 {'_id': '904122957185867778',
  'tweet': {'standard_text': 'Video – 2 killed , 4 injured after inhaling toxic fumes at a Pennsylvania power plant – CBS\xa0News'}},
 {'_id': '904124135718170624',
  'tweet': {'standard_text': 'To the crackhead that injured killed and stole my dogs..... I HATE YOU!!!!'}},
 {'_id': '904124307420438528',
  'tweet': {'standard_text': "News KENYA FIRE - At least 7 students killed , 10 injured in Kenyan girls' school fire"}},
 {'_id': '904125559411757056',
  'tweet': {'standard_text': '3 killed , 16 injured in Vereeniging road accident via ewnreporter'}},
 {'_id': '904125878241828864',
  'tweet': {'standard_text': 'RT JournoWes: LaTunaFire remains at more than 5,000 acres, 10% containment & no one has been injured or killed sa…'}},
 {'_id': '904128003298533376',
  'tweet': {'standard_text': 'One teen killed , six injured at barricade:'}},
 {'_id': '904128057849634816',
  'tweet': {'standard_text': '2 killed , 4 injured after workers inhale toxic fumes at Pennsylvania power plant'}},
 {'_id': '904128621715103745',
  'tweet': {'standard_text': 'georgebkk: travel Thai visa One teen killed , six injured at barricade:'}},
 {'_id': '904130840426422272',
  'tweet': {'standard_text': '2 killed , 4 injured after inhaling toxic fumes at power plant'}},
 {'_id': '904130840837459970',
  'tweet': {'standard_text': '2 killed , 4 injured after inhaling toxic fumes at power plant'}},
 {'_id': '904131265611300864',
  'tweet': {'standard_text': "Knife-wielding 'terrorists' killed 29 people and injured 130 others at a China train station."}}]

In [95]:

    

def reference_similatity(reference):
    doc_reference = nlp(reference['description'])
    print(reference['description'])
    tweets = []
    filter_dict = {'event_id':reference['id'],'tweet.lang':'en'}
    #filter_dict = {'event_id':reference['id'],'tweet.lang':'en','tweet.media.card_url':None}
    #filter_dict = {'event_id':reference['id'],'tweet.lang':'en','tweet.media.card_url':{'$ne':None}}
    #filter_dict = {'event_id':reference['id'],'tweet.lang':'en','f':'&f=news'}
    query_dict = {'tweet.standard_text':1}
    records  = [i for i in db.pos.find(filter_dict,query_dict)]+[i for i in db.paper.find(filter_dict,query_dict)]
    print(len(records))
    for tweet in records:
        tweet_id = tweet['_id']
        tweet_text = tweet['tweet']['standard_text'] 
        doc_tweet = nlp(tweet_text)
        coverage_num,jacard_similarity = entity_coverage(doc_reference,doc_tweet)
        char_match,token_match = fuzzy_string_matching(reference['description'],tweet_text)
        tweets.append((tweet_id,doc_reference.similarity(doc_tweet),tweet_text))
        #tweets.append((tweet_id,doc_reference.similarity(doc_tweet),coverage_num,jacard_similarity,char_match,token_match,tweet_text))  
    #tweets = [(,doc.similarity(nlp(tweet['tweet']['standard_text'])),entity_coverage,tweet['tweet']['standard_text']) for tweet in db.pos.find({'event_id':record['id']},{'tweet.standard_text':1})] + \
    #[(tweet['_id'],doc.similarity(nlp(tweet['tweet']['standard_text'])),entity_coverage,tweet['tweet']['standard_text']) for tweet in db.paper.find({'event_id':record['id']},{'tweet.standard_text':1})]
    tweets = sorted(tweets,key=lambda x:x[1],reverse=True)
    for i in tweets:
        print(i)


    

In [96]:

    

reference_similatity(df_events.iloc[11])


    

    




Syrian state television reports the Syrian Army reaches a point 3 kilometers from Deir ez-Zor, a city besieged by the Islamic State of Iraq and the Levant since 2014. (Reuters)
Two Russian soldiers, escorting the military staff ceasefire monitoring convoy, are killed in Deir ez-Zor Province by ISIL shelling. (Reuters)
207
('904744262499049472', 0.92306943746711545, 'Syrian army , allies 3 km from Deir al-Zor enclave: state TV ,IS is holding 93,000 civilians for years')
('904832661579026434', 0.91734328901533679, 'Two Russia |n contract soldiers were killed in the Syria |n province of Deir ez-Zor as a result ISIS mortar attack')
('904756045486202880', 0.91117196504155551, 'Syrian government forces are closing in on the eastern city of Deir al-Zour, which has been besieged by ISIS.')
('904832034144702466', 0.90087469217912353, 'Ministry of Defence: two Russian soldiers have died in the province Deir el-Zor when firing...')
('904838519742283777', 0.89892350959985856, 'Syrian government forces are closing in on the eastern city of Deir al-Zour, which has been besieged by so-called...')
('904725261438115840', 0.89591376934947298, 'Syrian army thrusts east to break siege of Deir al-Zor. 3 kms to go, says Syrian state TV— ellen_fra & Laila Bassam')
('904799838709186560', 0.89444747706370664, "Syria : Assad 's army, allies thrust east to break siege Deir al-Zor, one of ISIL 's last major cities it holds")
('904832715538571265', 0.89381559623982154, 'world news Syrian army , allies thrust east to break siege in Deir al-Zor city hyperrts StopRussianAggression')
('904815201660305413', 0.89152495329864168, 'REUTERS HEADLINES broadcast by TheIsraelCompany : Syrian army , allies thrust east to break siege in Deir al-...')
('904743638592131072', 0.88757200863475194, 'Syrian troops close to breaking Islamic State siege in eastern city of Deir al-Zor city - Hindustan Times')
('904741303094837249', 0.88477938703982983, "Syrian army 'close to breaking' ISIL siege on parts of the eastern city of Deir Az Zor")
('904828512170737664', 0.88469880793732958, 'Defense Ministry reports 2 Russian soldiers killed in ISIS shelling in Deir ez-Zor province')
('904779227777687552', 0.88322772290637985, 'Syrian army reaches I.S. lines in Deir al-Zor The Syrian army and its allies are advancing rapidly on a pro-govern…')
('904833550645690368', 0.88148743184814604, 'The Syrian army prepares for a siege of fighters of ISIL in the city of Deir ez-Zor')
('904817368114847744', 0.87788820700487391, 'Syrian army & allies hours away from breaking IS siege in Deir Ezzor city marking another humiliating defeat for IS')
('904845884520783872', 0.8751515152078484, 'Syrian army , allies thrust east to break siege in Deir al-Zor city #')
('904741131505852416', 0.87513646624716168, "Beirut: Syria's army battled the Islamic State group on the edges of Deir Ezzor on Monday, seeking to break the si…")
('904770047192727552', 0.87256307583393555, 'ReutersWorld: Syrian army , allies thrust east to break siege in Deir al-Zor city')
('904815924695498752', 0.87185638757863859, "Two Russian soldiers killed by shelling in Syria's Deir al-Zor province: Ifax")
('904822083464699906', 0.87185638757863859, "Two Russian soldiers killed by shelling in Syria's Deir al-Zor province: Ifax")
('904741805434044416', 0.87165977096748182, 'Syrian army , allies thrust east to break siege in Deir al-Zor city')
('904742139141263360', 0.87165977096748182, 'Syrian army , allies thrust east to break siege in Deir al-Zor city')
('904748408384126976', 0.87165977096748182, 'Syrian army , allies thrust east to break siege in Deir al-Zor city')
('904773181839421440', 0.87165977096748182, 'Syrian army , allies thrust east to break siege in Deir al-Zor city')
('904775148754100228', 0.87165977096748182, 'Syrian army , allies thrust east to break siege in Deir al-Zor city')
('904776095026827264', 0.87165977096748182, 'Syrian army , allies thrust east to break siege in Deir al-Zor city')
('904792016764227584', 0.87165977096748182, 'Syrian army, allies thrust east to break siege in Deir al-Zor city')
('904793968352739328', 0.87165977096748182, 'Syrian army , allies thrust east to break siege in Deir al-Zor city')
('904805041483247616', 0.87165977096748182, 'Syrian army , allies thrust east to break siege in Deir al-Zor city')
('904832599322972160', 0.87165977096748182, 'Syrian army , allies thrust east to break siege in Deir al-Zor city')
('904843639276412928', 0.87165977096748182, 'Syrian army , allies thrust east to break siege in Deir al-Zor city')
('904846577184931840', 0.87165977096748182, 'Syrian army , allies thrust east to break siege in Deir al-Zor city')
('904849092517531651', 0.87165977096748182, 'Syrian army , allies thrust east to break siege in Deir al-Zor city')
('904851447837220864', 0.87165977096748182, 'Syrian army , allies thrust east to break siege in Deir al-Zor city')
('904787431081205760', 0.87165975556834907, 'topNews Syrian army , allies thrust east to break siege in Deir al-Zor city')
('904825355441246209', 0.87165975556834907, 'Syrian army , allies thrust east to break siege in Deir al-Zor\xa0city')
('904849608634982400', 0.87165975556834907, 'Syrian army , allies thrust east to break siege in Deir al-Zor city ReutersUS')
('904855273227120640', 0.87140119627408275, 'Syrian army , allies thrust east to break siege in Deir al-Zor city -')
('904827139714936837', 0.87075625186120675, 'Geolocated to north Deir Ezzor. Syrian Air Force helicopters are clearly flying north of the Euphrates river, in zones SDF wants to take.')
('904856351410495488', 0.87066299035133099, 'Deir Ezzor governor: Jubilation in city over Syrian army ’s advance via SanaAjel')
('904852842174918656', 0.86967237784297846, 'The Ministry of Defence has reported about death in Syria of two Russian soldiers — Ramble...')
('904783477643599872', 0.86967096595259519, '"With a sudden lunge through jihadist lines, the Syrian army and its allies on Monday came to within 3 km of...')
('904792234880532480', 0.86929208469063524, '2 Russian soldiers killed in ISIS shelling in Deir ez-Zor province – Defense Ministry')
('904794813714878465', 0.86929208469063524, '2 Russian soldiers killed in ISIS shelling in Deir ez-Zor province – Defense Ministry')
('904821948290555904', 0.86929208469063524, '2 Russian soldiers killed in ISIS shelling in Deir ez-Zor province – Defense Ministry')
('904833418063519745', 0.86929208469063524, '2 Russian soldiers killed in ISIS shelling in Deir ez-Zor province – Defense Ministry')
('904838421788712960', 0.86929208469063524, '2 Russian soldiers killed in ISIS shelling in Deir ez-Zor province – Defense Ministry')
('904850846567059456', 0.86929208469063524, '2 Russian soldiers killed in ISIS shelling in Deir ez-Zor province – Defense Ministry')
('904854617800011776', 0.86929208469063524, '2 Russian soldiers killed in ISIS shelling in Deir ez-Zor province – Defense Ministry')
('904849158099673088', 0.86860854766788187, "Two Russian soldiers killed by shelling in Syria's Deir al-Zor province")
('904848587376386048', 0.86829024007181865, 'Syrian Arab Army, allies race to relieve Deir al-Zor from Islamic State (via NewsfusionApps ISISWatch )')
('904826538230677504', 0.86743225977790295, "Two Russian soldiers killed by shelling in Syria's Deir al-Zor province - Ifax")
('904851448533368832', 0.86743225977790295, "Two Russian soldiers killed by shelling in Syria's Deir al-Zor province - Ifax")
('904819361952694272', 0.86705696486064365, "Syria's army battles the Islamic State group on the edges of Deir Ezzor by sarahussein")
('904833096750653440', 0.86600987236802685, 'BREAKING: Two Russian soldiers killed by ISIS in Deir Ezzor province')
('904838197456318465', 0.86481091859638903, 'Syrian forces poised to punch through, end siege in Deir al-Zor, liberate 93,000 civilians')
('904838275730419712', 0.86481091859638903, 'Syrian forces poised to punch through, end siege in Deir al-Zor, liberate 93,000 civilians')
('904844106114977792', 0.86481091859638903, 'Syrian forces poised to punch through, end siege in Deir al-Zor, liberate 93,000 civilians')
('904850523358187520', 0.86481091859638903, 'Syrian forces poised to punch through, end siege in Deir al-Zor, liberate 93,000 civilians')
('904845311058817024', 0.86433035673842029, 'AustralianArmy soldiers from Task Group Taji 5 watch on as two Iraqi Army Mi-17 helicopters depart during training in Iraq CJTFOIR')
('904856566024740864', 0.86336200776837935, 'Army advances towards lifting siege on Deir Ezzor city, Russian Aerospace Forces carry out 80 sorties against ISIS')
('904829055018598400', 0.8625660421107556, 'I liked a YouTube video Syrian War Report – September 4, 2017: Syrian Army Reaches Deir Ezzor City')
('904848133208846336', 0.86242582119764766, 'RT XHNews : Syrian government forces supported by Russian warplanes are advancing towards ISIS -held city of Deir …')
('904801672987373570', 0.85911270065485124, '2 Russian soldiers killed in ISIS shelling in Deir ez-Zor province – Defence Ministry')
('904839904470978560', 0.85900711821318987, 'The meeting of Great Men! They will be included in Syrian History Deir ez-Zor about 2B relieved after being under Siege for over 3Years!')
('904750947674587136', 0.85899320913900445, 'RT: reuters : Syrian army , allies thrust east to break siege in Deir al-Zor city')
('904735447296958465', 0.85898738470276026, 'According to a US Embassy statement, some 3 ,500 Syrian wounded have been treated in Israeli hospitals')
('904851273115025408', 0.85764841550713289, 'BREAKING: Two Russian soldiers killed by ISIS in DeirEzzor province')
('904757104468262913', 0.85578266734906272, 'Reuters: Syrian army , allies thrust east to break siege in Deir al-Zor city: BEIRUT…')
('904723386118656003', 0.85471857566352594, 'Syrian army thrusts east to break siege of Deir al-Zor city')
('904726787778895872', 0.85471857566352594, 'Syrian army thrusts east to break siege of Deir al-Zor city')
('904740280674525186', 0.85471857566352594, 'Syrian army thrusts east to break siege of Deir al-Zor city')
('904730918350237697', 0.85338399217936789, 'Syrian army , allies 3 kilometers away from besieged Dayr al-Zawr enclave Iran IranPower News')
('904723885815451649', 0.85213742697690742, 'Syrian forces are rapidly approaching Deir Ezzor, aiming to oust the IS jihadists and break a years-long siege')
('904829190028967936', 0.85098477225443381, 'the Tiger forces Republican guard and the 4th Mechanized are all Syrian and they are the 1s doing the fighting in Deir Ezzor')
('904800026173464576', 0.85001097921151703, 'Syrian Army flattens Saudi,Israeli,USA Sponsored ISIS positions, will enter Deir ez-Zor ‘within 48 hours’ – governor')
('904851554418745344', 0.84943056219384017, 'Smdh.OF COURSE raids happened bc agreement betw Trump & Iraq . Iraq USED TO reject Iraq deportations back; but sure NOWthey will. *CHILLS 3 /')
('904826498384896004', 0.84855794062747369, 'Russian invader Valery Dziouba was one of two soviet soldiers killed by ISIS in eastern Hama .')
('904792079594913792', 0.84745804954145898, 'Syrian army at edge of IS-held Deir Ezzor: monitor')
('904830815737139200', 0.84665539964201497, 'Jordan , Iraq reopen main border crossing for first time in 3 years – World Tribune: Window on the Real World')
('904847338744807425', 0.84339259587066084, 'The Syrian Army Is on the Doorstep of Deir Ez-Zor')
('904854583679135745', 0.84313462532405969, 'Syria army close to retaking Deir ez-Zorm from Islamic State -')
('904827779308371968', 0.84206477358335052, '" Syrian Army , Russian Attack Helicopters Destroyed 18 ISIS VBIEDs During Monday Clashes With ISIS Near Deir Ezzor"')
('904835232565571584', 0.84165717658490791, 'Video from 3 years ago: Syrian tanks devastated many cities before US or Russian bombing')
('904827251031543808', 0.84157280761275299, '"2 Russian Military Servicemen Killed In Syria’s Deir Ezzor Province" @jimmypedya headlines news feedly')
('904855512566689793', 0.84131124329433205, 'No he just let the taliban take over 2/ 3 of Afghanistan and ISIS take over Iraq , your not real bright are you, sparky.')
('904789979154538496', 0.84098698082508028, 'Syrian army at edge of IS-held Deir Ezzor: monitor via Yahoo!')
('904746855745163264', 0.84095101355812518, 'Syrian army , allies thrust east to break siege in Deir al-Zor city worldnews')
('904843372816474114', 0.84072596554977763, "BREAKING Angela Merkel's party hit by Russian cyberattacks—elections in 3 weeks (via the funder ).")
('904829379003469825', 0.84070059457525104, 'Syrian army , allies thrust east to break siege in Deir al-Zor city ISIS Syria')
('904843397646647297', 0.83805185631980861, 'Syria Deir _ ez _ Zor offensive Russian RuAF raids tonight on two locations on right bank, NW of DeZ city. IvanSidorenko1 Liveuamap')
('904735840034836483', 0.83572705860034768, "Syrian army 'close to breaking' ISIL siege on parts of Deir Az Zor")
('904850365111304192', 0.83519606702595861, 'Two Russian soldiers killed by ISIS in DeirEzzor province Syria')
('904835655913562112', 0.83389631906334294, 'Syrian Army Drives Daesh Out of Key Positions on Palmyra- Deir ez-Zor Highway')
('904729802522861568', 0.83325418107550542, 'Syrian army , allies 3 kilometers away from besieged Dayr al-Zawr enclave Iran')
('904738398862065664', 0.83251225319347577, '"Syrian army , allies 3 kilometers away from besieged Dayr al-Zawr enclave" @jimmypedya headlines news feedly')
('904750250279276544', 0.83101382384022393, 'I saw this on the BBC and thought you should see it: IS conflict: Syrian army closes in on Deir al-Zour -')
('904854456470261761', 0.83101382384022393, 'I saw this on the BBC and thought you should see it: IS conflict: Syrian army closes in on Deir al-Zour -')
('904773922259906561', 0.83035611289480815, 'Syrian army , allies 3 kilometers away from besieged Dayr al-Zawr enclave luxuryhomebuild …')
('904734554300219392', 0.83015505914167476, 'Syrian army , allies 3 kilometers away from besieged Dayr al-Zawr enclave fakenews propaganda')
('904723335178846208', 0.82749166721056533, 'Syrian Army Nears Deir Ezzor in Offensive Against Islamic State via yahooNewsUK')
('904759534614347777', 0.82703698931689085, 'Syrian Army flattens ISIS positions, will enter Deir ez-Zor ‘within 48 hours’ – Governor')
('904790304875847680', 0.82703698931689085, 'Syrian Army flattens ISIS positions, will enter Deir ez-Zor ‘within 48 hours’ – governor')
('904845334567890948', 0.8263277889723033, ': Two Russian soldiers killed in east Syria')
('904760734445428736', 0.82618496043015177, '3 Vietnam vets reunite to bury one of their sons killed in Iraq in LastFlagFlying . More exclusive fall movie pics:')
('904825697608359938', 0.82606215795013294, 'A NSW mother is missing in Iraq after she went to rescue her two children via The Daily Telegraph Sarah Crawford')
('904781720578392065', 0.82401181538516932, 'U.S.-led coalition says Islamic State Syria convoy split in two')
('904797433451823104', 0.82207086020836662, 'Two Russian soldiers killed in east Syria..')
('904808972305051648', 0.82207082035144852, 'Two Russian soldiers killed in east Syria')
('904843181145120768', 0.82207082035144852, 'Two Russian soldiers killed in east Syria')
('904850277177716736', 0.82207082035144852, 'Two Russian soldiers killed in east Syria')
('904850675632402432', 0.82207082035144852, 'Two Russian soldiers killed in east Syria')
('904732469798326272', 0.82112295789467993, 'Syrian army , allies 3 kilometers away from besieged Dayr al-Zawr\xa0enclave')
('904773266602065920', 0.82112280752063849, 'Syrian army , allies 3 kilometers away from besieged Dayr al-Zawr enclave')
('904774633840414720', 0.82112280752063849, 'Syrian army , allies 3 kilometers away from besieged Dayr al-Zawr enclave')
('904828425302679557', 0.8206031479338648, 'eNCA | Two Russian soldiers killed in east Syria -')
('904844179062358017', 0.8199793412618448, 'Two wives fm E Europe. If I write this in a novel, both will be Russian plants to lead Don and kids down pro-Russ path to take-over.')
('904825039735865345', 0.81964699517219441, 'Sea Prayer virtual reality film commemorates 2nd anniversary of the death of 3 -year-old Syrian refugee Alan Kurdi')
('904739646269263872', 0.81931753687572995, 'RT CostinCercel " Syrian Army flattens ISIS positions, will enter Deir ez-Zor ‘within 48 hours’ Thanks to Russia "')
('904729641117736961', 0.81700813447132814, 'BBC News - IS conflict: Syrian army closes in on Deir al-Zour The games up 4 Daeshbags in Deir al-Zour.Hell Awaits!')
('904827437770567680', 0.81590394078771156, 'But fact of ISIS lineage belies your thesis: "Al-Qaeda" in Iraq was never really defeated if they morphed into ISIS and exploded in size 2/ 3')
('904836428676435968', 0.81448202120935842, "No, I didn't forget Iraq : 3 ,000 fellow warriors died there while another 16,000 returned with Purple Hearts. It now time for you to shut up.")
('904836604119867392', 0.81109917425158029, 'Syrian Army Races East, Trying to Stop Deir Ezzor Defenses Falling Syria SyrianArmy DeirEzzor ISIS Daesh')
('904820375598415872', 0.80674631981866018, 'A NSW mother is missing in Iraq after she went to rescue her two children via dailytelegraph SCrawfordNews')
('904847472752779268', 0.80501725952780767, 'Begging For War Suddenly I am John Wayne Awaiting my two hours of glory...Like Iraq')
('904847717129748482', 0.80501725952780767, 'Begging For War Suddenly I am John Wayne Awaiting my two hours of glory...Like Iraq')
('904838534753796096', 0.80326166089009332, 'Syrian Army advances within 3km of Deir Ezzor: video')
('904849459409977345', 0.8031655292672788, 'Syrian Army flattens ISIS positions, will enter Deir ez-Zor ‘within 48 hours’:')
('904833227390500864', 0.80300380682799688, "Kurds don't fight ISIS , they send ISIS to deir Ezzor so the syrian army can fight them.")
('904832543618412544', 0.80149861475642692, 'Syrian army to break 6-years long siege of Deir al-Zor city')
('904831279648165889', 0.80051617805915209, 'Two Russian servicemen killed in Syria')
('904831279828410368', 0.80051617805915209, 'Two Russian servicemen killed in Syria')
('904834989191184384', 0.80045275919406667, "1850 Ireland and 1900 (Volga/ Russian Germans) for 3 /4 of my family. I'd never even thought about its his way.")
('904741496246685697', 0.79983464556000761, 'ISIS on ahdingossa Syrian Forces Close to Seizing Oil-Rich Deir el-Zour from Islamic State')
('904847888278323201', 0.79962431647282961, 'PetroSA: Russian geological SEO Rosgeo throws national oil company a R5-billion lifeline of the two governments in…')
('904846558713307139', 0.79915758119013203, 'I would say neither. They changed their name to just " Islamic State" in fall of 2014 . Problem is by then everyone was already used to')
('904846354144362496', 0.79796305041794202, 'By that measure far right extremists are 3 times worse terrorists as Islamic extremists & they should be dealt with first! realDonaldTrump')
('904844876193439745', 0.79684782285872557, 'No, that means you support a racist bigot, with 5 children among 3 women, demeans women, and got elected thru Russian influence. We all lost')
('904826638231330816', 0.79594152503394955, 'That\'s it? Two links? Fake Fox News and "atr"? I can post 2 links too where Ocare is doing good. Aside from GOP & Russian Trump sabotaging.')
('904828807881703424', 0.79474838302485695, 'Syrian Army Pushing through Western Outskirt of Deir Ezzur City Iran Syria Iraq')
('904850903626379264', 0.79421881692860086, 'Three Going On In Russia: 1. Sanctions. 2. Serious business. 3 . Creativity. = Squirrels at play- Shot by a Russian photographer today.')
('904755178301259776', 0.79207912415013793, 'bbc - IS conflict: Syrian army closes in on Deir al-Zour. DeirEzzor UN US')
('904831447583870980', 0.79155792204490216, 'They read him like two dollars Russian Hoe that why there no trade deals... Russian investigation coming this Fall')
('904751225341706240', 0.7904832191927883, 'I saw this on the BBC and thought you should see it: Emotional moment Syrian family reunited after two years apart -')
('904743308337012737', 0.78953540226410412, 'Syrian Forces Close to Seizing Oil-Rich Deir el-Zour from Islamic State')
('904786971339436032', 0.78953540226410412, 'Syrian Forces Close to Seizing Oil-Rich Deir el-Zour from Islamic State')
('904824054980296704', 0.7891039205219571, 'Two killed after vintage Russian aircraft explodes into a fireball during an aerobatic routine - The Sun :')
('904737427125886979', 0.78817841181173742, 'BBC News - IS conflict: Syrian army closes in on Deir al-Zour')
('904740563605475328', 0.78817841181173742, 'BBC News - IS conflict: Syrian army closes in on Deir al-Zour')
('904847168988700674', 0.78817841181173742, 'BBC News - IS conflict: Syrian army closes in on Deir al-Zour')
('904830792580435968', 0.78650611910337942, 'Syrian army at edge of IS-held Deir Ezzor: monitor | | Iran')
('904845474796056576', 0.78602677018341782, 'Syrian army , allies race to relieve Deir Al-Zor')
('904849426988105728', 0.78597675870032868, 'Kemerovo I have bypassed two Kuzbass cities in the Russian rating of neighborliness')
('904730358633005057', 0.78197042652567905, 'IS conflict: Syrian army closes in on Deir al-Zour')
('904786222366806017', 0.78197042652567905, 'IS conflict: Syrian army closes in on Deir al-Zour')
('904802568366325760', 0.78197042652567905, 'IS conflict: Syrian army closes in on Deir al-Zour')
('904829197234954240', 0.78197042652567905, 'IS conflict: Syrian army closes in on Deir al-Zour')
('904838524423344129', 0.78197042652567905, 'IS conflict: Syrian army closes in on Deir al-Zour')
('904838556488577025', 0.78197042652567905, 'IS conflict: Syrian army closes in on Deir al-Zour')
('904843596406243328', 0.78197042652567905, 'IS conflict: Syrian army closes in on Deir al-Zour')
('904790591292268548', 0.7792896874203552, 'Syrian army , allies race to relieve Deir ezzor')
('904828775438979073', 0.77861849887870282, "I have two modes when drunk:flirty or crying. And there's no way to tell which is coming. It's the worst Russian roulette you'll ever play.")
('904825716725891072', 0.77745104333161041, 'Deir Ezzor: Syrian government enclave under IS siege')
('904853680767324160', 0.77616826198315969, 'Russian wheat exports up 11. 3 % on year at 5 mil mt, Ministry of Agriculture data shows. Full story:')
('904835882334769152', 0.77383723297142903, '1. Russian hacking. 2. Crosscheck and gerrymandering. 3 . GOP state led voter suppression. Hillary is our rightful POTUS. Illegitimate45')
('904831480525934593', 0.77313453291642242, 'An exquisite illustrated storybook film by Khaled Housseini inspired by the 3 yr old Syrian boy who, among so...')
('904848366135336963', 0.77217144835693863, "HumanCapitalFlight 2/ 3 of Russian oligarchs' children study abroad..assuming they study. #")
('904845446094483456', 0.77188595134541504, 'Daesh convoy in Syrian desert splits into two groups')
('904837350181744640', 0.76900000094528076, 'Some cruel joke swear, took 4 tests 3 negs one pos... (based on texts) some Russian roulette shit')
('904819933468643330', 0.76749673301700116, 'ICRC: RT UNICEF: Tears of joy. Syrian family reunited after two years apart BBCWorld')
('904723877682712576', 0.76629834676975583, 'BBC News - Emotional moment Syrian family reunited after two years apart')
('904817357716983809', 0.76555723599214742, 'RUSSIAN AWAY US seizes 3 Moscow posts after expelling envoys')
('904826453547765760', 0.76454940830442686, 'IS conflict: Syrian army closes in on Deir al-Zour [BBC]')
('904742109617557505', 0.75775472023209067, 'Vintage Russian aircraft somersaults through the sky before crashing into the ground and exploding leaving two dead')
('904762279144689664', 0.75775472023209067, 'Vintage Russian aircraft somersaults through the sky before crashing into the ground and exploding leaving two dead')
('904763305746612224', 0.75620938023665796, "Deir Az Zor : Syrian army 'close to breaking' ISIL siege AJENews")
('904745357334257664', 0.7562093733061992, "Deir Az Zor : Syrian army 'close to breaking' ISIL siege")
('904749183990505472', 0.7562093733061992, "Deir Az Zor : Syrian army 'close to breaking' ISIL siege")
('904753358258438150', 0.7562093733061992, "Deir Az Zor : Syrian army 'close to breaking' ISIL siege")
('904756751286886400', 0.7562093733061992, "Deir Az Zor : Syrian army 'close to breaking' ISIL siege")
('904780041866813440', 0.7562093733061992, "Deir Az Zor : Syrian army 'close to breaking' ISIL siege")
('904818896242221057', 0.7562093733061992, "Deir Az Zor : Syrian army 'close to breaking' ISIL siege")
('904839964415987714', 0.75221058219792469, "Anyone out there get Russian t v ? I don't even own a TV. Gave it up two years ago. I'll check online see if I can watch it. TrumpRussia")
('904829613389570049', 0.75055641124183603, 'We will be Islamic country soon')
('904737562123784193', 0.74903234234661664, '" Deir Az Zor : Syrian army \'close to breaking\' ISIL siege" Aljazeera')
('904751283407638528', 0.74864881593992816, 'Tears of joy. Syrian family reunited after two years apart BBCWorld')
('904855116565622784', 0.7377234292093704, 'Video: Syrian Army flattens ISIS: ’48 hours to enter Deir ez-Zor’')
('904766429311971328', 0.72789924253415783, 'Emotional moment Syrian family reunited after two years apart')
('904791255607119876', 0.72789924253415783, 'Emotional moment Syrian family reunited after two years apart')
('904814288602357762', 0.72789924253415783, 'Emotional moment Syrian family reunited after two years apart')
('904842644387495936', 0.72197593345299182, "Video: Syrian Army flattens ISIS: '48 hours to enter Deir ez-Zor'")
('904848254327717888', 0.72036575513139733, 'Deir Az Zor : Syrian army ‘close to breaking’ ISIL siege')
('904747022103740416', 0.71902321522330348, 'IS conflict: Syrian army closes in on Deir al-Zour IHL')
('904847730857652224', 0.71809263196196371, "Syrian Army flattens ISIS: '48 hours to enter Deir ez-Zor'")
('904825794102530048', 0.71455276817976332, 'Two Siamese cats discovered microphones hidden by Russian spies in Holland’s embassy in Moscow catfacts freethetigers meow')
('904724109828947969', 0.69780100287210145, 'Syrian army moves closer to ISIL -held Deir Az Zor')
('904789549443842049', 0.69342223331812425, 'RT euronews " Syrian forces advance on ISIL -besieged Deir al Zor "')
('904769300556267520', 0.69339640106174349, 'Syrian army reaches I.S. lines in Deir al-Zor')
('904830306863198208', 0.68871060907431914, 'Azerbaijan Armenia Russia Garabagh Two Russian servicemen killed in Syria')
('904760819141005312', 0.68049000367253065, 'Iraq 2014 | What We Left Behind via newyorker')
('904725774942568450', 0.67935396472324894, 'ISIS : Syrian army closes in on Deir al-Zour BlackIrishI BlissTabitha jayme0509 ramburner1 HawksEyeView_SI')
('904778721147715585', 0.67897482008913257, 'Syrian army reaches I.S. lines in Deir al-Zor news worldnews')
('904781814849564673', 0.67383731792083246, 'Syrian regime moves closer to ISIL -besieged Deir Az Zor ”')
('904842182376525824', 0.6435203534294105, 'Russian fighters bomb ISIS hardware near Deir Ez-Zor')
('904830424240857088', 0.61718056053339876, 'Yeeeees : 3 Trust me, Im Russian')
('904849336143577093', 0.55906687269312527, 'Two russian girls eating pussies')
('904852951675625472', 0.55070779840970918, 'two words: russian cursive')

In [97]:

    

def get_ents_set(doc):
    ents = [ent.text for ent in doc.ents]
    return set(ents)


    

In [42]:

    

def entity_coverage(doc_reference,doc_tweet):
    ents_reference = get_ents_set(doc_reference)
    ents_tweet = get_ents_set(doc_tweet)
    coverage_num = len(ents_reference & ents_tweet)
    jacard_similarity = coverage_num/len(ents_reference | ents_tweet)
    return coverage_num,jacard_similarity


    

In [60]:

    

def fuzzy_string_matching(text_reference,text_tweet):
    return fuzz.ratio(text_reference,text_tweet),fuzz.token_set_ratio(text_reference,text_tweet)


    

In [ ]: