In [1]:
from bs4 import BeautifulSoup

In [98]:
soup = BeautifulSoup(x.html(),'lxml')

In [99]:
tables = soup.find_all('table',attrs={'class':'vevent'})

In [144]:
t = tables[4]

In [145]:
pdiv = t.find_previous_sibling()

In [146]:
date = pdiv.get('id')

In [147]:
date


Out[147]:
'2017_September_05'

In [148]:
td = t.find('td',attrs={'class':'description'})

In [181]:
for type_,ul in zip([dl.get_text() for dl in td.find_all('dl')],[ul for ul in td.find_all('ul',recursive=False)]):
    for li in ul.find_all('li',recursive=False):
        print (date,type_,li.get_text())


('2017_September_05', u'Armed conflicts and attacks', u'Syrian Civil War\nSiege of Deir ez-Zor (2014\u201317)\nThe Syrian Army lifts the 28-month-long siege of Deir ez-Zor by the Islamic State of Iraq and the Levant. (Reuters) (BBC News)')
('2017_September_05', u'Armed conflicts and attacks', u'2016\u201317 Rohingya persecution in Myanmar\nMore than 123,000 Rohingya refugees have fled Myanmar and crossed into Bangladesh due to escalating violence by the Myanmar Army. (BBC News)\nBurmese State Counselor Aung San Suu Kyi receives international criticism and pressure urging her to help the Rohingya. (Reuters)')
('2017_September_05', u'Disasters and accidents', u'2017 Atlantic hurricane season\nHurricane Irma\nHurricane Irma strengthens to a maximum Category 5 hurricane, becoming the strongest Atlantic hurricane since 2005\'s Hurricane Wilma in terms of maximum sustained winds, described as "extremely dangerous." The National Hurricane Center (NHC) predicts that Irma could strengthen even more due to favorable conditions. (Washington Post)\nThe NHC says Irma is the most powerful Atlantic storm ever recorded outside the Caribbean Sea and the Gulf of Mexico. Ahead of Irma\'s landing,  President Donald Trump issues a state of emergency declaration for Florida, Puerto Rico, and the U.S. Virgin Islands.  (Reuters) (The Washington Examiner)')
('2017_September_05', u'International relations', u'India\u2013Myanmar relations\nIndian Prime Minister Narendra Modi heads to Myanmar for a state visit. (The Hindu)')
('2017_September_05', u'International relations', u'2017 North Korea crisis\nJapan\u2013United States relations, South Korea\u2013United States relations\nAmid tensions from South Korea, U.S. President Donald Trump announces that he is "allowing Japan and South Korea to buy a substantially increased amount of highly sophisticated military equipment from the United States." (The Independent)')
('2017_September_05', u'Law and crime', u'Journalists killed in India\nProminent Indian journalist and Lankesh Patrike editor Gauri Lankesh is shot dead by unidentified men outside her house in the city of Bengaluru. (Reuters)')
('2017_September_05', u'Law and crime', u'Australian Marriage Law Postal Survey (Same-sex marriage)\nThe High Court of Australia receives a complaint against the Australian Marriage Law Postal Survey citing the postal survey as "unique and offensive." (The Guardian)')
('2017_September_05', u'Politics and elections', u'Politics of Taiwan\nTainan Mayor William Lai Ching-te is appointed as the new Premier by the President of the Republic of China Tsai Ing-wen. (AP)')
('2017_September_05', u'Politics and elections', u'Deferred Action for Childhood Arrivals\nThe Trump Administration announces that, during the next six months, it will be ending the DACA program that has halted the deportation of about 800,000 people who were brought to the U.S. illegally as children.  Congress is called upon to pass legislation to correct the situation. (The New York Times)')
('2017_September_05', u'Sports', u'2017\u201318 NBA season\nBillionaire investor Tilman Fertitta buys the Houston Rockets NBA basketball team for US$2.2 billion. (Houston Chronicle) (ESPN)')

In [2]:
months = ["January","February","March","April","May","June","July","August","September","October","November","December"]

In [ ]:


In [3]:
import re

In [4]:
import requests
headers = {'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.81 Safari/537.36'}

In [5]:
import pdb

In [6]:
def trans_date(date_str):
    month = date_str.split('_')[1]
    month_num = months.index(month)+1
    date_str = date_str.replace(month,'0'+str(month_num) if month_num<10 else str(month_num))
    date_str = date_str.replace('_','-')
    return date_str

In [7]:
trans_date('2017_October_01')


Out[7]:
'2017-10-01'

In [8]:
def get_events_from_page(page_url):
    res = requests.get(page_url,headers=headers)
    soup = BeautifulSoup(res.text,'lxml')
    #pdb.set_trace()
    tables = soup.find_all('table',attrs={'class':'vevent'})
    events = []
    for t in tables:
        date = t.find_previous_sibling().get('id')
        td = t.find('td',attrs={'class':'description'})
        for type_,ul in zip([dl.get_text() for dl in td.find_all('dl',recursive=False)],[ul for ul in td.find_all('ul',recursive=False)]):
            for li in ul.find_all('li',recursive=False):
                #pdb.set_trace()
                try:
                    events.append({'date':trans_date(date),
                                   'class':type_.strip(),
                                   'title':li.a.get_text(),
                                   'description':li.ul.get_text().strip()})
                except:
                    pass
    return events

In [268]:
a = get_events_from_page('https://en.wikipedia.org/wiki/Portal:Current_events/July_2016')

In [269]:
len(a)


Out[269]:
233

In [270]:
a


Out[270]:
[{'class': u'Armed conflicts and attacks',
  'date': '2016-07-01',
  'description': u'Gunmen storm a restaurant, exchange fire with security, and take at least twenty hostages in the Gulshan neighborhood of Dhaka, Bangladesh. Two policemen are killed. The Islamic State claims responsibility. (ABC News) (CNN) (USA Today)',
  'title': u'2016 Gulshan attack'},
 {'class': u'Armed conflicts and attacks',
  'date': '2016-07-01',
  'description': u'At least six people are shot and killed and several people wounded by suspected Al-Shabaab militants who ambushed two buses in Mandera, Kenya, near the Somalian border. (Al Jazeera) (AP)',
  'title': u'War in Somalia'},
 {'class': u'Armed conflicts and attacks',
  'date': '2016-07-01',
  'description': u"A Palestinian woman, in what was construed to be an attempt to stab Israeli Border Police officers at the entrance to the Tomb of the Patriarchs in Hebron, is shot and later dies. (PNN)(The Times of Israel)\nIn response to violence in the Hebron area, including the killing yesterday of a 13-year-old Jewish girl asleep in her home in the Kiryat Arba settlement by a Palestinian assailant, the Israel Defense Forces places a closure on Hebron, brings additional troops to the area and announces a cut to tax funds it collects for the Palestinian Authority, saying it would deduct the amount the Palestinian leadership pays to the families of militants. (Al Jazeera) (The Times of Israel)\nA 63 year old Palestinian man dies during a clash at the Qalandiya checkpoint; Palestinian sources claim his death was due to tear gas inhalation while Israeli sources say the man suffered a fatal heart attack. (Ma'an) (The Times of Israel)\nRabbi Michael Mark is murdered and his two children badly injured when terrorists shoot at his car on the highway south of Hebron.(The Jerusalem Post) (The Times of Israel)\nPalestinian militants fire two rockets into Israel from the Gaza Strip; one explodes outside a preschool in the town of Sderot, causing damage to property but no injuries, and the second rocket exploded in an open area in the Sha'ar Hanegev region. (The Times of Israel)",
  'title': u'Israeli\u2013Palestinian conflict'},
 {'class': u'International relations',
  'date': '2016-07-01',
  'description': u"The European Union extends economic sanctions on Russia until 31 January 2017 over Moscow's continued support for separatist rebels in eastern Ukraine. (Reuters)",
  'title': u'Russia\u2013European Union relations'},
 {'class': u'International relations',
  'date': '2016-07-01',
  'description': u'The report of the diplomatic Quartet to the Security Council \u2014 the United Nations, Russia, the United States, and the European Union \u2014 calls on Israel to end settlement construction and expansion policy, and calls on Palestinians to act decisively to stop incitement to violence and to clearly condemn terrorist acts. Special Coordinator for the Middle East Peace Process Nickolay Mladenov tells the UNSC, \u201cThe main objective of this report is not about assigning blame ... (rather) it focuses on the major threats to achieving a negotiated peace." (AP) (UN News Centre) (The Times of Israel)',
  'title': u'Israel-Palestine relations'},
 {'class': u'International relations',
  'date': '2016-07-01',
  'description': u"Following his meeting at Finnish President Sauli Niinist\xf6's residence, Russian President Vladimir Putin says that Russia would respect Finland's choice whether to join NATO, though as part of NATO's military infrastructure, Finnish Defence Forces would overnight be at the borders of the Russian Federation. Putin and Niinist\xf6 call for improved security in Baltic airspace where Russian and Western aircraft have had close encounters in recent months. (Oneindia) (Reuters)",
  'title': u'Finland\u2013Russia relations'},
 {'class': u'Law and crime',
  'date': '2016-07-01',
  'description': u'Philippines President Rodrigo Duterte urges Filipino citizens to murder people suspected of using illegal drugs. (The Guardian)',
  'title': u'Crime in the Philippines'},
 {'class': u'Politics and elections',
  'date': '2016-07-01',
  'description': u'Austria\u2019s highest court orders a repeat of the presidential elections narrowly lost in May by right-wing populist Freedom Party of Austria candidate Norbert Hofer. The margin of votes was less than 1%. (The Telegraph)',
  'title': u'Austrian presidential election, 2016'},
 {'class': u'Politics and elections',
  'date': '2016-07-01',
  'description': u"Thousands of people take to the streets of Hong Kong, demanding answers over China's alleged abduction of five booksellers in late 2015 and that Chief Executive Leung Chun-ying resign from office. (The Guardian)",
  'title': u'Causeway Bay Books disappearances'},
 {'class': u'Armed conflicts and attacks',
  'date': '2016-07-02',
  'description': u'13 hostages were rescued following a standoff with the Rapid Action Battalion. (Reuters)\nTwenty hostages and two policemen were killed, and six gunmen are dead with one captured alive. (CNN)',
  'title': u'2016 Gulshan attack'},
 {'class': u'Armed conflicts and attacks',
  'date': '2016-07-02',
  'description': u'At least two people are killed and dozens more are wounded in a suicide bombing in Jalalabad, Nangarhar Province. (Radio Free Europe/Radio Liberty)',
  'title': u'War in Afghanistan'},
 {'class': u'Armed conflicts and attacks',
  'date': '2016-07-02',
  'description': u'According to The Pentagon, two senior ISIL leaders have been killed by a United States-led coalition airstrike in Mosul. (The Guardian) (BBC)',
  'title': u'Iraqi Civil War'},
 {'class': u'Armed conflicts and attacks',
  'date': '2016-07-02',
  'description': u'Syrian government airstrikes in Damascus kill at least 30 people. According to the rebels, the airstrikes were revenge for the killing of a Syrian Air Force pilot by Al-Nusra Front yesterday. The airstrikes reportedly targeted a medical centre and a school. (The Guardian)',
  'title': u'Syrian Civil War'},
 {'class': u'Armed conflicts and attacks',
  'date': '2016-07-02',
  'description': u'Turkish police arrest a total of 41 people in connection with the Istanbul Atat\xfcrk Airport terrorist attack. (UPI.com)',
  'title': u'2016 Istanbul airport attack'},
 {'class': u'Politics and elections',
  'date': '2016-07-02',
  'description': u'Australians vote to elect a government for the next three years. Prime Minister Malcolm Turnbull, leader of the conservative Liberal Party-led Coalition, dissolved both houses of Parliament in May in a bid for a Senate more friendly to his agenda. The final result showed the Coalition with a bare 1 seat majority in the House with 76 seats and the Australian Labor Party with 69 seats. (Reuters) (Sydney Morning Herald)',
  'title': u'Australian federal election, 2016'},
 {'class': u'Politics and elections',
  'date': '2016-07-02',
  'description': u'In pro-EU rallies, thousands of people protest in London against the United Kingdom withdrawal from the European Union and ask that Article 50 not be invoked. (The Guardian)',
  'title': u'Aftermath of the United Kingdom European Union membership referendum, 2016'},
 {'class': u'Armed conflicts and attacks',
  'date': '2016-07-03',
  'description': u'Twin bombings in a pair of crowded commercial areas in Baghdad kill at least 125 people. ISIL claims responsibility. (AP) (The Independent) (BBC)',
  'title': u'Iraqi Civil War'},
 {'class': u'International relations',
  'date': '2016-07-03',
  'description': u'The European Union warns Switzerland that it will lose access to its single market if it goes through with its plan to impose restrictions on the free movement of EU citizens. In a referendum held back in February 9, 2014, Swiss voters approved limiting the movement of economic migrants from the EU in the country. Current laws allow migrants to enter Switzerland without visas. (The Guardian)',
  'title': u'Switzerland\u2013European Union relations'},
 {'class': u'Law and crime',
  'date': '2016-07-03',
  'description': u'Ten people are killed by the Philippine National Police within a week after Rodrigo Duterte took office as president and made statements encouraging the murder of illegal drug addicts. (Channel News Asia)',
  'title': u'Human rights in the Philippines'},
 {'class': u'Politics and elections',
  'date': '2016-07-03',
  'description': u'Former French President Nicolas Sarkozy resigns as leader of The Republicans party and announces his candidacy for President again. (The Guardian)',
  'title': u'French presidential election, 2017'},
 {'class': u'Sport',
  'date': '2016-07-03',
  'description': u'Lewis Hamilton wins the Austrian Grand Prix at the Red Bull Ring in Spielberg, Austria. (BBC)',
  'title': u'2016 Formula One season'},
 {'class': u'Sport',
  'date': '2016-07-03',
  'description': u'The Russian Olympic Committee appeals the ban of its track and field team from participating in the upcoming summer Olympics by the International Association of Athletics Federations (IAAF), because of large scale doping, with the Court of Arbitration for Sport. A hearing on the case will take place on July 19. (Radio Free Europe/Radio Liberty)',
  'title': u'2016 Summer Olympics'},
 {'class': u'Armed conflicts and attacks',
  'date': '2016-07-04',
  'description': u'Malaysia confirms it has suffered its first bombing by the Islamic State of Iraq and the Levant during the previous week raising fears about the spread of influence of the militant group. (The New York Times)',
  'title': u'Terrorism in Malaysia'},
 {'class': u'Armed conflicts and attacks',
  'date': '2016-07-04',
  'description': u'The Communist Party of the Philippines agrees to fight drug lords after its legal wing Bagong Alyansang Makabayan joins the government cabinet of Rodrigo Duterte. (GMA News) (The Philippine Star)',
  'title': u'CPP\u2013NPA\u2013NDF rebellion'},
 {'class': u'Armed conflicts and attacks',
  'date': '2016-07-04',
  'description': u"Israel's military attacks two Syrian Army targets in Syrian territory from the Golan Heights after a fire from the fighting damages a security fence on the border. (Reuters)",
  'title': u'Syrian Civil War'},
 {'class': u'Armed conflicts and attacks',
  'date': '2016-07-04',
  'description': u"A suspected suicide bomber detonates his belt near the U.S. consulate in Jeddah, killing himself and wounding two guards. (BBC)\nA suicide bomber detonates a device near the security headquarters of the Prophet's Mosque in Medina, Saudi Arabia, killing four security guards. (Daily Mail) (BBC) (Pakistan Today)",
  'title': u'2016 Saudi Arabia bombings'},
 {'class': u'Armed conflicts and attacks',
  'date': '2016-07-04',
  'description': u'The Baghdad Provincial Council deputy head reports the death toll has passed 200, with at least 225 more injured. Iraqi Prime Minister Haider al-Abadi, visiting the Kerrada site, is jeered by onlookers who blame his government for not stopping ISIL-related bloodshed. (UPI)',
  'title': u'July 2016 Baghdad bombings'},
 {'class': u'Business and economics',
  'date': '2016-07-04',
  'description': u'Standard Life Investments suspends trading in its UK property fund, in response to increased withdrawal calls since the vote, to protect the interests of all investors in the fund. (BBC)',
  'title': u'United Kingdom withdrawal from the European Union'},
 {'class': u'International relations',
  'date': '2016-07-04',
  'description': u'A Turkish ship carrying humanitarian aid for the Gaza Strip docks at the Israeli port city of Ashdod, the first warming move between the two countries since relations began to improve. Relations between Turkey and Israel were tense since the 2010 Gaza flotilla raid but ended last week when Turkey agreed to normalize ties in a diplomatic treaty. (The Guardian)',
  'title': u'Blockade of the Gaza Strip'},
 {'class': u'International relations',
  'date': '2016-07-04',
  'description': u"Turkish Foreign Minister Mevl\xfct \xc7avu\u015fo\u011flu says Turkey's cooperation with Russia in combating the Islamic State in Syria does not include Russia's use of its Incirlik Air Base, which some have interpreted from his comments yesterday. (Reuters)",
  'title': u'Russia\u2013Turkey relations'},
 {'class': u'Politics and elections',
  'date': '2016-07-04',
  'description': u'Nigel Farage resigns as leader of the UK Independence Party (UKIP) but says he and other MEPs/MPs of the party would continue to carry out their term in the European Parliament. (BBC via Radio New Zealand)',
  'title': u'United Kingdom withdrawal from the European Union'},
 {'class': u'Politics and elections',
  'date': '2016-07-04',
  'description': u'The election deadlock continues as neither Australian Prime Minister Malcolm Turnbull nor opposition leader Bill Shorten has the minimum number of Lower House seats (76) needed to form a government. Shorten leads Turnbull, 71-67, with only six seats undecided. (UPI) (Newstalk ZB)',
  'title': u'Australian federal election, 2016'},
 {'class': u'Politics and elections',
  'date': '2016-07-04',
  'description': u"Zhejiang governor Li Qiang is transferred to neighbouring Jiangsu province to serve as party chief. He is succeeded as Zhejiang governor by Xinjiang deputy party chief Che Jun. (Sina)\nXian Hui is named acting Chairwoman of Ningxia Hui Autonomous Region, succeeding Liu Hui. It is the first time in the history of the People's Republic that a woman has succeeded another woman in a provincial government leadership position. (Sina)",
  'title': u'Communist Party of China'},
 {'class': u'Armed conflicts and attacks',
  'date': '2016-07-05',
  'description': u'Iraqi forces, backed by U.S. airstrikes, advance north towards ISIL-held Qayyarah Airfield West south of Mosul. (Reuters)\nThe United Nations reports a mortar attack on a camp for displaced Iraqis south of Baghdad kills three people, including two children, and injures 13 others, two critically. (The New York Times)',
  'title': u'Iraqi Civil War'},
 {'class': u'Armed conflicts and attacks',
  'date': '2016-07-05',
  'description': u'A suicide bomber attacks a police station in Solo, Indonesia, killing himself and injuring a police officer. Solo is the birthplace of President Joko Widodo. (Time) (Reuters)',
  'title': u'Terrorism in Indonesia'},
 {'class': u'Armed conflicts and attacks',
  'date': '2016-07-05',
  'description': u'Suspected Allied Democratic Forces rebels hack nine people to death in the village of Oicha, Democratic Republic of the Congo. (Reuters)',
  'title': u'Allied Democratic Forces insurgency'},
 {'class': u'Armed conflicts and attacks',
  'date': '2016-07-05',
  'description': u'A suicide bomb explosion in Kurdish-controlled Al-Hasakah in northeastern Syria kills at least 16 people, including three children and two women. ISIL claims responsibility. (Reuters)',
  'title': u'Syrian civil war'},
 {'class': u'Business and economics',
  'date': '2016-07-05',
  'description': u'The Central Bank of Russia announces it will start buying the Chinese yuan in an attempt to diversify its foreign-currency reserves away from the Euro and the U.S. dollar and to become less vulnerable to Western sanctions. (Radio Free Europe/Radio Liberty)',
  'title': u'International sanctions during the Ukrainian crisis'},
 {'class': u'Law and crime',
  'date': '2016-07-05',
  'description': u'Dozens of petty criminals and alleged drug addicts die in the Philippines within a few days after Rodrigo Duterte took the presidency. He actively encouraged extrajudicial killing of drug addicts and petty criminals before and after the election. (Al Jazeera) (The Guardian)',
  'title': u'Human rights in the Philippines'},
 {'class': u'Politics and elections',
  'date': '2016-07-05',
  'description': u'Hungarian Prime Minister Viktor Orb\xe1n announces that the country will hold a referendum on October 2 where voters will get to choose whether or not to accept European Union-imposed quotas on the distribution of migrants. (BBC)',
  'title': u'European migrant crisis'},
 {'class': u'Politics and elections',
  'date': '2016-07-05',
  'description': u'The FBI recommends, with numerous reservations about her actions, no charges be filed against Hillary Clinton over her emails. (BBC) (The New York Times)',
  'title': u'Hillary Clinton email investigation'},
 {'class': u'Politics and elections',
  'date': '2016-07-05',
  'description': u'Iraqi Interior Minister Mohammed Salem Al-Ghabban says he has submitted his resignation. Al-Ghabban cited a lack of "coordination among security systems" demonstrated by Sunday\'s bombings in Baghdad that killed over 200 people and injured at least 175 others. No report yet that Prime Minister Haider al-Abadi has accepted this resignation. (CNN) (The New York Times)',
  'title': u'July 2016 Baghdad bombings'},
 {'class': u'Politics and elections',
  'date': '2016-07-05',
  'description': u'Home Secretary Theresa May gets 165 votes after the first ballot of Conservative members of parliament to select a new Leader and the next Prime Minister. Her nearest rival, Minister of State for Energy Andrea Leadsom, picked up 66 votes. Secretary of State for Justice Michael Gove won 48 votes, and Secretary of State for Work and Pensions Stephen Crabb got 34 votes. Former Secretary of State for Defence Liam Fox is eliminated from consideration as he received just 16. Later, Crabb pulled out of the race and endorsed front-runner Theresa May. Fox also endorsed May. (Reuters) (Sky News) (Reuters\xb2)',
  'title': u'Conservative Party (UK) leadership election, 2016'},
 {'class': u'Armed conflicts and attacks',
  'date': '2016-07-06',
  'description': u"Twenty Philippine Army soldiers are under investigation after they are suspected of mutilating the body of a New People's Army member. The communist militant's eyes were gouged, his tongue was cut off and his throat was slit. (GMA News)",
  'title': u'CPP-NPA-NDF rebellion'},
 {'class': u'Armed conflicts and attacks',
  'date': '2016-07-06',
  'description': u'A suicide car bomb, that targeted the Central Security headquarters in the port city of Aden, Yemen, kills at least 26 people. Following the blast, militants stormed the compound and clashed with soldiers. No group immediately claimed responsibility but it is similar to an Islamic State in Iraq and the Levant attack on the port of Al Mukalla. (AP via The New York Times) (Reuters)',
  'title': u'Yemeni Civil War'},
 {'class': u'Business and economy',
  'date': '2016-07-06',
  'description': u"Standard & Poor's drops Australia's debt outlook from stable to negative following the country's surprisingly close election this weekend. (The Guardian)",
  'title': u'Australian federal election, 2016'},
 {'class': u'International relations',
  'date': '2016-07-06',
  'description': u'China threatens the United States regarding the South China Sea and it will not accept a verdict in the Permanent Court of Arbitration. (ABC News Go) (Forbes)',
  'title': u'International reactions to Philippines v. China'},
 {'class': u'Law and crime',
  'date': '2016-07-06',
  'description': u'South African athlete Oscar Pistorius is sentenced to six years imprisonment for the murder of Reeva Steenkamp. (CNN)',
  'title': u'Oscar Pistorius trial'},
 {'class': u'Politics and elections',
  'date': '2016-07-06',
  'description': u'A report about the United Kingdom\'s involvement in the Iraq War is released. The chairman of the inquiry, Sir John Chilcot, said that the invasion was not a "last resort" and was based on "flawed intelligence and assessments". The inquiry was announced on 15 June 2009 by then-Prime Minister Gordon Brown. The final hearing was held on 2 February 2011. (The Guardian) (BBC) (BBC\xb2)',
  'title': u'Iraq Inquiry'},
 {'class': u'Politics and elections',
  'date': '2016-07-06',
  'description': u"Loretta Lynch, the Attorney General of the United States, closes the investigation into Hillary Clinton's use of a personal email system to conduct official business without charges being laid. (CNN)",
  'title': u'Hillary Clinton email controversy'},
 {'class': u'Politics and elections',
  'date': '2016-07-06',
  'description': u'National "Stay Away" protests, organized over the Internet via the WhatsApp social messaging platform, take place in Zimbabwe, following fears of an economic collapse amid calls for President Robert Mugabe\'s resignation.(BBC News)',
  'title': u'2016 Zimbabwe protests'},
 {'class': u'Armed conflicts and attacks',
  'date': '2016-07-07',
  'description': u'A group of radical Islamists hurl homemade bombs at police officers in the Kishoreganj District in central Bangladesh killing at least one officer and injuring several others. (AP via ABC News)',
  'title': u'Terrorism in Bangladesh'},
 {'class': u'Armed conflicts and attacks',
  'date': '2016-07-07',
  'description': u'The Eastern Bloc of the FARC-EP, a unit of the FARC rebel group, announces its rejection of the recent ceasefire signed by its leader Timole\xf3n Jim\xe9nez and the Colombian government of President Juan Manuel Santos, declaring it will not lay down its arms and demobilize and will continue to fight the government. (Reuters via The Guardian)',
  'title': u'Colombian conflict'},
 {'class': u'Armed conflicts and attacks',
  'date': '2016-07-07',
  'description': u'At least 35 people are killed and 60 others injured in an attack on a Shiite mausoleum located in north of Baghdad. Islamic State in Iraq and the Levant have claimed responsibility. (Fox News), (Reuters)',
  'title': u'Iraqi Civil War (2014\u2013present)'},
 {'class': u'Disasters and accidents',
  'date': '2016-07-07',
  'description': u'Taiwan and China brace for the impact of Typhoon Nepartak, the first major typhoon of 2016. Thousands of people have been evacuated in Taiwan while over 35,000 military personnel have been placed on standby to help with relief efforts. (CNN) (ABC News)\nTyphoon Nepartak is expected to make landfall on mainland China on Friday and will make flooding worse. Nearly 200 people have died in flood waters in China in the past week with 41 people missing, 1.6 million relocated and almost 50,000 houses collapsed. (The Telegraph)',
  'title': u'2016 Pacific typhoon season'},
 {'class': u'International relations',
  'date': '2016-07-07',
  'description': u'U.S. Secretary of State John Kerry arrives in Kiev, Ukraine, where he is expected to discuss ways of ending the conflict in the east of the country. The visit, along with his visit to Georgia yesterday, comes ahead of the NATO summit in Warsaw that will take place tomorrow. (Radio Free Europe/Radio Liberty)',
  'title': u'Ukraine\u2013United States relations'},
 {'class': u'International relations',
  'date': '2016-07-07',
  'description': u'The United States tentatively approves eight, of 12 that applied, airlines for flights from 10 U.S. cities to Havana as early as this fall: Alaska Airlines, American Airlines, Delta Air Lines, Frontier Airlines, JetBlue Airways, Southwest Airlines, Spirit Airlines, and United Airlines. Last month, flights were approved from five U.S. cities to nine other international airports. (Reuters) (US DOT)',
  'title': u'Cuba\u2013United States relations'},
 {'class': u'Law and crime',
  'date': '2016-07-07',
  'description': u'The Philippines House of Representatives seeks to investigate the rise in deaths of suspected drug dealers in the country. (GMA News)',
  'title': u'Human rights in the Philippines'},
 {'class': u'Law and crime',
  'date': '2016-07-07',
  'description': u'Germany\'s parliament passes a new law saying that it is rape to have sex with a person who says "No" to the sex. Under the previous law, sex was not considered rape unless the victim fought back. The new law also classifies groping as a sex crime, makes it easier to deport migrants who commit sex offences, and makes it easier to prosecute assaults committed by a large group. (BBC)',
  'title': u'Rape in Germany'},
 {'class': u'Law and crime',
  'date': '2016-07-07',
  'description': u'In a so-called VatiLeaks case, a Vatican City Court dismisses charges of publishing confidential information against two Italian journalists stating it lacked jurisdiction in this case. Emiliano Fittipaldi and Gianluigi Nuzzi wrote books on the inner workings of the Vatican. The court did convict a Vatican priest to 18 months, and assessed a 10-month suspended sentence on an Italian communications expert, for conspiring to pass documents to the journalists; a fifth defendant was cleared of all charges. (AP) (The Guardian) (Catholic News)',
  'title': u'Vatican leaks scandal'},
 {'class': u'Law and crime',
  'date': '2016-07-07',
  'description': u'Minnesota Governor Mark Dayton requests a Justice Department investigation of last night\'s fatal police shooting of Philando Castile (an African-American) during a traffic stop. Castile\'s girlfriend, Lavish Reynolds, streamed the incident live over Facebook. It showed Castile had been shot several times and was slumped against Reynolds; he died in Minneapolis\'s Hennepin County Medical Center. (The New York Times) (CBS News)\nGov. Dayton says, "Would this have happened if those passengers, the driver and the passengers, were white? I don\'t think it would have. So I\'m forced to confront, and I think all of us in Minnesota are forced to confront, that this kind of racism exists." (NPR) (Wall Street Journal)',
  'title': u'Shooting of Philando Castile'},
 {'class': u'Law and crime',
  'date': '2016-07-07',
  'description': u'Eleven police officers are shot, with five dying of gunshot wounds in Dallas, Texas during a protest against two black men killed within a day of each other. Police kill the shooter with a bomb delivered on a bomb defusing robot. (New York Post)(KTLA)(NBC DFW), (ABC News Australia)',
  'title': u'2016 shooting of Dallas police officers'},
 {'class': u'Politics and elections',
  'date': '2016-07-07',
  'description': u'Zimbabwean authorities arrest dozens of protesters as anti-government protests spread across the country. (Al Jazeera)',
  'title': u'2016 Zimbabwe protests'},
 {'class': u'Politics and elections',
  'date': '2016-07-07',
  'description': u"Australian Prime Minister Malcolm Turnbull's Liberal/National coalition, behind Bill Shorten's Labor Party in the first 48 hours following Saturday's election, is now ahead of Labor in the Lower House, 74-71 seats; 76 seats are needed to form a government. Minor parties and independents have won five seats; mail-in and absentee votes are still being counted. Turnbull is on the road today seeking support from a small handful of independent and small party lawmakers. (Reuters) (The Australian) (Daily Mail)",
  'title': u'Australian federal election, 2016'},
 {'class': u'Politics and elections',
  'date': '2016-07-07',
  'description': u'Conservative Party members of parliament vote in a second ballot to choose the next leader of the party to replace outgoing Prime Minister David Cameron. The Home Secretary Theresa May secured the highest number votes with 199, while Department of Energy and Climate Change minister Andrea Leadsom received the second highest amount, with 84. The Secretary of State for Justice Michael Gove received just 46 votes and is eliminated from the race, meaning a run-off contest will now take place between May and Leadsom, with the result to be announced on September 9. (AP via Daily Mail) (BBC)',
  'title': u'Conservative Party (UK) leadership election, 2016'},
 {'class': u'Politics and elections',
  'date': '2016-07-07',
  'description': u'The United States Department of State reopens an investigation into the possible mishandling of classified information by former Secretary of State and presumptive Democratic Party presidential nominee Hillary Clinton and her top aides. (AP via Seattle Times)',
  'title': u'Hillary Clinton email scandal'},
 {'class': u'Armed conflicts and attacks',
  'date': '2016-07-08',
  'description': u'At least eight unidentified militants and one security officer are killed in ongoing clashes in rural Dagestan, Russia. (Radio Free Europe/Radio Liberty)',
  'title': u'Insurgency in the North Caucasus'},
 {'class': u'Armed conflicts and attacks',
  'date': '2016-07-08',
  'description': u'According to a Nigerian Army spokesman, a Boko Haram suicide bomber kills six people inside a mosque in the town of Damboa, Borno State. A second suicide bomber attempted to enter the same mosque but failed to gain entry and detonated his belt, killing only himself. (Reuters)',
  'title': u'Boko Haram insurgency'},
 {'class': u'Armed conflicts and attacks',
  'date': '2016-07-08',
  'description': u'Airstrikes hit Darkush in Idlib Province near the Turkish border, killing at least 23 people, according to the Syrian Observatory for Human Rights. At least 25 died when rebels shelled government-held areas of Aleppo. Friday is the last day of a 72-hour Syrian Army-sponsored ceasefire. (Reuters) (Reuters\xb2)',
  'title': u'Syrian Civil War'},
 {'class': u'Armed conflicts and attacks',
  'date': '2016-07-08',
  'description': u"Five soldiers between the two rival Sudan People's Liberation Army factions are killed in clashes in the capital Juba, with reports of heavy gunfire continuing, despite President Salva Kiir Mayardit and rebel leader Riek Machar having reached a peace deal back in April. (BBC)",
  'title': u'South Sudanese Civil War'},
 {'class': u'Disasters and accidents',
  'date': '2016-07-08',
  'description': u'Typhoon Nepartak hits eastern Taiwan causing three deaths so far, 124 injuries, thousands of people to be evacuated, disrupting transport and power supplies. (Reuters) (CNN)',
  'title': u'2016 Pacific typhoon season'},
 {'class': u'Health and medicine',
  'date': '2016-07-08',
  'description': u'The death of an elderly Utah woman with a Zika virus infection in late June is the first reported Zika-related death in the continental United States. The woman had traveled to an area where Zika is spreading. The exact cause of death has not been determined; the lady had an underlying medical condition. (NBC News) (Time)',
  'title': u'Zika virus epidemic'},
 {'class': u'International relations',
  'date': '2016-07-08',
  'description': u"NATO members gather in Warsaw, Poland, for a two-day summit. Much of the focus will be on Russia and NATO's commitments in bolstering its presence in Eastern Europe. Other issues to be discussed will include Europe's migrant crises, ISIL extremists, and the United Kingdom's withdrawal from the European Union. (Radio Free Europe/Radio Liberty)",
  'title': u'2016 Warsaw summit'},
 {'class': u'International relations',
  'date': '2016-07-08',
  'description': u"American and South Korean military officials agree to deploy a U.S. Terminal High Altitude Area Defense (THAAD) missile defense system in the country to counter North Korea's growing threats and use of ballistic missile and nuclear tests. (The Guardian)",
  'title': u'South Korea\u2013United States relations'},
 {'class': u'Law and crime',
  'date': '2016-07-08',
  'description': u'The left-wing political party, Bagong Alyansang Makabayan, which is part of the current government, asks Philippine President Rodrigo Duterte to investigate the recent vigilante killings of suspected drug pushers. (GMA News)\nOpposition senator and human rights lawyer Leila de Lima has condemned the killings and urged the Philippine Congress to investigate. (Channel News Asia)',
  'title': u'Human rights in the Philippines'},
 {'class': u'Law and crime',
  'date': '2016-07-08',
  'description': u'Police kill Micah X Johnson, the gunman believed responsible for the killing of five police officers and shooting of seven more in Dallas, Texas following a standoff. (Los Angeles Times)',
  'title': u'2016 shooting of Dallas police officers'},
 {'class': u'Politics and elections',
  'date': '2016-07-08',
  'description': u"Protests in Harare and elsewhere in the country continue despite police intimidation and pleas by President Robert Mugabe for them to stop. (Voice of America Zimbabwe)\nPresident Mugabe, in a live national television broadcast from a stadium in Bindura, blames Western sanctions for his country's inability to pay government workers on time. (AP via The Washington Post) (AFP via Yahoo! News)\n#ThisFlag protest leader Pastor Evan Mawarire says the movement, which uses WhatsApp, Facebook, and Twitter, will hold a two-day strike next week if demands, that include sacking corrupt ministers, payment of delayed salaries, lifting of roadblocks that residents say are used by police to extract bribes, etc., are not met. A drought has aggravated the country's situation as have banks that have a daily withdrawal ceiling as low as $50. (Reuters via CNBC Africa) (Ventures Africa)",
  'title': u'2016 Zimbabwe protests'},
 {'class': u'Armed conflicts and attacks',
  'date': '2016-07-09',
  'description': u'2016 Kashmir unrest\n\nAt least three people are killed and dozens more are wounded when protesters clash with Indian Army soldiers. The protesters defied an army curfew and took part in the funeral of a top rebel commander of Hizbul Mujahideen. (Radio Free Europe/Radio Liberty)',
  'title': u'Insurgency in Jammu and Kashmir'},
 {'class': u'Armed conflicts and attacks',
  'date': '2016-07-09',
  'description': u"According to a military spokesman of the rebel SPLM-IO, at least 115 soldiers between the two rival Sudan People's Liberation Army factions are killed in clashes in the capital, Juba. (Reuters)",
  'title': u'South Sudanese Civil War'},
 {'class': u'Armed conflicts and attacks',
  'date': '2016-07-09',
  'description': u"A Russian Air Force Mi-24 helicopter crashes near Palmyra, Syria, while assisting Syrian Army forces in the area. Russia's Ministry of Defence confirms two Russian pilots, Riafa'at Habibulin and Yevgeny Dolgin, are killed. (The Independent)",
  'title': u'Syrian civil war'},
 {'class': u'Disasters and accidents',
  'date': '2016-07-09',
  'description': u'Taiwan is recovering after Typhoon Nepartak hit yesterday killing two people and injuring 72. (PA via BT)',
  'title': u'Typhoon Nepartak (2016)'},
 {'class': u'International relations',
  'date': '2016-07-09',
  'description': u'Ahead of a court decision by the Permanent Court of Arbitration, China holds a naval drill in the disputed South China Sea raising tensions in the region. (The New York Times)',
  'title': u'International reactions to Philippines v. China'},
 {'class': u'International relations',
  'date': '2016-07-09',
  'description': u'South Korea claims that North Korea has fired a ballistic missile from a submarine. (Yonhap)',
  'title': u'North Korea and weapons of mass destruction'},
 {'class': u'International relations',
  'date': '2016-07-09',
  'description': u'NATO members in Warsaw, Poland, agree to extend their mission in Afghanistan into 2017 and announced that they will continue to fund the Afghan Armed Forces to 2020. (Radio Free Europe/Radio Liberty)',
  'title': u'2016 Warsaw summit'},
 {'class': u'International relations',
  'date': '2016-07-09',
  'description': u'Following the death of Alton Sterling and Philando Castile and the shooting of white police officers in Dallas, Texas, The Bahamas issues a rare travel warning for its citizens traveling to the United States. (BBC)',
  'title': u'Bahamian\u2013United States relations'},
 {'class': u'Law and crime',
  'date': '2016-07-09',
  'description': u'The government of the Philippines demands its critics to provide proof if there are human rights violations conducted during its drug war. (Rappler)\nMore than a hundred suspected drug pushers have been killed since Rodrigo Duterte took office as president of the country, a situation likened to the dictatorship of Ferdinand Marcos. (Philippine Daily Inquirer)',
  'title': u'Human rights in the Philippines'},
 {'class': u'Law and crime',
  'date': '2016-07-09',
  'description': u"This week, the Gambia and Tanzania outlawed child marriages. Friday, the Tanzanian High Court \u2014 in a case filed by the Msichana Initiative, a lobbying group that advocates for girls' right to education \u2014 ruled in favor of protecting girls from the harms of early marriage. And during a feast ending the Muslim holy month of Ramadan on Wednesday, The Gambian President Yahya Jammeh announced that child and forced marriages are banned. Jammeh called on the National Assembly to quickly take up the issue. (AP) (BBC) (Human Rights Watch)",
  'title': u'Child marriage'},
 {'class': u'Law and crime',
  'date': '2016-07-09',
  'description': u'Multiple gunshots strike the public safety headquarters of the San Antonio, Texas, Police Department. No injuries are reported. (USA Today)',
  'title': u'2016 shooting of Dallas police officers'},
 {'class': u'Politics and elections',
  'date': '2016-07-09',
  'description': u'The Zimbabwe economy is crippled after a week of protests as a result of dissatisfaction with the regime of Robert Mugabe. (The Telegraph)',
  'title': u'2016 Zimbabwe protests'},
 {'class': u'Politics and elections',
  'date': '2016-07-09',
  'description': u'Voters in Nauru go to the polls for an election. The President of Nauru Baron Waqa is reelected in his constituency. (Radio New Zealand), (ABC News Australia)',
  'title': u'Nauruan parliamentary election, 2016'},
 {'class': u'Politics and elections',
  'date': '2016-07-09',
  'description': u"Green Party presumptive presidential candidate Jill Stein invites Bernie Sanders to take over her place as the party's nominee. (International Business Times)",
  'title': u'United States presidential elections, 2016'},
 {'class': u'Sport',
  'date': '2016-07-09',
  'description': u'In tennis, American defending champion Serena Williams defeats Angelique Kerber of Germany in straight sets to win her seventh Wimbledon Championships title and her 22nd Grand Slam title, tying the Open era record held by Steffi Graf. (BBC)',
  'title': u"2016 Wimbledon Championships \u2013 Women's Singles"},
 {'class': u'Armed conflicts and attacks',
  'date': '2016-07-10',
  'description': u'An estimated 272 people have been killed between rival Sudan People\'s Liberation Army factions as clashes in the capital, Juba, continue. According to a spokesman for rebel leader and Vice President Riek Machar, South Sudan is "back to war" despite a peace deal being reached last April. (Al Jazeera) (BBC)',
  'title': u'South Sudanese Civil War'},
 {'class': u'International relations',
  'date': '2016-07-10',
  'description': u'The United Arab Emirates and Bahrain issue a travel warning for their citizens heading to American cities where protests are occuring following ethnic tensions in the country. This comes after The Bahamas issued its own travel warning yesterday, after the shooting of Dallas police officers. (Al Jazeera)',
  'title': u'Foreign relations of the United States'},
 {'class': u'Politics and elections',
  'date': '2016-07-10',
  'description': u'Voters in Japan go to the polls with the ruling Liberal Democratic Party and its Komeito partners looking for a two-thirds majority that will allow them to put forward a referendum to change the constitution. The governing parties won 70 of the 121 seats. (AAP via SBS), (Bloomberg), (AP via Crossroads Today)',
  'title': u'Japanese House of Councillors election, 2016'},
 {'class': u'Politics and elections',
  'date': '2016-07-10',
  'description': u'The Leader of the Opposition, Bill Shorten, concedes defeat as continued counting in marginal seats indicates that the governing coalition of the Liberal Party and National Party will either win a narrow majority or be able to govern with the support of independents. Malcolm Turnbull will continue as Prime Minister of Australia and has claimed victory. (Reuters), (The Guardian)',
  'title': u'Australian federal election, 2016'},
 {'class': u'Politics and elections',
  'date': '2016-07-10',
  'description': u'Protests continue in multiple cities across the United States following the death of two black men by police. Numerous confrontations and arrests have been reported. (Reuters)',
  'title': u'Shooting of Alton Sterling'},
 {'class': u'Sport',
  'date': '2016-07-10',
  'description': u"In tennis, Andy Murray of Scotland defeats Canadian Milos Raonic in the finals of the men's singles to win his second Wimbledon championship. It is the first time that a men's Wimbledon final has not featured Roger Federer, Rafael Nadal or Novak Djokovic since 2002. (Wimbledon official live updates)",
  'title': u"2016 Wimbledon Championships \u2013 Men's Singles"},
 {'class': u'Sport',
  'date': '2016-07-10',
  'description': u'In football, host nation France plays against Portugal for the 2016 UEFA European Championship at the Stade de France in Saint-Denis. Portugal wins 1\u20130 with an \xc9der goal in extra time. (UEFA official) (BBC), (ABC News Australia)',
  'title': u'UEFA Euro 2016 Final'},
 {'class': u'Armed conflicts and attacks',
  'date': '2016-07-11',
  'description': u'At least 10 Somali Armed Forces soldiers and 12 Al-Shabaab militants are killed in fighting near the town of Afgooye, Lower Shabelle. (Al Jazeera)',
  'title': u'War in Somalia'},
 {'class': u'Armed conflicts and attacks',
  'date': '2016-07-11',
  'description': u'2016 Kashmir unrest\n\nAt least 30 civilians have been killed by Indian Army soldiers as anti-government protests and violence spread across the state. (CNN-News18)',
  'title': u'Insurgency in Jammu and Kashmir'},
 {'class': u'Armed conflicts and attacks',
  'date': '2016-07-11',
  'description': u'Rebels in Aleppo launch an assault on government-controlled areas in the city. This comes after the Syrian Army blocked off the only road leading into opposition-controlled areas. According to state media, at least eight people have been killed and dozens more are wounded. (BBC)',
  'title': u'Syrian Civil War'},
 {'class': u'Armed conflicts and attacks',
  'date': '2016-07-11',
  'description': u'The United States announces it will be sending 560 more United States Armed Forces soldiers to Iraq to help fight against ISIL. (BBC)',
  'title': u'Iraqi Civil War'},
 {'class': u'Armed conflicts and attacks',
  'date': '2016-07-11',
  'description': u"Israeli forces level land east of Khan Yunis in the southern Gaza Strip. (Ma'an News)",
  'title': u'Israeli\u2013Palestinian conflict'},
 {'class': u'Law and crime',
  'date': '2016-07-11',
  'description': u'At least three people die, including the shooter, in a shooting at the Berrien County courthouse in the U.S. state of Michigan. (Washington Examiner)',
  'title': u'St. Joseph courthouse shooting'},
 {'class': u'Politics and elections',
  'date': '2016-07-11',
  'description': u'Protest leader, Evan Mawarire, asks the international community to put pressure on the regime of Robert Mugabe. (The Guardian)',
  'title': u'2016 Zimbabwe protests'},
 {'class': u'Politics and elections',
  'date': '2016-07-11',
  'description': u'Andrea Leadsom withdraws from the leadership election meaning that Theresa May will become the next leader of the Conservative Party and next Prime Minister of the United Kingdom. (The Guardian)\nPrime Minister David Cameron announces he will step down on Wednesday, July 13. (BBC)',
  'title': u'Conservative Party (UK) leadership election, 2016'},
 {'class': u'Politics and elections',
  'date': '2016-07-11',
  'description': u'The former Shadow Leader of the House of Commons Angela Eagle formally announces her bid to become the new leader of the Labour Party after Jeremy Corbyn refused calls to resign. (ITV News)',
  'title': u'Labour Party (UK) leadership crisis, 2016'},
 {'class': u'Armed conflicts and attacks',
  'date': '2016-07-12',
  'description': u'After five days of fighting that left 272 people dead and thousands more displaced in the capital, Juba, President Salva Kiir Mayardit and rebel leader and Vice-President Riek Machar declare a ceasefire. (Al Jazeera)',
  'title': u'South Sudanese Civil War'},
 {'class': u'Armed conflicts and attacks',
  'date': '2016-07-12',
  'description': u'A car bomb explodes at an outdoor market in Baghdad, killing at least 12 people and injuring 20 more. Bombings elsewhere in Iraq kill five more people. (Al Jazeera) (AP)',
  'title': u'Iraqi Civil War'},
 {'class': u'Disasters and accidents',
  'date': '2016-07-12',
  'description': u'23 people are killed and 50 others are injured after two trains collide head-on near Andria in the province of Puglia, southern Italy. (BBC)',
  'title': u'Andria train collision'},
 {'class': u'International relations',
  'date': '2016-07-12',
  'description': u'The Permanent Court of Arbitration rules in favor of the Philippines against China over territorial disputes in the South China Sea. (The New York Times), (The Guardian)\nThe court unanimously says that China has "no historical rights" based on the so-called "nine-dash line" map. (CNN), (BBC)\nChina rejects the tribunal\'s ruling, declaring it null and void. Chinese President Xi Jinping reiterates the South China Sea has been Chinese territory since "ancient times." (AP)',
  'title': u'Philippines v. China'},
 {'class': u'Law and crime',
  'date': '2016-07-12',
  'description': u'Evan Mawarire, the most visible leader of the protests in Zimbabwe, is arrested and charged with "inciting public violence and disturbing peace." (BBC)',
  'title': u'2016 Zimbabwe protests'},
 {'class': u'Politics and elections',
  'date': '2016-07-12',
  'description': u'Vermont Senator Bernie Sanders endorses Democratic Party rival Hillary Clinton for president at a rally in New Hampshire, effectively suspending his presidential campaign. (NBC News)',
  'title': u'United States presidential election, 2016'},
 {'class': u'Politics and elections',
  'date': '2016-07-12',
  'description': u'Citibank notifies the Venezuelan government it will close the accounts of the Venezuelan Central Bank and the Bank of Venezuela in 30 days after conducting a "periodic risk management review." Venezuela relies on Citibank to conduct foreign currency transactions due to the country\'s strict currency controls. (UPI)',
  'title': u'2016 state of emergency in Venezuela'},
 {'class': u'Armed conflicts and attacks',
  'date': '2016-07-13',
  'description': u'A car bomb explodes at a checkpoint north of Baghdad, killing at least seven people. (Radio Free Europe/Radio Liberty)',
  'title': u'Iraqi Civil War'},
 {'class': u'Armed conflicts and attacks',
  'date': '2016-07-13',
  'description': u'The death toll from anti-government protests in the Indian state rises to 36 with over a thousand more being treated for injuries. Chief Minister Mehbooba Mufti has called for calm. (BBC)',
  'title': u'Insurgency in Jammu and Kashmir'},
 {'class': u'Armed conflicts and attacks',
  'date': '2016-07-13',
  'description': u'Russian war planes allegedly bomb a refugee camp near the border with Jordan, killing at least eight people. (BBC)\nAccording to the Syrian Observatory for Human Rights, airstrikes in the towns of Al-Rastan and Ariha kill at least 24 people, despite a temporary ceasefire being declared by the Syrian government. (Reuters)',
  'title': u'Syrian Civil War'},
 {'class': u'Armed conflicts and attacks',
  'date': '2016-07-13',
  'description': u'The United States deploys more than three dozen soldiers to Juba, South Sudan, to protect the U.S. embassy and help evacuate non-essential embassy staff due to local clashes. (UPI)',
  'title': u'South Sudanese Civil War'},
 {'class': u'International relations',
  'date': '2016-07-13',
  'description': u"South Korea announces plans to deploy a Terminal High Altitude Area Defense anti-ballistic missile system in Seongju County in the country's south in order to deter against a North Korean attack. (CNN)",
  'title': u'North Korea\u2013South Korea relations'},
 {'class': u'Law and crime',
  'date': '2016-07-13',
  'description': u'A Zimbabwean court dismisses charges against opposition leader Pastor Evan Mawarire who was accused of trying to overthrow the government. (Reuters)',
  'title': u'2016 Zimbabwe protests'},
 {'class': u'Politics and elections',
  'date': '2016-07-13',
  'description': u"The new Prime Minister of the United Kingdom Theresa May begins forming her ministry following the end of the Second Cameron ministry. The former Mayor of London Boris Johnson is appointed Secretary of State for Foreign and Commonwealth Affairs, Philip Hammond is appointed as Chancellor of the Exchequer, Amber Rudd is appointed Home Secretary filling Theresa May's former position, Liam Fox is appointed as Secretary of State for International Trade and David Davis is appointed as minister for the newly created Secretary of State for Exiting the European Union. (BBC)",
  'title': u'May ministry'},
 {'class': u'Politics and elections',
  'date': '2016-07-13',
  'description': u'Baron Waqa is reelected President of Nauru, winning 16 of the 18 votes in Parliament. (Radio New Zealand)',
  'title': u'Nauruan parliamentary election, 2016'},
 {'class': u'Politics and elections',
  'date': '2016-07-13',
  'description': u'In a report by Amnesty International, the group documents findings of Egyptian officials forcibly making hundreds of people disappear and face torture in the past year in an attempt to crack down on dissent in the country. (BBC)',
  'title': u'Human rights in Egypt'},
 {'class': u'Armed conflicts and attacks',
  'date': '2016-07-14',
  'description': u'A suspected militant of Al-Shabaab shoots and kills four Kenyan police officers at a police station in Kapenguria, Kenya, where he was being held. The militant was later killed following a shootout with police. (Reuters)',
  'title': u'War in Somalia'},
 {'class': u'Armed conflicts and attacks',
  'date': '2016-07-14',
  'description': u'A truck plows into a mass of revelers during the Bastille Day fireworks on the Promenade des Anglais in Nice, France, with a death toll as high as 80 and 100 people injured. (Le Parisien) (Fox News)',
  'title': u'2016 Nice attack'},
 {'class': u'Arts and culture',
  'date': '2016-07-14',
  'description': u'The nominees are announced in Los Angeles. Game of Thrones, Veep and The People v. O. J. Simpson: American Crime Story lead in nominations. (Los Angeles Times) (The Washington Post)',
  'title': u'68th Primetime Emmy Awards'},
 {'class': u'Politics and elections',
  'date': '2016-07-14',
  'description': u'Elizabeth Truss is named Secretary of State for Justice and first ever female Lord Chancellor of the United Kingdom as former chancellor Michael Gove is ousted from the cabinet. (The Lawyer)',
  'title': u'May ministry'},
 {'class': u'Armed conflicts and attacks',
  'date': '2016-07-15',
  'description': u'The President of France Fran\xe7ois Hollande extends a state of emergency against terrorism for three months following the attack which has now claimed over 80 lives. He had previously announced that it would be lifted on 26 July. (Sputnik News), (BBC News)\nAuthorities identify the attacker as a 31-year-old Tunisian resident in France, who was killed during the exchange of gunfire with police. The man was known to police in connection with petty crime, but was not on the French intelligence service watch list. (Reuters) (BBC)',
  'title': u'2016 Nice attack'},
 {'class': u'Armed conflicts and attacks',
  'date': '2016-07-15',
  'description': u'The Prime Minister of Turkey Binali Y\u0131ld\u0131r\u0131m calls for "calm" over what he states was an "illegal attempt" by sections of the Turkish military to claim power. (The Telegraph)\nThe Turkish military claims to have gained control of the government with gunfire reported on the streets of Ankara. (CNBC)\nThe President of Turkey Recep Tayyip Erdo\u011fan urges supporters to oppose the coup d\'etat by taking to the streets. (Washington Post)\nThe death toll from the coup is reported to be at least 42 in Ankara and 60 across Turkey. 130 soldiers have been taken into custody and one Turkish Army general has been killed. (Reuters via Trust), (NBC News)',
  'title': u"2016 Turkish coup d'\xe9tat attempt"},
 {'class': u'International relations',
  'date': '2016-07-15',
  'description': u'Russia is to deploy its advanced long range S-400 surface-to-air missile system to Feodosia in the Crimean Peninsula next month for the first time. (Radio Free Europe/Radio Liberty) (RT)',
  'title': u'Ukrainian crisis'},
 {'class': u'Politics and elections',
  'date': '2016-07-15',
  'description': u'Donald Trump chooses Indiana governor Mike Pence as his vice presidential running mate. (CNN)',
  'title': u'Republican Party vice presidential candidate selection, 2016'},
 {'class': u'Sport',
  'date': '2016-07-15',
  'description': u'The largest video fighting game tournament in the world begins its three day event running from July 15\u201317. Over 15,000 players registered to compete with record breaking entrants for Street Fighter 5 at 5,107, Super Smash Bros for Wii U at 2,662 and Super Smash Bros. Melee at 2,372 entrants. (ESPN)',
  'title': u'Evolution Championship Series'},
 {'class': u'Armed conflicts and attacks',
  'date': '2016-07-16',
  'description': u'The President of Turkey Recep Tayyip Erdo\u011fan returns to Istanbul Atat\xfcrk Airport indicating that the coup may be faltering. (The New York Times)\nErdo\u011fan declares that the coup is over with over 1,500 members of the Turkish military in detention. (CNN) (AP)\nGeneral \xdcmit D\xfcndar reports at least 194 people were killed: two soldiers, 41 police officers, 47 civilians, and 104 alleged coup plotters. An official later adds that 1,440 people have been reported wounded. (NBC News)\nGreek security forces arrest eight people after a Turkish military helicopter lands at Alexandroupoli Airport. The helicopter crew have reportedly requested political asylum. (Reuters via Yahoo! News) (Ekathimerini)\nSecretary of State John Kerry says the United States would consider an extradition request for the cleric blamed for the attempted coup. Fethullah G\xfclen left Turkey in 1999 and now lives in Pennsylvania. G\xfclen denies any involvement. Kerry adds no request has been received. (NBC News) (AP)\nThe United States Federal Aviation Administration issues an order that prohibits all airlines from flying from Turkey to the U.S., and bans U.S. commercial and private aircraft from flying to Turkey. (Reuters)',
  'title': u"2016 Turkish coup d'\xe9tat attempt"},
 {'class': u'Armed conflicts and attacks',
  'date': '2016-07-16',
  'description': u'ISIL claims responsibility for the attack, and four people allegedly involved in the attack are arrested overnight. (AP, Fox News via West Australian)',
  'title': u'2016 Nice attack'},
 {'class': u'Armed conflicts and attacks',
  'date': '2016-07-17',
  'description': u'Syrian Government forces capture the main road into the parts of Aleppo controlled by rebels. (Reuters)',
  'title': u'Syrian Civil War'},
 {'class': u'Armed conflicts and attacks',
  'date': '2016-07-17',
  'description': u'UN-backed talks are underway in Tunis regarding the future of Libya, including discussions on forming a unified army for the Government of National Accord. (RTE)',
  'title': u'Second Libyan Civil War'},
 {'class': u'Law and crime',
  'date': '2016-07-17',
  'description': u'At least three Baton Rouge Police Department officers are shot dead and several others are wounded by a gunman in Baton Rouge, Louisiana. (USA Today) (BBC)',
  'title': u'2016 shooting of Baton Rouge police officers'},
 {'class': u'Politics and elections',
  'date': '2016-07-17',
  'description': u"Vietnamese police bus away about two dozen would-be protestors who gathered at Ho\xe0n Ki\u1ebfm Lake in central Hanoi to rally in support of the Permanent Court of Arbitration's South China Sea decision, and against China's rejection of the tribunal's ruling. This preemptive action, in effect, squashed the No-U group protest. (AP) (AAP via News.com.au)",
  'title': u'Territorial disputes in the South China Sea'},
 {'class': u'Politics and elections',
  'date': '2016-07-17',
  'description': u'Voters in S\xe3o Tom\xe9 and Pr\xedncipe go to the polls for a presidential election with former Prime Minister Evaristo Carvalho elected in the first round. (Africa News)',
  'title': u'S\xe3o Tom\xe9 and Pr\xedncipe presidential election, 2016'},
 {'class': u'Sport',
  'date': '2016-07-17',
  'description': u"In golf, Henrik Stenson wins the 2016 Open Championship defeating Phil Mickelson after a closely fought battle in the final round at the Royal Troon Golf Club in Scotland. Stenson's total of 264, 20 under par, is the lowest stroke total ever recorded in any major championship and ties Jason Day's performance at last year's PGA Championship for the lowest score in relation to par at a major. It is also Stenson's first victory in a major tournament. (The Guardian)",
  'title': u'2016 Open Championship'},
 {'class': u'Armed conflicts and attacks',
  'date': '2016-07-18',
  'description': u'At least six Yemeni soldiers are killed in suicide bombings near the city of Al Mukalla. (Reuters)',
  'title': u'Yemeni Civil War'},
 {'class': u'Armed conflicts and attacks',
  'date': '2016-07-18',
  'description': u'Three policemen and one civilian are killed in Almaty, Kazakhstan, in an attack on a police station by a radical Islamist. One gunman is reportedly still at large. (RT)',
  'title': u'2016 shooting of Almaty police officers'},
 {'class': u'Armed conflicts and attacks',
  'date': '2016-07-18',
  'description': u'Four people have been injured after a man with an axe attacks people on board a train near W\xfcrzburg in Germany. The axeman has reportedly been shot by police and is reportedly a 17-year-old Afghan man who pledged allegiance to ISIL before the attack. (BBC), (Indian Express), (Daily Mail)',
  'title': u'2016 W\xfcrzburg train attack'},
 {'class': u'Armed conflicts and attacks',
  'date': '2016-07-18',
  'description': u"General Ak\u0131n \xd6zt\xfcrk, Turkish Air Force chief until August 2015, confesses to planning last week's attempted coup d'\xe9tat, according to the state-run Anadolu News Agency. (UPI)",
  'title': u"2016 Turkish coup d'\xe9tat attempt"},
 {'class': u'International relations',
  'date': '2016-07-18',
  'description': u'South Korea warns that North Korea could hold another nuclear test at any time. (Yonhap News)',
  'title': u'North Korea\u2013South Korea relations'},
 {'class': u'International relations',
  'date': '2016-07-18',
  'description': u"The United States Secretary of State John Kerry warns Turkey it could lose its NATO membership if it fails to uphold the principles of democracy. Around 8,000 police officers have been sacked in the country following the attempted coup, with more than 6,000 people in the army, the judiciary and other state bodies detained as part of President Recep Tayyip Erdo\u011fan's crackdown. (The Independent)",
  'title': u'Turkey\u2013United States relations'},
 {'class': u'Politics and elections',
  'date': '2016-07-18',
  'description': u'The 2016 Republican National Convention gets underway in Cleveland, Ohio. With the day\'s theme being national and local security, notable speakers include reality television star Willie Robertson, actor Scott Baio, former Governor of Texas Rick Perry, former Mayor of New York City Rudy Giuliani, Milwaukee County sheriff David Clarke, retired United States Army lieutenant general Michael T. Flynn and Melania Trump, wife of presumptive Republican nominee for President of the United States Donald Trump. (ABC News)\nOpposing protests and rallies in Cleveland on the first day of the convention remain peaceful. (UPI)\nAfter giving her speech, Melania Trump is accused of plagiarizing parts of Michelle Obama\'s speech that she gave during the 2008 Democratic National Convention. The Trump campaign quickly responded to the controversy saying that the whole thing was "absurd". (CNN)',
  'title': u'United States presidential election, 2016'},
 {'class': u'Armed conflicts and attacks',
  'date': '2016-07-19',
  'description': u'The British-based Syrian Observatory for Human Rights reports U.S.-led coalition airstrikes on the Islamic State-held city of Manbij in northern Syria kill at least 56 civilians, including 11 children, and injure dozens more. (Reuters)',
  'title': u'Syrian Civil War'},
 {'class': u'Armed conflicts and attacks',
  'date': '2016-07-19',
  'description': u'2016 Nampala attack: At least 12 soldiers are killed and 27 others injured after unidentified militants briefly overrun a government base in central Mali, near the border with Mauritania. The attackers also burn and loot the adjacent village of Nampala. (Reuters)',
  'title': u'Northern Mali conflict'},
 {'class': u'Arts and culture',
  'date': '2016-07-19',
  'description': u'Roger Ailes is reported to be leaving Fox News after charges of sexual harassment are filed against him by former Fox and Friends host Gretchen Carlson. (Variety)',
  'title': u'Fox News Channel controversies'},
 {'class': u'International relations',
  'date': '2016-07-19',
  'description': u'The South Korean Joint Chiefs of Staff and the United States Strategic Command report North Korea fires three ballistic missiles into the Sea of Japan. The short-range ballistic missiles flew about 500 to 600 kilometers (310 to 375 miles) before crashing in the sea, sufficient to target the southern part of South Korea where the U.S. Terminal High Altitude Area Defense system is set to be installed. (AP) (BBC)',
  'title': u'North Korea\u2013South Korea relations'},
 {'class': u'Politics and elections',
  'date': '2016-07-19',
  'description': u'United States president Barack Obama and his vice president Joe Biden endorse Kamala Harris over Loretta Sanchez, both of whom are Democrats. (The Los Angeles Times)',
  'title': u'United States Senate election in California, 2016'},
 {'class': u'Politics and elections',
  'date': '2016-07-19',
  'description': u'Angela Eagle drops out of the race and announces her endorsement of Owen Smith, the last remaining candidate to challenge incumbent party leader Jeremy Corbyn. (BBC)',
  'title': u'Labour Party (UK) leadership election, 2016'},
 {'class': u'Politics and elections',
  'date': '2016-07-19',
  'description': u"Donald Trump is nominated as the Republican Party candidate for this year's presidential election. (NBC News)",
  'title': u'2016 Republican National Convention'},
 {'class': u'Armed conflicts and attacks',
  'date': '2016-07-20',
  'description': u'Prominent Ukrainian-based journalist Pavel Sheremet of the Ukrayinska Pravda is killed in a car bombing in Kiev. (The Guardian)',
  'title': u'Ukrainian crisis'},
 {'class': u'Armed conflicts and attacks',
  'date': '2016-07-20',
  'description': u'At least three French Armed Forces soldiers are killed after their helicopter is shot down near the city of Benghazi, Libya. The soldiers were killed while carrying out "dangerous intelligence operations", according to French President Fran\xe7ois Hollande. (BBC)',
  'title': u'Libyan Civil War (2014\u2013present)'},
 {'class': u'Armed conflicts and attacks',
  'date': '2016-07-20',
  'description': u'Following the Nampala attack, the government of Mali declares a three-day mourning period for the 17 soldiers killed in the attack, as well as a state of emergency for three months across the country. (Reuters)',
  'title': u'Northern Mali conflict'},
 {'class': u'Armed conflicts and attacks',
  'date': '2016-07-20',
  'description': u"Turkish President Recep Tayyip Erdogan declares a state of emergency for three months following Friday night's failed coup. (BBC News)",
  'title': u"2016 Turkish coup d'\xe9tat attempt"},
 {'class': u'Disasters and accidents',
  'date': '2016-07-20',
  'description': u'A M\xe9decins Sans Fronti\xe8res (Doctors Without Borders) ship rescues 209 people, including 50 children, from two dinghies in distress in the Mediterranean Sea but finds 22 dead bodies, 21 women and a child, lying in a pool of fuel in the bottom of one of the boats. The dinghies were 17 miles east of Tripoli, Libya. Cause of death is unknown, though an MSF official says it could be that fumes from mixed fuel and water rendered the people unconscious. The MSF ship is expected to arrive in the Sicilian port of Trapani on Friday.(Reuters)',
  'title': u'European migrant crisis'},
 {'class': u'Law and crime',
  'date': '2016-07-20',
  'description': u"The United States Court of Appeals for the Fifth Circuit affirms previous rulings that Texas' 2011 voter ID law is discriminatory and does not comply with the Voting Rights Act, and orders a lower court to resolve the problem in time for the November elections. (Reuters) (The Texas Tribune)",
  'title': u'United States presidential election, 2016'},
 {'class': u'Politics',
  'date': '2016-07-20',
  'description': u'Senator Ted Cruz, the runner-up in the race for the Republican Party nomination, is booed off stage after giving a speech and refusing to endorse Donald Trump. (Vox)',
  'title': u'2016 Republican National Convention'},
 {'class': u'Business and economy',
  'date': '2016-07-21',
  'description': u'21st Century Fox announced that the head of Fox News, Roger Ailes, is resigning. Rupert Murdoch will replace him as interim Chief Executive and Chairman. (Mediaite) (CNN) (New York Post)',
  'title': u'Fox News Channel controversies'},
 {'class': u'Disasters and accidents',
  'date': '2016-07-21',
  'description': u'Torrential precipitation and widespread flash floods from an extratropical cyclone resulted in at least 78 deaths over North China and nearby regions. (Xinhua)',
  'title': u'July 2016 North China cyclone'},
 {'class': u'International relations',
  'date': '2016-07-21',
  'description': u'Waves of Chinese protestors gather outside of KFC restaurants and Apple Stores across China to express their anti-U.S. sentiments after the ruling, while Chinese state media asks for calm. (AP via ABC News) (BBC) (USA Today)',
  'title': u'International reactions to Philippines v. China'},
 {'class': u'Law and crime',
  'date': '2016-07-21',
  'description': u"The Philippines' current crackdown on controlled substances passes a death toll of 500. (Abscbn)",
  'title': u'Philippine Drug War'},
 {'class': u'Politics',
  'date': '2016-07-21',
  'description': u'Donald Trump accepts the nomination of the Republican Party in the 2016 US Presidential election. (New York Times)',
  'title': u'2016 Republican National Convention'},
 {'class': u'Sports',
  'date': '2016-07-21',
  'description': u"Russia's track and field athletes will not be allowed to compete in the Rio Olympic Games, even if they did not fail drug tests, after an appeal to the Court of Arbitration for Sport is rejected. (BBC)",
  'title': u'2016 Summer Olympics'},
 {'class': u'Armed conflicts and attacks',
  'date': '2016-07-22',
  'description': u"Thousands gather in Istanbul's Taksim Square in a show of support for the new powers that President Recep Tayyip Erdo\u011fan has received in order to crackdown on dissent following the attempted coup. (The Telegraph)",
  'title': u"2016 Turkish coup d'\xe9tat attempt"},
 {'class': u'Armed conflicts and attacks',
  'date': '2016-07-22',
  'description': u'Ten people, including the gunman, are killed and 21 wounded in a shooting in and around the Olympia-Einkaufszentrum shopping mall in the Moosach district of Munich, Germany, next to the Olympic stadium.(CNN) (Reuters) (AP via KMGH-TV) (The New York Times) (BBC)',
  'title': u'2016 Munich shooting'},
 {'class': u'International relations',
  'date': '2016-07-22',
  'description': u'Taiwanese President Tsai Ing-wen rejects a supposed deadline set by Chinese President Xi Jinping for accepting the 1992 Consensus, saying such a deadline is against the will of the people. (Focus Taiwan) (The Washington Post)',
  'title': u'Cross-Strait relations'},
 {'class': u'Politics and elections',
  'date': '2016-07-22',
  'description': u'Presumptive Democratic Party presidential nominee Hillary Clinton selects Senator Tim Kaine (Virginia) as her vice-presidential running mate. (Fox News)',
  'title': u'United States presidential election, 2016'},
 {'class': u'Armed conflicts and attacks',
  'date': '2016-07-23',
  'description': u'Munich Police Chief Hubertus Andrae identifies the attacker as a dual national, 18-year-old Iranian-German who apparently acted alone. He was found dead of a suspected self-inflicted gunshot wound to his head. (Reuters)\nChief Andrae reports there are "no indications" the teenage gunman, who killed nine people and apparently himself, had links to ISIL or any other terror group. The attack injured 27. (NBC News)',
  'title': u'2016 Munich shooting'},
 {'class': u'Armed conflicts and attacks',
  'date': '2016-07-23',
  'description': u'Afghan officials report at least 80 people are killed and more than 230 injured as twin suicide blasts hit a large demonstration of the Shiite Hazara community over a power route line at Dehmazang Circle, Kabul, Afghanistan. The Islamic State claims responsibility. (Reuters) (News World India) (India Today) (Pakistan Today)\nAfghan President Ashraf Ghani declares Sunday, 24 July 2016, a day of national mourning.(BBC)',
  'title': u'July 2016 Kabul bombing'},
 {'class': u'Disasters and accidents',
  'date': '2016-07-23',
  'description': u'Floods kill at least 154 in China and 300,000 people are being evacuated in Hebei province. Villagers have accused local officials of covering up the extent of the damage. (AP via The Daily Mail)',
  'title': u'July 2016 North China cyclone'},
 {'class': u'Disasters and accidents',
  'date': '2016-07-23',
  'description': u'An article in New York magazine speculates that the pilot may have deliberately crashed the plane into the Southern Ocean. (New York magazine)',
  'title': u'Malaysia Airlines Flight 370'},
 {'class': u'Disasters and accidents',
  'date': '2016-07-23',
  'description': u"Hawaii Governor David Ige declares a state of emergency as Tropical Storm Darby is on track to reach the U.S. state's Big Island today with maximum sustained winds approaching 50 mph (80 kph). The storm, which previously was a Category 3 hurricane, could produce up to 15 inches (40 cm) of rain. (Reuters) (NHC)",
  'title': u'2016 Pacific hurricane season'},
 {'class': u'Disasters and accidents',
  'date': '2016-07-23',
  'description': u'The search intensifies for the Indian Air Force Antonov An-32 military transport with 29 people aboard that went missing over the Bay of Bengal shortly after its Friday morning takeoff from Tambaram. (UPI)',
  'title': u'2016 Indian Air Force An-32 disappearance'},
 {'class': u'International relations',
  'date': '2016-07-23',
  'description': u'Cambodia and Laos cause discord at the Association of Southeast Asian Nations meeting after they block a joint statement by the group regarding the South China Sea. (Reuters), (The New York Times)',
  'title': u'Philippines v. China'},
 {'class': u'Politics and elections',
  'date': '2016-07-23',
  'description': u'Emails released by Wikileaks show that the Democratic National Committee (DNC), led by Debbie Wasserman Schultz, supported the candidacy of Hillary Clinton over Bernie Sanders. (ABC News), (The Washington Post), (CNN), (Al Jazeera)\nInfluential UC Berkeley professor Robert Reich calls on the DNC to fire Schultz. (Mediaite)\nJeff Weaver, the campaign manager of Bernie Sanders, says that someone, alluding to Debbie Wasserman Schultz, must be held accountable for the revelations in the leaks. (ABC News)',
  'title': u'2016 Democratic National Committee email leak'},
 {'class': u'Politics and elections',
  'date': '2016-07-23',
  'description': u'Robert Mugabe calls war veterans, who are calling for him to resign, traitors. (The Washington Post)',
  'title': u'2016 Zimbabwe protests'},
 {'class': u'Politics and elections',
  'date': '2016-07-23',
  'description': u"President Recep Tayyip Erdo\u011fan's order under the state of emergency declared Wednesday after the failed military coup calls for closure of thousands of private schools, charities, and other institutions suspected of having links to Fethullah G\xfclen. (Reuters)",
  'title': u'Turkish state of emergency'},
 {'class': u'Armed conflicts and attacks',
  'date': '2016-07-24',
  'description': u'At least 21 are killed and more than 35 people injured in a suicide attack at one of the residential entrances to district Kadhimiyah, in northern Baghdad, Iraq. ISIL claims responsibility, as published in its official media. (CNN) (Iraqi News)',
  'title': u'Iraqi Civil War'},
 {'class': u'Armed conflicts and attacks',
  'date': '2016-07-24',
  'description': u'Fifteen people are injured following a suicide bombing in Ansbach, Germany. (BBC)',
  'title': u'2016 Ansbach bombing'},
 {'class': u'Armed conflicts and attacks',
  'date': '2016-07-24',
  'description': u'Syrian opposition activists report government airstrikes on Aleppo Province kill at least five people and hit at least five medical facilities. (AP via The New York Times)',
  'title': u'Syrian civil war'},
 {'class': u'Disasters and accidents',
  'date': '2016-07-24',
  'description': u'A dead body is found in the area of the Sand Fire near Santa Clarita that has doubled in size to more than 22,000 acres. Officials order mandatory evacuations for 1,500 homes in Little Tujunga, Sand, and Placerita canyons. The South Coast Air Quality Management District issues an alert for unhealthy air quality in affected areas. (NBC News) (Los Angeles Times)',
  'title': u'2016 California wildfires'},
 {'class': u'Health',
  'date': '2016-07-24',
  'description': u"The leader of the Australian delegation says the country's athletes would not be checking in at the athletes' village because it's unsafe for human habitation. Australian athletes have been staying at nearby Rio de Janeiro hotels. (The New York Times)",
  'title': u'2016 Summer Olympics'},
 {'class': u'International relations',
  'date': '2016-07-24',
  'description': u'Cambodia continues to block consensus in the Association of Southeast Asian Nations meeting. The country has political and economical dependencies with China. (BBC)',
  'title': u'Philippines v. China'},
 {'class': u'Politics and elections',
  'date': '2016-07-24',
  'description': u"Debbie Wasserman Schultz resigns from her position as Chairperson of the Democratic National Committee. Wasserman Schultz had reportedly been removed from a speaking role after a WikiLeaks email leak proved her implicit support of Hillary Clinton's campaign during the primaries. Her resignation will take effect upon the close of the convention. Donna Brazile will serve as interim chair. (Reuters) (The Washington Post) (CNN) (Fox News)",
  'title': u'2016 Democratic National Convention'},
 {'class': u'Politics and elections',
  'date': '2016-07-24',
  'description': u'In an unusual show of unity, tens of thousands of supporters of Turkey\'s ruling party and the country\'s main opposition parties hold a cross-party "Republic and Democracy" rally in support of democracy in Istanbul\'s central Taksim Square. (Reuters)',
  'title': u"2016 Turkish coup d'\xe9tat attempt"},
 {'class': u'Sport',
  'date': '2016-07-24',
  'description': u'The International Olympic Committee states it considers all Russian athletes tainted by the country\u2019s state-run doping scheme but individual athletes can compete in 2016 Games if they are able to convince individual sports federations of their innocence. A daunting task for these organizations to handle hundreds of appeals just 12 days before the start of the Rio Games on 5 August. (The New York Times) (AP & NBC News) (rt.com)',
  'title': u'2016 Summer Olympics'},
 {'class': u'Armed conflicts and attacks',
  'date': '2016-07-25',
  'description': u'A suicide car bomb kills at least 21 people and wounds more than 32 at the entrance to Al Khalis, northeast of Baghdad. Most died inside their vehicles while waiting to enter the town. (Al Jazeera)',
  'title': u'Iraqi Civil War (2014\u2013present)'},
 {'class': u'Armed conflicts and attacks',
  'date': '2016-07-25',
  'description': u"Saudi Arabia's Interior Ministry says five border guards have been killed following clashes with Houthi militants on the border with Yemen. (The New York Times)",
  'title': u'Conflict in Najran'},
 {'class': u'Law and crime',
  'date': '2016-07-25',
  'description': u'At least two people are killed after as many as 17 people are injured during a mass shooting at Club Blu, a nightclub in Fort Myers, Florida, with most of the victims reportedly minors. Two suspects and a person of interest have been detained in connection with the shooting. (BBC), (News-Press)',
  'title': u'2016 Fort Myers nightclub shooting'},
 {'class': u'Law and crime',
  'date': '2016-07-25',
  'description': u'Turkey orders the detention of 42 journalists including well-known writer Nazli Ilicak. Earlier, Turkish satirical magazine LeMan\xa0(tr) was prevented from publishing its post-coup edition\u2014a cartoon on the cover showed Turkish soldiers facing off against anti-coup protesters, both pushed toward each other by giant hands. (Reuters) (CNN)\nAmnesty International reports detainees in Turkey are being subjected to beatings and torture, including rape, in official and unofficial detention centres in the country. (NPR) (AI)',
  'title': u'2016 Turkish purges'},
 {'class': u'Politics and elections',
  'date': '2016-07-25',
  'description': u'High profile speakers include Senator Cory Booker, Senator Al Franken, Senator Kristen Gillibrand and Senator Jeanne Shaheen. (The Washington Post)\nHeadliners include Senator Elizabeth Warren, Democratic presidential candidate Senator Bernie Sanders and First Lady Michelle Obama. (Los Angeles Times)\nSenator Bernie Sanders is booed by his own supporters during his keynote speech after telling them, "We must elect Hillary Clinton." (ABC News)\nCelebrity appearances and performances include the boyband Boyz II Men, singer/songwriter Paul Simon and singer/actress Demi Lovato; among the speakers include Sanders supporter/comedian Sarah Silverman and Latina activist/actress Eva Longoria. (Variety), (CNN)',
  'title': u'2016 Democratic National Convention'},
 {'class': u'Armed conflicts and attacks',
  'date': '2016-07-26',
  'description': u"Al-Shabaab claim responsibility for a double suicide car bombing attack in Mogadishu near an African Union base at the city's airport. At least 13 people are killed, including seven UN guards. (AP) (Al-Jazeera)",
  'title': u'War in Somalia (2009\u2013present)'},
 {'class': u'Armed conflicts and attacks',
  'date': '2016-07-26',
  'description': u'Two men armed with knives take a priest, two nuns and two parishioners hostage in a church near Rouen in the French region of Normandy. French police kill the hostage-takers after the 86-year-old priest, Jacques Hamel, was killed by slitting his throat. (NBC News) (BBC), (Haaretz)\nFollowing the murder of the Catholic priest, French President Fran\xe7ois Hollande vows to wage war against the Islamic State of Iraq and the Levant "by every means". The organization released a video showing the attackers pledging allegiance to ISIL during the attack. (Local)',
  'title': u'2016 Normandy church attack'},
 {'class': u'Disasters and accidents',
  'date': '2016-07-26',
  'description': u'Sand Fire\n\nA state of emergency is declared in Los Angeles County, California, due to the threat posed by the Sand Fire. (Los Angeles Daily News)\nOfficials evacuate 10,000 homes as the blaze grows to more than 35,000 acres. (Los Angeles Times)\n\n\nSoberanes Fire\n\nCalifornia officials declare a state of emergency for this Garrapata State Park blaze in Monterey County that, in its fifth day, has grown to 23,500 acres, and is only 10 percent contained. Six major Big Sur State Parks remain closed. (San Francisco Chronicle) (AP)',
  'title': u'2016 California wildfires'},
 {'class': u'International relations',
  'date': '2016-07-26',
  'description': u'The Palestinian National Authority says it is planning to sue the United Kingdom over the 1917 Balfour Declaration. Palestinian Foreign Affairs Minister Riyad al-Maliki says the document led to mass Jewish immigration to British Mandatory Palestine, "at the expense of our Palestinian people". (BBC)',
  'title': u'Palestine\u2013United Kingdom relations'},
 {'class': u'Law and crime',
  'date': '2016-07-26',
  'description': u'At least 19 people are killed and 26 others are injured in a knife attack at a care centre for disabled people in the city of Sagamihara, Kanagawa Prefecture, Japan. A man has handed himself in to the police, according to local media. (BBC)',
  'title': u'Sagamihara stabbings'},
 {'class': u'Politics and elections',
  'date': '2016-07-26',
  'description': u"Hillary Clinton is nominated as the Democratic Party candidate for this year's presidential election. She becomes the first woman ever to be nominated for president by a major party in the United States.(CNN)(The Guardian) (NBC News)\nFormer president Bill Clinton, husband of Hillary Clinton, describes Hillary as a fighter who works relentlessly to make the world better. (The Sacramento Bee), (Time) (NPR)\nOther Democratic leaders speaking include Donna Brazile, Virginia Governor Terry McAuliffe, Senator Barbara Boxer, Senator Tom Harkin, Senator Amy Klobuchar, Senator Harry Reid, Senator Chuck Schumer, House Minority Leader Nancy Pelosi, former Attorney General Eric Holder, former U.S. Secretary of State Madeleine Albright and former U.S. President Jimmy Carter. (The Chicago Sun Times), (CBS)\nCelebrity appearances include Elizabeth Banks, Andra Day, Lena Dunham, America Ferrera, Tony Goldwin, Alicia Keys, Debra Messing, Meryl Streep, and Mallory Weggemann. (The Chicago Sun Times), (CBS)",
  'title': u'2016 Democratic National Convention'},
 {'class': u'Armed conflicts and attacks',
  'date': '2016-07-27',
  'description': u'At least 44 people are killed in a twin bombing in the Kurdish-controlled city of Al-Qamishli on the Syria\u2013Turkey border. ISIL claims responsibility for the attack. (The Guardian)',
  'title': u'Syrian civil war'},
 {'class': u'Arts and culture',
  'date': '2016-07-27',
  'description': u'The six Ramon Magsaysay Awardees for 2016 are: Indian Dalit activist Bezwada Wilson, Ombudsman of the Philippines Conchita Carpio-Morales, Indian Carnatic musician T.M. Krishna, Indonesian charity organization Dompet Dhuafa, Lao humanitarian group Vientiane Rescue, and the Japan Overseas Cooperation Volunteers. (Times of India) (AP via The Jakarta Post)',
  'title': u'Ramon Magsaysay Award'},
 {'class': u'Arts and culture',
  'date': '2016-07-27',
  'description': u'Taylor Swift is snubbed while Adele and Beyonc\xe9 lead in nominations. (The Washington Times), (Billboard)',
  'title': u'2016 MTV Video Music Awards'},
 {'class': u'Disasters and accidents',
  'date': '2016-07-27',
  'description': u'Investigators hypothesise that the plane may be further to the north than the current search area. (BBC)',
  'title': u'Malaysia Airlines Flight 370'},
 {'class': u'Health',
  'date': '2016-07-27',
  'description': u'Four Florida residents may have been infected by the virus locally. (The Independent)',
  'title': u'2015\u201316 Zika virus epidemic'},
 {'class': u'International relations',
  'date': '2016-07-27',
  'description': u"During a press conference, Republican nominee Donald Trump calls on Russia to find Hillary Clinton's 30,000 missing emails. (BBC) (Reuters) (The New York Times)",
  'title': u'United States presidential election, 2016'},
 {'class': u'Law and crime',
  'date': '2016-07-27',
  'description': u'The United Nations urge Indonesian president Joko Widodo to halt all executions. (Al Jazeera)',
  'title': u'Capital punishment in Indonesia'},
 {'class': u'Politics and elections',
  'date': '2016-07-27',
  'description': u"U.S. President Barack Obama's advocates for Hillary Clinton stating there has never been anyone more qualified than Hillary to serve as president. (ABC News) (Irish Mirror)\nNotable speakers include Former New York City mayor Michael Bloomberg, Former Defense secretary Leon Panetta, California governor Jerry Brown, Senate Minority Leader Harry Reid, Senator Sherrod Brown, Senator Chris Murphy, Atlanta mayor Kasim Reed, Detroit mayor Mike Duggan, Flint, Michigan mayor Karen Weaver, New York City mayor Bill de Blasio, Former San Francisco mayor Gavin Newsom, U.S. representative Ruben Gallego, former Maryland governor Martin O'Malley, Vice President Joe Biden, and longtime politician/activist the Reverend Jesse Jackson. (Fox)\nCelebrity appearances include Angela Bassett, Lee Daniels, Huey Dunbar, Ra\xfal Esparza, La India, Star Jones, Lenny Kravitz, Sigourney Weaver, and will.i.am. (Fox)\nFormer representative and gun violence victim advocate Gabrielle Giffords and her husband astronaut Mark Kelly will make an appearance. (Fox)\nVirginia Senator and vice presidential candidate Tim Kaine will speak. (Fox)",
  'title': u'2016 Democratic National Convention'},
 {'class': u'Politics and elections',
  'date': '2016-07-27',
  'description': u'Robert Mugabe says that his former supporters are to suffer severe punishment for supporting the protests. (Fox News)',
  'title': u'2016 Zimbabwe protests'},
 {'class': u'Politics and elections',
  'date': '2016-07-27',
  'description': u'Prime Minister Najib Razak gets emergency powers amid a widening scandal about corruption. (The Guardian)',
  'title': u'1Malaysia Development Berhad scandal'},
 {'class': u'Armed conflicts and attacks',
  'date': '2016-07-28',
  'description': u'Russian Defense Minister Sergei Shoigu says Russia and Syria will open humanitarian corridors to Aleppo, and will provide a way for rebel fighters to lay down their arms. (CBS News)\nThe Syrian government offers an amnesty for rebels who surrender within three months. (Huffington Post)\nAbu Mohammad al-Julani, the leader of Al-Nusra Front, announces that the group is splitting from Al-Qaeda with their approval and that new operations will be carried out under the name Jabhat Fateh al-Sham. (The Guardian)',
  'title': u'Syrian civil war'},
 {'class': u'International relations',
  'date': '2016-07-28',
  'description': u'North Korea says the United States has "crossed the red line in our showdown" after placing sanctions on its leader Kim Jong-un over human rights abuses, and considers the sanctions to be a "declaration of war" by the United States. (Sky News)',
  'title': u'North Korea\u2013United States relations'},
 {'class': u'Politics and elections',
  'date': '2016-07-28',
  'description': u"Hillary Clinton accepts her party's nomination as the candidate for this year's presidential election. (The New York Times)\nNotable speakers include Governor Mark Dayton of Minnesota, Governor Andrew Cuomo of New York, House Minority Leader Nancy Pelosi, Governor John Hickenlooper of Colorado, Governor Tom Wolf of Pennsylvania, retired General John R. Allen and Chelsea Clinton. (Fox)\nCelebrity appearances include singer Carole King, basketball player Kareem Abdul-Jabaar, actress Chlo\xeb Grace Moretz and singer Katy Perry. (Fox)",
  'title': u'2016 Democratic National Convention'},
 {'class': u'Armed conflicts and attacks',
  'date': '2016-07-29',
  'description': u"The United Nations suspends aid to dangerous areas of Nigeria's northeastern Borno State following Thursday's attack by Boko Haram (ISIL in Africa) on a humanitarian convoy where two soldiers and three civilians were wounded. However, UNICEF noted, aid will continue to the capital Maiduguri. (AP via The New York Times)",
  'title': u'Boko Haram insurgency'},
 {'class': u'Armed conflicts and attacks',
  'date': '2016-07-29',
  'description': u'British-based Syrian Observatory for Human Rights reports a Save the Children maternity hospital in Kafr Takharim in rebel-held Idlib Governorate is hit by airstrikes. No report on casualties. SOHR also says at least five people are killed and more than 25 injured from airstrikes in another part of the northern Idlib countryside. (Reuters)',
  'title': u'Syrian civil war'},
 {'class': u'Armed conflicts and attacks',
  'date': '2016-07-29',
  'description': u'The Front for the Liberation of the Enclave of Cabinda (FLEC) says it has killed nine Angolan soldiers and wounded 14 others in recent clashes in the oil-rich Cabinda Province. (Reuters)',
  'title': u'Cabinda War'},
 {'class': u'International relations',
  'date': '2016-07-29',
  'description': u'The United States military says it will shortly return 4,000 hectares of its Okinawa military installation, 17 percent of the area it controls, to the Japanese government. (Reuters)',
  'title': u'Japan\u2013United States relations'},
 {'class': u'Armed conflicts and attacks',
  'date': '2016-07-30',
  'description': u"The Turkish military kills 35 Kurdistan Workers' Party militants after they attempt to capture a base in Turkey's Hakk\xe2ri Province. (Reuters via ABC News Australia)",
  'title': u'Kurdish\u2013Turkish conflict (1978\u2013present)'},
 {'class': u'Armed conflicts and attacks',
  'date': '2016-07-30',
  'description': u'Niger, following attacks by Boko Haram on Bosso and Yebi in the Lake Chad region, extends for three months a state of emergency in Diffa. (Reuters)',
  'title': u'Boko Haram insurgency'},
 {'class': u'Disasters and accidents',
  'date': '2016-07-30',
  'description': u'At least 16 people are killed after a hot air balloon catches fire and crashes near Lockhart, Texas. (BBC)',
  'title': u'2016 Lockhart hot air balloon crash'},
 {'class': u'Disasters and accidents',
  'date': '2016-07-30',
  'description': u"The deadly, nine-day old Soberanes Fire, blazing south of Carmel-by-the-Sea, has destroyed 57 homes and is expected to grow to 170,000 acres (265 square miles) before it's done. It is 15 percent contained. Six major Big Sur State Parks remain closed through August 6. (Reuters) (Los Angeles Times)",
  'title': u'2016 California wildfires'},
 {'class': u'Armed conflicts and attacks',
  'date': '2016-07-31',
  'description': u'Islamic State militants attack two gas facilities northwest of Kirkuk in northern Iraq, killing at least five people and wounding six policemen. Three assailants detonated their explosive vests and a fourth was killed in clash with security forces. (Reuters)',
  'title': u'Iraqi Civil War (2014\u2013present)'},
 {'class': u'Armed conflicts and attacks',
  'date': '2016-07-31',
  'description': u'An Al-Shabaab attack on the Central Investigations Department kills at least nine people. (AP via The New York Times)',
  'title': u'War in Somalia (2009\u2013present)'},
 {'class': u'Armed conflicts and attacks',
  'date': '2016-07-31',
  'description': u'A two-week standoff comes to an end in Yerevan, Armenia, after the remaining gunmen holed up in a police station surrender to police. The National Security Service says "twenty terrorists have been arrested" following the "complete liberation" of the police compound. (Radio Free Europe/Radio Liberty)',
  'title': u'2016 Yerevan hostage crisis'},
 {'class': u'Disasters and accidents',
  'date': '2016-07-31',
  'description': u'Hong Kong braces for the storm. (Reuters)',
  'title': u'Typhoon Nida (2016)'},
 {'class': u'Law and crime',
  'date': '2016-07-31',
  'description': u'The Turkish government dismisses nearly 1,400 military personnel for alleged links to exiled cleric Fethullah G\xfclen. (Reuters)',
  'title': u'2016 Turkish purges'},
 {'class': u'Politics and elections',
  'date': '2016-07-31',
  'description': u"Yuriko Koike, of the Liberal Democratic Party (LDP) but not their preferred candidate, is elected the Tokyo prefecture's first female governor, replacing Yoichi Masuzoe who resigned last month. The former Minister of Defense defeated the LDP-Komeito ruling coalition's Hiroya Masuda and the opposition bloc's Shuntaro Torigoe. Turnout is over 40 percent. (Japan Times)",
  'title': u'Tokyo, Japan, gubernatorial election, 2016'},
 {'class': u'Politics and elections',
  'date': '2016-07-31',
  'description': u'Democratic presidential election nominee Hillary Clinton accuses Russian intelligence services of being behind recent hacks into Democratic National Committee computers, while also accusing Republican presidential nominee Donald Trump of supporting Russian President Vladimir Putin. (Reuters)',
  'title': u'Democratic National Committee cyber attacks'},
 {'class': u'Politics and elections',
  'date': '2016-07-31',
  'description': u'Thousands of people protest in the capital Kinshasa after the national electoral commission decides to postpone the election. (France 24) (AP via The Washington Post)',
  'title': u'Democratic Republic of the Congo general election, 2016'},
 {'class': u'Sport',
  'date': '2016-07-31',
  'description': u'American Jimmy Walker wins the 98th PGA Championship, his first major championship. (BBC)',
  'title': u'2016 PGA Championship'},
 {'class': u'Sport',
  'date': '2016-07-31',
  'description': u"Ariya Jutanugarn of Thailand wins the Women's British Open for her first major championship. (AP via ESPN)",
  'title': u"2016 Women's British Open"}]

In [77]:
years = [2010+i for i in range(8)]

In [78]:
years


Out[78]:
[2010, 2011, 2012, 2013, 2014, 2015, 2016, 2017]

In [79]:
Portal_url = 'https://en.wikipedia.org/wiki/Portal:Current_events/'

In [80]:
page_urls = [Portal_url+m+'_'+str(y) for y in years for m in months ]

In [81]:
page_urls[6:]


Out[81]:
['https://en.wikipedia.org/wiki/Portal:Current_events/July_2010',
 'https://en.wikipedia.org/wiki/Portal:Current_events/August_2010',
 'https://en.wikipedia.org/wiki/Portal:Current_events/September_2010',
 'https://en.wikipedia.org/wiki/Portal:Current_events/October_2010',
 'https://en.wikipedia.org/wiki/Portal:Current_events/November_2010',
 'https://en.wikipedia.org/wiki/Portal:Current_events/December_2010',
 'https://en.wikipedia.org/wiki/Portal:Current_events/January_2011',
 'https://en.wikipedia.org/wiki/Portal:Current_events/February_2011',
 'https://en.wikipedia.org/wiki/Portal:Current_events/March_2011',
 'https://en.wikipedia.org/wiki/Portal:Current_events/April_2011',
 'https://en.wikipedia.org/wiki/Portal:Current_events/May_2011',
 'https://en.wikipedia.org/wiki/Portal:Current_events/June_2011',
 'https://en.wikipedia.org/wiki/Portal:Current_events/July_2011',
 'https://en.wikipedia.org/wiki/Portal:Current_events/August_2011',
 'https://en.wikipedia.org/wiki/Portal:Current_events/September_2011',
 'https://en.wikipedia.org/wiki/Portal:Current_events/October_2011',
 'https://en.wikipedia.org/wiki/Portal:Current_events/November_2011',
 'https://en.wikipedia.org/wiki/Portal:Current_events/December_2011',
 'https://en.wikipedia.org/wiki/Portal:Current_events/January_2012',
 'https://en.wikipedia.org/wiki/Portal:Current_events/February_2012',
 'https://en.wikipedia.org/wiki/Portal:Current_events/March_2012',
 'https://en.wikipedia.org/wiki/Portal:Current_events/April_2012',
 'https://en.wikipedia.org/wiki/Portal:Current_events/May_2012',
 'https://en.wikipedia.org/wiki/Portal:Current_events/June_2012',
 'https://en.wikipedia.org/wiki/Portal:Current_events/July_2012',
 'https://en.wikipedia.org/wiki/Portal:Current_events/August_2012',
 'https://en.wikipedia.org/wiki/Portal:Current_events/September_2012',
 'https://en.wikipedia.org/wiki/Portal:Current_events/October_2012',
 'https://en.wikipedia.org/wiki/Portal:Current_events/November_2012',
 'https://en.wikipedia.org/wiki/Portal:Current_events/December_2012',
 'https://en.wikipedia.org/wiki/Portal:Current_events/January_2013',
 'https://en.wikipedia.org/wiki/Portal:Current_events/February_2013',
 'https://en.wikipedia.org/wiki/Portal:Current_events/March_2013',
 'https://en.wikipedia.org/wiki/Portal:Current_events/April_2013',
 'https://en.wikipedia.org/wiki/Portal:Current_events/May_2013',
 'https://en.wikipedia.org/wiki/Portal:Current_events/June_2013',
 'https://en.wikipedia.org/wiki/Portal:Current_events/July_2013',
 'https://en.wikipedia.org/wiki/Portal:Current_events/August_2013',
 'https://en.wikipedia.org/wiki/Portal:Current_events/September_2013',
 'https://en.wikipedia.org/wiki/Portal:Current_events/October_2013',
 'https://en.wikipedia.org/wiki/Portal:Current_events/November_2013',
 'https://en.wikipedia.org/wiki/Portal:Current_events/December_2013',
 'https://en.wikipedia.org/wiki/Portal:Current_events/January_2014',
 'https://en.wikipedia.org/wiki/Portal:Current_events/February_2014',
 'https://en.wikipedia.org/wiki/Portal:Current_events/March_2014',
 'https://en.wikipedia.org/wiki/Portal:Current_events/April_2014',
 'https://en.wikipedia.org/wiki/Portal:Current_events/May_2014',
 'https://en.wikipedia.org/wiki/Portal:Current_events/June_2014',
 'https://en.wikipedia.org/wiki/Portal:Current_events/July_2014',
 'https://en.wikipedia.org/wiki/Portal:Current_events/August_2014',
 'https://en.wikipedia.org/wiki/Portal:Current_events/September_2014',
 'https://en.wikipedia.org/wiki/Portal:Current_events/October_2014',
 'https://en.wikipedia.org/wiki/Portal:Current_events/November_2014',
 'https://en.wikipedia.org/wiki/Portal:Current_events/December_2014',
 'https://en.wikipedia.org/wiki/Portal:Current_events/January_2015',
 'https://en.wikipedia.org/wiki/Portal:Current_events/February_2015',
 'https://en.wikipedia.org/wiki/Portal:Current_events/March_2015',
 'https://en.wikipedia.org/wiki/Portal:Current_events/April_2015',
 'https://en.wikipedia.org/wiki/Portal:Current_events/May_2015',
 'https://en.wikipedia.org/wiki/Portal:Current_events/June_2015',
 'https://en.wikipedia.org/wiki/Portal:Current_events/July_2015',
 'https://en.wikipedia.org/wiki/Portal:Current_events/August_2015',
 'https://en.wikipedia.org/wiki/Portal:Current_events/September_2015',
 'https://en.wikipedia.org/wiki/Portal:Current_events/October_2015',
 'https://en.wikipedia.org/wiki/Portal:Current_events/November_2015',
 'https://en.wikipedia.org/wiki/Portal:Current_events/December_2015',
 'https://en.wikipedia.org/wiki/Portal:Current_events/January_2016',
 'https://en.wikipedia.org/wiki/Portal:Current_events/February_2016',
 'https://en.wikipedia.org/wiki/Portal:Current_events/March_2016',
 'https://en.wikipedia.org/wiki/Portal:Current_events/April_2016',
 'https://en.wikipedia.org/wiki/Portal:Current_events/May_2016',
 'https://en.wikipedia.org/wiki/Portal:Current_events/June_2016',
 'https://en.wikipedia.org/wiki/Portal:Current_events/July_2016',
 'https://en.wikipedia.org/wiki/Portal:Current_events/August_2016',
 'https://en.wikipedia.org/wiki/Portal:Current_events/September_2016',
 'https://en.wikipedia.org/wiki/Portal:Current_events/October_2016',
 'https://en.wikipedia.org/wiki/Portal:Current_events/November_2016',
 'https://en.wikipedia.org/wiki/Portal:Current_events/December_2016',
 'https://en.wikipedia.org/wiki/Portal:Current_events/January_2017',
 'https://en.wikipedia.org/wiki/Portal:Current_events/February_2017',
 'https://en.wikipedia.org/wiki/Portal:Current_events/March_2017',
 'https://en.wikipedia.org/wiki/Portal:Current_events/April_2017',
 'https://en.wikipedia.org/wiki/Portal:Current_events/May_2017',
 'https://en.wikipedia.org/wiki/Portal:Current_events/June_2017',
 'https://en.wikipedia.org/wiki/Portal:Current_events/July_2017',
 'https://en.wikipedia.org/wiki/Portal:Current_events/August_2017',
 'https://en.wikipedia.org/wiki/Portal:Current_events/September_2017',
 'https://en.wikipedia.org/wiki/Portal:Current_events/October_2017',
 'https://en.wikipedia.org/wiki/Portal:Current_events/November_2017',
 'https://en.wikipedia.org/wiki/Portal:Current_events/December_2017']

In [254]:
import pymongo

In [255]:
client = pymongo.MongoClient('101.132.182.124:27027')

In [256]:
db = client.tweet

In [ ]:
requests = [UpdateOne({'_id': _id,'cluster':None}, {'$set': {'cluster':{'cluster_label':clusters[index],'cluster_hash':cluster_hash}}}) for index,_id in tqdm(enumerate(ids))]
result = db.test.bulk_write(requests)
print(result.bulk_api_result)

In [93]:
def get_events_from_page(page_url):
    res = requests.get(page_url,headers=headers)
    soup = BeautifulSoup(res.text,'lxml')
    #pdb.set_trace()
    tables = soup.find_all('table',attrs={'class':'vevent'})
    events = []
    for t in tables:
        if t.find_previous_sibling().get('id') == None:
            date = (datetime.strptime(date,'%Y-%m-%d')+timedelta(days=1)).strftime('%Y-%m-%d')
        else:
            try:
                date = trans_date(t.find_previous_sibling().get('id'))
            except:
                date = (datetime.strptime(date,'%Y-%m-%d')+timedelta(days=1)).strftime('%Y-%m-%d')
        print date
        #pdb.set_trace()
        td = t.find('td',attrs={'class':'description'})
        if td.dl == None:
            types = [p.get_text() for p in td.find_all('p',recursive=False)]
        else:
            types = [dl.get_text() for dl in td.find_all('dl',recursive=False)]
        for type_,ul in zip(types,[ul for ul in td.find_all('ul',recursive=False)]):
            for li in ul.find_all('li',recursive=False):
                if li.ul == None:
                    events.append({'date':date,
                               'class':type_.strip(),
                               'title':'',
                               'description':li.get_text().strip()})
                else:
                    events.append({'date':date,
                                   'class':type_.strip(),
                                   'title':li.a.get_text(),
                                   'description':li.ul.get_text().strip()})
    print len(events)
    return events
a = get_events_from_page('https://en.wikipedia.org/wiki/Portal:Current_events/November_2012')

In [66]:
from datetime import datetime, timedelta

In [67]:
(datetime.strptime(trans_date('2017_August_17'),'%Y-%m-%d')+timedelta(days=1)).strftime('%Y-%m-%d')


Out[67]:
'2017-08-18'

In [94]:
for i in page_urls[-20:]:
    get_events_from_page(i)


2016-05-01
2016-05-02
2016-05-03
2016-05-04
2016-05-05
2016-05-06
2016-05-07
2016-05-08
2016-05-09
2016-05-10
2016-05-11
2016-05-12
2016-05-13
2016-05-14
2016-05-15
2016-05-16
2016-05-17
2016-05-18
2016-05-19
2016-05-20
2016-05-21
2016-05-22
2016-05-23
2016-05-24
2016-05-25
2016-05-26
2016-05-27
2016-05-28
2016-05-29
2016-05-30
2016-05-31
402
2016-06-01
2016-06-02
2016-06-03
2016-06-04
2016-06-05
2016-06-06
2016-06-07
2016-06-08
2016-06-09
2016-06-10
2016-06-11
2016-06-12
2016-06-13
2016-06-14
2016-06-15
2016-06-16
2016-06-17
2016-06-18
2016-06-19
2016-06-20
2016-06-21
2016-06-22
2016-06-23
2016-06-24
2016-06-25
2016-06-26
2016-06-27
2016-06-28
2016-06-29
2016-06-30
396
2016-07-01
2016-07-02
2016-07-03
2016-07-04
2016-07-05
2016-07-06
2016-07-07
2016-07-08
2016-07-09
2016-07-10
2016-07-11
2016-07-12
2016-07-13
2016-07-14
2016-07-15
2016-07-16
2016-07-17
2016-07-18
2016-07-19
2016-07-20
2016-07-21
2016-07-22
2016-07-23
2016-07-24
2016-07-25
2016-07-26
2016-07-27
2016-07-28
2016-07-29
2016-07-30
2016-07-31
364
2016-08-01
2016-08-02
2016-08-03
2016-08-04
2016-08-05
2016-08-06
2016-08-07
2016-08-08
2016-08-09
2016-08-10
2016-08-11
2016-08-12
2016-08-13
2016-08-14
2016-08-15
2016-08-16
2016-08-17
2016-08-18
2016-08-19
2016-08-20
2016-08-21
2016-08-22
2016-08-23
2016-08-24
2016-08-25
2016-08-26
2016-08-27
2016-08-28
2016-08-29
2016-08-30
2016-08-31
295
2016-09-01
2016-09-02
2016-09-03
2016-09-04
2016-09-05
2016-09-06
2016-09-07
2016-09-08
2016-09-09
2016-09-10
2016-09-11
2016-09-12
2016-09-13
2016-09-14
2016-09-15
2016-09-16
2016-09-17
2016-09-18
2016-09-19
2016-09-20
2016-09-21
2016-09-22
2016-09-23
2016-09-24
2016-09-25
2016-09-26
2016-09-27
2016-09-28
2016-09-29
2016-09-30
338
2016-10-01
2016-10-02
2016-10-03
2016-10-04
2016-10-05
2016-10-06
2016-10-07
2016-10-08
2016-10-09
2016-10-10
2016-10-11
2016-10-12
2016-10-13
2016-10-14
2016-10-15
2016-10-16
2016-10-17
2016-10-18
2016-10-19
2016-10-20
2016-10-21
2016-10-22
2016-10-23
2016-10-24
2016-10-25
2016-10-26
2016-10-27
2016-10-28
2016-10-29
2016-10-30
2016-10-31
329
2016-11-01
2016-11-02
2016-11-03
2016-11-04
2016-11-05
2016-11-06
2016-11-07
2016-11-08
2016-11-09
2016-11-10
2016-11-11
2016-11-12
2016-11-13
2016-11-14
2016-11-15
2016-11-16
2016-11-17
2016-11-18
2016-11-19
2016-11-20
2016-11-21
2016-11-22
2016-11-23
2016-11-24
2016-11-25
2016-11-26
2016-11-27
2016-11-28
2016-11-29
2016-11-30
342
2016-12-01
2016-12-02
2016-12-03
2016-12-04
2016-12-05
2016-12-06
2016-12-07
2016-12-08
2016-12-09
2016-12-10
2016-12-11
2016-12-12
2016-12-13
2016-12-14
2016-12-15
2016-12-16
2016-12-17
2016-12-18
2016-12-19
2016-12-20
2016-12-21
2016-12-22
2016-12-23
2016-12-24
2016-12-25
2016-12-26
2016-12-27
2016-12-28
2016-12-29
2016-12-30
2016-12-31
362
2017-01-01
2017-01-02
2017-01-03
2017-01-04
2017-01-05
2017-01-06
2017-01-07
2017-01-08
2017-01-09
2017-01-10
2017-01-11
2017-01-12
2017-01-13
2017-01-14
2017-01-15
2017-01-16
2017-01-17
2017-01-18
2017-01-19
2017-01-20
2017-01-21
2017-01-22
2017-01-23
2017-01-24
2017-01-25
2017-01-26
2017-01-27
2017-01-28
2017-01-29
2017-01-30
2017-01-31
318
2017-02-01
2017-02-02
2017-02-03
2017-02-04
2017-02-05
2017-02-06
2017-02-07
2017-02-08
2017-02-09
2017-02-10
2017-02-11
2017-02-12
2017-02-13
2017-02-14
2017-02-15
2017-02-16
2017-02-17
2017-02-18
2017-02-19
2017-02-20
2017-02-21
2017-02-22
2017-02-23
2017-02-24
2017-02-25
2017-02-26
2017-02-27
2017-02-28
296
2017-03-01
2017-03-02
2017-03-03
2017-03-04
2017-03-05
2017-03-06
2017-03-07
2017-03-08
2017-03-09
2017-03-10
2017-03-11
2017-03-12
2017-03-13
2017-03-14
2017-03-15
2017-03-16
2017-03-17
2017-03-18
2017-03-19
2017-03-20
2017-03-21
2017-03-22
2017-03-23
2017-03-24
2017-03-25
2017-03-26
2017-03-27
2017-03-28
2017-03-29
2017-03-30
2017-03-31
258
2017-04-01
2017-04-02
2017-04-03
2017-04-04
2017-04-05
2017-04-06
2017-04-07
2017-04-08
2017-04-09
2017-04-10
2017-04-11
2017-04-12
2017-04-13
2017-04-14
2017-04-15
2017-04-16
2017-04-17
2017-04-18
2017-04-19
2017-04-20
2017-04-21
2017-04-22
2017-04-23
2017-04-24
2017-04-25
2017-04-26
2017-04-27
2017-04-28
2017-04-29
2017-04-30
325
2017-05-01
2017-05-02
2017-05-03
2017-05-04
2017-05-05
2017-05-06
2017-05-07
2017-05-08
2017-05-09
2017-05-10
2017-05-11
2017-05-12
2017-05-13
2017-05-14
2017-05-15
2017-05-16
2017-05-17
2017-05-18
2017-05-19
2017-05-20
2017-05-21
2017-05-22
2017-05-23
2017-05-24
2017-05-25
2017-05-26
2017-05-27
2017-05-28
2017-05-29
2017-05-30
2017-05-31
290
2017-06-01
2017-06-02
2017-06-03
2017-06-04
2017-06-05
2017-06-06
2017-06-07
2017-06-08
2017-06-09
2017-06-10
2017-06-11
2017-06-12
2017-06-13
2017-06-14
2017-06-15
2017-06-16
2017-06-17
2017-06-18
2017-06-19
2017-06-20
2017-06-21
2017-06-22
2017-06-23
2017-06-24
2017-06-25
2017-06-26
2017-06-27
2017-06-28
2017-06-29
2017-06-30
305
2017-07-01
2017-07-02
2017-07-03
2017-07-04
2017-07-05
2017-07-06
2017-07-07
2017-07-08
2017-07-09
2017-07-10
2017-07-11
2017-07-12
2017-07-13
2017-07-14
2017-07-15
2017-07-16
2017-07-17
2017-07-18
2017-07-19
2017-07-20
2017-07-21
2017-07-22
2017-07-23
2017-07-24
2017-07-25
2017-07-26
2017-07-27
2017-07-28
2017-07-29
2017-07-30
2017-07-31
296
2017-08-01
2017-08-02
2017-08-03
2017-08-04
2017-08-05
2017-08-06
2017-08-07
2017-08-08
2017-08-09
2017-08-10
2017-08-11
2017-08-12
2017-08-13
2017-08-14
2017-08-15
2017-08-16
2017-08-17
2017-08-18
2017-08-19
2017-08-20
2017-08-21
2017-08-22
2017-08-23
2017-08-24
2017-08-25
2017-08-26
2017-08-27
2017-08-28
2017-08-29
2017-08-30
2017-08-31
268
2017-09-01
2017-09-02
2017-09-03
2017-09-04
2017-09-05
2017-09-06
2017-09-07
2017-09-08
2017-09-09
2017-09-10
2017-09-11
2017-09-12
2017-09-13
2017-09-14
2017-09-15
2017-09-16
2017-09-17
2017-09-18
2017-09-19
2017-09-20
2017-09-21
2017-09-22
2017-09-23
2017-09-24
2017-09-25
2017-09-26
2017-09-27
2017-09-28
2017-09-29
2017-09-30
277
0
0
0

In [96]:
400*90


Out[96]:
36000

In [99]:
page_urls[6:-2]


Out[99]:
['https://en.wikipedia.org/wiki/Portal:Current_events/July_2010',
 'https://en.wikipedia.org/wiki/Portal:Current_events/August_2010',
 'https://en.wikipedia.org/wiki/Portal:Current_events/September_2010',
 'https://en.wikipedia.org/wiki/Portal:Current_events/October_2010',
 'https://en.wikipedia.org/wiki/Portal:Current_events/November_2010',
 'https://en.wikipedia.org/wiki/Portal:Current_events/December_2010',
 'https://en.wikipedia.org/wiki/Portal:Current_events/January_2011',
 'https://en.wikipedia.org/wiki/Portal:Current_events/February_2011',
 'https://en.wikipedia.org/wiki/Portal:Current_events/March_2011',
 'https://en.wikipedia.org/wiki/Portal:Current_events/April_2011',
 'https://en.wikipedia.org/wiki/Portal:Current_events/May_2011',
 'https://en.wikipedia.org/wiki/Portal:Current_events/June_2011',
 'https://en.wikipedia.org/wiki/Portal:Current_events/July_2011',
 'https://en.wikipedia.org/wiki/Portal:Current_events/August_2011',
 'https://en.wikipedia.org/wiki/Portal:Current_events/September_2011',
 'https://en.wikipedia.org/wiki/Portal:Current_events/October_2011',
 'https://en.wikipedia.org/wiki/Portal:Current_events/November_2011',
 'https://en.wikipedia.org/wiki/Portal:Current_events/December_2011',
 'https://en.wikipedia.org/wiki/Portal:Current_events/January_2012',
 'https://en.wikipedia.org/wiki/Portal:Current_events/February_2012',
 'https://en.wikipedia.org/wiki/Portal:Current_events/March_2012',
 'https://en.wikipedia.org/wiki/Portal:Current_events/April_2012',
 'https://en.wikipedia.org/wiki/Portal:Current_events/May_2012',
 'https://en.wikipedia.org/wiki/Portal:Current_events/June_2012',
 'https://en.wikipedia.org/wiki/Portal:Current_events/July_2012',
 'https://en.wikipedia.org/wiki/Portal:Current_events/August_2012',
 'https://en.wikipedia.org/wiki/Portal:Current_events/September_2012',
 'https://en.wikipedia.org/wiki/Portal:Current_events/October_2012',
 'https://en.wikipedia.org/wiki/Portal:Current_events/November_2012',
 'https://en.wikipedia.org/wiki/Portal:Current_events/December_2012',
 'https://en.wikipedia.org/wiki/Portal:Current_events/January_2013',
 'https://en.wikipedia.org/wiki/Portal:Current_events/February_2013',
 'https://en.wikipedia.org/wiki/Portal:Current_events/March_2013',
 'https://en.wikipedia.org/wiki/Portal:Current_events/April_2013',
 'https://en.wikipedia.org/wiki/Portal:Current_events/May_2013',
 'https://en.wikipedia.org/wiki/Portal:Current_events/June_2013',
 'https://en.wikipedia.org/wiki/Portal:Current_events/July_2013',
 'https://en.wikipedia.org/wiki/Portal:Current_events/August_2013',
 'https://en.wikipedia.org/wiki/Portal:Current_events/September_2013',
 'https://en.wikipedia.org/wiki/Portal:Current_events/October_2013',
 'https://en.wikipedia.org/wiki/Portal:Current_events/November_2013',
 'https://en.wikipedia.org/wiki/Portal:Current_events/December_2013',
 'https://en.wikipedia.org/wiki/Portal:Current_events/January_2014',
 'https://en.wikipedia.org/wiki/Portal:Current_events/February_2014',
 'https://en.wikipedia.org/wiki/Portal:Current_events/March_2014',
 'https://en.wikipedia.org/wiki/Portal:Current_events/April_2014',
 'https://en.wikipedia.org/wiki/Portal:Current_events/May_2014',
 'https://en.wikipedia.org/wiki/Portal:Current_events/June_2014',
 'https://en.wikipedia.org/wiki/Portal:Current_events/July_2014',
 'https://en.wikipedia.org/wiki/Portal:Current_events/August_2014',
 'https://en.wikipedia.org/wiki/Portal:Current_events/September_2014',
 'https://en.wikipedia.org/wiki/Portal:Current_events/October_2014',
 'https://en.wikipedia.org/wiki/Portal:Current_events/November_2014',
 'https://en.wikipedia.org/wiki/Portal:Current_events/December_2014',
 'https://en.wikipedia.org/wiki/Portal:Current_events/January_2015',
 'https://en.wikipedia.org/wiki/Portal:Current_events/February_2015',
 'https://en.wikipedia.org/wiki/Portal:Current_events/March_2015',
 'https://en.wikipedia.org/wiki/Portal:Current_events/April_2015',
 'https://en.wikipedia.org/wiki/Portal:Current_events/May_2015',
 'https://en.wikipedia.org/wiki/Portal:Current_events/June_2015',
 'https://en.wikipedia.org/wiki/Portal:Current_events/July_2015',
 'https://en.wikipedia.org/wiki/Portal:Current_events/August_2015',
 'https://en.wikipedia.org/wiki/Portal:Current_events/September_2015',
 'https://en.wikipedia.org/wiki/Portal:Current_events/October_2015',
 'https://en.wikipedia.org/wiki/Portal:Current_events/November_2015',
 'https://en.wikipedia.org/wiki/Portal:Current_events/December_2015',
 'https://en.wikipedia.org/wiki/Portal:Current_events/January_2016',
 'https://en.wikipedia.org/wiki/Portal:Current_events/February_2016',
 'https://en.wikipedia.org/wiki/Portal:Current_events/March_2016',
 'https://en.wikipedia.org/wiki/Portal:Current_events/April_2016',
 'https://en.wikipedia.org/wiki/Portal:Current_events/May_2016',
 'https://en.wikipedia.org/wiki/Portal:Current_events/June_2016',
 'https://en.wikipedia.org/wiki/Portal:Current_events/July_2016',
 'https://en.wikipedia.org/wiki/Portal:Current_events/August_2016',
 'https://en.wikipedia.org/wiki/Portal:Current_events/September_2016',
 'https://en.wikipedia.org/wiki/Portal:Current_events/October_2016',
 'https://en.wikipedia.org/wiki/Portal:Current_events/November_2016',
 'https://en.wikipedia.org/wiki/Portal:Current_events/December_2016',
 'https://en.wikipedia.org/wiki/Portal:Current_events/January_2017',
 'https://en.wikipedia.org/wiki/Portal:Current_events/February_2017',
 'https://en.wikipedia.org/wiki/Portal:Current_events/March_2017',
 'https://en.wikipedia.org/wiki/Portal:Current_events/April_2017',
 'https://en.wikipedia.org/wiki/Portal:Current_events/May_2017',
 'https://en.wikipedia.org/wiki/Portal:Current_events/June_2017',
 'https://en.wikipedia.org/wiki/Portal:Current_events/July_2017',
 'https://en.wikipedia.org/wiki/Portal:Current_events/August_2017',
 'https://en.wikipedia.org/wiki/Portal:Current_events/September_2017',
 'https://en.wikipedia.org/wiki/Portal:Current_events/October_2017']

In [ ]: