This notebook serves to sort English Wikipedia section headers by frequency as related to this research project.
In [1]:
import numpy as np
import pandas as pd
In [2]:
# read in headers file by chunks of 100000 to conserve memory
# https://stackoverflow.com/questions/25962114/how-to-read-a-6-gb-csv-file-with-pandas
tp = pd.read_csv('enwiki_20161101_headings_2.tsv', sep='\t', header=0, dtype={'page_id': np.int32, 'page_title': object, 'page_ns': np.int16, 'heading_level': np.int8, 'heading_text': object}, iterator=True, chunksize=100000)
In [3]:
# concatenate all rows into a pandas dataframe
en_DF = pd.concat([chunk for chunk in tp])
In [6]:
en_DF.head()
Out[6]:
In [7]:
en_DF.page_ns.unique()
Out[7]:
In [8]:
# determine number of unique articles
len(en_DF.page_title.unique())
Out[8]:
In [9]:
# remove leading and trailing whitespace from heading_text column
en_DF['heading_text'] = pd.core.strings.str_strip(en_DF['heading_text'])
In [10]:
# groupby heading_text and count the number of unique page_titles each heading appears in
# sort in descending order
# this returns a pandas series object
article_count = en_DF.groupby('heading_text')['page_title'].apply(lambda x: len(x.unique())).sort_values(ascending=False)
In [11]:
# turn pandas series object into pandas dataframe
en_article_count_DF = pd.DataFrame({'section_title':article_count.index, 'number_of_articles':article_count.values})
In [12]:
en_article_count_DF.head()
Out[12]:
In [13]:
# add a column for the percentage of articles that header appears in
en_article_count_DF['article_percentage'] = (en_article_count_DF['number_of_articles']/5275388)*100
In [14]:
# set pandas options to display 100 rows
# round percentage to 2 decimal places and show top 100 results
pd.options.display.max_rows = 100
en_article_count_DF.round({'article_percentage': 2}).head(100)
Out[14]: