In [53]:

    

from google_ngram import *
import re
import numpy as np
import pandas as pd
import time
from math import sqrt
import matplotlib.pyplot as plt
%matplotlib inline

redo_web_lookup = False


    

In [21]:

    

poem = """Turning and turning in the widening gyre
The falcon cannot hear the falconer;
Things fall apart; the centre cannot hold;
Mere anarchy is loosed upon the world,
The blood-dimmed tide is loosed, and everywhere
The ceremony of innocence is drowned;
The best lack all conviction, while the worst
Are full of passionate intensity.
Surely some revelation is at hand;
Surely the Second Coming is at hand.
The Second Coming! Hardly are those words out
When a vast image out of Spiritus Mundi
Troubles my sight: somewhere in sands of the desert
A shape with lion body and the head of a man,
A gaze blank and pitiless as the sun,
Is moving its slow thighs, while all about it
Reel shadows of the indignant desert birds.
The darkness drops again; but now I know
That twenty centuries of stony sleep
Were vexed to nightmare by a rocking cradle,
And what rough beast, its hour come round at last,
Slouches towards Bethlehem to be born?"""


    

In [23]:

    

if redo_web_lookup:
    text = re.sub('\r', '', poem)
    text = text.lower()
    text = re.sub('\-', ' ', text) # treat hyphen as space
    text = text.replace(';', '\n').replace('\.+', '\n').replace(':', '\n') # treat clause-breakers as line-breakers
    text = re.sub('[^a-z \n]', '', text)
    text = text.split('\n')
    for i in range(len(text)):
        text[i] = text[i].split(' ')
        text[i] = [x for x in text[i] if x != ''] # remove empty items
    text = [x for x in text if len(x) > 0] # remove empty lists
    print text


    

    




[['turning', 'and', 'turning', 'in', 'the', 'widening', 'gyre'], ['the', 'falcon', 'cannot', 'hear', 'the', 'falconer'], ['things', 'fall', 'apart'], ['the', 'centre', 'cannot', 'hold'], ['mere', 'anarchy', 'is', 'loosed', 'upon', 'the', 'world'], ['the', 'blood', 'dimmed', 'tide', 'is', 'loosed', 'and', 'everywhere'], ['the', 'ceremony', 'of', 'innocence', 'is', 'drowned'], ['the', 'best', 'lack', 'all', 'conviction', 'while', 'the', 'worst'], ['are', 'full', 'of', 'passionate', 'intensity'], ['surely', 'some', 'revelation', 'is', 'at', 'hand'], ['surely', 'the', 'second', 'coming', 'is', 'at', 'hand'], ['the', 'second', 'coming', 'hardly', 'are', 'those', 'words', 'out'], ['when', 'a', 'vast', 'image', 'out', 'of', 'spiritus', 'mundi'], ['troubles', 'my', 'sight'], ['somewhere', 'in', 'sands', 'of', 'the', 'desert'], ['a', 'shape', 'with', 'lion', 'body', 'and', 'the', 'head', 'of', 'a', 'man'], ['a', 'gaze', 'blank', 'and', 'pitiless', 'as', 'the', 'sun'], ['is', 'moving', 'its', 'slow', 'thighs', 'while', 'all', 'about', 'it'], ['reel', 'shadows', 'of', 'the', 'indignant', 'desert', 'birds'], ['the', 'darkness', 'drops', 'again'], ['but', 'now', 'i', 'know'], ['that', 'twenty', 'centuries', 'of', 'stony', 'sleep'], ['were', 'vexed', 'to', 'nightmare', 'by', 'a', 'rocking', 'cradle'], ['and', 'what', 'rough', 'beast', 'its', 'hour', 'come', 'round', 'at', 'last'], ['slouches', 'towards', 'bethlehem', 'to', 'be', 'born']]

In [19]:

    

if redo_web_lookup:
    # using exec and eval instead of nested lists because
    # lists have zero-based counting and n-grams have 1-based counting.

    for i in range(1, 6): #1-gram, 2-gram ... 5-gram
        exec "grams_" + str(i) + " = []" # creates lists grams_1, grams_2 ...
        for sentence in text:
            nwords = len(sentence)
            if i < nwords:
                for pos in range(len(sentence)-i+1):
                    exec "grams_" + str(i) + ".append(' '.join(sentence[pos:pos+i]))"
        exec "print grams_" + str(i)


    

    




['turning', 'and', 'turning', 'in', 'the', 'widening', 'gyre', 'the', 'falcon', 'cannot', 'hear', 'the', 'falconer', 'things', 'fall', 'apart', 'the', 'centre', 'cannot', 'hold', 'mere', 'anarchy', 'is', 'loosed', 'upon', 'the', 'world', 'the', 'blood', 'dimmed', 'tide', 'is', 'loosed', 'and', 'everywhere', 'the', 'ceremony', 'of', 'innocence', 'is', 'drowned', 'the', 'best', 'lack', 'all', 'conviction', 'while', 'the', 'worst', 'are', 'full', 'of', 'passionate', 'intensity', 'surely', 'some', 'revelation', 'is', 'at', 'hand', 'surely', 'the', 'second', 'coming', 'is', 'at', 'hand', 'the', 'second', 'coming', 'hardly', 'are', 'those', 'words', 'out', 'when', 'a', 'vast', 'image', 'out', 'of', 'spiritus', 'mundi', 'troubles', 'my', 'sight', 'somewhere', 'in', 'sands', 'of', 'the', 'desert', 'a', 'shape', 'with', 'lion', 'body', 'and', 'the', 'head', 'of', 'a', 'man', 'a', 'gaze', 'blank', 'and', 'pitiless', 'as', 'the', 'sun', 'is', 'moving', 'its', 'slow', 'thighs', 'while', 'all', 'about', 'it', 'reel', 'shadows', 'of', 'the', 'indignant', 'desert', 'birds', 'the', 'darkness', 'drops', 'again', 'but', 'now', 'i', 'know', 'that', 'twenty', 'centuries', 'of', 'stony', 'sleep', 'were', 'vexed', 'to', 'nightmare', 'by', 'a', 'rocking', 'cradle', 'and', 'what', 'rough', 'beast', 'its', 'hour', 'come', 'round', 'at', 'last', 'slouches', 'towards', 'bethlehem', 'to', 'be', 'born']
['turning and', 'and turning', 'turning in', 'in the', 'the widening', 'widening gyre', 'the falcon', 'falcon cannot', 'cannot hear', 'hear the', 'the falconer', 'things fall', 'fall apart', 'the centre', 'centre cannot', 'cannot hold', 'mere anarchy', 'anarchy is', 'is loosed', 'loosed upon', 'upon the', 'the world', 'the blood', 'blood dimmed', 'dimmed tide', 'tide is', 'is loosed', 'loosed and', 'and everywhere', 'the ceremony', 'ceremony of', 'of innocence', 'innocence is', 'is drowned', 'the best', 'best lack', 'lack all', 'all conviction', 'conviction while', 'while the', 'the worst', 'are full', 'full of', 'of passionate', 'passionate intensity', 'surely some', 'some revelation', 'revelation is', 'is at', 'at hand', 'surely the', 'the second', 'second coming', 'coming is', 'is at', 'at hand', 'the second', 'second coming', 'coming hardly', 'hardly are', 'are those', 'those words', 'words out', 'when a', 'a vast', 'vast image', 'image out', 'out of', 'of spiritus', 'spiritus mundi', 'troubles my', 'my sight', 'somewhere in', 'in sands', 'sands of', 'of the', 'the desert', 'a shape', 'shape with', 'with lion', 'lion body', 'body and', 'and the', 'the head', 'head of', 'of a', 'a man', 'a gaze', 'gaze blank', 'blank and', 'and pitiless', 'pitiless as', 'as the', 'the sun', 'is moving', 'moving its', 'its slow', 'slow thighs', 'thighs while', 'while all', 'all about', 'about it', 'reel shadows', 'shadows of', 'of the', 'the indignant', 'indignant desert', 'desert birds', 'the darkness', 'darkness drops', 'drops again', 'but now', 'now i', 'i know', 'that twenty', 'twenty centuries', 'centuries of', 'of stony', 'stony sleep', 'were vexed', 'vexed to', 'to nightmare', 'nightmare by', 'by a', 'a rocking', 'rocking cradle', 'and what', 'what rough', 'rough beast', 'beast its', 'its hour', 'hour come', 'come round', 'round at', 'at last', 'slouches towards', 'towards bethlehem', 'bethlehem to', 'to be', 'be born']
['turning and turning', 'and turning in', 'turning in the', 'in the widening', 'the widening gyre', 'the falcon cannot', 'falcon cannot hear', 'cannot hear the', 'hear the falconer', 'the centre cannot', 'centre cannot hold', 'mere anarchy is', 'anarchy is loosed', 'is loosed upon', 'loosed upon the', 'upon the world', 'the blood dimmed', 'blood dimmed tide', 'dimmed tide is', 'tide is loosed', 'is loosed and', 'loosed and everywhere', 'the ceremony of', 'ceremony of innocence', 'of innocence is', 'innocence is drowned', 'the best lack', 'best lack all', 'lack all conviction', 'all conviction while', 'conviction while the', 'while the worst', 'are full of', 'full of passionate', 'of passionate intensity', 'surely some revelation', 'some revelation is', 'revelation is at', 'is at hand', 'surely the second', 'the second coming', 'second coming is', 'coming is at', 'is at hand', 'the second coming', 'second coming hardly', 'coming hardly are', 'hardly are those', 'are those words', 'those words out', 'when a vast', 'a vast image', 'vast image out', 'image out of', 'out of spiritus', 'of spiritus mundi', 'somewhere in sands', 'in sands of', 'sands of the', 'of the desert', 'a shape with', 'shape with lion', 'with lion body', 'lion body and', 'body and the', 'and the head', 'the head of', 'head of a', 'of a man', 'a gaze blank', 'gaze blank and', 'blank and pitiless', 'and pitiless as', 'pitiless as the', 'as the sun', 'is moving its', 'moving its slow', 'its slow thighs', 'slow thighs while', 'thighs while all', 'while all about', 'all about it', 'reel shadows of', 'shadows of the', 'of the indignant', 'the indignant desert', 'indignant desert birds', 'the darkness drops', 'darkness drops again', 'but now i', 'now i know', 'that twenty centuries', 'twenty centuries of', 'centuries of stony', 'of stony sleep', 'were vexed to', 'vexed to nightmare', 'to nightmare by', 'nightmare by a', 'by a rocking', 'a rocking cradle', 'and what rough', 'what rough beast', 'rough beast its', 'beast its hour', 'its hour come', 'hour come round', 'come round at', 'round at last', 'slouches towards bethlehem', 'towards bethlehem to', 'bethlehem to be', 'to be born']
['turning and turning in', 'and turning in the', 'turning in the widening', 'in the widening gyre', 'the falcon cannot hear', 'falcon cannot hear the', 'cannot hear the falconer', 'mere anarchy is loosed', 'anarchy is loosed upon', 'is loosed upon the', 'loosed upon the world', 'the blood dimmed tide', 'blood dimmed tide is', 'dimmed tide is loosed', 'tide is loosed and', 'is loosed and everywhere', 'the ceremony of innocence', 'ceremony of innocence is', 'of innocence is drowned', 'the best lack all', 'best lack all conviction', 'lack all conviction while', 'all conviction while the', 'conviction while the worst', 'are full of passionate', 'full of passionate intensity', 'surely some revelation is', 'some revelation is at', 'revelation is at hand', 'surely the second coming', 'the second coming is', 'second coming is at', 'coming is at hand', 'the second coming hardly', 'second coming hardly are', 'coming hardly are those', 'hardly are those words', 'are those words out', 'when a vast image', 'a vast image out', 'vast image out of', 'image out of spiritus', 'out of spiritus mundi', 'somewhere in sands of', 'in sands of the', 'sands of the desert', 'a shape with lion', 'shape with lion body', 'with lion body and', 'lion body and the', 'body and the head', 'and the head of', 'the head of a', 'head of a man', 'a gaze blank and', 'gaze blank and pitiless', 'blank and pitiless as', 'and pitiless as the', 'pitiless as the sun', 'is moving its slow', 'moving its slow thighs', 'its slow thighs while', 'slow thighs while all', 'thighs while all about', 'while all about it', 'reel shadows of the', 'shadows of the indignant', 'of the indignant desert', 'the indignant desert birds', 'that twenty centuries of', 'twenty centuries of stony', 'centuries of stony sleep', 'were vexed to nightmare', 'vexed to nightmare by', 'to nightmare by a', 'nightmare by a rocking', 'by a rocking cradle', 'and what rough beast', 'what rough beast its', 'rough beast its hour', 'beast its hour come', 'its hour come round', 'hour come round at', 'come round at last', 'slouches towards bethlehem to', 'towards bethlehem to be', 'bethlehem to be born']
['turning and turning in the', 'and turning in the widening', 'turning in the widening gyre', 'the falcon cannot hear the', 'falcon cannot hear the falconer', 'mere anarchy is loosed upon', 'anarchy is loosed upon the', 'is loosed upon the world', 'the blood dimmed tide is', 'blood dimmed tide is loosed', 'dimmed tide is loosed and', 'tide is loosed and everywhere', 'the ceremony of innocence is', 'ceremony of innocence is drowned', 'the best lack all conviction', 'best lack all conviction while', 'lack all conviction while the', 'all conviction while the worst', 'surely some revelation is at', 'some revelation is at hand', 'surely the second coming is', 'the second coming is at', 'second coming is at hand', 'the second coming hardly are', 'second coming hardly are those', 'coming hardly are those words', 'hardly are those words out', 'when a vast image out', 'a vast image out of', 'vast image out of spiritus', 'image out of spiritus mundi', 'somewhere in sands of the', 'in sands of the desert', 'a shape with lion body', 'shape with lion body and', 'with lion body and the', 'lion body and the head', 'body and the head of', 'and the head of a', 'the head of a man', 'a gaze blank and pitiless', 'gaze blank and pitiless as', 'blank and pitiless as the', 'and pitiless as the sun', 'is moving its slow thighs', 'moving its slow thighs while', 'its slow thighs while all', 'slow thighs while all about', 'thighs while all about it', 'reel shadows of the indignant', 'shadows of the indignant desert', 'of the indignant desert birds', 'that twenty centuries of stony', 'twenty centuries of stony sleep', 'were vexed to nightmare by', 'vexed to nightmare by a', 'to nightmare by a rocking', 'nightmare by a rocking cradle', 'and what rough beast its', 'what rough beast its hour', 'rough beast its hour come', 'beast its hour come round', 'its hour come round at', 'hour come round at last', 'slouches towards bethlehem to be', 'towards bethlehem to be born']

In [41]:

    

if redo_web_lookup:
    for i in [1, 2, 3, 4, 5]:
        exec("df_" + str(i) + "= pd.DataFrame()")
        for term in eval("grams_" + str(i)):
            try:
                g = Gngram([term], years=[1880,1980], case_insensitive=True)
                exec("df_" + str(i) + " = pd.concat([df_" + str(i) +", g.df_parents], axis=1)")
                print term, 'added; ',
            except SyntaxError: # unexpected EOF when parsing html if search term not found
                print '\n'+term, '################## skipped'
            time.sleep(10) # so Google doesn't deny access
        exec ("df_" + str(i) + ".to_csv('second_coming_" + str(i) + "_grams.csv')")


    

    




turning added;  and added;  turning added;  in added;  the added;  widening added;  gyre added;  the added;  falcon added;  cannot added;  hear added;  the added;  falconer added;  things added;  fall added;  apart added;  the added;  centre added;  cannot added;  hold added;  mere added;  anarchy added;  is added;  loosed added;  upon added;  the added;  world added;  the added;  blood added;  dimmed added;  tide added;  is added;  loosed added;  and added;  everywhere added;  the added;  ceremony added;  of added;  innocence added;  is added;  drowned added;  the added;  best added;  lack added;  all added;  conviction added;  while added;  the added;  worst added;  are added;  full added;  of added;  passionate added;  intensity added;  surely added;  some added;  revelation added;  is added;  at added;  hand added;  surely added;  the added;  second added;  coming added;  is added;  at added;  hand added;  the added;  second added;  coming added;  hardly added;  are added;  those added;  words added;  out added;  when added;  a added;  vast added;  image added;  out added;  of added;  spiritus added;  mundi added;  troubles added;  my added;  sight added;  somewhere added;  in added;  sands added;  of added;  the added;  desert added;  a added;  shape added;  with added;  lion added;  body added;  and added;  the added;  head added;  of added;  a added;  man added;  a added;  gaze added;  blank added;  and added;  pitiless added;  as added;  the added;  sun added;  is added;  moving added;  its added;  slow added;  thighs added;  while added;  all added;  about added;  it added;  reel added;  shadows added;  of added;  the added;  indignant added;  desert added;  birds added;  the added;  darkness added;  drops added;  again added;  but added;  now added;  i added;  know added;  that added;  twenty added;  centuries added;  of added;  stony added;  sleep added;  were added;  vexed added;  to added;  nightmare added;  by added;  a added;  rocking added;  cradle added;  and added;  what added;  rough added;  beast added;  its added;  hour added;  come added;  round added;  at added;  last added;  slouches added;  towards added;  bethlehem added;  to added;  be added;  born added;  turning and added;  and turning added;  turning in added;  in the added;  the widening added;  widening gyre added;  the falcon added;  falcon cannot added;  cannot hear added;  hear the added;  the falconer added;  things fall added;  fall apart added;  the centre added;  centre cannot added;  cannot hold added;  mere anarchy added;  anarchy is added;  is loosed added;  loosed upon added;  upon the added;  the world added;  the blood added;  blood dimmed added;  dimmed tide added;  tide is added;  is loosed added;  loosed and added;  and everywhere added;  the ceremony added;  ceremony of added;  of innocence added;  innocence is added;  is drowned added;  the best added;  best lack added;  lack all added;  all conviction added;  conviction while added;  while the added;  the worst added;  are full added;  full of added;  of passionate added;  passionate intensity added;  surely some added;  some revelation added;  revelation is added;  is at added;  at hand added;  surely the added;  the second added;  second coming added;  coming is added;  is at added;  at hand added;  the second added;  second coming added;  coming hardly added;  hardly are added;  are those added;  those words added;  words out added;  when a added;  a vast added;  vast image added;  image out added;  out of added;  of spiritus added;  spiritus mundi added;  troubles my added;  my sight added;  somewhere in added;  in sands added;  sands of added;  of the added;  the desert added;  a shape added;  shape with added;  with lion added;  lion body added;  body and added;  and the added;  the head added;  head of added;  of a added;  a man added;  a gaze added;  gaze blank added;  blank and added;  and pitiless added;  pitiless as added;  as the added;  the sun added;  is moving added;  moving its added;  its slow added;  slow thighs added;  thighs while added;  while all added;  all about added;  about it added;  reel shadows added;  shadows of added;  of the added;  the indignant added;  indignant desert added;  desert birds added;  the darkness added;  darkness drops added;  drops again added;  but now added;  now i added;  i know added;  that twenty added;  twenty centuries added;  centuries of added;  of stony added;  stony sleep added;  were vexed added;  vexed to added;  to nightmare added;  nightmare by added;  by a added;  a rocking added;  rocking cradle added;  and what added;  what rough added;  rough beast added;  beast its added;  its hour added;  hour come added;  come round added;  round at added;  at last added;  slouches towards added;  towards bethlehem added;  bethlehem to added;  to be added;  be born added;  turning and turning added;  and turning in added;  turning in the added;  in the widening added;  the widening gyre added;  the falcon cannot added;  falcon cannot hear added;  cannot hear the added;  hear the falconer added;  the centre cannot added;  centre cannot hold added;  mere anarchy is added;  anarchy is loosed added;  is loosed upon added;  loosed upon the added;  upon the world added;  the blood dimmed added;  blood dimmed tide added;  dimmed tide is added;  tide is loosed added;  is loosed and added;  loosed and everywhere added;  the ceremony of added;  ceremony of innocence added;  of innocence is added;  innocence is drowned added;  the best lack added;  best lack all added;  lack all conviction added;  all conviction while added;  conviction while the added;  while the worst added;  are full of added;  full of passionate added;  of passionate intensity added;  surely some revelation added;  some revelation is added;  revelation is at added;  is at hand added;  surely the second added;  the second coming added;  second coming is added;  coming is at added;  is at hand added;  the second coming added;  
second coming hardly ################## skipped

coming hardly are ################## skipped
hardly are those added;  are those words added;  those words out added;  when a vast added;  a vast image added;  vast image out added;  image out of added;  out of spiritus added;  of spiritus mundi added;  somewhere in sands added;  in sands of added;  sands of the added;  of the desert added;  a shape with added;  shape with lion added;  with lion body added;  lion body and added;  body and the added;  and the head added;  the head of added;  head of a added;  of a man added;  a gaze blank added;  gaze blank and added;  blank and pitiless added;  and pitiless as added;  pitiless as the added;  as the sun added;  is moving its added;  moving its slow added;  its slow thighs added;  
slow thighs while ################## skipped

thighs while all ################## skipped
while all about added;  all about it added;  reel shadows of added;  shadows of the added;  of the indignant added;  the indignant desert added;  indignant desert birds added;  the darkness drops added;  darkness drops again added;  but now i added;  now i know added;  that twenty centuries added;  twenty centuries of added;  centuries of stony added;  of stony sleep added;  were vexed to added;  vexed to nightmare added;  to nightmare by added;  nightmare by a added;  by a rocking added;  a rocking cradle added;  and what rough added;  what rough beast added;  rough beast its added;  beast its hour added;  its hour come added;  hour come round added;  come round at added;  round at last added;  slouches towards bethlehem added;  towards bethlehem to added;  bethlehem to be added;  to be born added;  turning and turning in added;  and turning in the added;  turning in the widening added;  in the widening gyre added;  the falcon cannot hear added;  falcon cannot hear the added;  cannot hear the falconer added;  mere anarchy is loosed added;  anarchy is loosed upon added;  is loosed upon the added;  loosed upon the world added;  the blood dimmed tide added;  blood dimmed tide is added;  dimmed tide is loosed added;  tide is loosed and added;  is loosed and everywhere added;  the ceremony of innocence added;  ceremony of innocence is added;  of innocence is drowned added;  the best lack all added;  best lack all conviction added;  lack all conviction while added;  all conviction while the added;  conviction while the worst added;  are full of passionate added;  full of passionate intensity added;  surely some revelation is added;  some revelation is at added;  revelation is at hand added;  surely the second coming added;  the second coming is added;  second coming is at added;  coming is at hand added;  
the second coming hardly ################## skipped

second coming hardly are ################## skipped

coming hardly are those ################## skipped
hardly are those words added;  are those words out added;  when a vast image added;  a vast image out added;  vast image out of added;  image out of spiritus added;  out of spiritus mundi added;  somewhere in sands of added;  in sands of the added;  sands of the desert added;  a shape with lion added;  shape with lion body added;  with lion body and added;  lion body and the added;  body and the head added;  and the head of added;  the head of a added;  head of a man added;  a gaze blank and added;  gaze blank and pitiless added;  blank and pitiless as added;  and pitiless as the added;  pitiless as the sun added;  is moving its slow added;  moving its slow thighs added;  
its slow thighs while ################## skipped

slow thighs while all ################## skipped

thighs while all about ################## skipped
while all about it added;  reel shadows of the added;  shadows of the indignant added;  of the indignant desert added;  the indignant desert birds added;  that twenty centuries of added;  twenty centuries of stony added;  centuries of stony sleep added;  were vexed to nightmare added;  vexed to nightmare by added;  to nightmare by a added;  nightmare by a rocking added;  by a rocking cradle added;  and what rough beast added;  
what rough beast its ################## skipped
rough beast its hour added;  beast its hour come added;  its hour come round added;  hour come round at added;  come round at last added;  slouches towards bethlehem to added;  towards bethlehem to be added;  bethlehem to be born added;  turning and turning in the added;  and turning in the widening added;  turning in the widening gyre added;  
the falcon cannot hear the ################## skipped

falcon cannot hear the falconer ################## skipped
mere anarchy is loosed upon added;  anarchy is loosed upon the added;  is loosed upon the world added;  the blood dimmed tide is added;  blood dimmed tide is loosed added;  dimmed tide is loosed and added;  tide is loosed and everywhere added;  the ceremony of innocence is added;  ceremony of innocence is drowned added;  the best lack all conviction added;  best lack all conviction while added;  lack all conviction while the added;  all conviction while the worst added;  surely some revelation is at added;  some revelation is at hand added;  surely the second coming is added;  the second coming is at added;  second coming is at hand added;  
the second coming hardly are ################## skipped

second coming hardly are those ################## skipped

coming hardly are those words ################## skipped
hardly are those words out added;  when a vast image out added;  a vast image out of added;  vast image out of spiritus added;  image out of spiritus mundi added;  somewhere in sands of the added;  in sands of the desert added;  a shape with lion body added;  shape with lion body and added;  with lion body and the added;  lion body and the head added;  body and the head of added;  and the head of a added;  the head of a man added;  a gaze blank and pitiless added;  gaze blank and pitiless as added;  blank and pitiless as the added;  and pitiless as the sun added;  is moving its slow thighs added;  
moving its slow thighs while ################## skipped

its slow thighs while all ################## skipped

slow thighs while all about ################## skipped

thighs while all about it ################## skipped
reel shadows of the indignant added;  shadows of the indignant desert added;  of the indignant desert birds added;  that twenty centuries of stony added;  twenty centuries of stony sleep added;  were vexed to nightmare by added;  vexed to nightmare by a added;  to nightmare by a rocking added;  nightmare by a rocking cradle added;  
and what rough beast its ################## skipped

what rough beast its hour ################## skipped

rough beast its hour come ################## skipped

beast its hour come round ################## skipped

its hour come round at ################## skipped

hour come round at last ################## skipped

slouches towards bethlehem to be ################## skipped

towards bethlehem to be born ################## skipped

In [62]:

    

redo_graphs = True
filenames = ['second_coming_1_grams.csv', 'second_coming_2_grams.csv', 'second_coming_3_grams.csv', 
             'second_coming_4_grams.csv', 'second_coming_5_grams.csv']


    

In [68]:

    

if redo_graphs:
    # 1-grams
    ngram_number = 1
    df = pd.read_csv(filenames[ngram_number-1], index_col=0)
    max_y = sqrt(df.max().max())
    num_charts = len(df.columns)
    num_cols = 5
    num_rows = num_charts / num_cols
    if num_charts % num_cols != 0:
        num_rows += 1

    f, axarr = plt.subplots(num_rows, num_cols, figsize=(16, 58))
    plt.subplots_adjust(hspace = .25, wspace=.3)
    #plt.locator_params(axis = 'y', nbins = 2)
    #f.axes.get_xaxis().set_visible(False)
    c_row = 0
    c_col = 0
    for column in df.columns:
        alpha = sqrt(df[column].max()) / max_y
        axarr[c_row, c_col].plot(df.index, df[column])
        title = df[column].name.replace(' (All)', '')
        title = re.sub('\.[0-9]{,2}$', '', title)
        axarr[c_row, c_col].set_title(title, fontdict ={'fontsize': 10})
        axarr[c_row, c_col].set_ylim(0, df[column].max())
        axarr[c_row, c_col].set_xlim(1880, 1980)
        axarr[c_row, c_col].get_xaxis().set_visible(False)
        axarr[c_row, c_col].locator_params(axis = 'y', nbins=4)
        #axarr[c_row, c_col].yaxis.set_ticks(np.arange(0, df[column].max()))
        axarr[c_row, c_col].fill_between(df.index, df[column], color='b', alpha=alpha, interpolate=True)
        c_col = (c_col + 1) % num_cols
        if c_col == 0: # new row
            c_row += 1


    

In [69]:

    

if redo_graphs:
    # 2-grams
    ngram_number = 2
    df = pd.read_csv(filenames[ngram_number-1], index_col=0)
    max_y = sqrt(df.max().max())
    num_charts = len(df.columns)
    num_cols = 5
    num_rows = num_charts / num_cols
    if num_charts % num_cols != 0:
        num_rows += 1

    f, axarr = plt.subplots(num_rows, num_cols, figsize=(16, num_rows * 2))
    plt.subplots_adjust(hspace = .25, wspace=.3)
    #plt.locator_params(axis = 'y', nbins = 2)
    #f.axes.get_xaxis().set_visible(False)
    c_row = 0
    c_col = 0
    for column in df.columns:
        alpha = sqrt(df[column].max()) / max_y
        axarr[c_row, c_col].plot(df.index, df[column])
        title = df[column].name.replace(' (All)', '')
        title = re.sub('\.[0-9]{,2}$', '', title)
        axarr[c_row, c_col].set_title(title, fontdict ={'fontsize': 10})
        axarr[c_row, c_col].set_ylim(0, df[column].max())
        axarr[c_row, c_col].set_xlim(1880, 1980)
        axarr[c_row, c_col].get_xaxis().set_visible(False)
        axarr[c_row, c_col].locator_params(axis = 'y', nbins=4)
        #axarr[c_row, c_col].yaxis.set_ticks(np.arange(0, df[column].max()))
        axarr[c_row, c_col].fill_between(df.index, df[column], color='b', alpha=alpha, interpolate=True)
        c_col = (c_col + 1) % num_cols
        if c_col == 0: # new row
            c_row += 1


    

In [70]:

    

if redo_graphs:
    # 3-grams
    ngram_number = 3
    df = pd.read_csv(filenames[ngram_number - 1], index_col=0)
    max_y = sqrt(df.max().max())
    num_charts = len(df.columns)
    num_cols = 5
    num_rows = num_charts / num_cols
    if num_charts % num_cols != 0:
        num_rows += 1

    f, axarr = plt.subplots(num_rows, num_cols, figsize=(16, num_rows * 2))
    plt.subplots_adjust(hspace = .25, wspace=.3)
    #plt.locator_params(axis = 'y', nbins = 2)
    #f.axes.get_xaxis().set_visible(False)
    c_row = 0
    c_col = 0
    for column in df.columns:
        alpha = sqrt(df[column].max()) / max_y
        axarr[c_row, c_col].plot(df.index, df[column])
        title = df[column].name.replace(' (All)', '')
        title = re.sub('\.[0-9]{,2}$', '', title)
        axarr[c_row, c_col].set_title(title, fontdict ={'fontsize': 10})
        axarr[c_row, c_col].set_ylim(0, df[column].max())
        axarr[c_row, c_col].set_xlim(1880, 1980)
        axarr[c_row, c_col].get_xaxis().set_visible(False)
        axarr[c_row, c_col].locator_params(axis = 'y', nbins=4)
        #axarr[c_row, c_col].yaxis.set_ticks(np.arange(0, df[column].max()))
        axarr[c_row, c_col].fill_between(df.index, df[column], color='b', alpha=alpha, interpolate=True)
        c_col = (c_col + 1) % num_cols
        if c_col == 0: # new row
            c_row += 1


    

In [71]:

    

if redo_graphs:
    # 4-grams
    ngram_number = 4
    df = pd.read_csv(filenames[ngram_number-1], index_col=0)
    max_y = sqrt(df.max().max())
    num_charts = len(df.columns)
    num_cols = 5
    num_rows = num_charts / num_cols
    if num_charts % num_cols != 0:
        num_rows += 1

    f, axarr = plt.subplots(num_rows, num_cols, figsize=(16, num_rows * 2))
    plt.subplots_adjust(hspace = .25, wspace=.3)
    #plt.locator_params(axis = 'y', nbins = 2)
    #f.axes.get_xaxis().set_visible(False)
    c_row = 0
    c_col = 0
    for column in df.columns:
        alpha = sqrt(df[column].max()) / max_y
        axarr[c_row, c_col].plot(df.index, df[column])
        title = df[column].name.replace(' (All)', '')
        title = re.sub('\.[0-9]{,2}$', '', title)
        axarr[c_row, c_col].set_title(title, fontdict ={'fontsize': 10})
        axarr[c_row, c_col].set_ylim(0, df[column].max())
        axarr[c_row, c_col].set_xlim(1880, 1980)
        axarr[c_row, c_col].get_xaxis().set_visible(False)
        axarr[c_row, c_col].locator_params(axis = 'y', nbins=4)
        #axarr[c_row, c_col].yaxis.set_ticks(np.arange(0, df[column].max()))
        axarr[c_row, c_col].fill_between(df.index, df[column], color='b', alpha=alpha, interpolate=True)
        c_col = (c_col + 1) % num_cols
        if c_col == 0: # new row
            c_row += 1


    

In [72]:

    

if redo_graphs:
    # 5-grams
    ngram_number = 5
    df = pd.read_csv(filenames[ngram_number-1], index_col=0)
    max_y = sqrt(df.max().max())
    num_charts = len(df.columns)
    num_cols = 5
    num_rows = num_charts / num_cols
    if num_charts % num_cols != 0:
        num_rows += 1

    f, axarr = plt.subplots(num_rows, num_cols, figsize=(16, num_rows * 2))
    plt.subplots_adjust(hspace = .25, wspace=.3)
    #plt.locator_params(axis = 'y', nbins = 2)
    #f.axes.get_xaxis().set_visible(False)
    c_row = 0
    c_col = 0
    for column in df.columns:
        alpha = sqrt(df[column].max()) / max_y
        axarr[c_row, c_col].plot(df.index, df[column])
        title = df[column].name.replace(' (All)', '')
        title = re.sub('\.[0-9]{,2}$', '', title)
        axarr[c_row, c_col].set_title(title, fontdict ={'fontsize': 10})
        axarr[c_row, c_col].set_ylim(0, df[column].max())
        axarr[c_row, c_col].set_xlim(1880, 1980)
        axarr[c_row, c_col].get_xaxis().set_visible(False)
        axarr[c_row, c_col].locator_params(axis = 'y', nbins=4)
        #axarr[c_row, c_col].yaxis.set_ticks(np.arange(0, df[column].max()))
        axarr[c_row, c_col].fill_between(df.index, df[column], color='b', alpha=alpha, interpolate=True)
        c_col = (c_col + 1) % num_cols
        if c_col == 0: # new row
            c_row += 1


    

In [ ]: