In [36]:

    

from selenium import webdriver
from selenium.webdriver.common.keys import Keys
from bs4 import BeautifulSoup


    

In [37]:

    

def get_source_code(page_source):
    soup = BeautifulSoup(page_source, "lxml")
    for p_elem in soup.find_all('textarea'):
        #print(p_elem.get_text())
        with open("code.txt", "a") as f:
            f.write(p_elem.get_text())


    

In [38]:

    

driver = webdriver.Firefox()
driver.get("http://pythonforengineers.com/articles/")


    

In [39]:

    

search_field = driver.find_element_by_name("s")
search_field.send_keys("build reddit bot")


    

In [40]:

    

search_field.send_keys(Keys.RETURN)


    

In [41]:

    

link = driver.find_element_by_link_text("Build a Reddit Bot Part 1")
link.click()
get_source_code(driver.page_source)


    

In [42]:

    

while(True):
    try:
        link2 = driver.find_element_by_partial_link_text("Next")
        link2.click()
        get_source_code(driver.page_source)
    except:
        break


    

In [43]:

    

driver.close()


    

In [ ]: