In [6]:

    

from selenium import webdriver
from selenium.webdriver.common.keys import Keys

driver = webdriver.Firefox()
driver.get("http://www.python.org")
assert "Python" in driver.title
elem = driver.find_element_by_name("q")
elem.clear()
elem.send_keys("pycon")
elem.send_keys(Keys.RETURN)
assert "No results found." not in driver.page_source


    

In [5]:

    

driver.close()


    

In [25]:

    

driver.get('http://www.yelp.com')


    

In [26]:

    

text_input = driver.find_element_by_id('find_desc')


    

In [27]:

    

text_input.clear()
text_input.send_keys('Chinese Food')


    

In [28]:

    

text_input.submit()


    

In [13]:

    

submit_button = driver.find_element_by_id('header-search-submit')


    

In [14]:

    

submit_button.click()


    

In [16]:

    

import bs4
soup = bs4.BeautifulSoup(driver.page_source)


    

    




C:\Users\Zac\Anaconda2\lib\site-packages\bs4\__init__.py:181: UserWarning: No parser was explicitly specified, so I'm using the best available HTML parser for this system ("lxml"). This usually isn't a problem, but if you run this code on another system, or in a different virtual environment, it may use a different parser and behave differently.

The code that caused this warning is on line 174 of the file C:\Users\Zac\Anaconda2\lib\runpy.py. To get rid of this warning, change code that looks like this:

 BeautifulSoup([your markup])

to this:

 BeautifulSoup([your markup], "lxml")

  markup_type=markup_type))

In [18]:

    

search_results = soup.findAll(attrs={'class': 'regular-search-result'})


    

In [19]:

    

[(res.find(attrs={'class':'biz-name js-analytics-click'}).span.text.strip(), 
  res.find(attrs={'class': 'biz-phone'}).text.strip()) for res in search_results]


    

    
Out[19]:





[(u'Old Peking', u'(201) 435-4664'),
 (u'Food King', u'(201) 332-5422'),
 (u'Hunan House I', u'(201) 876-9908'),
 (u'Chaan Teng', u'(212) 235-1886'),
 (u'Great Wall', u'(201) 433-1388'),
 (u'Shanghai Best', u'(201) 333-6868'),
 (u'The Wei', u'(718) 260-8888'),
 (u'Grand Sichuan', u'(201) 451-1960'),
 (u'Taste Of North China', u'(201) 521-1800'),
 (u'The Old Spot', u'(201) 205-2972')]

In [22]:

    

driver.save_screenshot('screen.png')


    

    
Out[22]:





True

In [24]:

    

driver.get('http://www.reddit.com')


    

In [29]:

    

#stealing requests
import requests
requests.get('https://api.stackexchange.com/2.2/search/advanced?key=U4DMV*8nvpm3EOpvf69Rxw((&site=stackoverflow&order=desc&sort=activity&q=how%20to%20find%20duplicates%20in%20a%20list&tagged=python&filter=default')


    

    
Out[29]:





<Response [200]>

In [30]:

    

r = requests.get('https://www.yelp.com/search_suggest/v2/prefetch?loc=Hoboken%2C+NJ&loc_name_param=find_loc&prefix=chinese+food+ta')


    

In [32]:

    

r.json()


    

    
Out[32]:





{u'response': [{u'prefix': u'chinese food ta',
   u'suggestions': [{u'ajax_data': u'null',
     u'css_class': None,
     u'is_bookmarked': None,
     u'is_typeahead': False,
     u'name_param': u'find_desc',
     u'num_checkins': None,
     u'query': u'Chinese food take out',
     u'redirect_url': None,
     u'subtitle': None,
     u'thumbnail': None,
     u'title': u'Chinese food take out',
     u'type': u'common',
     u'version': 1},
    {u'ajax_data': u'null',
     u'css_class': None,
     u'is_bookmarked': None,
     u'is_typeahead': False,
     u'name_param': u'find_desc',
     u'num_checkins': None,
     u'query': u'Chinese food takeout',
     u'redirect_url': None,
     u'subtitle': None,
     u'thumbnail': None,
     u'title': u'Chinese food takeout',
     u'type': u'common',
     u'version': 1},
    {u'ajax_data': u'null',
     u'css_class': None,
     u'is_bookmarked': None,
     u'is_typeahead': False,
     u'name_param': u'find_desc',
     u'num_checkins': None,
     u'query': u'Best chinese food take out',
     u'redirect_url': None,
     u'subtitle': None,
     u'thumbnail': None,
     u'title': u'Best chinese food take out',
     u'type': u'common',
     u'version': 1},
    {u'ajax_data': u'null',
     u'css_class': None,
     u'is_bookmarked': None,
     u'is_typeahead': False,
     u'name_param': u'find_desc',
     u'num_checkins': None,
     u'query': u'Great Wall Chinese Food Take-Out',
     u'redirect_url': u'/biz/great-wall-chinese-food-take-out-brooklyn',
     u'subtitle': u'84 Manhattan Avenue, Brooklyn',
     u'thumbnail': {u'color': None,
      u'key': u'https://s3-media1.fl.yelpcdn.com/assets/srv0/yelp_styleguide/8c6e08f3f2bb/assets/img/default_avatars/business_30_square.png',
      u'type': u'image-rounded'},
     u'title': u'Great Wall Chinese Food Take-Out',
     u'type': u'business',
     u'version': 1},
    {u'ajax_data': u'null',
     u'css_class': None,
     u'is_bookmarked': None,
     u'is_typeahead': False,
     u'name_param': u'find_desc',
     u'num_checkins': None,
     u'query': u'Chinese',
     u'redirect_url': None,
     u'subtitle': None,
     u'thumbnail': None,
     u'title': u'Chinese',
     u'type': u'category',
     u'version': 1},
    {u'ajax_data': u'null',
     u'css_class': None,
     u'is_bookmarked': None,
     u'is_typeahead': False,
     u'name_param': u'find_desc',
     u'num_checkins': None,
     u'query': u'Restaurants',
     u'redirect_url': None,
     u'subtitle': None,
     u'thumbnail': None,
     u'title': u'Restaurants',
     u'type': u'category',
     u'version': 1},
    {u'ajax_data': u'null',
     u'css_class': None,
     u'is_bookmarked': None,
     u'is_typeahead': False,
     u'name_param': u'find_desc',
     u'num_checkins': None,
     u'query': u'Thai',
     u'redirect_url': None,
     u'subtitle': None,
     u'thumbnail': None,
     u'title': u'Thai',
     u'type': u'category',
     u'version': 1}],
   u'typeahead': {u'typeahead_text': u'Chinese food take out', u'url': None},
   u'unique_request_id': u'4b5294baf6a5dcb6_0',
   u'unique_suggestion_id': u'4b5294baf6a5dcb6_0'}],
 u'unique_request_id': u'4b5294baf6a5dcb6'}

In [ ]: