In [1]:

    

from selenium import webdriver
from bs4 import BeautifulSoup
import pandas as pd
import time


    

In [2]:

    

class ProductListing:
    def __init__(self, soup):
        self.soup = soup
    
    def name(self):
        return self.soup.a.string
    
    def link(self):
        return self.soup.a.get("href")
    
    def itemId(self):
        return self.soup.a.get("href").split('/')[-1]
    
    def rating(self):
        return self.soup.find('span', class_='bv-text-link').string
    
    def price(self):
        return self.soup.find("span", class_="actual-price").meta.string.strip()


    

In [3]:

    

# setup chrome webdriver
path_to_chromedriver = '/Users/csiu/lib/chromedriver'
browser = webdriver.Chrome(executable_path = path_to_chromedriver)


    

In [4]:

    

# get info
url = 'http://www.gouletpens.com/noodlers-bottled-ink/c/128/?pageSize=500'
browser.get(url)


    

In [5]:

    

element = browser.find_elements_by_xpath('//div[@class="mz-productlisting-info"]')
time.sleep(10) ## to give the browser time for js to generate content (?)


    

In [6]:

    

data = {
    "name":[],
    "itemId":[],
    "rating":[],
    "price":[]
}
for e in element:
    htmlData = e.get_attribute('innerHTML')
    soup = BeautifulSoup(htmlData, 'html.parser')
    
    pl = ProductListing(soup)
    data["name"].append(pl.name())
    data["itemId"].append(pl.itemId())
    data["rating"].append(pl.rating())
    data["price"].append(pl.price())


    

In [7]:

    

df = pd.DataFrame(data)
df.sort(columns="rating", ascending=False, inplace=True)
df.head(10)


    

    
Out[7]:






  

    

      

      
itemId
      
name
      
price
      
rating
    
  
  

    

      
120
      
 N19007
      
                Noodler's Violet (3oz Bottled Ink)
      
 $12.50
      
 5.0
    
    

      
115
      
 N19176
      
               Noodler's Sunrise (3oz Bottled Ink)
      
 $10.00
      
 5.0
    
    

      
113
      
 N19175
      
      Noodler's St. Patty's Eire (3oz Bottled Ink)
      
 $12.50
      
 5.0
    
    

      
117
      
 N19005
      
             Noodler's Turquoise (3oz Bottled Ink)
      
 $12.50
      
 4.9
    
    

      
111
      
 N19036
      
           Noodler's Shah's Rose (3oz Bottled Ink)
      
 $12.50
      
 4.9
    
    

      
79 
      
 N19029
      
      Noodler's Navajo Turquoise (3oz Bottled Ink)
      
 $12.50
      
 4.9
    
    

      
64 
      
 N19171
      
              Noodler's Hellfire (3oz Bottled Ink)
      
 $12.50
      
 4.8
    
    

      
109
      
 N19032
      
          Noodler's Saguaro Wine (3oz Bottled Ink)
      
 $12.50
      
 4.8
    
    

      
17 
      
 N19024
      
                Noodler's Beaver (3oz Bottled Ink)
      
 $12.50
      
 4.8
    
    

      
125
      
 N19819
      
 Noodler's X-Feather (4.5oz Bottled Ink) with F...
      
 $19.00
      
 4.8
    
  

In [8]:

    

df.to_csv("goulet-ranks.csv", index=False)