Scraping Tutorial https://www.youtube.com/watch?v=XjNm9bazxn8&index=5&list=WL
In [4]:
import requests
from bs4 import BeautifulSoup
In [20]:
def trade_spider(max_pages):
page = 1
while page < max_pages:
url = "http://www.imagefap.com/gallery.php?type=1&gen=44&userid=&search=&page=" + str(page)
source_code = requests.get(url)
plain_text = source_code.text
soup = BeautifulSoup(plain_text)
for link in soup.findAll('a', {'class': 'gal_title'}):
href = "http://www.imagefap.com" + link.get('href')
title = link.get('title')
# print(href)
get_single_item_data(href)
page += 1
In [19]:
trade_spider(3)
In [32]:
def get_single_item_data(item_url):
source_code = requests.get(item_url)
plain_text = source_code.text
soup = BeautifulSoup(plain_text)
for item_name in soup.findAll('div', {'id': 'cnt_cats'}):
links = item_name.find('a').contents[0]
print(links)
In [33]:
trade_spider(2)
In [ ]: