練習

In [1]:

    

import requests
import re
import os

from PIL import Image
from bs4 import BeautifulSoup
from fake_useragent import UserAgent
from urllib.parse import urljoin
from urllib.parse import urlparse
from tldextract import extract
from pprint import pprint

url = 'http://aiacademy.tw/'


    

In [2]:

    

fu = UserAgent()
headers = {'User-Agent': fu.random}
resp = requests.get(url, headers=headers)
soup = BeautifulSoup(resp.text, 'lxml')


    

In [3]:

    

def invalid_href(url):
    check_anchor = re.match('.*#.*', url)
    check_protocol = re.match('[^https|http].*', urlparse(url).scheme)
    check_js = re.match('javascript.*', url)
    return any([check_anchor, check_protocol, check_js])

def inner_href(url, domain):
    return extract(url).domain == domain


    

In [4]:

    

wait_list = [url]
view_list = []
domain = extract(url).domain

while wait_list:

    link = wait_list.pop()
    if link in view_list:
        continue
    
    if invalid_href(link):
        continue
    
    if not inner_href(link, domain):
        continue
    
    print(link)
    view_list.append(link)
    
    page_resp = requests.get(link, headers=headers)
    page_soup = BeautifulSoup(page_resp.text, 'lxml')

    # search new links in current page
    links = page_soup.find_all('a')
    links = [link['href'] for link in links if link.has_attr('href')]
    links = [urljoin(page_resp.url, link) for link in links]
    links = list(filter(lambda x: x not in view_list, links))
    wait_list += links
    wait_list = list(set(wait_list))

print('view list:')
pprint(view_list)


    

    




http://aiacademy.tw/
http://aiacademy.tw/admission-mgr2/
http://aiacademy.tw/opening/
http://aiacademy.tw/category/general/
http://aiacademy.tw/registration-tech1/
http://aiacademy.tw/registration/
http://aiacademy.tw/announcement-180129/
http://aiacademy.tw/registration-mgr1/
http://aiacademy.tw/admission-tech2/
http://aiacademy.tw/curriculum-tech2/
http://aiacademy.tw/registration-tech2/
http://aiacademy.tw/wp-content/uploads/2018/01/registration-tech2.pdf
http://aiacademy.tw/parking/
http://aiacademy.tw/category/class-tech/
http://aiacademy.tw/category/news/
http://aiacademy.tw/category/news/page/3/
http://aiacademy.tw/refund/
http://aiacademy.tw/class-tech/
http://aiacademy.tw/mgr-class-overview/
http://aiacademy.tw/tech-leader-lecturers/
http://aiacademy.tw/lecturer/swc/
http://aiacademy.tw/lecturer/tjw/
http://aiacademy.tw/lecturer/whm/
http://aiacademy.tw/general
http://aiacademy.tw/curriculum/
http://aiacademy.tw/mgr-registration/
http://aiacademy.tw/curriculum-tech1/
http://aiacademy.tw/registration-mgr2/
http://aiacademy.tw/category/curriculum/
http://aiacademy.tw/organization/
http://aiacademy.tw/wp-content/uploads/2018/01/孔校長致詞.pdf
http://aiacademy.tw/mgr-lecturers/
http://aiacademy.tw/lecturer/hjh/
http://aiacademy.tw/mgr-admission-rule/
http://aiacademy.tw/ta/
http://aiacademy.tw/press-180127/
http://aiacademy.tw/category/%e9%87%8d%e5%a4%a7%e6%b4%bb%e5%8b%95/
http://aiacademy.tw/admission-mgr2
http://aiacademy.tw/category/faq/
http://aiacademy.tw/class-tech
http://aiacademy.tw/lecturer/huk/
http://aiacademy.tw/leave-of-absence-rule/
http://aiacademy.tw/about
http://aiacademy.tw/certificate-rule
http://aiacademy.tw/lecturer/albert/
http://aiacademy.tw/wp-content/uploads/2018/01/registration-mgr2.pdf
http://aiacademy.tw/vision/
http://aiacademy.tw/exam_notes/
http://aiacademy.tw/category/news
http://aiacademy.tw/press-180122/
http://aiacademy.tw/history/
http://aiacademy.tw/presence-rule-tech/
http://aiacademy.tw/support-us/
http://aiacademy.tw/admission-tech2
http://aiacademy.tw/category/faq
http://aiacademy.tw/lecturer/hungyilee/
http://aiacademy.tw/%e5%ad%b8%e5%93%a1%e7%b7%a8%e8%99%9f%e5%8f%8a%e7%ac%ac%e4%b8%80%e6%9c%9f%e9%96%8b%e5%ad%b8%e5%85%b8%e7%a6%ae%e8%a1%8c%e5%89%8d%e9%80%9a%e7%9f%a5%e5%b7%b2%e5%af%84%e5%87%ba%e8%ab%8b%e6%b3%a8%e6%84%8f/
http://aiacademy.tw/wp-content/uploads/2018/01/台灣人工智慧學校創校緣起及招生狀況1.pdf
http://aiacademy.tw/class-1st-overview/
http://aiacademy.tw/opening-presentation/
http://aiacademy.tw/certificate-rule/
http://aiacademy.tw/lecturer/ycw/
http://aiacademy.tw/wp-content/uploads/2018/01/AI-vs-Startups.pdf
http://aiacademy.tw/lecturer/ilt/
http://aiacademy.tw/lecturer/lin/
http://aiacademy.tw/class-overview/
http://aiacademy.tw/calendar/
http://aiacademy.tw/registration-mgr/
http://aiacademy.tw/mgr-class-enrollment-notice-0110/
http://aiacademy.tw/lecturer/swh/
http://aiacademy.tw/admission-tech1/
http://aiacademy.tw/enrollment_1st_term/
http://aiacademy.tw/mgr-class-1st-overview
http://aiacademy.tw/faq/
http://aiacademy.tw/mgr-curriculum/
http://aiacademy.tw/lecturer/sunmin/
http://aiacademy.tw/class-1st-overview
http://aiacademy.tw/lecturer/cph/
http://aiacademy.tw/rent/
http://aiacademy.tw/curriculum-mgr2/
http://aiacademy.tw/lecturer/ysc/
http://aiacademy.tw/lecturer/shw/
http://aiacademy.tw/calendar-am071/
http://aiacademy.tw/opening
http://aiacademy.tw/admission-mgr1/
http://aiacademy.tw/lecturers
http://aiacademy.tw/category/class-mgr/
http://aiacademy.tw/mgr-class-1st-overview/
http://aiacademy.tw/wp-content/uploads/2017/12/台灣人工智慧學校經理人周末研修班第一期－報名表格.pdf
http://aiacademy.tw/class-mgr/
http://aiacademy.tw/curriculum-mgr/
http://aiacademy.tw/admission-rule/
http://aiacademy.tw/category/misc/
http://aiacademy.tw/policy-tech/
http://aiacademy.tw/class-mgr
http://aiacademy.tw/absence-rule-manager/
http://aiacademy.tw/mgr-class-overview
http://aiacademy.tw/category/admission/
http://aiacademy.tw/class-2018jan-written-examination-list/
http://aiacademy.tw/presence-rule-mgr/
http://aiacademy.tw/curriculum-mgr1/
http://aiacademy.tw/lecturer/iac/
http://aiacademy.tw/category/lecturers/
http://aiacademy.tw/lecturer/weichao-chen/
http://aiacademy.tw/wp-content/uploads/2017/10/aiacademy.tw-registration.pdf
http://aiacademy.tw/job-fair/
http://aiacademy.tw/corporate-partner
http://aiacademy.tw/category/news/page/2/
http://aiacademy.tw/class-enrollment-notice-1225/
http://aiacademy.tw/aia-examination-notice-20171216/
http://aiacademy.tw/about/
http://aiacademy.tw/corporate-partner/
view list:
['http://aiacademy.tw/',
 'http://aiacademy.tw/admission-mgr2/',
 'http://aiacademy.tw/opening/',
 'http://aiacademy.tw/category/general/',
 'http://aiacademy.tw/registration-tech1/',
 'http://aiacademy.tw/registration/',
 'http://aiacademy.tw/announcement-180129/',
 'http://aiacademy.tw/registration-mgr1/',
 'http://aiacademy.tw/admission-tech2/',
 'http://aiacademy.tw/curriculum-tech2/',
 'http://aiacademy.tw/registration-tech2/',
 'http://aiacademy.tw/wp-content/uploads/2018/01/registration-tech2.pdf',
 'http://aiacademy.tw/parking/',
 'http://aiacademy.tw/category/class-tech/',
 'http://aiacademy.tw/category/news/',
 'http://aiacademy.tw/category/news/page/3/',
 'http://aiacademy.tw/refund/',
 'http://aiacademy.tw/class-tech/',
 'http://aiacademy.tw/mgr-class-overview/',
 'http://aiacademy.tw/tech-leader-lecturers/',
 'http://aiacademy.tw/lecturer/swc/',
 'http://aiacademy.tw/lecturer/tjw/',
 'http://aiacademy.tw/lecturer/whm/',
 'http://aiacademy.tw/general',
 'http://aiacademy.tw/curriculum/',
 'http://aiacademy.tw/mgr-registration/',
 'http://aiacademy.tw/curriculum-tech1/',
 'http://aiacademy.tw/registration-mgr2/',
 'http://aiacademy.tw/category/curriculum/',
 'http://aiacademy.tw/organization/',
 'http://aiacademy.tw/wp-content/uploads/2018/01/孔校長致詞.pdf',
 'http://aiacademy.tw/mgr-lecturers/',
 'http://aiacademy.tw/lecturer/hjh/',
 'http://aiacademy.tw/mgr-admission-rule/',
 'http://aiacademy.tw/ta/',
 'http://aiacademy.tw/press-180127/',
 'http://aiacademy.tw/category/%e9%87%8d%e5%a4%a7%e6%b4%bb%e5%8b%95/',
 'http://aiacademy.tw/admission-mgr2',
 'http://aiacademy.tw/category/faq/',
 'http://aiacademy.tw/class-tech',
 'http://aiacademy.tw/lecturer/huk/',
 'http://aiacademy.tw/leave-of-absence-rule/',
 'http://aiacademy.tw/about',
 'http://aiacademy.tw/certificate-rule',
 'http://aiacademy.tw/lecturer/albert/',
 'http://aiacademy.tw/wp-content/uploads/2018/01/registration-mgr2.pdf',
 'http://aiacademy.tw/vision/',
 'http://aiacademy.tw/exam_notes/',
 'http://aiacademy.tw/category/news',
 'http://aiacademy.tw/press-180122/',
 'http://aiacademy.tw/history/',
 'http://aiacademy.tw/presence-rule-tech/',
 'http://aiacademy.tw/support-us/',
 'http://aiacademy.tw/admission-tech2',
 'http://aiacademy.tw/category/faq',
 'http://aiacademy.tw/lecturer/hungyilee/',
 'http://aiacademy.tw/%e5%ad%b8%e5%93%a1%e7%b7%a8%e8%99%9f%e5%8f%8a%e7%ac%ac%e4%b8%80%e6%9c%9f%e9%96%8b%e5%ad%b8%e5%85%b8%e7%a6%ae%e8%a1%8c%e5%89%8d%e9%80%9a%e7%9f%a5%e5%b7%b2%e5%af%84%e5%87%ba%e8%ab%8b%e6%b3%a8%e6%84%8f/',
 'http://aiacademy.tw/wp-content/uploads/2018/01/台灣人工智慧學校創校緣起及招生狀況1.pdf',
 'http://aiacademy.tw/class-1st-overview/',
 'http://aiacademy.tw/opening-presentation/',
 'http://aiacademy.tw/certificate-rule/',
 'http://aiacademy.tw/lecturer/ycw/',
 'http://aiacademy.tw/wp-content/uploads/2018/01/AI-vs-Startups.pdf',
 'http://aiacademy.tw/lecturer/ilt/',
 'http://aiacademy.tw/lecturer/lin/',
 'http://aiacademy.tw/class-overview/',
 'http://aiacademy.tw/calendar/',
 'http://aiacademy.tw/registration-mgr/',
 'http://aiacademy.tw/mgr-class-enrollment-notice-0110/',
 'http://aiacademy.tw/lecturer/swh/',
 'http://aiacademy.tw/admission-tech1/',
 'http://aiacademy.tw/enrollment_1st_term/',
 'http://aiacademy.tw/mgr-class-1st-overview',
 'http://aiacademy.tw/faq/',
 'http://aiacademy.tw/mgr-curriculum/',
 'http://aiacademy.tw/lecturer/sunmin/',
 'http://aiacademy.tw/class-1st-overview',
 'http://aiacademy.tw/lecturer/cph/',
 'http://aiacademy.tw/rent/',
 'http://aiacademy.tw/curriculum-mgr2/',
 'http://aiacademy.tw/lecturer/ysc/',
 'http://aiacademy.tw/lecturer/shw/',
 'http://aiacademy.tw/calendar-am071/',
 'http://aiacademy.tw/opening',
 'http://aiacademy.tw/admission-mgr1/',
 'http://aiacademy.tw/lecturers',
 'http://aiacademy.tw/category/class-mgr/',
 'http://aiacademy.tw/mgr-class-1st-overview/',
 'http://aiacademy.tw/wp-content/uploads/2017/12/台灣人工智慧學校經理人周末研修班第一期－報名表格.pdf',
 'http://aiacademy.tw/class-mgr/',
 'http://aiacademy.tw/curriculum-mgr/',
 'http://aiacademy.tw/admission-rule/',
 'http://aiacademy.tw/category/misc/',
 'http://aiacademy.tw/policy-tech/',
 'http://aiacademy.tw/class-mgr',
 'http://aiacademy.tw/absence-rule-manager/',
 'http://aiacademy.tw/mgr-class-overview',
 'http://aiacademy.tw/category/admission/',
 'http://aiacademy.tw/class-2018jan-written-examination-list/',
 'http://aiacademy.tw/presence-rule-mgr/',
 'http://aiacademy.tw/curriculum-mgr1/',
 'http://aiacademy.tw/lecturer/iac/',
 'http://aiacademy.tw/category/lecturers/',
 'http://aiacademy.tw/lecturer/weichao-chen/',
 'http://aiacademy.tw/wp-content/uploads/2017/10/aiacademy.tw-registration.pdf',
 'http://aiacademy.tw/job-fair/',
 'http://aiacademy.tw/corporate-partner',
 'http://aiacademy.tw/category/news/page/2/',
 'http://aiacademy.tw/class-enrollment-notice-1225/',
 'http://aiacademy.tw/aia-examination-notice-20171216/',
 'http://aiacademy.tw/about/',
 'http://aiacademy.tw/corporate-partner/']