用Python 3开发网络爬虫

By Terrill Yang (Github: https://github.com/yttty)

用Python 3开发网络爬虫 - Chapter 06 开始采集

一个抓取Wikipedia的例子

我们从最简单的抓取开始，先写一段可以获取Wikipedia任何页面的代码：



In [1]:

    
from urllib.request import urlopen
from bs4 import BeautifulSoup

html = urlopen("https://en.wikipedia.org/wiki/Python_(programming_language)")
bsObj = BeautifulSoup(html.read(), "html.parser")
for link in bsObj.findAll("a"):
    if 'href' in link.attrs:
        print(link.attrs['href'])









    



/wiki/Wikipedia:Good_articles
#mw-head
#p-search
/wiki/File:Python_logo_and_wordmark.svg
/wiki/Programming_paradigm
/wiki/Multi-paradigm_programming_language
/wiki/Object-oriented_programming
/wiki/Imperative_programming
/wiki/Functional_programming
/wiki/Procedural_programming
/wiki/Reflective_programming
/wiki/Software_design
/wiki/Guido_van_Rossum
/wiki/Software_developer
/wiki/Python_Software_Foundation
#cite_note-1
/wiki/Software_release_life_cycle
#cite_note-2
#cite_note-3
/wiki/Type_system
/wiki/Duck_typing
/wiki/Dynamic_typing
/wiki/Strong_typing
/wiki/Operating_system
/wiki/Cross-platform
/wiki/Software_license
/wiki/Python_Software_Foundation_License
/wiki/Filename_extension
#cite_note-4
#cite_note-5
https://www.python.org/
/wiki/Programming_language_implementation
/wiki/CPython
/wiki/IronPython
/wiki/Jython
/wiki/MicroPython
/wiki/PyPy
/wiki/Dialect_(computing)
/wiki/Cython
/wiki/RPython
/wiki/Stackless_Python
/wiki/ABC_(programming_language)
#cite_note-faq-created-6
/wiki/ALGOL_68
#cite_note-98-interview-7
/wiki/C_(programming_language)
#cite_note-AutoNT-1-8
/wiki/C%2B%2B
#cite_note-classmix-9
/wiki/Dylan_(programming_language)
#cite_note-AutoNT-2-10
/wiki/Haskell_(programming_language)
#cite_note-AutoNT-3-11
/wiki/Icon_(programming_language)
#cite_note-AutoNT-4-12
/wiki/Java_(programming_language)
#cite_note-AutoNT-5-13
/wiki/Lisp_(programming_language)
#cite_note-AutoNT-6-14
/wiki/Modula-3
#cite_note-classmix-9
/wiki/Perl
/wiki/Boo_(programming_language)
/wiki/Cobra_(programming_language)
/wiki/CoffeeScript
#cite_note-15
/wiki/D_(programming_language)
/wiki/F_Sharp_(programming_language)
/wiki/Falcon_(programming_language)
/wiki/Genie_(programming_language)
#cite_note-16
/wiki/Go_(programming_language)
/wiki/Groovy_(programming_language)
/wiki/JavaScript
#cite_note-17
#cite_note-18
/wiki/Julia_(programming_language)
#cite_note-Julia-19
/wiki/Nim_(programming_language)
/wiki/Ruby_(programming_language)
#cite_note-bini-20
/wiki/Swift_(programming_language)
#cite_note-lattner2014-21
/wiki/File:Wikibooks-logo-en-noslogan.svg
https://en.wikibooks.org/wiki/Python_Programming
/wiki/High-level_programming_language
/wiki/General-purpose_programming_language
/wiki/Guido_van_Rossum
/wiki/Interpreted_language
/wiki/Readability
/wiki/Whitespace_character
/wiki/Code_block
/wiki/Source_lines_of_code
/wiki/C%2B%2B
/wiki/Java_(programming_language)
#cite_note-Summerfield-22
#cite_note-23
#cite_note-AutoNT-7-24
/wiki/Dynamic_type
/wiki/Memory_management
/wiki/Programming_paradigm
/wiki/Object-oriented_programming
/wiki/Imperative_programming
/wiki/Functional_programming
/wiki/Procedural_programming
/wiki/Standard_library
#cite_note-About-25
/wiki/Operating_system
/wiki/CPython
/wiki/Reference_implementation
/wiki/Open_source
#cite_note-26
/wiki/Python_Software_Foundation
#History
#Features_and_philosophy
#Syntax_and_semantics
#Indentation
#Statements_and_control_flow
#Expressions
#Methods
#Typing
#Mathematics
#Libraries
#Development_environments
#Implementations
#Development
#Naming
#Uses
#Languages_influenced_by_Python
#See_also
#References
#Further_reading
#External_links
/w/index.php?title=Python_(programming_language)&action=edit&section=1
/wiki/File:Guido_van_Rossum_OSCON_2006.jpg
/wiki/File:Guido_van_Rossum_OSCON_2006.jpg
/wiki/Guido_van_Rossum
/wiki/History_of_Python
#cite_note-venners-interview-pt-1-27
#cite_note-timeline-of-python-28
/wiki/Guido_van_Rossum
/wiki/Centrum_Wiskunde_%26_Informatica
/wiki/Netherlands
/wiki/ABC_(programming_language)
/wiki/SETL
#cite_note-AutoNT-12-29
/wiki/Exception_handling
/wiki/Amoeba_(operating_system)
#cite_note-faq-created-6
/wiki/Benevolent_dictator_for_life
#cite_note-30
/wiki/ABC_(programming_language)
/wiki/Unix
/wiki/C_(programming_language)
/wiki/Hacker_(programmer_subculture)
/wiki/Monty_Python%27s_Flying_Circus
/wiki/Cycle_detection
/wiki/Garbage_collection_(computer_science)
/wiki/Unicode
#cite_note-newin-2.0-31
#cite_note-3.0-release-32
/wiki/Backporting
#cite_note-pep-3000-33
#cite_note-34
#cite_note-35
/wiki/Go_(programming_language)
/wiki/Transcompiler
/wiki/The_Register
#cite_note-36
/w/index.php?title=Python_(programming_language)&action=edit&section=2
/wiki/Multi-paradigm_programming_language
/wiki/Object-oriented_programming
/wiki/Structured_programming
/wiki/Functional_programming
/wiki/Aspect-oriented_programming
/wiki/Metaprogramming
#cite_note-AutoNT-13-37
/wiki/Metaobject
#cite_note-AutoNT-14-38
/wiki/Design_by_contract
#cite_note-AutoNT-15-39
#cite_note-AutoNT-16-40
/wiki/Logic_programming
#cite_note-AutoNT-17-41
/wiki/Dynamic_typing
/wiki/Reference_counting
/wiki/Memory_management
/wiki/Name_resolution_(programming_languages)
/wiki/Late_binding
/wiki/Functional_programming
/wiki/Lisp_(programming_language)
/wiki/List_comprehension
/wiki/Associative_array
/wiki/Generator_(computer_programming)
#cite_note-AutoNT-59-42
/wiki/Haskell_(programming_language)
/wiki/Standard_ML
#cite_note-AutoNT-18-43
/wiki/Zen_of_Python
/wiki/Aphorism
#cite_note-PEP20-44
/wiki/ABC_(programming_language)
#cite_note-venners-interview-pt-1-27
/wiki/Perl
/wiki/Alex_Martelli
#cite_note-AutoNT-19-45
/wiki/There_is_more_than_one_way_to_do_it
#cite_note-PEP20-44
/wiki/Premature_optimization
#cite_note-AutoNT-20-46
/wiki/PyPy
/wiki/Just-in-time_compilation
/wiki/Cython
/wiki/Monty_Python
#cite_note-47
/wiki/Foobar
#cite_note-48
#cite_note-49
/wiki/Neologism
#cite_note-AutoNT-27-50
#cite_note-AutoNT-25-51
/w/index.php?title=Python_(programming_language)&action=edit&section=3
/wiki/Python_syntax_and_semantics
/wiki/C_(programming_language)
/wiki/Pascal_(programming_language)
#cite_note-AutoNT-52-52
/w/index.php?title=Python_(programming_language)&action=edit&section=4
/wiki/Python_syntax_and_semantics#Indentation
/wiki/Whitespace_character
/wiki/Block_(programming)
/wiki/Curly_bracket_programming_language
#cite_note-AutoNT-53-53
/wiki/Off-side_rule
/w/index.php?title=Python_(programming_language)&action=edit&section=5
/wiki/Statement_(computer_science)
/wiki/Imperative_programming
/wiki/C_(programming_language)
/wiki/Memory_management#ALLOCATION
/wiki/Variable_(computer_science)
/wiki/Type_system
/wiki/Pointer_(computer_programming)
/wiki/Object_(computer_science)
/wiki/Type_system
/wiki/Dynamic_type
/wiki/If-then-else
/wiki/Foreach#Python
/wiki/While_loop#Python
/wiki/Exception_handling_syntax#Python
/wiki/Class_(computer_science)
/wiki/Object-oriented_programming
/wiki/Function_(computing)
/wiki/Method_(computing)
/wiki/Lock_(computer_science)
/wiki/Computer_file
/wiki/Resource_Acquisition_Is_Initialization
/wiki/NOP
/wiki/Assertion_(programming)
/wiki/Generator_(computer_science)#Python
/wiki/Coroutine
#cite_note-diff_py2-3-54
/wiki/Tail_call
/wiki/First-class_continuations
#cite_note-AutoNT-55-55
#cite_note-AutoNT-56-56
/wiki/Coroutine
/wiki/Generator_(computer_science)
#cite_note-AutoNT-57-57
/wiki/Lazy_evaluation
/wiki/Iterator
#cite_note-AutoNT-58-58
/w/index.php?title=Python_(programming_language)&action=edit&section=6
/wiki/Expression_(computer_science)
/wiki/C_(programming_language)
/wiki/Java_(programming_language)
#cite_note-PEP465-59
#cite_note-Python3.5Changelog-60
#cite_note-61
#cite_note-62
/wiki/List_comprehension#Python
/wiki/Generator_(computer_science)
#cite_note-AutoNT-59-42
/wiki/Anonymous_function
/wiki/Lambda_(programming)
#cite_note-AutoNT-60-63
/wiki/%3F:
/wiki/List_(computer_science)
/wiki/Tuple
/wiki/Immutable
/wiki/Printf
/wiki/C_(programming_language)
/wiki/String_literal
/wiki/Unix_shell
/wiki/Perl
/wiki/Escape_character
/wiki/String_interpolation
/wiki/Here_document
/wiki/Ruby_(programming_language)
/wiki/Raw_string
/wiki/Regular_expression
/wiki/Microsoft_Windows
/wiki/C_Sharp_(programming_language)
/wiki/Array_index
/wiki/Array_slicing
/wiki/Zero-based
/wiki/Shallow_copy
/wiki/Common_Lisp
/wiki/Scheme_(programming_language)
/wiki/Ruby_(programming_language)
/wiki/List_comprehensions
/wiki/Conditional_(programming)
/wiki/Lambda_(programming)
/w/index.php?title=Python_(programming_language)&action=edit&section=7
/wiki/Method_(programming)
/wiki/Function_(programming)
/wiki/Syntactic_sugar
/wiki/This_(computer_programming)
/wiki/Instance_data
/wiki/C%2B%2B
/wiki/Java_(programming_language)
/wiki/Objective-C
/wiki/Ruby_(programming_language)
#cite_note-AutoNT-61-64
/w/index.php?title=Python_(programming_language)&action=edit&section=8
/wiki/Duck_typing
/wiki/Compile_time
/wiki/Type_system#Dynamic_type_checking_and_runtime_type_information
/wiki/Strongly_typed_programming_language
/wiki/Class_(computer_science)
/wiki/Object-oriented_programming
/wiki/Object_(computer_science)
/wiki/Metaclass
/wiki/Metaprogramming
/wiki/Reflection_(computer_science)
#cite_note-classy-65
/wiki/Gradual_typing
#cite_note-66
#cite_note-67
/wiki/Immutable_object
/wiki/Character_string
/wiki/Byte
/wiki/List_(computer_science)
/wiki/Set_(computer_science)
/wiki/Set_(computer_science)
/wiki/Associative_array
/wiki/Integer_(computer_science)
#cite_note-pep0237-68
/wiki/Floating_point
/wiki/Complex_number
/wiki/Boolean_value
/wiki/Ellipsis_(programming_operator)
/wiki/NumPy
/w/index.php?title=Python_(programming_language)&action=edit&section=9
#cite_note-69
#cite_note-pep0238-70
/wiki/Half-open_interval
#cite_note-AutoNT-62-71
/wiki/Rounding
/wiki/Rounding#Tie-breaking
#cite_note-AutoNT-63-72
/wiki/Round_to_even
#cite_note-AutoNT-64-73
#cite_note-AutoNT-65-74
/wiki/Wikipedia:Citing_sources
/wiki/Arbitrary_precision_arithmetic
#cite_note-75
#cite_note-76
#cite_note-77
/w/index.php?title=Python_(programming_language)&action=edit&section=10
/wiki/Standard_library
#cite_note-AutoNT-86-78
#cite_note-About-25
/wiki/MIME
/wiki/Hypertext_Transfer_Protocol
/wiki/Graphical_user_interface
/wiki/Relational_database
/wiki/Pseudorandom_number_generator
#cite_note-AutoNT-88-79
/wiki/Regular_expression
/wiki/Unit_testing
/wiki/Web_Server_Gateway_Interface
#cite_note-AutoNT-89-80
/wiki/Python_Package_Index
#cite_note-81
/w/index.php?title=Python_(programming_language)&action=edit&section=11
/wiki/Comparison_of_integrated_development_environments#Python
/wiki/Read%E2%80%93eval%E2%80%93print_loop
/wiki/Command_line_interpreter
/wiki/IDLE_(Python)
/wiki/IPython
/wiki/Syntax_highlighting
/wiki/Integrated_development_environment
/wiki/Python_IDE
/wiki/Web_browser
/wiki/SageMath
/wiki/PythonAnywhere
#cite_note-82
/w/index.php?title=Python_(programming_language)&action=edit&section=12
/wiki/List_of_Python_software#Python_implementations
/wiki/CPython
/wiki/C_(programming_language)
/wiki/C89_(C_version)
#cite_note-AutoNT-66-83
/wiki/Bytecode
#cite_note-AutoNT-67-84
/wiki/Virtual_machine
#cite_note-AutoNT-68-85
/wiki/Microsoft_Windows
/wiki/Unix-like
#cite_note-AutoNT-69-86
/wiki/PyPy
#cite_note-AutoNT-70-87
/wiki/Just-in-time_compilation
#cite_note-AutoNT-71-88
/wiki/Multi-core_processor
/wiki/Software_transactional_memory
#cite_note-AutoNT-72-89
/wiki/Stackless_Python
/wiki/Microthread
#cite_note-AutoNT-73-90
/wiki/MicroPython
/wiki/Microcontroller
/wiki/Unladen_Swallow
/wiki/LLVM
#cite_note-AutoNT-74-91
/wiki/Psyco
/wiki/Just-in-time_compilation
/wiki/Run-time_algorithm_specialisation
/wiki/Data_type
/wiki/Nokia
/wiki/Series_60
/wiki/PyS60
/wiki/Symbian
/wiki/N900
/wiki/GTK
#cite_note-92
/wiki/Object_language
/wiki/Jython
/wiki/Java_virtual_machine
/wiki/IronPython
/wiki/Common_Language_Runtime
/wiki/RPython
/wiki/C_(programming_language)
/wiki/Java_bytecode
/wiki/Common_Intermediate_Language
/wiki/Pyjamas_(software)
/wiki/JavaScript
/wiki/Shed_Skin
/wiki/C%2B%2B
/wiki/Cython
/wiki/Pyrex_(programming_language)
/wiki/C_(programming_language)
/w/index.php?title=Grumpy_(compiler)&action=edit&redlink=1
/wiki/Go_(programming_language)
#cite_note-93
/w/index.php?title=Python_(programming_language)&action=edit&section=13
#cite_note-PepCite000-94
/wiki/Benevolent_dictator_for_life
#cite_note-PepCite000-94
/wiki/Roundup_(issue_tracker)
/wiki/Bug_tracker
#cite_note-AutoNT-21-95
/wiki/Self-hosted
/wiki/Mercurial
/wiki/GitHub
#cite_note-py_dev_guide-96
/wiki/Ported
#cite_note-release-schedule-97
#cite_note-AutoNT-22-98
/wiki/Beta_release
/wiki/Unit_test
/wiki/BuildBot
/wiki/Continuous_integration
#cite_note-AutoNT-23-99
#cite_note-100
//en.wikipedia.org/w/index.php?title=Python_(programming_language)&action=edit
/wiki/Python_Package_Index
/wiki/Academic_conference
/wiki/PyCon
/wiki/Pyladies
/w/index.php?title=Python_(programming_language)&action=edit&section=14
/wiki/Monty_Python%27s_Flying_Circus
#cite_note-AutoNT-24-101
#cite_note-tutorial-chapter1-102
/wiki/Metasyntactic_variable
/wiki/Spam_(Monty_Python)
/wiki/Foobar
#cite_note-tutorial-chapter1-102
#cite_note-AutoNT-26-103
/wiki/Pygame
/wiki/Language_binding
/wiki/Simple_DirectMedia_Layer
/wiki/PyS60
/wiki/Symbian
/wiki/S60_(software_platform)
/wiki/PyQt
/wiki/PyGTK
/wiki/Qt_(software)
/wiki/GTK
/wiki/PyPy
/w/index.php?title=Python_(programming_language)&action=edit&section=15
/wiki/List_of_Python_software
/wiki/TIOBE_Programming_Community_Index
//en.wikipedia.org/w/index.php?title=Python_(programming_language)&action=edit
#cite_note-104
#cite_note-AutoNT-34-105
/wiki/Syntax_(programming_languages)
/wiki/C_(programming_language)
#cite_note-AutoNT-28-106
/wiki/Wikipedia
/wiki/Google
#cite_note-quotes-about-python-107
/wiki/Yahoo!
#cite_note-AutoNT-29-108
/wiki/CERN
#cite_note-AutoNT-30-109
/wiki/NASA
#cite_note-AutoNT-31-110
/wiki/Industrial_Light_%26_Magic
#cite_note-AutoNT-32-111
/wiki/ITA_Software
#cite_note-AutoNT-33-112
/wiki/Reddit
/wiki/Scripting_language
/wiki/Web_application
/wiki/Mod_wsgi
/wiki/Apache_web_server
#cite_note-AutoNT-35-113
/wiki/Web_Server_Gateway_Interface
/wiki/Web_framework
/wiki/Django_(web_framework)
/wiki/Pylons_(web_framework)
/wiki/Pyramid_(web_framework)
/wiki/TurboGears
/wiki/Web2py
/wiki/Tornado_(web_server)
/wiki/Flask_(web_framework)
/wiki/Bottle_(web_framework)
/wiki/Zope
/wiki/Pyjamas_(software)
/wiki/IronPython
/wiki/SQLAlchemy
/wiki/Data_mapper_pattern
/wiki/Twisted_(software)
/wiki/Dropbox_(service)
/wiki/NumPy
/wiki/SciPy
/wiki/Matplotlib
#cite_note-cise-114
#cite_note-millman-115
/wiki/Biopython
/wiki/Astropy
/wiki/SageMath
/wiki/Mathematical_software
/w/index.php?title=Electronic_Notebook&action=edit&redlink=1
/wiki/Mathematics
/wiki/Algebra
/wiki/Combinatorics
/wiki/Numerical_mathematics
/wiki/Number_theory
/wiki/Calculus
/wiki/Java_(programming_language)
/wiki/DataMelt
#cite_note-schekanov-116
#cite_note-schekanov2-117
/wiki/Finite_element_method
/wiki/Abaqus
/wiki/FreeCAD
/wiki/3ds_Max
/wiki/Blender_(software)
/wiki/Cinema_4D
/wiki/Lightwave
/wiki/Houdini_(software)
/wiki/Maya_(software)
/wiki/Modo_(software)
/wiki/MotionBuilder
/wiki/Autodesk_Softimage
/wiki/Nuke_(software)
/wiki/GIMP
#cite_note-118
/wiki/Inkscape
/wiki/Scribus
/wiki/Paint_Shop_Pro
#cite_note-AutoNT-38-119
/wiki/Musical_notation
/wiki/Scorewriter
/wiki/Capella_(notation_program)
/wiki/GNU_Debugger
/wiki/Prettyprint
/wiki/Esri
/wiki/ArcGIS
#cite_note-AutoNT-39-120
#cite_note-AutoNT-40-121
#cite_note-AutoNT-41-122
/wiki/Programming_language
/wiki/Google_App_Engine
/wiki/Java_(software_platform)
/wiki/Go_(programming_language)
#cite_note-AutoNT-42-123
/wiki/Algorithmic_trading
#cite_note-124
#cite_note-125
/wiki/Artificial_intelligence
#cite_note-AutoNT-43-126
#cite_note-AutoNT-44-127
#cite_note-AutoNT-45-128
#cite_note-AutoNT-46-129
/wiki/Natural_language_processing
#cite_note-AutoNT-47-130
/wiki/Linux_distribution
/wiki/AmigaOS_4
/wiki/FreeBSD
/wiki/NetBSD
/wiki/OpenBSD
/wiki/MacOS
/wiki/Ubuntu_(operating_system)
/wiki/Ubiquity_(software)
/wiki/Red_Hat_Linux
/wiki/Fedora_(operating_system)
/wiki/Anaconda_(installer)
/wiki/Gentoo_Linux
/wiki/Package_management_system
/wiki/Portage_(software)
/wiki/Information_security
#cite_note-AutoNT-49-131
#cite_note-AutoNT-50-132
/wiki/Sugar_(software)
/wiki/One_Laptop_per_Child
/wiki/Sugar_Labs
#cite_note-AutoNT-51-133
/wiki/Raspberry_Pi
/wiki/Single-board_computer
/wiki/LibreOffice
#cite_note-134
/w/index.php?title=Python_(programming_language)&action=edit&section=16
/wiki/Boo_(programming_language)
/wiki/Static_typing
/wiki/Duck_typing
/wiki/.NET_Framework
#cite_note-AutoNT-90-135
/wiki/Cobra_(programming_language)
#cite_note-AutoNT-91-136
/wiki/Design_by_contract
/wiki/Unit_testing
/wiki/Static_typing
#cite_note-AutoNT-92-137
/wiki/ECMAScript
/wiki/Iterator
/wiki/Generator_(computer_science)
/wiki/List_comprehension
#cite_note-AutoNT-93-138
/wiki/Go_(programming_language)
#cite_note-AutoNT-94-139
/wiki/Groovy_(programming_language)
/wiki/Java_(programming_language)
#cite_note-AutoNT-95-140
/wiki/Julia_(programming_language)
/wiki/Hygienic_macro
#cite_note-Julia-19
https://github.com/stevengj/PyCall.jl
https://github.com/JuliaLang/pyjulia
/wiki/OCaml
/wiki/Haskell_(programming_language)
#cite_note-AutoNT-96-141
/wiki/Ruby_(programming_language)
/wiki/Yukihiro_Matsumoto
#cite_note-linuxdevcenter-142
/wiki/CoffeeScript
/wiki/Swift_(programming_language)
#cite_note-143
/wiki/Tcl
#cite_note-AutoNT-99-144
/wiki/Erlang_(programming_language)
#cite_note-AutoNT-100-145
/wiki/TIOBE_index
#cite_note-AutoNT-101-146
/w/index.php?title=Python_(programming_language)&action=edit&section=17
/wiki/Portal:Free_software
/wiki/Portal:Python_programming
/wiki/Comparison_of_integrated_development_environments#Python
/wiki/Comparison_of_programming_languages
/wiki/List_of_programming_languages
/wiki/Off-side_rule
/w/index.php?title=Python_(programming_language)&action=edit&section=18
#cite_ref-1
http://python-history.blogspot.com/2009/01/brief-timeline-of-python.html
/wiki/Blogger_(service)
#cite_ref-2
http://blog.python.org/2016/06/python-352-and-python-345-are-now.html
#cite_ref-3
http://blog.python.org/2016/12/python-2713-released.html
#cite_ref-4
https://www.python.org/dev/peps/pep-0488/
#cite_ref-5
https://www.python.org/dev/peps/pep-0441/
#cite_ref-faq-created_6-0
#cite_ref-faq-created_6-1
https://docs.python.org/faq/general.html#why-was-python-created-in-the-first-place
#cite_ref-98-interview_7-0
http://www.amk.ca/python/writing/gvr-interview
#cite_ref-AutoNT-1_8-0
http://citeseerx.ist.psu.edu/viewdoc/summary?doi=10.1.1.38.2023
#cite_ref-classmix_9-0
#cite_ref-classmix_9-1
https://docs.python.org/tutorial/classes.html
#cite_ref-AutoNT-2_10-0
https://www.python.org/download/releases/2.3/mro/
#cite_ref-AutoNT-3_11-0
https://docs.python.org/howto/functional.html
#cite_ref-AutoNT-4_12-0
https://www.python.org/dev/peps/pep-0255/
#cite_ref-AutoNT-5_13-0
https://www.python.org/dev/peps/pep-0318/
#cite_ref-AutoNT-6_14-0
https://docs.python.org/3.2/tutorial/controlflow.html
#cite_ref-15
http://coffeescript.org/
#cite_ref-16
https://wiki.gnome.org/action/show/Projects/Genie
#cite_ref-17
http://www.2ality.com/2013/02/javascript-influences.html
#cite_ref-18
http://speakingjs.com/es5/ch03.html
#cite_ref-Julia_19-0
#cite_ref-Julia_19-1
http://julialang.org/blog/2012/02/why-we-created-julia
#cite_ref-bini_20-0
/wiki/International_Standard_Book_Number
/wiki/Special:BookSources/978-1-59059-881-8
#cite_ref-lattner2014_21-0
http://nondot.org/sabre/
#cite_ref-Summerfield_22-0
#cite_ref-23
https://books.google.com/books?id=3JfE7TGUwvgC&pg=PT100
/wiki/International_Standard_Book_Number
/wiki/Special:BookSources/9780735636972
#cite_ref-AutoNT-7_24-0
http://cutter.rexx.com/~dkuhlman/python_book_01.html
#cite_ref-About_25-0
#cite_ref-About_25-1
https://www.python.org/about
#cite_ref-26
https://docs.python.org/3/license.html
#cite_ref-venners-interview-pt-1_27-0
#cite_ref-venners-interview-pt-1_27-1
http://www.artima.com/intv/pythonP.html
#cite_ref-timeline-of-python_28-0
http://python-history.blogspot.com/2009/01/brief-timeline-of-python.html
#cite_ref-AutoNT-12_29-0
/wiki/Guido_van_Rossum
http://mail.python.org/pipermail/python-dev/2000-August/008881.html
#cite_ref-30
https://www.python.org/doc/essays/foreword/
#cite_ref-newin-2.0_31-0
https://docs.python.org/whatsnew/2.0.html
#cite_ref-3.0-release_32-0
https://www.python.org/download/releases/3.0/
#cite_ref-pep-3000_33-0
https://www.python.org/dev/peps/pep-3000/
#cite_ref-34
http://legacy.python.org/dev/peps/pep-0373/
#cite_ref-35
https://www.python.org/dev/peps/pep-0466/
#cite_ref-36
http://www.theregister.co.uk/2017/01/05/googles_grumpy_makes_python_go/
#cite_ref-AutoNT-13_37-0
http://www.webcitation.org/5lubkaJRc
https://www.python.org/community/pycon/dc2004/papers/24/metaclasses-pycon.pdf
#cite_ref-AutoNT-14_38-0
https://docs.python.org/3.0/reference/datamodel.html#special-method-names
#cite_ref-AutoNT-15_39-0
http://www.nongnu.org/pydbc/
#cite_ref-AutoNT-16_40-0
http://www.wayforward.net/pycontract/
#cite_ref-AutoNT-17_41-0
https://sites.google.com/site/pydatalog/
#cite_ref-AutoNT-59_42-0
#cite_ref-AutoNT-59_42-1
https://www.python.org/dev/peps/pep-0289/
#cite_ref-AutoNT-18_43-0
https://docs.python.org/3/library/itertools.html
#cite_ref-PEP20_44-0
#cite_ref-PEP20_44-1
https://www.python.org/dev/peps/pep-0020/
#cite_ref-AutoNT-19_45-0
http://shop.oreilly.com/product/9780596007973.do
/wiki/O%27Reilly_Media
/wiki/International_Standard_Book_Number
/wiki/Special:BookSources/978-0-596-00797-3
#cite_ref-AutoNT-20_46-0
http://ebeab.com/2014/01/21/python-culture/
#cite_ref-47
https://docs.python.org/2/faq/general.html#why-is-it-called-python
#cite_ref-48
http://insidetech.monster.com/training/articles/8114-15-ways-python-is-a-powerful-force-on-the-web
#cite_ref-49
https://docs.python.org/2/library/pprint.html
#cite_ref-AutoNT-27_50-0
http://python.net/~goodger/projects/pycon/2007/idiomatic/handout.html
#cite_ref-AutoNT-25_51-0
http://python.net/crew/mwh/hacks/objectthink.html
#cite_ref-AutoNT-52_52-0
https://docs.python.org/faq/general.html#is-python-a-good-language-for-beginning-programmers
#cite_ref-AutoNT-53_53-0
http://www.secnetix.de/~olli/Python/block_indentation.hawk
#cite_ref-diff_py2-3_54-0
http://inventwithpython.com/appendixa.html
/wiki/International_Standard_Book_Number
/wiki/Special:BookSources/978-0-9821060-1-3
#cite_ref-AutoNT-55_55-0
http://neopythonic.blogspot.be/2009/04/tail-recursion-elimination.html
#cite_ref-AutoNT-56_56-0
http://www.artima.com/weblogs/viewpost.jsp?thread=147358
#cite_ref-AutoNT-57_57-0
https://www.python.org/dev/peps/pep-0342/
#cite_ref-AutoNT-58_58-0
https://www.python.org/dev/peps/pep-0380/
#cite_ref-PEP465_59-0
https://www.python.org/dev/peps/pep-0465/
#cite_ref-Python3.5Changelog_60-0
https://www.python.org/downloads/release/python-351/
#cite_ref-61
http://docs.oracle.com/javase/specs/jls/se8/html/jls-15.html#jls-15.21.1
/wiki/Oracle_Corporation
#cite_ref-62
http://docs.oracle.com/javase/specs/jls/se8/html/jls-15.html#jls-15.21.3
#cite_ref-AutoNT-60_63-0
https://www.python.org/dev/peps/pep-0308/
#cite_ref-AutoNT-61_64-0
https://docs.python.org/faq/design.html#why-must-self-be-used-explicitly-in-method-definitions-and-calls
#cite_ref-classy_65-0
https://docs.python.org/reference/datamodel.html#new-style-and-classic-classes
#cite_ref-66
https://lwn.net/Articles/627418/
#cite_ref-67
http://mypy-lang.org/
#cite_ref-pep0237_68-0
https://www.python.org/dev/peps/pep-0237/
#cite_ref-69
http://legacy.python.org/dev/peps/pep-0465/
#cite_ref-pep0238_70-0
https://www.python.org/dev/peps/pep-0238/
#cite_ref-AutoNT-62_71-0
http://python-history.blogspot.com/2010/08/why-pythons-integer-division-floors.html
#cite_ref-AutoNT-63_72-0
https://docs.python.org/library/functions.html#round
#cite_ref-AutoNT-64_73-0
https://docs.python.org/py3k/library/functions.html#round
#cite_ref-AutoNT-65_74-0
#cite_ref-75
https://docs.python.org/2.7/library/stdtypes.html
#cite_ref-76
https://www.python.org/dev/peps/pep-0327/
#cite_ref-77
https://docs.python.org/2.6/whatsnew/2.6.html
#cite_ref-AutoNT-86_78-0
http://www.oracle.com/technetwork/articles/piotrowski-pythoncore-084049.html
#cite_ref-AutoNT-88_79-0
https://www.python.org/dev/peps/pep-0327/
#cite_ref-AutoNT-89_80-0
https://www.python.org/dev/peps/pep-0333/
#cite_ref-81
http://www.modulecounts.com/
#cite_ref-82
https://www.enthought.com/products/canopy/
#cite_ref-AutoNT-66_83-0
https://www.python.org/dev/peps/pep-0007/
#cite_ref-AutoNT-67_84-0
https://docs.python.org/3/library/dis.html#python-bytecode-instructions
#cite_ref-AutoNT-68_85-0
http://www.troeger.eu/teaching/pythonvm08.pdf
#cite_ref-AutoNT-69_86-0
http://www.oreilly.com/pub/a/oreilly/frank/rossum_1099.html
#cite_ref-AutoNT-70_87-0
http://pypy.org/compat.html
#cite_ref-AutoNT-71_88-0
http://speed.pypy.org/
#cite_ref-AutoNT-72_89-0
http://morepypy.blogspot.be/2012/06/stm-with-threads.html
#cite_ref-AutoNT-73_90-0
http://doc.pypy.org/en/latest/stackless.html
#cite_ref-AutoNT-74_91-0
https://code.google.com/p/unladen-swallow/wiki/ProjectPlan
#cite_ref-92
http://www.stochasticgeometry.ie/2010/04/29/python-on-the-nokia-n900/
#cite_ref-93
/wiki/ArXiv
//arxiv.org/abs/1404.6388
#cite_ref-PepCite000_94-0
#cite_ref-PepCite000_94-1
https://www.python.org/dev/peps/pep-0001/
#cite_ref-AutoNT-21_95-0
https://web.archive.org/web/20090601134342/http://www.python.org/dev/intro/
https://www.python.org/dev/intro/
#cite_ref-py_dev_guide_96-0
https://docs.python.org/devguide/
#cite_ref-release-schedule_97-0
https://mail.python.org/pipermail/python-dev/2002-April/022739.html
#cite_ref-AutoNT-22_98-0
https://www.python.org/dev/peps/pep-0006/
#cite_ref-AutoNT-23_99-0
https://www.python.org/dev/buildbot/
#cite_ref-100
http://www.modulecounts.com/#
#cite_ref-AutoNT-24_101-0
https://docs.python.org/2/faq/general.html#why-is-it-called-python
#cite_ref-tutorial-chapter1_102-0
#cite_ref-tutorial-chapter1_102-1
https://docs.python.org/tutorial/appetite.html
#cite_ref-AutoNT-26_103-0
https://stackoverflow.com/questions/5033906/in-python-should-i-use-else-after-a-return-in-an-if-block
/wiki/Stack_Overflow
#cite_ref-104
http://www.tiobe.com/tiobe-index/
#cite_ref-AutoNT-34_105-0
http://www.tiobe.com/index.php/paperinfo/tpci/Python.html
#cite_ref-AutoNT-28_106-0
http://page.mi.fu-berlin.de/prechelt/Biblio/jccpprt_computer2000.pdf
#cite_ref-quotes-about-python_107-0
https://www.python.org/about/quotes/
#cite_ref-AutoNT-29_108-0
https://wiki.python.org/moin/OrganizationsUsingPython
#cite_ref-AutoNT-30_109-0
http://cdsweb.cern.ch/journal/CERNBulletin/2006/31/News%20Articles/974627?ln=en
#cite_ref-AutoNT-31_110-0
https://www.python.org/about/success/usa/
#cite_ref-AutoNT-32_111-0
https://www.python.org/about/success/ilm/
#cite_ref-AutoNT-33_112-0
http://www.eweek.com/c/a/Application-Development/Python-Slithers-into-Systems/
#cite_ref-AutoNT-35_113-0
http://w3techs.com/technologies/details/pl-python/all/all
#cite_ref-cise_114-0
https://www.h2desk.com/blog/python-scientific-computing/
#cite_ref-millman_115-0
http://www.computer.org/csdl/mags/cs/2011/02/mcs2011020009.html
#cite_ref-schekanov_116-0
http://www.springer.com/la/book/9783319285290
/wiki/International_Standard_Book_Number
/wiki/Special:BookSources/978-3-319-28531-3
#cite_ref-schekanov2_117-0
http://www.springer.com/la/book/9783319285290
/wiki/International_Standard_Book_Number
/wiki/Special:BookSources/978-3-319-28531-3
#cite_ref-118
http://gimp-win.sourceforge.net/faq.html
#cite_ref-AutoNT-38_119-0
https://web.archive.org/web/20080319061519/http://www.jasc.com/support/customercare/articles/psp9components.asp
http://www.jasc.com/support/customercare/articles/psp9components.asp
#cite_ref-AutoNT-39_120-0
http://webhelp.esri.com/arcgisdesktop/9.2/index.cfm?TopicName=About_getting_started_with_writing_geoprocessing_scripts
#cite_ref-AutoNT-40_121-0
http://community.eveonline.com/news/dev-blogs/stackless-python-2.7/
/wiki/CCP_Games
#cite_ref-AutoNT-41_122-0
http://www.webcitation.org/5ru5VItfv
/wiki/Firaxis_Games
http://www.2kgames.com/civ4/blog_03.htm
#cite_ref-AutoNT-42_123-0
http://www.webcitation.org/5ru5FHxfV
https://code.google.com/apis/documents/docs/1.0/developers_guide_python.html
#cite_ref-124
http://www.quantinsti.com/blog/python-best-programming-language-algorithmic-trading/
#cite_ref-125
http://www.quantinsti.com/blog/ibpy-tutorial-implement-python-interactive-brokers-api/
#cite_ref-AutoNT-43_126-0
https://web.archive.org/web/20121101045354/http://wiki.python.org/moin/PythonForArtificialIntelligence
https://wiki.python.org/moin/PythonForArtificialIntelligence
#cite_ref-AutoNT-44_127-0
http://www.ainewsletter.com/newsletters/aix_0508.htm#python_ai_ai
#cite_ref-AutoNT-45_128-0
https://pypi.python.org/pypi/PyAIML
#cite_ref-AutoNT-46_129-0
/wiki/Stuart_J._Russell
/wiki/Peter_Norvig
http://aima.cs.berkeley.edu/
/wiki/International_Standard_Book_Number
/wiki/Special:BookSources/978-0-13-604259-4
#cite_ref-AutoNT-47_130-0
http://www.nltk.org
#cite_ref-AutoNT-49_131-0
http://www.immunitysec.com/products-immdbg.shtml
#cite_ref-AutoNT-50_132-0
http://oss.coresecurity.com/
#cite_ref-AutoNT-51_133-0
http://sugarlabs.org/go/Sugar
#cite_ref-134
http://www.libreoffice.org/download/4-0-new-features-and-fixes/
/wiki/The_Document_Foundation
#cite_ref-AutoNT-90_135-0
http://boo.codehaus.org/Gotchas+for+Python+Users
#cite_ref-AutoNT-91_136-0
http://cobra-language.com/docs/acknowledgements/
#cite_ref-AutoNT-92_137-0
http://cobra-language.com/docs/python/
#cite_ref-AutoNT-93_138-0
http://wiki.ecmascript.org/doku.php?id=proposals:iterators_and_generators
#cite_ref-AutoNT-94_139-0
http://www.techcrunch.com/2009/11/10/google-go-language/
#cite_ref-AutoNT-95_140-0
http://radio.weblogs.com/0112098/2003/08/29.html
#cite_ref-AutoNT-96_141-0
http://people.csail.mit.edu/mikelin/ocaml+twt/
#cite_ref-linuxdevcenter_142-0
http://www.linuxdevcenter.com/pub/a/linux/2001/11/29/ruby.html
#cite_ref-143
/wiki/Chris_Lattner
http://nondot.org/sabre
#cite_ref-AutoNT-99_144-0
http://www.tcl.tk/cgi-bin/tct/tip/3.html
#cite_ref-AutoNT-100_145-0
http://www.erlang.org/eeps/eep-0001.html
#cite_ref-AutoNT-101_146-0
http://www.tiobe.com/index.php/content/paperinfo/tpci/
/w/index.php?title=Python_(programming_language)&action=edit&section=19
http://www.greenteapress.com/thinkpython/html/
/wiki/International_Standard_Book_Number
/wiki/Special:BookSources/978-0-521-72596-5
http://www.computerworld.com.au/index.php/id;66665771
http://shop.oreilly.com/product/0636920028154.do
/wiki/International_Standard_Book_Number
/wiki/Special:BookSources/978-0-596-15806-4
http://diveintopython.net
/wiki/International_Standard_Book_Number
/wiki/Special:BookSources/978-1-59059-356-1
http://diveintopython3.net
/wiki/International_Standard_Book_Number
/wiki/Special:BookSources/978-1-4302-2415-0
http://www.qtrac.eu/py3book.html
/wiki/International_Standard_Book_Number
/wiki/Special:BookSources/978-0-321-68056-3
/w/index.php?title=Python_(programming_language)&action=edit&section=20
/wiki/Wikipedia:Wikimedia_sister_projects
https://commons.wikimedia.org/wiki/Category:Python_(programming_language)
https://en.wikiquote.org/wiki/Python
https://en.wikibooks.org/wiki/Python_Programming
https://en.wikiversity.org/wiki/Python
https://www.python.org/
news://comp.lang.python
/wiki/Usenet
https://groups.google.com/group/comp.lang.python
/wiki/Google_Groups
http://python-history.blogspot.co.uk/
https://mail.python.org/mailman/listinfo/python-list
https://www.dmoz.org/Computers/Programming/Languages/Python
/wiki/DMOZ
/wiki/Template:Programming_languages
/wiki/Template_talk:Programming_languages
//en.wikipedia.org/w/index.php?title=Template:Programming_languages&action=edit
/wiki/Programming_language
/wiki/Comparison_of_programming_languages
/wiki/Timeline_of_programming_languages
/wiki/History_of_programming_languages
/wiki/Assembly_language
/wiki/BASIC
/wiki/C_(programming_language)
/wiki/C%2B%2B
/wiki/C_Sharp_(programming_language)
/wiki/COBOL
/wiki/Fortran
/wiki/Go_(programming_language)
/wiki/Haskell_(programming_language)
/wiki/Java_(programming_language)
/wiki/JavaScript
/wiki/Lisp_(programming_language)
/wiki/Lua_(programming_language)
/wiki/Objective-C
/wiki/Pascal_(programming_language)
/wiki/Perl
/wiki/PHP
/wiki/Ruby_(programming_language)
/wiki/Shell_script
/wiki/Smalltalk
/wiki/Swift_(programming_language)
/wiki/Visual_Basic_.NET
/wiki/List_of_programming_languages
/wiki/Category:Programming_languages
/wiki/Comparison_of_programming_languages
/wiki/List_of_programming_languages
/wiki/Categorical_list_of_programming_languages
/wiki/Generational_list_of_programming_languages
/wiki/Non-English-based_programming_languages
/wiki/Template:Python_(programming_language)
/wiki/Template_talk:Python_(programming_language)
//en.wikipedia.org/w/index.php?title=Template:Python_(programming_language)&action=edit
/wiki/CLPython
/wiki/CPython
/wiki/Cython
/wiki/MicroPython
/wiki/IronPython
/wiki/Jython
/wiki/Psyco
/wiki/PyPy
/wiki/Python_for_S60
/wiki/Stackless_Python
/wiki/Unladen_Swallow
/wiki/File:Python-logo-notext.svg
/wiki/Comparison_of_integrated_development_environments#Python
/wiki/Eric_Python_IDE
/wiki/IDLE_(Python)
/wiki/PyDev
/wiki/Stani%27s_Python_Editor
/wiki/Ninja-IDE
/wiki/List_of_integrated_development_environments_for_Python#Python
/wiki/Web_Server_Gateway_Interface
/wiki/List_of_Python_software
/wiki/Python_Software_Foundation
/wiki/Python_Conference
/wiki/Template:Python_web_frameworks
/wiki/Template_talk:Python_web_frameworks
//en.wikipedia.org/w/index.php?title=Template:Python_web_frameworks&action=edit
/wiki/Web_framework
/wiki/Zope#BlueBream
/wiki/CherryPy
/wiki/Django_(web_framework)
/wiki/Flask_(web_framework)
/wiki/Grok_(web_framework)
/wiki/Nagare_(web_framework)
/wiki/Nevow
/wiki/Pyjamas_(software)
/wiki/Pylons_project#Pylons_Framework
/wiki/Pylons_project#Pyramid
/wiki/Quixote_(web_framework)
/wiki/Spyce
/wiki/TACTIC_(web_framework)
/wiki/Tornado_(web_server)
/wiki/TurboGears
/wiki/Twisted_(software)
/wiki/Webware_for_Python
/wiki/Web2py
/wiki/Zope#Zope_2
/wiki/Category:Python_web_frameworks
/wiki/Template:FOSS
/wiki/Template_talk:FOSS
//en.wikipedia.org/w/index.php?title=Template:FOSS&action=edit
/wiki/Free_and_open-source_software
/wiki/Alternative_terms_for_free_software
/wiki/Comparison_of_open-source_and_closed-source_software
/wiki/Comparison_of_source_code_hosting_facilities
/wiki/Free_software
/wiki/List_of_free_software_project_directories
/wiki/Gratis_versus_libre
/wiki/Long-term_support
/wiki/Open-source_software
/wiki/Open-source_software_development
/wiki/Outline_of_free_software
/wiki/List_of_free_and_open-source_software_packages
/wiki/Comparison_of_free_software_for_audio
/wiki/List_of_open-source_bioinformatics_software
/wiki/List_of_open-source_codecs
/wiki/List_of_collaborative_software#Open_source_software
/wiki/Comparison_of_open-source_configuration_management_software
/wiki/Device_driver#Open_drivers
/wiki/Free_and_open-source_graphics_device_driver
/wiki/Comparison_of_open-source_wireless_drivers
/wiki/Comparison_of_free_geophysics_software
/wiki/List_of_open-source_health_software
/wiki/List_of_open-source_software_for_mathematics
/wiki/Comparison_of_open-source_operating_systems
/wiki/Comparison_of_open-source_programming_language_licensing
/wiki/List_of_open-source_routing_platforms
/wiki/List_of_statistical_packages#Open-source_statistical_packages
/wiki/List_of_free_television_software
/wiki/List_of_open-source_video_games
/wiki/List_of_free_software_web_applications
/wiki/List_of_content_management_systems#Open_source_software
/wiki/Comparison_of_free_software_e-commerce_web_application_frameworks
/wiki/List_of_word_processors#Free_and_open-source_software
/wiki/List_of_free_and_open-source_Android_applications
/wiki/List_of_free_and_open-source_iOS_applications
/wiki/List_of_commercial_open-source_applications_and_services
/wiki/List_of_trademarked_open-source_software
/wiki/List_of_formerly_proprietary_software
/wiki/Free_software_movement
/wiki/History_of_free_and_open-source_software
/wiki/Open-source_software_movement
/wiki/List_of_free_and_open-source_software_organizations
/wiki/List_of_free-software_events
/wiki/Free_software_license
/wiki/Apache_License
/wiki/Apple_Public_Source_License
/wiki/Artistic_License
/wiki/Beerware
/wiki/Boost_Software_License
/wiki/BSD_licenses
/wiki/CC0
/wiki/Common_Development_and_Distribution_License
/wiki/Eclipse_Public_License
/wiki/GNU_General_Public_License
/wiki/GNU_Lesser_General_Public_License
/wiki/ISC_license
/wiki/MIT_License
/wiki/Mozilla_Public_License
/wiki/Shared_source#Open_Source_licenses
/wiki/WTFPL
/wiki/Zlib_License
/wiki/Comparison_of_free_and_open-source_software_licenses
/wiki/Contributor_License_Agreement
/wiki/Copyfree
/wiki/Copyleft
/wiki/Debian_Free_Software_Guidelines
/wiki/Definition_of_Free_Cultural_Works
/wiki/Free_license
/wiki/The_Free_Software_Definition
/wiki/The_Open_Source_Definition
/wiki/Open-source_license
/wiki/Permissive_free_software_licence
/wiki/Public_domain
/wiki/Viral_license
/wiki/Binary_blob
/wiki/Digital_rights_management
/wiki/Hardware_restriction
/wiki/License_proliferation
/wiki/Mozilla_Corporation_software_rebranded_by_the_Debian_project
/wiki/Proprietary_software
/wiki/SCO/Linux_controversies
/wiki/UEFI_Secure_Boot#Secure_boot
/wiki/Software_patents_and_free_software
/wiki/Open-source_software_security
/wiki/Trusted_Computing
/wiki/The_Cathedral_and_the_Bazaar
/wiki/Fork_(software_development)
/wiki/Microsoft_Open_Specification_Promise
/wiki/Revolution_OS
/wiki/Book:Free_and_Open_Source_Software
/wiki/Category:Free_software
https://commons.wikimedia.org/wiki/Category:Free_software
/wiki/Portal:Free_software
/wiki/Help:Authority_control
/wiki/Library_of_Congress_Control_Number
http://id.loc.gov/authorities/subjects/sh96008834
/wiki/Integrated_Authority_File
http://d-nb.info/gnd/4434275-5
https://en.wikipedia.org/w/index.php?title=Python_(programming_language)&oldid=767686216
/wiki/Help:Category
/wiki/Category:Class-based_programming_languages
/wiki/Category:Computational_notebook
/wiki/Category:Computer_science_in_the_Netherlands
/wiki/Category:Cross-platform_free_software
/wiki/Category:Dutch_inventions
/wiki/Category:Dynamically_typed_programming_languages
/wiki/Category:Educational_programming_languages
/wiki/Category:High-level_programming_languages
/wiki/Category:Information_technology_in_the_Netherlands
/wiki/Category:Object-oriented_programming_languages
/wiki/Category:Programming_languages_created_in_1991
/wiki/Category:Python_(programming_language)
/wiki/Category:Scripting_languages
/wiki/Category:Text-oriented_programming_languages
/wiki/Category:Wikipedia_articles_needing_page_number_citations_from_January_2012
/wiki/Category:Articles_containing_potentially_dated_statements_from_August_2016
/wiki/Category:All_articles_containing_potentially_dated_statements
/wiki/Category:Articles_with_DMOZ_links
/wiki/Category:Wikipedia_articles_with_LCCN_identifiers
/wiki/Category:Wikipedia_articles_with_GND_identifiers
/wiki/Category:Good_articles
/wiki/Category:Use_dmy_dates_from_August_2015
/wiki/Special:MyTalk
/wiki/Special:MyContributions
/w/index.php?title=Special:CreateAccount&returnto=Python+%28programming+language%29
/w/index.php?title=Special:UserLogin&returnto=Python+%28programming+language%29
/wiki/Python_(programming_language)
/wiki/Talk:Python_(programming_language)
#
/wiki/Python_(programming_language)
/w/index.php?title=Python_(programming_language)&action=edit
/w/index.php?title=Python_(programming_language)&action=history
#
/wiki/Main_Page
/wiki/Main_Page
/wiki/Portal:Contents
/wiki/Portal:Featured_content
/wiki/Portal:Current_events
/wiki/Special:Random
https://donate.wikimedia.org/wiki/Special:FundraiserRedirector?utm_source=donate&utm_medium=sidebar&utm_campaign=C13_en.wikipedia.org&uselang=en
//shop.wikimedia.org
/wiki/Help:Contents
/wiki/Wikipedia:About
/wiki/Wikipedia:Community_portal
/wiki/Special:RecentChanges
//en.wikipedia.org/wiki/Wikipedia:Contact_us
/wiki/Special:WhatLinksHere/Python_(programming_language)
/wiki/Special:RecentChangesLinked/Python_(programming_language)
/wiki/Wikipedia:File_Upload_Wizard
/wiki/Special:SpecialPages
/w/index.php?title=Python_(programming_language)&oldid=767686216
/w/index.php?title=Python_(programming_language)&action=info
https://www.wikidata.org/wiki/Q28865
/w/index.php?title=Special:CiteThisPage&page=Python_%28programming_language%29&id=767686216
/w/index.php?title=Special:Book&bookcmd=book_creator&referer=Python+%28programming+language%29
/w/index.php?title=Special:Book&bookcmd=render_article&arttitle=Python+%28programming+language%29&returnto=Python+%28programming+language%29&oldid=767686216&writer=rdf2latex
/w/index.php?title=Python_(programming_language)&printable=yes
https://commons.wikimedia.org/wiki/Category:Python_(programming_language)
https://en.wikibooks.org/wiki/Python_Programming
https://en.wikiquote.org/wiki/Python
https://en.wikiversity.org/wiki/Python_Concepts
https://af.wikipedia.org/wiki/Python_(programmeertaal)
https://als.wikipedia.org/wiki/Python_(Programmiersprache)
https://ar.wikipedia.org/wiki/%D8%A8%D8%A7%D9%8A%D8%AB%D9%88%D9%86
https://an.wikipedia.org/wiki/Python
https://as.wikipedia.org/wiki/%E0%A6%AA%E0%A6%BE%E0%A6%87%E0%A6%A5%E0%A6%A8
https://az.wikipedia.org/wiki/Python_(proqramla%C5%9Fd%C4%B1rma_dili)
https://azb.wikipedia.org/wiki/%D9%BE%D8%A7%DB%8C%D8%AA%D9%88%D9%86
https://bn.wikipedia.org/wiki/%E0%A6%AA%E0%A6%BE%E0%A6%87%E0%A6%A5%E0%A6%A8_(%E0%A6%AA%E0%A7%8D%E0%A6%B0%E0%A7%8B%E0%A6%97%E0%A7%8D%E0%A6%B0%E0%A6%BE%E0%A6%AE%E0%A6%BF%E0%A6%82_%E0%A6%AD%E0%A6%BE%E0%A6%B7%E0%A6%BE)
https://zh-min-nan.wikipedia.org/wiki/Python
https://be.wikipedia.org/wiki/Python,_%D0%BC%D0%BE%D0%B2%D0%B0_%D0%BF%D1%80%D0%B0%D0%B3%D1%80%D0%B0%D0%BC%D0%B0%D0%B2%D0%B0%D0%BD%D0%BD%D1%8F
https://bg.wikipedia.org/wiki/Python
https://bs.wikipedia.org/wiki/Python_(programski_jezik)
https://ca.wikipedia.org/wiki/Python
https://ceb.wikipedia.org/wiki/Python_(programming_language)
https://cs.wikipedia.org/wiki/Python
https://da.wikipedia.org/wiki/Python_(programmeringssprog)
https://de.wikipedia.org/wiki/Python_(Programmiersprache)
https://et.wikipedia.org/wiki/Python_(programmeerimiskeel)
https://el.wikipedia.org/wiki/Python
https://es.wikipedia.org/wiki/Python
https://eo.wikipedia.org/wiki/Python_(programlingvo)
https://eu.wikipedia.org/wiki/Python_(informatika)
https://fa.wikipedia.org/wiki/%D9%BE%D8%A7%DB%8C%D8%AA%D9%88%D9%86_(%D8%B2%D8%A8%D8%A7%D9%86_%D8%A8%D8%B1%D9%86%D8%A7%D9%85%D9%87%E2%80%8C%D9%86%D9%88%DB%8C%D8%B3%DB%8C)
https://fr.wikipedia.org/wiki/Python_(langage)
https://gl.wikipedia.org/wiki/Python
https://gu.wikipedia.org/wiki/%E0%AA%AA%E0%AA%BE%E0%AA%AF%E0%AA%A5%E0%AB%8B%E0%AA%A8(%E0%AA%AA%E0%AB%8D%E0%AA%B0%E0%AB%8B%E0%AA%97%E0%AB%8D%E0%AA%B0%E0%AA%BE%E0%AA%AE%E0%AA%BF%E0%AA%82%E0%AA%97_%E0%AA%AD%E0%AA%BE%E0%AA%B7%E0%AA%BE)
https://ko.wikipedia.org/wiki/%ED%8C%8C%EC%9D%B4%EC%8D%AC
https://hy.wikipedia.org/wiki/Python
https://hi.wikipedia.org/wiki/%E0%A4%AA%E0%A4%BE%E0%A4%87%E0%A4%A5%E0%A4%A8_(%E0%A4%AA%E0%A5%8D%E0%A4%B0%E0%A5%8B%E0%A4%97%E0%A5%8D%E0%A4%B0%E0%A4%BE%E0%A4%AE%E0%A4%A8_%E0%A4%AD%E0%A4%BE%E0%A4%B7%E0%A4%BE)
https://hr.wikipedia.org/wiki/Python_(programski_jezik)
https://id.wikipedia.org/wiki/Python_(bahasa_pemrograman)
https://ia.wikipedia.org/wiki/Python_(linguage_de_programmation)
https://is.wikipedia.org/wiki/Python_(forritunarm%C3%A1l)
https://it.wikipedia.org/wiki/Python
https://he.wikipedia.org/wiki/%D7%A4%D7%99%D7%99%D7%AA%D7%95%D7%9F
https://ka.wikipedia.org/wiki/%E1%83%9E%E1%83%98%E1%83%97%E1%83%9D%E1%83%9C%E1%83%98_(%E1%83%9E%E1%83%A0%E1%83%9D%E1%83%92%E1%83%A0%E1%83%90%E1%83%9B%E1%83%98%E1%83%A0%E1%83%94%E1%83%91%E1%83%98%E1%83%A1_%E1%83%94%E1%83%9C%E1%83%90)
https://kk.wikipedia.org/wiki/Python
https://la.wikipedia.org/wiki/Python
https://lv.wikipedia.org/wiki/Python
https://lt.wikipedia.org/wiki/Python
https://jbo.wikipedia.org/wiki/paiton
https://lmo.wikipedia.org/wiki/Python
https://hu.wikipedia.org/wiki/Python_(programoz%C3%A1si_nyelv)
https://mk.wikipedia.org/wiki/%D0%9F%D0%B0%D1%98%D1%82%D0%BE%D0%BD_(%D0%BF%D1%80%D0%BE%D0%B3%D1%80%D0%B0%D0%BC%D1%81%D0%BA%D0%B8_%D1%98%D0%B0%D0%B7%D0%B8%D0%BA)
https://ml.wikipedia.org/wiki/%E0%B4%AA%E0%B5%88%E0%B4%A4%E0%B5%8D%E0%B4%A4%E0%B5%BA_(%E0%B4%AA%E0%B5%8D%E0%B4%B0%E0%B5%8B%E0%B4%97%E0%B5%8D%E0%B4%B0%E0%B4%BE%E0%B4%AE%E0%B4%BF%E0%B4%99%E0%B5%8D%E0%B4%99%E0%B5%8D_%E0%B4%AD%E0%B4%BE%E0%B4%B7)
https://mr.wikipedia.org/wiki/%E0%A4%AA%E0%A4%BE%E0%A4%AF%E0%A4%A5%E0%A5%89%E0%A4%A8
https://ms.wikipedia.org/wiki/Python
https://mn.wikipedia.org/wiki/Python
https://my.wikipedia.org/wiki/Python_(programming_language)
https://nl.wikipedia.org/wiki/Python_(programmeertaal)
https://ne.wikipedia.org/wiki/%E0%A4%AA%E0%A4%BE%E0%A4%87%E0%A4%A5%E0%A4%A8_%E0%A4%AA%E0%A5%8D%E0%A4%B0%E0%A5%8B%E0%A4%97%E0%A5%8D%E0%A4%B0%E0%A4%BE%E0%A4%AE%E0%A4%BF%E0%A4%99%E0%A5%8D%E0%A4%97_%E0%A4%AD%E0%A4%BE%E0%A4%B7%E0%A4%BE
https://ja.wikipedia.org/wiki/Python
https://no.wikipedia.org/wiki/Python
https://nn.wikipedia.org/wiki/Python
https://or.wikipedia.org/wiki/%E0%AC%AA%E0%AC%BE%E0%AC%87%E0%AC%A5%E0%AC%A8%E0%AD%8D_(%E0%AC%AA%E0%AD%8D%E0%AC%B0%E0%AD%8B%E0%AC%97%E0%AD%8D%E0%AC%B0%E0%AC%BE%E0%AC%AE%E0%AC%BF%E0%AC%82_%E0%AC%AD%E0%AC%BE%E0%AC%B7%E0%AC%BE)
https://uz.wikipedia.org/wiki/Python
https://pnb.wikipedia.org/wiki/%D9%BE%D8%A7%D8%A6%DB%8C%D8%AA%DA%BE%D9%86_(%DA%A9%D9%85%D9%BE%DB%8C%D9%88%D9%B9%D8%B1_%D8%A8%D9%88%D9%84%DB%8C)
https://km.wikipedia.org/wiki/%E1%9E%95%E1%9E%B6%E1%9E%99%E1%9E%90%E1%9E%BB%E1%9E%93
https://nds.wikipedia.org/wiki/Python_(Programmeerspraak)
https://pl.wikipedia.org/wiki/Python
https://pt.wikipedia.org/wiki/Python
https://ro.wikipedia.org/wiki/Python
https://ru.wikipedia.org/wiki/Python
https://sco.wikipedia.org/wiki/Python_(programmin_leid)
https://sq.wikipedia.org/wiki/Python
https://si.wikipedia.org/wiki/%E0%B6%B4%E0%B6%BA%E0%B7%92%E0%B6%AD%E0%B6%B1%E0%B7%8A
https://simple.wikipedia.org/wiki/Python_(programming_language)
https://sk.wikipedia.org/wiki/Python_(programovac%C3%AD_jazyk)
https://sl.wikipedia.org/wiki/Python_(programski_jezik)
https://ckb.wikipedia.org/wiki/%D9%BE%D8%A7%DB%8C%D8%AA%D9%86_(%D8%B2%D9%85%D8%A7%D9%86%DB%8C_%D8%A8%DB%95%D8%B1%D9%86%D8%A7%D9%85%DB%95%D8%B3%D8%A7%D8%B2%DB%8C)
https://sr.wikipedia.org/wiki/%D0%9F%D0%B0%D1%98%D1%82%D0%BE%D0%BD_(%D0%BF%D1%80%D0%BE%D0%B3%D1%80%D0%B0%D0%BC%D1%81%D0%BA%D0%B8_%D1%98%D0%B5%D0%B7%D0%B8%D0%BA)
https://sh.wikipedia.org/wiki/Python_programski_jezik
https://fi.wikipedia.org/wiki/Python_(ohjelmointikieli)
https://sv.wikipedia.org/wiki/Python_(programspr%C3%A5k)
https://tl.wikipedia.org/wiki/Python_(wikang_pamprograma)
https://ta.wikipedia.org/wiki/%E0%AE%AA%E0%AF%88%E0%AE%A4%E0%AF%8D%E0%AE%A4%E0%AE%BE%E0%AE%A9%E0%AF%8D
https://te.wikipedia.org/wiki/%E0%B0%AA%E0%B1%88%E0%B0%A5%E0%B0%BE%E0%B0%A8%E0%B1%8D_(%E0%B0%95%E0%B0%82%E0%B0%AA%E0%B1%8D%E0%B0%AF%E0%B1%82%E0%B0%9F%E0%B0%B0%E0%B1%8D_%E0%B0%AD%E0%B0%BE%E0%B0%B7)
https://th.wikipedia.org/wiki/%E0%B8%A0%E0%B8%B2%E0%B8%A9%E0%B8%B2%E0%B9%84%E0%B8%9E%E0%B8%97%E0%B8%AD%E0%B8%99
https://tg.wikipedia.org/wiki/Python
https://tr.wikipedia.org/wiki/Python_(programlama_dili)
https://bug.wikipedia.org/wiki/Python
https://uk.wikipedia.org/wiki/Python
https://ur.wikipedia.org/wiki/%D9%BE%D8%A7%D8%A6%DB%8C%D8%AA%DA%BE%D9%86_%D9%BE%D8%B1%D9%88%DA%AF%D8%B1%D8%A7%D9%85%D9%86%DA%AF_%D8%B2%D8%A8%D8%A7%D9%86
https://vi.wikipedia.org/wiki/Python_(ng%C3%B4n_ng%E1%BB%AF_l%E1%BA%ADp_tr%C3%ACnh)
https://wuu.wikipedia.org/wiki/Python
https://zh-yue.wikipedia.org/wiki/Python
https://zh.wikipedia.org/wiki/Python
https://www.wikidata.org/wiki/Q28865#sitelinks-wikipedia
//en.wikipedia.org/wiki/Wikipedia:Text_of_Creative_Commons_Attribution-ShareAlike_3.0_Unported_License
//creativecommons.org/licenses/by-sa/3.0/
//wikimediafoundation.org/wiki/Terms_of_Use
//wikimediafoundation.org/wiki/Privacy_policy
//www.wikimediafoundation.org/
https://wikimediafoundation.org/wiki/Privacy_policy
/wiki/Wikipedia:About
/wiki/Wikipedia:General_disclaimer
//en.wikipedia.org/wiki/Wikipedia:Contact_us
https://www.mediawiki.org/wiki/Special:MyLanguage/How_to_contribute
https://wikimediafoundation.org/wiki/Cookie_statement
//en.m.wikipedia.org/w/index.php?title=Python_(programming_language)&mobileaction=toggle_view_mobile
https://wikimediafoundation.org/
//www.mediawiki.org/

可以发现，所有指向Wikipedia词条的链接都是/wiki/开头，所以我们可以用正则表达式来过滤出这些词条，就像这样



In [2]:

    
from urllib.request import urlopen
from bs4 import BeautifulSoup
import re

html = urlopen("https://en.wikipedia.org/wiki/Python_(programming_language)")
bsObj = BeautifulSoup(html.read(), "html.parser")
for link in bsObj.find("div", {"id":"bodyContent"}).findAll("a", href=re.compile("^(/wiki/)((?!:).)*$")):
    if 'href' in link.attrs:
        print(link.attrs['href'])









    



/wiki/Programming_paradigm
/wiki/Multi-paradigm_programming_language
/wiki/Object-oriented_programming
/wiki/Imperative_programming
/wiki/Functional_programming
/wiki/Procedural_programming
/wiki/Reflective_programming
/wiki/Software_design
/wiki/Guido_van_Rossum
/wiki/Software_developer
/wiki/Python_Software_Foundation
/wiki/Software_release_life_cycle
/wiki/Type_system
/wiki/Duck_typing
/wiki/Dynamic_typing
/wiki/Strong_typing
/wiki/Operating_system
/wiki/Cross-platform
/wiki/Software_license
/wiki/Python_Software_Foundation_License
/wiki/Filename_extension
/wiki/Programming_language_implementation
/wiki/CPython
/wiki/IronPython
/wiki/Jython
/wiki/MicroPython
/wiki/PyPy
/wiki/Dialect_(computing)
/wiki/Cython
/wiki/RPython
/wiki/Stackless_Python
/wiki/ABC_(programming_language)
/wiki/ALGOL_68
/wiki/C_(programming_language)
/wiki/C%2B%2B
/wiki/Dylan_(programming_language)
/wiki/Haskell_(programming_language)
/wiki/Icon_(programming_language)
/wiki/Java_(programming_language)
/wiki/Lisp_(programming_language)
/wiki/Modula-3
/wiki/Perl
/wiki/Boo_(programming_language)
/wiki/Cobra_(programming_language)
/wiki/CoffeeScript
/wiki/D_(programming_language)
/wiki/F_Sharp_(programming_language)
/wiki/Falcon_(programming_language)
/wiki/Genie_(programming_language)
/wiki/Go_(programming_language)
/wiki/Groovy_(programming_language)
/wiki/JavaScript
/wiki/Julia_(programming_language)
/wiki/Nim_(programming_language)
/wiki/Ruby_(programming_language)
/wiki/Swift_(programming_language)
/wiki/High-level_programming_language
/wiki/General-purpose_programming_language
/wiki/Guido_van_Rossum
/wiki/Interpreted_language
/wiki/Readability
/wiki/Whitespace_character
/wiki/Code_block
/wiki/Source_lines_of_code
/wiki/C%2B%2B
/wiki/Java_(programming_language)
/wiki/Dynamic_type
/wiki/Memory_management
/wiki/Programming_paradigm
/wiki/Object-oriented_programming
/wiki/Imperative_programming
/wiki/Functional_programming
/wiki/Procedural_programming
/wiki/Standard_library
/wiki/Operating_system
/wiki/CPython
/wiki/Reference_implementation
/wiki/Open_source
/wiki/Python_Software_Foundation
/wiki/Guido_van_Rossum
/wiki/History_of_Python
/wiki/Guido_van_Rossum
/wiki/Centrum_Wiskunde_%26_Informatica
/wiki/Netherlands
/wiki/ABC_(programming_language)
/wiki/SETL
/wiki/Exception_handling
/wiki/Amoeba_(operating_system)
/wiki/Benevolent_dictator_for_life
/wiki/ABC_(programming_language)
/wiki/Unix
/wiki/C_(programming_language)
/wiki/Hacker_(programmer_subculture)
/wiki/Monty_Python%27s_Flying_Circus
/wiki/Cycle_detection
/wiki/Garbage_collection_(computer_science)
/wiki/Unicode
/wiki/Backporting
/wiki/Go_(programming_language)
/wiki/Transcompiler
/wiki/The_Register
/wiki/Multi-paradigm_programming_language
/wiki/Object-oriented_programming
/wiki/Structured_programming
/wiki/Functional_programming
/wiki/Aspect-oriented_programming
/wiki/Metaprogramming
/wiki/Metaobject
/wiki/Design_by_contract
/wiki/Logic_programming
/wiki/Dynamic_typing
/wiki/Reference_counting
/wiki/Memory_management
/wiki/Name_resolution_(programming_languages)
/wiki/Late_binding
/wiki/Functional_programming
/wiki/Lisp_(programming_language)
/wiki/List_comprehension
/wiki/Associative_array
/wiki/Generator_(computer_programming)
/wiki/Haskell_(programming_language)
/wiki/Standard_ML
/wiki/Zen_of_Python
/wiki/Aphorism
/wiki/ABC_(programming_language)
/wiki/Perl
/wiki/Alex_Martelli
/wiki/There_is_more_than_one_way_to_do_it
/wiki/Premature_optimization
/wiki/PyPy
/wiki/Just-in-time_compilation
/wiki/Cython
/wiki/Monty_Python
/wiki/Foobar
/wiki/Neologism
/wiki/Python_syntax_and_semantics
/wiki/C_(programming_language)
/wiki/Pascal_(programming_language)
/wiki/Python_syntax_and_semantics#Indentation
/wiki/Whitespace_character
/wiki/Block_(programming)
/wiki/Curly_bracket_programming_language
/wiki/Off-side_rule
/wiki/Statement_(computer_science)
/wiki/Imperative_programming
/wiki/C_(programming_language)
/wiki/Memory_management#ALLOCATION
/wiki/Variable_(computer_science)
/wiki/Type_system
/wiki/Pointer_(computer_programming)
/wiki/Object_(computer_science)
/wiki/Type_system
/wiki/Dynamic_type
/wiki/If-then-else
/wiki/Foreach#Python
/wiki/While_loop#Python
/wiki/Exception_handling_syntax#Python
/wiki/Class_(computer_science)
/wiki/Object-oriented_programming
/wiki/Function_(computing)
/wiki/Method_(computing)
/wiki/Lock_(computer_science)
/wiki/Computer_file
/wiki/Resource_Acquisition_Is_Initialization
/wiki/NOP
/wiki/Assertion_(programming)
/wiki/Generator_(computer_science)#Python
/wiki/Coroutine
/wiki/Tail_call
/wiki/First-class_continuations
/wiki/Coroutine
/wiki/Generator_(computer_science)
/wiki/Lazy_evaluation
/wiki/Iterator
/wiki/Expression_(computer_science)
/wiki/C_(programming_language)
/wiki/Java_(programming_language)
/wiki/List_comprehension#Python
/wiki/Generator_(computer_science)
/wiki/Anonymous_function
/wiki/Lambda_(programming)
/wiki/List_(computer_science)
/wiki/Tuple
/wiki/Immutable
/wiki/Printf
/wiki/C_(programming_language)
/wiki/String_literal
/wiki/Unix_shell
/wiki/Perl
/wiki/Escape_character
/wiki/String_interpolation
/wiki/Here_document
/wiki/Ruby_(programming_language)
/wiki/Raw_string
/wiki/Regular_expression
/wiki/Microsoft_Windows
/wiki/C_Sharp_(programming_language)
/wiki/Array_index
/wiki/Array_slicing
/wiki/Zero-based
/wiki/Shallow_copy
/wiki/Common_Lisp
/wiki/Scheme_(programming_language)
/wiki/Ruby_(programming_language)
/wiki/List_comprehensions
/wiki/Conditional_(programming)
/wiki/Lambda_(programming)
/wiki/Method_(programming)
/wiki/Function_(programming)
/wiki/Syntactic_sugar
/wiki/This_(computer_programming)
/wiki/Instance_data
/wiki/C%2B%2B
/wiki/Java_(programming_language)
/wiki/Objective-C
/wiki/Ruby_(programming_language)
/wiki/Duck_typing
/wiki/Compile_time
/wiki/Type_system#Dynamic_type_checking_and_runtime_type_information
/wiki/Strongly_typed_programming_language
/wiki/Class_(computer_science)
/wiki/Object-oriented_programming
/wiki/Object_(computer_science)
/wiki/Metaclass
/wiki/Metaprogramming
/wiki/Reflection_(computer_science)
/wiki/Gradual_typing
/wiki/Immutable_object
/wiki/Character_string
/wiki/Byte
/wiki/List_(computer_science)
/wiki/Set_(computer_science)
/wiki/Set_(computer_science)
/wiki/Associative_array
/wiki/Integer_(computer_science)
/wiki/Floating_point
/wiki/Complex_number
/wiki/Boolean_value
/wiki/Ellipsis_(programming_operator)
/wiki/NumPy
/wiki/Half-open_interval
/wiki/Rounding
/wiki/Rounding#Tie-breaking
/wiki/Round_to_even
/wiki/Arbitrary_precision_arithmetic
/wiki/Standard_library
/wiki/MIME
/wiki/Hypertext_Transfer_Protocol
/wiki/Graphical_user_interface
/wiki/Relational_database
/wiki/Pseudorandom_number_generator
/wiki/Regular_expression
/wiki/Unit_testing
/wiki/Web_Server_Gateway_Interface
/wiki/Python_Package_Index
/wiki/Comparison_of_integrated_development_environments#Python
/wiki/Read%E2%80%93eval%E2%80%93print_loop
/wiki/Command_line_interpreter
/wiki/IDLE_(Python)
/wiki/IPython
/wiki/Syntax_highlighting
/wiki/Integrated_development_environment
/wiki/Python_IDE
/wiki/Web_browser
/wiki/SageMath
/wiki/PythonAnywhere
/wiki/List_of_Python_software#Python_implementations
/wiki/CPython
/wiki/C_(programming_language)
/wiki/C89_(C_version)
/wiki/Bytecode
/wiki/Virtual_machine
/wiki/Microsoft_Windows
/wiki/Unix-like
/wiki/PyPy
/wiki/Just-in-time_compilation
/wiki/Multi-core_processor
/wiki/Software_transactional_memory
/wiki/Stackless_Python
/wiki/Microthread
/wiki/MicroPython
/wiki/Microcontroller
/wiki/Unladen_Swallow
/wiki/LLVM
/wiki/Psyco
/wiki/Just-in-time_compilation
/wiki/Run-time_algorithm_specialisation
/wiki/Data_type
/wiki/Nokia
/wiki/Series_60
/wiki/PyS60
/wiki/Symbian
/wiki/N900
/wiki/GTK
/wiki/Object_language
/wiki/Jython
/wiki/Java_virtual_machine
/wiki/IronPython
/wiki/Common_Language_Runtime
/wiki/RPython
/wiki/C_(programming_language)
/wiki/Java_bytecode
/wiki/Common_Intermediate_Language
/wiki/Pyjamas_(software)
/wiki/JavaScript
/wiki/Shed_Skin
/wiki/C%2B%2B
/wiki/Cython
/wiki/Pyrex_(programming_language)
/wiki/C_(programming_language)
/wiki/Go_(programming_language)
/wiki/Benevolent_dictator_for_life
/wiki/Roundup_(issue_tracker)
/wiki/Bug_tracker
/wiki/Self-hosted
/wiki/Mercurial
/wiki/GitHub
/wiki/Ported
/wiki/Beta_release
/wiki/Unit_test
/wiki/BuildBot
/wiki/Continuous_integration
/wiki/Python_Package_Index
/wiki/Academic_conference
/wiki/PyCon
/wiki/Pyladies
/wiki/Monty_Python%27s_Flying_Circus
/wiki/Metasyntactic_variable
/wiki/Spam_(Monty_Python)
/wiki/Foobar
/wiki/Pygame
/wiki/Language_binding
/wiki/Simple_DirectMedia_Layer
/wiki/PyS60
/wiki/Symbian
/wiki/S60_(software_platform)
/wiki/PyQt
/wiki/PyGTK
/wiki/Qt_(software)
/wiki/GTK
/wiki/PyPy
/wiki/List_of_Python_software
/wiki/TIOBE_Programming_Community_Index
/wiki/Syntax_(programming_languages)
/wiki/C_(programming_language)
/wiki/Wikipedia
/wiki/Google
/wiki/Yahoo!
/wiki/CERN
/wiki/NASA
/wiki/Industrial_Light_%26_Magic
/wiki/ITA_Software
/wiki/Reddit
/wiki/Scripting_language
/wiki/Web_application
/wiki/Mod_wsgi
/wiki/Apache_web_server
/wiki/Web_Server_Gateway_Interface
/wiki/Web_framework
/wiki/Django_(web_framework)
/wiki/Pylons_(web_framework)
/wiki/Pyramid_(web_framework)
/wiki/TurboGears
/wiki/Web2py
/wiki/Tornado_(web_server)
/wiki/Flask_(web_framework)
/wiki/Bottle_(web_framework)
/wiki/Zope
/wiki/Pyjamas_(software)
/wiki/IronPython
/wiki/SQLAlchemy
/wiki/Data_mapper_pattern
/wiki/Twisted_(software)
/wiki/Dropbox_(service)
/wiki/NumPy
/wiki/SciPy
/wiki/Matplotlib
/wiki/Biopython
/wiki/Astropy
/wiki/SageMath
/wiki/Mathematical_software
/wiki/Mathematics
/wiki/Algebra
/wiki/Combinatorics
/wiki/Numerical_mathematics
/wiki/Number_theory
/wiki/Calculus
/wiki/Java_(programming_language)
/wiki/DataMelt
/wiki/Finite_element_method
/wiki/Abaqus
/wiki/FreeCAD
/wiki/3ds_Max
/wiki/Blender_(software)
/wiki/Cinema_4D
/wiki/Lightwave
/wiki/Houdini_(software)
/wiki/Maya_(software)
/wiki/Modo_(software)
/wiki/MotionBuilder
/wiki/Autodesk_Softimage
/wiki/Nuke_(software)
/wiki/GIMP
/wiki/Inkscape
/wiki/Scribus
/wiki/Paint_Shop_Pro
/wiki/Musical_notation
/wiki/Scorewriter
/wiki/Capella_(notation_program)
/wiki/GNU_Debugger
/wiki/Prettyprint
/wiki/Esri
/wiki/ArcGIS
/wiki/Programming_language
/wiki/Google_App_Engine
/wiki/Java_(software_platform)
/wiki/Go_(programming_language)
/wiki/Algorithmic_trading
/wiki/Artificial_intelligence
/wiki/Natural_language_processing
/wiki/Linux_distribution
/wiki/AmigaOS_4
/wiki/FreeBSD
/wiki/NetBSD
/wiki/OpenBSD
/wiki/MacOS
/wiki/Ubuntu_(operating_system)
/wiki/Ubiquity_(software)
/wiki/Red_Hat_Linux
/wiki/Fedora_(operating_system)
/wiki/Anaconda_(installer)
/wiki/Gentoo_Linux
/wiki/Package_management_system
/wiki/Portage_(software)
/wiki/Information_security
/wiki/Sugar_(software)
/wiki/One_Laptop_per_Child
/wiki/Sugar_Labs
/wiki/Raspberry_Pi
/wiki/Single-board_computer
/wiki/LibreOffice
/wiki/Boo_(programming_language)
/wiki/Static_typing
/wiki/Duck_typing
/wiki/.NET_Framework
/wiki/Cobra_(programming_language)
/wiki/Design_by_contract
/wiki/Unit_testing
/wiki/Static_typing
/wiki/ECMAScript
/wiki/Iterator
/wiki/Generator_(computer_science)
/wiki/List_comprehension
/wiki/Go_(programming_language)
/wiki/Groovy_(programming_language)
/wiki/Java_(programming_language)
/wiki/Julia_(programming_language)
/wiki/Hygienic_macro
/wiki/OCaml
/wiki/Haskell_(programming_language)
/wiki/Ruby_(programming_language)
/wiki/Yukihiro_Matsumoto
/wiki/CoffeeScript
/wiki/Swift_(programming_language)
/wiki/Tcl
/wiki/Erlang_(programming_language)
/wiki/TIOBE_index
/wiki/Comparison_of_integrated_development_environments#Python
/wiki/Comparison_of_programming_languages
/wiki/List_of_programming_languages
/wiki/Off-side_rule
/wiki/Blogger_(service)
/wiki/International_Standard_Book_Number
/wiki/International_Standard_Book_Number
/wiki/Guido_van_Rossum
/wiki/O%27Reilly_Media
/wiki/International_Standard_Book_Number
/wiki/International_Standard_Book_Number
/wiki/Oracle_Corporation
/wiki/ArXiv
/wiki/Stack_Overflow
/wiki/International_Standard_Book_Number
/wiki/International_Standard_Book_Number
/wiki/CCP_Games
/wiki/Firaxis_Games
/wiki/Stuart_J._Russell
/wiki/Peter_Norvig
/wiki/International_Standard_Book_Number
/wiki/The_Document_Foundation
/wiki/Chris_Lattner
/wiki/International_Standard_Book_Number
/wiki/International_Standard_Book_Number
/wiki/International_Standard_Book_Number
/wiki/International_Standard_Book_Number
/wiki/International_Standard_Book_Number
/wiki/Usenet
/wiki/Google_Groups
/wiki/DMOZ
/wiki/Programming_language
/wiki/Comparison_of_programming_languages
/wiki/Timeline_of_programming_languages
/wiki/History_of_programming_languages
/wiki/Assembly_language
/wiki/BASIC
/wiki/C_(programming_language)
/wiki/C%2B%2B
/wiki/C_Sharp_(programming_language)
/wiki/COBOL
/wiki/Fortran
/wiki/Go_(programming_language)
/wiki/Haskell_(programming_language)
/wiki/Java_(programming_language)
/wiki/JavaScript
/wiki/Lisp_(programming_language)
/wiki/Lua_(programming_language)
/wiki/Objective-C
/wiki/Pascal_(programming_language)
/wiki/Perl
/wiki/PHP
/wiki/Ruby_(programming_language)
/wiki/Shell_script
/wiki/Smalltalk
/wiki/Swift_(programming_language)
/wiki/Visual_Basic_.NET
/wiki/List_of_programming_languages
/wiki/Comparison_of_programming_languages
/wiki/List_of_programming_languages
/wiki/Categorical_list_of_programming_languages
/wiki/Generational_list_of_programming_languages
/wiki/Non-English-based_programming_languages
/wiki/CLPython
/wiki/CPython
/wiki/Cython
/wiki/MicroPython
/wiki/IronPython
/wiki/Jython
/wiki/Psyco
/wiki/PyPy
/wiki/Python_for_S60
/wiki/Stackless_Python
/wiki/Unladen_Swallow
/wiki/Comparison_of_integrated_development_environments#Python
/wiki/Eric_Python_IDE
/wiki/IDLE_(Python)
/wiki/PyDev
/wiki/Stani%27s_Python_Editor
/wiki/Ninja-IDE
/wiki/List_of_integrated_development_environments_for_Python#Python
/wiki/Web_Server_Gateway_Interface
/wiki/List_of_Python_software
/wiki/Python_Software_Foundation
/wiki/Python_Conference
/wiki/Web_framework
/wiki/Zope#BlueBream
/wiki/CherryPy
/wiki/Django_(web_framework)
/wiki/Flask_(web_framework)
/wiki/Grok_(web_framework)
/wiki/Nagare_(web_framework)
/wiki/Nevow
/wiki/Pyjamas_(software)
/wiki/Pylons_project#Pylons_Framework
/wiki/Pylons_project#Pyramid
/wiki/Quixote_(web_framework)
/wiki/Spyce
/wiki/TACTIC_(web_framework)
/wiki/Tornado_(web_server)
/wiki/TurboGears
/wiki/Twisted_(software)
/wiki/Webware_for_Python
/wiki/Web2py
/wiki/Zope#Zope_2
/wiki/Free_and_open-source_software
/wiki/Alternative_terms_for_free_software
/wiki/Comparison_of_open-source_and_closed-source_software
/wiki/Comparison_of_source_code_hosting_facilities
/wiki/Free_software
/wiki/List_of_free_software_project_directories
/wiki/Gratis_versus_libre
/wiki/Long-term_support
/wiki/Open-source_software
/wiki/Open-source_software_development
/wiki/Outline_of_free_software
/wiki/List_of_free_and_open-source_software_packages
/wiki/Comparison_of_free_software_for_audio
/wiki/List_of_open-source_bioinformatics_software
/wiki/List_of_open-source_codecs
/wiki/List_of_collaborative_software#Open_source_software
/wiki/Comparison_of_open-source_configuration_management_software
/wiki/Device_driver#Open_drivers
/wiki/Free_and_open-source_graphics_device_driver
/wiki/Comparison_of_open-source_wireless_drivers
/wiki/Comparison_of_free_geophysics_software
/wiki/List_of_open-source_health_software
/wiki/List_of_open-source_software_for_mathematics
/wiki/Comparison_of_open-source_operating_systems
/wiki/Comparison_of_open-source_programming_language_licensing
/wiki/List_of_open-source_routing_platforms
/wiki/List_of_statistical_packages#Open-source_statistical_packages
/wiki/List_of_free_television_software
/wiki/List_of_open-source_video_games
/wiki/List_of_free_software_web_applications
/wiki/List_of_content_management_systems#Open_source_software
/wiki/Comparison_of_free_software_e-commerce_web_application_frameworks
/wiki/List_of_word_processors#Free_and_open-source_software
/wiki/List_of_free_and_open-source_Android_applications
/wiki/List_of_free_and_open-source_iOS_applications
/wiki/List_of_commercial_open-source_applications_and_services
/wiki/List_of_trademarked_open-source_software
/wiki/List_of_formerly_proprietary_software
/wiki/Free_software_movement
/wiki/History_of_free_and_open-source_software
/wiki/Open-source_software_movement
/wiki/List_of_free_and_open-source_software_organizations
/wiki/List_of_free-software_events
/wiki/Free_software_license
/wiki/Apache_License
/wiki/Apple_Public_Source_License
/wiki/Artistic_License
/wiki/Beerware
/wiki/Boost_Software_License
/wiki/BSD_licenses
/wiki/CC0
/wiki/Common_Development_and_Distribution_License
/wiki/Eclipse_Public_License
/wiki/GNU_General_Public_License
/wiki/GNU_Lesser_General_Public_License
/wiki/ISC_license
/wiki/MIT_License
/wiki/Mozilla_Public_License
/wiki/Shared_source#Open_Source_licenses
/wiki/WTFPL
/wiki/Zlib_License
/wiki/Comparison_of_free_and_open-source_software_licenses
/wiki/Contributor_License_Agreement
/wiki/Copyfree
/wiki/Copyleft
/wiki/Debian_Free_Software_Guidelines
/wiki/Definition_of_Free_Cultural_Works
/wiki/Free_license
/wiki/The_Free_Software_Definition
/wiki/The_Open_Source_Definition
/wiki/Open-source_license
/wiki/Permissive_free_software_licence
/wiki/Public_domain
/wiki/Viral_license
/wiki/Binary_blob
/wiki/Digital_rights_management
/wiki/Hardware_restriction
/wiki/License_proliferation
/wiki/Mozilla_Corporation_software_rebranded_by_the_Debian_project
/wiki/Proprietary_software
/wiki/SCO/Linux_controversies
/wiki/UEFI_Secure_Boot#Secure_boot
/wiki/Software_patents_and_free_software
/wiki/Open-source_software_security
/wiki/Trusted_Computing
/wiki/The_Cathedral_and_the_Bazaar
/wiki/Fork_(software_development)
/wiki/Microsoft_Open_Specification_Promise
/wiki/Revolution_OS
/wiki/Library_of_Congress_Control_Number
/wiki/Integrated_Authority_File

上面的函数还不太能用于实际抓取，我们稍作改进，变成下面这个样子，就可以初步用于抓取页面的所有链接了。因为我们不能无限制地抓取下去，我便设置了10个链接的上限。



In [ ]:

    
from urllib.request import urlopen
from bs4 import BeautifulSoup
import datetime
import random
import re

count = 0
random.seed(datetime.datetime.now())
def getLinks(articleUrl):
    html = urlopen("http://en.wikipedia.org"+articleUrl)
    bsObj = BeautifulSoup(html, "html.parser")
    return bsObj.find("div", {"id":"bodyContent"}).findAll("a", href=re.compile("^(/wiki/)((?!:).)*$"))
links = getLinks("/wiki/Python_(programming_language)")
while len(links) > 0 and count < 10:
    newArticle = links[random.randint(0, len(links)-1)].attrs["href"]
    print(newArticle)
    count = count + 1
    links = getLinks(newArticle)









    



/wiki/C_(programming_language)
/wiki/Free-form_language
/wiki/Computer_programming
/wiki/Virtual_reality
/wiki/Visual_music
/wiki/Oscilloscope
/wiki/Digital_storage_oscilloscope
/wiki/Analog_signal
/wiki/Mobile_telephony
/wiki/Roaming

为了避免一个页面被采集两次，链接去重是非常重要的，下面的代码用Python的set来保存已经采集的链接。下面这段代码将无限制地运行下去，除非set集为空，然而这几乎是不可能的。



In [ ]:

    
from urllib.request import urlopen
from bs4 import BeautifulSoup
import re

pages = set()
def getLinks(pageUrl):
    global pages
    html = urlopen("http://en.wikipedia.org"+pageUrl)
    bsObj = BeautifulSoup(html, "html.parser")
    try:
        print(bsObj.h1.get_text())
        print(bsObj.find(id ="mw-content-text").findAll("p")[0])
        print(bsObj.find(id="ca-edit").find("span").find("a").attrs['href'])
    except AttributeError:
        print("This page is missing something! No worries though!")
    
    for link in bsObj.findAll("a", href=re.compile("^(/wiki/)")):
        if 'href' in link.attrs:
            if link.attrs['href'] not in pages:
                #We have encountered a new page
                newPage = link.attrs['href']
                print("----------------\n"+newPage)
                pages.add(newPage)
                getLinks(newPage)
getLinks("")









    



Main Page
<p><b><a href="/wiki/Henry_Ho%CA%BBolulu_Pitman" title="Henry Hoʻolulu Pitman">Henry Hoʻolulu Pitman</a></b> (1845–1863) was one of more than one hundred <a href="/wiki/Native_Hawaiians" title="Native Hawaiians">Native Hawaiians</a> and <a href="/wiki/Hawaiian_Islands" title="Hawaiian Islands">Hawaiian</a>-born combatants who fought in the <a href="/wiki/American_Civil_War" title="American Civil War">American Civil War</a> while Hawaii was still an <a href="/wiki/Kingdom_of_Hawaii" title="Kingdom of Hawaii">independent kingdom</a>. His father was a merchant from Massachusetts and his mother, <a href="/wiki/Kinooleoliliha" title="Kinooleoliliha">Kinoʻoleoliliha</a>, was a Hawaiian noble. He returned to the United States with his father for his education, but ran away from school without his family's knowledge and enlisted in the <a href="/wiki/Union_Army" title="Union Army">Union Army</a> as a <a href="/wiki/Private_(rank)" title="Private (rank)">private</a>. Despite his mixed-race ancestry, he avoided the racial segregation imposed on other Hawaiian recruits of the time and was assigned to a white regiment. He fought in the <a href="/wiki/Battle_of_Antietam" title="Battle of Antietam">Battle of Antietam</a> and the <a href="/wiki/Maryland_Campaign" title="Maryland Campaign">Maryland Campaign</a> and befriended <a href="/wiki/Robert_G._Carter" title="Robert G. Carter">Robert G. Carter</a>, a <a href="/wiki/Memoir" title="Memoir">memoirist</a> of the Civil War. On the march to <a href="/wiki/Battle_of_Fredericksburg" title="Battle of Fredericksburg">Fredericksburg</a>, he was separated from his regiment and captured by <a href="/wiki/Confederate_States_Army" title="Confederate States Army">Confederate</a> guerrilla forces. He was marched to Richmond and incarcerated in <a href="/wiki/Libby_Prison" title="Libby Prison">Libby Prison</a>, where he contracted a lung disease from the harsh conditions. He died on February<span class="nowrap"> </span>27, 1863, after his release on parole in a prisoner exchange. His legacy has sparked renewed interest in the role <a href="/wiki/Hawaii_and_the_American_Civil_War" title="Hawaii and the American Civil War">Hawaiians played in the Civil War</a>. (<a href="/wiki/Henry_Ho%CA%BBolulu_Pitman" title="Henry Hoʻolulu Pitman"><b>Full article...</b></a>)</p>
This page is missing something! No worries though!
----------------
/wiki/Wikipedia
Wikipedia
<p><b>Wikipedia</b> (<span class="nowrap"><span class="noexcerpt"><a href="//upload.wikimedia.org/wikipedia/commons/0/01/GT_Wikipedia_BE.ogg" title="Listen"><img alt="Listen" data-file-height="11" data-file-width="11" height="11" src="//upload.wikimedia.org/wikipedia/commons/thumb/3/3b/Speakerlink-new.svg/11px-Speakerlink-new.svg.png" srcset="//upload.wikimedia.org/wikipedia/commons/thumb/3/3b/Speakerlink-new.svg/17px-Speakerlink-new.svg.png 1.5x, //upload.wikimedia.org/wikipedia/commons/thumb/3/3b/Speakerlink-new.svg/22px-Speakerlink-new.svg.png 2x" width="11"/></a><sup><span class="IPA" style="color:#00e;font:bold 80% sans-serif;padding:0 .1em"><a href="/wiki/File:GT_Wikipedia_BE.ogg" title="File:GT Wikipedia BE.ogg">i</a></span></sup></span><span class="IPA nopopups"><a href="/wiki/Help:IPA_for_English" title="Help:IPA for English">/<span style="border-bottom:1px dotted"><span title="/ˌ/ secondary stress follows">ˌ</span><span title="'w' in 'wind'">w</span><span title="/ɪ/ short 'i' in 'bid'">ɪ</span><span title="'k' in 'kind'">k</span><span title="/ɪ/ or /ə/ 'e' in 'roses'">ᵻ</span><span title="/ˈ/ primary stress follows">ˈ</span><span title="'p' in 'pie'">p</span><span title="/iː/ long 'e' in 'seed'">iː</span><span title="'d' in 'dye'">d</span><span title="/i/ 'y' in 'happy'">i</span><span title="/ə/ 'a' in 'about'">ə</span></span>/</a></span></span> or <span class="nowrap"><span class="noexcerpt"><a href="//upload.wikimedia.org/wikipedia/commons/4/4c/GT_Wikipedia_AE.ogg" title="Listen"><img alt="Listen" data-file-height="11" data-file-width="11" height="11" src="//upload.wikimedia.org/wikipedia/commons/thumb/3/3b/Speakerlink-new.svg/11px-Speakerlink-new.svg.png" srcset="//upload.wikimedia.org/wikipedia/commons/thumb/3/3b/Speakerlink-new.svg/17px-Speakerlink-new.svg.png 1.5x, //upload.wikimedia.org/wikipedia/commons/thumb/3/3b/Speakerlink-new.svg/22px-Speakerlink-new.svg.png 2x" width="11"/></a><sup><span class="IPA" style="color:#00e;font:bold 80% sans-serif;padding:0 .1em"><a href="/wiki/File:GT_Wikipedia_AE.ogg" title="File:GT Wikipedia AE.ogg">i</a></span></sup></span><span class="IPA nopopups"><a href="/wiki/Help:IPA_for_English" title="Help:IPA for English">/<span style="border-bottom:1px dotted"><span title="/ˌ/ secondary stress follows">ˌ</span><span title="'w' in 'wind'">w</span><span title="/ɪ/ short 'i' in 'bid'">ɪ</span><span title="'k' in 'kind'">k</span><span title="/i/ 'y' in 'happy'">i</span><span title="/ˈ/ primary stress follows">ˈ</span><span title="'p' in 'pie'">p</span><span title="/iː/ long 'e' in 'seed'">iː</span><span title="'d' in 'dye'">d</span><span title="/i/ 'y' in 'happy'">i</span><span title="/ə/ 'a' in 'about'">ə</span></span>/</a></span></span> <span title="English pronunciation respelling"><a href="/wiki/Help:Pronunciation_respelling_key" title="Help:Pronunciation respelling key"><i><b><span class="smallcaps"><span style="font-variant: small-caps; text-transform: lowercase;">WIK</span></span></b>-i-<b><span class="smallcaps"><span style="font-variant: small-caps; text-transform: lowercase;">PEE</span></span></b>-dee-ə</i></a></span>) is a free <a href="/wiki/Online_encyclopedia" title="Online encyclopedia">online encyclopedia</a> that aims to allow anyone to edit articles.<sup class="reference" id="cite_ref-6"><a href="#cite_note-6">[3]</a></sup> Wikipedia is the largest and most popular general <a href="/wiki/Reference_work" title="Reference work">reference work</a> on the Internet<sup class="reference" id="cite_ref-Tancer_7-0"><a href="#cite_note-Tancer-7">[4]</a></sup><sup class="reference" id="cite_ref-Woodson_8-0"><a href="#cite_note-Woodson-8">[5]</a></sup><sup class="reference" id="cite_ref-9"><a href="#cite_note-9">[6]</a></sup> and is ranked among the ten most popular websites.<sup class="reference" id="cite_ref-Alexa_siteinfo_10-0"><a href="#cite_note-Alexa_siteinfo-10">[7]</a></sup> Wikipedia is owned by the nonprofit <a href="/wiki/Wikimedia_Foundation" title="Wikimedia Foundation">Wikimedia Foundation</a>.<sup class="reference" id="cite_ref-11"><a href="#cite_note-11">[8]</a></sup><sup class="reference" id="cite_ref-12"><a href="#cite_note-12">[9]</a></sup><sup class="reference" id="cite_ref-13"><a href="#cite_note-13">[10]</a></sup></p>
This page is missing something! No worries though!
----------------
/wiki/Wikipedia:Protection_policy#semi
Wikipedia:Protection policy
<p><br/></p>
This page is missing something! No worries though!
----------------
/wiki/Wikipedia:Requests_for_page_protection
Wikipedia:Requests for page protection
<p>This page is for requesting that a page, file or template be <b>fully protected</b>, <b>create protected</b> (<a href="/wiki/Wikipedia:Protection_policy#Creation_protection" title="Wikipedia:Protection policy">salted</a>), <b>extended confirmed protected</b>, <b>semi-protected</b>, added to <b>pending changes</b>, <b>move-protected</b>, <b>template protected</b> (template-specific), <b>upload protected</b> (file-specific), or <b>unprotected</b>. Please read up on the <a href="/wiki/Wikipedia:Protection_policy" title="Wikipedia:Protection policy">protection policy</a>. Full protection is used to stop edit warring between multiple users or to prevent vandalism to <a href="/wiki/Wikipedia:High-risk_templates" title="Wikipedia:High-risk templates">high-risk templates</a>, semi-protection and pending changes are usually used only to prevent IP and new user vandalism (see the <a href="/wiki/Wikipedia:Rough_guide_to_semi-protection" title="Wikipedia:Rough guide to semi-protection">rough guide to semi-protection</a>), and move protection is used to stop <a href="/wiki/Wikipedia:Moving_a_page" title="Wikipedia:Moving a page">pagemove</a> revert wars.</p>
/w/index.php?title=Wikipedia:Requests_for_page_protection&action=edit
----------------
/wiki/Wikipedia:Requests_for_permissions
Wikipedia:Requests for permissions
<p><span class="sysop-show" id="coordinates"><a href="/wiki/Wikipedia:Requests_for_permissions/Administrator_instructions" title="Wikipedia:Requests for permissions/Administrator instructions">Administrator instructions</a></span></p>
This page is missing something! No worries though!
----------------
/wiki/Wikipedia:Requesting_copyright_permission
Wikipedia:Requesting copyright permission
<p>To use copyrighted material on Wikipedia, it is <i>not enough</i> that we have permission to use it on Wikipedia alone. That's because Wikipedia itself states all its material may be used by anyone, for any purpose. So we have to be sure all material is in fact licensed for that purpose, whoever provided it.</p>
/w/index.php?title=Wikipedia:Requesting_copyright_permission&action=edit
----------------
/wiki/Wikipedia:User_access_levels
Wikipedia:User access levels
<p>The <b>user access level</b> of an editor affects their ability to perform certain actions on Wikipedia; it depends on which <i>rights</i> (also called <i>permissions</i>, <i><a href="/wiki/Internet_forum#User_groups" title="Internet forum">user groups</a></i>, <a class="mw-redirect" href="/wiki/Bit_(computing)" title="Bit (computing)"><i>bits</i></a> or <a class="mw-redirect" href="/wiki/Flag_(computing)" title="Flag (computing)"><i>flags</i></a>) are assigned to accounts. This is determined by whether the editor is <a class="mw-redirect" href="/wiki/Wikipedia:Logging_in" title="Wikipedia:Logging in">logged into</a> an account, and whether the account has a sufficient age and number of edits for certain automatic rights, and what additional rights have been assigned manually to the account.</p>
This page is missing something! No worries though!
----------------
/wiki/Wikipedia:Requests_for_adminship
Wikipedia:Requests for adminship
<p><input class="mw-inputbox-input searchboxInput mw-ui-input mw-ui-input-inline" dir="ltr" name="search" placeholder="" size="30" type="text" value=""/><input name="prefix" type="hidden" value="Wikipedia:Requests for adminship/"/><br/>
<input class="mw-ui-button" name="fulltext" type="submit" value="Search RfA"/><input name="fulltext" type="hidden" value="Search"/></p>
This page is missing something! No worries though!
----------------
/wiki/Wikipedia:Requested_articles
Wikipedia:Requested articles
<p>First, check that the article you're looking for doesn't already exist:</p>
This page is missing something! No worries though!
----------------
/wiki/Wikipedia:Recent_additions
Wikipedia:Recent additions
<p>This is a selection of recently created <a href="/wiki/Special:NewPages" title="Special:NewPages">new articles</a>, greatly expanded former <a href="/wiki/Wikipedia:Stub" title="Wikipedia:Stub">stub articles</a>, and recently promoted <a class="mw-redirect" href="/wiki/Wikipedia:GA" title="Wikipedia:GA">Good Articles</a> that were featured on the <a href="/wiki/Main_Page" title="Main Page">Main Page</a> as part of <a href="/wiki/Template:Did_you_know" title="Template:Did you know">Did you know?</a> You can <b><a href="/wiki/Template_talk:Did_you_know" title="Template talk:Did you know">submit new pages for consideration</a></b>. (Archives are grouped by month of Main page appearance<a href="/wiki/Template:DYK_archive_header" title="Template:DYK archive header">.</a>)</p>
This page is missing something! No worries though!
----------------
/wiki/Wikipedia:Shortcut
Wikipedia:Shortcut
<p>A <b>shortcut</b> is a specialized type of <a href="/wiki/Wikipedia:Redirect" title="Wikipedia:Redirect">redirect page</a> that provides an abbreviated <a class="mw-redirect" href="/wiki/Wikilink" title="Wikilink">wikilink</a> to a project page or one of its sections, usually from the <b><a href="/wiki/Wikipedia:Project_namespace" title="Wikipedia:Project namespace">Wikipedia namespace</a></b>. They are commonly used on community pages and talk pages, but should not be used in articles themselves. If there is a shortcut for a page or section, it is usually displayed in an information box labelled <i>Shortcuts:</i>, as can be seen at the top of this page.</p>
This page is missing something! No worries though!
----------------
/wiki/Wikipedia:Keyboard_shortcuts
Wikipedia:Keyboard shortcuts
<p>The <a href="/wiki/MediaWiki" title="MediaWiki">MediaWiki</a> software contains many <a href="/wiki/Keyboard_shortcut" title="Keyboard shortcut">keyboard shortcuts</a>. You can use them to access certain features of Wikipedia more quickly.</p>
This page is missing something! No worries though!
----------------
/wiki/Wikipedia:WikiProject_Kansas
Wikipedia:WikiProject Kansas
<p><span style="font-size:100%;font-weight:bold;border: none; margin: 0; padding:0; padding-bottom:.1em; color:#FFD700;"><a class="image" href="/wiki/File:Seal_of_Kansas.svg"><img alt="Seal of Kansas.svg" data-file-height="600" data-file-width="600" height="48" src="//upload.wikimedia.org/wikipedia/commons/thumb/4/45/Seal_of_Kansas.svg/48px-Seal_of_Kansas.svg.png" srcset="//upload.wikimedia.org/wikipedia/commons/thumb/4/45/Seal_of_Kansas.svg/72px-Seal_of_Kansas.svg.png 1.5x, //upload.wikimedia.org/wikipedia/commons/thumb/4/45/Seal_of_Kansas.svg/96px-Seal_of_Kansas.svg.png 2x" width="48"/></a><br/>
<i>Welcome</i></span></p>
/w/index.php?title=Wikipedia:WikiProject_Kansas&action=edit
----------------
/wiki/Wikipedia:Kansas
Wikipedia:WikiProject Kansas
<p><span style="font-size:100%;font-weight:bold;border: none; margin: 0; padding:0; padding-bottom:.1em; color:#FFD700;"><a class="image" href="/wiki/File:Seal_of_Kansas.svg"><img alt="Seal of Kansas.svg" data-file-height="600" data-file-width="600" height="48" src="//upload.wikimedia.org/wikipedia/commons/thumb/4/45/Seal_of_Kansas.svg/48px-Seal_of_Kansas.svg.png" srcset="//upload.wikimedia.org/wikipedia/commons/thumb/4/45/Seal_of_Kansas.svg/72px-Seal_of_Kansas.svg.png 1.5x, //upload.wikimedia.org/wikipedia/commons/thumb/4/45/Seal_of_Kansas.svg/96px-Seal_of_Kansas.svg.png 2x" width="48"/></a><br/>
<i>Welcome</i></span></p>
/w/index.php?title=Wikipedia:WikiProject_Kansas&action=edit
----------------
/wiki/File:Seal_of_Kansas.svg
File:Seal of Kansas.svg
<p>The Great Seal of the State of Kansas</p>
This page is missing something! No worries though!
----------------
/wiki/File:Commons-logo.svg
File:Commons-logo.svg
<p><span class="licensetpl_short">PD</span> <span class="licensetpl_long">Public domain</span> <span class="licensetpl_link_req">false</span> <span class="licensetpl_attr_req">false</span></p>
/w/index.php?title=File:Commons-logo.svg&action=edit
----------------
/wiki/Wikipedia:Criteria_for_speedy_deletion#F8
Wikipedia:Criteria for speedy deletion
<p>The <b>criteria for speedy deletion</b> (<b>CSD</b>) specify the only cases in which <a href="/wiki/Wikipedia:Administrators" title="Wikipedia:Administrators">administrators</a> have broad <a href="/wiki/Wikipedia:Consensus" title="Wikipedia:Consensus">consensus</a> to bypass <a href="/wiki/Wikipedia:Deletion_process" title="Wikipedia:Deletion process">deletion discussion</a>, at their discretion, and immediately delete Wikipedia pages or media. They cover only the cases specified in the rules here.</p>
This page is missing something! No worries though!
----------------
/wiki/Help:Edit_summary
Help:Edit summary
<p>An <b>edit summary</b> is a brief explanation of an <a href="/wiki/Help:Editing" title="Help:Editing">edit</a> to a Wikipedia page. When you edit a page, there is a small text entry field labeled <i>Edit summary</i> located under the main edit box and above the <i>Save page</i> button:</p>
This page is missing something! No worries though!
----------------
/wiki/Wikipedia:Information_pages

我们接下来可以写一个随机找外链的小程序



In [1]:

    
from urllib.request import urlopen
from bs4 import BeautifulSoup
import re
import datetime
import random

pages = set()
random.seed(datetime.datetime.now())

#Retrieves a list of all Internal links found on a page
def getInternalLinks(bsObj, includeUrl):
    internalLinks = []
    #Finds all links that begin with a "/"
    for link in bsObj.findAll("a", href=re.compile("^(/|.*"+includeUrl+")")):
        if link.attrs['href'] is not None:
            if link.attrs['href'] not in internalLinks:
                internalLinks.append(link.attrs['href'])
    return internalLinks
            
#Retrieves a list of all external links found on a page
def getExternalLinks(bsObj, excludeUrl):
    externalLinks = []
    #Finds all links that start with "http" or "www" that do
    #not contain the current URL
    for link in bsObj.findAll("a", href=re.compile("^(http|www)((?!"+excludeUrl+").)*$")):
        if link.attrs['href'] is not None:
            if link.attrs['href'] not in externalLinks:
                externalLinks.append(link.attrs['href'])
    return externalLinks

def splitAddress(address):
    addressParts = address.replace("http://", "").split("/")
    return addressParts

def getRandomExternalLink(startingPage):
    html = urlopen(startingPage)
    bsObj = BeautifulSoup(html, "html.parser")
    externalLinks = getExternalLinks(bsObj, splitAddress(startingPage)[0])
    if len(externalLinks) == 0:
        internalLinks = getInternalLinks(startingPage)
        return getNextExternalLink(internalLinks[random.randint(0, 
                                  len(internalLinks)-1)])
    else:
        return externalLinks[random.randint(0, len(externalLinks)-1)]
    
def followExternalOnly(startingSite):
    externalLink = getRandomExternalLink(startingSite)
    print("Random external link is: "+externalLink)
    #followExternalOnly(externalLink)
            
followExternalOnly("http://oreilly.com")
followExternalOnly("http://oreilly.com")
followExternalOnly("http://oreilly.com")









    



Random external link is: http://twitter.com/oreillymedia
Random external link is: https://www.youtube.com/user/OreillyMedia
Random external link is: http://twitter.com/oreillymedia

如果我们的目标是获取页面上所有的外链，并记录之，我们可以用下面的函数来办



In [2]:

    
allExtLinks = set()
allIntLinks = set()

def getAllExternalLinks(siteUrl):
    html = urlopen(siteUrl)
    bsObj = BeautifulSoup(html.read(), "html.parser")
    internalLinks = getInternalLinks(bsObj, splitAddress(siteUrl)[0])
    externalLinks = getExternalLinks(bsObj, splitAddress(siteUrl)[0])
    for link in externalLinks:
        if link not in allExtLinks:
            allExtLinks.add(link)
            print(link)
    for link in internalLinks:
        if link not in allIntLinks:
            print(link)
            allIntLinks.add(link)
            getAllExternalLinks(link)

getAllExternalLinks("http://oreilly.com")









    



https://www.safaribooksonline.com/static/corp/safari_transformation_whitepaper.pdf
https://www.safaribooksonline.com/static/corp/safari_transformation_whitepaper.pdf#page=7
http://twitter.com/oreillymedia
http://fb.co/OReilly
https://www.linkedin.com/company/o%27reilly-media
https://www.youtube.com/user/OreillyMedia
https://www.oreilly.com
https://www.oreilly.com
http://www.oreilly.com/ideas
http://www.oreilly.com/learning
http://www.oreilly.com/conferences/
http://shop.oreilly.com/
http://members.oreilly.com
https://www.oreilly.com/topics
https://www.safaribooksonline.com/?utm_medium=content&utm_source=oreilly.com&utm_campaign=lgen&utm_content=20170201+homepage+free+trial
https://www.safaribooksonline.com/enterprise/?utm_medium=content&utm_source=oreilly.com&utm_campaign=lgen&utm_content=20170201+homepage+enterprise
https://www.safaribooksonline.com/accounts/login/?utm_medium=content&utm_source=oreilly.com&utm_campaign=lgen&utm_content=20170203+homepage+sign+in
https://www.oreilly.com/people/129b9-ted-malaska?utm_medium=content&utm_source=oreilly.com&utm_campaign=lgen&utm_content=20170224+homepage+ted+malaska
https://www.safaribooksonline.com/search/?query=ted%20malaska&formats=live%20online%20training&highlight=true&extended_publisher_data=true&include_courses=true&field=authors&sort=date_added&utm_medium=content&utm_source=oreilly.com&utm_campaign=lgen&utm_content=20170224+homepage+ted+malaska+training
https://www.safaribooksonline.com/live-training/?utm_medium=content&utm_source=oreilly.com&utm_campaign=lgen&utm_content=20170201+homepage+take+a+live+online+course
https://www.safaribooksonline.com/learning-paths/?utm_medium=content&utm_source=oreilly.com&utm_campaign=lgen&utm_content=20170201+homepage+follow+a+path
https://www.safaribooksonline.com/?utm_medium=content&utm_source=oreilly.com&utm_campaign=lgen&utm_content=20170201+homepage+explore
http://www.oreilly.com/live-training/?view=grid
https://www.safaribooksonline.com/your-experience/?utm_medium=content&utm_source=oreilly.com&utm_campaign=lgen&utm_content=20170201+homepage+safari+platform
https://www.oreilly.com/ideas/8-data-trends-on-our-radar-for-2017?utm_medium=referral&utm_source=oreilly.com&utm_campaign=lgen&utm_content=link+2017+trends
https://www.oreilly.com/ideas?utm_medium=referral&utm_source=oreilly.com&utm_campaign=lgen&utm_content=link+read+latest+articles
http://www.oreilly.com/about/
http://www.oreilly.com/work-with-us.html
http://shop.oreilly.com/category/customer-service.do
http://www.oreilly.com/about/contact.html
http://www.oreilly.com/emails/newsletters/
http://www.oreilly.com/terms/
http://www.oreilly.com/privacy.html
http://www.oreilly.com/about/editorial_independence.html
/topics/ai






    



---------------------------------------------------------------------------
ValueError                                Traceback (most recent call last)
<ipython-input-2-dcf40a7752e9> in <module>()
     17             getAllExternalLinks(link)
     18 
---> 19 getAllExternalLinks("http://oreilly.com")

<ipython-input-2-dcf40a7752e9> in getAllExternalLinks(siteUrl)
     15             print(link)
     16             allIntLinks.add(link)
---> 17             getAllExternalLinks(link)
     18 
     19 getAllExternalLinks("http://oreilly.com")

<ipython-input-2-dcf40a7752e9> in getAllExternalLinks(siteUrl)
     15             print(link)
     16             allIntLinks.add(link)
---> 17             getAllExternalLinks(link)
     18 
     19 getAllExternalLinks("http://oreilly.com")

<ipython-input-2-dcf40a7752e9> in getAllExternalLinks(siteUrl)
      3 
      4 def getAllExternalLinks(siteUrl):
----> 5     html = urlopen(siteUrl)
      6     bsObj = BeautifulSoup(html.read(), "html.parser")
      7     internalLinks = getInternalLinks(bsObj, splitAddress(siteUrl)[0])

/usr/lib/python3.4/urllib/request.py in urlopen(url, data, timeout, cafile, capath, cadefault, context)
    159     else:
    160         opener = _opener
--> 161     return opener.open(url, data, timeout)
    162 
    163 def install_opener(opener):

/usr/lib/python3.4/urllib/request.py in open(self, fullurl, data, timeout)
    446         # accept a URL or a Request object
    447         if isinstance(fullurl, str):
--> 448             req = Request(fullurl, data)
    449         else:
    450             req = fullurl

/usr/lib/python3.4/urllib/request.py in __init__(self, url, data, headers, origin_req_host, unverifiable, method)
    264                  origin_req_host=None, unverifiable=False,
    265                  method=None):
--> 266         self.full_url = url
    267         self.headers = {}
    268         self.unredirected_hdrs = {}

/usr/lib/python3.4/urllib/request.py in full_url(self, url)
    290         self._full_url = unwrap(url)
    291         self._full_url, self.fragment = splittag(self._full_url)
--> 292         self._parse()
    293 
    294     @full_url.deleter

/usr/lib/python3.4/urllib/request.py in _parse(self)
    319         self.type, rest = splittype(self._full_url)
    320         if self.type is None:
--> 321             raise ValueError("unknown url type: %r" % self.full_url)
    322         self.host, self.selector = splithost(rest)
    323         if self.host:

ValueError: unknown url type: '/topics/ai'

使用Scrapy

写网络爬虫的挑战之一就是你要不断重复一些简单的任务：找出页面上所有的链接，区分外链和内链，跳转到新的页面。掌握这些基本模式非常有用，从零开始也是完全可行的，但是这里有一个工具可以帮助你自动处理这些细节。 https://scrapy.org 关于这个框架的使用，请参考其文档，这里不作更多叙述了。

如果以后有时间再继续补充。



In [1]:

    
import scrapy

class BlogSpider(scrapy.Spider):
    name = 'blogspider'
    start_urls = ['https://blog.scrapinghub.com']

    def parse(self, response):
        for title in response.css('h2.entry-title'):
            yield {'title': title.css('a ::text').extract_first()}

        next_page = response.css('div.prev-post > a ::attr(href)').extract_first()
        if next_page:
            yield scrapy.Request(response.urljoin(next_page), callback=self.parse)



In [ ]: