In [1]:

    

# https://realpython.com/modern-web-automation-with-python-and-selenium/

from selenium import webdriver
from selenium.webdriver import Firefox
from selenium.webdriver.firefox.options import Options
opts = Options()
opts.set_headless()
assert opts.headless  # Operating in headless mode
browser = webdriver.Firefox(options=opts)
browser.get('https://duckduckgo.com')

search_form = browser.find_element_by_id('search_form_input_homepage')
search_form.send_keys('real python')
search_form.submit()


    

    




C:\Anaconda3\lib\site-packages\ipykernel_launcher.py:7: DeprecationWarning: use setter for headless property instead of set_headless
  import sys

In [2]:

    

results = browser.find_elements_by_class_name('result__a')


    

In [13]:

    

len(results)


    

    
Out[13]:





11

In [4]:

    

results[0]


    

    
Out[4]:





<selenium.webdriver.firefox.webelement.FirefoxWebElement (session="2ce9f585-0f7a-4c48-b61c-58072d144b91", element="fb842026-2971-42fe-a9a3-ad6b3bc0f22e")>

In [7]:

    

type(results[0])


    

    
Out[7]:





selenium.webdriver.firefox.webelement.FirefoxWebElement

In [14]:

    

results[0].tag_name


    

    
Out[14]:





'a'

In [15]:

    

results[0].get_attribute('innerHTML')


    

    
Out[15]:





'<b>Python</b> at Amazon - Buy Books at Amazon.com and Save'

In [16]:

    

results[0].text


    

    
Out[16]:





'Python at Amazon - Buy Books at Amazon.com and Save'

In [17]:

    

for e in results:
    print(e.text)


    

    




Python at Amazon - Buy Books at Amazon.com and Save
Python Tutorials - Real Python
Learn Python Programming Online - Real Python
Course Overviews - Real Python
Itertools in Python 3, By Example - Real Python
Practical Introduction to Web Scraping in Python - Real Python
Real Python · GitHub
Cool New Features in Python 3.7 - Real Python
11 Beginner Tips for Learning Python Programming - Real Python
Python's range() Function (Guide) - Real Python

In [ ]:

    




    

In [18]:

    

driver = webdriver.Firefox()
driver.get('http://www.w3c.org')
element = driver.find_element_by_name('q')
element.send_keys('hi mom')

element_text = element.text
element_attribute_value = element.get_attribute('value')


    

In [20]:

    

element.tag_name


    

    
Out[20]:





'input'

In [19]:

    

print('element.text: {0}'.format(element_text))
print('element.get_attribute(\'value\'): {0}'.format(element_attribute_value))


    

    




<selenium.webdriver.firefox.webelement.FirefoxWebElement (session="a75c3a98-0cce-4318-b59b-458020d87721", element="b5a2b7d0-0e1f-42fe-a184-66b26394b75a")>
element.text: 
element.get_attribute('value'): hi mom

In [21]:

    

element = driver.find_element_by_css_selector('.description.expand_description > p')
element_text = element.text
element_attribute_value = element.get_attribute('value')

print (element.tag_name)
print ('element.text: {0}'.format(element_text))
print ('element.get_attribute(\'value\'): {0}'.format(element_attribute_value))


    

    




p
element.text: The CSS Working Group has published Selectors Level 3 as a W3C Recommendation. Selectors are patterns that match against elements in a tree, and as such form one of several technologies that can be used to select nodes in an HTML or XML document. Selectors have been optimized for use with HTML and XML, and are designed to be usable in performance-critical code.
element.get_attribute('value'): None

In [22]:

    

everything = driver.execute_script(
    'var element = arguments[0];'
    'var attributes = {};'
    'for (index = 0; index < element.attributes.length; ++index) {'
    '    attributes[element.attributes[index].name] = element.attributes[index].value };'
    'var properties = [];'
    'properties[0] = attributes;'
    'var element_text = element.textContent;'
    'properties[1] = element_text;'
    'var styles = getComputedStyle(element);'
    'var computed_styles = {};'
    'for (index = 0; index < styles.length; ++index) {'
    '    var value_ = styles.getPropertyValue(styles[index]);'
    '    computed_styles[styles[index]] = value_ };'
    'properties[2] = computed_styles;'
    'return properties;', element)


    

In [23]:

    

everything


    

    
Out[23]:





[{},
 'The CSS Working Group has published Selectors Level 3 as a W3C Recommendation. Selectors are patterns that match against elements in a tree, and as such form one of several technologies that can be used to select nodes in an HTML or XML document. Selectors have been optimized for use with HTML and XML, and are designed to be usable in performance-critical code.',
 {'-moz-appearance': 'none',
  '-moz-binding': 'none',
  '-moz-box-align': 'stretch',
  '-moz-box-direction': 'normal',
  '-moz-box-flex': '0',
  '-moz-box-ordinal-group': '1',
  '-moz-box-orient': 'horizontal',
  '-moz-box-pack': 'start',
  '-moz-float-edge': 'content-box',
  '-moz-force-broken-image-icon': '0',
  '-moz-image-region': 'auto',
  '-moz-orient': 'inline',
  '-moz-outline-radius-bottomleft': '0px',
  '-moz-outline-radius-bottomright': '0px',
  '-moz-outline-radius-topleft': '0px',
  '-moz-outline-radius-topright': '0px',
  '-moz-stack-sizing': 'stretch-to-fit',
  '-moz-tab-size': '8',
  '-moz-text-size-adjust': 'auto',
  '-moz-user-focus': 'none',
  '-moz-user-input': 'auto',
  '-moz-user-modify': 'read-only',
  '-moz-user-select': 'auto',
  '-moz-window-dragging': 'default',
  '-webkit-text-fill-color': 'rgb(51, 51, 51)',
  '-webkit-text-stroke-color': 'rgb(51, 51, 51)',
  '-webkit-text-stroke-width': '0px',
  'align-content': 'normal',
  'align-items': 'normal',
  'align-self': 'auto',
  'animation-delay': '0s',
  'animation-direction': 'normal',
  'animation-duration': '0s',
  'animation-fill-mode': 'none',
  'animation-iteration-count': '1',
  'animation-name': 'none',
  'animation-play-state': 'running',
  'animation-timing-function': 'ease',
  'backface-visibility': 'visible',
  'background-attachment': 'scroll',
  'background-blend-mode': 'normal',
  'background-clip': 'border-box',
  'background-color': 'rgba(0, 0, 0, 0)',
  'background-image': 'none',
  'background-origin': 'padding-box',
  'background-position': '0% 0%',
  'background-position-x': '0%',
  'background-position-y': '0%',
  'background-repeat': 'repeat',
  'background-size': 'auto auto',
  'block-size': '106.25px',
  'border-block-end-color': 'rgb(51, 51, 51)',
  'border-block-end-style': 'none',
  'border-block-end-width': '0px',
  'border-block-start-color': 'rgb(51, 51, 51)',
  'border-block-start-style': 'none',
  'border-block-start-width': '0px',
  'border-bottom-color': 'rgb(51, 51, 51)',
  'border-bottom-left-radius': '0px',
  'border-bottom-right-radius': '0px',
  'border-bottom-style': 'none',
  'border-bottom-width': '0px',
  'border-collapse': 'separate',
  'border-image-outset': '0',
  'border-image-repeat': 'stretch stretch',
  'border-image-slice': '100%',
  'border-image-source': 'none',
  'border-image-width': '1',
  'border-inline-end-color': 'rgb(51, 51, 51)',
  'border-inline-end-style': 'none',
  'border-inline-end-width': '0px',
  'border-inline-start-color': 'rgb(51, 51, 51)',
  'border-inline-start-style': 'none',
  'border-inline-start-width': '0px',
  'border-left-color': 'rgb(51, 51, 51)',
  'border-left-style': 'none',
  'border-left-width': '0px',
  'border-right-color': 'rgb(51, 51, 51)',
  'border-right-style': 'none',
  'border-right-width': '0px',
  'border-spacing': '0px 0px',
  'border-top-color': 'rgb(51, 51, 51)',
  'border-top-left-radius': '0px',
  'border-top-right-radius': '0px',
  'border-top-style': 'none',
  'border-top-width': '0px',
  'bottom': 'auto',
  'box-decoration-break': 'slice',
  'box-shadow': 'none',
  'box-sizing': 'content-box',
  'caption-side': 'top',
  'caret-color': 'rgb(51, 51, 51)',
  'clear': 'none',
  'clip': 'auto',
  'clip-path': 'none',
  'clip-rule': 'nonzero',
  'color': 'rgb(51, 51, 51)',
  'color-adjust': 'economy',
  'color-interpolation': 'srgb',
  'color-interpolation-filters': 'linearrgb',
  'column-count': 'auto',
  'column-fill': 'balance',
  'column-gap': 'normal',
  'column-rule-color': 'rgb(51, 51, 51)',
  'column-rule-style': 'none',
  'column-rule-width': '0px',
  'column-width': 'auto',
  'content': 'none',
  'counter-increment': 'none',
  'counter-reset': 'none',
  'cursor': 'auto',
  'direction': 'ltr',
  'display': 'block',
  'dominant-baseline': 'auto',
  'empty-cells': 'show',
  'fill': 'rgb(0, 0, 0)',
  'fill-opacity': '1',
  'fill-rule': 'nonzero',
  'filter': 'none',
  'flex-basis': 'auto',
  'flex-direction': 'row',
  'flex-grow': '0',
  'flex-shrink': '1',
  'flex-wrap': 'nowrap',
  'float': 'none',
  'flood-color': 'rgb(0, 0, 0)',
  'flood-opacity': '1',
  'font-family': '"\'Helvetica Neue\'", Helvetica, Arial, Verdana, Geneva, sans-serif',
  'font-feature-settings': 'normal',
  'font-kerning': 'auto',
  'font-language-override': 'normal',
  'font-optical-sizing': 'auto',
  'font-size': '14.1667px',
  'font-size-adjust': 'none',
  'font-stretch': '100%',
  'font-style': 'normal',
  'font-synthesis': 'weight style',
  'font-variant': 'normal',
  'font-variant-alternates': 'normal',
  'font-variant-caps': 'normal',
  'font-variant-east-asian': 'normal',
  'font-variant-ligatures': 'normal',
  'font-variant-numeric': 'normal',
  'font-variant-position': 'normal',
  'font-variation-settings': 'normal',
  'font-weight': '400',
  'grid-auto-columns': 'auto',
  'grid-auto-flow': 'row',
  'grid-auto-rows': 'auto',
  'grid-column-end': 'auto',
  'grid-column-start': 'auto',
  'grid-row-end': 'auto',
  'grid-row-start': 'auto',
  'grid-template-areas': 'none',
  'grid-template-columns': 'none',
  'grid-template-rows': 'none',
  'height': '106.25px',
  'hyphens': 'manual',
  'image-orientation': 'none',
  'image-rendering': 'auto',
  'ime-mode': 'auto',
  'inline-size': '595.2px',
  'inset-block-end': 'auto',
  'inset-block-start': 'auto',
  'inset-inline-end': 'auto',
  'inset-inline-start': 'auto',
  'isolation': 'auto',
  'justify-content': 'normal',
  'justify-items': 'normal',
  'justify-self': 'auto',
  'left': 'auto',
  'letter-spacing': 'normal',
  'lighting-color': 'rgb(255, 255, 255)',
  'line-height': '21.25px',
  'list-style-image': 'none',
  'list-style-position': 'outside',
  'list-style-type': 'none',
  'margin-block-end': '0px',
  'margin-block-start': '0px',
  'margin-bottom': '0px',
  'margin-inline-end': '0px',
  'margin-inline-start': '0px',
  'margin-left': '0px',
  'margin-right': '0px',
  'margin-top': '0px',
  'marker-end': 'none',
  'marker-mid': 'none',
  'marker-start': 'none',
  'mask': 'none',
  'mask-clip': 'border-box',
  'mask-composite': 'add',
  'mask-image': 'none',
  'mask-mode': 'match-source',
  'mask-origin': 'border-box',
  'mask-position': '0% 0%',
  'mask-position-x': '0%',
  'mask-position-y': '0%',
  'mask-repeat': 'repeat',
  'mask-size': 'auto auto',
  'mask-type': 'luminance',
  'max-block-size': 'none',
  'max-height': 'none',
  'max-inline-size': 'none',
  'max-width': 'none',
  'min-block-size': '0px',
  'min-height': '0px',
  'min-inline-size': '0px',
  'min-width': '0px',
  'mix-blend-mode': 'normal',
  'object-fit': 'fill',
  'object-position': '50% 50%',
  'opacity': '1',
  'order': '0',
  'outline-color': 'rgb(51, 51, 51)',
  'outline-offset': '0px',
  'outline-style': 'none',
  'outline-width': '0px',
  'overflow': 'visible',
  'overflow-wrap': 'normal',
  'overflow-x': 'visible',
  'overflow-y': 'visible',
  'overscroll-behavior-x': 'auto',
  'overscroll-behavior-y': 'auto',
  'padding-block-end': '0px',
  'padding-block-start': '20px',
  'padding-bottom': '0px',
  'padding-inline-end': '20px',
  'padding-inline-start': '20px',
  'padding-left': '20px',
  'padding-right': '20px',
  'padding-top': '20px',
  'page-break-after': 'auto',
  'page-break-before': 'auto',
  'page-break-inside': 'auto',
  'paint-order': 'normal',
  'perspective': 'none',
  'perspective-origin': '317.6px 63.1333px',
  'pointer-events': 'auto',
  'position': 'static',
  'quotes': '"“" "”" "‘" "’"',
  'resize': 'none',
  'right': 'auto',
  'row-gap': 'normal',
  'ruby-align': 'space-around',
  'ruby-position': 'over',
  'scroll-behavior': 'auto',
  'scroll-snap-coordinate': 'none',
  'scroll-snap-destination': '0px 0px',
  'scroll-snap-points-x': 'none',
  'scroll-snap-points-y': 'none',
  'scroll-snap-type-x': 'none',
  'scroll-snap-type-y': 'none',
  'shape-image-threshold': '0',
  'shape-margin': '0px',
  'shape-outside': 'none',
  'shape-rendering': 'auto',
  'stop-color': 'rgb(0, 0, 0)',
  'stop-opacity': '1',
  'stroke': 'none',
  'stroke-dasharray': 'none',
  'stroke-dashoffset': '0px',
  'stroke-linecap': 'butt',
  'stroke-linejoin': 'miter',
  'stroke-miterlimit': '4',
  'stroke-opacity': '1',
  'stroke-width': '1px',
  'table-layout': 'auto',
  'text-align': 'start',
  'text-align-last': 'auto',
  'text-anchor': 'start',
  'text-combine-upright': 'none',
  'text-decoration': 'none',
  'text-decoration-color': 'rgb(51, 51, 51)',
  'text-decoration-line': 'none',
  'text-decoration-style': 'solid',
  'text-emphasis-color': 'rgb(51, 51, 51)',
  'text-emphasis-position': 'over right',
  'text-emphasis-style': 'none',
  'text-indent': '0px',
  'text-justify': 'auto',
  'text-orientation': 'mixed',
  'text-overflow': 'clip',
  'text-rendering': 'auto',
  'text-shadow': 'none',
  'text-transform': 'none',
  'top': 'auto',
  'touch-action': 'auto',
  'transform': 'none',
  'transform-box': 'border-box',
  'transform-origin': '317.6px 63.1333px',
  'transform-style': 'flat',
  'transition-delay': '0s',
  'transition-duration': '0s',
  'transition-property': 'all',
  'transition-timing-function': 'ease',
  'unicode-bidi': 'isolate',
  'vector-effect': 'none',
  'vertical-align': 'baseline',
  'visibility': 'visible',
  'white-space': 'normal',
  'width': '595.2px',
  'will-change': 'auto',
  'word-break': 'normal',
  'word-spacing': '0px',
  'writing-mode': 'horizontal-tb',
  'z-index': 'auto'}]

In [ ]:

    

driver.quit()