In [1]:

    

import selenium
from selenium import webdriver
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.support.ui import Select
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.by import By


    

In [20]:

    

def data_download_compendex(username,password,keywords,no_of_pages):
    """
    Function to download results from Compendex
    """
    driver = webdriver.Chrome()
    driver.get("https://apps.webofknowledge.com/WOS_GeneralSearch_input.do;jsessionid=0649C3B3705A1155F3BFFAD0CF93CB63?product=WOS&search_mode=GeneralSearch&SID=3CgenZXtDdie9nA2KFd&preferencesSaved=")
    if (driver.title == "UW NetID Weblogin"):
        username_textbox = driver.find_element_by_id('weblogin_netid')
        username_textbox.send_keys(username)
        password_textbox = driver.find_element_by_id('weblogin_password')
        password_textbox.send_keys(password)
        submit_button = driver.find_element_by_name('submit').click()
    driver.find_element_by_id("value(input1)").send_keys(keywords[0])
    driver.find_element_by_id("addSearchRow1").click()
    print "added another field"
    for i in range(len(keywords)-1):
        if i<2:
            dropdown1 = driver.find_element_by_id("bool_default")
            Select(dropdown1).select_by_visible_text("OR")
            driver.find_element_by_id("srchWrd%d"%(i+2)).send_keys(keywords[i+1])
        else:
            dropdown1 = driver.find_element_by_id('connector')
            Select(dropdown1).select_by_visible_text("OR")
            driver.find_element_by_name("searchWords").send_keys(keywords[i+1])
        driver.find_element_by_id("addSearchRow1").click()
    driver.find_element_by_id("submitsearch_tool").click()
    WebDriverWait(driver,5).until(EC.presence_of_element_located((By.ID, "pageSizeVal_top")))
    dropdown1=driver.find_element_by_id("pageSizeVal_top")
    Select(dropdown1).select_by_visible_text("100")
    WebDriverWait(driver,5).until(EC.presence_of_element_located((By.ID, "pageckbx")))
    driver.find_element_by_id("pageckbx").click()
    WebDriverWait(driver,5).until(EC.presence_of_element_located((By.ID, "downloadli")))
    WebDriverWait(driver, 30).until(EC.visibility_of_element_located((By.ID,'downloadli')));
    driver.find_element_by_id("downloadli").click()
    WebDriverWait(driver,5).until(EC.presence_of_element_located((By.ID, "rdCsv")))
    driver.find_element_by_id("rdCsv").click()
    driver.find_element_by_id("rdDet").click()
    driver.find_element_by_id("savePrefsButton").click()
    for i in range(no_of_pages-1):
        driver.find_element_by_class_name("blackpipeleft").click()
        WebDriverWait(driver,5).until(EC.presence_of_element_located((By.ID, "pageckbx")))
        driver.find_element_by_id("pageckbx").click()
        WebDriverWait(driver,5).until(EC.presence_of_element_located((By.ID, "downloadli")))
        WebDriverWait(driver, 30).until(EC.visibility_of_element_located((By.ID,'downloadli')));
        driver.find_element_by_id("downloadli").click()
    return


    

In [21]:

    

username = ''
password = ''
keywords = ['nuclear','atomic','uranium','centrifuge']
no_of_pages = 5
data_download_compendex(username,password,keywords,5)


    

    




added another field






    




---------------------------------------------------------------------------
ElementNotVisibleException                Traceback (most recent call last)
<ipython-input-21-c153645da8dd> in <module>()
      3 keywords = ['nuclear','atomic','uranium','centrifuge']
      4 no_of_pages = 5
----> 5 data_download_compendex(username,password,keywords,5)

<ipython-input-20-5375d13d8783> in data_download_compendex(username, password, keywords, no_of_pages)
     17         if i<2:
     18             dropdown1 = driver.find_element_by_id("bool_default")
---> 19             Select(dropdown1).select_by_visible_text("OR")
     20             driver.find_element_by_id("srchWrd%d"%(i+2)).send_keys(keywords[i+1])
     21         else:

/Users/peacenick36/anaconda/lib/python2.7/site-packages/selenium/webdriver/support/select.pyc in select_by_visible_text(self, text)
    118         matched = False
    119         for opt in opts:
--> 120             self._setSelected(opt)
    121             if not self.is_multiple:
    122                 return

/Users/peacenick36/anaconda/lib/python2.7/site-packages/selenium/webdriver/support/select.pyc in _setSelected(self, option)
    210     def _setSelected(self, option):
    211         if not option.is_selected():
--> 212             option.click()
    213 
    214     def _unsetSelected(self, option):

/Users/peacenick36/anaconda/lib/python2.7/site-packages/selenium/webdriver/remote/webelement.pyc in click(self)
     75     def click(self):
     76         """Clicks the element."""
---> 77         self._execute(Command.CLICK_ELEMENT)
     78 
     79     def submit(self):

/Users/peacenick36/anaconda/lib/python2.7/site-packages/selenium/webdriver/remote/webelement.pyc in _execute(self, command, params)
    491             params = {}
    492         params['id'] = self._id
--> 493         return self._parent.execute(command, params)
    494 
    495     def find_element(self, by=By.ID, value=None):

/Users/peacenick36/anaconda/lib/python2.7/site-packages/selenium/webdriver/remote/webdriver.pyc in execute(self, driver_command, params)
    247         response = self.command_executor.execute(driver_command, params)
    248         if response:
--> 249             self.error_handler.check_response(response)
    250             response['value'] = self._unwrap_value(
    251                 response.get('value', None))

/Users/peacenick36/anaconda/lib/python2.7/site-packages/selenium/webdriver/remote/errorhandler.pyc in check_response(self, response)
    191         elif exception_class == UnexpectedAlertPresentException and 'alert' in value:
    192             raise exception_class(message, screen, stacktrace, value['alert'].get('text'))
--> 193         raise exception_class(message, screen, stacktrace)
    194 
    195     def _value_or_default(self, obj, key, default):

ElementNotVisibleException: Message: element not visible: Element is not currently visible and may not be manipulated
  (Session info: chrome=57.0.2987.133)
  (Driver info: chromedriver=2.29.461585 (0be2cd95f834e9ee7c46bcc7cf405b483f5ae83b),platform=Mac OS X 10.12.4 x86_64)

In [ ]: