In [1]:
import pyfiglet
phrase = 'Oauth2'
print "x = \"\"\""
print pyfiglet.figlet_format(phrase, font='standard')
print " \"\"\""
In [ ]:
#Download from list of #urls
# import urllib
# import urllib2
# import requests
# url = 'http://www.blog.pythonlibrary.org/wp-content/uploads/2012/06/wxDbViewer.zip'
# print "downloading with urllib"
# urllib.urlretrieve(url, "code.zip")
# print "downloading with urllib2"
# f = urllib2.urlopen(url)
# data = f.read()
# with open("code2.zip", "wb") as code:
# code.write(data)
# print "downloading with requests"
# r = requests.get(url)
# with open("code3.zip", "wb") as code:
# code.write(r.content)
###
import urllib
urllist = ['http://www.oshannonland.com/wp-content/uploads/2012/06/20-Spider-Man-20.mp3',
'http://www.oshannonland.com/wp-content/uploads/2012/06/21-Spider-Man-21.mp3',
'http://www.oshannonland.com/wp-content/uploads/2012/06/22-Spider-Man-22.mp3',
'http://www.oshannonland.com/wp-content/uploads/2012/06/23-Spider-Man-231.mp3',
'http://www.oshannonland.com/wp-content/uploads/2012/06/24-Spider-Man-24.mp3',
'http://www.oshannonland.com/wp-content/uploads/2012/06/25-Spider-Man-25.mp3',
'http://www.oshannonland.com/wp-content/uploads/2012/06/26-Spider-Man-26.mp3',
'http://www.oshannonland.com/wp-content/uploads/2012/06/spider-man-27.mp3',
'http://www.oshannonland.com/wp-content/uploads/2012/06/spider-man-28.mp3',
'http://www.oshannonland.com/wp-content/uploads/2012/06/spider-man-29.mp3',
'http://www.oshannonland.com/wp-content/uploads/2012/06/spider-man-30.mp3',
'http://www.oshannonland.com/wp-content/uploads/2012/06/spider-man-31.mp3',
'http://www.oshannonland.com/wp-content/uploads/2012/06/spider-man-32.mp3',
'http://www.oshannonland.com/wp-content/uploads/2012/06/spider-man-33.mp3']
for url in urllist:
urllib.urlretrieve(url, url.rsplit('/', 1)[-1])
print url.rsplit('/', 1)[-1] + ' downloaded successfully'
###
# from text file:
download_full_path = 'c:/eraseme/'
urlfile_full_path = 'c:/eraseme/urls.txt'
overwrite = False
import os
import urllib
os.chdir(download_full_path)
# For every line in the file
errors=[]
for url in open(urlfile_full_path):
try:# Split on the rightmost / and take everything on the right side of that
name = url.rsplit('/', 1)[-1]
# Combine the name and the downloads directory to get the local filename
filename = os.path.join(download_full_path, name)
# Download the file if it does not exist
if not os.path.isfile(filename) or overwrite == True:
urllib.urlretrieve(url, filename)
print filename
except:
errors.append(url)
print "Errors:",
for url in errors:
print url,
In [ ]:
x = """
_ _ _ _ ___ _ __ _ _ __ _ _
| |__ _ _| | | __ __| | / / | _ _ _ __| |___ / _|_ __ ___ _ __ ___ | |___ _| |_ / _(_) | ___
| '_ \| | | | | |/ / / _` | / /| | | | | | '__| / __| | |_| '__/ _ \| '_ ` _ \ | __\ \/ / __| | |_| | |/ _ \
| |_) | |_| | | < | (_| |/ / | | | |_| | | | \__ \ | _| | | (_) | | | | | | | |_ > <| |_ | _| | | __/
|_.__/ \__,_|_|_|\_\ \__,_/_/ |_| \__,_|_| |_|___/ |_| |_| \___/|_| |_| |_| \__/_/\_\\__| |_| |_|_|\___|
"""
download_full_path = 'c:/eraseme/'
urlfile_full_path = 'c:/eraseme/urls.txt'
overwrite = False
import os
import urllib
os.chdir(download_full_path)
# For every line in the file
errors=[]
for url in open(urlfile_full_path):
try:# Split on the rightmost / and take everything on the right side of that
name = url.rsplit('/', 1)[-1]
# Combine the name and the downloads directory to get the local filename
filename = os.path.join(download_full_path, name)
# Download the file if it does not exist
if not os.path.isfile(filename) or overwrite == True:
urllib.urlretrieve(url, filename)
print filename
except:
errors.append(url)
print "Errors:",
for url in errors:
print url,
In [ ]:
x = """
___ _ _ ____
/ _ \ __ _ _ _| |_| |__ |___ \
| | | |/ _` | | | | __| '_ \ __) |
| |_| | (_| | |_| | |_| | | |/ __/
\___/ \__,_|\__,_|\__|_| |_|_____|
"""
#oauth2
import urlparse
import oauth2 as oauth
consumer_key = 'azkeVAunE4IK2ChXoGifruSQ4YpxLwafX1dLZrLAkzLJrc2IcE'
consumer_secret = 'DhQLUrawaPosoCaMmrSTMyvBfQjaiSUQGV1mJ9vW9yvmu2GURB'
request_token_url = 'http://www.tumblr.com/oauth/request_token'
access_token_url = 'http://www.tumblr.com/oauth/access_token'
authorize_url = 'http://www.tumblr.com/oauth/authorize'
consumer = oauth.Consumer(consumer_key, consumer_secret)
client = oauth.Client(consumer)
# Step 1: Get a request token. This is a temporary token that is used for
# having the user authorize an access token and to sign the request to obtain
# said access token.
resp, content = client.request(request_token_url, "GET")
if resp['status'] != '200':
raise Exception("Invalid response %s." % resp['status'])
request_token = dict(urlparse.parse_qsl(content))
print "Request Token:"
print " - oauth_token = %s" % request_token['oauth_token']
print " - oauth_token_secret = %s" % request_token['oauth_token_secret']
print
# Step 2: Redirect to the provider. Since this is a CLI script we do not
# redirect. In a web application you would redirect the user to the URL
# below.
print "Go to the following link in your browser:"
print "%s?oauth_token=%s" % (authorize_url, request_token['oauth_token'])
print
# After the user has granted access to you, the consumer, the provider will
# redirect you to whatever URL you have told them to redirect to. You can
# usually define this in the oauth_callback argument as well.
accepted = 'n'
while accepted.lower() == 'n':
accepted = raw_input('Have you authorized me? (y/n) ')
oauth_verifier = raw_input('What is the PIN? ')
# Step 3: Once the consumer has redirected the user back to the oauth_callback
# URL you can request the access token the user has approved. You use the
# request token to sign this request. After this is done you throw away the
# request token and use the access token returned. You should store this
# access token somewhere safe, like a database, for future use.
token = oauth.Token(request_token['oauth_token'],
request_token['oauth_token_secret'])
token.set_verifier(oauth_verifier)
client = oauth.Client(consumer, token)
resp, content = client.request(access_token_url, "POST")
access_token = dict(urlparse.parse_qsl(content))
print "Access Token:"
print " - oauth_token = %s" % access_token['oauth_token']
print " - oauth_token_secret = %s" % access_token['oauth_token_secret']
print
print "You may now access protected resources using the access tokens above."
print
In [ ]:
x = """
____ _____ _____ ____
/ ___|| ___|_ _| _ \
\___ \| |_ | | | |_) |
___) | _| | | | __/
|____/|_| |_| |_|
"""
#SFTP transfer
# Transfer local to #ftp
import pysftp
sftpconn = pysftp.Connection('ssh.REDACTED.com',
username='REDACTED', password='REDACTED$)
sftpconn.put('C:/Users/David/Documents/Images_API/sked/sched.json', 'sked/sched.json')
# # Transfer ftp to local
import pysftp
sftpconn = pysftp.Connection('ssh.REDACTED.com',
username='REDACTED', password='REDACTED')
sftpconn.get('sked/sched.json', 'C:/Users/David/Documents/Images_API/sked/sched.json')
In [ ]:
#BeautifulSoup
from bs4 import BeautifulSoup
soup = BeautifulSoup(html_string) #or soup = BeautifulSoup(open("index.html"))
#to find first such tag:
tag = soup.find('p')
#attributes:
tag.name
tag['class'] # an attr
tag.contents
tag.attrs # returns a dict
#to find all tags (returns list)
tags = soup.findAll('p')
#to find <select name="HSpeaker" multiple size="4">
hspeaker = soup.find('select',{'name':'HSpeaker'})
#to create list all option tags under the above:
options = hspeaker.findAll('option')
print soup.find(id="link3")
#extract all urls in <a href=""> tags
for link in soup.find_all('a'):
print(link.get('href'))
#extract all text
print(soup.get_text())
# more:
# http://omz-software.com/pythonista/docs/ios/beautifulsoup_guide.html
In [ ]:
# mechanize
import mechanize
br = mechanize.Browser()
br.set_handle_robots(False) # ignore robots # no cookies
br.set_handle_equiv(True)
br.set_handle_redirect(True)
br.set_handle_referer(True)
# Follows refresh 0 but not hangs on refresh > 0:
br.set_handle_refresh(mechanize._http.HTTPRefreshProcessor(), max_time=1)
user_agents = ['Mozilla/5.0 (Windows; U; Windows NT 5.1; en-GB; rv:1.8.1.6) Gecko/20070725 Firefox/2.0.0.6',
'Mozilla/5.0 (X11; U; Linux i686; en-US) AppleWebKit/534.3 (KHTML, like Gecko) Chrome/6.0.472.63 Safari/534.3',
'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1)',
'Opera/9.00 (Windows NT 5.1; U; en)']
agent = 2
br.addheaders = [('User-agent', user_agents[agent])]
# Want debugging messages?
#br.set_debug_http(True)
#br.set_debug_redirects(True)
#br.set_debug_responses(True)
#alternate to use cookies
# import cookielib
# cj = cookielib.LWPCookieJar()
# br.set_cookiejar(cj)
#########################################
response = br.open('http://www.example.com') # can only be used once
soup = BeautifulSoup(response.read())
assert response.code == 200 # headers
#forms
for form in br.forms():
print form
br.select_form(nr=0) #first form
br.select_form("form1") # works when form has a name
br.form = list(br.forms())[0] # use when form is unnamed
# submit form
# eg if form control is <CheckboxControl(SenateSection=[*1])>
br.form['SenateSection']=1
br.submit()
print br.response().read()
#list all controls
for control in br.form.controls:
print control
print "type=%s, name=%s value=%s" % (control.type, control.name, br[control.name])
#find by name
control = br.form.find_control("controlname")
#allowed values
if control.type == "select": # means it's class ClientForm.SelectControl
for item in control.items:
print " name=%s values=%s" % (item.name, str([label.text for label in item.get_labels()]))
#select-type controls must be set with a list, even if only one item:
print control.value
print control # selected value is starred
control.value = ["ItemName"]
print control
br[control.name] = ["ItemName"] # equivalent and more normal
#text controls can be set as string
if control.type == "text": # means it's class ClientForm.TextControl
control.value = "stuff here"
br["controlname"] = "stuff here" # equivalent
# Looking at some results in link format
for link in br.links(url_regex='stockrt'):
print link
#or
print link.text, link.url
#links
# Testing presence of link (if the link is not found you would have to
# handle a LinkNotFoundError exception)
br.find_link(text='Weekend codes')
# Actually clicking the link
req = br.click_link(text='Weekend codes')
br.open(req)
print br.response().read()
print br.geturl()
# Back
br.back()
print br.response().read()
print br.geturl()
#
br.follow_link(text='Sign out')
#or
resp = br.follow_link(...)
# follow lots of links
all_links = [l for l in br.links(url_regex='\?v=c&th=')] # or text_regex
# Select the first 3
for link in all_links[0:3]:
print link
# Open each message
br.follow_link(msg_link)
#
request = br.click_link(link)
response = br.follow_link(link)
print response.geturl()
print response.get_data()
#
for link in br.links():
print link.text, link.url
# Download
f = br.retrieve('http://www.google.com.br/intl/pt-BR_br/images/logo.gif')[0]
# Proxy and user/password
br.set_proxies({"http": "joe:password@myproxy.example.com:3128"})
# Proxy
br.set_proxies({"http": "myproxy.example.com:3128"})
# Proxy password
br.add_proxy_password("joe", "password")
# more, incl password protected auth
# http://stockrt.github.io/p/emulating-a-browser-in-python-with-mechanize/
#pydoc
# http://joesourcecode.com/Documentation/mechanize0.2.5/