In [1]:
#url is a download address
#python requests_download_via_socket5.py url

In [8]:
import sys
import requests
import re
import os
import time

In [2]:
#url = str(sys.argv[1])
#url='http://www.chnaus.com/forum.php?mod=viewthread&tid=198569
fp=open('south_download.list','rb')

In [3]:
headers={'Content-Type':'application','User-Agent':'Mozilla/5.0 (Windows; U; Windows NT 5.1; de; rv:1.9.2.3) Gecko/20100401 Firefox/3.6.3'}

In [4]:
proxies = {'http': "socks5://127.0.0.1:9050"}

In [6]:
def download_file(url,headers,proxies):
    req=requests.get(url=url,headers=headers,proxies=proxies)
    content=req.content
    pattern=re.compile(r'\d{6}')
    result=pattern.search(url)
    if result:
        fn=result.group()
        with open(fn,'wb') as fp:
            fp.write(content)
            print(url+' was fininsh!\n')
            os.system('sed -i 1d south_download.list')

In [7]:
counter =0
for link in fp.readlines():
    counter = counter + 1
    download_file(link.decode('utf-8'),headers=headers,proxies=proxies)
    if counter == 20:
        os.system('service tor restart') 
        time.sleep(40)
        counter =0


http://www.chnaus.com/forum.php?mod=viewthread&tid=147698
 was fininsh!

http://www.chnaus.com/forum.php?mod=viewthread&tid=157801
 was fininsh!

http://www.chnaus.com/forum.php?mod=viewthread&tid=177413
 was fininsh!

http://www.chnaus.com/forum.php?mod=viewthread&tid=236557
 was fininsh!

http://www.chnaus.com/forum.php?mod=viewthread&tid=164732
 was fininsh!

http://www.chnaus.com/forum.php?mod=viewthread&tid=150656
 was fininsh!

http://www.chnaus.com/forum.php?mod=viewthread&tid=224164
 was fininsh!

http://www.chnaus.com/forum.php?mod=viewthread&tid=177793
 was fininsh!

http://www.chnaus.com/forum.php?mod=viewthread&tid=257404
 was fininsh!

http://www.chnaus.com/forum.php?mod=viewthread&tid=342582
 was fininsh!

http://www.chnaus.com/forum.php?mod=viewthread&tid=167681
 was fininsh!

http://www.chnaus.com/forum.php?mod=viewthread&tid=171536
 was fininsh!

http://www.chnaus.com/forum.php?mod=viewthread&tid=172496
 was fininsh!

http://www.chnaus.com/forum.php?mod=viewthread&tid=144301
 was fininsh!

---------------------------------------------------------------------------
TypeError                                 Traceback (most recent call last)
/home/jun_gentoo/anaconda3/lib/python3.5/site-packages/requests/packages/urllib3/connectionpool.py in _make_request(self, conn, method, url, timeout, chunked, **httplib_request_kw)
    383             try:  # Python 2.7, use buffering of HTTP responses
--> 384                 httplib_response = conn.getresponse(buffering=True)
    385             except TypeError:  # Python 2.6 and older, Python 3

TypeError: getresponse() got an unexpected keyword argument 'buffering'

During handling of the above exception, another exception occurred:

KeyboardInterrupt                         Traceback (most recent call last)
<ipython-input-7-f426256c6040> in <module>()
      1 for link in fp.readlines():
----> 2     download_file(link.decode('utf-8'),headers=headers,proxies=proxies)

<ipython-input-6-42e4bec348ca> in download_file(url, headers, proxies)
      1 def download_file(url,headers,proxies):
----> 2     req=requests.get(url=url,headers=headers,proxies=proxies)
      3     content=req.content
      4     pattern=re.compile(r'\d{6}')
      5     result=pattern.search(url)

/home/jun_gentoo/anaconda3/lib/python3.5/site-packages/requests/api.py in get(url, params, **kwargs)
     68 
     69     kwargs.setdefault('allow_redirects', True)
---> 70     return request('get', url, params=params, **kwargs)
     71 
     72 

/home/jun_gentoo/anaconda3/lib/python3.5/site-packages/requests/api.py in request(method, url, **kwargs)
     54     # cases, and look like a memory leak in others.
     55     with sessions.Session() as session:
---> 56         return session.request(method=method, url=url, **kwargs)
     57 
     58 

/home/jun_gentoo/anaconda3/lib/python3.5/site-packages/requests/sessions.py in request(self, method, url, params, data, headers, cookies, files, auth, timeout, allow_redirects, proxies, hooks, stream, verify, cert, json)
    486         }
    487         send_kwargs.update(settings)
--> 488         resp = self.send(prep, **send_kwargs)
    489 
    490         return resp

/home/jun_gentoo/anaconda3/lib/python3.5/site-packages/requests/sessions.py in send(self, request, **kwargs)
    607 
    608         # Send the request
--> 609         r = adapter.send(request, **kwargs)
    610 
    611         # Total elapsed time of the request (approximately)

/home/jun_gentoo/anaconda3/lib/python3.5/site-packages/requests/adapters.py in send(self, request, stream, timeout, verify, cert, proxies)
    421                     decode_content=False,
    422                     retries=self.max_retries,
--> 423                     timeout=timeout
    424                 )
    425 

/home/jun_gentoo/anaconda3/lib/python3.5/site-packages/requests/packages/urllib3/connectionpool.py in urlopen(self, method, url, body, headers, retries, redirect, assert_same_host, timeout, pool_timeout, release_conn, chunked, **response_kw)
    592                                                   timeout=timeout_obj,
    593                                                   body=body, headers=headers,
--> 594                                                   chunked=chunked)
    595 
    596             # If we're going to release the connection in ``finally:``, then

/home/jun_gentoo/anaconda3/lib/python3.5/site-packages/requests/packages/urllib3/connectionpool.py in _make_request(self, conn, method, url, timeout, chunked, **httplib_request_kw)
    385             except TypeError:  # Python 2.6 and older, Python 3
    386                 try:
--> 387                     httplib_response = conn.getresponse()
    388                 except Exception as e:
    389                     # Remove the TypeError from the exception chain in Python 3;

/home/jun_gentoo/anaconda3/lib/python3.5/http/client.py in getresponse(self)
   1196         try:
   1197             try:
-> 1198                 response.begin()
   1199             except ConnectionError:
   1200                 self.close()

/home/jun_gentoo/anaconda3/lib/python3.5/http/client.py in begin(self)
    319             raise UnknownProtocol(version)
    320 
--> 321         self.headers = self.msg = parse_headers(self.fp)
    322 
    323         if self.debuglevel > 0:

/home/jun_gentoo/anaconda3/lib/python3.5/http/client.py in parse_headers(fp, _class)
    204     headers = []
    205     while True:
--> 206         line = fp.readline(_MAXLINE + 1)
    207         if len(line) > _MAXLINE:
    208             raise LineTooLong("header line")

/home/jun_gentoo/anaconda3/lib/python3.5/socket.py in readinto(self, b)
    574         while True:
    575             try:
--> 576                 return self._sock.recv_into(b)
    577             except timeout:
    578                 self._timeout_occurred = True

KeyboardInterrupt: 

In [ ]: