In [18]:
import app
from app.scripts import *
from app import db, models
import urllib2, json
In [19]:
sample_data= models.Article.query.all()
for article in sample_data:
if article.abstract == '':
print article, article.title, article.abstract
<Article ID: 3908057, The Potential of UAV-based Remote Sensing for Supporting Precision Agriculture in Indonesia > The Potential of UAV-based Remote Sensing for Supporting Precision Agriculture in Indonesia
<Article ID: 4095151, Life cycle sustainability assessment of UK electricity scenarios to 2070 > Life cycle sustainability assessment of UK electricity scenarios to 2070
<Article ID: 4266988, Long term air quality monitoring in a net-zero energy residence designed with low emitting interior products > Long term air quality monitoring in a net-zero energy residence designed with low emitting interior products
<Article ID: 4280391, The role of ESR spectroscopy in advancing catalytic science: some recent developments> The role of ESR spectroscopy in advancing catalytic science: some recent developments
<Article ID: 4428078, What does the scientific literature tell us about the ventilationhealth relationship in public and residential buildings? > What does the scientific literature tell us about the ventilationhealth relationship in public and residential buildings?
<Article ID: 4463364, Rejection of GM crops is not a failure for science> Rejection of GM crops is not a failure for science
<Article ID: 4467417, Feature: Iranian scientists rely on ingenuity and smuggling to survive sanctions> Feature: Iranian scientists rely on ingenuity and smuggling to survive sanctions
<Article ID: 4469089, Sustainable development agenda: 2030> Sustainable development agenda: 2030
<Article ID: 4469094, This week's section> This week's section
<Article ID: 4488083, Reproducibility will not cure what ails science> Reproducibility will not cure what ails science
<Article ID: 4490985, Intel to end sponsorship of Science Talent Search> Intel to end sponsorship of Science Talent Search
<Article ID: 4498139, Spotlight on Faculty Positions> Spotlight on Faculty Positions
<Article ID: 4498960, Hadoop neural network for parallel and distributed feature selection > Hadoop neural network for parallel and distributed feature selection
<Article ID: 4499324, What are the 25 schools responsible for the most important advances in science?> What are the 25 schools responsible for the most important advances in science?
<Article ID: 4499426, Gender disparity in Polish science by year (19752014) and by discipline > Gender disparity in Polish science by year (19752014) and by discipline
<Article ID: 4499642, Towards a diagnostic adaptation science> Towards a diagnostic adaptation science
<Article ID: 4500405, Helpfulness of user-generated reviews as a function of review sentiment, product type and information quality > Helpfulness of user-generated reviews as a function of review sentiment, product type and information quality
<Article ID: 4501207, Beyond sarcasm: The metalanguage and structures of mock politeness > Beyond sarcasm: The metalanguage and structures of mock politeness
<Article ID: 4501323, Notch signals to Yap too> Notch signals to Yap too
<Article ID: 4501503, Yung Sik Kim . Questioning Science in East Asian Contexts: Essays on Science, Confucianism, and the Comparative History of Science . (Science and Religion in East Asia, 1.) vii + 284 pp., bibl., index. Leiden: Brill, 2014. 103 (cloth).> Yung Sik Kim . Questioning Science in East Asian Contexts: Essays on Science, Confucianism, and the Comparative History of Science . (Science and Religion in East Asia, 1.) vii + 284 pp., bibl., index. Leiden: Brill, 2014. 103 (cloth).
<Article ID: 4501614, The effect of TiO2/aluminosilicate nanocomposite additives on the mechanical and thermal properties of polyacrylic coatings> The effect of TiO2/aluminosilicate nanocomposite additives on the mechanical and thermal properties of polyacrylic coatings
<Article ID: 4503550, Examining the impacts of organizational culture and top management support of knowledge sharing on the success of software process improvement > Examining the impacts of organizational culture and top management support of knowledge sharing on the success of software process improvement
In [27]:
data= urllib2.urlopen('http://eutils.ncbi.nlm.nih.gov/entrez/eutils/esearch.fcgi?db=pubmed&reldate=10&datetype=edat&retmax=100000&usehistory=y&retmode=json')
print len(json.load(data)['esearchresult']['idlist'])
---------------------------------------------------------------------------
KeyboardInterrupt Traceback (most recent call last)
<ipython-input-27-13ed47607ce0> in <module>()
----> 1 data= urllib2.urlopen('http://eutils.ncbi.nlm.nih.gov/entrez/eutils/esearch.fcgi?db=pubmed&reldate=10&datetype=edat&retmax=100000&usehistory=y&retmode=json')
2 print len(json.load(data)['esearchresult']['idlist'])
/System/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/urllib2.pyc in urlopen(url, data, timeout)
125 if _opener is None:
126 _opener = build_opener()
--> 127 return _opener.open(url, data, timeout)
128
129 def install_opener(opener):
/System/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/urllib2.pyc in open(self, fullurl, data, timeout)
402 req = meth(req)
403
--> 404 response = self._open(req, data)
405
406 # post-process response
/System/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/urllib2.pyc in _open(self, req, data)
420 protocol = req.get_type()
421 result = self._call_chain(self.handle_open, protocol, protocol +
--> 422 '_open', req)
423 if result:
424 return result
/System/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/urllib2.pyc in _call_chain(self, chain, kind, meth_name, *args)
380 func = getattr(handler, meth_name)
381
--> 382 result = func(*args)
383 if result is not None:
384 return result
/System/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/urllib2.pyc in http_open(self, req)
1212
1213 def http_open(self, req):
-> 1214 return self.do_open(httplib.HTTPConnection, req)
1215
1216 http_request = AbstractHTTPHandler.do_request_
/System/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/urllib2.pyc in do_open(self, http_class, req)
1185 else:
1186 try:
-> 1187 r = h.getresponse(buffering=True)
1188 except TypeError: # buffering kw not supported
1189 r = h.getresponse()
/System/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/httplib.pyc in getresponse(self, buffering)
1043 response = self.response_class(*args, **kwds)
1044
-> 1045 response.begin()
1046 assert response.will_close != _UNKNOWN
1047 self.__state = _CS_IDLE
/System/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/httplib.pyc in begin(self)
407 # read until we get a non-100 response
408 while True:
--> 409 version, status, reason = self._read_status()
410 if status != CONTINUE:
411 break
/System/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/httplib.pyc in _read_status(self)
363 def _read_status(self):
364 # Initialize with Simple-Response defaults
--> 365 line = self.fp.readline(_MAXLINE + 1)
366 if len(line) > _MAXLINE:
367 raise LineTooLong("header line")
/System/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/socket.pyc in readline(self, size)
474 while True:
475 try:
--> 476 data = self._sock.recv(self._rbufsize)
477 except error, e:
478 if e.args[0] == EINTR:
KeyboardInterrupt:
In [29]:
import time
start= time.time()
for i in range(101)[1:]:
print 'http://api.altmetric.com/v1/citations/1m?num_results=100&page={0}'.format(str(i))
response= urllib2.urlopen('http://api.altmetric.com/v1/citations/1m?num_results=100&page={0}'.format(str(i)))
print time.time()-start
http://api.altmetric.com/v1/citations/1m?num_results=100&page=1
2.07585906982
http://api.altmetric.com/v1/citations/1m?num_results=100&page=2
3.91931295395
http://api.altmetric.com/v1/citations/1m?num_results=100&page=3
5.55645513535
http://api.altmetric.com/v1/citations/1m?num_results=100&page=4
7.81083202362
http://api.altmetric.com/v1/citations/1m?num_results=100&page=5
9.44721698761
http://api.altmetric.com/v1/citations/1m?num_results=100&page=6
11.2907321453
http://api.altmetric.com/v1/citations/1m?num_results=100&page=7
13.8348090649
http://api.altmetric.com/v1/citations/1m?num_results=100&page=8
15.4911880493
http://api.altmetric.com/v1/citations/1m?num_results=100&page=9
---------------------------------------------------------------------------
KeyboardInterrupt Traceback (most recent call last)
<ipython-input-29-104ff4e6c821> in <module>()
3 for i in range(101)[1:]:
4 print 'http://api.altmetric.com/v1/citations/1m?num_results=100&page={0}'.format(str(i))
----> 5 response= urllib2.urlopen('http://api.altmetric.com/v1/citations/1m?num_results=100&page={0}'.format(str(i)))
6 print time.time()-start
/System/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/urllib2.pyc in urlopen(url, data, timeout)
125 if _opener is None:
126 _opener = build_opener()
--> 127 return _opener.open(url, data, timeout)
128
129 def install_opener(opener):
/System/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/urllib2.pyc in open(self, fullurl, data, timeout)
402 req = meth(req)
403
--> 404 response = self._open(req, data)
405
406 # post-process response
/System/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/urllib2.pyc in _open(self, req, data)
420 protocol = req.get_type()
421 result = self._call_chain(self.handle_open, protocol, protocol +
--> 422 '_open', req)
423 if result:
424 return result
/System/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/urllib2.pyc in _call_chain(self, chain, kind, meth_name, *args)
380 func = getattr(handler, meth_name)
381
--> 382 result = func(*args)
383 if result is not None:
384 return result
/System/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/urllib2.pyc in http_open(self, req)
1212
1213 def http_open(self, req):
-> 1214 return self.do_open(httplib.HTTPConnection, req)
1215
1216 http_request = AbstractHTTPHandler.do_request_
/System/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/urllib2.pyc in do_open(self, http_class, req)
1185 else:
1186 try:
-> 1187 r = h.getresponse(buffering=True)
1188 except TypeError: # buffering kw not supported
1189 r = h.getresponse()
/System/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/httplib.pyc in getresponse(self, buffering)
1043 response = self.response_class(*args, **kwds)
1044
-> 1045 response.begin()
1046 assert response.will_close != _UNKNOWN
1047 self.__state = _CS_IDLE
/System/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/httplib.pyc in begin(self)
407 # read until we get a non-100 response
408 while True:
--> 409 version, status, reason = self._read_status()
410 if status != CONTINUE:
411 break
/System/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/httplib.pyc in _read_status(self)
363 def _read_status(self):
364 # Initialize with Simple-Response defaults
--> 365 line = self.fp.readline(_MAXLINE + 1)
366 if len(line) > _MAXLINE:
367 raise LineTooLong("header line")
/System/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/socket.pyc in readline(self, size)
474 while True:
475 try:
--> 476 data = self._sock.recv(self._rbufsize)
477 except error, e:
478 if e.args[0] == EINTR:
KeyboardInterrupt:
In [ ]:
Content source: alexherns/sciral-ocf-dev
Similar notebooks: