In [1]:

    

import requests
import re


    

In [2]:

    

url='https://www.propertyvalue.com.au'
header={'Content-Type':'text/html','User-Agent':'Mozilla/5.0 (X11; Linux x86_64; rv:50.0) Gecko/20100101 Firefox/50.0','Accept':'*/*','Accept-Encoding':'gzip, deflate'}


    

In [3]:

    

session=requests.Session()


    

In [4]:

    

session.headers.update(header)


    

In [5]:

    

req=session.get(url)


    

In [6]:

    

print(req.headers)


    

    




{'Cache-control': 'no-cache="set-cookie"', 'Content-Encoding': 'gzip', 'Content-Type': 'text/html;charset=UTF-8', 'Date': 'Wed, 29 Mar 2017 05:56:01 GMT', 'Server': 'TTserver', 'Set-Cookie': 'JSESSIONID=5B96EADAD9DEB61112E17226D71A3564; Path=/; HttpOnly, freemium_guest_id=6d7180932ac025c530f0ce7b18e8bee7; Expires=Wed, 05-Apr-2017 05:56:01 GMT; Path=/; HttpOnly, AWSELB=B313D93916C938D1385E130AEAAD6234F536BFACDD495AD0DDB6E5FA71B3955E38449857BC6FE88046F64B56F5663C7CBD28BC5A4901EDDBEC6CCF3186C175C9331D47419E;PATH=/, visid_incap_488260=cAGA2WUsSoma+z67KxnIAnFM21gAAAAAQUIPAAAAAADa4DDLbAg1YQs3l11GllEc; expires=Wed, 28 Mar 2018 12:16:29 GMT; path=/; Domain=.propertyvalue.com.au, nlbi_488260=ILjiJM3htRGlZBDmfi2xygAAAACN32GfNKCa+8ZyDvLho9cL; path=/; Domain=.propertyvalue.com.au, incap_ses_136_488260=hMixHx1ovgUlzclcNyzjAXFM21gAAAAAUGNY0iJ5PsXlYjSel2zKkA==; path=/; Domain=.propertyvalue.com.au, ___utmvmwFuItpE=slDaGwfoBdS; path=/; Max-Age=900, ___utmvawFuItpE=rIQ\x01NvCY; path=/; Max-Age=900, ___utmvbwFuItpE=eZU\r\n    XaTOiald: qtm; path=/; Max-Age=900', 'Vary': 'Accept-Encoding', 'transfer-encoding': 'chunked', 'Connection': 'keep-alive', 'X-Iinfo': '5-16289198-16289206 NNNN CT(2 2 0) RT(1490766961057 68) q(0 0 0 0) r(1 1) U2', 'X-CDN': 'Incapsula'}

In [7]:

    

h=req.headers


    

In [8]:

    

XIinfo=h.get('X-Iinfo')


    

In [9]:

    

c=req.cookies


    

In [10]:

    

print(c)


    

    




<RequestsCookieJar[<Cookie incap_ses_136_488260=hMixHx1ovgUlzclcNyzjAXFM21gAAAAAUGNY0iJ5PsXlYjSel2zKkA== for .propertyvalue.com.au/>, <Cookie nlbi_488260=ILjiJM3htRGlZBDmfi2xygAAAACN32GfNKCa+8ZyDvLho9cL for .propertyvalue.com.au/>, <Cookie visid_incap_488260=cAGA2WUsSoma+z67KxnIAnFM21gAAAAAQUIPAAAAAADa4DDLbAg1YQs3l11GllEc for .propertyvalue.com.au/>, <Cookie AWSELB=B313D93916C938D1385E130AEAAD6234F536BFACDD495AD0DDB6E5FA71B3955E38449857BC6FE88046F64B56F5663C7CBD28BC5A4901EDDBEC6CCF3186C175C9331D47419E for www.propertyvalue.com.au/>, <Cookie JSESSIONID=5B96EADAD9DEB61112E17226D71A3564 for www.propertyvalue.com.au/>, <Cookie ___utmvawFuItpE=rIQ
NvCY for www.propertyvalue.com.au/>, <Cookie ___utmvbwFuItpE=eZU
    XaTOiald: qtm for www.propertyvalue.com.au/>, <Cookie ___utmvmwFuItpE=slDaGwfoBdS for www.propertyvalue.com.au/>, <Cookie freemium_guest_id=6d7180932ac025c530f0ce7b18e8bee7 for www.propertyvalue.com.au/>]>

In [11]:

    

session.cookies.update(c)


    

In [12]:

    

print(XIinfo)


    

    




5-16289198-16289206 NNNN CT(2 2 0) RT(1490766961057 68) q(0 0 0 0) r(1 1) U2

In [13]:

    

pattern=re.compile(r'(?:RT\()(\d+)')


    

In [14]:

    

r=pattern.search(XIinfo)


    

In [15]:

    

print(r)


    

    




<_sre.SRE_Match object; span=(35, 51), match='RT(1490766961057'>

In [16]:

    

print(r.groups())


    

    




('1490766961057',)

In [17]:

    

XIinfo=r.groups()[0]


    

In [18]:

    

print(XIinfo)


    

    




1490766961057

In [16]:

    

address='108 burrumarra ave ngunnawal'
address=address.replace(' ','%20')
print(address)


    

    




108%20burrumarra%20ave%20ngunnawal

In [17]:

    

payload={'op':'myrp_widgets.freemiumInvestorSLAS.singleLineSearchHandler','subop':'getSuggestionList','searchCategories':'1%2C2%2C3%2C4','searchString':address,'maxSuggestionResults':5}


    

In [18]:

    

req=session.get(url,params=payload)


    

In [19]:

    

print(req.url)


    

    




https://www.propertyvalue.com.au/?searchCategories=1%252C2%252C3%252C4&op=myrp_widgets.freemiumInvestorSLAS.singleLineSearchHandler&searchString=108%2520burrumarra%2520ave%2520ngunnawal&subop=getSuggestionList&maxSuggestionResults=5

In [20]:

    

print(req.headers)


    

    




{'Cache-Control': 'no-cache', 'Content-Length': '786', 'Content-Type': 'text/html', 'Connection': 'close', 'X-Iinfo': '10-30501584-0 0NNN RT(1491376021893 59) q(0 -1 -1 -1) r(0 -1) B12(4,316,0) U18'}

In [21]:

    

print(req.content)


    

    




b'<html style="height:100%"><head><META NAME="ROBOTS" CONTENT="NOINDEX, NOFOLLOW"><meta name="format-detection" content="telephone=no"><meta name="viewport" content="initial-scale=1.0"><meta http-equiv="X-UA-Compatible" content="IE=edge,chrome=1"><script type="text/javascript" src="/_Incapsula_Resource?SWJIYLWA=2977d8d74f63d7f8fedbea018b7a1d05"></script></head><body style="margin:0px;height:100%"><iframe src="/_Incapsula_Resource?CWUDNSAI=24&xinfo=10-30501584-0 0NNN RT(1491376021893 59) q(0 -1 -1 -1) r(0 -1) B12(4,316,0) U18&incident_id=607000030055603457-288871292686237754&edet=12&cinfo=04000000" frameborder=0 width="100%" height="100%" marginheight="0px" marginwidth="0px">Request unsuccessful. Incapsula incident ID: 607000030055603457-288871292686237754</iframe></body></html>'

In [ ]: