In [1]:

    

import datetime as dt
from steemdata import SteemData

import pandas as pd
import numpy as np

import plotly.plotly as py
import plotly.graph_objs as go
import cufflinks as cf
    
# helpers
from toolz import keyfilter

def keep(d, whitelist):
    return keyfilter(lambda k: k in whitelist, d)

def omit(d, blacklist):
    return keyfilter(lambda k: k not in blacklist, d)


    

In [2]:

    

db = SteemData().db


    

In [ ]:

    




    

In [3]:

    

import json
import requests as rs
from steem.post import Post

def get_csrf(url):
    resp = rs.get(url)

    if resp.status_code == 200:
        body = resp.text
        start_index = body.find('"csrf":"')
        csrf = body[start_index + 8:start_index + 44]
        return csrf, resp.cookies

def get_view_count(url_partial, csrf, cookies):
    payload = {
        "csrf": csrf,
        "page": url_partial,
        "ref": "",
    }

    url = 'https://steemit.com/api/v1/page_view'
    resp = rs.post(url, json=payload, cookies=cookies)

    if resp.status_code == 200 and resp.text:
        return json.loads(resp.text)
    return {}

def get_views(identifier):
    p = Post(identifier)
    csrf, cookies = get_csrf('https://steemit.com' + p.url)
    return get_view_count(p.url, csrf, cookies).get('views', -1)


    

In [ ]:

    




    

In [4]:

    

# time constraints
time_constraints = {
    '$gte': dt.datetime.now() - dt.timedelta(days=8),
}
conditions = {
    'created': time_constraints,
#     'net_rshares': {'$gt': 0},
#     'author_reputation': {'$gt': 0},
    'net_votes': {'$gt': 100},
    'children': {'$gt': 3},
}
projection = {
    '_id': 0,
    'identifier': 1,
    'net_votes': 1,
    'net_rshares': 1,
    'author_reputation': 1,
    'title': 1,
    'author': 1,
    'pending_payout_value': 1,
    'total_payout_value': 1,
}
posts = list(db['Posts'].find(conditions, projection=projection))


    

In [5]:

    

def filter_spam(post):
    if int(post['net_rshares']) < 0 or int(post['author_reputation']) < 0:
        return False
    return True


    

In [6]:

    

posts = list(filter(filter_spam, posts))


    

In [7]:

    

len(posts)


    

    
Out[7]:





2476

In [8]:

    

# posts2 = []
# posts_gen = iter(posts)


    

In [9]:

    

# from contextlib import suppress
# import time

# for p in posts_gen:
#     with suppress(Exception):
#         views = get_views(p['identifier'])
#         time.sleep(0.01)
#         print(views)
        
#         posts2.append({
#             **p,
#             'views': views
#         })


    

In [10]:

    

from funcy import silent
posts2 = [{**x, 'views': silent(get_views)(x['identifier'])} for x in posts]


    

In [11]:

    

posts3 = [{
    **x,
    'pending_payout_value': x['pending_payout_value']['amount'],
    'total_payout_value': x['total_payout_value']['amount'],
} for x in posts2]


    

In [12]:

    

df = pd.DataFrame(posts3)


    

In [13]:

    

df2 = df[['author', 'title', 'net_votes', 'pending_payout_value', 'views']].dropna()
df2['author'] = df2['author'].apply(lambda x: "@%s" % x)
df2['views'] = df2['views'].apply(int)
df2['title'] = df2['title'].apply(lambda x: x[:30])


    

In [16]:

    

df3 = df2.sort_values('views', ascending=False).head(25)
df3.head(30)


    

    
Out[16]:







  

    

      

      
author
      
title
      
net_votes
      
pending_payout_value
      
views
    
  
  

    

      
1112
      
@kingscrown
      
PSA Freewallet and Jaxx Gettin
      
1427
      
2086.976
      
5057
    
    

      
2085
      
@eosio
      
Draft EOS Token Sale Smart Con
      
173
      
270.892
      
4779
    
    

      
1894
      
@ludorum
      
# 정말 도움이 필요합니다. 저희 집에 불이 났습니다.
      
1209
      
4253.241
      
3955
    
    

      
656
      
@roelandp
      
SteemFest² Updates - Hotel(s)
      
1552
      
5474.683
      
3801
    
    

      
960
      
@grantcardone
      
When Will Cryptocurrency Repla
      
210
      
288.464
      
3460
    
    

      
1664
      
@razvanelulmarin
      
EOS ICO announced. What's gonn
      
542
      
1618.029
      
3414
    
    

      
817
      
@furion
      
Analyzing the crypto market wi
      
1136
      
1840.047
      
3196
    
    

      
1549
      
@jerrybanfield
      
$200 A Day Mining Ethereum or
      
549
      
251.302
      
3182
    
    

      
1163
      
@theywillkillyou
      
Transitioning From Full-Time Y
      
917
      
1864.176
      
2886
    
    

      
2424
      
@jrcornel
      
Dear Coinbase, I Forgive You
      
625
      
0.000
      
2864
    
    

      
1806
      
@kingscrown
      
Crazy things happening in Cryp
      
880
      
1453.710
      
2724
    
    

      
413
      
@kingscrown
      
IOTA - The Cryptocurrency That
      
1096
      
1728.440
      
2598
    
    

      
542
      
@allasyummyfood
      
INTRODUCING 8 NEW SOCIAL MEDIA
      
838
      
1741.495
      
2507
    
    

      
876
      
@jerrybanfield
      
Steem Video Ad #2 on Facebook
      
1242
      
1752.426
      
2501
    
    

      
1875
      
@trafalgar
      
Is The World Becoming More Vio
      
1036
      
2406.923
      
2455
    
    

      
766
      
@kingscrown
      
Bancor Protocol - Some Facts a
      
1146
      
1683.459
      
2417
    
    

      
1489
      
@timsaid
      
MyPictureSunday#8 - Testing my
      
1073
      
1740.261
      
2356
    
    

      
1862
      
@thecryptofiend
      
My Thoughts on the New Apple i
      
496
      
821.067
      
2309
    
    

      
331
      
@furion
      
Build yourself a Trezor
      
1042
      
1972.835
      
2271
    
    

      
2332
      
@someguy123
      
Uber now accepts Litecoin (but
      
293
      
0.000
      
2104
    
    

      
193
      
@maa
      
비트코인이 망하지 않을 이유 10가지(10 reason
      
216
      
218.765
      
1983
    
    

      
627
      
@oldstone
      
EEA, 중앙권력을 파괴하기 위한 트로이 목마인가 아니
      
177
      
256.997
      
1889
    
    

      
2455
      
@jerrybanfield
      
My Top 10 Cryptocurrency Fails
      
546
      
0.000
      
1864
    
    

      
1175
      
@joseph
      
1897 - Bitcoin Down - Coinbase
      
863
      
1350.059
      
1843
    
    

      
1310
      
@trafalgar
      
How Effective is Our Prison Sy
      
1160
      
2136.411
      
1829
    
  

In [15]:

    

from tabulate import tabulate
print(tabulate(df3.values.tolist(), df3.columns.tolist(), tablefmt="html"))


    

    




<table>
<thead>
<tr><th>author          </th><th>title                                             </th><th style="text-align: right;">  net_votes</th><th style="text-align: right;">  pending_payout_value</th><th style="text-align: right;">  views</th></tr>
</thead>
<tbody>
<tr><td>@kingscrown     </td><td>PSA Freewallet and Jaxx Gettin                    </td><td style="text-align: right;">       1427</td><td style="text-align: right;">              2086.98 </td><td style="text-align: right;">   5057</td></tr>
<tr><td>@eosio          </td><td>Draft EOS Token Sale Smart Con                    </td><td style="text-align: right;">        173</td><td style="text-align: right;">               270.892</td><td style="text-align: right;">   4779</td></tr>
<tr><td>@ludorum        </td><td># 정말 도움이 필요합니다. 저희 집에 불이 났습니다.</td><td style="text-align: right;">       1209</td><td style="text-align: right;">              4253.24 </td><td style="text-align: right;">   3955</td></tr>
<tr><td>@roelandp       </td><td>SteemFest² Updates - Hotel(s)                     </td><td style="text-align: right;">       1552</td><td style="text-align: right;">              5474.68 </td><td style="text-align: right;">   3801</td></tr>
<tr><td>@grantcardone   </td><td>When Will Cryptocurrency Repla                    </td><td style="text-align: right;">        210</td><td style="text-align: right;">               288.464</td><td style="text-align: right;">   3460</td></tr>
<tr><td>@razvanelulmarin</td><td>EOS ICO announced. What's gonn                    </td><td style="text-align: right;">        542</td><td style="text-align: right;">              1618.03 </td><td style="text-align: right;">   3414</td></tr>
<tr><td>@furion         </td><td>Analyzing the crypto market wi                    </td><td style="text-align: right;">       1136</td><td style="text-align: right;">              1840.05 </td><td style="text-align: right;">   3196</td></tr>
<tr><td>@jerrybanfield  </td><td>$200 A Day Mining Ethereum or                     </td><td style="text-align: right;">        549</td><td style="text-align: right;">               251.302</td><td style="text-align: right;">   3182</td></tr>
<tr><td>@theywillkillyou</td><td>Transitioning From Full-Time Y                    </td><td style="text-align: right;">        917</td><td style="text-align: right;">              1864.18 </td><td style="text-align: right;">   2886</td></tr>
<tr><td>@jrcornel       </td><td>Dear Coinbase, I Forgive You                      </td><td style="text-align: right;">        625</td><td style="text-align: right;">                 0    </td><td style="text-align: right;">   2864</td></tr>
<tr><td>@kingscrown     </td><td>Crazy things happening in Cryp                    </td><td style="text-align: right;">        880</td><td style="text-align: right;">              1453.71 </td><td style="text-align: right;">   2724</td></tr>
<tr><td>@kingscrown     </td><td>IOTA - The Cryptocurrency That                    </td><td style="text-align: right;">       1096</td><td style="text-align: right;">              1728.44 </td><td style="text-align: right;">   2598</td></tr>
<tr><td>@allasyummyfood </td><td>INTRODUCING 8 NEW SOCIAL MEDIA                    </td><td style="text-align: right;">        838</td><td style="text-align: right;">              1741.49 </td><td style="text-align: right;">   2507</td></tr>
<tr><td>@jerrybanfield  </td><td>Steem Video Ad #2 on Facebook                     </td><td style="text-align: right;">       1242</td><td style="text-align: right;">              1752.43 </td><td style="text-align: right;">   2501</td></tr>
<tr><td>@trafalgar      </td><td>Is The World Becoming More Vio                    </td><td style="text-align: right;">       1036</td><td style="text-align: right;">              2406.92 </td><td style="text-align: right;">   2455</td></tr>
<tr><td>@kingscrown     </td><td>Bancor Protocol - Some Facts a                    </td><td style="text-align: right;">       1146</td><td style="text-align: right;">              1683.46 </td><td style="text-align: right;">   2417</td></tr>
<tr><td>@timsaid        </td><td>MyPictureSunday#8 - Testing my                    </td><td style="text-align: right;">       1073</td><td style="text-align: right;">              1740.26 </td><td style="text-align: right;">   2356</td></tr>
<tr><td>@thecryptofiend </td><td>My Thoughts on the New Apple i                    </td><td style="text-align: right;">        496</td><td style="text-align: right;">               821.067</td><td style="text-align: right;">   2309</td></tr>
<tr><td>@furion         </td><td>Build yourself a Trezor                           </td><td style="text-align: right;">       1042</td><td style="text-align: right;">              1972.84 </td><td style="text-align: right;">   2271</td></tr>
<tr><td>@someguy123     </td><td>Uber now accepts Litecoin (but                    </td><td style="text-align: right;">        293</td><td style="text-align: right;">                 0    </td><td style="text-align: right;">   2104</td></tr>
<tr><td>@maa            </td><td>비트코인이 망하지 않을 이유 10가지(10 reason      </td><td style="text-align: right;">        216</td><td style="text-align: right;">               218.765</td><td style="text-align: right;">   1983</td></tr>
<tr><td>@oldstone       </td><td>EEA, 중앙권력을 파괴하기 위한 트로이 목마인가 아니</td><td style="text-align: right;">        177</td><td style="text-align: right;">               256.997</td><td style="text-align: right;">   1889</td></tr>
<tr><td>@jerrybanfield  </td><td>My Top 10 Cryptocurrency Fails                    </td><td style="text-align: right;">        546</td><td style="text-align: right;">                 0    </td><td style="text-align: right;">   1864</td></tr>
<tr><td>@joseph         </td><td>1897 - Bitcoin Down - Coinbase                    </td><td style="text-align: right;">        863</td><td style="text-align: right;">              1350.06 </td><td style="text-align: right;">   1843</td></tr>
<tr><td>@trafalgar      </td><td>How Effective is Our Prison Sy                    </td><td style="text-align: right;">       1160</td><td style="text-align: right;">              2136.41 </td><td style="text-align: right;">   1829</td></tr>
</tbody>
</table>

In [ ]: