notebook.community

Edit and run 





In [1]:



    


from __future__ import print_function
from pyspark import SparkContext
from pyspark.streaming import StreamingContext



    











In [2]:



    


sc = SparkContext(appName="TwitterSentimentAnalysis")
sc.setLogLevel("ERROR")
ssc = StreamingContext(sc, 10)



    











In [3]:



    


socket_stream = ssc.socketTextStream("172.31.20.58", 1982)
lines = socket_stream.window(20)



    











In [4]:



    


hashtags = lines.flatMap(lambda text : text.split(" ")).filter(lambda word : word.lower().startswith('#')).map(lambda word : word.lower(), ).reduceByKey(lambda a,b : a+b)



    











In [5]:



    


author_count_sorted_dstream = hashtags.transform(lambda foo : foo.sortBy(lambda x : x[0].lower()).sortBy(lambda x : x[1].lower(),ascending=False))



    











In [6]:



    


author_count_sorted_dstream.pprint()



    











In [ ]:



    


ssc.start()



    











In [ ]:



    


ssc.awaitTermination()



    


















    






-------------------------------------------
Time: 2017-10-08 14:04:10
-------------------------------------------

-------------------------------------------
Time: 2017-10-08 14:04:20
-------------------------------------------

-------------------------------------------
Time: 2017-10-08 14:04:30
-------------------------------------------

-------------------------------------------
Time: 2017-10-08 14:04:40
-------------------------------------------

-------------------------------------------
Time: 2017-10-08 14:04:50
-------------------------------------------

-------------------------------------------
Time: 2017-10-08 14:05:00
-------------------------------------------

-------------------------------------------
Time: 2017-10-08 14:05:10
-------------------------------------------

-------------------------------------------
Time: 2017-10-08 14:05:20
-------------------------------------------

-------------------------------------------
Time: 2017-10-08 14:05:30
-------------------------------------------

-------------------------------------------
Time: 2017-10-08 14:05:40
-------------------------------------------

-------------------------------------------
Time: 2017-10-08 14:05:50
-------------------------------------------

-------------------------------------------
Time: 2017-10-08 14:06:00
-------------------------------------------

-------------------------------------------
Time: 2017-10-08 14:06:10
-------------------------------------------

-------------------------------------------
Time: 2017-10-08 14:06:20
-------------------------------------------

-------------------------------------------
Time: 2017-10-08 14:06:30
-------------------------------------------

-------------------------------------------
Time: 2017-10-08 14:06:40
-------------------------------------------

-------------------------------------------
Time: 2017-10-08 14:06:50
-------------------------------------------

-------------------------------------------
Time: 2017-10-08 14:07:00
-------------------------------------------

-------------------------------------------
Time: 2017-10-08 14:07:10
-------------------------------------------

-------------------------------------------
Time: 2017-10-08 14:07:20
-------------------------------------------

-------------------------------------------
Time: 2017-10-08 14:07:30
-------------------------------------------

-------------------------------------------
Time: 2017-10-08 14:07:40
-------------------------------------------

-------------------------------------------
Time: 2017-10-08 14:07:50
-------------------------------------------

-------------------------------------------
Time: 2017-10-08 14:08:00
-------------------------------------------

-------------------------------------------
Time: 2017-10-08 14:08:10
-------------------------------------------

-------------------------------------------
Time: 2017-10-08 14:08:20
-------------------------------------------

-------------------------------------------
Time: 2017-10-08 14:08:30
-------------------------------------------

-------------------------------------------
Time: 2017-10-08 14:08:40
-------------------------------------------

-------------------------------------------
Time: 2017-10-08 14:08:50
-------------------------------------------

-------------------------------------------
Time: 2017-10-08 14:09:00
-------------------------------------------

-------------------------------------------
Time: 2017-10-08 14:09:10
-------------------------------------------

-------------------------------------------
Time: 2017-10-08 14:09:20
-------------------------------------------

-------------------------------------------
Time: 2017-10-08 14:09:30
-------------------------------------------

-------------------------------------------
Time: 2017-10-08 14:09:40
-------------------------------------------

-------------------------------------------
Time: 2017-10-08 14:09:50
-------------------------------------------

-------------------------------------------
Time: 2017-10-08 14:10:00
-------------------------------------------

-------------------------------------------
Time: 2017-10-08 14:10:10
-------------------------------------------

-------------------------------------------
Time: 2017-10-08 14:10:20
-------------------------------------------

-------------------------------------------
Time: 2017-10-08 14:10:30
-------------------------------------------

-------------------------------------------
Time: 2017-10-08 14:10:40
-------------------------------------------

-------------------------------------------
Time: 2017-10-08 14:10:50
-------------------------------------------

-------------------------------------------
Time: 2017-10-08 14:11:00
-------------------------------------------

-------------------------------------------
Time: 2017-10-08 14:11:10
-------------------------------------------

-------------------------------------------
Time: 2017-10-08 14:11:20
-------------------------------------------

-------------------------------------------
Time: 2017-10-08 14:11:30
-------------------------------------------

-------------------------------------------
Time: 2017-10-08 14:11:40
-------------------------------------------

-------------------------------------------
Time: 2017-10-08 14:11:50
-------------------------------------------

-------------------------------------------
Time: 2017-10-08 14:12:00
-------------------------------------------

-------------------------------------------
Time: 2017-10-08 14:12:10
-------------------------------------------

-------------------------------------------
Time: 2017-10-08 14:12:20
-------------------------------------------

-------------------------------------------
Time: 2017-10-08 14:12:30
-------------------------------------------

-------------------------------------------
Time: 2017-10-08 14:12:40
-------------------------------------------

-------------------------------------------
Time: 2017-10-08 14:12:50
-------------------------------------------

-------------------------------------------
Time: 2017-10-08 14:13:00
-------------------------------------------

-------------------------------------------
Time: 2017-10-08 14:13:10
-------------------------------------------

-------------------------------------------
Time: 2017-10-08 14:13:20
-------------------------------------------

-------------------------------------------
Time: 2017-10-08 14:13:30
-------------------------------------------

-------------------------------------------
Time: 2017-10-08 14:13:40
-------------------------------------------

-------------------------------------------
Time: 2017-10-08 14:13:50
-------------------------------------------

-------------------------------------------
Time: 2017-10-08 14:14:00
-------------------------------------------

-------------------------------------------
Time: 2017-10-08 14:14:10
-------------------------------------------

-------------------------------------------
Time: 2017-10-08 14:14:20
-------------------------------------------

-------------------------------------------
Time: 2017-10-08 14:14:30
-------------------------------------------

-------------------------------------------
Time: 2017-10-08 14:14:40
-------------------------------------------

-------------------------------------------
Time: 2017-10-08 14:14:50
-------------------------------------------

-------------------------------------------
Time: 2017-10-08 14:15:00
-------------------------------------------

-------------------------------------------
Time: 2017-10-08 14:15:10
-------------------------------------------

-------------------------------------------
Time: 2017-10-08 14:15:20
-------------------------------------------

-------------------------------------------
Time: 2017-10-08 14:15:30
-------------------------------------------

-------------------------------------------
Time: 2017-10-08 14:15:40
-------------------------------------------

-------------------------------------------
Time: 2017-10-08 14:15:50
-------------------------------------------

-------------------------------------------
Time: 2017-10-08 14:16:00
-------------------------------------------

-------------------------------------------
Time: 2017-10-08 14:16:10
-------------------------------------------

-------------------------------------------
Time: 2017-10-08 14:16:20
-------------------------------------------

-------------------------------------------
Time: 2017-10-08 14:16:30
-------------------------------------------

-------------------------------------------
Time: 2017-10-08 14:16:40
-------------------------------------------

-------------------------------------------
Time: 2017-10-08 14:16:50
-------------------------------------------

-------------------------------------------
Time: 2017-10-08 14:17:00
-------------------------------------------

-------------------------------------------
Time: 2017-10-08 14:17:10
-------------------------------------------


















In [ ]:



    






    











In [ ]:



    






    











In [ ]:



    






    











In [ ]:



    






    











In [ ]:



    






    











In [ ]:

Content source: dalonlobo/GL-Mini-Projects

Similar notebooks:

notebook.community | gallery | about