In [1]:
!module list
Currently Loaded Modulefiles:
1) anaconda3/4.2.0 2) matlab/2015a 3) zeromq/4.1.5
In [ ]:
!mkdir ~/software
!cd ~/software; wget https://archive.apache.org/dist/spark/spark-2.1.0/spark-2.1.0-bin-hadoop2.7.tgz
In [ ]:
!cd ~/software; tar -xzf spark-2.1.0-bin-hadoop2.7.tgz
In [2]:
import os
import subprocess
def generate_compute_file(node_file=os.environ['PBS_NODEFILE'],
slaves_file='/home/lngo/software/spark-2.1.0-bin-hadoop2.7/conf/slaves'):
node_list = subprocess.check_output(['cat', node_file]).decode("utf-8").strip().split('\n')
node_list = sorted(list(set(node_list)))
print("Executors: ")
with open(slaves_file, 'w') as slave_file_fp:
for node in node_list:
slave_file_fp.write('{}\n'.format(node))
print(node)
generate_compute_file()
Executors:
node0052.palmetto.clemson.edu
node0121.palmetto.clemson.edu
node0139.palmetto.clemson.edu
node1774.palmetto.clemson.edu
In [3]:
!more ~/software/spark-2.1.0-bin-hadoop2.7/conf/slaves
node0052.palmetto.clemson.edu
node0121.palmetto.clemson.edu
node0139.palmetto.clemson.edu
node1774.palmetto.clemson.edu
In [4]:
!sh ~/software/spark-2.1.0-bin-hadoop2.7/sbin/stop-all.sh
!echo '------------------------------------------------------'
!sh ~/software/spark-2.1.0-bin-hadoop2.7/sbin/start-all.sh
node1774.palmetto.clemson.edu: stopping org.apache.spark.deploy.worker.Worker
node0052.palmetto.clemson.edu: stopping org.apache.spark.deploy.worker.Worker
node0121.palmetto.clemson.edu: stopping org.apache.spark.deploy.worker.Worker
node0139.palmetto.clemson.edu: stopping org.apache.spark.deploy.worker.Worker
stopping org.apache.spark.deploy.master.Master
------------------------------------------------------
starting org.apache.spark.deploy.master.Master, logging to /home/lngo/software/spark-2.1.0-bin-hadoop2.7/logs/spark-lngo-org.apache.spark.deploy.master.Master-1-node1774.palmetto.clemson.edu.out
node1774.palmetto.clemson.edu: starting org.apache.spark.deploy.worker.Worker, logging to /home/lngo/software/spark-2.1.0-bin-hadoop2.7/logs/spark-lngo-org.apache.spark.deploy.worker.Worker-1-node1774.palmetto.clemson.edu.out
node0052.palmetto.clemson.edu: starting org.apache.spark.deploy.worker.Worker, logging to /home/lngo/software/spark-2.1.0-bin-hadoop2.7/logs/spark-lngo-org.apache.spark.deploy.worker.Worker-1-node0052.palmetto.clemson.edu.out
node0139.palmetto.clemson.edu: starting org.apache.spark.deploy.worker.Worker, logging to /home/lngo/software/spark-2.1.0-bin-hadoop2.7/logs/spark-lngo-org.apache.spark.deploy.worker.Worker-1-node0139.palmetto.clemson.edu.out
node0121.palmetto.clemson.edu: starting org.apache.spark.deploy.worker.Worker, logging to /home/lngo/software/spark-2.1.0-bin-hadoop2.7/logs/spark-lngo-org.apache.spark.deploy.worker.Worker-1-node0121.palmetto.clemson.edu.out
In [5]:
import sys
import os
import json
sys.path.insert(0, '/home/lngo/software/spark-2.1.0-bin-hadoop2.7/python')
sys.path.insert(0, '/home/lngo/software/spark-2.1.0-bin-hadoop2.7/python/lib/py4j-0.10.4-src.zip')
os.environ["SPARK_HOME"] = "/home/lngo/software/spark-2.1.0-bin-hadoop2.7/"
os.environ['PYSPARK_PYTHON'] = '/software/anaconda/4.2.0/bin/python'
os.environ['PYSPARK_SUBMIT_ARGS'] = '--packages org.apache.spark:spark-streaming-kafka-0-8_2.11:2.0.0 pyspark-shell'
import pyspark
from pyspark.streaming import StreamingContext
from pyspark.streaming.kafka import KafkaUtils
conf = pyspark.SparkConf()
conf.setMaster("spark://node1774:7077")
conf.set("spark.driver.memory","8g")
conf.set("spark.executor.memory","50g")
conf.set("spark.num.executors","4")
conf.set("spark.executor.cores","12")
sc = pyspark.SparkContext(conf=conf, appName='PythonStreaming')
ssc = StreamingContext(sc, 2)
In [6]:
sc
Out[6]:
<pyspark.context.SparkContext at 0x2b8c39a1d810>
In [7]:
ssc
Out[7]:
<pyspark.streaming.context.StreamingContext at 0x2b8c288fdad0>
In [8]:
brokers = 'node1774.palmetto.clemson.edu:9092'
topic = 'test'
def parseTweet(x):
textContent = 'NoContent'
try:
jsonData = json.loads(json.loads(x))
textContent = jsonData['text']
except TypeError:
textContent = 'TypeError'
return textContent
kvs = KafkaUtils.createDirectStream(ssc, [topic],{'metadata.broker.list': brokers})
lines = kvs.map(lambda x: x[1])
counts = lines.map(parseTweet).flatMap(lambda line: line.split(' ')) \
.map(lambda word: (word, 1)) \
.reduceByKey(lambda a, b: a+b)
counts.pprint()
ssc.start()
ssc.awaitTermination()
-------------------------------------------
Time: 2017-11-27 13:07:48
-------------------------------------------
-------------------------------------------
Time: 2017-11-27 13:07:50
-------------------------------------------
(u'we', 1)
(u'UGA,', 1)
(u'@JBarnesRTR', 1)
(u'I', 1)
(u'Oklahoma', 1)
(u'mean', 1)
(u'looked', 1)
(u'all', 1)
(u'at', 1)
(u'https://t.co/8yQRTR5Odc', 1)
...
-------------------------------------------
Time: 2017-11-27 13:07:52
-------------------------------------------
-------------------------------------------
Time: 2017-11-27 13:07:54
-------------------------------------------
-------------------------------------------
Time: 2017-11-27 13:07:56
-------------------------------------------
-------------------------------------------
Time: 2017-11-27 13:07:58
-------------------------------------------
-------------------------------------------
Time: 2017-11-27 13:08:00
-------------------------------------------
(u'LSU.', 1)
(u'Clemson', 1)
(u'to', 1)
(u'and', 1)
(u'that', 1)
(u'@AUFAMILY', 1)
(u'Tell', 1)
-------------------------------------------
Time: 2017-11-27 13:08:02
-------------------------------------------
(u'', 1)
(u'educated', 1)
(u'roster,', 1)
(u'@TheDutchColeman:', 1)
(u'prospects\u2026', 1)
(u'an', 1)
(u'RT', 1)
(u'current', 1)
(u'player', 1)
(u'that.', 1)
...
-------------------------------------------
Time: 2017-11-27 13:08:04
-------------------------------------------
-------------------------------------------
Time: 2017-11-27 13:08:06
-------------------------------------------
(u'', 2)
(u'defense', 1)
(u'DC).', 1)
(u'defenses', 1)
(u'top', 1)
(u'&', 1)
(u'Venables', 1)
(u'Clemson', 1)
(u'is', 2)
(u'than', 1)
...
-------------------------------------------
Time: 2017-11-27 13:08:08
-------------------------------------------
-------------------------------------------
Time: 2017-11-27 13:08:10
-------------------------------------------
-------------------------------------------
Time: 2017-11-27 13:08:12
-------------------------------------------
-------------------------------------------
Time: 2017-11-27 13:08:14
-------------------------------------------
-------------------------------------------
Time: 2017-11-27 13:08:16
-------------------------------------------
-------------------------------------------
Time: 2017-11-27 13:08:18
-------------------------------------------
-------------------------------------------
Time: 2017-11-27 13:08:20
-------------------------------------------
-------------------------------------------
Time: 2017-11-27 13:08:22
-------------------------------------------
(u'poll:', 1)
(u'https://t.co/oGeW8cOUGk', 1)
(u'Clemson', 1)
(u'falls', 1)
(u'OU', 1)
(u'2nd', 1)
(u'AP', 1)
(u'No.', 1)
(u'#Fitness', 1)
(u'after', 1)
...
-------------------------------------------
Time: 2017-11-27 13:08:24
-------------------------------------------
-------------------------------------------
Time: 2017-11-27 13:08:26
-------------------------------------------
-------------------------------------------
Time: 2017-11-27 13:08:28
-------------------------------------------
-------------------------------------------
Time: 2017-11-27 13:08:30
-------------------------------------------
(u'', 2)
(u'be:', 1)
(u'will', 1)
(u'14\nOklahoma', 1)
(u'1.\u2026', 1)
(u'38\n\nAll-Americans:\n\nClemson', 1)
(u'#Championship', 1)
(u'#CFPlayoff', 1)
(u'@_ElleJohnson:', 1)
(u'Sat,', 1)
...
-------------------------------------------
Time: 2017-11-27 13:08:32
-------------------------------------------
-------------------------------------------
Time: 2017-11-27 13:08:34
-------------------------------------------
-------------------------------------------
Time: 2017-11-27 13:08:36
-------------------------------------------
-------------------------------------------
Time: 2017-11-27 13:08:38
-------------------------------------------
(u'', 1)
(u'14\nOklahoma', 1)
(u'38\n\nAll-Americans:\n\nClemson', 1)
(u'RT', 1)
(u'23\nOklahoma', 1)
(u'winners:\n\nClemson\u2026', 1)
(u'63\n\nHeisman', 1)
(u'10-win', 1)
(u'seasons:\n\nClemson', 1)
(u'@blinkinriley:', 1)
...
-------------------------------------------
Time: 2017-11-27 13:08:40
-------------------------------------------
(u'@MBUSA', 1)
(u'To', 1)
(u'beat', 1)
(u'https://t.co/slGRhmKMac', 1)
(u'new', 1)
(u'Clemson', 1)
(u'@espn:', 1)
(u'fell,', 1)
(u'1?\n\nWhile', 1)
(u'RT', 1)
...
-------------------------------------------
Time: 2017-11-27 13:08:42
-------------------------------------------
(u'@QualkTalk', 1)
(u'for', 1)
(u'Clemson', 1)
(u'my', 1)
(u'some', 1)
(u'by', 1)
(u'looks\u2026', 1)
(u'https://t.co/ACf20q5LkZ', 1)
(u'rampage', 1)
(u'uga', 1)
...
-------------------------------------------
Time: 2017-11-27 13:08:44
-------------------------------------------
(u'so', 1)
(u'beat', 2)
(u'State\n9.', 1)
(u'RT', 1)
(u'Can\u2019t', 1)
(u'as', 1)
(u'State\u2026', 1)
(u'https://t.co/gsIoykKfbz', 1)
(u'Penn', 1)
(u'@SportsCenter:', 1)
...
-------------------------------------------
Time: 2017-11-27 13:08:46
-------------------------------------------
-------------------------------------------
Time: 2017-11-27 13:08:48
-------------------------------------------
(u'', 1)
(u'?\nMinutes', 1)
(u'secure', 1)
(u'program\u2019s', 1)
(u'UW', 1)
(u'you', 1)
(u'Clemson', 1)
(u'is', 1)
(u'https:/\u2026', 1)
(u'any', 1)
...
-------------------------------------------
Time: 2017-11-27 13:08:50
-------------------------------------------
(u'etc', 2)
(u'Clemson', 1)
(u'Tee', 1)
(u'lol', 1)
(u'DC,', 1)
(u'Martin,', 1)
(u'HC,', 1)
(u'St', 1)
(u'Iowa', 1)
(u'https://t.co/erG0WHkS6x', 1)
-------------------------------------------
Time: 2017-11-27 13:08:52
-------------------------------------------
(u'', 1)
(u'14\nOklahoma', 1)
(u'38\n\nAll-Americans:\n\nClemson', 1)
(u'RT', 1)
(u'23\nOklahoma', 1)
(u'winners:\n\nClemson\u2026', 1)
(u'63\n\nHeisman', 1)
(u'10-win', 1)
(u'seasons:\n\nClemson', 1)
(u'@blinkinriley:', 1)
...
-------------------------------------------
Time: 2017-11-27 13:08:54
-------------------------------------------
(u'better', 1)
(u'ra\u2026', 1)
(u'UCF', 1)
(u'https://t.co/TjbXtRLkVQ', 1)
(u'boy', 1)
(u'notice', 1)
(u'play', 1)
(u'no', 1)
(u'you', 1)
(u'worked', 1)
...
-------------------------------------------
Time: 2017-11-27 13:08:56
-------------------------------------------
-------------------------------------------
Time: 2017-11-27 13:08:58
-------------------------------------------
-------------------------------------------
Time: 2017-11-27 13:09:00
-------------------------------------------
(u'\u2705\nLouisville', 1)
(u'@TigerJustin13:', 1)
(u'#Savage', 1)
(u'\n\nOhio', 1)
(u'\u2705\nFSU', 1)
(u'\u2705\nAuburn', 1)
(u'Saturday', 1)
(u'RT', 1)
(u'#Clemson', 1)
(u'9-0', 1)
...
-------------------------------------------
Time: 2017-11-27 13:09:02
-------------------------------------------
(u'salary', 1)
(u'based\u2026', 1)
(u'4th', 1)
(u'That\u2019s', 1)
(u'for', 1)
(u'LSU,', 1)
(u'assistants', 1)
(u'Clemson', 1)
(u'is', 1)
(u'RT', 1)
...
-------------------------------------------
Time: 2017-11-27 13:09:04
-------------------------------------------
(u'', 1)
(u'#VolTwitter', 1)
(u'support.', 1)
(u'Love', 1)
(u'literally', 1)
(u'tweet', 1)
(u'Unfortunately,', 1)
(u'my', 1)
(u'RT', 1)
(u'to', 1)
...
-------------------------------------------
Time: 2017-11-27 13:09:06
-------------------------------------------
-------------------------------------------
Time: 2017-11-27 13:09:08
-------------------------------------------
(u'', 1)
(u'FOUR', 1)
(u'tickets', 1)
(u'11/29!\n\n\u2795https://t.co/N9tWJXpo2T\u2026', 1)
(u'chance', 1)
(u'https://t.co/stlLYLRg2i', 1)
(u'#math', 1)
(u'#CFPlayoff', 1)
(u'\U0001f418\u2764\ufe0f', 1)
(u'(2pm)', 1)
...
-------------------------------------------
Time: 2017-11-27 13:09:10
-------------------------------------------
(u'will', 1)
(u'https://t.co/PUkaFXOKkc', 1)
(u'played', 1)
(u'Oklahoma', 1)
(u'one', 1)
(u"let's", 1)
(u'loss', 1)
(u'hasnt', 1)
(u'soul\u2026', 1)
(u'the', 1)
...
-------------------------------------------
Time: 2017-11-27 13:09:12
-------------------------------------------
(u'', 1)
(u'Tennessee\u2026', 1)
(u'@nickmcclure_:', 1)
(u'beat', 5)
(u'Michigan', 1)
(u'RT', 1)
(u'Central', 1)
(u'Akron\nAkron', 1)
(u'Tennessee\nMid', 1)
(u'Mid', 1)
...
-------------------------------------------
Time: 2017-11-27 13:09:14
-------------------------------------------
-------------------------------------------
Time: 2017-11-27 13:09:16
-------------------------------------------
-------------------------------------------
Time: 2017-11-27 13:09:18
-------------------------------------------
-------------------------------------------
Time: 2017-11-27 13:09:20
-------------------------------------------
(u'', 1)
(u'will', 1)
(u'Like', 1)
(u'like', 1)
(u'https://t.co/j4hkivn071', 1)
(u'take', 1)
(u'Miami', 1)
(u'I', 2)
(u'is', 1)
(u'top-15', 1)
...
-------------------------------------------
Time: 2017-11-27 13:09:22
-------------------------------------------
(u'defense', 1)
(u'for', 1)
(u'Bentley', 1)
(u'Clemson', 1)
(u'life', 1)
(u'made', 1)
(u'football:', 1)
(u'Carolina', 1)
(u'Jake', 1)
(u'https://t.co/uSHDa8jdPU', 1)
...
-------------------------------------------
Time: 2017-11-27 13:09:24
-------------------------------------------
(u'Clemson\u2019s', 1)
(u'@LarryWilliamsTI', 1)
(u'@dixiefriedsport', 1)
(u'play', 2)
(u'I\u2019m', 1)
(u'too.', 1)
(u'guy', 1)
(u'hard', 1)
(u'by', 1)
(u'listen', 1)
...
-------------------------------------------
Time: 2017-11-27 13:09:26
-------------------------------------------
(u'prospect', 1)
(u'', 1)
(u'better', 1)
(u'I.....', 1)
(u'point', 1)
(u'State\n9.', 1)
(u'Clemson', 1)
(u'you', 1)
(u'than', 1)
(u'he', 1)
...
-------------------------------------------
Time: 2017-11-27 13:09:28
-------------------------------------------
-------------------------------------------
Time: 2017-11-27 13:09:30
-------------------------------------------
(u'State\n9.', 1)
(u'RT', 1)
(u'State\u2026', 1)
(u'Penn', 1)
(u'@SportsCenter:', 1)
(u'Georgia\n7.', 1)
(u'Miami\n8.', 1)
(u'Oklahoma\n3.', 1)
(u'AP', 1)
(u'Poll:\n1.', 1)
...
-------------------------------------------
Time: 2017-11-27 13:09:32
-------------------------------------------
-------------------------------------------
Time: 2017-11-27 13:09:34
-------------------------------------------
-------------------------------------------
Time: 2017-11-27 13:09:36
-------------------------------------------
(u'', 1)
(u'hi', 1)
(u'Clemson,', 1)
(u'RT', 1)
(u'say', 1)
(u'@SEC_Exposed:', 1)
(u'UCLA', 1)
(u'and', 1)
(u'TCU,', 1)
(u'Cal', 1)
...
-------------------------------------------
Time: 2017-11-27 13:09:38
-------------------------------------------
-------------------------------------------
Time: 2017-11-27 13:09:40
-------------------------------------------
-------------------------------------------
Time: 2017-11-27 13:09:42
-------------------------------------------
-------------------------------------------
Time: 2017-11-27 13:09:44
-------------------------------------------
(u'BABEYYY', 1)
(u'YEAH', 1)
(u'https://t.co/Zpfwc3QYFl', 1)
-------------------------------------------
Time: 2017-11-27 13:09:46
-------------------------------------------
(u'beer', 1)
(u'stuff', 1)
(u"shouldn't", 1)
(u'"Students', 1)
(u'Clemson', 1)
(u'thats', 1)
(u'to', 1)
(u'band', 1)
(u'every', 1)
(u'at', 1)
...
-------------------------------------------
Time: 2017-11-27 13:09:48
-------------------------------------------
-------------------------------------------
Time: 2017-11-27 13:09:50
-------------------------------------------
-------------------------------------------
Time: 2017-11-27 13:09:52
-------------------------------------------
-------------------------------------------
Time: 2017-11-27 13:09:54
-------------------------------------------
-------------------------------------------
Time: 2017-11-27 13:09:56
-------------------------------------------
-------------------------------------------
Time: 2017-11-27 13:09:58
-------------------------------------------
-------------------------------------------
Time: 2017-11-27 13:10:00
-------------------------------------------
-------------------------------------------
Time: 2017-11-27 13:10:02
-------------------------------------------
-------------------------------------------
Time: 2017-11-27 13:10:04
-------------------------------------------
(u'for', 1)
(u'https://t.co/KHSZHDYdH4', 1)
(u'Yea', 1)
(u'with', 1)
(u'one', 1)
(u'confused', 1)
(u'me', 1)
(u'real', 1)
(u'they', 1)
(u'that', 1)
-------------------------------------------
Time: 2017-11-27 13:10:06
-------------------------------------------
-------------------------------------------
Time: 2017-11-27 13:10:08
-------------------------------------------
-------------------------------------------
Time: 2017-11-27 13:10:10
-------------------------------------------
(u'take', 1)
(u'like', 1)
(u"It's", 1)
(u'football?!', 1)
(u'top', 1)
(u'Clemson', 1)
(u'able', 1)
(u'loss...', 1)
(u'25', 1)
(u'10', 2)
...
-------------------------------------------
Time: 2017-11-27 13:10:12
-------------------------------------------
-------------------------------------------
Time: 2017-11-27 13:10:14
-------------------------------------------
(u'Clemson/SC', 1)
(u'State\n9.', 1)
(u'control', 1)
(u'RT', 1)
(u'field.DID', 1)
(u'game', 1)
(u'Tennessee.The', 1)
(u'@GamecocksOnline', 1)
(u'from', 1)
(u'State\u2026', 1)
...
-------------------------------------------
Time: 2017-11-27 13:10:16
-------------------------------------------
(u'State\n9.', 1)
(u'RT', 1)
(u'State\u2026', 1)
(u'Penn', 1)
(u'@SportsCenter:', 1)
(u'Georgia\n7.', 1)
(u'Miami\n8.', 1)
(u'Oklahoma\n3.', 1)
(u'AP', 1)
(u'Poll:\n1.', 1)
...
-------------------------------------------
Time: 2017-11-27 13:10:18
-------------------------------------------
-------------------------------------------
Time: 2017-11-27 13:10:20
-------------------------------------------
(u'lit', 1)
(u'\U0001f436\U0001f3c8\u2764\ufe0f\U0001f43e', 1)
(u'https://t.co/lvVyi6Kzs3', 1)
(u'it\u2019s', 1)
(u'Saturday', 1)
-------------------------------------------
Time: 2017-11-27 13:10:22
-------------------------------------------
(u'', 1)
(u'14\nOklahoma', 1)
(u'38\n\nAll-Americans:\n\nClemson', 1)
(u'RT', 1)
(u'23\nOklahoma', 1)
(u'winners:\n\nClemson\u2026', 1)
(u'63\n\nHeisman', 1)
(u'10-win', 1)
(u'seasons:\n\nClemson', 1)
(u'@blinkinriley:', 1)
...
-------------------------------------------
Time: 2017-11-27 13:10:24
-------------------------------------------
-------------------------------------------
Time: 2017-11-27 13:10:26
-------------------------------------------
(u'field', 1)
(u'own', 1)
(u'stuff', 1)
(u'throwing', 1)
(u'game?', 1)
(u'Clemson', 1)
(u'field\n\nDo', 1)
(u'remember', 1)
(u'in', 1)
(u'not', 1)
...
-------------------------------------------
Time: 2017-11-27 13:10:28
-------------------------------------------
-------------------------------------------
Time: 2017-11-27 13:10:30
-------------------------------------------
(u'Swinney', 1)
(u'under', 1)
(u'Tigers', 1)
(u'Clemson', 1)
(u'https://t.co/vC5yT8hBaN', 1)
(u'Looking', 1)
(u'game', 1)
(u'Dabo', 1)
(u'at', 1)
(u'in', 1)
...
-------------------------------------------
Time: 2017-11-27 13:10:32
-------------------------------------------
-------------------------------------------
Time: 2017-11-27 13:10:34
-------------------------------------------
(u'State\nMiami\nOklahoma\nTexas\nOklahoma', 1)
(u'', 1)
(u'@MitchMossRadio:', 1)
(u'top', 1)
(u'State\nOhio', 1)
(u'State\nMichigan', 1)
(u'RT', 1)
(u'State\nU\u2026', 1)
(u'in', 1)
(u'miami', 1)
...
-------------------------------------------
Time: 2017-11-27 13:10:36
-------------------------------------------
-------------------------------------------
Time: 2017-11-27 13:10:38
-------------------------------------------
(u'talking', 1)
(u'game\u2026', 1)
(u'last', 1)
(u'2015:', 1)
(u'Hunter', 1)
(u'before', 1)
(u'them', 1)
(u'smack', 1)
(u'game', 1)
(u'remember', 1)
...
-------------------------------------------
Time: 2017-11-27 13:10:40
-------------------------------------------
-------------------------------------------
Time: 2017-11-27 13:10:42
-------------------------------------------
-------------------------------------------
Time: 2017-11-27 13:10:44
-------------------------------------------
-------------------------------------------
Time: 2017-11-27 13:10:46
-------------------------------------------
-------------------------------------------
Time: 2017-11-27 13:10:48
-------------------------------------------
---------------------------------------------------------------------------
KeyboardInterrupt Traceback (most recent call last)
<ipython-input-8-a854cf99126a> in <module>()
17 counts.pprint()
18 ssc.start()
---> 19 ssc.awaitTermination()
/home/lngo/software/spark-2.1.0-bin-hadoop2.7/python/pyspark/streaming/context.pyc in awaitTermination(self, timeout)
204 """
205 if timeout is None:
--> 206 self._jssc.awaitTermination()
207 else:
208 self._jssc.awaitTerminationOrTimeout(int(timeout * 1000))
/home/lngo/software/spark-2.1.0-bin-hadoop2.7/python/lib/py4j-0.10.4-src.zip/py4j/java_gateway.py in __call__(self, *args)
1129 proto.END_COMMAND_PART
1130
-> 1131 answer = self.gateway_client.send_command(command)
1132 return_value = get_return_value(
1133 answer, self.gateway_client, self.target_id, self.name)
/home/lngo/software/spark-2.1.0-bin-hadoop2.7/python/lib/py4j-0.10.4-src.zip/py4j/java_gateway.py in send_command(self, command, retry, binary)
881 connection = self._get_connection()
882 try:
--> 883 response = connection.send_command(command)
884 if binary:
885 return response, self._create_connection_guard(connection)
/home/lngo/software/spark-2.1.0-bin-hadoop2.7/python/lib/py4j-0.10.4-src.zip/py4j/java_gateway.py in send_command(self, command)
1026
1027 try:
-> 1028 answer = smart_decode(self.stream.readline()[:-1])
1029 logger.debug("Answer received: {0}".format(answer))
1030 if answer.startswith(proto.RETURN_MESSAGE):
/software/anaconda/4.2.0/lib/python2.7/socket.pyc in readline(self, size)
449 while True:
450 try:
--> 451 data = self._sock.recv(self._rbufsize)
452 except error, e:
453 if e.args[0] == EINTR:
KeyboardInterrupt:
To stop the Spark job, call sc.stop()
In [9]:
ssc.stop()
sc.stop()
-------------------------------------------
Time: 2017-11-27 13:10:52
-------------------------------------------
In [ ]:
Content source: linhbngo/cpsc-4770_6770
Similar notebooks: