Testing Spark with Python 3

In [1]:

    

import sys
print (sys.version)
import os
os.environ['PYSPARK_PYTHON'] = 'python3'


    

    




3.4.3 (default, Oct 14 2015, 20:33:09) 
[GCC 4.8.4]

In [2]:

    

import pyspark
sc = pyspark.SparkContext('local[*]')


    

In [3]:

    

sc.master


    

    
Out[3]:





'local[*]'

In [4]:

    

rdd = sc.parallelize(range(1, 1000))


    

In [5]:

    

rdd.count()


    

    
Out[5]:





999

In [6]:

    

%matplotlib inline


    

In [7]:

    

# Check matplotlib plotting
import matplotlib.pyplot as plt
import matplotlib.cm as cm
from math import log

# function for generating plot layout
def preparePlot(xticks, yticks, figsize=(10.5, 6), hideLabels=False, gridColor='#999999', gridWidth=1.0):
    plt.close()
    fig, ax = plt.subplots(figsize=figsize, facecolor='white', edgecolor='white')
    ax.axes.tick_params(labelcolor='#999999', labelsize='10')
    for axis, ticks in [(ax.get_xaxis(), xticks), (ax.get_yaxis(), yticks)]:
        axis.set_ticks_position('none')
        axis.set_ticks(ticks)
        axis.label.set_color('#999999')
        if hideLabels: axis.set_ticklabels([])
    plt.grid(color=gridColor, linewidth=gridWidth, linestyle='-')
    map(lambda position: ax.spines[position].set_visible(False), ['bottom', 'top', 'left', 'right'])
    return fig, ax

# generate layout and plot data
x = range(1, 50)
y = [log(x1 ** 2) for x1 in x]
fig, ax = preparePlot(range(5, 60, 10), range(0, 12, 1))
plt.scatter(x, y, s=14**2, c='#d6ebf2', edgecolors='#8cbfd0', alpha=0.75)
ax.set_xlabel(r'$range(1, 50)$'), ax.set_ylabel(r'$\log_e(x^2)$')
pass


    

In [8]:

    

import matplotlib.pyplot as plt
import numpy as np


    

In [9]:

    

x = np.linspace(0, 3*np.pi, 500)
plt.plot(x, np.sin(x**2))
plt.title('A simple chirp');