In [3]:

    

# import pyspark
# from pyspark import SparkConf
# from pyspark import SparkContext

# sc = SparkContext(appName="Jon's PySpark")

sc


    

    




---------------------------------------------------------------------------
NameError                                 Traceback (most recent call last)
<ipython-input-3-ce17cdb9128a> in <module>()
      5 # sc = SparkContext(appName="Jon's PySpark")
      6 
----> 7 sc

NameError: name 'sc' is not defined

In [5]:

    

from utils import header
print(header.create_header(sc))


    

    




---------------------------------------------------------------------------
NameError                                 Traceback (most recent call last)
<ipython-input-5-f92f434efc2f> in <module>()
      1 from utils import header
----> 2 print(header.create_header(sc))

NameError: name 'sc' is not defined

In [2]:

    

ob_rdds = []
for month in range(1, 13):
    dtg = "1970{:02}".format(month)
    obs = sc.textFile("/user/schiefjm/weather/gsod/1970")\
            .filter(lambda line: "STN" not in line)\
            .filter(lambda line: "034700" in line)\
            .filter(lambda line: dtg in line)
    ob_rdds.append([dtg, obs.take(obs.count())])


    

In [51]:

    

def get_stations(year):
    """ Given a year from 1929 to 2009 inclusive returns a set of the 
    stations in the data set for that year. 
    
    :input:
        year: a string or integer between 1929 and 2009 inclusive
        
    :output:
        returns a list of tuples of the station id and number of obs for
        said station
    """
    # validate the year
    year = int(year)
    if 1929 <= year <= 2009:
        year = str(year)
    else:
        print("Please enter a valid year from 1929 to 2009.")
        return 0
    
    # get the stations and number of obs for each
    return sc.textFile("/user/schiefjm/weather/gsod/" + year)\
             .filter(lambda line: "STN" not in line)\
             .map(lambda line: (line.split()[0], 1))\
             .reduceByKey(lambda x, y: x + y)\
             .collect()

for year in range(1929, 1932):
    stations = get_stations(year)
    total_obs = 0
    for station in stations:
        total_obs += int(station[1])
    print("\n{} has {} stations and {} total observations"\
          .format(year, len(stations), total_obs))
    print("-" * 70)
    for station in stations[:5]:
        print("{}\t{}".format(station[0], station[1]))


    

    




1929 has 21 stations and 2081 total observations
----------------------------------------------------------------------
038040	91
039730	89
990061	44
033790	148
037950	149

1930 has 23 stations and 7285 total observations
----------------------------------------------------------------------
039530	363
030750	357
030910	190
038640	361
035590	1

1931 has 31 stations and 9913 total observations
----------------------------------------------------------------------
105130	196
030260	358
122050	359
104160	144
037950	352

In [ ]:

    

counts = file.flatMap(lambda line: line.split(" ")) \
             .map(lambda word: (word, 1)) \
             .reduceByKey(lambda a, b: a + b)


    

In [1]:

    

import this


    

    




The Zen of Python, by Tim Peters

Beautiful is better than ugly.
Explicit is better than implicit.
Simple is better than complex.
Complex is better than complicated.
Flat is better than nested.
Sparse is better than dense.
Readability counts.
Special cases aren't special enough to break the rules.
Although practicality beats purity.
Errors should never pass silently.
Unless explicitly silenced.
In the face of ambiguity, refuse the temptation to guess.
There should be one-- and preferably only one --obvious way to do it.
Although that way may not be obvious at first unless you're Dutch.
Now is better than never.
Although never is often better than *right* now.
If the implementation is hard to explain, it's a bad idea.
If the implementation is easy to explain, it may be a good idea.
Namespaces are one honking great idea -- let's do more of those!

In [2]:

    

from __future__ import braces


    

    




  File "<ipython-input-2-2aebb3fc8ecf>", line 1
    from __future__ import braces
SyntaxError: not a chance

In [ ]: