In [1]:

    

sc


    

    
Out[1]:





<pyspark.context.SparkContext at 0x7f51ffe5c290>

In [3]:

    

def sparkcontext_info(sc):
    settings = []
    
    settings.append("Spark Configuration\n")
    settings.append("=" * 72 + "\n")
    for name, value in sc._conf.getAll():
        if len(value) > 68:
            value = value[:32] + " .. " + value[-32:]
        settings.append("{:<30}\n    {:>}\n".format(name, value))
    settings.append("=" * 72 + "\n")
    
    return "".join(settings)

print(sparkcontext_info(sc))


    

    




Spark Configuration
========================================================================
spark.executor.extraLibraryPath
    /opt/cloudera/parcels/CDH-5.3.1- .. 5.3.1.p0.5/lib/hadoop/lib/native
spark.executor.memory         
    2g
spark.driver.extraLibraryPath 
    /opt/cloudera/parcels/CDH-5.3.1- .. 5.3.1.p0.5/lib/hadoop/lib/native
spark.executor.instances      
    8
spark.serializer.objectStreamReset
    100
spark.eventLog.enabled        
    true
spark.yarn.historyServer.address
    http://c1-master.ec2.internal:18088
spark.cores.max               
    8
spark.rdd.compress            
    True
spark.app.name                
    PySparkShell
spark.eventLog.dir            
    hdfs://c1-master.ec2.internal:8020/user/spark/applicationHistory
spark.master                  
    yarn-client
========================================================================

In [4]:

    

obs = sc.textFile("/user/schiefjm/weather/gsod/1929")
obs.count()


    

    
Out[4]:





2102

In [ ]: