Scala

Version



In [1]:

    
util.Properties.versionMsg









    





Intitializing Scala interpreter ...






    





Spark Web UI available at http://docker-jupyter:4040
SparkContext available as 'sc' (version = 2.4.5, master = local[*], app id = local-1583323961465)
SparkSession available as 'spark'







    Out[1]:





res0: String = Scala library version 2.11.12 -- Copyright 2002-2017, LAMP/EPFL

Spark



In [2]:

    
import org.apache.spark._
import org.apache.spark.sql._

var spark = SparkSession.builder().master("local[*]").appName("spark")
.config("spark.driver.memory", "8g")
.config("spark.executor.memory", "8g")
.config("spark.python.worker.memory", "8g")
.getOrCreate()









    Out[2]:





import org.apache.spark._
import org.apache.spark.sql._
spark: org.apache.spark.sql.SparkSession = org.apache.spark.sql.SparkSession@56809094



In [3]:

    
val df = spark.read.json("/usr/local/spark/examples/src/main/resources/people.json")
df.show









    



+----+-------+
| age|   name|
+----+-------+
|null|Michael|
|  30|   Andy|
|  19| Justin|
+----+-------+







    Out[3]:





df: org.apache.spark.sql.DataFrame = [age: bigint, name: string]



In [4]:

    
spark.stop()



In [ ]: