This example shows how Toree enables sharing of the SQLContext across the variety of languages that it supports (Scala, Python, R, SQL). To demostrate, this notebook will load data using one language and read it from another. Refer to the Spark documentation for details about the DataFrame and SQL APIs.
In [ ]:
import org.apache.spark.{SparkConf}
val conf = new SparkConf()
conf.setAppName("Scala Spark Test")
kernel.createSparkContext(conf)
In [ ]:
val people = spark.read.json("/opt/datasets/people.json")
people.registerTempTable("people")
people.show()
In [ ]:
%%PySpark
people= spark.table("people")
people.show()
In [ ]:
%%PySpark
cars = spark.read.json("/opt/cars.json")
cars.registerTempTable("cars")
cars.show()
In [ ]:
val cars = spark.table("cars")
cars.show()
In [ ]:
%%sql
select * from cars where manufacturer == 'Audi'