In [ ]:
%dep
z.load("alvsanand:spark-generic-connector:0.2.0-spark_2x-s_2.11")
In [ ]:
import org.apache.spark.streaming.sgc._
import es.alvsanand.sgc.google.cloud_storage._
In [ ]:
val parameters = CloudStorageParameters("CREDENTIALS_ZIP_URL", "BUCKET_NAME")
In [ ]:
val rdd = sc.createSgcRDD(CloudStorageSgcConnectorFactory, parameters)
In [ ]:
rdd.partitions.map(_.asInstanceOf[SgcRDDPartition[CloudStorageSlot]].slot)
rdd.take(10).foreach(println)