In [ ]:
%dep
z.load("alvsanand:spark-generic-connector:0.2.0-spark_2x-s_2.11")
In [ ]:
import es.alvsanand.sgc.ftp.{FTPCredentials, FTPSlot, ProxyConfiguration}
import es.alvsanand.sgc.ftp.normal.{FTPSgcConnectorFactory, FTPParameters}
import org.apache.spark.streaming.sgc._
In [ ]:
val parameters = FTPParameters("HOST", PORT, "DIRECTORY", FTPCredentials("USER", Option("PASSWORD"))
In [ ]:
val parameters = FTPParameters("HOST", PORT, "DIRECTORY", FTPCredentials("USER", Option("PASSWORD"), activeMode = true)
In [ ]:
val parameters = FTPParameters("HOST", PORT, "DIRECTORY", FTPCredentials("USER", Option("PASSWORD")),
proxy = Option(ProxyConfiguration("PROXYHOST", 80, Option("PROXY_USER"),
Option("PROXY_PASSWORD"))
In [ ]:
val rdd = sc.createSgcRDD(FTPSgcConnectorFactory, parameters)
In [ ]:
rdd.partitions.map(_.asInstanceOf[SgcRDDPartition[FTPSlot]].slot)
rdd.take(10).foreach(println)