In [1]:

    

sc


    

    
Out[1]:





<pyspark.context.SparkContext at 0x7f38131f87f0>

In [58]:

    

nodes = "c18node14.acis.ufl.edu,c18node2.acis.ufl.edu,c18node6.acis.ufl.edu,c18node10.acis.ufl.edu,c18node12.acis.ufl.edu"
index = "idigbio"
#{"bool": 
#    {"must": [
#        {"term":{"recordset":"95773ebb-2f5f-43f0-a652-bfd8d5f4707a"}}
#        ]
#    }
#},
query = """{"query": {
    "bool": {
        "must": [
            {
                "term": {"recordset": "95773ebb-2f5f-43f0-a652-bfd8d5f4707a"}
            }
        ]
}}}"""


    

In [59]:

    

field_set = ["uuid", "licenselogourl"]
fields = ",".join(field_set)


    

In [60]:

    

df = (sqlContext.read.format("org.elasticsearch.spark.sql")
    .option("es.read.field.include", fields)
    .option("es.nodes", nodes)
    .option("es.query", query)
    .load("{0}/mediarecords".format(index))
)


    

In [61]:

    

print(df.count())
df.printSchema()


    

    




55
root
 |-- licenselogourl: string (nullable = true)
 |-- uuid: string (nullable = true)

In [63]:

    

df.head(10)


    

    
Out[63]:





[Row(licenselogourl='https://i.creativecommons.org/l/by-nc-sa/4.0/88x31.png', uuid='586f354a-b85b-4a14-865a-6a609df94a0c'),
 Row(licenselogourl='https://i.creativecommons.org/l/by-nc-sa/4.0/88x31.png', uuid='1b5dd966-623b-4304-8d63-a9b757b4b608'),
 Row(licenselogourl='https://i.creativecommons.org/l/by-nc-sa/4.0/88x31.png', uuid='2269ae53-03aa-463e-97ac-ee5afe05660e'),
 Row(licenselogourl='https://i.creativecommons.org/l/by-nc-sa/4.0/88x31.png', uuid='09077a63-88a9-476c-a889-7cf2c43f16ee'),
 Row(licenselogourl='https://i.creativecommons.org/l/by-nc-sa/4.0/88x31.png', uuid='17e3d421-3122-4839-86fd-4748edb003b2'),
 Row(licenselogourl='https://i.creativecommons.org/l/by-nc-sa/4.0/88x31.png', uuid='196f0c7a-224f-475c-94de-daf474d5cda2'),
 Row(licenselogourl='https://i.creativecommons.org/l/by-nc-sa/4.0/88x31.png', uuid='2822622f-3d03-41cc-9080-d42af9fefc37'),
 Row(licenselogourl='https://i.creativecommons.org/l/by-nc-sa/4.0/88x31.png', uuid='4160a3e0-e9c0-452b-a29c-a83ebfc40c75'),
 Row(licenselogourl='https://i.creativecommons.org/l/by-nc-sa/4.0/88x31.png', uuid='5ae928ab-d713-43e1-bf85-326f7d702613'),
 Row(licenselogourl='https://i.creativecommons.org/l/by-nc-sa/4.0/88x31.png', uuid='5c35c040-90f8-4d47-a562-9da4ee90f521')]