In [ ]:

    

import os, csv, datetime
import numpy as np
from IPython.parallel import Client


    

In [ ]:

    

# Code to run in parallel
def pe(filename):
    import numpy as np
    import csv, os, socket
    with open(filename) as csvfile:
        readcsv = csv.reader(csvfile, delimiter=",")
        data = []
        xint = []
        for row in readcsv:
            amp = row[0]
            data.append(float(amp))
    TS = np.array(data)

    perms = dict()
    m=5
    t=2
    for a in range(len(TS) - t*(m-1)):
        v = tuple(np.argsort(TS[a:(a + t*(m-1) + 1):t]))
        if v in perms:
            perms[v] += 1
        else:
            perms[v] = 1

    c = np.array(list(perms.values()))
    p = c / float(np.sum(c))
    pe = -np.sum(np.dot(p, np.log(p)))
    pe = pe / np.log(np.math.factorial(m))
    with open("/home/data/laser/pe.log","a") as fw:
        fw.writelines("{0},{1},{2},{3}\n".format(socket.gethostname(), os.getpid(), filename, pe))
    return pe


    

In [ ]:

    

# Code that distribute jobs
def main():
    profile_dir = "/home/data/ipython"
    results_file = "/home/data/laser/pe_results.csv"
    logs_file = "/home/data/laser/pe.log"

    # Reset logs
    with open(logs_file,"w") as fw:
        fw.writelines("host_name,process_id,file_name,pe_result\n")

    # Parallel balanced
    rc = Client(profile_dir=profile_dir)
    lview = rc.load_balanced_view()

    t1 = datetime.datetime.now()
    FB = np.arange(0., 351.)
    INJ = np.arange(0., 251.)

    list_pe = []
    files = []
    for aa in FB:
        for bb in INJ:
            filename = "/home/data/laser/ts/FB_%03d_INJ_%03d.csv" % (FB[aa], INJ[bb])
            if not os.path.exists(filename):
                continue
            files.append(filename)

    r = lview.map_async(pe, files)
    list_pe = r.get()

    t2 = datetime.datetime.now()
    print "Time : %s " % (t2-t1)
    print "Parallel process: {0}".format(len(rc.ids))
    print "Files processed: {0} ".format(len(list_pe))
    print ""
    
    # Save results to csv file
    with open(results_file, "w") as fw:
        fw.writelines(("pe\n"))
        for line in list_pe:
            fw.writelines("{0}\n".format(line))
    print "Results save to:", results_file
    print "Logs save to:", logs_file
    print "Results sample: ", list_pe[:10], "..."


    

In [ ]:

    

# Run code
main()


    

In [ ]:

    

# Check logs
logs_file = "/home/data/laser/pe.log"
fr = open(logs_file,"rb")
for line in fr:
    print line,


    

In [ ]: