In [42]:

    

import os
import pandas as pd
import re
import numpy as np
import itertools
import csv

directory = os.path.join("output")
combinedOutput = "output_combined.csv"

count = 0
pattern = re.compile("output")
for root,dirs,files in os.walk(directory):
    for file in files:
        if  not pattern.match(file): 
            continue
        fileAppend = "output/" + file
        if count == 0: 
            totalData = pd.read_csv(fileAppend)
            count +=1
        else:
            loadData = pd.read_csv(fileAppend)
            if not loadData.empty:
                totalData = totalData.append(loadData)


    

In [43]:

    

Ns, Ks, Ds = np.unique(totalData["n"].values),np.unique(totalData["k"].values), np.unique(totalData["d"].values)
alg, nodes, tasks, gpu = np.unique(totalData["algorithm"].values),np.unique(totalData["Nodes N"].values), np.unique(totalData["Nodes n"].values), np.unique(totalData["GPUs"].values)


    

In [44]:

    

newArray = [["Algorithm","N", "D","K","processes","problemSize","time", "GFLOPs"]]
for N, D, K, T, A in [x for x in list(itertools.product(Ns, Ds, Ks, tasks, alg))]:
    problemParams = [A,N,D,K,T, N*D*K]
    whichVals = (totalData["n"] == N) & (totalData["d"] == D) & (totalData["k"] == K) & (totalData["algorithm"] == A) & (totalData["Nodes n"] == T)
    if not np.any(whichVals): continue
    ops = N*K*(D*3-1) * np.mean(totalData["convergence"][whichVals].values)/1E9
    time = np.mean(totalData["time"][whichVals].values)
    newArray.append(problemParams +  [time] + [ops/time])


    

In [45]:

    

with open(combinedOutput, 'w') as f:
        writer = csv.writer(f, delimiter=',')
        writer.writerows([o for o in newArray])
        f.close()


    

In [ ]:

    




    

In [ ]: