In [1]:
%pylab inline
In [47]:
# Make sure files are there
!ls test-*-*.e*
In [37]:
%%file maketxt.awk
BEGIN { STATE = 0 }
/-------.*----/ { STATE=STATE+1; next }
STATE==1 && /Job name.*/ {
split($3, a, /-/)
print a[1]
print a[2]
next
}
STATE==2 && /Nresolved.*/ { print $3 ; next; }
STATE==2 && /radix sort.*/ { print $3; STATE=3 ; next }
/Application.*/ {STATE=4; next }
STATE==3 { print $2 }
STATE==4 { }
In [38]:
%%file maketxt.py
import numpy
from sys import stdin, stdout
x = numpy.loadtxt(stdin)
x = x.reshape(-1, 11)
numpy.savetxt(stdout, x, fmt='%.8g')
In [39]:
!for i in test-*-*.e*; do awk -f 'maketxt.awk' $i; done |python maketxt.py > timing-full.txt
In [26]:
# Grepped from BlueTides log files with
#grep -B 3 'Group catalogues saved. took' *.e* | awk 'NR % 5==1{printf("%d ", $4)}; NR % 5 ==4{print $6;}'
# then marked runs before the switch to MP-sort with 0 and after 1. (BlueTides log files contains too much junk to be included here)
In [27]:
%%file FOF-time.txt
0 131518536 1182.93
0 337978816 1443.26
0 809041153 1712.31
0 1797890470 2413.39
1 3741494450 1708.57
1 6251709054 1687.02
1 7234944269 1820.08
1 13145107052 2540.98
1 22349137220 2754.68
In [19]:
import numpy
from matplotlib.figure import Figure
from matplotlib.backends.backend_agg import FigureCanvasAgg
g = numpy.loadtxt('timing-full.txt', )
legends = ['Total', 'LocalSort1', 'FindEdges', 'SolveLayout', 'Exchange', 'LocalSort2']
# converto ranks
g[:, 0] *= 32
labels = {100000:'100K',
1000000: '1M',
10000000: '10M'}
print g.shape
for load in [100000, 1000000, 10000000]:
f = g[numpy.int32(g[:, 1]) == load, :]
arg = f[:, 0].argsort()
f = f[arg]
print labels[load], ' & ', '&'.join([' %3.4g ' % a for a in f[:, 3]]), r'\\'
In [14]:
# used by plotting cells.
def plotstrong(ax, x, f, xlim, title):
""" plots a data column """
Total = f[:, 3]
print f[:, 0]
LocalSort1 = f[:, 4]
FindEdges = f[:, 5] + f[:, 6]
SolveLayout = f[:, 7] + f[:, 8]
Exchange = f[:, 9]
LocalSort2= f[:, 10]
for row, marker in zip(legends, 'o>^*sd'):
ax.plot(x, locals()[row], marker+'-', label=row,
markersize=8,
markeredgewidth=1, lw=1)
ax.set_xscale('log')
ax.set_yscale('log')
ax.set_xlim(*xlim)
ax.set_ylim(1e-2, .4e4)
In [40]:
# This figure did not make it into the paper
# Alltoall Time per message as a function of message size.
#
fig = Figure()
canvas = FigureCanvasAgg(fig)
ax = fig.add_subplot(111)
print g.shape
for msgcount in numpy.unique(g[:, 0]):
f = g[numpy.int32(g[:, 0]) == msgcount]
msgsize = 1.0 * f[:, 1] / f[:, 0]
ax.plot(msgsize, f[:, 9] / msgcount, 'o', label='%g' % msgcount)
ax.set_yscale('log')
ax.set_xscale('log')
ax.legend(ncol=2, loc='upper left')
ax.set_ylim(1e-3, 1e-2)
ax.set_xlim(1e-1 * 1, .3e5 * 1)
MSGsplit = 1536 * 1, 0.002
style = {'color': 'gray', 'lw':2, 'ls':'--'}
ax.axvline(MSGsplit[0], **style)
ax.axhline(MSGsplit[1], **style)
ax.set_ylabel('Average Alltoallv Time per Message [Sec]')
ax.set_xlabel('Average Alltoallv Message Size [8 Bytes]')
ax.annotate("Region I", (0.1, 0.45), xycoords='axes fraction')
ax.annotate("Region II", (0.1, 0.05), xycoords='axes fraction')
ax.annotate("Region III", (0.8, 0.45), xycoords='axes fraction')
fig.savefig('exchange-by-msgsize.png')
fig
Out[40]:
In [21]:
# Weak scaling figures
fig, axes = subplots(1, 3, sharex=True, sharey=True, squeeze=True, figsize=(8, 3))
labels = {100000:'100K',
1000000: '1M',
10000000: '10M'}
for ax, load in zip(axes, [100000, 1000000, 10000000]):
f = g[numpy.int32(g[:, 1]) == load, :]
arg = f[:, 0].argsort()
f = f[arg]
plotstrong(ax, f[:, 0], f, xlim=(.5e3,
4e5),
title="Weak Scaling with %d items per MPI rank" % load
)
ax.set_title(x=0.95, y=0.01, label=labels[load], ha='right', fontsize='small')
l = ax.get_legend_handles_labels()
fig.legend(*l, ncol=6, loc='upper center', fontsize='small', numpoints=1, frameon=True)
axes[1].set_xlabel('Number of MPI ranks')
axes[0].set_ylabel("Walltime [sec]")
#ax.set_title(title)
fig.tight_layout()
fig.savefig('weak.png', dpi=200)
In [91]:
# Strong scaling figures
fig, axes = subplots(2, 3, sharex=True, sharey=True, squeeze=True, figsize=(8, 5.5))
for ax, np in zip(numpy.reshape(axes, -1), [2048, 4096, 8192, 16384, 80000, 160000]):
f = g[numpy.int32(g[:, 0]) == np, :]
arg = f[:, 1].argsort()
f = f[arg]
plotstrong(ax, f[:, 1], f, xlim=(.5e5,
5e7),
title="Strong scaling with %d MPI ranks" % np
)
ax.set_title(x=0.95, y=0.01, label='%d Ranks' % np, ha='right', fontsize='small')
l = ax.get_legend_handles_labels()
fig.legend(*l, ncol=6, loc='upper center', fontsize='small', numpoints=1, frameon=True)
axes[1, 1].set_xlabel('Number of Items per rank')
axes[0, 0].set_ylabel("Walltime [sec]")
#ax.set_title(title)
axes[1, 0].set_ylabel("Walltime [sec]")
fig.tight_layout()
fig.savefig('strong.png', dpi=200)
In [28]:
# improvements in BlueTides: before and after.
import numpy
from matplotlib.figure import Figure
from matplotlib.backends.backend_agg import FigureCanvasAgg
gg = numpy.loadtxt('FOF-time.txt', unpack=False)
old = gg[gg[:, 0]==0]
new = gg[gg[:, 0]==1]
print 700*1e9 / 1500.
def io(x):
t = x / 466666666.667
t = t.clip(1000, 1000)
return t
print io(new[-1, 1])
print gg[:, 1] / 81000. ** 2
figure = Figure((8, 3), dpi=200)
canvas = FigureCanvasAgg(figure)
ax = figure.add_subplot(111)
ax.plot(old[:, 1], old[:, 2], 'b-', lw=2)
ax.plot(old[:, 1], old[:, 2] - io(old[:, 1]), 'bo ', label='Merge-sort')
#plot(old[:, 1], old[:, 2], 'o ')
#plot(old[:, 1], io(old[:, 1]))
ax.plot(new[:, 1], new[:, 2], 'g-', lw=2)
ax.plot(new[:, 1], new[:, 2] - io(new[:, 1]), 'gs ', label='MP-Sort')
#plot(new[:, 1], new[:, 2], 'o ')
#plot(new[:, 1], io(new[:, 1]))
fakex = new[0, 1]
fakey = 4000.
#ax.plot(fakex, fakey, '^ ', label='Est. Merge-sort')
x = logspace(8, 11)
ax.set_xlim(1e8, 5e10)
ax.plot(x, x * 1e-6 * numpy.log(x) / 20, '--', label='N log N', color='gray')
ax.set_xscale('log')
ax.set_yscale('log')
ax.set_ylim(100, 1e4)
ax.legend(loc='upper center', ncol=4, numpoints=1, fontsize='small')
ax.set_xlabel('Number of Sorted Particles')
ax.set_ylabel('Sorting Walltime [sec] (Recovered)')
figure.tight_layout()
figure.savefig('MPSortInBlueTides.png', dpi=200)
display(figure)
In [ ]: