In [ ]:

    

import gc
from copy import deepcopy

%matplotlib inline 
from matplotlib import pyplot as plt
import numpy as np

try:
    from run_for_memory_profile import run_memory_profile, generate_fits_files
except ImportError:
    raise ImportError('Please install memory_profiler before running this notebook.')

from ccdproc.version import get_git_devstr
from astropy import __version__ as apy_version


    

In [ ]:

    

print('Astropy version: ', apy_version)


    

In [ ]:

    

image_size = 4000   # Square image, so 4000 x 4000
num_files = 10
sampling_interval = 0.01 # sec
memory_limit = 1000000000  # bytes, roughly 1GB

commit = get_git_devstr(sha=True)[:7]
print(commit)


    

In [ ]:

    

generate_fits_files(num_files, size=image_size)


    

In [ ]:

    

runs = {
    'average': {
        'times': [],
        'memory': [],
        'image_size': 0.
    },
    'median': {
        'times': [],
        'memory': [],
        'image_size': 0.
    },
    'sum': {
        'times': [],
        'memory': [],
        'image_size': 0.
    }
}
runs_clip = deepcopy(runs)


    

In [ ]:

    

_, _ = run_memory_profile(num_files, sampling_interval, size=image_size, 
                          memory_limit=memory_limit, combine_method='average')


    

In [ ]:

    

n_repetitions = 4


    

In [ ]:

    

def run_them(runs, clipping=False):
    for combine_method in runs.keys():
        for _ in range(n_repetitions):
            mem_use, img_size = run_memory_profile(num_files, sampling_interval, size=image_size, 
                                                   memory_limit=memory_limit, combine_method=combine_method,
                                                   sigma_clip=clipping)
            gc.collect()
            runs[combine_method]['times'].append(np.arange(len(mem_use)) * sampling_interval)
            runs[combine_method]['memory'].append(mem_use)
            runs[combine_method]['image_size'] = img_size
            runs[combine_method]['memory_limit'] = memory_limit
            runs[combine_method]['clipping'] = clipping


    

In [ ]:

    

run_them(runs)


    

In [ ]:

    

styles = ['solid', 'dashed', 'dotted']


    

In [ ]:

    

plt.figure(figsize=(20, 10))

for idx, method in enumerate(runs.keys()):
    style = styles[idx % len(styles)]
    for i, data in enumerate(zip(runs[method]['times'], runs[method]['memory'])):
        time, mem_use = data 
        if i == 0:
            label = 'Memory use in {} combine (repeated runs same style)'.format(method)
            alpha = 1.0
        else:
            label = ''
            alpha = 0.4
        plt.plot(time, mem_use, linestyle=style, label=label, alpha=alpha)

plt.vlines(-40 * sampling_interval, mem_use[0], mem_use[0] + memory_limit/1e6, colors='red', label='Memory use limit')
plt.vlines(-20 * sampling_interval, mem_use[0], mem_use[0] + runs[method]['image_size']/1e6, label='size of one image')

plt.grid()
clipped = 'ON' if runs[method]['clipping'] else 'OFF'

plt.title('ccdproc commit {}; {} repetitions per method; sigma_clip {}'.format(commit, n_repetitions, clipped),
          fontsize=20)
plt.xlabel('Time (sec)', fontsize=20)
plt.ylabel('Memory use (MB)', fontsize=20)

plt.legend(fontsize=20)
plt.savefig('commit_{}_reps_{}_clip_{}_memlim_{}GB.png'.format(commit, n_repetitions, clipped, memory_limit/1e9))


    

In [ ]:

    

run_them(runs_clip, clipping=True)


    

In [ ]:

    

plt.figure(figsize=(20, 10))

for idx, method in enumerate(runs_clip.keys()):
    style = styles[idx % len(styles)]
    for i, data in enumerate(zip(runs_clip[method]['times'], runs_clip[method]['memory'])):
        time, mem_use = data 
        if i == 0:
            label = 'Memory use in {} combine (repeated runs same style)'.format(method)
            alpha = 1.0
        else:
            label = ''
            alpha = 0.4
        plt.plot(time, mem_use, linestyle=style, label=label, alpha=alpha)

plt.vlines(-40 * sampling_interval, mem_use[0], mem_use[0] + memory_limit/1e6, colors='red', label='Memory use limit')
plt.vlines(-20 * sampling_interval, mem_use[0], mem_use[0] + runs_clip[method]['image_size']/1e6, label='size of one image')

plt.grid()
clipped = 'ON' if runs_clip[method]['clipping'] else 'OFF'

plt.title('ccdproc commit {}; {} repetitions per method; sigma_clip {}'.format(commit, n_repetitions, clipped),
          fontsize=20)
plt.xlabel('Time (sec)', fontsize=20)
plt.ylabel('Memory use (MB)', fontsize=20)

plt.legend(fontsize=20)
plt.savefig('commit_{}_reps_{}_clip_{}_memlim_{}GB.png'.format(commit, n_repetitions, clipped, memory_limit/1e9))