Tutorial for running liknking_EC2 see: https://github.com/JonasWallin/linkingEC2/blob/master/script/running%20MPI4py.ipynb
In [1]:
%%bash
. ~/.bashrc
pip install --upgrade git+https://git@github.com/JonasWallin/linkingEC2
In [2]:
from linkingEC2 import LinkingHandler
from ConfigParser import ConfigParser
config = ConfigParser()
starfigconfig_folder = "/Users/jonaswallin/.starcluster/"
config.read(starfigconfig_folder + "config")
acess_key_id = config.get('aws info', 'aws_access_key_id' , 0)
aws_secret_key = config.get('aws info', 'aws_secret_access_key', 0)
aws_region_name = config.get('aws info', 'aws_region_name' , 0)
my_key_loc = config.get('key mykeyABC', 'key_location',0)
linker = LinkingHandler(aws_secret_access_key = aws_secret_key,
aws_access_key_id = acess_key_id,
aws_region_name = aws_region_name,
key_location = my_key_loc,
key_name = 'mykeyABC' )
In [3]:
start_cluster= False
spot_cluster = True
n_nodes = 1
type_node = 'c4.8xlarge'
if spot_cluster:
linker.connect_spot_instance()
elif start_cluster:
linker.start_cluster('ami-d05e75b8', type_node, ['linking_EC2'], n_nodes)
else:
linker.connect_cluster()
In [10]:
PACKAGES_APT = [' libatlas3-base',
'libatlas-base-dev',
'python-dev',
'openmpi-bin',
'libopenmpi-dev',
'python-numpy',
'python-sklearn',
'python-matplotlib',
'git',
'python-scipy',
'r-base',
'r-base-core']
PACKAGES_PIP = ['cython',
'mpi4py',
'simplejson',
'rpy2']
#Adding later version of R
#http://philipp-burckhardt.com/2014/05/25/installing-r-rstudio-on-ubuntu/
command = 'sudo add-apt-repository "deb http://cran.rstudio.com/bin/linux/ubuntu trusty/"'
linker.send_command_ssh(command = 'gpg --keyserver pgpkeys.mit.edu --recv-key 51716619E084DAB9')
linker.send_command_ssh(command = 'gpg -a --export 51716619E084DAB9 | sudo apt-key add -')
linker.send_command_ssh(command = command)
linker.apt_install(PACKAGES_APT)
#problem with memory installing scipy:
#http://naokiwatanabe.blogspot.se/2014/12/install-numpy-schipy-matplotlib-and-etc.html
linker.send_command_ssh(command = 'sudo /bin/dd if=/dev/zero of=/var/swap.1 bs=1M count=1024')
linker.send_command_ssh(command = 'sudo /sbin/mkswap /var/swap.1')
linker.send_command_ssh(command = 'sudo /sbin/swapon /var/swap.1')
linker.pip_install('-U scipy')
linker.send_command_ssh(command = 'sudo swapoff /var/swap.1')
linker.send_command_ssh(command = 'sudo sudo rm /var/swap.1')
linker.pip_install(PACKAGES_PIP)
linker.pip_install(['git+https://git@github.com/JonasWallin/BayesFlow'])
import os
os.system('say "your packages is downloaded"')
Out[10]:
In [11]:
#linker.send_command_ssh( command = 'rm ~/covs_.npy')
#linker.send_command_ssh( command = 'rm ~/means_.npy')
#linker.send_command_ssh( command = 'rm ~/weights_.npy')
#linker.send_command_ssh( command = 'rm ~/article_util.py')
#linker.send_command_ssh( command = 'rm ~/article_simulatedata.py')
#linker.send_command_ssh( command = 'rm ~/article_estimate_largerdata1_mpi.py')
linker.send_command_ssh( command = 'wget https://raw.githubusercontent.com/JonasWallin/BayesFlow/master/examples/article1/covs_.npy')
linker.send_command_ssh( command = 'wget https://raw.githubusercontent.com/JonasWallin/BayesFlow/master/examples/article1/means_.npy')
linker.send_command_ssh( command = 'wget https://raw.githubusercontent.com/JonasWallin/BayesFlow/master/examples/article1/weights_.npy')
linker.send_command_ssh( command = 'wget https://raw.githubusercontent.com/JonasWallin/BayesFlow/master/examples/article1/article_util.py')
linker.send_command_ssh( command = 'wget https://raw.githubusercontent.com/JonasWallin/BayesFlow/master/examples/article1/article_simulatedata.py')
linker.send_command_ssh( command = 'wget https://raw.githubusercontent.com/JonasWallin/BayesFlow/master/examples/article1/article_estimate_largerdata1_mpi.py')
Out[11]:
In [32]:
print( linker.get_ssh_login() )
In [13]:
import numpy as np
tot_process = np.sum([node['n_process'] for node in linker.nodes])
command = 'mpirun -hostfile nodefile -n %d python article_estimate_largerdata1_mpi.py'%(tot_process)
linker.send_command_ssh(nodes = 0, command = command)
linker.copy_files_from_node('simulation_result.npy')
linker.copy_files_from_node('mus_sim.npy')
linker.copy_files_from_node('sim_data.npy')
In [10]:
linker.terminate_cluster()
In [3]:
print(linker.conn.get_all_spot_instance_requests())
In [3]:
spot_instance = linker.conn.get_all_spot_instance_requests()
nodes = []
reservation = []
print(spot_instance)
for i,spot in enumerate(spot_instance):
print(spot.instance_id)
if spot.instance_id is not None:
res = linker.conn.get_all_instances(instance_ids=[spot.instance_id])
node_alias = 'node{0:03d}'.format(i+1)
reservation.append(res[0].instances[0])
public_dns = res[0].instances[0].public_dns_name
private_dns = res[0].instances[0].private_dns_name
private_ip_address = res[0].instances[0].private_ip_address
nodes.append({'name' :node_alias,
'public_dns' :public_dns,
'private_dns':private_dns,
'private_ip' :private_ip_address})
In [4]:
linker.reservation = reservation
linker.nodes = nodes
In [5]:
linker.test_ssh_in()
In [6]:
from linkingEC2.linkingEC2 import get_number_processes
get_number_processes(nodes = linker.nodes,
my_key = linker.my_key_location,
user = linker.user,
silent = linker.silent)
In [7]:
#copying the ssh keys to nodes
linker.copy_files_to_nodes(file_name = linker.my_key_location,
destination = '~/.ssh/id_rsa')
linker._ssh_disable_StrictHostKeyChecking()
In [8]:
print("ssh -i {keyloc} -o 'StrictHostKeyChecking no' ubuntu@{hostname}".format(
keyloc = linker.my_key_location,
hostname = linker.nodes[0]['public_dns']))
In [9]:
linker.setup_nodefile()
In [22]:
linker.send_command_ssh( command = 'rm ~/covs_.npy')
linker.send_command_ssh( command = 'rm ~/means_.npy')
linker.send_command_ssh( command = 'rm ~/weights_.npy')
linker.send_command_ssh( command = 'rm ~/article_util.py')
linker.send_command_ssh( command = 'rm ~/article_simulatedata.py')
linker.send_command_ssh( command = 'rm ~/article_estimate_largerdata1_mpi.py')
Out[22]:
In [23]:
linker.send_command_ssh( command = 'wget https://raw.githubusercontent.com/JonasWallin/BayesFlow/master/examples/article1/covs_.npy')
linker.send_command_ssh( command = 'wget https://raw.githubusercontent.com/JonasWallin/BayesFlow/master/examples/article1/means_.npy')
linker.send_command_ssh( command = 'wget https://raw.githubusercontent.com/JonasWallin/BayesFlow/master/examples/article1/weights_.npy')
Out[23]:
In [30]:
linker.copy_files_from_node('simulation_result.npy')
linker.copy_files_from_node('mus_sim.npy')
In [ ]: