Running simple example through EC2

start downloading linking_EC2

Tutorial for running liknking_EC2 see: https://github.com/JonasWallin/linkingEC2/blob/master/script/running%20MPI4py.ipynb


In [1]:
%%bash
. ~/.bashrc
pip install  --upgrade git+https://git@github.com/JonasWallin/linkingEC2


Collecting git+https://git@github.com/JonasWallin/linkingEC2
  Cloning https://git@github.com/JonasWallin/linkingEC2 to /var/folders/ct/zpjk2t696k5bv03sqq9xrpd00000gn/T/pip-pXLfWY-build
Requirement already up-to-date: boto in /Users/jonaswallin/anaconda/lib/python2.7/site-packages (from linkingEC2==0.1)
Installing collected packages: linkingEC2
  Found existing installation: linkingEC2 0.1
    Uninstalling linkingEC2-0.1:
      Successfully uninstalled linkingEC2-0.1
  Running setup.py install for linkingEC2
Successfully installed linkingEC2-0.1

In [2]:
from linkingEC2 import LinkingHandler
from ConfigParser import ConfigParser

config = ConfigParser()
starfigconfig_folder = "/Users/jonaswallin/.starcluster/"
config.read(starfigconfig_folder + "config")

acess_key_id     = config.get('aws info', 'aws_access_key_id'    , 0)
aws_secret_key   = config.get('aws info', 'aws_secret_access_key', 0)
aws_region_name  = config.get('aws info', 'aws_region_name'      , 0)
my_key_loc       = config.get('key mykeyABC', 'key_location',0)


linker = LinkingHandler(aws_secret_access_key = aws_secret_key,
                        aws_access_key_id     = acess_key_id,
                        aws_region_name       = aws_region_name,
                        key_location          = my_key_loc,
                        key_name              = 'mykeyABC' )
start the actual cluster

In [3]:
start_cluster= False
spot_cluster = True
n_nodes = 1
type_node =  'c4.8xlarge'
if spot_cluster:
    linker.connect_spot_instance()
elif start_cluster:
    linker.start_cluster('ami-d05e75b8', type_node, ['linking_EC2'], n_nodes)
else:
    linker.connect_cluster()


---------------------------------------------------------------------------
AttributeError                            Traceback (most recent call last)
<ipython-input-3-7da223fc27d2> in <module>()
      4 type_node =  'c4.8xlarge'
      5 if spot_cluster:
----> 6     linker.connect_spot_instance()
      7 elif start_cluster:
      8     linker.start_cluster('ami-d05e75b8', type_node, ['linking_EC2'], n_nodes)

/Users/jonaswallin/anaconda/lib/python2.7/site-packages/linkingEC2/linkingEC2class.pyc in connect_spot_instance(self, extra_build)
     89 
     90 
---> 91                 self.nodes = get_dns_name(self.reservation, self.conn, silent = self.silent)
     92 
     93                 if extra_build:

/Users/jonaswallin/anaconda/lib/python2.7/site-packages/linkingEC2/linkingEC2.pyc in get_dns_name(reservations, conn, silent)
     45 	"""
     46         nodes = []
---> 47         for i in range(len(reservations.instances)):
     48                 instance_id  = reservations.instances[i].id
     49                 res = conn.get_all_instances(instance_ids=[instance_id])

AttributeError: 'SpotInstanceRequest' object has no attribute 'instances'

In [10]:
PACKAGES_APT = [' libatlas3-base',
                'libatlas-base-dev',
                'python-dev', 
                'openmpi-bin', 
                'libopenmpi-dev', 
                'python-numpy',
                'python-sklearn',
               'python-matplotlib',
               'git',
                'python-scipy',
               'r-base',
               'r-base-core']
PACKAGES_PIP = ['cython',
                'mpi4py',
                'simplejson',
               'rpy2']

#Adding later version of R
#http://philipp-burckhardt.com/2014/05/25/installing-r-rstudio-on-ubuntu/
command = 'sudo add-apt-repository "deb http://cran.rstudio.com/bin/linux/ubuntu trusty/"'
linker.send_command_ssh(command = 'gpg --keyserver pgpkeys.mit.edu --recv-key 51716619E084DAB9')
linker.send_command_ssh(command = 'gpg -a --export 51716619E084DAB9 | sudo apt-key add -')
linker.send_command_ssh(command = command)



linker.apt_install(PACKAGES_APT)

#problem with memory installing scipy:
#http://naokiwatanabe.blogspot.se/2014/12/install-numpy-schipy-matplotlib-and-etc.html
linker.send_command_ssh(command = 'sudo /bin/dd if=/dev/zero of=/var/swap.1 bs=1M count=1024')
linker.send_command_ssh(command = 'sudo /sbin/mkswap /var/swap.1')
linker.send_command_ssh(command = 'sudo /sbin/swapon /var/swap.1')
linker.pip_install('-U scipy')

linker.send_command_ssh(command = 'sudo swapoff /var/swap.1')
linker.send_command_ssh(command = 'sudo sudo rm /var/swap.1')



linker.pip_install(PACKAGES_PIP)
linker.pip_install(['git+https://git@github.com/JonasWallin/BayesFlow'])
import os
os.system('say "your packages is downloaded"')


node001, sshing in  gpg --keyserver pgpkeys.mit.edu --recv-key 51716619E084DAB9 
node001 done
node002, sshing in  gpg --keyserver pgpkeys.mit.edu --recv-key 51716619E084DAB9 
node002 done
node003, sshing in  gpg --keyserver pgpkeys.mit.edu --recv-key 51716619E084DAB9 
node003 done
node004, sshing in  gpg --keyserver pgpkeys.mit.edu --recv-key 51716619E084DAB9 
node004 done
node001, sshing in  gpg -a --export 51716619E084DAB9 | sudo apt-key add - 
node001 done
node002, sshing in  gpg -a --export 51716619E084DAB9 | sudo apt-key add - 
node002 done
node003, sshing in  gpg -a --export 51716619E084DAB9 | sudo apt-key add - 
node003 done
node004, sshing in  gpg -a --export 51716619E084DAB9 | sudo apt-key add - 
node004 done
node001, sshing in  sudo add-apt-repository "deb http://cran.rstudio.com/bin/linux/ubuntu trusty/" 
node001 done
node002, sshing in  sudo add-apt-repository "deb http://cran.rstudio.com/bin/linux/ubuntu trusty/" 
node002 done
node003, sshing in  sudo add-apt-repository "deb http://cran.rstudio.com/bin/linux/ubuntu trusty/" 
node003 done
node004, sshing in  sudo add-apt-repository "deb http://cran.rstudio.com/bin/linux/ubuntu trusty/" 
node004 done
installing the packages :  libatlas3-base libatlas-base-dev python-dev openmpi-bin libopenmpi-dev python-numpy python-sklearn python-matplotlib git python-scipy r-base r-base-core

update apt-get for node002
update apt-get for node001
update apt-get for node003
update apt-get for node004
node002 is done
node001 is done
node003 is done
node004 is done
upgrade apt-get for node002
upgrade apt-get for node001
upgrade apt-get for node003
upgrade apt-get for node004
node002 is done
node001 is done
node003 is done
node004 is done
node001, sshing in  sudo /bin/dd if=/dev/zero of=/var/swap.1 bs=1M count=1024 
installing package for node002
installing package for node001
installing package for node003
installing package for node004
node002 is done
node001 is done
node003 is done
node004 is done
node001 done
node002, sshing in  sudo /bin/dd if=/dev/zero of=/var/swap.1 bs=1M count=1024 
node002 done
node003, sshing in  sudo /bin/dd if=/dev/zero of=/var/swap.1 bs=1M count=1024 
node003 done
node004, sshing in  sudo /bin/dd if=/dev/zero of=/var/swap.1 bs=1M count=1024 
node004 done
node001, sshing in  sudo /sbin/mkswap /var/swap.1 
node001 done
node002, sshing in  sudo /sbin/mkswap /var/swap.1 
node002 done
node003, sshing in  sudo /sbin/mkswap /var/swap.1 
node003 done
node004, sshing in  sudo /sbin/mkswap /var/swap.1 
node004 done
node001, sshing in  sudo /sbin/swapon /var/swap.1 
node001 done
node002, sshing in  sudo /sbin/swapon /var/swap.1 
node002 done
node003, sshing in  sudo /sbin/swapon /var/swap.1 
node003 done
node004, sshing in  sudo /sbin/swapon /var/swap.1 
node004 done
installing the packages : python-pip

installing the packages : -U scipy

installing package for node001
installing package for node002
installing package for node003
installing package for node004
node001 is done
node002 is done
node003 is done
node004 is done
node001, sshing in  sudo swapoff /var/swap.1 
installing package for node001
installing package for node003
installing package for node002
installing package for node004
node001 is done
node003 is done
node002 is done
node004 is done
node001 done
node002, sshing in  sudo swapoff /var/swap.1 
node002 done
node003, sshing in  sudo swapoff /var/swap.1 
node003 done
node004, sshing in  sudo swapoff /var/swap.1 
node004 done
node001, sshing in  sudo sudo rm /var/swap.1 
node001 done
node002, sshing in  sudo sudo rm /var/swap.1 
node002 done
node003, sshing in  sudo sudo rm /var/swap.1 
node003 done
node004, sshing in  sudo sudo rm /var/swap.1 
node004 done
installing the packages : cython mpi4py simplejson rpy2

installing the packages : git+https://git@github.com/JonasWallin/BayesFlow

installing package for node002
installing package for node001
installing package for node003
installing package for node004
node002 is done
node001 is done
node003 is done
node004 is done
installing package for node001
installing package for node002
installing package for node003
installing package for node004
node001 is done
node002 is done
node003 is done
node004 is done
Out[10]:
0

In [11]:
#linker.send_command_ssh( command = 'rm ~/covs_.npy')
#linker.send_command_ssh( command = 'rm ~/means_.npy')
#linker.send_command_ssh( command = 'rm ~/weights_.npy')
#linker.send_command_ssh( command = 'rm ~/article_util.py')
#linker.send_command_ssh( command = 'rm ~/article_simulatedata.py')
#linker.send_command_ssh( command = 'rm ~/article_estimate_largerdata1_mpi.py')
linker.send_command_ssh( command = 'wget https://raw.githubusercontent.com/JonasWallin/BayesFlow/master/examples/article1/covs_.npy')
linker.send_command_ssh( command = 'wget https://raw.githubusercontent.com/JonasWallin/BayesFlow/master/examples/article1/means_.npy')
linker.send_command_ssh( command = 'wget https://raw.githubusercontent.com/JonasWallin/BayesFlow/master/examples/article1/weights_.npy')
linker.send_command_ssh( command = 'wget https://raw.githubusercontent.com/JonasWallin/BayesFlow/master/examples/article1/article_util.py')
linker.send_command_ssh( command = 'wget https://raw.githubusercontent.com/JonasWallin/BayesFlow/master/examples/article1/article_simulatedata.py')
linker.send_command_ssh( command = 'wget https://raw.githubusercontent.com/JonasWallin/BayesFlow/master/examples/article1/article_estimate_largerdata1_mpi.py')


node001, sshing in  wget https://raw.githubusercontent.com/JonasWallin/BayesFlow/master/examples/article1/covs_.npy 
node001 done
node002, sshing in  wget https://raw.githubusercontent.com/JonasWallin/BayesFlow/master/examples/article1/covs_.npy 
node002 done
node003, sshing in  wget https://raw.githubusercontent.com/JonasWallin/BayesFlow/master/examples/article1/covs_.npy 
node003 done
node004, sshing in  wget https://raw.githubusercontent.com/JonasWallin/BayesFlow/master/examples/article1/covs_.npy 
node004 done
node001, sshing in  wget https://raw.githubusercontent.com/JonasWallin/BayesFlow/master/examples/article1/means_.npy 
node001 done
node002, sshing in  wget https://raw.githubusercontent.com/JonasWallin/BayesFlow/master/examples/article1/means_.npy 
node002 done
node003, sshing in  wget https://raw.githubusercontent.com/JonasWallin/BayesFlow/master/examples/article1/means_.npy 
node003 done
node004, sshing in  wget https://raw.githubusercontent.com/JonasWallin/BayesFlow/master/examples/article1/means_.npy 
node004 done
node001, sshing in  wget https://raw.githubusercontent.com/JonasWallin/BayesFlow/master/examples/article1/weights_.npy 
node001 done
node002, sshing in  wget https://raw.githubusercontent.com/JonasWallin/BayesFlow/master/examples/article1/weights_.npy 
node002 done
node003, sshing in  wget https://raw.githubusercontent.com/JonasWallin/BayesFlow/master/examples/article1/weights_.npy 
node003 done
node004, sshing in  wget https://raw.githubusercontent.com/JonasWallin/BayesFlow/master/examples/article1/weights_.npy 
node004 done
node001, sshing in  wget https://raw.githubusercontent.com/JonasWallin/BayesFlow/master/examples/article1/article_util.py 
node001 done
node002, sshing in  wget https://raw.githubusercontent.com/JonasWallin/BayesFlow/master/examples/article1/article_util.py 
node002 done
node003, sshing in  wget https://raw.githubusercontent.com/JonasWallin/BayesFlow/master/examples/article1/article_util.py 
node003 done
node004, sshing in  wget https://raw.githubusercontent.com/JonasWallin/BayesFlow/master/examples/article1/article_util.py 
node004 done
node001, sshing in  wget https://raw.githubusercontent.com/JonasWallin/BayesFlow/master/examples/article1/article_simulatedata.py 
node001 done
node002, sshing in  wget https://raw.githubusercontent.com/JonasWallin/BayesFlow/master/examples/article1/article_simulatedata.py 
node002 done
node003, sshing in  wget https://raw.githubusercontent.com/JonasWallin/BayesFlow/master/examples/article1/article_simulatedata.py 
node003 done
node004, sshing in  wget https://raw.githubusercontent.com/JonasWallin/BayesFlow/master/examples/article1/article_simulatedata.py 
node004 done
node001, sshing in  wget https://raw.githubusercontent.com/JonasWallin/BayesFlow/master/examples/article1/article_estimate_largerdata1_mpi.py 
node001 done
node002, sshing in  wget https://raw.githubusercontent.com/JonasWallin/BayesFlow/master/examples/article1/article_estimate_largerdata1_mpi.py 
node002 done
node003, sshing in  wget https://raw.githubusercontent.com/JonasWallin/BayesFlow/master/examples/article1/article_estimate_largerdata1_mpi.py 
node003 done
node004, sshing in  wget https://raw.githubusercontent.com/JonasWallin/BayesFlow/master/examples/article1/article_estimate_largerdata1_mpi.py 
node004 done
Out[11]:
0

In [32]:
print( linker.get_ssh_login() )


---------------------------------------------------------------------------
AttributeError                            Traceback (most recent call last)
<ipython-input-32-96db7df1eaf1> in <module>()
----> 1 print( linker.get_ssh_login() )

/Users/jonaswallin/anaconda/lib/python2.7/site-packages/linkingEC2/linkingEC2class.pyc in get_ssh_login(self, node)
    247                         return
    248 
--> 249                 if node < 0 or node >= len(self.reservation.instances):
    250 			print("node {number} does not exists choose between [{low} {high}]".format(
    251                                    number = node,

AttributeError: 'list' object has no attribute 'instances'

login to main node and run:

mpirun -hostfile nodefile -n 2 python article_estimate_largerdata1_mpi.py

In [13]:
import numpy as np
tot_process = np.sum([node['n_process'] for node in linker.nodes])
command = 'mpirun -hostfile nodefile -n %d python article_estimate_largerdata1_mpi.py'%(tot_process)
linker.send_command_ssh(nodes = 0, command = command)
linker.copy_files_from_node('simulation_result.npy')
linker.copy_files_from_node('mus_sim.npy')
linker.copy_files_from_node('sim_data.npy')


node001, sshing in  mpirun -hostfile nodefile -n 128 python article_estimate_largerdata1_mpi.py 
node001 done
copying simulation_result.npy to local computer .
copying files to node001 done
copying mus_sim.npy to local computer .
copying files to node001 done
copying sim_data.npy to local computer .
copying files to node001 done

Terimante the cluster:


In [10]:
linker.terminate_cluster()

In [3]:
print(linker.conn.get_all_spot_instance_requests())


[SpotInstanceRequest:sir-0243sbqw, SpotInstanceRequest:sir-0245snpl, SpotInstanceRequest:sir-02478em0, SpotInstanceRequest:sir-023ztzqh, SpotInstanceRequest:sir-024364vv, SpotInstanceRequest:sir-0244lqqq]

In [3]:
spot_instance = linker.conn.get_all_spot_instance_requests()
nodes = []
reservation = []
print(spot_instance)
for i,spot in enumerate(spot_instance):
    print(spot.instance_id)
    if spot.instance_id is not None:
        res = linker.conn.get_all_instances(instance_ids=[spot.instance_id])
        node_alias  = 'node{0:03d}'.format(i+1)
        reservation.append(res[0].instances[0])
        public_dns  = res[0].instances[0].public_dns_name
        private_dns = res[0].instances[0].private_dns_name
        private_ip_address = res[0].instances[0].private_ip_address
        nodes.append({'name'  :node_alias,
                     'public_dns' :public_dns,
                      'private_dns':private_dns,
                     'private_ip' :private_ip_address})


[SpotInstanceRequest:sir-0241jc3q, SpotInstanceRequest:sir-0243z3ha, SpotInstanceRequest:sir-024666jf, SpotInstanceRequest:sir-0246xgte]
i-fb694a2e
i-756a49a0
i-67694ab2
i-81694a54

In [4]:
linker.reservation = reservation
linker.nodes = nodes

In [5]:
linker.test_ssh_in()


checking if ssh into node001 works:
*
checking if ssh into node002 works:
*
checking if ssh into node003 works:
*
checking if ssh into node004 works:
*

In [6]:
from linkingEC2.linkingEC2 import get_number_processes
get_number_processes(nodes  = linker.nodes,
                    my_key =  linker.my_key_location,
								 user   = linker.user,
								 silent = linker.silent)


collecting node001 number of processes:done
collecting node002 number of processes:done
collecting node003 number of processes:done
collecting node004 number of processes:done

In [7]:
#copying the ssh keys to nodes
	
			linker.copy_files_to_nodes(file_name = linker.my_key_location, 
									 destination = '~/.ssh/id_rsa')
			linker._ssh_disable_StrictHostKeyChecking()


copying ~/.ssh/mykeyABC.rsa to external ~/.ssh/id_rsa
copying files to node001 done
copying files to node002 done
copying files to node003 done
copying files to node004 done
disable StrictHostKeyChecking  in node001:

In [8]:
print("ssh  -i {keyloc} -o 'StrictHostKeyChecking no'  ubuntu@{hostname}".format(
		keyloc = linker.my_key_location,
		hostname =  linker.nodes[0]['public_dns']))


ssh  -i ~/.ssh/mykeyABC.rsa -o 'StrictHostKeyChecking no'  ubuntu@ec2-54-175-42-84.compute-1.amazonaws.com

In [9]:
linker.setup_nodefile()


appending to ~/nodefile in node001 done
appending to /etc/hosts in node001 done

In [22]:
linker.send_command_ssh( command = 'rm ~/covs_.npy')
linker.send_command_ssh( command = 'rm ~/means_.npy')
linker.send_command_ssh( command = 'rm ~/weights_.npy')
linker.send_command_ssh( command = 'rm ~/article_util.py')
linker.send_command_ssh( command = 'rm ~/article_simulatedata.py')
linker.send_command_ssh( command = 'rm ~/article_estimate_largerdata1_mpi.py')


node004, sshing in  rm ~/covs_.npy 
node004 done
node005, sshing in  rm ~/covs_.npy 
node005 done
node006, sshing in  rm ~/covs_.npy 
node006 done
node004, sshing in  rm ~/means_.npy 
node004 done
node005, sshing in  rm ~/means_.npy 
node005 done
node006, sshing in  rm ~/means_.npy 
node006 done
node004, sshing in  rm ~/weights_.npy 
node004 done
node005, sshing in  rm ~/weights_.npy 
node005 done
node006, sshing in  rm ~/weights_.npy 
node006 done
Out[22]:
0

In [23]:
linker.send_command_ssh( command = 'wget https://raw.githubusercontent.com/JonasWallin/BayesFlow/master/examples/article1/covs_.npy')
linker.send_command_ssh( command = 'wget https://raw.githubusercontent.com/JonasWallin/BayesFlow/master/examples/article1/means_.npy')
linker.send_command_ssh( command = 'wget https://raw.githubusercontent.com/JonasWallin/BayesFlow/master/examples/article1/weights_.npy')


node004, sshing in  wget https://raw.githubusercontent.com/JonasWallin/BayesFlow/master/examples/article1/covs_.npy 
node004 done
node005, sshing in  wget https://raw.githubusercontent.com/JonasWallin/BayesFlow/master/examples/article1/covs_.npy 
node005 done
node006, sshing in  wget https://raw.githubusercontent.com/JonasWallin/BayesFlow/master/examples/article1/covs_.npy 
node006 done
node004, sshing in  wget https://raw.githubusercontent.com/JonasWallin/BayesFlow/master/examples/article1/means_.npy 
node004 done
node005, sshing in  wget https://raw.githubusercontent.com/JonasWallin/BayesFlow/master/examples/article1/means_.npy 
node005 done
node006, sshing in  wget https://raw.githubusercontent.com/JonasWallin/BayesFlow/master/examples/article1/means_.npy 
node006 done
node004, sshing in  wget https://raw.githubusercontent.com/JonasWallin/BayesFlow/master/examples/article1/weights_.npy 
node004 done
node005, sshing in  wget https://raw.githubusercontent.com/JonasWallin/BayesFlow/master/examples/article1/weights_.npy 
node005 done
node006, sshing in  wget https://raw.githubusercontent.com/JonasWallin/BayesFlow/master/examples/article1/weights_.npy 
node006 done
Out[23]:
0

In [30]:
linker.copy_files_from_node('simulation_result.npy')
linker.copy_files_from_node('mus_sim.npy')


copying simulation_result.npy to local computer .
copying files to node004 done
copying mus_sim.npy to local computer .
copying files to node004 done

In [ ]: