Op voorhand heb ik 10 cores gevraagd en met volgende commando haal ik de cores op:
In [3]:
from __future__ import division
from IPython.parallel import Client
c = Client()
c.ids
Out[3]:
Ik heb in dus 10 cores aangevraagd in dit geval.
Volgende deel is nodig om he bij ons werkende te krijgen (zie biomath-wiki):
In [4]:
%%px --local
# Execute on all cores but also for serial calculations
import dill # Load dill, More general pickling class
# fallback to pickle instead of cPickle, so that dill can take over
import pickle
from IPython.kernel.zmq import serialize
serialize.pickle = pickle
Ik wil een aantal modules ter beschikking in elke core en dus doe ik sync-import:
In [3]:
with c[:].sync_imports():
#bio-intense custom developments
import biointense
import numpy as np
import time
import pySTAN
from copy import deepcopy
%load_ext autoreload
%autoreload 2
import sys
import os
import pandas as pd
from scipy import optimize
Om ervoor te zorgen dat alle cores mijn eigen te schrijven hulp- functie/definitie kan lezen, registreer ik kort een extra magic function (anders zou ik alles in 1 grote definitie moeten schrijven):
In [4]:
def pxlocal(line, cell):
ip = get_ipython()
ip.run_cell_magic("px", line, cell)
ip.run_cell(cell)
get_ipython().register_magic_function(pxlocal, "cell")
Met die magic function bij mijn definitie wordt dat dan ook beschikbaar in all cores:
In [5]:
%%pxlocal
#DOELFUNCTIE
def all_peaks_optimize(parameters, name):
"""
Parameters
-----------
name : char
id-name of the experiment
parameters : list
List with parameter values => [ks rmax]
"""
import pandas as pd
import biointense
import numpy as np
import time
import pySTAN
import sys
import os
import pandas as pd
from scipy import optimize
tauvalues = {'0131A': 20.00000001,
'0206A': 20.00000001,
'0226A': 199.999999601,
'0307A': 199.999999992,
'0319A': 21.2911210166,
'0326A': 92.8429882969,
'0401A': 38.9426243836,
'0409A': 89.1280050836,
'0417A': 120.916492359,
'0508A': 79.8659969447,
'1018A': 149.21015342,
'1025A': 193.611650726,
'1031A': 138.371608153,
'1107A': 79.4790380957,
'1114A': 125.880552489,
'1122A': 129.27346531,
'1128A': 123.383245389,
'1205A': 193.267121328,
'1212A': 170.381661059,
'1220A': 159.541718233}
#select the relevant peaks of the experiment
datapath1 = "/home/data/stvhoey/Fitten_profielen"
cols = ["id", "S", "DO","X", "kla", "Y", "ratio", "Ycalc" ]
piekdata = pd.read_table(os.path.join(datapath1,"alles2.lvm"), names = cols)
piekdata = piekdata.set_index("id")
#alle namen
allnames = [nm[:-1] for nm in piekdata.index.tolist()]
piekdata["shortname"] = allnames
#unieke namen
names = []
[names.append(i) for i in allnames if not i in names]
exp_peaks = piekdata.groupby("shortname").get_group(name)
SSE = 0
#voor elke piek een model opstellen en SSE optellen
for index, row in exp_peaks.iterrows():
#get names of the experiments
idi = index
Si = row["S"]
DOi= row["DO"]
Xi = row["X"]
klai = row["kla"]
Yi = row["Y"]
ratioi = row["ratio"]
initcond = {'SS':Si, 'DO':DOi, 'XX':Xi}
Modelname = 'Basicrespiromodel' + index
#hier de nieuwe pars geven
Parameters = {'tau':tauvalues[name], 'mumax':parameters[1], 'ks':parameters[0]}
System = {'dSS':'-(1-exp(-t/tau))*((mumax/(3600*24))*XX/('+str(Yi)+')*SS/((SS+ks)))',
'dDO':'('+str(klai)+'*('+ str(DOi)+'-DO)-(1-exp(-t/tau))*(mumax/(3600*24))*XX/'+str(Yi)+'*(1-'+str(Yi)+')*SS/((SS+ks)))',
'dXX' :'(1-exp(-t/tau))*(mumax*XX/(3600*24))*SS/(SS+ks)-(0.24/(24*3600))*XX'
}
M1 = biointense.odegenerator(System, Parameters, Modelname = Modelname)
M1.set_measured_states(['DO'])
M1.set_initial_conditions(initcond)
#-------------------------------------------------
#HIER NOG AAN TE PASSEN MET INLEZEN GOEIE FILES
#-------------------------------------------------
datapath = "/home/data/stvhoey/pieken/pieken"
data = pd.read_csv(os.path.join(datapath,index+"_pieken.csv"))
#-------------------------------------------------
time_col = 'NewTime'
lasttime = np.ceil(data[time_col].values[-1])
data = data.set_index(time_col)
M1.set_time({'start':0,'end':lasttime,'nsteps':20})
dataprofile1 = biointense.ode_measurements(data[["DO"]])
Fitprofile1 = biointense.ode_optimizer(M1,dataprofile1)
# initial_parset = {'tau':80, 'mumax':1.0, 'ks' :30}
# #the bugfix: !!!!
# initial_parset = collections.OrderedDict(sorted(initial_parset.items(),
# key=lambda t: t[0]))#effectieve volgorde is: ks rmax tau
SSEp = Fitprofile1.get_WSSE()[0]
SSE +=SSEp
print "-"*40
print "SSE of experiment ", name, " is ", SSE
print "-"*40
print "Current parameter values are \n",
print "ks: ", Fitprofile1.get_all_parameters()['ks']
print "mumax: ", Fitprofile1.get_all_parameters()['mumax']
print "-"*40
return SSE
In [6]:
datapath1 = "/home/data/stvhoey/Fitten_profielen"
In [7]:
#INFO IVM PIEKEN
#------------------
cols = ["id", "S", "DO","X", "kla", "Y", "ratio", "Ycalc" ]
piekdata = pd.read_table(os.path.join(datapath1,"alles2.lvm"), names = cols)
piekdata = piekdata.set_index("id")
#alle namen
allnames = [nm[:-1] for nm in piekdata.index.tolist()]
piekdata["shortname"] = allnames
#unieke namen
names = []
[names.append(i) for i in allnames if not i in names]
Out[7]:
In [8]:
names
Out[8]:
1 van de 2 volgende optimalisaties wil ik uitvoeren voor elk van mijn experimenten en dus dit zijn de functies die ik wil mappen (een for-loop schrijven)
In [9]:
def run_on_kernels(name):
from scipy import optimize
maxiters = 1
method = 'Nelder-Mead'
initial_parset = [3., 1.0] #{'ks' :30, 'mumax':1.0}
res = optimize.minimize(all_peaks_optimize, initial_parset,
args=(name,), method= method, jac = False,
options = {"maxiter": maxiters})
return [res, res.x, name]
In [12]:
def run_on_kernels2(name):
from scipy import optimize
maxiters = 500
method = 'TNC'
initial_parset = [3., 1.0] #{'ks' :30, 'mumax':1.0}
res = optimize.minimize(all_peaks_optimize, initial_parset,
args=(name,), method= method, jac = False, bounds = [(1, 30.), (0.1, 10.)],
options = {"maxiter": maxiters, "eps": 0.001})
return [res, res.x, name]
Hier begint het échte werk: parallel runnen van de optimalisatie, door eerst een balanced view op te vragen waarop ik dan mijn functie kan mappen (de opdrachten verspreiden):
In [2]:
lview = rc.load_balanced_view()
En hier gebeurt de parallele magie, ik map de functie voor elke naam in de lijst 'names'. De return van mijn run_on_kernels2-functie wordt dan telkens teruggegeven en die komen allemaal in hres terecht (maw, zorg voor iets van unieke ID die je ook terug meegeeft). Ik schrijf die hres-info dan in een file als alles resultaten binnen zijn.
In [ ]:
lview.block = False
hres = lview.map(run_on_kernels2, names)
f = open("2014_optimalpars_allpeaks_newlatenight", "w")
for j, parcombi in enumerate(hres.result):
f.write(parcombi[-1] + "\n")
for value in parcombi[1]:
f.write(str(value)+"\n")
f.close()
In [ ]:
In [25]:
In [ ]:
In [ ]:
In [18]:
maxiters = 1
method = 'Nelder-Mead'
optimal_pars = {}
#for name in names:
for name in ['0131A']:
print "*"*40
print "working on experiment ", name, "..."
print "*"*40
initial_parset = [30., 1.0] #{'ks' :30, 'mumax':1.0}
res = optimize.minimize(all_peaks_optimize, initial_parset,
args=(name,), method= method, jac = False,
options = {"maxiter": maxiters, "eps": 0.001}) #bounds = [(1, 50), (0.1, 10)]
print res.message
optimal_pars[name] = res.x
print "...done!"
print "*"*40
#
f = open("2014_optimalpars_allpeaks", "w")
for j, experiment in enumerate(optimal_pars.keys()):
f.write(experiment + "\n")
for key, value in parset.items():
f.write(str(value)+"\n")
f.write(str(wsses[j])+"\n")
f.close()
In [ ]: