In [3]:
import os
import sys
import random
import time
from random import seed, randint
import argparse
import platform
from datetime import datetime
import imp
import numpy as np
import fileinput
from itertools import product
import pandas as pd
from scipy.interpolate import griddata
from scipy.interpolate import interp2d
import seaborn as sns
from os import listdir
import matplotlib.pyplot as plt
import seaborn as sns
from scipy.interpolate import griddata
import matplotlib as mpl
sys.path.insert(0,'..')
from notebookFunctions import *
# from .. import notebookFunctions
%matplotlib inline
plt.rcParams['figure.figsize'] = (10,6.180) #golden ratio
# %matplotlib notebook
%load_ext autoreload
%autoreload 2
In [39]:
data = pd.read_feather("/Users/weilu/Research/server/may_2018/second_start_topology/rerun_5_20_May_231514.feather")
dic = {"T0":300, "T1":335, "T2":373, "T3":417, "T4":465, "T5":519, "T6":579, "T7":645, "T8":720, "T9":803, "T10":896, "T11":1000}
a = data
a["Temp"] = a["Temp"].apply(lambda x: dic[x])
rerun7 = data
t = a.query("Temp < 400").groupby(["BiasTo","Temp"])[["DisReal","Run"]].mean().reset_index()
t["Diff"] = t["DisReal"]-t["BiasTo"].apply(pd.to_numeric)
t["BiasTo"] = t["BiasTo"].apply(pd.to_numeric)
fg = sns.FacetGrid(data=t, hue='Temp', size=8, aspect=1.61)
fg.map(plt.scatter, 'BiasTo', 'Diff').add_legend()
Out[39]:
In [64]:
second.query("TempT == 373 and z_h6 < -10").plot.hexbin("DisReal", "Qw", cmap="seismic", sharex=False)
Out[64]:
In [61]:
data.query("Temp == 373 and z_h6 < -10").plot.hexbin("DisReal", "Qw", cmap="seismic", sharex=False)
Out[61]:
In [5]:
data = pd.read_feather("/Users/weilu/Research/server/may_2018/second_long/rerun_5_19_May_220121.feather")
dic = {"T0":300, "T1":335, "T2":373, "T3":417, "T4":465, "T5":519, "T6":579, "T7":645, "T8":720, "T9":803, "T10":896, "T11":1000}
a = data
a["Temp"] = a["Temp"].apply(lambda x: dic[x])
rerun7 = data
t = a.query("Temp < 400").groupby(["BiasTo","Temp"])[["DisReal","Run"]].mean().reset_index()
t["Diff"] = t["DisReal"]-t["BiasTo"].apply(pd.to_numeric)
t["BiasTo"] = t["BiasTo"].apply(pd.to_numeric)
fg = sns.FacetGrid(data=t, hue='Temp', size=8, aspect=1.61)
fg.map(plt.scatter, 'BiasTo', 'Diff').add_legend()
Out[5]:
In [65]:
# data = pd.read_feather("/Users/weilu/Research/server/may_2018/second_start_topology/rerun_5_20_May_231514.feather")
rerun5 = pd.read_feather("/Users/weilu/Research/server/may_2018/second_start_topology/rerun_5_20_May_231514.feather")
rerun4 = pd.read_feather("/Users/weilu/Research/server/may_2018/second_start_topology/rerun_4_20_May_231514.feather")
data = pd.concat([rerun5, rerun4])
dic = {"T0":300, "T1":335, "T2":373, "T3":417, "T4":465, "T5":519, "T6":579, "T7":645, "T8":720, "T9":803, "T10":896, "T11":1000}
data["TempT"] = data["Temp"].apply(lambda x: dic[x])
chosen = data.query("TempT < 420")
chosen.reset_index().to_feather("/Users/weilu/Research/server/may_2018/03_week/all_data_folder/second_toplogy_may21.feather")
In [6]:
rerun5 = pd.read_feather("/Users/weilu/Research/server/may_2018/second_long/rerun_5_19_May_220121.feather")
rerun4 = pd.read_feather("/Users/weilu/Research/server/may_2018/second_long/rerun_5_19_May_220121.feather")
data = pd.concat([rerun5, rerun4])
dic = {"T0":300, "T1":335, "T2":373, "T3":417, "T4":465, "T5":519, "T6":579, "T7":645, "T8":720, "T9":803, "T10":896, "T11":1000}
data["TempT"] = data["Temp"].apply(lambda x: dic[x])
chosen = data.query("TempT < 420")
chosen.reset_index().to_feather("/Users/weilu/Research/server/may_2018/03_week/all_data_folder/second_long_may19.feather")
In [9]:
data["BiasTo"] = data["BiasTo"].apply(pd.to_numeric)
In [10]:
second = pd.read_feather("/Users/weilu/Research/server/may_2018/03_week/all_data_folder/second_may19.feather")
In [13]:
second["BiasTo"] = second["BiasTo"].apply(pd.to_numeric)
In [67]:
data["Lipid"]
Out[67]:
In [11]:
data["BiasTo"].unique()
Out[11]:
In [15]:
second["BiasTo"].unique()
Out[15]:
In [19]:
chosen = data.query("TempT < 420")
In [27]:
new_chosen = chosen.query("BiasTo != 100.0 and BiasTo != 106")
In [28]:
all_data = pd.concat([new_chosen,second])
In [31]:
all_data.reset_index().to_feather("/Users/weilu/Research/server/may_2018/03_week/all_data_folder/second_start_extended_combined_may19.feather")
In [37]:
all_data.query("TempT == 417 and DisReal < 80").plot.hexbin("DisReal", "Qw", cmap="seismic", sharex=False)
Out[37]:
In [50]:
t =all_data.query("TempT < 427 and DisReal > 50 and DisReal < 60 and z_average < -4 and z_average > -10")
t.plot.hexbin("DisReal", "z_average", cmap="seismic", sharex=False)
Out[50]:
In [54]:
t.groupby("TempT").count()
Out[54]:
In [52]:
select(t)
Out[52]:
In [29]:
a = all_data.groupby("BiasTo")["AMH_4H"].count()
In [26]:
chosen.query("BiasTo == 100.0").plot.hexbin("DisReal", "Qw", cmap="seismic", sharex=False)
Out[26]:
In [25]:
second.query("BiasTo == 100.0").plot.hexbin("DisReal", "Qw", cmap="seismic", sharex=False)
Out[25]:
In [24]:
all_data.query("BiasTo == 100.0").plot.hexbin("DisReal", "Qw", cmap="seismic", sharex=False)
Out[24]:
In [30]:
a.sort_values()
Out[30]:
In [ ]:
In [122]:
# data = pd.read_feather("/Users/weilu/Research/server/may_2018/second_long/rerun_1_10_May_231258.feather")
data = pd.read_feather("/Users/weilu/Research/server/may_2018/second_start_topology/rerun_5_14_May_143016.feather")
dic = {"T0":300, "T1":335, "T2":373, "T3":417, "T4":465, "T5":519, "T6":579, "T7":645, "T8":720, "T9":803, "T10":896, "T11":1000}
a = data
a["Temp"] = a["Temp"].apply(lambda x: dic[x])
short2 = data
# t = a.query("Temp < 400").groupby(["BiasTo","Temp"])[["DisReal","Run"]].mean().reset_index()
# t["Diff"] = t["DisReal"]-t["BiasTo"].apply(pd.to_numeric)
# t["BiasTo"] = t["BiasTo"].apply(pd.to_numeric)
# fg = sns.FacetGrid(data=t, hue='Temp', size=8, aspect=1.61)
# fg.map(plt.scatter, 'BiasTo', 'Diff').add_legend()
In [ ]:
# data = pd.read_feather("/Users/weilu/Research/server/may_2018/second_long/rerun_1_10_May_231258.feather")
data = pd.read_feather("/Users/weilu/Research/server/may_2018/second_start_topology_lon")
dic = {"T0":300, "T1":335, "T2":373, "T3":417, "T4":465, "T5":519, "T6":579, "T7":645, "T8":720, "T9":803, "T10":896, "T11":1000}
a = data
a["Temp"] = a["Temp"].apply(lambda x: dic[x])
short = data
# t = a.query("Temp < 400").groupby(["BiasTo","Temp"])[["DisReal","Run"]].mean().reset_index()
# t["Diff"] = t["DisReal"]-t["BiasTo"].apply(pd.to_numeric)
# t["BiasTo"] = t["BiasTo"].apply(pd.to_numeric)
# fg = sns.FacetGrid(data=t, hue='Temp', size=8, aspect=1.61)
# fg.map(plt.scatter, 'BiasTo', 'Diff').add_legend()
In [113]:
# data = pd.read_feather("/Users/weilu/Research/server/may_2018/second_long/rerun_1_10_May_231258.feather")
data = pd.read_feather("/Users/weilu/Research/server/may_2018/02_week/all_data_folder/second_rerun3_with_goEnergyrerun_7_14_May_124103.feather")
dic = {"T0":300, "T1":335, "T2":373, "T3":417, "T4":465, "T5":519, "T6":579, "T7":645, "T8":720, "T9":803, "T10":896, "T11":1000}
a = data
a["Temp"] = a["Temp"].apply(lambda x: dic[x])
short = data
# t = a.query("Temp < 400").groupby(["BiasTo","Temp"])[["DisReal","Run"]].mean().reset_index()
# t["Diff"] = t["DisReal"]-t["BiasTo"].apply(pd.to_numeric)
# t["BiasTo"] = t["BiasTo"].apply(pd.to_numeric)
# fg = sns.FacetGrid(data=t, hue='Temp', size=8, aspect=1.61)
# fg.map(plt.scatter, 'BiasTo', 'Diff').add_legend()
In [115]:
data = pd.read_feather("/Users/weilu/Research/server/may_2018/second_long/rerun_5_14_May_155146.feather")
# data = pd.read_feather("/Users/weilu/Research/server/may_2018/02_week/all_data_folder/second_rerun3_with_goEnergyrerun_7_14_May_124103.feather")
dic = {"T0":300, "T1":335, "T2":373, "T3":417, "T4":465, "T5":519, "T6":579, "T7":645, "T8":720, "T9":803, "T10":896, "T11":1000}
a = data
a["Temp"] = a["Temp"].apply(lambda x: dic[x])
long = data
# t = a.query("Temp < 400").groupby(["BiasTo","Temp"])[["DisReal","Run"]].mean().reset_index()
# t["Diff"] = t["DisReal"]-t["BiasTo"].apply(pd.to_numeric)
# t["BiasTo"] = t["BiasTo"].apply(pd.to_numeric)
# fg = sns.FacetGrid(data=t, hue='Temp', size=8, aspect=1.61)
# fg.map(plt.scatter, 'BiasTo', 'Diff').add_legend()
In [28]:
data2['BiasTo'].unique()
Out[28]:
In [24]:
data = pd.read_feather("/Users/weilu/Research/server/may_2018/02_week/all_data_folder/second_longrerun_5_14_May_155146.feather")
In [30]:
d = data.query("BiasTo != '100.0'")
In [36]:
d.reset_index().to_feather("/Users/weilu/Research/server/may_2018/02_week/all_data_folder/second_longrerun_5.feather")
In [116]:
test = pd.concat([short,long])
In [111]:
test
Out[111]:
In [117]:
test.query("Temp == 300")
Out[117]:
In [121]:
test.query("Temp == 300").plot.hexbin("DisReal", "Qw", cmap="seismic", sharex=False)
Out[121]:
In [125]:
test.query("Temp == 373").plot.hexbin("DisReal", "z_h6", cmap="seismic", sharex=False)
Out[125]:
In [124]:
test.query("Temp == 335").plot.hexbin("DisReal", "z_h6", cmap="seismic", sharex=False)
Out[124]:
In [123]:
test.query("Temp == 300").plot.hexbin("DisReal", "z_h6", cmap="seismic", sharex=False)
Out[123]:
In [2]:
data = pd.read_feather("/Users/weilu/Research/server/may_2018/second_long/rerun_1_10_May_231258.feather")
dic = {"T0":300, "T1":335, "T2":373, "T3":417, "T4":465, "T5":519, "T6":579, "T7":645, "T8":720, "T9":803, "T10":896, "T11":1000}
a = data
a["Temp"] = a["Temp"].apply(lambda x: dic[x])
rerun1 = data
t = a.query("Temp < 400").groupby(["BiasTo","Temp"])[["DisReal","Run"]].mean().reset_index()
t["Diff"] = t["DisReal"]-t["BiasTo"].apply(pd.to_numeric)
t["BiasTo"] = t["BiasTo"].apply(pd.to_numeric)
fg = sns.FacetGrid(data=t, hue='Temp', size=8, aspect=1.61)
fg.map(plt.scatter, 'BiasTo', 'Diff').add_legend()
Out[2]:
In [5]:
data = pd.read_feather("/Users/weilu/Research/server/may_2018/second_long/rerun_3_11_May_131422.feather")
dic = {"T0":300, "T1":335, "T2":373, "T3":417, "T4":465, "T5":519, "T6":579, "T7":645, "T8":720, "T9":803, "T10":896, "T11":1000}
a = data
a["Temp"] = a["Temp"].apply(lambda x: dic[x])
rerun3 = data
t = a.query("Temp < 400").groupby(["BiasTo","Temp"])[["DisReal","Run"]].mean().reset_index()
t["Diff"] = t["DisReal"]-t["BiasTo"].apply(pd.to_numeric)
t["BiasTo"] = t["BiasTo"].apply(pd.to_numeric)
fg = sns.FacetGrid(data=t, hue='Temp', size=8, aspect=1.61)
fg.map(plt.scatter, 'BiasTo', 'Diff').add_legend()
Out[5]:
In [22]:
data = pd.read_feather("/Users/weilu/Research/server/may_2018/second_long/rerun_5_14_May_155146.feather")
dic = {"T0":300, "T1":335, "T2":373, "T3":417, "T4":465, "T5":519, "T6":579, "T7":645, "T8":720, "T9":803, "T10":896, "T11":1000}
a = data
a["Temp"] = a["Temp"].apply(lambda x: dic[x])
rerun5 = data
t = a.query("Temp < 400").groupby(["BiasTo","Temp"])[["DisReal","Run"]].mean().reset_index()
t["Diff"] = t["DisReal"]-t["BiasTo"].apply(pd.to_numeric)
t["BiasTo"] = t["BiasTo"].apply(pd.to_numeric)
fg = sns.FacetGrid(data=t, hue='Temp', size=8, aspect=1.61)
fg.map(plt.scatter, 'BiasTo', 'Diff').add_legend()
Out[22]:
In [101]:
rerun5.query("Temp == 300").plot.hexbin("z_h6", "Qw", cmap="seismic", sharex=False)
Out[101]:
In [92]:
rerun5.query("Temp == 335").plot.hexbin("z_h6", "Qw", cmap="seismic", sharex=False)
Out[92]:
In [103]:
t = rerun5.query("Temp == 335 and z_h6 > -10")
select(t, i=1)
Out[103]:
In [ ]:
In [93]:
rerun5.query("Temp == 335 and z_h6 > -10").plot.hexbin("z_h6", "Qw", cmap="seismic", sharex=False)
Out[93]:
In [76]:
rerun5.query("Temp == 335").plot.hexbin("DisReal", "Qw", cmap="seismic", sharex=False)
Out[76]:
In [77]:
rerun5.query("Temp == 335 and DisReal > 150").plot.hexbin("DisReal", "Qw", cmap="seismic", sharex=False)
Out[77]:
In [91]:
t= rerun5.query("Temp == 335 and DisReal > 150 and DisReal < 250")
t.plot.hexbin("DisReal", "Qw", cmap="seismic", sharex=False)
Out[91]:
In [90]:
select(t.query("DisReal < 250"))
Out[90]:
In [83]:
t = rerun5.query("Temp == 335 and DisReal > 110 and DisReal < 170")
t.plot.hexbin("DisReal", "Qw", cmap="seismic", sharex=False)
Out[83]:
In [86]:
t.shape
Out[86]:
In [85]:
t.query("Lipid1 > -0.1").shape
Out[85]:
In [84]:
t.plot.hexbin("DisReal", "Lipid1", cmap="seismic", sharex=False)
Out[84]:
In [72]:
t.shape
Out[72]:
In [75]:
t.query("z_h5 < 0 and z_h4 < 0").plot.hexbin("DisReal", "Qw", cmap="seismic", sharex=False)
Out[75]:
In [66]:
t.query("z_h5 < 0 and z_h4 < 0").plot.hexbin("z_h4", "Qw", cmap="seismic", sharex=False)
Out[66]:
In [59]:
t.plot.hexbin("z_h5", "Qw", cmap="seismic", sharex=False)
Out[59]:
In [64]:
t.plot.hexbin("z_h4", "Qw", cmap="seismic", sharex=False)
Out[64]:
In [61]:
select(t.query("z_h5 > 0"))
Out[61]:
In [56]:
t = rerun5.query("Dis_h56 > 60").query("Temp == 335 and DisReal > 110 and DisReal < 170")
select(t)
Out[56]:
In [48]:
t = rerun5.query("Temp == 335 and DisReal > 110 and DisReal < 170")
select(t)
Out[48]:
In [44]:
rerun5.query("Temp == 335 and DisReal > 150").plot.hexbin("DisReal", "Qw", cmap="seismic", sharex=False)
Out[44]:
In [23]:
rerun5.query("Temp == 335 and DisReal < 120").plot.hexbin("DisReal", "Qw", cmap="seismic", sharex=False)
Out[23]:
In [16]:
rerun3.query("Temp == 335 and DisReal < 120").plot.hexbin("DisReal", "Qw", cmap="seismic", sharex=False)
Out[16]:
In [17]:
rerun3.query("Temp == 300 and DisReal < 120").plot.hexbin("DisReal", "Qw", cmap="seismic", sharex=False)
Out[17]:
In [18]:
rerun3.query("Temp == 300 and DisReal < 120").plot.hexbin("z_h6", "Qw", cmap="seismic", sharex=False)
Out[18]:
In [20]:
rerun1.query("Temp == 300 and DisReal < 120").plot.hexbin("z_h6", "Qw", cmap="seismic", sharex=False)
Out[20]:
In [14]:
t = rerun3.query("Temp == 300 and DisReal < 200 and Qw > 0.25")
select(t)
Out[14]:
In [10]:
rerun1.query("Temp == 300 and DisReal < 200").plot.hexbin("DisReal", "Qw", cmap="seismic", sharex=False)
Out[10]:
In [11]:
rerun1.query("Temp == 335 and DisReal < 200").plot.hexbin("DisReal", "Qw", cmap="seismic", sharex=False)
Out[11]:
In [42]:
pre = "/Users/weilu/Research/server/may_2018/02_week"
temp = 310
location = pre + "/second_long/_280-350/2d_z_qw/force_0.1/"
location2 = location + f"pmf-{temp}.dat"
path, f = shortest_path(location2, start=(1, 10), end=(28,20),save=False, xlabel="z_H6", ylabel="Qw", zmax=20,res=30)
f_original =f
# plt.savefig("/Users/weilu/papers/figures/2d_z6_qw.png", dpi=300)
# plt.savefig("/Users/weilu/papers/figures/shortest_path.png", dpi=300)
location3 = location + f"evpb-{temp}.dat"
(xi,yi,zi) = plot2d(location3, zmax=120)
plt.plot(xi[path[:,1]], yi[path[:,0]], 'r.-')
# plt.savefig("/Users/weilu/papers/figures/2d_expected_dis.png", dpi=300)
plt.figure()
f_on_path = [zi[tuple(p)] for p in reversed(path)]
plt.plot(f_on_path)
# plt.savefig("/Users/weilu/papers/figures/shortest_path_expected_dis.png", dpi=300)
Out[42]:
In [41]:
pre = "/Users/weilu/Research/server/may_2018/02_week"
temp = 310
location = pre + "/second_long/_280-350/2d_z_qw/energy_z_h6/"
location2 = location + f"pmf-{temp}.dat"
path, f = shortest_path(location2, start=(15, 1), end=(2,28),save=False, xlabel="z_H6", ylabel="Qw", zmax=20,res=30)
f_original =f
# plt.savefig("/Users/weilu/papers/figures/2d_z6_qw.png", dpi=300)
# plt.savefig("/Users/weilu/papers/figures/shortest_path.png", dpi=300)
location3 = location + f"evpb-{temp}.dat"
(xi,yi,zi) = plot2d(location3, zmax=120)
plt.plot(xi[path[:,1]], yi[path[:,0]], 'r.-')
# plt.savefig("/Users/weilu/papers/figures/2d_expected_dis.png", dpi=300)
plt.figure()
f_on_path = [zi[tuple(p)] for p in reversed(path)]
plt.plot(f_on_path)
# plt.savefig("/Users/weilu/papers/figures/shortest_path_expected_dis.png", dpi=300)
Out[41]:
In [ ]: