### 设计环境

#### env_data

``````

In [1]:

import pandas as pd

def fix_data(path):

def high2low(tmp, freq):
"""处理从RiceQuant下载的分钟线数据，
从分钟线数据合成低频数据
2017-08-11
"""
# 分别处理bar数据
tmp_open = tmp['open'].resample(freq).ohlc()
tmp_open = tmp_open['open'].dropna()

tmp_high = tmp['high'].resample(freq).ohlc()
tmp_high = tmp_high['high'].dropna()

tmp_low = tmp['low'].resample(freq).ohlc()
tmp_low = tmp_low['low'].dropna()

tmp_close = tmp['close'].resample(freq).ohlc()
tmp_close = tmp_close['close'].dropna()

tmp_price = pd.concat([tmp_open, tmp_high, tmp_low, tmp_close], axis=1)

# 处理成交量
tmp_volume = tmp['volume'].resample(freq).sum()
tmp_volume.dropna(inplace=True)

return pd.concat([tmp_price, tmp_volume], axis=1)

``````

#### env_factor

``````

In [2]:

import pandas as pd
import talib

def get_factors(index,
opening,
closing,
highest,
lowest,
volume,
rolling=26,
drop=False,
normalization=True):
tmp = pd.DataFrame()

# 累积/派发线（Accumulation / Distribution Line，该指标将每日的成交量通过价格加权累计，
# 用以计算成交量的动量。属于趋势型因子

# 平均动向指数，DMI因子的构成部分。属于趋势型因子

# 相对平均动向指数，DMI因子的构成部分。属于趋势型因子

# 绝对价格振荡指数
tmp['APO'] = talib.APO(closing, fastperiod=12, slowperiod=26)

# Aroon通过计算自价格达到近期最高值和最低值以来所经过的期间数，
# 帮助投资者预测证券价格从趋势到区域区域或反转的变化，
# Aroon指标分为Aroon、AroonUp和AroonDown3个具体指标。属于趋势型因子
tmp['AROONDown'], tmp['AROONUp'] = talib.AROON(highest, lowest, timeperiod=14)
tmp['AROONOSC'] = talib.AROONOSC(highest, lowest, timeperiod=14)

# 均幅指标（Average TRUE Ranger），取一定时间周期内的股价波动幅度的移动平均值，
# 是显示市场变化率的指标，主要用于研判买卖时机。属于超买超卖型因子。
tmp['ATR14'] = talib.ATR(highest, lowest, closing, timeperiod=14)
tmp['ATR6'] = talib.ATR(highest, lowest, closing, timeperiod=6)

# 布林带
tmp['Boll_Up'], tmp['Boll_Mid'], tmp['Boll_Down'] = \
talib.BBANDS(closing, timeperiod=20, nbdevup=2, nbdevdn=2, matype=0)

# 均势指标
tmp['BOP'] = talib.BOP(opening, highest, lowest, closing)

# 5日顺势指标（Commodity Channel Index），专门测量股价是否已超出常态分布范围。属于超买超卖型因子。
tmp['CCI5'] = talib.CCI(highest, lowest, closing, timeperiod=5)
tmp['CCI10'] = talib.CCI(highest, lowest, closing, timeperiod=10)
tmp['CCI20'] = talib.CCI(highest, lowest, closing, timeperiod=20)
tmp['CCI88'] = talib.CCI(highest, lowest, closing, timeperiod=88)

# 钱德动量摆动指标（Chande Momentum Osciliator），与其他动量指标摆动指标如
# 相对强弱指标（RSI）和随机指标（KDJ）不同，
# 钱德动量指标在计算公式的分子中采用上涨日和下跌日的数据。属于超买超卖型因子
tmp['CMO_Close'] = talib.CMO(closing, timeperiod=14)
tmp['CMO_Open'] = talib.CMO(closing, timeperiod=14)

# DEMA双指数移动平均线
tmp['DEMA6'] = talib.DEMA(closing, timeperiod=6)
tmp['DEMA12'] = talib.DEMA(closing, timeperiod=12)
tmp['DEMA26'] = talib.DEMA(closing, timeperiod=26)

# DX 动向指数
tmp['DX'] = talib.DX(highest, lowest, closing, timeperiod=14)

# EMA 指数移动平均线
tmp['EMA6'] = talib.EMA(closing, timeperiod=6)
tmp['EMA12'] = talib.EMA(closing, timeperiod=12)
tmp['EMA26'] = talib.EMA(closing, timeperiod=26)

# KAMA 适应性移动平均线
tmp['KAMA'] = talib.KAMA(closing, timeperiod=30)

# MACD
tmp['MACD_DIF'], tmp['MACD_DEA'], tmp['MACD_bar'] = \
talib.MACD(closing, fastperiod=12, slowperiod=24, signalperiod=9)

# 中位数价格 不知道是什么意思
tmp['MEDPRICE'] = talib.MEDPRICE(highest, lowest)

# 负向指标 负向运动
tmp['MiNUS_DI'] = talib.MINUS_DI(highest, lowest, closing, timeperiod=14)
tmp['MiNUS_DM'] = talib.MINUS_DM(highest, lowest, timeperiod=14)

# 动量指标（Momentom Index），动量指数以分析股价波动的速度为目的，研究股价在波动过程中各种加速，
# 减速，惯性作用以及股价由静到动或由动转静的现象。属于趋势型因子
tmp['MOM'] = talib.MOM(closing, timeperiod=10)

# 归一化平均值范围
tmp['NATR'] = talib.NATR(highest, lowest, closing, timeperiod=14)

# OBV 	能量潮指标（On Balance Volume，OBV），以股市的成交量变化来衡量股市的推动力，
# 从而研判股价的走势。属于成交量型因子
tmp['OBV'] = talib.OBV(closing, volume)

# PLUS_DI 更向指示器
tmp['PLUS_DI'] = talib.PLUS_DI(highest, lowest, closing, timeperiod=14)
tmp['PLUS_DM'] = talib.PLUS_DM(highest, lowest, timeperiod=14)

# PPO 价格振荡百分比
tmp['PPO'] = talib.PPO(closing, fastperiod=6, slowperiod=26, matype=0)

# ROC 6日变动速率（Price Rate of Change），以当日的收盘价和N天前的收盘价比较，
# 通过计算股价某一段时间内收盘价变动的比例，应用价格的移动比较来测量价位动量。属于超买超卖型因子。
tmp['ROC6'] = talib.ROC(closing, timeperiod=6)
tmp['ROC20'] = talib.ROC(closing, timeperiod=20)
# 12日量变动速率指标（Volume Rate of Change），以今天的成交量和N天前的成交量比较，
# 通过计算某一段时间内成交量变动的幅度，应用成交量的移动比较来测量成交量运动趋向，
# 达到事先探测成交量供需的强弱，进而分析成交量的发展趋势及其将来是否有转势的意愿，
# 属于成交量的反趋向指标。属于成交量型因子
tmp['VROC6'] = talib.ROC(volume, timeperiod=6)
tmp['VROC20'] = talib.ROC(volume, timeperiod=20)

# ROC 6日变动速率（Price Rate of Change），以当日的收盘价和N天前的收盘价比较，
# 通过计算股价某一段时间内收盘价变动的比例，应用价格的移动比较来测量价位动量。属于超买超卖型因子。
tmp['ROCP6'] = talib.ROCP(closing, timeperiod=6)
tmp['ROCP20'] = talib.ROCP(closing, timeperiod=20)
# 12日量变动速率指标（Volume Rate of Change），以今天的成交量和N天前的成交量比较，
# 通过计算某一段时间内成交量变动的幅度，应用成交量的移动比较来测量成交量运动趋向，
# 达到事先探测成交量供需的强弱，进而分析成交量的发展趋势及其将来是否有转势的意愿，
# 属于成交量的反趋向指标。属于成交量型因子
tmp['VROCP6'] = talib.ROCP(volume, timeperiod=6)
tmp['VROCP20'] = talib.ROCP(volume, timeperiod=20)

# RSI
tmp['RSI'] = talib.RSI(closing, timeperiod=14)

# SAR 抛物线转向
tmp['SAR'] = talib.SAR(highest, lowest, acceleration=0.02, maximum=0.2)

# TEMA
tmp['TEMA6'] = talib.TEMA(closing, timeperiod=6)
tmp['TEMA12'] = talib.TEMA(closing, timeperiod=12)
tmp['TEMA26'] = talib.TEMA(closing, timeperiod=26)

# TRANGE 真实范围
tmp['TRANGE'] = talib.TRANGE(highest, lowest, closing)

# TYPPRICE 典型价格
tmp['TYPPRICE'] = talib.TYPPRICE(highest, lowest, closing)

# TSF 时间序列预测
tmp['TSF'] = talib.TSF(closing, timeperiod=14)

# ULTOSC 极限振子
tmp['ULTOSC'] = talib.ULTOSC(highest, lowest, closing, timeperiod1=7, timeperiod2=14, timeperiod3=28)

# 威廉指标
tmp['WILLR'] = talib.WILLR(highest, lowest, closing, timeperiod=14)

# 标准化
if normalization:
factors_list = tmp.columns.tolist()[1:]

if rolling >= 26:
for i in factors_list:
tmp[i] = (tmp[i] - tmp[i].rolling(window=rolling, center=False).mean())\
/ tmp[i].rolling(window=rolling, center=False).std()
elif rolling < 26 & rolling > 0:
print('Recommended rolling range greater than 26')
elif rolling <= 0:
for i in factors_list:
tmp[i] = (tmp[i] - tmp[i].mean()) / tmp[i].std()

if drop:
tmp.dropna(inplace=True)

``````

#### env 训练环境构建

``````

In [4]:

import numpy as np
import pandas as pd

quotes = fix_data('env2_simple_HS300.csv')
quotes = high2low(quotes, '5min')
daily_quotes = high2low(quotes, '1d')

Index = quotes.index
High = quotes.high.values
Low = quotes.low.values
Close = quotes.close.values
Open = quotes.open.values
Volume = quotes.volume.values
factors = get_factors(Index, Open, Close, High, Low, Volume, rolling=188, drop=True)

daily_quotes['returns'] = np.log(daily_quotes['close'].shift(-1) / daily_quotes['close'])
daily_quotes.dropna(inplace=True)

start_date = pd.to_datetime('2011-01-12')
end_date = pd.to_datetime('2016-12-29')
daily_quotes = daily_quotes.loc[start_date:end_date]
daily_quotes = daily_quotes.iloc[5:]
factors = factors.loc[start_date:end_date]

fac_list = []
for i in range(len(daily_quotes)):
s = i * 50
e = (i + 5) * 50
f = np.array(factors.iloc[s:e])
fac_list.append(np.expand_dims(f, axis=0))

fac_array = np.concatenate(fac_list, axis=0)
shape = [fac_array.shape[0], 5, 50, fac_array.shape[2]]
fac_array = fac_array.reshape(shape)
fac_array = np.transpose(fac_array, [0, 2, 3, 1])

DATE_QUOTES = daily_quotes
DATA_FAC = fac_array

class Account(object):
def __init__(self):
self.data_close = DATE_QUOTES['close']
self.data_open = DATE_QUOTES['open']
self.data_observation = DATA_FAC
self.action_space = ['long', 'short', 'close']
self.free = 1e-4
self.reset()

def reset(self):
self.step_counter = 0
self.cash = 1e5
self.position = 0
self.total_value = self.cash + self.position
self.flags = 0
return self._get_initial_state()

def _get_initial_state(self):
return self.data_observation[0]

def get_action_space(self):
return self.action_space

def long(self):
self.flags = 1
quotes = self.data_open[self.step_counter] * 10
self.cash -= quotes * (1 + self.free)
self.position = quotes

def short(self):
self.flags = -1
quotes = self.data_open[self.step_counter] * 10
self.cash += quotes * (1 - self.free)
self.position = - quotes

def keep(self):
quotes = self.data_open[self.step_counter] * 10
self.position = quotes * self.flags

def close_long(self):
self.flags = 0
quotes = self.data_open[self.step_counter] * 10
self.cash += quotes * (1 - self.free)
self.position = 0

def close_short(self):
self.flags = 0
quotes = self.data_open[self.step_counter] * 10
self.cash -= quotes * (1 + self.free)
self.position = 0

def step_op(self, action):

if action == 'long':
if self.flags == 0:
self.long()
elif self.flags == -1:
self.close_short()
self.long()
else:
self.keep()

elif action == 'close':
if self.flags == 1:
self.close_long()
elif self.flags == -1:
self.close_short()
else:
pass

elif action == 'short':
if self.flags == 0:
self.short()
elif self.flags == 1:
self.close_long()
self.short()
else:
self.keep()
else:
raise ValueError("action should be elements of ['long', 'short', 'close']")

position = self.data_close[self.step_counter] * 10 * self.flags
reward = self.cash + position - self.total_value
self.step_counter += 1
self.total_value = position + self.cash
next_observation = self.data_observation[self.step_counter]

done = False
if self.total_value < 4000:
done = True
if self.step_counter > 1000:
done = True

return reward, next_observation, done

def step(self, action):
if action == 0:
return self.step_op('long')
elif action == 1:
return self.step_op('short')
elif action == 2:
return self.step_op('close')
else:
raise ValueError("action should be one of [0,1,2]")

``````

#### A2C util

``````

In [5]:

import tensorflow as tf
from sonnet.python.modules.basic import Linear as sntLinear
from sonnet.python.modules.conv import Conv2D as sntConv2D

def swich(input):
return input * tf.nn.sigmoid(input)

def Linear(name, output_size):
initializers = {"w": tf.truncated_normal_initializer(stddev=0.1),
"b": tf.constant_initializer(value=0.1)}
regularizers = {"w": tf.contrib.layers.l2_regularizer(scale=0.1),
"b": tf.contrib.layers.l2_regularizer(scale=0.1)}
return sntLinear(output_size,
initializers=initializers,
regularizers=regularizers,
name=name)

def Conv2D(name, output_channels, kernel_shape, stride):
initializers = {"w": tf.truncated_normal_initializer(stddev=0.1),
"b": tf.constant_initializer(value=0.1)}
regularizers = {"w": tf.contrib.layers.l2_regularizer(scale=0.1),
"b": tf.contrib.layers.l2_regularizer(scale=0.1)}
return sntConv2D(output_channels,
kernel_shape,
stride,
initializers=initializers,
regularizers=regularizers,
name=name)

``````

#### TrainableNet

``````

In [7]:

import tensorflow as tf
from sonnet.python.modules.base import AbstractModule
#from util import Linear, Conv2D, swich

class ConvNet(AbstractModule):
def __init__(self, name):
super().__init__(name=name)

def _build(self, inputs, output_size):
network = Conv2D('input_layer', 16, [8, 8], [4, 4])(inputs)
network = swich(network)
network = Conv2D('hidden_layer', 32, [4, 4], [2, 2])(network)
network = swich(network)
network = tf.contrib.layers.flatten(network)
network = Linear('final_layer', 64)(network)
network = swich(network)
return Linear('output_layer', output_size)(network)

def get_regularization(self):
return self.get_variables(tf.GraphKeys.REGULARIZATION_LOSSES)

``````

#### Access

``````

In [8]:

LEARNING_RATE = 1e-4
DECAY_RATE = .9

class Access(object):
def __init__(self, state_size, action_size):
with tf.variable_scope('Access'):
# placeholder
self.inputs = tf.placeholder(tf.float32, [None] + state_size, "states")
self.actions = tf.placeholder(tf.int32, [None], "actions")
self.targets = tf.placeholder(tf.float32, [None], "discounted_rewards")

# network interface
self.actor = ConvNet('actor')
self.critic = ConvNet('critic')
self.policy = tf.nn.softmax(self.actor(self.inputs, action_size))
self.value = self.critic(self.inputs, 1)

# global optimizer
self.optimizer_actor = tf.train.RMSPropOptimizer(
LEARNING_RATE, DECAY_RATE, name='optimizer_actor')
self.optimizer_critic = tf.train.RMSPropOptimizer(
LEARNING_RATE, DECAY_RATE, name='optimizer_critic')

# saver
var_list = self.get_trainable()
var_list = list(var_list[0] + var_list[1])
self.saver = tf.train.Saver(var_list=var_list)

def get_trainable(self):
return [self.actor.get_variables(), self.critic.get_variables()]

def save(self, sess, path):
self.saver.save(sess, path)

def restore(self, sess, path):
var_list = list(self.get_trainable()[0] + self.get_trainable()[1])
saver = tf.train.Saver(var_list=var_list)
saver.restore(sess, path)

``````

#### ConvACNet

``````

In [9]:

_EPSILON = 1e-6  # avoid nan

ENTROPY_BETA = 0.1
POLICY_BETA = 1
VALUE_BETA = 1
ACTOR_NORM_BETA = 1e-3
CRITIC_NORM_BETA = 0.1

# local network for advantage actor-critic which are also know as A2C
class ConvACNet(object):
def __init__(self, access, state_size, action_size, scope_name):
self.Access = access
self.action_size = action_size
self.action_space = list(range(action_size))

with tf.variable_scope(scope_name):
# placeholder
self.inputs = tf.placeholder(tf.float32, [None] + state_size, "states")
self.actions = tf.placeholder(tf.int32, [None], "actions")
self.targets = tf.placeholder(tf.float32, [None], "discounted_rewards")

# network interface
self.actor = ConvNet('actor')
self.critic = ConvNet('critic')
self.policy = tf.nn.softmax(self.actor(self.inputs, self.action_size))
self.value = self.critic(self.inputs, 1)
self.policy_step = tf.squeeze(self.policy, axis=0)
self.greedy_action = tf.argmax(self.policy_step)

# losses
self._build_losses()

# async framework
self._build_async_interface()

self._build_interface()
print('graph %s' % (str(scope_name)))

def _build_losses(self):
# value loss
targets = tf.expand_dims(self.targets, axis=1)
# policy loss
action_gather = tf.one_hot(self.actions, self.action_size)
policy_action = tf.reduce_sum(self.policy * action_gather,
axis=1, keep_dims=True)
log_policy_action = tf.log(policy_action + _EPSILON)
self.policy_loss = -tf.reduce_mean(
# entropy loss
entropy_loss = tf.reduce_sum(self.policy * tf.log(self.policy + _EPSILON),
axis=1, keep_dims=True)
self.entropy_loss = tf.reduce_mean(entropy_loss)
# regularization
# total loss
self.actor_loss = self.policy_loss + ENTROPY_BETA * self.entropy_loss + self.actor_norm
self.critic_loss = self.value_loss + self.critic_norm

self.a_actor_loss = self.actor_loss
self.a_policy_mean = -tf.reduce_mean(log_policy_action)
self.a_policy_loss = self.policy_loss
self.a_entropy_loss = ENTROPY_BETA * self.entropy_loss
self.a_actor_norm = self.actor_norm
self.a_critic_loss = self.critic_loss
self.a_value_loss = self.value_loss
self.a_critic_norm = self.critic_norm
self.a_value_mean = tf.reduce_mean(self.value)

def _build_interface(self):
self.a_interface = [self.a_actor_loss,
self.a_policy_mean,
self.a_policy_loss,
self.a_entropy_loss,
self.a_actor_norm,
self.a_critic_loss,
self.a_value_loss,
self.a_critic_norm,
self.a_value_mean,

def _build_async_interface(self):
global_actor_params, global_critic_params = self.Access.get_trainable()
local_actor_params, local_critic_params = self.get_trainable()
# Set up optimizer with global norm clipping.
# update Access
self.update_global = [actor_apply, critic_apply]
# update ACNet
assign_list = []
for gv, lv in zip(global_actor_params, local_actor_params):
assign_list.append(tf.assign(lv, gv))
for gv, lv in zip(global_critic_params, local_critic_params):
assign_list.append(tf.assign(lv, gv))
self.update_local = assign_list

def get_trainable(self):
return [self.actor.get_variables(), self.critic.get_variables()]

def get_policy(self, sess, inputs):
return sess.run(self.policy, {self.inputs: inputs})

def get_stochastic_action(self, sess, inputs, epsilon=0.95):
# get stochastic action for train
if np.random.uniform() < epsilon:
policy = sess.run(self.policy_step, {self.inputs: np.expand_dims(inputs, axis=0)})
return np.random.choice(self.action_space, 1, p=policy)[0]
else:
return np.random.randint(self.action_size)

def get_deterministic_policy_action(self, sess, inputs):
# get deterministic action for test
return sess.run(self.greedy_action, {self.inputs: np.expand_dims(inputs, axis=0)})

def get_value(self, sess, inputs):
return sess.run(self.value, {self.inputs: inputs})

def train_step(self, sess, inputs, actions, targets):
feed_dict = {self.inputs: inputs,
self.actions: actions,
self.targets: targets}
sess.run(self.update_global, feed_dict)

def init_network(self, sess):
"""
init or update local network
:param sess:
:return:
"""
sess.run(self.update_local)

def get_losses(self, sess, inputs, actions, targets):
"""
get all loss functions of network
:param sess:
:param inputs:
:param actions:
:param targets:
:return:
"""
feed_dict = {self.inputs: inputs,
self.actions: actions,
self.targets: targets}
return sess.run(self.a_interface, feed_dict)

``````

#### A3C Framework

``````

In [10]:

MAX_EPISODE_LENGTH = 1000
MAX_EPISODES = 1000
GAMMA = .9

class ExplorerFramework(object):
def __init__(self, access, name, observation, action_size):
self.Access = access
self.AC = ConvACNet(self.Access, observation, action_size, name)
self.env = Account()
self.name = name

def get_bootstrap(self, done, sess, next_state):
if done:
terminal = 0
else:
terminal = self.AC.get_value(
sess, np.expand_dims(next_state, axis=0))[0][0]
return terminal

def get_output(self, sess, inputs, actions, targets):
return self.AC.get_losses(sess, inputs, actions, targets)

def run(self, sess, max_episodes, t_max=32):
episode = 0
while episode < max_episodes:
episode += 1
_ = self.run_episode(sess, t_max)

def run_episode(self, sess, t_max=32):
t_start = t = 0
episode_score = 0
buffer_state = []
buffer_action = []
buffer_reward = []

self.AC.init_network(sess)
state = self.env.reset()
while True:
t += 1
action = self.AC.get_stochastic_action(sess, state)
reward, next_state, done = self.env.step(action)
# buffer for loop
episode_score += reward
buffer_state.append(state)
buffer_action.append(action)
buffer_reward.append(reward)
state = next_state

if t - t_start == t_max or done:
t_start = t
terminal = self.get_bootstrap(done, sess, next_state)

buffer_target = []
for r in buffer_reward[::-1]:
terminal = r + GAMMA * terminal
buffer_target.append(terminal)
buffer_target.reverse()

inputs = np.stack(buffer_state, axis=0)
actions = np.squeeze(np.vstack(buffer_action), axis=1)
targets = np.squeeze(np.vstack(buffer_target), axis=1)
buffer_state = []
buffer_action = []
buffer_reward = []
self.AC.train_step(sess, inputs, actions, targets)

# update local network
self.AC.init_network(sess)

if done or t > MAX_EPISODE_LENGTH:
if self.name == 'W0':
outputs = tuple(self.get_output(sess, inputs, actions, targets))
print('actor: %f, actor_grad: %f, policy mean: %f, policy: %f, entropy: %f, actor_norm: %f, '
'critic: %f, critic_grad: %f, value: %f, critic_norm: %f, value_mean: %f, advantage: %f'
% outputs)
return episode_score

``````

#### 异步调参 or main A3C框架训练虚拟交易员

``````

In [11]:

import multiprocessing
import numpy as np

NUMS_CPU = multiprocessing.cpu_count()
state_size = [50, 58, 5]
action_size = 3
max_episodes = 10
GD = {}

class Worker(ExplorerFramework):
def __init__(self, access, name, observation, action_size):
super().__init__(access, name, observation, action_size)

def run(self, sess, max_episodes, t_max=32):
episode_score_list = []
episode = 0
while episode < max_episodes:
episode += 1
episode_socre = self.run_episode(sess, t_max)
episode_score_list.append(episode_socre)
GD[str(self.name)] = episode_score_list
if self.name == 'W0':
print('Episode: %f, score: %f' % (episode, episode_socre))
print('\n')

with tf.Session() as sess:
with tf.device("/cpu:0"):
A = Access(state_size, action_size)
F_list = []
for i in range(NUMS_CPU):
F_list.append(Worker(A, 'W%i' % i, state_size, action_size))
COORD = tf.train.Coordinator()
sess.run(tf.global_variables_initializer())
sess.graph.finalize()

for ac in F_list:
job = lambda: ac.run(sess, max_episodes)
t.start()
A.save(sess, 'model/saver_1.ckpt')

``````
``````

graph W0
graph W1
graph W2
graph W3
graph W4
graph W5
graph W6
graph W7
actor: -1177.737549, actor_grad: 8884.086914, policy mean: 1.116935, policy: -1177.769775, entropy: -0.017581, actor_norm: 0.049780, critic: 383562.968750, critic_grad: 31605.031250, value: 383558.031250, critic_norm: 4.947569, value_mean: 11.702258, advantage: -312.262329
Episode: 1.000000, score: 16787.198081

actor: -841.090881, actor_grad: 10532.248047, policy mean: 0.753624, policy: -841.112793, entropy: -0.027915, actor_norm: 0.049801, critic: 940003.750000, critic_grad: 217604.812500, value: 939998.750000, critic_norm: 4.978029, value_mean: 56.190865, advantage: -811.252686
Episode: 2.000000, score: 32410.422356

actor: -855.148804, actor_grad: 15613.357422, policy mean: 0.521149, policy: -855.161560, entropy: -0.037080, actor_norm: 0.049843, critic: 2016700.625000, critic_grad: 1537697.625000, value: 2016695.500000, critic_norm: 5.062683, value_mean: 306.390656, advantage: -1318.331543
Episode: 3.000000, score: 35136.165880

actor: -205.749939, actor_grad: 10438.278320, policy mean: 0.241953, policy: -205.776779, entropy: -0.023044, actor_norm: 0.049882, critic: 322208.281250, critic_grad: 736913.750000, value: 322203.125000, critic_norm: 5.150226, value_mean: 692.951599, advantage: -295.116669
Episode: 4.000000, score: 73439.384087

actor: -313.678650, actor_grad: 8555.097656, policy mean: 0.414298, policy: -313.699188, entropy: -0.029387, actor_norm: 0.049935, critic: 378348.812500, critic_grad: 1226384.250000, value: 378343.625000, critic_norm: 5.192396, value_mean: 870.241455, advantage: -400.466156
Episode: 5.000000, score: 113897.104198

actor: -1208.263672, actor_grad: 20052.359375, policy mean: 0.705939, policy: -1208.291626, entropy: -0.022128, actor_norm: 0.049993, critic: 1218572.500000, critic_grad: 3359059.000000, value: 1218567.250000, critic_norm: 5.221607, value_mean: 969.730347, advantage: -1013.713013
Episode: 6.000000, score: 106334.782018

actor: -83.064049, actor_grad: 3190.728027, policy mean: 0.457838, policy: -83.091057, entropy: -0.023037, actor_norm: 0.050052, critic: 603505.375000, critic_grad: 948188.687500, value: 603500.062500, critic_norm: 5.288424, value_mean: 1277.071655, advantage: 41.689358
Episode: 7.000000, score: 146797.368627

actor: -31.733437, actor_grad: 2639.299316, policy mean: 0.241683, policy: -31.757759, entropy: -0.025793, actor_norm: 0.050115, critic: 411183.625000, critic_grad: 1041092.875000, value: 411178.343750, critic_norm: 5.295101, value_mean: 1213.799683, advantage: -196.832336
Episode: 8.000000, score: 129270.505542

actor: -1253.637695, actor_grad: 15902.787109, policy mean: 0.927319, policy: -1253.667969, entropy: -0.019903, actor_norm: 0.050188, critic: 1516764.000000, critic_grad: 5172120.500000, value: 1516758.625000, critic_norm: 5.335513, value_mean: 1352.247192, advantage: -1174.552856
Episode: 9.000000, score: 153602.794563

actor: -297.400757, actor_grad: 4390.443848, policy mean: 0.692185, policy: -297.436523, entropy: -0.014480, actor_norm: 0.050233, critic: 420766.718750, critic_grad: 1429039.875000, value: 420761.375000, critic_norm: 5.335571, value_mean: 1299.355835, advantage: -306.331421
Episode: 10.000000, score: 146433.930931

``````
``````

In [12]:

import pandas as pd
import seaborn as sns
%matplotlib inline

tmp = pd.DataFrame(GD)
tmp.iloc[:500,:1].plot(figsize=(16,6))

``````
``````

Out[12]:

<matplotlib.axes._subplots.AxesSubplot at 0x1bdff567a20>

``````
``````

In [13]:

tmp.plot(figsize=(16,6))

``````
``````

Out[13]:

<matplotlib.axes._subplots.AxesSubplot at 0x1bdff592518>

``````

#### 策略测试

``````

In [14]:

state_size = [50, 58, 5]
action_size = 3

tf.reset_default_graph()
with tf.Session() as sess:
with tf.device("/cpu:0"):
A = Access(state_size, action_size)
W = ConvACNet(A, state_size, action_size, 'AC')
A.restore(sess,'model/saver_1.ckpt')
W.init_network(sess)
env = Account()
state = env.reset()
net_value = []
reward_list = []
for _ in range(1400):
action = W.get_deterministic_policy_action(sess, state)
reward, state, done = env.step(action)
reward_list.append(reward)
net_value.append(env.total_value)

``````
``````

graph AC
INFO:tensorflow:Restoring parameters from model/saver_1.ckpt

``````
``````

In [15]:

pd.Series(net_value[1000:]).plot(figsize=(16,6))

``````
``````

Out[15]:

<matplotlib.axes._subplots.AxesSubplot at 0x1bdffaa47b8>

``````
``````

In [ ]:

``````