作为刚入门强化学习的小白，最近几天在写一些基础的代码，使用DQN训练CartPole问题。
DQN是2013年DeepMind提出来的使用Q-learning与神经网络相结合的方法，其实和Q-learning的思想相同，只不过是计算的时候使用神经网络计算Q值。Q-learning简要说一下，就是使用函数逼近的方法，在选择动作时使用epsilon-greedy的方法，在更新Q函数的时候使用Qmax。
这份代码参考了：
https://blog.****.net/xinbolai1993/article/details/78280732 博主讲的很详细，感谢~
再插一句，初始化参数和调参也是一个技术活，会影响到更新的速度，我一开始b初始化为0.01，参数更新步长为0.001，结果得2000个episode才能训练完（Average Reward=200），但是我把b初始化为0.1，更新步长为0.005，平均600个episode就能训练好。
全部代码如下：
import tensorflow as tf
import numpy as np
from collections import deque
import random
import gym
EPISODE = 10000
class DeepQNetwork:
learning_rate = 0.001
gamma = 0.9
action_list = None
# 执行步数
step_index = 0
# epsilon的范围
initial_epsilon = 0.5
final_epsilon = 0.01
explore = 10000
# 经验回放存储
memory_size = 10000
BATCH = 32
# 神经网络
state_input = None
Q_val = None
y_input = None
optimizer = None
cost = 0
session = tf.Session()
cost_history = []
def __init__(self, env):
self.replay_memory_store = deque()
self.state_dim = env.observation_space.shape[0]
self.action_dim = env.action_space.n
self.action_list = np.identity(self.action_dim)
self.epsilon = self.initial_epsilon # epsilon_greedy-policy
self.create_network()
self.create_training_method()
self.session = tf.InteractiveSession()
self.session.run(tf.global_variables_initializer())
def print_loss(self):
import matplotlib.pyplot as plt
print(len(self.cost_history))
plt.plot(np.arange(len(self.cost_history)), self.cost_history)
plt.ylabel('Cost')
plt.xlabel('step')
plt.show()
def create_network(self):
self.state_input = tf.placeholder(shape=[None, self.state_dim], dtype=tf.float32)
# 第一层
neuro_layer_1 = 20
w1 = tf.Variable(tf.random_normal([self.state_dim, neuro_layer_1]))
b1 = tf.Variable(tf.zeros([neuro_layer_1]) + 0.1)
l1 = tf.nn.relu(tf.matmul(self.state_input, w1) + b1)
# 第二层
w2 = tf.Variable(tf.random_normal([neuro_layer_1, self.action_dim]))
b2 = tf.Variable(tf.zeros([self.action_dim]) + 0.1)
# 输出层
self.Q_val = tf.matmul(l1, w2) + b2
def egreedy_action(self, state):
self.epsilon -= (0.5 - 0.01) / 10000
Q_val_output = self.session.run(self.Q_val, feed_dict={self.state_input: [state]})[0]
if random.random() <= self.epsilon:
return random.randint(0, self.action_dim - 1) # 左闭右闭区间，np.random.randint为左闭右开区间
else:
return np.argmax(Q_val_output)
def max_action(self, state):
Q_val_output = self.session.run(self.Q_val, feed_dict={self.state_input: [state]})[0]
action = np.argmax(Q_val_output)
return action
def create_training_method(self):
self.action_input = tf.placeholder(shape=[None, self.action_dim], dtype=tf.float32)
self.y_input = tf.placeholder(shape=[None], dtype=tf.float32) # ???是[None]吗？
Q_action = tf.reduce_sum(tf.multiply(self.Q_val, self.action_input), reduction_indices=1)
self.loss = tf.reduce_mean(tf.square(self.y_input - Q_action))
self.optimizer = tf.train.AdamOptimizer(0.0005).minimize(self.loss)
def perceive(self, state, action, reward, next_state, done):
cur_action = self.action_list[action:action + 1]
self.replay_memory_store.append((state, cur_action[0], reward, next_state, done))
if len(self.replay_memory_store) > self.memory_size:
self.replay_memory_store.popleft()
if len(self.replay_memory_store) > self.BATCH:
self.train_Q_network()
def train_Q_network(self):
self.step_index += 1
# obtain random mini-batch from replay memory
mini_batch = random.sample(self.replay_memory_store, self.BATCH)
state_batch = [data[0] for data in mini_batch]
action_batch = [data[1] for data in mini_batch]
reward_batch = [data[2] for data in mini_batch]
next_state_batch = [data[3] for data in mini_batch]
# calculate y
y_batch = []
Q_val_batch = self.session.run(self.Q_val, feed_dict={self.state_input: next_state_batch}) # 预估下一个状态的Q值
for i in range(0, self.BATCH):
done = mini_batch[i][4]
if done:
y_batch.append(reward_batch[i])
else:
y_batch.append(reward_batch[i] + self.gamma * np.max(Q_val_batch[i])) # 选择最优的Q函数进行更新
_, self.cost = self.session.run([self.optimizer, self.loss], feed_dict={
self.y_input: y_batch,
self.state_input: state_batch,
self.action_input: action_batch
})
TEST = 10
STEP = 300
def main():
env = gym.make('CartPole-v0')
agent = DeepQNetwork(env)
for i in range(EPISODE):
# if i % 50 == 0:
# env.render()
# print(i)
state = env.reset()
for step in range(STEP):
# env.render()
action = agent.egreedy_action(state)
next_state, reward, done, _ = env.step(action)
agent.perceive(state, action, reward, next_state, done)
state = next_state
if done:
break
if i % 100 == 0:
total_reward = 0
for _ in range(TEST):
state = env.reset()
for _ in range(STEP):
env.render()
action = agent.max_action(state) # direct action for test
state, reward, done, _ = env.step(action)
total_reward += reward
if done:
break
ave_reward = total_reward / TEST
print('episode: ', i, 'Evaluation Average Reward:', ave_reward)
if ave_reward >= 200:
break
# agent.print_loss()
if __name__ == '__main__':
main()
训练结果如下：
