![在这里插入图片描述](https://img-blog.csdnimg.cn/20201226175726283.png?x-oss-process=image/watermark,type_ZmFuZ3poZW5naGVpdGk,shadow_10,text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzQyMTM4OTI3,size_16,color_FFFFFF,t_70#pic_center)
建立RL_brain.py
import torch
import torch.nn as nn
import torch.nn.functional as F
import numpy as np
class Net(nn.Module):
def __init__(self, N_STATES, N_ACTIONS):
nn.Module.__init__(self)
self.input_num = N_STATES
self.output_num = N_ACTIONS
self.fc1 = nn.Linear(self.input_num, 50)
self.fc1.weight.data.normal_(0, 0.1)
self.out = nn.Linear(50, self.output_num)
self.out.weight.data.normal_(0, 0.1)
def forward(self, state):
x = self.fc1(state)
x = F.relu(x)
actions_value = self.out(x)
return actions_value
class DQN(object):
def __init__(self, N_STATES, N_ACTIONS, MEMORY_CAPACITY = 2000, EPSILON = 0.9, LR = 0.1, BATCH_SIZE = 32 , TARGET_REPLACE_ITER = 100, GAMMA = 0.8):
self.n_states = N_STATES
self.n_actions = N_ACTIONS
self.memory_capacity = MEMORY_CAPACITY
self.epsilon = EPSILON
self.batch_size = BATCH_SIZE
self.taget_replace_iter = TARGET_REPLACE_ITER
self.gamma = GAMMA
net = Net(N_STATES, N_ACTIONS)
self.eval_net, self.target_net = net, net
self.learn_step_counter = 0
self.memory_counter = 0
self.memory = np.zeros((MEMORY_CAPACITY, N_STATES * 2 + 2), dtype=float)
self.optimizer = torch.optim.Adam(self.eval_net.parameters(), lr=LR)
self.loss_func = nn.MSELoss()
def choose_action(self, state):
state = torch.FloatTensor(state)
state = torch.unsqueeze(state, 0)
if np.random.uniform() < self.epsilon:
actions_value = self.eval_net.forward(state)
action = torch.max(actions_value, 1)[1].numpy()
action = action[0]
else:
action = np.random.choice(self.n_actions)
return action
def store_transition(self, s, a, r, s_):
transition = np.hstack((s, a, r, s_))
index = self.memory_counter % self.memory_capacity
self.memory[index, :] = transition
self.memory_counter += 1
def learn(self):
if self.learn_step_counter % self.taget_replace_iter == 0:
self.target_net.load_state_dict(self.eval_net.state_dict())
self.learn_step_counter += 1
sample_index = np.random.choice(self.memory_capacity, self.batch_size)
batch_memory = self.memory[sample_index, :]
b_s = torch.FloatTensor(batch_memory[:, :self.n_states])
b_a = torch.LongTensor(batch_memory[:, self.n_states:self.n_states+1].astype(int))
b_r = torch.FloatTensor(batch_memory[:, self.n_states+1:self.n_states+2])
b_s_ = torch.FloatTensor(batch_memory[:, -self.n_states:])
q_eval = self.eval_net(b_s).gather(1, b_a)
q_next = self.target_net(b_s_).detach()
q_target = b_r + self.gamma * q_next.max(1)[0].view(self.batch_size, 1)
loss = self.loss_func(q_eval, q_target)
self.optimizer.zero_grad()
loss.backward()
self.optimizer.step()
import time
import gym
from DQN_brain import *
start = time.time()
env = gym.make('CartPole-v1')
N_ACTIONS = env.action_space.n
N_STATES = env.observation_space.shape[0]
MEMORY_CAPACITY = 1000
dqn = DQN(N_STATES, N_ACTIONS, MEMORY_CAPACITY)
for i_episode in range(400):
s = env.reset()
ep_r = 0
for t in range(1000):
env.render()
a = dqn.choose_action(s)
s_, r, done, info = env.step(a)
x, x_dot, theta, theta_dot = s_
r1 = (env.x_threshold - abs(x)) / env.x_threshold - 0.8
r2 = (env.theta_threshold_radians - abs(theta)) / env.theta_threshold_radians - 0.5
r = r1 + r2
dqn.store_transition(s, a, r, s_)
ep_r += r
if dqn.memory_counter == MEMORY_CAPACITY:
print("记忆库已经满,开始学习")
if dqn.memory_counter > MEMORY_CAPACITY:
dqn.learn()
if done:
print('Ep: ', i_episode, '| Ep_r: ', ep_r)
if done:
break
s = s_
env.close()
interval = time.time() - start
print("Time: %.4f" % interval)
本文内容由网友自发贡献,版权归原作者所有,本站不承担相应法律责任。如您发现有涉嫌抄袭侵权的内容,请联系:hwhale#tublm.com(使用前将#替换为@)