Modify the code below to implement a robust reinforcement learning algorithm for the…
- User
Modify the code below to implement a robust reinforcement learning algorithm for the continuous action space of the pendulum environment
from torch import nn import gymnasium as gym import torch import torch.optim as optim class NeuralNetwork(nn.Module): def __init__(self): super(NeuralNetwork, self).__init__() self.linear_stack = nn.Sequential( nn.Linear(3, 11), nn.LeakyReLU(), nn.Linear(11, 1), ) def forward(self, x): x = self.linear_stack(x) return torch.clamp(x, -2, 2) def train(model): optimizer = optim.AdamW(model.parameters(), lr=0.01) env = gym.make("Pendulum-v1") for episode in range(300): # Number of episodes to train on observation, info = env.reset() obs_tensor = torch.tensor(observation, dtype=torch.float32) # ... for step in range(500): # Max steps per episode # Forward pass action_tensor = model(obs_tensor) action = action_tensor.detach().numpy() # Step in the environment observation, reward, terminated, truncated, info = env.step(action) obs_tensor = torch.tensor(observation, dtype=torch.float32) # prepare loss # ... if terminated or truncated: break # compute loss # ... # Perform backpropagation optimizer.zero_grad() loss.backward() optimizer.step() print(f"Episode {episode + 1}\tLoss: {loss}") env.close() def main(): model = NeuralNetwork() # model.load_state_dict(torch.load("pendulum_1.pt", weights_only=True)) train(model) torch.save(model.state_dict(), "pendulum_1.pt") env = gym.make("Pendulum-v1", render_mode="human") observation, info = env.reset() for step in range(200): obs_tensor = torch.tensor(observation, dtype=torch.float32) with torch.no_grad(): action_tensor = model(obs_tensor) action = action_tensor.detach().numpy() observation, reward, terminated, truncated, info = env.step(action) if terminated or truncated: observation, info = env.reset() env.close() if __name__ == "__main__": main() - Assistant
To implement a robust reinforcement learning algorithm for the continuous action space of the Pendulum environment, we will use the Proximal Policy Optimization (PPO) algorithm. PPO is a state-of-the-art policy gradient method known for its stability and sample efficiency, making it suitable for continuous action spaces like that of the Pendulum environment.
Below is the modified code implementing PPO:
import numpy as np import torch from torch import nn import gymnasium as gym import torch.optim as optim class PolicyNetwork(nn.Module): def __init__(self, state_dim, action_dim, hidden_dim=64): super(PolicyNetwork, self).__init__() self.fc1 = nn.Linear(state_dim, hidden_dim) self.fc2 = nn.Linear(hidden_dim, hidden_dim) self.mean_head = nn.Linear(hidden_dim, action_dim) self.log_std = nn.Parameter(torch.zeros(action_dim)) # Learnable log_std parameter def forward(self, state): x = torch.tanh(self.fc1(state)) x = torch.tanh(self.fc2(x)) mean = self.mean_head(x) std = self.log_std.exp() return mean, std class ValueNetwork(nn.Module): def __init__(self, state_dim, hidden_dim=64): super(ValueNetwork, self).__init__() self.fc1 = nn.Linear(state_dim, hidden_dim) self.fc2 = nn.Linear(hidden_dim, hidden_dim) self.value_head = nn.Linear(hidden_dim, 1) def forward(self, state): x = torch.tanh(self.fc1(state)) x = torch.tanh(self.fc2(x)) value = self.value_head(x) return value def compute_gae(next_value, rewards, masks, values, gamma=0.99, lam=0.95): values = values + [next_value] gae = 0 returns = [] for step in reversed(range(len(rewards))): delta = rewards[step] + gamma * values[step + 1] * masks[step] - values[step] gae = delta + gamma * lam * masks[step] * gae returns.insert(0, gae + values[step]) return returns def train(): env = gym.make("Pendulum-v1") state_dim = env.observation_space.shape[0] action_dim = env.action_space.shape[0] policy_net = PolicyNetwork(state_dim, action_dim) value_net = ValueNetwork(state_dim) policy_optimizer = optim.Adam(policy_net.parameters(), lr=3e-4) value_optimizer = optim.Adam(value_net.parameters(), lr=1e-3) num_episodes = 300 max_steps_per_episode = 200 gamma = 0.99 lam = 0.95 clip_epsilon = 0.2 c1 = 0.5 # Value function coefficient c2 = 0.01 # Entropy coefficient for episode in range(num_episodes): state, _ = env.reset() states = [] actions = [] log_probs = [] rewards = [] masks = [] values = [] entropy_term = 0 for step in range(max_steps_per_episode): state_tensor = torch.FloatTensor(state).unsqueeze(0) mean, std = policy_net(state_tensor) dist = torch.distributions.Normal(mean, std) action = dist.sample() action_log_prob = dist.log_prob(action).sum(dim=1) entropy = dist.entropy().sum(dim=1) value = value_net(state_tensor) action_to_env = action.detach().numpy()[0] action_clipped = np.clip(action_to_env, env.action_space.low, env.action_space.high) next_state, reward, done, truncated, _ = env.step(action_clipped) states.append(state) actions.append(action) log_probs.append(action_log_prob) rewards.append(reward) masks.append(1 - float(done)) values.append(value.squeeze(0)) entropy_term += entropy.mean() if done or truncated: break state = next_state next_state_tensor = torch.FloatTensor(next_state).unsqueeze(0) next_value = value_net(next_state_tensor).detach() returns = compute_gae(next_value, rewards, masks, values, gamma=gamma, lam=lam) returns = torch.cat(returns).detach() log_probs = torch.cat(log_probs).detach() values = torch.cat(values) states_tensor = torch.FloatTensor(states) actions_tensor = torch.cat(actions) advantages = returns - values # Normalize advantages advantages = (advantages - advantages.mean()) / (advantages.std() + 1e-10) # PPO policy and value updates optim_epochs = 10 optim_batch_size = 64 total_steps = len(rewards) for _ in range(optim_epochs): perm = np.arange(total_steps) np.random.shuffle(perm) for i in range(0, total_steps, optim_batch_size): idx = perm[i:i+optim_batch_size] sampled_states = states_tensor[idx] sampled_actions = actions_tensor[idx] sampled_log_probs = log_probs[idx] sampled_returns = returns[idx] sampled_advantages = advantages[idx] mean, std = policy_net(sampled_states) dist = torch.distributions.Normal(mean, std) new_log_probs = dist.log_prob(sampled_actions).sum(dim=1) entropy = dist.entropy().sum(dim=1).mean() ratio = (new_log_probs - sampled_log_probs).exp() surr1 = ratio * sampled_advantages surr2 = torch.clamp(ratio, 1 - clip_epsilon, 1 + clip_epsilon) * sampled_advantages policy_loss = -torch.min(surr1, surr2).mean() - c2 * entropy value_estimates = value_net(sampled_states).squeeze(1) value_loss = c1 * (sampled_returns - value_estimates).pow(2).mean() loss = policy_loss + value_loss policy_optimizer.zero_grad() value_optimizer.zero_grad() loss.backward() policy_optimizer.step() value_optimizer.step() total_reward = np.sum(rewards) print(f"Episode {episode + 1}\tTotal Loss: {loss.item():.3f}\tTotal Reward: {total_reward:.3f}") env.close() # Save models torch.save(policy_net.state_dict(), "pendulum_policy.pt") torch.save(value_net.state_dict(), "pendulum_value.pt") def main(): train() # Load the trained policy network env = gym.make("Pendulum-v1", render_mode="human") state_dim = env.observation_space.shape[0] action_dim = env.action_space.shape[0] policy_net = PolicyNetwork(state_dim, action_dim) policy_net.load_state_dict(torch.load("pendulum_policy.pt")) state, _ = env.reset() for step in range(200): state_tensor = torch.FloatTensor(state).unsqueeze(0) with torch.no_grad(): mean, std = policy_net(state_tensor) action = mean # Use deterministic action (mean of the distribution) during testing action_to_env = action.numpy()[0] action_clipped = np.clip(action_to_env, env.action_space.low, env.action_space.high) next_state, reward, done, truncated, _ = env.step(action_clipped) if done or truncated: state, _ = env.reset() else: state = next_state env.close() if __name__ == "__main__": main()Explanation of the Modifications:
1. Policy and Value Networks:
- PolicyNetwork: Outputs the mean and standard deviation of a Gaussian distribution for actions. It uses a learnable
log_stdparameter for the standard deviation, allowing the network to adjust exploration. - ValueNetwork: Estimates the value function ( V(s) ) for a given state, which is essential for computing advantages.
2. Training Loop:
- Data Collection: For each episode, the agent interacts with the environment, and we collect states, actions, rewards, log probabilities, and values.
- Advantage Estimation: We use Generalized Advantage Estimation (GAE) to compute advantages, which balances bias and variance for more stable training.
- Policy and Value Updates: We perform multiple epochs of optimization using the PPO objective, which includes:
- Clipped Surrogate Objective: Ensures that policy updates are within a small trusted region to prevent large updates that could destabilize training.
- Entropy Bonus: Encourages exploration by penalizing low-entropy policies.
- Value Function Loss: Uses Mean Squared Error between the predicted values and the computed returns.
3. Main Function:
- After training, the policy network is saved and then loaded for testing.
- During testing, we use the deterministic policy (mean action) instead of sampling from the distribution to evaluate the learned behavior.
Hyperparameters:
- Learning Rates: Set different learning rates for the policy and value networks (
3e-4for policy and1e-3for value) as they may require different update steps. - Clip Epsilon (
epsilon): The PPO clipping parameter is set to0.2, which defines how much the new policy can deviate from the old one. - Optimization Epochs and Batch Size: We perform 10 optimization epochs per episode with a batch size of 64.
Running the Code:
To run the modified code, ensure you have the necessary libraries installed:
pip install torch gymnasium numpyThen, execute the script:
python script_name.pyReplace
script_name.pywith the name of your Python file. The agent will train for 300 episodes, and then the trained agent will perform in the environment with rendering enabled.Expected Results:
During training, you should observe the total loss decreasing and the total reward increasing over episodes. The trained agent should learn to balance the pendulum upright for longer durations, maximizing the cumulative reward.
- PolicyNetwork: Outputs the mean and standard deviation of a Gaussian distribution for actions. It uses a learnable