import gym import numpy as np import matplotlib.pyplot as plt class GridWorld(gym.Env):…
- User
import gym import numpy as np import matplotlib.pyplot as plt
class GridWorld(gym.Env): def init(self): self.grid = np.array([ [-1, -1, -1, 0], [-1, 0, 0, 0], [-1, 0, 0, 0], [-1, -1, -1, 10] ])
self.observation_space = gym.spaces.Discrete(16) self.action_space = gym.spaces.Discrete(4) self.reward_range = (-5, 10) self.agent_pos = (0, 0) self.rewards = [] def step(self, action): # Get the current state and action current_state = self.agent_pos action = action # Calculate the next state if action == 0: # up next_state = (current_state[0] - 1, current_state[1]) elif action == 1: # down next_state = (current_state[0] + 1, current_state[1]) elif action == 2: # left next_state = (current_state[0], current_state[1] - 1) elif action == 3: # right next_state = (current_state[0], current_state[1] + 1) # Check if the next state is valid if next_state[0] < 0 or next_state[0] >= self.grid.shape[0] or next_state[1] < 0 or next_state[1] >= self.grid.shape[1]: next_state = current_state # Get the reward reward = self.grid[next_state[0]][next_state[1]] # Check if the episode is done done = reward == 10 # Update the agent's position self.agent_pos = next_state # Return the next state, reward, and done return next_state, reward, done def reset(self): # Reset the agent's position to the start self.agent_pos = (0, 0) return self.agent_pos def render(self): plt.figure(figsize=(5, 5)) # Add grid lines for each cell plt.grid(True, which='both', color='black', linewidth=5) plt.imshow(self.grid, cmap='viridis') plt.xticks([]) plt.yticks([]) # Mark start and end positions with text labels plt.text(0, 0, 'START', horizontalalignment='center', verticalalignment='center', fontsize=14, color='white') plt.text(3, 3, 'END', horizontalalignment='center', verticalalignment='center', fontsize=14, color='white') # Mark agent position with a red circle plt.scatter([self.agent_pos[1]], [self.agent_pos[0]], c='red', s=100) plt.show() def plot_rewards(self): plt.plot(self.rewards) plt.title("Rewards per Episode") plt.xlabel("Episode") plt.ylabel("Reward") plt.show() ------------------------------------------------environment--------------------------------------------------------------------------------------------------------------------------------import gym import numpy as np
Initialize environment, Q-table, and hyperparameters
env = GridWorld() n_states = env.observation_space.n n_actions = env.action_space.n Q = np.zeros((n_states, n_actions))
SARSA hyperparameters
episodes = 500 alpha = 0.1 gamma = 0.99 epsilon = 1.0 epsilon_decay = 0.995 epsilon_min = 0.01 max_timesteps = 100
SARSA Algorithm
total_rewards = [] for episode in range(episodes): state_tuple = env.reset() state = state_tuple[0] * 4 + state_tuple[1] # Convert the state tuple to single integer action = np.argmax(Q[state, :] + np.random.randn(1, n_actions) * (1.0 / (episode + 1)))
total_reward = 0 for t in range(max_timesteps): next_state_tuple, reward, done = env.step(action) next_state = next_state_tuple[0] * 4 + next_state_tuple[1] # Convert the next_state tuple to single integer next_action = np.argmax(Q[next_state, :] + np.random.randn(1, n_actions) * (1.0 / (episode + 1))) # Update Q-value target = reward + gamma * Q[next_state, next_action] Q[state, action] += alpha * (target - Q[state, action]) state, action = next_state, next_action total_reward += reward if done: break epsilon = max(epsilon_min, epsilon * epsilon_decay) total_rewards.append(total_reward)--------------------I have implemented saarsa like this -------------#Plot the SARSA algorithm results:
plt.plot(total_rewards) plt.xlabel('Episode') plt.ylabel('Total Reward per Episode') plt.title('SARSA: Total Reward per Episode') plt.show()----------------------------------------------------------------------------------------------------------------------------I also need the graph for epsilon delay ----------------------------------------• • Applying SARSA to solve the environment defined in Part 1. Plots should include epsilon decay and total reward per episode. --give code to Run your environment for at least 10 episodes, where the agent chooses only greedy actions from the learned policy. Plot should include the total reward per episode.
- Assistant
You can plot the epsilon decay graph by adding a list to store the epsilon value in each episode and then plotting that list along with the total rewards per episode.
Here is the modified code that includes a list to store the epsilon values:
import gym import numpy as np # Initialize environment, Q-table, and hyperparameters env = GridWorld() n_states = env.observation_space.n n_actions = env.action_space.n Q = np.zeros((n_states, n_actions)) # SARSA hyperparameters episodes = 500 alpha = 0.1 gamma = 0.99 epsilon = 1.0 epsilon_decay = 0.995 epsilon_min = 0.01 max_timesteps = 100 # Lists for plotting epsilon decay and total rewards total_rewards = [] epsilon_values = [] # SARSA Algorithm for episode in range(episodes): state_tuple = env.reset() state = state_tuple[0] * 4 + state_tuple[1] # Convert the state tuple to single integer action = np.argmax(Q[state, :] + np.random.randn(1, n_actions) * (1.0 / (episode + 1))) total_reward = 0 for t in range(max_timesteps): next_state_tuple, reward, done = env.step(action) next_state = next_state_tuple[0] * 4 + next_state_tuple[1] # Convert the next_state tuple to single integer next_action = np.argmax(Q[next_state, :] + np.random.randn(1, n_actions) * (1.0 / (episode + 1))) # Update Q-value target = reward + gamma * Q[next_state, next_action] Q[state, action] += alpha * (target - Q[state, action]) state, action = next_state, next_action total_reward += reward if done: break epsilon = max(epsilon_min, epsilon * epsilon_decay) epsilon_values.append(epsilon) total_rewards.append(total_reward) # Plot epsilon decay plt.plot(epsilon_values) plt.xlabel('Episode') plt.ylabel('Epsilon') plt.title('SARSA: Epsilon Decay per Episode') plt.show() # Plot total rewards plt.plot(total_rewards) plt.xlabel('Episode') plt.ylabel('Total Reward per Episode') plt.title('SARSA: Total Reward per Episode') plt.show()For running the environment for at least 10 episodes with only greedy actions and plotting the total reward per episode, you can create a separate loop and store the rewards in a list:
greedy_rewards = [] greedy_episodes = 10 for episode in range(greedy_episodes): state_tuple = env.reset() state = state_tuple[0] * 4 + state_tuple[1] total_reward = 0 for t in range(max_timesteps): action = np.argmax(Q[state, :]) next_state_tuple, reward, done = env.step(action) next_state = next_state_tuple[0] * 4 + next_state_tuple[1] state = next_state total_reward += reward if done: break greedy_rewards.append(total_reward) # Plot greedy rewards plt.plot(greedy_rewards) plt.xlabel('Episode') plt.ylabel('Total Reward per Episode') plt.title('Greedy Policy: Total Reward per Episode') plt.show()This will run the environment with the learned policy and plot the total reward per episode for 10 episodes using only greedy actions.
- User
Now that i have implemented saarsa s you said it really worked well , Now i need to implement Part III: Solve your environment using Qlearning [30 points] In this part, we apply Q-learning algorithm to solve the env defined in Part 1. Q-learning is an off-policy reinforcement learning algorithm. It has many similarities with SARSA. The algorithm involves taking action in the current state, observing the reward and next state, and updating the Q-value estimate of the previous state-action pair based on the maximum expected Q-value of the next state. STEPS:
- Apply Q-learning algorithm to solve the environment that was defined in Part I. You can modify your code from Part II.
- Try hyperparameter tuning on at least two parameters to get better results for Qlearning. You explore some of the hyperparameter tuning libraries, e.g. Optuna or make it manually. Parameters to tune: f. Discount factor (γ) g. Epsilon decay rate h. Epsilon min/max values i. Number of episodes j. Max timesteps Try at least 3 different values for each of the parameters that you choose.
- Provide the reward graphs and your explanation for each result. In total, you should have at least 3 graphs and your explanations. Make your suggestion on the most efficient hyperparameters values for your problem setup-------------------------------------------------------------------------------------------------------------------------------Applying Q-learning to solve the environment defined in Part 1. code to plot epsilon decay and total reward per episode. • code to Run your environment for at least 10 episodes, where the agent chooses only greedy actions from the learned policy. Plot should include the total reward per episode.
- Assistant
- Apply Q-learning algorithm to solve the environment that was defined in Part I:
import gym import numpy as np import matplotlib.pyplot as plt # Initialize environment, Q-table, and hyperparameters env = GridWorld() n_states = env.observation_space.n n_actions = env.action_space.n Q = np.zeros((n_states, n_actions)) # Q-Learning hyperparameters episodes = 500 alpha = 0.1 gamma = 0.99 epsilon = 1.0 epsilon_decay = 0.995 epsilon_min = 0.01 max_timesteps = 100 # Lists for plotting epsilon decay and total rewards total_rewards = [] epsilon_values = [] # Q-Learning Algorithm for episode in range(episodes): state_tuple = env.reset() state = state_tuple[0] * 4 + state_tuple[1] # Convert the state tuple to single integer total_reward = 0 for t in range(max_timesteps): action = np.argmax(Q[state, :] + np.random.randn(1, n_actions) * (1.0 / (episode + 1))) next_state_tuple, reward, done = env.step(action) next_state = next_state_tuple[0] * 4 + next_state_tuple[1] # Convert the next_state tuple to single integer # Update Q-value target = reward + gamma * np.max(Q[next_state, :]) Q[state, action] += alpha * (target - Q[state, action]) state = next_state total_reward += reward if done: break epsilon = max(epsilon_min, epsilon * epsilon_decay) epsilon_values.append(epsilon) total_rewards.append(total_reward) # Plot epsilon decay plt.plot(epsilon_values) plt.xlabel('Episode') plt.ylabel('Epsilon') plt.title('Q-Learning: Epsilon Decay per Episode') plt.show() # Plot total rewards plt.plot(total_rewards) plt.xlabel('Episode') plt.ylabel('Total Reward per Episode') plt.title('Q-Learning: Total Reward per Episode') plt.show()- Run the environment with only greedy actions for at least 10 episodes and plot the total rewards per episode:
greedy_rewards = [] greedy_episodes = 10 for episode in range(greedy_episodes): state_tuple = env.reset() state = state_tuple[0] * 4 + state_tuple[1] total_reward = 0 for t in range(max_timesteps): action = np.argmax(Q[state, :]) next_state_tuple, reward, done = env.step(action) next_state = next_state_tuple[0] * 4 + next_state_tuple[1] state = next_state total_reward += reward if done: break greedy_rewards.append(total_reward) # Plot greedy rewards plt.plot(greedy_rewards) plt.xlabel('Episode') plt.ylabel('Total Reward per Episode') plt.title('Greedy Policy: Total Reward per Episode') plt.show()- For hyperparameter tuning, you can manually change the hyperparameters in the Q-Learning algorithm, such as:
- Discount factor (gamma): Change the value in the range of (0, 1), e.g. 0.9, 0.95, 0.99, etc.
- Epsilon decay rate: Change the value in the range of (0, 1), e.g. 0.995, 0.99, 0.98, etc.
- Epsilon min/max values: Change the initial epsilon value and epsilon_min value, e.g. epsilon=1.0 and epsilon_min=0.05 or epsilon=0.9 and epsilon_min=0.01, etc.
- Number of episodes: Change the episodes value to the desired number, e.g. 500, 1000, 1500, etc.
- Max timesteps: Change the max_timesteps value to the desired number, e.g. 100, 200, 300, etc.
After changing the hyperparameters, run the Q-Learning algorithm, plot the resulting graphs and compare the results. The most efficient hyperparameter values will depend on the problem setup and the results obtained. You can also use libraries like Optuna for optimizing the hyperparameters.
Please note that running multiple variations of hyperparameters and analyzing their results is beyond the scope of this answer. You should try it on your own and compare the results to find the best set of hyperparameters for your problem.
- User
I have to write the code for all the hyperparameters, which is a bit lengthy, can you give me a code with the hyperparameters change dynamic code
- Assistant
Certainly! Here's an example of how you can create a function called
run_q_learningthat takes hyperparameters as arguments and returns the learned Q-table and total rewards for each episode. You can use this function to try different hyperparameter combinations more conveniently without repeating the main Q-Learning code.import gym import numpy as np import matplotlib.pyplot as plt def run_q_learning(episodes, alpha, gamma, epsilon, epsilon_decay, epsilon_min, max_timesteps): # Initialize environment, Q-table, and hyperparameters env = GridWorld() n_states = env.observation_space.n n_actions = env.action_space.n Q = np.zeros((n_states, n_actions)) # Lists for plotting epsilon decay and total rewards total_rewards = [] epsilon_values = [] # Q-Learning Algorithm for episode in range(episodes): state_tuple = env.reset() state = state_tuple[0] * 4 + state_tuple[1] # Convert the state tuple to single integer total_reward = 0 for t in range(max_timesteps): action = np.argmax(Q[state, :] + np.random.randn(1, n_actions) * (1.0 / (episode + 1))) next_state_tuple, reward, done = env.step(action) next_state = next_state_tuple[0] * 4 + next_state_tuple[1] # Convert the next_state tuple to single integer # Update Q-value target = reward + gamma * np.max(Q[next_state, :]) Q[state, action] += alpha * (target - Q[state, action]) state = next_state total_reward += reward if done: break epsilon = max(epsilon_min, epsilon * epsilon_decay) epsilon_values.append(epsilon) total_rewards.append(total_reward) return Q, total_rewards, epsilon_valuesNow you can run this function with different hyperparameter combinations and analyze the results. For example:
# Example 1: Default hyperparameters Q1, rewards1, epsilon_values1 = run_q_learning(episodes=500, alpha=0.1, gamma=0.99, epsilon=1.0, epsilon_decay=0.995, epsilon_min=0.01, max_timesteps=100) # Example 2: Different discount factor and epsilon decay rate Q2, rewards2, epsilon_values2 = run_q_learning(episodes=500, alpha=0.1, gamma=0.95, epsilon=1.0, epsilon_decay=0.99, epsilon_min=0.01, max_timesteps=100) # Example 3: Larger number of episodes and max timesteps Q3, rewards3, epsilon_values3 = run_q_learning(episodes=1000, alpha=0.1, gamma=0.99, epsilon=1.0, epsilon_decay=0.995, epsilon_min=0.01, max_timesteps=200) # Plot rewards and epsilon values for each example plt.figure(figsize=(12, 6)) plt.subplot(2, 3, 1) plt.plot(rewards1) plt.title("Example 1: Rewards per Episode") plt.subplot(2, 3, 2) plt.plot(rewards2) plt.title("Example 2: Rewards per Episode") plt.subplot(2, 3, 3) plt.plot(rewards3) plt.title("Example 3: Rewards per Episode") plt.subplot(2, 3, 4) plt.plot(epsilon_values1) plt.title("Example 1: Epsilon Decay per Episode") plt.subplot(2, 3, 5) plt.plot(epsilon_values2) plt.title("Example 2: Epsilon Decay per Episode") plt.subplot(2, 3, 6) plt.plot(epsilon_values3) plt.title("Example 3: Epsilon Decay per Episode") plt.tight_layout() plt.show()This will allow you to easily try different hyperparameter combinations and analyze the impact on rewards and epsilon values. I encourage you to try additional combinations and analyze their resulting reward graphs and epsilon decay graphs to find the most efficient set of hyperparameters for your particular problem setup.
- User
It really worked, but now I need -------------------------three parameters to change ---------------------------------------------------------------------------------------------------------------case 1 default parameters -- Q1, rewards1, epsilon_values1 = run_q_learning(episodes=500, alpha=0.1, gamma=0.99, epsilon=1.0, epsilon_decay=0.995, epsilon_min=0.01, max_timesteps=100)
-- case 2 -- Gamma - a. 0.4, 0.8, 1 both epsion delay and total rewards graph ----case 3 - max_time_steps -- 25, 150, 200 - Assistant
I can certainly help you with that! Here's an example of how to run the
run_q_learningfunction with the given hyperparameter combinations for each of the three cases:Case 1: Default parameters - Already provided in the previous response. Repeating it here for clarity:
Q1, rewards1, epsilon_values1 = run_q_learning(episodes=500, alpha=0.1, gamma=0.99, epsilon=1.0, epsilon_decay=0.995, epsilon_min=0.01, max_timesteps=100)Case 2: Different gamma values - 0.4, 0.8, and 1:
Q2a, rewards2a, epsilon_values2a = run_q_learning(episodes=500, alpha=0.1, gamma=0.4, epsilon=1.0, epsilon_decay=0.995, epsilon_min=0.01, max_timesteps=100) Q2b, rewards2b, epsilon_values2b = run_q_learning(episodes=500, alpha=0.1, gamma=0.8, epsilon=1.0, epsilon_decay=0.995, epsilon_min=0.01, max_timesteps=100) Q2c, rewards2c, epsilon_values2c = run_q_learning(episodes=500, alpha=0.1, gamma=1.0, epsilon=1.0, epsilon_decay=0.995, epsilon_min=0.01, max_timesteps=100)Case 3: Different max_timesteps values - 25, 150, and 200:
Q3a, rewards3a, epsilon_values3a = run_q_learning(episodes=500, alpha=0.1, gamma=0.99, epsilon=1.0, epsilon_decay=0.995, epsilon_min=0.01, max_timesteps=25) Q3b, rewards3b, epsilon_values3b = run_q_learning(episodes=500, alpha=0.1, gamma=0.99, epsilon=1.0, epsilon_decay=0.995, epsilon_min=0.01, max_timesteps=150) Q3c, rewards3c, epsilon_values3c = run_q_learning(episodes=500, alpha=0.1, gamma=0.99, epsilon=1.0, epsilon_decay=0.995, epsilon_min=0.01, max_timesteps=200)You can use the same plotting method as shown in the previous response to visualize the rewards and epsilon values for each of these cases. Remember to update the titles and variable names accordingly in the plotting code.
- User
#Case 1: Default parameters
Q1, rewards1, epsilon_values1 = run_q_learning(episodes=500, alpha=0.1, gamma=0.99, epsilon=1.0, epsilon_decay=0.995, epsilon_min=0.01, max_timesteps=100)
#Case 2: Different gamma values - 0.4, 0.8, and 1:
Q2a, rewards2a, epsilon_values2a = run_q_learning(episodes=500, alpha=0.1, gamma=0.4, epsilon=1.0, epsilon_decay=0.995, epsilon_min=0.01, max_timesteps=100) Q2b, rewards2b, epsilon_values2b = run_q_learning(episodes=500, alpha=0.1, gamma=0.8, epsilon=1.0, epsilon_decay=0.995, epsilon_min=0.01, max_timesteps=100) Q2c, rewards2c, epsilon_values2c = run_q_learning(episodes=500, alpha=0.1, gamma=1.0, epsilon=1.0, epsilon_decay=0.995, epsilon_min=0.01, max_timesteps=100)
#Case 3: Different max_timesteps values - 25, 150, and 200:
Q3a, rewards3a, epsilon_values3a = run_q_learning(episodes=500, alpha=0.1, gamma=0.99, epsilon=1.0, epsilon_decay=0.995, epsilon_min=0.01, max_timesteps=25) Q3b, rewards3b, epsilon_values3b = run_q_learning(episodes=500, alpha=0.1, gamma=0.99, epsilon=1.0, epsilon_decay=0.995, epsilon_min=0.01, max_timesteps=150) Q3c, rewards3c, epsilon_values3c = run_q_learning(episodes=500, alpha=0.1, gamma=0.99, epsilon=1.0, epsilon_decay=0.995, epsilon_min=0.01, max_timesteps=200)
Plot rewards and epsilon values for each example
plt.figure(figsize=(12, 6))-----------------------give the code to plot the same
- Assistant
Here's the code to plot the rewards and epsilon values for each of the cases:
plt.figure(figsize=(15, 15)) # Case 1: Default parameters plt.subplot(3, 3, 1) plt.plot(rewards1) plt.title("Case 1: Rewards per Episode") plt.subplot(3, 3, 4) plt.plot(epsilon_values1) plt.title("Case 1: Epsilon Decay per Episode") # Case 2: Different gamma values - 0.4, 0.8, and 1 plt.subplot(3, 3, 2) plt.plot(rewards2a) plt.title("Case 2 (gamma=0.4): Rewards per Episode") plt.subplot(3, 3, 5) plt.plot(epsilon_values2a) plt.title("Case 2 (gamma=0.4): Epsilon Decay per Episode") plt.subplot(3, 3, 3) plt.plot(rewards2b) plt.title("Case 2 (gamma=0.8): Rewards per Episode") plt.subplot(3, 3, 6) plt.plot(epsilon_values2b) plt.title("Case 2 (gamma=0.8): Epsilon Decay per Episode") plt.subplot(3, 3, 9) plt.plot(rewards2c) plt.title("Case 2 (gamma=1.0): Rewards per Episode") plt.subplot(3, 3, 12) plt.plot(epsilon_values2c) plt.title("Case 2 (gamma=1.0): Epsilon Decay per Episode") # Case 3: Different max_timesteps values - 25, 150, and 200 plt.subplot(3, 3, 7) plt.plot(rewards3a) plt.title("Case 3 (max_timesteps=25): Rewards per Episode") plt.subplot(3, 3, 10) plt.plot(epsilon_values3a) plt.title("Case 3 (max_timesteps=25): Epsilon Decay per Episode") plt.subplot(3, 3, 8) plt.plot(rewards3b) plt.title("Case 3 (max_timesteps=150): Rewards per Episode") plt.subplot(3, 3, 11) plt.plot(epsilon_values3b) plt.title("Case 3 (max_timesteps=150): Epsilon Decay per Episode") plt.subplot(3, 3, 9) plt.plot(rewards3c) plt.title("Case 3 (max_timesteps=200): Rewards per Episode") plt.subplot(3, 3, 12) plt.plot(epsilon_values3c) plt.title("Case 3 (max_timesteps=200): Epsilon Decay per Episode") # Adjust plot layout plt.tight_layout() plt.show()This will plot the rewards and epsilon decay graphs for each of the cases with different hyperparameter combinations. Remember to analyze the graphs to observe the impact of the varying hyperparameters on the Q-Learning algorithm performance.
- User
import gym import numpy as np import matplotlib.pyplot as plt
def run_q_learning(episodes, alpha, gamma, epsilon, epsilon_decay, epsilon_min, max_timesteps): # Initialize environment, Q-table, and hyperparameters env = GridWorld() n_states = env.observation_space.n n_actions = env.action_space.n Q = np.zeros((n_states, n_actions))
# Lists for plotting epsilon decay and total rewards total_rewards = [] epsilon_values = [] # Q-Learning Algorithm for episode in range(episodes): state_tuple = env.reset() state = state_tuple[0] * 4 + state_tuple[1] # Convert the state tuple to single integer total_reward = 0 for t in range(max_timesteps): action = np.argmax(Q[state, :] + np.random.randn(1, n_actions) * (1.0 / (episode + 1))) next_state_tuple, reward, done = env.step(action) next_state = next_state_tuple[0] * 4 + next_state_tuple[1] # Convert the next_state tuple to single integer # Update Q-value target = reward + gamma * np.max(Q[next_state, :]) Q[state, action] += alpha * (target - Q[state, action]) state = next_state total_reward += reward if done: break epsilon = max(epsilon_min, epsilon * epsilon_decay) epsilon_values.append(epsilon) total_rewards.append(total_reward) return Q, total_rewards, epsilon_values#Case 1: Default parameters
Q1, rewards1, epsilon_values1 = run_q_learning(episodes=500, alpha=0.1, gamma=0.99, epsilon=1.0, epsilon_decay=0.995, epsilon_min=0.01, max_timesteps=100)
#Case 2: Different gamma values - 0.4, 0.8, and 1:
Q2a, rewards2a, epsilon_values2a = run_q_learning(episodes=500, alpha=0.1, gamma=0.4, epsilon=1.0, epsilon_decay=0.995, epsilon_min=0.01, max_timesteps=100) Q2b, rewards2b, epsilon_values2b = run_q_learning(episodes=500, alpha=0.1, gamma=0.8, epsilon=1.0, epsilon_decay=0.995, epsilon_min=0.01, max_timesteps=100) Q2c, rewards2c, epsilon_values2c = run_q_learning(episodes=500, alpha=0.1, gamma=1.0, epsilon=1.0, epsilon_decay=0.995, epsilon_min=0.01, max_timesteps=100)
#Case 3: Different max_timesteps values - 200, 550, and 1000:
Q3a, rewards3a, epsilon_values3a = run_q_learning(episodes=500, alpha=0.1, gamma=0.99, epsilon=1.0, epsilon_decay=0.995, epsilon_min=0.01, max_timesteps=200) Q3b, rewards3b, epsilon_values3b = run_q_learning(episodes=500, alpha=0.1, gamma=0.99, epsilon=1.0, epsilon_decay=0.995, epsilon_min=0.01, max_timesteps=550) Q3c, rewards3c, epsilon_values3c = run_q_learning(episodes=500, alpha=0.1, gamma=0.99, epsilon=1.0, epsilon_decay=0.995, epsilon_min=0.01, max_timesteps=1000)
Plot rewards and epsilon values for each example
plt.figure(figsize=(12, 6))
Case 2: Different gamma values - 0.4, 0.8, and 1
plt.subplot(3, 3, 2) plt.plot(rewards2a) plt.title('Case 2 (gamma=0.4): Rewards per Episode') plt.subplot(3, 3, 5) plt.plot(epsilon_values2a) plt.title('Case 2 (gamma=0.4): Epsilon Decay per Episode')
plt.subplot(3, 3, 3) plt.plot(rewards2b) plt.title('Case 2 (gamma=0.8): Rewards per Episode') plt.subplot(3, 3, 6) plt.plot(epsilon_values2b) plt.title('Case 2 (gamma=0.8): Epsilon Decay per Episode')
plt.subplot(3, 3, 9) plt.plot(rewards2c) plt.title('Case 2 (gamma=1.0): Rewards per Episode') plt.subplot(3, 3, 12) plt.plot(epsilon_values2c) plt.title('Case 2 (gamma=1.0): Epsilon Decay per Episode')
Case 3: Different max_timesteps values - 25, 150, and 200
plt.subplot(3, 3, 7) plt.plot(rewards3a) plt.title('Case 3 (max_timesteps=25): Rewards per Episode') plt.subplot(3, 3, 10) plt.plot(epsilon_values3a) plt.title('Case 3 (max_timesteps=25): Epsilon Decay per Episode')
plt.subplot(3, 3, 8) plt.plot(rewards3b) plt.title('Case 3 (max_timesteps=150): Rewards per Episode') plt.subplot(3, 3, 11) plt.plot(epsilon_values3b) plt.title('Case 3 (max_timesteps=150): Epsilon Decay per Episode')
plt.subplot(3, 3, 9) plt.plot(rewards3c) plt.title('Case 3 (max_timesteps=200): Rewards per Episode') plt.subplot(3, 3, 12) plt.plot(epsilon_values3c) plt.title('Case 3 (max_timesteps=200): Epsilon Decay per Episode')
Adjust plot layout
plt.tight_layout() plt.show() ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- ValueError Traceback (most recent call last) in <cell line: 85>() 83 plt.plot(rewards2c) 84 plt.title('Case 2 (gamma=1.0): Rewards per Episode') ---> 85 plt.subplot(3, 3, 12) 86 plt.plot(epsilon_values2c) 87 plt.title('Case 2 (gamma=1.0): Epsilon Decay per Episode')
1 frames /usr/local/lib/python3.10/dist-packages/matplotlib/gridspec.py in _from_subplot_args(figure, args) 596 else: 597 if not isinstance(num, Integral) or num < 1 or num > rowscols: --> 598 raise ValueError( 599 f"num must be an integer with 1 <= num <= {rowscols}, " 600 f"not {num!r}"
ValueError: num must be an integer with 1 <= num <= 9, not 12---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
- Assistant
Apologies for the confusion. The error is caused by an incorrect subplot index. The indices should be sequential for a 3x3 grid (i.e., 1 to 9), but the code mistakenly has 12 as an index. Here's the corrected plotting code:
plt.figure(figsize=(15, 15)) # Case 1: Default parameters plt.subplot(3, 3, 1) plt.plot(rewards1) plt.title("Case 1: Rewards per Episode") plt.subplot(3, 3, 4) plt.plot(epsilon_values1) plt.title("Case 1: Epsilon Decay per Episode") # Case 2: Different gamma values - 0.4, 0.8, and 1 plt.subplot(3, 3, 2) plt.plot(rewards2a) plt.title("Case 2 (gamma=0.4): Rewards per Episode") plt.subplot(3, 3, 5) plt.plot(epsilon_values2a) plt.title("Case 2 (gamma=0.4): Epsilon Decay per Episode") plt.subplot(3, 3, 3) plt.plot(rewards2b) plt.title("Case 2 (gamma=0.8): Rewards per Episode") plt.subplot(3, 3, 6) plt.plot(epsilon_values2b) plt.title("Case 2 (gamma=0.8): Epsilon Decay per Episode") plt.subplot(3, 3, 9) plt.plot(rewards2c) plt.title("Case 2 (gamma=1.0): Rewards per Episode") plt.subplot(3, 3, 7) plt.plot(epsilon_values2c) plt.title("Case 2 (gamma=1.0): Epsilon Decay per Episode") # Case 3: Different max_timesteps values - 25, 150, and 200 plt.subplot(3, 3, 8) plt.plot(rewards3a) plt.title("Case 3 (max_timesteps=25): Rewards per Episode") plt.subplot(3, 3, 11) plt.plot(epsilon_values3a) plt.title("Case 3 (max_timesteps=25): Epsilon Decay per Episode") plt.subplot(3, 3, 9) plt.plot(rewards3b) plt.title("Case 3 (max_timesteps=150): Rewards per Episode") plt.subplot(3, 3, 12) plt.plot(epsilon_values3b) plt.title("Case 3 (max_timesteps=150): Epsilon Decay per Episode") plt.subplot(3, 3, 10) plt.plot(rewards3c) plt.title("Case 3 (max_timesteps=200): Rewards per Episode") plt.subplot(3, 3, 13) plt.plot(epsilon_values3c)
Other models, same or similar prompt
Responses were produced in different, uncontrolled contexts (system prompts, settings, prior turns). This is not a benchmark.