import gym import numpy as np import matplotlib.pyplot as plt def runsarsa(episodes,…
- User
import gym import numpy as np import matplotlib.pyplot as plt
def run_sarsa(episodes, alpha, gamma, epsilon, epsilon_decay, epsilon_min, max_timesteps): # Initialize environment, Q-table, and hyperparameters env = GridWorld() n_states = env.observation_space.n n_actions = env.action_space.n Q = np.zeros((n_states, n_actions))
# Lists for plotting epsilon decay and total rewards total_rewards = [] epsilon_values = [] # SARSA Algorithm for episode in range(episodes): state_tuple = env.reset() state = state_tuple[0] * 4 + state_tuple[1] # Convert the state tuple to single integer action = np.argmax(Q[state, :] + np.random.randn(1, n_actions) * (1.0 / (episode + 1))) total_reward = 0 for t in range(max_timesteps): next_state_tuple, reward, done = env.step(action) next_state = next_state_tuple[0] * 4 + next_state_tuple[1] # Convert the next_state tuple to single integer next_action = np.argmax(Q[next_state, :] + np.random.randn(1, n_actions) * (1.0 / (episode + 1))) # Update Q-value target = reward + gamma * Q[next_state, next_action] Q[state, action] += alpha * (target - Q[state, action]) state, action = next_state, next_action total_reward += reward if done: break epsilon = max(epsilon_min, epsilon * epsilon_decay) epsilon_values.append(epsilon) total_rewards.append(total_reward) return Q, total_rewards, epsilon_values-------------------------This is my sarsa algorithm ---------------------Now implement n - step sarsa-------------n-step SARSA [5 points]Modify your SARSA and implement a 2-step bootstrapping SARSA. Compare the results in graphs with SARSA
- Assistant
In order to implement a n-step SARSA, we will modify the update rule of the original SARSA algorithm to consider n steps ahead for bootstrapping. For this implementation, we'll use n=2 (2-step bootstrapping SARSA). Here's the modified version of the run_sarsa function:
def run_n_step_sarsa(episodes, alpha, gamma, epsilon, epsilon_decay, epsilon_min, max_timesteps, n=2): # Initialize environment, Q-table, and hyperparameters env = GridWorld() n_states = env.observation_space.n n_actions = env.action_space.n Q = np.zeros((n_states, n_actions)) # Lists for plotting epsilon decay and total rewards total_rewards = [] epsilon_values = [] # n-step SARSA Algorithm for episode in range(episodes): state_tuple = env.reset() state = state_tuple[0] * 4 + state_tuple[1] # Convert the state tuple to a single integer action = np.argmax(Q[state, :] + np.random.randn(1, n_actions) * (1.0 / (episode + 1))) total_reward = 0 t = 0 tmax = max_timesteps T = float("inf") states = [state] actions = [action] rewards = [0] while True: if t < T: next_state_tuple, reward, done = env.step(action) next_state = next_state_tuple[0] * 4 + next_state_tuple[1] # Convert the next_state tuple to a single integer rewards.append(reward) if done: T = t + 1 else: next_action = np.argmax(Q[next_state, :] + np.random.randn(1, n_actions) * (1.0 / (episode + 1))) actions.append(next_action) states.append(next_state) tau = t - n + 1 if tau >= 0: G = sum([gamma**(i-tau-1) * rewards[i] for i in range(tau+1, min(tau+n, T)+1)]) if tau + n < T: G += gamma**n * Q[states[tau+n], actions[tau+n]] Q[states[tau], actions[tau]] += alpha * (G - Q[states[tau], actions[tau]]) state, action = next_state, next_action total_reward += reward t += 1 if tau == T - 1: break epsilon = max(epsilon_min, epsilon * epsilon_decay) epsilon_values.append(epsilon) total_rewards.append(total_reward) return Q, total_rewards, epsilon_valuesNow let's compare the results of 2-step SARSA with the original SARSA in graphs:
# Set hyperparameters for both the algorithms episodes = 500 alpha = 0.1 gamma = 0.99 epsilon = 1.0 epsilon_decay = 0.995 epsilon_min = 0.01 max_timesteps = 100 # Run the original SARSA Q_sarsa, total_rewards_sarsa, epsilon_values_sarsa = run_sarsa(episodes, alpha, gamma, epsilon, epsilon_decay, epsilon_min, max_timesteps) # Run the 2-step SARSA Q_n_step_sarsa, total_rewards_n_step_sarsa, epsilon_values_n_step_sarsa = run_n_step_sarsa(episodes, alpha, gamma, epsilon, epsilon_decay, epsilon_min, max_timesteps, n=2) # Plot the total rewards for both the algorithms plt.plot(total_rewards_sarsa, label="SARSA") plt.plot(total_rewards_n_step_sarsa, label="2-Step SARSA") plt.xlabel("Episode") plt.ylabel("Total Reward") plt.legend() plt.show() # Plot the epsilon decay for both the algorithms plt.plot(epsilon_values_sarsa, label="SARSA") plt.plot(epsilon_values_n_step_sarsa, label="2-Step SARSA") plt.xlabel("Episode") plt.ylabel("Epsilon") plt.legend() plt.show()Running the above code will show you the comparison between the original SARSA and 2-step SARSA in terms of total rewards and epsilon decay. You can analyze the results and make conclusions about their performance.
Other models, same or similar prompt
Responses were produced in different, uncontrolled contexts (system prompts, settings, prior turns). This is not a benchmark.