Skip to content
TheChatArchiveThe Chat Archive
gpt-4-0314· openaiHugging Face Suzhen/CodeChat

import gym import numpy as np import matplotlib.pyplot as plt def runsarsa(episodes,…

  1. User

    import gym import numpy as np import matplotlib.pyplot as plt

    def run_sarsa(episodes, alpha, gamma, epsilon, epsilon_decay, epsilon_min, max_timesteps): # Initialize environment, Q-table, and hyperparameters env = GridWorld() n_states = env.observation_space.n n_actions = env.action_space.n Q = np.zeros((n_states, n_actions))

    # Lists for plotting epsilon decay and total rewards
    total_rewards = []
    epsilon_values = []
    
    # SARSA Algorithm
    for episode in range(episodes):
        state_tuple = env.reset()
        state = state_tuple[0] * 4 + state_tuple[1]  # Convert the state tuple to single integer
        action = np.argmax(Q[state, :] + np.random.randn(1, n_actions) * (1.0 / (episode + 1)))
    
        total_reward = 0
        for t in range(max_timesteps):
            next_state_tuple, reward, done = env.step(action)
            next_state = next_state_tuple[0] * 4 + next_state_tuple[1]  # Convert the next_state tuple to single integer
            next_action = np.argmax(Q[next_state, :] + np.random.randn(1, n_actions) * (1.0 / (episode + 1)))
    
            # Update Q-value
            target = reward + gamma * Q[next_state, next_action]
            Q[state, action] += alpha * (target - Q[state, action])
    
            state, action = next_state, next_action
            total_reward += reward
    
            if done:
                break
    
        epsilon = max(epsilon_min, epsilon * epsilon_decay)
        epsilon_values.append(epsilon)
        total_rewards.append(total_reward)
    
    return Q, total_rewards, epsilon_values-------------------------This is my sarsa algorithm ---------------------Now implement n - step sarsa-------------n-step SARSA [5 points]
    

    Modify your SARSA and implement a 2-step bootstrapping SARSA. Compare the results in graphs with SARSA

  2. Assistant

    In order to implement a n-step SARSA, we will modify the update rule of the original SARSA algorithm to consider n steps ahead for bootstrapping. For this implementation, we'll use n=2 (2-step bootstrapping SARSA). Here's the modified version of the run_sarsa function:

    def run_n_step_sarsa(episodes, alpha, gamma, epsilon, epsilon_decay, epsilon_min, max_timesteps, n=2):
        # Initialize environment, Q-table, and hyperparameters
        env = GridWorld()
        n_states = env.observation_space.n
        n_actions = env.action_space.n
        Q = np.zeros((n_states, n_actions))
    
        # Lists for plotting epsilon decay and total rewards
        total_rewards = []
        epsilon_values = []
    
        # n-step SARSA Algorithm
        for episode in range(episodes):
            state_tuple = env.reset()
            state = state_tuple[0] * 4 + state_tuple[1]  # Convert the state tuple to a single integer
            action = np.argmax(Q[state, :] + np.random.randn(1, n_actions) * (1.0 / (episode + 1)))
    
            total_reward = 0
            t = 0
            tmax = max_timesteps
            T = float("inf")
            states = [state]
            actions = [action]
            rewards = [0]
    
            while True:
                if t < T:
                    next_state_tuple, reward, done = env.step(action)
                    next_state = next_state_tuple[0] * 4 + next_state_tuple[1]  # Convert the next_state tuple to a single integer
                    rewards.append(reward)
    
                    if done:
                        T = t + 1
                    else:
                        next_action = np.argmax(Q[next_state, :] + np.random.randn(1, n_actions) * (1.0 / (episode + 1)))
                        actions.append(next_action)
                        states.append(next_state)
                
                tau = t - n + 1
                if tau >= 0:
                    G = sum([gamma**(i-tau-1) * rewards[i] for i in range(tau+1, min(tau+n, T)+1)])
                    if tau + n < T:
                        G += gamma**n * Q[states[tau+n], actions[tau+n]]
                    Q[states[tau], actions[tau]] += alpha * (G - Q[states[tau], actions[tau]])
                
                state, action = next_state, next_action
                total_reward += reward
                t += 1
    
                if tau == T - 1:
                    break
    
            epsilon = max(epsilon_min, epsilon * epsilon_decay)
            epsilon_values.append(epsilon)
            total_rewards.append(total_reward)
    
        return Q, total_rewards, epsilon_values
    

    Now let's compare the results of 2-step SARSA with the original SARSA in graphs:

    # Set hyperparameters for both the algorithms
    episodes = 500
    alpha = 0.1
    gamma = 0.99
    epsilon = 1.0
    epsilon_decay = 0.995
    epsilon_min = 0.01
    max_timesteps = 100
    
    # Run the original SARSA
    Q_sarsa, total_rewards_sarsa, epsilon_values_sarsa = run_sarsa(episodes, alpha, gamma, epsilon, epsilon_decay, epsilon_min, max_timesteps)
    
    # Run the 2-step SARSA
    Q_n_step_sarsa, total_rewards_n_step_sarsa, epsilon_values_n_step_sarsa = run_n_step_sarsa(episodes, alpha, gamma, epsilon, epsilon_decay, epsilon_min, max_timesteps, n=2)
    
    # Plot the total rewards for both the algorithms
    plt.plot(total_rewards_sarsa, label="SARSA")
    plt.plot(total_rewards_n_step_sarsa, label="2-Step SARSA")
    plt.xlabel("Episode")
    plt.ylabel("Total Reward")
    plt.legend()
    plt.show()
    
    # Plot the epsilon decay for both the algorithms
    plt.plot(epsilon_values_sarsa, label="SARSA")
    plt.plot(epsilon_values_n_step_sarsa, label="2-Step SARSA")
    plt.xlabel("Episode")
    plt.ylabel("Epsilon")
    plt.legend()
    plt.show()
    

    Running the above code will show you the comparison between the original SARSA and 2-step SARSA in terms of total rewards and epsilon decay. You can analyze the results and make conclusions about their performance.

Responses were produced in different, uncontrolled contexts (system prompts, settings, prior turns). This is not a benchmark.