Skip to content
TheChatArchiveThe Chat Archive
gpt-4-1106-preview· OpenAIAi2 WildChat-4.8M (AI2)

import gym import tensorflow as tf from tensorflow.keras import layers import numpy as…

  1. User

    import gym import tensorflow as tf from tensorflow.keras import layers import numpy as np import matplotlib.pyplot as plt

    problem = "Pendulum-v1" env = gym.make(problem)

    num_states = env.observation_space.shape[0] print("Size of State Space -> {}".format(num_states)) num_actions = env.action_space.shape[0] print("Size of Action Space -> {}".format(num_actions))

    upper_bound = env.action_space.high[0] lower_bound = env.action_space.low[0]

    print("Max Value of Action -> {}".format(upper_bound)) print("Min Value of Action -> {}".format(lower_bound))

    class OUActionNoise: def init(self, mean, std_deviation, theta=0.15, dt=1e-2, x_initial=None): self.theta = theta self.mean = mean self.std_dev = std_deviation self.dt = dt self.x_initial = x_initial self.reset()

    def __call__(self):
        # Formula taken from https://www.wikipedia.org/wiki/Ornstein-Uhlenbeck_process.
        x = (
            self.x_prev
            + self.theta * (self.mean - self.x_prev) * self.dt
            + self.std_dev * np.sqrt(self.dt) * np.random.normal(size=self.mean.shape)
        )
        # Store x into x_prev
        # Makes next noise dependent on current one
        self.x_prev = x
        return x
    
    def reset(self):
        if self.x_initial is not None:
            self.x_prev = self.x_initial
        else:
            self.x_prev = np.zeros_like(self.mean)
    

    class Buffer: def init(self, buffer_capacity=100000, batch_size=64): # Number of "experiences" to store at max self.buffer_capacity = buffer_capacity # Num of tuples to train on. self.batch_size = batch_size

        # Its tells us num of times record() was called.
        self.buffer_counter = 0
    
        # Instead of list of tuples as the exp.replay concept go
        # We use different np.arrays for each tuple element
        self.state_buffer = np.zeros((self.buffer_capacity, num_states))
        self.action_buffer = np.zeros((self.buffer_capacity, num_actions))
        self.reward_buffer = np.zeros((self.buffer_capacity, 1))
        self.next_state_buffer = np.zeros((self.buffer_capacity, num_states))
    
    # Takes (s,a,r,s') obervation tuple as input
    def record(self, obs_tuple):
        # Set index to zero if buffer_capacity is exceeded,
        # replacing old records
        index = self.buffer_counter % self.buffer_capacity
    
        self.state_buffer[index] = obs_tuple[0]
        self.action_buffer[index] = obs_tuple[1]
        self.reward_buffer[index] = obs_tuple[2]
        self.next_state_buffer[index] = obs_tuple[3]
    
        self.buffer_counter += 1
    
    # Eager execution is turned on by default in TensorFlow 2. Decorating with tf.function allows
    # TensorFlow to build a static graph out of the logic and computations in our function.
    # This provides a large speed up for blocks of code that contain many small TensorFlow operations such as this one.
    @tf.function
    def update(
        self, state_batch, action_batch, reward_batch, next_state_batch,
    ):
        # Training and updating Actor & Critic networks.
        # See Pseudo Code.
        with tf.GradientTape() as tape:
            target_actions = target_actor(next_state_batch, training=True)
            y = reward_batch + gamma * target_critic(
                [next_state_batch, target_actions], training=True
            )
            critic_value = critic_model([state_batch, action_batch], training=True)
            critic_loss = tf.math.reduce_mean(tf.math.square(y - critic_value))
    
        critic_grad = tape.gradient(critic_loss, critic_model.trainable_variables)
        critic_optimizer.apply_gradients(
            zip(critic_grad, critic_model.trainable_variables)
        )
    
        with tf.GradientTape() as tape:
            actions = actor_model(state_batch, training=True)
            critic_value = critic_model([state_batch, actions], training=True)
            # Used `-value` as we want to maximize the value given
            # by the critic for our actions
            actor_loss = -tf.math.reduce_mean(critic_value)
    
        actor_grad = tape.gradient(actor_loss, actor_model.trainable_variables)
        actor_optimizer.apply_gradients(
            zip(actor_grad, actor_model.trainable_variables)
        )
    
    # We compute the loss and update parameters
    def learn(self):
        # Get sampling range
        record_range = min(self.buffer_counter, self.buffer_capacity)
        # Randomly sample indices
        batch_indices = np.random.choice(record_range, self.batch_size)
    
        # Convert to tensors
        state_batch = tf.convert_to_tensor(self.state_buffer[batch_indices])
        action_batch = tf.convert_to_tensor(self.action_buffer[batch_indices])
        reward_batch = tf.convert_to_tensor(self.reward_buffer[batch_indices])
        reward_batch = tf.cast(reward_batch, dtype=tf.float32)
        next_state_batch = tf.convert_to_tensor(self.next_state_buffer[batch_indices])
    
        self.update(state_batch, action_batch, reward_batch, next_state_batch)
    

    This update target parameters slowly

    Based on rate tau, which is much less than one.

    @tf.function def update_target(target_weights, weights, tau): for (a, b) in zip(target_weights, weights): a.assign(b * tau + a * (1 - tau))

    def get_actor(): # Initialize weights between -3e-3 and 3-e3 last_init = tf.random_uniform_initializer(minval=-0.003, maxval=0.003)

    inputs = layers.Input(shape=(num_states,))
    out = layers.Dense(256, activation="relu")(inputs)
    out = layers.Dense(256, activation="relu")(out)
    outputs = layers.Dense(1, activation="tanh", kernel_initializer=last_init)(out)
    
    # Our upper bound is 2.0 for Pendulum.
    outputs = outputs * upper_bound
    model = tf.keras.Model(inputs, outputs)
    return model
    

    def get_critic(): # State as input state_input = layers.Input(shape=(num_states)) state_out = layers.Dense(16, activation="relu")(state_input) state_out = layers.Dense(32, activation="relu")(state_out)

    # Action as input
    action_input = layers.Input(shape=(num_actions))
    action_out = layers.Dense(32, activation="relu")(action_input)
    
    # Both are passed through seperate layer before concatenating
    concat = layers.Concatenate()([state_out, action_out])
    
    out = layers.Dense(256, activation="relu")(concat)
    out = layers.Dense(256, activation="relu")(out)
    outputs = layers.Dense(1)(out)
    
    # Outputs single value for give state-action
    model = tf.keras.Model([state_input, action_input], outputs)
    
    return model
    

    def policy(state, noise_object): sampled_actions = tf.squeeze(actor_model(state)) noise = noise_object() # Adding noise to action sampled_actions = sampled_actions.numpy() + noise

    # We make sure action is within bounds
    legal_action = np.clip(sampled_actions, lower_bound, upper_bound)
    
    return [np.squeeze(legal_action)]
    

    std_dev = 0.2 ou_noise = OUActionNoise(mean=np.zeros(1), std_deviation=float(std_dev) * np.ones(1))

    actor_model = get_actor() critic_model = get_critic()

    target_actor = get_actor() target_critic = get_critic()

    Making the weights equal initially

    target_actor.set_weights(actor_model.get_weights()) target_critic.set_weights(critic_model.get_weights())

    Learning rate for actor-critic models

    critic_lr = 0.002 actor_lr = 0.001

    critic_optimizer = tf.keras.optimizers.Adam(critic_lr) actor_optimizer = tf.keras.optimizers.Adam(actor_lr)

    total_episodes = 100

    Discount factor for future rewards

    gamma = 0.99

    Used to update target networks

    tau = 0.005

    buffer = Buffer(50000, 64)

    To store reward history of each episode

    ep_reward_list = []

    To store average reward history of last few episodes

    avg_reward_list = []

    Takes about 4 min to train

    for ep in range(total_episodes):

    prev_state = env.reset()
    episodic_reward = 0
    
    while True:
        # Uncomment this to see the Actor in action
        # But not in a python notebook.
        # env.render()
    
        tf_prev_state = tf.expand_dims(tf.convert_to_tensor(prev_state), 0)
    
        action = policy(tf_prev_state, ou_noise)
        # Recieve state and reward from environment.
        state, reward, done, info = env.step(action)
    
        buffer.record((prev_state, action, reward, state))
        episodic_reward += reward
    
        buffer.learn()
        update_target(target_actor.variables, actor_model.variables, tau)
        update_target(target_critic.variables, critic_model.variables, tau)
    
        # End this episode when `done` is True
        if done:
            break
    
        prev_state = state
    
    ep_reward_list.append(episodic_reward)
    
    # Mean of last 40 episodes
    avg_reward = np.mean(ep_reward_list[-40:])
    print("Episode * {} * Avg Reward is ==> {}".format(ep, avg_reward))
    avg_reward_list.append(avg_reward)
    

    Plotting graph

    Episodes versus Avg. Rewards

    plt.plot(avg_reward_list) plt.xlabel("Episode") plt.ylabel("Avg. Epsiodic Reward") plt.show()

    Save the weights

    actor_model.save_weights("pendulum_actor.h5") critic_model.save_weights("pendulum_critic.h5")

    target_actor.save_weights("pendulum_target_actor.h5") target_critic.save_weights("pendulum_target_critic.h5")

    How to test above code

  2. Assistant

    The code you have provided appears to be an implementation of the Deep Deterministic Policy Gradient (DDPG) algorithm, applied to the "Pendulum-v1" problem from OpenAI's Gym library. The given code includes the training loop but not the testing part. Below, I will show you how to add testing for the trained model.

    Here is the code to test the trained agent:

    # Testing the Agent
    
    def test_agent(num_episodes=10, render=True):
        for ep in range(num_episodes):
            state = env.reset()
            episodic_reward = 0
            while True:
                if render:
                    env.render()  # You can comment this line out if you don't want to render the environment
                tf_state = tf.expand_dims(tf.convert_to_tensor(state), 0)
                action = policy(tf_state, ou_noise)
                state, reward, done, _ = env.step(action)
                episodic_reward += reward
                if done:
                    break
    
            print("Test Episode: {}\nTest Reward: {:.2f}".format(ep, episodic_reward))
        env.close()
    
    # Load the trained model weights before testing
    actor_model.load_weights("pendulum_actor.h5")
    critic_model.load_weights("pendulum_critic.h5")
    
    # Test the trained agent
    test_agent()
    

    You can call test_agent() after the training loop. It will load the trained weights for both the actor and critic models, then run the agent through a specified number of episodes. The render flag determines whether you see the simulation on the screen. Note that env.render() might not work if you are running the code in a notebook environment or a headless server; it's usually used in local development environments that support GUI.

    The test_agent function will test the agent for num_episodes episodes and print out the rewards after each test episode. This gives you an indication of how well your agent has learned to balance the pendulum.

    Note: In the provided code, the OUActionNoise is instantiated but should not be used during testing, as it introduces randomness to the agent's policy. When testing, usually, you want the deterministic output of the policy. If you want the deterministic behavior in test mode, you can modify the policy function to ignore noise:

    def policy(state, noise_object=None, add_noise=False):
        sampled_actions = tf.squeeze(actor_model(state))
        if add_noise and noise_object is not None:
            noise = noise_object()
            sampled_actions = sampled_actions.numpy() + noise
        else:
            sampled_actions = sampled_actions.numpy()
    
        legal_action = np.clip(sampled_actions, lower_bound, upper_bound)
        return [np.squeeze(legal_action)]
    

    Then, in the test_agent function, you can call action = policy(tf_state, add_noise=False) to get the deterministic actions.

Responses were produced in different, uncontrolled contexts (system prompts, settings, prior turns). This is not a benchmark.