Skip to content
TheChatArchiveThe Chat Archive
gpt-3.5-turbo-0301· OpenAIAi2 WildChat-4.8M (AI2)

import torch import random import numpy as np from collections import deque from game…

  1. User

    import torch import random import numpy as np from collections import deque from game import SnakeGameAI, Direction, Point from model import Linear_QNet, QTrainer from helper import plot

    MAX_MEMORY = 100_000 BATCH_SIZE = 1000 LR = 0.002

    class Agent:

    def __init__(self):
        self.n_games = 0
        self.epsilon = 0  # randomness
        self.gamma = 0.9  # discount rate
        self.memory = deque(maxlen=MAX_MEMORY)  # popleft()
        self.model = Linear_QNet(11, 256, 3)
        self.trainer = QTrainer(self.model, lr=LR, gamma=self.gamma)
    
    def get_state(self, game):
        head2 = game.snake2[0]
    
        point_l = Point(head2.x - 20, head2.y)
        point_r = Point(head2.x + 20, head2.y)
        point_u = Point(head2.x, head2.y - 20)
        point_d = Point(head2.x, head2.y + 20)
    
        dir_l = game.direction == Direction.LEFT
        dir_r = game.direction == Direction.RIGHT
        dir_u = game.direction == Direction.UP
        dir_d = game.direction == Direction.DOWN
    
        state = [
            # Danger straight
            (dir_r and game.is_collision(point_r)) or
            (dir_l and game.is_collision(point_l)) or
            (dir_u and game.is_collision(point_u)) or
            (dir_d and game.is_collision(point_d)),
    
            # Danger right
            (dir_u and game.is_collision(point_r)) or
            (dir_d and game.is_collision(point_l)) or
            (dir_l and game.is_collision(point_u)) or
            (dir_r and game.is_collision(point_d)),
    
            # Danger left
            (dir_d and game.is_collision(point_r)) or
            (dir_u and game.is_collision(point_l)) or
            (dir_r and game.is_collision(point_u)) or
            (dir_l and game.is_collision(point_d)),
    
            # Move direction
            dir_l,
            dir_r,
            dir_u,
            dir_d,
    
            # Food location
            game.food.x < game.head.x,  # food left
            game.food.x > game.head.x,  # food right
            game.food.y < game.head.y,  # food up
            game.food.y > game.head.y  # food down
        ]
    
        return np.array(state, dtype=int)
    
    def remember(self, state, action, reward, next_state, done):
        self.memory.append((state, action, reward, next_state, done))  # popleft if MAX_MEMORY is reached
    
    def train_long_memory(self):
        if len(self.memory) > BATCH_SIZE:
            mini_sample = random.sample(self.memory, BATCH_SIZE)  # list of tuples
        else:
            mini_sample = self.memory
    
        states, actions, rewards, next_states, dones = zip(*mini_sample)
        self.trainer.train_step(states, actions, rewards, next_states, dones)
        # for state, action, reward, nexrt_state, done in mini_sample:
        #    self.trainer.train_step(state, action, reward, next_state, done)
    
    def train_short_memory(self, state, action, reward, next_state, done):
        self.trainer.train_step(state, action, reward, next_state, done)
    
    def get_action(self, state):
        # random moves: tradeoff exploration / exploitation
        self.epsilon = 80 - self.n_games
        final_move = [0, 0, 0]
        if random.randint(0, 200) < self.epsilon:
            move = random.randint(0, 2)
            final_move[move] = 1
        else:
            state0 = torch.tensor(state, dtype=torch.float)
            prediction = self.model(state0)
            move = torch.argmax(prediction).item()
            final_move[move] = 1
    
        return final_move
    

    def train(): plot_scores = [] plot_mean_scores = [] total_score = 0 record = 0 agent = Agent() game = SnakeGameAI() while True: # get old state state_old = agent.get_state(game)

        # get move
        final_move = agent.get_action(state_old)
    
        # perform move and get new state
        reward, done, score = game.play_step(final_move)
        state_new = agent.get_state(game)
    
        # train short memory
        agent.train_short_memory(state_old, final_move, reward, state_new, done)
    
        # remember
        agent.remember(state_old, final_move, reward, state_new, done)
    
        if done:
            # train long memory, plot result
            game.reset()
            agent.n_games += 1
            agent.train_long_memory()
    
            if score > record:
                record = score
                agent.model.save()
    
            print('Game', agent.n_games, 'Score', score, 'Record:', record)
    
            plot_scores.append(score)
            total_score += score
            mean_score = total_score / agent.n_games
            plot_mean_scores.append(mean_score)
            plot(plot_scores, plot_mean_scores)
    

    if name == 'main': train()

  2. Assistant

    There are a few components in this code:

    • The Agent class, which represents the AI agent that plays the game.
    • The SnakeGameAI class, which represents the actual game environment.
    • The Linear_QNet class, which is a simple linear neural network used as the Q-value function approximator.
    • The QTrainer class, which implements the Q-learning algorithm using gradient descent to minimize the mean squared error between the predicted Q-values and the target Q-values.
    • The train function, which is the main training loop that iteratively plays the game, collects experience tuples, trains the Q-value estimator on them, and plots the results.

    Here's a breakdown of the key components in more detail:

    1. The Agent class

    The Agent class is used to represent the AI agent that is trained to play the game. It has the following key properties and methods:

    • n_games: The number of games the agent has played.
    • epsilon: The exploration rate of the agent. This controls the balance between exploration and exploitation during training (i.e. how often the agent takes random actions vs. how often it takes the action with the highest Q-value).
    • gamma: The discount factor for future rewards.
    • memory: A deque (double-ended queue) that stores the agent's experience tuples (i.e. (state, action, reward, next_state, done)), which are used for training the Q-value estimator.
    • model: The Q-value estimator (i.e. the neural network).
    • trainer: The Q-learning trainer, which implements the Q-learning algorithm using gradient descent to update the Q-value estimator.

    The key methods in this class are:

    • get_state: Returns a feature vector representing the current state of the game board. This consists of a binary representation of various game board configurations such as the position of the snake's head and the location of the food.
    • remember: Adds a new experience tuple to the agent's memory.
    • train_long_memory: Trains the Q-value estimator on a batch of the agent's experience tuples (i.e. a mini-batch sampled from the memory).
    • train_short_memory: Performs a single update step on the Q-value estimator using a single experience tuple (i.e. a single (state, action, reward, next_state, done) tuple).
    • get_action: Returns the action that the agent should take based on the current state of the game board. This is done using an epsilon-greedy policy, where the agent takes a random action with probability epsilon and takes the action with the highest Q-value with probability 1-epsilon.
    1. The SnakeGameAI class

    The SnakeGameAI class represents the game environment, which the agent interacts with. It has the following key properties and methods:

    • screen_width, screen_height: The dimensions of the game board.
    • snake_block, snake_list, snake_block2, snake_list2: The positions of the snakes on the game board.
    • food: The position of the food on the game board.
    • direction, direction2: The current directions of the snakes.
    • score, score2: The current scores of the snakes.
    • game_over: A flag that is set to True when the game is over.

    The key methods in this class are:

    • draw_snake: Draws the snake on the game board.
    • draw_food: Draws the food on the game board.
    • update_snake: Updates the position of the snake based on its current direction.
    • update_food: Randomly places the food on the game board.
    • reset: Resets the game state (i.e. snake positions, score, and game over flag).
    • is_collision: Checks if a given point collides with the snake.
    • play_step: Takes an action as input and performs a single step of the game. This involves updating the game state, checking for collisions, and calculating the reward.
    1. The Linear_QNet class

    The Linear_QNet class is a simple linear neural network used as the Q-value function approximator. It has the following key properties and methods:

    • fc1, fc2, fc3: The three fully connected layers of the network.
    • forward: Computes the forward pass of the network (i.e. the Q-values for a given state).
    1. The QTrainer class

    The QTrainer class implements the Q-learning algorithm using gradient descent to update the Q-value estimator. It has the following key properties and methods:

    • model: The Q-value estimator (i.e. the neural network).
    • lr: The learning rate for the optimizer.
    • optimizer: The optimizer to use for gradient descent (e.g. Adam).
    • loss_fn: The loss function to use (e.g. mean squared error).

    The key methods in this class are:

    • train_step: Updates the Q-values for a batch of experience tuples using gradient descent. This involves computing the target Q-values, computing the predicted Q-values using the current network weights, computing the loss between the predicted and target Q-values, backpropagating the loss through the network, and applying the updates to the network weights using the optimizer.
  3. User

    there is a problem snake can eat food but snake2 just walk through it

  4. Assistant

    If the second snake is not eating the food and walking through it, it's likely because the game logic is not accounting for the second snake's position when checking for collisions with the food.

    You need to modify the SnakeGameAI class to consider the position of snake2 when checking for food collisions. One way to do this is to add the position of the second snake to the is_collision function.

    Here's an example of how you could modify the is_collision function:

    def is_collision(self, pt, pt2):
        if pt in self.snake_list[:-1] or pt in self.snake_list2[:-1]:
            return True
        return False
    

    With this modification, the is_collision function checks if the given point pt or pt2 is in either snake's body, and returns True if there is a collision.

    You also need to modify the play_step function to handle the case where either snake eats the food, and update the score accordingly. Here's an example of how you could do this:

    if new_head == self.food:
        self.score += 1
        self.food = self.get_random_location()
    elif new_head == self.food2:
        self.score2 += 1
        self.food2 = self.get_random_location()
    else:
        self.snake_list.pop()
        self.snake_list2.pop()
    

    With these modifications, the second snake should be able to eat the food and the score should be updated accordingly.