import torch import random import numpy as np from collections import deque from game…
- User
import torch import random import numpy as np from collections import deque from game import SnakeGameAI, Direction, Point from model import Linear_QNet, QTrainer from helper import plot
MAX_MEMORY = 100_000 BATCH_SIZE = 1000 LR = 0.002
class Agent:
def __init__(self): self.n_games = 0 self.epsilon = 0 # randomness self.gamma = 0.9 # discount rate self.memory = deque(maxlen=MAX_MEMORY) # popleft() self.model = Linear_QNet(11, 256, 3) self.trainer = QTrainer(self.model, lr=LR, gamma=self.gamma) def get_state(self, game): head2 = game.snake2[0] point_l = Point(head2.x - 20, head2.y) point_r = Point(head2.x + 20, head2.y) point_u = Point(head2.x, head2.y - 20) point_d = Point(head2.x, head2.y + 20) dir_l = game.direction == Direction.LEFT dir_r = game.direction == Direction.RIGHT dir_u = game.direction == Direction.UP dir_d = game.direction == Direction.DOWN state = [ # Danger straight (dir_r and game.is_collision(point_r)) or (dir_l and game.is_collision(point_l)) or (dir_u and game.is_collision(point_u)) or (dir_d and game.is_collision(point_d)), # Danger right (dir_u and game.is_collision(point_r)) or (dir_d and game.is_collision(point_l)) or (dir_l and game.is_collision(point_u)) or (dir_r and game.is_collision(point_d)), # Danger left (dir_d and game.is_collision(point_r)) or (dir_u and game.is_collision(point_l)) or (dir_r and game.is_collision(point_u)) or (dir_l and game.is_collision(point_d)), # Move direction dir_l, dir_r, dir_u, dir_d, # Food location game.food.x < game.head.x, # food left game.food.x > game.head.x, # food right game.food.y < game.head.y, # food up game.food.y > game.head.y # food down ] return np.array(state, dtype=int) def remember(self, state, action, reward, next_state, done): self.memory.append((state, action, reward, next_state, done)) # popleft if MAX_MEMORY is reached def train_long_memory(self): if len(self.memory) > BATCH_SIZE: mini_sample = random.sample(self.memory, BATCH_SIZE) # list of tuples else: mini_sample = self.memory states, actions, rewards, next_states, dones = zip(*mini_sample) self.trainer.train_step(states, actions, rewards, next_states, dones) # for state, action, reward, nexrt_state, done in mini_sample: # self.trainer.train_step(state, action, reward, next_state, done) def train_short_memory(self, state, action, reward, next_state, done): self.trainer.train_step(state, action, reward, next_state, done) def get_action(self, state): # random moves: tradeoff exploration / exploitation self.epsilon = 80 - self.n_games final_move = [0, 0, 0] if random.randint(0, 200) < self.epsilon: move = random.randint(0, 2) final_move[move] = 1 else: state0 = torch.tensor(state, dtype=torch.float) prediction = self.model(state0) move = torch.argmax(prediction).item() final_move[move] = 1 return final_movedef train(): plot_scores = [] plot_mean_scores = [] total_score = 0 record = 0 agent = Agent() game = SnakeGameAI() while True: # get old state state_old = agent.get_state(game)
# get move final_move = agent.get_action(state_old) # perform move and get new state reward, done, score = game.play_step(final_move) state_new = agent.get_state(game) # train short memory agent.train_short_memory(state_old, final_move, reward, state_new, done) # remember agent.remember(state_old, final_move, reward, state_new, done) if done: # train long memory, plot result game.reset() agent.n_games += 1 agent.train_long_memory() if score > record: record = score agent.model.save() print('Game', agent.n_games, 'Score', score, 'Record:', record) plot_scores.append(score) total_score += score mean_score = total_score / agent.n_games plot_mean_scores.append(mean_score) plot(plot_scores, plot_mean_scores)if name == 'main': train()
- Assistant
There are a few components in this code:
- The Agent class, which represents the AI agent that plays the game.
- The SnakeGameAI class, which represents the actual game environment.
- The Linear_QNet class, which is a simple linear neural network used as the Q-value function approximator.
- The QTrainer class, which implements the Q-learning algorithm using gradient descent to minimize the mean squared error between the predicted Q-values and the target Q-values.
- The train function, which is the main training loop that iteratively plays the game, collects experience tuples, trains the Q-value estimator on them, and plots the results.
Here's a breakdown of the key components in more detail:
- The Agent class
The Agent class is used to represent the AI agent that is trained to play the game. It has the following key properties and methods:
- n_games: The number of games the agent has played.
- epsilon: The exploration rate of the agent. This controls the balance between exploration and exploitation during training (i.e. how often the agent takes random actions vs. how often it takes the action with the highest Q-value).
- gamma: The discount factor for future rewards.
- memory: A deque (double-ended queue) that stores the agent's experience tuples (i.e. (state, action, reward, next_state, done)), which are used for training the Q-value estimator.
- model: The Q-value estimator (i.e. the neural network).
- trainer: The Q-learning trainer, which implements the Q-learning algorithm using gradient descent to update the Q-value estimator.
The key methods in this class are:
- get_state: Returns a feature vector representing the current state of the game board. This consists of a binary representation of various game board configurations such as the position of the snake's head and the location of the food.
- remember: Adds a new experience tuple to the agent's memory.
- train_long_memory: Trains the Q-value estimator on a batch of the agent's experience tuples (i.e. a mini-batch sampled from the memory).
- train_short_memory: Performs a single update step on the Q-value estimator using a single experience tuple (i.e. a single (state, action, reward, next_state, done) tuple).
- get_action: Returns the action that the agent should take based on the current state of the game board. This is done using an epsilon-greedy policy, where the agent takes a random action with probability epsilon and takes the action with the highest Q-value with probability 1-epsilon.
- The SnakeGameAI class
The SnakeGameAI class represents the game environment, which the agent interacts with. It has the following key properties and methods:
- screen_width, screen_height: The dimensions of the game board.
- snake_block, snake_list, snake_block2, snake_list2: The positions of the snakes on the game board.
- food: The position of the food on the game board.
- direction, direction2: The current directions of the snakes.
- score, score2: The current scores of the snakes.
- game_over: A flag that is set to True when the game is over.
The key methods in this class are:
- draw_snake: Draws the snake on the game board.
- draw_food: Draws the food on the game board.
- update_snake: Updates the position of the snake based on its current direction.
- update_food: Randomly places the food on the game board.
- reset: Resets the game state (i.e. snake positions, score, and game over flag).
- is_collision: Checks if a given point collides with the snake.
- play_step: Takes an action as input and performs a single step of the game. This involves updating the game state, checking for collisions, and calculating the reward.
- The Linear_QNet class
The Linear_QNet class is a simple linear neural network used as the Q-value function approximator. It has the following key properties and methods:
- fc1, fc2, fc3: The three fully connected layers of the network.
- forward: Computes the forward pass of the network (i.e. the Q-values for a given state).
- The QTrainer class
The QTrainer class implements the Q-learning algorithm using gradient descent to update the Q-value estimator. It has the following key properties and methods:
- model: The Q-value estimator (i.e. the neural network).
- lr: The learning rate for the optimizer.
- optimizer: The optimizer to use for gradient descent (e.g. Adam).
- loss_fn: The loss function to use (e.g. mean squared error).
The key methods in this class are:
- train_step: Updates the Q-values for a batch of experience tuples using gradient descent. This involves computing the target Q-values, computing the predicted Q-values using the current network weights, computing the loss between the predicted and target Q-values, backpropagating the loss through the network, and applying the updates to the network weights using the optimizer.
- User
there is a problem snake can eat food but snake2 just walk through it
- Assistant
If the second snake is not eating the food and walking through it, it's likely because the game logic is not accounting for the second snake's position when checking for collisions with the food.
You need to modify the
SnakeGameAIclass to consider the position ofsnake2when checking for food collisions. One way to do this is to add the position of the second snake to theis_collisionfunction.Here's an example of how you could modify the
is_collisionfunction:def is_collision(self, pt, pt2): if pt in self.snake_list[:-1] or pt in self.snake_list2[:-1]: return True return FalseWith this modification, the
is_collisionfunction checks if the given pointptorpt2is in either snake's body, and returnsTrueif there is a collision.You also need to modify the
play_stepfunction to handle the case where either snake eats the food, and update the score accordingly. Here's an example of how you could do this:if new_head == self.food: self.score += 1 self.food = self.get_random_location() elif new_head == self.food2: self.score2 += 1 self.food2 = self.get_random_location() else: self.snake_list.pop() self.snake_list2.pop()With these modifications, the second snake should be able to eat the food and the score should be updated accordingly.