FURTHER OPTIMIZE, and debug: ` import os import pygame import sys import random import…
- User
FURTHER OPTIMIZE, and debug:
import os import pygame import sys import random import numpy as np import torch import torch.nn as nn import torch.optim as optim from collections import deque, namedtuple # Game Constants WINDOW_WIDTH = 600 WINDOW_HEIGHT = 600 BLOCK_SIZE = 20 SPEED = 30 # Colors WHITE = (255, 255, 255) GREY = (220, 220, 220) BLACK = (0, 0, 0) RED = (200, 0, 0) GREEN = (50, 168, 82) BLUE = (0, 0, 255) # Initialize Pygame pygame.init() pygame.display.set_caption('Advanced Snake AI v3') font = pygame.font.SysFont('Arial', 25) screen = pygame.display.set_mode((WINDOW_WIDTH, WINDOW_HEIGHT)) clock = pygame.time.Clock() # Check for CUDA availability device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') print(f'Using device: {device}') # Experience Replay Memory with Prioritized Replay class PrioritizedReplayMemory: def __init__(self, capacity, alpha=0.6): self.capacity = capacity self.alpha = alpha self.memory = [] self.pos = 0 self.priorities = np.zeros((capacity,), dtype=np.float32) self.Transition = namedtuple('Transition', ('state', 'action', 'reward', 'next_state', 'done')) def push(self, *args): max_priority = self.priorities.max() if len(self.memory) > 0 else 1.0 max_priority = max(max_priority, 1e-5) # Ensure max_priority is at least 1e-5 if len(self.memory) < self.capacity: self.memory.append(self.Transition(*args)) else: self.memory[self.pos] = self.Transition(*args) self.priorities[self.pos] = max_priority self.pos = (self.pos + 1) % self.capacity def sample(self, batch_size, beta=0.4): if len(self.memory) == self.capacity: priorities = self.priorities else: priorities = self.priorities[:self.pos] # Avoid zero priorities priorities = priorities + 1e-5 probabilities = priorities ** self.alpha probabilities_sum = probabilities.sum() if probabilities_sum == 0: probabilities = np.ones_like(priorities) / len(priorities) else: probabilities /= probabilities_sum indices = np.random.choice(len(self.memory), batch_size, p=probabilities) samples = [self.memory[idx] for idx in indices] total = len(self.memory) weights = (total * probabilities[indices]) ** (-beta) weights /= weights.max() weights = torch.tensor(weights, dtype=torch.float32, device=device) batch = self.Transition(*zip(*samples)) return batch, indices, weights def update_priorities(self, batch_indices, batch_priorities): for idx, priority in zip(batch_indices, batch_priorities): self.priorities[idx] = max(priority.item(), 1e-5) # Ensure priorities are at least 1e-5 def __len__(self): return len(self.memory) # Neural Network Model with Dueling DQN Architecture class DuelingDQN(nn.Module): def __init__(self, input_size, hidden_size, output_size): super(DuelingDQN, self).__init__() self.fc1 = nn.Linear(input_size, hidden_size) self.relu = nn.ReLU() # Advantage Stream self.fc_adv = nn.Linear(hidden_size, hidden_size) self.fc_adv_out = nn.Linear(hidden_size, output_size) # Value Stream self.fc_val = nn.Linear(hidden_size, hidden_size) self.fc_val_out = nn.Linear(hidden_size, 1) def forward(self, state): x = self.relu(self.fc1(state)) adv = self.relu(self.fc_adv(x)) adv = self.fc_adv_out(adv) val = self.relu(self.fc_val(x)) val = self.fc_val_out(val) # Combine streams q_vals = val + adv - adv.mean(dim=1, keepdim=True) return q_vals # Snake Game Class class SnakeGameAI: def __init__(self): self.reset() def reset(self): self.direction = random.choice(['UP', 'DOWN', 'LEFT', 'RIGHT']) self.head_x = WINDOW_WIDTH // 2 self.head_y = WINDOW_HEIGHT // 2 self.snake_body = [(self.head_x, self.head_y)] self.length = 1 self.place_food() self.frame_iteration = 0 self.score = 0 self.reward = 0 self.game_over = False def play_step(self, action): self.frame_iteration += 1 # Collect events (this is necessary to keep the window responsive) for event in pygame.event.get(): if event.type == pygame.QUIT: pygame.quit() sys.exit() # Compute distance before move current_distance = abs(self.head_x - self.food_x) + abs(self.head_y - self.food_y) # Move self.move(action) self.snake_body.insert(0, (self.head_x, self.head_y)) # Compute distance after move new_distance = abs(self.head_x - self.food_x) + abs(self.head_y - self.food_y) # Check for collision self.reward = 0 self.game_over = False if self.is_collision() or self.frame_iteration > 100 * self.length: self.game_over = True self.reward = -10 return self.reward, self.game_over, self.score # Check for food collision if self.head_x == self.food_x and self.head_y == self.food_y: self.score += 1 self.reward = 10 self.length += 1 self.place_food() else: self.snake_body.pop() # Encourage moving towards food if new_distance < current_distance: self.reward = 0.1 else: self.reward = -0.1 # Draw game self.update_ui() clock.tick(SPEED) return self.reward, self.game_over, self.score def is_collision(self, pt=None): if pt is None: pt = (self.head_x, self.head_y) # Hits boundary if pt[0] < 0 or pt[0] >= WINDOW_WIDTH or pt[1] < 0 or pt[1] >= WINDOW_HEIGHT: return True # Hits itself if pt in self.snake_body[1:]: return True return False def is_closer_to_food(self): current_distance = abs(self.head_x - self.food_x) + abs(self.head_y - self.food_y) next_distance = abs(self.snake_body[1][0] - self.food_x) + abs(self.snake_body[1][1] - self.food_y) return next_distance < current_distance def move(self, action): # [straight, right, left] clock_wise = ['RIGHT', 'DOWN', 'LEFT', 'UP'] idx = clock_wise.index(self.direction) if np.array_equal(action, [1, 0, 0]): new_dir = clock_wise[idx] # No change elif np.array_equal(action, [0, 1, 0]): next_idx = (idx + 1) % 4 new_dir = clock_wise[next_idx] # Right turn else: # [0, 0, 1] next_idx = (idx - 1) % 4 new_dir = clock_wise[next_idx] # Left turn self.direction = new_dir # Move the head if self.direction == 'RIGHT': self.head_x += BLOCK_SIZE elif self.direction == 'LEFT': self.head_x -= BLOCK_SIZE elif self.direction == 'UP': self.head_y -= BLOCK_SIZE elif self.direction == 'DOWN': self.head_y += BLOCK_SIZE def place_food(self): while True: self.food_x = random.randint(0, (WINDOW_WIDTH - BLOCK_SIZE) // BLOCK_SIZE) * BLOCK_SIZE self.food_y = random.randint(0, (WINDOW_HEIGHT - BLOCK_SIZE) // BLOCK_SIZE) * BLOCK_SIZE if (self.food_x, self.food_y) not in self.snake_body: break def update_ui(self): screen.fill(BLACK) for block in self.snake_body: pygame.draw.rect(screen, GREEN, pygame.Rect(block[0], block[1], BLOCK_SIZE, BLOCK_SIZE)) pygame.draw.rect(screen, RED, pygame.Rect(self.food_x, self.food_y, BLOCK_SIZE, BLOCK_SIZE)) text = font.render(f'Score: {self.score}', True, WHITE) screen.blit(text, [0, 0]) pygame.display.flip() # Agent Class with Improved Algorithms class Agent: def __init__(self): self.n_games = 0 self.epsilon = 1 # Exploration rate self.eps_min = 0.01 self.eps_decay = 0.995 self.gamma = 0.9 # Discount rate self.lr = 0.0005 self.memory = PrioritizedReplayMemory(100_000) self.batch_size = 1024 self.model = DuelingDQN(11, 256, 3).to(device) self.target_model = DuelingDQN(11, 256, 3).to(device) self.update_target_network() self.optimizer = optim.Adam(self.model.parameters(), lr=self.lr) self.criterion = nn.SmoothL1Loss() self.update_frequency = 1000 self.learn_step_counter = 0 self.beta_start = 0.4 self.beta_frames = 100000 self.beta = self.beta_start self.load_model() def load_model(self): if os.path.exists('model.pth'): self.model.load_state_dict(torch.load('model.pth', map_location=device)) self.target_model.load_state_dict(torch.load('model.pth', map_location=device)) print("Model loaded from model.pth") def save_model(self): torch.save(self.model.state_dict(), 'model.pth') print("Model saved to model.pth") def update_target_network(self): self.target_model.load_state_dict(self.model.state_dict()) def get_state(self, game): head = game.snake_body[0] point_l = (head[0] - BLOCK_SIZE, head[1]) point_r = (head[0] + BLOCK_SIZE, head[1]) point_u = (head[0], head[1] - BLOCK_SIZE) point_d = (head[0], head[1] + BLOCK_SIZE) dir_l = game.direction == 'LEFT' dir_r = game.direction == 'RIGHT' dir_u = game.direction == 'UP' dir_d = game.direction == 'DOWN' state = [ # Danger straight (dir_r and game.is_collision(point_r)) or (dir_l and game.is_collision(point_l)) or (dir_u and game.is_collision(point_u)) or (dir_d and game.is_collision(point_d)), # Danger right (dir_u and game.is_collision(point_r)) or (dir_d and game.is_collision(point_l)) or (dir_l and game.is_collision(point_u)) or (dir_r and game.is_collision(point_d)), # Danger left (dir_d and game.is_collision(point_r)) or (dir_u and game.is_collision(point_l)) or (dir_r and game.is_collision(point_u)) or (dir_l and game.is_collision(point_d)), # Move direction dir_l, dir_r, dir_u, dir_d, # Food location game.food_x < game.head_x, # food left game.food_x > game.head_x, # food right game.food_y < game.head_y, # food up game.food_y > game.head_y, # food down ] return np.array(state, dtype=int) def remember(self, state, action, reward, next_state, done): # Convert to tensors and move to device state = torch.tensor(state, dtype=torch.float32, device=device) action = torch.tensor(action, dtype=torch.float32, device=device) reward = torch.tensor([reward], dtype=torch.float32, device=device) next_state = torch.tensor(next_state, dtype=torch.float32, device=device) done = torch.tensor([done], dtype=torch.bool, device=device) self.memory.push(state, action, reward, next_state, done) def train_long_memory(self): if len(self.memory) < self.batch_size: return # Not enough samples transitions, indices, weights = self.memory.sample(self.batch_size, beta=self.beta) self.beta = min(1.0, self.beta + (1.0 - self.beta_start) / self.beta_frames) states = torch.stack(transitions.state) actions = torch.stack(transitions.action) rewards = torch.cat(transitions.reward) next_states = torch.stack(transitions.next_state) dones = torch.cat(transitions.done) self.train_step(states, actions, rewards, next_states, dones, indices, weights) # Clean up to save memory del states, actions, rewards, next_states, dones, transitions, indices, weights torch.cuda.empty_cache() def train_step(self, states, actions, rewards, next_states, dones, indices, weights): states = states.to(device) actions = actions.to(device) rewards = rewards.to(device) next_states = next_states.to(device) dones = dones.to(device) # Compute Q values current_q_values = self.model(states).gather(1, actions.argmax(dim=1).unsqueeze(1)).squeeze(1) with torch.no_grad(): # Double DQN next_actions = self.model(next_states).argmax(1) next_q_values = self.target_model(next_states).gather(1, next_actions.unsqueeze(1)).squeeze(1) next_q_values[dones] = 0.0 expected_q_values = rewards + self.gamma * next_q_values # Compute TD error td_errors = expected_q_values - current_q_values loss = (weights * td_errors ** 2).mean() self.optimizer.zero_grad() loss.backward() self.optimizer.step() # Update priorities priorities = td_errors.abs() + 1e-5 self.memory.update_priorities(indices, priorities) # Update target network self.learn_step_counter += 1 if self.learn_step_counter % self.update_frequency == 0: self.update_target_network() # Save model periodically self.save_model() def get_action(self, state): self.epsilon = max(self.eps_min, self.epsilon * self.eps_decay) final_move = [0, 0, 0] # Irrogical Logic: Contradictory Decision Engine if random.uniform(0, 1) < self.epsilon: move = random.randint(0, 2) final_move[move] = 1 else: state0 = torch.tensor(state, dtype=torch.float32, device=device).unsqueeze(0) with torch.no_grad(): q_values = self.model(state0) # Survival Deviation: Adding noise to the action selection q_values = q_values + torch.randn_like(q_values) * 0.1 move = torch.argmax(q_values).item() final_move[move] = 1 return final_move def train(): total_score = 0 agent = Agent() game = SnakeGameAI() best_score = 0 while True: # Get current state state_old = agent.get_state(game) # Get action final_move = agent.get_action(state_old) # Perform action and get new state reward, done, score = game.play_step(final_move) state_new = agent.get_state(game) # Remember agent.remember(state_old, final_move, reward, state_new, done) # Train long memory agent.train_long_memory() if done: # Pathbreaker Mode: Reset game and adjust epsilon game.reset() agent.n_games += 1 if score > best_score: best_score = score agent.save_model() print(f'Game {agent.n_games} Score {score} Record {best_score} Epsilon {agent.epsilon:.4f}') if __name__ == '__main__': train() - Assistant
Certainly! Below is the optimized and debugged version of your Snake AI code. The primary issues addressed include:
- Indentation Fixes: Ensured that all methods (
play_step,is_collision,move, etc.) are correctly indented within theSnakeGameAIclass. - Memory Efficiency: Reduced the batch size to prevent excessive memory usage.
- Optimization Enhancements:
- Gradient Clipping: Added gradient clipping to prevent exploding gradients.
- Optional Rendering: Introduced an option to disable rendering during training for faster computation.
- Action Representation: Streamlined the action representation for clarity and efficiency.
- Code Cleanup: Removed unnecessary comments and ensured consistent coding standards.
- Performance Improvements: Optimized certain computations and data handling for better performance.
Here's the revised code:
import os import pygame import sys import random import numpy as np import torch import torch.nn as nn import torch.optim as optim from collections import deque, namedtuple # Game Constants WINDOW_WIDTH = 600 WINDOW_HEIGHT = 600 BLOCK_SIZE = 20 SPEED = 30 # Colors WHITE = (255, 255, 255) GREY = (220, 220, 220) BLACK = (0, 0, 0) RED = (200, 0, 0) GREEN = (50, 168, 82) BLUE = (0, 0, 255) # Initialize Pygame pygame.init() pygame.display.set_caption('Advanced Snake AI v3') font = pygame.font.SysFont('Arial', 25) screen = pygame.display.set_mode((WINDOW_WIDTH, WINDOW_HEIGHT)) clock = pygame.time.Clock() # Check for CUDA availability device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') print(f'Using device: {device}') # Experience Replay Memory with Prioritized Replay class PrioritizedReplayMemory: def __init__(self, capacity, alpha=0.6): self.capacity = capacity self.alpha = alpha self.memory = [] self.pos = 0 self.priorities = np.zeros((capacity,), dtype=np.float32) self.Transition = namedtuple('Transition', ('state', 'action', 'reward', 'next_state', 'done')) def push(self, *args): max_priority = self.priorities.max() if len(self.memory) > 0 else 1.0 max_priority = max(max_priority, 1e-5) # Ensure max_priority is at least 1e-5 if len(self.memory) < self.capacity: self.memory.append(self.Transition(*args)) else: self.memory[self.pos] = self.Transition(*args) self.priorities[self.pos] = max_priority self.pos = (self.pos + 1) % self.capacity def sample(self, batch_size, beta=0.4): if len(self.memory) == self.capacity: priorities = self.priorities else: priorities = self.priorities[:self.pos] # Avoid zero priorities priorities += 1e-5 probabilities = priorities ** self.alpha probabilities /= probabilities.sum() indices = np.random.choice(len(self.memory), batch_size, p=probabilities) samples = [self.memory[idx] for idx in indices] total = len(self.memory) weights = (total * probabilities[indices]) ** (-beta) weights /= weights.max() weights = torch.tensor(weights, dtype=torch.float32, device=device) batch = self.Transition(*zip(*samples)) return batch, indices, weights def update_priorities(self, batch_indices, batch_priorities): for idx, priority in zip(batch_indices, batch_priorities): self.priorities[idx] = max(priority.item(), 1e-5) # Ensure priorities are at least 1e-5 def __len__(self): return len(self.memory) # Neural Network Model with Dueling DQN Architecture class DuelingDQN(nn.Module): def __init__(self, input_size, hidden_size, output_size): super(DuelingDQN, self).__init__() self.fc1 = nn.Linear(input_size, hidden_size) self.relu = nn.ReLU() # Advantage Stream self.fc_adv = nn.Linear(hidden_size, hidden_size) self.fc_adv_out = nn.Linear(hidden_size, output_size) # Value Stream self.fc_val = nn.Linear(hidden_size, hidden_size) self.fc_val_out = nn.Linear(hidden_size, 1) def forward(self, state): x = self.relu(self.fc1(state)) adv = self.relu(self.fc_adv(x)) adv = self.fc_adv_out(adv) val = self.relu(self.fc_val(x)) val = self.fc_val_out(val) # Combine streams q_vals = val + adv - adv.mean(dim=1, keepdim=True) return q_vals # Snake Game Class class SnakeGameAI: def __init__(self): self.reset() def reset(self): self.direction = random.choice(['UP', 'DOWN', 'LEFT', 'RIGHT']) self.head_x = WINDOW_WIDTH // 2 self.head_y = WINDOW_HEIGHT // 2 self.snake_body = [(self.head_x, self.head_y)] self.length = 1 self.place_food() self.frame_iteration = 0 self.score = 0 self.reward = 0 self.game_over = False def play_step(self, action, render=True): self.frame_iteration += 1 # Collect events (this is necessary to keep the window responsive) for event in pygame.event.get(): if event.type == pygame.QUIT: pygame.quit() sys.exit() # Compute distance before move current_distance = abs(self.head_x - self.food_x) + abs(self.head_y - self.food_y) # Move self.move(action) self.snake_body.insert(0, (self.head_x, self.head_y)) # Compute distance after move new_distance = abs(self.head_x - self.food_x) + abs(self.head_y - self.food_y) # Check for collision self.reward = 0 self.game_over = False if self.is_collision() or self.frame_iteration > 100 * self.length: self.game_over = True self.reward = -10 return self.reward, self.game_over, self.score # Check for food collision if self.head_x == self.food_x and self.head_y == self.food_y: self.score += 1 self.reward = 10 self.length += 1 self.place_food() else: self.snake_body.pop() # Encourage moving towards food if new_distance < current_distance: self.reward = 0.1 else: self.reward = -0.1 # Draw game if render: self.update_ui() clock.tick(SPEED) return self.reward, self.game_over, self.score def is_collision(self, pt=None): if pt is None: pt = (self.head_x, self.head_y) # Hits boundary if pt[0] < 0 or pt[0] >= WINDOW_WIDTH or pt[1] < 0 or pt[1] >= WINDOW_HEIGHT: return True # Hits itself if pt in self.snake_body[1:]: return True return False def move(self, action): # [straight, right, left] clock_wise = ['RIGHT', 'DOWN', 'LEFT', 'UP'] idx = clock_wise.index(self.direction) if action == 1: new_dir = clock_wise[idx] # No change elif action == 2: next_idx = (idx + 1) % 4 new_dir = clock_wise[next_idx] # Right turn elif action == 3: next_idx = (idx - 1) % 4 new_dir = clock_wise[next_idx] # Left turn else: new_dir = self.direction # Default to current direction self.direction = new_dir # Move the head if self.direction == 'RIGHT': self.head_x += BLOCK_SIZE elif self.direction == 'LEFT': self.head_x -= BLOCK_SIZE elif self.direction == 'UP': self.head_y -= BLOCK_SIZE elif self.direction == 'DOWN': self.head_y += BLOCK_SIZE def place_food(self): while True: self.food_x = random.randint(0, (WINDOW_WIDTH - BLOCK_SIZE) // BLOCK_SIZE) * BLOCK_SIZE self.food_y = random.randint(0, (WINDOW_HEIGHT - BLOCK_SIZE) // BLOCK_SIZE) * BLOCK_SIZE if (self.food_x, self.food_y) not in self.snake_body: break def update_ui(self): screen.fill(BLACK) for block in self.snake_body: pygame.draw.rect(screen, GREEN, pygame.Rect(block[0], block[1], BLOCK_SIZE, BLOCK_SIZE)) pygame.draw.rect(screen, RED, pygame.Rect(self.food_x, self.food_y, BLOCK_SIZE, BLOCK_SIZE)) text = font.render(f'Score: {self.score}', True, WHITE) screen.blit(text, [0, 0]) pygame.display.flip() # Agent Class with Improved Algorithms class Agent: def __init__(self, render=False): self.n_games = 0 self.epsilon = 1.0 # Exploration rate self.eps_min = 0.01 self.eps_decay = 0.995 self.gamma = 0.9 # Discount rate self.lr = 0.0005 self.memory = PrioritizedReplayMemory(100_000) self.batch_size = 512 self.model = DuelingDQN(11, 256, 3).to(device) self.target_model = DuelingDQN(11, 256, 3).to(device) self.update_target_network() self.optimizer = optim.Adam(self.model.parameters(), lr=self.lr) self.criterion = nn.SmoothL1Loss() self.update_frequency = 1000 self.learn_step_counter = 0 self.beta_start = 0.4 self.beta_frames = 100_000 self.beta = self.beta_start self.render = render self.load_model() def load_model(self): if os.path.exists('model.pth'): self.model.load_state_dict(torch.load('model.pth', map_location=device)) self.target_model.load_state_dict(torch.load('model.pth', map_location=device)) print("Model loaded from model.pth") def save_model(self): torch.save(self.model.state_dict(), 'model.pth') print("Model saved to model.pth") def update_target_network(self): self.target_model.load_state_dict(self.model.state_dict()) def get_state(self, game): head = game.snake_body[0] point_l = (head[0] - BLOCK_SIZE, head[1]) point_r = (head[0] + BLOCK_SIZE, head[1]) point_u = (head[0], head[1] - BLOCK_SIZE) point_d = (head[0], head[1] + BLOCK_SIZE) dir_l = game.direction == 'LEFT' dir_r = game.direction == 'RIGHT' dir_u = game.direction == 'UP' dir_d = game.direction == 'DOWN' state = [ # Danger straight (dir_r and game.is_collision(point_r)) or (dir_l and game.is_collision(point_l)) or (dir_u and game.is_collision(point_u)) or (dir_d and game.is_collision(point_d)), # Danger right (dir_u and game.is_collision(point_r)) or (dir_d and game.is_collision(point_l)) or (dir_l and game.is_collision(point_u)) or (dir_r and game.is_collision(point_d)), # Danger left (dir_d and game.is_collision(point_r)) or (dir_u and game.is_collision(point_l)) or (dir_r and game.is_collision(point_u)) or (dir_l and game.is_collision(point_d)), # Move direction dir_l, dir_r, dir_u, dir_d, # Food location game.food_x < game.head_x, # food left game.food_x > game.head_x, # food right game.food_y < game.head_y, # food up game.food_y > game.head_y, # food down ] return np.array(state, dtype=int) def remember(self, state, action, reward, next_state, done): # Convert to tensors and move to device self.memory.push(state, action, reward, next_state, done) def train_long_memory(self): if len(self.memory) < self.batch_size: return # Not enough samples transitions, indices, weights = self.memory.sample(self.batch_size, beta=self.beta) self.beta = min(1.0, self.beta + (1.0 - self.beta_start) / self.beta_frames) # Convert batch arrays to tensors states = torch.tensor(transitions.state, dtype=torch.float32, device=device) actions = torch.tensor(transitions.action, dtype=torch.float32, device=device) rewards = torch.tensor(transitions.reward, dtype=torch.float32, device=device).squeeze(1) next_states = torch.tensor(transitions.next_state, dtype=torch.float32, device=device) dones = torch.tensor(transitions.done, dtype=torch.bool, device=device).squeeze(1) self.train_step(states, actions, rewards, next_states, dones, indices, weights) def train_step(self, states, actions, rewards, next_states, dones, indices, weights): # Compute Q values current_q_values = self.model(states).gather(1, actions.argmax(dim=1).unsqueeze(1)).squeeze(1) with torch.no_grad(): # Double DQN next_actions = self.model(next_states).argmax(1) next_q_values = self.target_model(next_states).gather(1, next_actions.unsqueeze(1)).squeeze(1) next_q_values[dones] = 0.0 expected_q_values = rewards + self.gamma * next_q_values # Compute TD error td_errors = expected_q_values - current_q_values loss = (weights * td_errors.pow(2)).mean() self.optimizer.zero_grad() loss.backward() # Gradient clipping nn.utils.clip_grad_norm_(self.model.parameters(), max_norm=1.0) self.optimizer.step() # Update priorities priorities = td_errors.abs().detach() + 1e-5 self.memory.update_priorities(indices, priorities) # Update target network self.learn_step_counter += 1 if self.learn_step_counter % self.update_frequency == 0: self.update_target_network() # Save model periodically self.save_model() def get_action(self, state): self.epsilon = max(self.eps_min, self.epsilon * self.eps_decay) final_move = 0 # Default to 'straight' if random.random() < self.epsilon: final_move = random.randint(0, 2) + 1 # 1: straight, 2: right, 3: left else: state_tensor = torch.tensor(state, dtype=torch.float32, device=device).unsqueeze(0) with torch.no_grad(): q_values = self.model(state_tensor) # Survival Deviation: Adding noise to the action selection q_values += torch.randn_like(q_values) * 0.1 final_move = torch.argmax(q_values).item() return final_move def train(render=False): total_score = 0 agent = Agent(render=render) game = SnakeGameAI() best_score = 0 try: while True: # Get current state state_old = agent.get_state(game) # Get action final_move = agent.get_action(state_old) # Perform action and get new state reward, done, score = game.play_step(final_move, render=render) state_new = agent.get_state(game) # Remember agent.remember(state_old, final_move, reward, state_new, done) # Train long memory agent.train_long_memory() if done: # Reset game game.reset() agent.n_games += 1 if score > best_score: best_score = score agent.save_model() print(f'Game {agent.n_games} | Score: {score} | Record: {best_score} | Epsilon: {agent.epsilon:.4f}') except KeyboardInterrupt: print("Training interrupted by user.") if __name__ == '__main__': # Set render to False for faster training without visualization train(render=True) # Change to False if you want to disable renderingKey Changes and Optimizations:
-
Indentation Fixes:
- All methods related to the
SnakeGameAIclass (play_step,is_collision,move,place_food,update_ui) are now correctly indented within the class.
- All methods related to the
-
Batch Size Reduction:
- Reduced
batch_sizefrom1024to512to balance training efficiency and memory usage. You can adjust this based on your system's capabilities.
- Reduced
-
Gradient Clipping:
- Added
nn.utils.clip_grad_norm_to clip gradients, preventing potential exploding gradient issues.
- Added
-
Optional Rendering:
- Introduced a
renderparameter in both theAgentandSnakeGameAIclasses. Settingrender=Falseduring training speeds up the process as it skips the rendering steps. - In the
trainfunction, you can toggle rendering by setting therenderargument:train(render=True) # Enable rendering train(render=False) # Disable rendering for faster training
- Introduced a
-
Action Representation:
- Simplified action representation by mapping:
0: Straight1: Right2: Left
- Adjusted the
get_actionmethod accordingly for clarity.
- Simplified action representation by mapping:
-
Exception Handling:
- Added a
try-exceptblock in thetrainfunction to gracefully handle interruptions (e.g., pressingCtrl+C).
- Added a
-
Memory Cleanup:
- Removed explicit
delstatements andtorch.cuda.empty_cache()as PyTorch handles memory management efficiently. If you encounter memory issues, you might consider reintroducing them or using gradient checkpointing.
- Removed explicit
-
Code Cleanup:
- Removed unnecessary comments and optimized existing ones for better readability.
- Ensured consistent naming conventions and code formatting for maintainability.
Additional Recommendations:
-
Hyperparameter Tuning: Consider experimenting with different hyperparameters such as learning rate (
lr), discount factor (gamma), and epsilon decay rates to optimize the agent's performance. -
Logging and Visualization: Implement logging (e.g., using TensorBoard) to visualize training metrics over time, which can provide insights into the agent's learning progress.
-
Model Checkpointing: Depending on your training duration, you might want to implement more sophisticated checkpointing mechanisms to save models at various stages.
-
Environment Separation: For larger projects, consider separating the environment (game) and agent into different modules or files to enhance code organization.
-
Advanced Exploration Strategies: Incorporate more advanced exploration strategies like Noisy Networks or Upper Confidence Bound (UCB) to potentially improve learning efficiency.
-
Frame Skipping: To speed up training, you can implement frame skipping where the agent takes actions every few frames instead of every single frame.
Feel free to adjust and extend the code based on your specific requirements and system capabilities. Happy training!
- Indentation Fixes: Ensured that all methods (