Skip to content
TheChatArchiveThe Chat Archive
o1-mini-2024-09-12· OpenAIAi2 WildChat-4.8M (AI2)

FURTHER OPTIMIZE, and debug: ` import os import pygame import sys import random import…

  1. User

    FURTHER OPTIMIZE, and debug:

    import os
    import pygame
    import sys
    import random
    import numpy as np
    import torch
    import torch.nn as nn
    import torch.optim as optim
    from collections import deque, namedtuple
    
    # Game Constants
    WINDOW_WIDTH = 600
    WINDOW_HEIGHT = 600
    BLOCK_SIZE = 20
    SPEED = 30
    
    # Colors
    WHITE = (255, 255, 255)
    GREY = (220, 220, 220)
    BLACK = (0, 0, 0)
    RED = (200, 0, 0)
    GREEN = (50, 168, 82)
    BLUE = (0, 0, 255)
    
    # Initialize Pygame
    pygame.init()
    pygame.display.set_caption('Advanced Snake AI v3')
    font = pygame.font.SysFont('Arial', 25)
    screen = pygame.display.set_mode((WINDOW_WIDTH, WINDOW_HEIGHT))
    clock = pygame.time.Clock()
    
    # Check for CUDA availability
    device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
    print(f'Using device: {device}')
    
    # Experience Replay Memory with Prioritized Replay
    class PrioritizedReplayMemory:
        def __init__(self, capacity, alpha=0.6):
            self.capacity = capacity
            self.alpha = alpha
            self.memory = []
            self.pos = 0
            self.priorities = np.zeros((capacity,), dtype=np.float32)
            self.Transition = namedtuple('Transition', ('state', 'action', 'reward', 'next_state', 'done'))
    
        def push(self, *args):
            max_priority = self.priorities.max() if len(self.memory) > 0 else 1.0
            max_priority = max(max_priority, 1e-5)  # Ensure max_priority is at least 1e-5
            if len(self.memory) < self.capacity:
                self.memory.append(self.Transition(*args))
            else:
                self.memory[self.pos] = self.Transition(*args)
            self.priorities[self.pos] = max_priority
            self.pos = (self.pos + 1) % self.capacity
    
        def sample(self, batch_size, beta=0.4):
            if len(self.memory) == self.capacity:
                priorities = self.priorities
            else:
                priorities = self.priorities[:self.pos]
    
            # Avoid zero priorities
            priorities = priorities + 1e-5
            probabilities = priorities ** self.alpha
            probabilities_sum = probabilities.sum()
            if probabilities_sum == 0:
                probabilities = np.ones_like(priorities) / len(priorities)
            else:
                probabilities /= probabilities_sum
    
            indices = np.random.choice(len(self.memory), batch_size, p=probabilities)
            samples = [self.memory[idx] for idx in indices]
    
            total = len(self.memory)
            weights = (total * probabilities[indices]) ** (-beta)
            weights /= weights.max()
            weights = torch.tensor(weights, dtype=torch.float32, device=device)
    
            batch = self.Transition(*zip(*samples))
            return batch, indices, weights
    
        def update_priorities(self, batch_indices, batch_priorities):
            for idx, priority in zip(batch_indices, batch_priorities):
                self.priorities[idx] = max(priority.item(), 1e-5)  # Ensure priorities are at least 1e-5
    
        def __len__(self):
            return len(self.memory)
    
    # Neural Network Model with Dueling DQN Architecture
    class DuelingDQN(nn.Module):
        def __init__(self, input_size, hidden_size, output_size):
            super(DuelingDQN, self).__init__()
            self.fc1 = nn.Linear(input_size, hidden_size)
            self.relu = nn.ReLU()
    
            # Advantage Stream
            self.fc_adv = nn.Linear(hidden_size, hidden_size)
            self.fc_adv_out = nn.Linear(hidden_size, output_size)
    
            # Value Stream
            self.fc_val = nn.Linear(hidden_size, hidden_size)
            self.fc_val_out = nn.Linear(hidden_size, 1)
    
        def forward(self, state):
            x = self.relu(self.fc1(state))
    
            adv = self.relu(self.fc_adv(x))
            adv = self.fc_adv_out(adv)
    
            val = self.relu(self.fc_val(x))
            val = self.fc_val_out(val)
    
            # Combine streams
            q_vals = val + adv - adv.mean(dim=1, keepdim=True)
            return q_vals
    
    # Snake Game Class
    class SnakeGameAI:
        def __init__(self):
            self.reset()
    
        def reset(self):
            self.direction = random.choice(['UP', 'DOWN', 'LEFT', 'RIGHT'])
            self.head_x = WINDOW_WIDTH // 2
            self.head_y = WINDOW_HEIGHT // 2
            self.snake_body = [(self.head_x, self.head_y)]
            self.length = 1
    
            self.place_food()
            self.frame_iteration = 0
            self.score = 0
            self.reward = 0
            self.game_over = False
    
    def play_step(self, action):
        self.frame_iteration += 1
    
        # Collect events (this is necessary to keep the window responsive)
        for event in pygame.event.get():
            if event.type == pygame.QUIT:
                pygame.quit()
                sys.exit()
    
        # Compute distance before move
        current_distance = abs(self.head_x - self.food_x) + abs(self.head_y - self.food_y)
    
        # Move
        self.move(action)
        self.snake_body.insert(0, (self.head_x, self.head_y))
    
        # Compute distance after move
        new_distance = abs(self.head_x - self.food_x) + abs(self.head_y - self.food_y)
    
        # Check for collision
        self.reward = 0
        self.game_over = False
        if self.is_collision() or self.frame_iteration > 100 * self.length:
            self.game_over = True
            self.reward = -10
            return self.reward, self.game_over, self.score
    
        # Check for food collision
        if self.head_x == self.food_x and self.head_y == self.food_y:
            self.score += 1
            self.reward = 10
            self.length += 1
            self.place_food()
        else:
            self.snake_body.pop()
            # Encourage moving towards food
            if new_distance < current_distance:
                self.reward = 0.1
            else:
                self.reward = -0.1
    
        # Draw game
        self.update_ui()
        clock.tick(SPEED)
    
        return self.reward, self.game_over, self.score
    
        def is_collision(self, pt=None):
            if pt is None:
                pt = (self.head_x, self.head_y)
            # Hits boundary
            if pt[0] < 0 or pt[0] >= WINDOW_WIDTH or pt[1] < 0 or pt[1] >= WINDOW_HEIGHT:
                return True
            # Hits itself
            if pt in self.snake_body[1:]:
                return True
            return False
    
        def is_closer_to_food(self):
            current_distance = abs(self.head_x - self.food_x) + abs(self.head_y - self.food_y)
            next_distance = abs(self.snake_body[1][0] - self.food_x) + abs(self.snake_body[1][1] - self.food_y)
            return next_distance < current_distance
    
        def move(self, action):
            # [straight, right, left]
            clock_wise = ['RIGHT', 'DOWN', 'LEFT', 'UP']
            idx = clock_wise.index(self.direction)
    
            if np.array_equal(action, [1, 0, 0]):
                new_dir = clock_wise[idx]  # No change
            elif np.array_equal(action, [0, 1, 0]):
                next_idx = (idx + 1) % 4
                new_dir = clock_wise[next_idx]  # Right turn
            else:  # [0, 0, 1]
                next_idx = (idx - 1) % 4
                new_dir = clock_wise[next_idx]  # Left turn
            self.direction = new_dir
    
            # Move the head
            if self.direction == 'RIGHT':
                self.head_x += BLOCK_SIZE
            elif self.direction == 'LEFT':
                self.head_x -= BLOCK_SIZE
            elif self.direction == 'UP':
                self.head_y -= BLOCK_SIZE
            elif self.direction == 'DOWN':
                self.head_y += BLOCK_SIZE
    
        def place_food(self):
            while True:
                self.food_x = random.randint(0, (WINDOW_WIDTH - BLOCK_SIZE) // BLOCK_SIZE) * BLOCK_SIZE
                self.food_y = random.randint(0, (WINDOW_HEIGHT - BLOCK_SIZE) // BLOCK_SIZE) * BLOCK_SIZE
                if (self.food_x, self.food_y) not in self.snake_body:
                    break
    
        def update_ui(self):
            screen.fill(BLACK)
            for block in self.snake_body:
                pygame.draw.rect(screen, GREEN, pygame.Rect(block[0], block[1], BLOCK_SIZE, BLOCK_SIZE))
            pygame.draw.rect(screen, RED, pygame.Rect(self.food_x, self.food_y, BLOCK_SIZE, BLOCK_SIZE))
            text = font.render(f'Score: {self.score}', True, WHITE)
            screen.blit(text, [0, 0])
            pygame.display.flip()
    
    # Agent Class with Improved Algorithms
    class Agent:
        def __init__(self):
            self.n_games = 0
            self.epsilon = 1  # Exploration rate
            self.eps_min = 0.01
            self.eps_decay = 0.995
            self.gamma = 0.9  # Discount rate
            self.lr = 0.0005
            self.memory = PrioritizedReplayMemory(100_000)
            self.batch_size = 1024
            self.model = DuelingDQN(11, 256, 3).to(device)
            self.target_model = DuelingDQN(11, 256, 3).to(device)
            self.update_target_network()
            self.optimizer = optim.Adam(self.model.parameters(), lr=self.lr)
            self.criterion = nn.SmoothL1Loss()
            self.update_frequency = 1000
            self.learn_step_counter = 0
            self.beta_start = 0.4
            self.beta_frames = 100000
            self.beta = self.beta_start
            self.load_model()
    
        def load_model(self):
            if os.path.exists('model.pth'):
                self.model.load_state_dict(torch.load('model.pth', map_location=device))
                self.target_model.load_state_dict(torch.load('model.pth', map_location=device))
                print("Model loaded from model.pth")
    
        def save_model(self):
            torch.save(self.model.state_dict(), 'model.pth')
            print("Model saved to model.pth")
    
        def update_target_network(self):
            self.target_model.load_state_dict(self.model.state_dict())
    
        def get_state(self, game):
            head = game.snake_body[0]
            point_l = (head[0] - BLOCK_SIZE, head[1])
            point_r = (head[0] + BLOCK_SIZE, head[1])
            point_u = (head[0], head[1] - BLOCK_SIZE)
            point_d = (head[0], head[1] + BLOCK_SIZE)
    
            dir_l = game.direction == 'LEFT'
            dir_r = game.direction == 'RIGHT'
            dir_u = game.direction == 'UP'
            dir_d = game.direction == 'DOWN'
    
            state = [
                # Danger straight
                (dir_r and game.is_collision(point_r)) or
                (dir_l and game.is_collision(point_l)) or
                (dir_u and game.is_collision(point_u)) or
                (dir_d and game.is_collision(point_d)),
    
                # Danger right
                (dir_u and game.is_collision(point_r)) or
                (dir_d and game.is_collision(point_l)) or
                (dir_l and game.is_collision(point_u)) or
                (dir_r and game.is_collision(point_d)),
    
                # Danger left
                (dir_d and game.is_collision(point_r)) or
                (dir_u and game.is_collision(point_l)) or
                (dir_r and game.is_collision(point_u)) or
                (dir_l and game.is_collision(point_d)),
    
                # Move direction
                dir_l,
                dir_r,
                dir_u,
                dir_d,
    
                # Food location
                game.food_x < game.head_x,  # food left
                game.food_x > game.head_x,  # food right
                game.food_y < game.head_y,  # food up
                game.food_y > game.head_y,  # food down
            ]
            return np.array(state, dtype=int)
    
        def remember(self, state, action, reward, next_state, done):
            # Convert to tensors and move to device
            state = torch.tensor(state, dtype=torch.float32, device=device)
            action = torch.tensor(action, dtype=torch.float32, device=device)
            reward = torch.tensor([reward], dtype=torch.float32, device=device)
            next_state = torch.tensor(next_state, dtype=torch.float32, device=device)
            done = torch.tensor([done], dtype=torch.bool, device=device)
            self.memory.push(state, action, reward, next_state, done)
    
        def train_long_memory(self):
            if len(self.memory) < self.batch_size:
                return  # Not enough samples
            transitions, indices, weights = self.memory.sample(self.batch_size, beta=self.beta)
            self.beta = min(1.0, self.beta + (1.0 - self.beta_start) / self.beta_frames)
    
            states = torch.stack(transitions.state)
            actions = torch.stack(transitions.action)
            rewards = torch.cat(transitions.reward)
            next_states = torch.stack(transitions.next_state)
            dones = torch.cat(transitions.done)
    
            self.train_step(states, actions, rewards, next_states, dones, indices, weights)
    
            # Clean up to save memory
            del states, actions, rewards, next_states, dones, transitions, indices, weights
            torch.cuda.empty_cache()
    
        def train_step(self, states, actions, rewards, next_states, dones, indices, weights):
            states = states.to(device)
            actions = actions.to(device)
            rewards = rewards.to(device)
            next_states = next_states.to(device)
            dones = dones.to(device)
    
            # Compute Q values
            current_q_values = self.model(states).gather(1, actions.argmax(dim=1).unsqueeze(1)).squeeze(1)
    
            with torch.no_grad():
                # Double DQN
                next_actions = self.model(next_states).argmax(1)
                next_q_values = self.target_model(next_states).gather(1, next_actions.unsqueeze(1)).squeeze(1)
                next_q_values[dones] = 0.0
    
            expected_q_values = rewards + self.gamma * next_q_values
    
            # Compute TD error
            td_errors = expected_q_values - current_q_values
            loss = (weights * td_errors ** 2).mean()
    
            self.optimizer.zero_grad()
            loss.backward()
            self.optimizer.step()
    
            # Update priorities
            priorities = td_errors.abs() + 1e-5
            self.memory.update_priorities(indices, priorities)
    
            # Update target network
            self.learn_step_counter += 1
            if self.learn_step_counter % self.update_frequency == 0:
                self.update_target_network()
                # Save model periodically
                self.save_model()
    
        def get_action(self, state):
            self.epsilon = max(self.eps_min, self.epsilon * self.eps_decay)
            final_move = [0, 0, 0]
    
            # Irrogical Logic: Contradictory Decision Engine
            if random.uniform(0, 1) < self.epsilon:
                move = random.randint(0, 2)
                final_move[move] = 1
            else:
                state0 = torch.tensor(state, dtype=torch.float32, device=device).unsqueeze(0)
                with torch.no_grad():
                    q_values = self.model(state0)
                    # Survival Deviation: Adding noise to the action selection
                    q_values = q_values + torch.randn_like(q_values) * 0.1
                    move = torch.argmax(q_values).item()
                final_move[move] = 1
            return final_move
    
    def train():
        total_score = 0
        agent = Agent()
        game = SnakeGameAI()
        best_score = 0
        while True:
            # Get current state
            state_old = agent.get_state(game)
    
            # Get action
            final_move = agent.get_action(state_old)
    
            # Perform action and get new state
            reward, done, score = game.play_step(final_move)
            state_new = agent.get_state(game)
    
            # Remember
            agent.remember(state_old, final_move, reward, state_new, done)
    
            # Train long memory
            agent.train_long_memory()
    
            if done:
                # Pathbreaker Mode: Reset game and adjust epsilon
                game.reset()
                agent.n_games += 1
    
                if score > best_score:
                    best_score = score
                    agent.save_model()
    
                print(f'Game {agent.n_games} Score {score} Record {best_score} Epsilon {agent.epsilon:.4f}')
    
    if __name__ == '__main__':
        train()
    
  2. Assistant

    Certainly! Below is the optimized and debugged version of your Snake AI code. The primary issues addressed include:

    1. Indentation Fixes: Ensured that all methods (play_step, is_collision, move, etc.) are correctly indented within the SnakeGameAI class.
    2. Memory Efficiency: Reduced the batch size to prevent excessive memory usage.
    3. Optimization Enhancements:
      • Gradient Clipping: Added gradient clipping to prevent exploding gradients.
      • Optional Rendering: Introduced an option to disable rendering during training for faster computation.
      • Action Representation: Streamlined the action representation for clarity and efficiency.
    4. Code Cleanup: Removed unnecessary comments and ensured consistent coding standards.
    5. Performance Improvements: Optimized certain computations and data handling for better performance.

    Here's the revised code:

    import os
    import pygame
    import sys
    import random
    import numpy as np
    import torch
    import torch.nn as nn
    import torch.optim as optim
    from collections import deque, namedtuple
    
    # Game Constants
    WINDOW_WIDTH = 600
    WINDOW_HEIGHT = 600
    BLOCK_SIZE = 20
    SPEED = 30
    
    # Colors
    WHITE = (255, 255, 255)
    GREY = (220, 220, 220)
    BLACK = (0, 0, 0)
    RED = (200, 0, 0)
    GREEN = (50, 168, 82)
    BLUE = (0, 0, 255)
    
    # Initialize Pygame
    pygame.init()
    pygame.display.set_caption('Advanced Snake AI v3')
    font = pygame.font.SysFont('Arial', 25)
    screen = pygame.display.set_mode((WINDOW_WIDTH, WINDOW_HEIGHT))
    clock = pygame.time.Clock()
    
    # Check for CUDA availability
    device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
    print(f'Using device: {device}')
    
    # Experience Replay Memory with Prioritized Replay
    class PrioritizedReplayMemory:
        def __init__(self, capacity, alpha=0.6):
            self.capacity = capacity
            self.alpha = alpha
            self.memory = []
            self.pos = 0
            self.priorities = np.zeros((capacity,), dtype=np.float32)
            self.Transition = namedtuple('Transition', ('state', 'action', 'reward', 'next_state', 'done'))
    
        def push(self, *args):
            max_priority = self.priorities.max() if len(self.memory) > 0 else 1.0
            max_priority = max(max_priority, 1e-5)  # Ensure max_priority is at least 1e-5
            if len(self.memory) < self.capacity:
                self.memory.append(self.Transition(*args))
            else:
                self.memory[self.pos] = self.Transition(*args)
            self.priorities[self.pos] = max_priority
            self.pos = (self.pos + 1) % self.capacity
    
        def sample(self, batch_size, beta=0.4):
            if len(self.memory) == self.capacity:
                priorities = self.priorities
            else:
                priorities = self.priorities[:self.pos]
    
            # Avoid zero priorities
            priorities += 1e-5
            probabilities = priorities ** self.alpha
            probabilities /= probabilities.sum()
    
            indices = np.random.choice(len(self.memory), batch_size, p=probabilities)
            samples = [self.memory[idx] for idx in indices]
    
            total = len(self.memory)
            weights = (total * probabilities[indices]) ** (-beta)
            weights /= weights.max()
            weights = torch.tensor(weights, dtype=torch.float32, device=device)
    
            batch = self.Transition(*zip(*samples))
            return batch, indices, weights
    
        def update_priorities(self, batch_indices, batch_priorities):
            for idx, priority in zip(batch_indices, batch_priorities):
                self.priorities[idx] = max(priority.item(), 1e-5)  # Ensure priorities are at least 1e-5
    
        def __len__(self):
            return len(self.memory)
    
    # Neural Network Model with Dueling DQN Architecture
    class DuelingDQN(nn.Module):
        def __init__(self, input_size, hidden_size, output_size):
            super(DuelingDQN, self).__init__()
            self.fc1 = nn.Linear(input_size, hidden_size)
            self.relu = nn.ReLU()
    
            # Advantage Stream
            self.fc_adv = nn.Linear(hidden_size, hidden_size)
            self.fc_adv_out = nn.Linear(hidden_size, output_size)
    
            # Value Stream
            self.fc_val = nn.Linear(hidden_size, hidden_size)
            self.fc_val_out = nn.Linear(hidden_size, 1)
    
        def forward(self, state):
            x = self.relu(self.fc1(state))
    
            adv = self.relu(self.fc_adv(x))
            adv = self.fc_adv_out(adv)
    
            val = self.relu(self.fc_val(x))
            val = self.fc_val_out(val)
    
            # Combine streams
            q_vals = val + adv - adv.mean(dim=1, keepdim=True)
            return q_vals
    
    # Snake Game Class
    class SnakeGameAI:
        def __init__(self):
            self.reset()
    
        def reset(self):
            self.direction = random.choice(['UP', 'DOWN', 'LEFT', 'RIGHT'])
            self.head_x = WINDOW_WIDTH // 2
            self.head_y = WINDOW_HEIGHT // 2
            self.snake_body = [(self.head_x, self.head_y)]
            self.length = 1
    
            self.place_food()
            self.frame_iteration = 0
            self.score = 0
            self.reward = 0
            self.game_over = False
    
        def play_step(self, action, render=True):
            self.frame_iteration += 1
    
            # Collect events (this is necessary to keep the window responsive)
            for event in pygame.event.get():
                if event.type == pygame.QUIT:
                    pygame.quit()
                    sys.exit()
    
            # Compute distance before move
            current_distance = abs(self.head_x - self.food_x) + abs(self.head_y - self.food_y)
    
            # Move
            self.move(action)
            self.snake_body.insert(0, (self.head_x, self.head_y))
    
            # Compute distance after move
            new_distance = abs(self.head_x - self.food_x) + abs(self.head_y - self.food_y)
    
            # Check for collision
            self.reward = 0
            self.game_over = False
            if self.is_collision() or self.frame_iteration > 100 * self.length:
                self.game_over = True
                self.reward = -10
                return self.reward, self.game_over, self.score
    
            # Check for food collision
            if self.head_x == self.food_x and self.head_y == self.food_y:
                self.score += 1
                self.reward = 10
                self.length += 1
                self.place_food()
            else:
                self.snake_body.pop()
                # Encourage moving towards food
                if new_distance < current_distance:
                    self.reward = 0.1
                else:
                    self.reward = -0.1
    
            # Draw game
            if render:
                self.update_ui()
                clock.tick(SPEED)
    
            return self.reward, self.game_over, self.score
    
        def is_collision(self, pt=None):
            if pt is None:
                pt = (self.head_x, self.head_y)
            # Hits boundary
            if pt[0] < 0 or pt[0] >= WINDOW_WIDTH or pt[1] < 0 or pt[1] >= WINDOW_HEIGHT:
                return True
            # Hits itself
            if pt in self.snake_body[1:]:
                return True
            return False
    
        def move(self, action):
            # [straight, right, left]
            clock_wise = ['RIGHT', 'DOWN', 'LEFT', 'UP']
            idx = clock_wise.index(self.direction)
    
            if action == 1:
                new_dir = clock_wise[idx]  # No change
            elif action == 2:
                next_idx = (idx + 1) % 4
                new_dir = clock_wise[next_idx]  # Right turn
            elif action == 3:
                next_idx = (idx - 1) % 4
                new_dir = clock_wise[next_idx]  # Left turn
            else:
                new_dir = self.direction  # Default to current direction
    
            self.direction = new_dir
    
            # Move the head
            if self.direction == 'RIGHT':
                self.head_x += BLOCK_SIZE
            elif self.direction == 'LEFT':
                self.head_x -= BLOCK_SIZE
            elif self.direction == 'UP':
                self.head_y -= BLOCK_SIZE
            elif self.direction == 'DOWN':
                self.head_y += BLOCK_SIZE
    
        def place_food(self):
            while True:
                self.food_x = random.randint(0, (WINDOW_WIDTH - BLOCK_SIZE) // BLOCK_SIZE) * BLOCK_SIZE
                self.food_y = random.randint(0, (WINDOW_HEIGHT - BLOCK_SIZE) // BLOCK_SIZE) * BLOCK_SIZE
                if (self.food_x, self.food_y) not in self.snake_body:
                    break
    
        def update_ui(self):
            screen.fill(BLACK)
            for block in self.snake_body:
                pygame.draw.rect(screen, GREEN, pygame.Rect(block[0], block[1], BLOCK_SIZE, BLOCK_SIZE))
            pygame.draw.rect(screen, RED, pygame.Rect(self.food_x, self.food_y, BLOCK_SIZE, BLOCK_SIZE))
            text = font.render(f'Score: {self.score}', True, WHITE)
            screen.blit(text, [0, 0])
            pygame.display.flip()
    
    # Agent Class with Improved Algorithms
    class Agent:
        def __init__(self, render=False):
            self.n_games = 0
            self.epsilon = 1.0  # Exploration rate
            self.eps_min = 0.01
            self.eps_decay = 0.995
            self.gamma = 0.9  # Discount rate
            self.lr = 0.0005
            self.memory = PrioritizedReplayMemory(100_000)
            self.batch_size = 512
            self.model = DuelingDQN(11, 256, 3).to(device)
            self.target_model = DuelingDQN(11, 256, 3).to(device)
            self.update_target_network()
            self.optimizer = optim.Adam(self.model.parameters(), lr=self.lr)
            self.criterion = nn.SmoothL1Loss()
            self.update_frequency = 1000
            self.learn_step_counter = 0
            self.beta_start = 0.4
            self.beta_frames = 100_000
            self.beta = self.beta_start
            self.render = render
            self.load_model()
    
        def load_model(self):
            if os.path.exists('model.pth'):
                self.model.load_state_dict(torch.load('model.pth', map_location=device))
                self.target_model.load_state_dict(torch.load('model.pth', map_location=device))
                print("Model loaded from model.pth")
    
        def save_model(self):
            torch.save(self.model.state_dict(), 'model.pth')
            print("Model saved to model.pth")
    
        def update_target_network(self):
            self.target_model.load_state_dict(self.model.state_dict())
    
        def get_state(self, game):
            head = game.snake_body[0]
            point_l = (head[0] - BLOCK_SIZE, head[1])
            point_r = (head[0] + BLOCK_SIZE, head[1])
            point_u = (head[0], head[1] - BLOCK_SIZE)
            point_d = (head[0], head[1] + BLOCK_SIZE)
    
            dir_l = game.direction == 'LEFT'
            dir_r = game.direction == 'RIGHT'
            dir_u = game.direction == 'UP'
            dir_d = game.direction == 'DOWN'
    
            state = [
                # Danger straight
                (dir_r and game.is_collision(point_r)) or
                (dir_l and game.is_collision(point_l)) or
                (dir_u and game.is_collision(point_u)) or
                (dir_d and game.is_collision(point_d)),
    
                # Danger right
                (dir_u and game.is_collision(point_r)) or
                (dir_d and game.is_collision(point_l)) or
                (dir_l and game.is_collision(point_u)) or
                (dir_r and game.is_collision(point_d)),
    
                # Danger left
                (dir_d and game.is_collision(point_r)) or
                (dir_u and game.is_collision(point_l)) or
                (dir_r and game.is_collision(point_u)) or
                (dir_l and game.is_collision(point_d)),
    
                # Move direction
                dir_l,
                dir_r,
                dir_u,
                dir_d,
    
                # Food location
                game.food_x < game.head_x,  # food left
                game.food_x > game.head_x,  # food right
                game.food_y < game.head_y,  # food up
                game.food_y > game.head_y,  # food down
            ]
            return np.array(state, dtype=int)
    
        def remember(self, state, action, reward, next_state, done):
            # Convert to tensors and move to device
            self.memory.push(state, action, reward, next_state, done)
    
        def train_long_memory(self):
            if len(self.memory) < self.batch_size:
                return  # Not enough samples
            transitions, indices, weights = self.memory.sample(self.batch_size, beta=self.beta)
            self.beta = min(1.0, self.beta + (1.0 - self.beta_start) / self.beta_frames)
    
            # Convert batch arrays to tensors
            states = torch.tensor(transitions.state, dtype=torch.float32, device=device)
            actions = torch.tensor(transitions.action, dtype=torch.float32, device=device)
            rewards = torch.tensor(transitions.reward, dtype=torch.float32, device=device).squeeze(1)
            next_states = torch.tensor(transitions.next_state, dtype=torch.float32, device=device)
            dones = torch.tensor(transitions.done, dtype=torch.bool, device=device).squeeze(1)
    
            self.train_step(states, actions, rewards, next_states, dones, indices, weights)
    
        def train_step(self, states, actions, rewards, next_states, dones, indices, weights):
            # Compute Q values
            current_q_values = self.model(states).gather(1, actions.argmax(dim=1).unsqueeze(1)).squeeze(1)
    
            with torch.no_grad():
                # Double DQN
                next_actions = self.model(next_states).argmax(1)
                next_q_values = self.target_model(next_states).gather(1, next_actions.unsqueeze(1)).squeeze(1)
                next_q_values[dones] = 0.0
    
            expected_q_values = rewards + self.gamma * next_q_values
    
            # Compute TD error
            td_errors = expected_q_values - current_q_values
            loss = (weights * td_errors.pow(2)).mean()
    
            self.optimizer.zero_grad()
            loss.backward()
            # Gradient clipping
            nn.utils.clip_grad_norm_(self.model.parameters(), max_norm=1.0)
            self.optimizer.step()
    
            # Update priorities
            priorities = td_errors.abs().detach() + 1e-5
            self.memory.update_priorities(indices, priorities)
    
            # Update target network
            self.learn_step_counter += 1
            if self.learn_step_counter % self.update_frequency == 0:
                self.update_target_network()
                # Save model periodically
                self.save_model()
    
        def get_action(self, state):
            self.epsilon = max(self.eps_min, self.epsilon * self.eps_decay)
            final_move = 0  # Default to 'straight'
    
            if random.random() < self.epsilon:
                final_move = random.randint(0, 2) + 1  # 1: straight, 2: right, 3: left
            else:
                state_tensor = torch.tensor(state, dtype=torch.float32, device=device).unsqueeze(0)
                with torch.no_grad():
                    q_values = self.model(state_tensor)
                    # Survival Deviation: Adding noise to the action selection
                    q_values += torch.randn_like(q_values) * 0.1
                    final_move = torch.argmax(q_values).item()
    
            return final_move
    
    def train(render=False):
        total_score = 0
        agent = Agent(render=render)
        game = SnakeGameAI()
        best_score = 0
        try:
            while True:
                # Get current state
                state_old = agent.get_state(game)
    
                # Get action
                final_move = agent.get_action(state_old)
    
                # Perform action and get new state
                reward, done, score = game.play_step(final_move, render=render)
                state_new = agent.get_state(game)
    
                # Remember
                agent.remember(state_old, final_move, reward, state_new, done)
    
                # Train long memory
                agent.train_long_memory()
    
                if done:
                    # Reset game
                    game.reset()
                    agent.n_games += 1
    
                    if score > best_score:
                        best_score = score
                        agent.save_model()
    
                    print(f'Game {agent.n_games} | Score: {score} | Record: {best_score} | Epsilon: {agent.epsilon:.4f}')
        except KeyboardInterrupt:
            print("Training interrupted by user.")
    
    if __name__ == '__main__':
        # Set render to False for faster training without visualization
        train(render=True)  # Change to False if you want to disable rendering
    

    Key Changes and Optimizations:

    1. Indentation Fixes:

      • All methods related to the SnakeGameAI class (play_step, is_collision, move, place_food, update_ui) are now correctly indented within the class.
    2. Batch Size Reduction:

      • Reduced batch_size from 1024 to 512 to balance training efficiency and memory usage. You can adjust this based on your system's capabilities.
    3. Gradient Clipping:

      • Added nn.utils.clip_grad_norm_ to clip gradients, preventing potential exploding gradient issues.
    4. Optional Rendering:

      • Introduced a render parameter in both the Agent and SnakeGameAI classes. Setting render=False during training speeds up the process as it skips the rendering steps.
      • In the train function, you can toggle rendering by setting the render argument:
        train(render=True)  # Enable rendering
        train(render=False)  # Disable rendering for faster training
        
    5. Action Representation:

      • Simplified action representation by mapping:
        • 0: Straight
        • 1: Right
        • 2: Left
      • Adjusted the get_action method accordingly for clarity.
    6. Exception Handling:

      • Added a try-except block in the train function to gracefully handle interruptions (e.g., pressing Ctrl+C).
    7. Memory Cleanup:

      • Removed explicit del statements and torch.cuda.empty_cache() as PyTorch handles memory management efficiently. If you encounter memory issues, you might consider reintroducing them or using gradient checkpointing.
    8. Code Cleanup:

      • Removed unnecessary comments and optimized existing ones for better readability.
      • Ensured consistent naming conventions and code formatting for maintainability.

    Additional Recommendations:

    • Hyperparameter Tuning: Consider experimenting with different hyperparameters such as learning rate (lr), discount factor (gamma), and epsilon decay rates to optimize the agent's performance.

    • Logging and Visualization: Implement logging (e.g., using TensorBoard) to visualize training metrics over time, which can provide insights into the agent's learning progress.

    • Model Checkpointing: Depending on your training duration, you might want to implement more sophisticated checkpointing mechanisms to save models at various stages.

    • Environment Separation: For larger projects, consider separating the environment (game) and agent into different modules or files to enhance code organization.

    • Advanced Exploration Strategies: Incorporate more advanced exploration strategies like Noisy Networks or Upper Confidence Bound (UCB) to potentially improve learning efficiency.

    • Frame Skipping: To speed up training, you can implement frame skipping where the agent takes actions every few frames instead of every single frame.

    Feel free to adjust and extend the code based on your specific requirements and system capabilities. Happy training!

FURTHER OPTIMIZE, and debug: ` import os import pygame… · The Chat Archive