Skip to content
TheChatArchiveThe Chat Archive
o1-mini-2024-09-12· OpenAIAi2 WildChat-4.8M (AI2)

This is a training script for a model making predictions on a 1 min usd/euro chart. It…

  1. User

    This is a training script for a model making predictions on a 1 min usd/euro chart. It predicts if the price will be lower, much lower, higher, much higher, or the same price, 30 minutes into the future. The trained model will be used in a meratrader5 environment as part of an ea making trades, which is why it is saved as an onnx model. Improve its speed, effencienccy, and predictive ability without adding features. Make the loss calculation more appropriate for the task at hand.

    import os import torch import torch.nn as nn import torch.optim as optim from torch.utils.data import Dataset, DataLoader import pandas as pd import numpy as np from sklearn.preprocessing import MinMaxScaler from sklearn.model_selection import train_test_split import onnx import onnxruntime from tqdm import tqdm

    Configuration

    class Config: def init(self): self.task_name = 'short_term_forecast' self.seq_len = 180 # Input sequence length self.pred_len = 30 # Prediction length (30 minutes into the future) self.d_model = 1024 # Model dimension self.enc_in = 4 # Number of input features (Open, High, Low, Close) self.e_layers = 6 # Number of encoder layers self.dropout = 0.4 self.batch_size = 128 self.learning_rate = 0.00001 self.epochs = 100 self.device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') self.data_path = '/mnt/d/FX/fx_csv/22_23_1M.csv' self.model_save_path = '/mnt/d/FX/Mixer_Models' self.results_save_path = '/mnt/d/FX/Mixer_Results'

    configs = Config()

    class ScalerModule(nn.Module): def init(self, scaler): super(ScalerModule, self).init() self.min_ = nn.Parameter(torch.FloatTensor(scaler.min_), requires_grad=False) self.scale_ = nn.Parameter(torch.FloatTensor(scaler.scale_), requires_grad=False)

    def forward(self, x):
        return (x - self.min_) / self.scale_
    

    Model definition

    class ResBlock(nn.Module): def init(self, configs): super(ResBlock, self).init() self.temporal = nn.Sequential( nn.Linear(configs.seq_len, configs.d_model), nn.ReLU(), nn.Linear(configs.d_model, configs.seq_len), nn.Dropout(configs.dropout) ) self.channel = nn.Sequential( nn.Linear(configs.enc_in, configs.d_model), nn.ReLU(), nn.Linear(configs.d_model, configs.enc_in), nn.Dropout(configs.dropout) )

    def forward(self, x):
        # x: [B, L, D]
        x = x + self.temporal(x.transpose(1, 2)).transpose(1, 2)
        x = x + self.channel(x)
        return x
    

    class Model(nn.Module): def init(self, configs, scaler): super(Model, self).init() self.scaler = ScalerModule(scaler) self.task_name = configs.task_name self.layer = configs.e_layers self.model = nn.ModuleList([ResBlock(configs) for _ in range(configs.e_layers)]) self.pred_len = configs.pred_len self.projection = nn.Linear(configs.seq_len, configs.pred_len) self.final_layer = nn.Linear(configs.enc_in, 1) # Output a single value self.classifier = nn.Linear(configs.pred_len, 5) # 5 classes for prediction

    def forecast(self, x_enc):
        # x: [B, L, D]
        x_enc = self.scaler(x_enc)
        for i in range(self.layer):
            x_enc = self.model[i](x_enc)
        enc_out = self.projection(x_enc.transpose(1, 2)).transpose(1, 2)
        enc_out = self.final_layer(enc_out).squeeze(-1)
        return self.classifier(enc_out)
    
    def forward(self, x_enc):
        if self.task_name == 'short_term_forecast':
            return self.forecast(x_enc)
        else:
            raise ValueError('Only short_term_forecast task implemented')
    

    Dataset

    class ForexDataset(Dataset): def init(self, data, seq_len, pred_len): self.data = data self.seq_len = seq_len self.pred_len = pred_len

    def __len__(self):
        return len(self.data) - self.seq_len - self.pred_len + 1
    
    def __getitem__(self, idx):
        x = self.data[idx:idx+self.seq_len]
        y = self.data[idx+self.seq_len+self.pred_len-1, 3]  # Close price after pred_len
        
        # Calculate the price change
        current_price = self.data[idx+self.seq_len-1, 3]
        price_change = (y - current_price) / current_price
        
        # Classify the price change
        if price_change < -0.01:
            label = 0  # much lower
        elif -0.01 <= price_change < -0.001:
            label = 1  # lower
        elif -0.001 <= price_change <= 0.001:
            label = 2  # same
        elif 0.001 < price_change <= 0.01:
            label = 3  # higher
        else:
            label = 4  # much higher
        
        return torch.FloatTensor(x), torch.LongTensor([label])
    

    Data preprocessing

    def preprocess_data(configs): print("Loading and preprocessing data...") df = pd.read_csv(configs.data_path) df['DateTime'] = pd.to_datetime(df['DateTime']) df = df.set_index('DateTime')

    scaler = MinMaxScaler()
    data = scaler.fit_transform(df[['Open', 'High', 'Low', 'Close']])
    
    dataset = ForexDataset(data, configs.seq_len, configs.pred_len)
    train_size = int(0.8 * len(dataset))
    test_size = len(dataset) - train_size
    train_dataset, test_dataset = torch.utils.data.random_split(dataset, [train_size, test_size])
    
    train_loader = DataLoader(train_dataset, batch_size=configs.batch_size, shuffle=True)
    test_loader = DataLoader(test_dataset, batch_size=configs.batch_size, shuffle=False)
    
    return train_loader, test_loader, scaler
    

    def calculate_metrics(output, target): _, predicted = torch.max(output.data, 1) correct = (predicted == target).sum().item()

    # Convert output probabilities to continuous predictions
    continuous_pred = torch.sum(output * torch.tensor([-2, -1, 0, 1, 2]).to(output.device), dim=1)
    continuous_target = torch.tensor([-2, -1, 0, 1, 2]).to(target.device)[target]
    
    mae = torch.mean(torch.abs(continuous_pred - continuous_target)).item()
    mse = torch.mean((continuous_pred - continuous_target) ** 2).item()
    rmse = torch.sqrt(torch.tensor(mse)).item()
    
    return correct, mae, mse, rmse
    

    Training function

    def train(model, train_loader, optimizer, criterion, device, epoch): model.train() total_loss = 0 correct = 0 total = 0 total_mae = 0 total_mse = 0

    progress_bar = tqdm(train_loader, desc=f"Epoch {epoch+1}")
    for batch_x, batch_y in progress_bar:
        optimizer.zero_grad()
        batch_x, batch_y = batch_x.to(device), batch_y.to(device)
        output = model(batch_x)
        loss = criterion(output, batch_y.squeeze())
        loss.backward()
        optimizer.step()
        
        total_loss += loss.item()
        batch_correct, batch_mae, batch_mse, _ = calculate_metrics(output, batch_y.squeeze())
        correct += batch_correct
        total += batch_y.size(0)
        total_mae += batch_mae * batch_y.size(0)
        total_mse += batch_mse * batch_y.size(0)
        
        # Update progress bar
        avg_loss = total_loss / (progress_bar.n + 1)
        accuracy = 100 * correct / total
        avg_mae = total_mae / total
        avg_rmse = torch.sqrt(torch.tensor(total_mse / total)).item()
        progress_bar.set_postfix({
            'loss': f'{avg_loss:.4f}',
            'acc': f'{accuracy:.2f}%',
            'MAE': f'{avg_mae:.4f}',
            'RMSE': f'{avg_rmse:.4f}'
        })
    
    return (total_loss / len(train_loader), 100 * correct / total, 
            total_mae / total, torch.sqrt(torch.tensor(total_mse / total)).item())
    

    def evaluate(model, test_loader, criterion, device): model.eval() total_loss = 0 correct = 0 total = 0 total_mae = 0 total_mse = 0

    with torch.no_grad():
        for batch_x, batch_y in tqdm(test_loader, desc="Evaluating"):
            batch_x, batch_y = batch_x.to(device), batch_y.to(device)
            output = model(batch_x)
            loss = criterion(output, batch_y.squeeze())
            total_loss += loss.item()
            
            batch_correct, batch_mae, batch_mse, _ = calculate_metrics(output, batch_y.squeeze())
            correct += batch_correct
            total += batch_y.size(0)
            total_mae += batch_mae * batch_y.size(0)
            total_mse += batch_mse * batch_y.size(0)
    
    avg_loss = total_loss / len(test_loader)
    accuracy = 100 * correct / total
    avg_mae = total_mae / total
    avg_rmse = torch.sqrt(torch.tensor(total_mse / total)).item()
    return avg_loss, accuracy, avg_mae, avg_rmse
    

    Save model function

    def save_model(model, path, epoch, test_loss): if not os.path.exists(path): os.makedirs(path) models = sorted([f for f in os.listdir(path) if f.endswith('.onnx')]) if len(models) >= 3: os.remove(os.path.join(path, models[0]))

    model_path = os.path.join(path, f'model_epoch_{epoch}_loss_{test_loss:.4f}.onnx')
    dummy_input = torch.randn(1, configs.seq_len, configs.enc_in).to(configs.device)
    torch.onnx.export(model, dummy_input, model_path, export_params=True, opset_version=11,
                      input_names=['input'], output_names=['output'],
                      dynamic_axes={'input': {0: 'batch_size'}, 'output': {0: 'batch_size'}})
    print(f"Model saved: {model_path}")
    

    Main training loop

    def main(): print(f"Using device: {configs.device}") train_loader, test_loader, scaler = preprocess_data(configs)

    model = Model(configs, scaler).to(configs.device)
    optimizer = optim.Adam(model.parameters(), lr=configs.learning_rate)
    criterion = nn.CrossEntropyLoss()
    
    best_test_loss = float('inf')
    
    for epoch in range(configs.epochs):
        train_loss, train_acc, train_mae, train_rmse = train(model, train_loader, optimizer, criterion, configs.device, epoch)
        test_loss, test_acc, test_mae, test_rmse = evaluate(model, test_loader, criterion, configs.device)
        
        print(f"\nEpoch {epoch+1}/{configs.epochs}")
        print(f"Train Loss: {train_loss:.4f}, Train Accuracy: {train_acc:.2f}%")
        print(f"Train MAE: {train_mae:.4f}, Train RMSE: {train_rmse:.4f}")
        print(f"Test Loss: {test_loss:.4f}, Test Accuracy: {test_acc:.2f}%")
        print(f"Test MAE: {test_mae:.4f}, Test RMSE: {test_rmse:.4f}")
        
        if test_loss < best_test_loss:
            best_test_loss = test_loss
            save_model(model, configs.model_save_path, epoch+1, test_loss)
    
    # Save final results
    with open(os.path.join(configs.results_save_path, 'training_results.txt'), 'w') as f:
        f.write(f"Final Train Loss: {train_loss:.4f}\n")
        f.write(f"Final Train Accuracy: {train_acc:.2f}%\n")
        f.write(f"Final Train MAE: {train_mae:.4f}\n")
        f.write(f"Final Train RMSE: {train_rmse:.4f}\n")
        f.write(f"Final Test Loss: {test_loss:.4f}\n")
        f.write(f"Final Test Accuracy: {test_acc:.2f}%\n")
        f.write(f"Final Test MAE: {test_mae:.4f}\n")
        f.write(f"Final Test RMSE: {test_rmse:.4f}\n")
        f.write(f"Best Test Loss: {best_test_loss:.4f}\n")
    
    print("Training completed. Results saved.")
    

    if name == "main": main()

  2. Assistant

    To enhance the speed, efficiency, and predictive performance of your Forex prediction model while keeping the feature set unchanged, we can implement several optimizations and improvements:

    1. Mixed Precision Training: Utilize PyTorch's Automatic Mixed Precision (AMP) to accelerate training and reduce memory usage without sacrificing model accuracy.

    2. Optimized Loss Function for Ordinal Classification: Since your prediction task involves ordered categories (much lower < lower < same < higher < much higher), using an ordinal regression loss can better capture the inherent ordering in the classes compared to the standard CrossEntropyLoss.

    3. Model Architecture Refinement: Simplify the model where possible to reduce computational complexity while maintaining or improving predictive performance.

    4. DataLoader Improvements: Optimize data loading by enabling pinned memory and increasing the number of worker threads to speed up data fetching.

    5. Learning Rate Scheduler: Implement a learning rate scheduler to adjust the learning rate during training, which can lead to faster convergence and better performance.

    6. Efficient Metric Computation: Vectorize metric calculations to reduce computational overhead.

    Below is the revised training script incorporating these improvements:

    import os
    import torch
    import torch.nn as nn
    import torch.optim as optim
    from torch.utils.data import Dataset, DataLoader
    import pandas as pd
    import numpy as np
    from sklearn.preprocessing import MinMaxScaler
    from sklearn.model_selection import train_test_split
    import onnx
    import onnxruntime
    from tqdm import tqdm
    
    # Configuration
    class Config:
        def __init__(self):
            self.task_name = 'short_term_forecast'
            self.seq_len = 180  # Input sequence length
            self.pred_len = 30  # Prediction length (30 minutes into the future)
            self.d_model = 512  # Reduced model dimension for efficiency
            self.enc_in = 4  # Number of input features (Open, High, Low, Close)
            self.e_layers = 4  # Reduced number of encoder layers
            self.dropout = 0.3  # Slightly reduced dropout
            self.batch_size = 256  # Increased batch size for better GPU utilization
            self.learning_rate = 0.001  # Increased learning rate for faster convergence
            self.epochs = 100
            self.device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
            self.data_path = '/mnt/d/FX/fx_csv/22_23_1M.csv'
            self.model_save_path = '/mnt/d/FX/Mixer_Models'
            self.results_save_path = '/mnt/d/FX/Mixer_Results'
            self.num_workers = 4  # Number of workers for DataLoader
            self.scaler_path = '/mnt/d/FX/scaler.pkl'  # Path to save the scaler
    
    configs = Config()
    
    class ScalerModule(nn.Module):
        def __init__(self, scaler):
            super(ScalerModule, self).__init__()
            self.min_ = nn.Parameter(torch.FloatTensor(scaler.min_), requires_grad=False)
            self.scale_ = nn.Parameter(torch.FloatTensor(scaler.scale_), requires_grad=False)
    
        def forward(self, x):
            return (x - self.min_) / self.scale_
    
    # Optimized ResBlock with Layer Normalization
    class ResBlock(nn.Module):
        def __init__(self, configs):
            super(ResBlock, self).__init__()
            self.temporal = nn.Sequential(
                nn.Linear(configs.seq_len, configs.d_model),
                nn.ReLU(),
                nn.Linear(configs.d_model, configs.seq_len),
                nn.Dropout(configs.dropout)
            )
            self.channel = nn.Sequential(
                nn.Linear(configs.enc_in, configs.d_model),
                nn.ReLU(),
                nn.Linear(configs.d_model, configs.enc_in),
                nn.Dropout(configs.dropout)
            )
            self.layer_norm = nn.LayerNorm(configs.enc_in)
    
        def forward(self, x):
            # x: [B, L, D]
            residual = x
            x = self.temporal(x.transpose(1, 2)).transpose(1, 2)
            x = self.layer_norm(x + residual)
            residual = x
            x = self.channel(x)
            x = self.layer_norm(x + residual)
            return x
    
    # Ordinal Regression Loss
    class OrdinalLoss(nn.Module):
        def __init__(self):
            super(OrdinalLoss, self).__init__()
            self.loss_fn = nn.BCEWithLogitsLoss()
    
        def forward(self, outputs, targets, num_classes=5):
            # Convert targets to ordinal binary format
            targets = targets.unsqueeze(1)
            targets = (torch.arange(1, num_classes).to(targets.device) < targets).float()
            return self.loss_fn(outputs, targets)
    
    # Modified Model with Ordered Output
    class Model(nn.Module):
        def __init__(self, configs, scaler):
            super(Model, self).__init__()
            self.scaler = ScalerModule(scaler)
            self.task_name = configs.task_name
            self.layer = configs.e_layers
            self.model = nn.ModuleList([ResBlock(configs) for _ in range(configs.e_layers)])
            self.seq_len = configs.seq_len
            self.projection = nn.Linear(configs.seq_len, configs.pred_len)
            self.final_layer = nn.Linear(configs.enc_in, configs.d_model)
            self.relu = nn.ReLU()
            # Output layers for ordinal regression
            self.classifiers = nn.ModuleList([
                nn.Linear(configs.d_model, 1) for _ in range(configs.pred_len - 1)
            ])
    
        def forward(self, x_enc):
            # x: [B, L, D]
            x_enc = self.scaler(x_enc)
            for i in range(self.layer):
                x_enc = self.model[i](x_enc)
            enc_out = self.projection(x_enc.transpose(1, 2)).transpose(1, 2)
            enc_out = self.final_layer(enc_out)
            enc_out = self.relu(enc_out)
            logits = [classifier(enc_out) for classifier in self.classifiers]
            logits = torch.cat(logits, dim=-1)  # [B, pred_len -1]
            return logits
    
    # Dataset
    class ForexDataset(Dataset):
        def __init__(self, data, seq_len, pred_len):
            self.data = data
            self.seq_len = seq_len
            self.pred_len = pred_len
    
        def __len__(self):
            return len(self.data) - self.seq_len - self.pred_len + 1
    
        def __getitem__(self, idx):
            x = self.data[idx:idx+self.seq_len]
            y = self.data[idx+self.seq_len+self.pred_len-1, 3]  # Close price after pred_len
    
            # Calculate the price change
            current_price = self.data[idx+self.seq_len-1, 3]
            price_change = (y - current_price) / current_price
    
            # Classify the price change
            if price_change < -0.01:
                label = 0  # much lower
            elif -0.01 <= price_change < -0.001:
                label = 1  # lower
            elif -0.001 <= price_change <= 0.001:
                label = 2  # same
            elif 0.001 < price_change <= 0.01:
                label = 3  # higher
            else:
                label = 4  # much higher
    
            return torch.FloatTensor(x), torch.LongTensor([label])
    
    # Data preprocessing
    def preprocess_data(configs):
        print("Loading and preprocessing data...")
        df = pd.read_csv(configs.data_path)
        df['DateTime'] = pd.to_datetime(df['DateTime'])
        df = df.set_index('DateTime')
    
        scaler = MinMaxScaler()
        data = scaler.fit_transform(df[['Open', 'High', 'Low', 'Close']])
    
        # Save the scaler for future use
        import joblib
        joblib.dump(scaler, configs.scaler_path)
    
        dataset = ForexDataset(data, configs.seq_len, configs.pred_len)
        train_size = int(0.8 * len(dataset))
        test_size = len(dataset) - train_size
        train_dataset, test_dataset = torch.utils.data.random_split(dataset, [train_size, test_size])
    
        train_loader = DataLoader(
            train_dataset, batch_size=configs.batch_size, shuffle=True,
            num_workers=configs.num_workers, pin_memory=True
        )
        test_loader = DataLoader(
            test_dataset, batch_size=configs.batch_size, shuffle=False,
            num_workers=configs.num_workers, pin_memory=True
        )
    
        return train_loader, test_loader, scaler
    
    def calculate_metrics(outputs, targets):
        # Convert ordinal outputs to class predictions
        # Each output corresponds to a threshold; sum them to get class
        probs = torch.sigmoid(outputs)
        cumulative = torch.cumsum(probs, dim=1)
        predicted = (cumulative > 0.5).sum(dim=1)
    
        correct = (predicted == targets).sum().item()
    
        # For MAE and RMSE, convert classes to ordered values
        continuous_pred = predicted.float() - 2  # Classes 0-4 mapped to -2 to 2
        continuous_target = targets.float() - 2
    
        mae = torch.mean(torch.abs(continuous_pred - continuous_target)).item()
        mse = torch.mean((continuous_pred - continuous_target) ** 2).item()
        rmse = torch.sqrt(torch.tensor(mse)).item()
    
        return correct, mae, mse, rmse
    
    # Training function with Mixed Precision
    def train(model, train_loader, optimizer, criterion, device, epoch, scaler):
        model.train()
        scaler_amp = torch.cuda.amp.GradScaler()
        total_loss = 0
        correct = 0
        total = 0
        total_mae = 0
        total_mse = 0
    
        progress_bar = tqdm(train_loader, desc=f"Epoch {epoch+1}/{configs.epochs}")
        for batch_x, batch_y in progress_bar:
            optimizer.zero_grad()
            batch_x, batch_y = batch_x.to(device, non_blocking=True), batch_y.to(device, non_blocking=True)
            with torch.cuda.amp.autocast():
                outputs = model(batch_x)  # [B, pred_len -1]
                loss = criterion(outputs, batch_y.squeeze())
            
            scaler_amp.scale(loss).backward()
            scaler_amp.step(optimizer)
            scaler_amp.update()
    
            total_loss += loss.item() * batch_x.size(0)
            batch_correct, batch_mae, batch_mse, _ = calculate_metrics(outputs, batch_y.squeeze())
            correct += batch_correct
            total += batch_y.size(0)
            total_mae += batch_mae * batch_y.size(0)
            total_mse += batch_mse * batch_y.size(0)
    
            # Update progress bar
            avg_loss = total_loss / total
            accuracy = 100 * correct / total
            avg_mae = total_mae / total
            avg_rmse = np.sqrt(total_mse / total)
            progress_bar.set_postfix({
                'loss': f'{avg_loss:.4f}',
                'acc': f'{accuracy:.2f}%',
                'MAE': f'{avg_mae:.4f}',
                'RMSE': f'{avg_rmse:.4f}'
            })
    
        return (total_loss / len(train_loader.dataset), 100 * correct / total, 
                total_mae / total, np.sqrt(total_mse / total))
    
    # Evaluation function with Mixed Precision
    def evaluate(model, test_loader, criterion, device, scaler):
        model.eval()
        scaler_amp = torch.cuda.amp.GradScaler()
        total_loss = 0
        correct = 0
        total = 0
        total_mae = 0
        total_mse = 0
    
        with torch.no_grad():
            progress_bar = tqdm(test_loader, desc="Evaluating")
            for batch_x, batch_y in progress_bar:
                batch_x, batch_y = batch_x.to(device, non_blocking=True), batch_y.to(device, non_blocking=True)
                with torch.cuda.amp.autocast():
                    outputs = model(batch_x)
                    loss = criterion(outputs, batch_y.squeeze())
                
                total_loss += loss.item() * batch_x.size(0)
                batch_correct, batch_mae, batch_mse, _ = calculate_metrics(outputs, batch_y.squeeze())
                correct += batch_correct
                total += batch_y.size(0)
                total_mae += batch_mae * batch_y.size(0)
                total_mse += batch_mse * batch_y.size(0)
    
                # Update progress bar
                avg_loss = total_loss / total
                accuracy = 100 * correct / total
                avg_mae = total_mae / total
                avg_rmse = np.sqrt(total_mse / total)
                progress_bar.set_postfix({
                    'loss': f'{avg_loss:.4f}',
                    'acc': f'{accuracy:.2f}%',
                    'MAE': f'{avg_mae:.4f}',
                    'RMSE': f'{avg_rmse:.4f}'
                })
    
        return (total_loss / len(test_loader.dataset), 100 * correct / total, 
                total_mae / total, np.sqrt(total_mse / total))
    
    # Save model function with ONNX optimizations
    def save_model(model, path, epoch, test_loss, scaler):
        if not os.path.exists(path):
            os.makedirs(path)
        models = sorted([f for f in os.listdir(path) if f.endswith('.onnx')])
        if len(models) >= 3:
            os.remove(os.path.join(path, models[0]))
        
        model_path = os.path.join(path, f'model_epoch_{epoch}_loss_{test_loss:.4f}.onnx')
        dummy_input = torch.randn(1, configs.seq_len, configs.enc_in).to(configs.device)
        torch.onnx.export(
            model, dummy_input, model_path, export_params=True, opset_version=12,
            input_names=['input'], output_names=['output'],
            dynamic_axes={'input': {0: 'batch_size'}, 'output': {0: 'batch_size'}}
        )
        print(f"Model saved: {model_path}")
    
    # Main training loop with Learning Rate Scheduler
    def main():
        print(f"Using device: {configs.device}")
        train_loader, test_loader, scaler = preprocess_data(configs)
        
        model = Model(configs, scaler).to(configs.device)
        optimizer = optim.Adam(model.parameters(), lr=configs.learning_rate)
        criterion = OrdinalLoss()
        scheduler = optim.lr_scheduler.ReduceLROnPlateau(optimizer, mode='min', 
                                                         factor=0.5, patience=5, verbose=True)
        
        best_test_loss = float('inf')
        
        for epoch in range(configs.epochs):
            train_loss, train_acc, train_mae, train_rmse = train(
                model, train_loader, optimizer, criterion, configs.device, epoch, scaler
            )
            test_loss, test_acc, test_mae, test_rmse = evaluate(
                model, test_loader, criterion, configs.device, scaler
            )
            
            scheduler.step(test_loss)
    
            print(f"\nEpoch {epoch+1}/{configs.epochs}")
            print(f"Train Loss: {train_loss:.4f}, Train Accuracy: {train_acc:.2f}%")
            print(f"Train MAE: {train_mae:.4f}, Train RMSE: {train_rmse:.4f}")
            print(f"Test Loss: {test_loss:.4f}, Test Accuracy: {test_acc:.2f}%")
            print(f"Test MAE: {test_mae:.4f}, Test RMSE: {test_rmse:.4f}")
            
            if test_loss < best_test_loss:
                best_test_loss = test_loss
                save_model(model, configs.model_save_path, epoch+1, test_loss, scaler)
        
        # Save final results
        if not os.path.exists(configs.results_save_path):
            os.makedirs(configs.results_save_path)
        with open(os.path.join(configs.results_save_path, 'training_results.txt'), 'w') as f:
            f.write(f"Final Train Loss: {train_loss:.4f}\n")
            f.write(f"Final Train Accuracy: {train_acc:.2f}%\n")
            f.write(f"Final Train MAE: {train_mae:.4f}\n")
            f.write(f"Final Train RMSE: {train_rmse:.4f}\n")
            f.write(f"Final Test Loss: {test_loss:.4f}\n")
            f.write(f"Final Test Accuracy: {test_acc:.2f}%\n")
            f.write(f"Final Test MAE: {test_mae:.4f}\n")
            f.write(f"Final Test RMSE: {test_rmse:.4f}\n")
            f.write(f"Best Test Loss: {best_test_loss:.4f}\n")
        
        print("Training completed. Results saved.")
    
    if __name__ == "__main__":
        main()
    

    Detailed Changes and Improvements

    1. Mixed Precision Training:

      • Implementation: Introduced torch.cuda.amp's GradScaler and autocast to handle mixed precision.
      • Benefit: Accelerates training and reduces GPU memory usage, allowing for larger batch sizes or more complex models.
      from torch.cuda.amp import GradScaler, autocast
      
      # Within train and evaluate functions:
      scaler_amp = torch.cuda.amp.GradScaler()
      with torch.cuda.amp.autocast():
          outputs = model(batch_x)
          loss = criterion(outputs, batch_y.squeeze())
      
      scaler_amp.scale(loss).backward()
      scaler_amp.step(optimizer)
      scaler_amp.update()
      
    2. Optimized Ordinal Loss Function:

      • Implementation: Created a custom OrdinalLoss class that transforms the multiclass problem into multiple binary classification tasks, aligning with ordinal regression principles.
      • Benefit: Captures the ordinal nature of the output classes, potentially improving predictive performance.
      class OrdinalLoss(nn.Module):
          def __init__(self):
              super(OrdinalLoss, self).__init__()
              self.loss_fn = nn.BCEWithLogitsLoss()
      
          def forward(self, outputs, targets, num_classes=5):
              targets = targets.unsqueeze(1)
              targets = (torch.arange(1, num_classes).to(targets.device) < targets).float()
              return self.loss_fn(outputs, targets)
      
    3. Model Architecture Refinement:

      • Changes:
        • Reduced d_model from 1024 to 512 and e_layers from 6 to 4 to decrease computational load.
        • Added LayerNorm in ResBlock to stabilize training.
        • Modified the output layer to handle ordinal outputs by predicting multiple thresholds.
      • Benefit: Maintains modeling capacity while reducing computational requirements, leading to faster training and inference.
      class ResBlock(nn.Module):
          def __init__(self, configs):
              super(ResBlock, self).__init__()
              # Previous layers...
              self.layer_norm = nn.LayerNorm(configs.enc_in)
      
          def forward(self, x):
              # Apply layer normalization after residual connections
              x = self.layer_norm(x + residual)
              # Similarly for channel operations
      
    4. DataLoader Improvements:

      • Changes:
        • Increased batch_size to 256 for better GPU utilization.
        • Enabled pin_memory and set num_workers to 4 to speed up data loading.
      • Benefit: Reduces data loading bottlenecks, ensuring the GPU remains utilized effectively.
      train_loader = DataLoader(
          train_dataset, batch_size=configs.batch_size, shuffle=True,
          num_workers=configs.num_workers, pin_memory=True
      )
      test_loader = DataLoader(
          test_dataset, batch_size=configs.batch_size, shuffle=False,
          num_workers=configs.num_workers, pin_memory=True
      )
      
    5. Learning Rate Scheduler:

      • Implementation: Added ReduceLROnPlateau scheduler to adjust the learning rate based on the validation loss.
      • Benefit: Helps in converging faster and escaping local minima by reducing the learning rate when performance plateaus.
      scheduler = optim.lr_scheduler.ReduceLROnPlateau(optimizer, mode='min', 
                                                       factor=0.5, patience=5, verbose=True)
      
      # After evaluation:
      scheduler.step(test_loss)
      
    6. Efficient Metric Computation:

      • Changes:
        • Vectorized the conversion of ordinal outputs to class predictions.
        • Mapped class indices to continuous values (-2 to 2) to calculate MAE and RMSE.
      • Benefit: Reduces computational overhead and accelerates metric calculations.
      def calculate_metrics(outputs, targets):
          probs = torch.sigmoid(outputs)
          cumulative = torch.cumsum(probs, dim=1)
          predicted = (cumulative > 0.5).sum(dim=1)
          # Rest of the metric calculations...
      
    7. Model Saving Enhancements:

      • Changes:
        • Updated opset_version to 12 for better ONNX support.
        • Included scaler saving using joblib to ensure the scaler can be applied during inference.
      • Benefit: Ensures compatibility and reproducibility when deploying the model.
      import joblib
      joblib.dump(scaler, configs.scaler_path)
      
      torch.onnx.export(
          model, dummy_input, model_path, export_params=True, opset_version=12,
          input_names=['input'], output_names=['output'],
          dynamic_axes={'input': {0: 'batch_size'}, 'output': {0: 'batch_size'}}
      )
      

    Additional Recommendations

    1. Model Evaluation and Early Stopping:

      • Although a learning rate scheduler is implemented, consider adding an early stopping mechanism to halt training when the model stops improving, saving computational resources.
    2. Class Imbalance Handling:

      • Analyze the distribution of classes. If there's significant imbalance, consider implementing class weights in the loss function or using techniques like SMOTE to oversample minority classes.
    3. Hyperparameter Tuning:

      • Experiment with different hyperparameters such as learning rates, batch sizes, and model dimensions to find the optimal configuration for your specific dataset.
    4. Regularization Techniques:

      • Incorporate additional regularization methods like weight decay to prevent overfitting, especially if the model complexity increases.
    5. Logging and Monitoring:

      • Implement more sophisticated logging (e.g., using TensorBoard) to monitor training progress, loss curves, and other metrics in real-time.

    Conclusion

    The above modifications aim to optimize training speed and efficiency, improve the model's ability to understand the ordinal nature of the prediction task, and ensure that the model is scalable and deployable in a production environment like MetaTrader5. By leveraging mixed precision training, refining the loss function to align with the problem's nature, and optimizing data loading and model architecture, the model should exhibit enhanced performance and faster training times.