This is a training script for a model making predictions on a 1 min usd/euro chart. It…
- User
This is a training script for a model making predictions on a 1 min usd/euro chart. It predicts if the price will be lower, much lower, higher, much higher, or the same price, 30 minutes into the future. The trained model will be used in a meratrader5 environment as part of an ea making trades, which is why it is saved as an onnx model. Improve its speed, effencienccy, and predictive ability without adding features. Make the loss calculation more appropriate for the task at hand.
import os import torch import torch.nn as nn import torch.optim as optim from torch.utils.data import Dataset, DataLoader import pandas as pd import numpy as np from sklearn.preprocessing import MinMaxScaler from sklearn.model_selection import train_test_split import onnx import onnxruntime from tqdm import tqdm
Configuration
class Config: def init(self): self.task_name = 'short_term_forecast' self.seq_len = 180 # Input sequence length self.pred_len = 30 # Prediction length (30 minutes into the future) self.d_model = 1024 # Model dimension self.enc_in = 4 # Number of input features (Open, High, Low, Close) self.e_layers = 6 # Number of encoder layers self.dropout = 0.4 self.batch_size = 128 self.learning_rate = 0.00001 self.epochs = 100 self.device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') self.data_path = '/mnt/d/FX/fx_csv/22_23_1M.csv' self.model_save_path = '/mnt/d/FX/Mixer_Models' self.results_save_path = '/mnt/d/FX/Mixer_Results'
configs = Config()
class ScalerModule(nn.Module): def init(self, scaler): super(ScalerModule, self).init() self.min_ = nn.Parameter(torch.FloatTensor(scaler.min_), requires_grad=False) self.scale_ = nn.Parameter(torch.FloatTensor(scaler.scale_), requires_grad=False)
def forward(self, x): return (x - self.min_) / self.scale_Model definition
class ResBlock(nn.Module): def init(self, configs): super(ResBlock, self).init() self.temporal = nn.Sequential( nn.Linear(configs.seq_len, configs.d_model), nn.ReLU(), nn.Linear(configs.d_model, configs.seq_len), nn.Dropout(configs.dropout) ) self.channel = nn.Sequential( nn.Linear(configs.enc_in, configs.d_model), nn.ReLU(), nn.Linear(configs.d_model, configs.enc_in), nn.Dropout(configs.dropout) )
def forward(self, x): # x: [B, L, D] x = x + self.temporal(x.transpose(1, 2)).transpose(1, 2) x = x + self.channel(x) return xclass Model(nn.Module): def init(self, configs, scaler): super(Model, self).init() self.scaler = ScalerModule(scaler) self.task_name = configs.task_name self.layer = configs.e_layers self.model = nn.ModuleList([ResBlock(configs) for _ in range(configs.e_layers)]) self.pred_len = configs.pred_len self.projection = nn.Linear(configs.seq_len, configs.pred_len) self.final_layer = nn.Linear(configs.enc_in, 1) # Output a single value self.classifier = nn.Linear(configs.pred_len, 5) # 5 classes for prediction
def forecast(self, x_enc): # x: [B, L, D] x_enc = self.scaler(x_enc) for i in range(self.layer): x_enc = self.model[i](x_enc) enc_out = self.projection(x_enc.transpose(1, 2)).transpose(1, 2) enc_out = self.final_layer(enc_out).squeeze(-1) return self.classifier(enc_out) def forward(self, x_enc): if self.task_name == 'short_term_forecast': return self.forecast(x_enc) else: raise ValueError('Only short_term_forecast task implemented')Dataset
class ForexDataset(Dataset): def init(self, data, seq_len, pred_len): self.data = data self.seq_len = seq_len self.pred_len = pred_len
def __len__(self): return len(self.data) - self.seq_len - self.pred_len + 1 def __getitem__(self, idx): x = self.data[idx:idx+self.seq_len] y = self.data[idx+self.seq_len+self.pred_len-1, 3] # Close price after pred_len # Calculate the price change current_price = self.data[idx+self.seq_len-1, 3] price_change = (y - current_price) / current_price # Classify the price change if price_change < -0.01: label = 0 # much lower elif -0.01 <= price_change < -0.001: label = 1 # lower elif -0.001 <= price_change <= 0.001: label = 2 # same elif 0.001 < price_change <= 0.01: label = 3 # higher else: label = 4 # much higher return torch.FloatTensor(x), torch.LongTensor([label])Data preprocessing
def preprocess_data(configs): print("Loading and preprocessing data...") df = pd.read_csv(configs.data_path) df['DateTime'] = pd.to_datetime(df['DateTime']) df = df.set_index('DateTime')
scaler = MinMaxScaler() data = scaler.fit_transform(df[['Open', 'High', 'Low', 'Close']]) dataset = ForexDataset(data, configs.seq_len, configs.pred_len) train_size = int(0.8 * len(dataset)) test_size = len(dataset) - train_size train_dataset, test_dataset = torch.utils.data.random_split(dataset, [train_size, test_size]) train_loader = DataLoader(train_dataset, batch_size=configs.batch_size, shuffle=True) test_loader = DataLoader(test_dataset, batch_size=configs.batch_size, shuffle=False) return train_loader, test_loader, scalerdef calculate_metrics(output, target): _, predicted = torch.max(output.data, 1) correct = (predicted == target).sum().item()
# Convert output probabilities to continuous predictions continuous_pred = torch.sum(output * torch.tensor([-2, -1, 0, 1, 2]).to(output.device), dim=1) continuous_target = torch.tensor([-2, -1, 0, 1, 2]).to(target.device)[target] mae = torch.mean(torch.abs(continuous_pred - continuous_target)).item() mse = torch.mean((continuous_pred - continuous_target) ** 2).item() rmse = torch.sqrt(torch.tensor(mse)).item() return correct, mae, mse, rmseTraining function
def train(model, train_loader, optimizer, criterion, device, epoch): model.train() total_loss = 0 correct = 0 total = 0 total_mae = 0 total_mse = 0
progress_bar = tqdm(train_loader, desc=f"Epoch {epoch+1}") for batch_x, batch_y in progress_bar: optimizer.zero_grad() batch_x, batch_y = batch_x.to(device), batch_y.to(device) output = model(batch_x) loss = criterion(output, batch_y.squeeze()) loss.backward() optimizer.step() total_loss += loss.item() batch_correct, batch_mae, batch_mse, _ = calculate_metrics(output, batch_y.squeeze()) correct += batch_correct total += batch_y.size(0) total_mae += batch_mae * batch_y.size(0) total_mse += batch_mse * batch_y.size(0) # Update progress bar avg_loss = total_loss / (progress_bar.n + 1) accuracy = 100 * correct / total avg_mae = total_mae / total avg_rmse = torch.sqrt(torch.tensor(total_mse / total)).item() progress_bar.set_postfix({ 'loss': f'{avg_loss:.4f}', 'acc': f'{accuracy:.2f}%', 'MAE': f'{avg_mae:.4f}', 'RMSE': f'{avg_rmse:.4f}' }) return (total_loss / len(train_loader), 100 * correct / total, total_mae / total, torch.sqrt(torch.tensor(total_mse / total)).item())def evaluate(model, test_loader, criterion, device): model.eval() total_loss = 0 correct = 0 total = 0 total_mae = 0 total_mse = 0
with torch.no_grad(): for batch_x, batch_y in tqdm(test_loader, desc="Evaluating"): batch_x, batch_y = batch_x.to(device), batch_y.to(device) output = model(batch_x) loss = criterion(output, batch_y.squeeze()) total_loss += loss.item() batch_correct, batch_mae, batch_mse, _ = calculate_metrics(output, batch_y.squeeze()) correct += batch_correct total += batch_y.size(0) total_mae += batch_mae * batch_y.size(0) total_mse += batch_mse * batch_y.size(0) avg_loss = total_loss / len(test_loader) accuracy = 100 * correct / total avg_mae = total_mae / total avg_rmse = torch.sqrt(torch.tensor(total_mse / total)).item() return avg_loss, accuracy, avg_mae, avg_rmseSave model function
def save_model(model, path, epoch, test_loss): if not os.path.exists(path): os.makedirs(path) models = sorted([f for f in os.listdir(path) if f.endswith('.onnx')]) if len(models) >= 3: os.remove(os.path.join(path, models[0]))
model_path = os.path.join(path, f'model_epoch_{epoch}_loss_{test_loss:.4f}.onnx') dummy_input = torch.randn(1, configs.seq_len, configs.enc_in).to(configs.device) torch.onnx.export(model, dummy_input, model_path, export_params=True, opset_version=11, input_names=['input'], output_names=['output'], dynamic_axes={'input': {0: 'batch_size'}, 'output': {0: 'batch_size'}}) print(f"Model saved: {model_path}")Main training loop
def main(): print(f"Using device: {configs.device}") train_loader, test_loader, scaler = preprocess_data(configs)
model = Model(configs, scaler).to(configs.device) optimizer = optim.Adam(model.parameters(), lr=configs.learning_rate) criterion = nn.CrossEntropyLoss() best_test_loss = float('inf') for epoch in range(configs.epochs): train_loss, train_acc, train_mae, train_rmse = train(model, train_loader, optimizer, criterion, configs.device, epoch) test_loss, test_acc, test_mae, test_rmse = evaluate(model, test_loader, criterion, configs.device) print(f"\nEpoch {epoch+1}/{configs.epochs}") print(f"Train Loss: {train_loss:.4f}, Train Accuracy: {train_acc:.2f}%") print(f"Train MAE: {train_mae:.4f}, Train RMSE: {train_rmse:.4f}") print(f"Test Loss: {test_loss:.4f}, Test Accuracy: {test_acc:.2f}%") print(f"Test MAE: {test_mae:.4f}, Test RMSE: {test_rmse:.4f}") if test_loss < best_test_loss: best_test_loss = test_loss save_model(model, configs.model_save_path, epoch+1, test_loss) # Save final results with open(os.path.join(configs.results_save_path, 'training_results.txt'), 'w') as f: f.write(f"Final Train Loss: {train_loss:.4f}\n") f.write(f"Final Train Accuracy: {train_acc:.2f}%\n") f.write(f"Final Train MAE: {train_mae:.4f}\n") f.write(f"Final Train RMSE: {train_rmse:.4f}\n") f.write(f"Final Test Loss: {test_loss:.4f}\n") f.write(f"Final Test Accuracy: {test_acc:.2f}%\n") f.write(f"Final Test MAE: {test_mae:.4f}\n") f.write(f"Final Test RMSE: {test_rmse:.4f}\n") f.write(f"Best Test Loss: {best_test_loss:.4f}\n") print("Training completed. Results saved.")if name == "main": main()
- Assistant
To enhance the speed, efficiency, and predictive performance of your Forex prediction model while keeping the feature set unchanged, we can implement several optimizations and improvements:
-
Mixed Precision Training: Utilize PyTorch's Automatic Mixed Precision (AMP) to accelerate training and reduce memory usage without sacrificing model accuracy.
-
Optimized Loss Function for Ordinal Classification: Since your prediction task involves ordered categories (much lower < lower < same < higher < much higher), using an ordinal regression loss can better capture the inherent ordering in the classes compared to the standard
CrossEntropyLoss. -
Model Architecture Refinement: Simplify the model where possible to reduce computational complexity while maintaining or improving predictive performance.
-
DataLoader Improvements: Optimize data loading by enabling pinned memory and increasing the number of worker threads to speed up data fetching.
-
Learning Rate Scheduler: Implement a learning rate scheduler to adjust the learning rate during training, which can lead to faster convergence and better performance.
-
Efficient Metric Computation: Vectorize metric calculations to reduce computational overhead.
Below is the revised training script incorporating these improvements:
import os import torch import torch.nn as nn import torch.optim as optim from torch.utils.data import Dataset, DataLoader import pandas as pd import numpy as np from sklearn.preprocessing import MinMaxScaler from sklearn.model_selection import train_test_split import onnx import onnxruntime from tqdm import tqdm # Configuration class Config: def __init__(self): self.task_name = 'short_term_forecast' self.seq_len = 180 # Input sequence length self.pred_len = 30 # Prediction length (30 minutes into the future) self.d_model = 512 # Reduced model dimension for efficiency self.enc_in = 4 # Number of input features (Open, High, Low, Close) self.e_layers = 4 # Reduced number of encoder layers self.dropout = 0.3 # Slightly reduced dropout self.batch_size = 256 # Increased batch size for better GPU utilization self.learning_rate = 0.001 # Increased learning rate for faster convergence self.epochs = 100 self.device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') self.data_path = '/mnt/d/FX/fx_csv/22_23_1M.csv' self.model_save_path = '/mnt/d/FX/Mixer_Models' self.results_save_path = '/mnt/d/FX/Mixer_Results' self.num_workers = 4 # Number of workers for DataLoader self.scaler_path = '/mnt/d/FX/scaler.pkl' # Path to save the scaler configs = Config() class ScalerModule(nn.Module): def __init__(self, scaler): super(ScalerModule, self).__init__() self.min_ = nn.Parameter(torch.FloatTensor(scaler.min_), requires_grad=False) self.scale_ = nn.Parameter(torch.FloatTensor(scaler.scale_), requires_grad=False) def forward(self, x): return (x - self.min_) / self.scale_ # Optimized ResBlock with Layer Normalization class ResBlock(nn.Module): def __init__(self, configs): super(ResBlock, self).__init__() self.temporal = nn.Sequential( nn.Linear(configs.seq_len, configs.d_model), nn.ReLU(), nn.Linear(configs.d_model, configs.seq_len), nn.Dropout(configs.dropout) ) self.channel = nn.Sequential( nn.Linear(configs.enc_in, configs.d_model), nn.ReLU(), nn.Linear(configs.d_model, configs.enc_in), nn.Dropout(configs.dropout) ) self.layer_norm = nn.LayerNorm(configs.enc_in) def forward(self, x): # x: [B, L, D] residual = x x = self.temporal(x.transpose(1, 2)).transpose(1, 2) x = self.layer_norm(x + residual) residual = x x = self.channel(x) x = self.layer_norm(x + residual) return x # Ordinal Regression Loss class OrdinalLoss(nn.Module): def __init__(self): super(OrdinalLoss, self).__init__() self.loss_fn = nn.BCEWithLogitsLoss() def forward(self, outputs, targets, num_classes=5): # Convert targets to ordinal binary format targets = targets.unsqueeze(1) targets = (torch.arange(1, num_classes).to(targets.device) < targets).float() return self.loss_fn(outputs, targets) # Modified Model with Ordered Output class Model(nn.Module): def __init__(self, configs, scaler): super(Model, self).__init__() self.scaler = ScalerModule(scaler) self.task_name = configs.task_name self.layer = configs.e_layers self.model = nn.ModuleList([ResBlock(configs) for _ in range(configs.e_layers)]) self.seq_len = configs.seq_len self.projection = nn.Linear(configs.seq_len, configs.pred_len) self.final_layer = nn.Linear(configs.enc_in, configs.d_model) self.relu = nn.ReLU() # Output layers for ordinal regression self.classifiers = nn.ModuleList([ nn.Linear(configs.d_model, 1) for _ in range(configs.pred_len - 1) ]) def forward(self, x_enc): # x: [B, L, D] x_enc = self.scaler(x_enc) for i in range(self.layer): x_enc = self.model[i](x_enc) enc_out = self.projection(x_enc.transpose(1, 2)).transpose(1, 2) enc_out = self.final_layer(enc_out) enc_out = self.relu(enc_out) logits = [classifier(enc_out) for classifier in self.classifiers] logits = torch.cat(logits, dim=-1) # [B, pred_len -1] return logits # Dataset class ForexDataset(Dataset): def __init__(self, data, seq_len, pred_len): self.data = data self.seq_len = seq_len self.pred_len = pred_len def __len__(self): return len(self.data) - self.seq_len - self.pred_len + 1 def __getitem__(self, idx): x = self.data[idx:idx+self.seq_len] y = self.data[idx+self.seq_len+self.pred_len-1, 3] # Close price after pred_len # Calculate the price change current_price = self.data[idx+self.seq_len-1, 3] price_change = (y - current_price) / current_price # Classify the price change if price_change < -0.01: label = 0 # much lower elif -0.01 <= price_change < -0.001: label = 1 # lower elif -0.001 <= price_change <= 0.001: label = 2 # same elif 0.001 < price_change <= 0.01: label = 3 # higher else: label = 4 # much higher return torch.FloatTensor(x), torch.LongTensor([label]) # Data preprocessing def preprocess_data(configs): print("Loading and preprocessing data...") df = pd.read_csv(configs.data_path) df['DateTime'] = pd.to_datetime(df['DateTime']) df = df.set_index('DateTime') scaler = MinMaxScaler() data = scaler.fit_transform(df[['Open', 'High', 'Low', 'Close']]) # Save the scaler for future use import joblib joblib.dump(scaler, configs.scaler_path) dataset = ForexDataset(data, configs.seq_len, configs.pred_len) train_size = int(0.8 * len(dataset)) test_size = len(dataset) - train_size train_dataset, test_dataset = torch.utils.data.random_split(dataset, [train_size, test_size]) train_loader = DataLoader( train_dataset, batch_size=configs.batch_size, shuffle=True, num_workers=configs.num_workers, pin_memory=True ) test_loader = DataLoader( test_dataset, batch_size=configs.batch_size, shuffle=False, num_workers=configs.num_workers, pin_memory=True ) return train_loader, test_loader, scaler def calculate_metrics(outputs, targets): # Convert ordinal outputs to class predictions # Each output corresponds to a threshold; sum them to get class probs = torch.sigmoid(outputs) cumulative = torch.cumsum(probs, dim=1) predicted = (cumulative > 0.5).sum(dim=1) correct = (predicted == targets).sum().item() # For MAE and RMSE, convert classes to ordered values continuous_pred = predicted.float() - 2 # Classes 0-4 mapped to -2 to 2 continuous_target = targets.float() - 2 mae = torch.mean(torch.abs(continuous_pred - continuous_target)).item() mse = torch.mean((continuous_pred - continuous_target) ** 2).item() rmse = torch.sqrt(torch.tensor(mse)).item() return correct, mae, mse, rmse # Training function with Mixed Precision def train(model, train_loader, optimizer, criterion, device, epoch, scaler): model.train() scaler_amp = torch.cuda.amp.GradScaler() total_loss = 0 correct = 0 total = 0 total_mae = 0 total_mse = 0 progress_bar = tqdm(train_loader, desc=f"Epoch {epoch+1}/{configs.epochs}") for batch_x, batch_y in progress_bar: optimizer.zero_grad() batch_x, batch_y = batch_x.to(device, non_blocking=True), batch_y.to(device, non_blocking=True) with torch.cuda.amp.autocast(): outputs = model(batch_x) # [B, pred_len -1] loss = criterion(outputs, batch_y.squeeze()) scaler_amp.scale(loss).backward() scaler_amp.step(optimizer) scaler_amp.update() total_loss += loss.item() * batch_x.size(0) batch_correct, batch_mae, batch_mse, _ = calculate_metrics(outputs, batch_y.squeeze()) correct += batch_correct total += batch_y.size(0) total_mae += batch_mae * batch_y.size(0) total_mse += batch_mse * batch_y.size(0) # Update progress bar avg_loss = total_loss / total accuracy = 100 * correct / total avg_mae = total_mae / total avg_rmse = np.sqrt(total_mse / total) progress_bar.set_postfix({ 'loss': f'{avg_loss:.4f}', 'acc': f'{accuracy:.2f}%', 'MAE': f'{avg_mae:.4f}', 'RMSE': f'{avg_rmse:.4f}' }) return (total_loss / len(train_loader.dataset), 100 * correct / total, total_mae / total, np.sqrt(total_mse / total)) # Evaluation function with Mixed Precision def evaluate(model, test_loader, criterion, device, scaler): model.eval() scaler_amp = torch.cuda.amp.GradScaler() total_loss = 0 correct = 0 total = 0 total_mae = 0 total_mse = 0 with torch.no_grad(): progress_bar = tqdm(test_loader, desc="Evaluating") for batch_x, batch_y in progress_bar: batch_x, batch_y = batch_x.to(device, non_blocking=True), batch_y.to(device, non_blocking=True) with torch.cuda.amp.autocast(): outputs = model(batch_x) loss = criterion(outputs, batch_y.squeeze()) total_loss += loss.item() * batch_x.size(0) batch_correct, batch_mae, batch_mse, _ = calculate_metrics(outputs, batch_y.squeeze()) correct += batch_correct total += batch_y.size(0) total_mae += batch_mae * batch_y.size(0) total_mse += batch_mse * batch_y.size(0) # Update progress bar avg_loss = total_loss / total accuracy = 100 * correct / total avg_mae = total_mae / total avg_rmse = np.sqrt(total_mse / total) progress_bar.set_postfix({ 'loss': f'{avg_loss:.4f}', 'acc': f'{accuracy:.2f}%', 'MAE': f'{avg_mae:.4f}', 'RMSE': f'{avg_rmse:.4f}' }) return (total_loss / len(test_loader.dataset), 100 * correct / total, total_mae / total, np.sqrt(total_mse / total)) # Save model function with ONNX optimizations def save_model(model, path, epoch, test_loss, scaler): if not os.path.exists(path): os.makedirs(path) models = sorted([f for f in os.listdir(path) if f.endswith('.onnx')]) if len(models) >= 3: os.remove(os.path.join(path, models[0])) model_path = os.path.join(path, f'model_epoch_{epoch}_loss_{test_loss:.4f}.onnx') dummy_input = torch.randn(1, configs.seq_len, configs.enc_in).to(configs.device) torch.onnx.export( model, dummy_input, model_path, export_params=True, opset_version=12, input_names=['input'], output_names=['output'], dynamic_axes={'input': {0: 'batch_size'}, 'output': {0: 'batch_size'}} ) print(f"Model saved: {model_path}") # Main training loop with Learning Rate Scheduler def main(): print(f"Using device: {configs.device}") train_loader, test_loader, scaler = preprocess_data(configs) model = Model(configs, scaler).to(configs.device) optimizer = optim.Adam(model.parameters(), lr=configs.learning_rate) criterion = OrdinalLoss() scheduler = optim.lr_scheduler.ReduceLROnPlateau(optimizer, mode='min', factor=0.5, patience=5, verbose=True) best_test_loss = float('inf') for epoch in range(configs.epochs): train_loss, train_acc, train_mae, train_rmse = train( model, train_loader, optimizer, criterion, configs.device, epoch, scaler ) test_loss, test_acc, test_mae, test_rmse = evaluate( model, test_loader, criterion, configs.device, scaler ) scheduler.step(test_loss) print(f"\nEpoch {epoch+1}/{configs.epochs}") print(f"Train Loss: {train_loss:.4f}, Train Accuracy: {train_acc:.2f}%") print(f"Train MAE: {train_mae:.4f}, Train RMSE: {train_rmse:.4f}") print(f"Test Loss: {test_loss:.4f}, Test Accuracy: {test_acc:.2f}%") print(f"Test MAE: {test_mae:.4f}, Test RMSE: {test_rmse:.4f}") if test_loss < best_test_loss: best_test_loss = test_loss save_model(model, configs.model_save_path, epoch+1, test_loss, scaler) # Save final results if not os.path.exists(configs.results_save_path): os.makedirs(configs.results_save_path) with open(os.path.join(configs.results_save_path, 'training_results.txt'), 'w') as f: f.write(f"Final Train Loss: {train_loss:.4f}\n") f.write(f"Final Train Accuracy: {train_acc:.2f}%\n") f.write(f"Final Train MAE: {train_mae:.4f}\n") f.write(f"Final Train RMSE: {train_rmse:.4f}\n") f.write(f"Final Test Loss: {test_loss:.4f}\n") f.write(f"Final Test Accuracy: {test_acc:.2f}%\n") f.write(f"Final Test MAE: {test_mae:.4f}\n") f.write(f"Final Test RMSE: {test_rmse:.4f}\n") f.write(f"Best Test Loss: {best_test_loss:.4f}\n") print("Training completed. Results saved.") if __name__ == "__main__": main()Detailed Changes and Improvements
-
Mixed Precision Training:
- Implementation: Introduced
torch.cuda.amp'sGradScalerandautocastto handle mixed precision. - Benefit: Accelerates training and reduces GPU memory usage, allowing for larger batch sizes or more complex models.
from torch.cuda.amp import GradScaler, autocast # Within train and evaluate functions: scaler_amp = torch.cuda.amp.GradScaler() with torch.cuda.amp.autocast(): outputs = model(batch_x) loss = criterion(outputs, batch_y.squeeze()) scaler_amp.scale(loss).backward() scaler_amp.step(optimizer) scaler_amp.update() - Implementation: Introduced
-
Optimized Ordinal Loss Function:
- Implementation: Created a custom
OrdinalLossclass that transforms the multiclass problem into multiple binary classification tasks, aligning with ordinal regression principles. - Benefit: Captures the ordinal nature of the output classes, potentially improving predictive performance.
class OrdinalLoss(nn.Module): def __init__(self): super(OrdinalLoss, self).__init__() self.loss_fn = nn.BCEWithLogitsLoss() def forward(self, outputs, targets, num_classes=5): targets = targets.unsqueeze(1) targets = (torch.arange(1, num_classes).to(targets.device) < targets).float() return self.loss_fn(outputs, targets) - Implementation: Created a custom
-
Model Architecture Refinement:
- Changes:
- Reduced
d_modelfrom 1024 to 512 ande_layersfrom 6 to 4 to decrease computational load. - Added
LayerNorminResBlockto stabilize training. - Modified the output layer to handle ordinal outputs by predicting multiple thresholds.
- Reduced
- Benefit: Maintains modeling capacity while reducing computational requirements, leading to faster training and inference.
class ResBlock(nn.Module): def __init__(self, configs): super(ResBlock, self).__init__() # Previous layers... self.layer_norm = nn.LayerNorm(configs.enc_in) def forward(self, x): # Apply layer normalization after residual connections x = self.layer_norm(x + residual) # Similarly for channel operations - Changes:
-
DataLoader Improvements:
- Changes:
- Increased
batch_sizeto 256 for better GPU utilization. - Enabled
pin_memoryand setnum_workersto 4 to speed up data loading.
- Increased
- Benefit: Reduces data loading bottlenecks, ensuring the GPU remains utilized effectively.
train_loader = DataLoader( train_dataset, batch_size=configs.batch_size, shuffle=True, num_workers=configs.num_workers, pin_memory=True ) test_loader = DataLoader( test_dataset, batch_size=configs.batch_size, shuffle=False, num_workers=configs.num_workers, pin_memory=True ) - Changes:
-
Learning Rate Scheduler:
- Implementation: Added
ReduceLROnPlateauscheduler to adjust the learning rate based on the validation loss. - Benefit: Helps in converging faster and escaping local minima by reducing the learning rate when performance plateaus.
scheduler = optim.lr_scheduler.ReduceLROnPlateau(optimizer, mode='min', factor=0.5, patience=5, verbose=True) # After evaluation: scheduler.step(test_loss) - Implementation: Added
-
Efficient Metric Computation:
- Changes:
- Vectorized the conversion of ordinal outputs to class predictions.
- Mapped class indices to continuous values (
-2to2) to calculate MAE and RMSE.
- Benefit: Reduces computational overhead and accelerates metric calculations.
def calculate_metrics(outputs, targets): probs = torch.sigmoid(outputs) cumulative = torch.cumsum(probs, dim=1) predicted = (cumulative > 0.5).sum(dim=1) # Rest of the metric calculations... - Changes:
-
Model Saving Enhancements:
- Changes:
- Updated
opset_versionto 12 for better ONNX support. - Included
scalersaving usingjoblibto ensure the scaler can be applied during inference.
- Updated
- Benefit: Ensures compatibility and reproducibility when deploying the model.
import joblib joblib.dump(scaler, configs.scaler_path) torch.onnx.export( model, dummy_input, model_path, export_params=True, opset_version=12, input_names=['input'], output_names=['output'], dynamic_axes={'input': {0: 'batch_size'}, 'output': {0: 'batch_size'}} ) - Changes:
Additional Recommendations
-
Model Evaluation and Early Stopping:
- Although a learning rate scheduler is implemented, consider adding an early stopping mechanism to halt training when the model stops improving, saving computational resources.
-
Class Imbalance Handling:
- Analyze the distribution of classes. If there's significant imbalance, consider implementing class weights in the loss function or using techniques like SMOTE to oversample minority classes.
-
Hyperparameter Tuning:
- Experiment with different hyperparameters such as learning rates, batch sizes, and model dimensions to find the optimal configuration for your specific dataset.
-
Regularization Techniques:
- Incorporate additional regularization methods like weight decay to prevent overfitting, especially if the model complexity increases.
-
Logging and Monitoring:
- Implement more sophisticated logging (e.g., using TensorBoard) to monitor training progress, loss curves, and other metrics in real-time.
Conclusion
The above modifications aim to optimize training speed and efficiency, improve the model's ability to understand the ordinal nature of the prediction task, and ensure that the model is scalable and deployable in a production environment like MetaTrader5. By leveraging mixed precision training, refining the loss function to align with the problem's nature, and optimizing data loading and model architecture, the model should exhibit enhanced performance and faster training times.
-